<commit_before>package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"io"
	"log"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

const (
	ADDRESS   = "http://localhost:9900"
	LENGTH    = 6
	DELETE    = ADDRESS + "/del/{PASTE}/{DELKEY}\n"
	PASTEARGS = ADDRESS + "/p/{PASTE}/(python|language)\n"
	URLARGS   = ADDRESS + "/save/XML|JSON\n"
	SOURCE    = "Source: https://github.com/ewhal/Pastebin\n"
	TEXT      = "$ <command> | curl -F 'p=<-' " + ADDRESS + "/save" + "\n" + PASTEARGS + URLARGS + DELETE + SOURCE
	PORT      = ":9900"
	USERNAME  = ""
	PASS      = ""
	NAME      = ""
	DATABASE  = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

type Response struct {
	ID     string `json:"id"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	s := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	query, err := db.Query("select id from pastebin")
	for query.Next() {
		var id string
		err := query.Scan(&id)
		if err != nil {

		}
		if id == s {
			generateName()
		}
	}
	db.Close()

	return s

}
func hash(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

func save(raw string) []string {
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	sha := hash(raw)
	query, err := db.Query("select id, hash, data, delkey from pastebin")
	for query.Next() {
		var id, hash, paste, delkey string
		err := query.Scan(&id, &hash, &paste, &delkey)
		check(err)
		if hash == sha {
			url := ADDRESS + "/p/" + id
			return []string{id, hash, url, paste, delkey}
		}
	}
	id := generateName()
	url := ADDRESS + "/p/" + id
	delKey := uniuri.NewLen(40)
	paste := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, hash, data, delkey) values(?,?,?,?)")
	check(err)
	_, err = stmt.Exec(id, sha, paste, delKey)
	check(err)
	db.Close()
	return []string{id, sha, url, paste, delKey}

}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	check(err)

	stmt, err := db.Prepare("delete from pastebin where delkey=? id=?")
	check(err)

	res, err := stmt.Exec(html.EscapeString(delkey), html.EscapeString(paste))
	check(err)

	affect, err := res.RowsAffected()
	check(err)

	io.WriteString(w, string(affect))

	db.Close()

}
func saveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
		}

		values := save(paste)
		b := &Response{
			ID:     values[0],
			HASH:   values[1],
			URL:    values[2],
			SIZE:   len(values[3]),
			DELKEY: values[4],
		}

		switch output {
		case "json":

			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)

		default:
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

func langHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]
	s := getPaste(paste)
	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "full, style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,", "utf-8")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	io.WriteString(w, highlight)

}

func getPaste(paste string) string {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	var s string
	err = db.QueryRow("select data from pastebin where id=?", param1).Scan(&s)
	db.Close()
	check(err)

	if err == sql.ErrNoRows {
		return "Error invalid paste"
	} else {
		return html.UnescapeString(s)
	}

}
func pasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s := getPaste(paste)
	io.WriteString(w, s)

}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", pasteHandler)
	router.HandleFunc("/p/{pasteId}/{lang}", langHandler)
	router.HandleFunc("/save", saveHandler)
	router.HandleFunc("/save/{output}", saveHandler)
	router.HandleFunc("/del/{pasteId}/{delKey}", delHandler)
	router.PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.Dir("assets/"))))
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<commit_msg>Add lang handling<commit_after>package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"io"
	"log"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

const (
	ADDRESS   = "http://localhost:9900"
	LENGTH    = 6
	DELETE    = ADDRESS + "/del/{PASTE}/{DELKEY}\n"
	PASTEARGS = ADDRESS + "/p/{PASTE}/(python|language)\n"
	URLARGS   = ADDRESS + "/save/XML|JSON\n"
	SOURCE    = "Source: https://github.com/ewhal/Pastebin\n"
	TEXT      = "$ <command> | curl -F 'p=<-' " + ADDRESS + "/save" + "\n" + PASTEARGS + URLARGS + DELETE + SOURCE
	PORT      = ":9900"
	USERNAME  = ""
	PASS      = ""
	NAME      = ""
	DATABASE  = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

type Response struct {
	ID     string `json:"id"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	s := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	query, err := db.Query("select id from pastebin")
	for query.Next() {
		var id string
		err := query.Scan(&id)
		if err != nil {

		}
		if id == s {
			generateName()
		}
	}
	db.Close()

	return s

}
func hash(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

func save(raw string, lang string) []string {
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	sha := hash(raw)
	query, err := db.Query("select id, hash, data, delkey from pastebin")
	for query.Next() {
		var id, hash, paste, delkey string
		err := query.Scan(&id, &hash, &paste, &delkey)
		check(err)
		if hash == sha {
			url := ADDRESS + "/p/" + id
			return []string{id, hash, url, paste, delkey}
		}
	}
	id := generateName()
	if lang == "" {
		url := ADDRESS + "/p/" + id
	} else {
		url := ADDRESS + "/p/" + id + "/" lang
	}
	delKey := uniuri.NewLen(40)
	paste := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, hash, data, delkey) values(?,?,?,?)")
	check(err)
	_, err = stmt.Exec(id, sha, paste, delKey)
	check(err)
	db.Close()
	return []string{id, sha, url, paste, delKey}
}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	check(err)

	stmt, err := db.Prepare("delete from pastebin where delkey=? id=?")
	check(err)

	res, err := stmt.Exec(html.EscapeString(delkey), html.EscapeString(paste))
	check(err)

	affect, err := res.RowsAffected()
	check(err)

	io.WriteString(w, string(affect))

	db.Close()

}
func saveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		lang := r.FormValue("lang")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
		}
		values := save(paste, lang)
		b := &Response{
			ID:     values[0],
			HASH:   values[1],
			URL:    values[2],
			SIZE:   len(values[3]),
			DELKEY: values[4],
		}

		switch output {
		case "json":

			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)

		default:
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

func langHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]
	s := getPaste(paste)
	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "full, style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,", "utf-8")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	io.WriteString(w, highlight)

}

func getPaste(paste string) string {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	var s string
	err = db.QueryRow("select data from pastebin where id=?", param1).Scan(&s)
	db.Close()
	check(err)

	if err == sql.ErrNoRows {
		return "Error invalid paste"
	} else {
		return html.UnescapeString(s)
	}

}
func pasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s := getPaste(paste)
	io.WriteString(w, s)

}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", pasteHandler)
	router.HandleFunc("/p/{pasteId}/{lang}", langHandler)
	router.HandleFunc("/save", saveHandler)
	router.HandleFunc("/save/{output}", saveHandler)
	router.HandleFunc("/del/{pasteId}/{delKey}", delHandler)
	router.PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.Dir("assets/"))))
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>


<commit_before>// Copyright 2015 CodeIgnition. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"flag"
	"log"
	"net/http"
	"time"

	"github.com/codeignition/recon/policy"
	"github.com/nats-io/nats"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// TODO: If possible, extract the path constants into a separate package
// which can be used both by marksman and recond.

// Different URL paths.
// They are not hardcoded so that changing them is easier.
const (
	eventsAPIPath = "/api/events"
	agentsAPIPath = "/api/agents"
	agentAPIPath  = agentsAPIPath + "/"
	policyAPIPath = "/api/policy"
)

// Agent represents a recon daemon running on
// a machine.
type Agent struct {
	UID          string    `json:"uid"`
	HostName     string    `json:"host_name"`
	RegisteredAt time.Time `json:"registered_at"` // time at which the agent was registered
	UpdatedAt    time.Time `json:"updated_at"`    // time at which the agent metrics were last received / updated
}

// agentOutput is the structure that's present in the JSON API.
// It specifies a contract with the clients (e.g. frontend).
// So, be careful while changing it and update the clients accordingly.
type agentOutput struct {
	Agent
	Status string `json:"status"`
}

// TODO: Instead of using a global for each collection,
// abstract this into an interface, which makes it
// easier for testing.
var (
	// agents collection
	agentsC *mgo.Collection

	// events collection
	eventsC *mgo.Collection
)

// NATS variables
var (
	natsEncConn *nats.EncodedConn
)

// Command line flags
// prepend flag to variable names to not pollute the global namespace.
var (
	flagAddr    = flag.String("addr", ":8080", "serve HTTP on `address`")
	flagNatsURL = flag.String("nats", nats.DefaultURL, "nats URL")
)

func main() {
	log.SetFlags(0)
	log.SetPrefix("marksman: ")

	flag.Parse()

	mux := http.NewServeMux()

	mux.Handle("/", http.FileServer(http.Dir("./public")))
	mux.HandleFunc(eventsAPIPath, eventsHandler)
	mux.HandleFunc(agentsAPIPath, agentsHandler)
	mux.HandleFunc(agentAPIPath, agentHandler)
	mux.HandleFunc(policyAPIPath, policyHandler)

	session, err := mgo.Dial("127.0.0.1")
	if err != nil {
		panic(err)
	}
	defer session.Close()

	// Optional. Switch the session to a monotonic behavior.
	session.SetMode(mgo.Monotonic, true)
	agentsC = session.DB("recon-dev").C("agents")
	eventsC = session.DB("recon-dev").C("events")

	nc, err := nats.Connect(*flagNatsURL)
	if err != nil {
		log.Fatalln(err)
	}
	natsEncConn, err = nats.NewEncodedConn(nc, "json")
	if err != nil {
		log.Fatalln(err)
	}
	defer natsEncConn.Close()

	natsEncConn.Subscribe("policy_events", func(e *policy.Event) {
		uid := e.Policy.AgentUID
		var a Agent
		err := agentsC.Find(bson.M{"uid": uid}).One(&a)
		if err != nil {
			log.Printf("failed to find the agent %s: %s", uid, err)
			return
		}

		a.UpdatedAt = time.Now()
		err = agentsC.Update(bson.M{"uid": uid}, a)
		if err != nil {
			log.Printf("failed to update the agent %s: %s", uid, err)
		}

		err = eventsC.Insert(e)
		if err != nil {
			log.Printf("failed to insert event: %s", err)
		}

	})

	log.Println("Server started: http://localhost" + *flagAddr)
	log.Fatal(http.ListenAndServe(*flagAddr, mux))
}

// Status returns the status of the agent.
// It is calculated on-demand. If the agent was registered
// or its metrics were updated in the last 10 seconds, it
// returns "online", else "offline"
func (a Agent) Status() string {
	if time.Since(a.UpdatedAt) > 10*time.Second {
		return "offline"
	}
	return "online"
}

func agentHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		uid := r.URL.Path[len(agentAPIPath):]
		var a Agent
		err := agentsC.Find(bson.M{"uid": uid}).One(&a)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		b := agentOutput{
			Agent:  a,
			Status: a.Status(),
		}

		enc := json.NewEncoder(w)
		if err := enc.Encode(b); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	default:
		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		return
	}
}

func agentsHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	switch r.Method {
	case "GET":
		var agents []Agent
		err := agentsC.Find(nil).All(&agents)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		b := make([]agentOutput, len(agents))
		for i, a := range agents {
			b[i] = agentOutput{
				Agent:  a,
				Status: a.Status(),
			}
		}

		enc := json.NewEncoder(w)
		if err := enc.Encode(b); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	case "POST":
		var a Agent
		defer r.Body.Close()
		dec := json.NewDecoder(r.Body)
		if err := dec.Decode(&a); err != nil {
			http.Error(w, "unable to decode json", http.StatusBadRequest)
			return
		}
		if a.UID == "" {
			http.Error(w, "UID can't be empty", http.StatusBadRequest)
			return
		}
		// check whether the agent is already registered.
		// Insert it only if its not registered else just
		// update the UpdatedAt field for the agent in the DB.
		err := agentsC.Find(bson.M{"uid": a.UID}).One(&a)
		if err != nil {
			if err == mgo.ErrNotFound {
				a.RegisteredAt = time.Now()
				a.UpdatedAt = time.Now()
				if err := agentsC.Insert(a); err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
			} else {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		} else {
			a.UpdatedAt = time.Now()
			err = agentsC.Update(bson.M{"uid": a.UID}, a)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		}

		w.WriteHeader(http.StatusCreated)
		nurl := struct {
			NatsURL string `json:"nats_url"`
		}{
			NatsURL: *flagNatsURL,
		}
		enc := json.NewEncoder(w)
		if err := enc.Encode(nurl); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	default:
		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		return
	}
}
<commit_msg>add CORS middleware<commit_after>// Copyright 2015 CodeIgnition. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"flag"
	"log"
	"net/http"
	"time"

	"github.com/codeignition/recon/policy"
	"github.com/nats-io/nats"
	"github.com/rs/cors"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// TODO: If possible, extract the path constants into a separate package
// which can be used both by marksman and recond.

// Different URL paths.
// They are not hardcoded so that changing them is easier.
const (
	eventsAPIPath = "/api/events"
	agentsAPIPath = "/api/agents"
	agentAPIPath  = agentsAPIPath + "/"
	policyAPIPath = "/api/policy"
)

// Agent represents a recon daemon running on
// a machine.
type Agent struct {
	UID          string    `json:"uid"`
	HostName     string    `json:"host_name"`
	RegisteredAt time.Time `json:"registered_at"` // time at which the agent was registered
	UpdatedAt    time.Time `json:"updated_at"`    // time at which the agent metrics were last received / updated
}

// agentOutput is the structure that's present in the JSON API.
// It specifies a contract with the clients (e.g. frontend).
// So, be careful while changing it and update the clients accordingly.
type agentOutput struct {
	Agent
	Status string `json:"status"`
}

// TODO: Instead of using a global for each collection,
// abstract this into an interface, which makes it
// easier for testing.
var (
	// agents collection
	agentsC *mgo.Collection

	// events collection
	eventsC *mgo.Collection
)

// NATS variables
var (
	natsEncConn *nats.EncodedConn
)

// Command line flags
// prepend flag to variable names to not pollute the global namespace.
var (
	flagAddr    = flag.String("addr", ":8080", "serve HTTP on `address`")
	flagNatsURL = flag.String("nats", nats.DefaultURL, "nats URL")
)

func main() {
	log.SetFlags(0)
	log.SetPrefix("marksman: ")

	flag.Parse()

	mux := http.NewServeMux()

	mux.Handle("/", http.FileServer(http.Dir("./public")))
	mux.HandleFunc(eventsAPIPath, eventsHandler)
	mux.HandleFunc(agentsAPIPath, agentsHandler)
	mux.HandleFunc(agentAPIPath, agentHandler)
	mux.HandleFunc(policyAPIPath, policyHandler)

	session, err := mgo.Dial("127.0.0.1")
	if err != nil {
		panic(err)
	}
	defer session.Close()

	// Optional. Switch the session to a monotonic behavior.
	session.SetMode(mgo.Monotonic, true)
	agentsC = session.DB("recon-dev").C("agents")
	eventsC = session.DB("recon-dev").C("events")

	nc, err := nats.Connect(*flagNatsURL)
	if err != nil {
		log.Fatalln(err)
	}
	natsEncConn, err = nats.NewEncodedConn(nc, "json")
	if err != nil {
		log.Fatalln(err)
	}
	defer natsEncConn.Close()

	natsEncConn.Subscribe("policy_events", func(e *policy.Event) {
		uid := e.Policy.AgentUID
		var a Agent
		err := agentsC.Find(bson.M{"uid": uid}).One(&a)
		if err != nil {
			log.Printf("failed to find the agent %s: %s", uid, err)
			return
		}

		a.UpdatedAt = time.Now()
		err = agentsC.Update(bson.M{"uid": uid}, a)
		if err != nil {
			log.Printf("failed to update the agent %s: %s", uid, err)
		}

		err = eventsC.Insert(e)
		if err != nil {
			log.Printf("failed to insert event: %s", err)
		}

	})

	log.Println("Server started: http://localhost" + *flagAddr)
	handler := cors.Default().Handler(mux)
	log.Fatal(http.ListenAndServe(*flagAddr, handler))
}

// Status returns the status of the agent.
// It is calculated on-demand. If the agent was registered
// or its metrics were updated in the last 10 seconds, it
// returns "online", else "offline"
func (a Agent) Status() string {
	if time.Since(a.UpdatedAt) > 10*time.Second {
		return "offline"
	}
	return "online"
}

func agentHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		uid := r.URL.Path[len(agentAPIPath):]
		var a Agent
		err := agentsC.Find(bson.M{"uid": uid}).One(&a)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		b := agentOutput{
			Agent:  a,
			Status: a.Status(),
		}

		enc := json.NewEncoder(w)
		if err := enc.Encode(b); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	default:
		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		return
	}
}

func agentsHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	switch r.Method {
	case "GET":
		var agents []Agent
		err := agentsC.Find(nil).All(&agents)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		b := make([]agentOutput, len(agents))
		for i, a := range agents {
			b[i] = agentOutput{
				Agent:  a,
				Status: a.Status(),
			}
		}

		enc := json.NewEncoder(w)
		if err := enc.Encode(b); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	case "POST":
		var a Agent
		defer r.Body.Close()
		dec := json.NewDecoder(r.Body)
		if err := dec.Decode(&a); err != nil {
			http.Error(w, "unable to decode json", http.StatusBadRequest)
			return
		}
		if a.UID == "" {
			http.Error(w, "UID can't be empty", http.StatusBadRequest)
			return
		}
		// check whether the agent is already registered.
		// Insert it only if its not registered else just
		// update the UpdatedAt field for the agent in the DB.
		err := agentsC.Find(bson.M{"uid": a.UID}).One(&a)
		if err != nil {
			if err == mgo.ErrNotFound {
				a.RegisteredAt = time.Now()
				a.UpdatedAt = time.Now()
				if err := agentsC.Insert(a); err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
			} else {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		} else {
			a.UpdatedAt = time.Now()
			err = agentsC.Update(bson.M{"uid": a.UID}, a)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		}

		w.WriteHeader(http.StatusCreated)
		nurl := struct {
			NatsURL string `json:"nats_url"`
		}{
			NatsURL: *flagNatsURL,
		}
		enc := json.NewEncoder(w)
		if err := enc.Encode(nurl); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	default:
		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/boltdb/bolt"
	"github.com/cloudfoundry-community/firehose-to-syslog/caching"
	"github.com/cloudfoundry-community/firehose-to-syslog/events"
	"github.com/cloudfoundry-community/firehose-to-syslog/extrafields"
	"github.com/cloudfoundry-community/firehose-to-syslog/firehose"
	"github.com/cloudfoundry-community/firehose-to-syslog/logging"
	"github.com/cloudfoundry-community/go-cfclient"
	"github.com/pkg/profile"
	"gopkg.in/alecthomas/kingpin.v2"
	"log"
	"net/http"
	"os"
	"time"
)

var (
	debug             = kingpin.Flag("debug", "Enable debug mode. This disables forwarding to syslog").Default("false").OverrideDefaultFromEnvar("DEBUG").Bool()
	apiEndpoint       = kingpin.Flag("api-endpoint", "Api endpoint address. For bosh-lite installation of CF: https://api.10.244.0.34.xip.io").OverrideDefaultFromEnvar("API_ENDPOINT").Required().String()
	dopplerEndpoint   = kingpin.Flag("doppler-endpoint", "Overwrite default doppler endpoint return by /v2/info").OverrideDefaultFromEnvar("DOPPLER_ENDPOINT").String()
	syslogServer      = kingpin.Flag("syslog-server", "Syslog server.").OverrideDefaultFromEnvar("SYSLOG_ENDPOINT").String()
	subscriptionId    = kingpin.Flag("subscription-id", "Id for the subscription.").Default("firehose").OverrideDefaultFromEnvar("FIREHOSE_SUBSCRIPTION_ID").String()
	user              = kingpin.Flag("user", "Admin user.").Default("admin").OverrideDefaultFromEnvar("FIREHOSE_USER").String()
	password          = kingpin.Flag("password", "Admin password.").Default("admin").OverrideDefaultFromEnvar("FIREHOSE_PASSWORD").String()
	skipSSLValidation = kingpin.Flag("skip-ssl-validation", "Please don't").Default("false").OverrideDefaultFromEnvar("SKIP_SSL_VALIDATION").Bool()
	cfPush            = kingpin.Flag("cf-push", "Deploy to Cloud Foundry.").Default("false").OverrideDefaultFromEnvar("CF_PUSH").Bool()
	wantedEvents      = kingpin.Flag("events", fmt.Sprintf("Comma separated list of events you would like. Valid options are %s", events.GetListAuthorizedEventEvents())).Default("LogMessage").OverrideDefaultFromEnvar("EVENTS").String()
	boltDatabasePath  = kingpin.Flag("boltdb-path", "Bolt Database path ").Default("my.db").String()
	tickerTime        = kingpin.Flag("cc-pull-time", "CloudController Pooling time in sec").Default("60s").Duration()
	extraFields       = kingpin.Flag("extra-fields", "Extra fields you want to annotate your events with, example: '--extra-fields=env:dev,something:other ").Default("").String()
	modeProf          = kingpin.Flag("mode-prof", "Enable profiling mode, one of [cpu, mem, block]").Default("").String()
	pathProf          = kingpin.Flag("path-prof", "Set the Path to write profiling file").Default("").String()
)

const (
	version = "1.1.0 - c9af859"
)

func main() {
	kingpin.Version(version)
	kingpin.Parse()
	logging.LogStd(fmt.Sprintf("Starting firehose-to-syslog %s ", version), true)

	logging.SetupLogging(*syslogServer, *debug)

	if *cfPush == true {
		setupHTTP()
	}

	c := cfclient.Config{
		ApiAddress:        *apiEndpoint,
		Username:          *user,
		Password:          *password,
		SkipSslValidation: *skipSSLValidation,
	}
	cfClient := cfclient.NewClient(&c)

	if len(*dopplerEndpoint) > 0 {
		cfClient.Endpoint.DopplerEndpoint = *dopplerEndpoint
	}
	logging.LogStd(fmt.Sprintf("Using %s as doppler endpoint", cfClient.Endpoint.DopplerEndpoint), true)

	logging.LogStd("Setting up event routing!", true)
	err := events.SetupEventRouting(*wantedEvents)
	if err != nil {
		log.Fatal("Error setting up event routing: ", err)
		os.Exit(1)

	}

	//Use bolt for in-memory  - file caching
	db, err := bolt.Open(*boltDatabasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		log.Fatal("Error opening bolt db: ", err)
		os.Exit(1)

	}
	defer db.Close()

	if *modeProf != "" {
		switch *modeProf {
		case "cpu":
			defer profile.Start(profile.CPUProfile, profile.ProfilePath(*pathProf)).Stop()
		case "mem":
			defer profile.Start(profile.MemProfile, profile.ProfilePath(*pathProf)).Stop()
		case "block":
			defer profile.Start(profile.BlockProfile, profile.ProfilePath(*pathProf)).Stop()
		default:
			// do nothing
		}
	}

	caching.SetCfClient(cfClient)
	caching.SetAppDb(db)
	caching.CreateBucket()

	//Let's Update the database the first time
	logging.LogStd("Start filling app/space/org cache.", true)
	apps := caching.GetAllApp()
	logging.LogStd(fmt.Sprintf("Done filling cache! Found [%d] Apps", len(apps)), true)

	// Ticker Pooling the CC every X sec
	ccPooling := time.NewTicker(*tickerTime)

	go func() {
		for range ccPooling.C {
			apps = caching.GetAllApp()
		}
	}()

	// Parse extra fields from cmd call
	extraFields, err := extrafields.ParseExtraFields(*extraFields)
	if err != nil {
		log.Fatal("Error parsing extra fields: ", err)
		os.Exit(1)
	}

	if logging.Connect() || *debug {

		logging.LogStd("Connected to Syslog Server! Connecting to Firehose...", true)

		firehose := firehose.CreateFirehoseChan(cfClient.Endpoint.DopplerEndpoint, cfClient.GetToken(), *subscriptionId, *skipSSLValidation)
		if firehose != nil {
			logging.LogStd("Firehose Subscription Succesfull! Routing events...", true)
			events.RouteEvents(firehose, extraFields)
		} else {
			logging.LogError("Failed connecting to Firehose...Please check settings and try again!", "")
		}

	} else {
		logging.LogError("Failed connecting to the Syslog Server...Please check settings and try again!", "")
	}

}

func getInfo(w http.ResponseWriter, req *http.Request) {
	fmt.Fprintln(w,
		"Hello!\nWe have processed", events.GetTotalCountOfSelectedEvents(), "total events.",
		"\nWe have tapped the firehose at", fmt.Sprintf("wss://doppler.%s", *dopplerEndpoint))
	fmt.Fprintln(w, "\nOf those events, we have processed\n")
	for event, count := range events.GetSelectedEventsCount() {
		fmt.Fprintln(w, count, event, "events\n")
	}
}

func setupHTTP() {
	http.HandleFunc("/", getInfo)

	go func() {
		err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
		if err != nil {
			log.Fatal("ListenAndServe:", err)
		}
	}()
}
<commit_msg>Removed http handler and changed the flag for logging event...<commit_after>package main

import (
	"fmt"
	"github.com/boltdb/bolt"
	"github.com/cloudfoundry-community/firehose-to-syslog/caching"
	"github.com/cloudfoundry-community/firehose-to-syslog/events"
	"github.com/cloudfoundry-community/firehose-to-syslog/extrafields"
	"github.com/cloudfoundry-community/firehose-to-syslog/firehose"
	"github.com/cloudfoundry-community/firehose-to-syslog/logging"
	"github.com/cloudfoundry-community/go-cfclient"
	"github.com/pkg/profile"
	"gopkg.in/alecthomas/kingpin.v2"
	"log"
	"os"
	"time"
)

var (
	debug             = kingpin.Flag("debug", "Enable debug mode. This disables forwarding to syslog").Default("false").OverrideDefaultFromEnvar("DEBUG").Bool()
	apiEndpoint       = kingpin.Flag("api-endpoint", "Api endpoint address. For bosh-lite installation of CF: https://api.10.244.0.34.xip.io").OverrideDefaultFromEnvar("API_ENDPOINT").Required().String()
	dopplerEndpoint   = kingpin.Flag("doppler-endpoint", "Overwrite default doppler endpoint return by /v2/info").OverrideDefaultFromEnvar("DOPPLER_ENDPOINT").String()
	syslogServer      = kingpin.Flag("syslog-server", "Syslog server.").OverrideDefaultFromEnvar("SYSLOG_ENDPOINT").String()
	subscriptionId    = kingpin.Flag("subscription-id", "Id for the subscription.").Default("firehose").OverrideDefaultFromEnvar("FIREHOSE_SUBSCRIPTION_ID").String()
	user              = kingpin.Flag("user", "Admin user.").Default("admin").OverrideDefaultFromEnvar("FIREHOSE_USER").String()
	password          = kingpin.Flag("password", "Admin password.").Default("admin").OverrideDefaultFromEnvar("FIREHOSE_PASSWORD").String()
	skipSSLValidation = kingpin.Flag("skip-ssl-validation", "Please don't").Default("false").OverrideDefaultFromEnvar("SKIP_SSL_VALIDATION").Bool()
	showEventTotals   = kingpin.Flag("log-event-totals", "Logs the counters for all selected events since nozzle was last started.").Default("false").OverrideDefaultFromEnvar("LOG_EVENT_TOTALS").Bool()
	wantedEvents      = kingpin.Flag("events", fmt.Sprintf("Comma separated list of events you would like. Valid options are %s", events.GetListAuthorizedEventEvents())).Default("LogMessage").OverrideDefaultFromEnvar("EVENTS").String()
	boltDatabasePath  = kingpin.Flag("boltdb-path", "Bolt Database path ").Default("my.db").String()
	tickerTime        = kingpin.Flag("cc-pull-time", "CloudController Pooling time in sec").Default("60s").Duration()
	extraFields       = kingpin.Flag("extra-fields", "Extra fields you want to annotate your events with, example: '--extra-fields=env:dev,something:other ").Default("").String()
	modeProf          = kingpin.Flag("mode-prof", "Enable profiling mode, one of [cpu, mem, block]").Default("").String()
	pathProf          = kingpin.Flag("path-prof", "Set the Path to write profiling file").Default("").String()
)

const (
	version = "1.1.0 - c9af859"
)

func main() {
	kingpin.Version(version)
	kingpin.Parse()
	logging.LogStd(fmt.Sprintf("Starting firehose-to-syslog %s ", version), true)

	logging.SetupLogging(*syslogServer, *debug)

	c := cfclient.Config{
		ApiAddress:        *apiEndpoint,
		Username:          *user,
		Password:          *password,
		SkipSslValidation: *skipSSLValidation,
	}
	cfClient := cfclient.NewClient(&c)

	if len(*dopplerEndpoint) > 0 {
		cfClient.Endpoint.DopplerEndpoint = *dopplerEndpoint
	}
	logging.LogStd(fmt.Sprintf("Using %s as doppler endpoint", cfClient.Endpoint.DopplerEndpoint), true)

	logging.LogStd("Setting up event routing!", true)
	err := events.SetupEventRouting(*wantedEvents)
	if err != nil {
		log.Fatal("Error setting up event routing: ", err)
		os.Exit(1)

	}

	//Use bolt for in-memory  - file caching
	db, err := bolt.Open(*boltDatabasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		log.Fatal("Error opening bolt db: ", err)
		os.Exit(1)

	}
	defer db.Close()

	if *modeProf != "" {
		switch *modeProf {
		case "cpu":
			defer profile.Start(profile.CPUProfile, profile.ProfilePath(*pathProf)).Stop()
		case "mem":
			defer profile.Start(profile.MemProfile, profile.ProfilePath(*pathProf)).Stop()
		case "block":
			defer profile.Start(profile.BlockProfile, profile.ProfilePath(*pathProf)).Stop()
		default:
			// do nothing
		}
	}

	caching.SetCfClient(cfClient)
	caching.SetAppDb(db)
	caching.CreateBucket()

	//Let's Update the database the first time
	logging.LogStd("Start filling app/space/org cache.", true)
	apps := caching.GetAllApp()
	logging.LogStd(fmt.Sprintf("Done filling cache! Found [%d] Apps", len(apps)), true)

	// Ticker Pooling the CC every X sec
	ccPooling := time.NewTicker(*tickerTime)

	go func() {
		for range ccPooling.C {
			apps = caching.GetAllApp()
		}
	}()

	// Parse extra fields from cmd call
	extraFields, err := extrafields.ParseExtraFields(*extraFields)
	if err != nil {
		log.Fatal("Error parsing extra fields: ", err)
		os.Exit(1)
	}

	if *showEventTotals == true {
		logEventTotals()
	}

	if logging.Connect() || *debug {

		logging.LogStd("Connected to Syslog Server! Connecting to Firehose...", true)

		firehose := firehose.CreateFirehoseChan(cfClient.Endpoint.DopplerEndpoint, cfClient.GetToken(), *subscriptionId, *skipSSLValidation)
		if firehose != nil {
			logging.LogStd("Firehose Subscription Succesfull! Routing events...", true)
			events.RouteEvents(firehose, extraFields)
		} else {
			logging.LogError("Failed connecting to Firehose...Please check settings and try again!", "")
		}

	} else {
		logging.LogError("Failed connecting to the Syslog Server...Please check settings and try again!", "")
	}
}

func getEventTotals() string {
	var s string
	s = fmt.Sprintln(s,
		"\nWe have processed", events.GetTotalCountOfSelectedEvents(), "total events from the firehose at",
		*dopplerEndpoint)
	fmt.Sprintln(s, "\nOf those events, we have processed\n")
	for event, count := range events.GetSelectedEventsCount() {
		fmt.Sprintln(s, count, event, "events\n")
	}
	return s
}

func logEventTotals() {
	dur, err := time.ParseDuration("10s")
	if err != nil {
		log.Fatal("Error parsing duration for firehose status check: ", err)
		os.Exit(1)
	}

	firehoseEventTotals := time.NewTicker(dur)

	go func() {
		for range firehoseEventTotals.C {
			logging.LogStd(getEventTotals(), true)
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/confluentinc/confluent-kafka-go/kafka"
	log "github.com/sirupsen/logrus"
)

func main() {
	if err := rootCmd.Execute(); err != nil {
		log.Fatal(err)
	}
}

func run() {
	app, err := NewApplication()
	if err != nil {
		log.Fatal(err)
	}

	// create a producer
	producer, err := kafka.NewProducer(&kafka.ConfigMap{"bootstrap.servers": strings.Join(app.Brokers(), ",")})
	if err != nil {
		app.Log.Fatal("Unable to create producer:", err)
	}

	// generic logging for producer
	go func() {
		for e := range producer.Events() {
			switch ev := e.(type) {
			case *kafka.Message:
				m := ev
				if m.TopicPartition.Error != nil {
					app.Log.Fatal("delivery failed", m.TopicPartition.Error)
				} else {
					app.Log.Debugf("Delivered message to topic %s [%d] at offset %v",
						*m.TopicPartition.Topic, m.TopicPartition.Partition, m.TopicPartition.Offset)
				}
				return
			default:
				app.Log.Debugf("Ignored event: %s", ev)
			}
		}
	}()

	// create a proxy to let functions write data back to kafka
	writeProxy, err := NewWriteProxy(app, producer.ProduceChannel())
	if err != nil {
		app.Log.Fatal(err)
	}
	go http.ListenAndServe(app.GetString("write_proxy_listen"), writeProxy)
	app.Log.Infof("Write Proxy Started: %s", app.GetString("write_proxy_listen"))

	consumer, err := kafka.NewConsumer(&kafka.ConfigMap{
		"bootstrap.servers":               strings.Join(app.Brokers(), ","),
		"group.id":                        app.ConsumerGroupID(),
		"session.timeout.ms":              6000,
		"go.application.rebalance.enable": true,
		"default.topic.config":            kafka.ConfigMap{"auto.offset.reset": "earliest"},
		"metadata.max.age.ms":             1000,
		"enable.auto.commit":              false,
	})
	if err != nil {
		app.Log.Fatal("Failed to create consumer", err)
	}
	defer consumer.Close()

	err = consumer.SubscribeTopics([]string{app.Subscription()}, nil)
	if err != nil {
		app.Log.Fatal("Failed to subscribe topics %v, %s", app.Subscription(), err)
	}
	app.Log.Infof("Consumer Started")

	// periodically refresh metadata to know if there's any new topic created
	go refreshMetadata(consumer, app.Log)

	// start the consumer loop
	for {
		ev := consumer.Poll(100)
		if ev == nil {
			continue
		}
		switch e := ev.(type) {
		case kafka.AssignedPartitions:
			app.Log.Info(e)
			consumer.Assign(e.Partitions)
		case kafka.RevokedPartitions:
			app.Log.Info(e)
			consumer.Unassign()
		case *kafka.Message:
			app.Log.Debugf("Message Received %s", e.TopicPartition)

			parts := strings.Split(*e.TopicPartition.Topic, "_")
			object := parts[1]

			routePath := GetRouteEvent(object, "UPDATE")
			app.Log.Debugf("Looking up route %s", routePath)

			if _, ok := app.Routes[routePath]; !ok {
				app.Log.Debugf("Route not found")
				continue
			}

			err := app.Invoke(app.Routes[routePath], []byte(string(e.Value)))

			if err != nil {
				app.Log.WithField("route", app.Routes[routePath]).Errorf("Invocation Error: %v", err)
				continue
			}
			_, err = consumer.CommitMessage(e)
			if err != nil {
				app.Log.Errorf("Commit Error: %v %v", e, err)
			}
		case kafka.PartitionEOF:
			app.Log.Debugf("Reached %v", e)
		case kafka.Error:
			app.Log.Fatalf("Consumer Error %s", e)
		default:
			app.Log.Debugf("Unknown Message %v", e)
		}
	}

}

type WriteProxy struct {
	produceChan chan *kafka.Message
	app         *Application
}

func NewWriteProxy(app *Application, produceChan chan *kafka.Message) (*WriteProxy, error) {
	proxy := WriteProxy{produceChan, app}
	return &proxy, nil
}

func (p WriteProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		p.app.Log.Error("read body failed", err)
		w.WriteHeader(400)
		return
	}
	var write WriteRequest
	err = json.Unmarshal(body, &write)
	if err != nil {
		p.app.Log.Errorf("decode body failed %v", err)
		w.WriteHeader(400)
		return
	}
	topic := fmt.Sprintf("%s_%s", p.app.GetString("app_name"), write.Object)
	for key, value := range write.Data {
		p.produceChan <- &kafka.Message{
			TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
			Key:            []byte(key),
			Value:          value,
		}
	}
	fmt.Fprintf(w, "ok")
}

type WriteRequest struct {
	Object string                     `json:"object"`
	Data   map[string]json.RawMessage `json:"data"`
}

func refreshMetadata(consumer *kafka.Consumer, logger *log.Entry) {
	for {
		time.Sleep(5 * time.Second)
		metadata, err := consumer.GetMetadata(nil, true, 100)
		if err != nil {
			// somethimes it just timed out, ignore
			logger.Warn("Unable to refresh metadata: ", err)
			continue
		}
		keys := []string{}
		for k := range metadata.Topics {
			keys = append(keys, k)
		}
		// logger.Info("metadata: ", keys)
	}
}
<commit_msg>basic function concurrency<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"time"

	"github.com/confluentinc/confluent-kafka-go/kafka"
	log "github.com/sirupsen/logrus"
)

const CONCURRENT_FUNC = 5000

func main() {
	if err := rootCmd.Execute(); err != nil {
		log.Fatal(err)
	}
}

func run() {
	app, err := NewApplication()
	if err != nil {
		log.Fatal(err)
	}

	// create a producer
	producer, err := kafka.NewProducer(&kafka.ConfigMap{"bootstrap.servers": strings.Join(app.Brokers(), ",")})
	if err != nil {
		app.Log.Fatal("Unable to create producer:", err)
	}

	// generic logging for producer
	go func() {
		for e := range producer.Events() {
			switch ev := e.(type) {
			case *kafka.Message:
				m := ev
				if m.TopicPartition.Error != nil {
					app.Log.Fatal("delivery failed", m.TopicPartition.Error)
				} else {
					app.Log.Debugf("Delivered message to topic %s [%d] at offset %v",
						*m.TopicPartition.Topic, m.TopicPartition.Partition, m.TopicPartition.Offset)
				}
				return
			default:
				app.Log.Debugf("Ignored event: %s", ev)
			}
		}
	}()

	// create a proxy to let functions write data back to kafka
	writeProxy, err := NewWriteProxy(app, producer.ProduceChannel())
	if err != nil {
		app.Log.Fatal(err)
	}
	go http.ListenAndServe(app.GetString("write_proxy_listen"), writeProxy)
	app.Log.Infof("Write Proxy Started: %s", app.GetString("write_proxy_listen"))

	consumer, err := kafka.NewConsumer(&kafka.ConfigMap{
		"bootstrap.servers":               strings.Join(app.Brokers(), ","),
		"group.id":                        app.ConsumerGroupID(),
		"session.timeout.ms":              6000,
		"go.application.rebalance.enable": true,
		"default.topic.config":            kafka.ConfigMap{"auto.offset.reset": "earliest"},
		"metadata.max.age.ms":             1000,
		"enable.auto.commit":              false,
	})
	if err != nil {
		app.Log.Fatal("Failed to create consumer", err)
	}

	// TODO: lock this or use goroutine
	lastProcessedMessages := make(map[string]*kafka.Message)
	defer func() {
		for _, msg := range lastProcessedMessages {
			_, err := consumer.CommitMessage(msg)
			if err != nil {
				app.Log.Fatal(err)
			}
		}
		consumer.Close()
	}()

	err = consumer.SubscribeTopics([]string{app.Subscription()}, nil)
	if err != nil {
		app.Log.Fatal("Failed to subscribe topics %v, %s", app.Subscription(), err)
	}
	app.Log.Infof("Consumer Started")

	// close consumers when user press ctrl+c
	run := true
	c := make(chan os.Signal)
	signal.Notify(c, os.Interrupt)
	go func() {
		for _ = range c {
			fmt.Println("Exiting...")
			run = false
		}
	}()

	// periodically refresh metadata to know if there's any new topic created
	go refreshMetadata(consumer, app.Log)

	runFunc := make(chan *kafka.Message, CONCURRENT_FUNC)

	// start the consumer loop
	for run {
		ev := consumer.Poll(100)
		if ev == nil {
			continue
		}
		switch e := ev.(type) {
		case kafka.AssignedPartitions:
			app.Log.Info(e)
			consumer.Assign(e.Partitions)
		case kafka.RevokedPartitions:
			app.Log.Info(e)
			consumer.Unassign()
		case *kafka.Message:
			app.Log.Infof("Message Received %s", e.TopicPartition)

			parts := strings.Split(*e.TopicPartition.Topic, "_")
			object := parts[1]

			routePath := GetRouteEvent(object, "UPDATE")
			app.Log.Debugf("Looking up route %s", routePath)

			if _, ok := app.Routes[routePath]; !ok {
				app.Log.Debugf("Route not found")
				lastProcessedMessages[*e.TopicPartition.Topic] = e
				continue
			}

			// block when exceed CONCURRENT_FUNC
			runFunc <- e

			go func() {
				msg := <-runFunc
				parts := strings.Split(*msg.TopicPartition.Topic, "_")
				object := parts[1]

				routePath := GetRouteEvent(object, "UPDATE")

				err := app.Invoke(app.Routes[routePath], []byte(string(msg.Value)))

				if err != nil {
					app.Log.WithField("route", app.Routes[routePath]).Errorf("Invocation Error: %v", err)
					return
				}
				_, err = consumer.CommitMessage(msg)
				if err != nil {
					app.Log.Errorf("Commit Error: %v %v", msg, err)
					return
				}
				lastProcessedMessages[*e.TopicPartition.Topic] = e
			}()
		case kafka.PartitionEOF:
			app.Log.Debugf("Reached %v", e)
		case kafka.Error:
			app.Log.Fatalf("Consumer Error %s", e)
		default:
			app.Log.Debugf("Unknown Message %v", e)
		}
	}
}

type WriteProxy struct {
	produceChan chan *kafka.Message
	app         *Application
}

func NewWriteProxy(app *Application, produceChan chan *kafka.Message) (*WriteProxy, error) {
	proxy := WriteProxy{produceChan, app}
	return &proxy, nil
}

func (p WriteProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		p.app.Log.Error("read body failed", err)
		w.WriteHeader(400)
		return
	}
	var write WriteRequest
	err = json.Unmarshal(body, &write)
	if err != nil {
		p.app.Log.Errorf("decode body failed %v", err)
		w.WriteHeader(400)
		return
	}
	topic := fmt.Sprintf("%s_%s", p.app.GetString("app_name"), write.Object)
	for key, value := range write.Data {
		p.produceChan <- &kafka.Message{
			TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
			Key:            []byte(key),
			Value:          value,
		}
	}
	fmt.Fprintf(w, "ok")
}

type WriteRequest struct {
	Object string                     `json:"object"`
	Data   map[string]json.RawMessage `json:"data"`
}

func refreshMetadata(consumer *kafka.Consumer, logger *log.Entry) {
	for {
		time.Sleep(5 * time.Second)
		metadata, err := consumer.GetMetadata(nil, true, 100)
		if err != nil {
			// somethimes it just timed out, ignore
			logger.Warn("Unable to refresh metadata: ", err)
			continue
		}
		keys := []string{}
		for k := range metadata.Topics {
			keys = append(keys, k)
		}
		// logger.Info("metadata: ", keys)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/docker/go-plugins-helpers/volume"
)

var (
	Version string
	Build   string
)

var (
	version  = flag.Bool("version", false, "Version of docker-volume-location-plugin")
	location = flag.String("location", "/mnt/docker-volumes", "Where to save docker-volumes")
)

func main() {
	flag.Parse()

	if *version {
		fmt.Printf("hello-http-go version: %s, build: %s\n", Version, Build)
		os.Exit(0)
	}

	d := newVolumeLocationDriver(location)
	h := volume.NewHandler(d)
	fmt.Println(h.ServeUnix("root", "vlp"))
}
<commit_msg>Corrected application name in -version output<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/docker/go-plugins-helpers/volume"
)

var (
	Version string
	Build   string
)

var (
	version  = flag.Bool("version", false, "Version of docker-volume-location-plugin")
	location = flag.String("location", "/mnt/docker-volumes", "Where to save docker-volumes")
)

func main() {
	flag.Parse()

	if *version {
		fmt.Printf("docker-volume-location-plugin version: %s, build: %s\n", Version, Build)
		os.Exit(0)
	}

	d := newVolumeLocationDriver(location)
	h := volume.NewHandler(d)
	fmt.Println(h.ServeUnix("root", "vlp"))
}
<|endoftext|>


<commit_before>// https://rsdn.ru/article/baseserv/pe_coff.xml
// http://www.godevtool.com/Other/pdb.htm
// https://tech.yandex.ru/disk/doc/dg/reference/move-docpage/
package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/bozaro/pdbuploader/parse"
	"github.com/bozaro/pdbuploader/uploader"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

const (
	maxRetryCount     int = 5
	initialRetryDelay int = 100
)

type requestProvider func() (*http.Request, error)

type Uploader struct {
	client     *http.Client
	reqFactory uploader.RequestFactory
}

func NewUploader(client *http.Client, reqFactory uploader.RequestFactory) Uploader {
	return Uploader{
		client,
		reqFactory,
	}
}

func (this Uploader) doRequest(provider requestProvider) (*http.Response, error) {
	retryDelay := initialRetryDelay
	for pass := 0; ; pass++ {
		request, err := provider()
		if err != nil {
			return nil, err
		}
		response, err := this.client.Do(request)
		if (err == nil) && (response.StatusCode < 500 || response.StatusCode >= 600) {
			return response, nil
		}
		if pass >= maxRetryCount {
			return response, err
		}
		time.Sleep(time.Duration(retryDelay) * time.Millisecond)
		retryDelay *= 2
	}
}

func parentUrl(u *url.URL) *url.URL {
	if u == nil || u.Path == "" || u.Path == "/" {
		return nil
	}
	r := *u
	r.Path = path.Dir(r.Path)
	return &r
}

func (this Uploader) MkDir(u *url.URL) error {
	if u.Path == "" || u.Path == "/" {
		return errors.New(fmt.Sprintf("Can't create root path: %s", u.String()))
	}
	requestFunc := func() (*http.Request, error) {
		return this.reqFactory("MKCOL", u.String(), nil)
	}
	response, err := this.doRequest(requestFunc)
	if err != nil {
		return err
	}
	// Already created
	if response.StatusCode == 405 {
		return nil
	}
	if response.StatusCode == 409 {
		if err := this.MkDir(parentUrl(u)); err != nil {
			return err
		}
		response, err = this.doRequest(requestFunc)
		if err != nil {
			return err
		}
	}
	// Successfully created
	if response.StatusCode == 201 {
		return nil
	}
	return errors.New(fmt.Sprintf("Unexpected MKCOL status code %d [%s]: %s", response.StatusCode, response.Status, u.String()))
}

func (this Uploader) UploadFile(u *url.URL, content uploader.ContentProvider) error {
	if strings.HasSuffix(u.Path, "/") {
		return errors.New(fmt.Sprintf("Invalid target URL. Require file name in path: %s", u.String()))
	}
	requestFunc := func() (*http.Request, error) {
		reader, err := content.GetReader()
		if err != nil {
			return nil, err
		}
		req, err := this.reqFactory("PUT", u.String(), reader)
		if err != nil {
			return nil, err
		}
		req.Header.Set("Overwrite", "F") // todo
		return req, nil
	}
	response, err := this.doRequest(requestFunc)
	if err != nil {
		return err
	}
	if response.StatusCode == 409 {
		if err := this.MkDir(parentUrl(u)); err != nil {
			return err
		}
		response, err = this.doRequest(requestFunc)
		if err != nil {
			return err
		}
	}
	// Successfully uploaded
	if response.StatusCode == 201 {
		return nil
	}
	return errors.New(fmt.Sprintf("Unexpected PUT status code %d [%s]: %s", response.StatusCode, response.Status, u.String()))
}

func upload(reqFactory uploader.RequestFactory) {
	http_uploader := NewUploader(http.DefaultClient, reqFactory)

	u, _ := url.Parse("https://webdav.yandex.ru/PDB/foo/bar/blah.txt")
	err := http_uploader.UploadFile(u, uploader.NewBytesContentProvider([]byte("Example")))
	fmt.Println(err)
}

func main() {

	usernamePtr := flag.String("username", "bozaro", "Username")
	passwordPtr := flag.String("password", "", "Password")
	flag.Parse()

	upload(uploader.NewBasicRequestFactory(*usernamePtr, *passwordPtr))

	file, _ := os.Open("sample/hello.exe")
	{
		info, err := parse.ParseExe(file)
		fmt.Printf("EXE: %s\n", err)
		if err == nil {
			fmt.Printf("  Code ID: %s\n", info.CodeId)
			fmt.Printf("  Debug ID: %s\n", info.DebugId)
			fmt.Printf("  PDB: %s\n", info.PDBFileName)
		}
	}
	file, _ = os.Open("sample/hello.pdb")
	{
		debug_id, err := parse.ParsePdb(file)
		fmt.Printf("PDB: %s\n", err)
		if err == nil {
			fmt.Printf("  Debug ID: %s\n", debug_id)
		}
	}
}
<commit_msg>Add overwrite flag<commit_after>// https://rsdn.ru/article/baseserv/pe_coff.xml
// http://www.godevtool.com/Other/pdb.htm
// https://tech.yandex.ru/disk/doc/dg/reference/move-docpage/
package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/bozaro/pdbuploader/parse"
	"github.com/bozaro/pdbuploader/uploader"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

const (
	maxRetryCount int = 5
	initialRetryDelay int = 100
)

type requestProvider func() (*http.Request, error)

type Uploader struct {
	SupportPutOverwrite bool
	client              *http.Client
	reqFactory          uploader.RequestFactory
}

func NewUploader(client *http.Client, reqFactory uploader.RequestFactory) Uploader {
	return Uploader{
		false,
		client,
		reqFactory,
	}
}

func (this Uploader) doRequest(provider requestProvider) (*http.Response, error) {
	retryDelay := initialRetryDelay
	for pass := 0;; pass++ {
		request, err := provider()
		if err != nil {
			return nil, err
		}
		response, err := this.client.Do(request)
		if (err == nil) && (response.StatusCode < 500 || response.StatusCode >= 600) {
			return response, nil
		}
		if pass >= maxRetryCount {
			return response, err
		}
		time.Sleep(time.Duration(retryDelay) * time.Millisecond)
		retryDelay *= 2
	}
}

func parentUrl(u *url.URL) *url.URL {
	if u == nil || u.Path == "" || u.Path == "/" {
		return nil
	}
	r := *u
	r.Path = path.Dir(r.Path)
	return &r
}

func (this Uploader) MkDir(u *url.URL) error {
	if u.Path == "" || u.Path == "/" {
		return errors.New(fmt.Sprintf("Can't create root path: %s", u.String()))
	}
	requestFunc := func() (*http.Request, error) {
		return this.reqFactory("MKCOL", u.String(), nil)
	}
	response, err := this.doRequest(requestFunc)
	if err != nil {
		return err
	}
	// Already created
	if response.StatusCode == 405 {
		return nil
	}
	if response.StatusCode == 409 {
		if err := this.MkDir(parentUrl(u)); err != nil {
			return err
		}
		response, err = this.doRequest(requestFunc)
		if err != nil {
			return err
		}
	}
	// Successfully created
	if response.StatusCode == 201 {
		return nil
	}
	return errors.New(fmt.Sprintf("Unexpected MKCOL status code %d [%s]: %s", response.StatusCode, response.Status, u.String()))
}

func (this Uploader) UploadFile(u *url.URL, content uploader.ContentProvider, overwrite bool) error {
	if strings.HasSuffix(u.Path, "/") {
		return errors.New(fmt.Sprintf("Invalid target URL. Require file name in path: %s", u.String()))
	}
	if !(overwrite && this.SupportPutOverwrite) {
		response, err := this.doRequest(func() (*http.Request, error) {
			return this.reqFactory("HEAD", u.String(), nil)
		})
		if err != nil {
			return err
		}
		switch response.StatusCode {
		// File already exists
		case 200:
			return nil
		// Not found
		case 404:
			break
		default:
			return errors.New(fmt.Sprintf("Unexpected HEAD status code %d [%s]: %s", response.StatusCode, response.Status, u.String()))
		}
	}
	requestFunc := func() (*http.Request, error) {
		reader, err := content.GetReader()
		if err != nil {
			return nil, err
		}
		req, err := this.reqFactory("PUT", u.String(), reader)
		if err != nil {
			return nil, err
		}
		if !overwrite {
			req.Header.Set("Overwrite", "F")
		}
		return req, nil
	}
	response, err := this.doRequest(requestFunc)
	if err != nil {
		return err
	}
	if response.StatusCode == 409 {
		if err := this.MkDir(parentUrl(u)); err != nil {
			return err
		}
		response, err = this.doRequest(requestFunc)
		if err != nil {
			return err
		}
	}
	// Successfully uploaded
	if response.StatusCode == 201 {
		return nil
	}
	return errors.New(fmt.Sprintf("Unexpected PUT status code %d [%s]: %s", response.StatusCode, response.Status, u.String()))
}

func upload(reqFactory uploader.RequestFactory) {
	http_uploader := NewUploader(http.DefaultClient, reqFactory)

	u, _ := url.Parse("https://webdav.yandex.ru/PDB/foo/bar/blah.txt")
	err := http_uploader.UploadFile(u, uploader.NewBytesContentProvider([]byte("Example")), false)
	fmt.Println(err)
}

func main() {

	usernamePtr := flag.String("username", "bozaro", "Username")
	passwordPtr := flag.String("password", "", "Password")
	flag.Parse()

	upload(uploader.NewBasicRequestFactory(*usernamePtr, *passwordPtr))

	{
		file, _ := os.Open("sample/hello.exe")
		defer file.Close()

		info, err := parse.ParseExe(file)
		fmt.Printf("EXE: %s\n", err)
		if err == nil {
			fmt.Printf("  Code ID: %s\n", info.CodeId)
			fmt.Printf("  Debug ID: %s\n", info.DebugId)
			fmt.Printf("  PDB: %s\n", info.PDBFileName)
		}
	}
	{
		file, _ := os.Open("sample/hello.pdb")
		defer file.Close()

		debug_id, err := parse.ParsePdb(file)
		fmt.Printf("PDB: %s\n", err)
		if err == nil {
			fmt.Printf("  Debug ID: %s\n", debug_id)
		}
	}
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/line/line-bot-sdk-go/linebot"
)

var bot *linebot.Client

func main() {
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}

	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.ImageMessage:
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewImageMessage("http://wallpaper-gallery.net/images/image/image-13.jpg", "photoMsg")).Do(); err != nil {
					log.Print("Photo message")
				}
			case *linebot.TextMessage:
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(message.ID+":"+message.Text+" OK!!!!!")).Do(); err != nil {
					log.Print(err)
				}

			}
		}
	}
}
<commit_msg>aaaaaa<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/line/line-bot-sdk-go/linebot"
)

var bot *linebot.Client

func main() {
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}

	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.ImageMessage:
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewImageMessage("http://wallpaper-gallery.net/images/image/image-13.jpg", "photoMsg")).Do(); err != nil {
					log.Print("Photo message1")
				}
			case *linebot.TextMessage:
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(message.ID+":"+message.Text+" OK!!!!!")).Do(); err != nil {
					log.Print(err)
				}

			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	_ "io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/daviddengcn/go-colortext"
	"github.com/korovkin/limiter"
	_ "github.com/korovkin/worker"
)

type logger struct {
	ticket int
}

var (
	loggerMutex = new(sync.Mutex)
	loggerIndex = uint32(0)
)

var colors = []ct.Color{
	ct.Green,
	ct.Cyan,
	ct.Magenta,
	ct.Yellow,
	ct.Blue,
	ct.Red,
}

func (l *logger) Write(p []byte) (int, error) {
	buf := bytes.NewBuffer(p)
	wrote := 0
	for {
		line, err := buf.ReadBytes('\n')
		if len(line) > 1 {
			now := time.Now().Format("15:04:05")
			s := string(line)

			loggerMutex.Lock()
			ct.ChangeColor(colors[l.ticket%len(colors)], false, ct.None, false)
			fmt.Printf("%s %d | ", now, l.ticket)
			ct.ResetColor()
			fmt.Print(s)
			loggerMutex.Unlock()

			wrote += len(line)
		}
		if err != nil {
			break
		}
	}
	if len(p) > 0 && p[len(p)-1] != '\n' {
		fmt.Println()
	}

	return len(p), nil
}

func newLogger(ticket int) *logger {
	loggerMutex.Lock()
	defer loggerMutex.Unlock()
	l := &logger{ticket}
	return l
}

func executeCommand(ticket int, cmdLine string) bool {
	T_START := time.Now()
	logger := newLogger(ticket)

	defer func() {
		fmt.Fprintf(logger, "ticket: done: dt:"+time.Since(T_START).String()+"\n")
	}()

	cs := []string{"/bin/sh", "-c", cmdLine}
	cmd := exec.Command(cs[0], cs[1:]...)
	cmd.Stdin = nil
	cmd.Stdout = logger
	cmd.Stderr = logger
	cmd.Env = append(
		os.Environ(),
		fmt.Sprintf("PARALLEL_TICKER=%d", ticket),
	)

	fmt.Fprintf(logger, "ticket: run: '"+cmdLine+"'\n")

	err := cmd.Start()
	if err != nil {
		log.Fatalln("failed to start:", err)
		return true
	}

	err = cmd.Wait()
	return true
}

func main() {
	T_START := time.Now()
	logger := newLogger(0)
	defer func() {
		fmt.Fprintf(logger, "DONE: DT: "+time.Since(T_START).String()+"\n")
	}()

	flag_jobs := flag.Int(
		"j",
		2,
		"num of concurrent jobs")

	flag.Parse()
	fmt.Fprintf(logger, fmt.Sprintf("concurrency limit: %d", *flag_jobs))
	worker := limiter.NewConcurrencyLimiter(*flag_jobs)

	r := bufio.NewReaderSize(os.Stdin, 1*1024*1024)
	fmt.Fprintf(logger, "reading from stdin...\n")

	for {
		line, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}
		line = strings.TrimSpace(line)
		worker.ExecuteWithTicket(func(ticket int) {
			executeCommand(ticket, line)
		})
	}

	worker.Wait()
}
<commit_msg>add elapsed time<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	_ "io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/daviddengcn/go-colortext"
	"github.com/korovkin/limiter"
	_ "github.com/korovkin/worker"
)

type logger struct {
	ticket int
}

var (
	loggerMutex     = new(sync.Mutex)
	loggerIndex     = uint32(0)
	loggerStartTime = time.Now()
)

var loggerColors = []ct.Color{
	ct.Green,
	ct.Cyan,
	ct.Magenta,
	ct.Yellow,
	ct.Blue,
	ct.Red,
}

func (l *logger) Write(p []byte) (int, error) {
	buf := bytes.NewBuffer(p)
	wrote := 0
	for {
		line, err := buf.ReadBytes('\n')
		if len(line) > 1 {
			now := time.Now().Format("15:04:05")
			s := string(line)

			loggerMutex.Lock()
			ct.ChangeColor(loggerColors[l.ticket%len(loggerColors)], false, ct.None, false)
			fmt.Printf("[%14s %s %d] ", time.Since(loggerStartTime).String(), now, l.ticket)
			ct.ResetColor()
			fmt.Print(s)
			loggerMutex.Unlock()

			wrote += len(line)
		}
		if err != nil {
			break
		}
	}
	if len(p) > 0 && p[len(p)-1] != '\n' {
		fmt.Println()
	}

	return len(p), nil
}

func newLogger(ticket int) *logger {
	loggerMutex.Lock()
	defer loggerMutex.Unlock()
	l := &logger{ticket}
	return l
}

func executeCommand(ticket int, cmdLine string) bool {
	T_START := time.Now()
	logger := newLogger(ticket)

	defer func() {
		fmt.Fprintf(logger, "done: dt: "+time.Since(T_START).String()+"\n")
	}()

	cs := []string{"/bin/sh", "-c", cmdLine}
	cmd := exec.Command(cs[0], cs[1:]...)
	cmd.Stdin = nil
	cmd.Stdout = logger
	cmd.Stderr = logger
	cmd.Env = append(
		os.Environ(),
		fmt.Sprintf("PARALLEL_TICKER=%d", ticket),
	)

	fmt.Fprintf(logger, "run: '"+cmdLine+"'\n")

	err := cmd.Start()
	if err != nil {
		log.Fatalln("failed to start:", err)
		return true
	}

	err = cmd.Wait()
	return true
}

func main() {
	T_START := time.Now()
	logger := newLogger(0)
	defer func() {
		fmt.Fprintf(logger, "ALL DONE: dt: "+time.Since(T_START).String()+"\n")
	}()

	flag_jobs := flag.Int(
		"j",
		2,
		"num of concurrent jobs")

	flag.Parse()
	fmt.Fprintf(logger, fmt.Sprintf("concurrency limit: %d", *flag_jobs))
	worker := limiter.NewConcurrencyLimiter(*flag_jobs)

	r := bufio.NewReaderSize(os.Stdin, 1*1024*1024)
	fmt.Fprintf(logger, "reading from stdin...\n")

	for {
		line, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}
		line = strings.TrimSpace(line)
		worker.ExecuteWithTicket(func(ticket int) {
			executeCommand(ticket, line)
		})
	}

	worker.Wait()
}
<|endoftext|>


<commit_before>// @TODO
// - make it secure (maybe POST or JWT)

package main

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

var ()

// ServerHandler handles incoming requests.
type ServerHandler struct {
}

// CrawlingLocation keeps the location info in incoming JSON payload.
type CrawlingLocation struct {
	URL string `json:"url"`
}

// CrawlingInfoJSON is the format of the incoming JSON.
type CrawlingInfoJSON struct {
	Locations []CrawlingLocation `json:"locations"`
}

// CrawlingLocationResult keeps records of location crawl results.
type CrawlingLocationResult struct {
	URL     string            `json:"url"`
	Headers map[string]string `json:"headers"`
}

// CrawlingResponseJSON is the returned object.
type CrawlingResponseJSON struct {
	Locations []CrawlingLocationResult `json:"locations"`
}

func (sh *ServerHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	log.Println("incoming request")

	payload := &CrawlingInfoJSON{}
	b, err := ioutil.ReadAll(req.Body)
	if err != nil {
		rw.WriteHeader(400)
		io.WriteString(rw, "Unable to read payload.")
		return
	}

	if err = json.Unmarshal(b, payload); err != nil {
		rw.WriteHeader(400)
		io.WriteString(rw, "Unable to parse JSON. "+err.Error())
		return
	}

	responseJSON := &CrawlingResponseJSON{}
	for _, location := range payload.Locations {
		log.Println(location.URL)
		locationInfo := CrawlingLocationResult{URL: location.URL, Headers: make(map[string]string)}
		responseJSON.Locations = append(responseJSON.Locations, locationInfo)
	}

	jsonOut, err := json.Marshal(responseJSON)
	if err != nil {
		rw.WriteHeader(400)
		io.WriteString(rw, "Unable to generate JSON. "+err.Error())
		return
	}

	rw.WriteHeader(200)
	rw.Write(jsonOut)
}

func main() {
	log.Println("crawler has been started")

	server := &http.Server{
		// @TODO make it a configuration
		Addr:           "localhost:8080",
		Handler:        &ServerHandler{},
		ReadTimeout:    time.Second * 60,
		WriteTimeout:   time.Second * 60,
		MaxHeaderBytes: 0,
	}

	log.Println("server has been initialized")

	err := server.ListenAndServe()
	if err != nil {
		log.Panicln(err)
	}

	log.Println("listen loop has been started")

	// @todo figure out if this is effective
	for {
	}
}
<commit_msg>Add fetch return.<commit_after>// @TODO
// - make it secure (maybe POST or JWT)

package main

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

var ()

// ServerHandler handles incoming requests.
type ServerHandler struct {
}

// CrawlingLocation keeps the location info in incoming JSON payload.
type CrawlingLocation struct {
	URL string `json:"url"`
}

// CrawlingInfoJSON is the format of the incoming JSON.
type CrawlingInfoJSON struct {
	Locations []CrawlingLocation `json:"locations"`
}

// CrawlingLocationResult keeps records of location crawl results.
type CrawlingLocationResult struct {
	URL        string              `json:"url"`
	StatusCode int                 `json:"status_code"`
	Protocol   string              `json:"protocol"`
	Headers    map[string][]string `json:"headers"`
}

// CrawlingResponseJSON is the returned object.
type CrawlingResponseJSON struct {
	Locations []CrawlingLocationResult `json:"locations"`
}

func (sh *ServerHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	log.Println("incoming request")

	payload := &CrawlingInfoJSON{}
	b, err := ioutil.ReadAll(req.Body)
	if err != nil {
		rw.WriteHeader(400)
		io.WriteString(rw, "Unable to read payload.")
		return
	}

	if err = json.Unmarshal(b, payload); err != nil {
		rw.WriteHeader(400)
		io.WriteString(rw, "Unable to parse JSON. "+err.Error())
		return
	}

	responseJSON := &CrawlingResponseJSON{}
	for _, location := range payload.Locations {
		log.Println(location.URL)
		locationInfo := CrawlingLocationResult{URL: location.URL}

		resp, err := LoadURL(location.URL)
		if err != nil {
			log.Println("unable to fetch header for: " + location.URL)
		} else {
			locationInfo.Headers = resp.Header
			locationInfo.StatusCode = resp.StatusCode
			locationInfo.Protocol = resp.Proto
		}

		responseJSON.Locations = append(responseJSON.Locations, locationInfo)
	}

	jsonOut, err := json.Marshal(responseJSON)
	if err != nil {
		rw.WriteHeader(400)
		io.WriteString(rw, "Unable to generate JSON. "+err.Error())
		return
	}

	rw.WriteHeader(200)
	rw.Write(jsonOut)
}

// LoadURL loads the path and returns the header info.
func LoadURL(path string) (*http.Response, error) {
	resp, err := http.Get(path)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

func main() {
	log.Println("crawler has been started")

	server := &http.Server{
		// @TODO make it a configuration
		Addr:           "localhost:8080",
		Handler:        &ServerHandler{},
		ReadTimeout:    time.Second * 30,
		WriteTimeout:   time.Second * 30,
		MaxHeaderBytes: 0,
	}

	log.Println("server has been initialized")

	err := server.ListenAndServe()
	if err != nil {
		log.Panicln(err)
	}

	log.Println("listen loop has been started")

	// @todo figure out if this is effective
	for {
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/Felamande/jsvm"

	"github.com/codegangsta/cli"
	"github.com/peterh/liner"

	_ "github.com/Felamande/god/modules/go"
	"github.com/Felamande/god/modules/god"
	"github.com/Felamande/god/modules/hotkey"
	_ "github.com/Felamande/god/modules/log"
	_ "github.com/Felamande/jsvm/module/os"
	_ "github.com/Felamande/jsvm/module/path"
)

var version = "2.0.0-beta"

func init() {
	err := jsvm.Run("god.js")
	if err != nil {
		fmt.Println(err)
	}

}
func main() {

	app := cli.NewApp()
	app.Version = version
	app.Name = filepath.Base(os.Args[0])
	app.HelpName = app.Name
	app.Usage = "An automation tool for Go project."
	app.Action = enterAction
	app.Commands = append(app.Commands, initCmd, watchCmd, reloadCmd, exitCmd)

	for name, CbFunc := range god.SubCmd {
		app.Commands = append(app.Commands, newSubCmd(name, CbFunc))
	}

	go hotkey.ApplyAll()
	app.Run(os.Args)
}

func enterAction(c *cli.Context) {
	god.Init.Call()
	line := liner.NewLiner()
	defer line.Close()
	line.SetCtrlCAborts(false)

	historyFile, err := os.OpenFile(filepath.Join(os.TempDir(), ".god_hostory"), os.O_CREATE|os.O_RDWR, 0777)
	if err == nil {
		line.ReadHistory(historyFile)
	}

	for {
		argStr, err := line.Prompt(`(god) `)
		argStr = strings.TrimSpace(argStr)
		if len(argStr) == 0 {
			continue
		}
		line.AppendHistory(argStr)
		if err == liner.ErrPromptAborted {
			os.Exit(0)
		} else if err != nil {
			fmt.Println(err)
			continue
		}
		splits := strings.Split(argStr, " ")
		var args = append([]string{}, "god")
		for _, s := range splits {
			if s != "" {
				args = append(args, s)
			}
		}
		if len(args) == 0 {
			continue
		}
		if c.App.Command(args[1]) == nil {
			continue
		}

		c.App.Run(args)

	}

}

func newSubCmd(name string, CbFunc jsvm.Func) cli.Command {
	return cli.Command{
		Name: name,
		Action: func(c *cli.Context) {
			arg := c.Args()
			flags, nargs, err := parseArgs(arg, "-")
			if err != nil {
				fmt.Println(name+": ", err)
				return
			}
			CbFunc.Call(nargs, flags)
		},
		Usage: "user defined command " + name,

		SkipFlagParsing: true,
	}
}
<commit_msg>use a manager struct to manage subcommands<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"gopkg.in/fsnotify.v1"

	"github.com/Felamande/god/lib/jsvm"

	"github.com/codegangsta/cli"
	"github.com/peterh/liner"

	_ "github.com/Felamande/god/modules/go"
	"github.com/Felamande/god/modules/god"
	"github.com/Felamande/god/modules/hotkey"
	_ "github.com/Felamande/god/modules/log"
	_ "github.com/Felamande/god/modules/os"
	_ "github.com/Felamande/god/modules/path"
)

var version = "2.0.0-beta2"

//Cmder command manager
type Cmder struct {
	line      *liner.State
	history   *os.File
	w         *fsnotify.Watcher
	stopWChan chan bool
	watchm    *sync.Mutex
	wd        string

	global map[string]string
}

var cmder = &Cmder{
	stopWChan: make(chan bool, 1),
	watchm:    new(sync.Mutex),
	global:    make(map[string]string),
}

func init() {
	err := jsvm.Run("god.js")
	if err != nil {
		fmt.Println(err)
	}
	cmder.global["ps1"] = "(god) "
}
func main() {

	app := cli.NewApp()
	app.Version = version
	app.Name = filepath.Base(os.Args[0])
	app.HelpName = app.Name
	app.Usage = "An automation tool for Go project."
	app.Action = cmder.enterAction
	app.Commands = append(app.Commands, initCmd, watchCmd, reloadCmd, exitCmd, historyCmd)

	for name, CbFunc := range god.SubCmd {
		app.Commands = append(app.Commands, newSubCmd(name, CbFunc))
	}

	err := hotkey.Bind("ctrl+d", func() { cmder.exit(nil) })
	if err != nil {
		fmt.Println(err)

	}
	err = hotkey.Bind("ctrl+alt+p", func() { cmder.unwatch(nil) })
	if err != nil {
		fmt.Println(err)
	}

	go hotkey.ApplyAll()
	app.Run(os.Args)
}

func (c *Cmder) enterAction(ctx *cli.Context) {

	god.Init.Call()
	line := liner.NewLiner()
	defer line.Close()
	line.SetCtrlCAborts(false)
	c.line = line

	historyFile, err := os.OpenFile(filepath.Join(os.TempDir(), ".god_hostory"), os.O_CREATE|os.O_RDWR, 0777)

	if err != nil {
		fmt.Println(err)

	}
	c.history = historyFile
	line.ReadHistory(historyFile)

	for {
		argStr, err := line.Prompt(c.global["ps1"])
		argStr = strings.TrimSpace(argStr)
		if len(argStr) == 0 {
			continue
		}
		line.AppendHistory(argStr)
		if err == liner.ErrPromptAborted {
			os.Exit(0)
		} else if err != nil {
			fmt.Println(err)
			continue
		}

		splits := strings.Split(argStr, " ")
		var args = append([]string{}, "god")
		for _, s := range splits {
			if s != "" {
				args = append(args, s)
			}
		}
		if len(args) == 0 {
			continue
		}
		if ctx.App.Command(args[1]) == nil {
			continue
		}

		ctx.App.Run(args)

	}

}

func newSubCmd(name string, CbFunc jsvm.Func) cli.Command {
	return cli.Command{
		Name: name,
		Action: func(c *cli.Context) {
			arg := c.Args()
			flags, nargs, err := parseArgs(arg, "-")
			if err != nil {
				fmt.Println(name+": ", err)
				return
			}
			CbFunc.Call(nargs, flags)
		},
		Usage: "user defined command " + name,

		SkipFlagParsing: true,
	}
}
<|endoftext|>


<commit_before>package main
import (
  "flag"
  "os"
  "fmt"
  "strings"
  "strconv"
  "bytes"
  "github.com/parnurzeal/gorequest"
  "encoding/json"
  "appstore-hipchat-integration/models"
  "github.com/tbruyelle/hipchat-go/hipchat"
  "github.com/eknkc/amber"
)

var (
  token  = flag.String("token", "", "The HipChat AuthToken")
  roomId = flag.String("room", "", "The HipChat room id")
  test   = flag.Bool("t", false, "Enable auth_test parameter")
)

type Review struct {
  Title string
  Author string
  Stars []string
  Message string
}

func getJson() models.AutoGenerated {
  url := "https://itunes.apple.com/au/rss/customerreviews/id=1029783189/page=1/json"
  resp, body, err := gorequest.New().Get(url).End()
  fmt.Println(resp.Status)
  if err != nil {
  }
  feed := models.AutoGenerated{}
  json.Unmarshal([]byte(body), &feed)

  return feed;
}

func main() {
  flag.Parse()
  if *token == "" || *roomId == "" {
    flag.PrintDefaults()
    return
  }
  hipchat.AuthTest = *test

  compiler := amber.New()
  // Parse the input file
  err := compiler.ParseFile("./review.amber")
  if (err != nil) {
    fmt.Println("Unable to find review.amber")
    return
  }
  // Compile input file to Go template
  tpl, err := compiler.Compile()
  if (err != nil) {
    return
  }
  feed := getJson();
  i, err := strconv.Atoi(feed.Feed.Entry[1].ImRating.Label)
  if err == nil {
    var slice = make([]string, i)
    star := "https://s.mzstatic.com/htmlResources/B69E/web-storefront/images/star@2x.png"

    for i := range slice {
      slice[i] = star
    }
    review := Review{Title: feed.Feed.Entry[1].Title.Label, Author: feed.Feed.Entry[1].Author.Name.Label, Stars: slice, Message: feed.Feed.Entry[1].Content.Label}
    // Check built in html/template documentation for further details
    var buf bytes.Buffer
    tpl.Execute(&buf, review)
    notifRq := &hipchat.NotificationRequest{Message: strings.TrimSpace(buf.String())}
    c := hipchat.NewClient(*token)
    resp, err := c.Room.Notification(*roomId, notifRq)
    if err != nil {
      fmt.Fprintf(os.Stderr, "Error during room notification %q\n", err)
      fmt.Fprintf(os.Stderr, "Server returns %+v\n", resp)
      return
    }

    if hipchat.AuthTest {
      _, ok := hipchat.AuthTestResponse["success"]
      fmt.Println("Authentification succeed :", ok)
    } else {
      fmt.Println("Sent App Reviews")
    }
  }
}
<commit_msg>Cleaning up code<commit_after>package main
import (
  "flag"
  "os"
  "fmt"
  "strings"
  "strconv"
  "bytes"
  "github.com/parnurzeal/gorequest"
  "encoding/json"
  "appstore-hipchat-integration/models"
  "github.com/tbruyelle/hipchat-go/hipchat"
  "github.com/eknkc/amber"
)

var (
  token  = flag.String("token", "", "The HipChat AuthToken")
  roomId = flag.String("room", "", "The HipChat room id")
  test   = flag.Bool("t", false, "Enable auth_test parameter")
)

type Review struct {
  Title string
  Author string
  Stars []string
  Message string
}

func getJson() models.AutoGenerated {
  url := "https://itunes.apple.com/au/rss/customerreviews/id=1029783189/page=1/json"
  resp, body, err := gorequest.New().Get(url).End()
  fmt.Println(resp.Status)
  if err != nil {
  }
  feed := models.AutoGenerated{}
  json.Unmarshal([]byte(body), &feed)

  return feed;
}

func getReviewMessage(review Review) string {
  compiler := amber.New()
  // Parse the input file
  err := compiler.ParseFile("./review.amber")
  if (err != nil) {
    fmt.Println("Unable to find review.amber")
  }
  // Compile input file to Go template
  tpl, err := compiler.Compile()
  if (err != nil) {
    fmt.Println("Problem when compiling review.amber")
  }
  var buf bytes.Buffer
  tpl.Execute(&buf, review)
  return strings.TrimSpace(buf.String())
}

func main() {
  flag.Parse()
  if *token == "" || *roomId == "" {
    flag.PrintDefaults()
    return
  }
  hipchat.AuthTest = *test

  feed := getJson();
  entry := feed.Feed.Entry[2]
  i, err := strconv.Atoi(entry.ImRating.Label)
  if err == nil {
    var slice = make([]string, i)
    star := "https://s.mzstatic.com/htmlResources/B69E/web-storefront/images/star@2x.png"

    for i := range slice {
      slice[i] = star
    }
    review := Review{Title: entry.Title.Label, Author: entry.Author.Name.Label, Stars: slice, Message: entry.Content.Label}
    // Check built in html/template documentation for further details
    message := getReviewMessage(review);
    notifRq := &hipchat.NotificationRequest{Message: message}
    c := hipchat.NewClient(*token)
    resp, err := c.Room.Notification(*roomId, notifRq)
    if err != nil {
      fmt.Fprintf(os.Stderr, "Error during room notification %q\n", err)
      fmt.Fprintf(os.Stderr, "Server returns %+v\n", resp)
      return
    }

    if hipchat.AuthTest {
      _, ok := hipchat.AuthTestResponse["success"]
      fmt.Println("Authentification succeed :", ok)
    } else {
      fmt.Println("Sent App Reviews")
    }
  }
}
<|endoftext|>


<commit_before>// Copyright 2015 Florian Hülsmann <fh@cbix.de>

package main

import (
	"fmt"
	"github.com/mediocregopher/radix.v2/redis"
	"golang.org/x/net/icmp"
	"golang.org/x/net/ipv6"
	"net"
)

const (
	AssignedPrefixLength     = 64
	OnLinkPrefixLength       = 48
	DefaultValidLifetime     = 86400
	DefaultPreferredLifetime = 14400
)

var (
	pc *ipv6.PacketConn
	db *redis.Client
)

func main() {
	var err error
	// open redis connection
	if db, err = redis.Dial("tcp", "localhost:6379"); err != nil {
		panic(err)
	}
	defer db.Close()

	// open listening connection
	conn, err := net.ListenIP("ip6:ipv6-icmp", &net.IPAddr{net.IPv6unspecified, ""})
	if err != nil {
		panic(err)
	}
	defer conn.Close()
	pc = ipv6.NewPacketConn(conn)
	// RFC4861 requires the hop limit set to 255, but the default value in golang is 64
	pc.SetHopLimit(255)

	// only accept router solicitations
	filter := new(ipv6.ICMPFilter)
	filter.SetAll(true)
	filter.Accept(ipv6.ICMPTypeRouterSolicitation)
	filter.Accept(ipv6.ICMPTypeRouterAdvertisement)
	filter.Accept(ipv6.ICMPTypeNeighborSolicitation)
	filter.Accept(ipv6.ICMPTypeNeighborAdvertisement)
	if err = pc.SetICMPFilter(filter); err != nil {
		panic(err)
	}

	// read from socket
	buf := make([]byte, 512)
	for {
		n, _, srcAddr, err := pc.ReadFrom(buf)
		if err != nil {
			panic(err)
		}
		go handleND(srcAddr, buf[:n])
	}
}

// method handleND parses arbitrary ICMPv6 messages, currently only router solicitations
func handleND(src net.Addr, body []byte) {
	t := ipv6.ICMPType(body[0])
	fmt.Printf("%v from %v\n", t, src)
	switch t {
	case ipv6.ICMPTypeRouterSolicitation:
		handleRS(src, body)
	default:
		return
	}
}

// method handleRS parses a router solicitation and eventually replies with a unicasted router advertisement
func handleRS(src net.Addr, body []byte) {
	// parse ND options
	options, err := parseOptions(body[8:])
	if err != nil {
		fmt.Println(err)
	}

	// check if any of the options is a source LLA
	var lla *NDOptionLLA = nil
	for _, o := range options {
		if o == nil {
			continue
		}
		llaopt, ok := (*o).(*NDOptionLLA)
		if !ok {
			continue
		}
		lla = llaopt
		if int(lla.OptionType) != 1 {
			continue
		}
	}
	if lla == nil {
		fmt.Println("no source LLA option given")
		return
	}
	// lookup prefix from redis
	fmt.Printf("looking up prefix for %v ... ", net.HardwareAddr(lla.Addr))
	mackey := append([]byte("fahrrad/mac/"), lla.Addr...)
	prefix, err := db.Cmd("GET", mackey).Bytes()
	if err != nil {
		//fmt.Println(err)
		// i.e. key doesn't exist
		fmt.Printf("not found: %v\n", err)
		return
	}
	if len(prefix) != 16 {
		fmt.Printf("invalid length: %x\n", prefix)
		return
	}
	fmt.Printf("found %v/64\n", net.IP(prefix))
	// ICMPv6 RA header:
	msgbody := []byte{0x40, 0x00, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}

	/*
	   According to RFC 5942, the announced prefixes for on-link usage and autoconfiguration
	   can be separate from each other (there can be an arbitrary number of advertised on-link
	   and autoconf prefixes, respectively).
	   This allows us to use /48 for the link but tell the clients to use /64 for autoconf.
	*/
	// Prefix options:
	// autoconf (assigned prefix) option
	apopt := &NDOptionPrefix{
		PrefixLength:      AssignedPrefixLength,
		OnLink:            false,
		AutoConf:          true,
		ValidLifetime:     DefaultValidLifetime,
		PreferredLifetime: DefaultPreferredLifetime,
		Prefix:            net.IP(prefix).Mask(net.CIDRMask(AssignedPrefixLength, 128)),
	}
	// onlink prefix option
	olopt := &NDOptionPrefix{
		PrefixLength:      OnLinkPrefixLength,
		OnLink:            true,
		AutoConf:          false,
		ValidLifetime:     DefaultValidLifetime,
		PreferredLifetime: DefaultPreferredLifetime,
		Prefix:            net.IP(prefix).Mask(net.CIDRMask(OnLinkPrefixLength, 128)),
	}
	apoptbytes, err := apopt.Marshal()
	if err != nil {
		// this should never happen
		fmt.Println(err)
		return
	}
	msgbody = append(msgbody, apoptbytes...)
	oloptbytes, err := olopt.Marshal()
	if err != nil {
		// this should never happen
		fmt.Println(err)
		return
	}
	msgbody = append(msgbody, oloptbytes...)

	// at this point we could include a source LLA option, but RFC 4861 doesn't require that
	// this would work by taking net.InterfaceByName(src.(*net.IPAddr).Zone) and its HardwareAddr

	// code and checksum are 0, the latter is calculated by the kernel
	// TODO: data structure for RA/ND message body
	msg := &icmp.Message{ipv6.ICMPTypeRouterAdvertisement, 0, 0, &icmp.DefaultMessageBody{msgbody}}
	mb, err := msg.Marshal(nil)
	if err != nil {
		panic(err)
	}
	// send package
	_, err = pc.WriteTo(mb, nil, src)
	if err != nil {
		panic(err)
	}
}
<commit_msg>comment<commit_after>// Copyright 2015 Florian Hülsmann <fh@cbix.de>

package main

import (
	"fmt"
	"github.com/mediocregopher/radix.v2/redis"
	"golang.org/x/net/icmp"
	"golang.org/x/net/ipv6"
	"net"
)

const (
	AssignedPrefixLength     = 64
	OnLinkPrefixLength       = 48
	DefaultValidLifetime     = 86400
	DefaultPreferredLifetime = 14400
)

var (
	pc *ipv6.PacketConn
	db *redis.Client
)

func main() {
	var err error
	// open redis connection
	if db, err = redis.Dial("tcp", "localhost:6379"); err != nil {
		panic(err)
	}
	defer db.Close()

	// open listening connection
	conn, err := net.ListenIP("ip6:ipv6-icmp", &net.IPAddr{net.IPv6unspecified, ""})
	if err != nil {
		panic(err)
	}
	defer conn.Close()
	pc = ipv6.NewPacketConn(conn)
	// RFC4861 requires the hop limit set to 255, but the default value in golang is 64
	pc.SetHopLimit(255)

	// only accept neighbor discovery messages
	filter := new(ipv6.ICMPFilter)
	filter.SetAll(true)
	filter.Accept(ipv6.ICMPTypeRouterSolicitation)
	filter.Accept(ipv6.ICMPTypeRouterAdvertisement)
	filter.Accept(ipv6.ICMPTypeNeighborSolicitation)
	filter.Accept(ipv6.ICMPTypeNeighborAdvertisement)
	if err = pc.SetICMPFilter(filter); err != nil {
		panic(err)
	}

	// read from socket
	buf := make([]byte, 512)
	for {
		n, _, srcAddr, err := pc.ReadFrom(buf)
		if err != nil {
			panic(err)
		}
		go handleND(srcAddr, buf[:n])
	}
}

// method handleND parses arbitrary ICMPv6 messages, currently only router solicitations
func handleND(src net.Addr, body []byte) {
	t := ipv6.ICMPType(body[0])
	fmt.Printf("%v from %v\n", t, src)
	switch t {
	case ipv6.ICMPTypeRouterSolicitation:
		handleRS(src, body)
	default:
		return
	}
}

// method handleRS parses a router solicitation and eventually replies with a unicasted router advertisement
func handleRS(src net.Addr, body []byte) {
	// parse ND options
	options, err := parseOptions(body[8:])
	if err != nil {
		fmt.Println(err)
	}

	// check if any of the options is a source LLA
	var lla *NDOptionLLA = nil
	for _, o := range options {
		if o == nil {
			continue
		}
		llaopt, ok := (*o).(*NDOptionLLA)
		if !ok {
			continue
		}
		lla = llaopt
		if int(lla.OptionType) != 1 {
			continue
		}
	}
	if lla == nil {
		fmt.Println("no source LLA option given")
		return
	}
	// lookup prefix from redis
	fmt.Printf("looking up prefix for %v ... ", net.HardwareAddr(lla.Addr))
	mackey := append([]byte("fahrrad/mac/"), lla.Addr...)
	prefix, err := db.Cmd("GET", mackey).Bytes()
	if err != nil {
		//fmt.Println(err)
		// i.e. key doesn't exist
		fmt.Printf("not found: %v\n", err)
		return
	}
	if len(prefix) != 16 {
		fmt.Printf("invalid length: %x\n", prefix)
		return
	}
	fmt.Printf("found %v/64\n", net.IP(prefix))
	// ICMPv6 RA header:
	msgbody := []byte{0x40, 0x00, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}

	/*
	   According to RFC 5942, the announced prefixes for on-link usage and autoconfiguration
	   can be separate from each other (there can be an arbitrary number of advertised on-link
	   and autoconf prefixes, respectively).
	   This allows us to use /48 for the link but tell the clients to use /64 for autoconf.
	*/
	// Prefix options:
	// autoconf (assigned prefix) option
	apopt := &NDOptionPrefix{
		PrefixLength:      AssignedPrefixLength,
		OnLink:            false,
		AutoConf:          true,
		ValidLifetime:     DefaultValidLifetime,
		PreferredLifetime: DefaultPreferredLifetime,
		Prefix:            net.IP(prefix).Mask(net.CIDRMask(AssignedPrefixLength, 128)),
	}
	// onlink prefix option
	olopt := &NDOptionPrefix{
		PrefixLength:      OnLinkPrefixLength,
		OnLink:            true,
		AutoConf:          false,
		ValidLifetime:     DefaultValidLifetime,
		PreferredLifetime: DefaultPreferredLifetime,
		Prefix:            net.IP(prefix).Mask(net.CIDRMask(OnLinkPrefixLength, 128)),
	}
	apoptbytes, err := apopt.Marshal()
	if err != nil {
		// this should never happen
		fmt.Println(err)
		return
	}
	msgbody = append(msgbody, apoptbytes...)
	oloptbytes, err := olopt.Marshal()
	if err != nil {
		// this should never happen
		fmt.Println(err)
		return
	}
	msgbody = append(msgbody, oloptbytes...)

	// at this point we could include a source LLA option, but RFC 4861 doesn't require that
	// this would work by taking net.InterfaceByName(src.(*net.IPAddr).Zone) and its HardwareAddr

	// code and checksum are 0, the latter is calculated by the kernel
	// TODO: data structure for RA/ND message body
	msg := &icmp.Message{ipv6.ICMPTypeRouterAdvertisement, 0, 0, &icmp.DefaultMessageBody{msgbody}}
	mb, err := msg.Marshal(nil)
	if err != nil {
		panic(err)
	}
	// send package
	_, err = pc.WriteTo(mb, nil, src)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"strings"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
	"github.com/webhippie/dockerhub_exporter/exporter"

	_ "net/http/pprof"
)

var (
	// showVersion is a flag to display the current version.
	showVersion = flag.Bool("version", false, "Print version information")

	// listenAddress defines the local address binding for the server.
	listenAddress = flag.String("web.listen-address", ":9105", "Address to listen on for web interface and telemetry")

	// metricsPath defines the path to access the metrics.
	metricsPath = flag.String("web.telemetry-path", "/metrics", "Path to expose metrics of the exporter")

	// orgs defines the organizations to export.
	orgs StringSlice

	// repos defines the repositories to export.
	repos StringSlice
)

// init registers the collector version.
func init() {
	prometheus.MustRegister(version.NewCollector("dockerhub_exporter"))
}

// main simply initializes this tool.
func main() {
	flag.Var(&orgs, "dockerhub.org", "Organizations to watch on Docker Hub")
	flag.Var(&repos, "dockerhub.repo", "Repositories to watch on Docker Hub")

	flag.Parse()

	if *showVersion {
		fmt.Fprintln(os.Stdout, version.Print("dockerhub_exporter"))
		os.Exit(0)
	}

	log.Infoln("Starting Docker Hub exporter", version.Info())
	log.Infoln("Build context", version.BuildContext())

	e := exporter.NewExporter(orgs, repos)

	prometheus.MustRegister(e)
	prometheus.Unregister(prometheus.NewGoCollector())
	prometheus.Unregister(prometheus.NewProcessCollector(os.Getpid(), ""))

	http.Handle(*metricsPath, promhttp.Handler())

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, *metricsPath, http.StatusMovedPermanently)
	})

	log.Infof("Listening on %s", *listenAddress)

	if err := http.ListenAndServe(*listenAddress, nil); err != nil {
		log.Fatal(err)
	}
}

// StringSlice represents a custom string slice flag.
type StringSlice []string

// String represents the string slice as a string.
func (ss *StringSlice) String() string {
	return strings.Join(*ss, ",")
}

// Set appends the string slice value to the current list.
func (ss *StringSlice) Set(value string) error {
	if value != "" {
		*ss = append(*ss, strings.Split(strings.Replace(value, " ", "", -1), ",")...)
	}

	return nil
}
<commit_msg>Enforce a repo or an org flag<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"strings"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
	"github.com/webhippie/dockerhub_exporter/exporter"

	_ "net/http/pprof"
)

var (
	// showVersion is a flag to display the current version.
	showVersion = flag.Bool("version", false, "Print version information")

	// listenAddress defines the local address binding for the server.
	listenAddress = flag.String("web.listen-address", ":9105", "Address to listen on for web interface and telemetry")

	// metricsPath defines the path to access the metrics.
	metricsPath = flag.String("web.telemetry-path", "/metrics", "Path to expose metrics of the exporter")

	// orgs defines the organizations to export.
	orgs StringSlice

	// repos defines the repositories to export.
	repos StringSlice
)

// init registers the collector version.
func init() {
	prometheus.MustRegister(version.NewCollector("dockerhub_exporter"))
}

// main simply initializes this tool.
func main() {
	flag.Var(&orgs, "dockerhub.org", "Organizations to watch on Docker Hub")
	flag.Var(&repos, "dockerhub.repo", "Repositories to watch on Docker Hub")

	flag.Parse()

	if *showVersion {
		fmt.Fprintln(os.Stdout, version.Print("dockerhub_exporter"))
		os.Exit(0)
	}

	if len(orgs) == 0 && len(repos) == 0 {
		fmt.Fprintln(os.Stderr, "Please provide a repository or an organization")
		os.Exit(1)
	}

	log.Infoln("Starting Docker Hub exporter", version.Info())
	log.Infoln("Build context", version.BuildContext())

	e := exporter.NewExporter(orgs, repos)

	prometheus.MustRegister(e)
	prometheus.Unregister(prometheus.NewGoCollector())
	prometheus.Unregister(prometheus.NewProcessCollector(os.Getpid(), ""))

	http.Handle(*metricsPath, promhttp.Handler())

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, *metricsPath, http.StatusMovedPermanently)
	})

	log.Infof("Listening on %s", *listenAddress)

	if err := http.ListenAndServe(*listenAddress, nil); err != nil {
		log.Fatal(err)
	}
}

// StringSlice represents a custom string slice flag.
type StringSlice []string

// String represents the string slice as a string.
func (ss *StringSlice) String() string {
	return strings.Join(*ss, ",")
}

// Set appends the string slice value to the current list.
func (ss *StringSlice) Set(value string) error {
	if value != "" {
		*ss = append(*ss, strings.Split(strings.Replace(value, " ", "", -1), ",")...)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/gorilla/context"
	"log"
	"net"
	"net/http"
	"net/http/fcgi"
	"os"
	"path"
	"strconv"
)

var serveFcgi bool
var baseDir string
var tmpDir string
var port int
var smtpServer string
var smtpPort int
var smtpUsername string
var smtpPassword string
var reminderEmail string

func init() {
	flag.StringVar(&baseDir, "base", "./", "Base directory for server")
	flag.StringVar(&tmpDir, "tmp", "/tmp", "Directory to create temporary files in")
	flag.IntVar(&port, "port", 80, "Port to serve API/files on")
	flag.StringVar(&smtpServer, "smtp.server", "smtp.example.com", "SMTP server to send reminder emails from.")
	flag.IntVar(&smtpPort, "smtp.port", 587, "SMTP server port to connect to")
	flag.StringVar(&smtpUsername, "smtp.username", "moneygo", "SMTP username")
	flag.StringVar(&smtpPassword, "smtp.password", "password", "SMTP password")
	flag.StringVar(&reminderEmail, "email", "moneygo@example.com", "Email address to send reminder emails as.")
	flag.BoolVar(&serveFcgi, "fcgi", false, "Serve via fcgi rather than http.")
	flag.Parse()

	static_path := path.Join(baseDir, "static")

	// Ensure base directory is valid
	dir_err_str := "The base directory doesn't look like it contains the " +
		"'static' directory. Check to make sure you're passing the right " +
		"value to the -base argument."
	static_dir, err := os.Stat(static_path)
	if err != nil {
		log.Print(err)
		log.Fatal(dir_err_str)
	}
	if !static_dir.IsDir() {
		log.Fatal(dir_err_str)
	}
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, path.Join(baseDir, "static/index.html"))
}

func staticHandler(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, path.Join(baseDir, r.URL.Path))
}

func main() {
	servemux := http.NewServeMux()
	servemux.HandleFunc("/", rootHandler)
	servemux.HandleFunc("/static/", staticHandler)
	servemux.HandleFunc("/session/", SessionHandler)
	servemux.HandleFunc("/user/", UserHandler)
	servemux.HandleFunc("/security/", SecurityHandler)
	servemux.HandleFunc("/account/", AccountHandler)
	servemux.HandleFunc("/transaction/", TransactionHandler)

	listener, err := net.Listen("tcp", ":"+strconv.Itoa(port))
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("Serving on port %d out of directory: %s", port, baseDir)
	if serveFcgi {
		fcgi.Serve(listener, context.ClearHandler(servemux))
	} else {
		http.Serve(listener, context.ClearHandler(servemux))
	}
}
<commit_msg>Have `go generate` make JavaScript<commit_after>package main

//go:generate make

import (
	"flag"
	"github.com/gorilla/context"
	"log"
	"net"
	"net/http"
	"net/http/fcgi"
	"os"
	"path"
	"strconv"
)

var serveFcgi bool
var baseDir string
var tmpDir string
var port int
var smtpServer string
var smtpPort int
var smtpUsername string
var smtpPassword string
var reminderEmail string

func init() {
	flag.StringVar(&baseDir, "base", "./", "Base directory for server")
	flag.StringVar(&tmpDir, "tmp", "/tmp", "Directory to create temporary files in")
	flag.IntVar(&port, "port", 80, "Port to serve API/files on")
	flag.StringVar(&smtpServer, "smtp.server", "smtp.example.com", "SMTP server to send reminder emails from.")
	flag.IntVar(&smtpPort, "smtp.port", 587, "SMTP server port to connect to")
	flag.StringVar(&smtpUsername, "smtp.username", "moneygo", "SMTP username")
	flag.StringVar(&smtpPassword, "smtp.password", "password", "SMTP password")
	flag.StringVar(&reminderEmail, "email", "moneygo@example.com", "Email address to send reminder emails as.")
	flag.BoolVar(&serveFcgi, "fcgi", false, "Serve via fcgi rather than http.")
	flag.Parse()

	static_path := path.Join(baseDir, "static")

	// Ensure base directory is valid
	dir_err_str := "The base directory doesn't look like it contains the " +
		"'static' directory. Check to make sure you're passing the right " +
		"value to the -base argument."
	static_dir, err := os.Stat(static_path)
	if err != nil {
		log.Print(err)
		log.Fatal(dir_err_str)
	}
	if !static_dir.IsDir() {
		log.Fatal(dir_err_str)
	}
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, path.Join(baseDir, "static/index.html"))
}

func staticHandler(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, path.Join(baseDir, r.URL.Path))
}

func main() {
	servemux := http.NewServeMux()
	servemux.HandleFunc("/", rootHandler)
	servemux.HandleFunc("/static/", staticHandler)
	servemux.HandleFunc("/session/", SessionHandler)
	servemux.HandleFunc("/user/", UserHandler)
	servemux.HandleFunc("/security/", SecurityHandler)
	servemux.HandleFunc("/account/", AccountHandler)
	servemux.HandleFunc("/transaction/", TransactionHandler)

	listener, err := net.Listen("tcp", ":"+strconv.Itoa(port))
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("Serving on port %d out of directory: %s", port, baseDir)
	if serveFcgi {
		fcgi.Serve(listener, context.ClearHandler(servemux))
	} else {
		http.Serve(listener, context.ClearHandler(servemux))
	}
}
<|endoftext|>