<commit_before>// Package pastebin is a simple modern and powerful pastebin service
package pastebin

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	duration "github.com/channelmeter/iso8601duration"
	// uniuri is used for easy random string generation
	"github.com/dchest/uniuri"
	// pygments is used for syntax highlighting
	"github.com/ewhal/pygments"
	// mysql driver
	_ "github.com/go-sql-driver/mysql"
	// mux is used for url routing
	"github.com/gorilla/mux"
)

const (
	// ADDRESS that pastebin will return links for
	ADDRESS = "http://localhost:9900"
	// LENGTH of paste id
	LENGTH = 6
	// PORT that pastebin will listen on
	PORT = ":9900"
	// USERNAME for database
	USERNAME = ""
	// PASS database password
	PASS = ""
	// NAME database name
	NAME = ""
	// DATABASE connection String
	DATABASE = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

// Template pages
var templates = template.Must(template.ParseFiles("assets/paste.html", "assets/index.html", "assets/clone.html"))
var syntax, _ = ioutil.ReadFile("assets/syntax.html")

// Response API struct
type Response struct {
	ID     string `json:"id"`
	TITLE  string `json:"title"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

// Page generation struct
type Page struct {
	Title    string
	Body     []byte
	Raw      string
	Home     string
	Download string
	Clone    string
}

// check error handling function
func Check(err error) {
	if err != nil {
		log.Println(err)
	}
}

// GenerateName uses uniuri to generate a random string that isn't in the
// database
func GenerateName() string {
	// use uniuri to generate random string
	id := uniuri.NewLen(LENGTH)

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()
	// query database if id exists and if it does call generateName again
	query, err := db.Query("select id from pastebin where id=?", id)
	if err != sql.ErrNoRows {
		for query.Next() {
			GenerateName()
		}
	}

	return id

}

// Sha1 hashes paste into a sha1 hash
func Sha1(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

// DurationFromExpiry takes the expiry in string format and returns the duration
// that the paste will exist for
func DurationFromExpiry(expiry string) time.Duration {
	dura, err := duration.FromString(expiry) // dura is time.Duration type
	Check(err)

	duration := dura.ToDuration()

	return duration
}

// Save function handles the saving of each paste.
// raw string is the raw paste input
// lang string is the user specified language for syntax highlighting
// title string user customized title
// expiry string duration that the paste will exist for
// Returns Response struct
func Save(raw string, lang string, title string, expiry string) Response {

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()

	// hash paste data and query database to see if paste exists
	sha := Sha1(raw)
	query, err := db.Query("select id, title, hash, data, delkey from pastebin where hash=?", sha)

	if err != sql.ErrNoRows {
		for query.Next() {
			var id, title, hash, paste, delkey string
			err := query.Scan(&id, &title, &hash, &paste, &delkey)
			Check(err)
			url := ADDRESS + "/p/" + id
			return Response{id, title, hash, url, len(paste), delkey}
		}
	}
	id := GenerateName()
	url := ADDRESS + "/p/" + id
	if lang != "" {
		url += "/" + lang
	}

	const timeFormat = "2006-01-02 15:04:05"
	expiryTime := time.Now().Add(DurationFromExpiry(expiry)).Format(timeFormat)

	delKey := uniuri.NewLen(40)
	dataEscaped := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, title, hash, data, delkey, expiry) values(?,?,?,?,?,?)")
	Check(err)
	if title == "" {
		title = id
	}
	_, err = stmt.Exec(id, html.EscapeString(title), sha, dataEscaped, delKey, expiryTime)
	Check(err)

	return Response{id, title, sha, url, len(dataEscaped), delKey}
}

// DelHandler checks to see if delkey and pasteid exist in the database.
// if both exist and are correct the paste will be removed.
func DelHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()

	stmt, err := db.Prepare("delete from pastebin where delkey=? and id=?")
	Check(err)

	res, err := stmt.Exec(html.EscapeString(delkey), html.EscapeString(id))
	Check(err)

	_, err = res.RowsAffected()
	if err != sql.ErrNoRows {
		io.WriteString(w, id+" deleted")
	}
}

// SaveHandler Handles saving pastes and outputing responses
func SaveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		lang := r.FormValue("lang")
		title := r.FormValue("title")
		expiry := r.FormValue("expiry")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
			return
		}
		b := Save(paste, lang, title, expiry)

		switch output {
		case "json":
			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)

		case "html":
			w.Header().Set("Content-Type", "text/html")
			io.WriteString(w, "
URL: "+b.URL+"
")
			io.WriteString(w, "
Delete Key: "+b.DELKEY+"
")

		case "redirect":
			http.Redirect(w, r, b.URL, 301)

		default:
			w.Header().Set("Content-Type", "text/plain; charset=UTF-8; imeanit=yes")
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

// Highlight uses user specified input to call pygments library to highlight the
// paste
func Highlight(s string, lang string) (string, error) {

	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,noclasses=True,", "utf-8")
	if err != nil {
		return "", err
	}
	return highlight, nil

}

// GetPaste takes pasteid and language
// queries the database and returns paste data
func GetPaste(paste string, lang string) (string, string) {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()
	var title, s string
	var expiry string
	err = db.QueryRow("select title, data, expiry from pastebin where id=?", param1).Scan(&title, &s, &expiry)
	Check(err)
	if time.Now().Format("2006-01-02 15:04:05") > expiry {
		stmt, err := db.Prepare("delete from pastebin where id=?")
		Check(err)
		_, err = stmt.Exec(param1)
		Check(err)
		return "Error invalid paste", ""
	}

	if err == sql.ErrNoRows {
		return "Error invalid paste", ""
	}
	if lang != "" {
		high, err := Highlight(s, lang)
		Check(err)
		return high, html.UnescapeString(title)
	}
	return html.UnescapeString(s), html.UnescapeString(title)
}

// PasteHandler handles the generation of paste pages with the links
func PasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]

	s, title := GetPaste(paste, lang)

	// button links
	link := ADDRESS + "/raw/" + paste
	download := ADDRESS + "/download/" + paste
	clone := ADDRESS + "/clone/" + paste
	// Page struct
	p := &Page{
		Title:    title,
		Body:     []byte(s),
		Raw:      link,
		Home:     ADDRESS,
		Download: download,
		Clone:    clone,
	}
	if lang == "" {

		err := templates.ExecuteTemplate(w, "paste.html", p)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

	} else {
		fmt.Fprintf(w, string(syntax), p.Title, p.Title, s, p.Home, p.Download, p.Raw, p.Clone)

	}
}

// CloneHandler handles generating the clone pages
func CloneHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]

	s, title := GetPaste(paste, "")

	// Page links
	link := ADDRESS + "/raw/" + paste
	download := ADDRESS + "/download/" + paste
	clone := ADDRESS + "/clone/" + paste

	// Clone page struct
	p := &Page{
		Title:    title,
		Body:     []byte(s),
		Raw:      link,
		Home:     ADDRESS,
		Download: download,
		Clone:    clone,
	}
	err := templates.ExecuteTemplate(w, "clone.html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

}

// DownloadHandler forces downloads of selected pastes
func DownloadHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, _ := GetPaste(paste, "")

	// Set header to an attachment so browser will automatically download it
	w.Header().Set("Content-Disposition", "attachment; filename="+paste)
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	io.WriteString(w, s)

}

// RawHandler displays the pastes in text/plain format
func RawHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, _ := GetPaste(paste, "")

	w.Header().Set("Content-Type", "text/plain; charset=UTF-8; imeanit=yes")
	// simply write string to browser
	io.WriteString(w, s)

}

// RootHandler handles generating the root page
func RootHandler(w http.ResponseWriter, r *http.Request) {
	err := templates.ExecuteTemplate(w, "index.html", &Page{})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", PasteHandler).Methods("GET")
	router.HandleFunc("/raw/{pasteId}", RawHandler).Methods("GET")
	router.HandleFunc("/p/{pasteId}/{lang}", PasteHandler).Methods("GET")
	router.HandleFunc("/clone/{pasteId}", CloneHandler).Methods("GET")
	router.HandleFunc("/download/{pasteId}", DownloadHandler).Methods("GET")
	router.HandleFunc("/p", SaveHandler).Methods("POST")
	router.HandleFunc("/p/{output}", SaveHandler).Methods("POST")
	router.HandleFunc("/p/{pasteId}/{delKey}", DelHandler).Methods("DELETE")
	router.HandleFunc("/", RootHandler)
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<commit_msg>Add fallthrough<commit_after>// Package pastebin is a simple modern and powerful pastebin service
package pastebin

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	duration "github.com/channelmeter/iso8601duration"
	// uniuri is used for easy random string generation
	"github.com/dchest/uniuri"
	// pygments is used for syntax highlighting
	"github.com/ewhal/pygments"
	// mysql driver
	_ "github.com/go-sql-driver/mysql"
	// mux is used for url routing
	"github.com/gorilla/mux"
)

const (
	// ADDRESS that pastebin will return links for
	ADDRESS = "http://localhost:9900"
	// LENGTH of paste id
	LENGTH = 6
	// PORT that pastebin will listen on
	PORT = ":9900"
	// USERNAME for database
	USERNAME = ""
	// PASS database password
	PASS = ""
	// NAME database name
	NAME = ""
	// DATABASE connection String
	DATABASE = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

// Template pages
var templates = template.Must(template.ParseFiles("assets/paste.html", "assets/index.html", "assets/clone.html"))
var syntax, _ = ioutil.ReadFile("assets/syntax.html")

// Response API struct
type Response struct {
	ID     string `json:"id"`
	TITLE  string `json:"title"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

// Page generation struct
type Page struct {
	Title    string
	Body     []byte
	Raw      string
	Home     string
	Download string
	Clone    string
}

// check error handling function
func Check(err error) {
	if err != nil {
		log.Println(err)
	}
}

// GenerateName uses uniuri to generate a random string that isn't in the
// database
func GenerateName() string {
	// use uniuri to generate random string
	id := uniuri.NewLen(LENGTH)

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()
	// query database if id exists and if it does call generateName again
	query, err := db.Query("select id from pastebin where id=?", id)
	if err != sql.ErrNoRows {
		for query.Next() {
			GenerateName()
		}
	}

	return id

}

// Sha1 hashes paste into a sha1 hash
func Sha1(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

// DurationFromExpiry takes the expiry in string format and returns the duration
// that the paste will exist for
func DurationFromExpiry(expiry string) time.Duration {
	if expiry == "" {
		expiry = "P20Y"
	}
	dura, err := duration.FromString(expiry) // dura is time.Duration type
	Check(err)

	duration := dura.ToDuration()

	return duration
}

// Save function handles the saving of each paste.
// raw string is the raw paste input
// lang string is the user specified language for syntax highlighting
// title string user customized title
// expiry string duration that the paste will exist for
// Returns Response struct
func Save(raw string, lang string, title string, expiry string) Response {

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()

	// hash paste data and query database to see if paste exists
	sha := Sha1(raw)
	query, err := db.Query("select id, title, hash, data, delkey from pastebin where hash=?", sha)

	if err != sql.ErrNoRows {
		for query.Next() {
			var id, title, hash, paste, delkey string
			err := query.Scan(&id, &title, &hash, &paste, &delkey)
			Check(err)
			url := ADDRESS + "/p/" + id
			return Response{id, title, hash, url, len(paste), delkey}
		}
	}
	id := GenerateName()
	url := ADDRESS + "/p/" + id
	if lang != "" {
		url += "/" + lang
	}

	const timeFormat = "2006-01-02 15:04:05"
	expiryTime := time.Now().Add(DurationFromExpiry(expiry)).Format(timeFormat)

	delKey := uniuri.NewLen(40)
	dataEscaped := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, title, hash, data, delkey, expiry) values(?,?,?,?,?,?)")
	Check(err)
	if title == "" {
		title = id
	}
	_, err = stmt.Exec(id, html.EscapeString(title), sha, dataEscaped, delKey, expiryTime)
	Check(err)

	return Response{id, title, sha, url, len(dataEscaped), delKey}
}

// DelHandler checks to see if delkey and pasteid exist in the database.
// if both exist and are correct the paste will be removed.
func DelHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()

	stmt, err := db.Prepare("delete from pastebin where delkey=? and id=?")
	Check(err)

	res, err := stmt.Exec(html.EscapeString(delkey), html.EscapeString(id))
	Check(err)

	_, err = res.RowsAffected()
	if err != sql.ErrNoRows {
		io.WriteString(w, id+" deleted")
	}
}

// SaveHandler Handles saving pastes and outputing responses
func SaveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		lang := r.FormValue("lang")
		title := r.FormValue("title")
		expiry := r.FormValue("expiry")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
			return
		}
		b := Save(paste, lang, title, expiry)

		switch output {
		case "json":
			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)

		case "html":
			w.Header().Set("Content-Type", "text/html")
			io.WriteString(w, "
URL: "+b.URL+"
")
			io.WriteString(w, "
Delete Key: "+b.DELKEY+"
")

		case "redirect":
			http.Redirect(w, r, b.URL, 301)

		default:
			w.Header().Set("Content-Type", "text/plain; charset=UTF-8; imeanit=yes")
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

// Highlight uses user specified input to call pygments library to highlight the
// paste
func Highlight(s string, lang string) (string, error) {

	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,noclasses=True,", "utf-8")
	if err != nil {
		return "", err
	}
	return highlight, nil

}

// GetPaste takes pasteid and language
// queries the database and returns paste data
func GetPaste(paste string, lang string) (string, string) {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()
	var title, s string
	var expiry string
	err = db.QueryRow("select title, data, expiry from pastebin where id=?", param1).Scan(&title, &s, &expiry)
	Check(err)
	if time.Now().Format("2006-01-02 15:04:05") > expiry {
		stmt, err := db.Prepare("delete from pastebin where id=?")
		Check(err)
		_, err = stmt.Exec(param1)
		Check(err)
		return "Error invalid paste", ""
	}

	if err == sql.ErrNoRows {
		return "Error invalid paste", ""
	}
	if lang != "" {
		high, err := Highlight(s, lang)
		Check(err)
		return high, html.UnescapeString(title)
	}
	return html.UnescapeString(s), html.UnescapeString(title)
}

// PasteHandler handles the generation of paste pages with the links
func PasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]

	s, title := GetPaste(paste, lang)

	// button links
	link := ADDRESS + "/raw/" + paste
	download := ADDRESS + "/download/" + paste
	clone := ADDRESS + "/clone/" + paste
	// Page struct
	p := &Page{
		Title:    title,
		Body:     []byte(s),
		Raw:      link,
		Home:     ADDRESS,
		Download: download,
		Clone:    clone,
	}
	if lang == "" {

		err := templates.ExecuteTemplate(w, "paste.html", p)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

	} else {
		fmt.Fprintf(w, string(syntax), p.Title, p.Title, s, p.Home, p.Download, p.Raw, p.Clone)

	}
}

// CloneHandler handles generating the clone pages
func CloneHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]

	s, title := GetPaste(paste, "")

	// Page links
	link := ADDRESS + "/raw/" + paste
	download := ADDRESS + "/download/" + paste
	clone := ADDRESS + "/clone/" + paste

	// Clone page struct
	p := &Page{
		Title:    title,
		Body:     []byte(s),
		Raw:      link,
		Home:     ADDRESS,
		Download: download,
		Clone:    clone,
	}
	err := templates.ExecuteTemplate(w, "clone.html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

}

// DownloadHandler forces downloads of selected pastes
func DownloadHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, _ := GetPaste(paste, "")

	// Set header to an attachment so browser will automatically download it
	w.Header().Set("Content-Disposition", "attachment; filename="+paste)
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	io.WriteString(w, s)

}

// RawHandler displays the pastes in text/plain format
func RawHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, _ := GetPaste(paste, "")

	w.Header().Set("Content-Type", "text/plain; charset=UTF-8; imeanit=yes")
	// simply write string to browser
	io.WriteString(w, s)

}

// RootHandler handles generating the root page
func RootHandler(w http.ResponseWriter, r *http.Request) {
	err := templates.ExecuteTemplate(w, "index.html", &Page{})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", PasteHandler).Methods("GET")
	router.HandleFunc("/raw/{pasteId}", RawHandler).Methods("GET")
	router.HandleFunc("/p/{pasteId}/{lang}", PasteHandler).Methods("GET")
	router.HandleFunc("/clone/{pasteId}", CloneHandler).Methods("GET")
	router.HandleFunc("/download/{pasteId}", DownloadHandler).Methods("GET")
	router.HandleFunc("/p", SaveHandler).Methods("POST")
	router.HandleFunc("/p/{output}", SaveHandler).Methods("POST")
	router.HandleFunc("/p/{pasteId}/{delKey}", DelHandler).Methods("DELETE")
	router.HandleFunc("/", RootHandler)
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>


<commit_before>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"crypto/rand"
	"encoding/hex"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	indexTmpl = "index.html"
	formTmpl  = "form.html"
	idSize    = 8
	rawIdSize = idSize / 2
	randTries = 10

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."

	// Common error messages
	timedOut = "Request timed out."
)

var (
	siteUrl, listen, dataDir, maxSizeStr string
	lifeTime, timeout                    time.Duration
	maxSize                              ByteSize
	indexTemplate, formTemplate          *template.Template

	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KM]?B|[BKM])$`)
	startTime     = time.Now()
)

var workers [256]Worker
var post = make(chan PostRequest) // Posting is shared to balance load

type Id [rawIdSize]byte

type PasteInfo struct {
	Etag, ContentType string
	ModTime           time.Time
}

type GetRequest struct {
	w    http.ResponseWriter
	r    *http.Request
	done chan struct{}
	id   Id
}

type PostRequest struct {
	w       http.ResponseWriter
	r       *http.Request
	done    chan struct{}
	content []byte
	modTime time.Time
}

type Worker struct {
	n    byte // Its number, aka the first two hex chars
	get  chan GetRequest
	post chan PostRequest
	del  chan Id
	m    map[Id]PasteInfo
}

func (w Worker) recoverPaste(filePath string, fileInfo os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if fileInfo.IsDir() {
		return nil
	}
	id, err := IdFromPath(filePath)
	if err != nil {
		return errors.New("Found incompatible id at path " + filePath)
	}
	modTime := fileInfo.ModTime()
	deathTime := modTime.Add(lifeTime)
	if deathTime.Before(startTime) {
		err := os.Remove(filePath)
		if err != nil {
			return err
		}
	}
	if modTime.After(startTime) {
		modTime = startTime
	}
	pasteFile, err := os.Open(filePath)
	if err != nil {
		return err
	}
	buf := make([]byte, 512)
	_, err = pasteFile.Read(buf)
	pasteFile.Close()
	if err != nil && err != io.EOF {
		return err
	}
	w.m[id] = id.GenPasteInfo(modTime, buf)
	w.DeletePasteAfter(id, deathTime.Sub(startTime))
	return nil
}

func (w Worker) RandomId() (Id, error) {
	var id Id
	id[0] = w.n
	for try := 0; try < randTries; try++ {
		if _, err := rand.Read(id[1:]); err != nil {
			return id, err
		}
		if _, e := w.m[id]; !e {
			return id, nil
		}
	}
	return id, fmt.Errorf("Gave up trying to find an unused random id after %d tries", randTries)
}

func (w Worker) Work() {
	dir := hex.EncodeToString([]byte{w.n})
	if stat, err := os.Stat(dir); err == nil {
		if !stat.IsDir() {
			log.Fatalf("%s/%s exists but is not a directory!", dataDir, dir)
		}
	} else {
		if err := os.Mkdir(dir, 0700); err != nil {
			log.Fatalf("Could not create data directory %s/%s: %s", dataDir, dir, err)
		}
	}
	w.m = make(map[Id]PasteInfo)
	if err := filepath.Walk(dir, w.recoverPaste); err != nil {
		log.Fatalf("Could not recover data directory %s/%s: %s", dataDir, dir, err)
	}
	for {
		var done chan struct{}
		select {
		case request := <-w.get:
			done = request.done
			pasteInfo, e := w.m[request.id]
			if !e {
				http.Error(request.w, pasteNotFound, http.StatusNotFound)
				break
			}
			if inm := request.r.Header.Get("If-None-Match"); inm != "" {
				if pasteInfo.Etag == inm || inm == "*" {
					request.w.WriteHeader(http.StatusNotModified)
					break
				}
			}
			pasteFile, err := os.Open(request.id.Path())
			if err != nil {
				http.Error(request.w, unknownError, http.StatusInternalServerError)
				break
			}
			request.w.Header().Set("Etag", pasteInfo.Etag)
			request.w.Header().Set("Content-Type", pasteInfo.ContentType)
			http.ServeContent(request.w, request.r, "", pasteInfo.ModTime, pasteFile)
			pasteFile.Close()

		case request := <-w.post:
			done = request.done
			id, err := w.RandomId()
			if err != nil {
				log.Println(err)
				http.Error(request.w, unknownError, http.StatusInternalServerError)
				break
			}
			pastePath := id.Path()
			pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
			if err != nil {
				log.Printf("Could not create new paste file %s: %s", pastePath, err)
				http.Error(request.w, unknownError, http.StatusInternalServerError)
				break
			}
			_, err = pasteFile.Write(request.content)
			pasteFile.Close()
			if err != nil {
				log.Printf("Could not write data into %s: %s", pastePath, err)
				http.Error(request.w, unknownError, http.StatusInternalServerError)
				break
			}
			w.m[id] = id.GenPasteInfo(request.modTime, request.content)
			w.DeletePasteAfter(id, lifeTime)
			fmt.Fprintf(request.w, "%s/%s\n", siteUrl, id)

		case id := <-w.del:
			if err := os.Remove(id.Path()); err == nil {
				delete(w.m, id)
			} else {
				log.Printf("Could not remove %s: %s", id, err)
				w.DeletePasteAfter(id, 2*time.Minute)
			}
		}
		if done != nil {
			done <- struct{}{}
		}
	}
}

func init() {
	flag.StringVar(&siteUrl, "u", "http://localhost:8080", "URL of the site")
	flag.StringVar(&listen, "l", "localhost:8080", "Host and port to listen to")
	flag.StringVar(&dataDir, "d", "data", "Directory to store all the pastes in")
	flag.DurationVar(&lifeTime, "t", 12*time.Hour, "Lifetime of the pastes")
	flag.DurationVar(&timeout, "T", 200*time.Millisecond, "Timeout of requests")
	flag.StringVar(&maxSizeStr, "s", "1M", "Maximum size of POSTs in bytes")
}

func IdFromString(hexId string) (Id, error) {
	var id Id
	if len(hexId) != idSize {
		return id, errors.New("Invalid id")
	}
	b, err := hex.DecodeString(hexId)
	if err != nil || len(b) != rawIdSize {
		return id, errors.New("Invalid id")
	}
	copy(id[:], b)
	return id, nil
}

func IdFromPath(idPath string) (Id, error) {
	var id Id
	parts := strings.Split(idPath, string(filepath.Separator))
	if len(parts) != 2 {
		return id, errors.New("Found invalid number of directories at " + idPath)
	}
	return IdFromString(parts[0] + parts[1])
}

func (id Id) String() string {
	return hex.EncodeToString(id[:])
}

func (id Id) Path() string {
	hexId := id.String()
	return path.Join(hexId[0:2], hexId[2:])
}

func (id Id) GenPasteInfo(modTime time.Time, head []byte) (pasteInfo PasteInfo) {
	pasteInfo.ModTime = modTime
	pasteInfo.Etag = fmt.Sprintf("%d-%s", pasteInfo.ModTime.Unix(), id)
	pasteInfo.ContentType = http.DetectContentType(head)
	if pasteInfo.ContentType == "application/octet-stream" {
		pasteInfo.ContentType = "text-plain; charset=utf-8"
	}
	return
}

func (w Worker) DeletePasteAfter(id Id, duration time.Duration) {
	timer := time.NewTimer(duration)
	go func() {
		<-timer.C
		w.del <- id
	}()
}

type ByteSize int64

const (
	B ByteSize = 1 << (10 * iota)
	KB
	MB
)

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)

	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(KB)
	case "MB", "M":
		size *= float64(MB)
	}
	return ByteSize(size), nil
}

func (b ByteSize) String() string {
	switch {
	case b >= MB:
		return fmt.Sprintf("%.2fMB", float64(b)/float64(MB))
	case b >= KB:
		return fmt.Sprintf("%.2fKB", float64(b)/float64(KB))
	}
	return fmt.Sprintf("%dB", b)
}

func handler(w http.ResponseWriter, r *http.Request) {
	done := make(chan struct{})
	timer := time.NewTimer(timeout)
	switch r.Method {
	case "GET":
		switch r.URL.Path {
		case "/":
			indexTemplate.Execute(w, struct{ SiteUrl, LifeTime string }{
				siteUrl, lifeTime.String()})
			return
		case "/form":
			formTemplate.Execute(w, struct{ SiteUrl string }{siteUrl})
			return
		}
		id, err := IdFromString(r.URL.Path[1:])
		if err != nil {
			http.Error(w, invalidId, http.StatusBadRequest)
			return
		}
		select {
		case <-timer.C:
			http.Error(w, timedOut, http.StatusRequestTimeout)
		case workers[id[0]].get <- GetRequest{id: id, w: w, r: r, done: done}:
			// request is sent
			timer.Stop()
		}

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		if err := r.ParseMultipartForm(int64(maxSize)); err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		var content []byte
		if vs, found := r.Form["paste"]; found && len(vs[0]) > 0 {
			content = []byte(vs[0])
		} else {
			http.Error(w, missingForm, http.StatusBadRequest)
			return
		}
		select {
		case <-timer.C:
			http.Error(w, timedOut, http.StatusRequestTimeout)
		case post <- PostRequest{content: content, modTime: time.Now(), w: w, r: r, done: done}:
			// request is sent
			timer.Stop()
		}

	default:
		http.Error(w, "Unsupported action.", http.StatusBadRequest)
		return
	}
	<-done
}

func main() {
	var err error
	flag.Parse()
	if maxSize, err = parseByteSize(maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", maxSizeStr, err)
	}
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", indexTmpl, err)
	}
	if formTemplate, err = template.ParseFiles(formTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", formTmpl, err)
	}
	if err = os.MkdirAll(dataDir, 0700); err != nil {
		log.Fatalf("Could not create data directory %s: %s", dataDir, err)
	}
	if err = os.Chdir(dataDir); err != nil {
		log.Fatalf("Could not enter data directory %s: %s", dataDir, err)
	}
	log.Printf("maxSize  = %s", maxSize)
	log.Printf("siteUrl  = %s", siteUrl)
	log.Printf("listen   = %s", listen)
	log.Printf("dataDir  = %s", dataDir)
	log.Printf("lifeTime = %s", lifeTime)
	log.Printf("timeout  = %s", timeout)
	for n := range workers {
		w := &workers[n]
		w.n = byte(n)
		w.get = make(chan GetRequest)
		w.post = post
		w.del = make(chan Id)
		go w.Work()
	}
	http.HandleFunc("/", handler)
	log.Printf("Up and running!")
	log.Fatal(http.ListenAndServe(listen, nil))
}
<commit_msg>No need to duplicate post channels for every worker<commit_after>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"crypto/rand"
	"encoding/hex"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	indexTmpl = "index.html"
	formTmpl  = "form.html"
	idSize    = 8
	rawIdSize = idSize / 2
	randTries = 10

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."

	// Common error messages
	timedOut = "Request timed out."
)

var (
	siteUrl, listen, dataDir, maxSizeStr string
	lifeTime, timeout                    time.Duration
	maxSize                              ByteSize
	indexTemplate, formTemplate          *template.Template

	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KM]?B|[BKM])$`)
	startTime     = time.Now()
)

var workers [256]Worker
var post = make(chan PostRequest) // Posting is shared to balance load

type Id [rawIdSize]byte

type PasteInfo struct {
	Etag, ContentType string
	ModTime           time.Time
}

type GetRequest struct {
	w    http.ResponseWriter
	r    *http.Request
	done chan struct{}
	id   Id
}

type PostRequest struct {
	w       http.ResponseWriter
	r       *http.Request
	done    chan struct{}
	content []byte
	modTime time.Time
}

type Worker struct {
	n    byte // Its number, aka the first two hex chars
	get  chan GetRequest
	del  chan Id
	m    map[Id]PasteInfo
}

func (w Worker) recoverPaste(filePath string, fileInfo os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if fileInfo.IsDir() {
		return nil
	}
	id, err := IdFromPath(filePath)
	if err != nil {
		return errors.New("Found incompatible id at path " + filePath)
	}
	modTime := fileInfo.ModTime()
	deathTime := modTime.Add(lifeTime)
	if deathTime.Before(startTime) {
		err := os.Remove(filePath)
		if err != nil {
			return err
		}
	}
	if modTime.After(startTime) {
		modTime = startTime
	}
	pasteFile, err := os.Open(filePath)
	if err != nil {
		return err
	}
	buf := make([]byte, 512)
	_, err = pasteFile.Read(buf)
	pasteFile.Close()
	if err != nil && err != io.EOF {
		return err
	}
	w.m[id] = id.GenPasteInfo(modTime, buf)
	w.DeletePasteAfter(id, deathTime.Sub(startTime))
	return nil
}

func (w Worker) RandomId() (Id, error) {
	var id Id
	id[0] = w.n
	for try := 0; try < randTries; try++ {
		if _, err := rand.Read(id[1:]); err != nil {
			return id, err
		}
		if _, e := w.m[id]; !e {
			return id, nil
		}
	}
	return id, fmt.Errorf("Gave up trying to find an unused random id after %d tries", randTries)
}

func (w Worker) Work() {
	dir := hex.EncodeToString([]byte{w.n})
	if stat, err := os.Stat(dir); err == nil {
		if !stat.IsDir() {
			log.Fatalf("%s/%s exists but is not a directory!", dataDir, dir)
		}
	} else {
		if err := os.Mkdir(dir, 0700); err != nil {
			log.Fatalf("Could not create data directory %s/%s: %s", dataDir, dir, err)
		}
	}
	w.m = make(map[Id]PasteInfo)
	if err := filepath.Walk(dir, w.recoverPaste); err != nil {
		log.Fatalf("Could not recover data directory %s/%s: %s", dataDir, dir, err)
	}
	for {
		var done chan struct{}
		select {
		case request := <-w.get:
			done = request.done
			pasteInfo, e := w.m[request.id]
			if !e {
				http.Error(request.w, pasteNotFound, http.StatusNotFound)
				break
			}
			if inm := request.r.Header.Get("If-None-Match"); inm != "" {
				if pasteInfo.Etag == inm || inm == "*" {
					request.w.WriteHeader(http.StatusNotModified)
					break
				}
			}
			pasteFile, err := os.Open(request.id.Path())
			if err != nil {
				http.Error(request.w, unknownError, http.StatusInternalServerError)
				break
			}
			request.w.Header().Set("Etag", pasteInfo.Etag)
			request.w.Header().Set("Content-Type", pasteInfo.ContentType)
			http.ServeContent(request.w, request.r, "", pasteInfo.ModTime, pasteFile)
			pasteFile.Close()

		case request := <-post:
			done = request.done
			id, err := w.RandomId()
			if err != nil {
				log.Println(err)
				http.Error(request.w, unknownError, http.StatusInternalServerError)
				break
			}
			pastePath := id.Path()
			pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
			if err != nil {
				log.Printf("Could not create new paste file %s: %s", pastePath, err)
				http.Error(request.w, unknownError, http.StatusInternalServerError)
				break
			}
			_, err = pasteFile.Write(request.content)
			pasteFile.Close()
			if err != nil {
				log.Printf("Could not write data into %s: %s", pastePath, err)
				http.Error(request.w, unknownError, http.StatusInternalServerError)
				break
			}
			w.m[id] = id.GenPasteInfo(request.modTime, request.content)
			w.DeletePasteAfter(id, lifeTime)
			fmt.Fprintf(request.w, "%s/%s\n", siteUrl, id)

		case id := <-w.del:
			if err := os.Remove(id.Path()); err == nil {
				delete(w.m, id)
			} else {
				log.Printf("Could not remove %s: %s", id, err)
				w.DeletePasteAfter(id, 2*time.Minute)
			}
		}
		if done != nil {
			done <- struct{}{}
		}
	}
}

func init() {
	flag.StringVar(&siteUrl, "u", "http://localhost:8080", "URL of the site")
	flag.StringVar(&listen, "l", "localhost:8080", "Host and port to listen to")
	flag.StringVar(&dataDir, "d", "data", "Directory to store all the pastes in")
	flag.DurationVar(&lifeTime, "t", 12*time.Hour, "Lifetime of the pastes")
	flag.DurationVar(&timeout, "T", 200*time.Millisecond, "Timeout of requests")
	flag.StringVar(&maxSizeStr, "s", "1M", "Maximum size of POSTs in bytes")
}

func IdFromString(hexId string) (Id, error) {
	var id Id
	if len(hexId) != idSize {
		return id, errors.New("Invalid id")
	}
	b, err := hex.DecodeString(hexId)
	if err != nil || len(b) != rawIdSize {
		return id, errors.New("Invalid id")
	}
	copy(id[:], b)
	return id, nil
}

func IdFromPath(idPath string) (Id, error) {
	var id Id
	parts := strings.Split(idPath, string(filepath.Separator))
	if len(parts) != 2 {
		return id, errors.New("Found invalid number of directories at " + idPath)
	}
	return IdFromString(parts[0] + parts[1])
}

func (id Id) String() string {
	return hex.EncodeToString(id[:])
}

func (id Id) Path() string {
	hexId := id.String()
	return path.Join(hexId[0:2], hexId[2:])
}

func (id Id) GenPasteInfo(modTime time.Time, head []byte) (pasteInfo PasteInfo) {
	pasteInfo.ModTime = modTime
	pasteInfo.Etag = fmt.Sprintf("%d-%s", pasteInfo.ModTime.Unix(), id)
	pasteInfo.ContentType = http.DetectContentType(head)
	if pasteInfo.ContentType == "application/octet-stream" {
		pasteInfo.ContentType = "text-plain; charset=utf-8"
	}
	return
}

func (w Worker) DeletePasteAfter(id Id, duration time.Duration) {
	timer := time.NewTimer(duration)
	go func() {
		<-timer.C
		w.del <- id
	}()
}

type ByteSize int64

const (
	B ByteSize = 1 << (10 * iota)
	KB
	MB
)

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)

	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(KB)
	case "MB", "M":
		size *= float64(MB)
	}
	return ByteSize(size), nil
}

func (b ByteSize) String() string {
	switch {
	case b >= MB:
		return fmt.Sprintf("%.2fMB", float64(b)/float64(MB))
	case b >= KB:
		return fmt.Sprintf("%.2fKB", float64(b)/float64(KB))
	}
	return fmt.Sprintf("%dB", b)
}

func handler(w http.ResponseWriter, r *http.Request) {
	done := make(chan struct{})
	timer := time.NewTimer(timeout)
	switch r.Method {
	case "GET":
		switch r.URL.Path {
		case "/":
			indexTemplate.Execute(w, struct{ SiteUrl, LifeTime string }{
				siteUrl, lifeTime.String()})
			return
		case "/form":
			formTemplate.Execute(w, struct{ SiteUrl string }{siteUrl})
			return
		}
		id, err := IdFromString(r.URL.Path[1:])
		if err != nil {
			http.Error(w, invalidId, http.StatusBadRequest)
			return
		}
		select {
		case <-timer.C:
			http.Error(w, timedOut, http.StatusRequestTimeout)
		case workers[id[0]].get <- GetRequest{id: id, w: w, r: r, done: done}:
			// request is sent
			timer.Stop()
		}

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		if err := r.ParseMultipartForm(int64(maxSize)); err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		var content []byte
		if vs, found := r.Form["paste"]; found && len(vs[0]) > 0 {
			content = []byte(vs[0])
		} else {
			http.Error(w, missingForm, http.StatusBadRequest)
			return
		}
		select {
		case <-timer.C:
			http.Error(w, timedOut, http.StatusRequestTimeout)
		case post <- PostRequest{content: content, modTime: time.Now(), w: w, r: r, done: done}:
			// request is sent
			timer.Stop()
		}

	default:
		http.Error(w, "Unsupported action.", http.StatusBadRequest)
		return
	}
	<-done
}

func main() {
	var err error
	flag.Parse()
	if maxSize, err = parseByteSize(maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", maxSizeStr, err)
	}
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", indexTmpl, err)
	}
	if formTemplate, err = template.ParseFiles(formTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", formTmpl, err)
	}
	if err = os.MkdirAll(dataDir, 0700); err != nil {
		log.Fatalf("Could not create data directory %s: %s", dataDir, err)
	}
	if err = os.Chdir(dataDir); err != nil {
		log.Fatalf("Could not enter data directory %s: %s", dataDir, err)
	}
	log.Printf("maxSize  = %s", maxSize)
	log.Printf("siteUrl  = %s", siteUrl)
	log.Printf("listen   = %s", listen)
	log.Printf("dataDir  = %s", dataDir)
	log.Printf("lifeTime = %s", lifeTime)
	log.Printf("timeout  = %s", timeout)
	for n := range workers {
		w := &workers[n]
		w.n = byte(n)
		w.get = make(chan GetRequest)
		w.del = make(chan Id)
		go w.Work()
	}
	http.HandleFunc("/", handler)
	log.Printf("Up and running!")
	log.Fatal(http.ListenAndServe(listen, nil))
}
<|endoftext|>


<commit_before>package via

import (
	"fmt"
	"github.com/mrosset/util/file"
	"github.com/mrosset/util/json"
	"os"
	"path/filepath"
	"sort"
)

// Repo provides repo path string. This is the path that binary
// tarballs are downloaded and built too
type Repo string

// String provides stringer interface
func (r Repo) String() string {
	return string(r)
}

// Exists return true if the Repo path exists
func (r Repo) Exists() bool {
	return file.Exists(r.String())
}

// Ensure that the directory is created
func (r Repo) Ensure() error {
	if r.Exists() {
		return nil
	}
	return os.MkdirAll(r.String(), 0755)
}

// Expand returns the Repo path as a string that has been its
// environmental variables expanded.
func (r Repo) Expand() string {
	return os.ExpandEnv(string(r))
}

// NewRepo returns a new Repo who's parent is joined with dir
func NewRepo(parent, dir string) Repo {
	return Repo(filepath.Join(parent, dir))
}

// RepoFiles provides plan files map hash
type RepoFiles map[string][]string

// Returns a sorted slice key strings
func (rf RepoFiles) keys() []string {
	var (
		keys = []string{}
	)
	for k := range rf {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// Owns returns the first alphabetical plan Name of plan that contains file
func (rf RepoFiles) Owns(file string) string {
	for _, key := range rf.keys() {
		if filesContains(rf[key], file) {
			return key
		}
	}
	fmt.Println("warning: can not resolve", file)
	return ""
}

// Owners like owns but returns a slice of plan names instead of the first
// occurrence. The returned slice is sorted alphabetically
func (rf RepoFiles) Owners(file string) []string {
	owners := []string{}
	for _, key := range rf.keys() {
		if filesContains(rf[key], file) {
			owners = append(owners, key)
		}
	}
	return owners
}

// ReadRepoFiles reads files.json and returns a RepoFiles map hash
func ReadRepoFiles(config *Config) (RepoFiles, error) {
	files := RepoFiles{}
	if err := json.Read(&files, join(config.Plans, "files.json")); err != nil {
		return nil, err
	}
	return files, nil
}

// RepoCreate reads each plan's files creating a repo.json file that
// contains all plan's and groups. And also creating a files.json that
// contains a hash map of each plans files
//
// FIXME: this is pretty expensive and probably won't scale well. Also
// repo.json and files.json should probably not be kept in version control.
func RepoCreate(config *Config) error {
	var (
		repo  = []string{}
		files = map[string][]string{}
		rfile = join(config.Plans, "repo.json")
		ffile = join(config.Plans, "files.json")
	)
	e, err := PlanFiles(config)
	if err != nil {
		return err
	}
	for _, j := range e {
		p, err := ReadPath(j)
		if err != nil {
			return err
		}
		repo = append(repo, join(p.Group, p.Name+".json"))
		files[p.Name] = p.Files
	}
	err = json.Write(repo, rfile)
	if err != nil {
		return err
	}
	return json.Write(files, ffile)
}
<commit_msg>refine Repo documentation<commit_after>package via

import (
	"fmt"
	"github.com/mrosset/util/file"
	"github.com/mrosset/util/json"
	"os"
	"path/filepath"
	"sort"
)

// Repo provides repo path string. This is the path that binary
// tarballs are downloaded and built too
type Repo string

// String provides stringer interface
func (r Repo) String() string {
	return string(r)
}

// Exists return true if the Repo path exists
func (r Repo) Exists() bool {
	return file.Exists(r.String())
}

// Ensure that the Repo directory path is created
func (r Repo) Ensure() error {
	if r.Exists() {
		return nil
	}
	return os.MkdirAll(r.String(), 0755)
}

// Expand returns the Repo path as a string that has been its
// environmental variables expanded.
func (r Repo) Expand() string {
	return os.ExpandEnv(string(r))
}

// NewRepo returns a new Repo who's parent is joined with dir
func NewRepo(parent, dir string) Repo {
	return Repo(filepath.Join(parent, dir))
}

// RepoFiles provides plan files map hash
type RepoFiles map[string][]string

// Returns a sorted  key string slice
func (rf RepoFiles) keys() []string {
	var (
		keys = []string{}
	)
	for k := range rf {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// Owns returns the first alphabetical plan Name of plan that contains file
func (rf RepoFiles) Owns(file string) string {
	for _, key := range rf.keys() {
		if filesContains(rf[key], file) {
			return key
		}
	}
	fmt.Println("warning: can not resolve", file)
	return ""
}

// Owners like owns but returns a slice of plan names instead of the first
// occurrence. The returned slice is sorted alphabetically
func (rf RepoFiles) Owners(file string) []string {
	owners := []string{}
	for _, key := range rf.keys() {
		if filesContains(rf[key], file) {
			owners = append(owners, key)
		}
	}
	return owners
}

// ReadRepoFiles reads files.json and returns a RepoFiles map hash
func ReadRepoFiles(config *Config) (RepoFiles, error) {
	files := RepoFiles{}
	if err := json.Read(&files, join(config.Plans, "files.json")); err != nil {
		return nil, err
	}
	return files, nil
}

// RepoCreate reads each plan's files creating a repo.json file that
// contains all plan's and groups. And also creating a files.json that
// contains a hash map of each plans files
//
// FIXME: this is pretty expensive and probably won't scale well. Also
// repo.json and files.json should probably not be kept in version control.
func RepoCreate(config *Config) error {
	var (
		repo  = []string{}
		files = map[string][]string{}
		rfile = join(config.Plans, "repo.json")
		ffile = join(config.Plans, "files.json")
	)
	e, err := PlanFiles(config)
	if err != nil {
		return err
	}
	for _, j := range e {
		p, err := ReadPath(j)
		if err != nil {
			return err
		}
		repo = append(repo, join(p.Group, p.Name+".json"))
		files[p.Name] = p.Files
	}
	err = json.Write(repo, rfile)
	if err != nil {
		return err
	}
	return json.Write(files, ffile)
}
<|endoftext|>


<commit_before>package commands

import (
	"errors"
	"fmt"
	"io"

	//ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
	k "github.com/ipfs/go-ipfs/blocks/key"
	cmds "github.com/ipfs/go-ipfs/commands"
	"github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/filestore"
	"github.com/ipfs/go-ipfs/repo/fsrepo"
	b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58"
	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
)

var FileStoreCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Interact with filestore objects",
	},
	Subcommands: map[string]*cmds.Command{
		"ls":         lsFileStore,
		"verify":     verifyFileStore,
		"rm":         rmFilestoreObjs,
		"rm-invalid": rmInvalidObjs,
		//"rm-incomplete":      rmIncompleteObjs,
		"find-dangling-pins": findDanglingPins,
	},
}

var lsFileStore = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List objects in filestore",
		ShortDescription: `
List objects in the filestore.  If --quiet is specified only the
hashes are printed, otherwise the fields are as follows:
  <hash> <type> <filepath> <offset> <size>
where <type> is one of"
  leaf: to indicate a leaf node where the contents are stored
        to in the file itself
  root: to indicate a root node that represents the whole file
  other: some other kind of node that represent part of a file
and <filepath> is the part of the file the object represents.  The
part represented starts at <offset> and continues for <size> bytes.
If <offset> is the special value "-" than the "leaf" or "root" node
represents the whole file.
`,
	},
	Options: []cmds.Option{
		cmds.BoolOption("quiet", "q", "Write just hashes of objects."),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		_, fs, err := extractFilestore(req)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		quiet, _, err := res.Request().Option("quiet").Bool()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch, _ := filestore.List(fs, quiet)
		res.SetOutput(&chanWriter{ch, "", 0})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

type chanWriter struct {
	ch     <-chan filestore.ListRes
	buf    string
	offset int
}

func (w *chanWriter) Read(p []byte) (int, error) {
	if w.offset >= len(w.buf) {
		w.offset = 0
		res, more := <-w.ch
		if !more {
			return 0, io.EOF
		}
		if res.DataObj == nil {
			w.buf = res.MHash() + "\n"
		} else {
			w.buf = res.Format()
		}
	}
	sz := copy(p, w.buf[w.offset:])
	w.offset += sz
	return sz, nil
}

var verifyFileStore = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Verify objects in filestore",
		ShortDescription: `
Verify leaf nodes in the filestore, the output is:
  <status> <type> <filepath> <offset> <size>
where <type>, <filepath>, <offset> and <size> are the same as in the
"ls" command and <status> is one of:
  ok:      If the object is okay
  changed: If the object is invalid becuase the contents of the file
           have changed
  missing: If the file can not be found
  error:   If the file can be found but could not be read or some
           other error
`,
	},
	Run: func(req cmds.Request, res cmds.Response) {
		_, fs, err := extractFilestore(req)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch,_ := filestore.List(fs, false)
		rdr, wtr := io.Pipe()
		go func() {
			defer wtr.Close()
			for res := range ch {
				if !res.NoBlockData {
					continue
				}
				res.Status = filestore.Verify(fs, res.Key, res.DataObj)
				wtr.Write([]byte(res.Format()))
			}
		}()
		res.SetOutput(rdr)
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

var rmFilestoreObjs = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Remove objects from the filestore",
	},
	Arguments: []cmds.Argument{
		cmds.StringArg("hash", true, true, "Multi-hashes to remove."),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		node, fs, err := extractFilestore(req)
		_ = fs
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		hashes := req.Arguments()
		serr := res.Stderr()
		numErrors := 0
		for _, mhash := range hashes {
			key := k.B58KeyDecode(mhash)
			err = delFilestoreObj(req, node, fs, key)
			if err != nil {
				fmt.Fprintf(serr, "Error deleting %s: %s\n", mhash, err.Error())
				numErrors += 1
			}
		}
		if numErrors > 0 {
			res.SetError(errors.New("Could not delete some keys"), cmds.ErrNormal)
			return
		}
		return
	},
}

var rmInvalidObjs = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Remove invalid objects from the filestore",
		ShortDescription: `
Removes objects that have become invalid from the Filestrore up to the
reason specified in <level>.  If <level> is "changed" than remove any
blocks that have become invalid due to the contents of the underlying
file changing.  If <level> is "missing" also remove any blocks that
have become invalid because the underlying file is no longer available
due to a "No such file" or related error, but not if the file exists
but is unreadable for some reason.  If <level> is "all" remove any
blocks that fail to validate regardless of the reason.
`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("level", true, false, "one of changed, missing. or all").EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.BoolOption("quiet", "q", "Produce less output."),
		cmds.BoolOption("dry-run", "n", "Do everything except the actual delete."),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		node, fs, err := extractFilestore(req)
		_ = fs
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		args := req.Arguments()
		if len(args) != 1 {
			res.SetError(errors.New("invalid usage"), cmds.ErrNormal)
			return
		}
		mode := req.Arguments()[0]
		level := filestore.StatusMissing
		switch mode {
		case "changed":
			level = filestore.StatusChanged
		case "missing":
			level = filestore.StatusMissing
		case "all":
			level = filestore.StatusError
		default:
			res.SetError(errors.New("level must be one of: changed missing all"), cmds.ErrNormal)
		}
		quiet, _, err := res.Request().Option("quiet").Bool()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		dryRun, _, err := res.Request().Option("dry-run").Bool()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch,_ := filestore.List(fs, false)
		rdr, wtr := io.Pipe()
		go func() {
			defer wtr.Close()
			var toDel [][]byte
			for r := range ch {
				if !r.NoBlockData {
					continue
				}
				r.Status = filestore.Verify(fs, r.Key, r.DataObj)
				if r.Status >= level {
					toDel = append(toDel, r.RawHash())
					if !quiet {
						fmt.Fprintf(wtr, "will delete %s (part of %s)\n", r.MHash(), r.FilePath)
					}
				}
			}
			if dryRun {
				fmt.Fprintf(wtr, "Dry-run option specified.  Stopping.\n")
				fmt.Fprintf(wtr, "Would of deleted %d invalid objects.\n", len(toDel))
			} else {
				for _, key := range toDel {
					err = delFilestoreObj(req, node, fs, k.Key(key))
					if err != nil {
						mhash := b58.Encode(key)
						msg := fmt.Sprintf("Could not delete %s: %s\n", mhash, err.Error())
						res.SetError(errors.New(msg), cmds.ErrNormal)
						return

					}
				}
				fmt.Fprintf(wtr, "Deleted %d invalid objects.\n", len(toDel))
			}
		}()
		res.SetOutput(rdr)
		return
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

func delFilestoreObj(req cmds.Request, node *core.IpfsNode, fs *filestore.Datastore, key k.Key) error {
	err := fs.DeleteDirect(key.DsKey())
	if err != nil {
		return err
	}
	stillExists, err := node.Blockstore.Has(key)
	if err != nil {
		return err
	}
	if stillExists {
		return nil
	}
	_, pinned1, err := node.Pinning.IsPinnedWithType(key, "recursive")
	if err != nil {
		return err
	}
	_, pinned2, err := node.Pinning.IsPinnedWithType(key, "direct")
	if err != nil {
		return err
	}
	if pinned1 || pinned2 {
		println("unpinning")
		ctx, cancel := context.WithCancel(req.Context())
		defer cancel()
		err = node.Pinning.Unpin(ctx, key, true)
		if err != nil {
			return err
		}
		err := node.Pinning.Flush()
		if err != nil {
			return err
		}
	}
	return nil
}

func extractFilestore(req cmds.Request) (node *core.IpfsNode, fs *filestore.Datastore, err error) {
	node, err = req.InvocContext().GetNode()
	if err != nil {
		return
	}
	repo, ok := node.Repo.Self().(*fsrepo.FSRepo)
	if !ok {
		err = errors.New("Not a FSRepo")
		return
	}
	fs = repo.Filestore()
	return
}

var findDanglingPins = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List pinned objects that no longer exists",
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			return
		}
		r, w := io.Pipe()
		go func() {
			defer w.Close()
			err := listDanglingPins(n.Pinning.DirectKeys(), w, n.Blockstore)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			err = listDanglingPins(n.Pinning.RecursiveKeys(), w, n.Blockstore)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
		}()
		res.SetOutput(r)
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

func listDanglingPins(keys []k.Key, out io.Writer, d bs.Blockstore) error {
	for _, k := range keys {
		exists, err := d.Has(k)
		if err != nil {
			return err
		}
		if !exists {
			fmt.Fprintln(out, k.B58String())
		}
	}
	return nil
}
<commit_msg>Enhance "filestore rm" to give better output.<commit_after>package commands

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"

	//ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
	k "github.com/ipfs/go-ipfs/blocks/key"
	cmds "github.com/ipfs/go-ipfs/commands"
	"github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/filestore"
	"github.com/ipfs/go-ipfs/repo/fsrepo"
	b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58"
	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
)

var FileStoreCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Interact with filestore objects",
	},
	Subcommands: map[string]*cmds.Command{
		"ls":         lsFileStore,
		"verify":     verifyFileStore,
		"rm":         rmFilestoreObjs,
		"rm-invalid": rmInvalidObjs,
		//"rm-incomplete":      rmIncompleteObjs,
		"find-dangling-pins": findDanglingPins,
	},
}

var lsFileStore = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List objects in filestore",
		ShortDescription: `
List objects in the filestore.  If --quiet is specified only the
hashes are printed, otherwise the fields are as follows:
  <hash> <type> <filepath> <offset> <size>
where <type> is one of"
  leaf: to indicate a leaf node where the contents are stored
        to in the file itself
  root: to indicate a root node that represents the whole file
  other: some other kind of node that represent part of a file
and <filepath> is the part of the file the object represents.  The
part represented starts at <offset> and continues for <size> bytes.
If <offset> is the special value "-" than the "leaf" or "root" node
represents the whole file.
`,
	},
	Options: []cmds.Option{
		cmds.BoolOption("quiet", "q", "Write just hashes of objects."),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		_, fs, err := extractFilestore(req)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		quiet, _, err := res.Request().Option("quiet").Bool()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch, _ := filestore.List(fs, quiet)
		res.SetOutput(&chanWriter{ch, "", 0})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

type chanWriter struct {
	ch     <-chan filestore.ListRes
	buf    string
	offset int
}

func (w *chanWriter) Read(p []byte) (int, error) {
	if w.offset >= len(w.buf) {
		w.offset = 0
		res, more := <-w.ch
		if !more {
			return 0, io.EOF
		}
		if res.DataObj == nil {
			w.buf = res.MHash() + "\n"
		} else {
			w.buf = res.Format()
		}
	}
	sz := copy(p, w.buf[w.offset:])
	w.offset += sz
	return sz, nil
}

var verifyFileStore = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Verify objects in filestore",
		ShortDescription: `
Verify leaf nodes in the filestore, the output is:
  <status> <type> <filepath> <offset> <size>
where <type>, <filepath>, <offset> and <size> are the same as in the
"ls" command and <status> is one of:
  ok:      If the object is okay
  changed: If the object is invalid becuase the contents of the file
           have changed
  missing: If the file can not be found
  error:   If the file can be found but could not be read or some
           other error
`,
	},
	Run: func(req cmds.Request, res cmds.Response) {
		_, fs, err := extractFilestore(req)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch, _ := filestore.List(fs, false)
		rdr, wtr := io.Pipe()
		go func() {
			defer wtr.Close()
			for res := range ch {
				if !res.NoBlockData {
					continue
				}
				res.Status = filestore.Verify(fs, res.Key, res.DataObj)
				wtr.Write([]byte(res.Format()))
			}
		}()
		res.SetOutput(rdr)
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

var rmFilestoreObjs = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Remove objects from the filestore",
	},
	Arguments: []cmds.Argument{
		cmds.StringArg("hash", true, true, "Multi-hashes to remove."),
	},
	Options: []cmds.Option{
		cmds.BoolOption("quiet", "q", "Produce less output."),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		node, fs, err := extractFilestore(req)
		_ = fs
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		quiet, _, err := res.Request().Option("quiet").Bool()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		hashes := req.Arguments()
		rdr, wtr := io.Pipe()
		var rmWtr io.Writer = wtr
		if quiet {
			rmWtr = ioutil.Discard
		}
		go func() {
			numErrors := 0
			for _, mhash := range hashes {
				key := k.B58KeyDecode(mhash)
				err = delFilestoreObj(req, rmWtr, node, fs, key)
				if err != nil {
					fmt.Fprintf(wtr, "Error deleting %s: %s\n", mhash, err.Error())
					numErrors += 1
				}
			}
			if numErrors > 0 {
				wtr.CloseWithError(errors.New("Could not delete some keys."))
				return
			}
			wtr.Close()
		}()
		res.SetOutput(rdr)
		return
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

var rmInvalidObjs = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Remove invalid objects from the filestore",
		ShortDescription: `
Removes objects that have become invalid from the Filestrore up to the
reason specified in <level>.  If <level> is "changed" than remove any
blocks that have become invalid due to the contents of the underlying
file changing.  If <level> is "missing" also remove any blocks that
have become invalid because the underlying file is no longer available
due to a "No such file" or related error, but not if the file exists
but is unreadable for some reason.  If <level> is "all" remove any
blocks that fail to validate regardless of the reason.
`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("level", true, false, "one of changed, missing. or all").EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.BoolOption("quiet", "q", "Produce less output."),
		cmds.BoolOption("dry-run", "n", "Do everything except the actual delete."),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		node, fs, err := extractFilestore(req)
		_ = fs
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		args := req.Arguments()
		if len(args) != 1 {
			res.SetError(errors.New("invalid usage"), cmds.ErrNormal)
			return
		}
		mode := req.Arguments()[0]
		level := filestore.StatusMissing
		switch mode {
		case "changed":
			level = filestore.StatusChanged
		case "missing":
			level = filestore.StatusMissing
		case "all":
			level = filestore.StatusError
		default:
			res.SetError(errors.New("level must be one of: changed missing all"), cmds.ErrNormal)
		}
		quiet, _, err := res.Request().Option("quiet").Bool()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		dryRun, _, err := res.Request().Option("dry-run").Bool()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch, _ := filestore.List(fs, false)
		rdr, wtr := io.Pipe()
		var rmWtr io.Writer = wtr
		if quiet {
			rmWtr = ioutil.Discard
		}
		go func() {
			var toDel [][]byte
			for r := range ch {
				if !r.NoBlockData {
					continue
				}
				r.Status = filestore.Verify(fs, r.Key, r.DataObj)
				if r.Status >= level {
					toDel = append(toDel, r.RawHash())
					if !quiet {
						fmt.Fprintf(wtr, "will delete %s (part of %s)\n", r.MHash(), r.FilePath)
					}
				}
			}
			if dryRun {
				fmt.Fprintf(wtr, "Dry-run option specified.  Stopping.\n")
				fmt.Fprintf(wtr, "Would of deleted %d invalid objects.\n", len(toDel))
			} else {
				for _, key := range toDel {
					err = delFilestoreObj(req, rmWtr, node, fs, k.Key(key))
					if err != nil {
						mhash := b58.Encode(key)
						msg := fmt.Sprintf("Could not delete %s: %s\n", mhash, err.Error())
						wtr.CloseWithError(errors.New(msg))
						return

					}
				}
				fmt.Fprintf(wtr, "Deleted %d invalid objects.\n", len(toDel))
			}
			wtr.Close()
		}()
		res.SetOutput(rdr)
		return
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

func delFilestoreObj(req cmds.Request, out io.Writer, node *core.IpfsNode, fs *filestore.Datastore, key k.Key) error {
	err := fs.DeleteDirect(key.DsKey())
	if err != nil {
		return err
	}
	stillExists, err := node.Blockstore.Has(key)
	if err != nil {
		return err
	}
	fmt.Fprintf(out, "Deleted %s\n", key)
	if stillExists {
		return nil
	}
	_, pinned1, err := node.Pinning.IsPinnedWithType(key, "recursive")
	if err != nil {
		return err
	}
	_, pinned2, err := node.Pinning.IsPinnedWithType(key, "direct")
	if err != nil {
		return err
	}
	if pinned1 || pinned2 {
		ctx, cancel := context.WithCancel(req.Context())
		defer cancel()
		err = node.Pinning.Unpin(ctx, key, true)
		if err != nil {
			return err
		}
		err := node.Pinning.Flush()
		if err != nil {
			return err
		}
		fmt.Fprintf(out, "Unpinned %s\n", key)
	}
	return nil
}

func extractFilestore(req cmds.Request) (node *core.IpfsNode, fs *filestore.Datastore, err error) {
	node, err = req.InvocContext().GetNode()
	if err != nil {
		return
	}
	repo, ok := node.Repo.Self().(*fsrepo.FSRepo)
	if !ok {
		err = errors.New("Not a FSRepo")
		return
	}
	fs = repo.Filestore()
	return
}

var findDanglingPins = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List pinned objects that no longer exists",
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			return
		}
		r, w := io.Pipe()
		go func() {
			defer w.Close()
			err := listDanglingPins(n.Pinning.DirectKeys(), w, n.Blockstore)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			err = listDanglingPins(n.Pinning.RecursiveKeys(), w, n.Blockstore)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
		}()
		res.SetOutput(r)
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

func listDanglingPins(keys []k.Key, out io.Writer, d bs.Blockstore) error {
	for _, k := range keys {
		exists, err := d.Has(k)
		if err != nil {
			return err
		}
		if !exists {
			fmt.Fprintln(out, k.B58String())
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"reflect"

	"github.com/ligato/cn-infra/logging/logroot"
)

// Flavor is structure that contains a particular combination of plugins
// (fields of plugins)
type Flavor interface {
	// Plugins returns list of plugins.
	// Name of the plugin is supposed to be related to field name of Flavor struct
	Plugins() []*NamedPlugin
}

// ListPluginsInFlavor uses reflection to traverse top level fields of Flavor structure.
// It extracts all plugins and returns them as a slice of NamedPlugins.
func ListPluginsInFlavor(flavor Flavor) (plugins []*NamedPlugin) {
	uniqueness := map[PluginName]Plugin{}
	return listPluginsInFlavor(reflect.ValueOf(flavor), uniqueness)
}

// listPluginsInFlavor checks every field and tries to cast it to Plugin or inspect its type recursively.
func listPluginsInFlavor(flavorValue reflect.Value, uniqueness map[PluginName]Plugin) []*NamedPlugin {
	var res []*NamedPlugin

	flavorType := flavorValue.Type()

	if flavorType.Kind() == reflect.Ptr {
		flavorType = flavorType.Elem()
	}

	if flavorValue.Kind() == reflect.Ptr {
		flavorValue = flavorValue.Elem()
	}

	if !flavorValue.IsValid() {
		return res
	}

	pluginType := reflect.TypeOf((*Plugin)(nil)).Elem()

	if flavorType.Kind() == reflect.Struct {
		numField := flavorType.NumField()
		for i := 0; i < numField; i++ {
			field := flavorType.Field(i)

			exported := field.PkgPath == "" // PkgPath is empty for exported fields
			if !exported {
				continue
			}

			fieldVal := flavorValue.Field(i)
			plug := fieldPlugin(field, fieldVal, pluginType)
			if plug != nil {
				_, found := uniqueness[PluginName(field.Name)]
				if !found {
					uniqueness[PluginName(field.Name)] = plug
					res = append(res, &NamedPlugin{PluginName: PluginName(field.Name), Plugin: plug})
				}
			} else {
				// try to inspect flavor structure recursively
				res = append(res, listPluginsInFlavor(fieldVal, uniqueness)...)
			}
		}
	}

	return res
}

// fieldPlugin tries to cast given field to Plugin
func fieldPlugin(field reflect.StructField, fieldVal reflect.Value, pluginType reflect.Type) Plugin {
	switch fieldVal.Kind() {
	case reflect.Struct:
		ptrType := reflect.PtrTo(fieldVal.Type())
		if ptrType.Implements(pluginType) && fieldVal.CanAddr() {
			if plug, ok := fieldVal.Addr().Interface().(Plugin); ok {
				return plug
			}
		}
	case reflect.Ptr, reflect.Interface:
		if fieldVal.IsNil() {
			logroot.StandardLogger().WithField("fieldName", field.Name).Debug("Field is nil ", pluginType)
		} else if plug, ok := fieldVal.Interface().(Plugin); ok {
			return plug
		}

	}
	return nil
}
<commit_msg>Added error logging for cases where a field in a Flavor structure is neither a Plugin or another Flavor<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"errors"
	"reflect"

	"github.com/ligato/cn-infra/logging/logroot"
)

// Flavor is structure that contains a particular combination of plugins
// (fields of plugins)
type Flavor interface {
	// Plugins returns list of plugins.
	// Name of the plugin is supposed to be related to field name of Flavor struct
	Plugins() []*NamedPlugin
}

// ListPluginsInFlavor uses reflection to traverse top level fields of Flavor structure.
// It extracts all plugins and returns them as a slice of NamedPlugins.
func ListPluginsInFlavor(flavor Flavor) (plugins []*NamedPlugin) {
	uniqueness := map[PluginName]Plugin{}
	l, err := listPluginsInFlavor(reflect.ValueOf(flavor), uniqueness)
	if err != nil {
		logroot.StandardLogger().Error("Invalid argument - it does not satisfy the Flavor interface")
	}
	return l
}

// listPluginsInFlavor checks every field and tries to cast it to Plugin or inspect its type recursively.
func listPluginsInFlavor(flavorValue reflect.Value, uniqueness map[PluginName]Plugin) ([]*NamedPlugin, error) {
	var res []*NamedPlugin

	flavorType := flavorValue.Type()

	if flavorType.Kind() == reflect.Ptr {
		flavorType = flavorType.Elem()
	}

	if flavorValue.Kind() == reflect.Ptr {
		flavorValue = flavorValue.Elem()
	}

	if !flavorValue.IsValid() {
		return res, nil
	}

	if _, ok := flavorValue.Addr().Interface().(Flavor); !ok {
		return res, errors.New("does not satisfy the Flavor interface")
	}

	pluginType := reflect.TypeOf((*Plugin)(nil)).Elem()

	if flavorType.Kind() == reflect.Struct {
		numField := flavorType.NumField()
		for i := 0; i < numField; i++ {
			field := flavorType.Field(i)

			exported := field.PkgPath == "" // PkgPath is empty for exported fields
			if !exported {
				continue
			}

			fieldVal := flavorValue.Field(i)
			plug := fieldPlugin(field, fieldVal, pluginType)
			if plug != nil {
				_, found := uniqueness[PluginName(field.Name)]
				if !found {
					uniqueness[PluginName(field.Name)] = plug
					res = append(res, &NamedPlugin{PluginName: PluginName(field.Name), Plugin: plug})
				}
			} else {
				// try to inspect flavor structure recursively
				l, err := listPluginsInFlavor(fieldVal, uniqueness)
				if err != nil {
					logroot.StandardLogger().
						WithField("fieldName", field.Name).
						Error("Bad field: must satisfy either Plugin or Flavor interface")
				} else {
					res = append(res, l...)
				}
			}
		}
	}

	return res, nil
}

// fieldPlugin determines if a given field satisfies the Plugin interface.
// If yes, the plugin value is returned; if not, nil is returned
func fieldPlugin(field reflect.StructField, fieldVal reflect.Value, pluginType reflect.Type) Plugin {
	switch fieldVal.Kind() {
	case reflect.Struct:
		ptrType := reflect.PtrTo(fieldVal.Type())
		if ptrType.Implements(pluginType) && fieldVal.CanAddr() {
			if plug, ok := fieldVal.Addr().Interface().(Plugin); ok {
				return plug
			}
		}
	case reflect.Ptr, reflect.Interface:
		if fieldVal.IsNil() {
			logroot.StandardLogger().WithField("fieldName", field.Name).Debug("Field is nil ", pluginType)
		} else if plug, ok := fieldVal.Interface().(Plugin); ok {
			return plug
		}

	}
	return nil
}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2013, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"time"
	"syscall"
)

const (
	nanosecondsPerMillisecond float64 = 1000000.0
)

// Returns the time from the milliseconds since epoch. This returns the time in UTC.
func TimeFromMillis(timeInMillis int64) *time.Time {
	theTime := time.Unix(timeInMillis/1000, 0)
	location, _ := time.LoadLocation("UTC")
	theTime = theTime.In(location)
	return &theTime
}

// Convert a time struct to milliseconds since epoch.
func TimeToMillis(tv *time.Time) int64 { return tv.UnixNano() / 1e6 }

func NowInUtc() *time.Time {
	location, _ := time.LoadLocation("UTC")
	time := time.Now().In(location)
	return &time
}

// Convert a duration to milliseconds.
func DurationToMillis(dur *time.Duration) int64 { return int64(float64(dur.Nanoseconds()) / nanosecondsPerMillisecond) }

// Get the current time in millis since epoch. Source from stackoverflow:
// http://stackoverflow.com/questions/6161839/go-time-milliseconds
func CurrentTimeInMillis() int64 {
	tv := new(syscall.Timeval)
	syscall.Gettimeofday(tv)
	return (int64(tv.Sec)*1e3 + int64(tv.Usec)/1e3)
}

// Returns the current time in seconds since epoch (i.e., a unix timestamp). Source from stackoverflow:
// http://stackoverflow.com/questions/9539108/obtaining-a-unix-timestamp-in-go-language-current-time-in-seconds-since-epoch
func CurrentTimeInSeconds() int32 { return int32(time.Now().Unix()) }

// NowTimeUnixStr returns the date in unix date string format e.g., Wed Dec 11 19:03:18 EST 2013
func NowTimeUnixStr() string { return time.Now().Format(time.UnixDate) }

<commit_msg>added conv method.<commit_after>/**
 * (C) Copyright 2013, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"time"
	"syscall"
)

const (
	nanosecondsPerMillisecond float64 = 1000000.0
)

// Returns the time from the milliseconds since epoch. This returns the time in UTC.
func TimeFromMillis(timeInMillis int64) *time.Time {
	theTime := time.Unix(timeInMillis/1000, 0)
	location, _ := time.LoadLocation("UTC")
	theTime = theTime.In(location)
	return &theTime
}

// Convert a time struct to milliseconds since epoch.
func TimeToMillis(tv *time.Time) int64 { return tv.UnixNano() / 1e6 }

func NowInUtc() *time.Time {
	location, _ := time.LoadLocation("UTC")
	time := time.Now().In(location)
	return &time
}

func NowInUtcMinusSeconds(seconds int) *time.Time {
	now := NowInUtc()
	adjusted := now.Add((time.Duration(seconds)*time.Second)*-1)
	return &adjusted
}

// Convert a duration to milliseconds.
func DurationToMillis(dur *time.Duration) int64 { return int64(float64(dur.Nanoseconds()) / nanosecondsPerMillisecond) }

// Get the current time in millis since epoch. Source from stackoverflow:
// http://stackoverflow.com/questions/6161839/go-time-milliseconds
func CurrentTimeInMillis() int64 {
	tv := new(syscall.Timeval)
	syscall.Gettimeofday(tv)
	return (int64(tv.Sec)*1e3 + int64(tv.Usec)/1e3)
}

// Returns the current time in seconds since epoch (i.e., a unix timestamp). Source from stackoverflow:
// http://stackoverflow.com/questions/9539108/obtaining-a-unix-timestamp-in-go-language-current-time-in-seconds-since-epoch
func CurrentTimeInSeconds() int32 { return int32(time.Now().Unix()) }

// NowTimeUnixStr returns the date in unix date string format e.g., Wed Dec 11 19:03:18 EST 2013
func NowTimeUnixStr() string { return time.Now().Format(time.UnixDate) }

<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"
	"time"

	"github.com/dynport/gocloud/aws/s3"
	"github.com/dynport/gossh"
)

var logger = log.New(os.Stderr, "", 0)

const sshExample = "ubuntu@127.0.0.1"

func main() {
	dir := flag.String("dir", "", "Dir to build. Default: current directory")
	host := flag.String("host", os.Getenv("DEV_HOST"), "Host to build on. Example: "+sshExample)
	deploy := flag.String("deploy", "", "Deploy to host after building. Example: "+sshExample)
	bucket := flag.String("bucket", "", "Upload binary to s3 bucket after building")
	verbose := flag.Bool("verbose", false, "Build using -v flag")

	flag.Parse()
	logger.Printf("running with %q", *host)
	b := &build{Host: *host, Dir: *dir, DeployTo: *deploy, Bucket: *bucket, verbose: *verbose}
	e := b.Run()
	if e != nil {
		logger.Fatalf("ERROR: %s", e)
	}
}

type build struct {
	Host     string
	Dir      string
	Bucket   string
	DeployTo string
	verbose  bool
}

func benchmark(message string) func() {
	started := time.Now()
	return func() {
		logger.Printf("finished %s in %.06f", message, time.Since(started).Seconds())
	}
}

func (r *build) deps() ([]string, error) {
	s, e := r.exec("go", "list", "-f", `{{ join .Deps " " }}`)
	if e != nil {
		return nil, e
	}
	return strings.Fields(s), nil
}

func (r *build) exec(cmd string, vals ...string) (string, error) {
	c := exec.Command(cmd, vals...)
	c.Dir = r.Dir
	out, e := c.CombinedOutput()
	if e != nil {
		return "", e
	}
	return string(out), nil
}

func (r *build) currentPackage() (string, error) {
	s, e := r.exec("go", "list")
	if e != nil {
		return "", e
	}
	return strings.TrimSpace(s), nil
}

func (r *build) filesMap() (map[string]os.FileInfo, error) {
	cp, e := r.currentPackage()
	if e != nil {
		return nil, e
	}
	pkgs, e := r.deps()
	if e != nil {
		return nil, e
	}
	pkgs = append(pkgs, cp)
	files := map[string]os.FileInfo{}
	sum := int64(0)
	for _, p := range pkgs {
		if !strings.Contains(p, ".") {
			continue
		}
		prefix := os.ExpandEnv("$GOPATH/src")
		dbg.Printf("walking %q", p)
		e := filepath.Walk(os.ExpandEnv(prefix+"/"+p+"/"), func(p string, info os.FileInfo, e error) error {
			skip := func() bool {
				for _, s := range []string{".git", ".bzr", ".hg"} {
					if strings.Contains(p, "/"+s+"/") {
						return true
					}
				}
				return false
			}()
			if skip {
				return nil
			}
			if _, ok := files[p]; !ok {
				sum += info.Size()
				files[p] = info
			}
			return nil
		})
		if e != nil {
			return nil, e
		}
	}
	return files, nil
}

func (b *build) createArchive() (string, error) {
	defer benchmark("create archive")()
	var name string
	e := func() error {
		f, e := ioutil.TempFile("/tmp", "gobuild-archive-")
		if e != nil {
			return e
		}
		name = f.Name()
		defer f.Close()
		files, e := b.filesMap()
		if e != nil {
			return e
		}

		gz := gzip.NewWriter(f)
		sum := int64(0)
		defer gz.Close()
		t := tar.NewWriter(gz)
		defer t.Close()

		for p, info := range files {
			name := strings.TrimPrefix(p, os.ExpandEnv("$GOPATH/src/"))
			if info.IsDir() {
				continue
			}
			dbg.Printf("adding %q", p)
			h := &tar.Header{ModTime: info.ModTime(), Size: info.Size(), Mode: int64(info.Mode()), Name: name}
			e = t.WriteHeader(h)
			if e != nil {
				return e
			}
			e := func() error {
				f, e := os.Open(p)
				if e != nil {
					return e
				}
				defer f.Close()
				i, e := io.Copy(t, f)
				sum += i
				return e
			}()
			if e != nil {
				return e
			}
		}
		dbg.Printf("%s", sizePretty(sum))
		return nil
	}()
	return name, e
}

type buildConfig struct {
	Current string
	Sudo    bool
	Verbose bool
	Version string
}

func (b *buildConfig) Goroot() string {
	return "{{ .BuildHome }}/.go/go-{{ .Version }}/go"
}

func (b *buildConfig) Gopath() string {
	return "{{ .BuildHome }}/{{ .Current }}"
}

func (b *buildConfig) BuildHome() string {
	return "$HOME/.gobuild"
}

func (b *buildConfig) BinName() string {
	return path.Base(b.Current)
}

func (b *build) Run() error {
	defer benchmark("build")()
	currentPkg, e := b.currentPackage()
	if e != nil {
		return e
	}

	cfg, e := parseConfig(b.Host)
	if e != nil {
		return e
	}
	dbg.Printf("using config %#v", cfg)
	con, e := cfg.Connection()
	if e != nil {
		return e
	}
	defer con.Close()

	name, e := b.createArchive()
	if e != nil {
		return e
	}
	dbg.Printf("created archive at %q", name)
	defer os.RemoveAll(name)
	f, e := os.Open(name)
	if e != nil {
		return e
	}
	defer f.Close()

	ses, e := con.NewSession()
	if e != nil {
		return e
	}
	ses.Stdin = f
	ses.Stdout = os.Stdout
	ses.Stderr = os.Stderr

	buildCfg := &buildConfig{
		Current: currentPkg,
		Sudo:    cfg.User != "root",
		Verbose: b.verbose,
		Version: "1.3.1",
	}

	cmd := renderRecursive(buildCmd, buildCfg)
	e = ses.Run(cmd)
	if e != nil {
		return e
	}

	name = path.Base(currentPkg)
	var binPath string

	if b.Bucket != "" || b.DeployTo != "" {
		defer os.RemoveAll(binPath)
		e = func() error {
			ses, e := con.NewSession()
			if e != nil {
				return e
			}
			defer ses.Close()

			f, e := ioutil.TempFile("/tmp", "gobuild-bin-")
			if e != nil {
				return e
			}
			binPath = f.Name()
			defer f.Close()
			ses.Stdout = f
			ses.Stderr = os.Stderr

			cmd := renderRecursive("cat {{ .Gopath }}/bin/{{ .BinName }}", buildCfg)
			e = ses.Run(cmd)
			if e != nil {
				return e
			}
			return nil
		}()
		if e != nil {
			return e
		}
	}
	if name != "" {
		defer os.RemoveAll(binPath)
	}
	if b.Bucket != "" {
		e = func() error {
			f, e := os.Open(binPath)
			if e != nil {
				return e
			}
			defer f.Close()
			client := s3.NewFromEnv()
			client.CustomEndpointHost = "s3-eu-west-1.amazonaws.com"
			bucket, key := bucketAndKey(b.Bucket, name)
			logger.Printf("uploading to bucket=%q key=%q", bucket, key)
			return client.PutStream(bucket, key, f, nil)
		}()
		if e != nil {
			return e
		}
		logger.Printf("uploaded to bucket %q", b.Bucket)
	}

	if b.DeployTo != "" {
		e := func() error {
			cfg, e := parseConfig(b.DeployTo)
			if e != nil {
				return e
			}
			con, e := cfg.Connection()
			if e != nil {
				return e
			}
			defer con.Close()
			ses, e := con.NewSession()
			if e != nil {
				return e
			}
			defer ses.Close()
			f, e := os.Open(binPath)
			if e != nil {
				return e
			}
			defer f.Close()
			ses.Stdin = f
			ses.Stdout = os.Stdout
			ses.Stderr = os.Stderr
			s := struct {
				Name string
				Sudo bool
			}{
				Name: name, Sudo: cfg.User != "root",
			}
			cmd := renderRecursive("cd /usr/local/bin && cat - | {{ if .Sudo }}sudo {{ end}}tee {{ .Name }}.tmp > /dev/null && {{ if .Sudo }}sudo {{ end }}chmod 0755 {{ .Name }}.tmp && {{ if .Sudo }}sudo {{ end }}mv {{ .Name }}.tmp {{ .Name }}", s)
			dbg.Printf("%s", cmd)
			return ses.Run(cmd)
		}()
		if e != nil {
			return e
		}
	}
	return nil
}

func bucketAndKey(bucketWithPrefix, name string) (bucket, key string) {
	parts := strings.Split(bucketWithPrefix, "/")
	bucket = parts[0]
	key = name
	if len(parts) > 1 {
		key = strings.TrimSuffix(strings.Join(parts[1:], "/"), "/") + "/" + key
	}
	return bucket, key
}

func parseConfig(s string) (*gossh.Config, error) {
	cfg := &gossh.Config{}
	parts := strings.Split(s, "@")
	hostAndPort := ""
	switch len(parts) {
	case 0:
		return nil, fmt.Errorf("Host must be set")
	case 1:
		hostAndPort = parts[0]
	case 2:
		cfg.User = parts[0]
		hostAndPort = parts[1]
	case 3:
		return nil, fmt.Errorf("format of host %q not understood", s)
	}
	parts = strings.Split(hostAndPort, ":")
	cfg.Host = parts[0]
	if len(parts) == 2 {
		var e error
		cfg.Port, e = strconv.Atoi(parts[1])
		if e != nil {
			return nil, e
		}
	}
	return cfg, nil
}

const buildCmd = `#!/bin/bash
export BUILD_HOME={{ .BuildHome }}
export GOPATH={{ .Gopath }}
export GOROOT={{ .Goroot }}
export PATH=$GOROOT/bin:$PATH

if [ ! -f $GOROOT/bin/go ]; then
  echo "installing go {{ .Version }}"
  tmp=$(dirname $GOROOT)
  mkdir -p $tmp
  cd $tmp
  curl -sL "https://storage.googleapis.com/golang/go{{ .Version }}.linux-amd64.tar.gz" | tar xfz -
fi

set -xe
rm -Rf $GOPATH
mkdir -p $GOPATH/src
cd $GOPATH/src
tar xfz -
cd {{ .Current }}
go get {{ if .Verbose }}-v{{ end }} .
{{ with .Sudo }}sudo {{ end }}cp $GOPATH/bin/{{ .BinName }} /usr/local/bin/
`

func debugStream() io.Writer {
	if os.Getenv("DEBUG") == "true" {
		return os.Stderr
	}
	return ioutil.Discard
}

var dbg = log.New(debugStream(), "[DEBUG] ", log.Lshortfile)

func renderRecursive(tpl string, i interface{}) string {
	s := tpl
	for j := 0; j < 10; j++ {
		rendered := mustRender([]byte(s), i)
		if rendered == s {
			return rendered
		}
		s = rendered
	}
	logger.Fatal("rendering loop, rendered 10 times")
	return ""
}

func mustRender(raw []byte, i interface{}) string {
	out, e := render(raw, i)
	if e != nil {
		logger.Fatal(e)
	}
	return out
}

func render(raw []byte, i interface{}) (string, error) {
	tpl, e := template.New(string(raw)).Parse(string(raw))
	if e != nil {
		return "", e
	}
	buf := &bytes.Buffer{}
	e = tpl.Execute(buf, i)
	if e != nil {
		return "", e
	}
	return buf.String(), nil
}

var (
	oneKb = 1024.0
	oneMb = oneKb * 1024.0
	oneGb = oneMb * 1024.0
)

func sizePretty(raw int64) string {
	f := float64(raw)
	if f < oneKb {
		return fmt.Sprintf("%.0f", f)
	} else if f < oneMb {
		return fmt.Sprintf("%.2fKB", f/oneKb)
	} else if f < oneGb {
		return fmt.Sprintf("%.2fMB", f/oneMb)
	} else {
		return fmt.Sprintf("%.2fGB", f/oneGb)
	}
}
<commit_msg>allow public uploads to s3<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"
	"time"

	"github.com/dynport/gocloud/aws/s3"
	"github.com/dynport/gossh"
)

var logger = log.New(os.Stderr, "", 0)

const sshExample = "ubuntu@127.0.0.1"

func main() {
	dir := flag.String("dir", "", "Dir to build. Default: current directory")
	host := flag.String("host", os.Getenv("DEV_HOST"), "Host to build on. Example: "+sshExample)
	deploy := flag.String("deploy", "", "Deploy to host after building. Example: "+sshExample)
	bucket := flag.String("bucket", "", "Upload binary to s3 bucket after building")
	public := flag.Bool("public", false, "Upload to s3 and make public")
	verbose := flag.Bool("verbose", false, "Build using -v flag")

	flag.Parse()
	logger.Printf("running with %q", *host)
	b := &build{Host: *host, Dir: *dir, DeployTo: *deploy, Bucket: *bucket, verbose: *verbose, Public: *public}
	e := b.Run()
	if e != nil {
		logger.Fatalf("ERROR: %s", e)
	}
}

type build struct {
	Host     string
	Dir      string
	Bucket   string
	Public   bool
	DeployTo string
	verbose  bool
}

func benchmark(message string) func() {
	started := time.Now()
	return func() {
		logger.Printf("finished %s in %.06f", message, time.Since(started).Seconds())
	}
}

func (r *build) deps() ([]string, error) {
	s, e := r.exec("go", "list", "-f", `{{ join .Deps " " }}`)
	if e != nil {
		return nil, e
	}
	return strings.Fields(s), nil
}

func (r *build) exec(cmd string, vals ...string) (string, error) {
	c := exec.Command(cmd, vals...)
	c.Dir = r.Dir
	out, e := c.CombinedOutput()
	if e != nil {
		return "", e
	}
	return string(out), nil
}

func (r *build) currentPackage() (string, error) {
	s, e := r.exec("go", "list")
	if e != nil {
		return "", e
	}
	return strings.TrimSpace(s), nil
}

func (r *build) filesMap() (map[string]os.FileInfo, error) {
	cp, e := r.currentPackage()
	if e != nil {
		return nil, e
	}
	pkgs, e := r.deps()
	if e != nil {
		return nil, e
	}
	pkgs = append(pkgs, cp)
	files := map[string]os.FileInfo{}
	sum := int64(0)
	for _, p := range pkgs {
		if !strings.Contains(p, ".") {
			continue
		}
		prefix := os.ExpandEnv("$GOPATH/src")
		dbg.Printf("walking %q", p)
		e := filepath.Walk(os.ExpandEnv(prefix+"/"+p+"/"), func(p string, info os.FileInfo, e error) error {
			skip := func() bool {
				for _, s := range []string{".git", ".bzr", ".hg"} {
					if strings.Contains(p, "/"+s+"/") {
						return true
					}
				}
				return false
			}()
			if skip {
				return nil
			}
			if _, ok := files[p]; !ok {
				sum += info.Size()
				files[p] = info
			}
			return nil
		})
		if e != nil {
			return nil, e
		}
	}
	return files, nil
}

func (b *build) createArchive() (string, error) {
	defer benchmark("create archive")()
	var name string
	e := func() error {
		f, e := ioutil.TempFile("/tmp", "gobuild-archive-")
		if e != nil {
			return e
		}
		name = f.Name()
		defer f.Close()
		files, e := b.filesMap()
		if e != nil {
			return e
		}

		gz := gzip.NewWriter(f)
		sum := int64(0)
		defer gz.Close()
		t := tar.NewWriter(gz)
		defer t.Close()

		for p, info := range files {
			name := strings.TrimPrefix(p, os.ExpandEnv("$GOPATH/src/"))
			if info.IsDir() {
				continue
			}
			dbg.Printf("adding %q", p)
			h := &tar.Header{ModTime: info.ModTime(), Size: info.Size(), Mode: int64(info.Mode()), Name: name}
			e = t.WriteHeader(h)
			if e != nil {
				return e
			}
			e := func() error {
				f, e := os.Open(p)
				if e != nil {
					return e
				}
				defer f.Close()
				i, e := io.Copy(t, f)
				sum += i
				return e
			}()
			if e != nil {
				return e
			}
		}
		dbg.Printf("%s", sizePretty(sum))
		return nil
	}()
	return name, e
}

type buildConfig struct {
	Current string
	Sudo    bool
	Verbose bool
	Version string
}

func (b *buildConfig) Goroot() string {
	return "{{ .BuildHome }}/.go/go-{{ .Version }}/go"
}

func (b *buildConfig) Gopath() string {
	return "{{ .BuildHome }}/{{ .Current }}"
}

func (b *buildConfig) BuildHome() string {
	return "$HOME/.gobuild"
}

func (b *buildConfig) BinName() string {
	return path.Base(b.Current)
}

func (b *build) Run() error {
	defer benchmark("build")()
	currentPkg, e := b.currentPackage()
	if e != nil {
		return e
	}

	cfg, e := parseConfig(b.Host)
	if e != nil {
		return e
	}
	dbg.Printf("using config %#v", cfg)
	con, e := cfg.Connection()
	if e != nil {
		return e
	}
	defer con.Close()

	name, e := b.createArchive()
	if e != nil {
		return e
	}
	dbg.Printf("created archive at %q", name)
	defer os.RemoveAll(name)
	f, e := os.Open(name)
	if e != nil {
		return e
	}
	defer f.Close()

	ses, e := con.NewSession()
	if e != nil {
		return e
	}
	ses.Stdin = f
	ses.Stdout = os.Stdout
	ses.Stderr = os.Stderr

	buildCfg := &buildConfig{
		Current: currentPkg,
		Sudo:    cfg.User != "root",
		Verbose: b.verbose,
		Version: "1.3.1",
	}

	cmd := renderRecursive(buildCmd, buildCfg)
	e = ses.Run(cmd)
	if e != nil {
		return e
	}

	name = path.Base(currentPkg)
	var binPath string

	if b.Bucket != "" || b.DeployTo != "" {
		defer os.RemoveAll(binPath)
		e = func() error {
			ses, e := con.NewSession()
			if e != nil {
				return e
			}
			defer ses.Close()

			f, e := ioutil.TempFile("/tmp", "gobuild-bin-")
			if e != nil {
				return e
			}
			binPath = f.Name()
			defer f.Close()
			ses.Stdout = f
			ses.Stderr = os.Stderr

			cmd := renderRecursive("cat {{ .Gopath }}/bin/{{ .BinName }}", buildCfg)
			e = ses.Run(cmd)
			if e != nil {
				return e
			}
			return nil
		}()
		if e != nil {
			return e
		}
	}
	if name != "" {
		defer os.RemoveAll(binPath)
	}
	if b.Bucket != "" {
		e = func() error {
			f, e := os.Open(binPath)
			if e != nil {
				return e
			}
			defer f.Close()
			client := s3.NewFromEnv()
			client.CustomEndpointHost = "s3-eu-west-1.amazonaws.com"
			bucket, key := bucketAndKey(b.Bucket, name)
			logger.Printf("uploading to bucket=%q key=%q", bucket, key)

			opts := &s3.PutOptions{}
			if b.Public {
				opts.AmzAcl = "public-read"
			}
			return client.PutStream(bucket, key, f, opts)
		}()
		if e != nil {
			return e
		}
		logger.Printf("uploaded to bucket %q", b.Bucket)
	}

	if b.DeployTo != "" {
		e := func() error {
			cfg, e := parseConfig(b.DeployTo)
			if e != nil {
				return e
			}
			con, e := cfg.Connection()
			if e != nil {
				return e
			}
			defer con.Close()
			ses, e := con.NewSession()
			if e != nil {
				return e
			}
			defer ses.Close()
			f, e := os.Open(binPath)
			if e != nil {
				return e
			}
			defer f.Close()
			ses.Stdin = f
			ses.Stdout = os.Stdout
			ses.Stderr = os.Stderr
			s := struct {
				Name string
				Sudo bool
			}{
				Name: name, Sudo: cfg.User != "root",
			}
			cmd := renderRecursive("cd /usr/local/bin && cat - | {{ if .Sudo }}sudo {{ end}}tee {{ .Name }}.tmp > /dev/null && {{ if .Sudo }}sudo {{ end }}chmod 0755 {{ .Name }}.tmp && {{ if .Sudo }}sudo {{ end }}mv {{ .Name }}.tmp {{ .Name }}", s)
			dbg.Printf("%s", cmd)
			return ses.Run(cmd)
		}()
		if e != nil {
			return e
		}
	}
	return nil
}

func bucketAndKey(bucketWithPrefix, name string) (bucket, key string) {
	parts := strings.Split(bucketWithPrefix, "/")
	bucket = parts[0]
	key = name
	if len(parts) > 1 {
		key = strings.TrimSuffix(strings.Join(parts[1:], "/"), "/") + "/" + key
	}
	return bucket, key
}

func parseConfig(s string) (*gossh.Config, error) {
	cfg := &gossh.Config{}
	parts := strings.Split(s, "@")
	hostAndPort := ""
	switch len(parts) {
	case 0:
		return nil, fmt.Errorf("Host must be set")
	case 1:
		hostAndPort = parts[0]
	case 2:
		cfg.User = parts[0]
		hostAndPort = parts[1]
	case 3:
		return nil, fmt.Errorf("format of host %q not understood", s)
	}
	parts = strings.Split(hostAndPort, ":")
	cfg.Host = parts[0]
	if len(parts) == 2 {
		var e error
		cfg.Port, e = strconv.Atoi(parts[1])
		if e != nil {
			return nil, e
		}
	}
	return cfg, nil
}

const buildCmd = `#!/bin/bash
export BUILD_HOME={{ .BuildHome }}
export GOPATH={{ .Gopath }}
export GOROOT={{ .Goroot }}
export PATH=$GOROOT/bin:$PATH

if [ ! -f $GOROOT/bin/go ]; then
  echo "installing go {{ .Version }}"
  tmp=$(dirname $GOROOT)
  mkdir -p $tmp
  cd $tmp
  curl -sL "https://storage.googleapis.com/golang/go{{ .Version }}.linux-amd64.tar.gz" | tar xfz -
fi

set -xe
rm -Rf $GOPATH
mkdir -p $GOPATH/src
cd $GOPATH/src
tar xfz -
cd {{ .Current }}
go get {{ if .Verbose }}-v{{ end }} .
{{ with .Sudo }}sudo {{ end }}cp $GOPATH/bin/{{ .BinName }} /usr/local/bin/
`

func debugStream() io.Writer {
	if os.Getenv("DEBUG") == "true" {
		return os.Stderr
	}
	return ioutil.Discard
}

var dbg = log.New(debugStream(), "[DEBUG] ", log.Lshortfile)

func renderRecursive(tpl string, i interface{}) string {
	s := tpl
	for j := 0; j < 10; j++ {
		rendered := mustRender([]byte(s), i)
		if rendered == s {
			return rendered
		}
		s = rendered
	}
	logger.Fatal("rendering loop, rendered 10 times")
	return ""
}

func mustRender(raw []byte, i interface{}) string {
	out, e := render(raw, i)
	if e != nil {
		logger.Fatal(e)
	}
	return out
}

func render(raw []byte, i interface{}) (string, error) {
	tpl, e := template.New(string(raw)).Parse(string(raw))
	if e != nil {
		return "", e
	}
	buf := &bytes.Buffer{}
	e = tpl.Execute(buf, i)
	if e != nil {
		return "", e
	}
	return buf.String(), nil
}

var (
	oneKb = 1024.0
	oneMb = oneKb * 1024.0
	oneGb = oneMb * 1024.0
)

func sizePretty(raw int64) string {
	f := float64(raw)
	if f < oneKb {
		return fmt.Sprintf("%.0f", f)
	} else if f < oneMb {
		return fmt.Sprintf("%.2fKB", f/oneKb)
	} else if f < oneGb {
		return fmt.Sprintf("%.2fMB", f/oneMb)
	} else {
		return fmt.Sprintf("%.2fGB", f/oneGb)
	}
}
<|endoftext|>