<commit_before>package wiki

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/fcgi"
	"os"
	"regexp"
	"text/template"
	"time"

	"errors"

	"github.com/mahata/gwiki/util"
	"github.com/russross/blackfriday"
)

type Config struct {
	TxtDir   string `json:"txtDir"`
	ImgDir   string `json:"imgDir`
	Password string `json:"password`
}

var templates = template.Must(template.ParseFiles("login.html", "edit.html", "view.html", "upload-file.html"))

var validPath = regexp.MustCompile(`^/(edit|save|view|static)/(\S+)$`)
var config Config

type Page struct {
	Title   string
	Content []byte
	Login   bool
}

func isExist(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

func toHash(password string) string {
	converted := sha256.Sum256([]byte(password))
	return hex.EncodeToString(converted[:])
}

func (p *Page) save() error {
	archiveDir := config.TxtDir + "/" + p.Title
	if !isExist(archiveDir) {
		err := os.Mkdir(archiveDir, 0700)
		if err != nil {
			os.Stderr.WriteString("Failed to create a directory. Is your disk full?")
			panic(err)
		}
	}
	archiveFilePath := archiveDir + "/" + string(fmt.Sprint(time.Now().Unix()))
	archiveErr := ioutil.WriteFile(archiveFilePath, p.Content, 0600)
	if archiveErr != nil {
		return archiveErr
	}
	filePath := archiveDir + ".txt"

	return ioutil.WriteFile(filePath, p.Content, 0600)
}

func loadConf(confFile string) {
	file, err := ioutil.ReadFile(confFile)
	if err != nil {
		panic(err)
	}

	json.Unmarshal(file, &config)
}

func loadPage(title string) (*Page, error) {
	filename := config.TxtDir + "/" + title + ".txt"
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return &Page{Title: title, Content: content}, nil
}

func Run(useNginx bool) {
	loadConf("config.json")

	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/login", loginHandler)
	http.HandleFunc("/upload-file", uploadFileHandler)
	http.HandleFunc("/view/", makeHandler(viewHandler))
	http.HandleFunc("/edit/", makeHandler(editHandler))
	http.HandleFunc("/save/", makeHandler(saveHandler))
	http.HandleFunc("/static/", makeHandler(staticHandler))

	if useNginx {
		listen, err := net.Listen("tcp", "0.0.0.0:8080")
		if err != nil {
			os.Stderr.WriteString("Failed to copy the HTTP server. Is port 8080 available?")
			panic(err)
		}
		fcgi.Serve(listen, nil)
	} else {
		err := http.ListenAndServe(":8080", nil)
		if err != nil {
			os.Stderr.WriteString("Failed to copy the HTTP server. Is port 8080 available?")
			panic(err)
		}
	}
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "/view/index", http.StatusFound)
}

func loginHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		p := &Page{
			Title:   "Login Page",
			Content: []byte("Login:"),
		}
		renderTemplate(w, "login", p)
	} else {
		if r.PostFormValue("password") == config.Password {
			cookie := http.Cookie{
				Name:     "login",
				Value:    toHash(config.Password),
				HttpOnly: true,
				Expires:  time.Now().AddDate(0, 1, 0),
			}
			http.SetCookie(w, &cookie)
			http.Redirect(w, r, "/", http.StatusFound)
		} else {
			http.Error(w, "Login password doesn't match", http.StatusForbidden)
		}
	}
}

func calcExtension(contentType string) (string, error) {
	switch contentType {
	case "image/jpeg":
		return ".jpg", nil
	case "image/gif":
		return ".gif", nil
	case "image/png":
		return ".png", nil
	default:
		return "", errors.New("Only jpeg, gif and png files are allowed for the file upload")
	}
}

func uploadFileHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		http.Error(w, "Upload data have to be passed with POST.", http.StatusBadRequest)
	} else {
		r.ParseMultipartForm(32 << 20) // maxMemory
		file, handler, err := r.FormFile("upload-file")
		if err != nil {
			http.Error(w, "upload-file parameter must be passed", http.StatusBadRequest)
			return
		}
		defer file.Close()

		extension, err := calcExtension(handler.Header.Get(("Content-Type")))
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		uploadFileName := util.GetRandomString() + extension
		f, err := os.OpenFile(fmt.Sprintf("%s/%s", config.ImgDir, uploadFileName), os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			http.Error(w, "Can't create a file handler. Disk full?", http.StatusInternalServerError)
			return
		}
		defer f.Close()

		_, err = io.Copy(f, file)
		if err != nil {
			http.Error(w, "Can't upload the upload-file. Disk full?", http.StatusInternalServerError)
			return
		}

		p := &Page{
			Title:   "File Posted",
			Content: []byte(fmt.Sprintf("![AltText](/static/%s \"TitleText\")", uploadFileName)),
		}
		renderTemplate(w, "upload-file", p)
	}
}

func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) {
	err := templates.ExecuteTemplate(w, tmpl+".html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func viewHandler(w http.ResponseWriter, r *http.Request, title string) {
	p, err := loadPage(title)
	if err != nil {
		http.Redirect(w, r, "/edit/"+title, http.StatusFound)
		return
	}

	cookie, err := r.Cookie("login")
	if err != nil {
		p.Login = false
	} else {
		p.Login = cookie.Value == toHash(config.Password)
	}

	p.Content = blackfriday.MarkdownCommon([]byte(p.Content))
	renderTemplate(w, "view", p)
}

func editHandler(w http.ResponseWriter, r *http.Request, title string) {
	p, err := loadPage(title)
	if err != nil {
		// Looks like there's no page to edit
		ioutil.WriteFile(fmt.Sprintf("%s/%s.txt", config.TxtDir, title), []byte(""), 0600)
		p, _ = loadPage(title)
	}

	cookie, err := r.Cookie("login")
	if err != nil {
		p.Login = false
	} else {
		p.Login = cookie.Value == toHash(config.Password)
	}

	renderTemplate(w, "edit", p)
}

func saveHandler(w http.ResponseWriter, r *http.Request, title string) {
	content := r.FormValue("content")
	p := &Page{Title: title, Content: []byte(content)}
	err := p.save()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	http.Redirect(w, r, "/view/"+title, http.StatusFound)
}

func staticHandler(w http.ResponseWriter, r *http.Request, fpath string) {
	imgPath := config.ImgDir + "/" + fpath
	_, err := os.Stat(imgPath)
	if err == nil {
		http.ServeFile(w, r, imgPath)
	} else {
		http.ServeFile(w, r, config.ImgDir+"/not-found.png") // FixMe: Add this image when installing the app
	}
}

func makeHandler(fn func(http.ResponseWriter, *http.Request, string)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		m := validPath.FindStringSubmatch(r.URL.Path)
		if m == nil {
			http.NotFound(w, r)
			return
		}
		fn(w, r, m[2])
	}
}
<commit_msg>Add super clunky first version of SQLite integration<commit_after>package wiki

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/fcgi"
	"os"
	"regexp"
	"text/template"
	"time"

	"errors"

	"database/sql"

	"github.com/mahata/gwiki/util"
	_ "github.com/mattn/go-sqlite3"
	"github.com/russross/blackfriday"
)

type Config struct {
	TxtDir   string `json:"txtDir"`
	ImgDir   string `json:"imgDir`
	Password string `json:"password`
}

var templates = template.Must(template.ParseFiles("login.html", "edit.html", "view.html", "upload-file.html"))

var validPath = regexp.MustCompile(`^/(edit|save|view|static)/(\S+)$`)
var config Config

type Page struct {
	Title   string
	Content []byte
	Login   bool
}

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}

func isExist(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

func toHash(password string) string {
	converted := sha256.Sum256([]byte(password))
	return hex.EncodeToString(converted[:])
}

func (p *Page) save() error {
	archiveDir := config.TxtDir + "/" + p.Title
	if !isExist(archiveDir) {
		err := os.Mkdir(archiveDir, 0700)
		if err != nil {
			os.Stderr.WriteString("Failed to create a directory. Is your disk full?")
			panic(err)
		}
	}
	archiveFilePath := archiveDir + "/" + string(fmt.Sprint(time.Now().Unix()))
	archiveErr := ioutil.WriteFile(archiveFilePath, p.Content, 0600)
	if archiveErr != nil {
		return archiveErr
	}
	filePath := archiveDir + ".txt"

	// FixMe: WIP
	db, err := sql.Open("sqlite3", "./sample.sqlite3")
	checkErr(err)
	stmt, err := db.Prepare("INSERT INTO wiki (title, content, unixtime) values(?, ?, ?)")
	checkErr(err)
	res, err := stmt.Exec(p.Title, p.Content, string(fmt.Sprint(time.Now().Unix())))
	checkErr(err)
	id, err := res.LastInsertId()
	checkErr(err)
	fmt.Println(id)

	return ioutil.WriteFile(filePath, p.Content, 0600)
}

func loadConf(confFile string) {
	file, err := ioutil.ReadFile(confFile)
	if err != nil {
		panic(err)
	}

	json.Unmarshal(file, &config)
}

func loadPage(title string) (*Page, error) {
	filename := config.TxtDir + "/" + title + ".txt"
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	//db, err := sql.Open("sqlite3", "./sample.sqlite3")
	//checkErr(err)
	//
	//rows, err := db.Query("SELECT * FROM wiki")
	//checkErr(err)
	//for rows.Next() {
	//	var id int
	//	var title string
	//	var content string
	//	var unixtime int
	//	err = rows.Scan(&id, &title, &content, &unixtime)
	//	fmt.Println(id, title, content, unixtime)
	//}

	return &Page{Title: title, Content: content}, nil
}

func Run(useNginx bool) {
	loadConf("config.json")

	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/login", loginHandler)
	http.HandleFunc("/upload-file", uploadFileHandler)
	http.HandleFunc("/view/", makeHandler(viewHandler))
	http.HandleFunc("/edit/", makeHandler(editHandler))
	http.HandleFunc("/save/", makeHandler(saveHandler))
	http.HandleFunc("/static/", makeHandler(staticHandler))

	if useNginx {
		listen, err := net.Listen("tcp", "0.0.0.0:8080")
		if err != nil {
			os.Stderr.WriteString("Failed to copy the HTTP server. Is port 8080 available?")
			panic(err)
		}
		fcgi.Serve(listen, nil)
	} else {
		err := http.ListenAndServe(":8080", nil)
		if err != nil {
			os.Stderr.WriteString("Failed to copy the HTTP server. Is port 8080 available?")
			panic(err)
		}
	}
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "/view/index", http.StatusFound)
}

func loginHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		p := &Page{
			Title:   "Login Page",
			Content: []byte("Login:"),
		}
		renderTemplate(w, "login", p)
	} else {
		if r.PostFormValue("password") == config.Password {
			cookie := http.Cookie{
				Name:     "login",
				Value:    toHash(config.Password),
				HttpOnly: true,
				Expires:  time.Now().AddDate(0, 1, 0),
			}
			http.SetCookie(w, &cookie)
			http.Redirect(w, r, "/", http.StatusFound)
		} else {
			http.Error(w, "Login password doesn't match", http.StatusForbidden)
		}
	}
}

func calcExtension(contentType string) (string, error) {
	switch contentType {
	case "image/jpeg":
		return ".jpg", nil
	case "image/gif":
		return ".gif", nil
	case "image/png":
		return ".png", nil
	default:
		return "", errors.New("Only jpeg, gif and png files are allowed for the file upload")
	}
}

func uploadFileHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		http.Error(w, "Upload data have to be passed with POST.", http.StatusBadRequest)
	} else {
		r.ParseMultipartForm(32 << 20) // maxMemory
		file, handler, err := r.FormFile("upload-file")
		if err != nil {
			http.Error(w, "upload-file parameter must be passed", http.StatusBadRequest)
			return
		}
		defer file.Close()

		extension, err := calcExtension(handler.Header.Get(("Content-Type")))
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		uploadFileName := util.GetRandomString() + extension
		f, err := os.OpenFile(fmt.Sprintf("%s/%s", config.ImgDir, uploadFileName), os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			http.Error(w, "Can't create a file handler. Disk full?", http.StatusInternalServerError)
			return
		}
		defer f.Close()

		_, err = io.Copy(f, file)
		if err != nil {
			http.Error(w, "Can't upload the upload-file. Disk full?", http.StatusInternalServerError)
			return
		}

		p := &Page{
			Title:   "File Posted",
			Content: []byte(fmt.Sprintf("![AltText](/static/%s \"TitleText\")", uploadFileName)),
		}
		renderTemplate(w, "upload-file", p)
	}
}

func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) {
	err := templates.ExecuteTemplate(w, tmpl+".html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func viewHandler(w http.ResponseWriter, r *http.Request, title string) {
	p, err := loadPage(title)
	if err != nil {
		http.Redirect(w, r, "/edit/"+title, http.StatusFound)
		return
	}

	cookie, err := r.Cookie("login")
	if err != nil {
		p.Login = false
	} else {
		p.Login = cookie.Value == toHash(config.Password)
	}

	p.Content = blackfriday.MarkdownCommon([]byte(p.Content))
	renderTemplate(w, "view", p)
}

func editHandler(w http.ResponseWriter, r *http.Request, title string) {
	p, err := loadPage(title)
	if err != nil {
		// Looks like there's no page to edit
		ioutil.WriteFile(fmt.Sprintf("%s/%s.txt", config.TxtDir, title), []byte(""), 0600)
		p, _ = loadPage(title)
	}

	cookie, err := r.Cookie("login")
	if err != nil {
		p.Login = false
	} else {
		p.Login = cookie.Value == toHash(config.Password)
	}

	renderTemplate(w, "edit", p)
}

func saveHandler(w http.ResponseWriter, r *http.Request, title string) {
	content := r.FormValue("content")
	p := &Page{Title: title, Content: []byte(content)}
	err := p.save()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	http.Redirect(w, r, "/view/"+title, http.StatusFound)
}

func staticHandler(w http.ResponseWriter, r *http.Request, fpath string) {
	imgPath := config.ImgDir + "/" + fpath
	_, err := os.Stat(imgPath)
	if err == nil {
		http.ServeFile(w, r, imgPath)
	} else {
		http.ServeFile(w, r, config.ImgDir+"/not-found.png") // FixMe: Add this image when installing the app
	}
}

func makeHandler(fn func(http.ResponseWriter, *http.Request, string)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		m := validPath.FindStringSubmatch(r.URL.Path)
		if m == nil {
			http.NotFound(w, r)
			return
		}
		fn(w, r, m[2])
	}
}
<|endoftext|>


<commit_before>package cache

import (
	"sort"
	"sync"

	"github.com/raintank/metrictank/mdata/cache/accnt"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/worldping-api/pkg/log"
)

type CCacheMetric struct {
	sync.RWMutex

	// points at cached data chunks, indexed by their according time stamp
	chunks map[uint32]*CCacheChunk
}

func NewCCacheMetric() *CCacheMetric {
	return &CCacheMetric{
		chunks: make(map[uint32]*CCacheChunk),
	}
}

func (mc *CCacheMetric) Init(prev uint32, itergen chunk.IterGen) {
	mc.Add(prev, itergen)
}

func (mc *CCacheMetric) Del(ts uint32) int {
	mc.Lock()
	defer mc.Unlock()

	if _, ok := mc.chunks[ts]; !ok {
		return len(mc.chunks)
	}

	prev := mc.chunks[ts].Prev
	next := mc.chunks[ts].Next

	if _, ok := mc.chunks[prev]; prev != 0 && ok {
		mc.chunks[prev].Next = 0
	}
	if _, ok := mc.chunks[next]; next != 0 && ok {
		mc.chunks[next].Prev = 0
	}

	delete(mc.chunks, ts)

	return len(mc.chunks)
}

func (mc *CCacheMetric) Add(prev uint32, itergen chunk.IterGen) {
	ts := itergen.Ts()

	mc.Lock()
	defer mc.Unlock()

	if _, ok := mc.chunks[ts]; ok {
		// chunk is already present. no need to error on that, just ignore it
		return
	}

	mc.chunks[ts] = &CCacheChunk{
		Ts:    ts,
		Prev:  0,
		Next:  0,
		Itgen: itergen,
	}

	nextTs := mc.nextTs(ts)

	log.Debug("CCacheMetric Add: caching chunk ts %d, nextTs %d", ts, nextTs)

	// if previous chunk has not been passed we try to be smart and figure it out.
	// this is common in a scenario where a metric continuously gets queried
	// for a range that starts less than one chunkspan before now().
	if prev == 0 {
		ts, ok := mc.seekDesc(ts-1, mc.sortedTs())
		if ok {
			prev = ts
		}
	}

	// if the previous chunk is cached, link in both directions
	if _, ok := mc.chunks[prev]; ok {
		mc.chunks[prev].Next = ts
		mc.chunks[ts].Prev = prev
	}

	// if nextTs() can't figure out the end date it returns ts
	if nextTs > ts {
		// if the next chunk is cached, link in both directions
		if _, ok := mc.chunks[nextTs]; ok {
			mc.chunks[nextTs].Prev = ts
			mc.chunks[ts].Next = nextTs
		}
	}

	return
}

// get sorted slice of all chunk timestamps
// assumes we have at least read lock
func (mc *CCacheMetric) sortedTs() []uint32 {
	keys := make([]uint32, 0, len(mc.chunks))
	for k := range mc.chunks {
		keys = append(keys, k)
	}
	sort.Sort(accnt.Uint32Asc(keys))
	return keys
}

// nextTs takes a chunk's ts and returns the ts of the next chunk. (guessing if necessary)
// assumes we already have at least a read lock
func (mc *CCacheMetric) nextTs(ts uint32) uint32 {
	chunk := mc.chunks[ts]
	span := chunk.Itgen.Span()
	if span > 0 {
		// if the chunk is span-aware we don't need anything else
		return chunk.Ts + span
	}

	if chunk.Next == 0 {
		if chunk.Prev == 0 {
			// if a chunk has no next and no previous chunk we have to assume it's length is 0
			return chunk.Ts
		} else {
			// if chunk has no next chunk, but has a previous one, we assume the length of this one is same as the previous one
			return chunk.Ts + (chunk.Ts - chunk.Prev)
		}
	} else {
		// if chunk has a next chunk, then that's the ts we need
		return chunk.Next
	}
}

// seekAsc finds the t0 of the chunk that contains ts, by searching from old to recent
// if not found or can't be sure returns 0, false
// assumes we already have at least a read lock
func (mc *CCacheMetric) seekAsc(ts uint32, keys []uint32) (uint32, bool) {
	log.Debug("CCacheMetric seekAsc: seeking for %d in the keys %+d", ts, keys)

	for i := 0; i < len(keys) && keys[i] <= ts; i++ {
		if mc.nextTs(keys[i]) > ts {
			log.Debug("CCacheMetric seekAsc: seek found ts %d is between %d and %d", ts, keys[i], mc.nextTs(keys[i]))
			return keys[i], true
		}
	}

	log.Debug("CCacheMetric seekAsc: seekAsc unsuccessful")
	return 0, false
}

// seekDesc finds the t0 of the chunk that contains ts, by searching from recent to old
// if not found or can't be sure returns 0, false
// assumes we already have at least a read lock
func (mc *CCacheMetric) seekDesc(ts uint32, keys []uint32) (uint32, bool) {
	log.Debug("CCacheMetric seekDesc: seeking for %d in the keys %+d", ts, keys)

	for i := len(keys) - 1; i >= 0 && mc.nextTs(keys[i]) > ts; i-- {
		if keys[i] <= ts {
			log.Debug("CCacheMetric seekDesc: seek found ts %d is between %d and %d", ts, keys[i], mc.nextTs(keys[i]))
			return keys[i], true
		}
	}

	log.Debug("CCacheMetric seekDesc: seekDesc unsuccessful")
	return 0, false
}

func (mc *CCacheMetric) searchForward(from, until uint32, keys []uint32, res *CCSearchResult) {
	ts, ok := mc.seekAsc(from, keys)
	if !ok {
		return
	}

	// add all consecutive chunks to search results, starting at the one containing "from"
	for ; ts != 0; ts = mc.chunks[ts].Next {
		log.Debug("CCacheMetric searchForward: forward search adds chunk ts %d to start", ts)
		res.Start = append(res.Start, mc.chunks[ts].Itgen)
		nextTs := mc.nextTs(ts)
		res.From = nextTs

		if nextTs >= until {
			res.Complete = true
			break
		}
	}
}

func (mc *CCacheMetric) searchBackward(from, until uint32, keys []uint32, res *CCSearchResult) {
	ts, ok := mc.seekDesc(until-1, keys)
	if !ok {
		return
	}

	for ; ts != 0; ts = mc.chunks[ts].Prev {
		if ts < from {
			break
		}

		log.Debug("CCacheMetric searchBackward: backward search adds chunk ts %d to end", ts)
		res.End = append(res.End, mc.chunks[ts].Itgen)
		res.Until = ts
	}
}

// the idea of this method is that we first look for the chunks where the
// "from" and "until" ts are in. then we seek from the "from" towards "until"
// and add as many cunks as possible to the result, if this did not result
// in all chunks necessary to serve the request we do the same in the reverse
// order from "until" to "from"
// if the first seek in chronological direction already ends up with all the
// chunks we need to serve the request, the second one can be skipped.

// EXAMPLE:
// from ts:                    |
// until ts:                                                   |
// cache:            |---|---|---|   |   |   |   |   |---|---|---|---|---|---|
// chunks returned:          |---|                   |---|---|---|
//
func (mc *CCacheMetric) Search(res *CCSearchResult, from, until uint32) {
	mc.RLock()
	defer mc.RUnlock()

	if len(mc.chunks) < 1 {
		return
	}

	keys := mc.sortedTs()

	mc.searchForward(from, until, keys, res)
	if !res.Complete {
		mc.searchBackward(from, until, keys, res)
	}

	if !res.Complete && res.From > res.Until {
		log.Debug("CCacheMetric Search: Found from > until (%d/%d), printing chunks\n", res.From, res.Until)
		mc.debugMetric(keys)
	}
}

func (mc *CCacheMetric) debugMetric(keys []uint32) {
	log.Debug("CCacheMetric debugMetric: --- debugging metric ---\n")
	for _, key := range keys {
		log.Debug("CCacheMetric debugMetric: ts %d; prev %d; next %d\n", key, mc.chunks[key].Prev, mc.chunks[key].Next)
	}
	log.Debug("CCacheMetric debugMetric: ------------------------\n")
}
<commit_msg>don't unnecessarily regenerate keys list<commit_after>package cache

import (
	"sort"
	"sync"

	"github.com/raintank/metrictank/mdata/cache/accnt"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/worldping-api/pkg/log"
)

type CCacheMetric struct {
	sync.RWMutex

	// points at cached data chunks, indexed by their according time stamp
	chunks map[uint32]*CCacheChunk

	// the list of chunk time stamps in ascending order
	keys []uint32
}

func NewCCacheMetric() *CCacheMetric {
	return &CCacheMetric{
		chunks: make(map[uint32]*CCacheChunk),
	}
}

func (mc *CCacheMetric) Init(prev uint32, itergen chunk.IterGen) {
	mc.Add(prev, itergen)
}

func (mc *CCacheMetric) Del(ts uint32) int {
	mc.Lock()
	defer mc.Unlock()

	if _, ok := mc.chunks[ts]; !ok {
		return len(mc.chunks)
	}

	prev := mc.chunks[ts].Prev
	next := mc.chunks[ts].Next

	if _, ok := mc.chunks[prev]; prev != 0 && ok {
		mc.chunks[prev].Next = 0
	}
	if _, ok := mc.chunks[next]; next != 0 && ok {
		mc.chunks[next].Prev = 0
	}

	delete(mc.chunks, ts)

	// regenerate the list of sorted keys after deleting a chunk
	mc.generateKeys()

	return len(mc.chunks)
}

func (mc *CCacheMetric) Add(prev uint32, itergen chunk.IterGen) {
	ts := itergen.Ts()

	mc.Lock()
	defer mc.Unlock()

	if _, ok := mc.chunks[ts]; ok {
		// chunk is already present. no need to error on that, just ignore it
		return
	}

	mc.chunks[ts] = &CCacheChunk{
		Ts:    ts,
		Prev:  0,
		Next:  0,
		Itgen: itergen,
	}

	nextTs := mc.nextTs(ts)

	log.Debug("CCacheMetric Add: caching chunk ts %d, nextTs %d", ts, nextTs)

	// if previous chunk has not been passed we try to be smart and figure it out.
	// this is common in a scenario where a metric continuously gets queried
	// for a range that starts less than one chunkspan before now().
	if prev == 0 {
		ts, ok := mc.seekDesc(ts - 1)
		if ok {
			prev = ts
		}
	}

	// if the previous chunk is cached, link in both directions
	if _, ok := mc.chunks[prev]; ok {
		mc.chunks[prev].Next = ts
		mc.chunks[ts].Prev = prev
	}

	// if nextTs() can't figure out the end date it returns ts
	if nextTs > ts {
		// if the next chunk is cached, link in both directions
		if _, ok := mc.chunks[nextTs]; ok {
			mc.chunks[nextTs].Prev = ts
			mc.chunks[ts].Next = nextTs
		}
	}

	// regenerate the list of sorted keys after adding a chunk
	mc.generateKeys()

	return
}

// generate sorted slice of all chunk timestamps
// assumes we have at least read lock
func (mc *CCacheMetric) generateKeys() {
	keys := make([]uint32, 0, len(mc.chunks))
	for k := range mc.chunks {
		keys = append(keys, k)
	}
	sort.Sort(accnt.Uint32Asc(keys))
	mc.keys = keys
}

// nextTs takes a chunk's ts and returns the ts of the next chunk. (guessing if necessary)
// assumes we already have at least a read lock
func (mc *CCacheMetric) nextTs(ts uint32) uint32 {
	chunk := mc.chunks[ts]
	span := chunk.Itgen.Span()
	if span > 0 {
		// if the chunk is span-aware we don't need anything else
		return chunk.Ts + span
	}

	if chunk.Next == 0 {
		if chunk.Prev == 0 {
			// if a chunk has no next and no previous chunk we have to assume it's length is 0
			return chunk.Ts
		} else {
			// if chunk has no next chunk, but has a previous one, we assume the length of this one is same as the previous one
			return chunk.Ts + (chunk.Ts - chunk.Prev)
		}
	} else {
		// if chunk has a next chunk, then that's the ts we need
		return chunk.Next
	}
}

// seekAsc finds the t0 of the chunk that contains ts, by searching from old to recent
// if not found or can't be sure returns 0, false
// assumes we already have at least a read lock
func (mc *CCacheMetric) seekAsc(ts uint32) (uint32, bool) {
	log.Debug("CCacheMetric seekAsc: seeking for %d in the keys %+d", ts, mc.keys)

	for i := 0; i < len(mc.keys) && mc.keys[i] <= ts; i++ {
		if mc.nextTs(mc.keys[i]) > ts {
			log.Debug("CCacheMetric seekAsc: seek found ts %d is between %d and %d", ts, mc.keys[i], mc.nextTs(mc.keys[i]))
			return mc.keys[i], true
		}
	}

	log.Debug("CCacheMetric seekAsc: seekAsc unsuccessful")
	return 0, false
}

// seekDesc finds the t0 of the chunk that contains ts, by searching from recent to old
// if not found or can't be sure returns 0, false
// assumes we already have at least a read lock
func (mc *CCacheMetric) seekDesc(ts uint32) (uint32, bool) {
	log.Debug("CCacheMetric seekDesc: seeking for %d in the keys %+d", ts, mc.keys)

	for i := len(mc.keys) - 1; i >= 0 && mc.nextTs(mc.keys[i]) > ts; i-- {
		if mc.keys[i] <= ts {
			log.Debug("CCacheMetric seekDesc: seek found ts %d is between %d and %d", ts, mc.keys[i], mc.nextTs(mc.keys[i]))
			return mc.keys[i], true
		}
	}

	log.Debug("CCacheMetric seekDesc: seekDesc unsuccessful")
	return 0, false
}

func (mc *CCacheMetric) searchForward(from, until uint32, res *CCSearchResult) {
	ts, ok := mc.seekAsc(from)
	if !ok {
		return
	}

	// add all consecutive chunks to search results, starting at the one containing "from"
	for ; ts != 0; ts = mc.chunks[ts].Next {
		log.Debug("CCacheMetric searchForward: forward search adds chunk ts %d to start", ts)
		res.Start = append(res.Start, mc.chunks[ts].Itgen)
		nextTs := mc.nextTs(ts)
		res.From = nextTs

		if nextTs >= until {
			res.Complete = true
			break
		}
	}
}

func (mc *CCacheMetric) searchBackward(from, until uint32, res *CCSearchResult) {
	ts, ok := mc.seekDesc(until - 1)
	if !ok {
		return
	}

	for ; ts != 0; ts = mc.chunks[ts].Prev {
		if ts < from {
			break
		}

		log.Debug("CCacheMetric searchBackward: backward search adds chunk ts %d to end", ts)
		res.End = append(res.End, mc.chunks[ts].Itgen)
		res.Until = ts
	}
}

// the idea of this method is that we first look for the chunks where the
// "from" and "until" ts are in. then we seek from the "from" towards "until"
// and add as many cunks as possible to the result, if this did not result
// in all chunks necessary to serve the request we do the same in the reverse
// order from "until" to "from"
// if the first seek in chronological direction already ends up with all the
// chunks we need to serve the request, the second one can be skipped.

// EXAMPLE:
// from ts:                    |
// until ts:                                                   |
// cache:            |---|---|---|   |   |   |   |   |---|---|---|---|---|---|
// chunks returned:          |---|                   |---|---|---|
//
func (mc *CCacheMetric) Search(res *CCSearchResult, from, until uint32) {
	mc.RLock()
	defer mc.RUnlock()

	if len(mc.chunks) < 1 {
		return
	}

	mc.searchForward(from, until, res)
	if !res.Complete {
		mc.searchBackward(from, until, res)
	}

	if !res.Complete && res.From > res.Until {
		log.Debug("CCacheMetric Search: Found from > until (%d/%d), printing chunks\n", res.From, res.Until)
		mc.debugMetric()
	}
}

func (mc *CCacheMetric) debugMetric() {
	log.Debug("CCacheMetric debugMetric: --- debugging metric ---\n")
	for _, key := range mc.keys {
		log.Debug("CCacheMetric debugMetric: ts %d; prev %d; next %d\n", key, mc.chunks[key].Prev, mc.chunks[key].Next)
	}
	log.Debug("CCacheMetric debugMetric: ------------------------\n")
}
<|endoftext|>


<commit_before>package cache

import (
	"context"
	"sort"
	"sync"

	"github.com/grafana/metrictank/mdata/cache/accnt"
	"github.com/grafana/metrictank/mdata/chunk"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/raintank/worldping-api/pkg/log"
	"gopkg.in/raintank/schema.v1"
)

type CCacheMetric struct {
	sync.RWMutex

	// points at cached data chunks, indexed by their according time stamp
	chunks map[uint32]*CCacheChunk

	// the list of chunk time stamps in ascending order
	keys []uint32

	MKey schema.MKey
}

func NewCCacheMetric() *CCacheMetric {
	return &CCacheMetric{
		chunks: make(map[uint32]*CCacheChunk),
	}
}

func (mc *CCacheMetric) Init(MKey schema.MKey, prev uint32, itergen chunk.IterGen) {
	mc.Add(prev, itergen)
	mc.MKey = MKey
}

func (mc *CCacheMetric) Del(ts uint32) int {
	mc.Lock()
	defer mc.Unlock()

	if _, ok := mc.chunks[ts]; !ok {
		return len(mc.chunks)
	}

	prev := mc.chunks[ts].Prev
	next := mc.chunks[ts].Next

	if prev != 0 {
		if _, ok := mc.chunks[prev]; ok {
			mc.chunks[prev].Next = 0
		}
	}

	if next != 0 {
		if _, ok := mc.chunks[next]; ok {
			mc.chunks[next].Prev = 0
		}
	}

	delete(mc.chunks, ts)

	// regenerate the list of sorted keys after deleting a chunk
	mc.generateKeys()

	return len(mc.chunks)
}

func (mc *CCacheMetric) Add(prev uint32, itergen chunk.IterGen) {
	ts := itergen.Ts

	mc.Lock()
	defer mc.Unlock()

	if _, ok := mc.chunks[ts]; ok {
		// chunk is already present. no need to error on that, just ignore it
		return
	}

	mc.chunks[ts] = &CCacheChunk{
		Ts:    ts,
		Prev:  0,
		Next:  0,
		Itgen: itergen,
	}

	nextTs := mc.nextTs(ts)

	log.Debug("CCacheMetric Add: caching chunk ts %d, nextTs %d", ts, nextTs)

	// if previous chunk has not been passed we try to be smart and figure it out.
	// this is common in a scenario where a metric continuously gets queried
	// for a range that starts less than one chunkspan before now().
	if prev == 0 {
		res, ok := mc.seekDesc(ts - 1)
		if ok {
			prev = res
		}
	}

	// if the previous chunk is cached, link in both directions
	if _, ok := mc.chunks[prev]; ok {
		mc.chunks[prev].Next = ts
		mc.chunks[ts].Prev = prev
	}

	// if nextTs() can't figure out the end date it returns ts
	if nextTs > ts {
		// if the next chunk is cached, link in both directions
		if _, ok := mc.chunks[nextTs]; ok {
			mc.chunks[nextTs].Prev = ts
			mc.chunks[ts].Next = nextTs
		}
	}

	// regenerate the list of sorted keys after adding a chunk
	mc.generateKeys()

	return
}

// generate sorted slice of all chunk timestamps
// assumes we have at least read lock
func (mc *CCacheMetric) generateKeys() {
	keys := make([]uint32, 0, len(mc.chunks))
	for k := range mc.chunks {
		keys = append(keys, k)
	}
	sort.Sort(accnt.Uint32Asc(keys))
	mc.keys = keys
}

// nextTs takes a chunk's ts and returns the ts of the next chunk. (guessing if necessary)
// assumes we already have at least a read lock
func (mc *CCacheMetric) nextTs(ts uint32) uint32 {
	chunk := mc.chunks[ts]
	span := chunk.Itgen.Span
	if span > 0 {
		// if the chunk is span-aware we don't need anything else
		return chunk.Ts + span
	}

	if chunk.Next == 0 {
		if chunk.Prev == 0 {
			// if a chunk has no next and no previous chunk we have to assume it's length is 0
			return chunk.Ts
		} else {
			// if chunk has no next chunk, but has a previous one, we assume the length of this one is same as the previous one
			return chunk.Ts + (chunk.Ts - chunk.Prev)
		}
	} else {
		// if chunk has a next chunk, then that's the ts we need
		return chunk.Next
	}
}

// returns the last Ts of this metric cache
// since ranges are exclusive at the end this is actually the first Ts that is not cached
func (mc *CCacheMetric) lastTs() uint32 {
	mc.RLock()
	defer mc.RUnlock()
	return mc.nextTs(mc.keys[len(mc.keys)-1])
}

// seekAsc finds the t0 of the chunk that contains ts, by searching from old to recent
// if not found or can't be sure returns 0, false
// assumes we already have at least a read lock
func (mc *CCacheMetric) seekAsc(ts uint32) (uint32, bool) {
	log.Debug("CCacheMetric seekAsc: seeking for %d in the keys %+d", ts, mc.keys)

	for i := 0; i < len(mc.keys) && mc.keys[i] <= ts; i++ {
		if mc.nextTs(mc.keys[i]) > ts {
			log.Debug("CCacheMetric seekAsc: seek found ts %d is between %d and %d", ts, mc.keys[i], mc.nextTs(mc.keys[i]))
			return mc.keys[i], true
		}
	}

	log.Debug("CCacheMetric seekAsc: seekAsc unsuccessful")
	return 0, false
}

// seekDesc finds the t0 of the chunk that contains ts, by searching from recent to old
// if not found or can't be sure returns 0, false
// assumes we already have at least a read lock
func (mc *CCacheMetric) seekDesc(ts uint32) (uint32, bool) {
	log.Debug("CCacheMetric seekDesc: seeking for %d in the keys %+d", ts, mc.keys)

	for i := len(mc.keys) - 1; i >= 0 && mc.nextTs(mc.keys[i]) > ts; i-- {
		if mc.keys[i] <= ts {
			log.Debug("CCacheMetric seekDesc: seek found ts %d is between %d and %d", ts, mc.keys[i], mc.nextTs(mc.keys[i]))
			return mc.keys[i], true
		}
	}

	log.Debug("CCacheMetric seekDesc: seekDesc unsuccessful")
	return 0, false
}

func (mc *CCacheMetric) searchForward(ctx context.Context, metric schema.AMKey, from, until uint32, res *CCSearchResult) {
	ts, ok := mc.seekAsc(from)
	if !ok {
		return
	}

	// add all consecutive chunks to search results, starting at the one containing "from"
	for ; ts != 0; ts = mc.chunks[ts].Next {
		log.Debug("CCacheMetric searchForward: forward search adds chunk ts %d to start", ts)
		res.Start = append(res.Start, mc.chunks[ts].Itgen)
		nextTs := mc.nextTs(ts)
		res.From = nextTs

		if nextTs >= until {
			res.Complete = true
			break
		}
		if mc.chunks[ts].Next != 0 && ts >= mc.chunks[ts].Next {
			log.Warn("CCacheMetric: suspected bug suppressed. searchForward(%q, %d, %d, res) ts is %d while Next is %d", metric, from, until, ts, mc.chunks[ts].Next)
			span := opentracing.SpanFromContext(ctx)
			span.SetTag("searchForwardBug", true)
			searchFwdBug.Inc()
			break
		}
	}
}

func (mc *CCacheMetric) searchBackward(from, until uint32, res *CCSearchResult) {
	ts, ok := mc.seekDesc(until - 1)
	if !ok {
		return
	}

	for ; ts != 0; ts = mc.chunks[ts].Prev {
		if ts < from {
			break
		}

		log.Debug("CCacheMetric searchBackward: backward search adds chunk ts %d to end", ts)
		res.End = append(res.End, mc.chunks[ts].Itgen)
		res.Until = ts
	}
}

// the idea of this method is that we first look for the chunks where the
// "from" and "until" ts are in. then we seek from the "from" towards "until"
// and add as many cunks as possible to the result, if this did not result
// in all chunks necessary to serve the request we do the same in the reverse
// order from "until" to "from"
// if the first seek in chronological direction already ends up with all the
// chunks we need to serve the request, the second one can be skipped.

// EXAMPLE:
// from ts:                    |
// until ts:                                                   |
// cache:            |---|---|---|   |   |   |   |   |---|---|---|---|---|---|
// chunks returned:          |---|                   |---|---|---|
//
func (mc *CCacheMetric) Search(ctx context.Context, metric schema.AMKey, res *CCSearchResult, from, until uint32) {
	mc.RLock()
	defer mc.RUnlock()

	if len(mc.chunks) < 1 {
		return
	}

	mc.searchForward(ctx, metric, from, until, res)
	if !res.Complete {
		mc.searchBackward(from, until, res)
	}

	if !res.Complete && res.From > res.Until {
		log.Debug("CCacheMetric Search: Found from > until (%d/%d), printing chunks\n", res.From, res.Until)
		mc.debugMetric()
	}
}

func (mc *CCacheMetric) debugMetric() {
	log.Debug("CCacheMetric debugMetric: --- debugging metric ---\n")
	for _, key := range mc.keys {
		log.Debug("CCacheMetric debugMetric: ts %d; prev %d; next %d\n", key, mc.chunks[key].Prev, mc.chunks[key].Next)
	}
	log.Debug("CCacheMetric debugMetric: ------------------------\n")
}
<commit_msg>golint<commit_after>package cache

import (
	"context"
	"sort"
	"sync"

	"github.com/grafana/metrictank/mdata/cache/accnt"
	"github.com/grafana/metrictank/mdata/chunk"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/raintank/worldping-api/pkg/log"
	"gopkg.in/raintank/schema.v1"
)

// CCacheMetric caches data chunks
type CCacheMetric struct {
	sync.RWMutex

	// points at cached data chunks, indexed by their according time stamp
	chunks map[uint32]*CCacheChunk

	// the list of chunk time stamps in ascending order
	keys []uint32

	MKey schema.MKey
}

// NewCCacheMetric creates a CCacheMetric
func NewCCacheMetric() *CCacheMetric {
	return &CCacheMetric{
		chunks: make(map[uint32]*CCacheChunk),
	}
}

func (mc *CCacheMetric) Init(MKey schema.MKey, prev uint32, itergen chunk.IterGen) {
	mc.Add(prev, itergen)
	mc.MKey = MKey
}

func (mc *CCacheMetric) Del(ts uint32) int {
	mc.Lock()
	defer mc.Unlock()

	if _, ok := mc.chunks[ts]; !ok {
		return len(mc.chunks)
	}

	prev := mc.chunks[ts].Prev
	next := mc.chunks[ts].Next

	if prev != 0 {
		if _, ok := mc.chunks[prev]; ok {
			mc.chunks[prev].Next = 0
		}
	}

	if next != 0 {
		if _, ok := mc.chunks[next]; ok {
			mc.chunks[next].Prev = 0
		}
	}

	delete(mc.chunks, ts)

	// regenerate the list of sorted keys after deleting a chunk
	mc.generateKeys()

	return len(mc.chunks)
}

func (mc *CCacheMetric) Add(prev uint32, itergen chunk.IterGen) {
	ts := itergen.Ts

	mc.Lock()
	defer mc.Unlock()

	if _, ok := mc.chunks[ts]; ok {
		// chunk is already present. no need to error on that, just ignore it
		return
	}

	mc.chunks[ts] = &CCacheChunk{
		Ts:    ts,
		Prev:  0,
		Next:  0,
		Itgen: itergen,
	}

	nextTs := mc.nextTs(ts)

	log.Debug("CCacheMetric Add: caching chunk ts %d, nextTs %d", ts, nextTs)

	// if previous chunk has not been passed we try to be smart and figure it out.
	// this is common in a scenario where a metric continuously gets queried
	// for a range that starts less than one chunkspan before now().
	if prev == 0 {
		res, ok := mc.seekDesc(ts - 1)
		if ok {
			prev = res
		}
	}

	// if the previous chunk is cached, link in both directions
	if _, ok := mc.chunks[prev]; ok {
		mc.chunks[prev].Next = ts
		mc.chunks[ts].Prev = prev
	}

	// if nextTs() can't figure out the end date it returns ts
	if nextTs > ts {
		// if the next chunk is cached, link in both directions
		if _, ok := mc.chunks[nextTs]; ok {
			mc.chunks[nextTs].Prev = ts
			mc.chunks[ts].Next = nextTs
		}
	}

	// regenerate the list of sorted keys after adding a chunk
	mc.generateKeys()

	return
}

// generate sorted slice of all chunk timestamps
// assumes we have at least read lock
func (mc *CCacheMetric) generateKeys() {
	keys := make([]uint32, 0, len(mc.chunks))
	for k := range mc.chunks {
		keys = append(keys, k)
	}
	sort.Sort(accnt.Uint32Asc(keys))
	mc.keys = keys
}

// nextTs takes a chunk's ts and returns the ts of the next chunk. (guessing if necessary)
// assumes we already have at least a read lock
func (mc *CCacheMetric) nextTs(ts uint32) uint32 {
	chunk := mc.chunks[ts]
	span := chunk.Itgen.Span
	if span > 0 {
		// if the chunk is span-aware we don't need anything else
		return chunk.Ts + span
	}

	if chunk.Next == 0 {
		if chunk.Prev == 0 {
			// if a chunk has no next and no previous chunk we have to assume it's length is 0
			return chunk.Ts
		} else {
			// if chunk has no next chunk, but has a previous one, we assume the length of this one is same as the previous one
			return chunk.Ts + (chunk.Ts - chunk.Prev)
		}
	} else {
		// if chunk has a next chunk, then that's the ts we need
		return chunk.Next
	}
}

// lastTs returns the last Ts of this metric cache
// since ranges are exclusive at the end this is actually the first Ts that is not cached
func (mc *CCacheMetric) lastTs() uint32 {
	mc.RLock()
	defer mc.RUnlock()
	return mc.nextTs(mc.keys[len(mc.keys)-1])
}

// seekAsc finds the t0 of the chunk that contains ts, by searching from old to recent
// if not found or can't be sure returns 0, false
// assumes we already have at least a read lock
func (mc *CCacheMetric) seekAsc(ts uint32) (uint32, bool) {
	log.Debug("CCacheMetric seekAsc: seeking for %d in the keys %+d", ts, mc.keys)

	for i := 0; i < len(mc.keys) && mc.keys[i] <= ts; i++ {
		if mc.nextTs(mc.keys[i]) > ts {
			log.Debug("CCacheMetric seekAsc: seek found ts %d is between %d and %d", ts, mc.keys[i], mc.nextTs(mc.keys[i]))
			return mc.keys[i], true
		}
	}

	log.Debug("CCacheMetric seekAsc: seekAsc unsuccessful")
	return 0, false
}

// seekDesc finds the t0 of the chunk that contains ts, by searching from recent to old
// if not found or can't be sure returns 0, false
// assumes we already have at least a read lock
func (mc *CCacheMetric) seekDesc(ts uint32) (uint32, bool) {
	log.Debug("CCacheMetric seekDesc: seeking for %d in the keys %+d", ts, mc.keys)

	for i := len(mc.keys) - 1; i >= 0 && mc.nextTs(mc.keys[i]) > ts; i-- {
		if mc.keys[i] <= ts {
			log.Debug("CCacheMetric seekDesc: seek found ts %d is between %d and %d", ts, mc.keys[i], mc.nextTs(mc.keys[i]))
			return mc.keys[i], true
		}
	}

	log.Debug("CCacheMetric seekDesc: seekDesc unsuccessful")
	return 0, false
}

func (mc *CCacheMetric) searchForward(ctx context.Context, metric schema.AMKey, from, until uint32, res *CCSearchResult) {
	ts, ok := mc.seekAsc(from)
	if !ok {
		return
	}

	// add all consecutive chunks to search results, starting at the one containing "from"
	for ; ts != 0; ts = mc.chunks[ts].Next {
		log.Debug("CCacheMetric searchForward: forward search adds chunk ts %d to start", ts)
		res.Start = append(res.Start, mc.chunks[ts].Itgen)
		nextTs := mc.nextTs(ts)
		res.From = nextTs

		if nextTs >= until {
			res.Complete = true
			break
		}
		if mc.chunks[ts].Next != 0 && ts >= mc.chunks[ts].Next {
			log.Warn("CCacheMetric: suspected bug suppressed. searchForward(%q, %d, %d, res) ts is %d while Next is %d", metric, from, until, ts, mc.chunks[ts].Next)
			span := opentracing.SpanFromContext(ctx)
			span.SetTag("searchForwardBug", true)
			searchFwdBug.Inc()
			break
		}
	}
}

func (mc *CCacheMetric) searchBackward(from, until uint32, res *CCSearchResult) {
	ts, ok := mc.seekDesc(until - 1)
	if !ok {
		return
	}

	for ; ts != 0; ts = mc.chunks[ts].Prev {
		if ts < from {
			break
		}

		log.Debug("CCacheMetric searchBackward: backward search adds chunk ts %d to end", ts)
		res.End = append(res.End, mc.chunks[ts].Itgen)
		res.Until = ts
	}
}

// the idea of this method is that we first look for the chunks where the
// "from" and "until" ts are in. then we seek from the "from" towards "until"
// and add as many cunks as possible to the result, if this did not result
// in all chunks necessary to serve the request we do the same in the reverse
// order from "until" to "from"
// if the first seek in chronological direction already ends up with all the
// chunks we need to serve the request, the second one can be skipped.

// EXAMPLE:
// from ts:                    |
// until ts:                                                   |
// cache:            |---|---|---|   |   |   |   |   |---|---|---|---|---|---|
// chunks returned:          |---|                   |---|---|---|
//
func (mc *CCacheMetric) Search(ctx context.Context, metric schema.AMKey, res *CCSearchResult, from, until uint32) {
	mc.RLock()
	defer mc.RUnlock()

	if len(mc.chunks) < 1 {
		return
	}

	mc.searchForward(ctx, metric, from, until, res)
	if !res.Complete {
		mc.searchBackward(from, until, res)
	}

	if !res.Complete && res.From > res.Until {
		log.Debug("CCacheMetric Search: Found from > until (%d/%d), printing chunks\n", res.From, res.Until)
		mc.debugMetric()
	}
}

func (mc *CCacheMetric) debugMetric() {
	log.Debug("CCacheMetric debugMetric: --- debugging metric ---\n")
	for _, key := range mc.keys {
		log.Debug("CCacheMetric debugMetric: ts %d; prev %d; next %d\n", key, mc.chunks[key].Prev, mc.chunks[key].Next)
	}
	log.Debug("CCacheMetric debugMetric: ------------------------\n")
}
<|endoftext|>


<commit_before>package instance

import (
	"bytes"
	"fmt"
	"github.com/mitchellh/goamz/ec2"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
	"text/template"
)

type bundleCmdData struct {
	AccountId    string
	Architecture string
	CertPath     string
	KeyPath      string
	PrivatePath  string
}

type StepBundleVolume struct{}

func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepAction {
	comm := state["communicator"].(packer.Communicator)
	config := state["config"].(*Config)
	instance := state["instance"].(*ec2.Instance)
	ui := state["ui"].(packer.Ui)
	x509RemoteCertPath := state["x509RemoteCertPath"].(string)
	x509RemoteKeyPath := state["x509RemoteKeyPath"].(string)

	// Verify the AMI tools are available
	ui.Say("Checking for EC2 AMI tools...")
	cmd := &packer.RemoteCmd{Command: "ec2-ami-tools-version"}
	if err := comm.Start(cmd); err != nil {
		state["error"] = fmt.Errorf("Error checking for AMI tools: %s", err)
		ui.Error(state["error"].(error).Error())
		return multistep.ActionHalt
	}
	cmd.Wait()

	if cmd.ExitStatus != 0 {
		state["error"] = fmt.Errorf(
			"The EC2 AMI tools could not be detected. These must be manually\n" +
				"via a provisioner or some other means and are required for Packer\n" +
				"to create an instance-store AMI.")
		ui.Error(state["error"].(error).Error())
		return multistep.ActionHalt
	}

	// Bundle the volume
	var bundleCmd bytes.Buffer
	tData := bundleCmdData{
		AccountId:    config.AccountId,
		Architecture: instance.Architecture,
		CertPath:     x509RemoteCertPath,
		KeyPath:      x509RemoteKeyPath,
		PrivatePath:  config.X509UploadPath,
	}
	t := template.Must(template.New("bundleCmd").Parse(config.BundleVolCommand))
	t.Execute(&bundleCmd, tData)

	ui.Say("Bundling the volume...")
	cmd = new(packer.RemoteCmd)
	cmd.Command = bundleCmd.String()
	if err := cmd.StartWithUi(comm, ui); err != nil {
		state["error"] = fmt.Errorf("Error bundling volume: %s", err)
		ui.Error(state["error"].(error).Error())
		return multistep.ActionHalt
	}

	return multistep.ActionContinue
}

func (s *StepBundleVolume) Cleanup(map[string]interface{}) {}
<commit_msg>builder/amazon/instance: detect error bundling<commit_after>package instance

import (
	"bytes"
	"fmt"
	"github.com/mitchellh/goamz/ec2"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
	"text/template"
)

type bundleCmdData struct {
	AccountId    string
	Architecture string
	CertPath     string
	KeyPath      string
	PrivatePath  string
}

type StepBundleVolume struct{}

func (s *StepBundleVolume) Run(state map[string]interface{}) multistep.StepAction {
	comm := state["communicator"].(packer.Communicator)
	config := state["config"].(*Config)
	instance := state["instance"].(*ec2.Instance)
	ui := state["ui"].(packer.Ui)
	x509RemoteCertPath := state["x509RemoteCertPath"].(string)
	x509RemoteKeyPath := state["x509RemoteKeyPath"].(string)

	// Verify the AMI tools are available
	ui.Say("Checking for EC2 AMI tools...")
	cmd := &packer.RemoteCmd{Command: "ec2-ami-tools-version"}
	if err := comm.Start(cmd); err != nil {
		state["error"] = fmt.Errorf("Error checking for AMI tools: %s", err)
		ui.Error(state["error"].(error).Error())
		return multistep.ActionHalt
	}
	cmd.Wait()

	if cmd.ExitStatus != 0 {
		state["error"] = fmt.Errorf(
			"The EC2 AMI tools could not be detected. These must be manually\n" +
				"via a provisioner or some other means and are required for Packer\n" +
				"to create an instance-store AMI.")
		ui.Error(state["error"].(error).Error())
		return multistep.ActionHalt
	}

	// Bundle the volume
	var bundleCmd bytes.Buffer
	tData := bundleCmdData{
		AccountId:    config.AccountId,
		Architecture: instance.Architecture,
		CertPath:     x509RemoteCertPath,
		KeyPath:      x509RemoteKeyPath,
		PrivatePath:  config.X509UploadPath,
	}
	t := template.Must(template.New("bundleCmd").Parse(config.BundleVolCommand))
	t.Execute(&bundleCmd, tData)

	ui.Say("Bundling the volume...")
	cmd = new(packer.RemoteCmd)
	cmd.Command = bundleCmd.String()
	if err := cmd.StartWithUi(comm, ui); err != nil {
		state["error"] = fmt.Errorf("Error bundling volume: %s", err)
		ui.Error(state["error"].(error).Error())
		return multistep.ActionHalt
	}

	if cmd.ExitStatus != 0 {
		state["error"] = fmt.Errorf(
			"Volume bundling failed. Please see the output above for more\n" +
				"details on what went wrong.")
		ui.Error(state["error"].(error).Error())
		return multistep.ActionHalt
	}

	return multistep.ActionContinue
}

func (s *StepBundleVolume) Cleanup(map[string]interface{}) {}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/coreos/go-systemd/unit"
	"github.com/urfave/cli"
)

type FourGState struct {
	Enabled bool   `json:"enabled"`
	Configg *FourG `json:"config"`
}

type FourG struct {
	Network
}

//ToSystemdUnit implement UnitFile interface
func (f FourG) ToSystemdUnit() ([]*unit.UnitOption, error) {
	if f.Interface == "" {
		f.Interface = "eth1"
	}
	return f.Network.ToSystemdUnit()
}

func fourGState(i string) (*FourGState, error) {
	dir := os.Getenv("FCONF_CONFIGDIR")
	if dir == "" {
		dir = fconfConfigDir
	}
	b, err := ioutil.ReadFile(filepath.Join(dir,
		fmt.Sprintf(defaultFougGConfig, i)))
	if err != nil {
		return nil, err
	}
	f := &FourGState{}
	err = json.Unmarshal(b, f)
	if err != nil {
		return nil, err
	}
	if f.Configg == nil {
		return nil, ErrWrongStateFile
	}
	return f, nil
}

func FourgCMD(ctx *cli.Context) error {
	if ctx.IsSet(enableFlag) {
		return EnableFourg(ctx)
	}
	if ctx.IsSet(disableFlag) {
		return DisableFourg(ctx)
	}
	if ctx.IsSet(removeFlag) {
		return RemoveFourg(ctx)
	}
	if ctx.IsSet(configFlag) {
		return configFourgCMD(ctx)
	}
	return nil
}
func RemoveFourg(ctx *cli.Context) error {
	err := DisableFourg(ctx)
	if err != nil {
		return err
	}
	var i string
	if ctx.IsSet("interface") {
		i = ctx.String("interface")
	} else {
		i = ctx.Args().First()
	}
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	f, err := fourGState(i)
	if err != nil {
		return err
	}

	// removestate file
	stateFile := filepath.Join(stateDir(),
		fmt.Sprintf(defaultFougGConfig, i))
	err = removeFile(stateFile)
	if err != nil {
		return err
	}
	// remove systemd file
	unit := filepath.Join(networkBase,
		fmt.Sprintf(fourgService, i))
	err = removeFile(unit)
	if err != nil {
		return err
	}

	err = FlushInterface(f.Configg.Interface)
	if err != nil {
		return err
	}

	// reload systemd-networkd
	return restartService("systemd-networkd")
}
func EnableFourg(ctx *cli.Context) error {
	if ctx.IsSet(configFlag) {
		err := configFourgCMD(ctx)
		if err != nil {
			return err
		}
	}
	var i string
	if ctx.IsSet("interface") {
		i = ctx.String("interface")
	} else {
		i = ctx.Args().First()
	}
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	e, err := fourGState(i)
	if err != nil {
		return err
	}
	_, err = exec.Command("ip", "link", "set", "up", e.Configg.Interface).Output()
	if err != nil {
		return fmt.Errorf("ERROR: runnin ip link set up %s %v",
			e.Configg.Interface, err,
		)
	}
	err = restartService("systemd-networkd")
	if err != nil {
		return fmt.Errorf("ERROR: restarting systemd %v ", err)
	}
	e.Enabled = true
	data, err := json.Marshal(e)
	if err != nil {
		return err
	}
	return keepState(
		fmt.Sprintf(defaultFougGConfig, i), data)
}

func DisableFourg(ctx *cli.Context) error {
	var i string
	if ctx.IsSet("interface") {
		i = ctx.String("interface")
	} else {
		i = ctx.Args().First()
	}
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	e, err := fourGState(i)
	if err != nil {
		return err
	}
	_, err = exec.Command("ip", "link", "set", "down", "dev", e.Configg.Interface).Output()
	if err != nil {
		return err
	}
	unit := filepath.Join(networkBase,
		fmt.Sprintf(fourgService, e.Configg.Interface))
	err = removeFile(unit)
	if err != nil {
		return err
	}
	fmt.Println("successfully disabled 4G")
	e.Enabled = false
	data, err := json.Marshal(e)
	if err != nil {
		return err
	}
	ctx.Set("interface", i)
	return keepState(
		fmt.Sprintf(defaultFougGConfig, i), data)
}

func configFourgCMD(ctx *cli.Context) error {
	base := ctx.String("dir")
	name := ctx.String("name")
	src := ctx.String("config")
	if src == "" {
		return errors.New("fconf: missing configuration source file")
	}
	var b []byte
	var err error
	if src == "stdin" {
		b, err = ReadFromStdin()
		if err != nil {
			return err
		}
	} else {
		b, err = ioutil.ReadFile(src)
		if err != nil {
			return err
		}
	}
	e := FourG{}
	err = json.Unmarshal(b, &e)
	if err != nil {
		return err
	}
	if e.Interface == "" {
		e.Interface = "eth1"
	}
	err = checkDir(base)
	if err != nil {
		return err
	}
	if strings.Contains(name, "%s") {
		name = fmt.Sprintf(name, e.Interface)
	}
	filename := filepath.Join(base, name)
	err = CreateSystemdFile(e, filename, 0644)
	if err != nil {
		return err
	}
	fmt.Printf("successful written 4G configuration to %s \n", filename)
	state := &FourGState{Configg: &e}
	b, _ = json.Marshal(state)
	return keepState(
		fmt.Sprintf(defaultFougGConfig, e.Interface), b)
}
<commit_msg>4g: Create systemd unit file on enable.<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/coreos/go-systemd/unit"
	"github.com/urfave/cli"
)

type FourGState struct {
	Enabled bool   `json:"enabled"`
	Configg *FourG `json:"config"`
}

type FourG struct {
	Network
}

//ToSystemdUnit implement UnitFile interface
func (f FourG) ToSystemdUnit() ([]*unit.UnitOption, error) {
	if f.Interface == "" {
		f.Interface = "eth1"
	}
	return f.Network.ToSystemdUnit()
}

func fourGState(i string) (*FourGState, error) {
	dir := os.Getenv("FCONF_CONFIGDIR")
	if dir == "" {
		dir = fconfConfigDir
	}
	b, err := ioutil.ReadFile(filepath.Join(dir,
		fmt.Sprintf(defaultFougGConfig, i)))
	if err != nil {
		return nil, err
	}
	f := &FourGState{}
	err = json.Unmarshal(b, f)
	if err != nil {
		return nil, err
	}
	if f.Configg == nil {
		return nil, ErrWrongStateFile
	}
	return f, nil
}

func FourgCMD(ctx *cli.Context) error {
	if ctx.IsSet(enableFlag) {
		return EnableFourg(ctx)
	}
	if ctx.IsSet(disableFlag) {
		return DisableFourg(ctx)
	}
	if ctx.IsSet(removeFlag) {
		return RemoveFourg(ctx)
	}
	if ctx.IsSet(configFlag) {
		return configFourgCMD(ctx)
	}
	return nil
}
func RemoveFourg(ctx *cli.Context) error {
	err := DisableFourg(ctx)
	if err != nil {
		return err
	}
	var i string
	if ctx.IsSet("interface") {
		i = ctx.String("interface")
	} else {
		i = ctx.Args().First()
	}
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	f, err := fourGState(i)
	if err != nil {
		return err
	}

	// removestate file
	stateFile := filepath.Join(stateDir(),
		fmt.Sprintf(defaultFougGConfig, i))
	err = removeFile(stateFile)
	if err != nil {
		return err
	}
	// remove systemd file
	unit := filepath.Join(networkBase,
		fmt.Sprintf(fourgService, i))
	err = removeFile(unit)
	if err != nil {
		return err
	}

	err = FlushInterface(f.Configg.Interface)
	if err != nil {
		return err
	}

	// reload systemd-networkd
	return restartService("systemd-networkd")
}
func EnableFourg(ctx *cli.Context) error {
	if ctx.IsSet(configFlag) {
		err := configFourgCMD(ctx)
		if err != nil {
			return err
		}
	}
	var i string
	if ctx.IsSet("interface") {
		i = ctx.String("interface")
	} else {
		i = ctx.Args().First()
	}
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	e, err := fourGState(i)
	if err != nil {
		return err
	}
	unit := filepath.Join(networkBase,
		fmt.Sprintf(fourgService, e.Configg.Interface))
	_, err = os.Stat(unit)
	if os.IsNotExist(err) {
		err = CreateSystemdFile(e.Configg, unit, 0644)
		if err != nil {
			return err
		}
	}
	_, err = exec.Command("ip", "link", "set", "up", e.Configg.Interface).Output()
	if err != nil {
		return fmt.Errorf("ERROR: runnin ip link set up %s %v",
			e.Configg.Interface, err,
		)
	}
	err = restartService("systemd-networkd")
	if err != nil {
		return fmt.Errorf("ERROR: restarting systemd %v ", err)
	}
	e.Enabled = true
	data, err := json.Marshal(e)
	if err != nil {
		return err
	}
	return keepState(
		fmt.Sprintf(defaultFougGConfig, i), data)
}

func DisableFourg(ctx *cli.Context) error {
	var i string
	if ctx.IsSet("interface") {
		i = ctx.String("interface")
	} else {
		i = ctx.Args().First()
	}
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	e, err := fourGState(i)
	if err != nil {
		return err
	}
	_, err = exec.Command("ip", "link", "set", "down", "dev", e.Configg.Interface).Output()
	if err != nil {
		return err
	}
	unit := filepath.Join(networkBase,
		fmt.Sprintf(fourgService, e.Configg.Interface))
	err = removeFile(unit)
	if err != nil {
		return err
	}
	fmt.Println("successfully disabled 4G")
	e.Enabled = false
	data, err := json.Marshal(e)
	if err != nil {
		return err
	}
	ctx.Set("interface", i)
	return keepState(
		fmt.Sprintf(defaultFougGConfig, i), data)
}

func configFourgCMD(ctx *cli.Context) error {
	base := ctx.String("dir")
	name := ctx.String("name")
	src := ctx.String("config")
	if src == "" {
		return errors.New("fconf: missing configuration source file")
	}
	var b []byte
	var err error
	if src == "stdin" {
		b, err = ReadFromStdin()
		if err != nil {
			return err
		}
	} else {
		b, err = ioutil.ReadFile(src)
		if err != nil {
			return err
		}
	}
	e := FourG{}
	err = json.Unmarshal(b, &e)
	if err != nil {
		return err
	}
	if e.Interface == "" {
		e.Interface = "eth1"
	}
	err = checkDir(base)
	if err != nil {
		return err
	}
	if strings.Contains(name, "%s") {
		name = fmt.Sprintf(name, e.Interface)
	}
	filename := filepath.Join(base, name)
	err = CreateSystemdFile(e, filename, 0644)
	if err != nil {
		return err
	}
	fmt.Printf("successful written 4G configuration to %s \n", filename)
	state := &FourGState{Configg: &e}
	b, _ = json.Marshal(state)
	return keepState(
		fmt.Sprintf(defaultFougGConfig, e.Interface), b)
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package middleware

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/go-martini/martini"

	"github.com/gogits/cache"
	"github.com/gogits/git"
	"github.com/gogits/session"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/auth"
	"github.com/gogits/gogs/modules/base"
	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/setting"
)

// Context represents context of a request.
type Context struct {
	*Render
	c        martini.Context
	p        martini.Params
	Req      *http.Request
	Res      http.ResponseWriter
	Flash    *Flash
	Session  session.SessionStore
	Cache    cache.Cache
	User     *models.User
	IsSigned bool

	csrfToken string

	Repo struct {
		IsOwner     bool
		IsTrueOwner bool
		IsWatching  bool
		IsBranch    bool
		IsTag       bool
		IsCommit    bool
		HasAccess   bool
		Repository  *models.Repository
		Owner       *models.User
		Commit      *git.Commit
		Tag         *git.Tag
		GitRepo     *git.Repository
		BranchName  string
		TagName     string
		CommitId    string
		RepoLink    string
		CloneLink   struct {
			SSH   string
			HTTPS string
			Git   string
		}
		Mirror *models.Mirror
	}
}

// Query querys form parameter.
func (ctx *Context) Query(name string) string {
	ctx.Req.ParseForm()
	return ctx.Req.Form.Get(name)
}

// func (ctx *Context) Param(name string) string {
// 	return ctx.p[name]
// }

// HasError returns true if error occurs in form validation.
func (ctx *Context) HasApiError() bool {
	hasErr, ok := ctx.Data["HasError"]
	if !ok {
		return false
	}
	return hasErr.(bool)
}

func (ctx *Context) GetErrMsg() string {
	return ctx.Data["ErrorMsg"].(string)
}

// HasError returns true if error occurs in form validation.
func (ctx *Context) HasError() bool {
	hasErr, ok := ctx.Data["HasError"]
	if !ok {
		return false
	}
	ctx.Flash.ErrorMsg = ctx.Data["ErrorMsg"].(string)
	ctx.Data["Flash"] = ctx.Flash
	return hasErr.(bool)
}

// HTML calls render.HTML underlying but reduce one argument.
func (ctx *Context) HTML(status int, name base.TplName, htmlOpt ...HTMLOptions) {
	ctx.Render.HTML(status, string(name), ctx.Data, htmlOpt...)
}

// RenderWithErr used for page has form validation but need to prompt error to users.
func (ctx *Context) RenderWithErr(msg string, tpl base.TplName, form auth.Form) {
	if form != nil {
		auth.AssignForm(form, ctx.Data)
	}
	ctx.Flash.ErrorMsg = msg
	ctx.Data["Flash"] = ctx.Flash
	ctx.HTML(200, tpl)
}

// Handle handles and logs error by given status.
func (ctx *Context) Handle(status int, title string, err error) {
	if err != nil {
		log.Error("%s: %v", title, err)
		if martini.Dev != martini.Prod {
			ctx.Data["ErrorMsg"] = err
		}
	}

	switch status {
	case 404:
		ctx.Data["Title"] = "Page Not Found"
	case 500:
		ctx.Data["Title"] = "Internal Server Error"
	}
	ctx.HTML(status, base.TplName(fmt.Sprintf("status/%d", status)))
}

func (ctx *Context) GetCookie(name string) string {
	cookie, err := ctx.Req.Cookie(name)
	if err != nil {
		return ""
	}
	return cookie.Value
}

func (ctx *Context) SetCookie(name string, value string, others ...interface{}) {
	cookie := http.Cookie{}
	cookie.Name = name
	cookie.Value = value

	if len(others) > 0 {
		switch v := others[0].(type) {
		case int:
			cookie.MaxAge = v
		case int64:
			cookie.MaxAge = int(v)
		case int32:
			cookie.MaxAge = int(v)
		}
	}

	// default "/"
	if len(others) > 1 {
		if v, ok := others[1].(string); ok && len(v) > 0 {
			cookie.Path = v
		}
	} else {
		cookie.Path = "/"
	}

	// default empty
	if len(others) > 2 {
		if v, ok := others[2].(string); ok && len(v) > 0 {
			cookie.Domain = v
		}
	}

	// default empty
	if len(others) > 3 {
		switch v := others[3].(type) {
		case bool:
			cookie.Secure = v
		default:
			if others[3] != nil {
				cookie.Secure = true
			}
		}
	}

	// default false. for session cookie default true
	if len(others) > 4 {
		if v, ok := others[4].(bool); ok && v {
			cookie.HttpOnly = true
		}
	}

	ctx.Res.Header().Add("Set-Cookie", cookie.String())
}

// Get secure cookie from request by a given key.
func (ctx *Context) GetSecureCookie(Secret, key string) (string, bool) {
	val := ctx.GetCookie(key)
	if val == "" {
		return "", false
	}

	parts := strings.SplitN(val, "|", 3)

	if len(parts) != 3 {
		return "", false
	}

	vs := parts[0]
	timestamp := parts[1]
	sig := parts[2]

	h := hmac.New(sha1.New, []byte(Secret))
	fmt.Fprintf(h, "%s%s", vs, timestamp)

	if fmt.Sprintf("%02x", h.Sum(nil)) != sig {
		return "", false
	}
	res, _ := base64.URLEncoding.DecodeString(vs)
	return string(res), true
}

// Set Secure cookie for response.
func (ctx *Context) SetSecureCookie(Secret, name, value string, others ...interface{}) {
	vs := base64.URLEncoding.EncodeToString([]byte(value))
	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
	h := hmac.New(sha1.New, []byte(Secret))
	fmt.Fprintf(h, "%s%s", vs, timestamp)
	sig := fmt.Sprintf("%02x", h.Sum(nil))
	cookie := strings.Join([]string{vs, timestamp, sig}, "|")
	ctx.SetCookie(name, cookie, others...)
}

func (ctx *Context) CsrfToken() string {
	if len(ctx.csrfToken) > 0 {
		return ctx.csrfToken
	}

	token := ctx.GetCookie("_csrf")
	if len(token) == 0 {
		token = base.GetRandomString(30)
		ctx.SetCookie("_csrf", token)
	}
	ctx.csrfToken = token
	return token
}

func (ctx *Context) CsrfTokenValid() bool {
	token := ctx.Query("_csrf")
	if token == "" {
		token = ctx.Req.Header.Get("X-Csrf-Token")
	}
	if token == "" {
		return false
	} else if ctx.csrfToken != token {
		return false
	}
	return true
}

func (ctx *Context) ServeFile(file string, names ...string) {
	var name string
	if len(names) > 0 {
		name = names[0]
	} else {
		name = filepath.Base(file)
	}
	ctx.Res.Header().Set("Content-Description", "File Transfer")
	ctx.Res.Header().Set("Content-Type", "application/octet-stream")
	ctx.Res.Header().Set("Content-Disposition", "attachment; filename="+name)
	ctx.Res.Header().Set("Content-Transfer-Encoding", "binary")
	ctx.Res.Header().Set("Expires", "0")
	ctx.Res.Header().Set("Cache-Control", "must-revalidate")
	ctx.Res.Header().Set("Pragma", "public")
	http.ServeFile(ctx.Res, ctx.Req, file)
}

func (ctx *Context) ServeContent(name string, r io.ReadSeeker, params ...interface{}) {
	modtime := time.Now()
	for _, p := range params {
		switch v := p.(type) {
		case time.Time:
			modtime = v
		}
	}
	ctx.Res.Header().Set("Content-Description", "File Transfer")
	ctx.Res.Header().Set("Content-Type", "application/octet-stream")
	ctx.Res.Header().Set("Content-Disposition", "attachment; filename="+name)
	ctx.Res.Header().Set("Content-Transfer-Encoding", "binary")
	ctx.Res.Header().Set("Expires", "0")
	ctx.Res.Header().Set("Cache-Control", "must-revalidate")
	ctx.Res.Header().Set("Pragma", "public")
	http.ServeContent(ctx.Res, ctx.Req, name, modtime, r)
}

type Flash struct {
	url.Values
	ErrorMsg, SuccessMsg string
}

func (f *Flash) Error(msg string) {
	f.Set("error", msg)
	f.ErrorMsg = msg
}

func (f *Flash) Success(msg string) {
	f.Set("success", msg)
	f.SuccessMsg = msg
}

// InitContext initializes a classic context for a request.
func InitContext() martini.Handler {
	return func(res http.ResponseWriter, r *http.Request, c martini.Context, rd *Render) {
		ctx := &Context{
			c: c,
			// p:      p,
			Req:    r,
			Res:    res,
			Cache:  setting.Cache,
			Render: rd,
		}
		ctx.Data["PageStartTime"] = time.Now()

		// start session
		ctx.Session = setting.SessionManager.SessionStart(res, r)

		// Get flash.
		values, err := url.ParseQuery(ctx.GetCookie("gogs_flash"))
		if err != nil {
			log.Error("InitContext.ParseQuery(flash): %v", err)
		} else if len(values) > 0 {
			ctx.Flash = &Flash{Values: values}
			ctx.Flash.ErrorMsg = ctx.Flash.Get("error")
			ctx.Flash.SuccessMsg = ctx.Flash.Get("success")
			ctx.Data["Flash"] = ctx.Flash
			ctx.SetCookie("gogs_flash", "", -1)
		}
		ctx.Flash = &Flash{Values: url.Values{}}

		rw := res.(martini.ResponseWriter)
		rw.Before(func(martini.ResponseWriter) {
			ctx.Session.SessionRelease(res)

			if flash := ctx.Flash.Encode(); len(flash) > 0 {
				ctx.SetCookie("gogs_flash", flash, 0)
			}
		})

		// Get user from session if logined.
		user := auth.SignedInUser(ctx.req.Header, ctx.Session)
		ctx.User = user
		ctx.IsSigned = user != nil

		ctx.Data["IsSigned"] = ctx.IsSigned

		if user != nil {
			ctx.Data["SignedUser"] = user
			ctx.Data["SignedUserId"] = user.Id
			ctx.Data["SignedUserName"] = user.Name
			ctx.Data["IsAdmin"] = ctx.User.IsAdmin
		}

		// If request sends files, parse them here otherwise the Query() can't be parsed and the CsrfToken will be invalid.
		if strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") {
			if err = ctx.Req.ParseMultipartForm(setting.AttachmentMaxSize << 20); err != nil { // 32MB max size
				ctx.Handle(500, "issue.Comment(ctx.Req.ParseMultipartForm)", err)
				return
			}
		}

		// get or create csrf token
		ctx.Data["CsrfToken"] = ctx.CsrfToken()
		ctx.Data["CsrfTokenHtml"] = template.HTML(`<input type="hidden" name="_csrf" value="` + ctx.csrfToken + `">`)

		c.Map(ctx)

		c.Next()
	}
}
<commit_msg>Ignore EOF error when parsing form.<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package middleware

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/go-martini/martini"

	"github.com/gogits/cache"
	"github.com/gogits/git"
	"github.com/gogits/session"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/auth"
	"github.com/gogits/gogs/modules/base"
	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/setting"
)

// Context represents context of a request.
type Context struct {
	*Render
	c        martini.Context
	p        martini.Params
	Req      *http.Request
	Res      http.ResponseWriter
	Flash    *Flash
	Session  session.SessionStore
	Cache    cache.Cache
	User     *models.User
	IsSigned bool

	csrfToken string

	Repo struct {
		IsOwner     bool
		IsTrueOwner bool
		IsWatching  bool
		IsBranch    bool
		IsTag       bool
		IsCommit    bool
		HasAccess   bool
		Repository  *models.Repository
		Owner       *models.User
		Commit      *git.Commit
		Tag         *git.Tag
		GitRepo     *git.Repository
		BranchName  string
		TagName     string
		CommitId    string
		RepoLink    string
		CloneLink   struct {
			SSH   string
			HTTPS string
			Git   string
		}
		Mirror *models.Mirror
	}
}

// Query querys form parameter.
func (ctx *Context) Query(name string) string {
	ctx.Req.ParseForm()
	return ctx.Req.Form.Get(name)
}

// func (ctx *Context) Param(name string) string {
// 	return ctx.p[name]
// }

// HasError returns true if error occurs in form validation.
func (ctx *Context) HasApiError() bool {
	hasErr, ok := ctx.Data["HasError"]
	if !ok {
		return false
	}
	return hasErr.(bool)
}

func (ctx *Context) GetErrMsg() string {
	return ctx.Data["ErrorMsg"].(string)
}

// HasError returns true if error occurs in form validation.
func (ctx *Context) HasError() bool {
	hasErr, ok := ctx.Data["HasError"]
	if !ok {
		return false
	}
	ctx.Flash.ErrorMsg = ctx.Data["ErrorMsg"].(string)
	ctx.Data["Flash"] = ctx.Flash
	return hasErr.(bool)
}

// HTML calls render.HTML underlying but reduce one argument.
func (ctx *Context) HTML(status int, name base.TplName, htmlOpt ...HTMLOptions) {
	ctx.Render.HTML(status, string(name), ctx.Data, htmlOpt...)
}

// RenderWithErr used for page has form validation but need to prompt error to users.
func (ctx *Context) RenderWithErr(msg string, tpl base.TplName, form auth.Form) {
	if form != nil {
		auth.AssignForm(form, ctx.Data)
	}
	ctx.Flash.ErrorMsg = msg
	ctx.Data["Flash"] = ctx.Flash
	ctx.HTML(200, tpl)
}

// Handle handles and logs error by given status.
func (ctx *Context) Handle(status int, title string, err error) {
	if err != nil {
		log.Error("%s: %v", title, err)
		if martini.Dev != martini.Prod {
			ctx.Data["ErrorMsg"] = err
		}
	}

	switch status {
	case 404:
		ctx.Data["Title"] = "Page Not Found"
	case 500:
		ctx.Data["Title"] = "Internal Server Error"
	}
	ctx.HTML(status, base.TplName(fmt.Sprintf("status/%d", status)))
}

func (ctx *Context) GetCookie(name string) string {
	cookie, err := ctx.Req.Cookie(name)
	if err != nil {
		return ""
	}
	return cookie.Value
}

func (ctx *Context) SetCookie(name string, value string, others ...interface{}) {
	cookie := http.Cookie{}
	cookie.Name = name
	cookie.Value = value

	if len(others) > 0 {
		switch v := others[0].(type) {
		case int:
			cookie.MaxAge = v
		case int64:
			cookie.MaxAge = int(v)
		case int32:
			cookie.MaxAge = int(v)
		}
	}

	// default "/"
	if len(others) > 1 {
		if v, ok := others[1].(string); ok && len(v) > 0 {
			cookie.Path = v
		}
	} else {
		cookie.Path = "/"
	}

	// default empty
	if len(others) > 2 {
		if v, ok := others[2].(string); ok && len(v) > 0 {
			cookie.Domain = v
		}
	}

	// default empty
	if len(others) > 3 {
		switch v := others[3].(type) {
		case bool:
			cookie.Secure = v
		default:
			if others[3] != nil {
				cookie.Secure = true
			}
		}
	}

	// default false. for session cookie default true
	if len(others) > 4 {
		if v, ok := others[4].(bool); ok && v {
			cookie.HttpOnly = true
		}
	}

	ctx.Res.Header().Add("Set-Cookie", cookie.String())
}

// Get secure cookie from request by a given key.
func (ctx *Context) GetSecureCookie(Secret, key string) (string, bool) {
	val := ctx.GetCookie(key)
	if val == "" {
		return "", false
	}

	parts := strings.SplitN(val, "|", 3)

	if len(parts) != 3 {
		return "", false
	}

	vs := parts[0]
	timestamp := parts[1]
	sig := parts[2]

	h := hmac.New(sha1.New, []byte(Secret))
	fmt.Fprintf(h, "%s%s", vs, timestamp)

	if fmt.Sprintf("%02x", h.Sum(nil)) != sig {
		return "", false
	}
	res, _ := base64.URLEncoding.DecodeString(vs)
	return string(res), true
}

// Set Secure cookie for response.
func (ctx *Context) SetSecureCookie(Secret, name, value string, others ...interface{}) {
	vs := base64.URLEncoding.EncodeToString([]byte(value))
	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
	h := hmac.New(sha1.New, []byte(Secret))
	fmt.Fprintf(h, "%s%s", vs, timestamp)
	sig := fmt.Sprintf("%02x", h.Sum(nil))
	cookie := strings.Join([]string{vs, timestamp, sig}, "|")
	ctx.SetCookie(name, cookie, others...)
}

func (ctx *Context) CsrfToken() string {
	if len(ctx.csrfToken) > 0 {
		return ctx.csrfToken
	}

	token := ctx.GetCookie("_csrf")
	if len(token) == 0 {
		token = base.GetRandomString(30)
		ctx.SetCookie("_csrf", token)
	}
	ctx.csrfToken = token
	return token
}

func (ctx *Context) CsrfTokenValid() bool {
	token := ctx.Query("_csrf")
	if token == "" {
		token = ctx.Req.Header.Get("X-Csrf-Token")
	}
	if token == "" {
		return false
	} else if ctx.csrfToken != token {
		return false
	}
	return true
}

func (ctx *Context) ServeFile(file string, names ...string) {
	var name string
	if len(names) > 0 {
		name = names[0]
	} else {
		name = filepath.Base(file)
	}
	ctx.Res.Header().Set("Content-Description", "File Transfer")
	ctx.Res.Header().Set("Content-Type", "application/octet-stream")
	ctx.Res.Header().Set("Content-Disposition", "attachment; filename="+name)
	ctx.Res.Header().Set("Content-Transfer-Encoding", "binary")
	ctx.Res.Header().Set("Expires", "0")
	ctx.Res.Header().Set("Cache-Control", "must-revalidate")
	ctx.Res.Header().Set("Pragma", "public")
	http.ServeFile(ctx.Res, ctx.Req, file)
}

func (ctx *Context) ServeContent(name string, r io.ReadSeeker, params ...interface{}) {
	modtime := time.Now()
	for _, p := range params {
		switch v := p.(type) {
		case time.Time:
			modtime = v
		}
	}
	ctx.Res.Header().Set("Content-Description", "File Transfer")
	ctx.Res.Header().Set("Content-Type", "application/octet-stream")
	ctx.Res.Header().Set("Content-Disposition", "attachment; filename="+name)
	ctx.Res.Header().Set("Content-Transfer-Encoding", "binary")
	ctx.Res.Header().Set("Expires", "0")
	ctx.Res.Header().Set("Cache-Control", "must-revalidate")
	ctx.Res.Header().Set("Pragma", "public")
	http.ServeContent(ctx.Res, ctx.Req, name, modtime, r)
}

type Flash struct {
	url.Values
	ErrorMsg, SuccessMsg string
}

func (f *Flash) Error(msg string) {
	f.Set("error", msg)
	f.ErrorMsg = msg
}

func (f *Flash) Success(msg string) {
	f.Set("success", msg)
	f.SuccessMsg = msg
}

// InitContext initializes a classic context for a request.
func InitContext() martini.Handler {
	return func(res http.ResponseWriter, r *http.Request, c martini.Context, rd *Render) {
		ctx := &Context{
			c: c,
			// p:      p,
			Req:    r,
			Res:    res,
			Cache:  setting.Cache,
			Render: rd,
		}
		ctx.Data["PageStartTime"] = time.Now()

		// start session
		ctx.Session = setting.SessionManager.SessionStart(res, r)

		// Get flash.
		values, err := url.ParseQuery(ctx.GetCookie("gogs_flash"))
		if err != nil {
			log.Error("InitContext.ParseQuery(flash): %v", err)
		} else if len(values) > 0 {
			ctx.Flash = &Flash{Values: values}
			ctx.Flash.ErrorMsg = ctx.Flash.Get("error")
			ctx.Flash.SuccessMsg = ctx.Flash.Get("success")
			ctx.Data["Flash"] = ctx.Flash
			ctx.SetCookie("gogs_flash", "", -1)
		}
		ctx.Flash = &Flash{Values: url.Values{}}

		rw := res.(martini.ResponseWriter)
		rw.Before(func(martini.ResponseWriter) {
			ctx.Session.SessionRelease(res)

			if flash := ctx.Flash.Encode(); len(flash) > 0 {
				ctx.SetCookie("gogs_flash", flash, 0)
			}
		})

		// Get user from session if logined.
		user := auth.SignedInUser(ctx.req.Header, ctx.Session)
		ctx.User = user
		ctx.IsSigned = user != nil

		ctx.Data["IsSigned"] = ctx.IsSigned

		if user != nil {
			ctx.Data["SignedUser"] = user
			ctx.Data["SignedUserId"] = user.Id
			ctx.Data["SignedUserName"] = user.Name
			ctx.Data["IsAdmin"] = ctx.User.IsAdmin
		}

		// If request sends files, parse them here otherwise the Query() can't be parsed and the CsrfToken will be invalid.
		if r.Method == "POST" && strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") {
			if err = ctx.Req.ParseMultipartForm(setting.AttachmentMaxSize << 20); err != nil && !strings.Contains(err.Error(), "EOF") { // 32MB max size
				ctx.Handle(500, "issue.Comment(ctx.Req.ParseMultipartForm)", err)
				return
			}
		}

		// get or create csrf token
		ctx.Data["CsrfToken"] = ctx.CsrfToken()
		ctx.Data["CsrfTokenHtml"] = template.HTML(`<input type="hidden" name="_csrf" value="` + ctx.csrfToken + `">`)

		c.Map(ctx)

		c.Next()
	}
}
<|endoftext|>


<commit_before>package emitter

import (
	"fmt"
	"net"
	"net/http"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/metric"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

type PrometheusEmitter struct {
	buildsStarted     prometheus.Counter
	buildsFinished    prometheus.Counter
	buildsSucceeded   prometheus.Counter
	buildsErrored     prometheus.Counter
	buildsFailed      prometheus.Counter
	buildsAborted     prometheus.Counter
	buildsFinishedVec *prometheus.CounterVec
	buildDurationsVec *prometheus.HistogramVec

	workerContainers *prometheus.GaugeVec
	workerVolumes    *prometheus.GaugeVec

	httpRequestsDuration *prometheus.HistogramVec
}

type PrometheusConfig struct {
	BindIP   string `long:"prometheus-bind-ip" description:"IP to listen on to expose Prometheus metrics."`
	BindPort string `long:"prometheus-bind-port" description:"Port to listen on to expose Prometheus metrics."`
}

func init() {
	metric.RegisterEmitter(&PrometheusConfig{})
}

func (config *PrometheusConfig) Description() string { return "Prometheus" }
func (config *PrometheusConfig) IsConfigured() bool {
	return config.BindPort != "" && config.BindIP != ""
}
func (config *PrometheusConfig) bind() string {
	return fmt.Sprintf("%s:%s", config.BindIP, config.BindPort)
}

func (config *PrometheusConfig) NewEmitter() (metric.Emitter, error) {
	buildsStarted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "started_total",
		Help:      "Total number of Concourse builds started.",
	})
	prometheus.MustRegister(buildsStarted)

	buildsFinished := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "finished_total",
		Help:      "Total number of Concourse builds finished.",
	})
	prometheus.MustRegister(buildsFinished)

	buildsSucceeded := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "succeeded_total",
		Help:      "Total number of Concourse builds succeeded.",
	})
	prometheus.MustRegister(buildsSucceeded)

	buildsErrored := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "errored_total",
		Help:      "Total number of Concourse builds errored.",
	})
	prometheus.MustRegister(buildsErrored)

	buildsFailed := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "failed_total",
		Help:      "Total number of Concourse builds failed.",
	})
	prometheus.MustRegister(buildsFailed)

	buildsAborted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "aborted_total",
		Help:      "Total number of Concourse builds aborted.",
	})
	prometheus.MustRegister(buildsAborted)

	buildsFinishedVec := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "finished",
			Help:      "Count of builds finished across various dimensions.",
		},
		[]string{"team", "pipeline", "status"},
	)
	prometheus.MustRegister(buildsFinishedVec)
	buildDurationsVec := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "duration_seconds",
			Help:      "Build time in seconds",
		},
		[]string{"team", "pipeline"},
	)
	prometheus.MustRegister(buildDurationsVec)

	workerContainers := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "containers",
			Help:      "Number of containers per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerContainers)
	workerVolumes := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "volumes",
			Help:      "Number of volumes per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerVolumes)

	httpRequestsDuration := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "http_responses",
			Name:      "duration_seconds",
			Help:      "Response time in seconds",
		},
		[]string{"path", "method"},
	)
	prometheus.MustRegister(httpRequestsDuration)

	listener, err := net.Listen("tcp", config.bind())
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, promhttp.Handler())

	return &PrometheusEmitter{
		buildsStarted:     buildsStarted,
		buildsFinished:    buildsFinished,
		buildsFinishedVec: buildsFinishedVec,
		buildDurationsVec: buildDurationsVec,
		buildsSucceeded:   buildsSucceeded,
		buildsErrored:     buildsErrored,
		buildsFailed:      buildsFailed,
		buildsAborted:     buildsAborted,

		workerContainers: workerContainers,
		workerVolumes:    workerVolumes,

		httpRequestsDuration: httpRequestsDuration,
	}, nil
}

// Emit processes incoming metrics.
// In order to provide idiomatic Prometheus metrics, we'll have to convert the various
// Event types (differentiated by the less-than-ideal string Name field) into different
// Prometheus metrics.
func (emitter *PrometheusEmitter) Emit(logger lager.Logger, event metric.Event) {
	switch event.Name {
	case "build started":
		emitter.buildsStarted.Inc()
	case "build finished":
		emitter.buildFinishedMetrics(logger, event)
	case "worker containers":
		emitter.workerContainersMetrics(logger, event)
	case "worker volumes":
		emitter.workerVolumesMetrics(logger, event)
	case "http response time":
		emitter.httpResponseTimeMetrics(logger, event)
	default:
		// unless we have a specific metric, we do nothing
	}
}

func (emitter *PrometheusEmitter) buildFinishedMetrics(logger lager.Logger, event metric.Event) {
	// concourse_builds_finished_total
	emitter.buildsFinished.Inc()

	// concourse_builds_finished
	team, exists := event.Attributes["team_name"]
	if !exists {
		logger.Error("failed-to-find-team-name-in-event", fmt.Errorf("expected team_name to exist in event.Attributes"))
	}

	pipeline, exists := event.Attributes["pipeline"]
	if !exists {
		logger.Error("failed-to-find-pipeline-in-event", fmt.Errorf("expected pipeline to exist in event.Attributes"))
	}

	buildStatus, exists := event.Attributes["build_status"]
	if !exists {
		logger.Error("failed-to-find-build_status-in-event", fmt.Errorf("expected build_status to exist in event.Attributes"))
	}
	emitter.buildsFinishedVec.WithLabelValues(team, pipeline, buildStatus).Inc()

	// concourse_builds_(aborted|succeeded|failed|errored)_total
	switch buildStatus {
	case string(db.BuildStatusAborted):
		// concourse_builds_aborted_total
		emitter.buildsAborted.Inc()
	case string(db.BuildStatusSucceeded):
		// concourse_builds_succeeded_total
		emitter.buildsSucceeded.Inc()
	case string(db.BuildStatusFailed):
		// concourse_builds_failed_total
		emitter.buildsFailed.Inc()
	case string(db.BuildStatusErrored):
		// concourse_builds_errored_total
		emitter.buildsErrored.Inc()
	}

	// concourse_builds_duration_seconds
	duration, ok := event.Value.(float64)
	if !ok {
		logger.Error("build-finished-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}
	// seconds are the standard prometheus base unit for time
	duration = duration / 1000
	emitter.buildDurationsVec.WithLabelValues(team, pipeline).Observe(duration)
}

func (emitter *PrometheusEmitter) workerContainersMetrics(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	containers, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerContainers.WithLabelValues(worker).Set(float64(containers))
}

func (emitter *PrometheusEmitter) workerVolumesMetrics(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	volumes, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerVolumes.WithLabelValues(worker).Set(float64(volumes))
}

func (emitter *PrometheusEmitter) httpResponseTimeMetrics(logger lager.Logger, event metric.Event) {
	path, exists := event.Attributes["path"]
	if !exists {
		logger.Error("failed-to-find-path-in-event", fmt.Errorf("expected path to exist in event.Attributes"))
	}
	method, exists := event.Attributes["method"]
	if !exists {
		logger.Error("failed-to-find-method-in-event", fmt.Errorf("expected method to exist in event.Attributes"))
	}

	responseTime, ok := event.Value.(float64)
	if !ok {
		logger.Error("http-response-time-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}

	emitter.httpRequestsDuration.WithLabelValues(path, method).Observe(responseTime / 1000)
}
<commit_msg>prometheus: remove path from http durations metric<commit_after>package emitter

import (
	"fmt"
	"net"
	"net/http"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/metric"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

type PrometheusEmitter struct {
	buildsStarted     prometheus.Counter
	buildsFinished    prometheus.Counter
	buildsSucceeded   prometheus.Counter
	buildsErrored     prometheus.Counter
	buildsFailed      prometheus.Counter
	buildsAborted     prometheus.Counter
	buildsFinishedVec *prometheus.CounterVec
	buildDurationsVec *prometheus.HistogramVec

	workerContainers *prometheus.GaugeVec
	workerVolumes    *prometheus.GaugeVec

	httpRequestsDuration *prometheus.HistogramVec
}

type PrometheusConfig struct {
	BindIP   string `long:"prometheus-bind-ip" description:"IP to listen on to expose Prometheus metrics."`
	BindPort string `long:"prometheus-bind-port" description:"Port to listen on to expose Prometheus metrics."`
}

func init() {
	metric.RegisterEmitter(&PrometheusConfig{})
}

func (config *PrometheusConfig) Description() string { return "Prometheus" }
func (config *PrometheusConfig) IsConfigured() bool {
	return config.BindPort != "" && config.BindIP != ""
}
func (config *PrometheusConfig) bind() string {
	return fmt.Sprintf("%s:%s", config.BindIP, config.BindPort)
}

func (config *PrometheusConfig) NewEmitter() (metric.Emitter, error) {
	buildsStarted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "started_total",
		Help:      "Total number of Concourse builds started.",
	})
	prometheus.MustRegister(buildsStarted)

	buildsFinished := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "finished_total",
		Help:      "Total number of Concourse builds finished.",
	})
	prometheus.MustRegister(buildsFinished)

	buildsSucceeded := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "succeeded_total",
		Help:      "Total number of Concourse builds succeeded.",
	})
	prometheus.MustRegister(buildsSucceeded)

	buildsErrored := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "errored_total",
		Help:      "Total number of Concourse builds errored.",
	})
	prometheus.MustRegister(buildsErrored)

	buildsFailed := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "failed_total",
		Help:      "Total number of Concourse builds failed.",
	})
	prometheus.MustRegister(buildsFailed)

	buildsAborted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "aborted_total",
		Help:      "Total number of Concourse builds aborted.",
	})
	prometheus.MustRegister(buildsAborted)

	buildsFinishedVec := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "finished",
			Help:      "Count of builds finished across various dimensions.",
		},
		[]string{"team", "pipeline", "status"},
	)
	prometheus.MustRegister(buildsFinishedVec)
	buildDurationsVec := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "duration_seconds",
			Help:      "Build time in seconds",
		},
		[]string{"team", "pipeline"},
	)
	prometheus.MustRegister(buildDurationsVec)

	workerContainers := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "containers",
			Help:      "Number of containers per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerContainers)
	workerVolumes := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "volumes",
			Help:      "Number of volumes per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerVolumes)

	httpRequestsDuration := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "http_responses",
			Name:      "duration_seconds",
			Help:      "Response time in seconds",
		},
		[]string{"method"},
	)
	prometheus.MustRegister(httpRequestsDuration)

	listener, err := net.Listen("tcp", config.bind())
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, promhttp.Handler())

	return &PrometheusEmitter{
		buildsStarted:     buildsStarted,
		buildsFinished:    buildsFinished,
		buildsFinishedVec: buildsFinishedVec,
		buildDurationsVec: buildDurationsVec,
		buildsSucceeded:   buildsSucceeded,
		buildsErrored:     buildsErrored,
		buildsFailed:      buildsFailed,
		buildsAborted:     buildsAborted,

		workerContainers: workerContainers,
		workerVolumes:    workerVolumes,

		httpRequestsDuration: httpRequestsDuration,
	}, nil
}

// Emit processes incoming metrics.
// In order to provide idiomatic Prometheus metrics, we'll have to convert the various
// Event types (differentiated by the less-than-ideal string Name field) into different
// Prometheus metrics.
func (emitter *PrometheusEmitter) Emit(logger lager.Logger, event metric.Event) {
	switch event.Name {
	case "build started":
		emitter.buildsStarted.Inc()
	case "build finished":
		emitter.buildFinishedMetrics(logger, event)
	case "worker containers":
		emitter.workerContainersMetrics(logger, event)
	case "worker volumes":
		emitter.workerVolumesMetrics(logger, event)
	case "http response time":
		emitter.httpResponseTimeMetrics(logger, event)
	default:
		// unless we have a specific metric, we do nothing
	}
}

func (emitter *PrometheusEmitter) buildFinishedMetrics(logger lager.Logger, event metric.Event) {
	// concourse_builds_finished_total
	emitter.buildsFinished.Inc()

	// concourse_builds_finished
	team, exists := event.Attributes["team_name"]
	if !exists {
		logger.Error("failed-to-find-team-name-in-event", fmt.Errorf("expected team_name to exist in event.Attributes"))
	}

	pipeline, exists := event.Attributes["pipeline"]
	if !exists {
		logger.Error("failed-to-find-pipeline-in-event", fmt.Errorf("expected pipeline to exist in event.Attributes"))
	}

	buildStatus, exists := event.Attributes["build_status"]
	if !exists {
		logger.Error("failed-to-find-build_status-in-event", fmt.Errorf("expected build_status to exist in event.Attributes"))
	}
	emitter.buildsFinishedVec.WithLabelValues(team, pipeline, buildStatus).Inc()

	// concourse_builds_(aborted|succeeded|failed|errored)_total
	switch buildStatus {
	case string(db.BuildStatusAborted):
		// concourse_builds_aborted_total
		emitter.buildsAborted.Inc()
	case string(db.BuildStatusSucceeded):
		// concourse_builds_succeeded_total
		emitter.buildsSucceeded.Inc()
	case string(db.BuildStatusFailed):
		// concourse_builds_failed_total
		emitter.buildsFailed.Inc()
	case string(db.BuildStatusErrored):
		// concourse_builds_errored_total
		emitter.buildsErrored.Inc()
	}

	// concourse_builds_duration_seconds
	duration, ok := event.Value.(float64)
	if !ok {
		logger.Error("build-finished-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}
	// seconds are the standard prometheus base unit for time
	duration = duration / 1000
	emitter.buildDurationsVec.WithLabelValues(team, pipeline).Observe(duration)
}

func (emitter *PrometheusEmitter) workerContainersMetrics(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	containers, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerContainers.WithLabelValues(worker).Set(float64(containers))
}

func (emitter *PrometheusEmitter) workerVolumesMetrics(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	volumes, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerVolumes.WithLabelValues(worker).Set(float64(volumes))
}

func (emitter *PrometheusEmitter) httpResponseTimeMetrics(logger lager.Logger, event metric.Event) {
	method, exists := event.Attributes["method"]
	if !exists {
		logger.Error("failed-to-find-method-in-event", fmt.Errorf("expected method to exist in event.Attributes"))
	}

	responseTime, ok := event.Value.(float64)
	if !ok {
		logger.Error("http-response-time-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}

	emitter.httpRequestsDuration.WithLabelValues(method).Observe(responseTime / 1000)
}
<|endoftext|>


<commit_before>package teststat

import (
	"math"
	"strconv"
	"strings"
	"testing"

	"github.com/codahale/hdrhistogram"
)

// AssertCirconusNormalHistogram ensures the Circonus Histogram data captured in
// the result slice abides a normal distribution.
func AssertCirconusNormalHistogram(t *testing.T, mean, stdev, min, max int64, result []string) {
	if len(result) <= 0 {
		t.Fatal("no results")
	}

	// Circonus just dumps the raw counts. We need to do our own statistical analysis.
	h := hdrhistogram.New(min, max, 3)

	for _, s := range result {
		// "H[1.23e04]=123"
		toks := strings.Split(s, "=")
		if len(toks) != 2 {
			t.Fatalf("bad H value: %q", s)
		}

		var bucket string
		bucket = toks[0]
		bucket = bucket[2 : len(bucket)-1] // "H[1.23e04]" -> "1.23e04"
		f, err := strconv.ParseFloat(bucket, 64)
		if err != nil {
			t.Fatalf("error parsing H value: %q: %v", s, err)
		}

		count, err := strconv.ParseFloat(toks[1], 64)
		if err != nil {
			t.Fatalf("error parsing H count: %q: %v", s, err)
		}

		h.RecordValues(int64(f), int64(count))
	}

	// Apparently Circonus buckets observations by dropping a sigfig, so we have
	// very coarse tolerance.
	var tolerance int64 = 20
	for _, quantile := range []int{50, 90, 99} {
		want := normalValueAtQuantile(mean, stdev, quantile)
		have := h.ValueAtQuantile(float64(quantile))
		if int64(math.Abs(float64(want)-float64(have))) > tolerance {
			t.Errorf("quantile %d: want %d, have %d", quantile, want, have)
		}
	}
}
<commit_msg>metrics/circonus: increase tolerance<commit_after>package teststat

import (
	"math"
	"strconv"
	"strings"
	"testing"

	"github.com/codahale/hdrhistogram"
)

// AssertCirconusNormalHistogram ensures the Circonus Histogram data captured in
// the result slice abides a normal distribution.
func AssertCirconusNormalHistogram(t *testing.T, mean, stdev, min, max int64, result []string) {
	if len(result) <= 0 {
		t.Fatal("no results")
	}

	// Circonus just dumps the raw counts. We need to do our own statistical analysis.
	h := hdrhistogram.New(min, max, 3)

	for _, s := range result {
		// "H[1.23e04]=123"
		toks := strings.Split(s, "=")
		if len(toks) != 2 {
			t.Fatalf("bad H value: %q", s)
		}

		var bucket string
		bucket = toks[0]
		bucket = bucket[2 : len(bucket)-1] // "H[1.23e04]" -> "1.23e04"
		f, err := strconv.ParseFloat(bucket, 64)
		if err != nil {
			t.Fatalf("error parsing H value: %q: %v", s, err)
		}

		count, err := strconv.ParseFloat(toks[1], 64)
		if err != nil {
			t.Fatalf("error parsing H count: %q: %v", s, err)
		}

		h.RecordValues(int64(f), int64(count))
	}

	// Apparently Circonus buckets observations by dropping a sigfig, so we have
	// very coarse tolerance.
	var tolerance int64 = 30
	for _, quantile := range []int{50, 90, 99} {
		want := normalValueAtQuantile(mean, stdev, quantile)
		have := h.ValueAtQuantile(float64(quantile))
		if int64(math.Abs(float64(want)-float64(have))) > tolerance {
			t.Errorf("quantile %d: want %d, have %d", quantile, want, have)
		}
	}
}
<|endoftext|>