<commit_before>package diff

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"
)

const (
	tokenDiffGit        = "diff --git" // diff --git a/sample.old.txt b/sample.new.txt
	tokenOldFile        = "---"        // --- sample.old.txt	2016-10-13 05:09:35.820791185 +0900
	tokenNewFile        = "+++"        // +++ sample.new.txt	2016-10-13 05:15:26.839245048 +0900
	tokenStartHunk      = "@@"         // @@ -1,3 +1,4 @@
	tokenUnchangedLine  = " "          //  unchanged, contextual line
	tokenAddedLine      = "+"          // +added line
	tokenDeletedLine    = "-"          // -deleted line
	tokenNoNewlineAtEOF = `\`          // \ No newline at end of file
)

var (
	// ErrNoNewFile represents error which there are no expected new file line.
	ErrNoNewFile = errors.New("no expected new file line") // +++ newfile
	// ErrNoHunks represents error which there are no expected hunks.
	ErrNoHunks = errors.New("no expected hunks") // @@ -1,3 +1,4 @@
)

// ErrInvalidHunkRange represents invalid line of hunk range. @@ -1,3 +1,4 @@
type ErrInvalidHunkRange struct {
	invalid string
}

func (e *ErrInvalidHunkRange) Error() string {
	return fmt.Sprintf("invalid hunk range: %v", e.invalid)
}

// ParseMultiFile parses a multi-file unified diff.
func ParseMultiFile(r io.Reader) ([]*FileDiff, error) {
	return (&multiFileParser{r: bufio.NewReader(r)}).Parse()
}

type multiFileParser struct {
	r *bufio.Reader
}

func (p *multiFileParser) Parse() ([]*FileDiff, error) {
	var fds []*FileDiff
	fp := &fileParser{r: p.r}
	for {
		fd, err := fp.Parse()
		if err != nil || fd == nil {
			break
		}
		fds = append(fds, fd)
	}
	return fds, nil
}

// ParseFile parses a file unified diff.
func ParseFile(r io.Reader) (*FileDiff, error) {
	return (&fileParser{r: bufio.NewReader(r)}).Parse()
}

type fileParser struct {
	r *bufio.Reader
}

func (p *fileParser) Parse() (*FileDiff, error) {
	fd := &FileDiff{}
	fd.Extended = parseExtendedHeader(p.r)
	b, err := p.r.Peek(len(tokenOldFile))
	if err != nil {
		if err == io.EOF && len(fd.Extended) > 0 {
			return fd, nil
		}
		return nil, nil
	}
	if bytes.HasPrefix(b, []byte(tokenOldFile)) {
		// parse `--- sample.old.txt	2016-10-13 05:09:35.820791185 +0900`
		oldline, _ := readline(p.r) // ignore err because we know it can read something
		fd.PathOld, fd.TimeOld = parseFileHeader(oldline)
		// parse `+++ sample.new.txt	2016-10-13 05:09:35.820791185 +0900`
		if b, err := p.r.Peek(len(tokenNewFile)); err != nil || !bytes.HasPrefix(b, []byte(tokenNewFile)) {
			return nil, ErrNoNewFile
		}
		newline, _ := readline(p.r) // ignore err because we know it can read something
		fd.PathNew, fd.TimeNew = parseFileHeader(newline)
	}
	// parse hunks
	fd.Hunks, err = p.parseHunks()
	if err != nil {
		return nil, err
	}
	return fd, nil
}

func (p *fileParser) parseHunks() ([]*Hunk, error) {
	b, err := p.r.Peek(len(tokenOldFile))
	if err != nil {
		return nil, ErrNoHunks
	}
	if !bytes.HasPrefix(b, []byte(tokenStartHunk)) {
		b, err := p.r.Peek(len(tokenDiffGit))
		if err != nil {
			return nil, ErrNoHunks
		}
		if bytes.HasPrefix(b, []byte(tokenDiffGit)) {
			// git diff may contain a file diff with empty hunks.
			// e.g. delete an empty file.
			return []*Hunk{}, nil
		}
		return nil, ErrNoHunks
	}
	var hunks []*Hunk
	hp := &hunkParser{r: p.r}
	for {
		h, err := hp.Parse()
		if err != nil {
			return nil, err
		}
		if h == nil {
			break
		}
		hunks = append(hunks, h)
	}
	return hunks, nil
}

// parseFileHeader parses file header line and returns filename and timestamp.
// timestamp may be empty.
func parseFileHeader(line string) (filename, timestamp string) {
	// strip `+++ ` or `--- `
	ss := line[len(tokenOldFile)+1:]
	tabi := strings.LastIndex(ss, "\t")
	if tabi == -1 {
		return unescapeCString(ss), ""
	}
	return unescapeCString(ss[:tabi]), ss[tabi+1:]
}

func unescapeCString(str string) string {
	if !strings.HasPrefix(str, `"`) {
		// no need to unescape
		return str
	}
	str = strings.TrimPrefix(strings.TrimSuffix(str, `"`), `"`)

	res := make([]byte, 0, len(str))
	r := strings.NewReader(str)
LOOP:
	for {
		ch, err := r.ReadByte()
		if err != nil {
			break
		}
		if ch != '\\' {
			res = append(res, ch)
			continue
		}

		ch, err = r.ReadByte()
		if err != nil {
			break
		}
		switch ch {
		case 'a':
			res = append(res, '\a')
		case 'b':
			res = append(res, '\b')
		case 't':
			res = append(res, '\t')
		case 'n':
			res = append(res, '\n')
		case 'v':
			res = append(res, '\v')
		case 'f':
			res = append(res, '\f')
		case 'r':
			res = append(res, '\r')
		case '"':
			res = append(res, '"')
		case '\\':
			res = append(res, '\\')
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
			if err := r.UnreadByte(); err != nil {
				break LOOP
			}
			var oct [3]byte
			if n, _ := r.Read(oct[:]); n < 3 {
				res = append(res, oct[:n]...)
				break LOOP
			}
			ch, err := strconv.ParseUint(string(oct[:]), 8, 8)
			if err != nil {
				res = append(res, oct[:]...)
				break
			}
			res = append(res, byte(ch))
		default:
			res = append(res, ch)
		}
	}

	return string(res)
}

func parseExtendedHeader(r *bufio.Reader) []string {
	var es []string
	b, err := r.Peek(len(tokenDiffGit))
	if err != nil {
		return nil
	}
	// if starts with 'diff --git', parse extended header
	if bytes.HasPrefix(b, []byte(tokenDiffGit)) {
		diffgitline, _ := readline(r) // ignore err because we know it can read something
		es = append(es, diffgitline)
		for {
			b, err := r.Peek(len(tokenDiffGit))
			if err != nil || bytes.HasPrefix(b, []byte(tokenOldFile)) || bytes.HasPrefix(b, []byte(tokenDiffGit)) {
				break
			}
			line, _ := readline(r)
			es = append(es, string(line))
		}
	}
	return es
}

type hunkParser struct {
	r        *bufio.Reader
	lnumdiff int
}

func (p *hunkParser) Parse() (*Hunk, error) {
	if b, err := p.r.Peek(len(tokenStartHunk)); err != nil || !bytes.HasPrefix(b, []byte(tokenStartHunk)) {
		return nil, nil
	}
	rangeline, _ := readline(p.r)
	hr, err := parseHunkRange(rangeline)
	if err != nil {
		return nil, err
	}
	hunk := &Hunk{
		StartLineOld:  hr.lold,
		LineLengthOld: hr.sold,
		StartLineNew:  hr.lnew,
		LineLengthNew: hr.snew,
		Section:       hr.section,
	}
	lold := hr.lold
	lnew := hr.lnew
endhunk:
	for !p.done(lold, lnew, hr) {
		b, err := p.r.Peek(1)
		if err != nil {
			break
		}
		token := string(b)
		switch token {
		case tokenUnchangedLine, tokenAddedLine, tokenDeletedLine:
			p.lnumdiff++
			l, _ := readline(p.r)
			line := &Line{Content: l[len(token):]} // trim first token
			switch token {
			case tokenUnchangedLine:
				line.Type = LineUnchanged
				line.LnumDiff = p.lnumdiff
				line.LnumOld = lold
				line.LnumNew = lnew
				lold++
				lnew++
			case tokenAddedLine:
				line.Type = LineAdded
				line.LnumDiff = p.lnumdiff
				line.LnumNew = lnew
				lnew++
			case tokenDeletedLine:
				line.Type = LineDeleted
				line.LnumDiff = p.lnumdiff
				line.LnumOld = lold
				lold++
			}
			hunk.Lines = append(hunk.Lines, line)
		case tokenNoNewlineAtEOF:
			// skip \ No newline at end of file. just consume line
			readline(p.r)
		default:
			break endhunk
		}
	}
	p.lnumdiff++ // count up by an additional hunk
	return hunk, nil
}

func (p *hunkParser) done(lold, lnew int, hr *hunkrange) bool {
	end := (lold >= hr.lold+hr.sold && lnew >= hr.lnew+hr.snew)
	if b, err := p.r.Peek(1); err != nil || (string(b) != tokenNoNewlineAtEOF && end) {
		return true
	}
	return false
}

// @@ -l,s +l,s @@ optional section heading
type hunkrange struct {
	lold, sold, lnew, snew int
	section                string
}

// @@ -lold[,sold] +lnew[,snew] @@[ section]
// 0  1              2            3   4
func parseHunkRange(rangeline string) (*hunkrange, error) {
	ps := strings.SplitN(rangeline, " ", 5)
	invalidErr := &ErrInvalidHunkRange{invalid: rangeline}
	hunkrange := &hunkrange{}
	if len(ps) < 4 || ps[0] != "@@" || ps[3] != "@@" {
		return nil, invalidErr
	}
	old := ps[1] // -lold[,sold]
	if !strings.HasPrefix(old, "-") {
		return nil, invalidErr
	}
	lold, sold, err := parseLS(old[1:])
	if err != nil {
		return nil, invalidErr
	}
	hunkrange.lold = lold
	hunkrange.sold = sold
	new := ps[2] // +lnew[,snew]
	if !strings.HasPrefix(new, "+") {
		return nil, invalidErr
	}
	lnew, snew, err := parseLS(new[1:])
	if err != nil {
		return nil, invalidErr
	}
	hunkrange.lnew = lnew
	hunkrange.snew = snew
	if len(ps) == 5 {
		hunkrange.section = ps[4]
	}
	return hunkrange, nil
}

// l[,s]
func parseLS(ls string) (l, s int, err error) {
	ss := strings.SplitN(ls, ",", 2)
	l, err = strconv.Atoi(ss[0])
	if err != nil {
		return 0, 0, err
	}
	if len(ss) == 2 {
		s, err = strconv.Atoi(ss[1])
		if err != nil {
			return 0, 0, err
		}
	} else {
		s = 1
	}
	return l, s, nil
}

// readline reads lines from bufio.Reader with size limit. It consumes
// remaining content even if the line size reaches size limit.
func readline(r *bufio.Reader) (string, error) {
	line, isPrefix, err := r.ReadLine()
	if err != nil {
		return "", err
	}
	// consume all remaining line content
	for isPrefix {
		_, isPrefix, _ = r.ReadLine()
	}
	return string(line), nil
}
<commit_msg>rename unescapeCString to unquoteCStyle<commit_after>package diff

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"
)

const (
	tokenDiffGit        = "diff --git" // diff --git a/sample.old.txt b/sample.new.txt
	tokenOldFile        = "---"        // --- sample.old.txt	2016-10-13 05:09:35.820791185 +0900
	tokenNewFile        = "+++"        // +++ sample.new.txt	2016-10-13 05:15:26.839245048 +0900
	tokenStartHunk      = "@@"         // @@ -1,3 +1,4 @@
	tokenUnchangedLine  = " "          //  unchanged, contextual line
	tokenAddedLine      = "+"          // +added line
	tokenDeletedLine    = "-"          // -deleted line
	tokenNoNewlineAtEOF = `\`          // \ No newline at end of file
)

var (
	// ErrNoNewFile represents error which there are no expected new file line.
	ErrNoNewFile = errors.New("no expected new file line") // +++ newfile
	// ErrNoHunks represents error which there are no expected hunks.
	ErrNoHunks = errors.New("no expected hunks") // @@ -1,3 +1,4 @@
)

// ErrInvalidHunkRange represents invalid line of hunk range. @@ -1,3 +1,4 @@
type ErrInvalidHunkRange struct {
	invalid string
}

func (e *ErrInvalidHunkRange) Error() string {
	return fmt.Sprintf("invalid hunk range: %v", e.invalid)
}

// ParseMultiFile parses a multi-file unified diff.
func ParseMultiFile(r io.Reader) ([]*FileDiff, error) {
	return (&multiFileParser{r: bufio.NewReader(r)}).Parse()
}

type multiFileParser struct {
	r *bufio.Reader
}

func (p *multiFileParser) Parse() ([]*FileDiff, error) {
	var fds []*FileDiff
	fp := &fileParser{r: p.r}
	for {
		fd, err := fp.Parse()
		if err != nil || fd == nil {
			break
		}
		fds = append(fds, fd)
	}
	return fds, nil
}

// ParseFile parses a file unified diff.
func ParseFile(r io.Reader) (*FileDiff, error) {
	return (&fileParser{r: bufio.NewReader(r)}).Parse()
}

type fileParser struct {
	r *bufio.Reader
}

func (p *fileParser) Parse() (*FileDiff, error) {
	fd := &FileDiff{}
	fd.Extended = parseExtendedHeader(p.r)
	b, err := p.r.Peek(len(tokenOldFile))
	if err != nil {
		if err == io.EOF && len(fd.Extended) > 0 {
			return fd, nil
		}
		return nil, nil
	}
	if bytes.HasPrefix(b, []byte(tokenOldFile)) {
		// parse `--- sample.old.txt	2016-10-13 05:09:35.820791185 +0900`
		oldline, _ := readline(p.r) // ignore err because we know it can read something
		fd.PathOld, fd.TimeOld = parseFileHeader(oldline)
		// parse `+++ sample.new.txt	2016-10-13 05:09:35.820791185 +0900`
		if b, err := p.r.Peek(len(tokenNewFile)); err != nil || !bytes.HasPrefix(b, []byte(tokenNewFile)) {
			return nil, ErrNoNewFile
		}
		newline, _ := readline(p.r) // ignore err because we know it can read something
		fd.PathNew, fd.TimeNew = parseFileHeader(newline)
	}
	// parse hunks
	fd.Hunks, err = p.parseHunks()
	if err != nil {
		return nil, err
	}
	return fd, nil
}

func (p *fileParser) parseHunks() ([]*Hunk, error) {
	b, err := p.r.Peek(len(tokenOldFile))
	if err != nil {
		return nil, ErrNoHunks
	}
	if !bytes.HasPrefix(b, []byte(tokenStartHunk)) {
		b, err := p.r.Peek(len(tokenDiffGit))
		if err != nil {
			return nil, ErrNoHunks
		}
		if bytes.HasPrefix(b, []byte(tokenDiffGit)) {
			// git diff may contain a file diff with empty hunks.
			// e.g. delete an empty file.
			return []*Hunk{}, nil
		}
		return nil, ErrNoHunks
	}
	var hunks []*Hunk
	hp := &hunkParser{r: p.r}
	for {
		h, err := hp.Parse()
		if err != nil {
			return nil, err
		}
		if h == nil {
			break
		}
		hunks = append(hunks, h)
	}
	return hunks, nil
}

// parseFileHeader parses file header line and returns filename and timestamp.
// timestamp may be empty.
func parseFileHeader(line string) (filename, timestamp string) {
	// strip `+++ ` or `--- `
	ss := line[len(tokenOldFile)+1:]
	tabi := strings.LastIndex(ss, "\t")
	if tabi == -1 {
		return unquoteCStyle(ss), ""
	}
	return unquoteCStyle(ss[:tabi]), ss[tabi+1:]
}

func unquoteCStyle(str string) string {
	if !strings.HasPrefix(str, `"`) {
		// no need to unescape
		return str
	}
	str = strings.TrimPrefix(strings.TrimSuffix(str, `"`), `"`)

	res := make([]byte, 0, len(str))
	r := strings.NewReader(str)
LOOP:
	for {
		ch, err := r.ReadByte()
		if err != nil {
			break
		}
		if ch != '\\' {
			res = append(res, ch)
			continue
		}

		ch, err = r.ReadByte()
		if err != nil {
			break
		}
		switch ch {
		case 'a':
			res = append(res, '\a')
		case 'b':
			res = append(res, '\b')
		case 't':
			res = append(res, '\t')
		case 'n':
			res = append(res, '\n')
		case 'v':
			res = append(res, '\v')
		case 'f':
			res = append(res, '\f')
		case 'r':
			res = append(res, '\r')
		case '"':
			res = append(res, '"')
		case '\\':
			res = append(res, '\\')
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
			if err := r.UnreadByte(); err != nil {
				break LOOP
			}
			var oct [3]byte
			if n, _ := r.Read(oct[:]); n < 3 {
				res = append(res, oct[:n]...)
				break LOOP
			}
			ch, err := strconv.ParseUint(string(oct[:]), 8, 8)
			if err != nil {
				res = append(res, oct[:]...)
				break
			}
			res = append(res, byte(ch))
		default:
			res = append(res, ch)
		}
	}

	return string(res)
}

func parseExtendedHeader(r *bufio.Reader) []string {
	var es []string
	b, err := r.Peek(len(tokenDiffGit))
	if err != nil {
		return nil
	}
	// if starts with 'diff --git', parse extended header
	if bytes.HasPrefix(b, []byte(tokenDiffGit)) {
		diffgitline, _ := readline(r) // ignore err because we know it can read something
		es = append(es, diffgitline)
		for {
			b, err := r.Peek(len(tokenDiffGit))
			if err != nil || bytes.HasPrefix(b, []byte(tokenOldFile)) || bytes.HasPrefix(b, []byte(tokenDiffGit)) {
				break
			}
			line, _ := readline(r)
			es = append(es, string(line))
		}
	}
	return es
}

type hunkParser struct {
	r        *bufio.Reader
	lnumdiff int
}

func (p *hunkParser) Parse() (*Hunk, error) {
	if b, err := p.r.Peek(len(tokenStartHunk)); err != nil || !bytes.HasPrefix(b, []byte(tokenStartHunk)) {
		return nil, nil
	}
	rangeline, _ := readline(p.r)
	hr, err := parseHunkRange(rangeline)
	if err != nil {
		return nil, err
	}
	hunk := &Hunk{
		StartLineOld:  hr.lold,
		LineLengthOld: hr.sold,
		StartLineNew:  hr.lnew,
		LineLengthNew: hr.snew,
		Section:       hr.section,
	}
	lold := hr.lold
	lnew := hr.lnew
endhunk:
	for !p.done(lold, lnew, hr) {
		b, err := p.r.Peek(1)
		if err != nil {
			break
		}
		token := string(b)
		switch token {
		case tokenUnchangedLine, tokenAddedLine, tokenDeletedLine:
			p.lnumdiff++
			l, _ := readline(p.r)
			line := &Line{Content: l[len(token):]} // trim first token
			switch token {
			case tokenUnchangedLine:
				line.Type = LineUnchanged
				line.LnumDiff = p.lnumdiff
				line.LnumOld = lold
				line.LnumNew = lnew
				lold++
				lnew++
			case tokenAddedLine:
				line.Type = LineAdded
				line.LnumDiff = p.lnumdiff
				line.LnumNew = lnew
				lnew++
			case tokenDeletedLine:
				line.Type = LineDeleted
				line.LnumDiff = p.lnumdiff
				line.LnumOld = lold
				lold++
			}
			hunk.Lines = append(hunk.Lines, line)
		case tokenNoNewlineAtEOF:
			// skip \ No newline at end of file. just consume line
			readline(p.r)
		default:
			break endhunk
		}
	}
	p.lnumdiff++ // count up by an additional hunk
	return hunk, nil
}

func (p *hunkParser) done(lold, lnew int, hr *hunkrange) bool {
	end := (lold >= hr.lold+hr.sold && lnew >= hr.lnew+hr.snew)
	if b, err := p.r.Peek(1); err != nil || (string(b) != tokenNoNewlineAtEOF && end) {
		return true
	}
	return false
}

// @@ -l,s +l,s @@ optional section heading
type hunkrange struct {
	lold, sold, lnew, snew int
	section                string
}

// @@ -lold[,sold] +lnew[,snew] @@[ section]
// 0  1              2            3   4
func parseHunkRange(rangeline string) (*hunkrange, error) {
	ps := strings.SplitN(rangeline, " ", 5)
	invalidErr := &ErrInvalidHunkRange{invalid: rangeline}
	hunkrange := &hunkrange{}
	if len(ps) < 4 || ps[0] != "@@" || ps[3] != "@@" {
		return nil, invalidErr
	}
	old := ps[1] // -lold[,sold]
	if !strings.HasPrefix(old, "-") {
		return nil, invalidErr
	}
	lold, sold, err := parseLS(old[1:])
	if err != nil {
		return nil, invalidErr
	}
	hunkrange.lold = lold
	hunkrange.sold = sold
	new := ps[2] // +lnew[,snew]
	if !strings.HasPrefix(new, "+") {
		return nil, invalidErr
	}
	lnew, snew, err := parseLS(new[1:])
	if err != nil {
		return nil, invalidErr
	}
	hunkrange.lnew = lnew
	hunkrange.snew = snew
	if len(ps) == 5 {
		hunkrange.section = ps[4]
	}
	return hunkrange, nil
}

// l[,s]
func parseLS(ls string) (l, s int, err error) {
	ss := strings.SplitN(ls, ",", 2)
	l, err = strconv.Atoi(ss[0])
	if err != nil {
		return 0, 0, err
	}
	if len(ss) == 2 {
		s, err = strconv.Atoi(ss[1])
		if err != nil {
			return 0, 0, err
		}
	} else {
		s = 1
	}
	return l, s, nil
}

// readline reads lines from bufio.Reader with size limit. It consumes
// remaining content even if the line size reaches size limit.
func readline(r *bufio.Reader) (string, error) {
	line, isPrefix, err := r.ReadLine()
	if err != nil {
		return "", err
	}
	// consume all remaining line content
	for isPrefix {
		_, isPrefix, _ = r.ReadLine()
	}
	return string(line), nil
}
<|endoftext|>


<commit_before>package qtypes

import (
	"fmt"
	"strings"
)

type Dimensions struct {
	Map map[string]string
}

func NewDimensions() Dimensions {
	return Dimensions{
		Map: map[string]string{},
	}
}

func (dim *Dimensions) Add(key,val string) {
	dim.Map[key] = val
}

func (dim *Dimensions) String() string {
	res := []string{}
	for k,v := range dim.Map {
		res = append(res, fmt.Sprintf("%s=%s", k,v))
	}
	return strings.Join(res, ",")
}<commit_msg>sort dimensions<commit_after>package qtypes

import (
	"fmt"
	"strings"
	"sort"
)

type Dimensions struct {
	Map map[string]string
}

func NewDimensions() Dimensions {
	return Dimensions{
		Map: map[string]string{},
	}
}

func (dim *Dimensions) Add(key,val string) {
	dim.Map[key] = val
}

func (dim *Dimensions) String() string {
	res := []string{}
		for k,v := range dim.Map {
		res = append(res, fmt.Sprintf("%s=%s", k,v))
	}
	sort.Strings(res)
	return strings.Join(res, ",")
}<|endoftext|>


<commit_before>package auth

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"

	"github.com/chrisolsen/fbgraphapi"
	"golang.org/x/net/context"
)

// GetToken returns the *Token value for the raw token value contained within the auth cookie or auth header
func GetToken(c context.Context, r *http.Request) (*Token, error) {
	key, err := getTokenKeyFromCookie(r)
	if err != nil && err != http.ErrNoCookie {
		return nil, err
	}
	if err == http.ErrNoCookie {
		key, err = getTokenKeyFromHeader(r)
	}
	if err != nil {
		return nil, err
	}

	tstore := NewTokenStore()
	var token Token
	err = tstore.Get(c, key, &token)
	return &token, err
}

// ClearToken deletes the token in the response to the client as well as deletes the token
// in the database to ensure it is no longer usable
func ClearToken(c context.Context, w http.ResponseWriter, r *http.Request) error {
	var err error
	var key *datastore.Key

	key, err = getTokenKeyFromCookie(r)
	if err != nil {
		clearCookie(w)
		return datastore.Delete(c, key)
	}

	key, err = getTokenKeyFromHeader(r)
	if err != nil {
		clearHeader(w)
		return datastore.Delete(c, key)
	}

	return nil
}

// Authorize .
func Authorize(c context.Context, creds *Credentials) (*Token, error) {
	if len(creds.ProviderName) > 0 {
		// Calls the private method with an appengine urlGetter.
		// This allows for internal testing of the authenticate method while
		// stubbing out the external auth service
		return authorizeOut(c, creds, appEngineURLGetter{Ctx: c})
	}
	return authorizeIn(c, creds)
}

func authorizeIn(c context.Context, creds *Credentials) (*Token, error) {
	accountStore := NewAccountStore()
	tokenStore := NewTokenStore()

	accountKey, err := accountStore.GetAccountKeyByCredentials(c, creds)
	if err != nil {
		return nil, fmt.Errorf("getting account key by credentials: %v", err)
	}

	token, err := tokenStore.Create(c, accountKey)
	if err != nil {
		return nil, err
	}

	return token, nil
}

func authorizeOut(c context.Context, creds *Credentials, urlGetter urlGetter) (*Token, error) {
	var err error
	tokenStore := NewTokenStore()
	accountStore := NewAccountStore()

	switch creds.ProviderName {
	case "facebook":
		err = fbgraphapi.Authenticate(creds.ProviderToken, creds.ProviderID, urlGetter)
	default:
		return nil, errors.New("unknown auth provider")
	}
	if err != nil {
		return nil, fmt.Errorf("authenticate: %v", err)
	}

	accountKey, err := accountStore.GetAccountKeyByCredentials(c, creds)
	if err != nil {
		return nil, fmt.Errorf("getting account key by credentials: %v", err)
	}

	token, err := tokenStore.Create(c, accountKey)
	if err != nil {
		return nil, err
	}

	return token, nil
}

func clearCookie(w http.ResponseWriter) {
	http.SetCookie(w, &http.Cookie{
		Name:     cookieName,
		Expires:  time.Time{},
		HttpOnly: true,
		Secure:   !appengine.IsDevAppServer(),
		Value:    "o7awyeu;oiqejwuriueysdfia;lkjsd;faseufhsdjhvf", // ensure it is erased
	})
}

func getTokenKeyFromHeader(r *http.Request) (*datastore.Key, error) {
	h := r.Header.Get("Authorization")
	tokenKey, err := datastore.DecodeKey(h[len("token="):])
	if err != nil {
		return nil, fmt.Errorf("failed to decode token key: %v", err)
	}

	return tokenKey, nil
}

func getTokenKeyFromCookie(r *http.Request) (*datastore.Key, error) {
	cookie, err := r.Cookie(cookieName)
	if err != nil {
		return nil, fmt.Errorf("failed to get current cookie: %v", err)
	}

	tokenKey, err := datastore.DecodeKey(cookie.Value)
	if err != nil {
		return nil, fmt.Errorf("failed to decode token key: %v", err)
	}

	return tokenKey, nil
}

func clearHeader(w http.ResponseWriter) {
	w.Header().Del("Authorization")
}
<commit_msg>add Signup method<commit_after>package auth

import (
	"errors"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/chrisolsen/fbgraphapi"
	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
)

// GetToken returns the *Token value for the raw token value contained within the auth cookie or auth header
func GetToken(c context.Context, r *http.Request) (*Token, error) {
	key, err := getTokenKeyFromCookie(r)
	if err != nil && err != http.ErrNoCookie {
		return nil, err
	}
	if err == http.ErrNoCookie {
		key, err = getTokenKeyFromHeader(r)
	}
	if err != nil {
		return nil, err
	}

	tstore := NewTokenStore()
	var token Token
	err = tstore.Get(c, key, &token)
	return &token, err
}

func Signup(c context.Context, w http.ResponseWriter, r *http.Request, creds *Credentials, account *Account) error {
	astore := NewAccountStore()
	tstore := NewTokenStore()

	accountKey, err := astore.Create(c, creds, account)
	if err != nil {
		return errors.New("failed to create account")
	}
	token, err := tstore.Create(c, accountKey)

	if accepts(r, "json") {
		setHeaderToken(w, token.Key)
	} else {
		setAuthCookieToken(w, token.Key)
	}

	return nil
}

// Signout deletes the token in the response to the client as well as deletes the token
// in the database to ensure it is no longer usable
func Signout(c context.Context, w http.ResponseWriter, r *http.Request) error {
	var err error
	var key *datastore.Key

	key, err = getTokenKeyFromCookie(r)
	if err != nil {
		clearCookie(w)
		return datastore.Delete(c, key)
	}

	key, err = getTokenKeyFromHeader(r)
	if err != nil {
		clearHeader(w)
		return datastore.Delete(c, key)
	}

	return nil
}

// Authorize .
func Authorize(c context.Context, creds *Credentials) (*Token, error) {
	if len(creds.ProviderName) > 0 {
		// Calls the private method with an appengine urlGetter.
		// This allows for internal testing of the authenticate method while
		// stubbing out the external auth service
		return authorizeOut(c, creds, appEngineURLGetter{Ctx: c})
	}
	return authorizeIn(c, creds)
}

func authorizeIn(c context.Context, creds *Credentials) (*Token, error) {
	accountStore := NewAccountStore()
	tokenStore := NewTokenStore()

	accountKey, err := accountStore.GetAccountKeyByCredentials(c, creds)
	if err != nil {
		return nil, fmt.Errorf("getting account key by credentials: %v", err)
	}

	token, err := tokenStore.Create(c, accountKey)
	if err != nil {
		return nil, err
	}

	return token, nil
}

func authorizeOut(c context.Context, creds *Credentials, urlGetter urlGetter) (*Token, error) {
	var err error
	tokenStore := NewTokenStore()
	accountStore := NewAccountStore()

	switch creds.ProviderName {
	case "facebook":
		err = fbgraphapi.Authenticate(creds.ProviderToken, creds.ProviderID, urlGetter)
	default:
		return nil, errors.New("unknown auth provider")
	}
	if err != nil {
		return nil, fmt.Errorf("authenticate: %v", err)
	}

	accountKey, err := accountStore.GetAccountKeyByCredentials(c, creds)
	if err != nil {
		return nil, fmt.Errorf("getting account key by credentials: %v", err)
	}

	token, err := tokenStore.Create(c, accountKey)
	if err != nil {
		return nil, err
	}

	return token, nil
}

func clearCookie(w http.ResponseWriter) {
	http.SetCookie(w, &http.Cookie{
		Name:     cookieName,
		Expires:  time.Time{},
		HttpOnly: true,
		Secure:   !appengine.IsDevAppServer(),
		Value:    "o7awyeu;oiqejwuriueysdfia;lkjsd;faseufhsdjhvf", // ensure it is erased
	})
}

func setAuthCookieToken(w http.ResponseWriter, token *datastore.Key) {
	http.SetCookie(w, &http.Cookie{
		Name:     cookieName,
		Expires:  time.Now().Add(time.Hour * 24 * 14),
		HttpOnly: true,
		Secure:   !appengine.IsDevAppServer(),
		Value:    token.Encode(),
	})
}

func setHeaderToken(w http.ResponseWriter, tokenKey *datastore.Key) {
	w.Header().Set("Authorization", tokenKey.Encode())
}

func getTokenKeyFromHeader(r *http.Request) (*datastore.Key, error) {
	h := r.Header.Get("Authorization")
	tokenKey, err := datastore.DecodeKey(h[len("token="):])
	if err != nil {
		return nil, fmt.Errorf("failed to decode token key: %v", err)
	}

	return tokenKey, nil
}

func getTokenKeyFromCookie(r *http.Request) (*datastore.Key, error) {
	cookie, err := r.Cookie(cookieName)
	if err != nil {
		return nil, fmt.Errorf("failed to get current cookie: %v", err)
	}

	tokenKey, err := datastore.DecodeKey(cookie.Value)
	if err != nil {
		return nil, fmt.Errorf("failed to decode token key: %v", err)
	}

	return tokenKey, nil
}

func clearHeader(w http.ResponseWriter) {
	w.Header().Del("Authorization")
}

func accepts(r *http.Request, t string) bool {
	return strings.Index(r.Header.Get("Accept"), t) > 0
}
<|endoftext|>


<commit_before>package auth

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strings"
)

// Where we store the config file
const CONFIGFILE = ".dockercfg"

// Only used for user auth + account creation
const INDEXSERVER = "https://index.docker.io/v1/"

//const INDEXSERVER = "http://indexstaging-docker.dotcloud.com/"

var (
	ErrConfigFileMissing = errors.New("The Auth config file is missing")
)

type AuthConfig struct {
	Username string `json:"username,omitempty"`
	Password string `json:"password,omitempty"`
	Auth     string `json:"auth"`
	Email    string `json:"email"`
}

type ConfigFile struct {
	Configs  map[string]AuthConfig `json:"configs,omitempty"`
	rootPath string
}

func IndexServerAddress() string {
	return INDEXSERVER
}

// create a base64 encoded auth string to store in config
func encodeAuth(authConfig *AuthConfig) string {
	authStr := authConfig.Username + ":" + authConfig.Password
	msg := []byte(authStr)
	encoded := make([]byte, base64.StdEncoding.EncodedLen(len(msg)))
	base64.StdEncoding.Encode(encoded, msg)
	return string(encoded)
}

// decode the auth string
func decodeAuth(authStr string) (string, string, error) {
	decLen := base64.StdEncoding.DecodedLen(len(authStr))
	decoded := make([]byte, decLen)
	authByte := []byte(authStr)
	n, err := base64.StdEncoding.Decode(decoded, authByte)
	if err != nil {
		return "", "", err
	}
	if n > decLen {
		return "", "", fmt.Errorf("Something went wrong decoding auth config")
	}
	arr := strings.Split(string(decoded), ":")
	if len(arr) != 2 {
		return "", "", fmt.Errorf("Invalid auth configuration file")
	}
	password := strings.Trim(arr[1], "\x00")
	return arr[0], password, nil
}

// load up the auth config information and return values
// FIXME: use the internal golang config parser
func LoadConfig(rootPath string) (*ConfigFile, error) {
	configFile := ConfigFile{Configs: make(map[string]AuthConfig), rootPath: rootPath}
	confFile := path.Join(rootPath, CONFIGFILE)
	if _, err := os.Stat(confFile); err != nil {
		return &configFile, ErrConfigFileMissing
	}
	b, err := ioutil.ReadFile(confFile)
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(b, &configFile.Configs); err != nil {
		arr := strings.Split(string(b), "\n")
		if len(arr) < 2 {
			return nil, fmt.Errorf("The Auth config file is empty")
		}
		authConfig := AuthConfig{}
		origAuth := strings.Split(arr[0], " = ")
		authConfig.Username, authConfig.Password, err = decodeAuth(origAuth[1])
		if err != nil {
			return nil, err
		}
		origEmail := strings.Split(arr[1], " = ")
		authConfig.Email = origEmail[1]
		configFile.Configs[IndexServerAddress()] = authConfig
	} else {
		for k, authConfig := range configFile.Configs {
			authConfig.Username, authConfig.Password, err = decodeAuth(authConfig.Auth)
			if err != nil {
				return nil, err
			}
			authConfig.Auth = ""
			configFile.Configs[k] = authConfig
		}
	}
	return &configFile, nil
}

// save the auth config
func SaveConfig(configFile *ConfigFile) error {
	confFile := path.Join(configFile.rootPath, CONFIGFILE)
	if len(configFile.Configs) == 0 {
		os.Remove(confFile)
		return nil
	}
	for k, authConfig := range configFile.Configs {
		authConfig.Auth = encodeAuth(&authConfig)
		authConfig.Username = ""
		authConfig.Password = ""
		configFile.Configs[k] = authConfig
	}

	b, err := json.Marshal(configFile.Configs)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(confFile, b, 0600)
	if err != nil {
		return err
	}
	return nil
}

// try to register/login to the registry server
func Login(authConfig *AuthConfig) (string, error) {
	client := &http.Client{}
	reqStatusCode := 0
	var status string
	var reqBody []byte
	jsonBody, err := json.Marshal(authConfig)
	if err != nil {
		return "", fmt.Errorf("Config Error: %s", err)
	}

	// using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status.
	b := strings.NewReader(string(jsonBody))
	req1, err := http.Post(IndexServerAddress()+"users/", "application/json; charset=utf-8", b)
	if err != nil {
		return "", fmt.Errorf("Server Error: %s", err)
	}
	reqStatusCode = req1.StatusCode
	defer req1.Body.Close()
	reqBody, err = ioutil.ReadAll(req1.Body)
	if err != nil {
		return "", fmt.Errorf("Server Error: [%#v] %s", reqStatusCode, err)
	}

	if reqStatusCode == 201 {
		status = "Account created. Please use the confirmation link we sent" +
			" to your e-mail to activate it."
	} else if reqStatusCode == 403 {
		return "", fmt.Errorf("Login: Your account hasn't been activated. " +
			"Please check your e-mail for a confirmation link.")
	} else if reqStatusCode == 400 {
		if string(reqBody) == "\"Username or email already exists\"" {
			req, err := http.NewRequest("GET", IndexServerAddress()+"users/", nil)
			req.SetBasicAuth(authConfig.Username, authConfig.Password)
			resp, err := client.Do(req)
			if err != nil {
				return "", err
			}
			defer resp.Body.Close()
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return "", err
			}
			if resp.StatusCode == 200 {
				status = "Login Succeeded"
			} else if resp.StatusCode == 401 {
				return "", fmt.Errorf("Wrong login/password, please try again")
			} else {
				return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body,
					resp.StatusCode, resp.Header)
			}
		} else {
			return "", fmt.Errorf("Registration: %s", reqBody)
		}
	} else {
		return "", fmt.Errorf("Unexpected status code [%d] : %s", reqStatusCode, reqBody)
	}
	return status, nil
}
<commit_msg>Copy authConfigs on save so data is not modified<commit_after>package auth

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strings"
)

// Where we store the config file
const CONFIGFILE = ".dockercfg"

// Only used for user auth + account creation
const INDEXSERVER = "https://index.docker.io/v1/"

//const INDEXSERVER = "http://indexstaging-docker.dotcloud.com/"

var (
	ErrConfigFileMissing = errors.New("The Auth config file is missing")
)

type AuthConfig struct {
	Username string `json:"username,omitempty"`
	Password string `json:"password,omitempty"`
	Auth     string `json:"auth"`
	Email    string `json:"email"`
}

type ConfigFile struct {
	Configs  map[string]AuthConfig `json:"configs,omitempty"`
	rootPath string
}

func IndexServerAddress() string {
	return INDEXSERVER
}

// create a base64 encoded auth string to store in config
func encodeAuth(authConfig *AuthConfig) string {
	authStr := authConfig.Username + ":" + authConfig.Password
	msg := []byte(authStr)
	encoded := make([]byte, base64.StdEncoding.EncodedLen(len(msg)))
	base64.StdEncoding.Encode(encoded, msg)
	return string(encoded)
}

// decode the auth string
func decodeAuth(authStr string) (string, string, error) {
	decLen := base64.StdEncoding.DecodedLen(len(authStr))
	decoded := make([]byte, decLen)
	authByte := []byte(authStr)
	n, err := base64.StdEncoding.Decode(decoded, authByte)
	if err != nil {
		return "", "", err
	}
	if n > decLen {
		return "", "", fmt.Errorf("Something went wrong decoding auth config")
	}
	arr := strings.Split(string(decoded), ":")
	if len(arr) != 2 {
		return "", "", fmt.Errorf("Invalid auth configuration file")
	}
	password := strings.Trim(arr[1], "\x00")
	return arr[0], password, nil
}

// load up the auth config information and return values
// FIXME: use the internal golang config parser
func LoadConfig(rootPath string) (*ConfigFile, error) {
	configFile := ConfigFile{Configs: make(map[string]AuthConfig), rootPath: rootPath}
	confFile := path.Join(rootPath, CONFIGFILE)
	if _, err := os.Stat(confFile); err != nil {
		return &configFile, ErrConfigFileMissing
	}
	b, err := ioutil.ReadFile(confFile)
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(b, &configFile.Configs); err != nil {
		arr := strings.Split(string(b), "\n")
		if len(arr) < 2 {
			return nil, fmt.Errorf("The Auth config file is empty")
		}
		authConfig := AuthConfig{}
		origAuth := strings.Split(arr[0], " = ")
		authConfig.Username, authConfig.Password, err = decodeAuth(origAuth[1])
		if err != nil {
			return nil, err
		}
		origEmail := strings.Split(arr[1], " = ")
		authConfig.Email = origEmail[1]
		configFile.Configs[IndexServerAddress()] = authConfig
	} else {
		for k, authConfig := range configFile.Configs {
			authConfig.Username, authConfig.Password, err = decodeAuth(authConfig.Auth)
			if err != nil {
				return nil, err
			}
			authConfig.Auth = ""
			configFile.Configs[k] = authConfig
		}
	}
	return &configFile, nil
}

// save the auth config
func SaveConfig(configFile *ConfigFile) error {
	confFile := path.Join(configFile.rootPath, CONFIGFILE)
	if len(configFile.Configs) == 0 {
		os.Remove(confFile)
		return nil
	}

	configs := make(map[string]AuthConfig, len(configFile.Configs))
	for k, authConfig := range configFile.Configs {
		authCopy := authConfig

		authCopy.Auth = encodeAuth(&authCopy)
		authCopy.Username = ""
		authCopy.Password = ""

		configs[k] = authCopy
	}

	b, err := json.Marshal(configs)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(confFile, b, 0600)
	if err != nil {
		return err
	}
	return nil
}

// try to register/login to the registry server
func Login(authConfig *AuthConfig) (string, error) {
	client := &http.Client{}
	reqStatusCode := 0
	var status string
	var reqBody []byte
	jsonBody, err := json.Marshal(authConfig)
	if err != nil {
		return "", fmt.Errorf("Config Error: %s", err)
	}

	// using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status.
	b := strings.NewReader(string(jsonBody))
	req1, err := http.Post(IndexServerAddress()+"users/", "application/json; charset=utf-8", b)
	if err != nil {
		return "", fmt.Errorf("Server Error: %s", err)
	}
	reqStatusCode = req1.StatusCode
	defer req1.Body.Close()
	reqBody, err = ioutil.ReadAll(req1.Body)
	if err != nil {
		return "", fmt.Errorf("Server Error: [%#v] %s", reqStatusCode, err)
	}

	if reqStatusCode == 201 {
		status = "Account created. Please use the confirmation link we sent" +
			" to your e-mail to activate it."
	} else if reqStatusCode == 403 {
		return "", fmt.Errorf("Login: Your account hasn't been activated. " +
			"Please check your e-mail for a confirmation link.")
	} else if reqStatusCode == 400 {
		if string(reqBody) == "\"Username or email already exists\"" {
			req, err := http.NewRequest("GET", IndexServerAddress()+"users/", nil)
			req.SetBasicAuth(authConfig.Username, authConfig.Password)
			resp, err := client.Do(req)
			if err != nil {
				return "", err
			}
			defer resp.Body.Close()
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return "", err
			}
			if resp.StatusCode == 200 {
				status = "Login Succeeded"
			} else if resp.StatusCode == 401 {
				return "", fmt.Errorf("Wrong login/password, please try again")
			} else {
				return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body,
					resp.StatusCode, resp.Header)
			}
		} else {
			return "", fmt.Errorf("Registration: %s", reqBody)
		}
	} else {
		return "", fmt.Errorf("Unexpected status code [%d] : %s", reqStatusCode, reqBody)
	}
	return status, nil
}
<|endoftext|>


<commit_before>package hatchery

import (
	"context"
	"fmt"
	"os"
	"os/signal"
	"runtime"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/patrickmn/go-cache"

	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// Create creates hatchery
func Create(h Interface) error {
	ctx := context.Background()
	ctx, cancel := context.WithCancel(ctx)

	// Gracefully shutdown connections
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGKILL)
	defer func() {
		signal.Stop(c)
		cancel()
	}()

	go func() {
		select {
		case <-c:
			defer cancel()
			return
		case <-ctx.Done():
			return
		}
	}()

	if err := h.Init(); err != nil {
		return fmt.Errorf("Create> Init error: %v", err)
	}

	hostname, errh := os.Hostname()
	if errh != nil {
		return fmt.Errorf("Create> Cannot retrieve hostname: %s", errh)
	}

	go hearbeat(h, h.Configuration().API.Token, h.Configuration().API.MaxHeartbeatFailures)

	pbjobs := make(chan sdk.PipelineBuildJob, 1)
	wjobs := make(chan sdk.WorkflowNodeJobRun, 1)
	errs := make(chan error, 1)
	var nRoutines, workersStarted, nRegister int64
	lastSpawnWorker := time.Now()

	go func(ctx context.Context) {
		if err := h.CDSClient().QueuePolling(ctx, wjobs, pbjobs, errs, 2*time.Second, h.Configuration().Provision.GraceTimeQueued, nil); err != nil {
			log.Error("Queues polling stopped: %v", err)
			cancel()
		}
	}(ctx)

	// Create a cache with a default expiration time of 3 second, and which
	// purges expired items every minute
	spawnIDs := cache.New(3*time.Second, 60*time.Second)

	tickerProvision := time.NewTicker(time.Duration(h.Configuration().Provision.Frequency) * time.Second)
	tickerRegister := time.NewTicker(time.Duration(h.Configuration().Provision.RegisterFrequency) * time.Second)
	tickerCountWorkersStarted := time.NewTicker(time.Duration(2 * time.Second))
	tickerGetModels := time.NewTicker(time.Duration(3 * time.Second))
	defer func() {
		tickerProvision.Stop()
		tickerRegister.Stop()
		tickerCountWorkersStarted.Stop()
		tickerGetModels.Stop()
	}()

	var models []sdk.Model

	// Call WorkerModel Enabled first
	var errwm error
	models, errwm = h.CDSClient().WorkerModelsEnabled()
	if errwm != nil {
		log.Error("error on h.CDSClient().WorkerModelsEnabled() (init call): %v", errwm)
	}

	// hatchery is now fully Initialized
	h.SetInitialized()

	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-tickerCountWorkersStarted.C:
			// we refresh workersStarted 30" after the last spawn
			if lastSpawnWorker.Add(30*time.Second).Unix() < time.Now().Unix() {
				workersStarted = int64(h.WorkersStarted())
				if workersStarted > int64(h.Configuration().Provision.MaxWorker) {
					log.Debug("max workers reached. current:%d max:%d", workersStarted, int64(h.Configuration().Provision.MaxWorker))
				}
				log.Debug("workers already started:%d", workersStarted)
			}
		case <-tickerGetModels.C:
			var errwm error
			models, errwm = h.CDSClient().WorkerModelsEnabled()
			if errwm != nil {
				log.Error("error on h.CDSClient().WorkerModelsEnabled(): %v", errwm)
			}
		case j := <-pbjobs:
			if workersStarted > int64(h.Configuration().Provision.MaxWorker) {
				log.Debug("maxWorkersReached:%d", workersStarted)
				continue
			}
			lastSpawnWorker = time.Now()
			atomic.AddInt64(&workersStarted, 1)
			go func(job sdk.PipelineBuildJob) {
				if isRun, _, _ := receiveJob(h, false, job.ExecGroups, job.ID, job.QueuedSeconds, []int64{}, job.BookedBy, job.Job.Action.Requirements, models, &nRoutines, spawnIDs, hostname); isRun {
					spawnIDs.SetDefault(string(job.ID), job.ID)
				} else {
					atomic.AddInt64(&workersStarted, -1)
				}
			}(j)
		case j := <-wjobs:
			if workersStarted > int64(h.Configuration().Provision.MaxWorker) {
				log.Debug("maxWorkersReached:%d", workersStarted)
				continue
			}
			atomic.AddInt64(&workersStarted, 1)
			lastSpawnWorker = time.Now()
			go func(job sdk.WorkflowNodeJobRun) {
				// count + 1 here, and remove -1 if worker is not started
				// this avoid to spawn to many workers compare
				if isRun, temptToSpawn, _ := receiveJob(h, true, job.ExecGroups, job.ID, job.QueuedSeconds, job.SpawnAttempts, job.BookedBy, job.Job.Action.Requirements, models, &nRoutines, spawnIDs, hostname); isRun {
					spawnIDs.SetDefault(string(job.ID), job.ID)
				} else if temptToSpawn {
					atomic.AddInt64(&workersStarted, -1)
					found := false
					for _, hID := range job.SpawnAttempts {
						if hID == h.ID() {
							found = true
						}
					}

					if !found {
						if hCount, err := h.CDSClient().HatcheryCount(job.WorkflowNodeRunID); err == nil {
							if int64(len(job.SpawnAttempts)) < hCount {
								if _, errQ := h.CDSClient().QueueJobIncAttempts(job.ID); errQ != nil {
									log.Warning("Hatchery> Create> cannot inc spawn attempts %s", errQ)
								}
							}
						} else {
							log.Warning("Hatchery> Create> cannot get hatchery count %s", err)
						}
					}

				} else {
					atomic.AddInt64(&workersStarted, -1)
				}
			}(j)
		case err := <-errs:
			log.Error("%v", err)
		case <-tickerProvision.C:
			provisioning(h, h.Configuration().Provision.Disabled, models)
		case <-tickerRegister.C:
			if err := workerRegister(h, models, &nRegister); err != nil {
				log.Warning("Error on workerRegister: %s", err)
			}
		}
	}
}

// Register calls CDS API to register current hatchery
func Register(h Interface) error {
	newHatchery, uptodate, err := h.CDSClient().HatcheryRegister(*h.Hatchery())
	if err != nil {
		return sdk.WrapError(err, "register> Got HTTP exiting")
	}
	h.Hatchery().ID = newHatchery.ID
	h.Hatchery().GroupID = newHatchery.GroupID
	h.Hatchery().Model = newHatchery.Model
	h.Hatchery().Name = newHatchery.Name
	h.Hatchery().IsSharedInfra = newHatchery.IsSharedInfra

	log.Info("Register> Hatchery %s registered with id:%d", h.Hatchery().Name, h.Hatchery().ID)

	if !uptodate {
		log.Warning("-=-=-=-=- Please update your hatchery binary - Hatchery Version:%s %s %s -=-=-=-=-", sdk.VERSION, runtime.GOOS, runtime.GOARCH)
	}
	return nil
}

func hearbeat(m Interface, token string, maxFailures int) {
	var failures int
	for {
		time.Sleep(5 * time.Second)
		if m.Hatchery().ID == 0 {
			log.Info("hearbeat> %s Disconnected from CDS engine, trying to register...", m.Hatchery().Name)
			if err := Register(m); err != nil {
				log.Info("hearbeat> %s Cannot register: %s", m.Hatchery().Name, err)
				checkFailures(maxFailures, failures)
				continue
			}
			if m.Hatchery().ID == 0 {
				log.Error("hearbeat> Cannot register hatchery. ID %d", m.Hatchery().ID)
				checkFailures(maxFailures, failures)
				continue
			}
			log.Info("hearbeat> %s Registered back: ID %d with model ID %d", m.Hatchery().Name, m.Hatchery().ID, m.Hatchery().Model.ID)
		}

		if err := m.CDSClient().HatcheryRefresh(m.Hatchery().ID); err != nil {
			log.Info("heartbeat> %s cannot refresh beat: %s", m.Hatchery().Name, err)
			m.Hatchery().ID = 0
			checkFailures(maxFailures, failures)
			continue
		}
		failures = 0
	}
}

func checkFailures(maxFailures, nb int) {
	if nb > maxFailures {
		log.Error("Too many failures on try register. This hatchery is killed")
		os.Exit(10)
	}
}

// workerRegister is called by a ticker.
// the hatchery checks each worker model, and if a worker model needs to
// be registered, the hatchery calls SpawnWorker().
// each ticker can trigger 5 worker models (maximum)
// and 5 worker models can be spawned in same time, in the case of a spawn takes longer
// than a tick.
func workerRegister(h Interface, models []sdk.Model, nRegister *int64) error {
	if len(models) == 0 {
		return fmt.Errorf("workerRegister> No model returned by GetWorkerModels")
	}
	log.Debug("workerRegister> models received: %d", len(models))

	// currentRegister contains the register spawned in this ticker
	var currentRegister int64

	for k := range models {
		if *nRegister > 5 || currentRegister > 5 {
			return nil
		}

		if models[k].Type != h.ModelType() {
			continue
		}

		// if current hatchery is in same group than worker model -> do not avoid spawn, even if worker model is in error
		if models[k].NbSpawnErr > 5 && h.Hatchery().GroupID != models[k].ID {
			log.Warning("workerRegister> Too many errors on spawn with model %s, please check this worker model", models[k].Name)
			continue
		}

		if h.NeedRegistration(&models[k]) {
			if err := h.CDSClient().WorkerModelBook(models[k].ID); err != nil {
				log.Debug("workerRegister> WorkerModelBook on model %s err: %s", models[k].Name, err)
			} else {
				log.Info("workerRegister> spawning model %s (%d)", models[k].Name, models[k].ID)
				atomic.AddInt64(nRegister, 1)
				currentRegister++
				go func(m sdk.Model) {
					if _, errSpawn := h.SpawnWorker(SpawnArguments{Model: m, IsWorkflowJob: false, JobID: 0, Requirements: nil, RegisterOnly: true, LogInfo: "spawn for register"}); errSpawn != nil {
						log.Warning("workerRegister> cannot spawn worker for register:%s err:%v", m.Name, errSpawn)
						if err := h.CDSClient().WorkerModelSpawnError(m.ID, fmt.Sprintf("workerRegister> cannot spawn worker for register: %s", errSpawn)); err != nil {
							log.Error("workerRegister> error on call client.WorkerModelSpawnError on worker model %s for register: %s", m.Name, err)
						}
					}
					atomic.AddInt64(nRegister, -1)
				}(models[k])
			}
		} else {
			log.Debug("workerRegister> no need to register worker model %s (%d)", models[k].Name, models[k].ID)
		}
	}
	return nil
}
<commit_msg>fix (hatchery): check max workers (#2937)<commit_after>package hatchery

import (
	"context"
	"fmt"
	"os"
	"os/signal"
	"runtime"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/patrickmn/go-cache"

	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// Create creates hatchery
func Create(h Interface) error {
	ctx := context.Background()
	ctx, cancel := context.WithCancel(ctx)

	// Gracefully shutdown connections
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGKILL)
	defer func() {
		signal.Stop(c)
		cancel()
	}()

	go func() {
		select {
		case <-c:
			defer cancel()
			return
		case <-ctx.Done():
			return
		}
	}()

	if err := h.Init(); err != nil {
		return fmt.Errorf("Create> Init error: %v", err)
	}

	hostname, errh := os.Hostname()
	if errh != nil {
		return fmt.Errorf("Create> Cannot retrieve hostname: %s", errh)
	}

	go hearbeat(h, h.Configuration().API.Token, h.Configuration().API.MaxHeartbeatFailures)

	pbjobs := make(chan sdk.PipelineBuildJob, 1)
	wjobs := make(chan sdk.WorkflowNodeJobRun, 1)
	errs := make(chan error, 1)
	var nRoutines, workersStarted, nRegister int64
	lastSpawnWorker := time.Now()

	go func(ctx context.Context) {
		if err := h.CDSClient().QueuePolling(ctx, wjobs, pbjobs, errs, 2*time.Second, h.Configuration().Provision.GraceTimeQueued, nil); err != nil {
			log.Error("Queues polling stopped: %v", err)
			cancel()
		}
	}(ctx)

	// Create a cache with a default expiration time of 3 second, and which
	// purges expired items every minute
	spawnIDs := cache.New(3*time.Second, 60*time.Second)

	tickerProvision := time.NewTicker(time.Duration(h.Configuration().Provision.Frequency) * time.Second)
	tickerRegister := time.NewTicker(time.Duration(h.Configuration().Provision.RegisterFrequency) * time.Second)
	tickerCountWorkersStarted := time.NewTicker(time.Duration(2 * time.Second))
	tickerGetModels := time.NewTicker(time.Duration(3 * time.Second))
	defer func() {
		tickerProvision.Stop()
		tickerRegister.Stop()
		tickerCountWorkersStarted.Stop()
		tickerGetModels.Stop()
	}()

	var models []sdk.Model

	// Call WorkerModel Enabled first
	var errwm error
	models, errwm = h.CDSClient().WorkerModelsEnabled()
	if errwm != nil {
		log.Error("error on h.CDSClient().WorkerModelsEnabled() (init call): %v", errwm)
	}

	// hatchery is now fully Initialized
	h.SetInitialized()

	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-tickerCountWorkersStarted.C:
			// we refresh workersStarted 30" after the last spawn
			if lastSpawnWorker.Add(30*time.Second).Unix() < time.Now().Unix() {
				workersStarted = int64(h.WorkersStarted())
				if workersStarted > int64(h.Configuration().Provision.MaxWorker) {
					log.Debug("max workers reached. current:%d max:%d", workersStarted, int64(h.Configuration().Provision.MaxWorker))
				}
				log.Debug("workers already started:%d", workersStarted)
			}
		case <-tickerGetModels.C:
			var errwm error
			models, errwm = h.CDSClient().WorkerModelsEnabled()
			if errwm != nil {
				log.Error("error on h.CDSClient().WorkerModelsEnabled(): %v", errwm)
			}
		case j := <-pbjobs:
			if workersStarted >= int64(h.Configuration().Provision.MaxWorker) {
				log.Debug("maxWorkersReached:%d", workersStarted)
				continue
			}
			lastSpawnWorker = time.Now()
			atomic.AddInt64(&workersStarted, 1)
			go func(job sdk.PipelineBuildJob) {
				if isRun, _, _ := receiveJob(h, false, job.ExecGroups, job.ID, job.QueuedSeconds, []int64{}, job.BookedBy, job.Job.Action.Requirements, models, &nRoutines, spawnIDs, hostname); isRun {
					spawnIDs.SetDefault(string(job.ID), job.ID)
				} else {
					atomic.AddInt64(&workersStarted, -1)
				}
			}(j)
		case j := <-wjobs:
			if workersStarted >= int64(h.Configuration().Provision.MaxWorker) {
				log.Debug("maxWorkersReached:%d", workersStarted)
				continue
			}
			atomic.AddInt64(&workersStarted, 1)
			lastSpawnWorker = time.Now()
			go func(job sdk.WorkflowNodeJobRun) {
				// count + 1 here, and remove -1 if worker is not started
				// this avoid to spawn to many workers compare
				if isRun, temptToSpawn, _ := receiveJob(h, true, job.ExecGroups, job.ID, job.QueuedSeconds, job.SpawnAttempts, job.BookedBy, job.Job.Action.Requirements, models, &nRoutines, spawnIDs, hostname); isRun {
					spawnIDs.SetDefault(string(job.ID), job.ID)
				} else if temptToSpawn {
					atomic.AddInt64(&workersStarted, -1)
					found := false
					for _, hID := range job.SpawnAttempts {
						if hID == h.ID() {
							found = true
						}
					}

					if !found {
						if hCount, err := h.CDSClient().HatcheryCount(job.WorkflowNodeRunID); err == nil {
							if int64(len(job.SpawnAttempts)) < hCount {
								if _, errQ := h.CDSClient().QueueJobIncAttempts(job.ID); errQ != nil {
									log.Warning("Hatchery> Create> cannot inc spawn attempts %s", errQ)
								}
							}
						} else {
							log.Warning("Hatchery> Create> cannot get hatchery count %s", err)
						}
					}

				} else {
					atomic.AddInt64(&workersStarted, -1)
				}
			}(j)
		case err := <-errs:
			log.Error("%v", err)
		case <-tickerProvision.C:
			provisioning(h, h.Configuration().Provision.Disabled, models)
		case <-tickerRegister.C:
			if err := workerRegister(h, models, &nRegister); err != nil {
				log.Warning("Error on workerRegister: %s", err)
			}
		}
	}
}

// Register calls CDS API to register current hatchery
func Register(h Interface) error {
	newHatchery, uptodate, err := h.CDSClient().HatcheryRegister(*h.Hatchery())
	if err != nil {
		return sdk.WrapError(err, "register> Got HTTP exiting")
	}
	h.Hatchery().ID = newHatchery.ID
	h.Hatchery().GroupID = newHatchery.GroupID
	h.Hatchery().Model = newHatchery.Model
	h.Hatchery().Name = newHatchery.Name
	h.Hatchery().IsSharedInfra = newHatchery.IsSharedInfra

	log.Info("Register> Hatchery %s registered with id:%d", h.Hatchery().Name, h.Hatchery().ID)

	if !uptodate {
		log.Warning("-=-=-=-=- Please update your hatchery binary - Hatchery Version:%s %s %s -=-=-=-=-", sdk.VERSION, runtime.GOOS, runtime.GOARCH)
	}
	return nil
}

func hearbeat(m Interface, token string, maxFailures int) {
	var failures int
	for {
		time.Sleep(5 * time.Second)
		if m.Hatchery().ID == 0 {
			log.Info("hearbeat> %s Disconnected from CDS engine, trying to register...", m.Hatchery().Name)
			if err := Register(m); err != nil {
				log.Info("hearbeat> %s Cannot register: %s", m.Hatchery().Name, err)
				checkFailures(maxFailures, failures)
				continue
			}
			if m.Hatchery().ID == 0 {
				log.Error("hearbeat> Cannot register hatchery. ID %d", m.Hatchery().ID)
				checkFailures(maxFailures, failures)
				continue
			}
			log.Info("hearbeat> %s Registered back: ID %d with model ID %d", m.Hatchery().Name, m.Hatchery().ID, m.Hatchery().Model.ID)
		}

		if err := m.CDSClient().HatcheryRefresh(m.Hatchery().ID); err != nil {
			log.Info("heartbeat> %s cannot refresh beat: %s", m.Hatchery().Name, err)
			m.Hatchery().ID = 0
			checkFailures(maxFailures, failures)
			continue
		}
		failures = 0
	}
}

func checkFailures(maxFailures, nb int) {
	if nb > maxFailures {
		log.Error("Too many failures on try register. This hatchery is killed")
		os.Exit(10)
	}
}

// workerRegister is called by a ticker.
// the hatchery checks each worker model, and if a worker model needs to
// be registered, the hatchery calls SpawnWorker().
// each ticker can trigger 5 worker models (maximum)
// and 5 worker models can be spawned in same time, in the case of a spawn takes longer
// than a tick.
func workerRegister(h Interface, models []sdk.Model, nRegister *int64) error {
	if len(models) == 0 {
		return fmt.Errorf("workerRegister> No model returned by GetWorkerModels")
	}
	log.Debug("workerRegister> models received: %d", len(models))

	// currentRegister contains the register spawned in this ticker
	var currentRegister int64

	for k := range models {
		if *nRegister > 5 || currentRegister > 5 {
			return nil
		}

		if models[k].Type != h.ModelType() {
			continue
		}

		// if current hatchery is in same group than worker model -> do not avoid spawn, even if worker model is in error
		if models[k].NbSpawnErr > 5 && h.Hatchery().GroupID != models[k].ID {
			log.Warning("workerRegister> Too many errors on spawn with model %s, please check this worker model", models[k].Name)
			continue
		}

		if h.NeedRegistration(&models[k]) {
			if err := h.CDSClient().WorkerModelBook(models[k].ID); err != nil {
				log.Debug("workerRegister> WorkerModelBook on model %s err: %s", models[k].Name, err)
			} else {
				log.Info("workerRegister> spawning model %s (%d)", models[k].Name, models[k].ID)
				atomic.AddInt64(nRegister, 1)
				currentRegister++
				go func(m sdk.Model) {
					if _, errSpawn := h.SpawnWorker(SpawnArguments{Model: m, IsWorkflowJob: false, JobID: 0, Requirements: nil, RegisterOnly: true, LogInfo: "spawn for register"}); errSpawn != nil {
						log.Warning("workerRegister> cannot spawn worker for register:%s err:%v", m.Name, errSpawn)
						if err := h.CDSClient().WorkerModelSpawnError(m.ID, fmt.Sprintf("workerRegister> cannot spawn worker for register: %s", errSpawn)); err != nil {
							log.Error("workerRegister> error on call client.WorkerModelSpawnError on worker model %s for register: %s", m.Name, err)
						}
					}
					atomic.AddInt64(nRegister, -1)
				}(models[k])
			}
		} else {
			log.Debug("workerRegister> no need to register worker model %s (%d)", models[k].Name, models[k].ID)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package osin

import (
	"net/http"
	"time"
)

// AuthorizeRequestType is the type for OAuth param `response_type`
type AuthorizeRequestType string

const (
	CODE  AuthorizeRequestType = "code"
	TOKEN                      = "token"
)

// Authorize request information
type AuthorizeRequest struct {
	Type        AuthorizeRequestType
	Client      *Client
	Scope       string
	RedirectUri string
	State       string

	// Set if request is authorized
	Authorized bool

	// Token expiration in seconds. Change if different from default.
	// If type = TOKEN, this expiration will be for the ACCESS token.
	Expiration int32

	// Data to be passed to storage. Not used by the library.
	UserData interface{}
}

// Authorization data
type AuthorizeData struct {
	// Client information
	Client *Client

	// Authorization code
	Code string

	// Token expiration in seconds
	ExpiresIn int32

	// Requested scope
	Scope string

	// Redirect Uri from request
	RedirectUri string

	// State data from request
	State string

	// Date created
	CreatedAt time.Time

	// Data to be passed to storage. Not used by the library.
	UserData interface{}
}

// IsExpired is true if authorization expired
func (d *AuthorizeData) IsExpired() bool {
	return d.CreatedAt.Add(time.Duration(d.ExpiresIn) * time.Second).Before(time.Now())
}

// ExpireAt returns the expiration date
func (d *AuthorizeData) ExpireAt() time.Time {
	return d.CreatedAt.Add(time.Duration(d.ExpiresIn) * time.Second)
}

// AuthorizeTokenGen is the token generator interface
type AuthorizeTokenGen interface {
	GenerateAuthorizeToken(data *AuthorizeData) (string, error)
}

// HandleAuthorizeRequest is the main http.HandlerFunc for handling
// authorization requests
func (s *Server) HandleAuthorizeRequest(w *Response, r *http.Request) *AuthorizeRequest {
	r.ParseForm()

	requestType := AuthorizeRequestType(r.Form.Get("response_type"))
	if s.Config.AllowedAuthorizeTypes.Exists(requestType) {
		switch requestType {
		case CODE:
			return s.handleCodeRequest(w, r)
		case TOKEN:
			return s.handleTokenRequest(w, r)
		}
	}

	w.SetError(E_UNSUPPORTED_RESPONSE_TYPE, "")
	return nil
}

func (s *Server) handleCodeRequest(w *Response, r *http.Request) *AuthorizeRequest {
	// create the authorization request
	ret := &AuthorizeRequest{
		Type:        CODE,
		State:       r.Form.Get("state"),
		Scope:       r.Form.Get("scope"),
		RedirectUri: r.Form.Get("redirect_uri"),
		Authorized:  false,
		Expiration:  s.Config.AuthorizationExpiration,
	}

	var err error

	// must have a valid client
	ret.Client, err = s.Storage.GetClient(r.Form.Get("client_id"))
	if err != nil {
		w.SetErrorState(E_SERVER_ERROR, "", ret.State)
		w.InternalError = err
		return nil
	}
	if ret.Client == nil {
		w.SetErrorState(E_UNAUTHORIZED_CLIENT, "", ret.State)
		return nil
	}
	if ret.Client.RedirectUri == "" {
		w.SetErrorState(E_UNAUTHORIZED_CLIENT, "", ret.State)
		return nil
	}

	// force redirect response to client redirecturl first
	w.SetRedirect(ret.Client.RedirectUri)

	// check redirect uri
	if ret.RedirectUri == "" {
		ret.RedirectUri = ret.Client.RedirectUri
	}
	if err = ValidateUri(ret.Client.RedirectUri, ret.RedirectUri); err != nil {
		w.SetErrorState(E_INVALID_REQUEST, "", ret.State)
		w.InternalError = err
		return nil
	}

	return ret
}

func (s *Server) handleTokenRequest(w *Response, r *http.Request) *AuthorizeRequest {
	// create the authorization request
	ret := &AuthorizeRequest{
		Type:        TOKEN,
		State:       r.Form.Get("state"),
		Scope:       r.Form.Get("scope"),
		RedirectUri: r.Form.Get("redirect_uri"),
		Authorized:  false,
		// this type will generate a token directly, use access token expiration instead.
		Expiration: s.Config.AccessExpiration,
	}

	var err error

	// must have a valid client
	ret.Client, err = s.Storage.GetClient(r.Form.Get("client_id"))
	if err != nil {
		w.SetErrorState(E_SERVER_ERROR, "", ret.State)
		w.InternalError = err
		return nil
	}
	if ret.Client == nil {
		w.SetErrorState(E_UNAUTHORIZED_CLIENT, "", ret.State)
		return nil
	}
	if ret.Client.RedirectUri == "" {
		w.SetErrorState(E_UNAUTHORIZED_CLIENT, "", ret.State)
		return nil
	}

	// force redirect response to client redirecturl first
	w.SetRedirect(ret.Client.RedirectUri)

	// check redirect uri
	if ret.RedirectUri == "" {
		ret.RedirectUri = ret.Client.RedirectUri
	}
	if err = ValidateUri(ret.Client.RedirectUri, ret.RedirectUri); err != nil {
		w.SetErrorState(E_INVALID_REQUEST, "", ret.State)
		w.InternalError = err
		return nil
	}

	return ret
}

func (s *Server) FinishAuthorizeRequest(w *Response, r *http.Request, ar *AuthorizeRequest) {
	// don't process if is already an error
	if w.IsError {
		return
	}

	// force redirect response
	w.SetRedirect(ar.RedirectUri)

	if ar.Authorized {
		if ar.Type == TOKEN {
			w.SetRedirectFragment(true)

			// generate token directly
			ret := &AccessRequest{
				Type:            IMPLICIT,
				Code:            "",
				Client:          ar.Client,
				RedirectUri:     ar.RedirectUri,
				Scope:           ar.Scope,
				GenerateRefresh: false, // per the RFC, should NOT generate a refresh token in this case
				Authorized:      true,
				Expiration:      ar.Expiration,
				UserData:        ar.UserData,
			}

			s.FinishAccessRequest(w, r, ret)
		} else {
			// generate authorization token
			ret := &AuthorizeData{
				Client:      ar.Client,
				CreatedAt:   time.Now(),
				ExpiresIn:   ar.Expiration,
				RedirectUri: ar.RedirectUri,
				State:       ar.State,
				Scope:       ar.Scope,
				UserData:    ar.UserData,
			}

			// generate token code
			code, err := s.AuthorizeTokenGen.GenerateAuthorizeToken(ret)
			if err != nil {
				w.SetErrorState(E_SERVER_ERROR, "", ar.State)
				w.InternalError = err
				return
			}
			ret.Code = code

			// save authorization token
			if err = s.Storage.SaveAuthorize(ret); err != nil {
				w.SetErrorState(E_SERVER_ERROR, "", ar.State)
				w.InternalError = err
				return
			}

			// redirect with code
			w.Output["code"] = ret.Code
			w.Output["state"] = ret.State
		}
	} else {
		// redirect with error
		w.SetErrorState(E_ACCESS_DENIED, "", ar.State)
	}
}
<commit_msg>unescape uri in HandleAuthorizeRequest<commit_after>package osin

import (
	"net/http"
	"net/url"
	"time"
)

// AuthorizeRequestType is the type for OAuth param `response_type`
type AuthorizeRequestType string

const (
	CODE  AuthorizeRequestType = "code"
	TOKEN                      = "token"
)

// Authorize request information
type AuthorizeRequest struct {
	Type        AuthorizeRequestType
	Client      *Client
	Scope       string
	RedirectUri string
	State       string

	// Set if request is authorized
	Authorized bool

	// Token expiration in seconds. Change if different from default.
	// If type = TOKEN, this expiration will be for the ACCESS token.
	Expiration int32

	// Data to be passed to storage. Not used by the library.
	UserData interface{}
}

// Authorization data
type AuthorizeData struct {
	// Client information
	Client *Client

	// Authorization code
	Code string

	// Token expiration in seconds
	ExpiresIn int32

	// Requested scope
	Scope string

	// Redirect Uri from request
	RedirectUri string

	// State data from request
	State string

	// Date created
	CreatedAt time.Time

	// Data to be passed to storage. Not used by the library.
	UserData interface{}
}

// IsExpired is true if authorization expired
func (d *AuthorizeData) IsExpired() bool {
	return d.CreatedAt.Add(time.Duration(d.ExpiresIn) * time.Second).Before(time.Now())
}

// ExpireAt returns the expiration date
func (d *AuthorizeData) ExpireAt() time.Time {
	return d.CreatedAt.Add(time.Duration(d.ExpiresIn) * time.Second)
}

// AuthorizeTokenGen is the token generator interface
type AuthorizeTokenGen interface {
	GenerateAuthorizeToken(data *AuthorizeData) (string, error)
}

// HandleAuthorizeRequest is the main http.HandlerFunc for handling
// authorization requests
func (s *Server) HandleAuthorizeRequest(w *Response, r *http.Request) *AuthorizeRequest {
	r.ParseForm()

	requestType := AuthorizeRequestType(r.Form.Get("response_type"))
	if s.Config.AllowedAuthorizeTypes.Exists(requestType) {
		switch requestType {
		case CODE:
			return s.handleCodeRequest(w, r)
		case TOKEN:
			return s.handleTokenRequest(w, r)
		}
	}

	w.SetError(E_UNSUPPORTED_RESPONSE_TYPE, "")
	return nil
}

func (s *Server) handleCodeRequest(w *Response, r *http.Request) *AuthorizeRequest {
	// create the authorization request
	unescapedUri, err := url.QueryUnescape(r.Form.Get("redirect_uri"))
	if err != nil {
		unescapedUri = ""
	}
	ret := &AuthorizeRequest{
		Type:        CODE,
		State:       r.Form.Get("state"),
		Scope:       r.Form.Get("scope"),
		RedirectUri: unescapedUri,
		Authorized:  false,
		Expiration:  s.Config.AuthorizationExpiration,
	}

	// must have a valid client
	ret.Client, err = s.Storage.GetClient(r.Form.Get("client_id"))
	if err != nil {
		w.SetErrorState(E_SERVER_ERROR, "", ret.State)
		w.InternalError = err
		return nil
	}
	if ret.Client == nil {
		w.SetErrorState(E_UNAUTHORIZED_CLIENT, "", ret.State)
		return nil
	}
	if ret.Client.RedirectUri == "" {
		w.SetErrorState(E_UNAUTHORIZED_CLIENT, "", ret.State)
		return nil
	}

	// force redirect response to client redirecturl first
	w.SetRedirect(ret.Client.RedirectUri)

	// check redirect uri
	if ret.RedirectUri == "" {
		ret.RedirectUri = ret.Client.RedirectUri
	}
	if err = ValidateUri(ret.Client.RedirectUri, ret.RedirectUri); err != nil {
		w.SetErrorState(E_INVALID_REQUEST, "", ret.State)
		w.InternalError = err
		return nil
	}

	return ret
}

func (s *Server) handleTokenRequest(w *Response, r *http.Request) *AuthorizeRequest {
	// create the authorization request
	unescapedUri, err := url.QueryUnescape(r.Form.Get("redirect_uri"))
	if err != nil {
		unescapedUri = ""
	}
	ret := &AuthorizeRequest{
		Type:        TOKEN,
		State:       r.Form.Get("state"),
		Scope:       r.Form.Get("scope"),
		RedirectUri: unescapedUri,
		Authorized:  false,
		// this type will generate a token directly, use access token expiration instead.
		Expiration: s.Config.AccessExpiration,
	}

	// must have a valid client
	ret.Client, err = s.Storage.GetClient(r.Form.Get("client_id"))
	if err != nil {
		w.SetErrorState(E_SERVER_ERROR, "", ret.State)
		w.InternalError = err
		return nil
	}
	if ret.Client == nil {
		w.SetErrorState(E_UNAUTHORIZED_CLIENT, "", ret.State)
		return nil
	}
	if ret.Client.RedirectUri == "" {
		w.SetErrorState(E_UNAUTHORIZED_CLIENT, "", ret.State)
		return nil
	}

	// force redirect response to client redirecturl first
	w.SetRedirect(ret.Client.RedirectUri)

	// check redirect uri
	if ret.RedirectUri == "" {
		ret.RedirectUri = ret.Client.RedirectUri
	}
	if err = ValidateUri(ret.Client.RedirectUri, ret.RedirectUri); err != nil {
		w.SetErrorState(E_INVALID_REQUEST, "", ret.State)
		w.InternalError = err
		return nil
	}

	return ret
}

func (s *Server) FinishAuthorizeRequest(w *Response, r *http.Request, ar *AuthorizeRequest) {
	// don't process if is already an error
	if w.IsError {
		return
	}

	// force redirect response
	w.SetRedirect(ar.RedirectUri)

	if ar.Authorized {
		if ar.Type == TOKEN {
			w.SetRedirectFragment(true)

			// generate token directly
			ret := &AccessRequest{
				Type:            IMPLICIT,
				Code:            "",
				Client:          ar.Client,
				RedirectUri:     ar.RedirectUri,
				Scope:           ar.Scope,
				GenerateRefresh: false, // per the RFC, should NOT generate a refresh token in this case
				Authorized:      true,
				Expiration:      ar.Expiration,
				UserData:        ar.UserData,
			}

			s.FinishAccessRequest(w, r, ret)
		} else {
			// generate authorization token
			ret := &AuthorizeData{
				Client:      ar.Client,
				CreatedAt:   time.Now(),
				ExpiresIn:   ar.Expiration,
				RedirectUri: ar.RedirectUri,
				State:       ar.State,
				Scope:       ar.Scope,
				UserData:    ar.UserData,
			}

			// generate token code
			code, err := s.AuthorizeTokenGen.GenerateAuthorizeToken(ret)
			if err != nil {
				w.SetErrorState(E_SERVER_ERROR, "", ar.State)
				w.InternalError = err
				return
			}
			ret.Code = code

			// save authorization token
			if err = s.Storage.SaveAuthorize(ret); err != nil {
				w.SetErrorState(E_SERVER_ERROR, "", ar.State)
				w.InternalError = err
				return
			}

			// redirect with code
			w.Output["code"] = ret.Code
			w.Output["state"] = ret.State
		}
	} else {
		// redirect with error
		w.SetErrorState(E_ACCESS_DENIED, "", ar.State)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// evolution of: https://github.com/GoogleCloudPlatform/kubernetes/blob/release-0.6/pkg/client/cache/fifo.go
package queue

import (
	"fmt"
	"reflect"
	"sync"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/config"
)

type EventType int

const (
	ADD_EVENT EventType = iota
	UPDATE_EVENT
	DELETE_EVENT
	POP_EVENT
)

type Entry struct {
	Value UniqueCopyable
	Event EventType
}

type Copyable interface {
	// return an independent copy (deep clone) of the current object
	Copy() Copyable
}

type UniqueID interface {
	GetUID() string
}

type UniqueCopyable interface {
	Copyable
	UniqueID
}

func (e *Entry) Copy() Copyable {
	return &Entry{Value: e.Value.Copy().(UniqueCopyable), Event: e.Event}
}

// deliver a message
type pigeon func(msg *Entry)

func dead(msg *Entry) {
	// intentionally blank
}

// HistoricalFIFO receives adds and updates from a Reflector, and puts them in a queue for
// FIFO order processing. If multiple adds/updates of a single item happen while
// an item is in the queue before it has been processed, it will only be
// processed once, and when it is processed, the most recent version will be
// processed. This can't be done with a channel.
type HistoricalFIFO struct {
	lock    sync.RWMutex
	cond    sync.Cond
	items   map[string]*Entry // We depend on the property that items in the queue are in the set.
	queue   []string
	carrier pigeon // may be dead, but never nil
}

// panics if obj doesn't implement UniqueCopyable; otherwise returns the same, typecast object
func checkType(obj interface{}) UniqueCopyable {
	if v, ok := obj.(UniqueCopyable); !ok {
		panic(fmt.Sprintf("Illegal object type, expected UniqueCopyable: %T", obj))
	} else {
		return v
	}
}

// Add inserts an item, and puts it in the queue. The item is only enqueued
// if it doesn't already exist in the set.
func (f *HistoricalFIFO) Add(id string, v interface{}) {
	obj := checkType(v)
	notifications := []*Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; !exists {
		f.queue = append(f.queue, id)
	} else {
		if entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj)
	f.cond.Broadcast()
}

// Update is the same as Add in this implementation.
func (f *HistoricalFIFO) Update(id string, obj interface{}) {
	f.Add(id, obj)
}

// Add the item to the store, but only if there exists a prior entry for
// for the object in the store whose event type matches that given, and then
// only enqueued if it doesn't already exist in the set.
func (f *HistoricalFIFO) Readd(id string, v interface{}, t EventType) {
	obj := checkType(v)
	notifications := []*Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; exists {
		if entry.Event != t {
			return
		} else if entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj)
	f.cond.Broadcast()
}

// Delete removes an item. It doesn't add it to the queue, because
// this implementation assumes the consumer only cares about the objects,
// not the order in which they were created/added.
func (f *HistoricalFIFO) Delete(id string) {
	deleteEvent := (*Entry)(nil)
	defer func() {
		f.carrier(deleteEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	entry, exists := f.items[id]
	if exists {
		//TODO(jdef): set a timer to expunge the history for this object
		//or else, simply do garbage collection every n'th merge(), removing
		//expired DELETE entries
		deleteEvent = &Entry{Value: entry.Value, Event: DELETE_EVENT}
		f.items[id] = deleteEvent
	}
}

// List returns a list of all the items.
func (f *HistoricalFIFO) List() []interface{} {
	f.lock.RLock()
	defer f.lock.RUnlock()

	// TODO(jdef): slightly overallocates b/c of deleted items
	list := make([]interface{}, 0, len(f.queue))

	for _, entry := range f.items {
		if entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			continue
		}
		list = append(list, entry.Value.Copy())
	}
	return list
}

// ContainedIDs returns a util.StringSet containing all IDs of the stored items.
// This is a snapshot of a moment in time, and one should keep in mind that
// other go routines can add or remove items after you call this.
func (c *HistoricalFIFO) ContainedIDs() util.StringSet {
	c.lock.RLock()
	defer c.lock.RUnlock()
	set := util.StringSet{}
	for id, entry := range c.items {
		if entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			continue
		}
		set.Insert(id)
	}
	return set
}

// Get returns the requested item, or sets exists=false.
func (f *HistoricalFIFO) Get(id string) (interface{}, bool) {
	f.lock.RLock()
	defer f.lock.RUnlock()
	entry, exists := f.items[id]
	if exists && !(entry.Event == DELETE_EVENT || entry.Event == POP_EVENT) {
		return entry.Value.Copy(), true
	}
	return nil, false
}

// Pop waits until an item is ready and returns it. If multiple items are
// ready, they are returned in the order in which they were added/updated.
// The item is removed from the queue (and the store) before it is returned,
// so if you don't succesfully process it, you need to add it back with Add().
func (f *HistoricalFIFO) Pop() interface{} {
	popEvent := (*Entry)(nil)
	defer func() {
		f.carrier(popEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	for {
		for len(f.queue) == 0 {
			f.cond.Wait()
		}
		id := f.queue[0]
		f.queue = f.queue[1:]
		entry, ok := f.items[id]
		if !ok || entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			// Item may have been deleted subsequently.
			continue
		}
		value := entry.Value
		popEvent = &Entry{Value: value, Event: POP_EVENT}
		f.items[id] = popEvent
		return value.Copy()
	}
}

// Replace will delete the contents of 'f', using instead the given map.
// 'f' takes ownersip of the map, you should not reference the map again
// after calling this function. f's queue is reset, too; upon return, it
// will contain the items in the map, in no particular order.
func (f *HistoricalFIFO) Replace(idToObj map[string]interface{}) {
	notifications := make([]*Entry, 0, len(idToObj))
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	f.queue = f.queue[:0]
	for id, v := range f.items {
		if _, exists := idToObj[id]; !exists && v.Event != DELETE_EVENT {
			// a non-deleted entry in the items list that doesn't show up in the
			// new list: mark it as deleted
			e := &Entry{Value: v.Value, Event: DELETE_EVENT}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	for id, v := range idToObj {
		obj := checkType(v)
		f.queue = append(f.queue, id)
		n := f.merge(id, obj)
		notifications = append(notifications, n...)
	}
	if len(f.queue) > 0 {
		f.cond.Broadcast()
	}
}

// expects that caller has already locked around state
func (f *HistoricalFIFO) merge(id string, obj UniqueCopyable) (notifications []*Entry) {
	entry, exists := f.items[id]
	if !exists {
		e := &Entry{Value: obj.Copy().(UniqueCopyable), Event: ADD_EVENT}
		f.items[id] = e
		notifications = append(notifications, e)
	} else {
		if entry.Event != DELETE_EVENT && entry.Value.GetUID() != obj.GetUID() {
			// hidden DELETE!
			// (1) append a DELETE
			// (2) append an ADD
			// .. and notify listeners in that order
			e1 := &Entry{Value: entry.Value, Event: DELETE_EVENT}
			e2 := &Entry{Value: obj.Copy().(UniqueCopyable), Event: ADD_EVENT}
			f.items[id] = e2
			notifications = append(notifications, e1, e2)
		} else if !reflect.DeepEqual(obj, entry.Value) {
			//TODO(jdef): it would be nice if we could rely on resource versions
			//instead of doing a DeepEqual. Maybe someday we'll be able to.
			e := &Entry{Value: obj.Copy().(UniqueCopyable), Event: UPDATE_EVENT}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	return
}

// NewFIFO returns a Store which can be used to queue up items to
// process. If a non-nil Mux is provided, then modifications to the
// the FIFO are delivered on a channel specific to this fifo.
func NewFIFO(mux *config.Mux) *HistoricalFIFO {
	carrier := dead
	if mux != nil {
		//TODO(jdef): append a UUID to "fifo" here?
		ch := mux.Channel("fifo")
		carrier = func(msg *Entry) {
			if msg != nil {
				ch <- msg.Copy()
			}
		}
	}
	f := &HistoricalFIFO{
		items:   map[string]*Entry{},
		queue:   []string{},
		carrier: carrier,
	}
	f.cond.L = &f.lock
	return f
}
<commit_msg>refactor Entry into an interface; simply the storage map; initial implementation of garbage collection for deleted entries<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// evolution of: https://github.com/GoogleCloudPlatform/kubernetes/blob/release-0.6/pkg/client/cache/fifo.go
package queue

import (
	"fmt"
	"reflect"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
)

type EventType int

const (
	ADD_EVENT EventType = 1 << iota
	UPDATE_EVENT
	DELETE_EVENT
	POP_EVENT
)

type Entry interface {
	Copyable
	Value() UniqueCopyable
	// types is a logically OR'd combination of EventType, e.g. ADD_EVENT|UPDATE_EVENT
	Is(types EventType) bool
}

type entry struct {
	value UniqueCopyable
	event EventType
}

type deletedEntry struct {
	*entry
	expiration time.Time
}

type Copyable interface {
	// return an independent copy (deep clone) of the current object
	Copy() Copyable
}

type UniqueID interface {
	GetUID() string
}

type UniqueCopyable interface {
	Copyable
	UniqueID
}

func (e *entry) Value() UniqueCopyable {
	return e.value
}

func (e *entry) Copy() Copyable {
	if e == nil {
		return nil
	}
	return &entry{e.value.Copy().(UniqueCopyable), e.event}
}

func (e *entry) Is(types EventType) bool {
	return types&e.event != 0
}

func (e *deletedEntry) Copy() Copyable {
	if e == nil {
		return nil
	}
	return &deletedEntry{e.entry.Copy().(*entry), e.expiration}
}

// deliver a message
type pigeon func(msg Entry)

func dead(msg Entry) {
	// intentionally blank
}

// HistoricalFIFO receives adds and updates from a Reflector, and puts them in a queue for
// FIFO order processing. If multiple adds/updates of a single item happen while
// an item is in the queue before it has been processed, it will only be
// processed once, and when it is processed, the most recent version will be
// processed. This can't be done with a channel.
type HistoricalFIFO struct {
	lock      sync.RWMutex
	cond      sync.Cond
	items     map[string]Entry // We depend on the property that items in the queue are in the set.
	queue     []string
	carrier   pigeon // may be dead, but never nil
	gcc       int
	lingerTTL time.Duration
}

// panics if obj doesn't implement UniqueCopyable; otherwise returns the same, typecast object
func checkType(obj interface{}) UniqueCopyable {
	if v, ok := obj.(UniqueCopyable); !ok {
		panic(fmt.Sprintf("Illegal object type, expected UniqueCopyable: %T", obj))
	} else {
		return v
	}
}

// Add inserts an item, and puts it in the queue. The item is only enqueued
// if it doesn't already exist in the set.
func (f *HistoricalFIFO) Add(id string, v interface{}) {
	obj := checkType(v)
	notifications := []Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; !exists {
		f.queue = append(f.queue, id)
	} else {
		if entry.Is(DELETE_EVENT | POP_EVENT) {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj)
	f.cond.Broadcast()
}

// Update is the same as Add in this implementation.
func (f *HistoricalFIFO) Update(id string, obj interface{}) {
	f.Add(id, obj)
}

// Add the item to the store, but only if there exists a prior entry for
// for the object in the store whose event type matches that given, and then
// only enqueued if it doesn't already exist in the set.
func (f *HistoricalFIFO) Readd(id string, v interface{}, t EventType) {
	obj := checkType(v)
	notifications := []Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; exists {
		if !entry.Is(t) {
			return
		} else if entry.Is(DELETE_EVENT | POP_EVENT) {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj)
	f.cond.Broadcast()
}

// Delete removes an item. It doesn't add it to the queue, because
// this implementation assumes the consumer only cares about the objects,
// not the order in which they were created/added.
func (f *HistoricalFIFO) Delete(id string) {
	deleteEvent := (Entry)(nil)
	defer func() {
		f.carrier(deleteEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	item, exists := f.items[id]
	if exists && !item.Is(DELETE_EVENT) {
		e := item.(*entry)
		e.event = DELETE_EVENT
		deleteEvent = &deletedEntry{e, time.Now().Add(f.lingerTTL)}
		f.items[id] = deleteEvent
	}
}

// List returns a list of all the items.
func (f *HistoricalFIFO) List() []interface{} {
	f.lock.RLock()
	defer f.lock.RUnlock()

	// TODO(jdef): slightly overallocates b/c of deleted items
	list := make([]interface{}, 0, len(f.queue))

	for _, entry := range f.items {
		if entry.Is(DELETE_EVENT | POP_EVENT) {
			continue
		}
		list = append(list, entry.Value().Copy())
	}
	return list
}

// ContainedIDs returns a util.StringSet containing all IDs of the stored items.
// This is a snapshot of a moment in time, and one should keep in mind that
// other go routines can add or remove items after you call this.
func (c *HistoricalFIFO) ContainedIDs() util.StringSet {
	c.lock.RLock()
	defer c.lock.RUnlock()
	set := util.StringSet{}
	for id, entry := range c.items {
		if entry.Is(DELETE_EVENT | POP_EVENT) {
			continue
		}
		set.Insert(id)
	}
	return set
}

// Get returns the requested item, or sets exists=false.
func (f *HistoricalFIFO) Get(id string) (interface{}, bool) {
	f.lock.RLock()
	defer f.lock.RUnlock()
	entry, exists := f.items[id]
	if exists && !entry.Is(DELETE_EVENT|POP_EVENT) {
		return entry.Value().Copy(), true
	}
	return nil, false
}

// Pop waits until an item is ready and returns it. If multiple items are
// ready, they are returned in the order in which they were added/updated.
// The item is removed from the queue (and the store) before it is returned,
// so if you don't succesfully process it, you need to add it back with Add().
func (f *HistoricalFIFO) Pop() interface{} {
	popEvent := (Entry)(nil)
	defer func() {
		f.carrier(popEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	for {
		for len(f.queue) == 0 {
			f.cond.Wait()
		}
		id := f.queue[0]
		f.queue = f.queue[1:]
		item, ok := f.items[id]
		if !ok || item.Is(DELETE_EVENT|POP_EVENT) {
			// Item may have been deleted subsequently.
			continue
		}
		value := item.Value()
		popEvent = &entry{value, POP_EVENT}
		f.items[id] = popEvent
		return value.Copy()
	}
}

// Replace will delete the contents of 'f', using instead the given map.
// 'f' takes ownersip of the map, you should not reference the map again
// after calling this function. f's queue is reset, too; upon return, it
// will contain the items in the map, in no particular order.
func (f *HistoricalFIFO) Replace(idToObj map[string]interface{}) {
	notifications := make([]Entry, 0, len(idToObj))
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	f.queue = f.queue[:0]
	now := time.Now()
	for id, v := range f.items {
		if _, exists := idToObj[id]; !exists && !v.Is(DELETE_EVENT) {
			// a non-deleted entry in the items list that doesn't show up in the
			// new list: mark it as deleted
			ent := v.(*entry)
			ent.event = DELETE_EVENT
			e := &deletedEntry{ent, now.Add(f.lingerTTL)}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	for id, v := range idToObj {
		obj := checkType(v)
		f.queue = append(f.queue, id)
		n := f.merge(id, obj)
		notifications = append(notifications, n...)
	}
	if len(f.queue) > 0 {
		f.cond.Broadcast()
	}
}

// garbage collect DELETEd items whose TTL has expired; the IDs of such items are removed
// from the queue. This impl assumes that caller has acquired state lock.
func (f *HistoricalFIFO) gc() {
	now := time.Now()
	deleted := make(map[string]struct{})
	for id, v := range f.items {
		if v.Is(DELETE_EVENT) {
			ent := v.(*deletedEntry)
			if ent.expiration.Before(now) {
				delete(f.items, id)
				deleted[id] = struct{}{}
			}
		}
	}
	// remove deleted items from the queue, will likely (slightly) overallocate here
	queue := make([]string, 0, len(f.queue))
	for _, id := range f.queue {
		if _, exists := deleted[id]; !exists {
			queue = append(queue, id)
		}
	}
	f.queue = queue
}

// Assumes that the caller has acquired the state lock.
func (f *HistoricalFIFO) merge(id string, obj UniqueCopyable) (notifications []Entry) {
	item, exists := f.items[id]
	now := time.Now()
	if !exists {
		e := &entry{obj.Copy().(UniqueCopyable), ADD_EVENT}
		f.items[id] = e
		notifications = append(notifications, e)
	} else {
		if !item.Is(DELETE_EVENT) && item.Value().GetUID() != obj.GetUID() {
			// hidden DELETE!
			// (1) append a DELETE
			// (2) append an ADD
			// .. and notify listeners in that order
			ent := item.(*entry)
			ent.event = DELETE_EVENT
			e1 := &deletedEntry{ent, now.Add(f.lingerTTL)}
			e2 := &entry{obj.Copy().(UniqueCopyable), ADD_EVENT}
			f.items[id] = e2
			notifications = append(notifications, e1, e2)
		} else if !reflect.DeepEqual(obj, item.Value()) {
			//TODO(jdef): it would be nice if we could rely on resource versions
			//instead of doing a DeepEqual. Maybe someday we'll be able to.
			e := &entry{obj.Copy().(UniqueCopyable), UPDATE_EVENT}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	// check for garbage collection
	f.gcc++
	if f.gcc%256 == 0 { //TODO(jdef): extract constant
		f.gcc = 0
		f.gc()
	}
	return
}

// NewFIFO returns a Store which can be used to queue up items to
// process. If a non-nil Mux is provided, then modifications to the
// the FIFO are delivered on a channel specific to this fifo.
func NewFIFO(ch chan<- Entry) *HistoricalFIFO {
	carrier := dead
	if ch != nil {
		carrier = func(msg Entry) {
			if msg != nil {
				ch <- msg.Copy().(Entry)
			}
		}
	}
	f := &HistoricalFIFO{
		items:     map[string]Entry{},
		queue:     []string{},
		carrier:   carrier,
		lingerTTL: 5 * time.Minute, // TODO(jdef): extract constant
	}
	f.cond.L = &f.lock
	return f
}
<|endoftext|>


<commit_before>package content

import (
	"errors"
	"html/template"
	"io"

	"github.com/russross/blackfriday"

	"github.com/octavore/press/proto/press/models"
)

func (m *Module) render(w io.Writer, page *models.Page) error {
	contents, err := m.createContentMap(page)
	if err != nil {
		return err
	}
	return m.Templates.Render(w, page, contents)
}

type contentMap map[string]interface{}

func (m *Module) createContentMap(page *models.Page) (contentMap, error) {
	contents := map[string]interface{}{}
	var err error
	for _, c := range page.Contents {
		switch c.GetType().(type) {
		case *models.Content_Short:
			contents[c.GetKey()], err = renderTextualContent(c.GetValue(), c.GetShort())
		case *models.Content_Text:
			contents[c.GetKey()], err = renderTextualContent(c.GetValue(), c.GetText())
		case *models.Content_Multiple:
			contents[c.GetKey()] = c.GetValue()
		default:
			m.Logger.Warningf("unknown content type: %s", c.GetType())
		}
		// future: allow custom registere content type renderers
		if err != nil {
			return nil, err
		}
	}
	return contents, err
}

type textualContent interface {
	GetType() models.ContentTextType
}

func renderTextualContent(s string, t textualContent) (interface{}, error) {
	switch t.GetType() {
	case models.ContentTextType_text:
		return s, nil
	case models.ContentTextType_html:
		return template.HTML(s), nil
	case models.ContentTextType_markdown:
		data := blackfriday.MarkdownCommon([]byte(s))
		return template.HTML(string(data)), nil
	}
	return nil, errors.New("unknown content text type: " + t.GetType().String())
}
<commit_msg>server: Pass content title to page.<commit_after>package content

import (
	"errors"
	"html/template"
	"io"

	"github.com/russross/blackfriday"

	"github.com/octavore/press/proto/press/models"
)

func (m *Module) render(w io.Writer, page *models.Page) error {
	contents, err := m.createContentMap(page)
	if err != nil {
		return err
	}
	return m.Templates.Render(w, page, contents)
}

type contentMap map[string]interface{}

func (m *Module) createContentMap(page *models.Page) (contentMap, error) {
	contents := map[string]interface{}{}
	contents["title"] = page.GetName()
	var err error
	for _, c := range page.Contents {
		switch c.GetType().(type) {
		case *models.Content_Short:
			contents[c.GetKey()], err = renderTextualContent(c.GetValue(), c.GetShort())
		case *models.Content_Text:
			contents[c.GetKey()], err = renderTextualContent(c.GetValue(), c.GetText())
		case *models.Content_Multiple:
			contents[c.GetKey()] = c.GetValue()
		default:
			m.Logger.Warningf("unknown content type: %s", c.GetType())
		}
		// future: allow custom registered content type renderers
		if err != nil {
			return nil, err
		}
	}
	return contents, err
}

type textualContent interface {
	GetType() models.ContentTextType
}

func renderTextualContent(s string, t textualContent) (interface{}, error) {
	switch t.GetType() {
	case models.ContentTextType_text:
		return s, nil
	case models.ContentTextType_html:
		return template.HTML(s), nil
	case models.ContentTextType_markdown:
		data := blackfriday.MarkdownCommon([]byte(s))
		return template.HTML(string(data)), nil
	}
	return nil, errors.New("unknown content text type: " + t.GetType().String())
}
<|endoftext|>