<commit_before>package db

import (
    "log"
    "fmt"
    "gopkg.in/mgo.v2"
    "gopkg.in/mgo.v2/bson"
    "github.com/oliveroneill/hanserver/hanapi/imagedata"
)

// MongoInterface - a mongodb implementation of `DatabaseInterface`
type MongoInterface struct {
    DatabaseInterface
    session *mgo.Session
}

// NewMongoInterface - use to create a new mongo connection
func NewMongoInterface() DatabaseInterface {
    c := new(MongoInterface)
    // use for Docker:
    session, err := mgo.Dial("mongodb")
    // use locally:
    // session, err := mgo.Dial("localhost:27017")
    if err != nil {
        log.Fatal(err)
    }
    c.session = session
    // if geospatial index hasn't been set up this will create it
    getImageCollection(session).EnsureIndex(mgo.Index{Key: []string{"coordinates:2dsphere"}})
    return c
}

func getHanDB(session *mgo.Session) *mgo.Database {
    return session.DB("han")
}

func getRegionCollection(session *mgo.Session) *mgo.Collection {
    return getHanDB(session).C("regions")
}

func getImageCollection(session *mgo.Session) *mgo.Collection {
    return getHanDB(session).C("images")
}

// GetRegions returns the watched locations that are stored in the database
// These locations are queried to populate the database with images
func (c MongoInterface) GetRegions() []imagedata.Location {
    collection := getRegionCollection(c.session)
    var regions []imagedata.Location
    collection.Find(map[string]interface{}{}).All(&regions)
    return regions
}

// AddRegion adds this new location as a place to query images on
func (c MongoInterface) AddRegion(lat float64, lng float64) {
    collection := getRegionCollection(c.session)
    collection.Insert(map[string]interface{}{ "lat": lat, "lng": lng })
}

// AddImage adds new image data for the feed
func (c MongoInterface) AddImage(image imagedata.ImageData) {
    collection := getImageCollection(c.session)
    // insert if it's not already there
    _, err := collection.Upsert(map[string]interface{}{ "_id": image.ID }, image)
    if err != nil {
        log.Fatal(err)
    }
}

// GetImages returns images closest to the specified location
func (c MongoInterface) GetImages(lat float64, lng float64) []imagedata.ImageData {
    // Mongo allows us to aggregate based on distance from the query
    agg := []bson.M{
        bson.M{
            "$geoNear": bson.M{
                "spherical": true,
                "near": bson.M{
                    "type": "Point",
                    "coordinates": []float64{lng, lat},
                },
                "distanceField": "distance",
                // ensure that deleted images aren't in here
                "query": map[string]interface{}{"deleted": nil},
            },
        },
    }

    // convert to response data
    var response []imagedata.ImageData
    collection := getImageCollection(c.session)
    collection.Pipe(agg).All(&response)
    return response
}

// GetAllImages returns all images stored
func (c MongoInterface) GetAllImages() []imagedata.ImageData {
    var response []imagedata.ImageData
    collection := getImageCollection(c.session)
    err := collection.Find(nil).All(&response)
    if err != nil {
        panic(err)
    }
    return response
}

// SoftDelete will add a delete field to image so it's no longer visible in
// feed
func (c MongoInterface) SoftDelete(id string, reason string) {
    collection := getImageCollection(c.session)
    // update the image with a "deleted" field
    err := collection.UpdateId(
        id,
        bson.M{"$set": bson.M{"deleted": true, "deleted_reason": reason}},
    )
    if err != nil {
        fmt.Println(err)
    }
}

// Close will close the current mongo connection
func (c MongoInterface) Close() {
    c.session.Close()
}<commit_msg>fixed: bug where images are undeleted after update<commit_after>package db

import (
    "log"
    "fmt"
    "gopkg.in/mgo.v2"
    "gopkg.in/mgo.v2/bson"
    "github.com/oliveroneill/hanserver/hanapi/imagedata"
)

// MongoInterface - a mongodb implementation of `DatabaseInterface`
type MongoInterface struct {
    DatabaseInterface
    session *mgo.Session
}

// NewMongoInterface - use to create a new mongo connection
func NewMongoInterface() DatabaseInterface {
    c := new(MongoInterface)
    // use for Docker:
    session, err := mgo.Dial("mongodb")
    // use locally:
    // session, err := mgo.Dial("localhost:27017")
    if err != nil {
        log.Fatal(err)
    }
    c.session = session
    // if geospatial index hasn't been set up this will create it
    getImageCollection(session).EnsureIndex(mgo.Index{Key: []string{"coordinates:2dsphere"}})
    return c
}

func getHanDB(session *mgo.Session) *mgo.Database {
    return session.DB("han")
}

func getRegionCollection(session *mgo.Session) *mgo.Collection {
    return getHanDB(session).C("regions")
}

func getImageCollection(session *mgo.Session) *mgo.Collection {
    return getHanDB(session).C("images")
}

// GetRegions returns the watched locations that are stored in the database
// These locations are queried to populate the database with images
func (c MongoInterface) GetRegions() []imagedata.Location {
    collection := getRegionCollection(c.session)
    var regions []imagedata.Location
    collection.Find(map[string]interface{}{}).All(&regions)
    return regions
}

// AddRegion adds this new location as a place to query images on
func (c MongoInterface) AddRegion(lat float64, lng float64) {
    collection := getRegionCollection(c.session)
    collection.Insert(map[string]interface{}{ "lat": lat, "lng": lng })
}

// AddImage adds new image data for the feed
func (c MongoInterface) AddImage(image imagedata.ImageData) {
    collection := getImageCollection(c.session)
    // insert if it's not already there
    _, err := collection.Upsert(bson.M{ "_id": image.ID }, bson.M{"$set":image})
    if err != nil {
        log.Fatal(err)
    }
}

// GetImages returns images closest to the specified location
func (c MongoInterface) GetImages(lat float64, lng float64) []imagedata.ImageData {
    // Mongo allows us to aggregate based on distance from the query
    agg := []bson.M{
        bson.M{
            "$geoNear": bson.M{
                "spherical": true,
                "near": bson.M{
                    "type": "Point",
                    "coordinates": []float64{lng, lat},
                },
                "distanceField": "distance",
                // ensure that deleted images aren't in here
                "query": map[string]interface{}{"deleted": nil},
            },
        },
    }

    // convert to response data
    var response []imagedata.ImageData
    collection := getImageCollection(c.session)
    collection.Pipe(agg).All(&response)
    return response
}

// GetAllImages returns all images stored
func (c MongoInterface) GetAllImages() []imagedata.ImageData {
    var response []imagedata.ImageData
    collection := getImageCollection(c.session)
    err := collection.Find(nil).All(&response)
    if err != nil {
        panic(err)
    }
    return response
}

// SoftDelete will add a delete field to image so it's no longer visible in
// feed
func (c MongoInterface) SoftDelete(id string, reason string) {
    collection := getImageCollection(c.session)
    // update the image with a "deleted" field
    err := collection.UpdateId(
        id,
        bson.M{"$set": bson.M{"deleted": true, "deleted_reason": reason}},
    )
    if err != nil {
        fmt.Println(err)
    }
}

// Close will close the current mongo connection
func (c MongoInterface) Close() {
    c.session.Close()
}<|endoftext|>


<commit_before>package handlers

import (
	"io"
	"log"
	"net/http"
	"time"

	"github.com/zenazn/goji/web"
)

const STATIC_URL string = "http://redskull.iamtherealbill.com/static/"
const STATIC_ROOT string = "html/static/"

// Static serves static content such as images, JS and CSS files
func Static(c web.C, w http.ResponseWriter, req *http.Request) {
	//log.Printf("STATIC_URL: %s, %d", STATIC_URL, len(STATIC_URL))
	//log.Printf("PATH: %s, %d", req.URL.Path, len(STATIC_URL))
	static_file := req.URL.Path[7:]
	if len(static_file) != 0 {
		f, err := http.Dir(STATIC_ROOT).Open(static_file)
		if err == nil {
			content := io.ReadSeeker(f)
			http.ServeContent(w, req, static_file, time.Now(), content)
			return
		}
		log.Print(err)
	}
	http.NotFound(w, req)
}
<commit_msg>fixing STATIC_ROOT to use envconfig for base directory<commit_after>package handlers

import (
	"io"
	"log"
	"net/http"
	"time"

	"github.com/zenazn/goji/web"
)

const STATIC_URL string = "http://redskull.iamtherealbill.com/static/"
const STATIC_ROOT string = "html/static/"

// Static serves static content such as images, JS and CSS files
func Static(c web.C, w http.ResponseWriter, req *http.Request) {
	//log.Printf("STATIC_URL: %s, %d", STATIC_URL, len(STATIC_URL))
	//log.Printf("PATH: %s, %d", req.URL.Path, len(STATIC_URL))
	static_file := req.URL.Path[7:]
	root := TemplateBase + STATIC_ROOT
	if len(static_file) != 0 {
		f, err := http.Dir(root).Open(static_file)
		if err == nil {
			content := io.ReadSeeker(f)
			http.ServeContent(w, req, static_file, time.Now(), content)
			return
		}
		log.Print(err)
	}
	http.NotFound(w, req)
}
<|endoftext|>


<commit_before>// Copyright © 2014 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"net"
	"path/filepath"
	"reflect"
	"strings"
	"sync"

	"github.com/spf13/cast"
	bp "github.com/spf13/hugo/bufferpool"
	jww "github.com/spf13/jwalterweatherman"
	"github.com/spf13/viper"
)

// Filepath separator defined by os.Separator.
const FilePathSeparator = string(filepath.Separator)

// FindAvailablePort returns an available and valid TCP port.
func FindAvailablePort() (*net.TCPAddr, error) {
	l, err := net.Listen("tcp", ":0")
	if err == nil {
		defer l.Close()
		addr := l.Addr()
		if a, ok := addr.(*net.TCPAddr); ok {
			return a, nil
		}
		return nil, fmt.Errorf("Unable to obtain a valid tcp port. %v", addr)
	}
	return nil, err
}

// InStringArray checks if a string is an element of a slice of strings
// and returns a boolean value.
func InStringArray(arr []string, el string) bool {
	for _, v := range arr {
		if v == el {
			return true
		}
	}
	return false
}

// GuessType attempts to guess the type of file from a given string.
func GuessType(in string) string {
	switch strings.ToLower(in) {
	case "md", "markdown", "mdown":
		return "markdown"
	case "asciidoc", "adoc", "ad":
		return "asciidoc"
	case "mmark":
		return "mmark"
	case "rst":
		return "rst"
	case "html", "htm":
		return "html"
	}

	return "unknown"
}

// ReaderToBytes takes an io.Reader argument, reads from it
// and returns bytes.
func ReaderToBytes(lines io.Reader) []byte {
	if lines == nil {
		return []byte{}
	}
	b := bp.GetBuffer()
	defer bp.PutBuffer(b)

	b.ReadFrom(lines)

	bc := make([]byte, b.Len(), b.Len())
	copy(bc, b.Bytes())
	return bc
}

// ReaderToString is the same as ReaderToBytes, but returns a string.
func ReaderToString(lines io.Reader) string {
	if lines == nil {
		return ""
	}
	b := bp.GetBuffer()
	defer bp.PutBuffer(b)
	b.ReadFrom(lines)
	return b.String()
}

// StringToReader does the opposite of ReaderToString.
func StringToReader(in string) io.Reader {
	return strings.NewReader(in)
}

// BytesToReader does the opposite of ReaderToBytes.
func BytesToReader(in []byte) io.Reader {
	return bytes.NewReader(in)
}

// ReaderContains reports whether subslice is within r.
func ReaderContains(r io.Reader, subslice []byte) bool {

	if r == nil || len(subslice) == 0 {
		return false
	}

	bufflen := len(subslice) * 4
	halflen := bufflen / 2
	buff := make([]byte, bufflen)
	var err error
	var n, i int

	for {
		i++
		if i == 1 {
			n, err = io.ReadAtLeast(r, buff[:halflen], halflen)
		} else {
			if i != 2 {
				// shift left to catch overlapping matches
				copy(buff[:], buff[halflen:])
			}
			n, err = io.ReadAtLeast(r, buff[halflen:], halflen)
		}

		if n > 0 && bytes.Contains(buff, subslice) {
			return true
		}

		if err != nil {
			break
		}
	}
	return false
}

// ThemeSet checks whether a theme is in use or not.
func ThemeSet() bool {
	return viper.GetString("theme") != ""
}

// DistinctErrorLogger ignores duplicate log statements.
type DistinctErrorLogger struct {
	sync.RWMutex
	m map[string]bool
}

// Printf will ERROR log the string returned from fmt.Sprintf given the arguments,
// but not if it has been logged before.
func (l *DistinctErrorLogger) Printf(format string, v ...interface{}) {
	logStatement := fmt.Sprintf(format, v...)
	l.RLock()
	if l.m[logStatement] {
		l.RUnlock()
		return
	}
	l.RUnlock()

	l.Lock()
	if !l.m[logStatement] {
		jww.ERROR.Print(logStatement)
		l.m[logStatement] = true
	}
	l.Unlock()
}

// NewDistinctErrorLogger creates a new DistinctErrorLogger
func NewDistinctErrorLogger() *DistinctErrorLogger {
	return &DistinctErrorLogger{m: make(map[string]bool)}
}

// Avoid spamming the logs with errors
var deprecatedLogger = NewDistinctErrorLogger()

// Deprecated logs ERROR logs about a deprecation, but only once for a given set of arguments' values.
func Deprecated(object, item, alternative string) {
	deprecatedLogger.Printf("%s's %s is deprecated and will be removed in Hugo %s. Use %s instead.", object, item, NextHugoReleaseVersion(), alternative)
}

// SliceToLower goes through the source slice and lowers all values.
func SliceToLower(s []string) []string {
	if s == nil {
		return nil
	}

	l := make([]string, len(s))
	for i, v := range s {
		l[i] = strings.ToLower(v)
	}

	return l
}

// Md5String takes a string and returns its MD5 hash.
func Md5String(f string) string {
	h := md5.New()
	h.Write([]byte(f))
	return hex.EncodeToString(h.Sum([]byte{}))
}

// Seq creates a sequence of integers.
// It's named and used as GNU's seq.
// Examples:
// 3 => 1, 2, 3
// 1 2 4 => 1, 3
// -3 => -1, -2, -3
// 1 4 => 1, 2, 3, 4
// 1 -2 => 1, 0, -1, -2
func Seq(args ...interface{}) ([]int, error) {
	if len(args) < 1 || len(args) > 3 {
		return nil, errors.New("Seq, invalid number of args: 'first' 'increment' (optional) 'last' (optional)")
	}

	intArgs := cast.ToIntSlice(args)

	if len(intArgs) < 1 || len(intArgs) > 3 {
		return nil, errors.New("Invalid argument(s) to Seq")
	}

	var inc = 1
	var last int
	var first = intArgs[0]

	if len(intArgs) == 1 {
		last = first
		if last == 0 {
			return []int{}, nil
		} else if last > 0 {
			first = 1
		} else {
			first = -1
			inc = -1
		}
	} else if len(intArgs) == 2 {
		last = intArgs[1]
		if last < first {
			inc = -1
		}
	} else {
		inc = intArgs[1]
		last = intArgs[2]
		if inc == 0 {
			return nil, errors.New("'increment' must not be 0")
		}
		if first < last && inc < 0 {
			return nil, errors.New("'increment' must be > 0")
		}
		if first > last && inc > 0 {
			return nil, errors.New("'increment' must be < 0")
		}
	}

	// sanity check
	if last < -100000 {
		return nil, errors.New("size of result exeeds limit")
	}
	size := int(((last - first) / inc) + 1)

	// sanity check
	if size <= 0 || size > 2000 {
		return nil, errors.New("size of result exeeds limit")
	}

	seq := make([]int, size)
	val := first
	for i := 0; ; i++ {
		seq[i] = val
		val += inc
		if (inc < 0 && val < last) || (inc > 0 && val > last) {
			break
		}
	}

	return seq, nil
}

// DoArithmetic performs arithmetic operations (+,-,*,/) using reflection to
// determine the type of the two terms.
func DoArithmetic(a, b interface{}, op rune) (interface{}, error) {
	av := reflect.ValueOf(a)
	bv := reflect.ValueOf(b)
	var ai, bi int64
	var af, bf float64
	var au, bu uint64
	switch av.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		ai = av.Int()
		switch bv.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			bi = bv.Int()
		case reflect.Float32, reflect.Float64:
			af = float64(ai) // may overflow
			ai = 0
			bf = bv.Float()
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			bu = bv.Uint()
			if ai >= 0 {
				au = uint64(ai)
				ai = 0
			} else {
				bi = int64(bu) // may overflow
				bu = 0
			}
		default:
			return nil, errors.New("Can't apply the operator to the values")
		}
	case reflect.Float32, reflect.Float64:
		af = av.Float()
		switch bv.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			bf = float64(bv.Int()) // may overflow
		case reflect.Float32, reflect.Float64:
			bf = bv.Float()
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			bf = float64(bv.Uint()) // may overflow
		default:
			return nil, errors.New("Can't apply the operator to the values")
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		au = av.Uint()
		switch bv.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			bi = bv.Int()
			if bi >= 0 {
				bu = uint64(bi)
				bi = 0
			} else {
				ai = int64(au) // may overflow
				au = 0
			}
		case reflect.Float32, reflect.Float64:
			af = float64(au) // may overflow
			au = 0
			bf = bv.Float()
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			bu = bv.Uint()
		default:
			return nil, errors.New("Can't apply the operator to the values")
		}
	case reflect.String:
		as := av.String()
		if bv.Kind() == reflect.String && op == '+' {
			bs := bv.String()
			return as + bs, nil
		}
		return nil, errors.New("Can't apply the operator to the values")
	default:
		return nil, errors.New("Can't apply the operator to the values")
	}

	switch op {
	case '+':
		if ai != 0 || bi != 0 {
			return ai + bi, nil
		} else if af != 0 || bf != 0 {
			return af + bf, nil
		} else if au != 0 || bu != 0 {
			return au + bu, nil
		} else {
			return 0, nil
		}
	case '-':
		if ai != 0 || bi != 0 {
			return ai - bi, nil
		} else if af != 0 || bf != 0 {
			return af - bf, nil
		} else if au != 0 || bu != 0 {
			return au - bu, nil
		} else {
			return 0, nil
		}
	case '*':
		if ai != 0 || bi != 0 {
			return ai * bi, nil
		} else if af != 0 || bf != 0 {
			return af * bf, nil
		} else if au != 0 || bu != 0 {
			return au * bu, nil
		} else {
			return 0, nil
		}
	case '/':
		if bi != 0 {
			return ai / bi, nil
		} else if bf != 0 {
			return af / bf, nil
		} else if bu != 0 {
			return au / bu, nil
		} else {
			return nil, errors.New("Can't divide the value by 0")
		}
	default:
		return nil, errors.New("There is no such an operation")
	}
}
<commit_msg>Make deprecated warning for this release more alert<commit_after>// Copyright © 2014 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"net"
	"path/filepath"
	"reflect"
	"strings"
	"sync"

	"github.com/spf13/cast"
	bp "github.com/spf13/hugo/bufferpool"
	jww "github.com/spf13/jwalterweatherman"
	"github.com/spf13/viper"
)

// Filepath separator defined by os.Separator.
const FilePathSeparator = string(filepath.Separator)

// FindAvailablePort returns an available and valid TCP port.
func FindAvailablePort() (*net.TCPAddr, error) {
	l, err := net.Listen("tcp", ":0")
	if err == nil {
		defer l.Close()
		addr := l.Addr()
		if a, ok := addr.(*net.TCPAddr); ok {
			return a, nil
		}
		return nil, fmt.Errorf("Unable to obtain a valid tcp port. %v", addr)
	}
	return nil, err
}

// InStringArray checks if a string is an element of a slice of strings
// and returns a boolean value.
func InStringArray(arr []string, el string) bool {
	for _, v := range arr {
		if v == el {
			return true
		}
	}
	return false
}

// GuessType attempts to guess the type of file from a given string.
func GuessType(in string) string {
	switch strings.ToLower(in) {
	case "md", "markdown", "mdown":
		return "markdown"
	case "asciidoc", "adoc", "ad":
		return "asciidoc"
	case "mmark":
		return "mmark"
	case "rst":
		return "rst"
	case "html", "htm":
		return "html"
	}

	return "unknown"
}

// ReaderToBytes takes an io.Reader argument, reads from it
// and returns bytes.
func ReaderToBytes(lines io.Reader) []byte {
	if lines == nil {
		return []byte{}
	}
	b := bp.GetBuffer()
	defer bp.PutBuffer(b)

	b.ReadFrom(lines)

	bc := make([]byte, b.Len(), b.Len())
	copy(bc, b.Bytes())
	return bc
}

// ReaderToString is the same as ReaderToBytes, but returns a string.
func ReaderToString(lines io.Reader) string {
	if lines == nil {
		return ""
	}
	b := bp.GetBuffer()
	defer bp.PutBuffer(b)
	b.ReadFrom(lines)
	return b.String()
}

// StringToReader does the opposite of ReaderToString.
func StringToReader(in string) io.Reader {
	return strings.NewReader(in)
}

// BytesToReader does the opposite of ReaderToBytes.
func BytesToReader(in []byte) io.Reader {
	return bytes.NewReader(in)
}

// ReaderContains reports whether subslice is within r.
func ReaderContains(r io.Reader, subslice []byte) bool {

	if r == nil || len(subslice) == 0 {
		return false
	}

	bufflen := len(subslice) * 4
	halflen := bufflen / 2
	buff := make([]byte, bufflen)
	var err error
	var n, i int

	for {
		i++
		if i == 1 {
			n, err = io.ReadAtLeast(r, buff[:halflen], halflen)
		} else {
			if i != 2 {
				// shift left to catch overlapping matches
				copy(buff[:], buff[halflen:])
			}
			n, err = io.ReadAtLeast(r, buff[halflen:], halflen)
		}

		if n > 0 && bytes.Contains(buff, subslice) {
			return true
		}

		if err != nil {
			break
		}
	}
	return false
}

// ThemeSet checks whether a theme is in use or not.
func ThemeSet() bool {
	return viper.GetString("theme") != ""
}

// DistinctErrorLogger ignores duplicate log statements.
type DistinctErrorLogger struct {
	sync.RWMutex
	m map[string]bool
}

// Printf will ERROR log the string returned from fmt.Sprintf given the arguments,
// but not if it has been logged before.
func (l *DistinctErrorLogger) Printf(format string, v ...interface{}) {
	logStatement := fmt.Sprintf(format, v...)
	l.RLock()
	if l.m[logStatement] {
		l.RUnlock()
		return
	}
	l.RUnlock()

	l.Lock()
	if !l.m[logStatement] {
		jww.ERROR.Print(logStatement)
		l.m[logStatement] = true
	}
	l.Unlock()
}

// NewDistinctErrorLogger creates a new DistinctErrorLogger
func NewDistinctErrorLogger() *DistinctErrorLogger {
	return &DistinctErrorLogger{m: make(map[string]bool)}
}

// Avoid spamming the logs with errors
var deprecatedLogger = NewDistinctErrorLogger()

// Deprecated logs ERROR logs about a deprecation, but only once for a given set of arguments' values.
func Deprecated(object, item, alternative string) {
	//	deprecatedLogger.Printf("%s's %s is deprecated and will be removed in Hugo %s. Use %s instead.", object, item, NextHugoReleaseVersion(), alternative)
	deprecatedLogger.Printf("%s's %s is deprecated and will be removed VERY SOON. Use %s instead.", object, item, alternative)

}

// SliceToLower goes through the source slice and lowers all values.
func SliceToLower(s []string) []string {
	if s == nil {
		return nil
	}

	l := make([]string, len(s))
	for i, v := range s {
		l[i] = strings.ToLower(v)
	}

	return l
}

// Md5String takes a string and returns its MD5 hash.
func Md5String(f string) string {
	h := md5.New()
	h.Write([]byte(f))
	return hex.EncodeToString(h.Sum([]byte{}))
}

// Seq creates a sequence of integers.
// It's named and used as GNU's seq.
// Examples:
// 3 => 1, 2, 3
// 1 2 4 => 1, 3
// -3 => -1, -2, -3
// 1 4 => 1, 2, 3, 4
// 1 -2 => 1, 0, -1, -2
func Seq(args ...interface{}) ([]int, error) {
	if len(args) < 1 || len(args) > 3 {
		return nil, errors.New("Seq, invalid number of args: 'first' 'increment' (optional) 'last' (optional)")
	}

	intArgs := cast.ToIntSlice(args)

	if len(intArgs) < 1 || len(intArgs) > 3 {
		return nil, errors.New("Invalid argument(s) to Seq")
	}

	var inc = 1
	var last int
	var first = intArgs[0]

	if len(intArgs) == 1 {
		last = first
		if last == 0 {
			return []int{}, nil
		} else if last > 0 {
			first = 1
		} else {
			first = -1
			inc = -1
		}
	} else if len(intArgs) == 2 {
		last = intArgs[1]
		if last < first {
			inc = -1
		}
	} else {
		inc = intArgs[1]
		last = intArgs[2]
		if inc == 0 {
			return nil, errors.New("'increment' must not be 0")
		}
		if first < last && inc < 0 {
			return nil, errors.New("'increment' must be > 0")
		}
		if first > last && inc > 0 {
			return nil, errors.New("'increment' must be < 0")
		}
	}

	// sanity check
	if last < -100000 {
		return nil, errors.New("size of result exeeds limit")
	}
	size := int(((last - first) / inc) + 1)

	// sanity check
	if size <= 0 || size > 2000 {
		return nil, errors.New("size of result exeeds limit")
	}

	seq := make([]int, size)
	val := first
	for i := 0; ; i++ {
		seq[i] = val
		val += inc
		if (inc < 0 && val < last) || (inc > 0 && val > last) {
			break
		}
	}

	return seq, nil
}

// DoArithmetic performs arithmetic operations (+,-,*,/) using reflection to
// determine the type of the two terms.
func DoArithmetic(a, b interface{}, op rune) (interface{}, error) {
	av := reflect.ValueOf(a)
	bv := reflect.ValueOf(b)
	var ai, bi int64
	var af, bf float64
	var au, bu uint64
	switch av.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		ai = av.Int()
		switch bv.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			bi = bv.Int()
		case reflect.Float32, reflect.Float64:
			af = float64(ai) // may overflow
			ai = 0
			bf = bv.Float()
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			bu = bv.Uint()
			if ai >= 0 {
				au = uint64(ai)
				ai = 0
			} else {
				bi = int64(bu) // may overflow
				bu = 0
			}
		default:
			return nil, errors.New("Can't apply the operator to the values")
		}
	case reflect.Float32, reflect.Float64:
		af = av.Float()
		switch bv.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			bf = float64(bv.Int()) // may overflow
		case reflect.Float32, reflect.Float64:
			bf = bv.Float()
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			bf = float64(bv.Uint()) // may overflow
		default:
			return nil, errors.New("Can't apply the operator to the values")
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		au = av.Uint()
		switch bv.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			bi = bv.Int()
			if bi >= 0 {
				bu = uint64(bi)
				bi = 0
			} else {
				ai = int64(au) // may overflow
				au = 0
			}
		case reflect.Float32, reflect.Float64:
			af = float64(au) // may overflow
			au = 0
			bf = bv.Float()
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			bu = bv.Uint()
		default:
			return nil, errors.New("Can't apply the operator to the values")
		}
	case reflect.String:
		as := av.String()
		if bv.Kind() == reflect.String && op == '+' {
			bs := bv.String()
			return as + bs, nil
		}
		return nil, errors.New("Can't apply the operator to the values")
	default:
		return nil, errors.New("Can't apply the operator to the values")
	}

	switch op {
	case '+':
		if ai != 0 || bi != 0 {
			return ai + bi, nil
		} else if af != 0 || bf != 0 {
			return af + bf, nil
		} else if au != 0 || bu != 0 {
			return au + bu, nil
		} else {
			return 0, nil
		}
	case '-':
		if ai != 0 || bi != 0 {
			return ai - bi, nil
		} else if af != 0 || bf != 0 {
			return af - bf, nil
		} else if au != 0 || bu != 0 {
			return au - bu, nil
		} else {
			return 0, nil
		}
	case '*':
		if ai != 0 || bi != 0 {
			return ai * bi, nil
		} else if af != 0 || bf != 0 {
			return af * bf, nil
		} else if au != 0 || bu != 0 {
			return au * bu, nil
		} else {
			return 0, nil
		}
	case '/':
		if bi != 0 {
			return ai / bi, nil
		} else if bf != 0 {
			return af / bf, nil
		} else if bu != 0 {
			return au / bu, nil
		} else {
			return nil, errors.New("Can't divide the value by 0")
		}
	default:
		return nil, errors.New("There is no such an operation")
	}
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	_ "fmt"
	"log"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type LockedAdminStats struct {
	sync.Mutex
	LoggerAdminStats
}

type LogData struct {
	sync.Mutex
	esIndex elasticsearch.IIndex
	id      int
}

const schema = "LogData2"

type LoggerService struct {
	stats   LockedAdminStats
	logData LogData
}

func (logger *LoggerService) Init(sys *piazza.SystemConfig, esIndex elasticsearch.IIndex) {
	var err error

	logger.stats.CreatedOn = time.Now()
	/***
	err = esIndex.Delete()
	if err != nil {
		log.Fatal(err)
	}
	if esIndex.IndexExists() {
		log.Fatal("index still exists")
	}
	err = esIndex.Create()
	if err != nil {
		log.Fatal(err)
	}
	***/

	if !esIndex.IndexExists() {
		err = esIndex.Create("")
		if err != nil {
			log.Fatal(err)
		}

		mapping :=
			`{
			"LogData2":{
				"properties":{
					"service":{
						"type": "string",
						"store": true
					},
					"address":{
						"type": "string",
						"store": true
					},
					"createdOn":{
						"type": "date",
						"store": true
					},
					"severity":{
						"type": "string",
						"store": true
					},
					"message":{
						"type": "string",
						"store": true
					}
				}
			}
		}`

		err = esIndex.SetMapping(schema, piazza.JsonString(mapping))
		if err != nil {
			log.Fatal(err)
		}
	}

	logger.logData.esIndex = esIndex
}

func (logger *LoggerService) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}

	err := resp.SetType()
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusInternalServerError, Message: err.Error()}
	}

	return resp
}

func (logger *LoggerService) PostMessage(mssg *Message) *piazza.JsonResponse {
	err := mssg.Validate()
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusBadRequest, Message: err.Error()}
	}

	logger.logData.Lock()
	idStr := strconv.Itoa(logger.logData.id)
	logger.logData.id++
	logger.logData.Unlock()

	_, err = logger.logData.esIndex.PostData(schema, idStr, mssg)
	if err != nil {
		//log.Printf("POST failed (1): %#v %#v", err, indexResult)
		return &piazza.JsonResponse{
			StatusCode: http.StatusInternalServerError,
			Message:    err.Error(),
		}
	}

	/*	if !indexResult.Created {
		log.Printf("POST failed (2): %#v", *indexResult)
		resp := &piazza.JsonResponse{
			StatusCode: http.StatusInternalServerError,
			Message:    "POST of log data failed",
		}
		return resp
	}*/

	logger.stats.LoggerAdminStats.NumMessages++

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       mssg,
	}

	err = resp.SetType()
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusInternalServerError, Message: err.Error()}
	}

	return resp
}

func (logger *LoggerService) GetStats() *piazza.JsonResponse {
	logger.logData.Lock()
	t := logger.stats.LoggerAdminStats
	logger.logData.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}

	err := resp.SetType()
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusInternalServerError, Message: err.Error()}
	}

	return resp
}

func (logger *LoggerService) GetMessage(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	var formalPagination *piazza.JsonPagination
	{
		defaults := &piazza.JsonPagination{
			PerPage: 10,
			Page:    0,
			Order:   piazza.PaginationOrderDescending,
			SortBy:  "createdOn",
		}
		formalPagination, err = piazza.NewJsonPagination(params, defaults)
		if err != nil {
			return &piazza.JsonResponse{StatusCode: http.StatusBadRequest, Message: err.Error()}
		}
	}

	filterParams := logger.parseFilterParams(params)

	//log.Printf("size %d, from %d, key %s, format %v",
	//	format.Size, format.From, format.Key, format.Order)

	//log.Printf("filterParams: %v\n", filterParams)

	var searchResult *elasticsearch.SearchResult

	if len(filterParams) == 0 {
		searchResult, err = logger.logData.esIndex.FilterByMatchAll(schema, formalPagination)
	} else {
		var jsonString = logger.createQueryDslAsString(formalPagination, filterParams)
		searchResult, err = logger.logData.esIndex.SearchByJSON(schema, jsonString)
	}

	if err != nil {
		resp := &piazza.JsonResponse{
			StatusCode: http.StatusInternalServerError,
			Message:    "query failed: " + err.Error(),
		}
		return resp
	}

	// TODO: unsafe truncation
	count := int(searchResult.TotalHits())
	matched := int(searchResult.NumberMatched())
	lines := make([]Message, count)

	i := 0
	for _, hit := range *searchResult.GetHits() {
		if hit == nil {
			log.Printf("null source hit")
			continue
		}
		src := *hit.Source
		//log.Printf("source hit: %s", string(src))

		tmp := &Message{}
		err = json.Unmarshal(src, tmp)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			resp := &piazza.JsonResponse{
				StatusCode: http.StatusInternalServerError,
				Message:    "unmarshall failed: " + err.Error(),
			}
			return resp
		}

		// still needed?
		err = tmp.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}

		lines[i] = *tmp
		i++
	}

	bar := make([]interface{}, len(lines))

	for i, e := range lines {
		bar[i] = e
	}

	formalPagination.Count = matched
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       bar,
		Pagination: formalPagination,
	}

	err = resp.SetType()
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusInternalServerError, Message: err.Error()}
	}

	return resp
}

func (logger *LoggerService) parseFilterParams(params *piazza.HttpQueryParams) map[string]interface{} {

	var filterParams = map[string]interface{}{}

	before := params.Get("before")

	if before != "" {
		num, err := strconv.Atoi(before)
		if err == nil {
			filterParams["before"] = num
		}
	}

	after := params.Get("after")

	if after != "" {
		num, err := strconv.Atoi(after)
		if err == nil {
			filterParams["after"] = num
		}
	}

	service := params.Get("service")

	if service != "" {
		filterParams["service"] = service
	}

	contains := params.Get("contains")

	if contains != "" {
		filterParams["contains"] = contains
	}

	return filterParams
}

func (logger *LoggerService) createQueryDslAsString(
	format *piazza.JsonPagination,
	params map[string]interface{},
) string {
	// fmt.Printf("%d\n", len(params))

	must := []map[string]interface{}{}

	if params["service"] != nil {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"service": params["service"],
			},
		})
	}

	if params["contains"] != nil {
		must = append(must, map[string]interface{}{
			"multi_match": map[string]interface{}{
				"query":  params["contains"],
				"fields": []string{"address", "message", "service", "severity"},
			},
		})
	}

	if params["after"] != nil || params["before"] != nil {
		rangeParams := map[string]int{}

		if params["after"] != nil {
			rangeParams["gte"] = params["after"].(int)
		}

		if params["before"] != nil {
			rangeParams["lte"] = params["before"].(int)
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"stamp": rangeParams,
			},
		})
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": format.PerPage,
		"from": format.PerPage * format.Page,
	}

	dsl["sort"] = map[string]string{
		format.SortBy: string(format.Order),
	}

	output, _ := json.Marshal(dsl)
	return string(output)
}
<commit_msg>bumped schema number - debugging<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	_ "fmt"
	"log"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type LockedAdminStats struct {
	sync.Mutex
	LoggerAdminStats
}

type LogData struct {
	sync.Mutex
	esIndex elasticsearch.IIndex
	id      int
}

const schema = "LogData3"

type LoggerService struct {
	stats   LockedAdminStats
	logData LogData
}

func (logger *LoggerService) Init(sys *piazza.SystemConfig, esIndex elasticsearch.IIndex) {
	var err error

	logger.stats.CreatedOn = time.Now()
	/***
	err = esIndex.Delete()
	if err != nil {
		log.Fatal(err)
	}
	if esIndex.IndexExists() {
		log.Fatal("index still exists")
	}
	err = esIndex.Create()
	if err != nil {
		log.Fatal(err)
	}
	***/

	if !esIndex.IndexExists() {
		err = esIndex.Create("")
		if err != nil {
			log.Fatal(err)
		}

		mapping :=
			`{
			"LogData3":{
				"properties":{
					"service":{
						"type": "string",
						"store": true
					},
					"address":{
						"type": "string",
						"store": true
					},
					"createdOn":{
						"type": "date",
						"store": true
					},
					"severity":{
						"type": "string",
						"store": true
					},
					"message":{
						"type": "string",
						"store": true
					}
				}
			}
		}`

		err = esIndex.SetMapping(schema, piazza.JsonString(mapping))
		if err != nil {
			log.Fatal(err)
		}
	}

	logger.logData.esIndex = esIndex
}

func (logger *LoggerService) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}

	err := resp.SetType()
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusInternalServerError, Message: err.Error()}
	}

	return resp
}

func (logger *LoggerService) PostMessage(mssg *Message) *piazza.JsonResponse {
	err := mssg.Validate()
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusBadRequest, Message: err.Error()}
	}

	logger.logData.Lock()
	idStr := strconv.Itoa(logger.logData.id)
	logger.logData.id++
	logger.logData.Unlock()

	_, err = logger.logData.esIndex.PostData(schema, idStr, mssg)
	if err != nil {
		//log.Printf("POST failed (1): %#v %#v", err, indexResult)
		return &piazza.JsonResponse{
			StatusCode: http.StatusInternalServerError,
			Message:    err.Error(),
		}
	}

	/*	if !indexResult.Created {
		log.Printf("POST failed (2): %#v", *indexResult)
		resp := &piazza.JsonResponse{
			StatusCode: http.StatusInternalServerError,
			Message:    "POST of log data failed",
		}
		return resp
	}*/

	logger.stats.LoggerAdminStats.NumMessages++

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       mssg,
	}

	err = resp.SetType()
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusInternalServerError, Message: err.Error()}
	}

	return resp
}

func (logger *LoggerService) GetStats() *piazza.JsonResponse {
	logger.logData.Lock()
	t := logger.stats.LoggerAdminStats
	logger.logData.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}

	err := resp.SetType()
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusInternalServerError, Message: err.Error()}
	}

	return resp
}

func (logger *LoggerService) GetMessage(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	var formalPagination *piazza.JsonPagination
	{
		defaults := &piazza.JsonPagination{
			PerPage: 10,
			Page:    0,
			Order:   piazza.PaginationOrderDescending,
			SortBy:  "createdOn",
		}
		formalPagination, err = piazza.NewJsonPagination(params, defaults)
		if err != nil {
			return &piazza.JsonResponse{StatusCode: http.StatusBadRequest, Message: err.Error()}
		}
	}

	filterParams := logger.parseFilterParams(params)

	//log.Printf("size %d, from %d, key %s, format %v",
	//	format.Size, format.From, format.Key, format.Order)

	//log.Printf("filterParams: %v\n", filterParams)

	var searchResult *elasticsearch.SearchResult

	if len(filterParams) == 0 {
		searchResult, err = logger.logData.esIndex.FilterByMatchAll(schema, formalPagination)
	} else {
		var jsonString = logger.createQueryDslAsString(formalPagination, filterParams)
		searchResult, err = logger.logData.esIndex.SearchByJSON(schema, jsonString)
	}

	if err != nil {
		resp := &piazza.JsonResponse{
			StatusCode: http.StatusInternalServerError,
			Message:    "query failed: " + err.Error(),
		}
		return resp
	}

	// TODO: unsafe truncation
	count := int(searchResult.TotalHits())
	matched := int(searchResult.NumberMatched())
	lines := make([]Message, count)

	i := 0
	for _, hit := range *searchResult.GetHits() {
		if hit == nil {
			log.Printf("null source hit")
			continue
		}
		src := *hit.Source
		//log.Printf("source hit: %s", string(src))

		tmp := &Message{}
		err = json.Unmarshal(src, tmp)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			resp := &piazza.JsonResponse{
				StatusCode: http.StatusInternalServerError,
				Message:    "unmarshall failed: " + err.Error(),
			}
			return resp
		}

		// still needed?
		err = tmp.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}

		lines[i] = *tmp
		i++
	}

	bar := make([]interface{}, len(lines))

	for i, e := range lines {
		bar[i] = e
	}

	formalPagination.Count = matched
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       bar,
		Pagination: formalPagination,
	}

	err = resp.SetType()
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusInternalServerError, Message: err.Error()}
	}

	return resp
}

func (logger *LoggerService) parseFilterParams(params *piazza.HttpQueryParams) map[string]interface{} {

	var filterParams = map[string]interface{}{}

	before := params.Get("before")

	if before != "" {
		num, err := strconv.Atoi(before)
		if err == nil {
			filterParams["before"] = num
		}
	}

	after := params.Get("after")

	if after != "" {
		num, err := strconv.Atoi(after)
		if err == nil {
			filterParams["after"] = num
		}
	}

	service := params.Get("service")

	if service != "" {
		filterParams["service"] = service
	}

	contains := params.Get("contains")

	if contains != "" {
		filterParams["contains"] = contains
	}

	return filterParams
}

func (logger *LoggerService) createQueryDslAsString(
	format *piazza.JsonPagination,
	params map[string]interface{},
) string {
	// fmt.Printf("%d\n", len(params))

	must := []map[string]interface{}{}

	if params["service"] != nil {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"service": params["service"],
			},
		})
	}

	if params["contains"] != nil {
		must = append(must, map[string]interface{}{
			"multi_match": map[string]interface{}{
				"query":  params["contains"],
				"fields": []string{"address", "message", "service", "severity"},
			},
		})
	}

	if params["after"] != nil || params["before"] != nil {
		rangeParams := map[string]int{}

		if params["after"] != nil {
			rangeParams["gte"] = params["after"].(int)
		}

		if params["before"] != nil {
			rangeParams["lte"] = params["before"].(int)
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"stamp": rangeParams,
			},
		})
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": format.PerPage,
		"from": format.PerPage * format.Page,
	}

	dsl["sort"] = map[string]string{
		format.SortBy: string(format.Order),
	}

	output, _ := json.Marshal(dsl)
	return string(output)
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2014 AT&T
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package impl

import (
	"errors"
	"os"
	"fmt"
	"time"
	"strings"

	"code.google.com/p/gogoprotobuf/proto"
	"github.com/golang/glog"

	"github.com/att-innovate/charmander-scheduler/communication"
	"github.com/att-innovate/charmander-scheduler/mesosproto"
	"github.com/att-innovate/charmander-scheduler/scheduler"
	"github.com/att-innovate/charmander-scheduler/upid"

	managerInterface "github.com/att-innovate/charmander-scheduler/manager"
)

var taskRegistry = NewTaskRegistry()
var nodeRegistry = NewNodeRegistry()

func New(
	scheduler *scheduler.Scheduler,
	frameworkName string,
	master string,
	listening string) (manager, error) {

	framework := &mesosproto.FrameworkInfo{
		User: proto.String(""),
		Name: proto.String(frameworkName),
	}

	newManager := manager{
		scheduler:     scheduler,
		frameworkInfo: framework,
		master:        master,
		listening:     listening,
	}

	return newManager, nil
}

type manager struct {
	scheduler     *scheduler.Scheduler
	frameworkInfo *mesosproto.FrameworkInfo
	masterUPID    *upid.UPID
	selfUPID      *upid.UPID
	frameworkId   string
	master        string
	listening     string
}

func (self *manager) Start() error {
	if communication.MesosMasterReachable(self.master) == false {
		return errors.New("Mesos unreachable")
	}

	self.frameworkInfo.User = proto.String("root")

	// set default hostname
	if self.frameworkInfo.GetHostname() == "" {
		host, err := os.Hostname()
		if err != nil || host == "" {
			host = "unknown"
		}
		self.frameworkInfo.Hostname = proto.String(host)
	}

	if m, err := upid.Parse("master@" + self.master); err != nil {
		return err
	} else {
		self.masterUPID = m
	}

	self.selfUPID= &upid.UPID{
		ID: "scheduler",
		Host: self.GetListenerIP(),
		Port: fmt.Sprintf("%d", self.GetListenerPortForScheduler())}

	communication.InitRestHandler(self)

	self.announceFramework()

	return nil
}

func (self *manager) GetListenerIP() string {
	return self.listening
}

func (self *manager) GetListenerPortForScheduler() int {
	return 7070
}

func (self *manager) GetListenerPortForRESTApi() int {
	return 7075
}

func (self *manager) GetTaskRequests() []*managerInterface.Task {
	return taskRegistry.Tasks()
}

func (self *manager) ResourceRequirementsWouldMatch(offer *mesosproto.Offer, taskRequest *managerInterface.Task) bool {
	for _, resource := range offer.GetResources() {
		switch {
		case *resource.Name == "cpus":
			if resource.Scalar.GetValue() < float64(taskRequest.Cpus) { return false }
		case *resource.Name == "mem":
			if resource.Scalar.GetValue() < float64(taskRequest.Mem) { return false }
		}
	}
	for _, attribute := range offer.GetAttributes() {
		switch {
		case *attribute.Name == "nodetype":
			if len(taskRequest.NodeType) > 0 && attribute.Text.GetValue() != taskRequest.NodeType { return false }
		case *attribute.Name == "nodename":
			if len(taskRequest.NodeName) > 0 && attribute.Text.GetValue() != taskRequest.NodeName { return false }
		}
	}

	return true
}

func (self *manager) HandleFrameworkRegistered(frameworkId string) {
	self.frameworkId = frameworkId
	if self.scheduler != nil && self.scheduler.Registered != nil {
		self.scheduler.Registered(self, frameworkId)
	}
}

func (self *manager) HandleResourceOffered(offers []*mesosproto.Offer) {
	self.updateNodeRegistry(offers)

	offers= self.enforceSLAs(offers)

	if self.scheduler != nil && self.scheduler.ResourceOffers != nil {
		self.scheduler.ResourceOffers(self, offers)
	}
}

func (self *manager) updateNodeRegistry(offers []*mesosproto.Offer) {
	for _, offer := range offers {
		slaveID := offer.GetSlaveId().GetValue()
		if nodeRegistry.Exists(slaveID) {
			nodeRegistry.UpdateTimeOfLastOffer(slaveID)
			continue
		}

		node := &managerInterface.Node {
			ID: slaveID,
			Hostname: offer.GetHostname(),
			TimeOfLastOffer: time.Now().Unix(),
		}

		for _, attribute := range offer.GetAttributes() {
			if *attribute.Name == "nodename" {
				node.NodeName = attribute.Text.GetValue()
			} else if *attribute.Name == "nodetype" {
				node.NodeType = attribute.Text.GetValue()
			}
		}

		nodeRegistry.Register(slaveID, node)

		glog.Infoln("New Node Registered: ", node)
	}
}

func (self *manager) enforceSLAs(offers []*mesosproto.Offer) []*mesosproto.Offer {
	result := offers
	var taskRequests []*managerInterface.Task
	taskRequests = self.GetTaskRequests()

	for _, taskRequest := range taskRequests {
		if taskRequest.Running { continue }
		if len(taskRequest.Sla) == 0 {continue }

		if taskRequest.Sla == managerInterface.SLA_ONE_PER_NODE {
			for _, offer := range offers {
				if resolveNodeName(*offer) == taskRequest.NodeName {
					if self.ResourceRequirementsWouldMatch(offer, taskRequest) {
						self.AcceptOffer(offer.GetId(), offer.SlaveId, taskRequest)
						result = removeOfferFromList(result, offer)
					}
				}
			}
		}
	}

	return result
}

func resolveNodeName(offer mesosproto.Offer) string {
	for _, attribute := range offer.GetAttributes() {
		if *attribute.Name == "nodename" {
			return attribute.Text.GetValue()
		}
	}

	return "notfound"
}

func removeOfferFromList(offers []*mesosproto.Offer, offer *mesosproto.Offer) []*mesosproto.Offer {
	numberOfOffers := len(offers)
	if numberOfOffers == 1 { return []*mesosproto.Offer {}}

	result := make([]*mesosproto.Offer, numberOfOffers-1)
	i :=  0
	for _, value := range offers {
		if offer.GetId().GetValue() == value.GetId().GetValue() { continue }
		result[i] = value
		i++
	}

	return result
}

func (self *manager) HandleStatusMessage(statusMessage *mesosproto.StatusUpdateMessage) {
	glog.Infof("Status Update %v\n", statusMessage)
	status := statusMessage.GetUpdate().GetStatus()

	switch {
	case  *status.State == mesosproto.TaskState_TASK_RUNNING:
		task, _ := taskRegistry.Fetch(status.GetTaskId().GetValue())
		task.Running= true
		task.SlaveID= status.GetSlaveId().GetValue()
		glog.Infof("Running: %s on Slave: %s", task.InternalID, task.SlaveID)
	case  *status.State == mesosproto.TaskState_TASK_FAILED:
		taskRegistry.Delete(status.GetTaskId().GetValue())
		glog.Infoln("Task Failed: ", status.GetTaskId().GetValue())
	case  *status.State == mesosproto.TaskState_TASK_LOST:
		taskRegistry.Delete(status.GetTaskId().GetValue())
		glog.Infoln("Task Lost: ", status.GetTaskId().GetValue())
	case  *status.State == mesosproto.TaskState_TASK_FINISHED:
		taskRegistry.Delete(status.GetTaskId().GetValue())
		glog.Infoln("Task Finished: ", status.GetTaskId().GetValue())
	}

	self.acknowledgeStatusUpdate(statusMessage)
}

func (self *manager) HandleRunDockerImage(task *managerInterface.Task) {
	if task.Sla == managerInterface.SLA_ONE_PER_NODE {
		for _, node := range nodeRegistry.Nodes() {
			newTask := managerInterface.CopyTask(*task)
			newTask.NodeName= node.NodeName
			self.handleRunDockerImageImpl(&newTask)
		}
	} else {
		self.handleRunDockerImageImpl(task)
	}
}

func (self *manager) handleRunDockerImageImpl(task *managerInterface.Task) {
	id := fmt.Sprintf("%v-%v", strings.Replace(task.ID, " ", "", -1), time.Now().UnixNano())
	memory := float64(task.Mem)
	portResources := []*mesosproto.Value_Range{}
	cpus := float64(0.1)
	if task.Cpus > 0 {
		cpus= task.Cpus
	}

	dockerInfo := &mesosproto.ContainerInfo_DockerInfo {
		Image: &task.DockerImage,
	}
	containerInfo := &mesosproto.ContainerInfo {
		Type: mesosproto.ContainerInfo_DOCKER.Enum(),
		Docker: dockerInfo,
	}
	for _, volume := range task.Volumes {
		mode := mesosproto.Volume_RW
		if volume.Mode == "ro" {
			mode = mesosproto.Volume_RO
		}

		containerInfo.Volumes = append(containerInfo.Volumes, &mesosproto.Volume{
				ContainerPath: &volume.ContainerPath,
				HostPath:      &volume.HostPath,
				Mode:          &mode,
			})
	}
	for _, port := range task.Ports {
		dockerInfo.PortMappings = append(dockerInfo.PortMappings, &mesosproto.ContainerInfo_DockerInfo_PortMapping{
				ContainerPort: &port.ContainerPort,
				HostPort:      &port.HostPort,
			})
		portResources = append(portResources, &mesosproto.Value_Range{
				Begin: proto.Uint64(uint64(port.HostPort)),
				End:   proto.Uint64(uint64(port.HostPort)),
			})
	}
	if len(task.Ports) > 0 {
		dockerInfo.Network= mesosproto.ContainerInfo_DockerInfo_BRIDGE.Enum()
	}

	commandInfo := &mesosproto.CommandInfo{
		Shell: proto.Bool(false),
	}
	if len(task.Arguments) > 0 {
		commandInfo.Arguments = task.Arguments
	}
	resources := [] *mesosproto.Resource {
		&mesosproto.Resource{
			Name: proto.String("mem"),
			Scalar: &mesosproto.Value_Scalar{Value: &memory},
			Type: mesosproto.Value_SCALAR.Enum(),
		},
		&mesosproto.Resource{
			Name: proto.String("cpus"),
			Scalar: &mesosproto.Value_Scalar{Value: &cpus},
			Type: mesosproto.Value_SCALAR.Enum(),
		},
	}
	if len(task.Ports) > 0 {
		resources = append(resources,
			&mesosproto.Resource{
				Name: proto.String("ports"),
				Ranges: &mesosproto.Value_Ranges{ Range: portResources},
				Type: mesosproto.Value_RANGES.Enum(),
			},
		)
	}

	taskInfo := &mesosproto.TaskInfo {
		Name: &id,
		TaskId: &mesosproto.TaskID{Value: &id},
		Container: containerInfo,
		Command: commandInfo,
		Resources: resources,
	}

	task.InternalID = id
	task.CreatedAt = time.Now().Unix()
	task.TaskInfo = taskInfo
	task.RequestSent = false

	glog.Infoln("Task: ", task)

	taskRegistry.Register(id, task)

}

func (self *manager) announceFramework() {
	message := &mesosproto.RegisterFrameworkMessage{
		Framework: self.frameworkInfo,
	}

	glog.Infof("Registering with master %s [%s] ", self.masterUPID, message)
	messagePackage := communication.NewMessage(self.masterUPID, message, nil)
	if err := communication.SendMessageToMesos(self.selfUPID, messagePackage); err != nil {
		glog.Errorf("Failed to send RegisterFramework message: %v\n", err)
	}

}

func (self *manager) AcceptOffer(offerId *mesosproto.OfferID, slaveId *mesosproto.SlaveID, taskRequest *managerInterface.Task) {
	glog.Infoln("Working on: ", taskRequest.TaskInfo)
	taskRequest.TaskInfo.SlaveId= slaveId
	message := &mesosproto.LaunchTasksMessage{
		FrameworkId: &mesosproto.FrameworkID{Value: &self.frameworkId},
		OfferIds:    []*mesosproto.OfferID{offerId},
		Tasks:       []*mesosproto.TaskInfo{taskRequest.TaskInfo},
		Filters:     &mesosproto.Filters{},
	}

	messagePackage := communication.NewMessage(self.masterUPID, message, nil)
	if err := communication.SendMessageToMesos(self.selfUPID, messagePackage); err != nil {
		glog.Errorf("Failed to send AcceptOffer message: %v\n", err)
	} else {
		taskRequest.RequestSent = true
	}

}

func (self *manager) DeclineOffer(offerId *mesosproto.OfferID) {
	message := &mesosproto.LaunchTasksMessage{
		FrameworkId: &mesosproto.FrameworkID{Value: &self.frameworkId},
		OfferIds:    []*mesosproto.OfferID{offerId},
		Tasks:       []*mesosproto.TaskInfo{},
		Filters:     &mesosproto.Filters{},
	}

	messagePackage := communication.NewMessage(self.masterUPID, message, nil)
	if err := communication.SendMessageToMesos(self.selfUPID, messagePackage); err != nil {
		glog.Errorf("Failed to send DeclineOffer message: %v\n", err)
	}

}

func (self *manager) acknowledgeStatusUpdate(statusUpdate *mesosproto.StatusUpdateMessage) {
	message := &mesosproto.StatusUpdateAcknowledgementMessage{
		FrameworkId: statusUpdate.GetUpdate().FrameworkId,
		SlaveId:     statusUpdate.GetUpdate().Status.SlaveId,
		TaskId:      statusUpdate.GetUpdate().Status.TaskId,
		Uuid:        statusUpdate.GetUpdate().Uuid,
	}

	messagePackage := communication.NewMessage(self.masterUPID, message, nil)
	if err := communication.SendMessageToMesos(self.selfUPID, messagePackage); err != nil {
		glog.Errorf("Failed to send StatusAccept message: %v\n", err)
	}

}
<commit_msg>Added initial retry logic for mesos connection<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2014 AT&T
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package impl

import (
	"errors"
	"os"
	"fmt"
	"time"
	"strings"

	"code.google.com/p/gogoprotobuf/proto"
	"github.com/golang/glog"

	"github.com/att-innovate/charmander-scheduler/communication"
	"github.com/att-innovate/charmander-scheduler/mesosproto"
	"github.com/att-innovate/charmander-scheduler/scheduler"
	"github.com/att-innovate/charmander-scheduler/upid"

	managerInterface "github.com/att-innovate/charmander-scheduler/manager"
)

var taskRegistry = NewTaskRegistry()
var nodeRegistry = NewNodeRegistry()

func New(
	scheduler *scheduler.Scheduler,
	frameworkName string,
	master string,
	listening string) (manager, error) {

	framework := &mesosproto.FrameworkInfo{
		User: proto.String(""),
		Name: proto.String(frameworkName),
	}

	newManager := manager{
		scheduler:     scheduler,
		frameworkInfo: framework,
		master:        master,
		listening:     listening,
	}

	return newManager, nil
}

type manager struct {
	scheduler     *scheduler.Scheduler
	frameworkInfo *mesosproto.FrameworkInfo
	masterUPID    *upid.UPID
	selfUPID      *upid.UPID
	frameworkId   string
	master        string
	listening     string
}

func (self *manager) Start() error {
	retryCounter := 10

	for ; retryCounter > 0; retryCounter-- {
		if communication.MesosMasterReachable(self.master) { break }
		time.Sleep(6 * time.Second)
	}

	if retryCounter == 0 {
		return errors.New("Mesos unreachable")
	}

	self.frameworkInfo.User = proto.String("root")

	// set default hostname
	if self.frameworkInfo.GetHostname() == "" {
		host, err := os.Hostname()
		if err != nil || host == "" {
			host = "unknown"
		}
		self.frameworkInfo.Hostname = proto.String(host)
	}

	if m, err := upid.Parse("master@" + self.master); err != nil {
		return err
	} else {
		self.masterUPID = m
	}

	self.selfUPID= &upid.UPID{
		ID: "scheduler",
		Host: self.GetListenerIP(),
		Port: fmt.Sprintf("%d", self.GetListenerPortForScheduler())}

	communication.InitRestHandler(self)

	self.announceFramework()

	return nil
}

func (self *manager) GetListenerIP() string {
	return self.listening
}

func (self *manager) GetListenerPortForScheduler() int {
	return 7070
}

func (self *manager) GetListenerPortForRESTApi() int {
	return 7075
}

func (self *manager) GetTaskRequests() []*managerInterface.Task {
	return taskRegistry.Tasks()
}

func (self *manager) ResourceRequirementsWouldMatch(offer *mesosproto.Offer, taskRequest *managerInterface.Task) bool {
	for _, resource := range offer.GetResources() {
		switch {
		case *resource.Name == "cpus":
			if resource.Scalar.GetValue() < float64(taskRequest.Cpus) { return false }
		case *resource.Name == "mem":
			if resource.Scalar.GetValue() < float64(taskRequest.Mem) { return false }
		}
	}
	for _, attribute := range offer.GetAttributes() {
		switch {
		case *attribute.Name == "nodetype":
			if len(taskRequest.NodeType) > 0 && attribute.Text.GetValue() != taskRequest.NodeType { return false }
		case *attribute.Name == "nodename":
			if len(taskRequest.NodeName) > 0 && attribute.Text.GetValue() != taskRequest.NodeName { return false }
		}
	}

	return true
}

func (self *manager) HandleFrameworkRegistered(frameworkId string) {
	self.frameworkId = frameworkId
	if self.scheduler != nil && self.scheduler.Registered != nil {
		self.scheduler.Registered(self, frameworkId)
	}
}

func (self *manager) HandleResourceOffered(offers []*mesosproto.Offer) {
	self.updateNodeRegistry(offers)

	offers= self.enforceSLAs(offers)

	if self.scheduler != nil && self.scheduler.ResourceOffers != nil {
		self.scheduler.ResourceOffers(self, offers)
	}
}

func (self *manager) updateNodeRegistry(offers []*mesosproto.Offer) {
	for _, offer := range offers {
		slaveID := offer.GetSlaveId().GetValue()
		if nodeRegistry.Exists(slaveID) {
			nodeRegistry.UpdateTimeOfLastOffer(slaveID)
			continue
		}

		node := &managerInterface.Node {
			ID: slaveID,
			Hostname: offer.GetHostname(),
			TimeOfLastOffer: time.Now().Unix(),
		}

		for _, attribute := range offer.GetAttributes() {
			if *attribute.Name == "nodename" {
				node.NodeName = attribute.Text.GetValue()
			} else if *attribute.Name == "nodetype" {
				node.NodeType = attribute.Text.GetValue()
			}
		}

		nodeRegistry.Register(slaveID, node)

		glog.Infoln("New Node Registered: ", node)
	}
}

func (self *manager) enforceSLAs(offers []*mesosproto.Offer) []*mesosproto.Offer {
	result := offers
	var taskRequests []*managerInterface.Task
	taskRequests = self.GetTaskRequests()

	for _, taskRequest := range taskRequests {
		if taskRequest.Running { continue }
		if len(taskRequest.Sla) == 0 {continue }

		if taskRequest.Sla == managerInterface.SLA_ONE_PER_NODE {
			for _, offer := range offers {
				if resolveNodeName(*offer) == taskRequest.NodeName {
					if self.ResourceRequirementsWouldMatch(offer, taskRequest) {
						self.AcceptOffer(offer.GetId(), offer.SlaveId, taskRequest)
						result = removeOfferFromList(result, offer)
					}
				}
			}
		}
	}

	return result
}

func resolveNodeName(offer mesosproto.Offer) string {
	for _, attribute := range offer.GetAttributes() {
		if *attribute.Name == "nodename" {
			return attribute.Text.GetValue()
		}
	}

	return "notfound"
}

func removeOfferFromList(offers []*mesosproto.Offer, offer *mesosproto.Offer) []*mesosproto.Offer {
	numberOfOffers := len(offers)
	if numberOfOffers == 1 { return []*mesosproto.Offer {}}

	result := make([]*mesosproto.Offer, numberOfOffers-1)
	i :=  0
	for _, value := range offers {
		if offer.GetId().GetValue() == value.GetId().GetValue() { continue }
		result[i] = value
		i++
	}

	return result
}

func (self *manager) HandleStatusMessage(statusMessage *mesosproto.StatusUpdateMessage) {
	glog.Infof("Status Update %v\n", statusMessage)
	status := statusMessage.GetUpdate().GetStatus()

	switch {
	case  *status.State == mesosproto.TaskState_TASK_RUNNING:
		task, _ := taskRegistry.Fetch(status.GetTaskId().GetValue())
		task.Running= true
		task.SlaveID= status.GetSlaveId().GetValue()
		glog.Infof("Running: %s on Slave: %s", task.InternalID, task.SlaveID)
	case  *status.State == mesosproto.TaskState_TASK_FAILED:
		taskRegistry.Delete(status.GetTaskId().GetValue())
		glog.Infoln("Task Failed: ", status.GetTaskId().GetValue())
	case  *status.State == mesosproto.TaskState_TASK_LOST:
		taskRegistry.Delete(status.GetTaskId().GetValue())
		glog.Infoln("Task Lost: ", status.GetTaskId().GetValue())
	case  *status.State == mesosproto.TaskState_TASK_FINISHED:
		taskRegistry.Delete(status.GetTaskId().GetValue())
		glog.Infoln("Task Finished: ", status.GetTaskId().GetValue())
	}

	self.acknowledgeStatusUpdate(statusMessage)
}

func (self *manager) HandleRunDockerImage(task *managerInterface.Task) {
	if task.Sla == managerInterface.SLA_ONE_PER_NODE {
		for _, node := range nodeRegistry.Nodes() {
			newTask := managerInterface.CopyTask(*task)
			newTask.NodeName= node.NodeName
			self.handleRunDockerImageImpl(&newTask)
		}
	} else {
		self.handleRunDockerImageImpl(task)
	}
}

func (self *manager) handleRunDockerImageImpl(task *managerInterface.Task) {
	id := fmt.Sprintf("%v-%v", strings.Replace(task.ID, " ", "", -1), time.Now().UnixNano())
	memory := float64(task.Mem)
	portResources := []*mesosproto.Value_Range{}
	cpus := float64(0.1)
	if task.Cpus > 0 {
		cpus= task.Cpus
	}

	dockerInfo := &mesosproto.ContainerInfo_DockerInfo {
		Image: &task.DockerImage,
	}
	containerInfo := &mesosproto.ContainerInfo {
		Type: mesosproto.ContainerInfo_DOCKER.Enum(),
		Docker: dockerInfo,
	}
	for _, volume := range task.Volumes {
		mode := mesosproto.Volume_RW
		if volume.Mode == "ro" {
			mode = mesosproto.Volume_RO
		}

		containerInfo.Volumes = append(containerInfo.Volumes, &mesosproto.Volume{
				ContainerPath: &volume.ContainerPath,
				HostPath:      &volume.HostPath,
				Mode:          &mode,
			})
	}
	for _, port := range task.Ports {
		dockerInfo.PortMappings = append(dockerInfo.PortMappings, &mesosproto.ContainerInfo_DockerInfo_PortMapping{
				ContainerPort: &port.ContainerPort,
				HostPort:      &port.HostPort,
			})
		portResources = append(portResources, &mesosproto.Value_Range{
				Begin: proto.Uint64(uint64(port.HostPort)),
				End:   proto.Uint64(uint64(port.HostPort)),
			})
	}
	if len(task.Ports) > 0 {
		dockerInfo.Network= mesosproto.ContainerInfo_DockerInfo_BRIDGE.Enum()
	}

	commandInfo := &mesosproto.CommandInfo{
		Shell: proto.Bool(false),
	}
	if len(task.Arguments) > 0 {
		for _, argument := range task.Arguments {
			commandInfo.Arguments = append(commandInfo.Arguments, argument)
		}
	}
	resources := [] *mesosproto.Resource {
		&mesosproto.Resource{
			Name: proto.String("mem"),
			Scalar: &mesosproto.Value_Scalar{Value: &memory},
			Type: mesosproto.Value_SCALAR.Enum(),
		},
		&mesosproto.Resource{
			Name: proto.String("cpus"),
			Scalar: &mesosproto.Value_Scalar{Value: &cpus},
			Type: mesosproto.Value_SCALAR.Enum(),
		},
	}
	if len(task.Ports) > 0 {
		resources = append(resources,
			&mesosproto.Resource{
				Name: proto.String("ports"),
				Ranges: &mesosproto.Value_Ranges{ Range: portResources},
				Type: mesosproto.Value_RANGES.Enum(),
			},
		)
	}

	taskInfo := &mesosproto.TaskInfo {
		Name: &id,
		TaskId: &mesosproto.TaskID{Value: &id},
		Container: containerInfo,
		Command: commandInfo,
		Resources: resources,
	}

	task.InternalID = id
	task.CreatedAt = time.Now().Unix()
	task.TaskInfo = taskInfo
	task.RequestSent = false

	glog.Infoln("Task: ", task)

	taskRegistry.Register(id, task)

}

func (self *manager) announceFramework() {
	message := &mesosproto.RegisterFrameworkMessage{
		Framework: self.frameworkInfo,
	}

	glog.Infof("Registering with master %s [%s] ", self.masterUPID, message)
	messagePackage := communication.NewMessage(self.masterUPID, message, nil)
	if err := communication.SendMessageToMesos(self.selfUPID, messagePackage); err != nil {
		glog.Errorf("Failed to send RegisterFramework message: %v\n", err)
	}

}

func (self *manager) AcceptOffer(offerId *mesosproto.OfferID, slaveId *mesosproto.SlaveID, taskRequest *managerInterface.Task) {
	glog.Infoln("Working on: ", taskRequest.TaskInfo)
	taskRequest.TaskInfo.SlaveId= slaveId
	message := &mesosproto.LaunchTasksMessage{
		FrameworkId: &mesosproto.FrameworkID{Value: &self.frameworkId},
		OfferIds:    []*mesosproto.OfferID{offerId},
		Tasks:       []*mesosproto.TaskInfo{taskRequest.TaskInfo},
		Filters:     &mesosproto.Filters{},
	}

	messagePackage := communication.NewMessage(self.masterUPID, message, nil)
	if err := communication.SendMessageToMesos(self.selfUPID, messagePackage); err != nil {
		glog.Errorf("Failed to send AcceptOffer message: %v\n", err)
	} else {
		taskRequest.RequestSent = true
	}

}

func (self *manager) DeclineOffer(offerId *mesosproto.OfferID) {
	message := &mesosproto.LaunchTasksMessage{
		FrameworkId: &mesosproto.FrameworkID{Value: &self.frameworkId},
		OfferIds:    []*mesosproto.OfferID{offerId},
		Tasks:       []*mesosproto.TaskInfo{},
		Filters:     &mesosproto.Filters{},
	}

	messagePackage := communication.NewMessage(self.masterUPID, message, nil)
	if err := communication.SendMessageToMesos(self.selfUPID, messagePackage); err != nil {
		glog.Errorf("Failed to send DeclineOffer message: %v\n", err)
	}

}

func (self *manager) acknowledgeStatusUpdate(statusUpdate *mesosproto.StatusUpdateMessage) {
	message := &mesosproto.StatusUpdateAcknowledgementMessage{
		FrameworkId: statusUpdate.GetUpdate().FrameworkId,
		SlaveId:     statusUpdate.GetUpdate().Status.SlaveId,
		TaskId:      statusUpdate.GetUpdate().Status.TaskId,
		Uuid:        statusUpdate.GetUpdate().Uuid,
	}

	messagePackage := communication.NewMessage(self.masterUPID, message, nil)
	if err := communication.SendMessageToMesos(self.selfUPID, messagePackage); err != nil {
		glog.Errorf("Failed to send StatusAccept message: %v\n", err)
	}

}
<|endoftext|>


<commit_before>//  Copyright (c) 2012 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package rest

import (
	"encoding/json"
	"fmt"
	"net/http"
	"testing"

	"github.com/couchbase/sg-bucket"
	"github.com/couchbase/sync_gateway/channels"

	"github.com/couchbaselabs/go.assert"
)

func TestDesignDocs(t *testing.T) {
	var rt restTester
	response := rt.sendRequest("GET", "/db/_design/foo", "")
	assertStatus(t, response, 403)
	response = rt.sendRequest("PUT", "/db/_design/foo", `{"prop":true}`)
	assertStatus(t, response, 403)
	response = rt.sendRequest("DELETE", "/db/_design/foo", "")
	assertStatus(t, response, 403)

	response = rt.sendAdminRequest("GET", "/db/_design/foo", "")
	assertStatus(t, response, 404)
	response = rt.sendAdminRequest("PUT", "/db/_design/foo", `{"prop":true}`)
	assertStatus(t, response, 201)
	response = rt.sendAdminRequest("GET", "/db/_design/foo", "")
	assertStatus(t, response, 200)
	response = rt.sendAdminRequest("GET", "/db/_design%2ffoo", "")
	assertStatus(t, response, 200)
	response = rt.sendAdminRequest("GET", "/db/_design%2Ffoo", "")
	assertStatus(t, response, 200)
	response = rt.sendAdminRequest("DELETE", "/db/_design/foo", "")
	assertStatus(t, response, 200)
	response = rt.sendAdminRequest("PUT", "/db/_design/sync_gateway", "{}")
	assertStatus(t, response, 403)
	response = rt.sendAdminRequest("GET", "/db/_design/sync_gateway", "")
	assertStatus(t, response, 200)
}

func TestViewQuery(t *testing.T) {
	var rt restTester
	response := rt.sendAdminRequest("PUT", "/db/_design/foo", `{"views":{"bar": {"map": "function(doc) {emit(doc.key, doc.value);}"}}}`)
	assertStatus(t, response, 201)
	response = rt.sendRequest("PUT", "/db/doc1", `{"key":10, "value":"ten"}`)
	assertStatus(t, response, 201)
	response = rt.sendRequest("PUT", "/db/doc2", `{"key":7, "value":"seven"}`)
	assertStatus(t, response, 201)

	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar", ``)
	assertStatus(t, response, 200)
	var result sgbucket.ViewResult
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 2)
	assert.DeepEquals(t, result.Rows[0], &sgbucket.ViewRow{ID: "doc2", Key: 7.0, Value: "seven"})
	assert.DeepEquals(t, result.Rows[1], &sgbucket.ViewRow{ID: "doc1", Key: 10.0, Value: "ten"})

	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar?endkey=9", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 1)
	assert.DeepEquals(t, result.Rows[0], &sgbucket.ViewRow{ID: "doc2", Key: 7.0, Value: "seven"})

	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar?endkey=9&include_docs=true", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 1)
	assert.DeepEquals(t, *result.Rows[0].Doc, map[string]interface{}{"key": 7.0, "value": "seven"})
}

//Tests #1109, where design doc contains multiple views
func failingTestViewQueryMultipleViews(t *testing.T) {
	var rt restTester
	//Define three views
	response := rt.sendAdminRequest("PUT", "/db/_design/foo", `{"views": {"by_fname": {"map": "function (doc, meta) { emit(doc.fname, null); }"},"by_lname": {"map": "function (doc, meta) { emit(doc.lname, null); }"},"by_age": {"map": "function (doc, meta) { emit(doc.age, doc); }"}}}`)
	assertStatus(t, response, 201)
	response = rt.sendRequest("PUT", "/db/doc1", `{"fname": "Alice", "lname":"Ten", "age":10}`)
	assertStatus(t, response, 201)
	response = rt.sendRequest("PUT", "/db/doc2", `{"fname": "Bob", "lname":"Seven", "age":7}`)
	assertStatus(t, response, 201)

	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/by_age", ``)
	assertStatus(t, response, 200)
	var result sgbucket.ViewResult
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 2)
	assert.DeepEquals(t, result.Rows[0], &sgbucket.ViewRow{ID: "doc2", Key: 7.0, Value: interface {}(nil)})
	assert.DeepEquals(t, result.Rows[1], &sgbucket.ViewRow{ID: "doc1", Key: 10.0, Value: interface {}(nil)})

	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/by_fname", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 2)
	assert.DeepEquals(t, result.Rows[0], &sgbucket.ViewRow{ID: "doc1", Key: "Alice" , Value: interface {}(nil)})
	assert.DeepEquals(t, result.Rows[1], &sgbucket.ViewRow{ID: "doc2", Key: "Bob" , Value: interface {}(nil)})

	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/by_lname", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 2)
	assert.DeepEquals(t, result.Rows[0], &sgbucket.ViewRow{ID: "doc2", Key: "Seven" , Value: interface {}(nil)})
	assert.DeepEquals(t, result.Rows[1], &sgbucket.ViewRow{ID: "doc1", Key: "Ten" , Value: interface {}(nil)})
}

func TestUserViewQuery(t *testing.T) {
	rt := restTester{syncFn: `function(doc) {channel(doc.channel)}`}
	a := rt.ServerContext().Database("db").Authenticator()
	// Create a view:
	response := rt.sendAdminRequest("PUT", "/db/_design/foo", `{"views":{"bar": {"map": "function(doc) {emit(doc.key, doc.value);}"}}}`)
	assertStatus(t, response, 201)
	// Create docs:
	response = rt.sendRequest("PUT", "/db/doc1", `{"key":10, "value":"ten", "channel":"W"}`)
	assertStatus(t, response, 201)
	response = rt.sendRequest("PUT", "/db/doc2", `{"key":7, "value":"seven", "channel":"Q"}`)
	assertStatus(t, response, 201)
	// Create a user:
	quinn, _ := a.NewUser("quinn", "123456", channels.SetOf("Q", "q"))
	a.Save(quinn)

	// Have the user query the view:
	request, _ := http.NewRequest("GET", "/db/_design/foo/_view/bar?include_docs=true", nil)
	request.SetBasicAuth("quinn", "123456")
	response = rt.send(request)
	assertStatus(t, response, 200)
	var result sgbucket.ViewResult
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 1)
	row := result.Rows[0]
	assert.Equals(t, row.Key, float64(7))
	assert.Equals(t, row.Value, "seven")
	assert.DeepEquals(t, *row.Doc, map[string]interface{}{"key": 7.0, "value": "seven", "channel": "Q"})

	// Admin should see both rows:
	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 2)
	row = result.Rows[0]
	assert.Equals(t, row.Key, float64(7))
	assert.Equals(t, row.Value, "seven")
	assert.DeepEquals(t, *row.Doc, map[string]interface{}{"key": 7.0, "value": "seven", "channel": "Q"})
	row = result.Rows[1]
	assert.Equals(t, row.Key, float64(10))
	assert.Equals(t, row.Value, "ten")
	assert.DeepEquals(t, *row.Doc, map[string]interface{}{"key": 10.0, "value": "ten", "channel": "W"})

	// Make sure users are not allowed to query internal views:
	request, _ = http.NewRequest("GET", "/db/_design/sync_gateway/_view/access", nil)
	request.SetBasicAuth("quinn", "123456")
	response = rt.send(request)
	assertStatus(t, response, 403)
}

// This includes a fix for #857
func TestAdminReduceViewQuery(t *testing.T) {

	rt := restTester{syncFn: `function(doc) {channel(doc.channel)}`}

	// Create a view with a reduce:
	response := rt.sendAdminRequest("PUT", "/db/_design/foo", `{"views":{"bar": {"map": "function(doc) {emit(doc.key, doc.value);}", "reduce": "_count"}}}`)
	assertStatus(t, response, 201)

	for i := 0; i < 9; i++ {
		// Create docs:
		response = rt.sendRequest("PUT", fmt.Sprintf("/db/doc%v", i), `{"key":0, "value":"0", "channel":"W"}`)
		assertStatus(t, response, 201)

	}
	response = rt.sendRequest("PUT", fmt.Sprintf("/db/doc%v", 10), `{"key":1, "value":"0", "channel":"W"}`)
	assertStatus(t, response, 201)

	var result sgbucket.ViewResult

	// Admin view query:
	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar?reduce=true", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)

	// we should get 1 row with the reduce result
	assert.Equals(t, len(result.Rows), 1)
	row := result.Rows[0]
	value := row.Value.(float64)
	assert.True(t, value == 10)

	// todo support group reduce, see #955
	// // test group=true
	// response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar?reduce=true&group=true", ``)
	// assertStatus(t, response, 200)
	// json.Unmarshal(response.Body.Bytes(), &result)
	// // we should get 2 rows with the reduce result
	// assert.Equals(t, len(result.Rows), 2)
	// row = result.Rows[0]
	// value = row.Value.(float64)
	// assert.True(t, value == 9)
	// row = result.Rows[1]
	// value = row.Value.(float64)
	// assert.True(t, value == 1)
}

func TestAdminReduceSumQuery(t *testing.T) {

	rt := restTester{syncFn: `function(doc) {channel(doc.channel)}`}

	// Create a view with a reduce:
	response := rt.sendAdminRequest("PUT", "/db/_design/foo", `{"options":{"raw":true},"views":{"bar": {"map": "function(doc) {if (doc.key && doc.value) emit(doc.key, doc.value);}", "reduce": "_sum"}}}`)
	assertStatus(t, response, 201)

	for i := 0; i < 9; i++ {
		// Create docs:
		response = rt.sendRequest("PUT", fmt.Sprintf("/db/doc%v", i), `{"key":"A", "value":1}`)
		assertStatus(t, response, 201)

	}
	response = rt.sendRequest("PUT", fmt.Sprintf("/db/doc%v", 10), `{"key":"B", "value":99}`)
	assertStatus(t, response, 201)

	var result sgbucket.ViewResult

	// Admin view query:
	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar?reduce=true", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)

	// we should get 1 row with the reduce result
	assert.Equals(t, len(result.Rows), 1)
	row := result.Rows[0]
	value := row.Value.(float64)
	assert.Equals(t, value, 108.0)
}
<commit_msg>Added comment on why failing test currently fails<commit_after>//  Copyright (c) 2012 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package rest

import (
	"encoding/json"
	"fmt"
	"net/http"
	"testing"

	"github.com/couchbase/sg-bucket"
	"github.com/couchbase/sync_gateway/channels"

	"github.com/couchbaselabs/go.assert"
)

func TestDesignDocs(t *testing.T) {
	var rt restTester
	response := rt.sendRequest("GET", "/db/_design/foo", "")
	assertStatus(t, response, 403)
	response = rt.sendRequest("PUT", "/db/_design/foo", `{"prop":true}`)
	assertStatus(t, response, 403)
	response = rt.sendRequest("DELETE", "/db/_design/foo", "")
	assertStatus(t, response, 403)

	response = rt.sendAdminRequest("GET", "/db/_design/foo", "")
	assertStatus(t, response, 404)
	response = rt.sendAdminRequest("PUT", "/db/_design/foo", `{"prop":true}`)
	assertStatus(t, response, 201)
	response = rt.sendAdminRequest("GET", "/db/_design/foo", "")
	assertStatus(t, response, 200)
	response = rt.sendAdminRequest("GET", "/db/_design%2ffoo", "")
	assertStatus(t, response, 200)
	response = rt.sendAdminRequest("GET", "/db/_design%2Ffoo", "")
	assertStatus(t, response, 200)
	response = rt.sendAdminRequest("DELETE", "/db/_design/foo", "")
	assertStatus(t, response, 200)
	response = rt.sendAdminRequest("PUT", "/db/_design/sync_gateway", "{}")
	assertStatus(t, response, 403)
	response = rt.sendAdminRequest("GET", "/db/_design/sync_gateway", "")
	assertStatus(t, response, 200)
}

func TestViewQuery(t *testing.T) {
	var rt restTester
	response := rt.sendAdminRequest("PUT", "/db/_design/foo", `{"views":{"bar": {"map": "function(doc) {emit(doc.key, doc.value);}"}}}`)
	assertStatus(t, response, 201)
	response = rt.sendRequest("PUT", "/db/doc1", `{"key":10, "value":"ten"}`)
	assertStatus(t, response, 201)
	response = rt.sendRequest("PUT", "/db/doc2", `{"key":7, "value":"seven"}`)
	assertStatus(t, response, 201)

	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar", ``)
	assertStatus(t, response, 200)
	var result sgbucket.ViewResult
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 2)
	assert.DeepEquals(t, result.Rows[0], &sgbucket.ViewRow{ID: "doc2", Key: 7.0, Value: "seven"})
	assert.DeepEquals(t, result.Rows[1], &sgbucket.ViewRow{ID: "doc1", Key: 10.0, Value: "ten"})

	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar?endkey=9", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 1)
	assert.DeepEquals(t, result.Rows[0], &sgbucket.ViewRow{ID: "doc2", Key: 7.0, Value: "seven"})

	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar?endkey=9&include_docs=true", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 1)
	assert.DeepEquals(t, *result.Rows[0].Doc, map[string]interface{}{"key": 7.0, "value": "seven"})
}

//Tests #1109, where design doc contains multiple views
//Currently fails against walrus bucket as '_sync' property will exist in doc if it is emmitted in the map function
func failingTestViewQueryMultipleViews(t *testing.T) {
	var rt restTester
	//Define three views
	response := rt.sendAdminRequest("PUT", "/db/_design/foo", `{"views": {"by_fname": {"map": "function (doc, meta) { emit(doc.fname, null); }"},"by_lname": {"map": "function (doc, meta) { emit(doc.lname, null); }"},"by_age": {"map": "function (doc, meta) { emit(doc.age, doc); }"}}}`)
	assertStatus(t, response, 201)
	response = rt.sendRequest("PUT", "/db/doc1", `{"fname": "Alice", "lname":"Ten", "age":10}`)
	assertStatus(t, response, 201)
	response = rt.sendRequest("PUT", "/db/doc2", `{"fname": "Bob", "lname":"Seven", "age":7}`)
	assertStatus(t, response, 201)

	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/by_age", ``)
	assertStatus(t, response, 200)
	var result sgbucket.ViewResult
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 2)
	assert.DeepEquals(t, result.Rows[0], &sgbucket.ViewRow{ID: "doc2", Key: 7.0, Value: interface {}(nil)})
	assert.DeepEquals(t, result.Rows[1], &sgbucket.ViewRow{ID: "doc1", Key: 10.0, Value: interface {}(nil)})

	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/by_fname", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 2)
	assert.DeepEquals(t, result.Rows[0], &sgbucket.ViewRow{ID: "doc1", Key: "Alice" , Value: interface {}(nil)})
	assert.DeepEquals(t, result.Rows[1], &sgbucket.ViewRow{ID: "doc2", Key: "Bob" , Value: interface {}(nil)})

	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/by_lname", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 2)
	assert.DeepEquals(t, result.Rows[0], &sgbucket.ViewRow{ID: "doc2", Key: "Seven" , Value: interface {}(nil)})
	assert.DeepEquals(t, result.Rows[1], &sgbucket.ViewRow{ID: "doc1", Key: "Ten" , Value: interface {}(nil)})
}

func TestUserViewQuery(t *testing.T) {
	rt := restTester{syncFn: `function(doc) {channel(doc.channel)}`}
	a := rt.ServerContext().Database("db").Authenticator()
	// Create a view:
	response := rt.sendAdminRequest("PUT", "/db/_design/foo", `{"views":{"bar": {"map": "function(doc) {emit(doc.key, doc.value);}"}}}`)
	assertStatus(t, response, 201)
	// Create docs:
	response = rt.sendRequest("PUT", "/db/doc1", `{"key":10, "value":"ten", "channel":"W"}`)
	assertStatus(t, response, 201)
	response = rt.sendRequest("PUT", "/db/doc2", `{"key":7, "value":"seven", "channel":"Q"}`)
	assertStatus(t, response, 201)
	// Create a user:
	quinn, _ := a.NewUser("quinn", "123456", channels.SetOf("Q", "q"))
	a.Save(quinn)

	// Have the user query the view:
	request, _ := http.NewRequest("GET", "/db/_design/foo/_view/bar?include_docs=true", nil)
	request.SetBasicAuth("quinn", "123456")
	response = rt.send(request)
	assertStatus(t, response, 200)
	var result sgbucket.ViewResult
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 1)
	row := result.Rows[0]
	assert.Equals(t, row.Key, float64(7))
	assert.Equals(t, row.Value, "seven")
	assert.DeepEquals(t, *row.Doc, map[string]interface{}{"key": 7.0, "value": "seven", "channel": "Q"})

	// Admin should see both rows:
	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)
	assert.Equals(t, len(result.Rows), 2)
	row = result.Rows[0]
	assert.Equals(t, row.Key, float64(7))
	assert.Equals(t, row.Value, "seven")
	assert.DeepEquals(t, *row.Doc, map[string]interface{}{"key": 7.0, "value": "seven", "channel": "Q"})
	row = result.Rows[1]
	assert.Equals(t, row.Key, float64(10))
	assert.Equals(t, row.Value, "ten")
	assert.DeepEquals(t, *row.Doc, map[string]interface{}{"key": 10.0, "value": "ten", "channel": "W"})

	// Make sure users are not allowed to query internal views:
	request, _ = http.NewRequest("GET", "/db/_design/sync_gateway/_view/access", nil)
	request.SetBasicAuth("quinn", "123456")
	response = rt.send(request)
	assertStatus(t, response, 403)
}

// This includes a fix for #857
func TestAdminReduceViewQuery(t *testing.T) {

	rt := restTester{syncFn: `function(doc) {channel(doc.channel)}`}

	// Create a view with a reduce:
	response := rt.sendAdminRequest("PUT", "/db/_design/foo", `{"views":{"bar": {"map": "function(doc) {emit(doc.key, doc.value);}", "reduce": "_count"}}}`)
	assertStatus(t, response, 201)

	for i := 0; i < 9; i++ {
		// Create docs:
		response = rt.sendRequest("PUT", fmt.Sprintf("/db/doc%v", i), `{"key":0, "value":"0", "channel":"W"}`)
		assertStatus(t, response, 201)

	}
	response = rt.sendRequest("PUT", fmt.Sprintf("/db/doc%v", 10), `{"key":1, "value":"0", "channel":"W"}`)
	assertStatus(t, response, 201)

	var result sgbucket.ViewResult

	// Admin view query:
	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar?reduce=true", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)

	// we should get 1 row with the reduce result
	assert.Equals(t, len(result.Rows), 1)
	row := result.Rows[0]
	value := row.Value.(float64)
	assert.True(t, value == 10)

	// todo support group reduce, see #955
	// // test group=true
	// response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar?reduce=true&group=true", ``)
	// assertStatus(t, response, 200)
	// json.Unmarshal(response.Body.Bytes(), &result)
	// // we should get 2 rows with the reduce result
	// assert.Equals(t, len(result.Rows), 2)
	// row = result.Rows[0]
	// value = row.Value.(float64)
	// assert.True(t, value == 9)
	// row = result.Rows[1]
	// value = row.Value.(float64)
	// assert.True(t, value == 1)
}

func TestAdminReduceSumQuery(t *testing.T) {

	rt := restTester{syncFn: `function(doc) {channel(doc.channel)}`}

	// Create a view with a reduce:
	response := rt.sendAdminRequest("PUT", "/db/_design/foo", `{"options":{"raw":true},"views":{"bar": {"map": "function(doc) {if (doc.key && doc.value) emit(doc.key, doc.value);}", "reduce": "_sum"}}}`)
	assertStatus(t, response, 201)

	for i := 0; i < 9; i++ {
		// Create docs:
		response = rt.sendRequest("PUT", fmt.Sprintf("/db/doc%v", i), `{"key":"A", "value":1}`)
		assertStatus(t, response, 201)

	}
	response = rt.sendRequest("PUT", fmt.Sprintf("/db/doc%v", 10), `{"key":"B", "value":99}`)
	assertStatus(t, response, 201)

	var result sgbucket.ViewResult

	// Admin view query:
	response = rt.sendAdminRequest("GET", "/db/_design/foo/_view/bar?reduce=true", ``)
	assertStatus(t, response, 200)
	json.Unmarshal(response.Body.Bytes(), &result)

	// we should get 1 row with the reduce result
	assert.Equals(t, len(result.Rows), 1)
	row := result.Rows[0]
	value := row.Value.(float64)
	assert.Equals(t, value, 108.0)
}
<|endoftext|>


<commit_before>package keys_test

import (
	"fmt"
	"net/http"
	"net/http/httptest"

	. "github.com/LewisWatson/Ginkgo-BDD-Testing-Framework-Lightning-Talk/step2"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Keys", func() {

	var (
		maxAge       int64
		err          error
		serverTokens map[string]interface{}
	)

	BeforeEach(func() {

		/*
		 * given
		 */

		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set(HeaderCacheControl, "..., max-age=19008, ...")
			fmt.Fprintln(w, jsonKeys)
		}))
		defer ts.Close()

		url := ts.URL

		/*
		 * when
		 */

		serverTokens, maxAge, err = GetKeys(url)
	})

	/*
	 * then
	 */

	It("should not throw an error", func() {
		Expect(err).NotTo(HaveOccurred())
	})

	It("should return max age from response header", func() {
		Expect(maxAge).To(Equal(int64(19008)))
	})

	It("should return a max age greater than 10", func() {
		Expect(maxAge).To(BeNumerically(">", 10))
	})

	It("should return 4 keys", func() {
		Expect(len(serverTokens)).To(Equal(4))
	})

	Context("response doesn't contain max age", func() {

		BeforeEach(func() {

			/*
			 * given
			 */

			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				fmt.Fprintln(w, jsonKeys)
			}))
			defer ts.Close()

			url := ts.URL

			/*
			 * when
			 */

			serverTokens, maxAge, err = GetKeys(url)
		})

		It("should return ErrCacheControlHeaderLacksMaxAge", func() {
			Expect(err).To(Equal(ErrCacheControlHeaderLacksMaxAge))
		})

	})

	Context("no server response", func() {

		BeforeEach(func() {

			/*
			 * given
			 */

			url := "notgonnaresolv.org.uk.eu"

			/*
			 * when
			 */

			serverTokens, maxAge, err = GetKeys(url)
		})

		It("should return an error", func() {
			Expect(err).To(HaveOccurred())
		})

	})

})
<commit_msg>bug fix<commit_after>package keys_test

import (
	"fmt"
	"net/http"
	"net/http/httptest"

	. "github.com/LewisWatson/Ginkgo-BDD-Testing-Framework-Lightning-Talk/step2/complete"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Keys", func() {

	var (
		maxAge       int64
		err          error
		serverTokens map[string]interface{}
	)

	BeforeEach(func() {

		/*
		 * given
		 */

		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set(HeaderCacheControl, "..., max-age=19008, ...")
			fmt.Fprintln(w, jsonKeys)
		}))
		defer ts.Close()

		url := ts.URL

		/*
		 * when
		 */

		serverTokens, maxAge, err = GetKeys(url)
	})

	/*
	 * then
	 */

	It("should not throw an error", func() {
		Expect(err).NotTo(HaveOccurred())
	})

	It("should return max age from response header", func() {
		Expect(maxAge).To(Equal(int64(19008)))
	})

	It("should return a max age greater than 10", func() {
		Expect(maxAge).To(BeNumerically(">", 10))
	})

	It("should return 4 keys", func() {
		Expect(len(serverTokens)).To(Equal(4))
	})

	Context("response doesn't contain max age", func() {

		BeforeEach(func() {

			/*
			 * given
			 */

			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				fmt.Fprintln(w, jsonKeys)
			}))
			defer ts.Close()

			url := ts.URL

			/*
			 * when
			 */

			serverTokens, maxAge, err = GetKeys(url)
		})

		It("should return ErrCacheControlHeaderLacksMaxAge", func() {
			Expect(err).To(Equal(ErrCacheControlHeaderLacksMaxAge))
		})

	})

	Context("no server response", func() {

		BeforeEach(func() {

			/*
			 * given
			 */

			url := "notgonnaresolv.org.uk.eu"

			/*
			 * when
			 */

			serverTokens, maxAge, err = GetKeys(url)
		})

		It("should return an error", func() {
			Expect(err).To(HaveOccurred())
		})

	})

})
<|endoftext|>