<commit_before>// Copyright 2015 Comcast Cable Communications Management, LLC

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"errors"
	"github.com/jmoiron/sqlx"
	"strconv"
)

type ApiMethod int

const (
	GET ApiMethod = iota
	POST
	PUT
	DELETE
	OPTIONS
)

const API_PATH = "/api/2.0/"

func (m ApiMethod) String() string {
	switch m {
	case GET:
		return "GET"
	case POST:
		return "POST"
	case PUT:
		return "PUT"
	case DELETE:
		return "DELETE"
	case OPTIONS:
		return "OPTIONS"
	}
	return "INVALID"
}

type ApiMethods []ApiMethod

// String returns a comma-separated list of the methods, as expected in headers such as Access-Control-Allow-Methods
func (methods ApiMethods) String() string {
	var s string
	for _, method := range methods {
		s += method.String() + ","
	}
	if s != "" {
		s = s[:len(s)-1] // strip trailing ,
	}
	return s
}

type ApiHandlerFunc func(pathParams map[string]string, payload []byte, dbb *sqlx.DB) (interface{}, error)
type ApiHandlerFuncMap map[ApiMethod]ApiHandlerFunc

func (handlerMap ApiHandlerFuncMap) Methods() ApiMethods {
	var methods []ApiMethod
	for method, _ := range handlerMap {
		methods = append(methods, method)
	}
	return methods
}

func ApiHandlers() map[string]ApiHandlerFuncMap {
	return map[string]ApiHandlerFuncMap{
		"cdn":                                 ApiHandlerFuncMap{GET: emptyWrap(getCdns), POST: bodyWrap(postCdn)},
		"cdn/{id}":                            ApiHandlerFuncMap{GET: idWrap(getCdnById), PUT: idBodyWrap(putCdn), DELETE: idWrap(delCdn)},
		"asn":                                 ApiHandlerFuncMap{GET: emptyWrap(getAsns), POST: bodyWrap(postAsn)},
		"asn/{id}":                            ApiHandlerFuncMap{GET: idWrap(getAsnById), PUT: idBodyWrap(putAsn), DELETE: idWrap(delAsn)},
		"cachegroup":                          ApiHandlerFuncMap{GET: emptyWrap(getCachegroups), POST: bodyWrap(postCachegroup)},
		"cachegroup/{id}":                     ApiHandlerFuncMap{GET: idWrap(getCachegroupById), PUT: idBodyWrap(putCachegroup), DELETE: idWrap(delCachegroup)},
		"cachegroup_parameter":                ApiHandlerFuncMap{GET: emptyWrap(getCachegroupParameters), POST: bodyWrap(postCachegroupParameter)},
		"cachegroup_parameter/{id}":           ApiHandlerFuncMap{GET: idWrap(getCachegroupParameterById), PUT: idBodyWrap(putCachegroupParameter), DELETE: idWrap(delCachegroupParameter)},
		"deliveryservice":                     ApiHandlerFuncMap{GET: emptyWrap(getDeliveryservices), POST: bodyWrap(postDeliveryservice)},
		"deliveryserviceRegex/{id}":           ApiHandlerFuncMap{GET: idWrap(getDeliveryserviceRegexById), PUT: idBodyWrap(putDeliveryserviceRegex), DELETE: idWrap(delDeliveryserviceRegex)},
		"deliveryservice_regex":               ApiHandlerFuncMap{GET: emptyWrap(getDeliveryserviceRegexs), POST: bodyWrap(postDeliveryserviceRegex)},
		"deliveryservice_regex/{id}":          ApiHandlerFuncMap{GET: idWrap(getDeliveryserviceRegexById), PUT: idBodyWrap(putDeliveryserviceRegex), DELETE: idWrap(delDeliveryserviceRegex)},
		"deliveryservice_server":              ApiHandlerFuncMap{GET: emptyWrap(getDeliveryserviceServers), POST: bodyWrap(postDeliveryserviceServer)},
		"deliveryservice_server/{id}":         ApiHandlerFuncMap{GET: idWrap(getDeliveryserviceServerById), PUT: idBodyWrap(putDeliveryserviceServer), DELETE: idWrap(delDeliveryserviceServer)},
		"deliveryservice_tmuser":              ApiHandlerFuncMap{GET: emptyWrap(getDeliveryserviceTmusers), POST: bodyWrap(postDeliveryserviceTmuser)},
		"deliveryservice_tmuser/{id}":         ApiHandlerFuncMap{GET: idWrap(getDeliveryserviceTmuserById), PUT: idBodyWrap(putDeliveryserviceTmuser), DELETE: idWrap(delDeliveryserviceTmuser)},
		"division":                            ApiHandlerFuncMap{GET: emptyWrap(getDivisions), POST: bodyWrap(postDivision)},
		"division/{id}":                       ApiHandlerFuncMap{GET: idWrap(getDivisionById), PUT: idBodyWrap(putDivision), DELETE: idWrap(delDivision)},
		"federation":                          ApiHandlerFuncMap{GET: emptyWrap(getFederations), POST: bodyWrap(postFederation)},
		"federation/{id}":                     ApiHandlerFuncMap{GET: idWrap(getFederationById), PUT: idBodyWrap(putFederation), DELETE: idWrap(delFederation)},
		"federation_deliveryservice":          ApiHandlerFuncMap{GET: emptyWrap(getFederationDeliveryservices), POST: bodyWrap(postFederationDeliveryservice)},
		"federation_deliveryservice/{id}":     ApiHandlerFuncMap{GET: idWrap(getFederationDeliveryserviceById), PUT: idBodyWrap(putFederationDeliveryservice), DELETE: idWrap(delFederationDeliveryservice)},
		"federation_federation_resolver":      ApiHandlerFuncMap{GET: emptyWrap(getFederationFederationResolvers), POST: bodyWrap(postFederationFederationResolver)},
		"federation_federation_resolver/{id}": ApiHandlerFuncMap{GET: idWrap(getFederationFederationResolverById), PUT: idBodyWrap(putFederationFederationResolver), DELETE: idWrap(delFederationFederationResolver)},
		"federation_resolver":                 ApiHandlerFuncMap{GET: emptyWrap(getFederationResolvers), POST: bodyWrap(postFederationResolver)},
		"federation_resolver/{id}":            ApiHandlerFuncMap{GET: idWrap(getFederationResolverById), PUT: idBodyWrap(putFederationResolver), DELETE: idWrap(delFederationResolver)},
		"federation_tmuser":                   ApiHandlerFuncMap{GET: emptyWrap(getFederationTmusers), POST: bodyWrap(postFederationTmuser)},
		"federation_tmuser/{id}":              ApiHandlerFuncMap{GET: idWrap(getFederationTmuserById), PUT: idBodyWrap(putFederationTmuser), DELETE: idWrap(delFederationTmuser)},
		"job":                    ApiHandlerFuncMap{GET: emptyWrap(getJobs), POST: bodyWrap(postJob)},
		"job/{id}":               ApiHandlerFuncMap{GET: idWrap(getJobById), PUT: idBodyWrap(putJob), DELETE: idWrap(delJob)},
		"job_result":             ApiHandlerFuncMap{GET: emptyWrap(getJobResults), POST: bodyWrap(postJobResult)},
		"job_result/{id}":        ApiHandlerFuncMap{GET: idWrap(getJobResultById), PUT: idBodyWrap(putJobResult), DELETE: idWrap(delJobResult)},
		"job_status":             ApiHandlerFuncMap{GET: emptyWrap(getJobStatuss), POST: bodyWrap(postJobStatus)},
		"job_status/{id}":        ApiHandlerFuncMap{GET: idWrap(getJobStatusById), PUT: idBodyWrap(putJobStatus), DELETE: idWrap(delJobStatus)},
		"log":                    ApiHandlerFuncMap{GET: emptyWrap(getLogs), POST: bodyWrap(postLog)},
		"log/{id}":               ApiHandlerFuncMap{GET: idWrap(getLogById), PUT: idBodyWrap(putLog), DELETE: idWrap(delLog)},
		"parameter":              ApiHandlerFuncMap{GET: emptyWrap(getParameters), POST: bodyWrap(postParameter)},
		"parameter/{id}":         ApiHandlerFuncMap{GET: idWrap(getParameterById), PUT: idBodyWrap(putParameter), DELETE: idWrap(delParameter)},
		"phys_location":          ApiHandlerFuncMap{GET: emptyWrap(getPhysLocations), POST: bodyWrap(postPhysLocation)},
		"phys_location/{id}":     ApiHandlerFuncMap{GET: idWrap(getPhysLocationById), PUT: idBodyWrap(putPhysLocation), DELETE: idWrap(delPhysLocation)},
		"profile":                ApiHandlerFuncMap{GET: emptyWrap(getProfiles), POST: bodyWrap(postProfile)},
		"profile/{id}":           ApiHandlerFuncMap{GET: idWrap(getProfileById), PUT: idBodyWrap(putProfile), DELETE: idWrap(delProfile)},
		"profile_parameter":      ApiHandlerFuncMap{GET: emptyWrap(getProfileParameters), POST: bodyWrap(postProfileParameter)},
		"profile_parameter/{id}": ApiHandlerFuncMap{GET: idWrap(getProfileParameterById), PUT: idBodyWrap(putProfileParameter), DELETE: idWrap(delProfileParameter)},
		"regex":                  ApiHandlerFuncMap{GET: emptyWrap(getRegexs), POST: bodyWrap(postRegex)},
		"regex/{id}":             ApiHandlerFuncMap{GET: idWrap(getRegexById), PUT: idBodyWrap(putRegex), DELETE: idWrap(delRegex)},
		"region":                 ApiHandlerFuncMap{GET: emptyWrap(getRegions), POST: bodyWrap(postRegion)},
		"region/{id}":            ApiHandlerFuncMap{GET: idWrap(getRegionById), PUT: idBodyWrap(putRegion), DELETE: idWrap(delRegion)},
		"role":                   ApiHandlerFuncMap{GET: emptyWrap(getRoles), POST: bodyWrap(postRole)},
		"role/{id}":              ApiHandlerFuncMap{GET: idWrap(getRoleById), PUT: idBodyWrap(putRole), DELETE: idWrap(delRole)},
		"server":                 ApiHandlerFuncMap{GET: emptyWrap(getServers), POST: bodyWrap(postServer)},
		"server/{id}":            ApiHandlerFuncMap{GET: idWrap(getServerById), PUT: idBodyWrap(putServer), DELETE: idWrap(delServer)},
		"servercheck":            ApiHandlerFuncMap{GET: emptyWrap(getServerchecks), POST: bodyWrap(postServercheck)},
		"servercheck/{id}":       ApiHandlerFuncMap{GET: idWrap(getServercheckById), PUT: idBodyWrap(putServercheck), DELETE: idWrap(delServercheck)},
		"staticdnsentry":         ApiHandlerFuncMap{GET: emptyWrap(getStaticdnsentrys), POST: bodyWrap(postStaticdnsentry)},
		"staticdnsentry/{id}":    ApiHandlerFuncMap{GET: idWrap(getStaticdnsentryById), PUT: idBodyWrap(putStaticdnsentry), DELETE: idWrap(delStaticdnsentry)},
		"stats_summary":          ApiHandlerFuncMap{GET: emptyWrap(getStatsSummarys), POST: bodyWrap(postStatsSummary)},
		"stats_summary/{id}":     ApiHandlerFuncMap{GET: idWrap(getStatsSummaryById), PUT: idBodyWrap(putStatsSummary), DELETE: idWrap(delStatsSummary)},
		"status":                 ApiHandlerFuncMap{GET: emptyWrap(getStatuss), POST: bodyWrap(postStatus)},
		"status/{id}":            ApiHandlerFuncMap{GET: idWrap(getStatusById), PUT: idBodyWrap(putStatus), DELETE: idWrap(delStatus)},
		"tm_user":                ApiHandlerFuncMap{GET: emptyWrap(getTmUsers), POST: bodyWrap(postTmUser)},
		"tm_user/{id}":           ApiHandlerFuncMap{GET: idWrap(getTmUserById), PUT: idBodyWrap(putTmUser), DELETE: idWrap(delTmUser)},
		"to_extension":           ApiHandlerFuncMap{GET: emptyWrap(getToExtensions), POST: bodyWrap(postToExtension)},
		"to_extension/{id}":      ApiHandlerFuncMap{GET: idWrap(getToExtensionById), PUT: idBodyWrap(putToExtension), DELETE: idWrap(delToExtension)},
		"type":                   ApiHandlerFuncMap{GET: emptyWrap(getTypes), POST: bodyWrap(postType)},
		"type/{id}":              ApiHandlerFuncMap{GET: idWrap(getTypeById), PUT: idBodyWrap(putType), DELETE: idWrap(delType)},
	}
}

type EmptyHandlerFunc func(db *sqlx.DB) (interface{}, error)
type IntHandlerFunc func(id int, db *sqlx.DB) (interface{}, error)
type BodyHandlerFunc func(payload []byte, db *sqlx.DB) (interface{}, error)
type IntBodyHandlerFunc func(id int, payload []byte, db *sqlx.DB) (interface{}, error)

func idBodyWrap(f IntBodyHandlerFunc) ApiHandlerFunc {
	return func(pathParams map[string]string, payload []byte, db *sqlx.DB) (interface{}, error) {
		if strid, ok := pathParams["id"]; !ok {
			return nil, errors.New("Id missing")
		} else if id, err := strconv.Atoi(strid); err != nil {
			return nil, errors.New("Id is not an integer: " + strid)
		} else {
			return f(id, payload, db)
		}
	}
}

func idWrap(f IntHandlerFunc) ApiHandlerFunc {
	return idBodyWrap(func(id int, payload []byte, db *sqlx.DB) (interface{}, error) {
		return f(id, db)
	})
}

func bodyWrap(f BodyHandlerFunc) ApiHandlerFunc {
	return func(pathParams map[string]string, payload []byte, db *sqlx.DB) (interface{}, error) {
		return f(payload, db)
	}
}

func emptyWrap(f EmptyHandlerFunc) ApiHandlerFunc {
	return func(pathParams map[string]string, payload []byte, db *sqlx.DB) (interface{}, error) {
		return f(db)
	}
}
<commit_msg>add deliveryservice/{id} route<commit_after>// Copyright 2015 Comcast Cable Communications Management, LLC

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"errors"
	"github.com/jmoiron/sqlx"
	"strconv"
)

type ApiMethod int

const (
	GET ApiMethod = iota
	POST
	PUT
	DELETE
	OPTIONS
)

const API_PATH = "/api/2.0/"

func (m ApiMethod) String() string {
	switch m {
	case GET:
		return "GET"
	case POST:
		return "POST"
	case PUT:
		return "PUT"
	case DELETE:
		return "DELETE"
	case OPTIONS:
		return "OPTIONS"
	}
	return "INVALID"
}

type ApiMethods []ApiMethod

// String returns a comma-separated list of the methods, as expected in headers such as Access-Control-Allow-Methods
func (methods ApiMethods) String() string {
	var s string
	for _, method := range methods {
		s += method.String() + ","
	}
	if s != "" {
		s = s[:len(s)-1] // strip trailing ,
	}
	return s
}

type ApiHandlerFunc func(pathParams map[string]string, payload []byte, dbb *sqlx.DB) (interface{}, error)
type ApiHandlerFuncMap map[ApiMethod]ApiHandlerFunc

func (handlerMap ApiHandlerFuncMap) Methods() ApiMethods {
	var methods []ApiMethod
	for method, _ := range handlerMap {
		methods = append(methods, method)
	}
	return methods
}

func ApiHandlers() map[string]ApiHandlerFuncMap {
	return map[string]ApiHandlerFuncMap{
		"cdn":                                 ApiHandlerFuncMap{GET: emptyWrap(getCdns), POST: bodyWrap(postCdn)},
		"cdn/{id}":                            ApiHandlerFuncMap{GET: idWrap(getCdnById), PUT: idBodyWrap(putCdn), DELETE: idWrap(delCdn)},
		"asn":                                 ApiHandlerFuncMap{GET: emptyWrap(getAsns), POST: bodyWrap(postAsn)},
		"asn/{id}":                            ApiHandlerFuncMap{GET: idWrap(getAsnById), PUT: idBodyWrap(putAsn), DELETE: idWrap(delAsn)},
		"cachegroup":                          ApiHandlerFuncMap{GET: emptyWrap(getCachegroups), POST: bodyWrap(postCachegroup)},
		"cachegroup/{id}":                     ApiHandlerFuncMap{GET: idWrap(getCachegroupById), PUT: idBodyWrap(putCachegroup), DELETE: idWrap(delCachegroup)},
		"cachegroup_parameter":                ApiHandlerFuncMap{GET: emptyWrap(getCachegroupParameters), POST: bodyWrap(postCachegroupParameter)},
		"cachegroup_parameter/{id}":           ApiHandlerFuncMap{GET: idWrap(getCachegroupParameterById), PUT: idBodyWrap(putCachegroupParameter), DELETE: idWrap(delCachegroupParameter)},
		"deliveryservice":                     ApiHandlerFuncMap{GET: emptyWrap(getDeliveryservices), POST: bodyWrap(postDeliveryservice)},
		"deliveryservice/{id}":                ApiHandlerFuncMap{GET: idWrap(getDeliveryserviceById), PUT: idBodyWrap(putDeliveryservice), DELETE: idWrap(delDeliveryservice)},
		"deliveryserviceRegex/{id}":           ApiHandlerFuncMap{GET: idWrap(getDeliveryserviceRegexById), PUT: idBodyWrap(putDeliveryserviceRegex), DELETE: idWrap(delDeliveryserviceRegex)},
		"deliveryservice_regex":               ApiHandlerFuncMap{GET: emptyWrap(getDeliveryserviceRegexs), POST: bodyWrap(postDeliveryserviceRegex)},
		"deliveryservice_regex/{id}":          ApiHandlerFuncMap{GET: idWrap(getDeliveryserviceRegexById), PUT: idBodyWrap(putDeliveryserviceRegex), DELETE: idWrap(delDeliveryserviceRegex)},
		"deliveryservice_server":              ApiHandlerFuncMap{GET: emptyWrap(getDeliveryserviceServers), POST: bodyWrap(postDeliveryserviceServer)},
		"deliveryservice_server/{id}":         ApiHandlerFuncMap{GET: idWrap(getDeliveryserviceServerById), PUT: idBodyWrap(putDeliveryserviceServer), DELETE: idWrap(delDeliveryserviceServer)},
		"deliveryservice_tmuser":              ApiHandlerFuncMap{GET: emptyWrap(getDeliveryserviceTmusers), POST: bodyWrap(postDeliveryserviceTmuser)},
		"deliveryservice_tmuser/{id}":         ApiHandlerFuncMap{GET: idWrap(getDeliveryserviceTmuserById), PUT: idBodyWrap(putDeliveryserviceTmuser), DELETE: idWrap(delDeliveryserviceTmuser)},
		"division":                            ApiHandlerFuncMap{GET: emptyWrap(getDivisions), POST: bodyWrap(postDivision)},
		"division/{id}":                       ApiHandlerFuncMap{GET: idWrap(getDivisionById), PUT: idBodyWrap(putDivision), DELETE: idWrap(delDivision)},
		"federation":                          ApiHandlerFuncMap{GET: emptyWrap(getFederations), POST: bodyWrap(postFederation)},
		"federation/{id}":                     ApiHandlerFuncMap{GET: idWrap(getFederationById), PUT: idBodyWrap(putFederation), DELETE: idWrap(delFederation)},
		"federation_deliveryservice":          ApiHandlerFuncMap{GET: emptyWrap(getFederationDeliveryservices), POST: bodyWrap(postFederationDeliveryservice)},
		"federation_deliveryservice/{id}":     ApiHandlerFuncMap{GET: idWrap(getFederationDeliveryserviceById), PUT: idBodyWrap(putFederationDeliveryservice), DELETE: idWrap(delFederationDeliveryservice)},
		"federation_federation_resolver":      ApiHandlerFuncMap{GET: emptyWrap(getFederationFederationResolvers), POST: bodyWrap(postFederationFederationResolver)},
		"federation_federation_resolver/{id}": ApiHandlerFuncMap{GET: idWrap(getFederationFederationResolverById), PUT: idBodyWrap(putFederationFederationResolver), DELETE: idWrap(delFederationFederationResolver)},
		"federation_resolver":                 ApiHandlerFuncMap{GET: emptyWrap(getFederationResolvers), POST: bodyWrap(postFederationResolver)},
		"federation_resolver/{id}":            ApiHandlerFuncMap{GET: idWrap(getFederationResolverById), PUT: idBodyWrap(putFederationResolver), DELETE: idWrap(delFederationResolver)},
		"federation_tmuser":                   ApiHandlerFuncMap{GET: emptyWrap(getFederationTmusers), POST: bodyWrap(postFederationTmuser)},
		"federation_tmuser/{id}":              ApiHandlerFuncMap{GET: idWrap(getFederationTmuserById), PUT: idBodyWrap(putFederationTmuser), DELETE: idWrap(delFederationTmuser)},
		"job":                    ApiHandlerFuncMap{GET: emptyWrap(getJobs), POST: bodyWrap(postJob)},
		"job/{id}":               ApiHandlerFuncMap{GET: idWrap(getJobById), PUT: idBodyWrap(putJob), DELETE: idWrap(delJob)},
		"job_result":             ApiHandlerFuncMap{GET: emptyWrap(getJobResults), POST: bodyWrap(postJobResult)},
		"job_result/{id}":        ApiHandlerFuncMap{GET: idWrap(getJobResultById), PUT: idBodyWrap(putJobResult), DELETE: idWrap(delJobResult)},
		"job_status":             ApiHandlerFuncMap{GET: emptyWrap(getJobStatuss), POST: bodyWrap(postJobStatus)},
		"job_status/{id}":        ApiHandlerFuncMap{GET: idWrap(getJobStatusById), PUT: idBodyWrap(putJobStatus), DELETE: idWrap(delJobStatus)},
		"log":                    ApiHandlerFuncMap{GET: emptyWrap(getLogs), POST: bodyWrap(postLog)},
		"log/{id}":               ApiHandlerFuncMap{GET: idWrap(getLogById), PUT: idBodyWrap(putLog), DELETE: idWrap(delLog)},
		"parameter":              ApiHandlerFuncMap{GET: emptyWrap(getParameters), POST: bodyWrap(postParameter)},
		"parameter/{id}":         ApiHandlerFuncMap{GET: idWrap(getParameterById), PUT: idBodyWrap(putParameter), DELETE: idWrap(delParameter)},
		"phys_location":          ApiHandlerFuncMap{GET: emptyWrap(getPhysLocations), POST: bodyWrap(postPhysLocation)},
		"phys_location/{id}":     ApiHandlerFuncMap{GET: idWrap(getPhysLocationById), PUT: idBodyWrap(putPhysLocation), DELETE: idWrap(delPhysLocation)},
		"profile":                ApiHandlerFuncMap{GET: emptyWrap(getProfiles), POST: bodyWrap(postProfile)},
		"profile/{id}":           ApiHandlerFuncMap{GET: idWrap(getProfileById), PUT: idBodyWrap(putProfile), DELETE: idWrap(delProfile)},
		"profile_parameter":      ApiHandlerFuncMap{GET: emptyWrap(getProfileParameters), POST: bodyWrap(postProfileParameter)},
		"profile_parameter/{id}": ApiHandlerFuncMap{GET: idWrap(getProfileParameterById), PUT: idBodyWrap(putProfileParameter), DELETE: idWrap(delProfileParameter)},
		"regex":                  ApiHandlerFuncMap{GET: emptyWrap(getRegexs), POST: bodyWrap(postRegex)},
		"regex/{id}":             ApiHandlerFuncMap{GET: idWrap(getRegexById), PUT: idBodyWrap(putRegex), DELETE: idWrap(delRegex)},
		"region":                 ApiHandlerFuncMap{GET: emptyWrap(getRegions), POST: bodyWrap(postRegion)},
		"region/{id}":            ApiHandlerFuncMap{GET: idWrap(getRegionById), PUT: idBodyWrap(putRegion), DELETE: idWrap(delRegion)},
		"role":                   ApiHandlerFuncMap{GET: emptyWrap(getRoles), POST: bodyWrap(postRole)},
		"role/{id}":              ApiHandlerFuncMap{GET: idWrap(getRoleById), PUT: idBodyWrap(putRole), DELETE: idWrap(delRole)},
		"server":                 ApiHandlerFuncMap{GET: emptyWrap(getServers), POST: bodyWrap(postServer)},
		"server/{id}":            ApiHandlerFuncMap{GET: idWrap(getServerById), PUT: idBodyWrap(putServer), DELETE: idWrap(delServer)},
		"servercheck":            ApiHandlerFuncMap{GET: emptyWrap(getServerchecks), POST: bodyWrap(postServercheck)},
		"servercheck/{id}":       ApiHandlerFuncMap{GET: idWrap(getServercheckById), PUT: idBodyWrap(putServercheck), DELETE: idWrap(delServercheck)},
		"staticdnsentry":         ApiHandlerFuncMap{GET: emptyWrap(getStaticdnsentrys), POST: bodyWrap(postStaticdnsentry)},
		"staticdnsentry/{id}":    ApiHandlerFuncMap{GET: idWrap(getStaticdnsentryById), PUT: idBodyWrap(putStaticdnsentry), DELETE: idWrap(delStaticdnsentry)},
		"stats_summary":          ApiHandlerFuncMap{GET: emptyWrap(getStatsSummarys), POST: bodyWrap(postStatsSummary)},
		"stats_summary/{id}":     ApiHandlerFuncMap{GET: idWrap(getStatsSummaryById), PUT: idBodyWrap(putStatsSummary), DELETE: idWrap(delStatsSummary)},
		"status":                 ApiHandlerFuncMap{GET: emptyWrap(getStatuss), POST: bodyWrap(postStatus)},
		"status/{id}":            ApiHandlerFuncMap{GET: idWrap(getStatusById), PUT: idBodyWrap(putStatus), DELETE: idWrap(delStatus)},
		"tm_user":                ApiHandlerFuncMap{GET: emptyWrap(getTmUsers), POST: bodyWrap(postTmUser)},
		"tm_user/{id}":           ApiHandlerFuncMap{GET: idWrap(getTmUserById), PUT: idBodyWrap(putTmUser), DELETE: idWrap(delTmUser)},
		"to_extension":           ApiHandlerFuncMap{GET: emptyWrap(getToExtensions), POST: bodyWrap(postToExtension)},
		"to_extension/{id}":      ApiHandlerFuncMap{GET: idWrap(getToExtensionById), PUT: idBodyWrap(putToExtension), DELETE: idWrap(delToExtension)},
		"type":                   ApiHandlerFuncMap{GET: emptyWrap(getTypes), POST: bodyWrap(postType)},
		"type/{id}":              ApiHandlerFuncMap{GET: idWrap(getTypeById), PUT: idBodyWrap(putType), DELETE: idWrap(delType)},
	}
}

type EmptyHandlerFunc func(db *sqlx.DB) (interface{}, error)
type IntHandlerFunc func(id int, db *sqlx.DB) (interface{}, error)
type BodyHandlerFunc func(payload []byte, db *sqlx.DB) (interface{}, error)
type IntBodyHandlerFunc func(id int, payload []byte, db *sqlx.DB) (interface{}, error)

func idBodyWrap(f IntBodyHandlerFunc) ApiHandlerFunc {
	return func(pathParams map[string]string, payload []byte, db *sqlx.DB) (interface{}, error) {
		if strid, ok := pathParams["id"]; !ok {
			return nil, errors.New("Id missing")
		} else if id, err := strconv.Atoi(strid); err != nil {
			return nil, errors.New("Id is not an integer: " + strid)
		} else {
			return f(id, payload, db)
		}
	}
}

func idWrap(f IntHandlerFunc) ApiHandlerFunc {
	return idBodyWrap(func(id int, payload []byte, db *sqlx.DB) (interface{}, error) {
		return f(id, db)
	})
}

func bodyWrap(f BodyHandlerFunc) ApiHandlerFunc {
	return func(pathParams map[string]string, payload []byte, db *sqlx.DB) (interface{}, error) {
		return f(payload, db)
	}
}

func emptyWrap(f EmptyHandlerFunc) ApiHandlerFunc {
	return func(pathParams map[string]string, payload []byte, db *sqlx.DB) (interface{}, error) {
		return f(db)
	}
}
<|endoftext|>


<commit_before>package main

import (
	R "gorules"
	"net/http"

	"gopkg.in/gin-gonic/gin.v1"
)

// Message - structure of data sent to rules api.
type Message struct {
	Rule   interface{}            `form:"rule" json:"rule" binding:"required"`
	Action interface{}            `form:"action" json:"action"`
	Data   map[string]interface{} `form:"data" json:"data" binding:"required"`
}

func main() {
	router := gin.Default()

	router.POST("/rules", func(c *gin.Context) {
		var json Message
		if c.BindJSON(&json) == nil {
			result := R.NewRuleProcessor(json.Rule).Process(json.Data)
			c.JSON(http.StatusOK, gin.H{"status": result})
		}

		c.JSON(http.StatusOK, gin.H{"status": "Invalid data"})
	})

	router.GET("/status", func(c *gin.Context) {
		c.JSON(http.StatusOK, gin.H{"status": "Service running"})
	})

	router.GET("/operators/math", func(c *gin.Context) {
		operators := R.MathOperatorList()
		c.JSON(http.StatusOK, gin.H{"operators": operators})
	})

	router.GET("/operators/string", func(c *gin.Context) {
		operators := R.StringOperatorList()
		c.JSON(http.StatusOK, gin.H{"operators": operators})
	})

	router.Run(":8080")
}
<commit_msg>added apis for exposing opeartor list<commit_after>package main

import (
	R "gorules"
	"net/http"

	"fmt"

	"gopkg.in/gin-gonic/gin.v1"
)

// RuleMessage - structure of data sent to rules api.
type RuleMessage struct {
	Rule   interface{}            `form:"rule" json:"rule" binding:"required"`
	Action interface{}            `form:"action" json:"action"`
	Data   map[string]interface{} `form:"data" json:"data" binding:"required"`
}

// ExpressionMessage - structure for evaluating online expressions
type ExpressionMessage struct {
	Expression interface{}            `form:"expression" json:"expression" binding:"required"`
	Data       map[string]interface{} `form:"data" json:"data"`
}

func main() {
	router := gin.Default()

	router.GET("/status", func(c *gin.Context) {
		c.JSON(http.StatusOK, gin.H{"status": "Service running"})
	})

	router.GET("/operators/math", func(c *gin.Context) {
		operators := R.MathOperatorList()
		c.JSON(http.StatusOK, gin.H{"operators": operators})
	})

	router.GET("/operators/string", func(c *gin.Context) {
		operators := R.StringOperatorList()
		c.JSON(http.StatusOK, gin.H{"operators": operators})
	})

	router.POST("/rule", func(c *gin.Context) {
		var json RuleMessage
		if c.BindJSON(&json) == nil {
			result := R.NewRuleProcessor(json.Rule).Process(json.Data)
			c.JSON(http.StatusOK, gin.H{"status": result})
		}

		// c.JSON(http.StatusOK, gin.H{"status": "Invalid data"})
	})

	router.POST("/expression", func(c *gin.Context) {
		var json ExpressionMessage
		if c.BindJSON(&json) == nil {
			result, _ := R.NewValue(json.Expression.(string)).Evaluate(json.Data)
			fmt.Println("json", json.Expression, result)
			c.JSON(http.StatusOK, gin.H{"result": result})
		}

		// c.JSON(http.StatusOK, gin.H{"status": "Invalid data"})
	})

	router.Run(":8080")
}
<|endoftext|>


<commit_before>package api

import (
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/VoltFramework/volt/mesoslib"
	"github.com/VoltFramework/volt/mesosproto"
	"github.com/elazarl/go-bindata-assetfs"
	"github.com/gorilla/mux"
)

type API struct {
	sync.RWMutex

	m   *mesoslib.MesosLib
	log *logrus.Logger

	tasks  []*Task
	states map[string]*mesosproto.TaskState
}

func NewAPI(m *mesoslib.MesosLib) *API {
	return &API{
		m:      m,
		tasks:  make([]*Task, 0),
		states: make(map[string]*mesosproto.TaskState, 0),
	}
}

// Simple _ping endpoint, returns OK
func (api *API) _ping(w http.ResponseWriter, r *http.Request) {
	io.WriteString(w, "OK")
}

type Task struct {
	ID      string   `json:"id"`
	Command string   `json:"cmd"`
	Cpus    float64  `json:"cpus,string"`
	Mem     float64  `json:"mem,string"`
	Files   []string `json:"files"`

	SlaveId *string               `json:"slave_id",string`
	State   *mesosproto.TaskState `json:"state,string"`
}

func (api *API) writeError(w http.ResponseWriter, code int, message string) {
	api.m.Log.Warn(message)
	w.WriteHeader(code)
	data := struct {
		Code    int    `json:"code"`
		Message string `json:"message"`
	}{
		code,
		message,
	}
	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(&data); err != nil {
		api.writeError(w, http.StatusInternalServerError, err.Error())
	}
}

// Enpoint to call to add a new task
func (api *API) tasksAdd(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()
	var (
		defaultState mesosproto.TaskState = mesosproto.TaskState_TASK_STAGING
		task                              = Task{State: &defaultState}
	)

	if err := json.NewDecoder(r.Body).Decode(&task); err != nil {
		api.writeError(w, http.StatusBadRequest, err.Error())
		return
	}

	id := make([]byte, 6)
	n, err := rand.Read(id)
	if n != len(id) || err != nil {
		api.writeError(w, http.StatusInternalServerError, err.Error())
		return
	}
	task.ID = hex.EncodeToString(id)
	api.Lock()
	api.tasks = append(api.tasks, &task)
	api.states[task.ID] = task.State
	api.Unlock()

	f := func() error {
		offer, resources, err := api.m.RequestOffer(task.Cpus, task.Mem)
		if err != nil {
			return err
		}
		if offer != nil {
			task.SlaveId = offer.SlaveId.Value
			return api.m.LaunchTask(offer, resources, task.Command+" > volt_stdout 2> volt_stderr", task.ID)
		}
		return fmt.Errorf("No offer available")
	}
	if len(task.Files) > 0 {
		if err := f(); err != nil {
			api.writeError(w, http.StatusInternalServerError, err.Error())
			return
		}
		files, err := api.m.ReadFile(task.ID, task.Files...)
		if err != nil {
			api.writeError(w, http.StatusInternalServerError, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		if err := json.NewEncoder(w).Encode(files); err != nil {
			api.writeError(w, http.StatusInternalServerError, err.Error())
		}
	} else {
		go f()
		w.WriteHeader(http.StatusAccepted)
		io.WriteString(w, "OK")
	}
}

// Endpoint to list all the tasks
func (api *API) tasksList(w http.ResponseWriter, r *http.Request) {
	api.RLock()
	data := struct {
		Size  int     `json:"size"`
		Tasks []*Task `json:"tasks"`
	}{
		len(api.tasks),
		api.tasks,
	}
	api.RUnlock()
	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(&data); err != nil {
		api.writeError(w, http.StatusInternalServerError, err.Error())
	}
}

// Endpoint to delete a task
func (api *API) tasksDelete(w http.ResponseWriter, r *http.Request) {
	var (
		vars   = mux.Vars(r)
		id     = vars["id"]
		tasks  = make([]*Task, 0)
		states = make(map[string]*mesosproto.TaskState, len(api.states)-1)
	)

	api.Lock()
	for _, task := range api.tasks {
		if task != nil && task.ID != id {
			tasks = append(tasks, task)
			states[task.ID] = task.State
		}
	}
	api.tasks = tasks
	api.states = states
	api.Unlock()
	io.WriteString(w, "OK")
}

// Endpoint to kill a task
func (api *API) tasksKill(w http.ResponseWriter, r *http.Request) {
	var (
		vars = mux.Vars(r)
		id   = vars["id"]
	)
	if err := api.m.KillTask(id); err != nil {
		api.writeError(w, http.StatusInternalServerError, err.Error())
	} else {
		io.WriteString(w, "OK")
	}
}

func (api *API) getFile(w http.ResponseWriter, r *http.Request) {
	var (
		vars = mux.Vars(r)
		id   = vars["id"]
		file = vars["file"]
	)

	files, err := api.m.ReadFile(id, []string{file}...)
	if err != nil {
		api.writeError(w, http.StatusInternalServerError, err.Error())
		return
	}
	content, ok := files[file]
	if !ok {
		api.writeError(w, http.StatusNotFound, file+" not found")
		return
	}
	io.WriteString(w, content)
}

func (api *API) handleStates() {
	for {
		event := <-api.m.GetEvent(mesosproto.Event_UPDATE)
		ID := event.Update.Status.TaskId.GetValue()

		state, ok := api.states[ID]
		if !ok {
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Warn("Update received for unknown task.")
			continue
		}

		*state = *event.Update.Status.State
		switch *event.Update.Status.State {
		case mesosproto.TaskState_TASK_STAGING:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Info("Task was registered.")
		case mesosproto.TaskState_TASK_STARTING:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Info("Task is starting.")
		case mesosproto.TaskState_TASK_RUNNING:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Info("Task is running.")
		case mesosproto.TaskState_TASK_FINISHED:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Info("Task is finished.")
		case mesosproto.TaskState_TASK_FAILED:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Warn("Task has failed.")
		case mesosproto.TaskState_TASK_KILLED:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Warn("Task was killed.")
		case mesosproto.TaskState_TASK_LOST:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Warn("Task was lost.")
		}
	}
}

// Register all the routes and then serve the API
func (api *API) ListenAndServe(port int) error {
	r := mux.NewRouter()
	api.m.Log.WithFields(logrus.Fields{"port": port}).Info("Starting API...")

	endpoints := map[string]map[string]func(w http.ResponseWriter, r *http.Request){
		"DELETE": {
			"/tasks/{id}": api.tasksDelete,
		},
		"GET": {
			"/_ping":                  api._ping,
			"/tasks/{id}/file/{file}": api.getFile,
			"/tasks":                  api.tasksList,
		},
		"POST": {
			"/tasks": api.tasksAdd,
		},
		"PUT": {
			"/tasks/{id}/kill": api.tasksKill,
		},
	}

	for method, routes := range endpoints {
		for route, fct := range routes {
			_route := route
			_fct := fct
			_method := method

			api.m.Log.WithFields(logrus.Fields{"method": _method, "route": _route}).Debug("Registering API route...")
			r.Path(_route).Methods(_method).HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				api.m.Log.WithFields(logrus.Fields{"from": r.RemoteAddr}).Infof("[%s] %s", _method, _route)
				_fct(w, r)
			})
		}
	}
	r.PathPrefix("/").Handler(http.FileServer(&assetfs.AssetFS{Asset, AssetDir, "./static/"}))
	go api.handleStates()
	return http.ListenAndServe(fmt.Sprintf(":%d", port), r)
}
<commit_msg>delete also kills<commit_after>package api

import (
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/VoltFramework/volt/mesoslib"
	"github.com/VoltFramework/volt/mesosproto"
	"github.com/elazarl/go-bindata-assetfs"
	"github.com/gorilla/mux"
)

type API struct {
	sync.RWMutex

	m   *mesoslib.MesosLib
	log *logrus.Logger

	tasks  []*Task
	states map[string]*mesosproto.TaskState
}

func NewAPI(m *mesoslib.MesosLib) *API {
	return &API{
		m:      m,
		tasks:  make([]*Task, 0),
		states: make(map[string]*mesosproto.TaskState, 0),
	}
}

// Simple _ping endpoint, returns OK
func (api *API) _ping(w http.ResponseWriter, r *http.Request) {
	io.WriteString(w, "OK")
}

type Task struct {
	ID      string   `json:"id"`
	Command string   `json:"cmd"`
	Cpus    float64  `json:"cpus,string"`
	Mem     float64  `json:"mem,string"`
	Files   []string `json:"files"`

	SlaveId *string               `json:"slave_id",string`
	State   *mesosproto.TaskState `json:"state,string"`
}

func (api *API) writeError(w http.ResponseWriter, code int, message string) {
	api.m.Log.Warn(message)
	w.WriteHeader(code)
	data := struct {
		Code    int    `json:"code"`
		Message string `json:"message"`
	}{
		code,
		message,
	}
	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(&data); err != nil {
		api.writeError(w, http.StatusInternalServerError, err.Error())
	}
}

// Enpoint to call to add a new task
func (api *API) tasksAdd(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()
	var (
		defaultState mesosproto.TaskState = mesosproto.TaskState_TASK_STAGING
		task                              = Task{State: &defaultState}
	)

	if err := json.NewDecoder(r.Body).Decode(&task); err != nil {
		api.writeError(w, http.StatusBadRequest, err.Error())
		return
	}

	id := make([]byte, 6)
	n, err := rand.Read(id)
	if n != len(id) || err != nil {
		api.writeError(w, http.StatusInternalServerError, err.Error())
		return
	}
	task.ID = hex.EncodeToString(id)
	api.Lock()
	api.tasks = append(api.tasks, &task)
	api.states[task.ID] = task.State
	api.Unlock()

	f := func() error {
		offer, resources, err := api.m.RequestOffer(task.Cpus, task.Mem)
		if err != nil {
			return err
		}
		if offer != nil {
			task.SlaveId = offer.SlaveId.Value
			return api.m.LaunchTask(offer, resources, task.Command+" > volt_stdout 2> volt_stderr", task.ID)
		}
		return fmt.Errorf("No offer available")
	}
	if len(task.Files) > 0 {
		if err := f(); err != nil {
			api.writeError(w, http.StatusInternalServerError, err.Error())
			return
		}
		files, err := api.m.ReadFile(task.ID, task.Files...)
		if err != nil {
			api.writeError(w, http.StatusInternalServerError, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		if err := json.NewEncoder(w).Encode(files); err != nil {
			api.writeError(w, http.StatusInternalServerError, err.Error())
		}
	} else {
		go f()
		w.WriteHeader(http.StatusAccepted)
		io.WriteString(w, "OK")
	}
}

// Endpoint to list all the tasks
func (api *API) tasksList(w http.ResponseWriter, r *http.Request) {
	api.RLock()
	data := struct {
		Size  int     `json:"size"`
		Tasks []*Task `json:"tasks"`
	}{
		len(api.tasks),
		api.tasks,
	}
	api.RUnlock()
	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(&data); err != nil {
		api.writeError(w, http.StatusInternalServerError, err.Error())
	}
}

// Endpoint to delete a task
func (api *API) tasksDelete(w http.ResponseWriter, r *http.Request) {
	var (
		vars   = mux.Vars(r)
		id     = vars["id"]
		tasks  = make([]*Task, 0)
		states = make(map[string]*mesosproto.TaskState, len(api.states)-1)
	)

	if err := api.m.KillTask(id); err != nil {
		api.writeError(w, http.StatusInternalServerError, err.Error())
		return
	}

	api.Lock()
	for _, task := range api.tasks {
		if task != nil && task.ID != id {
			tasks = append(tasks, task)
			states[task.ID] = task.State
		}
	}
	api.tasks = tasks
	api.states = states
	api.Unlock()
	io.WriteString(w, "OK")
}

// Endpoint to kill a task
func (api *API) tasksKill(w http.ResponseWriter, r *http.Request) {
	var (
		vars = mux.Vars(r)
		id   = vars["id"]
	)
	if err := api.m.KillTask(id); err != nil {
		api.writeError(w, http.StatusInternalServerError, err.Error())
	} else {
		io.WriteString(w, "OK")
	}
}

func (api *API) getFile(w http.ResponseWriter, r *http.Request) {
	var (
		vars = mux.Vars(r)
		id   = vars["id"]
		file = vars["file"]
	)

	files, err := api.m.ReadFile(id, []string{file}...)
	if err != nil {
		api.writeError(w, http.StatusInternalServerError, err.Error())
		return
	}
	content, ok := files[file]
	if !ok {
		api.writeError(w, http.StatusNotFound, file+" not found")
		return
	}
	io.WriteString(w, content)
}

func (api *API) handleStates() {
	for {
		event := <-api.m.GetEvent(mesosproto.Event_UPDATE)
		ID := event.Update.Status.TaskId.GetValue()

		state, ok := api.states[ID]
		if !ok {
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Warn("Update received for unknown task.")
			continue
		}

		*state = *event.Update.Status.State
		switch *event.Update.Status.State {
		case mesosproto.TaskState_TASK_STAGING:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Info("Task was registered.")
		case mesosproto.TaskState_TASK_STARTING:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Info("Task is starting.")
		case mesosproto.TaskState_TASK_RUNNING:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Info("Task is running.")
		case mesosproto.TaskState_TASK_FINISHED:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Info("Task is finished.")
		case mesosproto.TaskState_TASK_FAILED:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Warn("Task has failed.")
		case mesosproto.TaskState_TASK_KILLED:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Warn("Task was killed.")
		case mesosproto.TaskState_TASK_LOST:
			api.m.Log.WithFields(logrus.Fields{"ID": ID, "message": event.Update.Status.GetMessage()}).Warn("Task was lost.")
		}
	}
}

// Register all the routes and then serve the API
func (api *API) ListenAndServe(port int) error {
	r := mux.NewRouter()
	api.m.Log.WithFields(logrus.Fields{"port": port}).Info("Starting API...")

	endpoints := map[string]map[string]func(w http.ResponseWriter, r *http.Request){
		"DELETE": {
			"/tasks/{id}": api.tasksDelete,
		},
		"GET": {
			"/_ping":                  api._ping,
			"/tasks/{id}/file/{file}": api.getFile,
			"/tasks":                  api.tasksList,
		},
		"POST": {
			"/tasks": api.tasksAdd,
		},
		"PUT": {
			"/tasks/{id}/kill": api.tasksKill,
		},
	}

	for method, routes := range endpoints {
		for route, fct := range routes {
			_route := route
			_fct := fct
			_method := method

			api.m.Log.WithFields(logrus.Fields{"method": _method, "route": _route}).Debug("Registering API route...")
			r.Path(_route).Methods(_method).HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				api.m.Log.WithFields(logrus.Fields{"from": r.RemoteAddr}).Infof("[%s] %s", _method, _route)
				_fct(w, r)
			})
		}
	}
	r.PathPrefix("/").Handler(http.FileServer(&assetfs.AssetFS{Asset, AssetDir, "./static/"}))
	go api.handleStates()
	return http.ListenAndServe(fmt.Sprintf(":%d", port), r)
}
<|endoftext|>


<commit_before>package pmb

import (
	"errors"
	"fmt"
	"math/rand"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/howeyc/gopass"
)

type PMBConfig map[string]string

type PMB struct {
	config PMBConfig
}

type Message struct {
	Contents map[string]interface{}
	Raw      string
	Done     chan error
}

type Connection struct {
	Out    chan Message
	In     chan Message
	uri    string
	prefix string
	Keys   []string
	Id     string
}

type Notification struct {
	Message string
	URL     string
	Level   float64
}

func GetPMB(primaryURI string) *PMB {
	config := getConfig(primaryURI)

	return &PMB{config: config}
}

func getConfig(primaryURI string) PMBConfig {
	config := make(PMBConfig)

	if len(primaryURI) > 0 {
		config["primary"] = primaryURI
	} else if primaryURI := os.Getenv("PMB_PRIMARY_URI"); len(primaryURI) > 0 {
		config["primary"] = primaryURI
	}

	if key := os.Getenv("PMB_KEY"); len(key) > 0 {
		config["key"] = key
	} else {
		config["key"] = ""
	}
	logrus.Debugf("Config: %s", config)

	return config
}

func (pmb *PMB) ConnectIntroducer(id string) (*Connection, error) {

	if len(pmb.config["primary"]) > 0 {
		logrus.Debugf("calling connectWithKey")
		return connectWithKey(pmb.config["primary"], id, "", pmb.config["key"], true, true)
	}

	return nil, errors.New("No URI found, use '-p' to specify one")
}

func (pmb *PMB) ConnectClient(id string, checkKey bool) (*Connection, error) {

	if len(pmb.config["primary"]) > 0 {
		logrus.Debugf("calling connectWithKey")
		return connectWithKey(pmb.config["primary"], id, "", pmb.config["key"], false, checkKey)
	}

	return nil, errors.New("No URI found, use '-p' to specify one")
}

func (pmb *PMB) ConnectSubClient(conn *Connection, sub string) (*Connection, error) {

	if len(pmb.config["primary"]) > 0 {
		logrus.Debugf("calling connectWithKey")
		return connectWithKey(pmb.config["primary"], conn.Id, sub, strings.Join(conn.Keys, ","), false, false)
	}

	return nil, errors.New("No URI found, use '-p' to specify one")
}

// Deprecated
func (pmb *PMB) GetConnection(id string, isIntroducer bool) (*Connection, error) {

	if len(pmb.config["primary"]) > 0 {
		logrus.Debugf("calling connectWithKey")
		return connectWithKey(pmb.config["primary"], id, "", pmb.config["key"], isIntroducer, true)
	}

	return nil, errors.New("No URI found, use '-p' to specify one")
}

func (pmb *PMB) CopyKey(id string) (*Connection, error) {

	if len(pmb.config["primary"]) > 0 {
		return copyKey(pmb.config["primary"], id)
	}

	return nil, errors.New("No URI found, use '-p' to specify one")
}

var charactersForRandom = []byte("1234567890abcdefghijklmnopqrstuvwxyz")

var randSeeded = false

func ensureRandSeeded() {
	if !randSeeded {
		logrus.Debugf("Initializing rand")
		rand.Seed(time.Now().UnixNano())
		randSeeded = true
	}
}

func GenerateRandomString(length int) string {
	ensureRandSeeded()
	random := make([]byte, length)
	for i, _ := range random {
		random[i] = charactersForRandom[rand.Intn(len(charactersForRandom))]
	}
	return string(random)
}

func GenerateRandomID(prefix string) string {
	return fmt.Sprintf("%s-%s", prefix, GenerateRandomString(12))
}

func SendNotification(conn *Connection, note Notification) error {
	notificationId := GenerateRandomID("notify")
	notifyData := map[string]interface{}{
		"type":            "Notification",
		"notification-id": notificationId,
		"message":         note.Message,
		"level":           note.Level,
		"url":             note.URL,
	}
	conn.Out <- Message{Contents: notifyData}

	timeout := time.After(2 * time.Second)
	for {
		select {
		case message := <-conn.In:
			data := message.Contents
			if data["type"].(string) == "NotificationDisplayed" && data["origin"].(string) == conn.Id {
				return nil
			}
		case _ = <-timeout:
			return fmt.Errorf("Unable to determine if message was displayed...")
		}
	}
}

func connect(URI string, id string, sub string) (*Connection, error) {
	if strings.HasPrefix(URI, "http") {
		return connectHTTP(URI, id, sub)
	} else if strings.HasPrefix(URI, "amqp") {
		return connectAMQP(URI, id, sub)
	}
	return nil, fmt.Errorf("Unknown PMB URI")
}

func copyKey(URI string, id string) (*Connection, error) {
	conn, err := connect(URI, id, "")
	if err != nil {
		return nil, err
	}

	data := map[string]interface{}{
		"type": "RequestAuth",
	}
	mess := Message{
		Contents: data,
		Done:     make(chan error),
	}
	conn.Out <- mess

	<-mess.Done

	return conn, nil
}

func connectWithKey(URI string, id string, sub string, key string, isIntroducer bool, checkKey bool) (*Connection, error) {
	logrus.Debugf("calling connect")
	conn, err := connect(URI, id, sub)
	if err != nil {
		return nil, err
	}

	if len(key) > 0 {
		// convert keys
		conn.Keys, err = parseKeys(key)
		if err != nil {
			return nil, err
		}

		// if we're not the introducer, check if the auth is valid
		if !isIntroducer && checkKey {
			err = testAuth(conn, id)
			if err != nil {
				return nil, err
			}
		}

		return conn, nil

	} else {

		// keep requesting auth until we can verify that it's valid
		for {
			conn.Keys = []string{}
			inkeys, err := requestKey(conn)
			if err != nil {
				return nil, err
			}

			// convert keys
			conn.Keys, err = parseKeys(inkeys)
			if err != nil {
				return nil, err
			}

			if !checkKey {
				break
			}

			err = testAuth(conn, id)
			if err != nil {
				logrus.Warningf("Error with key: %s", err)
			} else {
				break
			}
		}
	}

	return conn, nil
}

func parseKeys(keystring string) ([]string, error) {
	keyre := regexp.MustCompile("[a-z0-9]{32}")
	keys := keyre.FindAllString(keystring, -1)

	if len(keys) == 0 {
		return []string{}, fmt.Errorf("Auth key(s) invalid.")
	} else {
		logrus.Debugf("keys: %s", keys)
	}

	return keys, nil
}

func testAuth(conn *Connection, id string) error {

	conn.Out <- Message{Contents: map[string]interface{}{
		"type": "TestAuth",
	}}

	timeout := time.After(10 * time.Second)
	for {
		select {
		case message := <-conn.In:
			data := message.Contents
			if data["type"].(string) == "AuthValid" && data["origin"].(string) == id {
				return nil
			}
		case _ = <-timeout:
			return fmt.Errorf("Auth key was invalid.")
		}
	}
}

func requestKey(conn *Connection) (string, error) {
	data := map[string]interface{}{
		"type": "RequestAuth",
	}
	conn.Out <- Message{Contents: data}

	time.Sleep(200 * time.Millisecond)

	tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
	if err != nil {
		fmt.Errorf("failed to open /dev/tty", err)
	}

	key, err := gopass.GetPasswdPrompt("Enter key: ", true, tty, tty)
	if err != nil {
		if err == gopass.ErrInterrupted {
			return "", fmt.Errorf("interrupted")
		} else {
			return "", err
		}
	}

	return string(key), nil
}

func (pmb *PMB) PrimaryURI() string {
	return pmb.config["primary"]
}
<commit_msg>attempt to make windows prompting better<commit_after>package pmb

import (
	"errors"
	"fmt"
	"math/rand"
	"os"
	"regexp"
	"runtime"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/howeyc/gopass"
)

type PMBConfig map[string]string

type PMB struct {
	config PMBConfig
}

type Message struct {
	Contents map[string]interface{}
	Raw      string
	Done     chan error
}

type Connection struct {
	Out    chan Message
	In     chan Message
	uri    string
	prefix string
	Keys   []string
	Id     string
}

type Notification struct {
	Message string
	URL     string
	Level   float64
}

func GetPMB(primaryURI string) *PMB {
	config := getConfig(primaryURI)

	return &PMB{config: config}
}

func getConfig(primaryURI string) PMBConfig {
	config := make(PMBConfig)

	if len(primaryURI) > 0 {
		config["primary"] = primaryURI
	} else if primaryURI := os.Getenv("PMB_PRIMARY_URI"); len(primaryURI) > 0 {
		config["primary"] = primaryURI
	}

	if key := os.Getenv("PMB_KEY"); len(key) > 0 {
		config["key"] = key
	} else {
		config["key"] = ""
	}
	logrus.Debugf("Config: %s", config)

	return config
}

func (pmb *PMB) ConnectIntroducer(id string) (*Connection, error) {

	if len(pmb.config["primary"]) > 0 {
		logrus.Debugf("calling connectWithKey")
		return connectWithKey(pmb.config["primary"], id, "", pmb.config["key"], true, true)
	}

	return nil, errors.New("No URI found, use '-p' to specify one")
}

func (pmb *PMB) ConnectClient(id string, checkKey bool) (*Connection, error) {

	if len(pmb.config["primary"]) > 0 {
		logrus.Debugf("calling connectWithKey")
		return connectWithKey(pmb.config["primary"], id, "", pmb.config["key"], false, checkKey)
	}

	return nil, errors.New("No URI found, use '-p' to specify one")
}

func (pmb *PMB) ConnectSubClient(conn *Connection, sub string) (*Connection, error) {

	if len(pmb.config["primary"]) > 0 {
		logrus.Debugf("calling connectWithKey")
		return connectWithKey(pmb.config["primary"], conn.Id, sub, strings.Join(conn.Keys, ","), false, false)
	}

	return nil, errors.New("No URI found, use '-p' to specify one")
}

// Deprecated
func (pmb *PMB) GetConnection(id string, isIntroducer bool) (*Connection, error) {

	if len(pmb.config["primary"]) > 0 {
		logrus.Debugf("calling connectWithKey")
		return connectWithKey(pmb.config["primary"], id, "", pmb.config["key"], isIntroducer, true)
	}

	return nil, errors.New("No URI found, use '-p' to specify one")
}

func (pmb *PMB) CopyKey(id string) (*Connection, error) {

	if len(pmb.config["primary"]) > 0 {
		return copyKey(pmb.config["primary"], id)
	}

	return nil, errors.New("No URI found, use '-p' to specify one")
}

var charactersForRandom = []byte("1234567890abcdefghijklmnopqrstuvwxyz")

var randSeeded = false

func ensureRandSeeded() {
	if !randSeeded {
		logrus.Debugf("Initializing rand")
		rand.Seed(time.Now().UnixNano())
		randSeeded = true
	}
}

func GenerateRandomString(length int) string {
	ensureRandSeeded()
	random := make([]byte, length)
	for i, _ := range random {
		random[i] = charactersForRandom[rand.Intn(len(charactersForRandom))]
	}
	return string(random)
}

func GenerateRandomID(prefix string) string {
	return fmt.Sprintf("%s-%s", prefix, GenerateRandomString(12))
}

func SendNotification(conn *Connection, note Notification) error {
	notificationId := GenerateRandomID("notify")
	notifyData := map[string]interface{}{
		"type":            "Notification",
		"notification-id": notificationId,
		"message":         note.Message,
		"level":           note.Level,
		"url":             note.URL,
	}
	conn.Out <- Message{Contents: notifyData}

	timeout := time.After(2 * time.Second)
	for {
		select {
		case message := <-conn.In:
			data := message.Contents
			if data["type"].(string) == "NotificationDisplayed" && data["origin"].(string) == conn.Id {
				return nil
			}
		case _ = <-timeout:
			return fmt.Errorf("Unable to determine if message was displayed...")
		}
	}
}

func connect(URI string, id string, sub string) (*Connection, error) {
	if strings.HasPrefix(URI, "http") {
		return connectHTTP(URI, id, sub)
	} else if strings.HasPrefix(URI, "amqp") {
		return connectAMQP(URI, id, sub)
	}
	return nil, fmt.Errorf("Unknown PMB URI")
}

func copyKey(URI string, id string) (*Connection, error) {
	conn, err := connect(URI, id, "")
	if err != nil {
		return nil, err
	}

	data := map[string]interface{}{
		"type": "RequestAuth",
	}
	mess := Message{
		Contents: data,
		Done:     make(chan error),
	}
	conn.Out <- mess

	<-mess.Done

	return conn, nil
}

func connectWithKey(URI string, id string, sub string, key string, isIntroducer bool, checkKey bool) (*Connection, error) {
	logrus.Debugf("calling connect")
	conn, err := connect(URI, id, sub)
	if err != nil {
		return nil, err
	}

	if len(key) > 0 {
		// convert keys
		conn.Keys, err = parseKeys(key)
		if err != nil {
			return nil, err
		}

		// if we're not the introducer, check if the auth is valid
		if !isIntroducer && checkKey {
			err = testAuth(conn, id)
			if err != nil {
				return nil, err
			}
		}

		return conn, nil

	} else {

		// keep requesting auth until we can verify that it's valid
		for {
			conn.Keys = []string{}
			inkeys, err := requestKey(conn)
			if err != nil {
				return nil, err
			}

			// convert keys
			conn.Keys, err = parseKeys(inkeys)
			if err != nil {
				return nil, err
			}

			if !checkKey {
				break
			}

			err = testAuth(conn, id)
			if err != nil {
				logrus.Warningf("Error with key: %s", err)
			} else {
				break
			}
		}
	}

	return conn, nil
}

func parseKeys(keystring string) ([]string, error) {
	keyre := regexp.MustCompile("[a-z0-9]{32}")
	keys := keyre.FindAllString(keystring, -1)

	if len(keys) == 0 {
		return []string{}, fmt.Errorf("Auth key(s) invalid.")
	} else {
		logrus.Debugf("keys: %s", keys)
	}

	return keys, nil
}

func testAuth(conn *Connection, id string) error {

	conn.Out <- Message{Contents: map[string]interface{}{
		"type": "TestAuth",
	}}

	timeout := time.After(10 * time.Second)
	for {
		select {
		case message := <-conn.In:
			data := message.Contents
			if data["type"].(string) == "AuthValid" && data["origin"].(string) == id {
				return nil
			}
		case _ = <-timeout:
			return fmt.Errorf("Auth key was invalid.")
		}
	}
}

func requestKey(conn *Connection) (string, error) {
	data := map[string]interface{}{
		"type": "RequestAuth",
	}
	conn.Out <- Message{Contents: data}

	time.Sleep(200 * time.Millisecond)

	var key []byte
	var err error

	if runtime.GOOS == "windows" {
		fmt.Printf("Enter key: ")
		key, err = gopass.GetPasswd()
		if err != nil {
			if err == gopass.ErrInterrupted {
				return "", fmt.Errorf("interrupted")
			} else {
				return "", err
			}
		}
	} else {
		tty, errt := os.OpenFile("/dev/tty", os.O_RDWR, 0)
		if errt != nil {
			fmt.Errorf("failed to open /dev/tty", err)
		}

		key, err = gopass.GetPasswdPrompt("Enter key: ", true, tty, tty)
		if err != nil {
			if err == gopass.ErrInterrupted {
				return "", fmt.Errorf("interrupted")
			} else {
				return "", err
			}
		}
	}

	return string(key), nil
}

func (pmb *PMB) PrimaryURI() string {
	return pmb.config["primary"]
}
<|endoftext|>


<commit_before>package app

import (
	"html/template"
	"log"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
	"sync"

	"github.com/kvch/redme/model"
)

type ReqHandler func(http.ResponseWriter, *http.Request) error

var (
	db        *model.RedMeDB
	templates map[string]*template.Template
)

type PostsPage struct {
	Posts         []*model.RedMePost
	NumberOfPosts int
	Err           string
	Success       string
	LastId        int
}

type FeedsPage struct {
	Feeds   []*model.RedMeFeed
	Err     string
	Success string
}

func (fn ReqHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if err := fn(w, r); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func InitializeFedMe(path string) {
	var err error
	db, err = model.NewRedMeDBConn(path)
	if err != nil {
		log.Fatalln("Error while connecting to DB:", err)
	}

	initTemplates()
}

func initTemplates() {
	if templates == nil {
		templates = make(map[string]*template.Template)
	}
	templatesDir := "templates/"
	basePath := "templates/base.tmpl"
	successPath := "templates/success.tmpl"
	errorPath := "templates/error.tmpl"
	layouts, err := filepath.Glob(templatesDir + "*.tmpl")
	if err != nil {
		log.Fatal("Error while initializing templates:", err)
	}

	funcMap := template.FuncMap{
		"noescape": func(s string) template.HTML {
			return template.HTML(s)
		},
	}

	for _, layout := range layouts {
		if layout != basePath && layout != successPath && layout != errorPath {
			templates[filepath.Base(layout)] = template.Must(template.New("").Funcs(funcMap).ParseFiles(layout, basePath, errorPath, successPath))
		}
	}
}

func ShowUnreadPosts(w http.ResponseWriter, r *http.Request) error {
	posts, err := db.GetAllUnreadPosts()
	lastId := getLastId(posts)
	if err != nil {
		p := &PostsPage{Posts: nil, NumberOfPosts: 0, Success: "", Err: "Error while fetching posts", LastId: lastId}
		return renderTemplate(w, "index.tmpl", p)
	}
	p := &PostsPage{Posts: posts, NumberOfPosts: len(posts), Success: "", Err: "", LastId: lastId}
	return renderTemplate(w, "index.tmpl", p)
}

func MarkAllPostsRead(w http.ResponseWriter, r *http.Request) error {
	values, _ := url.ParseQuery(r.URL.RawQuery)
	id := values.Get("id")
	err := db.MarkAllPostsRead(id)
	if err != nil {
		posts, _ := db.GetAllUnreadPosts()
		lastId := getLastId(posts)
		p := &PostsPage{Posts: posts, NumberOfPosts: 0, Success: "", Err: "Error while marking posts as read", LastId: lastId}
		return renderTemplate(w, "index.tmpl", p)
	}

	p := &PostsPage{Posts: nil, NumberOfPosts: 0, Success: "", Err: "", LastId: 0}
	return renderTemplate(w, "index.tmpl", p)
}

func getLastId(posts []*model.RedMePost) int {
	lastId := 0
	if len(posts) > 0 {
		for _, p := range posts {
			if lastId < p.Id {
				lastId = p.Id
			}
		}
	}
	return lastId
}

func RefreshFeeds(w http.ResponseWriter, r *http.Request) error {
	feeds, err := db.GetAllFeeds()
	var wg sync.WaitGroup
	if err != nil {
		log.Println(err)
		p := &PostsPage{Posts: nil, NumberOfPosts: 0, Success: "", Err: "Error while fetching feeds from db", LastId: 0}
		return renderTemplate(w, "index.tmpl", p)
	}

	wg.Add(len(feeds))
	for _, f := range feeds {
		go func(f *model.RedMeFeed) {
			defer wg.Done()
			log.Println("Updating", f.Feed.Title)
			err := f.Feed.Update()
			if err != nil {
				log.Println("Error while updating", f.Feed.Title, err.Error())
			}
			for _, i := range f.Feed.Items {
				db.AddPost(f, i)
			}
			log.Println("Update finished", f.Feed.Title)
		}(f)
	}
	wg.Wait()
	w.Header().Set("Cache-Control", "no-cache")
	http.Redirect(w, r, "/", 302)
	return nil
}

func ListFeeds(w http.ResponseWriter, r *http.Request) error {
	feeds, err := db.GetAllFeeds()
	if err != nil {
		p := &FeedsPage{Feeds: nil, Success: "", Err: "Error while fetching feeds from db"}
		return renderTemplate(w, "add.tmpl", p)
	}
	p := &FeedsPage{Feeds: feeds, Success: "", Err: ""}
	return renderTemplate(w, "add.tmpl", p)
}

func AddFeed(w http.ResponseWriter, r *http.Request) error {
	r.ParseForm()

	var filters []string
	if (r.Form.Get("filters")) != "" {
		filters = strings.Split(r.Form.Get("filters"), ",")
	}
	newFeed, err := model.NewRedMeFeed(r.Form.Get("feed"), filters)
	if err != nil {
		log.Println(err)
		f, _ := db.GetAllFeeds()
		p := &FeedsPage{Feeds: f, Success: "", Err: "Error while adding feed"}
		return renderTemplate(w, "add.tmpl", p)
	}

	err = db.AddFeed(newFeed)
	if err != nil {
		log.Println(err)
		f, _ := db.GetAllFeeds()
		p := &FeedsPage{Feeds: f, Success: "", Err: "Error while saving feed to db"}
		return renderTemplate(w, "add.tmpl", p)
	}

	f, _ := db.GetAllFeeds()
	p := &FeedsPage{Feeds: f, Success: "Successfully added feed", Err: ""}
	return renderTemplate(w, "add.tmpl", p)
}

func renderTemplate(w http.ResponseWriter, name string, data interface{}) error {
	tmpl, ok := templates[name]
	if !ok {
		log.Fatal("Template does not exist:", name)
	}
	return tmpl.ExecuteTemplate(w, "base", data)
}
<commit_msg>redirect after marking all as read<commit_after>package app

import (
	"html/template"
	"log"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
	"sync"

	"github.com/kvch/redme/model"
)

type ReqHandler func(http.ResponseWriter, *http.Request) error

var (
	db        *model.RedMeDB
	templates map[string]*template.Template
)

type PostsPage struct {
	Posts         []*model.RedMePost
	NumberOfPosts int
	Err           string
	Success       string
	LastId        int
}

type FeedsPage struct {
	Feeds   []*model.RedMeFeed
	Err     string
	Success string
}

func (fn ReqHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if err := fn(w, r); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func InitializeFedMe(path string) {
	var err error
	db, err = model.NewRedMeDBConn(path)
	if err != nil {
		log.Fatalln("Error while connecting to DB:", err)
	}

	initTemplates()
}

func initTemplates() {
	if templates == nil {
		templates = make(map[string]*template.Template)
	}
	templatesDir := "templates/"
	basePath := "templates/base.tmpl"
	successPath := "templates/success.tmpl"
	errorPath := "templates/error.tmpl"
	layouts, err := filepath.Glob(templatesDir + "*.tmpl")
	if err != nil {
		log.Fatal("Error while initializing templates:", err)
	}

	funcMap := template.FuncMap{
		"noescape": func(s string) template.HTML {
			return template.HTML(s)
		},
	}

	for _, layout := range layouts {
		if layout != basePath && layout != successPath && layout != errorPath {
			templates[filepath.Base(layout)] = template.Must(template.New("").Funcs(funcMap).ParseFiles(layout, basePath, errorPath, successPath))
		}
	}
}

func ShowUnreadPosts(w http.ResponseWriter, r *http.Request) error {
	posts, err := db.GetAllUnreadPosts()
	lastId := getLastId(posts)
	if err != nil {
		p := &PostsPage{Posts: nil, NumberOfPosts: 0, Success: "", Err: "Error while fetching posts", LastId: lastId}
		return renderTemplate(w, "index.tmpl", p)
	}
	p := &PostsPage{Posts: posts, NumberOfPosts: len(posts), Success: "", Err: "", LastId: lastId}
	return renderTemplate(w, "index.tmpl", p)
}

func MarkAllPostsRead(w http.ResponseWriter, r *http.Request) error {
	values, _ := url.ParseQuery(r.URL.RawQuery)
	id := values.Get("id")
	err := db.MarkAllPostsRead(id)
	if err != nil {
		posts, _ := db.GetAllUnreadPosts()
		lastId := getLastId(posts)
		p := &PostsPage{Posts: posts, NumberOfPosts: 0, Success: "", Err: "Error while marking posts as read", LastId: lastId}
		return renderTemplate(w, "index.tmpl", p)
	}

	w.Header().Set("Cache-Control", "no-cache")
	http.Redirect(w, r, "/", 302)
	return nil
}

func getLastId(posts []*model.RedMePost) int {
	lastId := 0
	if len(posts) > 0 {
		for _, p := range posts {
			if lastId < p.Id {
				lastId = p.Id
			}
		}
	}
	return lastId
}

func RefreshFeeds(w http.ResponseWriter, r *http.Request) error {
	feeds, err := db.GetAllFeeds()
	var wg sync.WaitGroup
	if err != nil {
		log.Println(err)
		p := &PostsPage{Posts: nil, NumberOfPosts: 0, Success: "", Err: "Error while fetching feeds from db", LastId: 0}
		return renderTemplate(w, "index.tmpl", p)
	}

	wg.Add(len(feeds))
	for _, f := range feeds {
		go func(f *model.RedMeFeed) {
			defer wg.Done()
			log.Println("Updating", f.Feed.Title)
			err := f.Feed.Update()
			if err != nil {
				log.Println("Error while updating", f.Feed.Title, err.Error())
			}
			for _, i := range f.Feed.Items {
				db.AddPost(f, i)
			}
			log.Println("Update finished", f.Feed.Title)
		}(f)
	}
	wg.Wait()
	w.Header().Set("Cache-Control", "no-cache")
	http.Redirect(w, r, "/", 302)
	return nil
}

func ListFeeds(w http.ResponseWriter, r *http.Request) error {
	feeds, err := db.GetAllFeeds()
	if err != nil {
		p := &FeedsPage{Feeds: nil, Success: "", Err: "Error while fetching feeds from db"}
		return renderTemplate(w, "add.tmpl", p)
	}
	p := &FeedsPage{Feeds: feeds, Success: "", Err: ""}
	return renderTemplate(w, "add.tmpl", p)
}

func AddFeed(w http.ResponseWriter, r *http.Request) error {
	r.ParseForm()

	var filters []string
	if (r.Form.Get("filters")) != "" {
		filters = strings.Split(r.Form.Get("filters"), ",")
	}
	newFeed, err := model.NewRedMeFeed(r.Form.Get("feed"), filters)
	if err != nil {
		log.Println(err)
		f, _ := db.GetAllFeeds()
		p := &FeedsPage{Feeds: f, Success: "", Err: "Error while adding feed"}
		return renderTemplate(w, "add.tmpl", p)
	}

	err = db.AddFeed(newFeed)
	if err != nil {
		log.Println(err)
		f, _ := db.GetAllFeeds()
		p := &FeedsPage{Feeds: f, Success: "", Err: "Error while saving feed to db"}
		return renderTemplate(w, "add.tmpl", p)
	}

	f, _ := db.GetAllFeeds()
	p := &FeedsPage{Feeds: f, Success: "Successfully added feed", Err: ""}
	return renderTemplate(w, "add.tmpl", p)
}

func renderTemplate(w http.ResponseWriter, name string, data interface{}) error {
	tmpl, ok := templates[name]
	if !ok {
		log.Fatal("Template does not exist:", name)
	}
	return tmpl.ExecuteTemplate(w, "base", data)
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/globalsign/mgo"
	"github.com/globalsign/mgo/bson"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/log"
)

var (
	bulkMaxWaitMongoTime = 1 * time.Second
	bulkMaxNumberMsgs    = 1000
	bulkQueueMaxSize     = 10000

	buckets = append([]float64{0.1, 0.5}, prometheus.ExponentialBuckets(1, 1.6, 15)...)

	logsInQueue = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_queue_current",
		Help: "The current number of log entries in dispatcher queue.",
	})

	logsInAppQueues = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_app_queues_current",
		Help: "The current number of log entries in app queues.",
	})

	logsQueueBlockedTotal = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_queue_blocked_seconds_total",
		Help: "The total time spent blocked trying to add log to queue.",
	})

	logsQueueSize = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_dispatcher_queue_size",
		Help: "The max number of log entries in a dispatcher queue.",
	})

	logsEnqueued = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_enqueued_total",
		Help: "The number of log entries enqueued for processing.",
	})

	logsWritten = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_write_total",
		Help: "The number of log entries written to mongo.",
	})

	logsDropped = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_total",
		Help: "The number of log entries dropped due to full buffers.",
	})

	logsMongoFullLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_full_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})

	logsMongoLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})
)

func init() {
	prometheus.MustRegister(logsInQueue)
	prometheus.MustRegister(logsInAppQueues)
	prometheus.MustRegister(logsQueueSize)
	prometheus.MustRegister(logsEnqueued)
	prometheus.MustRegister(logsWritten)
	prometheus.MustRegister(logsDropped)
	prometheus.MustRegister(logsQueueBlockedTotal)
	prometheus.MustRegister(logsMongoFullLatency)
	prometheus.MustRegister(logsMongoLatency)
}

type LogListener struct {
	c       <-chan Applog
	logConn *db.LogStorage
	quit    chan struct{}
}

func isCappedPositionLost(err error) bool {
	if err == nil {
		return false
	}
	return strings.Contains(err.Error(), "CappedPositionLost")
}

func isSessionClosed(r interface{}) bool {
	return fmt.Sprintf("%v", r) == "Session already closed"
}

func NewLogListener(a *App, filterLog Applog) (*LogListener, error) {
	conn, err := db.LogConn()
	if err != nil {
		return nil, err
	}
	c := make(chan Applog, 10)
	quit := make(chan struct{})
	coll := conn.Logs(a.Name)
	var lastLog Applog
	err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	if err == mgo.ErrNotFound {
		// Tail cursors do not work correctly if the collection is empty (the
		// Next() call wouldn't block). So if the collection is empty we insert
		// the very first log line in it. This is quite rare in the real world
		// though so the impact of this extra log message is really small.
		err = a.Log("Logs initialization", "tsuru", "")
		if err != nil {
			return nil, err
		}
		err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	}
	if err != nil {
		return nil, err
	}
	lastId := lastLog.MongoID
	mkQuery := func() bson.M {
		m := bson.M{
			"_id": bson.M{"$gt": lastId},
		}
		if filterLog.Source != "" {
			m["source"] = filterLog.Source
		}
		if filterLog.Unit != "" {
			m["unit"] = filterLog.Unit
		}
		return m
	}
	query := coll.Find(mkQuery())
	tailTimeout := 10 * time.Second
	iter := query.Sort("$natural").Tail(tailTimeout)
	go func() {
		defer close(c)
		defer func() {
			if r := recover(); r != nil {
				if isSessionClosed(r) {
					return
				}
				panic(err)
			}
		}()
		for {
			var applog Applog
			for iter.Next(&applog) {
				lastId = applog.MongoID
				select {
				case c <- applog:
				case <-quit:
					iter.Close()
					return
				}
			}
			if iter.Timeout() {
				continue
			}
			if err := iter.Err(); err != nil {
				if !isCappedPositionLost(err) {
					log.Errorf("error tailing logs: %v", err)
					iter.Close()
					return
				}
			}
			iter.Close()
			query = coll.Find(mkQuery())
			iter = query.Sort("$natural").Tail(tailTimeout)
		}
	}()
	l := LogListener{c: c, logConn: conn, quit: quit}
	return &l, nil
}

func (l *LogListener) ListenChan() <-chan Applog {
	return l.c
}

func (l *LogListener) Close() {
	l.logConn.Close()
	if l.quit != nil {
		close(l.quit)
		l.quit = nil
	}
}

type LogDispatcher struct {
	mu             sync.RWMutex
	dispatchers    map[string]*appLogDispatcher
	msgCh          chan *msgWithTS
	shuttingDown   int32
	doneProcessing chan struct{}
}

type msgWithTS struct {
	msg        *Applog
	arriveTime time.Time
}

func NewlogDispatcher(chanSize int) *LogDispatcher {
	d := &LogDispatcher{
		dispatchers:    make(map[string]*appLogDispatcher),
		msgCh:          make(chan *msgWithTS, chanSize),
		doneProcessing: make(chan struct{}),
	}
	go d.runWriter()
	shutdown.Register(d)
	logsQueueSize.Set(float64(chanSize))
	return d
}

func (d *LogDispatcher) getMessageDispatcher(msg *Applog) *appLogDispatcher {
	appName := msg.AppName
	d.mu.RLock()
	appD, ok := d.dispatchers[appName]
	if !ok {
		d.mu.RUnlock()
		d.mu.Lock()
		appD, ok = d.dispatchers[appName]
		if !ok {
			appD = newAppLogDispatcher(appName)
			d.dispatchers[appName] = appD
		}
		d.mu.Unlock()
	} else {
		d.mu.RUnlock()
	}
	return appD
}

func (d *LogDispatcher) runWriter() {
	defer close(d.doneProcessing)
	for msgExtra := range d.msgCh {
		if msgExtra == nil {
			break
		}
		logsInQueue.Dec()
		appD := d.getMessageDispatcher(msgExtra.msg)
		appD.send(msgExtra)
	}
}

func (d *LogDispatcher) Send(msg *Applog) error {
	if atomic.LoadInt32(&d.shuttingDown) == 1 {
		return errors.New("log dispatcher is shutting down")
	}
	logsInQueue.Inc()
	logsEnqueued.Inc()
	msgExtra := &msgWithTS{msg: msg, arriveTime: time.Now()}
	select {
	case d.msgCh <- msgExtra:
	default:
		t0 := time.Now()
		d.msgCh <- msgExtra
		logsQueueBlockedTotal.Add(time.Since(t0).Seconds())
	}
	return nil
}

func (a *LogDispatcher) String() string {
	return "log dispatcher"
}

func (d *LogDispatcher) Shutdown(ctx context.Context) error {
	atomic.StoreInt32(&d.shuttingDown, 1)
	d.msgCh <- nil
	<-d.doneProcessing
	logsInQueue.Set(0)
	for _, appD := range d.dispatchers {
		appD.stopWait()
	}
	return nil
}

type appLogDispatcher struct {
	appName string
	*bulkProcessor
}

func newAppLogDispatcher(appName string) *appLogDispatcher {
	d := &appLogDispatcher{
		bulkProcessor: initBulkProcessor(bulkMaxWaitMongoTime, bulkMaxNumberMsgs),
		appName:       appName,
	}
	d.flushable = d
	go d.run()
	return d
}

func (d *appLogDispatcher) flush(msgs []interface{}, lastMessage *msgWithTS) bool {
	conn, err := db.LogConn()
	if err != nil {
		log.Errorf("[log flusher] unable to connect to mongodb: %s", err)
		return false
	}
	coll := conn.Logs(d.appName)
	err = coll.Insert(msgs...)
	coll.Close()
	if err != nil {
		log.Errorf("[log flusher] unable to insert logs: %s", err)
		return false
	}
	if lastMessage != nil {
		logsMongoLatency.Observe(time.Since(lastMessage.arriveTime).Seconds())
		logsMongoFullLatency.Observe(time.Since(lastMessage.msg.Date).Seconds())
	}
	logsWritten.Add(float64(len(msgs)))
	return true
}

type bulkProcessor struct {
	maxWaitTime time.Duration
	bulkSize    int
	finished    chan struct{}
	ch          chan *msgWithTS
	nextNotify  *time.Timer
	flushable   interface {
		flush([]interface{}, *msgWithTS) bool
	}
}

func initBulkProcessor(maxWait time.Duration, bulkSize int) *bulkProcessor {
	return &bulkProcessor{
		maxWaitTime: maxWait,
		bulkSize:    bulkSize,
		finished:    make(chan struct{}),
		ch:          make(chan *msgWithTS, bulkQueueMaxSize),
		nextNotify:  time.NewTimer(0),
	}
}

func (p *bulkProcessor) send(msg *msgWithTS) {
	select {
	case p.ch <- msg:
		logsInAppQueues.Set(float64(len(p.ch)))
	default:
		logsDropped.Inc()
		select {
		case <-p.nextNotify.C:
			log.Errorf("dropping log messages to mongodb due to full channel buffer. app: %q, len: %d", msg.msg.AppName, len(p.ch))
			p.nextNotify.Reset(time.Minute)
		default:
		}
	}
}

func (p *bulkProcessor) stopWait() {
	p.ch <- nil
	<-p.finished
}

func (p *bulkProcessor) run() {
	defer close(p.finished)
	t := time.NewTimer(p.maxWaitTime)
	pos := 0
	bulkBuffer := make([]interface{}, p.bulkSize)
	shouldReturn := false
	var lastMessage *msgWithTS
	for {
		var flush bool
		select {
		case msgExtra := <-p.ch:
			logsInAppQueues.Set(float64(len(p.ch)))
			if msgExtra == nil {
				flush = true
				shouldReturn = true
				break
			}
			if pos == p.bulkSize {
				flush = true
				break
			}
			lastMessage = msgExtra
			bulkBuffer[pos] = msgExtra.msg
			pos++
			flush = p.bulkSize == pos
		case <-t.C:
			flush = true
			t.Reset(p.maxWaitTime)
		}
		if flush && pos > 0 {
			if p.flushable.flush(bulkBuffer[:pos], lastMessage) {
				lastMessage = nil
				pos = 0
			}
		}
		if shouldReturn {
			return
		}
	}
}
<commit_msg>app: incoming log messages metrics per app<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/globalsign/mgo"
	"github.com/globalsign/mgo/bson"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/log"
)

var (
	bulkMaxWaitMongoTime = 1 * time.Second
	bulkMaxNumberMsgs    = 1000
	bulkQueueMaxSize     = 10000

	buckets = append([]float64{0.1, 0.5}, prometheus.ExponentialBuckets(1, 1.6, 15)...)

	logsInQueue = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_queue_current",
		Help: "The current number of log entries in dispatcher queue.",
	})

	logsInAppQueues = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "tsuru_logs_app_queues_current",
		Help: "The current number of log entries in app queues.",
	}, []string{"app"})

	logsQueueBlockedTotal = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_queue_blocked_seconds_total",
		Help: "The total time spent blocked trying to add log to queue.",
	})

	logsQueueSize = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_dispatcher_queue_size",
		Help: "The max number of log entries in a dispatcher queue.",
	})

	logsEnqueued = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_enqueued_total",
		Help: "The number of log entries enqueued for processing.",
	}, []string{"app"})

	logsWritten = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_write_total",
		Help: "The number of log entries written to mongo.",
	}, []string{"app"})

	logsDropped = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_total",
		Help: "The number of log entries dropped due to full buffers.",
	}, []string{"app"})

	logsMongoFullLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_full_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})

	logsMongoLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})
)

func init() {
	prometheus.MustRegister(logsInQueue)
	prometheus.MustRegister(logsInAppQueues)
	prometheus.MustRegister(logsQueueSize)
	prometheus.MustRegister(logsEnqueued)
	prometheus.MustRegister(logsWritten)
	prometheus.MustRegister(logsDropped)
	prometheus.MustRegister(logsQueueBlockedTotal)
	prometheus.MustRegister(logsMongoFullLatency)
	prometheus.MustRegister(logsMongoLatency)
}

type LogListener struct {
	c       <-chan Applog
	logConn *db.LogStorage
	quit    chan struct{}
}

func isCappedPositionLost(err error) bool {
	if err == nil {
		return false
	}
	return strings.Contains(err.Error(), "CappedPositionLost")
}

func isSessionClosed(r interface{}) bool {
	return fmt.Sprintf("%v", r) == "Session already closed"
}

func NewLogListener(a *App, filterLog Applog) (*LogListener, error) {
	conn, err := db.LogConn()
	if err != nil {
		return nil, err
	}
	c := make(chan Applog, 10)
	quit := make(chan struct{})
	coll := conn.Logs(a.Name)
	var lastLog Applog
	err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	if err == mgo.ErrNotFound {
		// Tail cursors do not work correctly if the collection is empty (the
		// Next() call wouldn't block). So if the collection is empty we insert
		// the very first log line in it. This is quite rare in the real world
		// though so the impact of this extra log message is really small.
		err = a.Log("Logs initialization", "tsuru", "")
		if err != nil {
			return nil, err
		}
		err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	}
	if err != nil {
		return nil, err
	}
	lastId := lastLog.MongoID
	mkQuery := func() bson.M {
		m := bson.M{
			"_id": bson.M{"$gt": lastId},
		}
		if filterLog.Source != "" {
			m["source"] = filterLog.Source
		}
		if filterLog.Unit != "" {
			m["unit"] = filterLog.Unit
		}
		return m
	}
	query := coll.Find(mkQuery())
	tailTimeout := 10 * time.Second
	iter := query.Sort("$natural").Tail(tailTimeout)
	go func() {
		defer close(c)
		defer func() {
			if r := recover(); r != nil {
				if isSessionClosed(r) {
					return
				}
				panic(err)
			}
		}()
		for {
			var applog Applog
			for iter.Next(&applog) {
				lastId = applog.MongoID
				select {
				case c <- applog:
				case <-quit:
					iter.Close()
					return
				}
			}
			if iter.Timeout() {
				continue
			}
			if err := iter.Err(); err != nil {
				if !isCappedPositionLost(err) {
					log.Errorf("error tailing logs: %v", err)
					iter.Close()
					return
				}
			}
			iter.Close()
			query = coll.Find(mkQuery())
			iter = query.Sort("$natural").Tail(tailTimeout)
		}
	}()
	l := LogListener{c: c, logConn: conn, quit: quit}
	return &l, nil
}

func (l *LogListener) ListenChan() <-chan Applog {
	return l.c
}

func (l *LogListener) Close() {
	l.logConn.Close()
	if l.quit != nil {
		close(l.quit)
		l.quit = nil
	}
}

type LogDispatcher struct {
	mu             sync.RWMutex
	dispatchers    map[string]*appLogDispatcher
	msgCh          chan *msgWithTS
	shuttingDown   int32
	doneProcessing chan struct{}
}

type msgWithTS struct {
	msg        *Applog
	arriveTime time.Time
}

func NewlogDispatcher(chanSize int) *LogDispatcher {
	d := &LogDispatcher{
		dispatchers:    make(map[string]*appLogDispatcher),
		msgCh:          make(chan *msgWithTS, chanSize),
		doneProcessing: make(chan struct{}),
	}
	go d.runWriter()
	shutdown.Register(d)
	logsQueueSize.Set(float64(chanSize))
	return d
}

func (d *LogDispatcher) getMessageDispatcher(msg *Applog) *appLogDispatcher {
	appName := msg.AppName
	d.mu.RLock()
	appD, ok := d.dispatchers[appName]
	if !ok {
		d.mu.RUnlock()
		d.mu.Lock()
		appD, ok = d.dispatchers[appName]
		if !ok {
			appD = newAppLogDispatcher(appName)
			d.dispatchers[appName] = appD
		}
		d.mu.Unlock()
	} else {
		d.mu.RUnlock()
	}
	return appD
}

func (d *LogDispatcher) runWriter() {
	defer close(d.doneProcessing)
	for msgExtra := range d.msgCh {
		if msgExtra == nil {
			break
		}
		logsInQueue.Dec()
		appD := d.getMessageDispatcher(msgExtra.msg)
		appD.send(msgExtra)
	}
}

func (d *LogDispatcher) Send(msg *Applog) error {
	if atomic.LoadInt32(&d.shuttingDown) == 1 {
		return errors.New("log dispatcher is shutting down")
	}
	logsInQueue.Inc()
	logsEnqueued.WithLabelValues(msg.AppName).Inc()
	msgExtra := &msgWithTS{msg: msg, arriveTime: time.Now()}
	select {
	case d.msgCh <- msgExtra:
	default:
		t0 := time.Now()
		d.msgCh <- msgExtra
		logsQueueBlockedTotal.Add(time.Since(t0).Seconds())
	}
	return nil
}

func (a *LogDispatcher) String() string {
	return "log dispatcher"
}

func (d *LogDispatcher) Shutdown(ctx context.Context) error {
	atomic.StoreInt32(&d.shuttingDown, 1)
	d.msgCh <- nil
	<-d.doneProcessing
	logsInQueue.Set(0)
	for _, appD := range d.dispatchers {
		appD.stopWait()
	}
	return nil
}

type appLogDispatcher struct {
	appName string
	*bulkProcessor
}

func newAppLogDispatcher(appName string) *appLogDispatcher {
	d := &appLogDispatcher{
		bulkProcessor: initBulkProcessor(bulkMaxWaitMongoTime, bulkMaxNumberMsgs, appName),
		appName:       appName,
	}
	d.flushable = d
	go d.run()
	return d
}

func (d *appLogDispatcher) flush(msgs []interface{}, lastMessage *msgWithTS) bool {
	conn, err := db.LogConn()
	if err != nil {
		log.Errorf("[log flusher] unable to connect to mongodb: %s", err)
		return false
	}
	coll := conn.Logs(d.appName)
	err = coll.Insert(msgs...)
	coll.Close()
	if err != nil {
		log.Errorf("[log flusher] unable to insert logs: %s", err)
		return false
	}
	if lastMessage != nil {
		logsMongoLatency.Observe(time.Since(lastMessage.arriveTime).Seconds())
		logsMongoFullLatency.Observe(time.Since(lastMessage.msg.Date).Seconds())
	}
	logsWritten.WithLabelValues(d.appName).Add(float64(len(msgs)))
	return true
}

type bulkProcessor struct {
	appName     string
	maxWaitTime time.Duration
	bulkSize    int
	finished    chan struct{}
	ch          chan *msgWithTS
	nextNotify  *time.Timer
	flushable   interface {
		flush([]interface{}, *msgWithTS) bool
	}
}

func initBulkProcessor(maxWait time.Duration, bulkSize int, appName string) *bulkProcessor {
	return &bulkProcessor{
		appName:     appName,
		maxWaitTime: maxWait,
		bulkSize:    bulkSize,
		finished:    make(chan struct{}),
		ch:          make(chan *msgWithTS, bulkQueueMaxSize),
		nextNotify:  time.NewTimer(0),
	}
}

func (p *bulkProcessor) send(msg *msgWithTS) {
	select {
	case p.ch <- msg:
		logsInAppQueues.WithLabelValues(p.appName).Set(float64(len(p.ch)))
	default:
		logsDropped.WithLabelValues(p.appName).Inc()
		select {
		case <-p.nextNotify.C:
			log.Errorf("dropping log messages to mongodb due to full channel buffer. app: %q, len: %d", msg.msg.AppName, len(p.ch))
			p.nextNotify.Reset(time.Minute)
		default:
		}
	}
}

func (p *bulkProcessor) stopWait() {
	p.ch <- nil
	<-p.finished
}

func (p *bulkProcessor) run() {
	defer close(p.finished)
	t := time.NewTimer(p.maxWaitTime)
	pos := 0
	bulkBuffer := make([]interface{}, p.bulkSize)
	shouldReturn := false
	var lastMessage *msgWithTS
	for {
		var flush bool
		select {
		case msgExtra := <-p.ch:
			logsInAppQueues.WithLabelValues(p.appName).Set(float64(len(p.ch)))
			if msgExtra == nil {
				flush = true
				shouldReturn = true
				break
			}
			if pos == p.bulkSize {
				flush = true
				break
			}
			lastMessage = msgExtra
			bulkBuffer[pos] = msgExtra.msg
			pos++
			flush = p.bulkSize == pos
		case <-t.C:
			flush = true
			t.Reset(p.maxWaitTime)
		}
		if flush && pos > 0 {
			if p.flushable.flush(bulkBuffer[:pos], lastMessage) {
				lastMessage = nil
				pos = 0
			}
		}
		if shouldReturn {
			return
		}
	}
}
<|endoftext|>


<commit_before>package archive

import (
	"fmt"
	"io"
	"os"
	"path"
	"time"

	"crypto"
	"crypto/sha512"

	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/packet"

	"pault.ag/go/blobstore"
	"pault.ag/go/debian/control"
	"pault.ag/go/debian/dependency"
	"pault.ag/go/debian/transput"
)

// New {{{

func New(path string, signer *openpgp.Entity) (*Archive, error) {
	store, err := blobstore.Load(path)
	if err != nil {
		return nil, err
	}

	fd, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer fd.Close()

	return &Archive{
		store:      *store,
		signingKey: signer,
	}, nil
}

// }}}

// Archive magic {{{

type Archive struct {
	store      blobstore.Store
	signingKey *openpgp.Entity
}

func (a Archive) Suite(name string) (*Suite, error) {
	/* Get the Release / InRelease */
	inRelease := Release{}
	components := map[string]*Component{}

	fd, err := a.store.OpenPath(path.Join("dists", name, "InRelease"))
	if err == nil {
		defer fd.Close()
		if err := control.Unmarshal(&inRelease, fd); err != nil {
			return nil, err
		}

		for _, name := range inRelease.Components {
			components[name] = &Component{Packages: []Package{}}
		}
	}

	suite := Suite{
		Name: name,

		release:    inRelease,
		Components: components,
	}

	suite.features.Hashes = []string{"sha256", "sha1"}

	return &suite, nil
}

func (a Archive) encodeHashedBySuite(path string, suite Suite, data interface{}) (*blobstore.Object, []control.FileHash, error) {

	hashers := []*transput.Hasher{}
	for _, algorithm := range suite.features.Hashes {
		hasher, err := transput.NewHasher(algorithm)
		if err != nil {
			return nil, nil, err
		}
		hashers = append(hashers, hasher)
	}

	return a.encodeHashed(path, hashers, data)
}

func (a Archive) encodeHashed(path string, hashers []*transput.Hasher, data interface{}) (*blobstore.Object, []control.FileHash, error) {

	writers := []io.Writer{}
	for _, hasher := range hashers {
		writers = append(writers, hasher)
	}

	obj, err := a.encode(data, io.MultiWriter(writers...))
	if err != nil {
		return nil, nil, err
	}

	fileHashs := []control.FileHash{}
	for _, hasher := range hashers {
		fileHashs = append(fileHashs, control.FileHashFromHasher(path, *hasher))
	}

	return obj, fileHashs, nil
}

func (a Archive) encodeSigned(
	data interface{},
) (*blobstore.Object, *blobstore.Object, error) {
	if a.signingKey == nil {
		return nil, nil, fmt.Errorf("No signing key loaded")
	}

	signature, err := a.store.Create()
	if err != nil {
		return nil, nil, err
	}
	defer signature.Close()

	hash := sha512.New()

	obj, err := a.encode(data, hash)
	if err != nil {
		return nil, nil, err
	}

	sig := new(packet.Signature)
	sig.SigType = packet.SigTypeBinary
	sig.PubKeyAlgo = a.signingKey.PrivateKey.PubKeyAlgo

	sig.Hash = crypto.SHA512

	sig.CreationTime = new(packet.Config).Now()
	sig.IssuerKeyId = &(a.signingKey.PrivateKey.KeyId)

	err = sig.Sign(hash, a.signingKey.PrivateKey, nil)
	if err != nil {
		return nil, nil, err
	}

	if err := sig.Serialize(signature); err != nil {
		return nil, nil, err
	}

	sigObj, err := a.store.Commit(*signature)
	if err != nil {
		return nil, nil, err
	}

	return obj, sigObj, nil
}

func (a Archive) encode(data interface{}, tee io.Writer) (*blobstore.Object, error) {
	writer, err := a.store.Create()
	if err != nil {
		return nil, err
	}
	defer writer.Close()

	var target io.Writer = writer
	if tee != nil {
		target = io.MultiWriter(writer, tee)
	}

	encoder, err := control.NewEncoder(target)
	if err != nil {
		return nil, err
	}

	if err := encoder.Encode(data); err != nil {
		return nil, err
	}

	obj, err := a.store.Commit(*writer)
	if err != nil {
		return nil, err
	}

	return obj, nil
}

func (a Archive) Engross(suite Suite) (map[string]blobstore.Object, error) {
	files := map[string]blobstore.Object{}

	release := Release{
		Description:   "",
		Origin:        "",
		Label:         "",
		Version:       "",
		Suite:         suite.Name,
		Codename:      "",
		Components:    suite.ComponenetNames(),
		Architectures: suite.Arches(),
		Date:          time.Now().Format(time.RFC1123Z),
		SHA256:        []control.SHA256FileHash{},
		SHA1:          []control.SHA1FileHash{},
		SHA512:        []control.SHA512FileHash{},
		MD5Sum:        []control.MD5FileHash{},
	}

	for name, component := range suite.Components {
		for arch, pkgs := range component.ByArch() {
			suitePath := path.Join(name, fmt.Sprintf("binary-%s", arch),
				"Packages")
			filePath := path.Join("dists", suite.Name, suitePath)

			obj, hashes, err := a.encodeHashedBySuite(suitePath, suite, pkgs)
			if err != nil {
				return nil, err
			}

			for _, hash := range hashes {
				if err := release.AddHash(hash); err != nil {
					return nil, err
				}
			}

			files[filePath] = *obj
		}
	}

	filePath := path.Join("dists", suite.Name, "Release")
	obj, sig, err := a.encodeSigned(release)
	if err != nil {
		return nil, err
	}
	files[filePath] = *obj
	files[fmt.Sprintf("%s.gpg", filePath)] = *sig

	return files, nil
}

func (a Archive) Link(blobs map[string]blobstore.Object) error {
	for p, obj := range blobs {
		if err := a.store.Link(obj, p); err != nil {
			return err
		}
	}
	return nil
}

func (a Archive) Decruft() error {
	return a.store.GC(blobstore.DumbGarbageCollector{})
}

// }}}

// Suite magic {{{

type Suite struct {
	Name string

	release    Release
	Components map[string]*Component

	features struct {
		Hashes []string
	}
}

func (s Suite) Arches() []dependency.Arch {
	ret := map[dependency.Arch]bool{}
	for _, component := range s.Components {
		for _, arch := range component.Arches() {
			ret[arch] = true
		}
	}
	r := []dependency.Arch{}
	for arch, _ := range ret {
		r = append(r, arch)
	}
	return r
}

func (s Suite) ComponenetNames() []string {
	ret := []string{}
	for name, _ := range s.Components {
		ret = append(ret, name)
	}
	return ret
}

func (s Suite) Add(name string, pkg Package) {
	if _, ok := s.Components[name]; !ok {
		s.Components[name] = &Component{Packages: []Package{}}
	}
	s.Components[name].Add(pkg)
}

// }}}

// Component magic {{{

type Component struct {
	Packages []Package
}

func (c *Component) ByArch() map[dependency.Arch][]Package {
	ret := map[dependency.Arch][]Package{}

	for _, pkg := range c.Packages {
		packages := ret[pkg.Architecture]
		ret[pkg.Architecture] = append(packages, pkg)
	}

	return ret
}

func (c *Component) Arches() []dependency.Arch {
	ret := []dependency.Arch{}
	for _, pkg := range c.Packages {
		ret = append(ret, pkg.Architecture)
	}
	return ret
}

func (c *Component) Add(p Package) {
	c.Packages = append(c.Packages, p)
}

// }}}

// vim: foldmethod=marker
<commit_msg>Update Pool<commit_after>package archive

import (
	"fmt"
	"io"
	"os"
	"path"
	"time"

	"crypto"
	"crypto/sha512"

	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/packet"

	"pault.ag/go/blobstore"
	"pault.ag/go/debian/control"
	"pault.ag/go/debian/dependency"
	"pault.ag/go/debian/transput"
)

// New {{{

func New(path string, signer *openpgp.Entity) (*Archive, error) {
	store, err := blobstore.Load(path)
	if err != nil {
		return nil, err
	}

	fd, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer fd.Close()

	return &Archive{
		store:      *store,
		signingKey: signer,
	}, nil
}

// }}}

// Archive magic {{{

type Archive struct {
	store      blobstore.Store
	signingKey *openpgp.Entity
}

func (a Archive) Suite(name string) (*Suite, error) {
	/* Get the Release / InRelease */
	inRelease := Release{}
	components := map[string]*Component{}

	fd, err := a.store.OpenPath(path.Join("dists", name, "InRelease"))
	if err == nil {
		defer fd.Close()
		if err := control.Unmarshal(&inRelease, fd); err != nil {
			return nil, err
		}

		for _, name := range inRelease.Components {
			components[name] = &Component{Packages: []Package{}}
		}
	}

	suite := Suite{
		Name: name,

		release:    inRelease,
		Components: components,
	}

	suite.Pool = Pool{
		store: a.store,
		suite: &suite,
	}

	suite.features.Hashes = []string{"sha256", "sha1"}

	return &suite, nil
}

func (a Archive) encodeHashedBySuite(path string, suite Suite, data interface{}) (*blobstore.Object, []control.FileHash, error) {

	hashers := []*transput.Hasher{}
	for _, algorithm := range suite.features.Hashes {
		hasher, err := transput.NewHasher(algorithm)
		if err != nil {
			return nil, nil, err
		}
		hashers = append(hashers, hasher)
	}

	return a.encodeHashed(path, hashers, data)
}

func (a Archive) encodeHashed(path string, hashers []*transput.Hasher, data interface{}) (*blobstore.Object, []control.FileHash, error) {

	writers := []io.Writer{}
	for _, hasher := range hashers {
		writers = append(writers, hasher)
	}

	obj, err := a.encode(data, io.MultiWriter(writers...))
	if err != nil {
		return nil, nil, err
	}

	fileHashs := []control.FileHash{}
	for _, hasher := range hashers {
		fileHashs = append(fileHashs, control.FileHashFromHasher(path, *hasher))
	}

	return obj, fileHashs, nil
}

func (a Archive) encodeSigned(
	data interface{},
) (*blobstore.Object, *blobstore.Object, error) {
	if a.signingKey == nil {
		return nil, nil, fmt.Errorf("No signing key loaded")
	}

	signature, err := a.store.Create()
	if err != nil {
		return nil, nil, err
	}
	defer signature.Close()

	hash := sha512.New()

	obj, err := a.encode(data, hash)
	if err != nil {
		return nil, nil, err
	}

	sig := new(packet.Signature)
	sig.SigType = packet.SigTypeBinary
	sig.PubKeyAlgo = a.signingKey.PrivateKey.PubKeyAlgo

	sig.Hash = crypto.SHA512

	sig.CreationTime = new(packet.Config).Now()
	sig.IssuerKeyId = &(a.signingKey.PrivateKey.KeyId)

	err = sig.Sign(hash, a.signingKey.PrivateKey, nil)
	if err != nil {
		return nil, nil, err
	}

	if err := sig.Serialize(signature); err != nil {
		return nil, nil, err
	}

	sigObj, err := a.store.Commit(*signature)
	if err != nil {
		return nil, nil, err
	}

	return obj, sigObj, nil
}

func (a Archive) encode(data interface{}, tee io.Writer) (*blobstore.Object, error) {
	writer, err := a.store.Create()
	if err != nil {
		return nil, err
	}
	defer writer.Close()

	var target io.Writer = writer
	if tee != nil {
		target = io.MultiWriter(writer, tee)
	}

	encoder, err := control.NewEncoder(target)
	if err != nil {
		return nil, err
	}

	if err := encoder.Encode(data); err != nil {
		return nil, err
	}

	obj, err := a.store.Commit(*writer)
	if err != nil {
		return nil, err
	}

	return obj, nil
}

func (a Archive) Engross(suite Suite) (map[string]blobstore.Object, error) {
	files := map[string]blobstore.Object{}

	release := Release{
		Description:   "",
		Origin:        "",
		Label:         "",
		Version:       "",
		Suite:         suite.Name,
		Codename:      "",
		Components:    suite.ComponenetNames(),
		Architectures: suite.Arches(),
		Date:          time.Now().Format(time.RFC1123Z),
		SHA256:        []control.SHA256FileHash{},
		SHA1:          []control.SHA1FileHash{},
		SHA512:        []control.SHA512FileHash{},
		MD5Sum:        []control.MD5FileHash{},
	}

	for name, component := range suite.Components {
		for arch, pkgs := range component.ByArch() {
			suitePath := path.Join(name, fmt.Sprintf("binary-%s", arch),
				"Packages")
			filePath := path.Join("dists", suite.Name, suitePath)

			obj, hashes, err := a.encodeHashedBySuite(suitePath, suite, pkgs)
			if err != nil {
				return nil, err
			}

			for _, hash := range hashes {
				if err := release.AddHash(hash); err != nil {
					return nil, err
				}
			}

			files[filePath] = *obj
		}
	}

	filePath := path.Join("dists", suite.Name, "Release")
	obj, sig, err := a.encodeSigned(release)
	if err != nil {
		return nil, err
	}
	files[filePath] = *obj
	files[fmt.Sprintf("%s.gpg", filePath)] = *sig

	return files, nil
}

func (a Archive) Link(blobs map[string]blobstore.Object) error {
	for p, obj := range blobs {
		if err := a.store.Link(obj, p); err != nil {
			return err
		}
	}
	return nil
}

func (a Archive) Decruft() error {
	return a.store.GC(blobstore.DumbGarbageCollector{})
}

// }}}

// Suite magic {{{

type Suite struct {
	Name string

	release    Release
	Components map[string]*Component
	Pool       Pool

	features struct {
		Hashes []string
	}
}

func (s Suite) Arches() []dependency.Arch {
	ret := map[dependency.Arch]bool{}
	for _, component := range s.Components {
		for _, arch := range component.Arches() {
			ret[arch] = true
		}
	}
	r := []dependency.Arch{}
	for arch, _ := range ret {
		r = append(r, arch)
	}
	return r
}

func (s Suite) ComponenetNames() []string {
	ret := []string{}
	for name, _ := range s.Components {
		ret = append(ret, name)
	}
	return ret
}

func (s Suite) Add(name string, pkg Package) {
	if _, ok := s.Components[name]; !ok {
		s.Components[name] = &Component{Packages: []Package{}}
	}
	s.Components[name].Add(pkg)
}

// }}}

// Component magic {{{

type Component struct {
	Packages []Package
}

func (c *Component) ByArch() map[dependency.Arch][]Package {
	ret := map[dependency.Arch][]Package{}

	for _, pkg := range c.Packages {
		packages := ret[pkg.Architecture]
		ret[pkg.Architecture] = append(packages, pkg)
	}

	return ret
}

func (c *Component) Arches() []dependency.Arch {
	ret := []dependency.Arch{}
	for _, pkg := range c.Packages {
		ret = append(ret, pkg.Architecture)
	}
	return ret
}

func (c *Component) Add(p Package) {
	c.Packages = append(c.Packages, p)
}

// }}}

// vim: foldmethod=marker
<|endoftext|>