<commit_before>package node

import (
	"net/http"
	"reflect"

	"github.com/gin-gonic/gin"
)

type (
	// RestAPI is a generic way to build a REST-based API for the cluster
	// database.
	RestAPI struct {
		node *Node
		typ  reflect.Type
	}
)

// NewRestAPI will instantiate a new RestAPI.
func NewRestAPI(typ interface{}, node *Node) *RestAPI {
	return &RestAPI{
		node: node,
		typ:  reflect.TypeOf(typ),
	}
}

func (r *RestAPI) new() interface{} {
	return reflect.New(r.typ).Interface()
}

func (r *RestAPI) list(c *gin.Context) {
	// Reflect is so beautiful. This will create a pointer to a slice of typ's.
	list := reflect.New(reflect.MakeSlice(reflect.SliceOf(r.typ), 0, 0).Type()).Interface()

	err := r.node.All(list, -1, 0, false)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	c.JSON(http.StatusOK, list)
}

func (r *RestAPI) create(c *gin.Context) {
	record := r.new()
	err := c.BindJSON(record)
	if err != nil {
		c.AbortWithError(http.StatusBadRequest, err)
		return
	}

	err = r.node.Save(record)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	c.Data(http.StatusAccepted, "text/plain", []byte("got it, thanks"))
}

func (r *RestAPI) replace(c *gin.Context) {
	// For now we simply treat this as a create. Should be safe.
	r.create(c)
}

func (r *RestAPI) delete(c *gin.Context) {
	record := r.new()
	err := c.BindJSON(record)
	if err != nil {
		c.AbortWithError(http.StatusBadRequest, err)
		return
	}

	err = r.node.Delete(record)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	c.Data(http.StatusAccepted, "text/plain", []byte("deleted"))
}

// Router can be used to assign a Gin routergroup.
func (r *RestAPI) Router(router *gin.RouterGroup) {
	router.GET("/", r.list)
	router.POST("", r.create)
	router.PUT("/:id", r.replace)
	router.DELETE("/:id", r.delete)
}
<commit_msg>Added source for reflect-hackery.<commit_after>package node

import (
	"net/http"
	"reflect"

	"github.com/gin-gonic/gin"
)

type (
	// RestAPI is a generic way to build a REST-based API for the cluster
	// database.
	RestAPI struct {
		node *Node
		typ  reflect.Type
	}
)

// NewRestAPI will instantiate a new RestAPI.
func NewRestAPI(typ interface{}, node *Node) *RestAPI {
	return &RestAPI{
		node: node,
		typ:  reflect.TypeOf(typ),
	}
}

func (r *RestAPI) new() interface{} {
	return reflect.New(r.typ).Interface()
}

func (r *RestAPI) list(c *gin.Context) {
	// Reflect is so beautiful. This will create a pointer to a slice of typ's.
	// Please see: http://stackoverflow.com/a/25386460/1156537
	list := reflect.New(reflect.MakeSlice(reflect.SliceOf(r.typ), 0, 0).Type()).Interface()

	err := r.node.All(list, -1, 0, false)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	c.JSON(http.StatusOK, list)
}

func (r *RestAPI) create(c *gin.Context) {
	record := r.new()
	err := c.BindJSON(record)
	if err != nil {
		c.AbortWithError(http.StatusBadRequest, err)
		return
	}

	err = r.node.Save(record)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	c.Data(http.StatusAccepted, "text/plain", []byte("got it, thanks"))
}

func (r *RestAPI) replace(c *gin.Context) {
	// For now we simply treat this as a create. Should be safe.
	r.create(c)
}

func (r *RestAPI) delete(c *gin.Context) {
	record := r.new()
	err := c.BindJSON(record)
	if err != nil {
		c.AbortWithError(http.StatusBadRequest, err)
		return
	}

	err = r.node.Delete(record)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	c.Data(http.StatusAccepted, "text/plain", []byte("deleted"))
}

// Router can be used to assign a Gin routergroup.
func (r *RestAPI) Router(router *gin.RouterGroup) {
	router.GET("/", r.list)
	router.POST("", r.create)
	router.PUT("/:id", r.replace)
	router.DELETE("/:id", r.delete)
}
<|endoftext|>


<commit_before>package creds

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
)

// Creds defines a set of AWS credentials
type Creds struct {
	AccessKey, SecretKey, SessionToken, Region string
}

// New initializes credentials from a map
func New(argCreds map[string]string) (Creds, error) {
	required := []string{"AccessKey", "SecretKey", "SessionToken"}
	for _, key := range required {
		elem, ok := argCreds[key]
		if !ok || elem == "" {
			return Creds{}, fmt.Errorf("Missing required key for Creds: %s", key)
		}
	}
	c := Creds{
		AccessKey:    argCreds["AccessKey"],
		SecretKey:    argCreds["SecretKey"],
		SessionToken: argCreds["SessionToken"],
		Region:       argCreds["Region"],
	}
	return c, nil
}

// NewFromStsSdk initializes a credential object from an AWS SDK Credentials object
func NewFromStsSdk(stsCreds *sts.Credentials) (Creds, error) {
	return New(map[string]string{
		"AccessKey":    *stsCreds.AccessKeyId,
		"SecretKey":    *stsCreds.SecretAccessKey,
		"SessionToken": *stsCreds.SessionToken,
	})
}

// NewFromEnv initializes credentials from the environment variables
func NewFromEnv() (Creds, error) {
	envCreds := make(map[string]string)
	for k, v := range Translations["envvar"] {
		if envCreds[v] == "" {
			envCreds[v] = os.Getenv(k)
		}
	}
	return New(envCreds)
}

// Translations defines common mappings for credential variables
var Translations = map[string]map[string]string{
	"envvar": {
		"AWS_ACCESS_KEY_ID":     "AccessKey",
		"AWS_SECRET_ACCESS_KEY": "SecretKey",
		"AWS_SESSION_TOKEN":     "SessionToken",
		"AWS_SECURITY_TOKEN":    "SessionToken",
		"AWS_DEFAULT_REGION":    "Region",
	},
	"console": {
		"sessionId":    "AccessKey",
		"sessionKey":   "SecretKey",
		"sessionToken": "SessionToken",
	},
}

func (c Creds) toMap() map[string]string {
	return map[string]string{
		"AccessKey":    c.AccessKey,
		"SecretKey":    c.SecretKey,
		"SessionToken": c.SessionToken,
	}
}

// Translate converts credentials based on a map of field names
func (c Creds) Translate(dictionary map[string]string) map[string]string {
	old := c.toMap()
	new := make(map[string]string)
	for k, v := range dictionary {
		new[k] = old[v]
	}
	return new
}

// ToSdk returns an AWS SDK Credentials object
func (c *Creds) ToSdk() *credentials.Credentials {
	return credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.SessionToken)
}

// ToEnvVars returns environment variables suitable for eval-ing into the shell
func (c Creds) ToEnvVars() []string {
	envCreds := c.Translate(Translations["envvar"])
	var res []string
	for k, v := range envCreds {
		res = append(res, fmt.Sprintf("export %s=%s", k, v))
	}
	sort.Strings(res)
	return res
}

var consoleTokenURL = "https://signin.%s.com"

type consoleTokenResponse struct {
	SigninToken string
}

func (c Creds) toConsoleToken() (string, error) {
	args := []string{"?Action=getSigninToken"}

	consoleCreds := c.Translate(Translations["console"])
	jsonCreds, err := json.Marshal(consoleCreds)
	if err != nil {
		return "", err
	}
	urlCreds := url.QueryEscape(string(jsonCreds))
	paramCreds := fmt.Sprintf("Session=%s", urlCreds)
	args = append(args, paramCreds)

	argString := strings.Join(args, "&")
	namespace, err := c.namespace()
	if err != nil {
		return "", err
	}
	baseURL := fmt.Sprintf(consoleTokenURL, namespace)
	url := strings.Join([]string{baseURL, "/federation", argString}, "")

	resp, err := http.Get(url)

	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	tokenObj := consoleTokenResponse{}
	if err := json.Unmarshal(body, &tokenObj); err != nil {
		return "", err
	}

	return tokenObj.SigninToken, nil
}

// ToConsoleURL returns a console URL for the role
func (c Creds) ToConsoleURL() (string, error) {
	consoleToken, err := c.toConsoleToken()
	if err != nil {
		return "", err
	}
	namespace, err := c.namespace()
	if err != nil {
		return "", err
	}
	baseURL := fmt.Sprintf(consoleTokenURL, namespace)
	targetURL := fmt.Sprintf("https://console.%s.com/", namespace)
	urlParts := []string{
		baseURL,
		"/federation",
		"?Action=login",
		"&Issuer=",
		"&Destination=",
		url.QueryEscape(targetURL),
		"&SigninToken=",
		consoleToken,
	}
	urlString := strings.Join(urlParts, "")
	return urlString, nil
}

// ToSignoutURL returns a signout URL for the console
func (c Creds) ToSignoutURL() (string, error) {
	namespace, err := c.namespace()
	if err != nil {
		return "", err
	}
	baseURL := fmt.Sprintf(consoleTokenURL, namespace)
	url := strings.Join([]string{baseURL, "/oauth?Action=logout"}, "")
	return url, nil
}

// Client returns an AWS STS client for these creds
func (c Creds) Client() *sts.STS {
	config := aws.NewConfig().WithCredentialsChainVerboseErrors(true)
	if c.AccessKey != "" {
		config.WithCredentials(c.ToSdk())
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{
		Config:            *config,
		SharedConfigState: session.SharedConfigEnable,
	}))
	return sts.New(sess)
}

func (c Creds) identity() (*sts.GetCallerIdentityOutput, error) {
	params := &sts.GetCallerIdentityInput{}
	client := c.Client()
	return client.GetCallerIdentity(params)
}

func (c Creds) partition() (string, error) {
	identity, err := c.identity()
	if err != nil {
		return "", err
	}
	pieces := strings.Split(*identity.Arn, ":")
	return pieces[1], nil
}

func (c Creds) namespace() (string, error) {
	partition, err := c.partition()
	if err != nil {
		return "", err
	}
	result, ok := namespaces[partition]
	if ok {
		return result, nil
	}
	return "", fmt.Errorf("Unknown partition: %s", partition)
}

var namespaces = map[string]string{
	"aws":        "aws.amazon",
	"aws-us-gov": "amazonaws-us-gov",
}

// AccountID returns the user's account ID
func (c Creds) AccountID() (string, error) {
	identity, err := c.identity()
	if err != nil {
		return "", err
	}
	return *identity.Account, nil
}

// MfaArn returns the user's virtual MFA token ARN
func (c Creds) MfaArn() (string, error) {
	identity, err := c.identity()
	if err != nil {
		return "", err
	}
	if strings.Index(*identity.Arn, ":user/") == -1 {
		return "", fmt.Errorf("Failed to parse MFA ARN for non-user: %s", *identity.Arn)
	}
	mfaArn := strings.Replace(*identity.Arn, ":user/", ":mfa/", 1)
	return mfaArn, nil
}

// SessionName returns the default session name
func (c Creds) SessionName() (string, error) {
	identity, err := c.identity()
	if err != nil {
		return "", err
	}
	arnChunks := strings.Split(*identity.Arn, "/")
	oldName := arnChunks[len(arnChunks)-1]
	return oldName, nil
}

// NextRoleArn returns the new role's ARN
func (c Creds) NextRoleArn(role, accountID string) (string, error) {
	if role == "" {
		return "", fmt.Errorf("Role name cannot be empty")
	}
	identity, err := c.identity()
	if err != nil {
		return "", err
	}
	partition, err := c.partition()
	if err != nil {
		return "", err
	}
	if accountID == "" {
		accountID = *identity.Account
	}
	arn := fmt.Sprintf("arn:%s:iam::%s:role/%s", partition, accountID, role)
	return arn, nil
}
<commit_msg>skip env vars that aren't set<commit_after>package creds

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
)

// Creds defines a set of AWS credentials
type Creds struct {
	AccessKey, SecretKey, SessionToken, Region string
}

// New initializes credentials from a map
func New(argCreds map[string]string) (Creds, error) {
	required := []string{"AccessKey", "SecretKey", "SessionToken"}
	for _, key := range required {
		elem, ok := argCreds[key]
		if !ok || elem == "" {
			return Creds{}, fmt.Errorf("Missing required key for Creds: %s", key)
		}
	}
	c := Creds{
		AccessKey:    argCreds["AccessKey"],
		SecretKey:    argCreds["SecretKey"],
		SessionToken: argCreds["SessionToken"],
		Region:       argCreds["Region"],
	}
	return c, nil
}

// NewFromStsSdk initializes a credential object from an AWS SDK Credentials object
func NewFromStsSdk(stsCreds *sts.Credentials) (Creds, error) {
	return New(map[string]string{
		"AccessKey":    *stsCreds.AccessKeyId,
		"SecretKey":    *stsCreds.SecretAccessKey,
		"SessionToken": *stsCreds.SessionToken,
	})
}

// NewFromEnv initializes credentials from the environment variables
func NewFromEnv() (Creds, error) {
	envCreds := make(map[string]string)
	for k, v := range Translations["envvar"] {
		if envCreds[v] == "" {
			envCreds[v] = os.Getenv(k)
		}
	}
	return New(envCreds)
}

// Translations defines common mappings for credential variables
var Translations = map[string]map[string]string{
	"envvar": {
		"AWS_ACCESS_KEY_ID":     "AccessKey",
		"AWS_SECRET_ACCESS_KEY": "SecretKey",
		"AWS_SESSION_TOKEN":     "SessionToken",
		"AWS_SECURITY_TOKEN":    "SessionToken",
		"AWS_DEFAULT_REGION":    "Region",
	},
	"console": {
		"sessionId":    "AccessKey",
		"sessionKey":   "SecretKey",
		"sessionToken": "SessionToken",
	},
}

func (c Creds) toMap() map[string]string {
	return map[string]string{
		"AccessKey":    c.AccessKey,
		"SecretKey":    c.SecretKey,
		"SessionToken": c.SessionToken,
	}
}

// Translate converts credentials based on a map of field names
func (c Creds) Translate(dictionary map[string]string) map[string]string {
	old := c.toMap()
	new := make(map[string]string)
	for k, v := range dictionary {
		new[k] = old[v]
	}
	return new
}

// ToSdk returns an AWS SDK Credentials object
func (c *Creds) ToSdk() *credentials.Credentials {
	return credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.SessionToken)
}

// ToEnvVars returns environment variables suitable for eval-ing into the shell
func (c Creds) ToEnvVars() []string {
	envCreds := c.Translate(Translations["envvar"])
	var res []string
	for k, v := range envCreds {
		if v != "" {
			res = append(res, fmt.Sprintf("export %s=%s", k, v))
		}
	}
	sort.Strings(res)
	return res
}

var consoleTokenURL = "https://signin.%s.com"

type consoleTokenResponse struct {
	SigninToken string
}

func (c Creds) toConsoleToken() (string, error) {
	args := []string{"?Action=getSigninToken"}

	consoleCreds := c.Translate(Translations["console"])
	jsonCreds, err := json.Marshal(consoleCreds)
	if err != nil {
		return "", err
	}
	urlCreds := url.QueryEscape(string(jsonCreds))
	paramCreds := fmt.Sprintf("Session=%s", urlCreds)
	args = append(args, paramCreds)

	argString := strings.Join(args, "&")
	namespace, err := c.namespace()
	if err != nil {
		return "", err
	}
	baseURL := fmt.Sprintf(consoleTokenURL, namespace)
	url := strings.Join([]string{baseURL, "/federation", argString}, "")

	resp, err := http.Get(url)

	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	tokenObj := consoleTokenResponse{}
	if err := json.Unmarshal(body, &tokenObj); err != nil {
		return "", err
	}

	return tokenObj.SigninToken, nil
}

// ToConsoleURL returns a console URL for the role
func (c Creds) ToConsoleURL() (string, error) {
	consoleToken, err := c.toConsoleToken()
	if err != nil {
		return "", err
	}
	namespace, err := c.namespace()
	if err != nil {
		return "", err
	}
	baseURL := fmt.Sprintf(consoleTokenURL, namespace)
	targetURL := fmt.Sprintf("https://console.%s.com/", namespace)
	urlParts := []string{
		baseURL,
		"/federation",
		"?Action=login",
		"&Issuer=",
		"&Destination=",
		url.QueryEscape(targetURL),
		"&SigninToken=",
		consoleToken,
	}
	urlString := strings.Join(urlParts, "")
	return urlString, nil
}

// ToSignoutURL returns a signout URL for the console
func (c Creds) ToSignoutURL() (string, error) {
	namespace, err := c.namespace()
	if err != nil {
		return "", err
	}
	baseURL := fmt.Sprintf(consoleTokenURL, namespace)
	url := strings.Join([]string{baseURL, "/oauth?Action=logout"}, "")
	return url, nil
}

// Client returns an AWS STS client for these creds
func (c Creds) Client() *sts.STS {
	config := aws.NewConfig().WithCredentialsChainVerboseErrors(true)
	if c.AccessKey != "" {
		config.WithCredentials(c.ToSdk())
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{
		Config:            *config,
		SharedConfigState: session.SharedConfigEnable,
	}))
	return sts.New(sess)
}

func (c Creds) identity() (*sts.GetCallerIdentityOutput, error) {
	params := &sts.GetCallerIdentityInput{}
	client := c.Client()
	return client.GetCallerIdentity(params)
}

func (c Creds) partition() (string, error) {
	identity, err := c.identity()
	if err != nil {
		return "", err
	}
	pieces := strings.Split(*identity.Arn, ":")
	return pieces[1], nil
}

func (c Creds) namespace() (string, error) {
	partition, err := c.partition()
	if err != nil {
		return "", err
	}
	result, ok := namespaces[partition]
	if ok {
		return result, nil
	}
	return "", fmt.Errorf("Unknown partition: %s", partition)
}

var namespaces = map[string]string{
	"aws":        "aws.amazon",
	"aws-us-gov": "amazonaws-us-gov",
}

// AccountID returns the user's account ID
func (c Creds) AccountID() (string, error) {
	identity, err := c.identity()
	if err != nil {
		return "", err
	}
	return *identity.Account, nil
}

// MfaArn returns the user's virtual MFA token ARN
func (c Creds) MfaArn() (string, error) {
	identity, err := c.identity()
	if err != nil {
		return "", err
	}
	if strings.Index(*identity.Arn, ":user/") == -1 {
		return "", fmt.Errorf("Failed to parse MFA ARN for non-user: %s", *identity.Arn)
	}
	mfaArn := strings.Replace(*identity.Arn, ":user/", ":mfa/", 1)
	return mfaArn, nil
}

// SessionName returns the default session name
func (c Creds) SessionName() (string, error) {
	identity, err := c.identity()
	if err != nil {
		return "", err
	}
	arnChunks := strings.Split(*identity.Arn, "/")
	oldName := arnChunks[len(arnChunks)-1]
	return oldName, nil
}

// NextRoleArn returns the new role's ARN
func (c Creds) NextRoleArn(role, accountID string) (string, error) {
	if role == "" {
		return "", fmt.Errorf("Role name cannot be empty")
	}
	identity, err := c.identity()
	if err != nil {
		return "", err
	}
	partition, err := c.partition()
	if err != nil {
		return "", err
	}
	if accountID == "" {
		accountID = *identity.Account
	}
	arn := fmt.Sprintf("arn:%s:iam::%s:role/%s", partition, accountID, role)
	return arn, nil
}
<|endoftext|>


<commit_before>package notificationsettings

import (
	"net/http"
	"net/url"
	"socialapi/models"
	"socialapi/request"
	"socialapi/workers/common/response"

	"github.com/koding/bongo"
)

// Create creates the notification settings with the channelId and accountId
func Create(u *url.URL, h http.Header, req *models.NotificationSettings, ctx *models.Context) (int, http.Header, interface{}, error) {
	if !ctx.IsLoggedIn() {
		return response.NewInvalidRequest(models.ErrNotLoggedIn)
	}

	channelId, err := fetchChannelId(u, c)
	if err != nil {
		return response.NewBadRequest(err)
	}

	req.AccountId = ctx.Client.Account.Id
	req.ChannelId = channelId
	// it looks like we need to add groupName into notification settings structure

	if err := req.Create(); err != nil {
		return response.NewBadRequest(err)
	}

	return response.NewOK(response.NewSuccessResponse(req))
}

// Get gets the notification settings with id
func Get(u *url.URL, header http.Header, _ interface{}) (int, http.Header, interface{}, error) {
	id, err := request.GetURIInt64(u, "id")
	if err != nil {
		return response.NewBadRequest(err)
	}

	n := models.NewNotificationSettings()
	err := n.ById(id)
	if err == bongo.RecordNotFound {
		return response.NewNotFound()
	}

	if err != nil {
		return response.NewBadRequest(err)
	}

	return response.NewOK(response.NewSuccessResponse(n))
}

func Update(u *url.URL, h http.Header, req *models.NotificationSettings, ctx *models.Context) (int, http.Header, interface{}, error) {
	id, err := request.GetURIInt64(u, "id")
	if err != nil {
		return response.NewBadRequest(err)
	}

	if !ctx.IsLoggedIn() {
		return response.NewInvalidRequest(models.ErrNotLoggedIn)
	}

	desktopSetting := req.DesktopSetting
	mobileSetting := req.MobileSetting
	isMuted := req.IsMuted
	isSuppressed := req.IsSuppressed

	if err := req.ById(id); err != nil {
		if err == bongo.RecordNotFound {
			return response.NewNotFound()
		}
		return response.NewBadRequest(err)
	}

	if req.Id == 0 {
		return response.NewBadRequest(err)
	}

	req.DesktopSetting = desktopSetting
	req.MobileSetting = mobileSetting
	req.IsMuted = isMuted
	req.IsSuppressed = isSuppressed

	if err := req.Update(); err != nil {
		return response.NewBadRequest(err)
	}

	return response.NewOK(response.NewSuccessResponse(req))
}

func fetchChannelId(u *url.URL, context *models.Context) (int64, error) {
	channelId, err := request.GetURIInt64(u, "id")
	if err != nil {
		return 0, err
	}

	c, err := models.Cache.Channel.ById(channelId)
	if err != nil {
		return 0, err
	}

	canOpen, err := c.CanOpen(context.Client.Account.Id)
	if err != nil {
		return 0, err
	}

	if !canOpen {
		return 0, models.ErrCannotOpenChannel
	}

	return c.Id, nil
}
<commit_msg>handler: Delete enpoint is added<commit_after>package notificationsettings

import (
	"net/http"
	"net/url"
	"socialapi/models"
	"socialapi/request"
	"socialapi/workers/common/response"

	"github.com/koding/bongo"
)

// Create creates the notification settings with the channelId and accountId
func Create(u *url.URL, h http.Header, req *models.NotificationSettings, ctx *models.Context) (int, http.Header, interface{}, error) {
	if !ctx.IsLoggedIn() {
		return response.NewInvalidRequest(models.ErrNotLoggedIn)
	}

	channelId, err := fetchChannelId(u, c)
	if err != nil {
		return response.NewBadRequest(err)
	}

	req.AccountId = ctx.Client.Account.Id
	req.ChannelId = channelId
	// it looks like we need to add groupName into notification settings structure

	if err := req.Create(); err != nil {
		return response.NewBadRequest(err)
	}

	return response.NewOK(response.NewSuccessResponse(req))
}

// Get gets the notification settings with id
func Get(u *url.URL, header http.Header, _ interface{}) (int, http.Header, interface{}, error) {
	id, err := request.GetURIInt64(u, "id")
	if err != nil {
		return response.NewBadRequest(err)
	}

	n := models.NewNotificationSettings()
	err := n.ById(id)
	if err == bongo.RecordNotFound {
		return response.NewNotFound()
	}

	if err != nil {
		return response.NewBadRequest(err)
	}

	return response.NewOK(response.NewSuccessResponse(n))
}

func Update(u *url.URL, h http.Header, req *models.NotificationSettings, ctx *models.Context) (int, http.Header, interface{}, error) {
	id, err := request.GetURIInt64(u, "id")
	if err != nil {
		return response.NewBadRequest(err)
	}

	if !ctx.IsLoggedIn() {
		return response.NewInvalidRequest(models.ErrNotLoggedIn)
	}

	desktopSetting := req.DesktopSetting
	mobileSetting := req.MobileSetting
	isMuted := req.IsMuted
	isSuppressed := req.IsSuppressed

	if err := req.ById(id); err != nil {
		if err == bongo.RecordNotFound {
			return response.NewNotFound()
		}
		return response.NewBadRequest(err)
	}

	if req.Id == 0 {
		return response.NewBadRequest(err)
	}

	req.DesktopSetting = desktopSetting
	req.MobileSetting = mobileSetting
	req.IsMuted = isMuted
	req.IsSuppressed = isSuppressed

	if err := req.Update(); err != nil {
		return response.NewBadRequest(err)
	}

	return response.NewOK(response.NewSuccessResponse(req))
}

func Delete(u *url.URL, h http.Header, _ interface{}) (int, http.Header, interface{}, error) {
	id, err := request.GetURIInt64(u, "id")
	if err != nil {
		return response.NewBadRequest(err)
	}

	ns := models.NotificationSettings()
	ns.Id = id

	if err := ns.ById(id); err != nil {
		if err == bongo.RecordNotFound {
			return response.NewNotFound()
		}
		return response.NewBadRequest(err)
	}

	if err := ns.Delete(); err != nil {
		return response.NewBadRequest(err)
	}

	return response.NewDeleted()
}

func fetchChannelId(u *url.URL, context *models.Context) (int64, error) {
	channelId, err := request.GetURIInt64(u, "id")
	if err != nil {
		return 0, err
	}

	c, err := models.Cache.Channel.ById(channelId)
	if err != nil {
		return 0, err
	}

	canOpen, err := c.CanOpen(context.Client.Account.Id)
	if err != nil {
		return 0, err
	}

	if !canOpen {
		return 0, models.ErrCannotOpenChannel
	}

	return c.Id, nil
}
<|endoftext|>


<commit_before>package notify

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"
)

type SlackSettings struct {
	BearerToken string `json:"bearerToken"`
	ChannelID   string `json:"channelId"`
}

type SlackNotifier struct {
	Settings *SlackSettings
}

func (s *SlackNotifier) Notify(text string) error {
	payload := map[string]interface{}{"channel": s.Settings.ChannelID, "text": "Hello.. :tada:"}
	bytes, _ := json.Marshal(payload)

	client := &http.Client{}
	r, _ := http.NewRequest(http.MethodPost, "https://slack.com/api/chat.postMessage", strings.NewReader(string(bytes)))
	r.Header.Add("Authorization", "Bearer "+s.Settings.BearerToken)
	r.Header.Add("Content-Type", "application/json")
	r.Header.Add("Content-Length", strconv.Itoa(len(string(bytes))))

	_, err := client.Do(r)
	return err
}

func (s *SlackNotifier) Initialize() {
}

func (ss *SlackSettings) Validate() error {
	errSlackProperty := func(property string) error {
		return fmt.Errorf("missing slack property %s", property)
	}
	switch {
	case ss.BearerToken == "":
		return errSlackProperty("bearerToken")
	case ss.ChannelID == "":
		return errSlackProperty("channelId")
	}
	return nil
}

func (s *SlackNotifier) String() string {
	return fmt.Sprintf("Slack Bot %s on ChannelID %s", s.Settings.BearerToken, s.Settings.ChannelID)
}
<commit_msg>Change Slack notification message<commit_after>package notify

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"
)

type SlackSettings struct {
	BearerToken string `json:"bearerToken"`
	ChannelID   string `json:"channelId"`
}

type SlackNotifier struct {
	Settings *SlackSettings
}

func (s *SlackNotifier) Notify(text string) error {
	payload := map[string]interface{}{"channel": s.Settings.ChannelID, "text": "GOSSM Notification: " + text + " not reached"}
	bytes, _ := json.Marshal(payload)

	client := &http.Client{}
	r, _ := http.NewRequest(http.MethodPost, "https://slack.com/api/chat.postMessage", strings.NewReader(string(bytes)))
	r.Header.Add("Authorization", "Bearer "+s.Settings.BearerToken)
	r.Header.Add("Content-Type", "application/json")
	r.Header.Add("Content-Length", strconv.Itoa(len(string(bytes))))

	_, err := client.Do(r)
	return err
}

func (s *SlackNotifier) Initialize() {
}

func (ss *SlackSettings) Validate() error {
	errSlackProperty := func(property string) error {
		return fmt.Errorf("missing slack property %s", property)
	}
	switch {
	case ss.BearerToken == "":
		return errSlackProperty("bearerToken")
	case ss.ChannelID == "":
		return errSlackProperty("channelId")
	}
	return nil
}

func (s *SlackNotifier) String() string {
	return fmt.Sprintf("Slack Bot %s on ChannelID %s", s.Settings.BearerToken, s.Settings.ChannelID)
}
<|endoftext|>


<commit_before>package crypto

import (
	"crypto/aes"
	"crypto/cipher"
	"fmt"

	"github.com/applee/go-common/crypto/padding"
)

func AesEncrypt(origData, key []byte) (crypted []byte, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("panic:%v", e)
		}
	}()
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	blockSize := block.BlockSize()
	origData = padding.PKCS7.Padding(origData, blockSize)
	blockMode := cipher.NewCBCEncrypter(block, key[:blockSize])
	crypted = make([]byte, len(origData))
	blockMode.CryptBlocks(crypted, origData)
	return crypted, nil
}

func AesDecrypt(crypted, key []byte) (original []byte, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("panic:%v", e)
		}
	}()
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	blockSize := block.BlockSize()
	blockMode := cipher.NewCBCDecrypter(block, key[:blockSize])
	origData := make([]byte, len(crypted))
	blockMode.CryptBlocks(origData, crypted)
	original, _ = padding.PKCS7.Unpadding(origData, blockSize)
	return original, nil
}
<commit_msg>fix(crypto): return error of unpadding in AesDecrypt<commit_after>package crypto

import (
	"crypto/aes"
	"crypto/cipher"
	"fmt"

	"github.com/applee/go-common/crypto/padding"
)

func AesEncrypt(origData, key []byte) (crypted []byte, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("panic:%v", e)
		}
	}()
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	blockSize := block.BlockSize()
	origData = padding.PKCS7.Padding(origData, blockSize)
	fmt.Println(origData)
	blockMode := cipher.NewCBCEncrypter(block, key[:blockSize])
	crypted = make([]byte, len(origData))
	blockMode.CryptBlocks(crypted, origData)
	return crypted, nil
}

func AesDecrypt(crypted, key []byte) (original []byte, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("panic:%v", e)
		}
	}()
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	blockSize := block.BlockSize()
	blockMode := cipher.NewCBCDecrypter(block, key[:blockSize])
	origData := make([]byte, len(crypted))
	blockMode.CryptBlocks(origData, crypted)
	return padding.PKCS7.Unpadding(origData, blockSize)
}
<|endoftext|>


<commit_before>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// +build !gogit

package git

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
)

// ResolveReference resolves a name to a reference
func (repo *Repository) ResolveReference(name string) (string, error) {
	stdout, err := NewCommand("show-ref", "--hash", name).RunInDir(repo.Path)
	if err != nil {
		if strings.Contains(err.Error(), "not a valid ref") {
			return "", ErrNotExist{name, ""}
		}
		return "", err
	}
	stdout = strings.TrimSpace(stdout)
	if stdout == "" {
		return "", ErrNotExist{name, ""}
	}

	return stdout, nil
}

// GetRefCommitID returns the last commit ID string of given reference (branch or tag).
func (repo *Repository) GetRefCommitID(name string) (string, error) {
	stdout, err := NewCommand("show-ref", "--verify", "--hash", name).RunInDir(repo.Path)
	if err != nil {
		if strings.Contains(err.Error(), "not a valid ref") {
			return "", ErrNotExist{name, ""}
		}
		return "", err
	}

	return strings.TrimSpace(stdout), nil
}

// IsCommitExist returns true if given commit exists in current repository.
func (repo *Repository) IsCommitExist(name string) bool {
	_, err := NewCommand("cat-file", "-e", name).RunInDir(repo.Path)
	return err == nil
}

func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
	stdoutReader, stdoutWriter := io.Pipe()
	defer func() {
		_ = stdoutReader.Close()
		_ = stdoutWriter.Close()
	}()

	go func() {
		stderr := strings.Builder{}
		err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, &stderr, strings.NewReader(id.String()+"\n"))
		if err != nil {
			_ = stdoutWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
		} else {
			_ = stdoutWriter.Close()
		}
	}()

	bufReader := bufio.NewReader(stdoutReader)
	_, typ, size, err := ReadBatchLine(bufReader)
	if err != nil {
		return nil, err
	}

	switch typ {
	case "tag":
		// then we need to parse the tag
		// and load the commit
		data, err := ioutil.ReadAll(io.LimitReader(bufReader, size))
		if err != nil {
			return nil, err
		}
		tag, err := parseTagData(data)
		if err != nil {
			return nil, err
		}
		tag.repo = repo

		commit, err := tag.Commit()
		if err != nil {
			return nil, err
		}

		commit.CommitMessage = strings.TrimSpace(tag.Message)
		commit.Author = tag.Tagger
		commit.Signature = tag.Signature

		return commit, nil
	case "commit":
		return CommitFromReader(repo, id, io.LimitReader(bufReader, size))
	default:
		_ = stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ))
		log("Unknown typ: %s", typ)
		return nil, ErrNotExist{
			ID: id.String(),
		}
	}
}
<commit_msg>Fix when a commit not found returned 500 (#14732)<commit_after>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// +build !gogit

package git

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
)

// ResolveReference resolves a name to a reference
func (repo *Repository) ResolveReference(name string) (string, error) {
	stdout, err := NewCommand("show-ref", "--hash", name).RunInDir(repo.Path)
	if err != nil {
		if strings.Contains(err.Error(), "not a valid ref") {
			return "", ErrNotExist{name, ""}
		}
		return "", err
	}
	stdout = strings.TrimSpace(stdout)
	if stdout == "" {
		return "", ErrNotExist{name, ""}
	}

	return stdout, nil
}

// GetRefCommitID returns the last commit ID string of given reference (branch or tag).
func (repo *Repository) GetRefCommitID(name string) (string, error) {
	stdout, err := NewCommand("show-ref", "--verify", "--hash", name).RunInDir(repo.Path)
	if err != nil {
		if strings.Contains(err.Error(), "not a valid ref") {
			return "", ErrNotExist{name, ""}
		}
		return "", err
	}

	return strings.TrimSpace(stdout), nil
}

// IsCommitExist returns true if given commit exists in current repository.
func (repo *Repository) IsCommitExist(name string) bool {
	_, err := NewCommand("cat-file", "-e", name).RunInDir(repo.Path)
	return err == nil
}

func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
	stdoutReader, stdoutWriter := io.Pipe()
	defer func() {
		_ = stdoutReader.Close()
		_ = stdoutWriter.Close()
	}()

	go func() {
		stderr := strings.Builder{}
		err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, &stderr, strings.NewReader(id.String()+"\n"))
		if err != nil {
			_ = stdoutWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
		} else {
			_ = stdoutWriter.Close()
		}
	}()

	bufReader := bufio.NewReader(stdoutReader)
	_, typ, size, err := ReadBatchLine(bufReader)
	if err != nil {
		if errors.Is(err, io.EOF) {
			return nil, ErrNotExist{ID: id.String()}
		}
		return nil, err
	}

	switch typ {
	case "missing":
		return nil, ErrNotExist{ID: id.String()}
	case "tag":
		// then we need to parse the tag
		// and load the commit
		data, err := ioutil.ReadAll(io.LimitReader(bufReader, size))
		if err != nil {
			return nil, err
		}
		tag, err := parseTagData(data)
		if err != nil {
			return nil, err
		}
		tag.repo = repo

		commit, err := tag.Commit()
		if err != nil {
			return nil, err
		}

		commit.CommitMessage = strings.TrimSpace(tag.Message)
		commit.Author = tag.Tagger
		commit.Signature = tag.Signature

		return commit, nil
	case "commit":
		return CommitFromReader(repo, id, io.LimitReader(bufReader, size))
	default:
		_ = stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ))
		log("Unknown typ: %s", typ)
		return nil, ErrNotExist{
			ID: id.String(),
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"../nsq"
	"bitly/notify"
	"errors"
	"log"
	"sync"
	"time"
)

const (
	msgTimeoutMs = 60000
)

type Channel struct {
	sync.RWMutex
	name                string
	backend             nsq.BackendQueue
	incomingMessageChan chan *nsq.Message
	memoryMsgChan       chan *nsq.Message
	clientMessageChan   chan *nsq.Message
	inFlightMessages    map[string]*nsq.Message
	requeueCount        int64
	getCount            int64
	putCount            int64
	timeoutCount        int64
	topicName           string
	exitChan            chan int
	clients             []*nsq.ServerClient
}

// Channel constructor
func NewChannel(topicName string, channelName string, inMemSize int64, dataPath string, maxBytesPerFile int64) *Channel {
	c := &Channel{
		topicName: topicName,
		name:      channelName,
		// backend names, for uniqueness, automatically include the topic... <topic>:<channel>
		backend:             NewDiskQueue(topicName+":"+channelName, dataPath, maxBytesPerFile),
		incomingMessageChan: make(chan *nsq.Message, 5),
		memoryMsgChan:       make(chan *nsq.Message, inMemSize),
		clientMessageChan:   make(chan *nsq.Message),
		inFlightMessages:    make(map[string]*nsq.Message),
		exitChan:            make(chan int),
		clients:             make([]*nsq.ServerClient, 0, 5),
	}
	go c.router()
	go c.messagePump()
	notify.Post("new_channel", c)
	return c
}

func (c *Channel) MemoryChan() chan *nsq.Message {
	return c.memoryMsgChan
}

func (c *Channel) BackendQueue() nsq.BackendQueue {
	return c.backend
}

// PutMessage writes to the appropriate incoming
// message channel
func (c *Channel) PutMessage(msg *nsq.Message) {
	c.putCount += 1
	c.incomingMessageChan <- msg
}

func (c *Channel) FinishMessage(id []byte) error {
	msg, err := c.popInFlightMessage(id)
	if err != nil {
		log.Printf("ERROR: failed to finish message(%s) - %s", id, err.Error())
	} else {
		c.endWaitToAutoRequeue(msg)
	}
	return err
}

func (c *Channel) RequeueMessage(id []byte, timeoutMs int) error {
	if timeoutMs == 0 {
		return c.doRequeue(id)
	}
	go func() {
		msg, err := c.getInFlightMessage(id)
		if err != nil {
			log.Printf("ERROR: failed to defer requeue message(%s) - %s", id, err.Error())
			return
		}
		c.endWaitToAutoRequeue(msg)
		<-time.After(time.Duration(timeoutMs) * time.Millisecond)
		c.doRequeue(id)
	}()
	return nil
}

func (c *Channel) doRequeue(id []byte) error {
	msg, err := c.popInFlightMessage(id)
	if err != nil {
		log.Printf("ERROR: failed to requeue message(%s) - %s", id, err.Error())
	} else {
		c.endWaitToAutoRequeue(msg)
		c.requeueCount += 1
		// decr putCount so that we don't double count requeue puts
		c.putCount -= 1
		c.PutMessage(msg)
	}
	return err
}

// Router handles the muxing of incoming Channel messages, either writing
// to the in-memory channel or to the backend
func (c *Channel) router() {
	for {
		select {
		case msg := <-c.incomingMessageChan:
			select {
			case c.memoryMsgChan <- msg:
			default:
				data, err := msg.Encode()
				if err != nil {
					log.Printf("ERROR: failed to Encode() message - %s", err.Error())
					continue
				}
				err = c.backend.Put(data)
				if err != nil {
					log.Printf("ERROR: t.backend.Put() - %s", err.Error())
					// TODO: requeue?
				}
			}
		case <-c.exitChan:
			return
		}
	}
}

// pushInFlightMessage atomically adds a message to the in-flight dictionary
func (c *Channel) pushInFlightMessage(msg *nsq.Message) error {
	c.Lock()
	defer c.Unlock()

	_, ok := c.inFlightMessages[string(msg.Id)]
	if ok {
		return errors.New("E_ID_ALREADY_IN_FLIGHT")
	}
	c.inFlightMessages[string(msg.Id)] = msg

	return nil
}

func (c *Channel) popInFlightMessage(id []byte) (*nsq.Message, error) {
	c.Lock()
	defer c.Unlock()

	msg, ok := c.inFlightMessages[string(id)]
	if !ok {
		return nil, errors.New("E_ID_NOT_IN_FLIGHT")
	}
	delete(c.inFlightMessages, string(id))

	return msg, nil
}

func (c *Channel) getInFlightMessage(id []byte) (*nsq.Message, error) {
	c.RLock()
	defer c.RUnlock()

	msg, ok := c.inFlightMessages[string(id)]
	if !ok {
		return nil, errors.New("E_ID_NOT_IN_FLIGHT")
	}

	return msg, nil
}

// waitToAutoRequeue sleeps until forcefully closed (doing nothing) or
// times out (requeue the message automatically)
func (c *Channel) waitToAutoRequeue(msg *nsq.Message) {
	// TODO: we need cleanup for these goroutines
	timer := time.NewTimer(time.Duration(msgTimeoutMs) * time.Millisecond)
	defer timer.Stop()

	select {
	case <-timer.C:
	case <-msg.UtilChan:
		return
	}

	c.timeoutCount += 1
	err := c.RequeueMessage(msg.Id, 0)
	if err != nil {
		log.Printf("ERROR: channel(%s) RequeueMessage(%s) - %s", c.name, msg.Id, err.Error())
	}
}

// endWaitToAutoRequeue forcefully closes the goroutine for auto-requeue
// for the specified message
func (c *Channel) endWaitToAutoRequeue(msg *nsq.Message) {
	select {
	case msg.UtilChan <- 1:
	default:
	}
}

// messagePump reads messages from either memory or backend and writes
// to the client output go channel
//
// it is also performs in-flight accounting and initiates the auto-requeue
// goroutine
func (c *Channel) messagePump() {
	var msg *nsq.Message
	var buf []byte
	var err error

	for {
		select {
		case msg = <-c.memoryMsgChan:
		case buf = <-c.backend.ReadChan():
			msg, err = nsq.DecodeMessage(buf)
			if err != nil {
				log.Printf("ERROR: failed to decode message - %s", err.Error())
				continue
			}
		case <-c.exitChan:
			return
		}

		msg.Retries += 1
		c.pushInFlightMessage(msg)
		c.clientMessageChan <- msg
		go c.waitToAutoRequeue(msg)
		c.getCount += 1
	}
}

func (c *Channel) flushInFlight() {
	for _, msg := range c.inFlightMessages {
		data, err := msg.Encode()
		if err != nil {
			log.Printf("ERROR: failed to Encode() message - %s", err.Error())
			continue
		}
		err = c.backend.Put(data)
		if err != nil {
			log.Printf("ERROR: t.backend.Put() - %s", err.Error())
		}
	}
}

func (c *Channel) Close() error {
	var err error

	log.Printf("CHANNEL(%s): closing", c.name)

	close(c.exitChan)
	FlushQueue(c)
	c.flushInFlight()

	err = c.backend.Close()
	if err != nil {
		return err
	}

	return nil
}

func (c *Channel) AddClient(client *nsq.ServerClient) {
	c.Lock()
	defer c.Unlock()

	found := false
	for _, cli := range c.clients {
		if cli == client {
			found = true
			break
		}
	}

	if !found {
		c.clients = append(c.clients, client)
	}
}

func (c *Channel) RemoveClient(client *nsq.ServerClient) {
	c.Lock()
	defer c.Unlock()

	finalClients := make([]*nsq.ServerClient, 0, len(c.clients))
	for _, cli := range c.clients {
		if cli != client {
			finalClients = append(finalClients, cli)
		}
	}

	c.clients = finalClients
}
<commit_msg>comment channel.go<commit_after>package main

import (
	"../nsq"
	"bitly/notify"
	"errors"
	"log"
	"sync"
	"time"
)

const (
	// the amount of time to wait for a client to respond to a message
	msgTimeoutMs = 60000
)

// Channel represents the concrete type for a NSQ channel (and also
// implements the Queue interface)
//
// There can be many channels per topic and each with there own distinct
// clients subscribed.
//
// Channels maintain all client and message, orchestrating in-flight
// messages, timeouts, requeueing, etc.
type Channel struct {
	sync.RWMutex // embed a r/w mutex

	topicName string
	name      string

	backend nsq.BackendQueue

	incomingMessageChan chan *nsq.Message
	memoryMsgChan       chan *nsq.Message
	clientMessageChan   chan *nsq.Message
	exitChan            chan int

	// state tracking
	clients          []*nsq.ServerClient
	inFlightMessages map[string]*nsq.Message

	// stat counters
	requeueCount int64
	getCount     int64
	putCount     int64
	timeoutCount int64
}

// NewChannel creates a new instance of the Channel type and returns a pointer
func NewChannel(topicName string, channelName string, inMemSize int64, dataPath string, maxBytesPerFile int64) *Channel {
	c := &Channel{
		topicName: topicName,
		name:      channelName,
		// backend names, for uniqueness, automatically include the topic... <topic>:<channel>
		backend:             NewDiskQueue(topicName+":"+channelName, dataPath, maxBytesPerFile),
		incomingMessageChan: make(chan *nsq.Message, 5),
		memoryMsgChan:       make(chan *nsq.Message, inMemSize),
		clientMessageChan:   make(chan *nsq.Message),
		exitChan:            make(chan int),
		clients:             make([]*nsq.ServerClient, 0, 5),
		inFlightMessages:    make(map[string]*nsq.Message),
	}
	go c.router()
	go c.messagePump()
	notify.Post("new_channel", c)
	return c
}

// MemoryChan implements the Queue interface
func (c *Channel) MemoryChan() chan *nsq.Message {
	return c.memoryMsgChan
}

// BackendQueue implements the Queue interface
func (c *Channel) BackendQueue() nsq.BackendQueue {
	return c.backend
}

// PutMessage writes to the appropriate incoming message channel
// (which will be routed asynchronously)
func (c *Channel) PutMessage(msg *nsq.Message) {
	c.putCount += 1
	c.incomingMessageChan <- msg
}

// FinishMessage successfully discards an in-flight message
func (c *Channel) FinishMessage(id []byte) error {
	msg, err := c.popInFlightMessage(id)
	if err != nil {
		log.Printf("ERROR: failed to finish message(%s) - %s", id, err.Error())
	} else {
		c.endWaitToAutoRequeue(msg)
	}
	return err
}

// RequeueMessage requeues a message based on `timeoutMs`, ie:
//
// `timeoutMs` == 0 - requeue a message immediately
// `timeoutMs`  > 0 - asynchronously wait for the specified timeout
//     and requeue a message (aka "deferred requeue")
//
func (c *Channel) RequeueMessage(id []byte, timeoutMs int) error {
	if timeoutMs == 0 {
		return c.doRequeue(id)
	}
	go func() {
		// TODO: we need cleanup for these goroutines
		msg, err := c.getInFlightMessage(id)
		if err != nil {
			log.Printf("ERROR: failed to defer requeue message(%s) - %s", id, err.Error())
			return
		}
		c.endWaitToAutoRequeue(msg)
		<-time.After(time.Duration(timeoutMs) * time.Millisecond)
		c.doRequeue(id)
	}()
	return nil
}

// doRequeue performs the low level operations to requeue a message
func (c *Channel) doRequeue(id []byte) error {
	msg, err := c.popInFlightMessage(id)
	if err != nil {
		log.Printf("ERROR: failed to requeue message(%s) - %s", id, err.Error())
	} else {
		c.endWaitToAutoRequeue(msg)
		c.requeueCount += 1
		// decr putCount so that we don't double count requeue puts
		c.putCount -= 1
		c.PutMessage(msg)
	}
	return err
}

// Router handles the muxing of incoming Channel messages, either writing
// to the in-memory channel or to the backend
func (c *Channel) router() {
	for {
		select {
		case msg := <-c.incomingMessageChan:
			select {
			case c.memoryMsgChan <- msg:
			default:
				data, err := msg.Encode()
				if err != nil {
					log.Printf("ERROR: failed to Encode() message - %s", err.Error())
					continue
				}
				err = c.backend.Put(data)
				if err != nil {
					log.Printf("ERROR: t.backend.Put() - %s", err.Error())
					// TODO: requeue?
				}
			}
		case <-c.exitChan:
			return
		}
	}
}

// pushInFlightMessage atomically adds a message to the in-flight dictionary
func (c *Channel) pushInFlightMessage(msg *nsq.Message) error {
	c.Lock()
	defer c.Unlock()

	_, ok := c.inFlightMessages[string(msg.Id)]
	if ok {
		return errors.New("E_ID_ALREADY_IN_FLIGHT")
	}
	c.inFlightMessages[string(msg.Id)] = msg

	return nil
}

// popInFlightMessage atomically removes a message from the in-flight dictionary
func (c *Channel) popInFlightMessage(id []byte) (*nsq.Message, error) {
	c.Lock()
	defer c.Unlock()

	msg, ok := c.inFlightMessages[string(id)]
	if !ok {
		return nil, errors.New("E_ID_NOT_IN_FLIGHT")
	}
	delete(c.inFlightMessages, string(id))

	return msg, nil
}

// getInFlightMessage retrieves a message from the in-flight dictionary by ID
func (c *Channel) getInFlightMessage(id []byte) (*nsq.Message, error) {
	c.RLock()
	defer c.RUnlock()

	msg, ok := c.inFlightMessages[string(id)]
	if !ok {
		return nil, errors.New("E_ID_NOT_IN_FLIGHT")
	}

	return msg, nil
}

// waitToAutoRequeue sleeps until forcefully closed (doing nothing) or
// times out (requeue the message automatically)
func (c *Channel) waitToAutoRequeue(msg *nsq.Message) {
	// TODO: we need cleanup for these goroutines
	timer := time.NewTimer(time.Duration(msgTimeoutMs) * time.Millisecond)
	defer timer.Stop()

	select {
	case <-timer.C:
	case <-msg.UtilChan:
		return
	}

	c.timeoutCount += 1
	err := c.RequeueMessage(msg.Id, 0)
	if err != nil {
		log.Printf("ERROR: channel(%s) RequeueMessage(%s) - %s", c.name, msg.Id, err.Error())
	}
}

// endWaitToAutoRequeue forcefully closes the goroutine for auto-requeue
// for the specified message
func (c *Channel) endWaitToAutoRequeue(msg *nsq.Message) {
	select {
	case msg.UtilChan <- 1:
	default:
	}
}

// messagePump reads messages from either memory or backend and writes
// to the client output go channel
//
// it is also performs in-flight accounting and initiates the auto-requeue
// goroutine
func (c *Channel) messagePump() {
	var msg *nsq.Message
	var buf []byte
	var err error

	for {
		select {
		case msg = <-c.memoryMsgChan:
		case buf = <-c.backend.ReadChan():
			msg, err = nsq.DecodeMessage(buf)
			if err != nil {
				log.Printf("ERROR: failed to decode message - %s", err.Error())
				continue
			}
		case <-c.exitChan:
			return
		}

		msg.Retries += 1
		c.pushInFlightMessage(msg)
		c.clientMessageChan <- msg
		go c.waitToAutoRequeue(msg)
		c.getCount += 1
	}
}

// flushInFlight writes all in-flight messages to the backend
func (c *Channel) flushInFlight() {
	for _, msg := range c.inFlightMessages {
		data, err := msg.Encode()
		if err != nil {
			log.Printf("ERROR: failed to Encode() message - %s", err.Error())
			continue
		}
		err = c.backend.Put(data)
		if err != nil {
			log.Printf("ERROR: t.backend.Put() - %s", err.Error())
		}
	}
}

// Close cleanly closes the Channel
func (c *Channel) Close() error {
	var err error

	log.Printf("CHANNEL(%s): closing", c.name)

	close(c.exitChan)
	FlushQueue(c)
	c.flushInFlight()

	err = c.backend.Close()
	if err != nil {
		return err
	}

	return nil
}

// AddClient adds the ServerClient the Channel's client list
func (c *Channel) AddClient(client *nsq.ServerClient) {
	c.Lock()
	defer c.Unlock()

	found := false
	for _, cli := range c.clients {
		if cli == client {
			found = true
			break
		}
	}

	if !found {
		c.clients = append(c.clients, client)
	}
}

// RemoveClient removes the ServerClient from the Channel's client list
func (c *Channel) RemoveClient(client *nsq.ServerClient) {
	c.Lock()
	defer c.Unlock()

	finalClients := make([]*nsq.ServerClient, 0, len(c.clients))
	for _, cli := range c.clients {
		if cli != client {
			finalClients = append(finalClients, cli)
		}
	}

	c.clients = finalClients
}
<|endoftext|>


<commit_before>package volman_test

import (
	"encoding/json"
	"os"
	"testing"

	"fmt"
	"path"
	"path/filepath"

	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/inigo/helpers"
	"code.cloudfoundry.org/inigo/world"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/ginkgoreporter"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/localip"
	"code.cloudfoundry.org/voldriver"
	"code.cloudfoundry.org/volman"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

var (
	componentMaker world.ComponentMaker

	gardenProcess ifrit.Process
	gardenClient  garden.Client

	volmanClient           volman.Manager
	driverSyncer           ifrit.Runner
	driverSyncerProcess    ifrit.Process
	localDriverRunner      ifrit.Runner
	localNodePluginRunner  ifrit.Runner
	localDriverProcess     ifrit.Process
	localNodePluginProcess ifrit.Process

	driverClient    voldriver.Driver
	localNodeClient volman.Plugin

	logger lager.Logger

	driverPluginsPath string
)

var _ = SynchronizedBeforeSuite(func() []byte {
	payload, err := json.Marshal(world.BuiltArtifacts{
		Executables: CompileTestedExecutables(),
	})
	Expect(err).NotTo(HaveOccurred())

	return payload
}, func(encodedBuiltArtifacts []byte) {
	var builtArtifacts world.BuiltArtifacts

	err := json.Unmarshal(encodedBuiltArtifacts, &builtArtifacts)
	Expect(err).NotTo(HaveOccurred())

	localIP, err := localip.LocalIP()
	Expect(err).NotTo(HaveOccurred())

	componentMaker = helpers.MakeComponentMaker(builtArtifacts, localIP)
	componentMaker.Setup()
})

var _ = AfterSuite(func() {
	componentMaker.Teardown()
})

var _ = BeforeEach(func() {
	logger = lagertest.NewTestLogger("volman-inigo-suite")

	gardenProcess = ginkgomon.Invoke(componentMaker.Garden())
	gardenClient = componentMaker.GardenClient()

	localDriverRunner, driverClient = componentMaker.VolmanDriver(logger)
	localDriverProcess = ginkgomon.Invoke(localDriverRunner)

	localNodePluginRunner/*, localNodeClient */= componentMaker.CsiLocalNodePlugin(logger)
	localNodePluginProcess = ginkgomon.Invoke(localNodePluginRunner)

	// make a dummy spec file not corresponding to a running driver just to make sure volman ignores it
	driverPluginsPath = path.Join(componentMaker.VolmanDriverConfigDir, fmt.Sprintf("node-%d", config.GinkgoConfig.ParallelNode))
	voldriver.WriteDriverSpec(logger, driverPluginsPath, "deaddriver", "json", []byte(`{"Name":"deaddriver","Addr":"https://127.0.0.1:1111"}`))

	volmanClient, driverSyncer = componentMaker.VolmanClient(logger)
	driverSyncerProcess = ginkgomon.Invoke(driverSyncer)
})

var _ = AfterEach(func() {
	destroyContainerErrors := helpers.CleanupGarden(gardenClient)

	helpers.StopProcesses(gardenProcess, driverSyncerProcess, localDriverProcess, localNodePluginProcess)

	Expect(destroyContainerErrors).To(
		BeEmpty(),
		"%d containers failed to be destroyed!",
		len(destroyContainerErrors),
	)

	os.Remove(filepath.Join(driverPluginsPath, "deaddriver.json"))
})

func TestVolman(t *testing.T) {
	helpers.RegisterDefaultTimeouts()

	RegisterFailHandler(Fail)

	RunSpecsWithDefaultAndCustomReporters(t, "Volman Integration Suite", []Reporter{
		ginkgoreporter.New(GinkgoWriter),
	})
}

func CompileTestedExecutables() world.BuiltExecutables {
	var err error

	builtExecutables := world.BuiltExecutables{}

	builtExecutables["garden"], err = gexec.BuildIn(os.Getenv("GARDEN_GOPATH"), "code.cloudfoundry.org/guardian/cmd/gdn", "-race", "-a", "-tags", "daemon")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["local-driver"], err = gexec.Build("code.cloudfoundry.org/localdriver/cmd/localdriver", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["local-node-plugin"], err = gexec.Build("github.com/jeffpak/local-node-plugin/cmd/localnodeplugin", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["auctioneer"], err = gexec.BuildIn(os.Getenv("AUCTIONEER_GOPATH"), "code.cloudfoundry.org/auctioneer/cmd/auctioneer", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["rep"], err = gexec.BuildIn(os.Getenv("REP_GOPATH"), "code.cloudfoundry.org/rep/cmd/rep", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["bbs"], err = gexec.BuildIn(os.Getenv("BBS_GOPATH"), "code.cloudfoundry.org/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["locket"], err = gexec.BuildIn(os.Getenv("LOCKET_GOPATH"), "code.cloudfoundry.org/locket/cmd/locket", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["file-server"], err = gexec.BuildIn(os.Getenv("FILE_SERVER_GOPATH"), "code.cloudfoundry.org/fileserver/cmd/file-server", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["route-emitter"], err = gexec.BuildIn(os.Getenv("ROUTE_EMITTER_GOPATH"), "code.cloudfoundry.org/route-emitter/cmd/route-emitter", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["router"], err = gexec.BuildIn(os.Getenv("ROUTER_GOPATH"), "code.cloudfoundry.org/gorouter", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["ssh-proxy"], err = gexec.Build("code.cloudfoundry.org/diego-ssh/cmd/ssh-proxy", "-race")
	Expect(err).NotTo(HaveOccurred())

	return builtExecutables
}
<commit_msg>make a dummy spec file to make sure volman ignores it<commit_after>package volman_test

import (
	"encoding/json"
	"os"
	"testing"

	"fmt"
	"path"
	"path/filepath"

	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/inigo/helpers"
	"code.cloudfoundry.org/inigo/world"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/ginkgoreporter"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/localip"
	"code.cloudfoundry.org/voldriver"
	"code.cloudfoundry.org/volman"
	"github.com/Kaixiang/csiplugin"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

var (
	componentMaker world.ComponentMaker

	gardenProcess ifrit.Process
	gardenClient  garden.Client

	volmanClient           volman.Manager
	driverSyncer           ifrit.Runner
	driverSyncerProcess    ifrit.Process
	localDriverRunner      ifrit.Runner
	localNodePluginRunner  ifrit.Runner
	localDriverProcess     ifrit.Process
	localNodePluginProcess ifrit.Process

	driverClient    voldriver.Driver
	localNodeClient volman.Plugin

	logger lager.Logger

	driverPluginsPath string
)

var _ = SynchronizedBeforeSuite(func() []byte {
	payload, err := json.Marshal(world.BuiltArtifacts{
		Executables: CompileTestedExecutables(),
	})
	Expect(err).NotTo(HaveOccurred())

	return payload
}, func(encodedBuiltArtifacts []byte) {
	var builtArtifacts world.BuiltArtifacts

	err := json.Unmarshal(encodedBuiltArtifacts, &builtArtifacts)
	Expect(err).NotTo(HaveOccurred())

	localIP, err := localip.LocalIP()
	Expect(err).NotTo(HaveOccurred())

	componentMaker = helpers.MakeComponentMaker(builtArtifacts, localIP)
	componentMaker.Setup()
})

var _ = AfterSuite(func() {
	componentMaker.Teardown()
})

var _ = BeforeEach(func() {
	logger = lagertest.NewTestLogger("volman-inigo-suite")

	gardenProcess = ginkgomon.Invoke(componentMaker.Garden())
	gardenClient = componentMaker.GardenClient()

	localDriverRunner, driverClient = componentMaker.VolmanDriver(logger)
	localDriverProcess = ginkgomon.Invoke(localDriverRunner)

	localNodePluginRunner /*, localNodeClient */ = componentMaker.CsiLocalNodePlugin(logger)
	localNodePluginProcess = ginkgomon.Invoke(localNodePluginRunner)

	// make a dummy spec file not corresponding to a running driver just to make sure volman ignores it
	driverPluginsPath = path.Join(componentMaker.VolmanDriverConfigDir, fmt.Sprintf("node-%d", config.GinkgoConfig.ParallelNode))
	voldriver.WriteDriverSpec(logger, driverPluginsPath, "deaddriver", "json", []byte(`{"Name":"deaddriver","Addr":"https://127.0.0.1:1111"}`))

	// make a dummy spec file not corresponding to a running node plugin just to make sure volman ignores it
	driverPluginsPath = path.Join(componentMaker.VolmanDriverConfigDir, fmt.Sprintf("local-node-plugin-%d", config.GinkgoConfig.ParallelNode))
	csiSpec := csiplugin.CsiPluginSpec{
		Name:    "dead-csi-plugin",
		Address: "127.0.0.1:2222",
	}
	csiplugin.WriteSpec(logger, driverPluginsPath, csiSpec)

	volmanClient, driverSyncer = componentMaker.VolmanClient(logger)
	driverSyncerProcess = ginkgomon.Invoke(driverSyncer)
})

var _ = AfterEach(func() {
	destroyContainerErrors := helpers.CleanupGarden(gardenClient)

	helpers.StopProcesses(gardenProcess, driverSyncerProcess, localDriverProcess, localNodePluginProcess)

	Expect(destroyContainerErrors).To(
		BeEmpty(),
		"%d containers failed to be destroyed!",
		len(destroyContainerErrors),
	)

	os.Remove(filepath.Join(driverPluginsPath, "deaddriver.json"))
})

func TestVolman(t *testing.T) {
	helpers.RegisterDefaultTimeouts()

	RegisterFailHandler(Fail)

	RunSpecsWithDefaultAndCustomReporters(t, "Volman Integration Suite", []Reporter{
		ginkgoreporter.New(GinkgoWriter),
	})
}

func CompileTestedExecutables() world.BuiltExecutables {
	var err error

	builtExecutables := world.BuiltExecutables{}

	builtExecutables["garden"], err = gexec.BuildIn(os.Getenv("GARDEN_GOPATH"), "code.cloudfoundry.org/guardian/cmd/gdn", "-race", "-a", "-tags", "daemon")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["local-driver"], err = gexec.Build("code.cloudfoundry.org/localdriver/cmd/localdriver", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["local-node-plugin"], err = gexec.Build("github.com/jeffpak/local-node-plugin/cmd/localnodeplugin", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["auctioneer"], err = gexec.BuildIn(os.Getenv("AUCTIONEER_GOPATH"), "code.cloudfoundry.org/auctioneer/cmd/auctioneer", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["rep"], err = gexec.BuildIn(os.Getenv("REP_GOPATH"), "code.cloudfoundry.org/rep/cmd/rep", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["bbs"], err = gexec.BuildIn(os.Getenv("BBS_GOPATH"), "code.cloudfoundry.org/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["locket"], err = gexec.BuildIn(os.Getenv("LOCKET_GOPATH"), "code.cloudfoundry.org/locket/cmd/locket", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["file-server"], err = gexec.BuildIn(os.Getenv("FILE_SERVER_GOPATH"), "code.cloudfoundry.org/fileserver/cmd/file-server", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["route-emitter"], err = gexec.BuildIn(os.Getenv("ROUTE_EMITTER_GOPATH"), "code.cloudfoundry.org/route-emitter/cmd/route-emitter", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["router"], err = gexec.BuildIn(os.Getenv("ROUTER_GOPATH"), "code.cloudfoundry.org/gorouter", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["ssh-proxy"], err = gexec.Build("code.cloudfoundry.org/diego-ssh/cmd/ssh-proxy", "-race")
	Expect(err).NotTo(HaveOccurred())

	return builtExecutables
}
<|endoftext|>


<commit_before>package farm

func uoH(x, y, mul uint64, r uint) uint64 {
	a := (x ^ y) * mul
	a ^= (a >> 47)
	b := (y ^ a) * mul
	return rotate64(b, r) * mul
}

// Hash64WithSeeds hashes a byte slice and two uint64 seeds and returns a uint64 hash value
func Hash64WithSeeds(s []byte, seed0, seed1 uint64) uint64 {
	slen := len(s)
	if slen <= 64 {
		return naHash64WithSeeds(s, seed0, seed1)
	}

	// For strings over 64 bytes we loop.  Internal state consists of
	// 64 bytes: u, v, w, x, y, and z.
	x := seed0
	y := seed1*k2 + 113
	z := shiftMix(y*k2) * k2
	v := uint128{seed0, seed1}
	var w uint128
	u := x - z
	x *= k2
	mul := k2 + (u & 0x82)

	// Set end so that after the loop we have 1 to 64 bytes left to process.
	endIdx := ((slen - 1) / 64) * 64
	last64Idx := endIdx + ((slen - 1) & 63) - 63
	last64 := s[last64Idx:]

	for len(s) > 64 {
		a0 := fetch64(s, 0)
		a1 := fetch64(s, 8)
		a2 := fetch64(s, 16)
		a3 := fetch64(s, 24)
		a4 := fetch64(s, 32)
		a5 := fetch64(s, 40)
		a6 := fetch64(s, 48)
		a7 := fetch64(s, 56)
		x += a0 + a1
		y += a2
		z += a3
		v.lo += a4
		v.hi += a5 + a1
		w.lo += a6
		w.hi += a7

		x = rotate64(x, 26)
		x *= 9
		y = rotate64(y, 29)
		z *= mul
		v.lo = rotate64(v.lo, 33)
		v.hi = rotate64(v.hi, 30)
		w.lo ^= x
		w.lo *= 9
		z = rotate64(z, 32)
		z += w.hi
		w.hi += z
		z *= 9
		u, y = y, u

		z += a0 + a6
		v.lo += a2
		v.hi += a3
		w.lo += a4
		w.hi += a5 + a6
		x += a1
		y += a7

		y += v.lo
		v.lo += x - y
		v.hi += w.lo
		w.lo += v.hi
		w.hi += x - y
		x += w.hi
		w.hi = rotate64(w.hi, 34)
		u, z = z, u
		s = s[64:]
	}
	// Make s point to the last 64 bytes of input.
	s = last64
	u *= 9
	v.hi = rotate64(v.hi, 28)
	v.lo = rotate64(v.lo, 20)
	w.lo += (uint64(slen-1) & 63)
	u += y
	y += u
	x = rotate64(y-x+v.lo+fetch64(s, 8), 37) * mul
	y = rotate64(y^v.hi^fetch64(s, 48), 42) * mul
	x ^= w.hi * 9
	y += v.lo + fetch64(s, 40)
	z = rotate64(z+w.lo, 33) * mul
	v.lo, v.hi = weakHashLen32WithSeeds(s, v.hi*mul, x+w.lo)
	w.lo, w.hi = weakHashLen32WithSeeds(s[32:], z+w.hi, y+fetch64(s, 16))
	return uoH(hashLen16Mul(v.lo+x, w.lo^y, mul)+z-u,
		uoH(v.hi+y, w.hi+z, k2, 30)^x,
		k2,
		31)
}

// Hash64WithSeed hashes a byte slice and a uint64 seed and returns a uint64 hash value
func Hash64WithSeed(s []byte, seed uint64) uint64 {
	if len(s) <= 64 {
		return naHash64WithSeed(s, seed)
	}
	return Hash64WithSeeds(s, 0, seed)
}

// Hash64 hashes a byte slice and returns a uint64 hash value
func Hash64(s []byte) uint64 {
	if len(s) <= 64 {
		return naHash64(s)
	}
	return Hash64WithSeeds(s, 81, 0)
}
<commit_msg>Align comment<commit_after>package farm

func uoH(x, y, mul uint64, r uint) uint64 {
	a := (x ^ y) * mul
	a ^= (a >> 47)
	b := (y ^ a) * mul
	return rotate64(b, r) * mul
}

// Hash64WithSeeds hashes a byte slice and two uint64 seeds and returns a uint64 hash value
func Hash64WithSeeds(s []byte, seed0, seed1 uint64) uint64 {
	slen := len(s)
	if slen <= 64 {
		return naHash64WithSeeds(s, seed0, seed1)
	}

	// For strings over 64 bytes we loop.
	// Internal state consists of 64 bytes: u, v, w, x, y, and z.
	x := seed0
	y := seed1*k2 + 113
	z := shiftMix(y*k2) * k2
	v := uint128{seed0, seed1}
	var w uint128
	u := x - z
	x *= k2
	mul := k2 + (u & 0x82)

	// Set end so that after the loop we have 1 to 64 bytes left to process.
	endIdx := ((slen - 1) / 64) * 64
	last64Idx := endIdx + ((slen - 1) & 63) - 63
	last64 := s[last64Idx:]

	for len(s) > 64 {
		a0 := fetch64(s, 0)
		a1 := fetch64(s, 8)
		a2 := fetch64(s, 16)
		a3 := fetch64(s, 24)
		a4 := fetch64(s, 32)
		a5 := fetch64(s, 40)
		a6 := fetch64(s, 48)
		a7 := fetch64(s, 56)
		x += a0 + a1
		y += a2
		z += a3
		v.lo += a4
		v.hi += a5 + a1
		w.lo += a6
		w.hi += a7

		x = rotate64(x, 26)
		x *= 9
		y = rotate64(y, 29)
		z *= mul
		v.lo = rotate64(v.lo, 33)
		v.hi = rotate64(v.hi, 30)
		w.lo ^= x
		w.lo *= 9
		z = rotate64(z, 32)
		z += w.hi
		w.hi += z
		z *= 9
		u, y = y, u

		z += a0 + a6
		v.lo += a2
		v.hi += a3
		w.lo += a4
		w.hi += a5 + a6
		x += a1
		y += a7

		y += v.lo
		v.lo += x - y
		v.hi += w.lo
		w.lo += v.hi
		w.hi += x - y
		x += w.hi
		w.hi = rotate64(w.hi, 34)
		u, z = z, u
		s = s[64:]
	}
	// Make s point to the last 64 bytes of input.
	s = last64
	u *= 9
	v.hi = rotate64(v.hi, 28)
	v.lo = rotate64(v.lo, 20)
	w.lo += (uint64(slen-1) & 63)
	u += y
	y += u
	x = rotate64(y-x+v.lo+fetch64(s, 8), 37) * mul
	y = rotate64(y^v.hi^fetch64(s, 48), 42) * mul
	x ^= w.hi * 9
	y += v.lo + fetch64(s, 40)
	z = rotate64(z+w.lo, 33) * mul
	v.lo, v.hi = weakHashLen32WithSeeds(s, v.hi*mul, x+w.lo)
	w.lo, w.hi = weakHashLen32WithSeeds(s[32:], z+w.hi, y+fetch64(s, 16))
	return uoH(hashLen16Mul(v.lo+x, w.lo^y, mul)+z-u,
		uoH(v.hi+y, w.hi+z, k2, 30)^x,
		k2,
		31)
}

// Hash64WithSeed hashes a byte slice and a uint64 seed and returns a uint64 hash value
func Hash64WithSeed(s []byte, seed uint64) uint64 {
	if len(s) <= 64 {
		return naHash64WithSeed(s, seed)
	}
	return Hash64WithSeeds(s, 0, seed)
}

// Hash64 hashes a byte slice and returns a uint64 hash value
func Hash64(s []byte) uint64 {
	if len(s) <= 64 {
		return naHash64(s)
	}
	return Hash64WithSeeds(s, 81, 0)
}
<|endoftext|>


<commit_before>// Package fastentity provides text sequence identification in documents.
package fastentity

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"
	"unicode"
)

var (
	// Maximum entity length.  NB: currently entities can be added which are longer
	// but they will be ignored in the search.
	// TODO: Maybe also error/warn on import?
	MaxEntityLen = 30
	// Number of entities to initially allocate when creating a Group.
	DefaultGroupSize = 1000
)

const (
	left  = 0
	right = 1
)

type pair [2]int

type Store struct {
	Lookup map[string]*Group
	sync.RWMutex
}

type Group struct {
	Name     string
	Entities map[string][][]rune
	MaxLen   int
	sync.RWMutex
}

// Pops the last element and adds the new element
// to the front of stack.
func shift(n pair, s []pair) (pair, []pair) {
	if len(s) == 0 {
		return pair{}, append(s, n)
	}
	if len(s) == cap(s) {
		return s[0], append(s[1:], n)
	}
	return s[0], append(s, n)
}

// Create a new entity group structure
func New(groups ...string) *Store {
	s := &Store{
		Lookup: make(map[string]*Group, len(groups)),
	}
	for _, name := range groups {
		g := &Group{
			Name:     name,
			Entities: make(map[string][][]rune, DefaultGroupSize),
		}
		s.Lookup[name] = g
	}
	return s
}

// Add a new entity to a particular group
func (s *Store) Add(name string, entities ...[]rune) {
	if s.Lookup == nil {
		panic("You need to initialize the s before adding to it...")
	}

	s.Lock()
	g, ok := s.Lookup[name]
	if !ok {
		g = &Group{
			Name:     name,
			Entities: make(map[string][][]rune, DefaultGroupSize),
		}
		s.Lookup[name] = g
	}
	s.Unlock()

	g.Lock()
	for _, e := range entities {
		h := hash([]rune(e))
		g.Entities[h] = append(g.Entities[h], e)
		if len(e) > g.MaxLen {
			g.MaxLen = len(e)
		}
	}
	g.Unlock()
}

// Take the string and turn it into a hash
func hash(rs []rune) string {
	if len(rs) > 2 {
		return fmt.Sprintf("%s%s%s%03d", string(unicode.ToLower(rs[0])), string(unicode.ToLower(rs[1])), string(unicode.ToLower(rs[2])), len(rs))
	}
	if len(rs) > 1 {
		return fmt.Sprintf("%s%s%03d", string(unicode.ToLower(rs[0])), string(unicode.ToLower(rs[1])), len(rs))
	}
	return fmt.Sprintf("%s%03d", string(unicode.ToLower(rs[0])), len(rs))
}

// Find all entities for all type keys
func (s *Store) FindAll(rs []rune) map[string][][]rune {
	result := make(map[string][][]rune, len(s.Lookup))
	for name, g := range s.Lookup {
		result[name] = g.Find(rs)
	}
	return result
}

// Find only the entities of a given type = "key"
func (g *Group) Find(rs []rune) [][]rune {
	g.RLock()
	ents := find(rs, g)
	g.RUnlock()
	return ents[g.Name]
}

// Lock free find for use internally
func find(rs []rune, groups ...*Group) map[string][][]rune {

	results := make(map[string][][]rune, len(groups))

	pairs := make([]pair, 0, 20)
	start := 0
	prevspace := true // First char of sequence is legit
	thisspace := false
	var p1, p2 pair
	for off, r := range rs {

		// What are we looking at?
		thisspace = unicode.IsPunct(r) || unicode.IsSpace(r)

		if prevspace && !thisspace {
			// Word is beginning at this rune
			start = off
		} else if thisspace && !prevspace {
			// Word is ending, shift the pairs stack
			_, pairs = shift(pair{start, off}, pairs)

			// Run the stack, check for entities working backwards from the current position
			if len(pairs) > 1 {
				p2 = pairs[len(pairs)-1]
				for i := len(pairs) - 1; i >= 0; i-- {
					p1 = pairs[i]
					if p2[right]-p1[left] > MaxEntityLen {
						break // Too long or short, can ignore it
					}
					for _, group := range groups {
						if p2[right]-p1[left] > group.MaxLen {
							continue
						}
						if ents, ok := group.Entities[hash(rs[p1[left]:p2[right]])]; ok {
							// We have at least one entity with this key
							for _, ent := range ents {
								if len(ent) != p2[right]-p1[left] {
									break
								}
								match := true
								for i, r := range ent {
									if unicode.ToLower(r) != unicode.ToLower(rs[p1[left]+i]) {
										match = false
										break
									}
								}
								if match {
									results[group.Name] = append(results[group.Name], rs[p1[left]:p2[right]])
								}
							}
						}
					}
				}
			}

		}

		// Mark prevspace for the next loop
		if thisspace {
			prevspace = true
		} else {
			prevspace = false
		}
	}

	return results
}

type incr struct {
	sync.Mutex
	n int
}

func (i *incr) incr() {
	i.Lock()
	i.n++
	i.Unlock()
}

var entityFileSuffix = ".entities.csv"

// Create a new store by loading entity files from a given directory. The format
// expected has the format "<GROUP>.entities.csv"
func Load(dir string) (*Store, error) {
	dir = strings.TrimRight(dir, "/")
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	s := New()
	var wg sync.WaitGroup
	count := &incr{}
	for _, fileInfo := range files {
		if strings.HasSuffix(fileInfo.Name(), entityFileSuffix) {
			wg.Add(1)
			go func(filename string, group string) {
				defer wg.Done()
				f, err := os.Open(filename)
				if err != nil {
					// TODO: Remove this, return an error instead?
					fmt.Printf("Unable to load \"%s\" entity file: %s: %s\n", group, filename, err.Error())
					return
				}
				defer f.Close()

				count.incr()
				r := bufio.NewScanner(f)
				for r.Scan() {
					s.Add(group, []rune(r.Text()))
				}
			}(fmt.Sprintf("%s/%s", dir, fileInfo.Name()), strings.TrimSuffix(fileInfo.Name(), entityFileSuffix))
		}
	}
	wg.Wait()
	if count.n == 0 {
		return s, errors.New("There are no entity files")
	}
	return s, nil
}

// Save the existing entities to disk. Each group becomes a file with the format
// the format "<GROUP>.entities.csv" in the dir specified.
func (s *Store) Save(dir string) error {
	s.RLock()
	defer s.RUnlock()
	dir = strings.TrimRight(dir, "/")
	for name, group := range s.Lookup {
		filename := fmt.Sprintf("%s/%s", dir, strings.Replace(name, "/", "_", -1)+entityFileSuffix)
		f, err := os.Create(filename)
		if err != nil {
			return err
		}
		w := bufio.NewWriter(f)
		for _, entities := range group.Entities {
			for _, e := range entities {
				w.WriteString(string(e) + "\n")
			}
		}
		w.Flush()
		f.Close()
	}
	return nil
}
<commit_msg>Spacing and remove unnecessary declaration.<commit_after>// Package fastentity provides text sequence identification in documents.
package fastentity

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"
	"unicode"
)

var (
	// Maximum entity length.  NB: currently entities can be added which are longer
	// but they will be ignored in the search.
	// TODO: Maybe also error/warn on import?
	MaxEntityLen = 30
	// Number of entities to initially allocate when creating a Group.
	DefaultGroupSize = 1000
)

const (
	left  = 0
	right = 1
)

type pair [2]int

type Store struct {
	Lookup map[string]*Group
	sync.RWMutex
}

type Group struct {
	Name     string
	Entities map[string][][]rune
	MaxLen   int
	sync.RWMutex
}

// Pops the last element and adds the new element
// to the front of stack.
func shift(n pair, s []pair) (pair, []pair) {
	if len(s) == 0 {
		return pair{}, append(s, n)
	}
	if len(s) == cap(s) {
		return s[0], append(s[1:], n)
	}
	return s[0], append(s, n)
}

// Create a new entity group structure
func New(groups ...string) *Store {
	s := &Store{
		Lookup: make(map[string]*Group, len(groups)),
	}
	for _, name := range groups {
		g := &Group{
			Name:     name,
			Entities: make(map[string][][]rune, DefaultGroupSize),
		}
		s.Lookup[name] = g
	}
	return s
}

// Add a new entity to a particular group
func (s *Store) Add(name string, entities ...[]rune) {
	if s.Lookup == nil {
		panic("You need to initialize the s before adding to it...")
	}

	s.Lock()
	g, ok := s.Lookup[name]
	if !ok {
		g = &Group{
			Name:     name,
			Entities: make(map[string][][]rune, DefaultGroupSize),
		}
		s.Lookup[name] = g
	}
	s.Unlock()

	g.Lock()
	for _, e := range entities {
		h := hash([]rune(e))
		g.Entities[h] = append(g.Entities[h], e)
		if len(e) > g.MaxLen {
			g.MaxLen = len(e)
		}
	}
	g.Unlock()
}

// Take the string and turn it into a hash
func hash(rs []rune) string {
	if len(rs) > 2 {
		return fmt.Sprintf("%s%s%s%03d", string(unicode.ToLower(rs[0])), string(unicode.ToLower(rs[1])), string(unicode.ToLower(rs[2])), len(rs))
	}
	if len(rs) > 1 {
		return fmt.Sprintf("%s%s%03d", string(unicode.ToLower(rs[0])), string(unicode.ToLower(rs[1])), len(rs))
	}
	return fmt.Sprintf("%s%03d", string(unicode.ToLower(rs[0])), len(rs))
}

// Find all entities for all type keys
func (s *Store) FindAll(rs []rune) map[string][][]rune {
	result := make(map[string][][]rune, len(s.Lookup))
	for name, g := range s.Lookup {
		result[name] = g.Find(rs)
	}
	return result
}

// Find only the entities of a given type = "key"
func (g *Group) Find(rs []rune) [][]rune {
	g.RLock()
	ents := find(rs, g)
	g.RUnlock()
	return ents[g.Name]
}

// Lock free find for use internally
func find(rs []rune, groups ...*Group) map[string][][]rune {
	results := make(map[string][][]rune, len(groups))
	pairs := make([]pair, 0, 20)
	start := 0
	prevspace := true // First char of sequence is legit
	thisspace := false

	for off, r := range rs {
		// What are we looking at?
		thisspace = unicode.IsPunct(r) || unicode.IsSpace(r)

		if prevspace && !thisspace {
			// Word is beginning at this rune
			start = off
		} else if thisspace && !prevspace {
			// Word is ending, shift the pairs stack
			_, pairs = shift(pair{start, off}, pairs)

			// Run the stack, check for entities working backwards from the current position
			if len(pairs) > 1 {
				p2 := pairs[len(pairs)-1]
				for i := len(pairs) - 1; i >= 0; i-- {
					p1 := pairs[i]
					if p2[right]-p1[left] > MaxEntityLen {
						break // Too long or short, can ignore it
					}
					for _, group := range groups {
						if p2[right]-p1[left] > group.MaxLen {
							continue
						}
						if ents, ok := group.Entities[hash(rs[p1[left]:p2[right]])]; ok {
							// We have at least one entity with this key
							for _, ent := range ents {
								if len(ent) != p2[right]-p1[left] {
									break
								}
								match := true
								for i, r := range ent {
									if unicode.ToLower(r) != unicode.ToLower(rs[p1[left]+i]) {
										match = false
										break
									}
								}
								if match {
									results[group.Name] = append(results[group.Name], rs[p1[left]:p2[right]])
								}
							}
						}
					}
				}
			}
		}

		// Mark prevspace for the next loop
		if thisspace {
			prevspace = true
		} else {
			prevspace = false
		}
	}
	return results
}

type incr struct {
	sync.Mutex
	n int
}

func (i *incr) incr() {
	i.Lock()
	i.n++
	i.Unlock()
}

var entityFileSuffix = ".entities.csv"

// Create a new store by loading entity files from a given directory. The format
// expected has the format "<GROUP>.entities.csv"
func Load(dir string) (*Store, error) {
	dir = strings.TrimRight(dir, "/")
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	s := New()
	var wg sync.WaitGroup
	count := &incr{}
	for _, fileInfo := range files {
		if strings.HasSuffix(fileInfo.Name(), entityFileSuffix) {
			wg.Add(1)
			go func(filename string, group string) {
				defer wg.Done()
				f, err := os.Open(filename)
				if err != nil {
					// TODO: Remove this, return an error instead?
					fmt.Printf("Unable to load \"%s\" entity file: %s: %s\n", group, filename, err.Error())
					return
				}
				defer f.Close()

				count.incr()
				r := bufio.NewScanner(f)
				for r.Scan() {
					s.Add(group, []rune(r.Text()))
				}
			}(fmt.Sprintf("%s/%s", dir, fileInfo.Name()), strings.TrimSuffix(fileInfo.Name(), entityFileSuffix))
		}
	}
	wg.Wait()
	if count.n == 0 {
		return s, errors.New("There are no entity files")
	}
	return s, nil
}

// Save the existing entities to disk. Each group becomes a file with the format
// the format "<GROUP>.entities.csv" in the dir specified.
func (s *Store) Save(dir string) error {
	s.RLock()
	defer s.RUnlock()
	dir = strings.TrimRight(dir, "/")
	for name, group := range s.Lookup {
		filename := fmt.Sprintf("%s/%s", dir, strings.Replace(name, "/", "_", -1)+entityFileSuffix)
		f, err := os.Create(filename)
		if err != nil {
			return err
		}
		w := bufio.NewWriter(f)
		for _, entities := range group.Entities {
			for _, e := range entities {
				w.WriteString(string(e) + "\n")
			}
		}
		w.Flush()
		f.Close()
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bs

import (
	"fmt"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/tsuru/config"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/monsterqueue"
	"github.com/tsuru/tsuru/iaas"
	"github.com/tsuru/tsuru/log"
	"github.com/tsuru/tsuru/queue"
)

const QueueTaskName = "run-bs"

// RegisterQueueTask registers the internal bs queue task for later execution.
func RegisterQueueTask(p DockerProvisioner) error {
	q, err := queue.Queue()
	if err != nil {
		return err
	}
	return q.RegisterTask(&runBs{provisioner: p})
}

type runBs struct {
	provisioner DockerProvisioner
}

func (t *runBs) Name() string {
	return QueueTaskName
}

func (t *runBs) Run(job monsterqueue.Job) {
	params := job.Parameters()
	dockerEndpoint := params["endpoint"].(string)
	machineID := params["machine"].(string)
	node := cluster.Node{Address: dockerEndpoint}
	err := t.waitDocker(dockerEndpoint)
	if err != nil {
		job.Error(err)
		t.destroyMachine(machineID)
		return
	}
	rawMetadata := params["metadata"].(monsterqueue.JobParams)
	metadata := make(map[string]string, len(rawMetadata))
	for key, value := range rawMetadata {
		metadata[key] = value.(string)
	}
	err = createContainer(dockerEndpoint, metadata["pool"], t.provisioner, true)
	if err != nil {
		node.CreationStatus = cluster.NodeCreationStatusError
		node.Metadata = map[string]string{"creationError": err.Error()}
		t.provisioner.Cluster().UpdateNode(node)
		job.Error(err)
		t.destroyMachine(machineID)
		return
	}
	node.CreationStatus = cluster.NodeCreationStatusCreated
	_, err = t.provisioner.Cluster().UpdateNode(node)
	if err != nil {
		job.Error(err)
		t.destroyMachine(machineID)
		return
	}
	job.Success(nil)
}

func (t *runBs) waitDocker(endpoint string) error {
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}
	timeout, _ := config.GetInt("docker:api-timeout")
	if timeout == 0 {
		timeout = 600
	}
	timeoutChan := time.After(time.Duration(timeout) * time.Second)
	pong := make(chan error, 1)
	exit := make(chan struct{})
	go func() {
		for {
			select {
			case <-exit:
				return
			default:
				err := client.Ping()
				if err == nil {
					pong <- nil
					return
				}
				if e, ok := err.(*docker.Error); ok && e.Status > 499 {
					pong <- err
					return
				}
			}
		}
	}()
	select {
	case err := <-pong:
		return err
	case <-timeoutChan:
		close(exit)
		return fmt.Errorf("Docker API at %q didn't respond after %d seconds", endpoint, timeout)
	}
}

func (t *runBs) destroyMachine(id string) {
	if id != "" {
		machine, err := iaas.FindMachineById(id)
		if err != nil {
			log.Errorf("failed to remove machine %q: %s", id, err)
			return
		}
		err = machine.Destroy()
		if err != nil {
			log.Errorf("failed to remove machine %q: %s", id, err)
			return
		}
	}
}
<commit_msg>provision/docker/bs: do not destroy machine when bs creation fails<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bs

import (
	"fmt"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/tsuru/config"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/monsterqueue"
	"github.com/tsuru/tsuru/queue"
)

const QueueTaskName = "run-bs"

// RegisterQueueTask registers the internal bs queue task for later execution.
func RegisterQueueTask(p DockerProvisioner) error {
	q, err := queue.Queue()
	if err != nil {
		return err
	}
	return q.RegisterTask(&runBs{provisioner: p})
}

type runBs struct {
	provisioner DockerProvisioner
}

func (t *runBs) Name() string {
	return QueueTaskName
}

func (t *runBs) Run(job monsterqueue.Job) {
	params := job.Parameters()
	dockerEndpoint := params["endpoint"].(string)
	node := cluster.Node{Address: dockerEndpoint}
	err := t.waitDocker(dockerEndpoint)
	if err != nil {
		job.Error(err)
		return
	}
	node.CreationStatus = cluster.NodeCreationStatusCreated
	rawMetadata := params["metadata"].(monsterqueue.JobParams)
	metadata := make(map[string]string, len(rawMetadata))
	for key, value := range rawMetadata {
		metadata[key] = value.(string)
	}
	err = createContainer(dockerEndpoint, metadata["pool"], t.provisioner, true)
	if err != nil {
		t.provisioner.Cluster().UpdateNode(node)
		job.Error(err)
		return
	}
	node.Metadata["LastSuccess"] = time.Now().Format(time.RFC3339)
	_, err = t.provisioner.Cluster().UpdateNode(node)
	if err != nil {
		job.Error(err)
		return
	}
	job.Success(nil)
}

func (t *runBs) waitDocker(endpoint string) error {
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}
	timeout, _ := config.GetInt("docker:api-timeout")
	if timeout == 0 {
		timeout = 600
	}
	timeoutChan := time.After(time.Duration(timeout) * time.Second)
	pong := make(chan error, 1)
	exit := make(chan struct{})
	go func() {
		for {
			select {
			case <-exit:
				return
			default:
				err := client.Ping()
				if err == nil {
					pong <- nil
					return
				}
				if e, ok := err.(*docker.Error); ok && e.Status > 499 {
					pong <- err
					return
				}
			}
		}
	}()
	select {
	case err := <-pong:
		return err
	case <-timeoutChan:
		close(exit)
		return fmt.Errorf("Docker API at %q didn't respond after %d seconds", endpoint, timeout)
	}
}
<|endoftext|>