<commit_before>package kloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"koding/db/mongodb"
	"koding/db/mongodb/modelhelper"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/terraformer"
	tf "koding/kites/terraformer"

	"labix.org/v2/mgo/bson"

	"golang.org/x/net/context"

	"github.com/koding/kite"
	"github.com/mitchellh/mapstructure"
)

type TerraformPlanRequest struct {
	StackTemplateId string `json:"stackTemplateId"`

	GroupName string `json:"groupName"`
}

type terraformCredentials struct {
	Creds []*terraformCredential
}

type terraformCredential struct {
	Provider   string
	Identifier string
	Data       map[string]string `mapstructure:"data"`
}

// region returns the region from the credential data
func (t *terraformCredential) region() (string, error) {
	// for now we support only aws
	if t.Provider != "aws" {
		return "", fmt.Errorf("provider '%s' is not supported", t.Provider)
	}

	region := t.Data["region"]
	if region == "" {
		return "", fmt.Errorf("region for identifer '%s' is not set", t.Identifier)
	}

	return region, nil
}

func (t *terraformCredential) awsCredentials() (string, string, error) {
	if t.Provider != "aws" {
		return "", "", fmt.Errorf("provider '%s' is not supported", t.Provider)
	}

	// we do not check for key existency here because the key might exists but
	// with an empty value, so just checking for the emptiness of the value is
	// better
	accessKey := t.Data["access_key"]
	if accessKey == "" {
		return "", "", fmt.Errorf("accessKey for identifier '%s' is not set", t.Identifier)
	}

	secretKey := t.Data["secret_key"]
	if secretKey == "" {
		return "", "", fmt.Errorf("secretKey for identifier '%s' is not set", t.Identifier)
	}

	return accessKey, secretKey, nil
}

// appendAWSVariable appends the credentials aws data to the given template and
// returns it back.
func (t *terraformCredential) appendAWSVariable(template string) (string, error) {
	var data struct {
		Output   map[string]map[string]interface{} `json:"output,omitempty"`
		Resource map[string]map[string]interface{} `json:"resource,omitempty"`
		Provider struct {
			Aws struct {
				Region    string `json:"region"`
				AccessKey string `json:"access_key"`
				SecretKey string `json:"secret_key"`
			} `json:"aws"`
		} `json:"provider"`
		Variable map[string]map[string]interface{} `json:"variable,omitempty"`
	}

	if err := json.Unmarshal([]byte(template), &data); err != nil {
		return "", err
	}

	credRegion := t.Data["region"]
	if credRegion == "" {
		return "", fmt.Errorf("region for identifier '%s' is not set", t.Identifier)
	}

	// if region is not added, add it via credRegion
	region := data.Provider.Aws.Region
	if region == "" {
		data.Provider.Aws.Region = credRegion
	} else if !isVariable(region) && region != credRegion {
		// compare with the provider block's region. Don't allow if they are
		// different.
		return "", fmt.Errorf("region in the provider block doesn't match the region in credential data. Provider block: '%s'. Credential data: '%s'", region, credRegion)
	}

	if data.Variable == nil {
		data.Variable = make(map[string]map[string]interface{})
	}

	accessKey, secretKey, err := t.awsCredentials()
	if err != nil {
		return "", err
	}

	data.Variable["aws_access_key"] = map[string]interface{}{
		"default": accessKey,
	}

	data.Variable["aws_secret_key"] = map[string]interface{}{
		"default": secretKey,
	}

	data.Variable["region"] = map[string]interface{}{
		"default": credRegion,
	}

	out, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		return "", err
	}

	return string(out), nil
}

func (k *Kloud) Plan(r *kite.Request) (interface{}, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var args *TerraformPlanRequest
	if err := r.Args.One().Unmarshal(&args); err != nil {
		return nil, err
	}

	if args.StackTemplateId == "" {
		return nil, errors.New("stackIdTemplate is not passed")
	}

	if args.GroupName == "" {
		return nil, errors.New("group name is not passed")
	}

	ctx := k.ContextCreator(context.Background())
	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("session context is not passed")
	}

	k.Log.Debug("Fetching template for id %s", args.StackTemplateId)
	stackTemplate, err := modelhelper.GetStackTemplate(args.StackTemplateId)
	if err != nil {
		return nil, err
	}

	k.Log.Debug("Fetching credentials for id %v", stackTemplate.Credentials)
	creds, err := fetchCredentials(r.Username, args.GroupName, sess.DB, flattenValues(stackTemplate.Credentials))
	if err != nil {
		return nil, err
	}

	// TODO(arslan): make one single persistent connection if needed, for now
	// this is ok.
	tfKite, err := terraformer.Connect(sess.Kite)
	if err != nil {
		return nil, err
	}
	defer tfKite.Close()

	var region string
	for _, cred := range creds.Creds {
		region, err = cred.region()
		if err != nil {
			return nil, err
		}

		k.Log.Debug("Appending AWS variable for\n%s", stackTemplate.Template.Content)
		stackTemplate.Template.Content, err = cred.appendAWSVariable(stackTemplate.Template.Content)
		if err != nil {
			return nil, err
		}
	}

	sess.Log.Debug("Plan: stack template before injecting Koding data")
	buildData, err := injectKodingData(ctx, stackTemplate.Template.Content, r.Username, creds)
	if err != nil {
		return nil, err
	}
	stackTemplate.Template.Content = buildData.Template

	k.Log.Debug("Calling plan with content\n%s", stackTemplate.Template.Content)
	plan, err := tfKite.Plan(&tf.TerraformRequest{
		Content:   stackTemplate.Template.Content,
		ContentID: r.Username + "-" + args.StackTemplateId,
		Variables: nil,
	})
	if err != nil {
		return nil, err
	}

	machines, err := machinesFromPlan(plan)
	if err != nil {
		return nil, err
	}
	machines.AppendRegion(region)

	return machines, nil
}

func fetchCredentials(username, groupname string, db *mongodb.MongoDB, identifiers []string) (*terraformCredentials, error) {
	// fetch jaccount from username
	account, err := modelhelper.GetAccount(username)
	if err != nil {
		return nil, err
	}

	// fetch jGroup from group slug name
	group, err := modelhelper.GetGroup(groupname)
	if err != nil {
		return nil, err
	}

	// validate if username belongs to groupname
	selector := modelhelper.Selector{
		"targetId": account.Id,
		"sourceId": group.Id,
		"as": bson.M{
			"$in": []string{"member"},
		},
	}

	count, err := modelhelper.RelationshipCount(selector)
	if err != nil || count == 0 {
		return nil, fmt.Errorf("username '%s' does not belong to group '%s'", username, groupname)
	}

	// 2- fetch credential from identifiers via args
	credentials, err := modelhelper.GetCredentialsFromIdentifiers(identifiers...)
	if err != nil {
		return nil, err
	}

	// 3- count relationship with credential id and jaccount id as user or
	// owner. Any non valid credentials will be discarded
	validKeys := make(map[string]string, 0)

	for _, cred := range credentials {
		selector := modelhelper.Selector{
			"targetId": cred.Id,
			"sourceId": bson.M{
				"$in": []bson.ObjectId{account.Id, group.Id},
			},
			"as": bson.M{
				"$in": []string{"owner", "user"},
			},
		}

		count, err := modelhelper.RelationshipCount(selector)
		if err != nil || count == 0 {
			// we return for any not validated identifier key.
			return nil, fmt.Errorf("credential with identifier '%s' is not validated", cred.Identifier)
		}

		validKeys[cred.Identifier] = cred.Provider
	}

	// 4- fetch credentialdata with identifier
	validIdentifiers := make([]string, 0)
	for pKey := range validKeys {
		validIdentifiers = append(identifiers, pKey)
	}

	credentialData, err := modelhelper.GetCredentialDatasFromIdentifiers(validIdentifiers...)
	if err != nil {
		return nil, err
	}

	// 5- return list of keys. We only support aws for now
	creds := &terraformCredentials{
		Creds: make([]*terraformCredential, 0),
	}

	for _, data := range credentialData {
		provider, ok := validKeys[data.Identifier]
		if !ok {
			return nil, fmt.Errorf("provider is not found for identifer: %s", data.Identifier)
		}
		// for now we only support aws
		if provider != "aws" {
			continue
		}

		cred := &terraformCredential{
			Provider:   provider,
			Identifier: data.Identifier,
		}

		if err := mapstructure.Decode(data.Meta, &cred.Data); err != nil {
			return nil, err
		}
		creds.Creds = append(creds.Creds, cred)

	}
	return creds, nil
}
<commit_msg>kloud: add data fetched from db<commit_after>package kloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"koding/db/models"
	"koding/db/mongodb"
	"koding/db/mongodb/modelhelper"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/terraformer"
	tf "koding/kites/terraformer"

	"labix.org/v2/mgo/bson"

	"golang.org/x/net/context"

	"github.com/koding/kite"
	"github.com/mitchellh/mapstructure"
)

type TerraformPlanRequest struct {
	StackTemplateId string `json:"stackTemplateId"`

	GroupName string `json:"groupName"`
}

type terraformData struct {
	Creds   []*terraformCredential
	Account *models.Account
	Group   *models.Group
	User    *models.User
}

type terraformCredential struct {
	Provider   string
	Identifier string
	Data       map[string]string `mapstructure:"data"`
}

// region returns the region from the credential data
func (t *terraformCredential) region() (string, error) {
	// for now we support only aws
	if t.Provider != "aws" {
		return "", fmt.Errorf("provider '%s' is not supported", t.Provider)
	}

	region := t.Data["region"]
	if region == "" {
		return "", fmt.Errorf("region for identifer '%s' is not set", t.Identifier)
	}

	return region, nil
}

func (t *terraformCredential) awsCredentials() (string, string, error) {
	if t.Provider != "aws" {
		return "", "", fmt.Errorf("provider '%s' is not supported", t.Provider)
	}

	// we do not check for key existency here because the key might exists but
	// with an empty value, so just checking for the emptiness of the value is
	// better
	accessKey := t.Data["access_key"]
	if accessKey == "" {
		return "", "", fmt.Errorf("accessKey for identifier '%s' is not set", t.Identifier)
	}

	secretKey := t.Data["secret_key"]
	if secretKey == "" {
		return "", "", fmt.Errorf("secretKey for identifier '%s' is not set", t.Identifier)
	}

	return accessKey, secretKey, nil
}

// appendAWSVariable appends the credentials aws data to the given template and
// returns it back.
func (t *terraformCredential) appendAWSVariable(template string) (string, error) {
	var data struct {
		Output   map[string]map[string]interface{} `json:"output,omitempty"`
		Resource map[string]map[string]interface{} `json:"resource,omitempty"`
		Provider struct {
			Aws struct {
				Region    string `json:"region"`
				AccessKey string `json:"access_key"`
				SecretKey string `json:"secret_key"`
			} `json:"aws"`
		} `json:"provider"`
		Variable map[string]map[string]interface{} `json:"variable,omitempty"`
	}

	if err := json.Unmarshal([]byte(template), &data); err != nil {
		return "", err
	}

	credRegion := t.Data["region"]
	if credRegion == "" {
		return "", fmt.Errorf("region for identifier '%s' is not set", t.Identifier)
	}

	// if region is not added, add it via credRegion
	region := data.Provider.Aws.Region
	if region == "" {
		data.Provider.Aws.Region = credRegion
	} else if !isVariable(region) && region != credRegion {
		// compare with the provider block's region. Don't allow if they are
		// different.
		return "", fmt.Errorf("region in the provider block doesn't match the region in credential data. Provider block: '%s'. Credential data: '%s'", region, credRegion)
	}

	if data.Variable == nil {
		data.Variable = make(map[string]map[string]interface{})
	}

	accessKey, secretKey, err := t.awsCredentials()
	if err != nil {
		return "", err
	}

	data.Variable["aws_access_key"] = map[string]interface{}{
		"default": accessKey,
	}

	data.Variable["aws_secret_key"] = map[string]interface{}{
		"default": secretKey,
	}

	data.Variable["region"] = map[string]interface{}{
		"default": credRegion,
	}

	out, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		return "", err
	}

	return string(out), nil
}

func (k *Kloud) Plan(r *kite.Request) (interface{}, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var args *TerraformPlanRequest
	if err := r.Args.One().Unmarshal(&args); err != nil {
		return nil, err
	}

	if args.StackTemplateId == "" {
		return nil, errors.New("stackIdTemplate is not passed")
	}

	if args.GroupName == "" {
		return nil, errors.New("group name is not passed")
	}

	ctx := k.ContextCreator(context.Background())
	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("session context is not passed")
	}

	k.Log.Debug("Fetching template for id %s", args.StackTemplateId)
	stackTemplate, err := modelhelper.GetStackTemplate(args.StackTemplateId)
	if err != nil {
		return nil, err
	}

	k.Log.Debug("Fetching credentials for id %v", stackTemplate.Credentials)
	creds, err := fetchCredentials(r.Username, args.GroupName, sess.DB, flattenValues(stackTemplate.Credentials))
	if err != nil {
		return nil, err
	}

	// TODO(arslan): make one single persistent connection if needed, for now
	// this is ok.
	tfKite, err := terraformer.Connect(sess.Kite)
	if err != nil {
		return nil, err
	}
	defer tfKite.Close()

	var region string
	for _, cred := range creds.Creds {
		region, err = cred.region()
		if err != nil {
			return nil, err
		}

		k.Log.Debug("Appending AWS variable for\n%s", stackTemplate.Template.Content)
		stackTemplate.Template.Content, err = cred.appendAWSVariable(stackTemplate.Template.Content)
		if err != nil {
			return nil, err
		}
	}

	sess.Log.Debug("Plan: stack template before injecting Koding data")
	buildData, err := injectKodingData(ctx, stackTemplate.Template.Content, r.Username, creds)
	if err != nil {
		return nil, err
	}
	stackTemplate.Template.Content = buildData.Template

	k.Log.Debug("Calling plan with content\n%s", stackTemplate.Template.Content)
	plan, err := tfKite.Plan(&tf.TerraformRequest{
		Content:   stackTemplate.Template.Content,
		ContentID: r.Username + "-" + args.StackTemplateId,
		Variables: nil,
	})
	if err != nil {
		return nil, err
	}

	machines, err := machinesFromPlan(plan)
	if err != nil {
		return nil, err
	}
	machines.AppendRegion(region)

	return machines, nil
}

func fetchCredentials(username, groupname string, db *mongodb.MongoDB, identifiers []string) (*terraformData, error) {
	// fetch jaccount from username
	account, err := modelhelper.GetAccount(username)
	if err != nil {
		return nil, err
	}

	// fetch jGroup from group slug name
	group, err := modelhelper.GetGroup(groupname)
	if err != nil {
		return nil, err
	}

	// fetch jUser from username
	user, err := modelhelper.GetUser(username)
	if err != nil {
		return nil, err
	}

	// validate if username belongs to groupnam
	selector := modelhelper.Selector{
		"targetId": account.Id,
		"sourceId": group.Id,
		"as": bson.M{
			"$in": []string{"member"},
		},
	}

	count, err := modelhelper.RelationshipCount(selector)
	if err != nil || count == 0 {
		return nil, fmt.Errorf("username '%s' does not belong to group '%s'", username, groupname)
	}

	// 2- fetch credential from identifiers via args
	credentials, err := modelhelper.GetCredentialsFromIdentifiers(identifiers...)
	if err != nil {
		return nil, err
	}

	// 3- count relationship with credential id and jaccount id as user or
	// owner. Any non valid credentials will be discarded
	validKeys := make(map[string]string, 0)

	for _, cred := range credentials {
		selector := modelhelper.Selector{
			"targetId": cred.Id,
			"sourceId": bson.M{
				"$in": []bson.ObjectId{account.Id, group.Id},
			},
			"as": bson.M{
				"$in": []string{"owner", "user"},
			},
		}

		count, err := modelhelper.RelationshipCount(selector)
		if err != nil || count == 0 {
			// we return for any not validated identifier key.
			return nil, fmt.Errorf("credential with identifier '%s' is not validated", cred.Identifier)
		}

		validKeys[cred.Identifier] = cred.Provider
	}

	// 4- fetch credentialdata with identifier
	validIdentifiers := make([]string, 0)
	for pKey := range validKeys {
		validIdentifiers = append(identifiers, pKey)
	}

	credentialData, err := modelhelper.GetCredentialDatasFromIdentifiers(validIdentifiers...)
	if err != nil {
		return nil, err
	}

	// 5- return list of keys. We only support aws for now
	data := &terraformData{
		Account: account,
		Group:   group,
		User:    user,
		Creds:   make([]*terraformCredential, 0),
	}

	for _, c := range credentialData {
		provider, ok := validKeys[c.Identifier]
		if !ok {
			return nil, fmt.Errorf("provider is not found for identifer: %s", c.Identifier)
		}
		// for now we only support aws
		if provider != "aws" {
			continue
		}

		cred := &terraformCredential{
			Provider:   provider,
			Identifier: c.Identifier,
		}

		if err := mapstructure.Decode(c.Meta, &cred.Data); err != nil {
			return nil, err
		}

		data.Creds = append(data.Creds, cred)
	}

	return data, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package reparentutil

import (
	"context"
	"sync"
	"time"

	"vitess.io/vitess/go/vt/proto/topodata"
	"vitess.io/vitess/go/vt/topo/topoproto"

	"k8s.io/apimachinery/pkg/util/sets"

	"vitess.io/vitess/go/event"
	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/vt/concurrency"
	"vitess.io/vitess/go/vt/logutil"
	"vitess.io/vitess/go/vt/topo"
	"vitess.io/vitess/go/vt/topotools/events"
	"vitess.io/vitess/go/vt/vterrors"
	"vitess.io/vitess/go/vt/vttablet/tmclient"

	replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata"
	"vitess.io/vitess/go/vt/proto/vtrpc"
)

// FindValidEmergencyReparentCandidates will find candidates for an emergency
// reparent, and, if successful, return a mapping of those tablet aliases (as
// raw strings) to their replication positions for later comparison.
func FindValidEmergencyReparentCandidates(
	statusMap map[string]*replicationdatapb.StopReplicationStatus,
	primaryStatusMap map[string]*replicationdatapb.PrimaryStatus,
) (map[string]mysql.Position, error) {
	replicationStatusMap := make(map[string]*mysql.ReplicationStatus, len(statusMap))
	positionMap := make(map[string]mysql.Position)

	// Build out replication status list from proto types.
	for alias, statuspb := range statusMap {
		status := mysql.ProtoToReplicationStatus(statuspb.After)
		replicationStatusMap[alias] = &status
	}

	// Determine if we're GTID-based. If we are, we'll need to look for errant
	// GTIDs below.
	var (
		isGTIDBased                bool
		isNonGTIDBased             bool
		emptyRelayPosErrorRecorder concurrency.FirstErrorRecorder
	)

	for alias, status := range replicationStatusMap {
		if _, ok := status.RelayLogPosition.GTIDSet.(mysql.Mysql56GTIDSet); ok {
			isGTIDBased = true
		} else {
			isNonGTIDBased = true
		}

		if status.RelayLogPosition.IsZero() {
			// Potentially bail. If any other tablet is detected to have
			// GTID-based relay log positions, we will return the error recorded
			// here.
			emptyRelayPosErrorRecorder.RecordError(vterrors.Errorf(vtrpc.Code_UNAVAILABLE, "encountered tablet %v with no relay log position, when at least one other tablet in the status map has GTID based relay log positions", alias))
		}
	}

	if isGTIDBased && emptyRelayPosErrorRecorder.HasErrors() {
		return nil, emptyRelayPosErrorRecorder.Error()
	}

	if isGTIDBased && isNonGTIDBased {
		return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "encountered mix of GTID-based and non GTID-based relay logs")
	}

	// Create relevant position list of errant GTID-based positions for later
	// comparison.
	for alias, status := range replicationStatusMap {
		// If we're not GTID-based, no need to search for errant GTIDs, so just
		// add the position to the map and continue.
		if !isGTIDBased {
			positionMap[alias] = status.Position

			continue
		}

		// This condition should really never happen, since we did the same cast
		// in the earlier loop, but let's be doubly sure.
		relayLogGTIDSet, ok := status.RelayLogPosition.GTIDSet.(mysql.Mysql56GTIDSet)
		if !ok {
			return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "we got a filled-in relay log position, but it's not of type Mysql56GTIDSet, even though we've determined we need to use GTID based assesment")
		}

		// We need to remove this alias's status from the list, otherwise the
		// GTID diff will always be empty.
		statusList := make([]*mysql.ReplicationStatus, 0, len(replicationStatusMap)-1)

		for a, s := range replicationStatusMap {
			if a != alias {
				statusList = append(statusList, s)
			}
		}

		errantGTIDs, err := status.FindErrantGTIDs(statusList)
		switch {
		case err != nil:
			// Could not look up GTIDs to determine if we have any. It's not
			// safe to continue.
			return nil, err
		case len(errantGTIDs) != 0:
			// This tablet has errant GTIDs. It's not a valid candidate for
			// reparent, so don't insert it into the final mapping.
			continue
		}

		pos := mysql.Position{GTIDSet: relayLogGTIDSet}
		positionMap[alias] = pos
	}

	for alias, primaryStatus := range primaryStatusMap {
		executedPosition, err := mysql.DecodePosition(primaryStatus.Position)
		if err != nil {
			return nil, vterrors.Wrapf(err, "could not decode a primary status executed position for tablet %v: %v", alias, err)
		}

		positionMap[alias] = executedPosition
	}

	return positionMap, nil
}

// ReplicaWasRunning returns true if a StopReplicationStatus indicates that the
// replica had running replication threads before being stopped. It returns an
// error if the Before state of replication is nil.
func ReplicaWasRunning(stopStatus *replicationdatapb.StopReplicationStatus) (bool, error) {
	if stopStatus == nil || stopStatus.Before == nil {
		return false, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "could not determine Before state of StopReplicationStatus %v", stopStatus)
	}

	return stopStatus.Before.IoThreadRunning || stopStatus.Before.SqlThreadRunning, nil
}

// StopReplicationAndBuildStatusMaps stops replication on all replicas, then
// collects and returns a mapping of TabletAlias (as string) to their current
// replication positions.
func StopReplicationAndBuildStatusMaps(
	ctx context.Context,
	tmc tmclient.TabletManagerClient,
	ev *events.Reparent,
	tabletMap map[string]*topo.TabletInfo,
	waitReplicasTimeout time.Duration,
	ignoredTablets sets.String,
	tabletToWaitFor *topodata.TabletAlias,
	logger logutil.Logger,
) (map[string]*replicationdatapb.StopReplicationStatus, map[string]*replicationdatapb.PrimaryStatus, error) {
	event.DispatchUpdate(ev, "stop replication on all replicas")

	var (
		statusMap        = map[string]*replicationdatapb.StopReplicationStatus{}
		primaryStatusMap = map[string]*replicationdatapb.PrimaryStatus{}
		m                sync.Mutex
		errChan          = make(chan concurrency.Error)
	)

	groupCtx, groupCancel := context.WithTimeout(ctx, waitReplicasTimeout)
	defer groupCancel()

	fillStatus := func(alias string, tabletInfo *topo.TabletInfo, mustWaitForTablet bool) {
		var concurrencyErr concurrency.Error
		var err error
		defer func() {
			concurrencyErr.Err = err
			concurrencyErr.MustWaitFor = mustWaitForTablet
			errChan <- concurrencyErr
		}()

		logger.Infof("getting replication position from %v", alias)

		_, stopReplicationStatus, err := tmc.StopReplicationAndGetStatus(groupCtx, tabletInfo.Tablet, replicationdatapb.StopReplicationMode_IOTHREADONLY)
		sqlErr, isSQLErr := mysql.NewSQLErrorFromError(err).(*mysql.SQLError)
		switch {
		case isSQLErr && sqlErr != nil && sqlErr.Number() == mysql.ERNotReplica:
			var primaryStatus *replicationdatapb.PrimaryStatus

			primaryStatus, err = tmc.DemotePrimary(groupCtx, tabletInfo.Tablet)
			if err != nil {
				msg := "replica %v thinks it's primary but we failed to demote it"
				err = vterrors.Wrapf(err, msg+": %v", alias, err)

				logger.Warningf(msg, alias)
				return
			}

			m.Lock()
			primaryStatusMap[alias] = primaryStatus
			m.Unlock()
		case err == nil:
			m.Lock()
			statusMap[alias] = stopReplicationStatus
			m.Unlock()
		default:
			logger.Warningf("failed to get replication status from %v: %v", alias, err)

			err = vterrors.Wrapf(err, "error when getting replication status for alias %v: %v", alias, err)
		}
	}

	tabletAliasToWaitFor := ""
	numErrosToWaitFor := 0
	if tabletToWaitFor != nil {
		tabletAliasToWaitFor = topoproto.TabletAliasString(tabletToWaitFor)
	}
	for alias, tabletInfo := range tabletMap {
		if !ignoredTablets.Has(alias) {
			mustWaitFor := tabletAliasToWaitFor == alias
			if mustWaitFor {
				numErrosToWaitFor++
			}
			go fillStatus(alias, tabletInfo, mustWaitFor)
		}
	}

	errgroup := concurrency.ErrorGroup{
		NumGoroutines:        len(tabletMap) - ignoredTablets.Len(),
		NumRequiredSuccesses: len(tabletMap) - ignoredTablets.Len() - 1,
		NumAllowedErrors:     1,
		NumErrorsToWaitFor:   numErrosToWaitFor,
	}

	errRecorder := errgroup.Wait(groupCancel, errChan)
	if len(errRecorder.Errors) > 1 {
		return nil, nil, vterrors.Wrapf(errRecorder.Error(), "encountered more than one error when trying to stop replication and get positions: %v", errRecorder.Error())
	}

	return statusMap, primaryStatusMap, nil
}

// WaitForRelayLogsToApply blocks execution waiting for the given tablet's relay
// logs to apply, unless the specified context is canceled or exceeded.
// Typically a caller will set a timeout of WaitReplicasTimeout on a context and
// use that context with this function.
func WaitForRelayLogsToApply(ctx context.Context, tmc tmclient.TabletManagerClient, tabletInfo *topo.TabletInfo, status *replicationdatapb.StopReplicationStatus) error {
	switch status.After.RelayLogPosition {
	case "":
		return tmc.WaitForPosition(ctx, tabletInfo.Tablet, status.After.FileRelayLogPosition)
	default:
		return tmc.WaitForPosition(ctx, tabletInfo.Tablet, status.After.RelayLogPosition)
	}
}
<commit_msg>refactor: fixed import lines<commit_after>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package reparentutil

import (
	"context"
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/util/sets"

	"vitess.io/vitess/go/event"
	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/vt/concurrency"
	"vitess.io/vitess/go/vt/logutil"
	replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	"vitess.io/vitess/go/vt/proto/vtrpc"
	"vitess.io/vitess/go/vt/topo"
	"vitess.io/vitess/go/vt/topo/topoproto"
	"vitess.io/vitess/go/vt/topotools/events"
	"vitess.io/vitess/go/vt/vterrors"
	"vitess.io/vitess/go/vt/vttablet/tmclient"
)

// FindValidEmergencyReparentCandidates will find candidates for an emergency
// reparent, and, if successful, return a mapping of those tablet aliases (as
// raw strings) to their replication positions for later comparison.
func FindValidEmergencyReparentCandidates(
	statusMap map[string]*replicationdatapb.StopReplicationStatus,
	primaryStatusMap map[string]*replicationdatapb.PrimaryStatus,
) (map[string]mysql.Position, error) {
	replicationStatusMap := make(map[string]*mysql.ReplicationStatus, len(statusMap))
	positionMap := make(map[string]mysql.Position)

	// Build out replication status list from proto types.
	for alias, statuspb := range statusMap {
		status := mysql.ProtoToReplicationStatus(statuspb.After)
		replicationStatusMap[alias] = &status
	}

	// Determine if we're GTID-based. If we are, we'll need to look for errant
	// GTIDs below.
	var (
		isGTIDBased                bool
		isNonGTIDBased             bool
		emptyRelayPosErrorRecorder concurrency.FirstErrorRecorder
	)

	for alias, status := range replicationStatusMap {
		if _, ok := status.RelayLogPosition.GTIDSet.(mysql.Mysql56GTIDSet); ok {
			isGTIDBased = true
		} else {
			isNonGTIDBased = true
		}

		if status.RelayLogPosition.IsZero() {
			// Potentially bail. If any other tablet is detected to have
			// GTID-based relay log positions, we will return the error recorded
			// here.
			emptyRelayPosErrorRecorder.RecordError(vterrors.Errorf(vtrpc.Code_UNAVAILABLE, "encountered tablet %v with no relay log position, when at least one other tablet in the status map has GTID based relay log positions", alias))
		}
	}

	if isGTIDBased && emptyRelayPosErrorRecorder.HasErrors() {
		return nil, emptyRelayPosErrorRecorder.Error()
	}

	if isGTIDBased && isNonGTIDBased {
		return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "encountered mix of GTID-based and non GTID-based relay logs")
	}

	// Create relevant position list of errant GTID-based positions for later
	// comparison.
	for alias, status := range replicationStatusMap {
		// If we're not GTID-based, no need to search for errant GTIDs, so just
		// add the position to the map and continue.
		if !isGTIDBased {
			positionMap[alias] = status.Position

			continue
		}

		// This condition should really never happen, since we did the same cast
		// in the earlier loop, but let's be doubly sure.
		relayLogGTIDSet, ok := status.RelayLogPosition.GTIDSet.(mysql.Mysql56GTIDSet)
		if !ok {
			return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "we got a filled-in relay log position, but it's not of type Mysql56GTIDSet, even though we've determined we need to use GTID based assesment")
		}

		// We need to remove this alias's status from the list, otherwise the
		// GTID diff will always be empty.
		statusList := make([]*mysql.ReplicationStatus, 0, len(replicationStatusMap)-1)

		for a, s := range replicationStatusMap {
			if a != alias {
				statusList = append(statusList, s)
			}
		}

		errantGTIDs, err := status.FindErrantGTIDs(statusList)
		switch {
		case err != nil:
			// Could not look up GTIDs to determine if we have any. It's not
			// safe to continue.
			return nil, err
		case len(errantGTIDs) != 0:
			// This tablet has errant GTIDs. It's not a valid candidate for
			// reparent, so don't insert it into the final mapping.
			continue
		}

		pos := mysql.Position{GTIDSet: relayLogGTIDSet}
		positionMap[alias] = pos
	}

	for alias, primaryStatus := range primaryStatusMap {
		executedPosition, err := mysql.DecodePosition(primaryStatus.Position)
		if err != nil {
			return nil, vterrors.Wrapf(err, "could not decode a primary status executed position for tablet %v: %v", alias, err)
		}

		positionMap[alias] = executedPosition
	}

	return positionMap, nil
}

// ReplicaWasRunning returns true if a StopReplicationStatus indicates that the
// replica had running replication threads before being stopped. It returns an
// error if the Before state of replication is nil.
func ReplicaWasRunning(stopStatus *replicationdatapb.StopReplicationStatus) (bool, error) {
	if stopStatus == nil || stopStatus.Before == nil {
		return false, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "could not determine Before state of StopReplicationStatus %v", stopStatus)
	}

	return stopStatus.Before.IoThreadRunning || stopStatus.Before.SqlThreadRunning, nil
}

// StopReplicationAndBuildStatusMaps stops replication on all replicas, then
// collects and returns a mapping of TabletAlias (as string) to their current
// replication positions.
func StopReplicationAndBuildStatusMaps(
	ctx context.Context,
	tmc tmclient.TabletManagerClient,
	ev *events.Reparent,
	tabletMap map[string]*topo.TabletInfo,
	waitReplicasTimeout time.Duration,
	ignoredTablets sets.String,
	tabletToWaitFor *topodatapb.TabletAlias,
	logger logutil.Logger,
) (map[string]*replicationdatapb.StopReplicationStatus, map[string]*replicationdatapb.PrimaryStatus, error) {
	event.DispatchUpdate(ev, "stop replication on all replicas")

	var (
		statusMap        = map[string]*replicationdatapb.StopReplicationStatus{}
		primaryStatusMap = map[string]*replicationdatapb.PrimaryStatus{}
		m                sync.Mutex
		errChan          = make(chan concurrency.Error)
	)

	groupCtx, groupCancel := context.WithTimeout(ctx, waitReplicasTimeout)
	defer groupCancel()

	fillStatus := func(alias string, tabletInfo *topo.TabletInfo, mustWaitForTablet bool) {
		var concurrencyErr concurrency.Error
		var err error
		defer func() {
			concurrencyErr.Err = err
			concurrencyErr.MustWaitFor = mustWaitForTablet
			errChan <- concurrencyErr
		}()

		logger.Infof("getting replication position from %v", alias)

		_, stopReplicationStatus, err := tmc.StopReplicationAndGetStatus(groupCtx, tabletInfo.Tablet, replicationdatapb.StopReplicationMode_IOTHREADONLY)
		sqlErr, isSQLErr := mysql.NewSQLErrorFromError(err).(*mysql.SQLError)
		switch {
		case isSQLErr && sqlErr != nil && sqlErr.Number() == mysql.ERNotReplica:
			var primaryStatus *replicationdatapb.PrimaryStatus

			primaryStatus, err = tmc.DemotePrimary(groupCtx, tabletInfo.Tablet)
			if err != nil {
				msg := "replica %v thinks it's primary but we failed to demote it"
				err = vterrors.Wrapf(err, msg+": %v", alias, err)

				logger.Warningf(msg, alias)
				return
			}

			m.Lock()
			primaryStatusMap[alias] = primaryStatus
			m.Unlock()
		case err == nil:
			m.Lock()
			statusMap[alias] = stopReplicationStatus
			m.Unlock()
		default:
			logger.Warningf("failed to get replication status from %v: %v", alias, err)

			err = vterrors.Wrapf(err, "error when getting replication status for alias %v: %v", alias, err)
		}
	}

	tabletAliasToWaitFor := ""
	numErrosToWaitFor := 0
	if tabletToWaitFor != nil {
		tabletAliasToWaitFor = topoproto.TabletAliasString(tabletToWaitFor)
	}
	for alias, tabletInfo := range tabletMap {
		if !ignoredTablets.Has(alias) {
			mustWaitFor := tabletAliasToWaitFor == alias
			if mustWaitFor {
				numErrosToWaitFor++
			}
			go fillStatus(alias, tabletInfo, mustWaitFor)
		}
	}

	errgroup := concurrency.ErrorGroup{
		NumGoroutines:        len(tabletMap) - ignoredTablets.Len(),
		NumRequiredSuccesses: len(tabletMap) - ignoredTablets.Len() - 1,
		NumAllowedErrors:     1,
		NumErrorsToWaitFor:   numErrosToWaitFor,
	}

	errRecorder := errgroup.Wait(groupCancel, errChan)
	if len(errRecorder.Errors) > 1 {
		return nil, nil, vterrors.Wrapf(errRecorder.Error(), "encountered more than one error when trying to stop replication and get positions: %v", errRecorder.Error())
	}

	return statusMap, primaryStatusMap, nil
}

// WaitForRelayLogsToApply blocks execution waiting for the given tablet's relay
// logs to apply, unless the specified context is canceled or exceeded.
// Typically a caller will set a timeout of WaitReplicasTimeout on a context and
// use that context with this function.
func WaitForRelayLogsToApply(ctx context.Context, tmc tmclient.TabletManagerClient, tabletInfo *topo.TabletInfo, status *replicationdatapb.StopReplicationStatus) error {
	switch status.After.RelayLogPosition {
	case "":
		return tmc.WaitForPosition(ctx, tabletInfo.Tablet, status.After.FileRelayLogPosition)
	default:
		return tmc.WaitForPosition(ctx, tabletInfo.Tablet, status.After.RelayLogPosition)
	}
}
<|endoftext|>


<commit_before>package aphgrpc

import (
	"fmt"
	"math"
	"net/http"
	"strings"
	"time"

	"gopkg.in/mgutz/dat.v1"
	"gopkg.in/mgutz/dat.v1/sqlx-runner"

	"github.com/dictyBase/go-genproto/dictybaseapis/api/jsonapi"
	"github.com/fatih/structs"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/timestamp"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	context "golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

const (
	DefaultPagenum  int64 = 1
	DefaultPagesize int64 = 10
)

// JSONAPIParamsInfo interface should be implement by all grpc-gateway services
// that supports JSON API specifications.
type JSONAPIParamsInfo interface {
	// Relationships that could be included
	AllowedInclude() []string
	// Attribute fields that are allowed
	AllowedFields() []string
	// Filter fields that are allowed
	AllowedFilter() []string
	// FilterToColumns provides mapping between filter and storage columns
	FilterToColumns() map[string]string
	// RequiredAttrs are the mandatory attributes for creating a new resource
	RequiredAttrs() []string
}

// JSONAPIResource interface provides information about HTTP resource. All
// grpc-gateway services that supports JSONAPI should implement this interface.
type JSONAPIResource interface {
	//GetResourceName returns canonical resource name
	GetResourceName() string
	// GetBaseURL returns the base url with the scheme
	GetBaseURL() string
	// GetPrefix returns the path that could be appended to base url
	GetPathPrefix() string
}

func NullToTime(nt dat.NullTime) *timestamp.Timestamp {
	var ts *timestamp.Timestamp
	if nt.Valid {
		ts, _ := ptypes.TimestampProto(nt.Time)
		return ts
	}
	return ts
}

func ProtoTimeStamp(ts *timestamp.Timestamp) time.Time {
	t, _ := ptypes.Timestamp(ts)
	return t
}

func TimestampProto(t time.Time) *timestamp.Timestamp {
	ts, _ := ptypes.TimestampProto(t)
	return ts
}

func NullToString(s dat.NullString) string {
	if s.Valid {
		return s.String
	}
	return ""
}

func NullToInt64(i dat.NullInt64) int64 {
	if i.Valid {
		return i.Int64
	}
	var i64 int64
	return i64
}

// GetTotalPageNum calculate total no of pages from total no. records and page size
func GetTotalPageNum(record, pagesize int64) int64 {
	total := int64(math.Floor(float64(record) / float64(pagesize)))
	if math.Mod(float64(record), float64(pagesize)) > 0 {
		total += 1
	}
	return total
}

// GenPaginatedLinks generates paginated resource links
// from various page properties.
func GenPaginatedLinks(url string, lastpage, pagenum, pagesize int64) map[string]string {
	var links map[string]string
	links["self"] = AppendPaginationParams(url, pagenum, pagesize)
	links["first"] = AppendPaginationParams(url, 1, pagesize)
	if pagenum != 1 {
		links["previous"] = AppendPaginationParams(url, pagenum-1, pagesize)
	}
	links["last"] = AppendPaginationParams(url, lastpage, pagesize)
	if pagenum != lastpage {
		links["next"] = AppendPaginationParams(url, pagenum+1, pagesize)
	}
	return links
}

func GenBaseLink(rs JSONAPIResource) string {
	return fmt.Sprintf(
		"%s/%s",
		strings.Trim(rs.GetBaseURL(), "/"),
		strings.Trim(rs.GetPathPrefix(), "/"),
	)
}

func GenSingleResourceLink(rs JSONAPIResource, id int64) string {
	return fmt.Sprintf(
		"%s/%s/%d",
		GenBaseLink(rs),
		rs.GetResourceName(),
		id,
	)
}

func GenMultiResourceLink(rs JSONAPIResource) string {
	return fmt.Sprintf(
		"%s/%s",
		GenBaseLink(rs),
		rs.GetResourceName(),
	)
}

func AppendPaginationParams(url string, pagenum, pagesize int64) string {
	return fmt.Sprintf("%s?pagenum=%d&pagesize=%d", url, pagenum, pagesize)
}

func GenPaginatedResourceLink(rs JSONAPIResource, pagenum, pagesize int64) string {
	return fmt.Sprintf(
		"%s/%s?pagenum=%d&pagesize=%d",
		GenBaseLink(rs),
		rs.GetResourceName(),
		pagenum,
		pagesize,
	)
}

func GenSelfRelationshipLink(rs JSONAPIResource, rel string, id int64) string {
	return fmt.Sprintf(
		"%s/%s/%d/relationships/%s",
		GenBaseLink(rs),
		rs.GetResourceName(),
		id,
		rel,
	)
}

func GenRelatedRelationshipLink(rs JSONAPIResource, rel string, id int64) string {
	return fmt.Sprintf(
		"%s/%s/%d/%s",
		GenBaseLink(rs),
		rs.GetResourceName(),
		id,
		rel,
	)
}

//GetDefinedTagsWithValue check for fields that are initialized and returns a map
//with the tag and their values
func GetDefinedTagsWithValue(i interface{}, key string) map[string]interface{} {
	m := make(map[string]interface{})
	s := structs.New(i)
	for _, f := range s.Fields() {
		if !f.IsZero() {
			m[f.Tag(key)] = f.Value()
		}
	}
	return m
}

//GetDefinedTags check for fields that are initialized and returns a slice of
//their matching tag values
func GetDefinedTags(i interface{}, tag string) []string {
	var v []string
	s := structs.New(i)
	for _, f := range s.Fields() {
		if !f.IsZero() {
			v = append(v, f.Tag(tag))
		}
	}
	return v
}

// HandleCreateResponse modifies the grpc gateway filter which adds the JSON API header and
// modifies the http status response for POST request
func HandleCreateResponse(ctx context.Context, w http.ResponseWriter, resp proto.Message) error {
	w.Header().Set("Content-Type", "application/vnd.api+json")
	md, ok := runtime.ServerMetadataFromContext(ctx)
	if ok {
		trMD := md.TrailerMD
		if _, ok := trMD["method"]; ok {
			switch trMD["method"][0] {
			case "POST":
				w.WriteHeader(http.StatusCreated)
			case "POST_NO_CONTENT":
				w.WriteHeader(http.StatusNoContent)
			}
		}
	}
	return nil
}

type Service struct {
	Dbh             *runner.DB
	PathPrefix      string
	Include         []string
	IncludeStr      string
	FieldsToColumns map[string]string
	FieldsStr       string
	Resource        string
	BaseURL         string
	FilToColumns    map[string]string
	FilterStr       string
	Params          *JSONAPIParams
	ListMethod      bool
	ReqAttrs        []string
}

func (s *Service) RequiredAttrs() []string {
	return s.ReqAttrs
}

func (s *Service) IsListMethod() bool {
	return s.ListMethod
}

func (s *Service) FilterToColumns() map[string]string {
	return s.FilToColumns
}

func (s *Service) AllowedFilter() []string {
	var f []string
	for k, _ := range s.FilterToColumns() {
		f = append(f, k)
	}
	return f
}

func (s *Service) AllowedInclude() []string {
	return s.Include
}

func (s *Service) AllowedFields() []string {
	var f []string
	for k, _ := range s.FieldsToColumns {
		f = append(f, k)
	}
	return f
}

func (s *Service) GetResourceName() string {
	return s.Resource
}

func (s *Service) GetBaseURL() string {
	return s.BaseURL
}

func (s *Service) GetPathPrefix() string {
	return s.PathPrefix
}

func (s *Service) SetBaseURL(ctx context.Context) error {
	md, ok := metadata.FromIncomingContext(ctx)
	if !ok {
		return ErrRetrieveMetadata
	}
	slice, ok := md["x-forwarded-host"]
	if !ok {
		return ErrXForwardedHost
	}
	s.BaseURL = slice[0]
	return nil
}

func (s *Service) MapFieldsToColumns(fields []string) []string {
	var columns []string
	for _, v := range fields {
		columns = append(columns, s.FieldsToColumns[v])
	}
	return columns
}

func (s *Service) GetCount(table string) (int64, error) {
	var count int64
	err := s.Dbh.Select("COUNT(*)").From(table).QueryScalar(&count)
	return count, err
}

func (s *Service) GetAllFilteredCount(table string) (int64, error) {
	var count int64
	err := s.Dbh.Select("COUNT(*)").
		From(table).
		Scope(
			FilterToWhereClause(s, s.Params.Filters),
			FilterToBindValue(s.Params.Filters)...,
		).QueryScalar(&count)
	return count, err
}

// GetRelatedPagination generates JSONAPI pagination links for relation resources
func (s *Service) GetRelatedPagination(id, record, pagenum, pagesize int64, relation string) (*jsonapi.PaginationLinks, int64) {
	pages := GetTotalPageNum(record, pagesize)
	baseLink := s.GenCollResourceRelSelfLink(id, relation)
	pageLinks := GenPaginatedLinks(baseLink, pages, pagenum, pagesize)
	jsapiLinks := &jsonapi.PaginationLinks{
		Self:  pageLinks["self"],
		Last:  pageLinks["last"],
		First: pageLinks["first"],
	}
	if _, ok := pageLinks["previous"]; ok {
		jsapiLinks.Prev = pageLinks["previous"]
	}
	if _, ok := pageLinks["next"]; ok {
		jsapiLinks.Next = pageLinks["next"]
	}
	return jsapiLinks, pages
}

// GetPagination generates JSONAPI pagination links along with fields, include and filter query parameters
func (s *Service) GetPagination(record, pagenum, pagesize int64) (*jsonapi.PaginationLinks, int64) {
	pages := GetTotalPageNum(record, pagesize)
	baseLink := s.GenCollResourceSelfLink()
	pageLinks := GenPaginatedLinks(baseLink, pages, pagenum, pagesize)
	pageType := []string{"self", "last", "first", "previous", "next"}

	if s.Params != nil {
		params := s.Params
		switch {
		case params.HasFields && params.HasInclude && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&include=%s&filter=%s", s.FieldsStr, s.IncludeStr, s.FilterStr)
				}
			}
		case params.HasFields && params.HasInclude:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
				}
			}
		case params.HasFields && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&filter=%s", s.FieldsStr, s.FilterStr)
				}
			}
		case params.HasInclude && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&include=%s&filter=%s", s.IncludeStr, s.FilterStr)
				}
			}
		case params.HasInclude:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&include=%s", s.IncludeStr)
				}
			}
		case params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&filter=%s", s.FilterStr)
				}
			}
		case params.HasFields:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s", s.FieldsStr)
				}
			}
		}
	}
	jsapiLinks := &jsonapi.PaginationLinks{
		Self:  pageLinks["self"],
		Last:  pageLinks["last"],
		First: pageLinks["first"],
	}
	if _, ok := pageLinks["previous"]; ok {
		jsapiLinks.Prev = pageLinks["previous"]
	}
	if _, ok := pageLinks["next"]; ok {
		jsapiLinks.Next = pageLinks["next"]
	}
	return jsapiLinks, pages
}

func (s *Service) GenCollResourceRelSelfLink(id int64, relation string) string {
	return fmt.Sprintf(
		"%s/%d/%s",
		GenMultiResourceLink(s),
		id,
		relation,
	)
}

func (s *Service) GenCollResourceSelfLink() string {
	link := GenMultiResourceLink(s)
	if s.Params == nil {
		return link
	}
	params := s.Params
	switch {
	case params.HasFields && params.HasFilter && params.HasInclude:
		link += fmt.Sprintf("?fields=%s&include=%s&filter=%s", s.FieldsStr, s.IncludeStr, s.FilterStr)
	case params.HasFields && params.HasFilter:
		link += fmt.Sprintf("?fields=%s&filter=%s", s.FieldsStr, s.FilterStr)
	case params.HasFields && params.HasInclude:
		link += fmt.Sprintf("?fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
	case params.HasFilter && params.HasInclude:
		link += fmt.Sprintf("?filter=%s&include=%s", s.FilterStr, s.IncludeStr)
	case params.HasInclude:
		link += fmt.Sprintf("?include=%s", s.IncludeStr)
	case params.HasFilter:
		link += fmt.Sprintf("?filter=%s", s.FilterStr)
	case params.HasFields:
		link += fmt.Sprintf("?fields=%s", s.FieldsStr)
	}
	return link
}

func (s *Service) GenResourceSelfLink(id int64) string {
	links := GenSingleResourceLink(s, id)
	if !s.IsListMethod() && s.Params != nil {
		params := s.Params
		switch {
		case params.HasFields && params.HasInclude:
			links += fmt.Sprintf("?fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
		case params.HasFields:
			links += fmt.Sprintf("?fields=%s", s.FieldsStr)
		case params.HasInclude:
			links += fmt.Sprintf("?include=%s", s.IncludeStr)
		}
	}
	return links
}
<commit_msg>Fixed addition of extra resource in the url<commit_after>package aphgrpc

import (
	"fmt"
	"math"
	"net/http"
	"strings"
	"time"

	"gopkg.in/mgutz/dat.v1"
	"gopkg.in/mgutz/dat.v1/sqlx-runner"

	"github.com/dictyBase/go-genproto/dictybaseapis/api/jsonapi"
	"github.com/fatih/structs"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/timestamp"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	context "golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

const (
	DefaultPagenum  int64 = 1
	DefaultPagesize int64 = 10
)

// JSONAPIParamsInfo interface should be implement by all grpc-gateway services
// that supports JSON API specifications.
type JSONAPIParamsInfo interface {
	// Relationships that could be included
	AllowedInclude() []string
	// Attribute fields that are allowed
	AllowedFields() []string
	// Filter fields that are allowed
	AllowedFilter() []string
	// FilterToColumns provides mapping between filter and storage columns
	FilterToColumns() map[string]string
	// RequiredAttrs are the mandatory attributes for creating a new resource
	RequiredAttrs() []string
}

// JSONAPIResource interface provides information about HTTP resource. All
// grpc-gateway services that supports JSONAPI should implement this interface.
type JSONAPIResource interface {
	//GetResourceName returns canonical resource name
	GetResourceName() string
	// GetBaseURL returns the base url with the scheme
	GetBaseURL() string
	// GetPrefix returns the path that could be appended to base url
	GetPathPrefix() string
}

func NullToTime(nt dat.NullTime) *timestamp.Timestamp {
	var ts *timestamp.Timestamp
	if nt.Valid {
		ts, _ := ptypes.TimestampProto(nt.Time)
		return ts
	}
	return ts
}

func ProtoTimeStamp(ts *timestamp.Timestamp) time.Time {
	t, _ := ptypes.Timestamp(ts)
	return t
}

func TimestampProto(t time.Time) *timestamp.Timestamp {
	ts, _ := ptypes.TimestampProto(t)
	return ts
}

func NullToString(s dat.NullString) string {
	if s.Valid {
		return s.String
	}
	return ""
}

func NullToInt64(i dat.NullInt64) int64 {
	if i.Valid {
		return i.Int64
	}
	var i64 int64
	return i64
}

// GetTotalPageNum calculate total no of pages from total no. records and page size
func GetTotalPageNum(record, pagesize int64) int64 {
	total := int64(math.Floor(float64(record) / float64(pagesize)))
	if math.Mod(float64(record), float64(pagesize)) > 0 {
		total += 1
	}
	return total
}

// GenPaginatedLinks generates paginated resource links
// from various page properties.
func GenPaginatedLinks(url string, lastpage, pagenum, pagesize int64) map[string]string {
	var links map[string]string
	links["self"] = AppendPaginationParams(url, pagenum, pagesize)
	links["first"] = AppendPaginationParams(url, 1, pagesize)
	if pagenum != 1 {
		links["previous"] = AppendPaginationParams(url, pagenum-1, pagesize)
	}
	links["last"] = AppendPaginationParams(url, lastpage, pagesize)
	if pagenum != lastpage {
		links["next"] = AppendPaginationParams(url, pagenum+1, pagesize)
	}
	return links
}

func GenBaseLink(rs JSONAPIResource) string {
	return fmt.Sprintf(
		"%s/%s",
		strings.Trim(rs.GetBaseURL(), "/"),
		strings.Trim(rs.GetPathPrefix(), "/"),
	)
}

func GenSingleResourceLink(rs JSONAPIResource, id int64) string {
	return fmt.Sprintf(
		"%s/%d",
		GenBaseLink(rs),
		id,
	)
}

func GenMultiResourceLink(rs JSONAPIResource) string {
	return fmt.Sprintf(
		"%s/%s",
		GenBaseLink(rs),
		rs.GetResourceName(),
	)
}

func AppendPaginationParams(url string, pagenum, pagesize int64) string {
	return fmt.Sprintf("%s?pagenum=%d&pagesize=%d", url, pagenum, pagesize)
}

func GenPaginatedResourceLink(rs JSONAPIResource, pagenum, pagesize int64) string {
	return fmt.Sprintf(
		"%s/%s?pagenum=%d&pagesize=%d",
		GenBaseLink(rs),
		rs.GetResourceName(),
		pagenum,
		pagesize,
	)
}

func GenSelfRelationshipLink(rs JSONAPIResource, rel string, id int64) string {
	return fmt.Sprintf(
		"%s/%s/%d/relationships/%s",
		GenBaseLink(rs),
		rs.GetResourceName(),
		id,
		rel,
	)
}

func GenRelatedRelationshipLink(rs JSONAPIResource, rel string, id int64) string {
	return fmt.Sprintf(
		"%s/%s/%d/%s",
		GenBaseLink(rs),
		rs.GetResourceName(),
		id,
		rel,
	)
}

//GetDefinedTagsWithValue check for fields that are initialized and returns a map
//with the tag and their values
func GetDefinedTagsWithValue(i interface{}, key string) map[string]interface{} {
	m := make(map[string]interface{})
	s := structs.New(i)
	for _, f := range s.Fields() {
		if !f.IsZero() {
			m[f.Tag(key)] = f.Value()
		}
	}
	return m
}

//GetDefinedTags check for fields that are initialized and returns a slice of
//their matching tag values
func GetDefinedTags(i interface{}, tag string) []string {
	var v []string
	s := structs.New(i)
	for _, f := range s.Fields() {
		if !f.IsZero() {
			v = append(v, f.Tag(tag))
		}
	}
	return v
}

// HandleCreateResponse modifies the grpc gateway filter which adds the JSON API header and
// modifies the http status response for POST request
func HandleCreateResponse(ctx context.Context, w http.ResponseWriter, resp proto.Message) error {
	w.Header().Set("Content-Type", "application/vnd.api+json")
	md, ok := runtime.ServerMetadataFromContext(ctx)
	if ok {
		trMD := md.TrailerMD
		if _, ok := trMD["method"]; ok {
			switch trMD["method"][0] {
			case "POST":
				w.WriteHeader(http.StatusCreated)
			case "POST_NO_CONTENT":
				w.WriteHeader(http.StatusNoContent)
			}
		}
	}
	return nil
}

type Service struct {
	Dbh             *runner.DB
	PathPrefix      string
	Include         []string
	IncludeStr      string
	FieldsToColumns map[string]string
	FieldsStr       string
	Resource        string
	BaseURL         string
	FilToColumns    map[string]string
	FilterStr       string
	Params          *JSONAPIParams
	ListMethod      bool
	ReqAttrs        []string
}

func (s *Service) RequiredAttrs() []string {
	return s.ReqAttrs
}

func (s *Service) IsListMethod() bool {
	return s.ListMethod
}

func (s *Service) FilterToColumns() map[string]string {
	return s.FilToColumns
}

func (s *Service) AllowedFilter() []string {
	var f []string
	for k, _ := range s.FilterToColumns() {
		f = append(f, k)
	}
	return f
}

func (s *Service) AllowedInclude() []string {
	return s.Include
}

func (s *Service) AllowedFields() []string {
	var f []string
	for k, _ := range s.FieldsToColumns {
		f = append(f, k)
	}
	return f
}

func (s *Service) GetResourceName() string {
	return s.Resource
}

func (s *Service) GetBaseURL() string {
	return s.BaseURL
}

func (s *Service) GetPathPrefix() string {
	return s.PathPrefix
}

func (s *Service) SetBaseURL(ctx context.Context) error {
	md, ok := metadata.FromIncomingContext(ctx)
	if !ok {
		return ErrRetrieveMetadata
	}
	slice, ok := md["x-forwarded-host"]
	if !ok {
		return ErrXForwardedHost
	}
	s.BaseURL = slice[0]
	return nil
}

func (s *Service) MapFieldsToColumns(fields []string) []string {
	var columns []string
	for _, v := range fields {
		columns = append(columns, s.FieldsToColumns[v])
	}
	return columns
}

func (s *Service) GetCount(table string) (int64, error) {
	var count int64
	err := s.Dbh.Select("COUNT(*)").From(table).QueryScalar(&count)
	return count, err
}

func (s *Service) GetAllFilteredCount(table string) (int64, error) {
	var count int64
	err := s.Dbh.Select("COUNT(*)").
		From(table).
		Scope(
			FilterToWhereClause(s, s.Params.Filters),
			FilterToBindValue(s.Params.Filters)...,
		).QueryScalar(&count)
	return count, err
}

// GetRelatedPagination generates JSONAPI pagination links for relation resources
func (s *Service) GetRelatedPagination(id, record, pagenum, pagesize int64, relation string) (*jsonapi.PaginationLinks, int64) {
	pages := GetTotalPageNum(record, pagesize)
	baseLink := s.GenCollResourceRelSelfLink(id, relation)
	pageLinks := GenPaginatedLinks(baseLink, pages, pagenum, pagesize)
	jsapiLinks := &jsonapi.PaginationLinks{
		Self:  pageLinks["self"],
		Last:  pageLinks["last"],
		First: pageLinks["first"],
	}
	if _, ok := pageLinks["previous"]; ok {
		jsapiLinks.Prev = pageLinks["previous"]
	}
	if _, ok := pageLinks["next"]; ok {
		jsapiLinks.Next = pageLinks["next"]
	}
	return jsapiLinks, pages
}

// GetPagination generates JSONAPI pagination links along with fields, include and filter query parameters
func (s *Service) GetPagination(record, pagenum, pagesize int64) (*jsonapi.PaginationLinks, int64) {
	pages := GetTotalPageNum(record, pagesize)
	baseLink := s.GenCollResourceSelfLink()
	pageLinks := GenPaginatedLinks(baseLink, pages, pagenum, pagesize)
	pageType := []string{"self", "last", "first", "previous", "next"}

	if s.Params != nil {
		params := s.Params
		switch {
		case params.HasFields && params.HasInclude && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&include=%s&filter=%s", s.FieldsStr, s.IncludeStr, s.FilterStr)
				}
			}
		case params.HasFields && params.HasInclude:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
				}
			}
		case params.HasFields && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&filter=%s", s.FieldsStr, s.FilterStr)
				}
			}
		case params.HasInclude && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&include=%s&filter=%s", s.IncludeStr, s.FilterStr)
				}
			}
		case params.HasInclude:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&include=%s", s.IncludeStr)
				}
			}
		case params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&filter=%s", s.FilterStr)
				}
			}
		case params.HasFields:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s", s.FieldsStr)
				}
			}
		}
	}
	jsapiLinks := &jsonapi.PaginationLinks{
		Self:  pageLinks["self"],
		Last:  pageLinks["last"],
		First: pageLinks["first"],
	}
	if _, ok := pageLinks["previous"]; ok {
		jsapiLinks.Prev = pageLinks["previous"]
	}
	if _, ok := pageLinks["next"]; ok {
		jsapiLinks.Next = pageLinks["next"]
	}
	return jsapiLinks, pages
}

func (s *Service) GenCollResourceRelSelfLink(id int64, relation string) string {
	return fmt.Sprintf(
		"%s/%d/%s",
		GenMultiResourceLink(s),
		id,
		relation,
	)
}

func (s *Service) GenCollResourceSelfLink() string {
	link := GenMultiResourceLink(s)
	if s.Params == nil {
		return link
	}
	params := s.Params
	switch {
	case params.HasFields && params.HasFilter && params.HasInclude:
		link += fmt.Sprintf("?fields=%s&include=%s&filter=%s", s.FieldsStr, s.IncludeStr, s.FilterStr)
	case params.HasFields && params.HasFilter:
		link += fmt.Sprintf("?fields=%s&filter=%s", s.FieldsStr, s.FilterStr)
	case params.HasFields && params.HasInclude:
		link += fmt.Sprintf("?fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
	case params.HasFilter && params.HasInclude:
		link += fmt.Sprintf("?filter=%s&include=%s", s.FilterStr, s.IncludeStr)
	case params.HasInclude:
		link += fmt.Sprintf("?include=%s", s.IncludeStr)
	case params.HasFilter:
		link += fmt.Sprintf("?filter=%s", s.FilterStr)
	case params.HasFields:
		link += fmt.Sprintf("?fields=%s", s.FieldsStr)
	}
	return link
}

func (s *Service) GenResourceSelfLink(id int64) string {
	links := GenSingleResourceLink(s, id)
	if !s.IsListMethod() && s.Params != nil {
		params := s.Params
		switch {
		case params.HasFields && params.HasInclude:
			links += fmt.Sprintf("?fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
		case params.HasFields:
			links += fmt.Sprintf("?fields=%s", s.FieldsStr)
		case params.HasInclude:
			links += fmt.Sprintf("?include=%s", s.IncludeStr)
		}
	}
	return links
}
<|endoftext|>


<commit_before>package aphgrpc

import (
	"fmt"
	"math"
	"net/http"
	"strings"
	"time"

	"gopkg.in/mgutz/dat.v2/dat"
	"gopkg.in/mgutz/dat.v2/sqlx-runner"

	"github.com/dictyBase/go-genproto/dictybaseapis/api/jsonapi"
	"github.com/fatih/structs"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/timestamp"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	context "golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

const (
	DefaultPagenum  int64 = 1
	DefaultPagesize int64 = 10
)

// JSONAPIParamsInfo interface should be implement by all grpc-gateway services
// that supports JSON API specifications.
type JSONAPIParamsInfo interface {
	// Relationships that could be included
	AllowedInclude() []string
	// Attribute fields that are allowed
	AllowedFields() []string
	// Filter fields that are allowed
	AllowedFilter() []string
	// FilterToColumns provides mapping between filter and storage columns
	FilterToColumns() map[string]string
	// RequiredAttrs are the mandatory attributes for creating a new resource
	RequiredAttrs() []string
}

// JSONAPIResource interface provides information about HTTP resource. All
// grpc-gateway services that supports JSONAPI should implement this interface.
type JSONAPIResource interface {
	//GetResourceName returns canonical resource name
	GetResourceName() string
	// GetBaseURL returns the base url with the scheme
	GetBaseURL() string
	// GetPrefix returns the path that could be appended to base url
	GetPathPrefix() string
}

func NullToTime(nt dat.NullTime) *timestamp.Timestamp {
	var ts *timestamp.Timestamp
	if nt.Valid {
		ts, _ := ptypes.TimestampProto(nt.Time)
		return ts
	}
	return ts
}

func ProtoTimeStamp(ts *timestamp.Timestamp) time.Time {
	t, _ := ptypes.Timestamp(ts)
	return t
}

func TimestampProto(t time.Time) *timestamp.Timestamp {
	ts, _ := ptypes.TimestampProto(t)
	return ts
}

func NullToString(s dat.NullString) string {
	if s.Valid {
		return s.String
	}
	return ""
}

func NullToInt64(i dat.NullInt64) int64 {
	if i.Valid {
		return i.Int64
	}
	var i64 int64
	return i64
}

// GetTotalPageNum calculate total no of pages from total no. records and page size
func GetTotalPageNum(record, pagesize int64) int64 {
	total := int64(math.Floor(float64(record) / float64(pagesize)))
	if math.Mod(float64(record), float64(pagesize)) > 0 {
		total += 1
	}
	return total
}

// GenPaginatedLinks generates paginated resource links
// from various page properties.
func GenPaginatedLinks(url string, lastpage, pagenum, pagesize int64) map[string]string {
	var links map[string]string
	links["self"] = AppendPaginationParams(url, pagenum, pagesize)
	links["first"] = AppendPaginationParams(url, 1, pagesize)
	if pagenum != 1 {
		links["previous"] = AppendPaginationParams(url, pagenum-1, pagesize)
	}
	links["last"] = AppendPaginationParams(url, lastpage, pagesize)
	if pagenum != lastpage {
		links["next"] = AppendPaginationParams(url, pagenum+1, pagesize)
	}
	return links
}

func GenBaseLink(rs JSONAPIResource) string {
	return fmt.Sprintf(
		"%s/%s",
		strings.Trim(rs.GetBaseURL(), "/"),
		strings.Trim(rs.GetPathPrefix(), "/"),
	)
}

func GenSingleResourceLink(rs JSONAPIResource, id int64) string {
	return fmt.Sprintf(
		"%s/%d",
		GenBaseLink(rs),
		id,
	)
}

func GenMultiResourceLink(rs JSONAPIResource) string {
	return GenBaseLink(rs)
}

func AppendPaginationParams(url string, pagenum, pagesize int64) string {
	return fmt.Sprintf("%s?pagenum=%d&pagesize=%d", url, pagenum, pagesize)
}

func GenPaginatedResourceLink(rs JSONAPIResource, pagenum, pagesize int64) string {
	return fmt.Sprintf(
		"%s/%s?pagenum=%d&pagesize=%d",
		GenBaseLink(rs),
		rs.GetResourceName(),
		pagenum,
		pagesize,
	)
}

func GenSelfRelationshipLink(rs JSONAPIResource, rel string, id int64) string {
	return fmt.Sprintf(
		"%s/%s/%d/relationships/%s",
		GenBaseLink(rs),
		rs.GetResourceName(),
		id,
		rel,
	)
}

func GenRelatedRelationshipLink(rs JSONAPIResource, rel string, id int64) string {
	return fmt.Sprintf(
		"%s/%s/%d/%s",
		GenBaseLink(rs),
		rs.GetResourceName(),
		id,
		rel,
	)
}

//GetDefinedTagsWithValue check for fields that are initialized and returns a map
//with the tag and their values
func GetDefinedTagsWithValue(i interface{}, key string) map[string]interface{} {
	m := make(map[string]interface{})
	s := structs.New(i)
	for _, f := range s.Fields() {
		if !f.IsZero() {
			m[f.Tag(key)] = f.Value()
		}
	}
	return m
}

//GetDefinedTags check for fields that are initialized and returns a slice of
//their matching tag values
func GetDefinedTags(i interface{}, tag string) []string {
	var v []string
	s := structs.New(i)
	for _, f := range s.Fields() {
		if !f.IsZero() {
			v = append(v, f.Tag(tag))
		}
	}
	return v
}

// HandleCreateResponse modifies the grpc gateway filter which adds the JSON API header and
// modifies the http status response for POST request
func HandleCreateResponse(ctx context.Context, w http.ResponseWriter, resp proto.Message) error {
	w.Header().Set("Content-Type", "application/vnd.api+json")
	md, ok := runtime.ServerMetadataFromContext(ctx)
	if ok {
		trMD := md.TrailerMD
		if _, ok := trMD["method"]; ok {
			switch trMD["method"][0] {
			case "POST":
				w.WriteHeader(http.StatusCreated)
			case "POST_NO_CONTENT":
				w.WriteHeader(http.StatusNoContent)
			}
		}
	}
	return nil
}

type Service struct {
	Dbh             *runner.DB
	PathPrefix      string
	Include         []string
	IncludeStr      string
	FieldsToColumns map[string]string
	FieldsStr       string
	Resource        string
	BaseURL         string
	FilToColumns    map[string]string
	FilterStr       string
	Params          *JSONAPIParams
	ListMethod      bool
	ReqAttrs        []string
}

func (s *Service) RequiredAttrs() []string {
	return s.ReqAttrs
}

func (s *Service) IsListMethod() bool {
	return s.ListMethod
}

func (s *Service) FilterToColumns() map[string]string {
	return s.FilToColumns
}

func (s *Service) AllowedFilter() []string {
	var f []string
	for k, _ := range s.FilterToColumns() {
		f = append(f, k)
	}
	return f
}

func (s *Service) AllowedInclude() []string {
	return s.Include
}

func (s *Service) AllowedFields() []string {
	var f []string
	for k, _ := range s.FieldsToColumns {
		f = append(f, k)
	}
	return f
}

func (s *Service) GetResourceName() string {
	return s.Resource
}

func (s *Service) GetBaseURL() string {
	return s.BaseURL
}

func (s *Service) GetPathPrefix() string {
	return s.PathPrefix
}

func (s *Service) SetBaseURL(ctx context.Context) error {
	md, ok := metadata.FromIncomingContext(ctx)
	if !ok {
		return ErrRetrieveMetadata
	}
	slice, ok := md["x-forwarded-host"]
	if !ok {
		return ErrXForwardedHost
	}
	s.BaseURL = fmt.Sprintf("http://%s", slice[0])
	return nil
}

func (s *Service) MapFieldsToColumns(fields []string) []string {
	var columns []string
	for _, v := range fields {
		columns = append(columns, s.FieldsToColumns[v])
	}
	return columns
}

func (s *Service) GetCount(table string) (int64, error) {
	var count int64
	err := s.Dbh.Select("COUNT(*)").From(table).QueryScalar(&count)
	return count, err
}

func (s *Service) GetAllFilteredCount(table string) (int64, error) {
	var count int64
	err := s.Dbh.Select("COUNT(*)").
		From(table).
		Scope(
			FilterToWhereClause(s, s.Params.Filters),
			FilterToBindValue(s.Params.Filters)...,
		).QueryScalar(&count)
	return count, err
}

// GetRelatedPagination generates JSONAPI pagination links for relation resources
func (s *Service) GetRelatedPagination(id, record, pagenum, pagesize int64, relation string) (*jsonapi.PaginationLinks, int64) {
	pages := GetTotalPageNum(record, pagesize)
	baseLink := s.GenCollResourceRelSelfLink(id, relation)
	pageLinks := GenPaginatedLinks(baseLink, pages, pagenum, pagesize)
	jsapiLinks := &jsonapi.PaginationLinks{
		Self:  pageLinks["self"],
		Last:  pageLinks["last"],
		First: pageLinks["first"],
	}
	if _, ok := pageLinks["previous"]; ok {
		jsapiLinks.Prev = pageLinks["previous"]
	}
	if _, ok := pageLinks["next"]; ok {
		jsapiLinks.Next = pageLinks["next"]
	}
	return jsapiLinks, pages
}

// GetPagination generates JSONAPI pagination links along with fields, include and filter query parameters
func (s *Service) GetPagination(record, pagenum, pagesize int64) (*jsonapi.PaginationLinks, int64) {
	pages := GetTotalPageNum(record, pagesize)
	baseLink := s.GenCollResourceSelfLink()
	pageLinks := GenPaginatedLinks(baseLink, pages, pagenum, pagesize)
	pageType := []string{"self", "last", "first", "previous", "next"}

	if s.Params != nil {
		params := s.Params
		switch {
		case params.HasFields && params.HasInclude && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&include=%s&filter=%s", s.FieldsStr, s.IncludeStr, s.FilterStr)
				}
			}
		case params.HasFields && params.HasInclude:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
				}
			}
		case params.HasFields && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&filter=%s", s.FieldsStr, s.FilterStr)
				}
			}
		case params.HasInclude && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&include=%s&filter=%s", s.IncludeStr, s.FilterStr)
				}
			}
		case params.HasInclude:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&include=%s", s.IncludeStr)
				}
			}
		case params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&filter=%s", s.FilterStr)
				}
			}
		case params.HasFields:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s", s.FieldsStr)
				}
			}
		}
	}
	jsapiLinks := &jsonapi.PaginationLinks{
		Self:  pageLinks["self"],
		Last:  pageLinks["last"],
		First: pageLinks["first"],
	}
	if _, ok := pageLinks["previous"]; ok {
		jsapiLinks.Prev = pageLinks["previous"]
	}
	if _, ok := pageLinks["next"]; ok {
		jsapiLinks.Next = pageLinks["next"]
	}
	return jsapiLinks, pages
}

func (s *Service) GenCollResourceRelSelfLink(id int64, relation string) string {
	return fmt.Sprintf(
		"%s/%d/%s",
		GenMultiResourceLink(s),
		id,
		relation,
	)
}

func (s *Service) GenCollResourceSelfLink() string {
	link := GenMultiResourceLink(s)
	if s.Params == nil {
		return link
	}
	params := s.Params
	switch {
	case params.HasFields && params.HasFilter && params.HasInclude:
		link += fmt.Sprintf("?fields=%s&include=%s&filter=%s", s.FieldsStr, s.IncludeStr, s.FilterStr)
	case params.HasFields && params.HasFilter:
		link += fmt.Sprintf("?fields=%s&filter=%s", s.FieldsStr, s.FilterStr)
	case params.HasFields && params.HasInclude:
		link += fmt.Sprintf("?fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
	case params.HasFilter && params.HasInclude:
		link += fmt.Sprintf("?filter=%s&include=%s", s.FilterStr, s.IncludeStr)
	case params.HasInclude:
		link += fmt.Sprintf("?include=%s", s.IncludeStr)
	case params.HasFilter:
		link += fmt.Sprintf("?filter=%s", s.FilterStr)
	case params.HasFields:
		link += fmt.Sprintf("?fields=%s", s.FieldsStr)
	}
	return link
}

func (s *Service) GenResourceSelfLink(id int64) string {
	links := GenSingleResourceLink(s, id)
	if !s.IsListMethod() && s.Params != nil {
		params := s.Params
		switch {
		case params.HasFields && params.HasInclude:
			links += fmt.Sprintf("?fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
		case params.HasFields:
			links += fmt.Sprintf("?fields=%s", s.FieldsStr)
		case params.HasInclude:
			links += fmt.Sprintf("?include=%s", s.IncludeStr)
		}
	}
	return links
}
<commit_msg>Removed the extra resource name to remove repetition in the url<commit_after>package aphgrpc

import (
	"fmt"
	"math"
	"net/http"
	"strings"
	"time"

	"gopkg.in/mgutz/dat.v2/dat"
	"gopkg.in/mgutz/dat.v2/sqlx-runner"

	"github.com/dictyBase/go-genproto/dictybaseapis/api/jsonapi"
	"github.com/fatih/structs"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/timestamp"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	context "golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

const (
	DefaultPagenum  int64 = 1
	DefaultPagesize int64 = 10
)

// JSONAPIParamsInfo interface should be implement by all grpc-gateway services
// that supports JSON API specifications.
type JSONAPIParamsInfo interface {
	// Relationships that could be included
	AllowedInclude() []string
	// Attribute fields that are allowed
	AllowedFields() []string
	// Filter fields that are allowed
	AllowedFilter() []string
	// FilterToColumns provides mapping between filter and storage columns
	FilterToColumns() map[string]string
	// RequiredAttrs are the mandatory attributes for creating a new resource
	RequiredAttrs() []string
}

// JSONAPIResource interface provides information about HTTP resource. All
// grpc-gateway services that supports JSONAPI should implement this interface.
type JSONAPIResource interface {
	//GetResourceName returns canonical resource name
	GetResourceName() string
	// GetBaseURL returns the base url with the scheme
	GetBaseURL() string
	// GetPrefix returns the path that could be appended to base url
	GetPathPrefix() string
}

func NullToTime(nt dat.NullTime) *timestamp.Timestamp {
	var ts *timestamp.Timestamp
	if nt.Valid {
		ts, _ := ptypes.TimestampProto(nt.Time)
		return ts
	}
	return ts
}

func ProtoTimeStamp(ts *timestamp.Timestamp) time.Time {
	t, _ := ptypes.Timestamp(ts)
	return t
}

func TimestampProto(t time.Time) *timestamp.Timestamp {
	ts, _ := ptypes.TimestampProto(t)
	return ts
}

func NullToString(s dat.NullString) string {
	if s.Valid {
		return s.String
	}
	return ""
}

func NullToInt64(i dat.NullInt64) int64 {
	if i.Valid {
		return i.Int64
	}
	var i64 int64
	return i64
}

// GetTotalPageNum calculate total no of pages from total no. records and page size
func GetTotalPageNum(record, pagesize int64) int64 {
	total := int64(math.Floor(float64(record) / float64(pagesize)))
	if math.Mod(float64(record), float64(pagesize)) > 0 {
		total += 1
	}
	return total
}

// GenPaginatedLinks generates paginated resource links
// from various page properties.
func GenPaginatedLinks(url string, lastpage, pagenum, pagesize int64) map[string]string {
	var links map[string]string
	links["self"] = AppendPaginationParams(url, pagenum, pagesize)
	links["first"] = AppendPaginationParams(url, 1, pagesize)
	if pagenum != 1 {
		links["previous"] = AppendPaginationParams(url, pagenum-1, pagesize)
	}
	links["last"] = AppendPaginationParams(url, lastpage, pagesize)
	if pagenum != lastpage {
		links["next"] = AppendPaginationParams(url, pagenum+1, pagesize)
	}
	return links
}

func GenBaseLink(rs JSONAPIResource) string {
	return fmt.Sprintf(
		"%s/%s",
		strings.Trim(rs.GetBaseURL(), "/"),
		strings.Trim(rs.GetPathPrefix(), "/"),
	)
}

func GenSingleResourceLink(rs JSONAPIResource, id int64) string {
	return fmt.Sprintf(
		"%s/%d",
		GenBaseLink(rs),
		id,
	)
}

func GenMultiResourceLink(rs JSONAPIResource) string {
	return GenBaseLink(rs)
}

func AppendPaginationParams(url string, pagenum, pagesize int64) string {
	return fmt.Sprintf("%s?pagenum=%d&pagesize=%d", url, pagenum, pagesize)
}

func GenPaginatedResourceLink(rs JSONAPIResource, pagenum, pagesize int64) string {
	return fmt.Sprintf(
		"%s?pagenum=%d&pagesize=%d",
		GenBaseLink(rs),
		pagenum,
		pagesize,
	)
}

func GenSelfRelationshipLink(rs JSONAPIResource, rel string, id int64) string {
	return fmt.Sprintf(
		"%s/%d/relationships/%s",
		GenBaseLink(rs),
		id,
		rel,
	)
}

func GenRelatedRelationshipLink(rs JSONAPIResource, rel string, id int64) string {
	return fmt.Sprintf(
		"%s/%d/%s",
		GenBaseLink(rs),
		id,
		rel,
	)
}

//GetDefinedTagsWithValue check for fields that are initialized and returns a map
//with the tag and their values
func GetDefinedTagsWithValue(i interface{}, key string) map[string]interface{} {
	m := make(map[string]interface{})
	s := structs.New(i)
	for _, f := range s.Fields() {
		if !f.IsZero() {
			m[f.Tag(key)] = f.Value()
		}
	}
	return m
}

//GetDefinedTags check for fields that are initialized and returns a slice of
//their matching tag values
func GetDefinedTags(i interface{}, tag string) []string {
	var v []string
	s := structs.New(i)
	for _, f := range s.Fields() {
		if !f.IsZero() {
			v = append(v, f.Tag(tag))
		}
	}
	return v
}

// HandleCreateResponse modifies the grpc gateway filter which adds the JSON API header and
// modifies the http status response for POST request
func HandleCreateResponse(ctx context.Context, w http.ResponseWriter, resp proto.Message) error {
	w.Header().Set("Content-Type", "application/vnd.api+json")
	md, ok := runtime.ServerMetadataFromContext(ctx)
	if ok {
		trMD := md.TrailerMD
		if _, ok := trMD["method"]; ok {
			switch trMD["method"][0] {
			case "POST":
				w.WriteHeader(http.StatusCreated)
			case "POST_NO_CONTENT":
				w.WriteHeader(http.StatusNoContent)
			}
		}
	}
	return nil
}

type Service struct {
	Dbh             *runner.DB
	PathPrefix      string
	Include         []string
	IncludeStr      string
	FieldsToColumns map[string]string
	FieldsStr       string
	Resource        string
	BaseURL         string
	FilToColumns    map[string]string
	FilterStr       string
	Params          *JSONAPIParams
	ListMethod      bool
	ReqAttrs        []string
}

func (s *Service) RequiredAttrs() []string {
	return s.ReqAttrs
}

func (s *Service) IsListMethod() bool {
	return s.ListMethod
}

func (s *Service) FilterToColumns() map[string]string {
	return s.FilToColumns
}

func (s *Service) AllowedFilter() []string {
	var f []string
	for k, _ := range s.FilterToColumns() {
		f = append(f, k)
	}
	return f
}

func (s *Service) AllowedInclude() []string {
	return s.Include
}

func (s *Service) AllowedFields() []string {
	var f []string
	for k, _ := range s.FieldsToColumns {
		f = append(f, k)
	}
	return f
}

func (s *Service) GetResourceName() string {
	return s.Resource
}

func (s *Service) GetBaseURL() string {
	return s.BaseURL
}

func (s *Service) GetPathPrefix() string {
	return s.PathPrefix
}

func (s *Service) SetBaseURL(ctx context.Context) error {
	md, ok := metadata.FromIncomingContext(ctx)
	if !ok {
		return ErrRetrieveMetadata
	}
	slice, ok := md["x-forwarded-host"]
	if !ok {
		return ErrXForwardedHost
	}
	s.BaseURL = fmt.Sprintf("http://%s", slice[0])
	return nil
}

func (s *Service) MapFieldsToColumns(fields []string) []string {
	var columns []string
	for _, v := range fields {
		columns = append(columns, s.FieldsToColumns[v])
	}
	return columns
}

func (s *Service) GetCount(table string) (int64, error) {
	var count int64
	err := s.Dbh.Select("COUNT(*)").From(table).QueryScalar(&count)
	return count, err
}

func (s *Service) GetAllFilteredCount(table string) (int64, error) {
	var count int64
	err := s.Dbh.Select("COUNT(*)").
		From(table).
		Scope(
			FilterToWhereClause(s, s.Params.Filters),
			FilterToBindValue(s.Params.Filters)...,
		).QueryScalar(&count)
	return count, err
}

// GetRelatedPagination generates JSONAPI pagination links for relation resources
func (s *Service) GetRelatedPagination(id, record, pagenum, pagesize int64, relation string) (*jsonapi.PaginationLinks, int64) {
	pages := GetTotalPageNum(record, pagesize)
	baseLink := s.GenCollResourceRelSelfLink(id, relation)
	pageLinks := GenPaginatedLinks(baseLink, pages, pagenum, pagesize)
	jsapiLinks := &jsonapi.PaginationLinks{
		Self:  pageLinks["self"],
		Last:  pageLinks["last"],
		First: pageLinks["first"],
	}
	if _, ok := pageLinks["previous"]; ok {
		jsapiLinks.Prev = pageLinks["previous"]
	}
	if _, ok := pageLinks["next"]; ok {
		jsapiLinks.Next = pageLinks["next"]
	}
	return jsapiLinks, pages
}

// GetPagination generates JSONAPI pagination links along with fields, include and filter query parameters
func (s *Service) GetPagination(record, pagenum, pagesize int64) (*jsonapi.PaginationLinks, int64) {
	pages := GetTotalPageNum(record, pagesize)
	baseLink := s.GenCollResourceSelfLink()
	pageLinks := GenPaginatedLinks(baseLink, pages, pagenum, pagesize)
	pageType := []string{"self", "last", "first", "previous", "next"}

	if s.Params != nil {
		params := s.Params
		switch {
		case params.HasFields && params.HasInclude && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&include=%s&filter=%s", s.FieldsStr, s.IncludeStr, s.FilterStr)
				}
			}
		case params.HasFields && params.HasInclude:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
				}
			}
		case params.HasFields && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s&filter=%s", s.FieldsStr, s.FilterStr)
				}
			}
		case params.HasInclude && params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&include=%s&filter=%s", s.IncludeStr, s.FilterStr)
				}
			}
		case params.HasInclude:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&include=%s", s.IncludeStr)
				}
			}
		case params.HasFilter:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&filter=%s", s.FilterStr)
				}
			}
		case params.HasFields:
			for _, v := range pageType {
				if _, ok := pageLinks[v]; ok {
					pageLinks[v] += fmt.Sprintf("&fields=%s", s.FieldsStr)
				}
			}
		}
	}
	jsapiLinks := &jsonapi.PaginationLinks{
		Self:  pageLinks["self"],
		Last:  pageLinks["last"],
		First: pageLinks["first"],
	}
	if _, ok := pageLinks["previous"]; ok {
		jsapiLinks.Prev = pageLinks["previous"]
	}
	if _, ok := pageLinks["next"]; ok {
		jsapiLinks.Next = pageLinks["next"]
	}
	return jsapiLinks, pages
}

func (s *Service) GenCollResourceRelSelfLink(id int64, relation string) string {
	return fmt.Sprintf(
		"%s/%d/%s",
		GenMultiResourceLink(s),
		id,
		relation,
	)
}

func (s *Service) GenCollResourceSelfLink() string {
	link := GenMultiResourceLink(s)
	if s.Params == nil {
		return link
	}
	params := s.Params
	switch {
	case params.HasFields && params.HasFilter && params.HasInclude:
		link += fmt.Sprintf("?fields=%s&include=%s&filter=%s", s.FieldsStr, s.IncludeStr, s.FilterStr)
	case params.HasFields && params.HasFilter:
		link += fmt.Sprintf("?fields=%s&filter=%s", s.FieldsStr, s.FilterStr)
	case params.HasFields && params.HasInclude:
		link += fmt.Sprintf("?fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
	case params.HasFilter && params.HasInclude:
		link += fmt.Sprintf("?filter=%s&include=%s", s.FilterStr, s.IncludeStr)
	case params.HasInclude:
		link += fmt.Sprintf("?include=%s", s.IncludeStr)
	case params.HasFilter:
		link += fmt.Sprintf("?filter=%s", s.FilterStr)
	case params.HasFields:
		link += fmt.Sprintf("?fields=%s", s.FieldsStr)
	}
	return link
}

func (s *Service) GenResourceSelfLink(id int64) string {
	links := GenSingleResourceLink(s, id)
	if !s.IsListMethod() && s.Params != nil {
		params := s.Params
		switch {
		case params.HasFields && params.HasInclude:
			links += fmt.Sprintf("?fields=%s&include=%s", s.FieldsStr, s.IncludeStr)
		case params.HasFields:
			links += fmt.Sprintf("?fields=%s", s.FieldsStr)
		case params.HasInclude:
			links += fmt.Sprintf("?include=%s", s.IncludeStr)
		}
	}
	return links
}
<|endoftext|>