<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hubbub

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/google/go-github/v31/github"
	"k8s.io/klog/v2"

	"github.com/google/triage-party/pkg/tag"
)

var (
	// wordRelRefRe parses relative issue references, like "fixes #3402"
	wordRelRefRe = regexp.MustCompile(`\s#(\d+)\b`)

	// puncRelRefRe parses relative issue references, like "fixes #3402."
	puncRelRefRe = regexp.MustCompile(`\s\#(\d+)[\.\!:\?]`)

	// absRefRe parses absolute issue references, like "fixes http://github.com/minikube/issues/432"
	absRefRe = regexp.MustCompile(`https*://github.com/(\w+)/(\w+)/[ip][us]\w+/(\d+)`)

	// codeRe matches code
	codeRe    = regexp.MustCompile("(?s)```.*?```")
	detailsRe = regexp.MustCompile(`(?s)<details>.*</details>`)
)

// GitHubItem is an interface that matches both GitHub Issues and PullRequests
type GitHubItem interface {
	GetAssignee() *github.User
	GetAuthorAssociation() string
	GetBody() string
	GetComments() int
	GetHTMLURL() string
	GetCreatedAt() time.Time
	GetID() int64
	GetMilestone() *github.Milestone
	GetNumber() int
	GetClosedAt() time.Time
	GetState() string
	GetTitle() string
	GetURL() string
	GetUpdatedAt() time.Time
	GetUser() *github.User
	String() string
}

// createConversation creates a conversation from an issue-like
func (h *Engine) createConversation(i GitHubItem, cs []*Comment, age time.Time) *Conversation {

	authorIsMember := false
	if h.isMember(i.GetUser().GetLogin(), i.GetAuthorAssociation()) {
		authorIsMember = true
	}

	co := &Conversation{
		ID:                   i.GetNumber(),
		URL:                  i.GetHTMLURL(),
		Author:               i.GetUser(),
		Title:                i.GetTitle(),
		State:                i.GetState(),
		Type:                 Issue,
		Seen:                 age,
		Created:              i.GetCreatedAt(),
		Updated:              i.GetUpdatedAt(),
		CommentsTotal:        i.GetComments(),
		ClosedAt:             i.GetClosedAt(),
		SelfInflicted:        authorIsMember,
		LatestAuthorResponse: i.GetCreatedAt(),
		Milestone:            i.GetMilestone(),
		Reactions:            map[string]int{},
		LastCommentAuthor:    i.GetUser(),
		LastCommentBody:      i.GetBody(),
		Tags:                 map[tag.Tag]bool{},
	}

	if co.CommentsTotal == 0 {
		co.CommentsTotal = len(cs)
	}

	// "https://github.com/kubernetes/minikube/issues/7179",
	urlParts := strings.Split(i.GetHTMLURL(), "/")
	co.Organization = urlParts[3]
	co.Project = urlParts[4]
	h.parseRefs(i.GetBody(), co, i.GetUpdatedAt())

	if i.GetAssignee() != nil {
		co.Assignees = append(co.Assignees, i.GetAssignee())
		co.Tags[tag.Assigned] = true
	}

	if !authorIsMember {
		co.LatestMemberResponse = i.GetCreatedAt()
	}

	lastQuestion := time.Time{}
	seenCommenters := map[string]bool{}
	seenClosedCommenters := map[string]bool{}
	seenMemberComment := false

	if h.debug[co.ID] {
		klog.Errorf("debug conversation: %s", formatStruct(co))
	}

	for _, c := range cs {
		h.parseRefs(c.Body, co, c.Updated)
		if h.debug[co.ID] {
			klog.Errorf("debug conversation comment: %s", formatStruct(c))
		}

		// We don't like their kind around here
		if isBot(c.User) {
			continue
		}

		co.LastCommentBody = c.Body
		co.LastCommentAuthor = c.User

		r := c.Reactions
		if r.GetTotalCount() > 0 {
			co.ReactionsTotal += r.GetTotalCount()
			for k, v := range reactions(r) {
				co.Reactions[k] += v
			}
		}

		if !i.GetClosedAt().IsZero() && c.Created.After(i.GetClosedAt().Add(30*time.Second)) {
			klog.V(1).Infof("#%d: comment after closed on %s: %+v", co.ID, i.GetClosedAt(), c)
			co.ClosedCommentsTotal++
			seenClosedCommenters[*c.User.Login] = true
		}

		if c.User.GetLogin() == i.GetUser().GetLogin() {
			co.LatestAuthorResponse = c.Created
		}

		if c.User.GetLogin() == i.GetAssignee().GetLogin() {
			co.LatestAssigneeResponse = c.Created
		}

		if h.isMember(c.User.GetLogin(), c.AuthorAssoc) && !isBot(c.User) {
			if !co.LatestMemberResponse.After(co.LatestAuthorResponse) && !authorIsMember {
				co.AccumulatedHoldTime += c.Created.Sub(co.LatestAuthorResponse)
			}
			co.LatestMemberResponse = c.Created
			if !seenMemberComment {
				co.Tags[tag.Commented] = true
				seenMemberComment = true
			}
		}

		if strings.Contains(c.Body, "?") {
			for _, line := range strings.Split(c.Body, "\n") {
				line = strings.TrimSpace(line)
				if strings.HasPrefix(line, ">") {
					continue
				}
				if strings.Contains(line, "?") {
					lastQuestion = c.Created
				}
			}
		}

		if !seenCommenters[*c.User.Login] {
			co.Commenters = append(co.Commenters, c.User)
			seenCommenters[*c.User.Login] = true
		}
	}

	if co.LatestMemberResponse.After(co.LatestAuthorResponse) {
		co.Tags[tag.Send] = true
		co.CurrentHoldTime = 0
	} else if !authorIsMember {
		co.Tags[tag.Recv] = true
		co.CurrentHoldTime += time.Since(co.LatestAuthorResponse)
		co.AccumulatedHoldTime += time.Since(co.LatestAuthorResponse)
	}

	if lastQuestion.After(co.LatestMemberResponse) {
		co.Tags[tag.RecvQ] = true
	}

	if co.Milestone != nil && co.Milestone.GetState() == "open" {
		co.Tags[tag.OpenMilestone] = true
	}

	if !co.LatestAssigneeResponse.IsZero() {
		co.Tags[tag.AssigneeUpdated] = true
	}

	if len(cs) > 0 {
		last := cs[len(cs)-1]
		assoc := strings.ToLower(last.AuthorAssoc)
		if assoc == "none" {
			if last.User.GetLogin() == i.GetUser().GetLogin() {
				co.Tags[tag.AuthorLast] = true
			}
		} else {
			co.Tags[tag.RoleLast(assoc)] = true
		}

		if last.Updated.After(co.Updated) {
			co.Updated = last.Updated
		}
	}

	if co.State == "closed" {
		co.Tags[tag.Closed] = true
	}

	co.CommentersTotal = len(seenCommenters)
	co.ClosedCommentersTotal = len(seenClosedCommenters)

	if co.AccumulatedHoldTime > time.Since(co.Created) {
		panic(fmt.Sprintf("accumulated %s is more than age %s", co.AccumulatedHoldTime, time.Since(co.Created)))
	}

	// Loose, but good enough
	months := time.Since(co.Created).Hours() / 24 / 30
	co.CommentersPerMonth = float64(co.CommentersTotal) / months
	co.ReactionsPerMonth = float64(co.ReactionsTotal) / months
	return co
}

// Return if a user or role should be considered a member
func (h *Engine) isMember(user string, role string) bool {
	if h.members[user] {
		return true
	}

	if h.memberRoles[strings.ToLower(role)] {
		return true
	}

	return false
}

// UpdateIssueRefs updates referenced issues within a conversation, adding it if necessary
func (co *Conversation) UpdateIssueRefs(rc *RelatedConversation) {
	for i, ex := range co.IssueRefs {
		if ex.URL == rc.URL {
			if ex.Seen.After(rc.Seen) {
				return
			}
			co.IssueRefs[i] = rc
			return
		}
	}

	co.IssueRefs = append(co.IssueRefs, rc)
}

// UpdatePullRequestRefs updates referenced PR's within a conversation, adding it if necessary
func (co *Conversation) UpdatePullRequestRefs(rc *RelatedConversation) {
	for i, ex := range co.PullRequestRefs {
		if ex.URL == rc.URL {
			if ex.Seen.After(rc.Seen) {
				return
			}
			co.PullRequestRefs[i] = rc
			return
		}
	}

	co.PullRequestRefs = append(co.PullRequestRefs, rc)
}

// parse any references and update mention time
func (h *Engine) parseRefs(text string, co *Conversation, t time.Time) {

	// remove code samples which mention unrelated issues
	text = codeRe.ReplaceAllString(text, "")
	text = detailsRe.ReplaceAllString(text, "")

	var ms [][]string
	ms = append(ms, wordRelRefRe.FindAllStringSubmatch(text, -1)...)
	ms = append(ms, puncRelRefRe.FindAllStringSubmatch(text, -1)...)

	seen := map[string]bool{}

	for _, m := range ms {
		i, err := strconv.Atoi(m[1])
		if err != nil {
			klog.Errorf("unable to parse int from %s: %v", m[1], err)
			continue
		}

		if i == co.ID {
			continue
		}

		rc := &RelatedConversation{
			Organization: co.Organization,
			Project:      co.Project,
			ID:           i,
			Seen:         t,
		}

		if t.After(h.mtimeRef(rc)) {
			klog.V(1).Infof("%s later referenced #%d at %s: %s", co.URL, i, t, text)
			h.updateMtimeLong(co.Organization, co.Project, i, t)
		}

		if !seen[fmt.Sprintf("%s/%d", rc.Project, rc.ID)] {
			co.UpdateIssueRefs(rc)
		}
		seen[fmt.Sprintf("%s/%d", rc.Project, rc.ID)] = true
	}

	for _, m := range absRefRe.FindAllStringSubmatch(text, -1) {
		org := m[1]
		project := m[2]
		i, err := strconv.Atoi(m[3])
		if err != nil {
			klog.Errorf("unable to parse int from %s: %v", err)
			continue
		}

		if i == co.ID && org == co.Organization && project == co.Project {
			continue
		}

		rc := &RelatedConversation{
			Organization: org,
			Project:      project,
			ID:           i,
			Seen:         t,
		}

		if t.After(h.mtimeRef(rc)) {
			klog.Infof("%s later referenced %s/%s #%d at %s: %s", co.URL, org, project, i, t, text)
			h.updateMtimeLong(org, project, i, t)
		}

		if !seen[fmt.Sprintf("%s/%d", rc.Project, rc.ID)] {
			co.UpdateIssueRefs(rc)
		}
		seen[fmt.Sprintf("%s/%d", rc.Project, rc.ID)] = true
	}
}
<commit_msg>Defer adding comment tags until all comments are fetched<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hubbub

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/google/go-github/v31/github"
	"k8s.io/klog/v2"

	"github.com/google/triage-party/pkg/tag"
)

var (
	// wordRelRefRe parses relative issue references, like "fixes #3402"
	wordRelRefRe = regexp.MustCompile(`\s#(\d+)\b`)

	// puncRelRefRe parses relative issue references, like "fixes #3402."
	puncRelRefRe = regexp.MustCompile(`\s\#(\d+)[\.\!:\?]`)

	// absRefRe parses absolute issue references, like "fixes http://github.com/minikube/issues/432"
	absRefRe = regexp.MustCompile(`https*://github.com/(\w+)/(\w+)/[ip][us]\w+/(\d+)`)

	// codeRe matches code
	codeRe    = regexp.MustCompile("(?s)```.*?```")
	detailsRe = regexp.MustCompile(`(?s)<details>.*</details>`)
)

// GitHubItem is an interface that matches both GitHub Issues and PullRequests
type GitHubItem interface {
	GetAssignee() *github.User
	GetAuthorAssociation() string
	GetBody() string
	GetComments() int
	GetHTMLURL() string
	GetCreatedAt() time.Time
	GetID() int64
	GetMilestone() *github.Milestone
	GetNumber() int
	GetClosedAt() time.Time
	GetState() string
	GetTitle() string
	GetURL() string
	GetUpdatedAt() time.Time
	GetUser() *github.User
	String() string
}

// createConversation creates a conversation from an issue-like
func (h *Engine) createConversation(i GitHubItem, cs []*Comment, age time.Time) *Conversation {
	klog.Infof("creating conversation for #%d with %d/%d comments (age: %s)", i.GetNumber(), len(cs), i.GetComments(), age)

	authorIsMember := false
	if h.isMember(i.GetUser().GetLogin(), i.GetAuthorAssociation()) {
		authorIsMember = true
	}

	co := &Conversation{
		ID:            i.GetNumber(),
		URL:           i.GetHTMLURL(),
		Author:        i.GetUser(),
		Title:         i.GetTitle(),
		State:         i.GetState(),
		Type:          Issue,
		Seen:          age,
		Created:       i.GetCreatedAt(),
		Updated:       i.GetUpdatedAt(),
		CommentsTotal: i.GetComments(),
		// How many comments were parsed
		CommentsSeen:         len(cs),
		ClosedAt:             i.GetClosedAt(),
		SelfInflicted:        authorIsMember,
		LatestAuthorResponse: i.GetCreatedAt(),
		Milestone:            i.GetMilestone(),
		Reactions:            map[string]int{},
		LastCommentAuthor:    i.GetUser(),
		LastCommentBody:      i.GetBody(),
		Tags:                 map[tag.Tag]bool{},
	}

	if co.CommentsTotal == 0 {
		co.CommentsTotal = len(cs)
	}

	// "https://github.com/kubernetes/minikube/issues/7179",
	urlParts := strings.Split(i.GetHTMLURL(), "/")
	co.Organization = urlParts[3]
	co.Project = urlParts[4]
	h.parseRefs(i.GetBody(), co, i.GetUpdatedAt())

	if i.GetAssignee() != nil {
		co.Assignees = append(co.Assignees, i.GetAssignee())
		co.Tags[tag.Assigned] = true
	}

	if !authorIsMember {
		co.LatestMemberResponse = i.GetCreatedAt()
	}

	lastQuestion := time.Time{}
	seenCommenters := map[string]bool{}
	seenClosedCommenters := map[string]bool{}
	seenMemberComment := false

	if h.debug[co.ID] {
		klog.Errorf("debug conversation: %s", formatStruct(co))
	}

	for _, c := range cs {
		h.parseRefs(c.Body, co, c.Updated)
		if h.debug[co.ID] {
			klog.Errorf("debug conversation comment: %s", formatStruct(c))
		}

		// We don't like their kind around here
		if isBot(c.User) {
			continue
		}

		co.LastCommentBody = c.Body
		co.LastCommentAuthor = c.User

		r := c.Reactions
		if r.GetTotalCount() > 0 {
			co.ReactionsTotal += r.GetTotalCount()
			for k, v := range reactions(r) {
				co.Reactions[k] += v
			}
		}

		if !i.GetClosedAt().IsZero() && c.Created.After(i.GetClosedAt().Add(30*time.Second)) {
			klog.V(1).Infof("#%d: comment after closed on %s: %+v", co.ID, i.GetClosedAt(), c)
			co.ClosedCommentsTotal++
			seenClosedCommenters[*c.User.Login] = true
		}

		if c.User.GetLogin() == i.GetUser().GetLogin() {
			co.LatestAuthorResponse = c.Created
		}

		if c.User.GetLogin() == i.GetAssignee().GetLogin() {
			co.LatestAssigneeResponse = c.Created
		}

		if h.isMember(c.User.GetLogin(), c.AuthorAssoc) && !isBot(c.User) {
			if !co.LatestMemberResponse.After(co.LatestAuthorResponse) && !authorIsMember {
				co.AccumulatedHoldTime += c.Created.Sub(co.LatestAuthorResponse)
			}
			co.LatestMemberResponse = c.Created
			if !seenMemberComment {
				co.Tags[tag.Commented] = true
				seenMemberComment = true
			}
		}

		if strings.Contains(c.Body, "?") {
			for _, line := range strings.Split(c.Body, "\n") {
				line = strings.TrimSpace(line)
				if strings.HasPrefix(line, ">") {
					continue
				}
				if strings.Contains(line, "?") {
					lastQuestion = c.Created
				}
			}
		}

		if !seenCommenters[*c.User.Login] {
			co.Commenters = append(co.Commenters, c.User)
			seenCommenters[*c.User.Login] = true
		}
	}

	if co.Milestone != nil && co.Milestone.GetState() == "open" {
		co.Tags[tag.OpenMilestone] = true
	}

	if !co.LatestAssigneeResponse.IsZero() {
		co.Tags[tag.AssigneeUpdated] = true
	}

	// Only add these tags if we've seen all the comments
	if len(cs) >= co.CommentsTotal {
		if co.LatestMemberResponse.After(co.LatestAuthorResponse) {
			co.Tags[tag.Send] = true
			co.CurrentHoldTime = 0
		} else if !authorIsMember {
			co.Tags[tag.Recv] = true
			co.CurrentHoldTime += time.Since(co.LatestAuthorResponse)
			co.AccumulatedHoldTime += time.Since(co.LatestAuthorResponse)
		}

		if lastQuestion.After(co.LatestMemberResponse) {
			co.Tags[tag.RecvQ] = true
		}
	}

	if len(cs) > 0 {
		last := cs[len(cs)-1]
		assoc := strings.ToLower(last.AuthorAssoc)
		if assoc == "none" {
			if last.User.GetLogin() == i.GetUser().GetLogin() {
				co.Tags[tag.AuthorLast] = true
			}
		} else {
			co.Tags[tag.RoleLast(assoc)] = true
		}

		if last.Updated.After(co.Updated) {
			co.Updated = last.Updated
		}
	}

	if co.State == "closed" {
		co.Tags[tag.Closed] = true
	}

	co.CommentersTotal = len(seenCommenters)
	co.ClosedCommentersTotal = len(seenClosedCommenters)

	if co.AccumulatedHoldTime > time.Since(co.Created) {
		panic(fmt.Sprintf("accumulated %s is more than age %s", co.AccumulatedHoldTime, time.Since(co.Created)))
	}

	// Loose, but good enough
	months := time.Since(co.Created).Hours() / 24 / 30
	co.CommentersPerMonth = float64(co.CommentersTotal) / months
	co.ReactionsPerMonth = float64(co.ReactionsTotal) / months

	tagNames := []string{}
	for k := range co.Tags {
		tagNames = append(tagNames, k.ID)
	}

	if len(tagNames) > 0 {
		klog.V(1).Infof("#%d tags based on %d/%d comments: %s", co.ID, co.CommentsSeen, co.CommentsTotal, tagNames)
	}
	return co
}

// Return if a user or role should be considered a member
func (h *Engine) isMember(user string, role string) bool {
	if h.members[user] {
		return true
	}

	if h.memberRoles[strings.ToLower(role)] {
		return true
	}

	klog.V(1).Infof("%s (%s) is not considered a member: members=%s memberRoles=%s", user, role, h.members, h.memberRoles)
	return false
}

// UpdateIssueRefs updates referenced issues within a conversation, adding it if necessary
func (co *Conversation) UpdateIssueRefs(rc *RelatedConversation) {
	for i, ex := range co.IssueRefs {
		if ex.URL == rc.URL {
			if ex.Seen.After(rc.Seen) {
				return
			}
			co.IssueRefs[i] = rc
			return
		}
	}

	co.IssueRefs = append(co.IssueRefs, rc)
}

// UpdatePullRequestRefs updates referenced PR's within a conversation, adding it if necessary
func (co *Conversation) UpdatePullRequestRefs(rc *RelatedConversation) {
	for i, ex := range co.PullRequestRefs {
		if ex.URL == rc.URL {
			if ex.Seen.After(rc.Seen) {
				return
			}
			co.PullRequestRefs[i] = rc
			return
		}
	}

	co.PullRequestRefs = append(co.PullRequestRefs, rc)
}

// parse any references and update mention time
func (h *Engine) parseRefs(text string, co *Conversation, t time.Time) {

	// remove code samples which mention unrelated issues
	text = codeRe.ReplaceAllString(text, "")
	text = detailsRe.ReplaceAllString(text, "")

	var ms [][]string
	ms = append(ms, wordRelRefRe.FindAllStringSubmatch(text, -1)...)
	ms = append(ms, puncRelRefRe.FindAllStringSubmatch(text, -1)...)

	seen := map[string]bool{}

	for _, m := range ms {
		i, err := strconv.Atoi(m[1])
		if err != nil {
			klog.Errorf("unable to parse int from %s: %v", m[1], err)
			continue
		}

		if i == co.ID {
			continue
		}

		rc := &RelatedConversation{
			Organization: co.Organization,
			Project:      co.Project,
			ID:           i,
			Seen:         t,
		}

		if t.After(h.mtimeRef(rc)) {
			klog.V(1).Infof("%s later referenced #%d at %s: %s", co.URL, i, t, text)
			h.updateMtimeLong(co.Organization, co.Project, i, t)
		}

		if !seen[fmt.Sprintf("%s/%d", rc.Project, rc.ID)] {
			co.UpdateIssueRefs(rc)
		}
		seen[fmt.Sprintf("%s/%d", rc.Project, rc.ID)] = true
	}

	for _, m := range absRefRe.FindAllStringSubmatch(text, -1) {
		org := m[1]
		project := m[2]
		i, err := strconv.Atoi(m[3])
		if err != nil {
			klog.Errorf("unable to parse int from %s: %v", err)
			continue
		}

		if i == co.ID && org == co.Organization && project == co.Project {
			continue
		}

		rc := &RelatedConversation{
			Organization: org,
			Project:      project,
			ID:           i,
			Seen:         t,
		}

		if t.After(h.mtimeRef(rc)) {
			klog.Infof("%s later referenced %s/%s #%d at %s: %s", co.URL, org, project, i, t, text)
			h.updateMtimeLong(org, project, i, t)
		}

		if !seen[fmt.Sprintf("%s/%d", rc.Project, rc.ID)] {
			co.UpdateIssueRefs(rc)
		}
		seen[fmt.Sprintf("%s/%d", rc.Project, rc.ID)] = true
	}
}
<|endoftext|>


<commit_before>/*
© Copyright IBM Corporation 2018, 2019

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package mqini provides information about queue managers
package mqini

import (
	"bufio"
	"path/filepath"
	"strings"

	"github.com/ibm-messaging/mq-container/internal/command"
)

// QueueManager describe high-level configuration information for a queue manager
type QueueManager struct {
	Name             string
	Prefix           string
	Directory        string
	DataPath         string
	InstallationName string
}

// getQueueManagerFromStanza parses a queue manager stanza
func getQueueManagerFromStanza(stanza string) (*QueueManager, error) {
	scanner := bufio.NewScanner(strings.NewReader(stanza))
	qm := QueueManager{}
	for scanner.Scan() {
		l := scanner.Text()
		l = strings.TrimSpace(l)
		t := strings.Split(l, "=")
		switch t[0] {
		case "Name":
			qm.Name = t[1]
		case "Prefix":
			qm.Prefix = t[1]
		case "Directory":
			qm.Directory = t[1]
		case "DataPath":
			qm.DataPath = t[1]
		case "InstallationName":
			qm.InstallationName = t[1]
		}
	}
	return &qm, scanner.Err()
}

// GetQueueManager returns queue manager configuration information
func GetQueueManager(name string) (*QueueManager, error) {
	// dspmqinf essentially returns a subset of mqs.ini, but it's simpler to parse
	out, _, err := command.Run("dspmqinf", "-o", "stanza", name)
	if err != nil {
		return nil, err
	}
	return getQueueManagerFromStanza(out)
}

// GetErrorLogDirectory returns the directory holding the error logs for the
// specified queue manager
func GetErrorLogDirectory(qm *QueueManager) string {
	return filepath.Join(GetDataDirectory(qm), "errors")
}

// GetDataDirectory returns the data directory for the specified queue manager
func GetDataDirectory(qm *QueueManager) string {
	if qm.DataPath != "" {
		// Data path has been set explicitly (e.g. for multi-instance queue manager)
		return qm.DataPath
	} else {
		return filepath.Join(qm.Prefix, "qmgrs", qm.Directory)
	}
}
<commit_msg>Don't run dspmqinf before crtmqdir<commit_after>/*
© Copyright IBM Corporation 2018, 2019

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package mqini provides information about queue managers
package mqini

import (
	"bufio"
	"errors"
	"os"
	"path/filepath"
	"strings"

	"github.com/ibm-messaging/mq-container/internal/command"
)

// QueueManager describe high-level configuration information for a queue manager
type QueueManager struct {
	Name             string
	Prefix           string
	Directory        string
	DataPath         string
	InstallationName string
}

// getQueueManagerFromStanza parses a queue manager stanza
func getQueueManagerFromStanza(stanza string) (*QueueManager, error) {
	scanner := bufio.NewScanner(strings.NewReader(stanza))
	qm := QueueManager{}
	for scanner.Scan() {
		l := scanner.Text()
		l = strings.TrimSpace(l)
		t := strings.Split(l, "=")
		switch t[0] {
		case "Name":
			qm.Name = t[1]
		case "Prefix":
			qm.Prefix = t[1]
		case "Directory":
			qm.Directory = t[1]
		case "DataPath":
			qm.DataPath = t[1]
		case "InstallationName":
			qm.InstallationName = t[1]
		}
	}
	return &qm, scanner.Err()
}

// GetQueueManager returns queue manager configuration information
func GetQueueManager(name string) (*QueueManager, error) {
	_, err := os.Stat("/var/mqm/mqs.ini")
	if err != nil {
		// Don't run dspmqinf, which will generate an FDC if mqs.ini isn't there yet
		return nil, errors.New("dspmqinf should not be run before crtmqdir")
	}
	// dspmqinf essentially returns a subset of mqs.ini, but it's simpler to parse
	out, _, err := command.Run("dspmqinf", "-o", "stanza", name)
	if err != nil {
		return nil, err
	}
	return getQueueManagerFromStanza(out)
}

// GetErrorLogDirectory returns the directory holding the error logs for the
// specified queue manager
func GetErrorLogDirectory(qm *QueueManager) string {
	return filepath.Join(GetDataDirectory(qm), "errors")
}

// GetDataDirectory returns the data directory for the specified queue manager
func GetDataDirectory(qm *QueueManager) string {
	if qm.DataPath != "" {
		// Data path has been set explicitly (e.g. for multi-instance queue manager)
		return qm.DataPath
	} else {
		return filepath.Join(qm.Prefix, "qmgrs", qm.Directory)
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2012 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goon

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"errors"
	"fmt"
	"net/http"
	"reflect"
	"strings"
	"fmt"
)

var (
	// LogErrors issues appengine.Context.Errorf on any error.
	LogErrors bool = true
)

// Goon holds the app engine context and request memory cache.
type Goon struct {
	context       appengine.Context
	cache         map[string]interface{}
	inTransaction bool
	toSet         map[string]interface{}
	toDelete      []string
}

func memkey(k *datastore.Key) string {
	return k.String()
}

// NewGoon creates a new Goon object from the given request.
func NewGoon(r *http.Request) *Goon {
	return FromContext(appengine.NewContext(r))
}

// FromContext creates a new Goon object from the given appengine Context.
func FromContext(c appengine.Context) *Goon {
	return &Goon{
		context: c,
		cache:   make(map[string]interface{}),
	}
}

func (g *Goon) error(err error) {
	if LogErrors {
		g.context.Errorf("goon: %v", err.Error())
	}
}

func (g *Goon) extractKeys(src interface{}) ([]*datastore.Key, error) {
	v := reflect.Indirect(reflect.ValueOf(src))
	if v.Kind() != reflect.Slice {
		return nil, errors.New("goon: value must be a slice or pointer-to-slice")
	}
	l := v.Len()

	keys := make([]*datastore.Key, l)
	for i := 0; i < l; i++ {
		vi := v.Index(i)
		key, err := g.getStructKey(vi.Interface())
		if err != nil {
			return nil, err
		}
		keys[i] = key
	}
	return keys, nil
}

// Key is the same as KeyError, except nil is returned on error.
func (g *Goon) Key(src interface{}) *datastore.Key {
	if k, err := g.KeyError(src); err == nil {
		return k
	}
	return nil
}

// Key returns the key src based on its properties.
//
// src must be a S or *S for some struct type S. The key is extracted based on
// various fields of S. If a field of type int64 or string has a struct tag
// named goon with value "id", it is used as the key's id. If a field of type
// *datastore.Key has a struct tag named goon with value "parent", it is used
// as the key's parent. If a field of type string has a struct tag named goon
// with value "kind", it is used as the key's kind. The "kind" field supports
// an optional second parameter which is the default kind name. If no kind
// field exists, the struct's name is used. These fields should all have
// their datastore field marked as "-".
//
// Example, with kind User:
//	type User struct {
//		Id    string `datastore:"-" goon:"id"`
//		Read  time.Time
//	}
// Example, with kind U if _kind is the empty string:
//	type User struct {
//		_kind string `goon:"kind,U"`
//		Id    string `datastore:"-" goon:"id"`
//		Read  time.Time
//	}
// To override the kind above to UserKind:
//	u := User{_kind: "UserKind"}
//
// An example with a parent:
//	type UserData struct {
//		Id     string         `datastore:"-" goon:"id"`
//		Parent *datastore.Key `datastore:"-" goon:"parent"`
//		Data   []byte
//	}
func (g *Goon) KeyError(src interface{}) (*datastore.Key, error) {
	return g.getStructKey(src)
}

// RunInTransaction runs f in a transaction. It calls f with a transaction
// context tg that f should use for all App Engine operations. Neither cache nor
// memcache are used or set during a transaction.
//
// Otherwise similar to appengine/datastore.RunInTransaction:
// https://developers.google.com/appengine/docs/go/datastore/reference#RunInTransaction
func (g *Goon) RunInTransaction(f func(tg *Goon) error, opts *datastore.TransactionOptions) error {
	var ng *Goon
	err := datastore.RunInTransaction(g.context, func(tc appengine.Context) error {
		ng = &Goon{
			context:       tc,
			inTransaction: true,
			toSet:         make(map[string]interface{}),
		}
		return f(ng)
	}, opts)

	if err == nil {
		for k, v := range ng.toSet {
			g.cache[k] = v
		}

		for _, k := range ng.toDelete {
			delete(g.cache, k)
		}
	} else {
		g.error(err)
	}

	return err
}

// Put saves the entity src into the datastore based on src's key. If k is an
// incomplete key, the returned key will be a unique key generated by the
// datastore.
func (g *Goon) Put(src interface{}) error {
	return g.PutMulti([]interface{}{src})
}

// PutMany is a wrapper around PutMulti.
func (g *Goon) PutMany(srcs ...interface{}) error {
	return g.PutMulti(srcs)
}

const putMultiLimit = 500

// PutMulti is a batch version of Put.
//
// src must satisfy the same conditions as the dst argument to GetMulti.
func (g *Goon) PutMulti(src interface{}) error {
	keys, err := g.extractKeys(src)
	if err != nil {
		return err
	}

	var memkeys []string
	for _, key := range keys {
		if !key.Incomplete() {
			memkeys = append(memkeys, memkey(key))
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i <= len(keys)/putMultiLimit; i++ {
		lo := i * putMultiLimit
		hi := (i + 1) * putMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		rkeys, err := datastore.PutMulti(g.context, keys[lo:hi], v.Slice(lo, hi).Interface())
		if err != nil {
			g.error(err)
			return err
		}

		for i, key := range keys[lo:hi] {
			vi := v.Index(lo + i).Interface()
			if key.Incomplete() {
				setStructKey(vi, rkeys[i])
			}
			if g.inTransaction {
				g.toSet[memkey(rkeys[i])] = vi
			}
		}
	}

	if !g.inTransaction {
		g.putMemoryMulti(src)
	}

	// Before returning, update the structs to have correct key info
	for _, e := range es {
		if e.Src != nil {
			setStructKey(e.Src, e.Key)
		}
	}

	return nil
}

func (g *Goon) putMemoryMulti(src interface{}) {
	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i < v.Len(); i++ {
		g.putMemory(v.Index(i).Interface())
	}
}

func (g *Goon) putMemory(src interface{}) {
	key, _ := g.getStructKey(src)
	g.cache[memkey(key)] = src
}

func (g *Goon) putMemcache(srcs []interface{}) error {
	items := make([]*memcache.Item, len(srcs))

	for i, src := range srcs {
		gob, err := toGob(src)
		if err != nil {
			g.error(err)
			return err
		}
		key, err := g.getStructKey(src)

		items[i] = &memcache.Item{
			Key:   memkey(key),
			Value: gob,
		}
	}

	err := memcache.SetMulti(g.context, items)

	if err != nil {
		g.error(err)
		return err
	}

	g.putMemoryMulti(srcs)
	return nil
}

// Get loads the entity based on dst's key into dst
// If there is no such entity for the key, Get returns
// datastore.ErrNoSuchEntity.
func (g *Goon) Get(dst interface{}) error {
	dsts := []interface{}{dst}
	if err := g.GetMulti(dsts); err != nil {
		// Look for an embedded error if it's multi
		if me, ok := err.(appengine.MultiError); ok {
			for i, merr := range me {
				if i == 0 {
					return merr
				}
			}
		}
		// Not multi, normal error
		return err
	}
	return nil
}

// GetMulti is a batch version of Get.
//
// dst has similar constraints as datastore.GetMulti.
func (g *Goon) GetMulti(dst interface{}) error {
	keys, err := g.extractKeys(dst)
	if err != nil {
		return err
	}

	if g.inTransaction {
		return datastore.GetMulti(g.context, keys, dst)
	}

	// Before returning, update the structs to have correct key info
	defer func() {
		for _, e := range es {
			if e.Src != nil {
				setStructKey(e.Src, e.Key)
			}
		}
	}()

	var dskeys []*datastore.Key
	var dsdst []interface{}
	var dixs []int

	var memkeys []string
	var mixs []int

	v := reflect.Indirect(reflect.ValueOf(dst))
	for i, key := range keys {
		m := memkey(key)
		if s, present := g.cache[m]; present {
			vi := v.Index(i)
			vi.Set(reflect.ValueOf(s))
		} else {
			memkeys = append(memkeys, m)
			mixs = append(mixs, i)
		}
	}

	memvalues, err := memcache.GetMulti(g.context, memkeys)
	if err != nil {
		g.error(errors.New(fmt.Sprintf("goon: ignored memcache error: %v", err.Error())))
		// ignore memcache errors
		//return err
	}

	for i, m := range memkeys {
		d := v.Index(mixs[i]).Interface()
		if s, present := memvalues[m]; present {
			err := fromGob(d, s.Value)
			if err != nil {
				g.error(err)
				return err
			}

			g.putMemory(d)
		} else {
			key, err := g.getStructKey(d)
			if err != nil {
				g.error(err)
				return err
			}
			dskeys = append(dskeys, key)
			dsdst = append(dsdst, d)
			dixs = append(dixs, mixs[i])
		}
	}

	gmerr := datastore.GetMulti(g.context, dskeys, dsdst)
	var ret error
	var multiErr appengine.MultiError
	var toCache []interface{}
	if gmerr != nil {
		merr, ok := gmerr.(appengine.MultiError)
		if !ok {
			g.error(gmerr)
			return gmerr
		}
		multiErr = make(appengine.MultiError, len(keys))
		for i, idx := range dixs {
			multiErr[idx] = merr[i]
			if merr[i] == nil {
				toCache = append(toCache, dsdst[i])
			}
		}
		ret = multiErr
	} else {
		toCache = dsdst
	}

	if len(dskeys) > 0 {
		if err := g.putMemcache(toCache); err != nil {
			g.error(err)
			return err
		}
	}

	return ret
}

func setStructKey(src interface{}, key *datastore.Key) error {
	v := reflect.Indirect(reflect.ValueOf(src))
	t := v.Type()
	k := t.Kind()

	if k != reflect.Struct {
		return errors.New(fmt.Sprintf("goon: Expected struct, got instead: %v", k))
	}

	for i := 0; i < v.NumField(); i++ {
		tf := t.Field(i)
		vf := v.Field(i)

		if !vf.CanSet() {
			continue
		}

		tag := tf.Tag.Get("goon")
		if tag != "" {
			tagValues := strings.Split(tag, ",")
			for _, tagValue := range tagValues {
				if tagValue == "id" {
					if vf.Kind() == reflect.Int64 {
						vf.SetInt(key.IntID())
					} else if vf.Kind() == reflect.String {
						vf.SetString(key.StringID())
					}
				} else if tagValue == "parent" {
					if vf.Type() == reflect.TypeOf(&datastore.Key{}) {
						vf.Set(reflect.ValueOf(key.Parent()))
					}
				}
			}
		}
	}

	return nil
}

// Delete deletes the entity for the given key.
func (g *Goon) Delete(key *datastore.Key) error {
	keys := []*datastore.Key{key}
	return g.DeleteMulti(keys)
}

// DeleteMulti is a batch version of Delete.
func (g *Goon) DeleteMulti(keys []*datastore.Key) error {
	memkeys := make([]string, len(keys))
	for i, k := range keys {
		mk := memkey(k)
		memkeys[i] = mk

		if g.inTransaction {
			g.toDelete = append(g.toDelete, mk)
		} else {
			delete(g.cache, mk)
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	return datastore.DeleteMulti(g.context, keys)
}

// NotFound returns true if err is an appengine.MultiError and err[idx] is an datastore.ErrNoSuchEntity.
func NotFound(err error, idx int) bool {
	if merr, ok := err.(appengine.MultiError); ok {
		return idx < len(merr) && merr[idx] == datastore.ErrNoSuchEntity
	}
	return false
}
<commit_msg>Text corrections<commit_after>/*
 * Copyright (c) 2012 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goon

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"errors"
	"fmt"
	"net/http"
	"reflect"
	"strings"
	"fmt"
)

var (
	// LogErrors issues appengine.Context.Errorf on any error.
	LogErrors bool = true
)

// Goon holds the app engine context and request memory cache.
type Goon struct {
	context       appengine.Context
	cache         map[string]interface{}
	inTransaction bool
	toSet         map[string]interface{}
	toDelete      []string
}

func memkey(k *datastore.Key) string {
	return k.String()
}

// NewGoon creates a new Goon object from the given request.
func NewGoon(r *http.Request) *Goon {
	return FromContext(appengine.NewContext(r))
}

// FromContext creates a new Goon object from the given appengine Context.
func FromContext(c appengine.Context) *Goon {
	return &Goon{
		context: c,
		cache:   make(map[string]interface{}),
	}
}

func (g *Goon) error(err error) {
	if LogErrors {
		g.context.Errorf("goon: %v", err.Error())
	}
}

func (g *Goon) extractKeys(src interface{}) ([]*datastore.Key, error) {
	v := reflect.Indirect(reflect.ValueOf(src))
	if v.Kind() != reflect.Slice {
		return nil, errors.New("goon: value must be a slice or pointer-to-slice")
	}
	l := v.Len()

	keys := make([]*datastore.Key, l)
	for i := 0; i < l; i++ {
		vi := v.Index(i)
		key, err := g.getStructKey(vi.Interface())
		if err != nil {
			return nil, err
		}
		keys[i] = key
	}
	return keys, nil
}

// Key is the same as KeyError, except nil is returned on error.
func (g *Goon) Key(src interface{}) *datastore.Key {
	if k, err := g.KeyError(src); err == nil {
		return k
	}
	return nil
}

// Key returns the key of src based on its properties.
//
// src must be a S or *S for some struct type S. The key is extracted based on
// various fields of S. If a field of type int64 or string has a struct tag
// named goon with value "id", it is used as the key's id. If a field of type
// *datastore.Key has a struct tag named goon with value "parent", it is used
// as the key's parent. If a field of type string has a struct tag named goon
// with value "kind", it is used as the key's kind. The "kind" field supports
// an optional second parameter which is the default kind name. If no kind
// field exists, the struct's name is used. These fields should all have
// their datastore field marked as "-".
//
// Example, with kind User:
//	type User struct {
//		Id    string `datastore:"-" goon:"id"`
//		Read  time.Time
//	}
// Example, with kind U if _kind is the empty string:
//	type User struct {
//		_kind string `goon:"kind,U"`
//		Id    string `datastore:"-" goon:"id"`
//		Read  time.Time
//	}
// To override the kind above to UserKind:
//	u := User{_kind: "UserKind"}
//
// An example with a parent:
//	type UserData struct {
//		Id     string         `datastore:"-" goon:"id"`
//		Parent *datastore.Key `datastore:"-" goon:"parent"`
//		Data   []byte
//	}
func (g *Goon) KeyError(src interface{}) (*datastore.Key, error) {
	return g.getStructKey(src)
}

// RunInTransaction runs f in a transaction. It calls f with a transaction
// context tg that f should use for all App Engine operations. Neither cache nor
// memcache are used or set during a transaction.
//
// Otherwise similar to appengine/datastore.RunInTransaction:
// https://developers.google.com/appengine/docs/go/datastore/reference#RunInTransaction
func (g *Goon) RunInTransaction(f func(tg *Goon) error, opts *datastore.TransactionOptions) error {
	var ng *Goon
	err := datastore.RunInTransaction(g.context, func(tc appengine.Context) error {
		ng = &Goon{
			context:       tc,
			inTransaction: true,
			toSet:         make(map[string]interface{}),
		}
		return f(ng)
	}, opts)

	if err == nil {
		for k, v := range ng.toSet {
			g.cache[k] = v
		}

		for _, k := range ng.toDelete {
			delete(g.cache, k)
		}
	} else {
		g.error(err)
	}

	return err
}

// Put saves the entity src into the datastore based on src's key. If k is an
// incomplete key, the returned key will be a unique key generated by the
// datastore.
func (g *Goon) Put(src interface{}) error {
	return g.PutMulti([]interface{}{src})
}

// PutMany is a wrapper around PutMulti.
func (g *Goon) PutMany(srcs ...interface{}) error {
	return g.PutMulti(srcs)
}

const putMultiLimit = 500

// PutMulti is a batch version of Put.
//
// src must satisfy the same conditions as the dst argument to GetMulti.
func (g *Goon) PutMulti(src interface{}) error {
	keys, err := g.extractKeys(src)
	if err != nil {
		return err
	}

	var memkeys []string
	for _, key := range keys {
		if !key.Incomplete() {
			memkeys = append(memkeys, memkey(key))
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i <= len(keys)/putMultiLimit; i++ {
		lo := i * putMultiLimit
		hi := (i + 1) * putMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		rkeys, err := datastore.PutMulti(g.context, keys[lo:hi], v.Slice(lo, hi).Interface())
		if err != nil {
			g.error(err)
			return err
		}

		for i, key := range keys[lo:hi] {
			vi := v.Index(lo + i).Interface()
			if key.Incomplete() {
				setStructKey(vi, rkeys[i])
			}
			if g.inTransaction {
				g.toSet[memkey(rkeys[i])] = vi
			}
		}
	}

	if !g.inTransaction {
		g.putMemoryMulti(src)
	}

	// Before returning, update the structs to have correct key info
	for _, e := range es {
		if e.Src != nil {
			setStructKey(e.Src, e.Key)
		}
	}

	return nil
}

func (g *Goon) putMemoryMulti(src interface{}) {
	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i < v.Len(); i++ {
		g.putMemory(v.Index(i).Interface())
	}
}

func (g *Goon) putMemory(src interface{}) {
	key, _ := g.getStructKey(src)
	g.cache[memkey(key)] = src
}

func (g *Goon) putMemcache(srcs []interface{}) error {
	items := make([]*memcache.Item, len(srcs))

	for i, src := range srcs {
		gob, err := toGob(src)
		if err != nil {
			g.error(err)
			return err
		}
		key, err := g.getStructKey(src)

		items[i] = &memcache.Item{
			Key:   memkey(key),
			Value: gob,
		}
	}

	err := memcache.SetMulti(g.context, items)

	if err != nil {
		g.error(err)
		return err
	}

	g.putMemoryMulti(srcs)
	return nil
}

// Get loads the entity based on dst's key into dst
// If there is no such entity for the key, Get returns
// datastore.ErrNoSuchEntity.
func (g *Goon) Get(dst interface{}) error {
	dsts := []interface{}{dst}
	if err := g.GetMulti(dsts); err != nil {
		// Look for an embedded error if it's multi
		if me, ok := err.(appengine.MultiError); ok {
			for i, merr := range me {
				if i == 0 {
					return merr
				}
			}
		}
		// Not multi, normal error
		return err
	}
	return nil
}

// GetMulti is a batch version of Get.
//
// dst has similar constraints as datastore.GetMulti.
func (g *Goon) GetMulti(dst interface{}) error {
	keys, err := g.extractKeys(dst)
	if err != nil {
		return err
	}

	if g.inTransaction {
		return datastore.GetMulti(g.context, keys, dst)
	}

	// Before returning, update the structs to have correct key info
	defer func() {
		for _, e := range es {
			if e.Src != nil {
				setStructKey(e.Src, e.Key)
			}
		}
	}()

	var dskeys []*datastore.Key
	var dsdst []interface{}
	var dixs []int

	var memkeys []string
	var mixs []int

	v := reflect.Indirect(reflect.ValueOf(dst))
	for i, key := range keys {
		m := memkey(key)
		if s, present := g.cache[m]; present {
			vi := v.Index(i)
			vi.Set(reflect.ValueOf(s))
		} else {
			memkeys = append(memkeys, m)
			mixs = append(mixs, i)
		}
	}

	memvalues, err := memcache.GetMulti(g.context, memkeys)
	if err != nil {
		g.error(errors.New(fmt.Sprintf("goon: ignored memcache error: %v", err.Error())))
		// ignore memcache errors
		//return err
	}

	for i, m := range memkeys {
		d := v.Index(mixs[i]).Interface()
		if s, present := memvalues[m]; present {
			err := fromGob(d, s.Value)
			if err != nil {
				g.error(err)
				return err
			}

			g.putMemory(d)
		} else {
			key, err := g.getStructKey(d)
			if err != nil {
				g.error(err)
				return err
			}
			dskeys = append(dskeys, key)
			dsdst = append(dsdst, d)
			dixs = append(dixs, mixs[i])
		}
	}

	gmerr := datastore.GetMulti(g.context, dskeys, dsdst)
	var ret error
	var multiErr appengine.MultiError
	var toCache []interface{}
	if gmerr != nil {
		merr, ok := gmerr.(appengine.MultiError)
		if !ok {
			g.error(gmerr)
			return gmerr
		}
		multiErr = make(appengine.MultiError, len(keys))
		for i, idx := range dixs {
			multiErr[idx] = merr[i]
			if merr[i] == nil {
				toCache = append(toCache, dsdst[i])
			}
		}
		ret = multiErr
	} else {
		toCache = dsdst
	}

	if len(dskeys) > 0 {
		if err := g.putMemcache(toCache); err != nil {
			g.error(err)
			return err
		}
	}

	return ret
}

func setStructKey(src interface{}, key *datastore.Key) error {
	v := reflect.Indirect(reflect.ValueOf(src))
	t := v.Type()
	k := t.Kind()

	if k != reflect.Struct {
		return errors.New(fmt.Sprintf("goon: Expected struct, got instead: %v", k))
	}

	for i := 0; i < v.NumField(); i++ {
		tf := t.Field(i)
		vf := v.Field(i)

		if !vf.CanSet() {
			continue
		}

		tag := tf.Tag.Get("goon")
		if tag != "" {
			tagValues := strings.Split(tag, ",")
			for _, tagValue := range tagValues {
				if tagValue == "id" {
					if vf.Kind() == reflect.Int64 {
						vf.SetInt(key.IntID())
					} else if vf.Kind() == reflect.String {
						vf.SetString(key.StringID())
					}
				} else if tagValue == "parent" {
					if vf.Type() == reflect.TypeOf(&datastore.Key{}) {
						vf.Set(reflect.ValueOf(key.Parent()))
					}
				}
			}
		}
	}

	return nil
}

// Delete deletes the entity for the given key.
func (g *Goon) Delete(key *datastore.Key) error {
	keys := []*datastore.Key{key}
	return g.DeleteMulti(keys)
}

// DeleteMulti is a batch version of Delete.
func (g *Goon) DeleteMulti(keys []*datastore.Key) error {
	memkeys := make([]string, len(keys))
	for i, k := range keys {
		mk := memkey(k)
		memkeys[i] = mk

		if g.inTransaction {
			g.toDelete = append(g.toDelete, mk)
		} else {
			delete(g.cache, mk)
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	return datastore.DeleteMulti(g.context, keys)
}

// NotFound returns true if err is an appengine.MultiError and err[idx] is a datastore.ErrNoSuchEntity.
func NotFound(err error, idx int) bool {
	if merr, ok := err.(appengine.MultiError); ok {
		return idx < len(merr) && merr[idx] == datastore.ErrNoSuchEntity
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package proxy

import (
	"fmt"
	"net"
	"strconv"
	"sync"
	"time"

	"github.com/cilium/cilium/common/addressing"
	"github.com/cilium/cilium/pkg/endpointmanager"
	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/logging"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/policy"
	"github.com/cilium/cilium/pkg/proxy/accesslog"

	"github.com/sirupsen/logrus"
	"github.com/spf13/viper"
)

var (
	log          = logging.DefaultLogger
	perFlowDebug = false
)

// Magic markers are attached to each packet. The lower 16 bits are used to
// identify packets which have gone through the proxy and to determine whether
// the packet is coming from a proxy at ingress or egress. The marking is
// compatible with Kubernetes's use of the packet mark.  The upper 16 bits can
// be used to carry the security identity.
const (
	magicMarkIngress int = 0x0FEA
	magicMarkEgress  int = 0x0FEB
	magicMarkK8sMasq int = 0x4000
	magicMarkK8sDrop int = 0x8000
)

// field names used while logging
const (
	fieldMarker          = "marker"
	fieldSocket          = "socket"
	fieldFd              = "fd"
	fieldProxyRedirectID = "id"
)

// Redirect is the generic proxy redirect interface that each proxy redirect
// type must export
type Redirect interface {
	ToPort() uint16
	UpdateRules(l4 *policy.L4Filter, completions policy.CompletionContainer) error
	getSource() ProxySource
	Close(completions policy.CompletionContainer)
	IsIngress() bool
}

// GetMagicMark returns the magic marker with which each packet must be marked.
// The mark is different depending on whether the proxy is injected at ingress
// or egress.
func GetMagicMark(isIngress bool, identity int) int {
	mark := 0

	if isIngress {
		mark = magicMarkIngress
	} else {
		mark = magicMarkEgress
	}

	if identity != 0 {
		mark |= identity << 16
	}

	return mark
}

// ProxySource returns information about the endpoint being proxied.
type ProxySource interface {
	GetID() uint64
	RLock()
	RUnlock()
	Lock()
	Unlock()
	GetLabels() []string
	GetLabelsSHA() string
	GetIdentity() policy.NumericIdentity
	ResolveIdentity(policy.NumericIdentity) *policy.Identity
	GetIPv4Address() string
	GetIPv6Address() string
}

// Proxy maintains state about redirects
type Proxy struct {
	// mutex is the lock required when modifying any proxy datastructure
	mutex lock.RWMutex

	// rangeMin is the minimum port used for proxy port allocation
	rangeMin uint16

	// rangeMax is the maximum port used for proxy port allocation.
	// If port is unspecified, the proxy will automatically allocate
	// ports out of the rangeMin-rangeMax range.
	rangeMax uint16

	// nextPort is the next available proxy port to use
	nextPort uint16

	// allocatedPorts is a map of all allocated proxy ports pointing
	// to the redirect rules attached to that port
	allocatedPorts map[uint16]Redirect

	// redirects is a map of all redirect configurations indexed by
	// the redirect identifier. Redirects may be implemented by different
	// proxies.
	redirects map[string]Redirect
}

// NewProxy creates a Proxy to keep track of redirects.
func NewProxy(minPort uint16, maxPort uint16) *Proxy {
	return &Proxy{
		rangeMin:       minPort,
		rangeMax:       maxPort,
		nextPort:       minPort,
		redirects:      make(map[string]Redirect),
		allocatedPorts: make(map[uint16]Redirect),
	}
}

func (p *Proxy) allocatePort() (uint16, error) {
	port := p.nextPort

	for {
		resPort := port
		port++
		if port >= p.rangeMax {
			port = p.rangeMin
		}

		if _, ok := p.allocatedPorts[resPort]; !ok {
			return resPort, nil
		}

		if port == p.nextPort {
			return 0, fmt.Errorf("no available proxy ports")
		}
	}
}

var gcOnce sync.Once

// localEndpointInfo fills the access log with the local endpoint info.
func localEndpointInfo(r Redirect, info *accesslog.EndpointInfo) {
	source := r.getSource()
	source.Lock()
	info.ID = source.GetID()
	info.IPv4 = source.GetIPv4Address()
	info.IPv6 = source.GetIPv6Address()
	info.Labels = source.GetLabels()
	info.LabelsSHA256 = source.GetLabelsSHA()
	info.Identity = uint64(source.GetIdentity())
	source.Unlock()
}

func fillInfo(r Redirect, l *accesslog.LogRecord, srcIPPort, dstIPPort string, srcIdentity uint32) {

	ingress := r.IsIngress()

	if ingress {
		// At ingress the local origin endpoint is the destination
		localEndpointInfo(r, &l.DestinationEndpoint)
	} else {
		// At egress, the local origin endpoint is the source
		localEndpointInfo(r, &l.SourceEndpoint)
	}

	l.IPVersion = accesslog.VersionIPv4
	ipstr, port, err := net.SplitHostPort(srcIPPort)
	if err == nil {
		ip := net.ParseIP(ipstr)
		if ip != nil && ip.To4() == nil {
			l.IPVersion = accesslog.VersionIPV6
		}

		p, err := strconv.ParseUint(port, 10, 16)
		if err == nil {
			l.SourceEndpoint.Port = uint16(p)
			if ingress {
				fillIngressSourceInfo(&l.SourceEndpoint, &ip, srcIdentity)
			}
		}
	}

	ipstr, port, err = net.SplitHostPort(dstIPPort)
	if err == nil {
		p, err := strconv.ParseUint(port, 10, 16)
		if err == nil {
			l.DestinationEndpoint.Port = uint16(p)
			if !ingress {
				fillEgressDestinationInfo(&l.DestinationEndpoint, ipstr)
			}
		}
	}
}

// fillIdentity resolves the labels of the specified identity if known
// locally and fills in the following info member fields:
//  - info.Identity
//  - info.Labels
//  - info.LabelsSHA256
func fillIdentity(info *accesslog.EndpointInfo, id policy.NumericIdentity) {
	info.Identity = uint64(id)

	if identity := policy.LookupIdentityByID(id); identity != nil {
		info.Labels = identity.Labels.GetModel()
		info.LabelsSHA256 = identity.GetLabelsSHA256()
	}
}

// fillEndpointInfo tries to resolve the IP address and fills the EndpointInfo
// fields with either ReservedIdentityHost or ReservedIdentityWorld
func fillEndpointInfo(info *accesslog.EndpointInfo, ip net.IP) {
	if ip.To4() != nil {
		info.IPv4 = ip.String()

		// first we try to resolve and check if the IP is
		// same as Host
		if node.IsHostIPv4(ip) {
			fillIdentity(info, policy.ReservedIdentityHost)
			return
		}

		// If Host IP check fails, we try to resolve and check
		// if IP belongs to the cluster.
		if node.GetIPv4ClusterRange().Contains(ip) {
			c := addressing.DeriveCiliumIPv4(ip)
			ep := endpointmanager.LookupIPv4(c.String())
			if ep != nil {
				// Needs to be Lock as ep.GetLabelsSHA()
				// might overwrite internal endpoint attributes
				ep.Lock()
				info.ID = uint64(ep.ID)
				info.Labels = ep.GetLabels()
				info.LabelsSHA256 = ep.GetLabelsSHA()
				info.Identity = uint64(ep.GetIdentity())
				ep.Unlock()
			} else {
				fillIdentity(info, policy.ReservedIdentityCluster)
			}
		} else {
			// If we are unable to resolve the HostIP as well
			// as the cluster IP we mark this as a 'world' identity.
			fillIdentity(info, policy.ReservedIdentityWorld)
		}
	} else {
		info.IPv6 = ip.String()

		if node.IsHostIPv6(ip) {
			fillIdentity(info, policy.ReservedIdentityHost)
			return
		}

		if node.GetIPv6ClusterRange().Contains(ip) {
			c := addressing.DeriveCiliumIPv6(ip)
			id := c.EndpointID()
			info.ID = uint64(id)

			ep := endpointmanager.LookupCiliumID(id)
			if ep != nil {
				// Needs to be Lock as ep.GetLabelsSHA()
				// might overwrite internal endpoint attributes
				ep.Lock()
				info.Labels = ep.GetLabels()
				info.LabelsSHA256 = ep.GetLabelsSHA()
				info.Identity = uint64(ep.GetIdentity())
				ep.Unlock()
			} else {
				fillIdentity(info, policy.ReservedIdentityCluster)
			}
		} else {
			fillIdentity(info, policy.ReservedIdentityWorld)
		}
	}
}

// fillIngressSourceInfo fills the EndpointInfo fields, by fetching
// the consumable from the consumable cache of endpoint using identity sent by
// source. This is needed in ingress proxy while logging the source endpoint
// info.  Since there will be 2 proxies on the same host, if both egress and
// ingress policies are set, the ingress policy cannot determine the source
// endpoint info based on ip address, as the ip address would be that of the
// egress proxy i.e host.
func fillIngressSourceInfo(info *accesslog.EndpointInfo, ip *net.IP, srcIdentity uint32) {

	if srcIdentity != 0 {
		if ip != nil {
			if ip.To4() != nil {
				info.IPv4 = ip.String()
			} else {
				info.IPv6 = ip.String()
			}
		}
		fillIdentity(info, policy.NumericIdentity(srcIdentity))
	} else {
		// source security identity 0 is possible when somebody else other than
		// the BPF datapath attempts to
		// connect to the proxy.
		// We should try to resolve if the identity belongs to reserved_host
		// or reserved_world.
		if ip != nil {
			fillEndpointInfo(info, *ip)
		} else {
			log.Warn("Missing security identity in source endpoint info")
		}
	}
}

// fillEgressDestinationInfo returns the destination EndpointInfo for a flow
// leaving the proxy at egress.
func fillEgressDestinationInfo(info *accesslog.EndpointInfo, ipstr string) {
	ip := net.ParseIP(ipstr)
	if ip != nil {
		fillEndpointInfo(info, ip)
	}
}

// CreateOrUpdateRedirect creates or updates a L4 redirect with corresponding
// proxy configuration. This will allocate a proxy port as required and launch
// a proxy instance. If the redirect is already in place, only the rules will be
// updated.
func (p *Proxy) CreateOrUpdateRedirect(l4 *policy.L4Filter, id string, source ProxySource,
	notifier accesslog.LogRecordNotifier, completions policy.CompletionContainer) (Redirect, error) {
	gcOnce.Do(func() {
		if lf := viper.GetString("access-log"); lf != "" {
			if err := accesslog.OpenLogfile(lf, notifier); err != nil {
				log.WithError(err).WithField(accesslog.FieldFilePath, lf).
					Warn("Cannot open L7 access log")
			}
		}

		if labels := viper.GetStringSlice("agent-labels"); len(labels) != 0 {
			accesslog.SetMetadata(labels)
		}

		go func() {
			for {
				time.Sleep(time.Duration(10) * time.Second)
				if deleted := GC(); deleted > 0 {
					log.WithField("count", deleted).
						Debug("Evicted entries from proxy table")
				}
			}
		}()
	})

	p.mutex.Lock()
	defer p.mutex.Unlock()

	scopedLog := log.WithField(fieldProxyRedirectID, id)

	if r, ok := p.redirects[id]; ok {
		err := r.UpdateRules(l4, completions)
		if err != nil {
			scopedLog.WithError(err).Error("Unable to update ", l4.L7Parser, " proxy")
			return nil, err
		}
		scopedLog.WithField(logfields.Object, logfields.Repr(r)).
			Debug("updated existing ", l4.L7Parser, " proxy instance")
		return r, nil
	}

	to, err := p.allocatePort()
	if err != nil {
		return nil, err
	}

	var redir Redirect

	switch l4.L7Parser {
	case policy.ParserTypeKafka:
		redir, err = createKafkaRedirect(kafkaConfiguration{
			policy:     l4,
			id:         id,
			source:     source,
			listenPort: to})
	case policy.ParserTypeHTTP:
		redir, err = createEnvoyRedirect(l4, id, source, to, completions)
	default:
		return nil, fmt.Errorf("Unsupported L7 parser type: %s", l4.L7Parser)
	}
	if err != nil {
		scopedLog.WithError(err).Error("Unable to create ", l4.L7Parser, " proxy")
		return nil, err
	}
	scopedLog.WithField(logfields.Object, logfields.Repr(redir)).
		Debug("Created new ", l4.L7Parser, " proxy instance")

	p.allocatedPorts[to] = redir
	p.redirects[id] = redir

	return redir, nil
}

// RemoveRedirect removes an existing redirect.
func (p *Proxy) RemoveRedirect(id string, completions policy.CompletionContainer) error {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	r, ok := p.redirects[id]
	if !ok {
		return fmt.Errorf("unable to find redirect %s", id)
	}

	log.WithField(fieldProxyRedirectID, id).
		Debug("removing proxy redirect")
	toPort := r.ToPort()
	r.Close(completions)

	delete(p.redirects, id)
	delete(p.allocatedPorts, toPort)

	return nil
}

// ChangeLogLevel changes proxy log level to correspond to the logrus log level 'level'.
func ChangeLogLevel(level logrus.Level) {
	if envoyProxy != nil {
		envoyProxy.ChangeLogLevel(level)
	}
}
<commit_msg>proxy: Delay release of redirect ports after use<commit_after>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package proxy

import (
	"fmt"
	"net"
	"strconv"
	"sync"
	"time"

	"github.com/cilium/cilium/common/addressing"
	"github.com/cilium/cilium/pkg/endpointmanager"
	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/logging"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/policy"
	"github.com/cilium/cilium/pkg/proxy/accesslog"

	"github.com/sirupsen/logrus"
	"github.com/spf13/viper"
)

var (
	log          = logging.DefaultLogger
	perFlowDebug = false
)

// Magic markers are attached to each packet. The lower 16 bits are used to
// identify packets which have gone through the proxy and to determine whether
// the packet is coming from a proxy at ingress or egress. The marking is
// compatible with Kubernetes's use of the packet mark.  The upper 16 bits can
// be used to carry the security identity.
const (
	magicMarkIngress int = 0x0FEA
	magicMarkEgress  int = 0x0FEB
	magicMarkK8sMasq int = 0x4000
	magicMarkK8sDrop int = 0x8000
)

// field names used while logging
const (
	fieldMarker          = "marker"
	fieldSocket          = "socket"
	fieldFd              = "fd"
	fieldProxyRedirectID = "id"

	// portReleaseDelay is the delay until a port is being released
	portReleaseDelay = time.Duration(5) * time.Minute
)

// Redirect is the generic proxy redirect interface that each proxy redirect
// type must export
type Redirect interface {
	ToPort() uint16
	UpdateRules(l4 *policy.L4Filter, completions policy.CompletionContainer) error
	getSource() ProxySource
	Close(completions policy.CompletionContainer)
	IsIngress() bool
}

// GetMagicMark returns the magic marker with which each packet must be marked.
// The mark is different depending on whether the proxy is injected at ingress
// or egress.
func GetMagicMark(isIngress bool, identity int) int {
	mark := 0

	if isIngress {
		mark = magicMarkIngress
	} else {
		mark = magicMarkEgress
	}

	if identity != 0 {
		mark |= identity << 16
	}

	return mark
}

// ProxySource returns information about the endpoint being proxied.
type ProxySource interface {
	GetID() uint64
	RLock()
	RUnlock()
	Lock()
	Unlock()
	GetLabels() []string
	GetLabelsSHA() string
	GetIdentity() policy.NumericIdentity
	ResolveIdentity(policy.NumericIdentity) *policy.Identity
	GetIPv4Address() string
	GetIPv6Address() string
}

// Proxy maintains state about redirects
type Proxy struct {
	// mutex is the lock required when modifying any proxy datastructure
	mutex lock.RWMutex

	// rangeMin is the minimum port used for proxy port allocation
	rangeMin uint16

	// rangeMax is the maximum port used for proxy port allocation.
	// If port is unspecified, the proxy will automatically allocate
	// ports out of the rangeMin-rangeMax range.
	rangeMax uint16

	// nextPort is the next available proxy port to use
	nextPort uint16

	// allocatedPorts is a map of all allocated proxy ports pointing
	// to the redirect rules attached to that port
	allocatedPorts map[uint16]Redirect

	// redirects is a map of all redirect configurations indexed by
	// the redirect identifier. Redirects may be implemented by different
	// proxies.
	redirects map[string]Redirect
}

// NewProxy creates a Proxy to keep track of redirects.
func NewProxy(minPort uint16, maxPort uint16) *Proxy {
	return &Proxy{
		rangeMin:       minPort,
		rangeMax:       maxPort,
		nextPort:       minPort,
		redirects:      make(map[string]Redirect),
		allocatedPorts: make(map[uint16]Redirect),
	}
}

func (p *Proxy) allocatePort() (uint16, error) {
	port := p.nextPort

	for {
		resPort := port
		port++
		if port >= p.rangeMax {
			port = p.rangeMin
		}

		if _, ok := p.allocatedPorts[resPort]; !ok {
			return resPort, nil
		}

		if port == p.nextPort {
			return 0, fmt.Errorf("no available proxy ports")
		}
	}
}

var gcOnce sync.Once

// localEndpointInfo fills the access log with the local endpoint info.
func localEndpointInfo(r Redirect, info *accesslog.EndpointInfo) {
	source := r.getSource()
	source.Lock()
	info.ID = source.GetID()
	info.IPv4 = source.GetIPv4Address()
	info.IPv6 = source.GetIPv6Address()
	info.Labels = source.GetLabels()
	info.LabelsSHA256 = source.GetLabelsSHA()
	info.Identity = uint64(source.GetIdentity())
	source.Unlock()
}

func fillInfo(r Redirect, l *accesslog.LogRecord, srcIPPort, dstIPPort string, srcIdentity uint32) {

	ingress := r.IsIngress()

	if ingress {
		// At ingress the local origin endpoint is the destination
		localEndpointInfo(r, &l.DestinationEndpoint)
	} else {
		// At egress, the local origin endpoint is the source
		localEndpointInfo(r, &l.SourceEndpoint)
	}

	l.IPVersion = accesslog.VersionIPv4
	ipstr, port, err := net.SplitHostPort(srcIPPort)
	if err == nil {
		ip := net.ParseIP(ipstr)
		if ip != nil && ip.To4() == nil {
			l.IPVersion = accesslog.VersionIPV6
		}

		p, err := strconv.ParseUint(port, 10, 16)
		if err == nil {
			l.SourceEndpoint.Port = uint16(p)
			if ingress {
				fillIngressSourceInfo(&l.SourceEndpoint, &ip, srcIdentity)
			}
		}
	}

	ipstr, port, err = net.SplitHostPort(dstIPPort)
	if err == nil {
		p, err := strconv.ParseUint(port, 10, 16)
		if err == nil {
			l.DestinationEndpoint.Port = uint16(p)
			if !ingress {
				fillEgressDestinationInfo(&l.DestinationEndpoint, ipstr)
			}
		}
	}
}

// fillIdentity resolves the labels of the specified identity if known
// locally and fills in the following info member fields:
//  - info.Identity
//  - info.Labels
//  - info.LabelsSHA256
func fillIdentity(info *accesslog.EndpointInfo, id policy.NumericIdentity) {
	info.Identity = uint64(id)

	if identity := policy.LookupIdentityByID(id); identity != nil {
		info.Labels = identity.Labels.GetModel()
		info.LabelsSHA256 = identity.GetLabelsSHA256()
	}
}

// fillEndpointInfo tries to resolve the IP address and fills the EndpointInfo
// fields with either ReservedIdentityHost or ReservedIdentityWorld
func fillEndpointInfo(info *accesslog.EndpointInfo, ip net.IP) {
	if ip.To4() != nil {
		info.IPv4 = ip.String()

		// first we try to resolve and check if the IP is
		// same as Host
		if node.IsHostIPv4(ip) {
			fillIdentity(info, policy.ReservedIdentityHost)
			return
		}

		// If Host IP check fails, we try to resolve and check
		// if IP belongs to the cluster.
		if node.GetIPv4ClusterRange().Contains(ip) {
			c := addressing.DeriveCiliumIPv4(ip)
			ep := endpointmanager.LookupIPv4(c.String())
			if ep != nil {
				// Needs to be Lock as ep.GetLabelsSHA()
				// might overwrite internal endpoint attributes
				ep.Lock()
				info.ID = uint64(ep.ID)
				info.Labels = ep.GetLabels()
				info.LabelsSHA256 = ep.GetLabelsSHA()
				info.Identity = uint64(ep.GetIdentity())
				ep.Unlock()
			} else {
				fillIdentity(info, policy.ReservedIdentityCluster)
			}
		} else {
			// If we are unable to resolve the HostIP as well
			// as the cluster IP we mark this as a 'world' identity.
			fillIdentity(info, policy.ReservedIdentityWorld)
		}
	} else {
		info.IPv6 = ip.String()

		if node.IsHostIPv6(ip) {
			fillIdentity(info, policy.ReservedIdentityHost)
			return
		}

		if node.GetIPv6ClusterRange().Contains(ip) {
			c := addressing.DeriveCiliumIPv6(ip)
			id := c.EndpointID()
			info.ID = uint64(id)

			ep := endpointmanager.LookupCiliumID(id)
			if ep != nil {
				// Needs to be Lock as ep.GetLabelsSHA()
				// might overwrite internal endpoint attributes
				ep.Lock()
				info.Labels = ep.GetLabels()
				info.LabelsSHA256 = ep.GetLabelsSHA()
				info.Identity = uint64(ep.GetIdentity())
				ep.Unlock()
			} else {
				fillIdentity(info, policy.ReservedIdentityCluster)
			}
		} else {
			fillIdentity(info, policy.ReservedIdentityWorld)
		}
	}
}

// fillIngressSourceInfo fills the EndpointInfo fields, by fetching
// the consumable from the consumable cache of endpoint using identity sent by
// source. This is needed in ingress proxy while logging the source endpoint
// info.  Since there will be 2 proxies on the same host, if both egress and
// ingress policies are set, the ingress policy cannot determine the source
// endpoint info based on ip address, as the ip address would be that of the
// egress proxy i.e host.
func fillIngressSourceInfo(info *accesslog.EndpointInfo, ip *net.IP, srcIdentity uint32) {

	if srcIdentity != 0 {
		if ip != nil {
			if ip.To4() != nil {
				info.IPv4 = ip.String()
			} else {
				info.IPv6 = ip.String()
			}
		}
		fillIdentity(info, policy.NumericIdentity(srcIdentity))
	} else {
		// source security identity 0 is possible when somebody else other than
		// the BPF datapath attempts to
		// connect to the proxy.
		// We should try to resolve if the identity belongs to reserved_host
		// or reserved_world.
		if ip != nil {
			fillEndpointInfo(info, *ip)
		} else {
			log.Warn("Missing security identity in source endpoint info")
		}
	}
}

// fillEgressDestinationInfo returns the destination EndpointInfo for a flow
// leaving the proxy at egress.
func fillEgressDestinationInfo(info *accesslog.EndpointInfo, ipstr string) {
	ip := net.ParseIP(ipstr)
	if ip != nil {
		fillEndpointInfo(info, ip)
	}
}

// CreateOrUpdateRedirect creates or updates a L4 redirect with corresponding
// proxy configuration. This will allocate a proxy port as required and launch
// a proxy instance. If the redirect is already in place, only the rules will be
// updated.
func (p *Proxy) CreateOrUpdateRedirect(l4 *policy.L4Filter, id string, source ProxySource,
	notifier accesslog.LogRecordNotifier, completions policy.CompletionContainer) (Redirect, error) {
	gcOnce.Do(func() {
		if lf := viper.GetString("access-log"); lf != "" {
			if err := accesslog.OpenLogfile(lf, notifier); err != nil {
				log.WithError(err).WithField(accesslog.FieldFilePath, lf).
					Warn("Cannot open L7 access log")
			}
		}

		if labels := viper.GetStringSlice("agent-labels"); len(labels) != 0 {
			accesslog.SetMetadata(labels)
		}

		go func() {
			for {
				time.Sleep(time.Duration(10) * time.Second)
				if deleted := GC(); deleted > 0 {
					log.WithField("count", deleted).
						Debug("Evicted entries from proxy table")
				}
			}
		}()
	})

	p.mutex.Lock()
	defer p.mutex.Unlock()

	scopedLog := log.WithField(fieldProxyRedirectID, id)

	if r, ok := p.redirects[id]; ok {
		err := r.UpdateRules(l4, completions)
		if err != nil {
			scopedLog.WithError(err).Error("Unable to update ", l4.L7Parser, " proxy")
			return nil, err
		}
		scopedLog.WithField(logfields.Object, logfields.Repr(r)).
			Debug("updated existing ", l4.L7Parser, " proxy instance")
		return r, nil
	}

	to, err := p.allocatePort()
	if err != nil {
		return nil, err
	}

	var redir Redirect

	switch l4.L7Parser {
	case policy.ParserTypeKafka:
		redir, err = createKafkaRedirect(kafkaConfiguration{
			policy:     l4,
			id:         id,
			source:     source,
			listenPort: to})
	case policy.ParserTypeHTTP:
		redir, err = createEnvoyRedirect(l4, id, source, to, completions)
	default:
		return nil, fmt.Errorf("Unsupported L7 parser type: %s", l4.L7Parser)
	}
	if err != nil {
		scopedLog.WithError(err).Error("Unable to create ", l4.L7Parser, " proxy")
		return nil, err
	}
	scopedLog.WithField(logfields.Object, logfields.Repr(redir)).
		Debug("Created new ", l4.L7Parser, " proxy instance")

	p.allocatedPorts[to] = redir
	p.redirects[id] = redir

	return redir, nil
}

// RemoveRedirect removes an existing redirect.
func (p *Proxy) RemoveRedirect(id string, completions policy.CompletionContainer) error {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	r, ok := p.redirects[id]
	if !ok {
		return fmt.Errorf("unable to find redirect %s", id)
	}

	log.WithField(fieldProxyRedirectID, id).
		Debug("removing proxy redirect")
	toPort := r.ToPort()
	r.Close(completions)

	delete(p.redirects, id)

	// delay the release and reuse of the port number so it is guaranteed
	// to be safe to listen on the port again
	go func() {
		time.Sleep(portReleaseDelay)

		p.mutex.Lock()
		delete(p.allocatedPorts, toPort)
		p.mutex.Unlock()

		log.WithField(fieldProxyRedirectID, id).
			Debugf("Delayed release of proxy port %d", toPort)
	}()

	return nil
}

// ChangeLogLevel changes proxy log level to correspond to the logrus log level 'level'.
func ChangeLogLevel(level logrus.Level) {
	if envoyProxy != nil {
		envoyProxy.ChangeLogLevel(level)
	}
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "io/ioutil"
)

var header =
`rot13 v1.0  String to ROT13 Conversion
Wrap Labs Software, January 2015`

func main() {
  fmt.Println(header + "\n")

  contents, _ := ioutil.ReadFile("caffeine.txt")
  fmt.Println(string(contents))
}
<commit_msg>002rot13.go: Read filename from command line and print content<commit_after>package main

import (
  "fmt"
  "io/ioutil"
  "os"
)

var header =
  "rot13 v1.0  String to ROT13 Conversion\n" +
  "Wrap Labs Software, January 2015\n"
var usage =
  "usage: rot13 <filename>\n"

func main() {
  fmt.Println(header)

  if len(os.Args[1:]) == 0 {
    fmt.Println(usage)
    os.Exit(-1)
  }

  contents, _ := ioutil.ReadFile(os.Args[1])
  fmt.Println(string(contents))
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Camlistore Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"strconv"
	"sync"

	"camlistore.org/pkg/blobserver"
	"camlistore.org/pkg/httputil"
	"camlistore.org/pkg/jsonconfig"
	"camlistore.org/pkg/types/clientconfig"
)

const helpHTML string = `

			<title>Help</title>

Help

Web User Interface

Search bar predicates.

Client Configuration

You will need to use the following client configuration in order to access this server using the Camlistore command line tools.

{{ . }}

Anything Else?

See the Camlistore online documentation and community contacts.

	`

// HelpHandler publishes information related to accessing the server
type HelpHandler struct {
	clientConfig *clientconfig.Config // generated from serverConfig
	serverConfig jsonconfig.Obj       // low-level config
	goTemplate   *template.Template   // for rendering
}

// setServerConfigOnce guards operation within SetServerConfig
var setServerConfigOnce sync.Once

// SetServerConfig enables the handler to receive the server config
// before InitHandler, which generates a client config from the server config, is called.
func (hh *HelpHandler) SetServerConfig(config jsonconfig.Obj) {
	setServerConfigOnce.Do(func() { hh.serverConfig = config })
}

func init() {
	blobserver.RegisterHandlerConstructor("help", newHelpFromConfig)
}

func (hh *HelpHandler) InitHandler(hl blobserver.FindHandlerByTyper) error {
	if hh.serverConfig == nil {
		return fmt.Errorf("HelpHandler's serverConfig must be set before calling its InitHandler")
	}

	clientConfig, err := clientconfig.GenerateClientConfig(hh.serverConfig)
	if err != nil {
		return fmt.Errorf("error generating client config: %v", err)
	}
	hh.clientConfig = clientConfig

	tmpl, err := template.New("help").Parse(helpHTML)
	if err != nil {
		return fmt.Errorf("error creating template: %v", err)
	}
	hh.goTemplate = tmpl

	return nil
}

func newHelpFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
	return &HelpHandler{}, nil
}

func (hh *HelpHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	suffix := httputil.PathSuffix(req)
	if !httputil.IsGet(req) {
		http.Error(rw, "Illegal help method.", http.StatusMethodNotAllowed)
		return
	}
	switch suffix {
	case "":
		if clientConfig := req.FormValue("clientConfig"); clientConfig != "" {
			if clientConfigOnly, err := strconv.ParseBool(clientConfig); err == nil && clientConfigOnly {
				httputil.ReturnJSON(rw, hh.clientConfig)
				return
			}
		}
		hh.serveHelpHTML(rw, req)
	default:
		http.Error(rw, "Illegal help path.", http.StatusNotFound)
	}
}

func (hh *HelpHandler) serveHelpHTML(rw http.ResponseWriter, req *http.Request) {
	jsonBytes, err := json.MarshalIndent(hh.clientConfig, "", "  ")
	if err != nil {
		httputil.ServeError(rw, req, fmt.Errorf("could not serialize client config JSON: %v", err))
		return
	}

	hh.goTemplate.Execute(rw, string(jsonBytes))
}
<commit_msg>pkg/server/help: fix server's hostname when needed<commit_after>/*
Copyright 2015 The Camlistore Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"net/http"
	"strconv"
	"strings"
	"sync"

	"camlistore.org/pkg/blobserver"
	"camlistore.org/pkg/httputil"
	"camlistore.org/pkg/jsonconfig"
	"camlistore.org/pkg/types/clientconfig"
)

const helpHTML string = `

			<title>Help</title>

Help

Web User Interface

Search bar predicates.

Client Configuration

You will need to use the following client configuration in order to access this server using the Camlistore command line tools.

{{ . }}

Anything Else?

See the Camlistore online documentation and community contacts.

	`

// HelpHandler publishes information related to accessing the server
type HelpHandler struct {
	clientConfig *clientconfig.Config // generated from serverConfig
	serverConfig jsonconfig.Obj       // low-level config
	goTemplate   *template.Template   // for rendering
}

// setServerConfigOnce guards operation within SetServerConfig
var setServerConfigOnce sync.Once

// SetServerConfig enables the handler to receive the server config
// before InitHandler, which generates a client config from the server config, is called.
func (hh *HelpHandler) SetServerConfig(config jsonconfig.Obj) {
	setServerConfigOnce.Do(func() { hh.serverConfig = config })
}

func init() {
	blobserver.RegisterHandlerConstructor("help", newHelpFromConfig)
}

// fixServerInConfig checks if cc contains a meaningful server (for a client).
// If not, a newly allocated clone of cc is returned, except req.Host is used for
// the hostname of the server. Otherwise, cc is returned.
func fixServerInConfig(cc *clientconfig.Config, req *http.Request) (*clientconfig.Config, error) {
	if cc == nil {
		return nil, errors.New("nil client config")
	}
	if len(cc.Servers) == 0 || cc.Servers["default"] == nil || cc.Servers["default"].Server == "" {
		return nil, errors.New("no Server in client config")
	}
	listen := strings.TrimPrefix(strings.TrimPrefix(cc.Servers["default"].Server, "http://"), "https://")
	if !(strings.HasPrefix(listen, "0.0.0.0") || strings.HasPrefix(listen, ":")) {
		return cc, nil
	}
	newCC := *cc
	server := newCC.Servers["default"]
	if req.TLS != nil {
		server.Server = "https://" + req.Host
	} else {
		server.Server = "http://" + req.Host
	}
	newCC.Servers["default"] = server
	return &newCC, nil
}

func (hh *HelpHandler) InitHandler(hl blobserver.FindHandlerByTyper) error {
	if hh.serverConfig == nil {
		return fmt.Errorf("HelpHandler's serverConfig must be set before calling its InitHandler")
	}

	clientConfig, err := clientconfig.GenerateClientConfig(hh.serverConfig)
	if err != nil {
		return fmt.Errorf("error generating client config: %v", err)
	}
	hh.clientConfig = clientConfig

	tmpl, err := template.New("help").Parse(helpHTML)
	if err != nil {
		return fmt.Errorf("error creating template: %v", err)
	}
	hh.goTemplate = tmpl

	return nil
}

func newHelpFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
	return &HelpHandler{}, nil
}

func (hh *HelpHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	suffix := httputil.PathSuffix(req)
	if !httputil.IsGet(req) {
		http.Error(rw, "Illegal help method.", http.StatusMethodNotAllowed)
		return
	}
	switch suffix {
	case "":
		cc, err := fixServerInConfig(hh.clientConfig, req)
		if err != nil {
			httputil.ServeError(rw, req, err)
			return
		}
		if clientConfig := req.FormValue("clientConfig"); clientConfig != "" {
			if clientConfigOnly, err := strconv.ParseBool(clientConfig); err == nil && clientConfigOnly {
				httputil.ReturnJSON(rw, cc)
				return
			}
		}
		hh.serveHelpHTML(cc, rw, req)
	default:
		http.Error(rw, "Illegal help path.", http.StatusNotFound)
	}
}

func (hh *HelpHandler) serveHelpHTML(cc *clientconfig.Config, rw http.ResponseWriter, req *http.Request) {
	jsonBytes, err := json.MarshalIndent(cc, "", "  ")
	if err != nil {
		httputil.ServeError(rw, req, fmt.Errorf("could not serialize client config JSON: %v", err))
		return
	}

	hh.goTemplate.Execute(rw, string(jsonBytes))
}
<|endoftext|>


<commit_before>package store

import (
	log "github.com/Sirupsen/logrus"
	"github.com/garyburd/redigo/redis"
	"github.com/ulule/limiter"
)

const (
	// DefaultPrefix is the default prefix to use for the key in the store.
	DefaultPrefix = "limiter"
)

// RedisStore is the redis store.
type RedisStore struct {
	// The prefix to use for the key.
	Prefix string

	// github.com/garyburd/redigo Pool instance.
	Pool *redis.Pool

	// The maximum number of retry under race conditions.
	MaxRetry int
}

// NewRedisStore returns an instance of redis store.
func NewRedisStore(pool *redis.Pool) (Store, error) {
	return NewRedisStoreWithOptions(pool, Options{
		Prefix: DefaultPrefix,
	})
}

// NewRedisStoreWithOptions returns an instance of redis store with custom options.
func NewRedisStoreWithOptions(pool *redis.Pool, options Options) (Store, error) {
	store := &RedisStore{
		Pool:   pool,
		Prefix: options.Prefix,
	}

	if _, err := store.ping(); err != nil {
		return nil, err
	}

	return store, nil
}

func (s *RedisStore) getConnection() redis.Conn {
	return s.Pool.Get()
}

// ping checks if redis is alive.
func (s *RedisStore) ping() (bool, error) {
	conn := s.getConnection()
	defer conn.Close()

	data, err := conn.Do("PING")
	if err != nil || data == nil {
		return false, err
	}

	return data == "PONG", nil
}

// Exists checks if a key exists in the store
func (s *RedisStore) Exists(key string) (bool, error) {
	conn := s.getConnection()
	defer conn.Close()

	return s.exists(conn, key)
}

// Get retreives a value from the store
func (s *RedisStore) Get(key string) (string, error) {
	conn := s.getConnection()
	defer conn.Close()

	return s.get(conn, key)
}

// Remove a value from the store
func (s *RedisStore) Remove(key string) error {
	conn := s.getConnection()
	defer conn.Close()

	return s.remove(conn, key)
}

// Set a value in the store
func (s *RedisStore) Set(key string, value string, expire int64) error {
	conn := s.getConnection()
	defer conn.Close()

	return s.set(conn, key, value, expire)
}

// ToLimiterStore converts a storage into a limmiter compliant storage
func (s *RedisStore) ToLimiterStore(prefix string) (limiter.Store, error) {
	// Alternatively, you can pass options to the store with the "WithOptions"
	// function. For example, for Redis store:
	return limiter.NewRedisStoreWithOptions(s.Pool, limiter.StoreOptions{
		Prefix:   prefix,
		MaxRetry: limiter.DefaultMaxRetry,
	})
}

func (s *RedisStore) Publish(topic string, data []byte) error {
	c := s.getConnection()
	_, err := c.Do("PUBLISH", topic, data)
	return err
}

func (s *RedisStore) Subscribe(topic string) *Subscription {
	sub := NewSubscription()

	go func() {
		for {
			// Get a connection from a pool
			c := s.getConnection()
			psc := redis.PubSubConn{Conn: c}

			// Set up subscriptions
			psc.Subscribe(topic)

			// While not a permanent error on the connection.
			for c.Err() == nil {
				switch v := psc.Receive().(type) {
				case redis.Message:
					log.WithField("channel", v.Channel).Debug("Received a message")
					sub.Message <- Message(v.Data)
				case error:
					log.WithError(v).Debug("An error ocurred when getting the message")
					return
				}
			}
			c.Close()
		}
	}()

	return sub
}

func (s *RedisStore) exists(conn redis.Conn, key string) (bool, error) {
	exists, err := redis.Bool(conn.Do("EXISTS", key))
	if err != nil {
		return false, err
	}

	return exists, nil
}

func (s *RedisStore) remove(conn redis.Conn, key string) error {
	_, err := conn.Do("DEL", key)
	return err
}

func (s *RedisStore) get(conn redis.Conn, key string) (string, error) {
	return redis.String(conn.Do("GET", key))
}

func (s *RedisStore) set(conn redis.Conn, key string, value string, expire int64) error {
	command, args := getSetCommandAndArgs(key, value, expire)
	if _, err := conn.Do(command, args...); err != nil {
		return err
	}

	return nil
}

func getSetCommandAndArgs(key string, value string, expire int64) (string, []interface{}) {
	var args []interface{}
	if expire == 0 {
		args = append(args, key)
		args = append(args, value)
		return "SET", args
	}

	args = append(args, key)
	args = append(args, expire)
	args = append(args, value)
	return "SETEX", args
}
<commit_msg>Added missing docs<commit_after>package store

import (
	log "github.com/Sirupsen/logrus"
	"github.com/garyburd/redigo/redis"
	"github.com/ulule/limiter"
)

const (
	// DefaultPrefix is the default prefix to use for the key in the store.
	DefaultPrefix = "limiter"
)

// RedisStore is the redis store.
type RedisStore struct {
	// The prefix to use for the key.
	Prefix string

	// github.com/garyburd/redigo Pool instance.
	Pool *redis.Pool

	// The maximum number of retry under race conditions.
	MaxRetry int
}

// NewRedisStore returns an instance of redis store.
func NewRedisStore(pool *redis.Pool) (Store, error) {
	return NewRedisStoreWithOptions(pool, Options{
		Prefix: DefaultPrefix,
	})
}

// NewRedisStoreWithOptions returns an instance of redis store with custom options.
func NewRedisStoreWithOptions(pool *redis.Pool, options Options) (Store, error) {
	store := &RedisStore{
		Pool:   pool,
		Prefix: options.Prefix,
	}

	if _, err := store.ping(); err != nil {
		return nil, err
	}

	return store, nil
}

func (s *RedisStore) getConnection() redis.Conn {
	return s.Pool.Get()
}

// ping checks if redis is alive.
func (s *RedisStore) ping() (bool, error) {
	conn := s.getConnection()
	defer conn.Close()

	data, err := conn.Do("PING")
	if err != nil || data == nil {
		return false, err
	}

	return data == "PONG", nil
}

// Exists checks if a key exists in the store
func (s *RedisStore) Exists(key string) (bool, error) {
	conn := s.getConnection()
	defer conn.Close()

	return s.exists(conn, key)
}

// Get retreives a value from the store
func (s *RedisStore) Get(key string) (string, error) {
	conn := s.getConnection()
	defer conn.Close()

	return s.get(conn, key)
}

// Remove a value from the store
func (s *RedisStore) Remove(key string) error {
	conn := s.getConnection()
	defer conn.Close()

	return s.remove(conn, key)
}

// Set a value in the store
func (s *RedisStore) Set(key string, value string, expire int64) error {
	conn := s.getConnection()
	defer conn.Close()

	return s.set(conn, key, value, expire)
}

// ToLimiterStore converts a storage into a limmiter compliant storage
func (s *RedisStore) ToLimiterStore(prefix string) (limiter.Store, error) {
	// Alternatively, you can pass options to the store with the "WithOptions"
	// function. For example, for Redis store:
	return limiter.NewRedisStoreWithOptions(s.Pool, limiter.StoreOptions{
		Prefix:   prefix,
		MaxRetry: limiter.DefaultMaxRetry,
	})
}

// Publish publishes to a topic in redis
func (s *RedisStore) Publish(topic string, data []byte) error {
	c := s.getConnection()
	_, err := c.Do("PUBLISH", topic, data)
	return err
}

// Subscribe subscribes to a topic in redis
func (s *RedisStore) Subscribe(topic string) *Subscription {
	sub := NewSubscription()

	go func() {
		for {
			// Get a connection from a pool
			c := s.getConnection()
			psc := redis.PubSubConn{Conn: c}

			// Set up subscriptions
			psc.Subscribe(topic)

			// While not a permanent error on the connection.
			for c.Err() == nil {
				switch v := psc.Receive().(type) {
				case redis.Message:
					log.WithField("channel", v.Channel).Debug("Received a message")
					sub.Message <- Message(v.Data)
				case error:
					log.WithError(v).Debug("An error ocurred when getting the message")
					return
				}
			}
			c.Close()
		}
	}()

	return sub
}

func (s *RedisStore) exists(conn redis.Conn, key string) (bool, error) {
	exists, err := redis.Bool(conn.Do("EXISTS", key))
	if err != nil {
		return false, err
	}

	return exists, nil
}

func (s *RedisStore) remove(conn redis.Conn, key string) error {
	_, err := conn.Do("DEL", key)
	return err
}

func (s *RedisStore) get(conn redis.Conn, key string) (string, error) {
	return redis.String(conn.Do("GET", key))
}

func (s *RedisStore) set(conn redis.Conn, key string, value string, expire int64) error {
	command, args := getSetCommandAndArgs(key, value, expire)
	if _, err := conn.Do(command, args...); err != nil {
		return err
	}

	return nil
}

func getSetCommandAndArgs(key string, value string, expire int64) (string, []interface{}) {
	var args []interface{}
	if expire == 0 {
		args = append(args, key)
		args = append(args, value)
		return "SET", args
	}

	args = append(args, key)
	args = append(args, expire)
	args = append(args, value)
	return "SETEX", args
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"net"

	"github.com/Sirupsen/logrus"
	"github.com/asaskevich/govalidator"
	"github.com/cad/ovpm"
	"github.com/cad/ovpm/permset"
	gcontext "golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
)

// AuthUnaryInterceptor is a interceptor function.
//
// See https://godoc.org/google.golang.org/grpc#UnaryServerInterceptor.
func AuthUnaryInterceptor(ctx gcontext.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
	var enableAuthCheck bool

	md, ok := metadata.FromIncomingContext(ctx)
	if !ok {
		return nil, fmt.Errorf("Expected 2 metadata items in context; got %v", md)
	}

	// We enable auth check if we find a non-loopback
	// or invalid IP in the headers coming from the grpc-gateway.
	for _, userAgentIP := range md["x-forwarded-for"] {
		// Check if the remote user IP addr is a proper IP addr.
		if !govalidator.IsIP(userAgentIP) {
			enableAuthCheck = true
			logrus.Debugf("grpc request user agent ip can not be fetched from x-forwarded-for metadata, enabling auth check module '%s'", userAgentIP)
			break
		}

		// Check if the remote user IP addr is a loopback IP addr.
		if ip := net.ParseIP(userAgentIP); !ip.IsLoopback() {
			enableAuthCheck = true
			logrus.Debugf("grpc request user agent ips include non-loopback ip, enabling auth check module '%s'", userAgentIP)
			break
		}

		// TODO(cad): We assume gRPC endpoints are for cli only therefore
		//            we are listening only on looback IP.
		//
		// But if we decide use gRPC endpoints publicly, we need to add
		// extra checks against gRPC remote peer IP to test if the request
		// is coming from a remote peer IP or also from a loopback ip.
	}

	if !enableAuthCheck {
		logrus.Debugf("rpc: auth-check not enabled: %s", md["x-forwarded-for"])
		ctx = NewUsernameContext(ctx, "root")
		permissions := permset.New(ovpm.AdminPerms()...)
		ctx = permset.NewContext(ctx, permissions)
	}

	if enableAuthCheck {
		switch info.FullMethod {
		// AuthService methods
		case "/pb.AuthService/Status":
			return authRequired(ctx, req, handler)

		// UserService methods
		case "/pb.UserService/List":
			return authRequired(ctx, req, handler)
		case "/pb.UserService/Create":
			return authRequired(ctx, req, handler)
		case "/pb.UserService/Update":
			return authRequired(ctx, req, handler)
		case "/pb.UserService/Delete":
			return authRequired(ctx, req, handler)
		case "/pb.UserService/Renew":
			return authRequired(ctx, req, handler)
		case "/pb.UserService/GenConfig":
			return authRequired(ctx, req, handler)

		// VPNService methods
		case "/pb.VPNService/Status":
			return authRequired(ctx, req, handler)
		case "/pb.VPNService/Init":
			return authRequired(ctx, req, handler)
		case "/pb.VPNService/Update":
			return authRequired(ctx, req, handler)

		// NetworkService methods
		case "/pb.NetworkService/Create":
			return authRequired(ctx, req, handler)
		case "/pb.NetworkService/List":
			return authRequired(ctx, req, handler)
		case "/pb.NetworkService/Delete":
			return authRequired(ctx, req, handler)
		case "/pb.NetworkService/GetAllTypes":
			return authRequired(ctx, req, handler)
		case "/pb.NetworkService/GetAssociatedUsers":
			return authRequired(ctx, req, handler)
		case "/pb.NetworkService/Associate":
			return authRequired(ctx, req, handler)
		case "/pb.NetworkService/Dissociate":
			return authRequired(ctx, req, handler)
		default:
			logrus.Debugln("rpc: auth is not required for this endpoint: '%s'", info.FullMethod)
		}
	}
	return handler(ctx, req)
}
<commit_msg>refactor(api): require auth check on VPNService/Restart endpoint<commit_after>package api

import (
	"fmt"
	"net"

	"github.com/Sirupsen/logrus"
	"github.com/asaskevich/govalidator"
	"github.com/cad/ovpm"
	"github.com/cad/ovpm/permset"
	gcontext "golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
)

// AuthUnaryInterceptor is a interceptor function.
//
// See https://godoc.org/google.golang.org/grpc#UnaryServerInterceptor.
func AuthUnaryInterceptor(ctx gcontext.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
	var enableAuthCheck bool

	md, ok := metadata.FromIncomingContext(ctx)
	if !ok {
		return nil, fmt.Errorf("Expected 2 metadata items in context; got %v", md)
	}

	// We enable auth check if we find a non-loopback
	// or invalid IP in the headers coming from the grpc-gateway.
	for _, userAgentIP := range md["x-forwarded-for"] {
		// Check if the remote user IP addr is a proper IP addr.
		if !govalidator.IsIP(userAgentIP) {
			enableAuthCheck = true
			logrus.Debugf("grpc request user agent ip can not be fetched from x-forwarded-for metadata, enabling auth check module '%s'", userAgentIP)
			break
		}

		// Check if the remote user IP addr is a loopback IP addr.
		if ip := net.ParseIP(userAgentIP); !ip.IsLoopback() {
			enableAuthCheck = true
			logrus.Debugf("grpc request user agent ips include non-loopback ip, enabling auth check module '%s'", userAgentIP)
			break
		}

		// TODO(cad): We assume gRPC endpoints are for cli only therefore
		//            we are listening only on looback IP.
		//
		// But if we decide use gRPC endpoints publicly, we need to add
		// extra checks against gRPC remote peer IP to test if the request
		// is coming from a remote peer IP or also from a loopback ip.
	}

	if !enableAuthCheck {
		logrus.Debugf("rpc: auth-check not enabled: %s", md["x-forwarded-for"])
		ctx = NewUsernameContext(ctx, "root")
		permissions := permset.New(ovpm.AdminPerms()...)
		ctx = permset.NewContext(ctx, permissions)
	}

	if enableAuthCheck {
		switch info.FullMethod {
		// AuthService methods
		case "/pb.AuthService/Status":
			return authRequired(ctx, req, handler)

		// UserService methods
		case "/pb.UserService/List":
			return authRequired(ctx, req, handler)
		case "/pb.UserService/Create":
			return authRequired(ctx, req, handler)
		case "/pb.UserService/Update":
			return authRequired(ctx, req, handler)
		case "/pb.UserService/Delete":
			return authRequired(ctx, req, handler)
		case "/pb.UserService/Renew":
			return authRequired(ctx, req, handler)
		case "/pb.UserService/GenConfig":
			return authRequired(ctx, req, handler)

		// VPNService methods
		case "/pb.VPNService/Status":
			return authRequired(ctx, req, handler)
		case "/pb.VPNService/Init":
			return authRequired(ctx, req, handler)
		case "/pb.VPNService/Update":
			return authRequired(ctx, req, handler)
		case "/pb.VPNService/Restart":
			return authRequired(ctx, req, handler)

		// NetworkService methods
		case "/pb.NetworkService/Create":
			return authRequired(ctx, req, handler)
		case "/pb.NetworkService/List":
			return authRequired(ctx, req, handler)
		case "/pb.NetworkService/Delete":
			return authRequired(ctx, req, handler)
		case "/pb.NetworkService/GetAllTypes":
			return authRequired(ctx, req, handler)
		case "/pb.NetworkService/GetAssociatedUsers":
			return authRequired(ctx, req, handler)
		case "/pb.NetworkService/Associate":
			return authRequired(ctx, req, handler)
		case "/pb.NetworkService/Dissociate":
			return authRequired(ctx, req, handler)
		default:
			logrus.Debugln("rpc: auth is not required for this endpoint: '%s'", info.FullMethod)
		}
	}
	return handler(ctx, req)
}
<|endoftext|>