<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/DanielRenne/GoCore/core/extensions"
	"github.com/DanielRenne/GoCore/core/zip"
	"io"
	"io/ioutil"
	"net/http"
	"os"
)

const gitRepoName = "GoCoreDep-master"
const manifestFileName = "downloadedManifest.json"
const manifestURL = "https://raw.githubusercontent.com/DanielRenne/GoCoreDep/master/coreManifest.json"

type version struct {
	Version         string   `json:"version"`
	ReleaseURL      string   `json:"releaseURL"`
	ReleaseFileName string   `json:"releaseFileName"`
	GoDirectories   []string `json:"goDirectories"`
}

type coreManifest struct {
	Versions []version `json:"versions"`
}

func main() {

	downloadManifest()

	manifest, errManifest := readManifest()
	if errManifest != nil {
		fmt.Println("Failed to parse manifest file:  " + errManifest.Error())
		return
	}

	v := getVersion(manifest)
	fmt.Println("")
	cleanupFiles()

	downloadRelease(v.ReleaseURL, v.ReleaseFileName+".zip")

	fmt.Println("Unzipping goCoreMaster.zip . . .")

	excludedFiles := []string{}
	excludeFiles(excludedFiles, v)

	errUnzip := zip.Unzip(v.ReleaseFileName+".zip", v.ReleaseFileName, excludedFiles)

	if errUnzip != nil {
		fmt.Println("Failed to Unzip goCoreMaster.zip:  " + errUnzip.Error())
		return
	}

	fmt.Println("Unzipped GoCoreDep Master successfully.")

	errRemoveRepoZip := os.Remove(v.ReleaseFileName + ".zip")

	if errRemoveRepoZip != nil {
		fmt.Println("Failed to Remove GoCore Repo Zip File:  " + errRemoveRepoZip.Error())
		return
	}

	cleanGoCore(v)
	fmt.Println("\nMoving Files . . .")

	//Copy the Files then Remove the Directory
	extensions.CopyFolder(v.ReleaseFileName+"/"+v.ReleaseFileName, "")

	fmt.Println("Moved Files Successfully.")
	fmt.Println("\nStarted Cleaning Files.")
	errRemoveDecompressedFiles := extensions.RemoveDirectory(v.ReleaseFileName)

	if errRemoveDecompressedFiles != nil {
		fmt.Println("Failed to Remove GoCore Decompressed Files:  " + errRemoveDecompressedFiles.Error())
		return
	}

	fmt.Println("Cleaned up Repo Files Successfully.")
}

func downloadRelease(url string, fileName string) {

	fmt.Println("Downloading GoCore Dependencies.  Please Wait . . .")

	out, errCreateFile := os.Create(fileName)

	if errCreateFile != nil {
		fmt.Println("Failed to create file handle:  " + errCreateFile.Error())
		return
	}

	resp, errHttpGet := http.Get(url)
	defer resp.Body.Close()

	if errHttpGet != nil {
		fmt.Println("Failed to Download GoCoreDep master zip file:  " + errHttpGet.Error())
		return
	}

	n, errCopyOut := io.Copy(out, resp.Body)

	if errCopyOut != nil {
		fmt.Println("Failed to Output to goCoreMaster.zip:  " + errCopyOut.Error())
		return
	}
	out.Close()

	fmt.Println("Downloaded GoCoreDep Master successfully:  " + extensions.PrintMegaBytes(n))
}

func downloadManifest() {

	out, errCreateFile := os.Create(manifestFileName)

	if errCreateFile != nil {
		fmt.Println("Failed to create file handle:  " + errCreateFile.Error())
		return
	}

	fmt.Println("Downloading Manifest.  Please Wait . . .")

	resp, errHttpGet := http.Get(manifestURL)
	defer resp.Body.Close()

	if errHttpGet != nil {
		fmt.Println("Failed to Download GoCoreDep manifest:  " + errHttpGet.Error())
		return
	}

	n, errCopyOut := io.Copy(out, resp.Body)

	if errCopyOut != nil {
		fmt.Println("Failed to Output to manifest:  " + errCopyOut.Error())
		return
	}
	out.Close()

	fmt.Println("Downloaded GoCoreDep manifest successfully:  " + extensions.PrintMegaBytes(n))
}

func readManifest() (coreManifest, error) {

	var manifest coreManifest

	jsonData, err := ioutil.ReadFile(manifestFileName)
	if err != nil {
		fmt.Println("Reading of " + manifestFileName + " failed:  " + err.Error())
		return manifest, err
	}

	errUnmarshal := json.Unmarshal(jsonData, &manifest)
	if errUnmarshal != nil {
		fmt.Println("Parsing / Unmarshaling of " + manifestFileName + " failed:  " + errUnmarshal.Error())
		return manifest, errUnmarshal
	}

	return manifest, nil
}

func cleanupFiles() {
	err := os.Remove(manifestFileName)

	if err != nil {
		fmt.Println("Failed to clean up " + manifestFileName + ":  " + err.Error())
	}

}

func cleanGoCore(v version) {

	for _, directory := range v.GoDirectories {
		fmt.Println("Removing Directory:  " + directory)
		os.RemoveAll(directory)
	}
}

func getVersion(manifest coreManifest) version {
	v := manifest.Versions[len(manifest.Versions)-1]

	if len(os.Args) > 1 {
		argVersion := os.Args[1]

		for _, vn := range manifest.Versions {
			if vn.Version == argVersion {
				v = vn
				break
			}
		}
	}

	return v
}

func excludeFiles(excludedFiles []string, v version) {

	_, err := os.Stat("webConfig.json")
	if err == nil {
		excludedFiles = append(excludedFiles, v.ReleaseFileName+"/webConfig.json")
	}

	_, err1 := os.Stat("src/github.com/DanielRenne/GoCore/core/app/app.go")
	if err1 == nil {
		excludedFiles = append(excludedFiles, v.ReleaseFileName+"/src/github.com/DanielRenne/GoCore/core/app/app.go")
	}

	_, err2 := os.Stat("/keys/cert.pem")
	if err2 == nil {
		excludedFiles = append(excludedFiles, v.ReleaseFileName+"/keys/cert.pem")
	}

	_, err3 := os.Stat("/keys/key.pem")
	if err3 == nil {
		excludedFiles = append(excludedFiles, v.ReleaseFileName+"/keys/key.pem")
	}
}
<commit_msg>Update getCore.go<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/DanielRenne/GoCore/core/extensions"
	"github.com/DanielRenne/GoCore/core/zip"
	"io"
	"io/ioutil"
	"net/http"
	"os"
)

const gitRepoName = "GoCoreDep-master"
const manifestFileName = "downloadedManifest.json"
const manifestURL = "https://raw.githubusercontent.com/DanielRenne/GoCoreDep/master/coreManifest.json"

type version struct {
	Version         string   `json:"version"`
	ReleaseURL      string   `json:"releaseURL"`
	ReleaseFileName string   `json:"releaseFileName"`
	GoDirectories   []string `json:"goDirectories"`
}

type coreManifest struct {
	Versions []version `json:"versions"`
}

func main() {

	downloadManifest()

	manifest, errManifest := readManifest()
	if errManifest != nil {
		fmt.Println("Failed to parse manifest file:  " + errManifest.Error())
		return
	}

	v := getVersion(manifest)
	fmt.Println("")
	cleanupFiles()

	downloadRelease(v.ReleaseURL, v.ReleaseFileName+".zip")

	fmt.Println("Unzipping goCoreMaster.zip . . .")

	excludedFiles := []string{}
	excludeFiles(excludedFiles, v)

	errUnzip := zip.Unzip(v.ReleaseFileName+".zip", v.ReleaseFileName, excludedFiles)

	if errUnzip != nil {
		fmt.Println("Failed to Unzip goCoreMaster.zip:  " + errUnzip.Error())
		return
	}

	fmt.Println("Unzipped GoCoreDep Master successfully.")

	errRemoveRepoZip := os.Remove(v.ReleaseFileName + ".zip")

	if errRemoveRepoZip != nil {
		fmt.Println("Failed to Remove GoCore Repo Zip File:  " + errRemoveRepoZip.Error())
		return
	}

	errRemoveRepoZip = os.Remove("coreManifest.json")

	if errRemoveRepoZip != nil {
		fmt.Println("Failed to Remove GoCore coreManifest.json: " + errRemoveRepoZip.Error())
		return
	}

	errRemoveRepoZip = os.Remove("README.md")

	if errRemoveRepoZip != nil {
		fmt.Println("Failed to Remove GoCore README.md: " + errRemoveRepoZip.Error())
		return
	}

	cleanGoCore(v)
	fmt.Println("\nMoving Files . . .")

	//Copy the Files then Remove the Directory
	extensions.CopyFolder(v.ReleaseFileName+"/"+v.ReleaseFileName, "")

	fmt.Println("Moved Files Successfully.")
	fmt.Println("\nStarted Cleaning Files.")
	errRemoveDecompressedFiles := extensions.RemoveDirectory(v.ReleaseFileName)

	if errRemoveDecompressedFiles != nil {
		fmt.Println("Failed to Remove GoCore Decompressed Files:  " + errRemoveDecompressedFiles.Error())
		return
	}

	fmt.Println("Cleaned up Repo Files Successfully.")
}

func downloadRelease(url string, fileName string) {

	fmt.Println("Downloading GoCore Dependencies.  Please Wait . . .")

	out, errCreateFile := os.Create(fileName)

	if errCreateFile != nil {
		fmt.Println("Failed to create file handle:  " + errCreateFile.Error())
		return
	}

	resp, errHttpGet := http.Get(url)
	defer resp.Body.Close()

	if errHttpGet != nil {
		fmt.Println("Failed to Download GoCoreDep master zip file:  " + errHttpGet.Error())
		return
	}

	n, errCopyOut := io.Copy(out, resp.Body)

	if errCopyOut != nil {
		fmt.Println("Failed to Output to goCoreMaster.zip:  " + errCopyOut.Error())
		return
	}
	out.Close()

	fmt.Println("Downloaded GoCoreDep Master successfully:  " + extensions.PrintMegaBytes(n))
}

func downloadManifest() {

	out, errCreateFile := os.Create(manifestFileName)

	if errCreateFile != nil {
		fmt.Println("Failed to create file handle:  " + errCreateFile.Error())
		return
	}

	fmt.Println("Downloading Manifest.  Please Wait . . .")

	resp, errHttpGet := http.Get(manifestURL)
	defer resp.Body.Close()

	if errHttpGet != nil {
		fmt.Println("Failed to Download GoCoreDep manifest:  " + errHttpGet.Error())
		return
	}

	n, errCopyOut := io.Copy(out, resp.Body)

	if errCopyOut != nil {
		fmt.Println("Failed to Output to manifest:  " + errCopyOut.Error())
		return
	}
	out.Close()

	fmt.Println("Downloaded GoCoreDep manifest successfully:  " + extensions.PrintMegaBytes(n))
}

func readManifest() (coreManifest, error) {

	var manifest coreManifest

	jsonData, err := ioutil.ReadFile(manifestFileName)
	if err != nil {
		fmt.Println("Reading of " + manifestFileName + " failed:  " + err.Error())
		return manifest, err
	}

	errUnmarshal := json.Unmarshal(jsonData, &manifest)
	if errUnmarshal != nil {
		fmt.Println("Parsing / Unmarshaling of " + manifestFileName + " failed:  " + errUnmarshal.Error())
		return manifest, errUnmarshal
	}

	return manifest, nil
}

func cleanupFiles() {
	err := os.Remove(manifestFileName)

	if err != nil {
		fmt.Println("Failed to clean up " + manifestFileName + ":  " + err.Error())
	}

}

func cleanGoCore(v version) {

	for _, directory := range v.GoDirectories {
		fmt.Println("Removing Directory:  " + directory)
		os.RemoveAll(directory)
	}
}

func getVersion(manifest coreManifest) version {
	v := manifest.Versions[len(manifest.Versions)-1]

	if len(os.Args) > 1 {
		argVersion := os.Args[1]

		for _, vn := range manifest.Versions {
			if vn.Version == argVersion {
				v = vn
				break
			}
		}
	}

	return v
}

func excludeFiles(excludedFiles []string, v version) {

	_, err := os.Stat("webConfig.json")
	if err == nil {
		excludedFiles = append(excludedFiles, v.ReleaseFileName+"/webConfig.json")
	}

	_, err1 := os.Stat("src/github.com/DanielRenne/GoCore/core/app/app.go")
	if err1 == nil {
		excludedFiles = append(excludedFiles, v.ReleaseFileName+"/src/github.com/DanielRenne/GoCore/core/app/app.go")
	}

	_, err2 := os.Stat("/keys/cert.pem")
	if err2 == nil {
		excludedFiles = append(excludedFiles, v.ReleaseFileName+"/keys/cert.pem")
	}

	_, err3 := os.Stat("/keys/key.pem")
	if err3 == nil {
		excludedFiles = append(excludedFiles, v.ReleaseFileName+"/keys/key.pem")
	}
}
<|endoftext|>


<commit_before>package github

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/gin-gonic/gin"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/rs/zerolog/log"
)

// EventHandler handles http events for Github integration
type EventHandler interface {
	Handle(*gin.Context)
	HandlePushEvent(PushEvent)
}

type eventHandlerImpl struct {
	eventsChannel                chan PushEvent
	githubWebhookSecret          string
	prometheusInboundEventTotals *prometheus.CounterVec
}

// NewGithubEventHandler returns a github.EventHandler to handle incoming webhook events
func NewGithubEventHandler(eventsChannel chan PushEvent, githubWebhookSecret string, prometheusInboundEventTotals *prometheus.CounterVec) EventHandler {
	return &eventHandlerImpl{
		eventsChannel:                eventsChannel,
		githubWebhookSecret:          githubWebhookSecret,
		prometheusInboundEventTotals: prometheusInboundEventTotals,
	}
}

func (h *eventHandlerImpl) Handle(c *gin.Context) {

	// https://developer.github.com/webhooks/
	eventType := c.GetHeader("X-Github-Event")
	h.prometheusInboundEventTotals.With(prometheus.Labels{"event": eventType, "source": "github"}).Inc()

	body, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		log.Error().Err(err).Msg("Reading body from Github webhook failed")
		c.String(http.StatusInternalServerError, "Reading body from Github webhook failed")
		return
	}

	// https://developer.github.com/webhooks/securing/
	signature := c.GetHeader("X-Hub-Signature")
	signature = strings.Replace(signature, "sha1=", "", 1)
	actualMAC, err := hex.DecodeString(signature)
	if err != nil {
		log.Error().Err(err).Msg("Decoding hexadecimal X-Hub-Signature to byte array failed")
	}

	// calculate expected MAC
	mac := hmac.New(sha1.New, []byte(h.githubWebhookSecret))
	mac.Write(body)
	expectedMAC := mac.Sum(nil)

	// compare actual and expected MAC
	if hmac.Equal(actualMAC, expectedMAC) {
		log.Debug().Str("expectedMAC", hex.EncodeToString(expectedMAC)).Str("actualMAC", hex.EncodeToString(actualMAC)).Msg("Valid HMAC signature")
	} else {
		log.Warn().Str("expectedMAC", hex.EncodeToString(expectedMAC)).Str("actualMAC", hex.EncodeToString(actualMAC)).Msg("Invalid HMAC signature")
	}

	// unmarshal json body
	var b interface{}
	err = json.Unmarshal(body, &b)
	if err != nil {
		log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body from Github webhook failed")
		c.String(http.StatusInternalServerError, "Deserializing body from Github webhook failed")
		return
	}

	log.Debug().
		Str("method", c.Request.Method).
		Str("url", c.Request.URL.String()).
		Interface("headers", c.Request.Header).
		Interface("body", b).
		Msgf("Received webhook event of type '%v' from GitHub...", eventType)

	switch eventType {
	case "push": // Any Git push to a Repository, including editing tags or branches. Commits via API actions that update references are also counted. This is the default event.

		// unmarshal json body
		var pushEvent PushEvent
		err := json.Unmarshal(body, &pushEvent)
		if err != nil {
			log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body to GithubPushEvent failed")
			return
		}

		h.HandlePushEvent(pushEvent)

	case
		"commit_comment",              // Any time a Commit is commented on.
		"create",                      // Any time a Branch or Tag is created.
		"delete",                      // Any time a Branch or Tag is deleted.
		"deployment",                  // Any time a Repository has a new deployment created from the API.
		"deployment_status",           // Any time a deployment for a Repository has a status update from the API.
		"fork",                        // Any time a Repository is forked.
		"gollum",                      // Any time a Wiki page is updated.
		"installation",                // Any time a GitHub App is installed or uninstalled.
		"installation_repositories",   // Any time a repository is added or removed from an installation.
		"issue_comment",               // Any time a comment on an issue is created, edited, or deleted.
		"issues",                      // Any time an Issue is assigned, unassigned, labeled, unlabeled, opened, edited, milestoned, demilestoned, closed, or reopened.
		"label",                       // Any time a Label is created, edited, or deleted.
		"marketplace_purchase",        // Any time a user purchases, cancels, or changes their GitHub Marketplace plan.
		"member",                      // Any time a User is added or removed as a collaborator to a Repository, or has their permissions modified.
		"membership",                  // Any time a User is added or removed from a team. Organization hooks only.
		"milestone",                   // Any time a Milestone is created, closed, opened, edited, or deleted.
		"organization",                // Any time a user is added, removed, or invited to an Organization. Organization hooks only.
		"org_block",                   // Any time an organization blocks or unblocks a user. Organization hooks only.
		"page_build",                  // Any time a Pages site is built or results in a failed build.
		"project_card",                // Any time a Project Card is created, edited, moved, converted to an issue, or deleted.
		"project_column",              // Any time a Project Column is created, edited, moved, or deleted.
		"project",                     // Any time a Project is created, edited, closed, reopened, or deleted.
		"public",                      // Any time a Repository changes from private to public.
		"pull_request_review_comment", // Any time a comment on a pull request's unified diff is created, edited, or deleted (in the Files Changed tab).
		"pull_request_review",         // Any time a pull request review is submitted, edited, or dismissed.
		"pull_request",                // Any time a pull request is assigned, unassigned, labeled, unlabeled, opened, edited, closed, reopened, or synchronized (updated due to a new push in the branch that the pull request is tracking). Also any time a pull request review is requested, or a review request is removed.
		"repository",                  // Any time a Repository is created, deleted (organization hooks only), made public, or made private.
		"release",                     // Any time a Release is published in a Repository.
		"status",                      // Any time a Repository has a status update from the API
		"team",                        // Any time a team is created, deleted, modified, or added to or removed from a repository. Organization hooks only
		"team_add",                    // Any time a team is added or modified on a Repository.
		"watch",                       // Any time a User stars a Repository.
		"integration_installation_repositories": // ?
		log.Debug().Str("event", eventType).Msgf("Not implemented Github webhook event of type '%v'", eventType)

	default:
		log.Warn().Str("event", eventType).Msgf("Unsupported Github webhook event of type '%v'", eventType)
	}

	c.String(http.StatusOK, "Aye aye!")
}

func (h *eventHandlerImpl) HandlePushEvent(pushEvent PushEvent) {

	log.Debug().Interface("pushEvent", pushEvent).Msgf("Deserialized GitHub push event for repository %v", pushEvent.Repository.FullName)

	// test making api calls for github app in the background
	h.eventsChannel <- pushEvent
}
}
<commit_msg>fix vs code save error<commit_after>package github

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/gin-gonic/gin"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/rs/zerolog/log"
)

// EventHandler handles http events for Github integration
type EventHandler interface {
	Handle(*gin.Context)
	HandlePushEvent(PushEvent)
}

type eventHandlerImpl struct {
	eventsChannel                chan PushEvent
	githubWebhookSecret          string
	prometheusInboundEventTotals *prometheus.CounterVec
}

// NewGithubEventHandler returns a github.EventHandler to handle incoming webhook events
func NewGithubEventHandler(eventsChannel chan PushEvent, githubWebhookSecret string, prometheusInboundEventTotals *prometheus.CounterVec) EventHandler {
	return &eventHandlerImpl{
		eventsChannel:                eventsChannel,
		githubWebhookSecret:          githubWebhookSecret,
		prometheusInboundEventTotals: prometheusInboundEventTotals,
	}
}

func (h *eventHandlerImpl) Handle(c *gin.Context) {

	// https://developer.github.com/webhooks/
	eventType := c.GetHeader("X-Github-Event")
	h.prometheusInboundEventTotals.With(prometheus.Labels{"event": eventType, "source": "github"}).Inc()

	body, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		log.Error().Err(err).Msg("Reading body from Github webhook failed")
		c.String(http.StatusInternalServerError, "Reading body from Github webhook failed")
		return
	}

	// https://developer.github.com/webhooks/securing/
	signature := c.GetHeader("X-Hub-Signature")
	signature = strings.Replace(signature, "sha1=", "", 1)
	actualMAC, err := hex.DecodeString(signature)
	if err != nil {
		log.Error().Err(err).Msg("Decoding hexadecimal X-Hub-Signature to byte array failed")
	}

	// calculate expected MAC
	mac := hmac.New(sha1.New, []byte(h.githubWebhookSecret))
	mac.Write(body)
	expectedMAC := mac.Sum(nil)

	// compare actual and expected MAC
	if hmac.Equal(actualMAC, expectedMAC) {
		log.Debug().Str("expectedMAC", hex.EncodeToString(expectedMAC)).Str("actualMAC", hex.EncodeToString(actualMAC)).Msg("Valid HMAC signature")
	} else {
		log.Warn().Str("expectedMAC", hex.EncodeToString(expectedMAC)).Str("actualMAC", hex.EncodeToString(actualMAC)).Msg("Invalid HMAC signature")
	}

	// unmarshal json body
	var b interface{}
	err = json.Unmarshal(body, &b)
	if err != nil {
		log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body from Github webhook failed")
		c.String(http.StatusInternalServerError, "Deserializing body from Github webhook failed")
		return
	}

	log.Debug().
		Str("method", c.Request.Method).
		Str("url", c.Request.URL.String()).
		Interface("headers", c.Request.Header).
		Interface("body", b).
		Msgf("Received webhook event of type '%v' from GitHub...", eventType)

	switch eventType {
	case "push": // Any Git push to a Repository, including editing tags or branches. Commits via API actions that update references are also counted. This is the default event.

		// unmarshal json body
		var pushEvent PushEvent
		err := json.Unmarshal(body, &pushEvent)
		if err != nil {
			log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body to GithubPushEvent failed")
			return
		}

		h.HandlePushEvent(pushEvent)

	case
		"commit_comment",              // Any time a Commit is commented on.
		"create",                      // Any time a Branch or Tag is created.
		"delete",                      // Any time a Branch or Tag is deleted.
		"deployment",                  // Any time a Repository has a new deployment created from the API.
		"deployment_status",           // Any time a deployment for a Repository has a status update from the API.
		"fork",                        // Any time a Repository is forked.
		"gollum",                      // Any time a Wiki page is updated.
		"installation",                // Any time a GitHub App is installed or uninstalled.
		"installation_repositories",   // Any time a repository is added or removed from an installation.
		"issue_comment",               // Any time a comment on an issue is created, edited, or deleted.
		"issues",                      // Any time an Issue is assigned, unassigned, labeled, unlabeled, opened, edited, milestoned, demilestoned, closed, or reopened.
		"label",                       // Any time a Label is created, edited, or deleted.
		"marketplace_purchase",        // Any time a user purchases, cancels, or changes their GitHub Marketplace plan.
		"member",                      // Any time a User is added or removed as a collaborator to a Repository, or has their permissions modified.
		"membership",                  // Any time a User is added or removed from a team. Organization hooks only.
		"milestone",                   // Any time a Milestone is created, closed, opened, edited, or deleted.
		"organization",                // Any time a user is added, removed, or invited to an Organization. Organization hooks only.
		"org_block",                   // Any time an organization blocks or unblocks a user. Organization hooks only.
		"page_build",                  // Any time a Pages site is built or results in a failed build.
		"project_card",                // Any time a Project Card is created, edited, moved, converted to an issue, or deleted.
		"project_column",              // Any time a Project Column is created, edited, moved, or deleted.
		"project",                     // Any time a Project is created, edited, closed, reopened, or deleted.
		"public",                      // Any time a Repository changes from private to public.
		"pull_request_review_comment", // Any time a comment on a pull request's unified diff is created, edited, or deleted (in the Files Changed tab).
		"pull_request_review",         // Any time a pull request review is submitted, edited, or dismissed.
		"pull_request",                // Any time a pull request is assigned, unassigned, labeled, unlabeled, opened, edited, closed, reopened, or synchronized (updated due to a new push in the branch that the pull request is tracking). Also any time a pull request review is requested, or a review request is removed.
		"repository",                  // Any time a Repository is created, deleted (organization hooks only), made public, or made private.
		"release",                     // Any time a Release is published in a Repository.
		"status",                      // Any time a Repository has a status update from the API
		"team",                        // Any time a team is created, deleted, modified, or added to or removed from a repository. Organization hooks only
		"team_add",                    // Any time a team is added or modified on a Repository.
		"watch",                       // Any time a User stars a Repository.
		"integration_installation_repositories": // ?
		log.Debug().Str("event", eventType).Msgf("Not implemented Github webhook event of type '%v'", eventType)

	default:
		log.Warn().Str("event", eventType).Msgf("Unsupported Github webhook event of type '%v'", eventType)
	}

	c.String(http.StatusOK, "Aye aye!")
}

func (h *eventHandlerImpl) HandlePushEvent(pushEvent PushEvent) {

	log.Debug().Interface("pushEvent", pushEvent).Msgf("Deserialized GitHub push event for repository %v", pushEvent.Repository.FullName)

	// test making api calls for github app in the background
	h.eventsChannel <- pushEvent
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////

package aead

import (
	"bytes"
	"encoding/binary"
	"errors"

	"github.com/golang/protobuf/proto"
	"github.com/google/tink/go/core/registry"
	"github.com/google/tink/go/tink"
	tinkpb "github.com/google/tink/proto/tink_go_proto"
)

const (
	lenDEK = 4
)

// KMSEnvelopeAEAD represents an instance of Envelope AEAD.
type KMSEnvelopeAEAD struct {
	dekTemplate *tinkpb.KeyTemplate
	remote      tink.AEAD
}

var _ tink.AEAD = (*KMSEnvelopeAEAD)(nil)

// NewKMSEnvelopeAEAD creates an new instance of KMSEnvelopeAEAD
func NewKMSEnvelopeAEAD(kt tinkpb.KeyTemplate, remote tink.AEAD) *KMSEnvelopeAEAD {
	return &KMSEnvelopeAEAD{
		remote:      remote,
		dekTemplate: &kt,
	}
}

// Encrypt implements the tink.AEAD interface for encryption.
func (a *KMSEnvelopeAEAD) Encrypt(pt, aad []byte) ([]byte, error) {
	dekM, err := registry.NewKey(a.dekTemplate)
	if err != nil {
		return nil, err
	}
	dek, err := proto.Marshal(dekM)
	if err != nil {
		return nil, err
	}
	encryptedDEK, err := a.remote.Encrypt(dek, []byte{0})
	if err != nil {
		return nil, err
	}
	p, err := registry.Primitive(a.dekTemplate.TypeUrl, dek)
	if err != nil {
		return nil, err
	}
	primitive, ok := p.(tink.AEAD)
	if !ok {
		return nil, errors.New("failed to convert AEAD primitive")
	}

	payload, err := primitive.Encrypt(pt, aad)
	if err != nil {
		return nil, err
	}
	return buildCipherText(encryptedDEK, payload)

}

// Decrypt implements the tink.AEAD interface for decryption.
func (a *KMSEnvelopeAEAD) Decrypt(ct, aad []byte) ([]byte, error) {
	b := bytes.NewBuffer(ct)
	bLen := b.Len()
	l := make([]byte, lenDEK)
	_, err := b.Read(l)
	ed, err := binary.ReadVarint(bytes.NewReader(l))
	if err != nil {
		return nil, errors.New("invalid ciphertext")
	}
	if ed <= 0 || ed > int64(len(ct)-lenDEK) {
		return nil, errors.New("invalid ciphertext")
	}
	encryptedDEK := make([]byte, ed)
	n, err := b.Read(encryptedDEK)
	if err != nil || int64(n) != ed {
		return nil, errors.New("invalid ciphertext")
	}
	pl := bLen - lenDEK - int(ed)
	payload := make([]byte, pl)
	n, err = b.Read(payload)
	if err != nil || n != pl {
		return nil, errors.New("invalid ciphertext")
	}

	dek, err := a.remote.Decrypt(encryptedDEK, []byte{0})
	if err != nil {
		return nil, errors.New("decryption failed")
	}
	p, err := registry.Primitive(a.dekTemplate.TypeUrl, dek)
	if err != nil {
		return nil, errors.New("decryption failed")
	}
	primitive, ok := p.(tink.AEAD)
	if !ok {
		return nil, errors.New("failed to convert AEAD primitive")
	}
	return primitive.Decrypt(payload, aad)
}

// buildCipherText builds the cipher text by appending the length DEK, encrypted DEK
// and the encrypted payload.
func buildCipherText(encryptedDEK, payload []byte) ([]byte, error) {
	buf := make([]byte, lenDEK)
	var b bytes.Buffer
	_ = binary.PutVarint(buf, int64(len(encryptedDEK)))
	_, err := b.Write(buf)
	if err != nil {
		return nil, err
	}
	_, err = b.Write(encryptedDEK)
	if err != nil {
		return nil, err
	}
	_, err = b.Write(payload)
	if err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}
<commit_msg>Propagate KMS errors to caller.<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////

package aead

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"

	"github.com/golang/protobuf/proto"
	"github.com/google/tink/go/core/registry"
	"github.com/google/tink/go/tink"
	tinkpb "github.com/google/tink/proto/tink_go_proto"
)

const (
	lenDEK = 4
)

// KMSEnvelopeAEAD represents an instance of Envelope AEAD.
type KMSEnvelopeAEAD struct {
	dekTemplate *tinkpb.KeyTemplate
	remote      tink.AEAD
}

var _ tink.AEAD = (*KMSEnvelopeAEAD)(nil)

// NewKMSEnvelopeAEAD creates an new instance of KMSEnvelopeAEAD
func NewKMSEnvelopeAEAD(kt tinkpb.KeyTemplate, remote tink.AEAD) *KMSEnvelopeAEAD {
	return &KMSEnvelopeAEAD{
		remote:      remote,
		dekTemplate: &kt,
	}
}

// Encrypt implements the tink.AEAD interface for encryption.
func (a *KMSEnvelopeAEAD) Encrypt(pt, aad []byte) ([]byte, error) {
	dekM, err := registry.NewKey(a.dekTemplate)
	if err != nil {
		return nil, err
	}
	dek, err := proto.Marshal(dekM)
	if err != nil {
		return nil, err
	}
	encryptedDEK, err := a.remote.Encrypt(dek, []byte{0})
	if err != nil {
		return nil, err
	}
	p, err := registry.Primitive(a.dekTemplate.TypeUrl, dek)
	if err != nil {
		return nil, err
	}
	primitive, ok := p.(tink.AEAD)
	if !ok {
		return nil, errors.New("kms_envelope_aead: failed to convert AEAD primitive")
	}

	payload, err := primitive.Encrypt(pt, aad)
	if err != nil {
		return nil, err
	}
	return buildCipherText(encryptedDEK, payload)

}

// Decrypt implements the tink.AEAD interface for decryption.
func (a *KMSEnvelopeAEAD) Decrypt(ct, aad []byte) ([]byte, error) {
	b := bytes.NewBuffer(ct)
	bLen := b.Len()
	l := make([]byte, lenDEK)
	_, err := b.Read(l)
	ed, err := binary.ReadVarint(bytes.NewReader(l))
	if err != nil {
		return nil, errors.New("kms_envelope_aead: invalid ciphertext")
	}
	if ed <= 0 || ed > int64(len(ct)-lenDEK) {
		return nil, errors.New("kms_envelope_aead: invalid ciphertext")
	}
	encryptedDEK := make([]byte, ed)
	n, err := b.Read(encryptedDEK)
	if err != nil || int64(n) != ed {
		return nil, errors.New("kms_envelope_aead: invalid ciphertext")
	}
	pl := bLen - lenDEK - int(ed)
	payload := make([]byte, pl)
	n, err = b.Read(payload)
	if err != nil || n != pl {
		return nil, errors.New("kms_envelope_aead: invalid ciphertext")
	}

	dek, err := a.remote.Decrypt(encryptedDEK, []byte{0})
	if err != nil {
		return nil, err
	}
	p, err := registry.Primitive(a.dekTemplate.TypeUrl, dek)
	if err != nil {
		return nil, fmt.Errorf("kms_envelope_aead: %s", err)
	}
	primitive, ok := p.(tink.AEAD)
	if !ok {
		return nil, errors.New("kms_envelope_aead: failed to convert AEAD primitive")
	}
	return primitive.Decrypt(payload, aad)
}

// buildCipherText builds the cipher text by appending the length DEK, encrypted DEK
// and the encrypted payload.
func buildCipherText(encryptedDEK, payload []byte) ([]byte, error) {
	buf := make([]byte, lenDEK)
	var b bytes.Buffer
	_ = binary.PutVarint(buf, int64(len(encryptedDEK)))
	_, err := b.Write(buf)
	if err != nil {
		return nil, err
	}
	_, err = b.Write(encryptedDEK)
	if err != nil {
		return nil, err
	}
	_, err = b.Write(payload)
	if err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}
<|endoftext|>


<commit_before>// +build darwin dragonfly freebsd linux netbsd openbsd solaris

package client

import (
	"fmt"
	"os"
	"os/exec"
	"syscall"
	"time"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/service"
)

// GetExtraFlags gets the extra fork-related flags for this platform
func GetExtraFlags() []cli.Flag {
	return []cli.Flag{
		cli.BoolFlag{
			Name:  "no-auto-fork, F",
			Usage: "Disable auto-fork of background service.",
		},
		cli.BoolFlag{
			Name:  "auto-fork",
			Usage: "Enable auto-fork of background service.",
		},
	}
}

// ForkServerNix forks a new background Keybase service, and waits until it's
// pingable. It will only do something useful on Unixes; it won't work on
// Windows (probably?). Returns an error if anything bad happens; otherwise,
// assume that the server was successfully started up.
func ForkServerNix(cl libkb.CommandLine) error {
	srv := service.NewService(true /* isDaemon */)

	// If we try to get an exclusive lock and succeed, it means we
	// need to relaunch the daemon since it's dead
	G.Log.Debug("Getting flock")
	err := srv.GetExclusiveLock()
	if err == nil {
		G.Log.Debug("Flocked! Server must have died")
		srv.ReleaseLock()
		err = spawnServer(cl)
		if err != nil {
			G.Log.Errorf("Error in spawning server process: %s", err)
			return err
		}
		err = pingLoop()
		if err != nil {
			G.Log.Errorf("Ping failure after server fork: %s", err)
			return err
		}
	} else {
		G.Log.Debug("The server is still up")
		err = nil
	}

	return err
}

func pingLoop() error {
	var err error
	for i := 0; i < 10; i++ {
		_, _, err = G.GetSocket()
		if err == nil {
			G.Log.Debug("Connected (%d)", i)
			return nil
		}
		G.ClearSocketError()
		G.Log.Debug("Failed to connect to socket (%d): %s", i, err)
		err = nil
		time.Sleep(200 * time.Millisecond)
	}
	return nil
}

func makeServerCommandLine(cl libkb.CommandLine) (arg0 string, args []string, err error) {
	// ForkExec requires an absolute path to the binary. LookPath() gets this
	// for us, or correctly leaves arg0 alone if it's already a path.
	arg0, err = exec.LookPath(os.Args[0])
	if err != nil {
		return
	}

	// Fixme: This isn't ideal, it would be better to specify when the args
	// are defined if they should be reexported to the server, and if so, then
	// we should automate the reconstruction of the argument vector.  Let's do
	// this when we yank out keybase/cli
	bools := []string{
		"debug",
		"api-dump-unsafe",
		"plain-logging",
	}

	strings := []string{
		"home",
		"server",
		"config",
		"session",
		"proxy",
		"username",
		"gpg-home",
		"gpg",
		"secret-keyring",
		"pid-file",
		"socket-file",
		"gpg-options",
		"local-rpc-debug-unsafe",
	}
	args = append(args, arg0)

	for _, b := range bools {
		if isSet, isTrue := cl.GetBool(b, true); isSet && isTrue {
			args = append(args, "--"+b)
		}
	}

	for _, s := range strings {
		if v := cl.GetGString(s); len(v) > 0 {
			args = append(args, "--"+s, v)
		}
	}

	args = append(args, "service")

	var chdir string
	chdir, err = G.Env.GetServiceSpawnDir()
	if err != nil {
		return
	}

	G.Log.Info("| Setting run directory for keybase service to %s", chdir)
	args = append(args, "--chdir", chdir)

	G.Log.Debug("| Made server args: %s %v", arg0, args)

	return
}

func spawnServer(cl libkb.CommandLine) (err error) {

	var files []uintptr
	var cmd string
	var args []string
	var devnull, log *os.File
	var pid int

	defer func() {
		if err != nil {
			if devnull != nil {
				devnull.Close()
			}
			if log != nil {
				log.Close()
			}
		}
	}()

	if devnull, err = os.OpenFile("/dev/null", os.O_RDONLY, 0); err != nil {
		return
	}
	files = append(files, devnull.Fd())

	if G.Env.GetSplitLogOutput() {
		files = append(files, uintptr(1), uintptr(2))
	} else {
		if _, log, err = libkb.OpenLogFile(); err != nil {
			return
		}
		files = append(files, log.Fd(), log.Fd())
	}

	attr := syscall.ProcAttr{
		Env:   os.Environ(),
		Sys:   &syscall.SysProcAttr{Setsid: true},
		Files: files,
	}

	cmd, args, err = makeServerCommandLine(cl)
	if err != nil {
		return err
	}

	pid, err = syscall.ForkExec(cmd, args, &attr)
	if err != nil {
		err = fmt.Errorf("Error in ForkExec: %s", err)
	} else {
		G.Log.Info("Forking background server with pid=%d", pid)
	}
	return err
}
<commit_msg>cli: pass `run-mode` through to forked daemon<commit_after>// +build darwin dragonfly freebsd linux netbsd openbsd solaris

package client

import (
	"fmt"
	"os"
	"os/exec"
	"syscall"
	"time"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/service"
)

// GetExtraFlags gets the extra fork-related flags for this platform
func GetExtraFlags() []cli.Flag {
	return []cli.Flag{
		cli.BoolFlag{
			Name:  "no-auto-fork, F",
			Usage: "Disable auto-fork of background service.",
		},
		cli.BoolFlag{
			Name:  "auto-fork",
			Usage: "Enable auto-fork of background service.",
		},
	}
}

// ForkServerNix forks a new background Keybase service, and waits until it's
// pingable. It will only do something useful on Unixes; it won't work on
// Windows (probably?). Returns an error if anything bad happens; otherwise,
// assume that the server was successfully started up.
func ForkServerNix(cl libkb.CommandLine) error {
	srv := service.NewService(true /* isDaemon */)

	// If we try to get an exclusive lock and succeed, it means we
	// need to relaunch the daemon since it's dead
	G.Log.Debug("Getting flock")
	err := srv.GetExclusiveLock()
	if err == nil {
		G.Log.Debug("Flocked! Server must have died")
		srv.ReleaseLock()
		err = spawnServer(cl)
		if err != nil {
			G.Log.Errorf("Error in spawning server process: %s", err)
			return err
		}
		err = pingLoop()
		if err != nil {
			G.Log.Errorf("Ping failure after server fork: %s", err)
			return err
		}
	} else {
		G.Log.Debug("The server is still up")
		err = nil
	}

	return err
}

func pingLoop() error {
	var err error
	for i := 0; i < 10; i++ {
		_, _, err = G.GetSocket()
		if err == nil {
			G.Log.Debug("Connected (%d)", i)
			return nil
		}
		G.ClearSocketError()
		G.Log.Debug("Failed to connect to socket (%d): %s", i, err)
		err = nil
		time.Sleep(200 * time.Millisecond)
	}
	return nil
}

func makeServerCommandLine(cl libkb.CommandLine) (arg0 string, args []string, err error) {
	// ForkExec requires an absolute path to the binary. LookPath() gets this
	// for us, or correctly leaves arg0 alone if it's already a path.
	arg0, err = exec.LookPath(os.Args[0])
	if err != nil {
		return
	}

	// Fixme: This isn't ideal, it would be better to specify when the args
	// are defined if they should be reexported to the server, and if so, then
	// we should automate the reconstruction of the argument vector.  Let's do
	// this when we yank out keybase/cli
	bools := []string{
		"debug",
		"api-dump-unsafe",
		"plain-logging",
	}

	strings := []string{
		"home",
		"server",
		"config",
		"session",
		"proxy",
		"username",
		"gpg-home",
		"gpg",
		"secret-keyring",
		"pid-file",
		"socket-file",
		"gpg-options",
		"local-rpc-debug-unsafe",
		"run-mode",
	}
	args = append(args, arg0)

	for _, b := range bools {
		if isSet, isTrue := cl.GetBool(b, true); isSet && isTrue {
			args = append(args, "--"+b)
		}
	}

	for _, s := range strings {
		if v := cl.GetGString(s); len(v) > 0 {
			args = append(args, "--"+s, v)
		}
	}

	args = append(args, "service")

	var chdir string
	chdir, err = G.Env.GetServiceSpawnDir()
	if err != nil {
		return
	}

	G.Log.Info("| Setting run directory for keybase service to %s", chdir)
	args = append(args, "--chdir", chdir)

	G.Log.Debug("| Made server args: %s %v", arg0, args)

	return
}

func spawnServer(cl libkb.CommandLine) (err error) {

	var files []uintptr
	var cmd string
	var args []string
	var devnull, log *os.File
	var pid int

	defer func() {
		if err != nil {
			if devnull != nil {
				devnull.Close()
			}
			if log != nil {
				log.Close()
			}
		}
	}()

	if devnull, err = os.OpenFile("/dev/null", os.O_RDONLY, 0); err != nil {
		return
	}
	files = append(files, devnull.Fd())

	if G.Env.GetSplitLogOutput() {
		files = append(files, uintptr(1), uintptr(2))
	} else {
		if _, log, err = libkb.OpenLogFile(); err != nil {
			return
		}
		files = append(files, log.Fd(), log.Fd())
	}

	attr := syscall.ProcAttr{
		Env:   os.Environ(),
		Sys:   &syscall.SysProcAttr{Setsid: true},
		Files: files,
	}

	cmd, args, err = makeServerCommandLine(cl)
	if err != nil {
		return err
	}

	pid, err = syscall.ForkExec(cmd, args, &attr)
	if err != nil {
		err = fmt.Errorf("Error in ForkExec: %s", err)
	} else {
		G.Log.Info("Forking background server with pid=%d", pid)
	}
	return err
}
<|endoftext|>


<commit_before>package fakes

import (
	"context"
	"fmt"
	"sync"

	"github.com/bazelbuild/remote-apis-sdks/go/digest"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
)

// ActionCache implements the RE ActionCache interface, storing fixed results.
type ActionCache struct {
	mu      sync.RWMutex
	results map[digest.Digest]*repb.ActionResult
	reads   map[digest.Digest]int
	writes  map[digest.Digest]int
}

// NewActionCache returns a new empty ActionCache.
func NewActionCache() *ActionCache {
	c := &ActionCache{}
	c.Clear()
	return c
}

// Clear removes all results from the cache.
func (c *ActionCache) Clear() {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.results = make(map[digest.Digest]*repb.ActionResult)
	c.reads = make(map[digest.Digest]int)
	c.writes = make(map[digest.Digest]int)
}

// PutAction sets a fake result for a given action, and returns the action digest.
func (c *ActionCache) PutAction(ac *repb.Action, res *repb.ActionResult) digest.Digest {
	d := digest.TestNewFromMessage(ac)
	c.Put(d, res)
	return d
}

// Put sets a fake result for a given action digest.
func (c *ActionCache) Put(d digest.Digest, res *repb.ActionResult) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.results[d] = res
}

// Get returns a previously saved fake result for the given action digest.
func (c *ActionCache) Get(d digest.Digest) *repb.ActionResult {
	c.mu.RLock()
	defer c.mu.RUnlock()
	res, ok := c.results[d]
	if !ok {
		return nil
	}
	return res
}

// GetReads returns the number of times GetActionResult was called for a given action digest.
// These include both successful and unsuccessful reads.
func (c *ActionCache) GetReads(d digest.Digest) int {
	return c.reads[d]
}

// GetWrites returns the number of times UpdateActionResult was called for a given action digest.
func (c *ActionCache) GetWrites(d digest.Digest) int {
	return c.writes[d]
}

// GetActionResult returns a stored result, if it was found.
func (c *ActionCache) GetActionResult(ctx context.Context, req *repb.GetActionResultRequest) (res *repb.ActionResult, err error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	dg, err := digest.NewFromProto(req.ActionDigest)
	if err != nil {
		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid digest received: %v", req.ActionDigest))
	}
	c.reads[dg]++
	if res, ok := c.results[dg]; ok {
		return res, nil
	}
	return nil, status.Error(codes.NotFound, "")
}

// UpdateActionResult sets/updates a given result.
func (c *ActionCache) UpdateActionResult(ctx context.Context, req *repb.UpdateActionResultRequest) (res *repb.ActionResult, err error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	dg, err := digest.NewFromProto(req.ActionDigest)
	if err != nil {
		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid digest received: %v", req.ActionDigest))
	}
	if req.ActionResult != nil {
		return nil, status.Error(codes.InvalidArgument, "no action result received")
	}
	c.results[dg] = req.ActionResult
	c.writes[dg]++
	return req.ActionResult, nil
}
<commit_msg>Fixing internal lint error: packages should have comments. (#21)<commit_after>// Package fakes contains configurable test fakes for an RE service.
package fakes

import (
	"context"
	"fmt"
	"sync"

	"github.com/bazelbuild/remote-apis-sdks/go/digest"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
)

// ActionCache implements the RE ActionCache interface, storing fixed results.
type ActionCache struct {
	mu      sync.RWMutex
	results map[digest.Digest]*repb.ActionResult
	reads   map[digest.Digest]int
	writes  map[digest.Digest]int
}

// NewActionCache returns a new empty ActionCache.
func NewActionCache() *ActionCache {
	c := &ActionCache{}
	c.Clear()
	return c
}

// Clear removes all results from the cache.
func (c *ActionCache) Clear() {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.results = make(map[digest.Digest]*repb.ActionResult)
	c.reads = make(map[digest.Digest]int)
	c.writes = make(map[digest.Digest]int)
}

// PutAction sets a fake result for a given action, and returns the action digest.
func (c *ActionCache) PutAction(ac *repb.Action, res *repb.ActionResult) digest.Digest {
	d := digest.TestNewFromMessage(ac)
	c.Put(d, res)
	return d
}

// Put sets a fake result for a given action digest.
func (c *ActionCache) Put(d digest.Digest, res *repb.ActionResult) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.results[d] = res
}

// Get returns a previously saved fake result for the given action digest.
func (c *ActionCache) Get(d digest.Digest) *repb.ActionResult {
	c.mu.RLock()
	defer c.mu.RUnlock()
	res, ok := c.results[d]
	if !ok {
		return nil
	}
	return res
}

// GetReads returns the number of times GetActionResult was called for a given action digest.
// These include both successful and unsuccessful reads.
func (c *ActionCache) GetReads(d digest.Digest) int {
	return c.reads[d]
}

// GetWrites returns the number of times UpdateActionResult was called for a given action digest.
func (c *ActionCache) GetWrites(d digest.Digest) int {
	return c.writes[d]
}

// GetActionResult returns a stored result, if it was found.
func (c *ActionCache) GetActionResult(ctx context.Context, req *repb.GetActionResultRequest) (res *repb.ActionResult, err error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	dg, err := digest.NewFromProto(req.ActionDigest)
	if err != nil {
		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid digest received: %v", req.ActionDigest))
	}
	c.reads[dg]++
	if res, ok := c.results[dg]; ok {
		return res, nil
	}
	return nil, status.Error(codes.NotFound, "")
}

// UpdateActionResult sets/updates a given result.
func (c *ActionCache) UpdateActionResult(ctx context.Context, req *repb.UpdateActionResultRequest) (res *repb.ActionResult, err error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	dg, err := digest.NewFromProto(req.ActionDigest)
	if err != nil {
		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid digest received: %v", req.ActionDigest))
	}
	if req.ActionResult != nil {
		return nil, status.Error(codes.InvalidArgument, "no action result received")
	}
	c.results[dg] = req.ActionResult
	c.writes[dg]++
	return req.ActionResult, nil
}
<|endoftext|>


<commit_before>package state

import ()

type actionDoc struct {
	Id      string `bson:"_id"`
	Name    string
	Payload map[string]interface{}
}

// Action represents an instruction to do some "action" and is expected to match
// an action definition in a charm.
type Action struct {
	st  *State
	doc actionDoc
}

func newAction(st *State, adoc actionDoc) *Action {
	return &Action{
		st:  st,
		doc: adoc,
	}
}

// Name returns the name of the Action
func (a *Action) Name() string {
	return a.doc.Name
}

// Id returns the id of the Action
func (a *Action) Id() string {
	return a.doc.Id
}

// Payload will contain a structure representing arguments or parameters to
// an action, and is expected to be validated by the Unit using the Charm
// definition of the Action
func (a *Action) Payload() map[string]interface{} {
	return a.doc.Payload
}
<commit_msg>Comment Action members 'Name', and 'Payload', pending clarification on these<commit_after>package state

import ()

type actionDoc struct {
	Id string `bson:"_id"`
	//Name    string
	//Payload map[string]interface{}
}

// Action represents an instruction to do some "action" and is expected to match
// an action definition in a charm.
type Action struct {
	st  *State
	doc actionDoc
}

func newAction(st *State, adoc actionDoc) *Action {
	return &Action{
		st:  st,
		doc: adoc,
	}
}

// Name returns the name of the Action
//func (a *Action) Name() string {
//	return a.doc.Name
//}

// Id returns the id of the Action
func (a *Action) Id() string {
	return a.doc.Id
}

// Payload will contain a structure representing arguments or parameters to
// an action, and is expected to be validated by the Unit using the Charm
// definition of the Action
//func (a *Action) Payload() map[string]interface{} {
//	return a.doc.Payload
//}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package servenv

import (
	"flag"
	"fmt"
	"math"
	"net"
	"time"

	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"

	"vitess.io/vitess/go/trace"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/keepalive"

	"context"

	"vitess.io/vitess/go/vt/grpccommon"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/vttls"
)

// This file handles gRPC server, on its own port.
// Clients register servers, based on service map:
//
// servenv.RegisterGRPCFlags()
// servenv.OnRun(func() {
//   if servenv.GRPCCheckServiceMap("XXX") {
//     pb.RegisterXXX(servenv.GRPCServer, XXX)
//   }
// }
//
// Note servenv.GRPCServer can only be used in servenv.OnRun,
// and not before, as it is initialized right before calling OnRun.
var (
	// GRPCPort is the port to listen on for gRPC. If not set or zero, don't listen.
	GRPCPort = flag.Int("grpc_port", 0, "Port to listen on for gRPC calls")

	// GRPCCert is the cert to use if TLS is enabled
	GRPCCert = flag.String("grpc_cert", "", "server certificate to use for gRPC connections, requires grpc_key, enables TLS")

	// GRPCKey is the key to use if TLS is enabled
	GRPCKey = flag.String("grpc_key", "", "server private key to use for gRPC connections, requires grpc_cert, enables TLS")

	// GRPCCA is the CA to use if TLS is enabled
	GRPCCA = flag.String("grpc_ca", "", "server CA to use for gRPC connections, requires TLS, and enforces client certificate check")

	// GRPCServerCA if specified will combine server cert and server CA
	GRPCServerCA = flag.String("grpc_server_ca", "", "path to server CA in PEM format, which will be combine with server cert, return full certificate chain to clients")

	// GRPCAuth which auth plugin to use (at the moment now only static is supported)
	GRPCAuth = flag.String("grpc_auth_mode", "", "Which auth plugin implementation to use (eg: static)")

	// GRPCServer is the global server to serve gRPC.
	GRPCServer *grpc.Server

	// GRPCMaxConnectionAge is the maximum age of a client connection, before GoAway is sent.
	// This is useful for L4 loadbalancing to ensure rebalancing after scaling.
	GRPCMaxConnectionAge = flag.Duration("grpc_max_connection_age", time.Duration(math.MaxInt64), "Maximum age of a client connection before GoAway is sent.")

	// GRPCMaxConnectionAgeGrace is an additional grace period after GRPCMaxConnectionAge, after which
	// connections are forcibly closed.
	GRPCMaxConnectionAgeGrace = flag.Duration("grpc_max_connection_age_grace", time.Duration(math.MaxInt64), "Additional grace period after grpc_max_connection_age, after which connections are forcibly closed.")

	// GRPCInitialConnWindowSize ServerOption that sets window size for a connection.
	// The lower bound for window size is 64K and any value smaller than that will be ignored.
	GRPCInitialConnWindowSize = flag.Int("grpc_server_initial_conn_window_size", 0, "gRPC server initial connection window size")

	// GRPCInitialWindowSize ServerOption that sets window size for stream.
	// The lower bound for window size is 64K and any value smaller than that will be ignored.
	GRPCInitialWindowSize = flag.Int("grpc_server_initial_window_size", 0, "gRPC server initial window size")

	// EnforcementPolicy MinTime that sets the keepalive enforcement policy on the server.
	// This is the minimum amount of time a client should wait before sending a keepalive ping.
	GRPCKeepAliveEnforcementPolicyMinTime = flag.Duration("grpc_server_keepalive_enforcement_policy_min_time", 10*time.Second, "gRPC server minimum keepalive time")

	// EnforcementPolicy PermitWithoutStream - If true, server allows keepalive pings
	// even when there are no active streams (RPCs). If false, and client sends ping when
	// there are no active streams, server will send GOAWAY and close the connection.
	GRPCKeepAliveEnforcementPolicyPermitWithoutStream = flag.Bool("grpc_server_keepalive_enforcement_policy_permit_without_stream", false, "gRPC server permit client keepalive pings even when there are no active streams (RPCs)")

	authPlugin Authenticator
)

// isGRPCEnabled returns true if gRPC server is set
func isGRPCEnabled() bool {
	if GRPCPort != nil && *GRPCPort != 0 {
		return true
	}

	if SocketFile != nil && *SocketFile != "" {
		return true
	}

	return false
}

// createGRPCServer create the gRPC server we will be using.
// It has to be called after flags are parsed, but before
// services register themselves.
func createGRPCServer() {
	// skip if not registered
	if !isGRPCEnabled() {
		log.Infof("Skipping gRPC server creation")
		return
	}

	grpccommon.EnableTracingOpt()

	var opts []grpc.ServerOption
	if GRPCPort != nil && *GRPCCert != "" && *GRPCKey != "" {
		config, err := vttls.ServerConfig(*GRPCCert, *GRPCKey, *GRPCCA, *GRPCServerCA)
		if err != nil {
			log.Exitf("Failed to log gRPC cert/key/ca: %v", err)
		}

		// create the creds server options
		creds := credentials.NewTLS(config)
		opts = []grpc.ServerOption{grpc.Creds(creds)}
	}
	// Override the default max message size for both send and receive
	// (which is 4 MiB in gRPC 1.0.0).
	// Large messages can occur when users try to insert or fetch very big
	// rows. If they hit the limit, they'll see the following error:
	// grpc: received message length XXXXXXX exceeding the max size 4194304
	// Note: For gRPC 1.0.0 it's sufficient to set the limit on the server only
	// because it's not enforced on the client side.
	log.Infof("Setting grpc max message size to %d", *grpccommon.MaxMessageSize)
	opts = append(opts, grpc.MaxRecvMsgSize(*grpccommon.MaxMessageSize))
	opts = append(opts, grpc.MaxSendMsgSize(*grpccommon.MaxMessageSize))

	if *GRPCInitialConnWindowSize != 0 {
		log.Infof("Setting grpc server initial conn window size to %d", int32(*GRPCInitialConnWindowSize))
		opts = append(opts, grpc.InitialConnWindowSize(int32(*GRPCInitialConnWindowSize)))
	}

	if *GRPCInitialWindowSize != 0 {
		log.Infof("Setting grpc server initial window size to %d", int32(*GRPCInitialWindowSize))
		opts = append(opts, grpc.InitialWindowSize(int32(*GRPCInitialWindowSize)))
	}

	ep := keepalive.EnforcementPolicy{
		MinTime:             *GRPCKeepAliveEnforcementPolicyMinTime,
		PermitWithoutStream: *GRPCKeepAliveEnforcementPolicyPermitWithoutStream,
	}
	opts = append(opts, grpc.KeepaliveEnforcementPolicy(ep))

	if GRPCMaxConnectionAge != nil {
		ka := keepalive.ServerParameters{
			MaxConnectionAge: *GRPCMaxConnectionAge,
		}
		if GRPCMaxConnectionAgeGrace != nil {
			ka.MaxConnectionAgeGrace = *GRPCMaxConnectionAgeGrace
		}
		opts = append(opts, grpc.KeepaliveParams(ka))
	}

	opts = append(opts, interceptors()...)

	GRPCServer = grpc.NewServer(opts...)
}

// We can only set a ServerInterceptor once, so we chain multiple interceptors into one
func interceptors() []grpc.ServerOption {
	interceptors := &serverInterceptorBuilder{}

	if *GRPCAuth != "" {
		log.Infof("enabling auth plugin %v", *GRPCAuth)
		pluginInitializer := GetAuthenticator(*GRPCAuth)
		authPluginImpl, err := pluginInitializer()
		if err != nil {
			log.Fatalf("Failed to load auth plugin: %v", err)
		}
		authPlugin = authPluginImpl
		interceptors.Add(authenticatingStreamInterceptor, authenticatingUnaryInterceptor)
	}

	if *grpccommon.EnableGRPCPrometheus {
		interceptors.Add(grpc_prometheus.StreamServerInterceptor, grpc_prometheus.UnaryServerInterceptor)
	}

	trace.AddGrpcServerOptions(interceptors.Add)

	return interceptors.Build()
}

func serveGRPC() {
	if *grpccommon.EnableGRPCPrometheus {
		grpc_prometheus.Register(GRPCServer)
		grpc_prometheus.EnableHandlingTimeHistogram()
	}
	// skip if not registered
	if GRPCPort == nil || *GRPCPort == 0 {
		return
	}

	// listen on the port
	log.Infof("Listening for gRPC calls on port %v", *GRPCPort)
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", *GRPCPort))
	if err != nil {
		log.Exitf("Cannot listen on port %v for gRPC: %v", *GRPCPort, err)
	}

	// and serve on it
	// NOTE: Before we call Serve(), all services must have registered themselves
	//       with "GRPCServer". This is the case because go/vt/servenv/run.go
	//       runs all OnRun() hooks after createGRPCServer() and before
	//       serveGRPC(). If this was not the case, the binary would crash with
	//       the error "grpc: Server.RegisterService after Server.Serve".
	go func() {
		err := GRPCServer.Serve(listener)
		if err != nil {
			log.Exitf("Failed to start grpc server: %v", err)
		}
	}()

	OnTermSync(func() {
		log.Info("Initiated graceful stop of gRPC server")
		GRPCServer.GracefulStop()
		log.Info("gRPC server stopped")
	})
}

// GRPCCheckServiceMap returns if we should register a gRPC service
// (and also logs how to enable / disable it)
func GRPCCheckServiceMap(name string) bool {
	// Silently fail individual services if gRPC is not enabled in
	// the first place (either on a grpc port or on the socket file)
	if !isGRPCEnabled() {
		return false
	}

	// then check ServiceMap
	return CheckServiceMap("grpc", name)
}

func authenticatingStreamInterceptor(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
	newCtx, err := authPlugin.Authenticate(stream.Context(), info.FullMethod)

	if err != nil {
		return err
	}

	wrapped := WrapServerStream(stream)
	wrapped.WrappedContext = newCtx
	return handler(srv, wrapped)
}

func authenticatingUnaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
	newCtx, err := authPlugin.Authenticate(ctx, info.FullMethod)
	if err != nil {
		return nil, err
	}

	return handler(newCtx, req)
}

// WrappedServerStream is based on the service stream wrapper from: https://github.com/grpc-ecosystem/go-grpc-middleware
type WrappedServerStream struct {
	grpc.ServerStream
	WrappedContext context.Context
}

// Context returns the wrapper's WrappedContext, overwriting the nested grpc.ServerStream.Context()
func (w *WrappedServerStream) Context() context.Context {
	return w.WrappedContext
}

// WrapServerStream returns a ServerStream that has the ability to overwrite context.
func WrapServerStream(stream grpc.ServerStream) *WrappedServerStream {
	if existing, ok := stream.(*WrappedServerStream); ok {
		return existing
	}
	return &WrappedServerStream{ServerStream: stream, WrappedContext: stream.Context()}
}

// serverInterceptorBuilder chains together multiple ServerInterceptors
type serverInterceptorBuilder struct {
	streamInterceptors []grpc.StreamServerInterceptor
	unaryInterceptors  []grpc.UnaryServerInterceptor
}

// Add adds interceptors to the builder
func (collector *serverInterceptorBuilder) Add(s grpc.StreamServerInterceptor, u grpc.UnaryServerInterceptor) {
	collector.streamInterceptors = append(collector.streamInterceptors, s)
	collector.unaryInterceptors = append(collector.unaryInterceptors, u)
}

// AddUnary adds a single unary interceptor to the builder
func (collector *serverInterceptorBuilder) AddUnary(u grpc.UnaryServerInterceptor) {
	collector.unaryInterceptors = append(collector.unaryInterceptors, u)
}

// Build returns DialOptions to add to the grpc.Dial call
func (collector *serverInterceptorBuilder) Build() []grpc.ServerOption {
	log.Infof("Building interceptors with %d unary interceptors and %d stream interceptors", len(collector.unaryInterceptors), len(collector.streamInterceptors))
	switch len(collector.unaryInterceptors) + len(collector.streamInterceptors) {
	case 0:
		return []grpc.ServerOption{}
	default:
		return []grpc.ServerOption{
			grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(collector.unaryInterceptors...)),
			grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(collector.streamInterceptors...)),
		}
	}
}
<commit_msg>support grpc reflection on APIs<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package servenv

import (
	"flag"
	"fmt"
	"math"
	"net"
	"time"

	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"

	"vitess.io/vitess/go/trace"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/keepalive"
	"google.golang.org/grpc/reflection"

	"context"

	"vitess.io/vitess/go/vt/grpccommon"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/vttls"
)

// This file handles gRPC server, on its own port.
// Clients register servers, based on service map:
//
// servenv.RegisterGRPCFlags()
// servenv.OnRun(func() {
//   if servenv.GRPCCheckServiceMap("XXX") {
//     pb.RegisterXXX(servenv.GRPCServer, XXX)
//   }
// }
//
// Note servenv.GRPCServer can only be used in servenv.OnRun,
// and not before, as it is initialized right before calling OnRun.
var (
	// GRPCPort is the port to listen on for gRPC. If not set or zero, don't listen.
	GRPCPort = flag.Int("grpc_port", 0, "Port to listen on for gRPC calls")

	// GRPCCert is the cert to use if TLS is enabled
	GRPCCert = flag.String("grpc_cert", "", "server certificate to use for gRPC connections, requires grpc_key, enables TLS")

	// GRPCKey is the key to use if TLS is enabled
	GRPCKey = flag.String("grpc_key", "", "server private key to use for gRPC connections, requires grpc_cert, enables TLS")

	// GRPCCA is the CA to use if TLS is enabled
	GRPCCA = flag.String("grpc_ca", "", "server CA to use for gRPC connections, requires TLS, and enforces client certificate check")

	// GRPCServerCA if specified will combine server cert and server CA
	GRPCServerCA = flag.String("grpc_server_ca", "", "path to server CA in PEM format, which will be combine with server cert, return full certificate chain to clients")

	// GRPCAuth which auth plugin to use (at the moment now only static is supported)
	GRPCAuth = flag.String("grpc_auth_mode", "", "Which auth plugin implementation to use (eg: static)")

	// GRPCServer is the global server to serve gRPC.
	GRPCServer *grpc.Server

	// GRPCMaxConnectionAge is the maximum age of a client connection, before GoAway is sent.
	// This is useful for L4 loadbalancing to ensure rebalancing after scaling.
	GRPCMaxConnectionAge = flag.Duration("grpc_max_connection_age", time.Duration(math.MaxInt64), "Maximum age of a client connection before GoAway is sent.")

	// GRPCMaxConnectionAgeGrace is an additional grace period after GRPCMaxConnectionAge, after which
	// connections are forcibly closed.
	GRPCMaxConnectionAgeGrace = flag.Duration("grpc_max_connection_age_grace", time.Duration(math.MaxInt64), "Additional grace period after grpc_max_connection_age, after which connections are forcibly closed.")

	// GRPCInitialConnWindowSize ServerOption that sets window size for a connection.
	// The lower bound for window size is 64K and any value smaller than that will be ignored.
	GRPCInitialConnWindowSize = flag.Int("grpc_server_initial_conn_window_size", 0, "gRPC server initial connection window size")

	// GRPCInitialWindowSize ServerOption that sets window size for stream.
	// The lower bound for window size is 64K and any value smaller than that will be ignored.
	GRPCInitialWindowSize = flag.Int("grpc_server_initial_window_size", 0, "gRPC server initial window size")

	// EnforcementPolicy MinTime that sets the keepalive enforcement policy on the server.
	// This is the minimum amount of time a client should wait before sending a keepalive ping.
	GRPCKeepAliveEnforcementPolicyMinTime = flag.Duration("grpc_server_keepalive_enforcement_policy_min_time", 10*time.Second, "gRPC server minimum keepalive time")

	// EnforcementPolicy PermitWithoutStream - If true, server allows keepalive pings
	// even when there are no active streams (RPCs). If false, and client sends ping when
	// there are no active streams, server will send GOAWAY and close the connection.
	GRPCKeepAliveEnforcementPolicyPermitWithoutStream = flag.Bool("grpc_server_keepalive_enforcement_policy_permit_without_stream", false, "gRPC server permit client keepalive pings even when there are no active streams (RPCs)")

	authPlugin Authenticator
)

// isGRPCEnabled returns true if gRPC server is set
func isGRPCEnabled() bool {
	if GRPCPort != nil && *GRPCPort != 0 {
		return true
	}

	if SocketFile != nil && *SocketFile != "" {
		return true
	}

	return false
}

// createGRPCServer create the gRPC server we will be using.
// It has to be called after flags are parsed, but before
// services register themselves.
func createGRPCServer() {
	// skip if not registered
	if !isGRPCEnabled() {
		log.Infof("Skipping gRPC server creation")
		return
	}

	grpccommon.EnableTracingOpt()

	var opts []grpc.ServerOption
	if GRPCPort != nil && *GRPCCert != "" && *GRPCKey != "" {
		config, err := vttls.ServerConfig(*GRPCCert, *GRPCKey, *GRPCCA, *GRPCServerCA)
		if err != nil {
			log.Exitf("Failed to log gRPC cert/key/ca: %v", err)
		}

		// create the creds server options
		creds := credentials.NewTLS(config)
		opts = []grpc.ServerOption{grpc.Creds(creds)}
	}
	// Override the default max message size for both send and receive
	// (which is 4 MiB in gRPC 1.0.0).
	// Large messages can occur when users try to insert or fetch very big
	// rows. If they hit the limit, they'll see the following error:
	// grpc: received message length XXXXXXX exceeding the max size 4194304
	// Note: For gRPC 1.0.0 it's sufficient to set the limit on the server only
	// because it's not enforced on the client side.
	log.Infof("Setting grpc max message size to %d", *grpccommon.MaxMessageSize)
	opts = append(opts, grpc.MaxRecvMsgSize(*grpccommon.MaxMessageSize))
	opts = append(opts, grpc.MaxSendMsgSize(*grpccommon.MaxMessageSize))

	if *GRPCInitialConnWindowSize != 0 {
		log.Infof("Setting grpc server initial conn window size to %d", int32(*GRPCInitialConnWindowSize))
		opts = append(opts, grpc.InitialConnWindowSize(int32(*GRPCInitialConnWindowSize)))
	}

	if *GRPCInitialWindowSize != 0 {
		log.Infof("Setting grpc server initial window size to %d", int32(*GRPCInitialWindowSize))
		opts = append(opts, grpc.InitialWindowSize(int32(*GRPCInitialWindowSize)))
	}

	ep := keepalive.EnforcementPolicy{
		MinTime:             *GRPCKeepAliveEnforcementPolicyMinTime,
		PermitWithoutStream: *GRPCKeepAliveEnforcementPolicyPermitWithoutStream,
	}
	opts = append(opts, grpc.KeepaliveEnforcementPolicy(ep))

	if GRPCMaxConnectionAge != nil {
		ka := keepalive.ServerParameters{
			MaxConnectionAge: *GRPCMaxConnectionAge,
		}
		if GRPCMaxConnectionAgeGrace != nil {
			ka.MaxConnectionAgeGrace = *GRPCMaxConnectionAgeGrace
		}
		opts = append(opts, grpc.KeepaliveParams(ka))
	}

	opts = append(opts, interceptors()...)

	GRPCServer = grpc.NewServer(opts...)
}

// We can only set a ServerInterceptor once, so we chain multiple interceptors into one
func interceptors() []grpc.ServerOption {
	interceptors := &serverInterceptorBuilder{}

	if *GRPCAuth != "" {
		log.Infof("enabling auth plugin %v", *GRPCAuth)
		pluginInitializer := GetAuthenticator(*GRPCAuth)
		authPluginImpl, err := pluginInitializer()
		if err != nil {
			log.Fatalf("Failed to load auth plugin: %v", err)
		}
		authPlugin = authPluginImpl
		interceptors.Add(authenticatingStreamInterceptor, authenticatingUnaryInterceptor)
	}

	if *grpccommon.EnableGRPCPrometheus {
		interceptors.Add(grpc_prometheus.StreamServerInterceptor, grpc_prometheus.UnaryServerInterceptor)
	}

	trace.AddGrpcServerOptions(interceptors.Add)

	return interceptors.Build()
}

func serveGRPC() {
	if *grpccommon.EnableGRPCPrometheus {
		grpc_prometheus.Register(GRPCServer)
		grpc_prometheus.EnableHandlingTimeHistogram()
	}
	// skip if not registered
	if GRPCPort == nil || *GRPCPort == 0 {
		return
	}

	// register reflection to support list calls :)
	reflection.Register(GRPCServer)

	// listen on the port
	log.Infof("Listening for gRPC calls on port %v", *GRPCPort)
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", *GRPCPort))
	if err != nil {
		log.Exitf("Cannot listen on port %v for gRPC: %v", *GRPCPort, err)
	}

	// and serve on it
	// NOTE: Before we call Serve(), all services must have registered themselves
	//       with "GRPCServer". This is the case because go/vt/servenv/run.go
	//       runs all OnRun() hooks after createGRPCServer() and before
	//       serveGRPC(). If this was not the case, the binary would crash with
	//       the error "grpc: Server.RegisterService after Server.Serve".
	go func() {
		err := GRPCServer.Serve(listener)
		if err != nil {
			log.Exitf("Failed to start grpc server: %v", err)
		}
	}()

	OnTermSync(func() {
		log.Info("Initiated graceful stop of gRPC server")
		GRPCServer.GracefulStop()
		log.Info("gRPC server stopped")
	})
}

// GRPCCheckServiceMap returns if we should register a gRPC service
// (and also logs how to enable / disable it)
func GRPCCheckServiceMap(name string) bool {
	// Silently fail individual services if gRPC is not enabled in
	// the first place (either on a grpc port or on the socket file)
	if !isGRPCEnabled() {
		return false
	}

	// then check ServiceMap
	return CheckServiceMap("grpc", name)
}

func authenticatingStreamInterceptor(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
	newCtx, err := authPlugin.Authenticate(stream.Context(), info.FullMethod)

	if err != nil {
		return err
	}

	wrapped := WrapServerStream(stream)
	wrapped.WrappedContext = newCtx
	return handler(srv, wrapped)
}

func authenticatingUnaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
	newCtx, err := authPlugin.Authenticate(ctx, info.FullMethod)
	if err != nil {
		return nil, err
	}

	return handler(newCtx, req)
}

// WrappedServerStream is based on the service stream wrapper from: https://github.com/grpc-ecosystem/go-grpc-middleware
type WrappedServerStream struct {
	grpc.ServerStream
	WrappedContext context.Context
}

// Context returns the wrapper's WrappedContext, overwriting the nested grpc.ServerStream.Context()
func (w *WrappedServerStream) Context() context.Context {
	return w.WrappedContext
}

// WrapServerStream returns a ServerStream that has the ability to overwrite context.
func WrapServerStream(stream grpc.ServerStream) *WrappedServerStream {
	if existing, ok := stream.(*WrappedServerStream); ok {
		return existing
	}
	return &WrappedServerStream{ServerStream: stream, WrappedContext: stream.Context()}
}

// serverInterceptorBuilder chains together multiple ServerInterceptors
type serverInterceptorBuilder struct {
	streamInterceptors []grpc.StreamServerInterceptor
	unaryInterceptors  []grpc.UnaryServerInterceptor
}

// Add adds interceptors to the builder
func (collector *serverInterceptorBuilder) Add(s grpc.StreamServerInterceptor, u grpc.UnaryServerInterceptor) {
	collector.streamInterceptors = append(collector.streamInterceptors, s)
	collector.unaryInterceptors = append(collector.unaryInterceptors, u)
}

// AddUnary adds a single unary interceptor to the builder
func (collector *serverInterceptorBuilder) AddUnary(u grpc.UnaryServerInterceptor) {
	collector.unaryInterceptors = append(collector.unaryInterceptors, u)
}

// Build returns DialOptions to add to the grpc.Dial call
func (collector *serverInterceptorBuilder) Build() []grpc.ServerOption {
	log.Infof("Building interceptors with %d unary interceptors and %d stream interceptors", len(collector.unaryInterceptors), len(collector.streamInterceptors))
	switch len(collector.unaryInterceptors) + len(collector.streamInterceptors) {
	case 0:
		return []grpc.ServerOption{}
	default:
		return []grpc.ServerOption{
			grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(collector.unaryInterceptors...)),
			grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(collector.streamInterceptors...)),
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"flag"
	"log"
	"os"
	"runtime"

	"github.com/BurntSushi/bcbgo/apps/hhsuite"
	"github.com/BurntSushi/bcbgo/hhfrag"
)

var (
	flagBlits = false
	flagSeqDB = "nr20"
	flagPdbDB = "pdb-select25"
	flagCpu   = runtime.NumCPU()

	seqDB hhsuite.Database
	pdbDB hhfrag.PDBDatabase
)

func init() {
	log.SetFlags(0)

	flag.BoolVar(&flagBlits, "blits", flagBlits,
		"When set, hhblits will be used in lieu of hhsearch.")
	flag.StringVar(&flagSeqDB, "seqdb", flagSeqDB,
		"The sequence database used to generate the query HHM.")
	flag.StringVar(&flagPdbDB, "pdbdb", flagPdbDB,
		"The PDB/HHM database used to assign fragments.")
	flag.IntVar(&flagCpu, "cpu", flagCpu,
		"The max number of CPUs to use.")

	flag.Usage = usage
	flag.Parse()

	seqDB = hhsuite.Database(flagSeqDB)
	pdbDB = hhfrag.PDBDatabase(flagPdbDB)

	runtime.GOMAXPROCS(flagCpu)
}

func usage() {
	log.Printf("Usage: hhfrag-map [flags] target-fasta out-fmap\n")
	flag.PrintDefaults()
}

func main() {
	if flag.NArg() != 2 {
		usage()
	}

	fasInp := flag.Arg(0)
	fmapOut := flag.Arg(1)

	conf := hhfrag.DefaultConfig
	conf.Blits = flagBlits
	fmap, err := conf.MapFromFasta(pdbDB, seqDB, fasInp)
	assert(err)

	out, err := os.Create(fmapOut)
	assert(err)
	w := gob.NewEncoder(out)
	err = w.Encode(fmap)
	assert(err)
}

func assert(err error) {
	if err != nil {
		log.Fatalln(err)
	}
}
<commit_msg>Allow hhfrag window parameters to be specified on the command line.<commit_after>package main

import (
	"encoding/gob"
	"flag"
	"log"
	"os"
	"runtime"

	"github.com/BurntSushi/bcbgo/apps/hhsuite"
	"github.com/BurntSushi/bcbgo/hhfrag"
)

var (
	hhfragConf = hhfrag.DefaultConfig
	flagSeqDB  = "nr20"
	flagPdbDB  = "pdb-select25"
	flagCpu    = runtime.NumCPU()

	seqDB hhsuite.Database
	pdbDB hhfrag.PDBDatabase
)

func init() {
	log.SetFlags(0)

	flag.BoolVar(&hhfragConf.Blits, "blits", hhfragConf.Blits,
		"When set, hhblits will be used in lieu of hhsearch.")
	flag.IntVar(&hhfragConf.WindowMin, "win-min", hhfragConf.WindowMin,
		"The minimum HMM window size for HHfrag.")
	flag.IntVar(&hhfragConf.WindowMax, "win-max", hhfragConf.WindowMax,
		"The maximum HMM window size for HHfrag.")
	flag.IntVar(&hhfragConf.WindowIncrement, "win-inc",
		hhfragConf.WindowIncrement,
		"The sliding window increment for HHfrag.")
	flag.StringVar(&flagSeqDB, "seqdb", flagSeqDB,
		"The sequence database used to generate the query HHM.")
	flag.StringVar(&flagPdbDB, "pdbdb", flagPdbDB,
		"The PDB/HHM database used to assign fragments.")
	flag.IntVar(&flagCpu, "cpu", flagCpu,
		"The max number of CPUs to use.")

	flag.Usage = usage
	flag.Parse()

	seqDB = hhsuite.Database(flagSeqDB)
	pdbDB = hhfrag.PDBDatabase(flagPdbDB)

	runtime.GOMAXPROCS(flagCpu)
}

func usage() {
	log.Printf("Usage: hhfrag-map [flags] target-fasta out-fmap\n")
	flag.PrintDefaults()
}

func main() {
	if flag.NArg() != 2 {
		usage()
	}

	fasInp := flag.Arg(0)
	fmapOut := flag.Arg(1)

	fmap, err := hhfragConf.MapFromFasta(pdbDB, seqDB, fasInp)
	assert(err)

	out, err := os.Create(fmapOut)
	assert(err)
	w := gob.NewEncoder(out)
	err = w.Encode(fmap)
	assert(err)
}

func assert(err error) {
	if err != nil {
		log.Fatalln(err)
	}
}
<|endoftext|>


<commit_before>package zeusmaster

import (
	"io"
	"os"
	"os/exec"
	"path"
	"runtime"
	"strings"

	slog "github.com/burke/zeus/go/shinylog"
)

var filesToWatch chan string
var filesChanged chan string

var watcherIn io.WriteCloser
var watcherOut io.ReadCloser
var watcherErr io.ReadCloser

var allWatchedFiles map[string]bool

func StartFileMonitor(tree *ProcessTree, quit chan bool) {
	// this is obscenely large, just because as long as we start
	// watching the files eventually, it's more of a priority to
	// get the slaves booted as quickly as possible.
	filesToWatch = make(chan string, 8192)
	filesChanged = make(chan string, 256)
	allWatchedFiles = make(map[string]bool)

	startWrapper()

	for {
		select {
		case <-quit:
			quit <- true
			return
		case path := <-filesToWatch:
			go handleLoadedFileNotification(path)
		case path := <-filesChanged:
			go handleChangedFileNotification(tree, path)
		}
	}
}

func executablePath() string {
	switch runtime.GOOS {
	case "darwin":
		return path.Join(path.Dir(os.Args[0]), "fsevents-wrapper")
	case "linux":
		gemRoot := path.Dir(path.Dir(os.Args[0]))
		return path.Join(gemRoot, "ext/inotify-wrapper/inotify-wrapper")
	}
	Error("Unsupported OS")
	return ""
}

func startWrapper() {
	cmd := exec.Command(executablePath())
	var err error
	if watcherIn, err = cmd.StdinPipe(); err != nil {
		Error(err.Error())
	}
	if watcherOut, err = cmd.StdoutPipe(); err != nil {
		Error(err.Error())
	}
	if watcherErr, err = cmd.StderrPipe(); err != nil {
		Error(err.Error())
	}

	cmd.Start()

	go func() {
		buf := make([]byte, 2048)
		for {
			n, err := watcherOut.Read(buf)
			if err != nil {
				errorFailedReadFromWatcher(err)
			}
			message := strings.TrimSpace(string(buf[:n]))
			files := strings.Split(message, "\n")
			for _, file := range files {
				filesChanged <- file
			}
		}
	}()

	go func() {
		err := cmd.Wait()
		ErrorFileMonitorWrapperCrashed(err)
	}()
}

func AddFile(file string) {
	filesToWatch <- file
}

func handleLoadedFileNotification(file string) {
	// a slave loaded a file. It's up to us here to make sure this file is watched.
	if !allWatchedFiles[file] {
		allWatchedFiles[file] = true
		startWatchingFile(file)
	}
}

func handleChangedFileNotification(tree *ProcessTree, file string) {
	slog.Yellow("Dependency change at " + file)
	tree.RestartNodesWithFeature(file)
}

func startWatchingFile(file string) {
	_, err := watcherIn.Write([]byte(file + "\n"))
	if err != nil {
		slog.Error(err)
	}
}
<commit_msg>Forcibly kill filesystem watcher on shudown<commit_after>package zeusmaster

import (
	"io"
	"os"
	"os/exec"
	"path"
	"runtime"
	"strings"

	slog "github.com/burke/zeus/go/shinylog"
)

var filesToWatch chan string
var filesChanged chan string

var watcherIn io.WriteCloser
var watcherOut io.ReadCloser
var watcherErr io.ReadCloser

var allWatchedFiles map[string]bool

func StartFileMonitor(tree *ProcessTree, quit chan bool) {
	// this is obscenely large, just because as long as we start
	// watching the files eventually, it's more of a priority to
	// get the slaves booted as quickly as possible.
	filesToWatch = make(chan string, 8192)
	filesChanged = make(chan string, 256)
	allWatchedFiles = make(map[string]bool)

	cmd := startWrapper()

	for {
		select {
		case <-quit:
			cmd.Process.Kill()
			quit <- true
			return
		case path := <-filesToWatch:
			go handleLoadedFileNotification(path)
		case path := <-filesChanged:
			go handleChangedFileNotification(tree, path)
		}
	}
}

func executablePath() string {
	switch runtime.GOOS {
	case "darwin":
		return path.Join(path.Dir(os.Args[0]), "fsevents-wrapper")
	case "linux":
		gemRoot := path.Dir(path.Dir(os.Args[0]))
		return path.Join(gemRoot, "ext/inotify-wrapper/inotify-wrapper")
	}
	Error("Unsupported OS")
	return ""
}

func startWrapper() *exec.Cmd {
	cmd := exec.Command(executablePath())
	var err error
	if watcherIn, err = cmd.StdinPipe(); err != nil {
		Error(err.Error())
	}
	if watcherOut, err = cmd.StdoutPipe(); err != nil {
		Error(err.Error())
	}
	if watcherErr, err = cmd.StderrPipe(); err != nil {
		Error(err.Error())
	}

	cmd.Start()

	go func() {
		buf := make([]byte, 2048)
		for {
			n, err := watcherOut.Read(buf)
			if err != nil {
				errorFailedReadFromWatcher(err)
			}
			message := strings.TrimSpace(string(buf[:n]))
			files := strings.Split(message, "\n")
			for _, file := range files {
				filesChanged <- file
			}
		}
	}()

	go func() {
		err := cmd.Wait()
		ErrorFileMonitorWrapperCrashed(err)
	}()

	return cmd
}

func AddFile(file string) {
	filesToWatch <- file
}

func handleLoadedFileNotification(file string) {
	// a slave loaded a file. It's up to us here to make sure this file is watched.
	if !allWatchedFiles[file] {
		allWatchedFiles[file] = true
		startWatchingFile(file)
	}
}

func handleChangedFileNotification(tree *ProcessTree, file string) {
	slog.Yellow("Dependency change at " + file)
	tree.RestartNodesWithFeature(file)
}

func startWatchingFile(file string) {
	_, err := watcherIn.Write([]byte(file + "\n"))
	if err != nil {
		slog.Error(err)
	}
}
<|endoftext|>


<commit_before>package highcharts

import (
    "testing"
    "net/http"
    "io/ioutil"
    "time"
    "code.google.com/p/go.net/websocket"
    "strconv"
)

func TestNewChart(t *testing.T){
    options := map[string]interface{}{
        "series":  []interface{}{
            map[string]interface{}{
                "name": "toto",
                "data": []int{1, 2, 3},
            },
        },
        "chart": map[string]interface{}{
            "type": "line",
        },
    }
    NewChart("/chart/", options)

    resp, err := http.Get("http://localhost:8080/chart/")
    if err != nil{
        t.Errorf("Error while loading chart page", err)
    }
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil{
        t.Errorf("Error while reading chart page", err)
    }
    // fmt.Println(string(body))

    resp, err = http.Get("http://localhost:8080/chart/data/")
    if err != nil{
        t.Errorf("Error while loading chart data page", err)
    }
    defer resp.Body.Close()
    body, err = ioutil.ReadAll(resp.Body)
    if err != nil{
        t.Errorf("Error while reading chart data page", err)
    }
    if string(body) != "{\"chart\":{\"type\":\"line\"},\"series\":[{\"data\":[1,2,3],\"name\":\"toto\"}]}"{
        t.Errorf("Error in data json")
    }

    resp, err = http.Get("http://localhost:8080/static/js/highcharts.js")
    if err != nil{
        t.Errorf("Error while reading static page", err)
    }

    resp, err = http.Get("http://localhost:8080/static/js/load.js")
    if err != nil{
        t.Errorf("Error while reading static page", err)
    }
}

func TestDynamicChart(t *testing.T){
    options := map[string]interface{}{
        "series":  []interface{}{
            map[string]interface{}{
                "name": "toto",
                "data": []int{1, 2, 3},
            },
        },
    }

    data := make(chan interface{})
    NewDynamicChart("/dynamic/", options, data)
    go func(){
        for i := 0; i < 10; i++{
            data<-i
            // Uncomment to test in browser
            // time.Sleep(1e9)
            time.Sleep(1)
        }
    }()
    resp, err := http.Get("http://localhost:8080/dynamic/")
    if err != nil{
        t.Errorf("Error while loading dynamic chart data page", err)
    }
    defer resp.Body.Close()
    _, err = ioutil.ReadAll(resp.Body)
    if err != nil{
        t.Errorf("Error while reading dynamic data page", err)
    }

    origin := "http://localhost/"
    url := "ws://localhost:8080/dynamic/streaming/"
    ws, err := websocket.Dial(url, "", origin)
    if err != nil {
        t.Errorf("Cannot open websocket", err)
    }
    var msg = make([]byte, 512)
    var n int
    for i := 0; i< 10; i++{
        n, err = ws.Read(msg)
        if err != nil {
            t.Errorf("Cannot receive on websocket", err)
        }
        strmsg := string(msg[:n])
        expmsg := strconv.Itoa(i)
        if strmsg != expmsg {
            t.Errorf("Did not receive the correct message on websocket exepected ", expmsg, " got ", strmsg)
        }
    }

    // Uncomment to test in browser
    // time.Sleep(1e11)
}
<commit_msg>Simplified dynamic test.<commit_after>package highcharts

import (
    "testing"
    "net/http"
    "io/ioutil"
    "time"
    "code.google.com/p/go.net/websocket"
    "strconv"
)

func TestNewChart(t *testing.T){
    options := map[string]interface{}{
        "series":  []interface{}{
            map[string]interface{}{
                "name": "toto",
                "data": []int{1, 2, 3},
            },
        },
        "chart": map[string]interface{}{
            "type": "line",
        },
    }
    NewChart("/chart/", options)

    resp, err := http.Get("http://localhost:8080/chart/")
    if err != nil{
        t.Errorf("Error while loading chart page", err)
    }
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil{
        t.Errorf("Error while reading chart page", err)
    }
    // fmt.Println(string(body))

    resp, err = http.Get("http://localhost:8080/chart/data/")
    if err != nil{
        t.Errorf("Error while loading chart data page", err)
    }
    defer resp.Body.Close()
    body, err = ioutil.ReadAll(resp.Body)
    if err != nil{
        t.Errorf("Error while reading chart data page", err)
    }
    if string(body) != "{\"chart\":{\"type\":\"line\"},\"series\":[{\"data\":[1,2,3],\"name\":\"toto\"}]}"{
        t.Errorf("Error in data json")
    }

    resp, err = http.Get("http://localhost:8080/static/js/highcharts.js")
    if err != nil{
        t.Errorf("Error while reading static page", err)
    }

    resp, err = http.Get("http://localhost:8080/static/js/load.js")
    if err != nil{
        t.Errorf("Error while reading static page", err)
    }
}

func TestDynamicChart(t *testing.T){
    data := make(chan interface{})
    NewDynamicChart("/dynamic/", nil, data)
    go func(){
        for i := 0; i < 10; i++{
            data<-i
            // Uncomment to test in browser
            // time.Sleep(1e9)
            time.Sleep(1)
        }
    }()
    resp, err := http.Get("http://localhost:8080/dynamic/")
    if err != nil{
        t.Errorf("Error while loading dynamic chart data page", err)
    }
    defer resp.Body.Close()
    _, err = ioutil.ReadAll(resp.Body)
    if err != nil{
        t.Errorf("Error while reading dynamic data page", err)
    }

    origin := "http://localhost/"
    url := "ws://localhost:8080/dynamic/streaming/"
    ws, err := websocket.Dial(url, "", origin)
    if err != nil {
        t.Errorf("Cannot open websocket", err)
    }
    var msg = make([]byte, 512)
    var n int
    for i := 0; i< 10; i++{
        n, err = ws.Read(msg)
        if err != nil {
            t.Errorf("Cannot receive on websocket", err)
        }
        strmsg := string(msg[:n])
        expmsg := strconv.Itoa(i)
        if strmsg != expmsg {
            t.Errorf("Did not receive the correct message on websocket exepected ", expmsg, " got ", strmsg)
        }
    }

    // Uncomment to test in browser
    // time.Sleep(1e11)
}
<|endoftext|>