<commit_before>//
// DISCLAIMER
//
// Copyright 2021 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Tomasz Mielech
//

package actions

import (
	"context"
	"time"

	"github.com/pkg/errors"
	"github.com/rs/zerolog"

	"github.com/arangodb-helper/arangodb/pkg/definitions"
)

var actions map[string]Action

// ActionTypes is the list of ActionType
type ActionTypes []ActionType

// Contains returns true if requested action is on list
func (a ActionTypes) Contains(t ActionType) bool {
	for _, b := range a {
		if b == t {
			return true
		}
	}

	return false
}

// ActionType keeps type of action
type ActionType string

// String returns string value of ActionType
func (a ActionType) String() string {
	return string(a)
}

const (
	// ActionTypeAll filter which run all action types
	ActionTypeAll ActionType = "All"
	// ActionTypePreStop PreStop Action Type
	ActionTypePreStop ActionType = "PreStop"
)

// Action describes how some actions should be started.
type Action interface {
	// Name returns name of the action.
	Name() string
	// Timeout returns how long it should wait for the action to be finished.
	Timeout() time.Duration
	// Condition returns true if this action should be launched.
	Condition(serverType definitions.ServerType) bool
}

// Progressor describes what to do when the specific moment of action occurs.
type Progressor interface {
	// Started is launched when the action starts.
	Started(actionName string)
	// Failed is launched when the action fails.
	Failed(err error)
	// Finished is launched when the action finishes.
	Finished()
	// Progress is launched whenever some progress occurs for the specific action.
	Progress(message string) error
}

// ActionPreStop describes how pre stop actions should be started.
type ActionPreStop interface {
	Action
	// PreStop runs action before server is stopped.
	PreStop(ctx context.Context, progress Progressor) error
}

// RegisterAction registers a new action if it does not exist.
func RegisterAction(action Action) {
	if action == nil {
		return
	}

	if actions == nil {
		actions = make(map[string]Action)
	}

	actions[action.Name()] = action
}

// StartAction starts actions based on type if actionType is on the limit list
func StartLimitedAction(logger zerolog.Logger, actionType ActionType, serverType definitions.ServerType, limit ActionTypes) {
	if !limit.Contains(actionType) && !limit.Contains(ActionTypeAll) {
		return
	}

	StartAction(logger, actionType, serverType)
}

// StartAction starts actions based on type
func StartAction(logger zerolog.Logger, actionType ActionType, serverType definitions.ServerType) {
	switch actionType {
	case ActionTypePreStop:
		log := logger.With().Str("action", actionType.String()).Logger()
		log.Info().Msgf("Starting actions")
		StartPreStopActions(log, serverType, &ProgressLog{
			LoggerOriginal: log,
			logger:         log,
		})
	}
}

// StartPreStopActions runs registered pre stop actions.
func StartPreStopActions(logger zerolog.Logger, serverType definitions.ServerType, progress Progressor) {
	if progress == nil {
		progress = &ProgressLog{
			LoggerOriginal: logger,
			logger:         logger,
		}
	}

	for _, anyAction := range actions {
		preStopAction, ok := anyAction.(ActionPreStop)
		if !ok {
			continue
		}

		if !preStopAction.Condition(serverType) {
			continue
		}

		logger.Info().Str("Name", anyAction.Name()).Msgf("Starting Action")

		ctxAction, cancelAction := context.WithTimeout(context.Background(), preStopAction.Timeout())

		progress.Started(preStopAction.Name())
		if err := preStopAction.PreStop(ctxAction, progress); err != nil {
			progress.Failed(err)
		} else {
			progress.Finished()
		}

		cancelAction()
	}
}

// ProgressLog describes simple logger for actions.
type ProgressLog struct {
	LoggerOriginal zerolog.Logger
	logger         zerolog.Logger
}

// Started is launched when the action starts.
func (p *ProgressLog) Started(actionName string) {
	p.logger = p.LoggerOriginal.With().Str("name", actionName).Logger()
	p.logger.Info().Msg("Action started")
}

// Failed is launched when the action fails.
func (p *ProgressLog) Failed(err error) {
	p.logger.Error().Err(err).Msg("Action failed")
}

// Finished is launched when the action finishes.
func (p *ProgressLog) Finished() {
	p.logger.Info().Msg("Action finished")
}

// Progress is launched whenever some progress occurs for the specific action.
func (p *ProgressLog) Progress(message string) error {
	p.logger.Info().Str("progress", message).Msg("Action progress")
	return errors.New(message)
}

// ProgressEmpty describes empty progress for the actions.
type ProgressEmpty struct{}

// Started is launched when the action starts.
func (p ProgressEmpty) Started(_ string) {}

// Failed is launched when the action fails.
func (p ProgressEmpty) Failed(_ error) {}

// Finished is launched when the action finishes.
func (p ProgressEmpty) Finished() {}

// Progress is launched whenever some progress occurs for the specific action.
func (p ProgressEmpty) Progress(message string) error {
	return errors.New(message)
}
<commit_msg>fix race condition on the actions' registry<commit_after>//
// DISCLAIMER
//
// Copyright 2021 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Tomasz Mielech
//

package actions

import (
	"context"
	"sync"
	"time"

	"github.com/pkg/errors"
	"github.com/rs/zerolog"

	"github.com/arangodb-helper/arangodb/pkg/definitions"
)

// Registry allows to register new actions.
type Registry struct {
	// mutex protects the internal fields of this structure.
	mutex sync.RWMutex
	// actions holds already registered actions.
	actions map[string]Action
}

var registry Registry

// ActionTypes is the list of ActionType
type ActionTypes []ActionType

// Contains returns true if requested action is on list
func (a ActionTypes) Contains(t ActionType) bool {
	for _, b := range a {
		if b == t {
			return true
		}
	}

	return false
}

// ActionType keeps type of action
type ActionType string

// String returns string value of ActionType
func (a ActionType) String() string {
	return string(a)
}

const (
	// ActionTypeAll filter which run all action types
	ActionTypeAll ActionType = "All"
	// ActionTypePreStop PreStop Action Type
	ActionTypePreStop ActionType = "PreStop"
)

// Action describes how some actions should be started.
type Action interface {
	// Name returns name of the action.
	Name() string
	// Timeout returns how long it should wait for the action to be finished.
	Timeout() time.Duration
	// Condition returns true if this action should be launched.
	Condition(serverType definitions.ServerType) bool
}

// Progressor describes what to do when the specific moment of action occurs.
type Progressor interface {
	// Started is launched when the action starts.
	Started(actionName string)
	// Failed is launched when the action fails.
	Failed(err error)
	// Finished is launched when the action finishes.
	Finished()
	// Progress is launched whenever some progress occurs for the specific action.
	Progress(message string) error
}

// ActionPreStop describes how pre stop actions should be started.
type ActionPreStop interface {
	Action
	// PreStop runs action before server is stopped.
	PreStop(ctx context.Context, progress Progressor) error
}

// RegisterAction registers a new action if it does not exist.
func RegisterAction(action Action) {
	if action == nil {
		return
	}

	registry.mutex.Lock()
	defer registry.mutex.Unlock()

	if registry.actions == nil {
		registry.actions = make(map[string]Action)
	}

	registry.actions[action.Name()] = action
}

// GetActions returns actions which are already registered.
func (r *Registry) GetActions() map[string]Action {
	registry.mutex.RLock()
	defer registry.mutex.RUnlock()

	return registry.actions
}

// StartAction starts actions based on type if actionType is on the limit list
func StartLimitedAction(logger zerolog.Logger, actionType ActionType, serverType definitions.ServerType, limit ActionTypes) {
	if !limit.Contains(actionType) && !limit.Contains(ActionTypeAll) {
		return
	}

	StartAction(logger, actionType, serverType)
}

// StartAction starts actions based on type
func StartAction(logger zerolog.Logger, actionType ActionType, serverType definitions.ServerType) {
	switch actionType {
	case ActionTypePreStop:
		log := logger.With().Str("action", actionType.String()).Logger()
		log.Info().Msgf("Starting actions")
		StartPreStopActions(log, serverType, &ProgressLog{
			LoggerOriginal: log,
			logger:         log,
		})
	}
}

// StartPreStopActions runs registered pre stop actions.
func StartPreStopActions(logger zerolog.Logger, serverType definitions.ServerType, progress Progressor) {
	if progress == nil {
		progress = &ProgressLog{
			LoggerOriginal: logger,
			logger:         logger,
		}
	}

	for _, anyAction := range registry.GetActions() {
		preStopAction, ok := anyAction.(ActionPreStop)
		if !ok {
			continue
		}

		if !preStopAction.Condition(serverType) {
			continue
		}

		logger.Info().Str("Name", anyAction.Name()).Msgf("Starting Action")

		ctxAction, cancelAction := context.WithTimeout(context.Background(), preStopAction.Timeout())

		progress.Started(preStopAction.Name())
		if err := preStopAction.PreStop(ctxAction, progress); err != nil {
			progress.Failed(err)
		} else {
			progress.Finished()
		}

		cancelAction()
	}
}

// ProgressLog describes simple logger for actions.
type ProgressLog struct {
	LoggerOriginal zerolog.Logger
	logger         zerolog.Logger
}

// Started is launched when the action starts.
func (p *ProgressLog) Started(actionName string) {
	p.logger = p.LoggerOriginal.With().Str("name", actionName).Logger()
	p.logger.Info().Msg("Action started")
}

// Failed is launched when the action fails.
func (p *ProgressLog) Failed(err error) {
	p.logger.Error().Err(err).Msg("Action failed")
}

// Finished is launched when the action finishes.
func (p *ProgressLog) Finished() {
	p.logger.Info().Msg("Action finished")
}

// Progress is launched whenever some progress occurs for the specific action.
func (p *ProgressLog) Progress(message string) error {
	p.logger.Info().Str("progress", message).Msg("Action progress")
	return errors.New(message)
}

// ProgressEmpty describes empty progress for the actions.
type ProgressEmpty struct{}

// Started is launched when the action starts.
func (p ProgressEmpty) Started(_ string) {}

// Failed is launched when the action fails.
func (p ProgressEmpty) Failed(_ error) {}

// Finished is launched when the action finishes.
func (p ProgressEmpty) Finished() {}

// Progress is launched whenever some progress occurs for the specific action.
func (p ProgressEmpty) Progress(message string) error {
	return errors.New(message)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net/http"
	"path/filepath"
	"regexp"

	dockerapi "github.com/docker/docker/api"
	dockerclient "github.com/docker/engine-api/client"
	"github.com/docker/go-plugins-helpers/authorization"
)

func newPlugin(dockerHost, certPath string, tlsVerify bool) (*novolume, error) {
	var transport *http.Transport
	if certPath != "" {
		tlsc := &tls.Config{}

		cert, err := tls.LoadX509KeyPair(filepath.Join(certPath, "cert.pem"), filepath.Join(certPath, "key.pem"))
		if err != nil {
			return nil, fmt.Errorf("Error loading x509 key pair: %s", err)
		}

		tlsc.Certificates = append(tlsc.Certificates, cert)
		tlsc.InsecureSkipVerify = !tlsVerify
		transport = &http.Transport{
			TLSClientConfig: tlsc,
		}
	}

	client, err := dockerclient.NewClient(dockerHost, dockerapi.DefaultVersion.String(), transport, nil)
	if err != nil {
		return nil, err
	}
	return &novolume{client: client}, nil
}

var (
	startRegExp = regexp.MustCompile(`/containers/(.*)/start$`)
)

type novolume struct {
	client *dockerclient.Client
}

func (p *novolume) AuthZReq(req authorization.Request) authorization.Response {
	if req.RequestMethod == "POST" && startRegExp.MatchString(req.RequestURI) {
		// this is deprecated in docker, remove once hostConfig is dropped to
		// being available at start time
		if req.RequestBody != nil {
			type vfrom struct {
				VolumesFrom []string
			}
			vf := &vfrom{}
			if err := json.NewDecoder(bytes.NewReader(req.RequestBody)).Decode(vf); err != nil {
				return authorization.Response{Err: err.Error()}
			}
			if len(vf.VolumesFrom) > 0 {
				goto noallow
			}
		}
		res := startRegExp.FindStringSubmatch(req.RequestURI)
		if len(res) < 1 {
			return authorization.Response{Err: "unable to find container name"}
		}
		container, err := p.client.ContainerInspect(res[1])
		if err != nil {
			return authorization.Response{Err: err.Error()}
		}
		bindDests := []string{}
		for _, m := range container.Mounts {
			if m.Driver != "" {
				goto noallow
			}
			bindDests = append(bindDests, m.Destination)
		}
		image, _, err := p.client.ImageInspectWithRaw(container.Image, false)
		if err != nil {
			return authorization.Response{Err: err.Error()}
		}
		if len(bindDests) == 0 && len(image.Config.Volumes) > 0 {
			goto noallow
		}
		for _, bd := range bindDests {
			if _, ok := image.Config.Volumes[bd]; !ok {
				goto noallow
			}
		}
		if len(container.HostConfig.VolumesFrom) > 0 {
			goto noallow
		}
	}
	return authorization.Response{Allow: true}

noallow:
	return authorization.Response{Msg: "volumes are not allowed"}
}

func (p *novolume) AuthZRes(req authorization.Request) authorization.Response {
	return authorization.Response{Allow: true}
}
<commit_msg>fix allow bind overrides VOLUME<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net/http"
	"path/filepath"
	"regexp"

	dockerapi "github.com/docker/docker/api"
	dockerclient "github.com/docker/engine-api/client"
	"github.com/docker/go-plugins-helpers/authorization"
)

func newPlugin(dockerHost, certPath string, tlsVerify bool) (*novolume, error) {
	var transport *http.Transport
	if certPath != "" {
		tlsc := &tls.Config{}

		cert, err := tls.LoadX509KeyPair(filepath.Join(certPath, "cert.pem"), filepath.Join(certPath, "key.pem"))
		if err != nil {
			return nil, fmt.Errorf("Error loading x509 key pair: %s", err)
		}

		tlsc.Certificates = append(tlsc.Certificates, cert)
		tlsc.InsecureSkipVerify = !tlsVerify
		transport = &http.Transport{
			TLSClientConfig: tlsc,
		}
	}

	client, err := dockerclient.NewClient(dockerHost, dockerapi.DefaultVersion.String(), transport, nil)
	if err != nil {
		return nil, err
	}
	return &novolume{client: client}, nil
}

var (
	startRegExp = regexp.MustCompile(`/containers/(.*)/start$`)
)

type novolume struct {
	client *dockerclient.Client
}

func (p *novolume) AuthZReq(req authorization.Request) authorization.Response {
	if req.RequestMethod == "POST" && startRegExp.MatchString(req.RequestURI) {
		// this is deprecated in docker, remove once hostConfig is dropped to
		// being available at start time
		if req.RequestBody != nil {
			type vfrom struct {
				VolumesFrom []string
			}
			vf := &vfrom{}
			if err := json.NewDecoder(bytes.NewReader(req.RequestBody)).Decode(vf); err != nil {
				return authorization.Response{Err: err.Error()}
			}
			if len(vf.VolumesFrom) > 0 {
				goto noallow
			}
		}
		res := startRegExp.FindStringSubmatch(req.RequestURI)
		if len(res) < 1 {
			return authorization.Response{Err: "unable to find container name"}
		}
		container, err := p.client.ContainerInspect(res[1])
		if err != nil {
			return authorization.Response{Err: err.Error()}
		}
		bindDests := []string{}
		for _, m := range container.Mounts {
			if m.Driver != "" {
				goto noallow
			}
			bindDests = append(bindDests, m.Destination)
		}
		image, _, err := p.client.ImageInspectWithRaw(container.Image, false)
		if err != nil {
			return authorization.Response{Err: err.Error()}
		}
		if len(bindDests) == 0 && len(image.Config.Volumes) > 0 {
			goto noallow
		}
		if len(image.Config.Volumes) > 0 {
			for _, bd := range bindDests {
				if _, ok := image.Config.Volumes[bd]; !ok {
					goto noallow
				}
			}
		}
		if len(container.HostConfig.VolumesFrom) > 0 {
			goto noallow
		}
		// TODO(runcom): FROM scratch ?!?!
	}
	return authorization.Response{Allow: true}

noallow:
	return authorization.Response{Msg: "volumes are not allowed"}
}

func (p *novolume) AuthZRes(req authorization.Request) authorization.Response {
	return authorization.Response{Allow: true}
}
<|endoftext|>


<commit_before>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/getgauge/common"
	"github.com/getgauge/gauge/config"
	"github.com/getgauge/gauge/conn"
	"github.com/getgauge/gauge/gauge_messages"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/version"
	"github.com/golang/protobuf/proto"
	"net"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	executionScope          = "execution"
	setupScope              = "setup"
	pluginConnectionPortEnv = "plugin_connection_port"
)

type pluginDescriptor struct {
	Id          string
	Version     string
	Name        string
	Description string
	Command     struct {
		Windows []string
		Linux   []string
		Darwin  []string
	}
	Scope               []string
	GaugeVersionSupport versionSupport
	pluginPath          string
}

type pluginHandler struct {
	pluginsMap map[string]*plugin
}

type plugin struct {
	connection net.Conn
	pluginCmd  *exec.Cmd
	descriptor *pluginDescriptor
}

func (plugin *plugin) kill(wg *sync.WaitGroup) error {
	defer wg.Done()
	if plugin.isStillRunning() {

		exited := make(chan bool, 1)
		go func() {
			for {
				if plugin.isStillRunning() {
					time.Sleep(100 * time.Millisecond)
				} else {
					exited <- true
					return
				}
			}
		}()

		select {
		case done := <-exited:
			if done {
				logger.Log.Debug("Plugin [%s] with pid [%d] has exited", plugin.descriptor.Name, plugin.pluginCmd.Process.Pid)
			}
		case <-time.After(config.PluginConnectionTimeout()):
			logger.Log.Warning("Plugin [%s] with pid [%d] did not exit after %.2f seconds. Forcefully killing it.", plugin.descriptor.Name, plugin.pluginCmd.Process.Pid, config.PluginConnectionTimeout().Seconds())
			return plugin.pluginCmd.Process.Kill()
		}
	}
	return nil
}

func (plugin *plugin) isStillRunning() bool {
	return plugin.pluginCmd.ProcessState == nil || !plugin.pluginCmd.ProcessState.Exited()
}

func isPluginInstalled(pluginName, pluginVersion string) bool {
	pluginsInstallDir, err := common.GetPluginsInstallDir(pluginName)
	if err != nil {
		return false
	}

	thisPluginDir := path.Join(pluginsInstallDir, pluginName)
	if !common.DirExists(thisPluginDir) {
		return false
	}

	if pluginVersion != "" {
		pluginJson := path.Join(thisPluginDir, pluginVersion, common.PluginJsonFile)
		if common.FileExists(pluginJson) {
			return true
		} else {
			return false
		}
	} else {
		return true
	}
}

func getPluginJsonPath(pluginName, version string) (string, error) {
	if !isPluginInstalled(pluginName, version) {
		return "", errors.New(fmt.Sprintf("%s %s is not installed", pluginName, version))
	}

	pluginInstallDir, err := common.GetPluginInstallDir(pluginName, "")
	if err != nil {
		return "", err
	}
	return filepath.Join(pluginInstallDir, common.PluginJsonFile), nil
}

func getPluginDescriptor(pluginId, pluginVersion string) (*pluginDescriptor, error) {
	pluginJson, err := getPluginJsonPath(pluginId, pluginVersion)
	if err != nil {
		return nil, err
	}
	return getPluginDescriptorFromJson(pluginJson)
}

func getPluginDescriptorFromJson(pluginJson string) (*pluginDescriptor, error) {
	pluginJsonContents, err := common.ReadFileContents(pluginJson)
	if err != nil {
		return nil, err
	}
	var pd pluginDescriptor
	if err = json.Unmarshal([]byte(pluginJsonContents), &pd); err != nil {
		return nil, errors.New(fmt.Sprintf("%s: %s", pluginJson, err.Error()))
	}
	pd.pluginPath = filepath.Dir(pluginJson)

	return &pd, nil
}

func startPlugin(pd *pluginDescriptor, action string, wait bool) (*exec.Cmd, error) {
	command := []string{}
	switch runtime.GOOS {
	case "windows":
		command = pd.Command.Windows
		break
	case "darwin":
		command = pd.Command.Darwin
		break
	default:
		command = pd.Command.Linux
		break
	}
	if len(command) == 0 {
		return nil, errors.New(fmt.Sprintf("Platform specific command not specified: %s.", runtime.GOOS))
	}

	pluginLogger := &pluginLogger{pluginName: pd.Name}
	cmd, err := common.ExecuteCommand(command, pd.pluginPath, pluginLogger, pluginLogger)

	if err != nil {
		return nil, err
	}

	if wait {
		return cmd, cmd.Wait()
	} else {
		go func() {
			cmd.Wait()
		}()
	}

	return cmd, nil
}

func setEnvForPlugin(action string, pd *pluginDescriptor, manifest *manifest, pluginEnvVars map[string]string) error {
	pluginEnvVars[fmt.Sprintf("%s_action", pd.Id)] = action
	pluginEnvVars["test_language"] = manifest.Language
	if err := setEnvironmentProperties(pluginEnvVars); err != nil {
		return err
	}
	return nil
}

func setEnvironmentProperties(properties map[string]string) error {
	for k, v := range properties {
		if err := common.SetEnvVariable(k, v); err != nil {
			return err
		}
	}
	return nil
}

func addPluginToTheProject(pluginName string, pluginArgs map[string]string, manifest *manifest) error {
	if !isPluginInstalled(pluginName, pluginArgs["version"]) {
		logger.Log.Info("Plugin %s %s is not installed. Downloading the plugin.... \n", pluginName, pluginArgs["version"])
		result := installPlugin(pluginName, pluginArgs["version"])
		if !result.success {
			logger.Log.Error(result.getMessage())
		}
	}
	pd, err := getPluginDescriptor(pluginName, pluginArgs["version"])
	if err != nil {
		return err
	}
	if isPluginAdded(manifest, pd) {
		return errors.New("Plugin " + pd.Name + " is already added")
	}

	action := setupScope
	if err := setEnvForPlugin(action, pd, manifest, pluginArgs); err != nil {
		return err
	}
	if _, err := startPlugin(pd, action, true); err != nil {
		return err
	}
	manifest.Plugins = append(manifest.Plugins, pd.Id)
	return manifest.save()
}

func isPluginAdded(manifest *manifest, descriptor *pluginDescriptor) bool {
	for _, pluginId := range manifest.Plugins {
		if pluginId == descriptor.Id {
			return true
		}
	}
	return false
}

func startPluginsForExecution(manifest *manifest) (*pluginHandler, []string) {
	warnings := make([]string, 0)
	handler := &pluginHandler{}
	envProperties := make(map[string]string)

	for _, pluginId := range manifest.Plugins {
		pd, err := getPluginDescriptor(pluginId, "")
		if err != nil {
			warnings = append(warnings, fmt.Sprintf("Error starting plugin %s. Failed to get plugin.json. %s", pluginId, err.Error()))
			continue
		}
		compatibilityErr := checkCompatiblity(version.CurrentGaugeVersion, &pd.GaugeVersionSupport)
		if compatibilityErr != nil {
			warnings = append(warnings, fmt.Sprintf("Compatible %s plugin version to current Gauge version %s not found", pd.Name, version.CurrentGaugeVersion))
			continue
		}
		if isExecutionScopePlugin(pd) {
			gaugeConnectionHandler, err := conn.NewGaugeConnectionHandler(0, nil)
			if err != nil {
				warnings = append(warnings, err.Error())
				continue
			}
			envProperties[pluginConnectionPortEnv] = strconv.Itoa(gaugeConnectionHandler.ConnectionPortNumber())
			setEnvForPlugin(executionScope, pd, manifest, envProperties)

			pluginCmd, err := startPlugin(pd, executionScope, false)
			if err != nil {
				warnings = append(warnings, fmt.Sprintf("Error starting plugin %s %s. %s", pd.Name, pd.Version, err.Error()))
				continue
			}
			pluginConnection, err := gaugeConnectionHandler.AcceptConnection(config.PluginConnectionTimeout(), make(chan error))
			if err != nil {
				warnings = append(warnings, fmt.Sprintf("Error starting plugin %s %s. Failed to connect to plugin. %s", pd.Name, pd.Version, err.Error()))
				pluginCmd.Process.Kill()
				continue
			}
			handler.addPlugin(pluginId, &plugin{connection: pluginConnection, pluginCmd: pluginCmd, descriptor: pd})
		}

	}
	return handler, warnings
}

func isExecutionScopePlugin(pd *pluginDescriptor) bool {
	for _, scope := range pd.Scope {
		if strings.ToLower(scope) == executionScope {
			return true
		}
	}
	return false
}

func (handler *pluginHandler) addPlugin(pluginId string, pluginToAdd *plugin) {
	if handler.pluginsMap == nil {
		handler.pluginsMap = make(map[string]*plugin)
	}
	handler.pluginsMap[pluginId] = pluginToAdd
}

func (handler *pluginHandler) removePlugin(pluginId string) {
	delete(handler.pluginsMap, pluginId)
}

func (handler *pluginHandler) notifyPlugins(message *gauge_messages.Message) {
	for id, plugin := range handler.pluginsMap {
		err := plugin.sendMessage(message)
		if err != nil {
			logger.Log.Error("Unable to connect to plugin %s %s. %s\n", plugin.descriptor.Name, plugin.descriptor.Version, err.Error())
			handler.killPlugin(id)
		}
	}
}

func (handler *pluginHandler) killPlugin(pluginId string) {
	plugin := handler.pluginsMap[pluginId]
	logger.Log.Debug("Killing Plugin %s %s\n", plugin.descriptor.Name, plugin.descriptor.Version)
	err := plugin.pluginCmd.Process.Kill()
	if err != nil {
		logger.Log.Error("Failed to kill plugin %s %s. %s\n", plugin.descriptor.Name, plugin.descriptor.Version, err.Error())
	}
	handler.removePlugin(pluginId)
}

func (handler *pluginHandler) gracefullyKillPlugins() {
	var wg sync.WaitGroup
	for _, plugin := range handler.pluginsMap {
		wg.Add(1)
		go plugin.kill(&wg)
	}
	wg.Wait()
}

func (plugin *plugin) sendMessage(message *gauge_messages.Message) error {
	messageId := common.GetUniqueId()
	message.MessageId = &messageId
	messageBytes, err := proto.Marshal(message)
	if err != nil {
		return err
	}
	err = conn.Write(plugin.connection, messageBytes)
	if err != nil {
		return errors.New(fmt.Sprintf("[Warning] Failed to send message to plugin: %d  %s", plugin.descriptor.Id, err.Error()))
	}
	return nil
}
<commit_msg>Not erroring when plugin is already added to project and --add-plugin is called<commit_after>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/getgauge/common"
	"github.com/getgauge/gauge/config"
	"github.com/getgauge/gauge/conn"
	"github.com/getgauge/gauge/gauge_messages"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/version"
	"github.com/golang/protobuf/proto"
	"net"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	executionScope          = "execution"
	setupScope              = "setup"
	pluginConnectionPortEnv = "plugin_connection_port"
)

type pluginDescriptor struct {
	Id          string
	Version     string
	Name        string
	Description string
	Command     struct {
		Windows []string
		Linux   []string
		Darwin  []string
	}
	Scope               []string
	GaugeVersionSupport versionSupport
	pluginPath          string
}

type pluginHandler struct {
	pluginsMap map[string]*plugin
}

type plugin struct {
	connection net.Conn
	pluginCmd  *exec.Cmd
	descriptor *pluginDescriptor
}

func (plugin *plugin) kill(wg *sync.WaitGroup) error {
	defer wg.Done()
	if plugin.isStillRunning() {

		exited := make(chan bool, 1)
		go func() {
			for {
				if plugin.isStillRunning() {
					time.Sleep(100 * time.Millisecond)
				} else {
					exited <- true
					return
				}
			}
		}()

		select {
		case done := <-exited:
			if done {
				logger.Log.Debug("Plugin [%s] with pid [%d] has exited", plugin.descriptor.Name, plugin.pluginCmd.Process.Pid)
			}
		case <-time.After(config.PluginConnectionTimeout()):
			logger.Log.Warning("Plugin [%s] with pid [%d] did not exit after %.2f seconds. Forcefully killing it.", plugin.descriptor.Name, plugin.pluginCmd.Process.Pid, config.PluginConnectionTimeout().Seconds())
			return plugin.pluginCmd.Process.Kill()
		}
	}
	return nil
}

func (plugin *plugin) isStillRunning() bool {
	return plugin.pluginCmd.ProcessState == nil || !plugin.pluginCmd.ProcessState.Exited()
}

func isPluginInstalled(pluginName, pluginVersion string) bool {
	pluginsInstallDir, err := common.GetPluginsInstallDir(pluginName)
	if err != nil {
		return false
	}

	thisPluginDir := path.Join(pluginsInstallDir, pluginName)
	if !common.DirExists(thisPluginDir) {
		return false
	}

	if pluginVersion != "" {
		pluginJson := path.Join(thisPluginDir, pluginVersion, common.PluginJsonFile)
		if common.FileExists(pluginJson) {
			return true
		} else {
			return false
		}
	} else {
		return true
	}
}

func getPluginJsonPath(pluginName, version string) (string, error) {
	if !isPluginInstalled(pluginName, version) {
		return "", errors.New(fmt.Sprintf("%s %s is not installed", pluginName, version))
	}

	pluginInstallDir, err := common.GetPluginInstallDir(pluginName, "")
	if err != nil {
		return "", err
	}
	return filepath.Join(pluginInstallDir, common.PluginJsonFile), nil
}

func getPluginDescriptor(pluginId, pluginVersion string) (*pluginDescriptor, error) {
	pluginJson, err := getPluginJsonPath(pluginId, pluginVersion)
	if err != nil {
		return nil, err
	}
	return getPluginDescriptorFromJson(pluginJson)
}

func getPluginDescriptorFromJson(pluginJson string) (*pluginDescriptor, error) {
	pluginJsonContents, err := common.ReadFileContents(pluginJson)
	if err != nil {
		return nil, err
	}
	var pd pluginDescriptor
	if err = json.Unmarshal([]byte(pluginJsonContents), &pd); err != nil {
		return nil, errors.New(fmt.Sprintf("%s: %s", pluginJson, err.Error()))
	}
	pd.pluginPath = filepath.Dir(pluginJson)

	return &pd, nil
}

func startPlugin(pd *pluginDescriptor, action string, wait bool) (*exec.Cmd, error) {
	command := []string{}
	switch runtime.GOOS {
	case "windows":
		command = pd.Command.Windows
		break
	case "darwin":
		command = pd.Command.Darwin
		break
	default:
		command = pd.Command.Linux
		break
	}
	if len(command) == 0 {
		return nil, errors.New(fmt.Sprintf("Platform specific command not specified: %s.", runtime.GOOS))
	}

	pluginLogger := &pluginLogger{pluginName: pd.Name}
	cmd, err := common.ExecuteCommand(command, pd.pluginPath, pluginLogger, pluginLogger)

	if err != nil {
		return nil, err
	}

	if wait {
		return cmd, cmd.Wait()
	} else {
		go func() {
			cmd.Wait()
		}()
	}

	return cmd, nil
}

func setEnvForPlugin(action string, pd *pluginDescriptor, manifest *manifest, pluginEnvVars map[string]string) error {
	pluginEnvVars[fmt.Sprintf("%s_action", pd.Id)] = action
	pluginEnvVars["test_language"] = manifest.Language
	if err := setEnvironmentProperties(pluginEnvVars); err != nil {
		return err
	}
	return nil
}

func setEnvironmentProperties(properties map[string]string) error {
	for k, v := range properties {
		if err := common.SetEnvVariable(k, v); err != nil {
			return err
		}
	}
	return nil
}

func addPluginToTheProject(pluginName string, pluginArgs map[string]string, manifest *manifest) error {
	if !isPluginInstalled(pluginName, pluginArgs["version"]) {
		logger.Log.Info("Plugin %s %s is not installed. Downloading the plugin.... \n", pluginName, pluginArgs["version"])
		result := installPlugin(pluginName, pluginArgs["version"])
		if !result.success {
			logger.Log.Error(result.getMessage())
		}
	}
	pd, err := getPluginDescriptor(pluginName, pluginArgs["version"])
	if err != nil {
		return err
	}
	if isPluginAdded(manifest, pd) {
		logger.Log.Info("Plugin " + pd.Name + " is already added")
		return nil
	}

	action := setupScope
	if err := setEnvForPlugin(action, pd, manifest, pluginArgs); err != nil {
		return err
	}
	if _, err := startPlugin(pd, action, true); err != nil {
		return err
	}
	manifest.Plugins = append(manifest.Plugins, pd.Id)
	return manifest.save()
}

func isPluginAdded(manifest *manifest, descriptor *pluginDescriptor) bool {
	for _, pluginId := range manifest.Plugins {
		if pluginId == descriptor.Id {
			return true
		}
	}
	return false
}

func startPluginsForExecution(manifest *manifest) (*pluginHandler, []string) {
	warnings := make([]string, 0)
	handler := &pluginHandler{}
	envProperties := make(map[string]string)

	for _, pluginId := range manifest.Plugins {
		pd, err := getPluginDescriptor(pluginId, "")
		if err != nil {
			warnings = append(warnings, fmt.Sprintf("Error starting plugin %s. Failed to get plugin.json. %s", pluginId, err.Error()))
			continue
		}
		compatibilityErr := checkCompatiblity(version.CurrentGaugeVersion, &pd.GaugeVersionSupport)
		if compatibilityErr != nil {
			warnings = append(warnings, fmt.Sprintf("Compatible %s plugin version to current Gauge version %s not found", pd.Name, version.CurrentGaugeVersion))
			continue
		}
		if isExecutionScopePlugin(pd) {
			gaugeConnectionHandler, err := conn.NewGaugeConnectionHandler(0, nil)
			if err != nil {
				warnings = append(warnings, err.Error())
				continue
			}
			envProperties[pluginConnectionPortEnv] = strconv.Itoa(gaugeConnectionHandler.ConnectionPortNumber())
			setEnvForPlugin(executionScope, pd, manifest, envProperties)

			pluginCmd, err := startPlugin(pd, executionScope, false)
			if err != nil {
				warnings = append(warnings, fmt.Sprintf("Error starting plugin %s %s. %s", pd.Name, pd.Version, err.Error()))
				continue
			}
			pluginConnection, err := gaugeConnectionHandler.AcceptConnection(config.PluginConnectionTimeout(), make(chan error))
			if err != nil {
				warnings = append(warnings, fmt.Sprintf("Error starting plugin %s %s. Failed to connect to plugin. %s", pd.Name, pd.Version, err.Error()))
				pluginCmd.Process.Kill()
				continue
			}
			handler.addPlugin(pluginId, &plugin{connection: pluginConnection, pluginCmd: pluginCmd, descriptor: pd})
		}

	}
	return handler, warnings
}

func isExecutionScopePlugin(pd *pluginDescriptor) bool {
	for _, scope := range pd.Scope {
		if strings.ToLower(scope) == executionScope {
			return true
		}
	}
	return false
}

func (handler *pluginHandler) addPlugin(pluginId string, pluginToAdd *plugin) {
	if handler.pluginsMap == nil {
		handler.pluginsMap = make(map[string]*plugin)
	}
	handler.pluginsMap[pluginId] = pluginToAdd
}

func (handler *pluginHandler) removePlugin(pluginId string) {
	delete(handler.pluginsMap, pluginId)
}

func (handler *pluginHandler) notifyPlugins(message *gauge_messages.Message) {
	for id, plugin := range handler.pluginsMap {
		err := plugin.sendMessage(message)
		if err != nil {
			logger.Log.Error("Unable to connect to plugin %s %s. %s\n", plugin.descriptor.Name, plugin.descriptor.Version, err.Error())
			handler.killPlugin(id)
		}
	}
}

func (handler *pluginHandler) killPlugin(pluginId string) {
	plugin := handler.pluginsMap[pluginId]
	logger.Log.Debug("Killing Plugin %s %s\n", plugin.descriptor.Name, plugin.descriptor.Version)
	err := plugin.pluginCmd.Process.Kill()
	if err != nil {
		logger.Log.Error("Failed to kill plugin %s %s. %s\n", plugin.descriptor.Name, plugin.descriptor.Version, err.Error())
	}
	handler.removePlugin(pluginId)
}

func (handler *pluginHandler) gracefullyKillPlugins() {
	var wg sync.WaitGroup
	for _, plugin := range handler.pluginsMap {
		wg.Add(1)
		go plugin.kill(&wg)
	}
	wg.Wait()
}

func (plugin *plugin) sendMessage(message *gauge_messages.Message) error {
	messageId := common.GetUniqueId()
	message.MessageId = &messageId
	messageBytes, err := proto.Marshal(message)
	if err != nil {
		return err
	}
	err = conn.Write(plugin.connection, messageBytes)
	if err != nil {
		return errors.New(fmt.Sprintf("[Warning] Failed to send message to plugin: %d  %s", plugin.descriptor.Id, err.Error()))
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os/exec"
	"strings"
	"sync"
	"time"

	"gopkg.in/yaml.v2"
)

const (
	pluginFlashWaitDu time.Duration = 50 * time.Millisecond

	pluginNameMain  string = "main"
	pluginMethodTCP        = "tcp"
	pluginMethodStd        = "std"
)

type plugin struct {
	Name        string            `yaml:"name"        json:"name"`
	Description string            `yaml:"description" json:"description"`
	Version     string            `yaml:"version"     json:"version"`
	Author      string            `yaml:"author"      json:"author"`
	Method      string            `yaml:"method"      json:"method"`
	Exec        []string          `yaml:"exec"        json:"-"`
	Nagomever   string            `yaml:"nagomever"   json:"-"`
	Depends     []string          `yaml:"depends"     json:"depends"`
	No          int               `yaml:"-"           json:"no"`
	Rw          *bufio.ReadWriter `yaml:"-"           json:"-"`
	Startc      chan struct{}     `yaml:"-"           json:"-"`
	flushTm     *time.Timer
	isEnable    bool
}

func (pl *plugin) Init(no int) {
	pl.flushTm = time.NewTimer(time.Hour)
	pl.Startc = make(chan struct{}, 1)
	pl.No = no
}

func (pl *plugin) Start(cv *CommentViewer) {
	if pl.No == 0 {
		log.Printf("plugin \"%s\" is not initialized\n", pl.Name)
		return
	}
	if pl.Name == "" {
		log.Printf("plugin \"%s\" no name is set\n", pl.Name)
		return
	}
	if pl.Rw == nil {
		log.Printf("plugin \"%s\" no rw\n", pl.Name)
		return
	}
	if pl.isEnable {
		return
	}
	pl.Enable()

	pl.Startc <- struct{}{}

	cv.wg.Add(1)
	go eachPluginRw(cv, pl.No-1)
}

func (pl *plugin) Enable() {
	if pl.isEnable {
		return
	}
	pl.isEnable = true

	// send message
	jmes, err := json.Marshal(Message{
		Domain:  DomainDirectngm,
		Command: CommDirectngmPlugEnabled,
	})
	if err != nil {
		log.Println(err)
		return
	}
	fmt.Fprintf(pl.Rw, "%s\n", jmes)
	pl.flushTm.Reset(0)
}

func (pl *plugin) Disable() {
	if !pl.isEnable {
		return
	}
	pl.isEnable = false

	// send message
	jmes, err := json.Marshal(Message{
		Domain:  DomainDirectngm,
		Command: CommDirectngmPlugDisabled,
	})
	if err != nil {
		log.Println(err)
		return
	}
	fmt.Fprintf(pl.Rw, "%s\n", jmes)
	pl.flushTm.Reset(0)
}

func (pl *plugin) IsEnable() bool {
	return pl.isEnable
}

func (pl *plugin) DependFilter(pln string) bool {
	f := false
	for _, d := range pl.Depends {
		if d == pln+DomainFilterSuffix {
			f = true
			break
		}
	}
	return f
}

func (pl *plugin) Depend(pln string) bool {
	f := false
	for _, d := range pl.Depends {
		if d == pln {
			f = true
			break
		}
	}
	return f
}

func (pl *plugin) loadPlugin(filePath string) error {
	d, err := ioutil.ReadFile(filePath)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(d, pl)
	if err != nil {
		return err
	}

	return nil
}

func (pl *plugin) savePlugin(filePath string) error {
	d, err := yaml.Marshal(pl)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(filePath, d, 0600)
	if err != nil {
		return err
	}

	return nil
}

// eachPluginRw manages plugins IO. It is launched when a plugin is leaded.
func eachPluginRw(cv *CommentViewer, n int) {
	defer cv.wg.Done()

	// wait for being enabled
	select {
	case <-cv.Pgns[n].Startc:
	case <-cv.Quit:
		return
	}

	// Run decoder.  It puts a message into "mes".
	dec := json.NewDecoder(cv.Pgns[n].Rw)
	mes := make(chan (*Message))
	cv.wg.Add(1)
	go func() {
		defer cv.wg.Done()
		for {
			m := new(Message)
			err := dec.Decode(m)
			if err != nil {
				if err != io.EOF {
					select {
					// ignore if quitting
					case <-cv.Quit:
					default:
						cv.CreateEvNewDialog(CtUIDialogTypeInfo, "plugin disconnect",
							fmt.Sprintf("plugin [%s] : connection disconnected", cv.Pgns[n].Name))
						log.Println(err)
					}
				}
				cv.Pgns[n].Rw = nil
				m = nil
			} else {
				m.prgno = n
			}

			select {
			case mes <- m:
				if m == nil {
					return
				}
			case <-cv.Quit:
				return
			}
		}
	}()

	for {
		if !cv.Pgns[n].IsEnable() {
			// wait for being enabled
			select {
			case <-cv.Pgns[n].Startc:
			case <-cv.Quit:
				return
			}
		}

		select {
		// Process received message
		case m := <-mes:
			if m == nil {
				// quit if UI plugin disconnect
				if cv.Pgns[n].Name == pluginNameMain {
					cv.Cmm.Disconnect()
					close(cv.Quit)
				}
				continue
			}

			// ignore if plugin is not enabled
			if cv.Pgns[n].IsEnable() {
				log.Printf("plugin message [%s] : %v", cv.Pgns[n].Name, m)
				cv.Evch <- m
			}

		// Flush plugin IO
		case <-cv.Pgns[n].flushTm.C:
			cv.Pgns[n].Rw.Flush()

		case <-cv.Quit:
			cv.Pgns[n].Rw = nil
			return
		}
	}
}

func sendPluginMessage(cv *CommentViewer) {
	defer cv.wg.Done()

	for {
	readLoop:
		select {
		case mes := <-cv.Evch:
			// Direct
			if mes.Domain == DomainDirectngm {
				plug := cv.Pgns[mes.prgno]
				if plug.Rw == nil {
					continue
				}
				jmes, err := json.Marshal(mes)
				if err != nil {
					log.Println(err)
					log.Println(mes)
					continue
				}
				plug.flushTm.Reset(pluginFlashWaitDu)
				_, err = fmt.Fprintf(plug.Rw, "%s\n", jmes)
				if err != nil {
					cv.CreateEvNewDialog(CtUIDialogTypeInfo, "plugin", "failed to send event : "+plug.Name)
					log.Println(err)

					plug.Rw = nil
					continue
				}
				continue
			}
			if mes.Domain == DomainDirect {
				go func() {
					nicoerr := processDirectMessage(cv, mes)
					if nicoerr != nil {
						log.Printf("plugin message error form [%s] : %s\n", cv.Pgns[mes.prgno].Name, nicoerr)
						log.Println(mes)
					}
				}()
				continue
			}

			// filter

			// Messages from filter plugin will not send same plugin.
			var st int
			if strings.HasSuffix(mes.Domain, DomainFilterSuffix) {
				st = mes.prgno + 1
				mes.Domain = strings.TrimSuffix(mes.Domain, DomainFilterSuffix)
			}
			for i := st; i < len(cv.Pgns); i++ {
				plug := cv.Pgns[i]

				if plug.Rw != nil && plug.DependFilter(mes.Domain) {
					plug.flushTm.Reset(pluginFlashWaitDu)

					// A message to filter plugin has filter domain.
					tmes := *mes
					tmes.Domain = DomainFilterSuffix + mes.Domain
					jmes, err := json.Marshal(tmes)
					if err != nil {
						log.Println(err)
						log.Println(mes)
						break readLoop
					}
					plug.flushTm.Reset(pluginFlashWaitDu)
					_, err = fmt.Fprintf(plug.Rw, "%s\n", jmes)
					if err != nil {
						cv.CreateEvNewDialog(CtUIDialogTypeInfo, "plugin", "failed to send event : "+plug.Name)
						log.Println(err)

						plug.Rw = nil
						continue
					}
					break readLoop
				}
			}

			jmes, err := json.Marshal(mes)
			if err != nil {
				log.Println(err)
				log.Println(mes)
				continue
			}

			var wg sync.WaitGroup

			// regular
			for i := range cv.Pgns {
				wg.Add(1)
				go func(i int) {
					defer wg.Done()
					plug := cv.Pgns[i]

					if plug.Rw != nil && plug.Depend(mes.Domain) {
						plug.flushTm.Reset(pluginFlashWaitDu)
						_, err := fmt.Fprintf(plug.Rw, "%s\n", jmes)
						if err != nil {
							cv.CreateEvNewDialog(CtUIDialogTypeInfo, "plugin", "failed to send event : "+plug.Name)
							log.Println(err)

							plug.Rw = nil
							return
						}
					}
				}(i)
			}

			go func() {
				nicoerr := processPluginMessage(cv, mes)
				if nicoerr != nil {
					log.Printf("plugin message error form [%s] : %s\n", cv.Pgns[mes.prgno].Name, nicoerr)
					log.Println(mes)
				}
			}()

			wg.Wait()

		case <-cv.Quit:
			return
		}

	}
}

func pluginTCPServer(cv *CommentViewer) {
	defer cv.wg.Done()

	adr, err := net.ResolveTCPAddr("tcp", ":"+cv.TCPPort)
	if err != nil {
		log.Panicln(err)
	}
	l, err := net.ListenTCP("tcp", adr)
	if err != nil {
		log.Panicln(err)
	}
	defer l.Close()

	for {
		l.SetDeadline(time.Now().Add(time.Second))
		select {
		default:
			conn, err := l.Accept()
			if err != nil {
				nerr, ok := err.(net.Error)
				if ok && nerr.Timeout() && nerr.Temporary() {
					continue
				}
				log.Println(err)
				continue
			}
			cv.wg.Add(1)
			go handleTCPPlugin(conn, cv)
		case <-cv.Quit:
			return
		}
	}
}

func handleTCPPlugin(c net.Conn, cv *CommentViewer) {
	defer cv.wg.Done()
	defer c.Close()

	rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))

	errc := make(chan struct{})

	cv.wg.Add(1)
	go func() {
		defer cv.wg.Done()
		for {
			select {
			default:
				errf := func(s interface{}) {
					// ignore if quitting
					select {
					case <-cv.Quit:
					default:
						log.Println(s)
					}
					close(errc)
				}

				dec := json.NewDecoder(rw)
				m := new(Message)
				err := dec.Decode(m)
				if err != nil {
					errf(err)
					return
				}
				if m.Domain != DomainDirect || m.Command != CommDirectNo {
					errf("send Direct.No message at first")
					return
				}

				var ct CtDirectNo
				if err := json.Unmarshal(m.Content, &ct); err != nil {
					errf(err)
					return
				}

				n := ct.No - 1
				if n < 0 || n >= len(cv.Pgns) {
					errf("received invalid plugin No.")
					return
				}
				if cv.Pgns[n].Rw != nil {
					errf("plugin is already connected")
					return
				}
				cv.Pgns[n].Rw = rw
				cv.Pgns[n].Start(cv)
				log.Println("loaded plugin ", cv.Pgns[n])
				break

			case <-cv.Quit:
				return
			}
			break
		}

	}()

	// wait for quitting or error in above go routine
	select {
	case <-errc:
	case <-cv.Quit:
	}
}

func handleSTDPlugin(p *plugin, cv *CommentViewer) {
	defer cv.wg.Done()

	if len(p.Exec) < 1 {
		log.Printf("exec is not specified in plugin [%s]\n", p.Name)
		return
	}

	cmd := exec.Command(p.Exec[0], p.Exec[1:]...)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		log.Println(err)
		return
	}
	defer stdin.Close()
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Println(err)
		return
	}
	defer stdout.Close()
	err = cmd.Start()
	if err != nil {
		log.Println(err)
		return
	}

	p.Rw = bufio.NewReadWriter(bufio.NewReader(stdout), bufio.NewWriter(stdin))
	p.Start(cv)
	log.Println("loaded plugin ", p)

	<-cv.Quit
}
<commit_msg>add plugin method isMain()<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os/exec"
	"strings"
	"sync"
	"time"

	"gopkg.in/yaml.v2"
)

const (
	pluginFlashWaitDu time.Duration = 50 * time.Millisecond

	pluginNameMain  string = "main"
	pluginMethodTCP        = "tcp"
	pluginMethodStd        = "std"
)

type plugin struct {
	Name        string            `yaml:"name"        json:"name"`
	Description string            `yaml:"description" json:"description"`
	Version     string            `yaml:"version"     json:"version"`
	Author      string            `yaml:"author"      json:"author"`
	Method      string            `yaml:"method"      json:"method"`
	Exec        []string          `yaml:"exec"        json:"-"`
	Nagomever   string            `yaml:"nagomever"   json:"-"`
	Depends     []string          `yaml:"depends"     json:"depends"`
	No          int               `yaml:"-"           json:"no"`
	Rw          *bufio.ReadWriter `yaml:"-"           json:"-"`
	Startc      chan struct{}     `yaml:"-"           json:"-"`
	flushTm     *time.Timer
	isEnable    bool
}

func (pl *plugin) Init(no int) {
	pl.flushTm = time.NewTimer(time.Hour)
	pl.Startc = make(chan struct{}, 1)
	pl.No = no
}

func (pl *plugin) Start(cv *CommentViewer) {
	if pl.No == 0 {
		log.Printf("plugin \"%s\" is not initialized\n", pl.Name)
		return
	}
	if pl.Name == "" {
		log.Printf("plugin \"%s\" no name is set\n", pl.Name)
		return
	}
	if pl.Rw == nil {
		log.Printf("plugin \"%s\" no rw\n", pl.Name)
		return
	}
	if pl.isEnable {
		return
	}
	pl.Enable()

	pl.Startc <- struct{}{}

	cv.wg.Add(1)
	go eachPluginRw(cv, pl.No-1)
}

func (pl *plugin) Enable() {
	if pl.isEnable {
		return
	}
	pl.isEnable = true

	// send message
	jmes, err := json.Marshal(Message{
		Domain:  DomainDirectngm,
		Command: CommDirectngmPlugEnabled,
	})
	if err != nil {
		log.Println(err)
		return
	}
	fmt.Fprintf(pl.Rw, "%s\n", jmes)
	pl.flushTm.Reset(0)
}

func (pl *plugin) Disable() {
	if !pl.isEnable {
		return
	}
	pl.isEnable = false

	// send message
	jmes, err := json.Marshal(Message{
		Domain:  DomainDirectngm,
		Command: CommDirectngmPlugDisabled,
	})
	if err != nil {
		log.Println(err)
		return
	}
	fmt.Fprintf(pl.Rw, "%s\n", jmes)
	pl.flushTm.Reset(0)
}

func (pl *plugin) IsEnable() bool {
	return pl.isEnable
}

func (pl *plugin) DependFilter(pln string) bool {
	f := false
	for _, d := range pl.Depends {
		if d == pln+DomainFilterSuffix {
			f = true
			break
		}
	}
	return f
}

func (pl *plugin) Depend(pln string) bool {
	f := false
	for _, d := range pl.Depends {
		if d == pln {
			f = true
			break
		}
	}
	return f
}

func (pl *plugin) loadPlugin(filePath string) error {
	d, err := ioutil.ReadFile(filePath)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(d, pl)
	if err != nil {
		return err
	}

	return nil
}

func (pl *plugin) savePlugin(filePath string) error {
	d, err := yaml.Marshal(pl)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(filePath, d, 0600)
	if err != nil {
		return err
	}

	return nil
}

func (pl *plugin) isMain() bool {
	return pl.No == 0
}

// eachPluginRw manages plugins IO. It is launched when a plugin is leaded.
func eachPluginRw(cv *CommentViewer, n int) {
	defer cv.wg.Done()

	// wait for being enabled
	select {
	case <-cv.Pgns[n].Startc:
	case <-cv.Quit:
		return
	}

	// Run decoder.  It puts a message into "mes".
	dec := json.NewDecoder(cv.Pgns[n].Rw)
	mes := make(chan (*Message))
	cv.wg.Add(1)
	go func() {
		defer cv.wg.Done()
		for {
			m := new(Message)
			err := dec.Decode(m)
			if err != nil {
				if err != io.EOF {
					select {
					// ignore if quitting
					case <-cv.Quit:
					default:
						cv.CreateEvNewDialog(CtUIDialogTypeInfo, "plugin disconnect",
							fmt.Sprintf("plugin [%s] : connection disconnected", cv.Pgns[n].Name))
						log.Println(err)
					}
				}
				cv.Pgns[n].Rw = nil
				m = nil
			} else {
				m.prgno = n
			}

			select {
			case mes <- m:
				if m == nil {
					return
				}
			case <-cv.Quit:
				return
			}
		}
	}()

	for {
		if !cv.Pgns[n].IsEnable() {
			// wait for being enabled
			select {
			case <-cv.Pgns[n].Startc:
			case <-cv.Quit:
				return
			}
		}

		select {
		// Process received message
		case m := <-mes:
			if m == nil {
				// quit if UI plugin disconnect
				if cv.Pgns[n].isMain() {
					cv.Cmm.Disconnect()
					close(cv.Quit)
				}
				continue
			}

			// ignore if plugin is not enabled
			if cv.Pgns[n].IsEnable() {
				log.Printf("plugin message [%s] : %v", cv.Pgns[n].Name, m)
				cv.Evch <- m
			}

		// Flush plugin IO
		case <-cv.Pgns[n].flushTm.C:
			cv.Pgns[n].Rw.Flush()

		case <-cv.Quit:
			cv.Pgns[n].Rw = nil
			return
		}
	}
}

func sendPluginMessage(cv *CommentViewer) {
	defer cv.wg.Done()

	for {
	readLoop:
		select {
		case mes := <-cv.Evch:
			// Direct
			if mes.Domain == DomainDirectngm {
				plug := cv.Pgns[mes.prgno]
				if plug.Rw == nil {
					continue
				}
				jmes, err := json.Marshal(mes)
				if err != nil {
					log.Println(err)
					log.Println(mes)
					continue
				}
				plug.flushTm.Reset(pluginFlashWaitDu)
				_, err = fmt.Fprintf(plug.Rw, "%s\n", jmes)
				if err != nil {
					cv.CreateEvNewDialog(CtUIDialogTypeInfo, "plugin", "failed to send event : "+plug.Name)
					log.Println(err)

					plug.Rw = nil
					continue
				}
				continue
			}
			if mes.Domain == DomainDirect {
				go func() {
					nicoerr := processDirectMessage(cv, mes)
					if nicoerr != nil {
						log.Printf("plugin message error form [%s] : %s\n", cv.Pgns[mes.prgno].Name, nicoerr)
						log.Println(mes)
					}
				}()
				continue
			}

			// filter

			// Messages from filter plugin will not send same plugin.
			var st int
			if strings.HasSuffix(mes.Domain, DomainFilterSuffix) {
				st = mes.prgno + 1
				mes.Domain = strings.TrimSuffix(mes.Domain, DomainFilterSuffix)
			}
			for i := st; i < len(cv.Pgns); i++ {
				plug := cv.Pgns[i]

				if plug.Rw != nil && plug.DependFilter(mes.Domain) {
					plug.flushTm.Reset(pluginFlashWaitDu)

					// A message to filter plugin has filter domain.
					tmes := *mes
					tmes.Domain = DomainFilterSuffix + mes.Domain
					jmes, err := json.Marshal(tmes)
					if err != nil {
						log.Println(err)
						log.Println(mes)
						break readLoop
					}
					plug.flushTm.Reset(pluginFlashWaitDu)
					_, err = fmt.Fprintf(plug.Rw, "%s\n", jmes)
					if err != nil {
						cv.CreateEvNewDialog(CtUIDialogTypeInfo, "plugin", "failed to send event : "+plug.Name)
						log.Println(err)

						plug.Rw = nil
						continue
					}
					break readLoop
				}
			}

			jmes, err := json.Marshal(mes)
			if err != nil {
				log.Println(err)
				log.Println(mes)
				continue
			}

			var wg sync.WaitGroup

			// regular
			for i := range cv.Pgns {
				wg.Add(1)
				go func(i int) {
					defer wg.Done()
					plug := cv.Pgns[i]

					if plug.Rw != nil && plug.Depend(mes.Domain) {
						plug.flushTm.Reset(pluginFlashWaitDu)
						_, err := fmt.Fprintf(plug.Rw, "%s\n", jmes)
						if err != nil {
							cv.CreateEvNewDialog(CtUIDialogTypeInfo, "plugin", "failed to send event : "+plug.Name)
							log.Println(err)

							plug.Rw = nil
							return
						}
					}
				}(i)
			}

			go func() {
				nicoerr := processPluginMessage(cv, mes)
				if nicoerr != nil {
					log.Printf("plugin message error form [%s] : %s\n", cv.Pgns[mes.prgno].Name, nicoerr)
					log.Println(mes)
				}
			}()

			wg.Wait()

		case <-cv.Quit:
			return
		}

	}
}

func pluginTCPServer(cv *CommentViewer) {
	defer cv.wg.Done()

	adr, err := net.ResolveTCPAddr("tcp", ":"+cv.TCPPort)
	if err != nil {
		log.Panicln(err)
	}
	l, err := net.ListenTCP("tcp", adr)
	if err != nil {
		log.Panicln(err)
	}
	defer l.Close()

	for {
		l.SetDeadline(time.Now().Add(time.Second))
		select {
		default:
			conn, err := l.Accept()
			if err != nil {
				nerr, ok := err.(net.Error)
				if ok && nerr.Timeout() && nerr.Temporary() {
					continue
				}
				log.Println(err)
				continue
			}
			cv.wg.Add(1)
			go handleTCPPlugin(conn, cv)
		case <-cv.Quit:
			return
		}
	}
}

func handleTCPPlugin(c net.Conn, cv *CommentViewer) {
	defer cv.wg.Done()
	defer c.Close()

	rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))

	errc := make(chan struct{})

	cv.wg.Add(1)
	go func() {
		defer cv.wg.Done()
		for {
			select {
			default:
				errf := func(s interface{}) {
					// ignore if quitting
					select {
					case <-cv.Quit:
					default:
						log.Println(s)
					}
					close(errc)
				}

				dec := json.NewDecoder(rw)
				m := new(Message)
				err := dec.Decode(m)
				if err != nil {
					errf(err)
					return
				}
				if m.Domain != DomainDirect || m.Command != CommDirectNo {
					errf("send Direct.No message at first")
					return
				}

				var ct CtDirectNo
				if err := json.Unmarshal(m.Content, &ct); err != nil {
					errf(err)
					return
				}

				n := ct.No - 1
				if n < 0 || n >= len(cv.Pgns) {
					errf("received invalid plugin No.")
					return
				}
				if cv.Pgns[n].Rw != nil {
					errf("plugin is already connected")
					return
				}
				cv.Pgns[n].Rw = rw
				cv.Pgns[n].Start(cv)
				log.Println("loaded plugin ", cv.Pgns[n])
				break

			case <-cv.Quit:
				return
			}
			break
		}

	}()

	// wait for quitting or error in above go routine
	select {
	case <-errc:
	case <-cv.Quit:
	}
}

func handleSTDPlugin(p *plugin, cv *CommentViewer) {
	defer cv.wg.Done()

	if len(p.Exec) < 1 {
		log.Printf("exec is not specified in plugin [%s]\n", p.Name)
		return
	}

	cmd := exec.Command(p.Exec[0], p.Exec[1:]...)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		log.Println(err)
		return
	}
	defer stdin.Close()
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Println(err)
		return
	}
	defer stdout.Close()
	err = cmd.Start()
	if err != nil {
		log.Println(err)
		return
	}

	p.Rw = bufio.NewReadWriter(bufio.NewReader(stdout), bufio.NewWriter(stdin))
	p.Start(cv)
	log.Println("loaded plugin ", p)

	<-cv.Quit
}
<|endoftext|>


<commit_before>// Copyright 2014 Unknown
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package pongo2 is a middleware that provides pongo2 template engine of Macaron.
package pongo2

import (
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Unknwon/macaron"
	"github.com/flosch/pongo2"
)

const (
	ContentType    = "Content-Type"
	ContentLength  = "Content-Length"
	ContentBinary  = "application/octet-stream"
	ContentJSON    = "application/json"
	ContentHTML    = "text/html"
	ContentXHTML   = "application/xhtml+xml"
	ContentXML     = "text/xml"
	defaultCharset = "UTF-8"
)

var (
	tplMap map[string]*pongo2.Template
	lock   sync.RWMutex // Go map is not safe.
)

func prepareCharset(charset string) string {
	if len(charset) != 0 {
		return "; charset=" + charset
	}

	return "; charset=" + defaultCharset
}

func getExt(s string) string {
	if strings.Index(s, ".") == -1 {
		return ""
	}
	return "." + strings.Join(strings.Split(s, ".")[1:], ".")
}

func compile(options Options) {
	lock.Lock()
	defer lock.Unlock()

	dir := options.Directory
	tplMap = make(map[string]*pongo2.Template)

	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		r, err := filepath.Rel(dir, path)
		if err != nil {
			return err
		}

		ext := getExt(r)

		for _, extension := range options.Extensions {
			if ext == extension {
				name := (r[0 : len(r)-len(ext)])
				// Bomb out if parse fails. We don't want any silent server starts.
				tplMap[name] = pongo2.Must(pongo2.FromFile(path))
				break
			}
		}

		return nil
	})
}

// Options represents a struct for specifying configuration options for the Render middleware.
type Options struct {
	// Directory to load templates. Default is "templates"
	Directory string
	// Extensions to parse template files from. Defaults to [".tmpl", ".html"]
	Extensions []string
	// Funcs is a slice of FuncMaps to apply to the template upon compilation. This is useful for helper functions. Defaults to [].
	// Funcs []template.FuncMap
	// Appends the given charset to the Content-Type header. Default is "UTF-8".
	Charset string
	// Outputs human readable JSON
	IndentJSON bool
	// Outputs human readable XML
	IndentXML bool
	// Prefixes the JSON output with the given bytes.
	PrefixJSON []byte
	// Prefixes the XML output with the given bytes.
	PrefixXML []byte
	// Allows changing of output to XHTML instead of HTML. Default is "text/html"
	HTMLContentType string
}

func prepareOptions(options []Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = options[0]
	}

	// Defaults
	if len(opt.Directory) == 0 {
		opt.Directory = "templates"
	}
	if len(opt.Extensions) == 0 {
		opt.Extensions = []string{".tmpl", ".html"}
	}
	if len(opt.HTMLContentType) == 0 {
		opt.HTMLContentType = ContentHTML
	}

	return opt
}

// Pongoer is a Middleware that maps a macaron.Render service into the Macaron handler chain.
// An single variadic pongo2.Options struct can be optionally provided to configure
// HTML rendering. The default directory for templates is "templates" and the default
// file extension is ".tmpl" and ".html".
//
// If MACARON_ENV is set to "" or "development" then templates will be recompiled on every request. For more performance, set the
// MACARON_ENV environment variable to "production".
func Pongoer(options ...Options) macaron.Handler {
	opt := prepareOptions(options)
	cs := prepareCharset(opt.Charset)
	compile(opt)

	return func(ctx *macaron.Context, rw http.ResponseWriter) {
		if macaron.Env == macaron.DEV {
			compile(opt)
		}
		r := &render{
			TplRender: &macaron.TplRender{
				ResponseWriter: rw,
				Opt: macaron.RenderOptions{
					IndentJSON: opt.IndentJSON,
					IndentXML:  opt.IndentXML,
					PrefixJSON: opt.PrefixJSON,
					PrefixXML:  opt.PrefixXML,
				},
				CompiledCharset: cs,
			},
			ResponseWriter:  rw,
			opt:             opt,
			compiledCharset: cs,
		}
		ctx.Render = r
		ctx.MapTo(r, (*macaron.Render)(nil))
	}
}

type render struct {
	*macaron.TplRender
	http.ResponseWriter
	opt             Options
	compiledCharset string

	startTime time.Time
}

func data2Context(data interface{}) pongo2.Context {
	return pongo2.Context(data.(map[string]interface{}))
}

func (r *render) HTML(status int, name string, data interface{}, _ ...macaron.HTMLOptions) {
	r.startTime = time.Now()

	lock.RLock()
	defer lock.RUnlock()

	t := tplMap[name]
	if t == nil {
		http.Error(r, "pongo2: \""+name+"\" is undefined", http.StatusInternalServerError)
		return
	}

	r.Header().Set(ContentType, r.opt.HTMLContentType+r.compiledCharset)
	r.WriteHeader(status)
	if err := t.ExecuteRW(r, pongo2.Context(data2Context(data))); err != nil {
		http.Error(r, err.Error(), http.StatusInternalServerError)
		return
	}
}

func (r *render) HTMLString(name string, data interface{}, _ ...macaron.HTMLOptions) (string, error) {
	lock.RLock()
	defer lock.RUnlock()

	t := tplMap[name]
	if t == nil {
		http.Error(r, "pongo2: \""+name+"\" is undefined", http.StatusInternalServerError)
		return "", nil
	}

	out, err := t.Execute(data2Context(data))
	if err != nil {
		return "", err
	}

	return out, nil
}
<commit_msg>Making changes according to a recent API-Change<commit_after>// Copyright 2014 Unknown
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package pongo2 is a middleware that provides pongo2 template engine of Macaron.
package pongo2

import (
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Unknwon/macaron"
	"github.com/flosch/pongo2"
)

const (
	ContentType    = "Content-Type"
	ContentLength  = "Content-Length"
	ContentBinary  = "application/octet-stream"
	ContentJSON    = "application/json"
	ContentHTML    = "text/html"
	ContentXHTML   = "application/xhtml+xml"
	ContentXML     = "text/xml"
	defaultCharset = "UTF-8"
)

var (
	tplMap map[string]*pongo2.Template
	lock   sync.RWMutex // Go map is not safe.
)

func prepareCharset(charset string) string {
	if len(charset) != 0 {
		return "; charset=" + charset
	}

	return "; charset=" + defaultCharset
}

func getExt(s string) string {
	if strings.Index(s, ".") == -1 {
		return ""
	}
	return "." + strings.Join(strings.Split(s, ".")[1:], ".")
}

func compile(options Options) {
	lock.Lock()
	defer lock.Unlock()

	dir := options.Directory
	tplMap = make(map[string]*pongo2.Template)

	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		r, err := filepath.Rel(dir, path)
		if err != nil {
			return err
		}

		ext := getExt(r)

		for _, extension := range options.Extensions {
			if ext == extension {
				name := (r[0 : len(r)-len(ext)])
				// Bomb out if parse fails. We don't want any silent server starts.
				tplMap[name] = pongo2.Must(pongo2.FromFile(path))
				break
			}
		}

		return nil
	})
}

// Options represents a struct for specifying configuration options for the Render middleware.
type Options struct {
	// Directory to load templates. Default is "templates"
	Directory string
	// Extensions to parse template files from. Defaults to [".tmpl", ".html"]
	Extensions []string
	// Funcs is a slice of FuncMaps to apply to the template upon compilation. This is useful for helper functions. Defaults to [].
	// Funcs []template.FuncMap
	// Appends the given charset to the Content-Type header. Default is "UTF-8".
	Charset string
	// Outputs human readable JSON
	IndentJSON bool
	// Outputs human readable XML
	IndentXML bool
	// Prefixes the JSON output with the given bytes.
	PrefixJSON []byte
	// Prefixes the XML output with the given bytes.
	PrefixXML []byte
	// Allows changing of output to XHTML instead of HTML. Default is "text/html"
	HTMLContentType string
}

func prepareOptions(options []Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = options[0]
	}

	// Defaults
	if len(opt.Directory) == 0 {
		opt.Directory = "templates"
	}
	if len(opt.Extensions) == 0 {
		opt.Extensions = []string{".tmpl", ".html"}
	}
	if len(opt.HTMLContentType) == 0 {
		opt.HTMLContentType = ContentHTML
	}

	return opt
}

// Pongoer is a Middleware that maps a macaron.Render service into the Macaron handler chain.
// An single variadic pongo2.Options struct can be optionally provided to configure
// HTML rendering. The default directory for templates is "templates" and the default
// file extension is ".tmpl" and ".html".
//
// If MACARON_ENV is set to "" or "development" then templates will be recompiled on every request. For more performance, set the
// MACARON_ENV environment variable to "production".
func Pongoer(options ...Options) macaron.Handler {
	opt := prepareOptions(options)
	cs := prepareCharset(opt.Charset)
	compile(opt)

	return func(ctx *macaron.Context, rw http.ResponseWriter) {
		if macaron.Env == macaron.DEV {
			compile(opt)
		}
		r := &render{
			TplRender: &macaron.TplRender{
				ResponseWriter: rw,
				Opt: macaron.RenderOptions{
					IndentJSON: opt.IndentJSON,
					IndentXML:  opt.IndentXML,
					PrefixJSON: opt.PrefixJSON,
					PrefixXML:  opt.PrefixXML,
				},
				CompiledCharset: cs,
			},
			ResponseWriter:  rw,
			opt:             opt,
			compiledCharset: cs,
		}
		ctx.Render = r
		ctx.MapTo(r, (*macaron.Render)(nil))
	}
}

type render struct {
	*macaron.TplRender
	http.ResponseWriter
	opt             Options
	compiledCharset string

	startTime time.Time
}

func data2Context(data interface{}) pongo2.Context {
	return pongo2.Context(data.(map[string]interface{}))
}

func (r *render) HTML(status int, name string, data interface{}, _ ...macaron.HTMLOptions) {
	r.startTime = time.Now()

	lock.RLock()
	defer lock.RUnlock()

	t := tplMap[name]
	if t == nil {
		http.Error(r, "pongo2: \""+name+"\" is undefined", http.StatusInternalServerError)
		return
	}

	r.Header().Set(ContentType, r.opt.HTMLContentType+r.compiledCharset)
	r.WriteHeader(status)
	if err := t.ExecuteWriter(pongo2.Context(data2Context(data)), r); err != nil {
		http.Error(r, err.Error(), http.StatusInternalServerError)
		return
	}
}

func (r *render) HTMLString(name string, data interface{}, _ ...macaron.HTMLOptions) (string, error) {
	lock.RLock()
	defer lock.RUnlock()

	t := tplMap[name]
	if t == nil {
		http.Error(r, "pongo2: \""+name+"\" is undefined", http.StatusInternalServerError)
		return "", nil
	}

	out, err := t.Execute(data2Context(data))
	if err != nil {
		return "", err
	}

	return out, nil
}
<|endoftext|>


<commit_before>/*
   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package utils

import (
	"crypto"
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/vmware/harbor/dao"
	"github.com/vmware/harbor/utils/log"

	"github.com/docker/distribution/registry/auth/token"
	"github.com/docker/libtrust"
)

const (
	issuer     = "registry-token-issuer"
	privateKey = "/etc/ui/private_key.pem"
	expiration = 5 //minute
)

// GetResourceActions ...
func GetResourceActions(scopes []string) []*token.ResourceActions {
	var res []*token.ResourceActions
	for _, s := range scopes {
		if s == "" {
			continue
		}
		items := strings.Split(s, ":")
		res = append(res, &token.ResourceActions{
			Type:    items[0],
			Name:    items[1],
			Actions: strings.Split(items[2], ","),
		})
	}
	return res
}

// FilterAccess modify the action list in access based on permission
// determine if the request needs to be authenticated.
func FilterAccess(username string, authenticated bool, a *token.ResourceActions) {

	if a.Type == "registry" && a.Name == "catalog" {
		return
	}

	//clear action list to assign to new acess element after perm check.
	a.Actions = []string{}
	if a.Type == "repository" {
		if strings.Contains(a.Name, "/") { //Only check the permission when the requested image has a namespace, i.e. project
			projectName := a.Name[0:strings.LastIndex(a.Name, "/")]
			var permission string
			if authenticated {
				isAdmin, err := dao.IsAdminRole(username)
				if err != nil {
					log.Errorf("Error occurred in IsAdminRole: %v")
				}
				if isAdmin {
					exist, err := dao.ProjectExists(projectName)
					if err != nil {
						log.Errorf("Error occurred in CheckExistProject: %v", err)
						return
					}
					if exist {
						permission = "RW"
					} else {
						permission = ""
						log.Infof("project %s does not exist, set empty permission for admin\n", projectName)
					}
				} else {
					permission, err = dao.GetPermission(username, projectName)
					if err != nil {
						log.Errorf("Error occurred in GetPermission: %v", err)
						return
					}
				}
			}
			if strings.Contains(permission, "W") {
				a.Actions = append(a.Actions, "push")
			}
			if strings.Contains(permission, "R") || dao.IsProjectPublic(projectName) {
				a.Actions = append(a.Actions, "pull")
			}
		}
	}
	log.Infof("current access, type: %s, name:%s, actions:%v \n", a.Type, a.Name, a.Actions)
}

// GenTokenForUI is for the UI process to call, so it won't establish a https connection from UI to proxy.
func GenTokenForUI(username string, service string, scopes []string) (string, error) {
	access := GetResourceActions(scopes)
	for _, a := range access {
		FilterAccess(username, true, a)
	}
	return MakeToken(username, service, access)
}

// MakeToken makes a valid jwt token based on parms.
func MakeToken(username, service string, access []*token.ResourceActions) (string, error) {
	pk, err := libtrust.LoadKeyFile(privateKey)
	if err != nil {
		return "", err
	}
	tk, err := makeTokenCore(issuer, username, service, expiration, access, pk)
	if err != nil {
		return "", err
	}
	rs := fmt.Sprintf("%s.%s", tk.Raw, base64UrlEncode(tk.Signature))
	return rs, nil
}

//make token core
func makeTokenCore(issuer, subject, audience string, expiration int,
	access []*token.ResourceActions, signingKey libtrust.PrivateKey) (*token.Token, error) {

	joseHeader := &token.Header{
		Type:       "JWT",
		SigningAlg: "RS256",
		KeyID:      signingKey.KeyID(),
	}

	jwtID, err := randString(16)
	if err != nil {
		return nil, fmt.Errorf("Error to generate jwt id: %s", err)
	}

	now := time.Now()

	claimSet := &token.ClaimSet{
		Issuer:     issuer,
		Subject:    subject,
		Audience:   audience,
		Expiration: now.Add(time.Duration(expiration) * time.Minute).Unix(),
		NotBefore:  now.Unix(),
		IssuedAt:   now.Unix(),
		JWTID:      jwtID,
		Access:     access,
	}

	var joseHeaderBytes, claimSetBytes []byte

	if joseHeaderBytes, err = json.Marshal(joseHeader); err != nil {
		return nil, fmt.Errorf("unable to marshal jose header: %s", err)
	}
	if claimSetBytes, err = json.Marshal(claimSet); err != nil {
		return nil, fmt.Errorf("unable to marshal claim set: %s", err)
	}

	encodedJoseHeader := base64UrlEncode(joseHeaderBytes)
	encodedClaimSet := base64UrlEncode(claimSetBytes)
	payload := fmt.Sprintf("%s.%s", encodedJoseHeader, encodedClaimSet)

	var signatureBytes []byte
	if signatureBytes, _, err = signingKey.Sign(strings.NewReader(payload), crypto.SHA256); err != nil {
		return nil, fmt.Errorf("unable to sign jwt payload: %s", err)
	}

	signature := base64UrlEncode(signatureBytes)
	tokenString := fmt.Sprintf("%s.%s", payload, signature)
	return token.NewToken(tokenString)
}

func randString(length int) (string, error) {
	const alphanum = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
	rb := make([]byte, length)
	_, err := rand.Read(rb)
	if err != nil {
		return "", err
	}
	for i, b := range rb {
		rb[i] = alphanum[int(b)%len(alphanum)]
	}
	return string(rb), nil
}

func base64UrlEncode(b []byte) string {
	return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=")
}
<commit_msg>fix error in go vet<commit_after>/*
   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package utils

import (
	"crypto"
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/vmware/harbor/dao"
	"github.com/vmware/harbor/utils/log"

	"github.com/docker/distribution/registry/auth/token"
	"github.com/docker/libtrust"
)

const (
	issuer     = "registry-token-issuer"
	privateKey = "/etc/ui/private_key.pem"
	expiration = 5 //minute
)

// GetResourceActions ...
func GetResourceActions(scopes []string) []*token.ResourceActions {
	var res []*token.ResourceActions
	for _, s := range scopes {
		if s == "" {
			continue
		}
		items := strings.Split(s, ":")
		res = append(res, &token.ResourceActions{
			Type:    items[0],
			Name:    items[1],
			Actions: strings.Split(items[2], ","),
		})
	}
	return res
}

// FilterAccess modify the action list in access based on permission
// determine if the request needs to be authenticated.
func FilterAccess(username string, authenticated bool, a *token.ResourceActions) {

	if a.Type == "registry" && a.Name == "catalog" {
		return
	}

	//clear action list to assign to new acess element after perm check.
	a.Actions = []string{}
	if a.Type == "repository" {
		if strings.Contains(a.Name, "/") { //Only check the permission when the requested image has a namespace, i.e. project
			projectName := a.Name[0:strings.LastIndex(a.Name, "/")]
			var permission string
			if authenticated {
				isAdmin, err := dao.IsAdminRole(username)
				if err != nil {
					log.Errorf("Error occurred in IsAdminRole: %v", err)
				}
				if isAdmin {
					exist, err := dao.ProjectExists(projectName)
					if err != nil {
						log.Errorf("Error occurred in CheckExistProject: %v", err)
						return
					}
					if exist {
						permission = "RW"
					} else {
						permission = ""
						log.Infof("project %s does not exist, set empty permission for admin\n", projectName)
					}
				} else {
					permission, err = dao.GetPermission(username, projectName)
					if err != nil {
						log.Errorf("Error occurred in GetPermission: %v", err)
						return
					}
				}
			}
			if strings.Contains(permission, "W") {
				a.Actions = append(a.Actions, "push")
			}
			if strings.Contains(permission, "R") || dao.IsProjectPublic(projectName) {
				a.Actions = append(a.Actions, "pull")
			}
		}
	}
	log.Infof("current access, type: %s, name:%s, actions:%v \n", a.Type, a.Name, a.Actions)
}

// GenTokenForUI is for the UI process to call, so it won't establish a https connection from UI to proxy.
func GenTokenForUI(username string, service string, scopes []string) (string, error) {
	access := GetResourceActions(scopes)
	for _, a := range access {
		FilterAccess(username, true, a)
	}
	return MakeToken(username, service, access)
}

// MakeToken makes a valid jwt token based on parms.
func MakeToken(username, service string, access []*token.ResourceActions) (string, error) {
	pk, err := libtrust.LoadKeyFile(privateKey)
	if err != nil {
		return "", err
	}
	tk, err := makeTokenCore(issuer, username, service, expiration, access, pk)
	if err != nil {
		return "", err
	}
	rs := fmt.Sprintf("%s.%s", tk.Raw, base64UrlEncode(tk.Signature))
	return rs, nil
}

//make token core
func makeTokenCore(issuer, subject, audience string, expiration int,
	access []*token.ResourceActions, signingKey libtrust.PrivateKey) (*token.Token, error) {

	joseHeader := &token.Header{
		Type:       "JWT",
		SigningAlg: "RS256",
		KeyID:      signingKey.KeyID(),
	}

	jwtID, err := randString(16)
	if err != nil {
		return nil, fmt.Errorf("Error to generate jwt id: %s", err)
	}

	now := time.Now()

	claimSet := &token.ClaimSet{
		Issuer:     issuer,
		Subject:    subject,
		Audience:   audience,
		Expiration: now.Add(time.Duration(expiration) * time.Minute).Unix(),
		NotBefore:  now.Unix(),
		IssuedAt:   now.Unix(),
		JWTID:      jwtID,
		Access:     access,
	}

	var joseHeaderBytes, claimSetBytes []byte

	if joseHeaderBytes, err = json.Marshal(joseHeader); err != nil {
		return nil, fmt.Errorf("unable to marshal jose header: %s", err)
	}
	if claimSetBytes, err = json.Marshal(claimSet); err != nil {
		return nil, fmt.Errorf("unable to marshal claim set: %s", err)
	}

	encodedJoseHeader := base64UrlEncode(joseHeaderBytes)
	encodedClaimSet := base64UrlEncode(claimSetBytes)
	payload := fmt.Sprintf("%s.%s", encodedJoseHeader, encodedClaimSet)

	var signatureBytes []byte
	if signatureBytes, _, err = signingKey.Sign(strings.NewReader(payload), crypto.SHA256); err != nil {
		return nil, fmt.Errorf("unable to sign jwt payload: %s", err)
	}

	signature := base64UrlEncode(signatureBytes)
	tokenString := fmt.Sprintf("%s.%s", payload, signature)
	return token.NewToken(tokenString)
}

func randString(length int) (string, error) {
	const alphanum = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
	rb := make([]byte, length)
	_, err := rand.Read(rb)
	if err != nil {
		return "", err
	}
	for i, b := range rb {
		rb[i] = alphanum[int(b)%len(alphanum)]
	}
	return string(rb), nil
}

func base64UrlEncode(b []byte) string {
	return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=")
}
<|endoftext|>


<commit_before>package fixity

import (
	"io"

	"github.com/leeola/errors"
	"github.com/leeola/fixity/q"
)

// Fixity implements writing, indexing and reading with a Fixity store.
//
// This interface will be implemented for multiple stores, such as a local on
// disk store and a remote over network store.
type Fixity interface {
	// Blob returns a raw blob of the given hash.
	//
	// Mainly useful for inspecting the underlying data structure.
	Blob(hash string) (io.ReadCloser, error)

	Head() (Block, error)

	Read(id string) (Content, error)

	ReadHash(hash string) (Content, error)

	// Remove marks the given block's content to be garbage collected eventually.
	//
	// Each Content, Blob and Chunk will be deleted if no other block in the
	// blockchain depends on it. This is a slow process.
	//
	// If the block is not a content block, an error will be returned.
	Remove(id string) error

	// Write a block for the given reader and index fields.
	Write(id string, r io.Reader, f ...Field) ([]string, error)

	// // Search for documents matching the given query.
	Search(*q.Query) ([]string, error)

	// TODO(leeola): Enable a close method to shutdown any
	//
	// // Close shuts down any connections that may need to be closed.
	// Close() error
}

type Block struct {
	Block             int    `json:"block"`
	PreviousBlockHash string `json:"previousBlockHash"`

	//Deletion  *Deletion  `json:"deletion,omitempty"`
	//Deletions  *Deletion  `json:"deletion,omitempty"`
	//Append  *Append  `json:"append,omitempty"`

	ContentHash string `json:"cotentHash,omitempty"`

	// BlockHash is the hash of the Block itself, provided by Fixity.
	BlockHash string `json:"-"`

	// Store allows block method(s) to load previous blocks and content.
	Store Store `json:"-"`
}

type Deletion struct {
	BlockHash   string `json:"blockHash"`
	ContentHash string `json:"contentHash,omitempty"`
}

type Deletions []Deletion

type Content struct {
	Id                  string `json:"id,omitempty"`
	PreviousContentHash string `json:"previousContentHash,omitempty"`
	BlobHash            string `json:"blobHash"`
	IndexedFields       Fields `json:"indexedFields,omitempty"`

	// ReadCloser allows the Content to be read from directly.
	io.ReadCloser `json:"-"`

	// ContentHash is the hash of the Content itself, provided by Fixity.
	ContentHash string `json:"-"`

	// Store allows block method(s) to load previous content.
	Store Store `json:"-"`
}

type Blob struct {
	ChunkHashes []string `json:"chunkHashes"`
	Size        int64    `json:"size"`
	RollSize    int      `json:"rollSize"`

	// ReadCloser allows the Blob to be read from directly.
	io.ReadCloser `json:"-"`
}

type Chunk struct {
	ChunkBytes []byte `json:"chunkBytes"`
	Size       int64  `json:"size"`
}

func (b *Block) PreviousBlock() (Block, error) {
	if b.Store == nil {
		return Block{}, errors.New("Store not set")
	}

	if b.PreviousBlockHash == "" {
		return Block{}, nil
	}

	var previousBlock Block
	err := readAndUnmarshal(b.Store, b.PreviousBlockHash, &previousBlock)
	if err != nil {
		return Block{}, err
	}

	previousBlock.BlockHash = b.PreviousBlockHash
	previousBlock.Store = b.Store

	return previousBlock, nil
}

func (b *Block) Content() (Content, error) {
	if b.Store == nil {
		return Content{}, errors.New("Store not set")
	}

	if b.ContentHash == "" {
		return Content{}, errors.New("contentHash is empty")
	}

	var c Content
	err := readAndUnmarshal(b.Store, b.ContentHash, &c)
	if err != nil {
		return Content{}, err
	}

	return c, nil
}
<commit_msg>feat: fleshed out the blockchain and skipping more.<commit_after>package fixity

import (
	"io"

	"github.com/leeola/errors"
	"github.com/leeola/fixity/q"
)

// Fixity implements user focused writing and reading of data.
//
// This interface will be implemented for multiple stores, such as a local on
// disk store and a remote over network store.
type Fixity interface {
	// Blob returns a raw blob of the given hash.
	//
	// Mainly useful for inspecting the underlying data structure.
	Blob(hash string) (io.ReadCloser, error)

	// Blockchain allows one to manage and inspect the Fixity Blockchain.
	//
	// The blockchain is low level and should be used with care. See Blockchain
	// docstring for further details.
	Blockchain() Blockchain

	// Delete the given id's content from the fixity store.
	//
	// Each Content, Blob and Chunk will be deleted if no other block in the
	// blockchain depends on it. Verifying this is done by the garbage
	// collector and is a slow process.
	//
	// All blocks for the given id will be removed from the blockchain.
	Delete(id string) error

	// Read the latest Content with the given id.
	Read(id string) (Content, error)

	// Read the Content with the given hash.
	ReadHash(hash string) (Content, error)

	// Write the given reader to the fixity store and index fields.
	Write(id string, r io.Reader, f ...Field) ([]string, error)

	// Search for documents matching the given query.
	Search(*q.Query) ([]string, error)

	// Close shuts down any connections that may need to be closed.
	Close() error
}

// Blockchain implements low level block management methods for Fixity.
//
// The fixity blockchain does not contain any traditional proof of work
// and does not have anything related to crypto or crypto currencies.
// The name was chosen in hopes to clearly express the ledger side of
// blockchains. While there will be many similarities in how the Fixity
// blockchain uses immutability and history, there will also be many
// differences from popular blockchains.
//
// A blockchain in Fixity serves as a ledger for what content exists
// on a distributed Fixity network. If a hash address cannot be found
// within one of the Blocks on the blockchain, such as the hash of
// a Content, Blob or Chunk, then it is considered available for
// garbage collection and will be removed.
//
// The Fixity blockchain consists of three main parts: The Block number,
// the PreviousBlockHash and some data effectively giving the Block a
// "type".
//
// The Block number is an ever incrementing value and with the help of
// PreviousBlockHash it provides a way for distributed nodes to achieve
// eventual consensus.
//
// The PreviousBlockHash provides a way to track the entire blockchain
// from the Head() Block. It also provides a way for the blockchain itself
// to be mutable, in an immutable environment. More on mutability soon.
//
// The Block type is the reason why the block exists. It may have added
// Content, removed Content, mutated the chain, etc.
//
// Mutability of the blockchain is achieved by appending new blocks that
// skip one or more blocks in their PreviousBlockHash. For example, with
// a blockchain of 5 blocks, Block 6 could be written with a
// PreviousBlockHash set to the hash of Block 4. Since Block 6 is the head,
// traversing the blockchain would look like: Block 6 -> Block 4 -> Block 3
// and so on. Note that Blocks start with 0 index, but for these examples
// we're not using zero index.
//
// To achieve mutability on blocks that aren't currently the Head()
// as Block 5 was in the previous example, all Blocks from the target
// Block to the Head() must be rewritten to the blockchain. In order.
// This means that removing old blocks can be costly and slow.
//
// Fixity strives to keep the ledger as a trustable and easy to verify
// chain. An alternative to block skipping would be to write a content
// deletion block, essentially writing to the ledger that content is
// to be garbage collected. However, verifying the ever growing blockchain
// would mean needing to reference these deletion blocks frequently to know
// what content should and shouldn't be looked into. In otherwords,
// content on the blockchain may have a deletion block for it further up
// the chain, so verifying content of the ledger becomes difficult.
//
// The chosen method of content skipping does most of the difficult work
// up front and results in a very clean ledger. It does this at the cost
// of needing to complicate the removal/skipping process.
//
// This interface focuses on all of the above functionality.
type Blockchain interface {
	// AppendBlocks locks the store and writes the given blocks in order.
	//
	// The field PreviousBlockHash's value of all blocks *must* be empty.
	//
	// The returned Block array will contain the new hashes of the given
	// blocks.
	AppendBlocks(appendTo Block, blocks []Block) ([]Block, error)

	// Head returns the latest block in the blockchain.
	Head() (Block, error)

	// SkipBlock removes the given block from the blockchain.
	SkipBlock(Block) ([]Block, error)
}

// Block serves as a ledger for mutations of the fixity datastore.
//
// Each block stores an always incrementing Block number and a hash of
// the previous block in the chain. These two fields allow a fixity
// store to be iterated through the always appending history.
//
// While the history is always appending, previous blocks may be skipped,
// effectively removing them from the history of the blockchain. This is
// done by writing a new block whose PreviousBlockHash value skips one or
// more previous blocks in the chain.
type Block struct {
	// Block is the ever incrementing block number for this block.
	//
	// Each block will be incremented from the previous block.
	Block int `json:"block"`

	// PreviousBlockHash is the hash of the block that came before this.
	//
	// Note that the blockchain itself is mutable, such that the
	// PreviousBlockHash isn't guaranteed to have the block number of Block-1.
	// If a block was skipped, the block numbers may differ.
	//
	// See FixityBlockchain.SkipBlock for more information on block skipping
	// and implications of that.
	PreviousBlockHash string `json:"previousBlockHash"`

	// Skip contains Skip data and makes this Block a Skip Block.
	Skip *Skip `json:"skip,omitempty"`

	// ContentHash contains the ContentHash and makes this block a Content block.
	ContentHash string `json:"cotentHash,omitempty"`

	// BlockHash is the hash of the Block itself, provided by Fixity.
	BlockHash string `json:"-"`

	// Store allows block method(s) to load previous blocks and content.
	Store Store `json:"-"`
}

// Skip blocks provide information about the block that was skipped.
type Skip struct {
	// BlockHash of the block to be skipped.
	BlockHash string `json:"blockHash"`
}

// Content stores blob, index and history information for Fixity content.
type Content struct {
	// Id provides a user friendly way to reference a chain of Contents.
	//
	// History of Content is tracked through the PreviousContentHash chain,
	// however that does not provide a clear single identity for users.
	// The id field allows this, can be indexed and assocoated and is
	// easy to conceptualize.
	Id string `json:"id,omitempty"`

	// PreviousContentHash stores the previous Content for this Content.
	//
	// This allows a single entity, such as a file or a database "record"
	// to be mutated through time. To reference this history of contents,
	// the Id is used.
	PreviousContentHash string `json:"previousContentHash,omitempty"`

	// BlobHash is the hash of the  Blob containing this content's data.
	BlobHash string `json:"blobHash"`

	// IndexedFields contains the indexed metadata for this content.
	//
	// This allows the content to be searched for and can be used to
	// store basic metadata about the content.
	IndexedFields Fields `json:"indexedFields,omitempty"`

	// ReadCloser allows the Content to be read from directly.
	//
	// TODO(leeola): Remove this in favor of a Blob() method which
	// returns the Blob & embedded reader. This saves creation of the
	// reader until it's been explicitly requested.
	//
	// This value is not stored.
	io.ReadCloser `json:"-"`

	// ContentHash is the hash of the Content itself, provided by Fixity.
	//
	// This value is not stored.
	ContentHash string `json:"-"`

	// Store allows block method(s) to load previous content.
	//
	// This value is not stored.
	Store Store `json:"-"`
}

// Blob stores a series of ordered ChunkHashes
type Blob struct {
	ChunkHashes []string `json:"chunkHashes"`
	Size        int64    `json:"size,omitempty"`
	RollSize    int      `json:"rollSize,omitempty"`

	// NextBlobHash is not currently supported / implemented anywhere, but
	// is required for very large storage. Eg, if there are so many chunks
	// for a given dataset that it cannot be stored in memory during writing
	// and reading, then we will need to split them up via NextBlobHash.
	//
	// // NextBlobHash stores another blob which is to be appended to this blob.
	// //
	// // This serves to allow very large blobs that cannot be loaded entirely
	// // into to memory to be split up into many parts.
	// NextBlobHash string `json:"nextBlobHash,omitempty"`

	// ReadCloser allows the Blob to be read from directly.
	//
	// This value is not stored.
	io.ReadCloser `json:"-"`
}

type Chunk struct {
	ChunkBytes []byte `json:"chunkBytes"`
	Size       int64  `json:"size"`
}

func (b *Block) PreviousBlock() (Block, error) {
	if b.Store == nil {
		return Block{}, errors.New("Store not set")
	}

	if b.PreviousBlockHash == "" {
		return Block{}, nil
	}

	var previousBlock Block
	err := readAndUnmarshal(b.Store, b.PreviousBlockHash, &previousBlock)
	if err != nil {
		return Block{}, err
	}

	previousBlock.BlockHash = b.PreviousBlockHash
	previousBlock.Store = b.Store

	return previousBlock, nil
}

func (b *Block) Content() (Content, error) {
	if b.Store == nil {
		return Content{}, errors.New("Store not set")
	}

	if b.ContentHash == "" {
		return Content{}, errors.New("contentHash is empty")
	}

	var c Content
	err := readAndUnmarshal(b.Store, b.ContentHash, &c)
	if err != nil {
		return Content{}, err
	}

	return c, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"strconv"
)

const (
	pathError = "Path parameter must be a number."
	rangeError = "Number must be from 1 to 100."
)

func handler(w http.ResponseWriter, r *http.Request) {
	s := r.URL.Path[1:]
	n, err := strconv.Atoi(s)
	if err != nil {
		fmt.Fprint(w, pathError)
		return
	}
	if n < 1 || n > 100 {
		fmt.Fprint(w, rangeError)
		return
	}
	f, b := n%3 == 0, n%5 == 0
	if f || b {
		s = ""
	}
	if f {
		s = "Fizz"
	}
	if b {
		s += "Buzz"
	}
	fmt.Fprintf(w, s)
}

func main() {
	args := struct{ port string }{}
	flag.StringVar(&args.port, "port", "80", "The port to serve on.")
	flag.Parse()
	http.HandleFunc("/", handler)
	if err := http.ListenAndServe(args.port, nil); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Format code<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"strconv"
)

const (
	pathError  = "Path parameter must be a number."
	rangeError = "Number must be from 1 to 100."
)

func handler(w http.ResponseWriter, r *http.Request) {
	s := r.URL.Path[1:]
	n, err := strconv.Atoi(s)
	if err != nil {
		fmt.Fprint(w, pathError)
		return
	}
	if n < 1 || n > 100 {
		fmt.Fprint(w, rangeError)
		return
	}
	f, b := n%3 == 0, n%5 == 0
	if f || b {
		s = ""
	}
	if f {
		s = "Fizz"
	}
	if b {
		s += "Buzz"
	}
	fmt.Fprintf(w, s)
}

func main() {
	args := struct{ port string }{}
	flag.StringVar(&args.port, "port", "80", "The port to serve on.")
	flag.Parse()
	http.HandleFunc("/", handler)
	if err := http.ListenAndServe(args.port, nil); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>