<commit_before>package rancher

import (
	"fmt"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	"github.com/docker/libcompose/config"
	"github.com/docker/libcompose/project"
	"github.com/rancher/go-rancher/v2"
)

type RancherVolumesFactory struct {
	Context *Context
}

func (f *RancherVolumesFactory) Create(projectName string, volumeConfigs map[string]*config.VolumeConfig, serviceConfigs *config.ServiceConfigs, volumeEnabled bool) (project.Volumes, error) {
	volumes := make([]*Volume, 0, len(volumeConfigs))
	for name, config := range volumeConfigs {
		volume := NewVolume(projectName, name, config, f.Context)
		volumes = append(volumes, volume)
	}
	return &Volumes{
		volumes:       volumes,
		volumeEnabled: volumeEnabled,
		Context:       f.Context,
	}, nil
}

type Volumes struct {
	volumes       []*Volume
	volumeEnabled bool
	Context       *Context
}

func (v *Volumes) Initialize(ctx context.Context) error {
	if !v.volumeEnabled {
		return nil
	}
	for _, volume := range v.volumes {
		if err := volume.EnsureItExists(ctx); err != nil {
			return err
		}
	}

	return nil
}

func (v *Volumes) Remove(ctx context.Context) error {
	if !v.volumeEnabled {
		return nil
	}
	for _, volume := range v.volumes {
		if err := volume.Remove(ctx); err != nil {
			return err
		}
	}
	return nil
}

type Volume struct {
	context       *Context
	name          string
	projectName   string
	driver        string
	driverOptions map[string]string
	external      bool
	perContainer  bool
}

func (v *Volume) Inspect(ctx context.Context) (*client.VolumeTemplate, error) {
	filters := map[string]interface{}{
		"name": v.name,
	}
	if !v.external {
		filters["stackId"] = v.context.Stack.Id
	}
	volumes, err := v.context.Client.VolumeTemplate.List(&client.ListOpts{
		Filters: filters,
	})
	if err != nil {
		return nil, err
	}

	if len(volumes.Data) > 0 {
		return &volumes.Data[0], nil
	}

	return nil, nil
}

func (v *Volume) Remove(ctx context.Context) error {
	volumeResource, err := v.Inspect(ctx)
	if err != nil {
		return err
	}
	err = v.context.Client.VolumeTemplate.Delete(volumeResource)
	return err
}

func (v *Volume) EnsureItExists(ctx context.Context) error {
	volumeResource, err := v.Inspect(ctx)
	if err != nil {
		return err
	}

	if v.external && volumeResource == nil {
		return fmt.Errorf("Volume %s declared as external, but could not be found. Please create the volume manually and try again.", v.name)
	}

	if volumeResource == nil {
		logrus.Infof("Creating volume template %s", v.name)
		return v.create(ctx)
	} else {
		logrus.Infof("Existing volume template found for %s", v.name)
	}

	if v.driver != "" && volumeResource.Driver != v.driver {
		return fmt.Errorf("Volume %q needs to be recreated - driver has changed", v.name)
	}
	return nil
}

func (v *Volume) create(ctx context.Context) error {
	driverOptions := map[string]interface{}{}
	for k, v := range v.driverOptions {
		driverOptions[k] = v
	}
	_, err := v.context.Client.VolumeTemplate.Create(&client.VolumeTemplate{
		Name:         v.name,
		Driver:       v.driver,
		DriverOpts:   driverOptions,
		StackId:      v.context.Stack.Id,
		PerContainer: v.perContainer,
	})
	return err
}

func NewVolume(projectName, name string, config *config.VolumeConfig, context *Context) *Volume {
	return &Volume{
		context:       context,
		name:          name,
		projectName:   projectName,
		driver:        config.Driver,
		driverOptions: config.DriverOpts,
		external:      config.External.External,
		perContainer:  config.PerContainer,
	}
}
<commit_msg>External volumes should lookup by volume and not template<commit_after>package rancher

import (
	"fmt"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	"github.com/docker/libcompose/config"
	"github.com/docker/libcompose/project"
	"github.com/rancher/go-rancher/v2"
)

type RancherVolumesFactory struct {
	Context *Context
}

func (f *RancherVolumesFactory) Create(projectName string, volumeConfigs map[string]*config.VolumeConfig, serviceConfigs *config.ServiceConfigs, volumeEnabled bool) (project.Volumes, error) {
	volumes := make([]*Volume, 0, len(volumeConfigs))
	for name, config := range volumeConfigs {
		volume := NewVolume(projectName, name, config, f.Context)
		volumes = append(volumes, volume)
	}
	return &Volumes{
		volumes:       volumes,
		volumeEnabled: volumeEnabled,
		Context:       f.Context,
	}, nil
}

type Volumes struct {
	volumes       []*Volume
	volumeEnabled bool
	Context       *Context
}

func (v *Volumes) Initialize(ctx context.Context) error {
	if !v.volumeEnabled {
		return nil
	}
	for _, volume := range v.volumes {
		if err := volume.EnsureItExists(ctx); err != nil {
			return err
		}
	}

	return nil
}

func (v *Volumes) Remove(ctx context.Context) error {
	if !v.volumeEnabled {
		return nil
	}
	for _, volume := range v.volumes {
		if err := volume.Remove(ctx); err != nil {
			return err
		}
	}
	return nil
}

type Volume struct {
	context       *Context
	name          string
	projectName   string
	driver        string
	driverOptions map[string]string
	external      bool
	perContainer  bool
}

// InspectTemplate looks up a volume template
func (v *Volume) InspectTemplate(ctx context.Context) (*client.VolumeTemplate, error) {
	volumes, err := v.context.Client.VolumeTemplate.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"name":    v.name,
			"stackId": v.context.Stack.Id,
		},
	})
	if err != nil {
		return nil, err
	}

	if len(volumes.Data) > 0 {
		return &volumes.Data[0], nil
	}

	return nil, nil
}

// InspectExternal looks up a volume
func (v *Volume) InspectExternal(ctx context.Context) (*client.Volume, error) {
	volumes, err := v.context.Client.Volume.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"name": v.name,
		},
	})
	if err != nil {
		return nil, err
	}

	if len(volumes.Data) > 0 {
		return &volumes.Data[0], nil
	}

	return nil, nil
}

func (v *Volume) Remove(ctx context.Context) error {
	if v.external {
		return nil
	}

	volumeResource, err := v.InspectTemplate(ctx)
	if err != nil {
		return err
	}
	return v.context.Client.VolumeTemplate.Delete(volumeResource)
}

func (v *Volume) EnsureItExists(ctx context.Context) error {
	if v.external {
		volumeResource, err := v.InspectExternal(ctx)
		if err != nil {
			return err
		}

		if volumeResource == nil {
			return fmt.Errorf("Volume %s declared as external, but could not be found. Please create the volume manually and try again.", v.name)
		}

		return nil
	}

	volumeResource, err := v.InspectTemplate(ctx)
	if err != nil {
		return err
	}

	if volumeResource == nil {
		logrus.Infof("Creating volume template %s", v.name)
		return v.create(ctx)
	} else {
		logrus.Infof("Existing volume template found for %s", v.name)
	}

	if v.driver != "" && volumeResource.Driver != v.driver {
		return fmt.Errorf("Volume %q needs to be recreated - driver has changed", v.name)
	}
	return nil
}

func (v *Volume) create(ctx context.Context) error {
	driverOptions := map[string]interface{}{}
	for k, v := range v.driverOptions {
		driverOptions[k] = v
	}
	_, err := v.context.Client.VolumeTemplate.Create(&client.VolumeTemplate{
		Name:         v.name,
		Driver:       v.driver,
		DriverOpts:   driverOptions,
		StackId:      v.context.Stack.Id,
		PerContainer: v.perContainer,
	})
	return err
}

func NewVolume(projectName, name string, config *config.VolumeConfig, context *Context) *Volume {
	return &Volume{
		context:       context,
		name:          name,
		projectName:   projectName,
		driver:        config.Driver,
		driverOptions: config.DriverOpts,
		external:      config.External.External,
		perContainer:  config.PerContainer,
	}
}
<|endoftext|>


<commit_before>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package scheme

import (
	os "os"

	announced "k8s.io/apimachinery/pkg/apimachinery/announced"
	registered "k8s.io/apimachinery/pkg/apimachinery/registered"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	runtime "k8s.io/apimachinery/pkg/runtime"
	schema "k8s.io/apimachinery/pkg/runtime/schema"
	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
	wardle "k8s.io/sample-apiserver/pkg/apis/wardle/install"
)

var Scheme = runtime.NewScheme()
var Codecs = serializer.NewCodecFactory(Scheme)
var ParameterCodec = runtime.NewParameterCodec(Scheme)

var Registry = registered.NewOrDie(os.Getenv("KUBE_API_VERSIONS"))
var GroupFactoryRegistry = make(announced.APIGroupFactoryRegistry)

func init() {
	v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})
	Install(GroupFactoryRegistry, Registry, Scheme)
}

// Install registers the API group and adds types to a scheme
func Install(groupFactoryRegistry announced.APIGroupFactoryRegistry, registry *registered.APIRegistrationManager, scheme *runtime.Scheme) {
	wardle.Install(groupFactoryRegistry, registry, scheme)
}
<commit_msg>generated<commit_after>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package scheme

import (
	os "os"

	registered "k8s.io/apimachinery/pkg/apimachinery/registered"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	runtime "k8s.io/apimachinery/pkg/runtime"
	schema "k8s.io/apimachinery/pkg/runtime/schema"
	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
	wardle "k8s.io/sample-apiserver/pkg/apis/wardle/install"
)

var Scheme = runtime.NewScheme()
var Codecs = serializer.NewCodecFactory(Scheme)
var ParameterCodec = runtime.NewParameterCodec(Scheme)

var Registry = registered.NewOrDie(os.Getenv("KUBE_API_VERSIONS"))

func init() {
	v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})
	Install(Registry, Scheme)
}

// Install registers the API group and adds types to a scheme
func Install(registry *registered.APIRegistrationManager, scheme *runtime.Scheme) {
	wardle.Install(registry, scheme)
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package safehttptest

import (
	"html/template"
	"io"
	"net/http"
	"strings"

	"github.com/google/go-safeweb/safehttp"
	"github.com/google/safehtml"
)

type testDispatcher struct{}

func (testDispatcher) Write(rw http.ResponseWriter, resp safehttp.Response) error {
	switch x := resp.(type) {
	case safehtml.HTML:
		_, err := rw.Write([]byte(x.String()))
		return err
	default:
		panic("not a safe response type")
	}
}

func (testDispatcher) ExecuteTemplate(rw http.ResponseWriter, t safehttp.Template, data interface{}) error {
	switch x := t.(type) {
	case *template.Template:
		return x.Execute(rw, data)
	default:
		panic("not a safe response type")
	}
}

// ResponseRecorder encapsulates a safehttp.ResponseWriter that records
// mutations for later inspection in tests. The safehttp.ResponseWriter
// should be passed as part of the handler function in tests.
type ResponseRecorder struct {
	safehttp.ResponseWriter
	rw *responseRecorder
	b  *strings.Builder
}

// NewResponseRecorder creates a ResponseRecorder from the default testDispatcher.
func NewResponseRecorder() *ResponseRecorder {
	var b strings.Builder
	rw := newResponseRecorder(&b)
	return &ResponseRecorder{
		rw:             rw,
		b:              &b,
		ResponseWriter: safehttp.NewResponseWriter(testDispatcher{}, rw),
	}
}

// NewResponseRecorderFromDispatcher creates a ResponseRecorder from a
// provided safehttp.Dispatcher.
func NewResponseRecorderFromDispatcher(d safehttp.Dispatcher) *ResponseRecorder {
	var b strings.Builder
	rw := newResponseRecorder(&b)
	return &ResponseRecorder{
		rw:             rw,
		b:              &b,
		ResponseWriter: safehttp.NewResponseWriter(d, rw),
	}
}

// Header returns the recorded response headers.
func (r *ResponseRecorder) Header() http.Header {
	return r.rw.Header()
}

// Status returns the recorded response status code.
func (r *ResponseRecorder) Status() int {
	return r.rw.status
}

// Body returns the recorded response body.
func (r *ResponseRecorder) Body() string {
	return r.b.String()
}

type responseRecorder struct {
	header http.Header
	writer io.Writer
	status int
}

func newResponseRecorder(w io.Writer) *responseRecorder {
	return &responseRecorder{
		header: http.Header{},
		writer: w,
		status: http.StatusOK,
	}
}

func (r *responseRecorder) Header() http.Header {
	return r.header
}

func (r *responseRecorder) WriteHeader(statusCode int) {
	r.status = statusCode
}

func (r *responseRecorder) Write(data []byte) (int, error) {
	return r.writer.Write(data)
}
<commit_msg>Rename responseRecorder to responseWriter for clarity<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package safehttptest

import (
	"html/template"
	"io"
	"net/http"
	"strings"

	"github.com/google/go-safeweb/safehttp"
	"github.com/google/safehtml"
)

type testDispatcher struct{}

func (testDispatcher) Write(rw http.ResponseWriter, resp safehttp.Response) error {
	switch x := resp.(type) {
	case safehtml.HTML:
		_, err := rw.Write([]byte(x.String()))
		return err
	default:
		panic("not a safe response type")
	}
}

func (testDispatcher) ExecuteTemplate(rw http.ResponseWriter, t safehttp.Template, data interface{}) error {
	switch x := t.(type) {
	case *template.Template:
		return x.Execute(rw, data)
	default:
		panic("not a safe response type")
	}
}

// ResponseRecorder encapsulates a safehttp.ResponseWriter that records
// mutations for later inspection in tests. The safehttp.ResponseWriter
// should be passed as part of the handler function in tests.
type ResponseRecorder struct {
	safehttp.ResponseWriter
	rw *responseWriter
	b  *strings.Builder
}

// NewResponseRecorder creates a ResponseRecorder from the default testDispatcher.
func NewResponseRecorder() *ResponseRecorder {
	var b strings.Builder
	rw := newResponseWriter(&b)
	return &ResponseRecorder{
		rw:             rw,
		b:              &b,
		ResponseWriter: safehttp.NewResponseWriter(testDispatcher{}, rw),
	}
}

// NewResponseRecorderFromDispatcher creates a ResponseRecorder from a
// provided safehttp.Dispatcher.
func NewResponseRecorderFromDispatcher(d safehttp.Dispatcher) *ResponseRecorder {
	var b strings.Builder
	rw := newResponseWriter(&b)
	return &ResponseRecorder{
		rw:             rw,
		b:              &b,
		ResponseWriter: safehttp.NewResponseWriter(d, rw),
	}
}

// Header returns the recorded response headers.
func (r *ResponseRecorder) Header() http.Header {
	return r.rw.Header()
}

// Status returns the recorded response status code.
func (r *ResponseRecorder) Status() int {
	return r.rw.status
}

// Body returns the recorded response body.
func (r *ResponseRecorder) Body() string {
	return r.b.String()
}

// responseWriter is an implementation of the http.ResponseWriter interface used
// for constructing an HTTP response.
type responseWriter struct {
	header http.Header
	writer io.Writer
	status int
}

func newResponseWriter(w io.Writer) *responseWriter {
	return &responseWriter{
		header: http.Header{},
		writer: w,
		status: http.StatusOK,
	}
}

func (r *responseWriter) Header() http.Header {
	return r.header
}

func (r *responseWriter) WriteHeader(statusCode int) {
	r.status = statusCode
}

func (r *responseWriter) Write(data []byte) (int, error) {
	return r.writer.Write(data)
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel

import (
	"encoding/json"
	"runtime"

	"golang.org/x/net/context"
)

// IntrospectionOptions are the options used when introspecting the Channel.
type IntrospectionOptions struct {
	// IncludeExchanges will include all the IDs in the message exchanges.
	IncludeExchanges bool `json:"includeExchanges"`

	// IncludeEmptyPeers will include peers, even if they have no connections.
	IncludeEmptyPeers bool `json:"includeEmptyPeers"`
}

// RuntimeState is a snapshot of the runtime state for a channel.
type RuntimeState struct {
	// LocalPeer is the local peer information (service name, host-port, etc).
	LocalPeer LocalPeerInfo `json:"localPeer"`

	// SubChannels contains information about any subchannels.
	SubChannels map[string]SubChannelRuntimeState `json:"subChannels"`

	// Peers contains information about all the peers on this channel.
	Peers map[string]PeerRuntimeState `json:"peers"`
}

// GoRuntimeStateOptions are the options used when getting Go runtime state.
type GoRuntimeStateOptions struct {
	// IncludeGoStacks will include all goroutine stacks.
	IncludeGoStacks bool `json:"includeGoStacks"`
}

// GoRuntimeState is a snapshot of runtime stats from the runtime.
type GoRuntimeState struct {
	MemStats      runtime.MemStats `json:"memStats"`
	NumGoroutines int              `json:"numGoRoutines"`
	NumCPU        int              `json:"numCPU"`
	NumCGo        int64            `json:"numCGo"`
	GoStacks      []byte           `json:"goStacks,omitempty"`
}

// SubChannelRuntimeState is the runtime state for a subchannel.
type SubChannelRuntimeState struct {
	Service string `json:"service"`
}

// ConnectionRuntimeState is the runtime state for a single connection.
type ConnectionRuntimeState struct {
	ConnectionState  string               `json:"connectionState"`
	LocalHostPort    string               `json:"localHostPort"`
	RemoteHostPort   string               `json:"remoteHostPort"`
	InboundExchange  ExchangeRuntimeState `json:"inboundExchange"`
	OutboundExchange ExchangeRuntimeState `json:"outboundExchange"`
}

// ExchangeRuntimeState is the runtime state for a message exchange set.
type ExchangeRuntimeState struct {
	Name      string   `json:"name"`
	Count     int      `json:"count"`
	Exchanges []uint32 `json:"exchanges,omitempty"`
}

// PeerRuntimeState is the runtime state for a single peer.
type PeerRuntimeState struct {
	HostPort    string                   `json:"hostPort"`
	Connections []ConnectionRuntimeState `json:"connections"`
}

// IntrospectState returns the RuntimeState for this channel.
// Note: this is purely for debugging and monitoring, and may slow down your Channel.
func (ch *Channel) IntrospectState(opts *IntrospectionOptions) *RuntimeState {
	return &RuntimeState{
		LocalPeer:   ch.PeerInfo(),
		SubChannels: ch.subChannels.IntrospectState(opts),
		Peers:       ch.peers.IntrospectState(opts),
	}
}

// IntrospectState returns the runtime state of the peer list.
func (l *PeerList) IntrospectState(opts *IntrospectionOptions) map[string]PeerRuntimeState {
	m := make(map[string]PeerRuntimeState)
	l.mut.RLock()
	for _, peer := range l.peers {
		peerState := peer.IntrospectState(opts)
		if len(peerState.Connections) > 0 || opts.IncludeEmptyPeers {
			m[peer.HostPort()] = peerState
		}
	}

	l.mut.RUnlock()
	return m
}

// IntrospectState returns the runtime state of the subchannels.
func (subChMap *subChannelMap) IntrospectState(opts *IntrospectionOptions) map[string]SubChannelRuntimeState {
	m := make(map[string]SubChannelRuntimeState)
	subChMap.mut.RLock()
	for k := range subChMap.subchannels {
		m[k] = SubChannelRuntimeState{
			Service: k,
		}
	}
	subChMap.mut.RUnlock()
	return m
}

// IntrospectState returns the runtime state for this peer.
func (p *Peer) IntrospectState(opts *IntrospectionOptions) PeerRuntimeState {
	p.mut.RLock()

	hostPort := p.hostPort
	conns := make([]ConnectionRuntimeState, len(p.connections))
	for i, conn := range p.connections {
		conns[i] = conn.IntrospectState(opts)
	}
	p.mut.RUnlock()

	return PeerRuntimeState{
		HostPort:    hostPort,
		Connections: conns,
	}
}

// IntrospectState returns the runtime state for this connection.
func (c *Connection) IntrospectState(opts *IntrospectionOptions) ConnectionRuntimeState {
	return ConnectionRuntimeState{
		ConnectionState:  c.state.String(),
		LocalHostPort:    c.conn.LocalAddr().String(),
		RemoteHostPort:   c.conn.RemoteAddr().String(),
		InboundExchange:  c.inbound.IntrospectState(opts),
		OutboundExchange: c.outbound.IntrospectState(opts),
	}
}

// IntrospectState returns the runtime state for this messsage exchange set.
func (mexset *messageExchangeSet) IntrospectState(opts *IntrospectionOptions) ExchangeRuntimeState {
	mexset.mut.RLock()
	state := ExchangeRuntimeState{
		Name:  mexset.name,
		Count: len(mexset.exchanges),
	}

	if opts.IncludeExchanges {
		state.Exchanges = make([]uint32, 0, len(mexset.exchanges))
		for k := range mexset.exchanges {
			state.Exchanges = append(state.Exchanges, k)
		}
	}

	mexset.mut.RUnlock()

	return state
}

func getStacks() []byte {
	var buf []byte
	for n := 4096; n < 10*1024*1024; n *= 2 {
		buf = make([]byte, n)
		stackLen := runtime.Stack(buf, true /* all */)
		if stackLen < n {
			return buf
		}
	}

	// return the first 10MB of stacks if we have more than 10MB.
	return buf
}
func (ch *Channel) handleIntrospection(arg3 []byte) interface{} {
	var opts IntrospectionOptions
	json.Unmarshal(arg3, &opts)
	return ch.IntrospectState(&opts)
}

func handleInternalRuntime(arg3 []byte) interface{} {
	var opts GoRuntimeStateOptions
	json.Unmarshal(arg3, &opts)

	state := GoRuntimeState{
		NumGoroutines: runtime.NumGoroutine(),
		NumCPU:        runtime.NumCPU(),
		NumCGo:        runtime.NumCgoCall(),
	}
	runtime.ReadMemStats(&state.MemStats)
	if opts.IncludeGoStacks {
		state.GoStacks = getStacks()
	}

	return state
}

// registerInternal registers the following internal handlers which return runtime state:
//  _gometa_introspect: TChannel internal state.
//  _gometa_runtime: Golang runtime stats.
func (ch *Channel) registerInternal() {
	endpoints := []struct {
		name    string
		handler func([]byte) interface{}
	}{
		{"_gometa_introspect", ch.handleIntrospection},
		{"_gometa_runtime", handleInternalRuntime},
	}

	for _, ep := range endpoints {
		// We need ep in our closure.
		ep := ep
		handler := func(ctx context.Context, call *InboundCall) {
			var arg2, arg3 []byte
			if err := NewArgReader(call.Arg2Reader()).Read(&arg2); err != nil {
				return
			}
			if err := NewArgReader(call.Arg3Reader()).Read(&arg3); err != nil {
				return
			}
			if err := NewArgWriter(call.Response().Arg2Writer()).Write(nil); err != nil {
				return
			}
			NewArgWriter(call.Response().Arg3Writer()).WriteJSON(ep.handler(arg3))
		}
		ch.Register(HandlerFunc(handler), ep.name)
	}
}
<commit_msg>Add isolated subchannels to introspection API<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel

import (
	"encoding/json"
	"runtime"

	"golang.org/x/net/context"
)

// IntrospectionOptions are the options used when introspecting the Channel.
type IntrospectionOptions struct {
	// IncludeExchanges will include all the IDs in the message exchanges.
	IncludeExchanges bool `json:"includeExchanges"`

	// IncludeEmptyPeers will include peers, even if they have no connections.
	IncludeEmptyPeers bool `json:"includeEmptyPeers"`
}

// RuntimeState is a snapshot of the runtime state for a channel.
type RuntimeState struct {
	// LocalPeer is the local peer information (service name, host-port, etc).
	LocalPeer LocalPeerInfo `json:"localPeer"`

	// SubChannels contains information about any subchannels.
	SubChannels map[string]SubChannelRuntimeState `json:"subChannels"`

	// RootPeers contains information about all the peers on this channel and their connections.
	RootPeers map[string]PeerRuntimeState `json:"rootPeers"`

	// Peers is the list of shared peers for this channel.
	Peers []string `json:"peers"`
}

// GoRuntimeStateOptions are the options used when getting Go runtime state.
type GoRuntimeStateOptions struct {
	// IncludeGoStacks will include all goroutine stacks.
	IncludeGoStacks bool `json:"includeGoStacks"`
}

// GoRuntimeState is a snapshot of runtime stats from the runtime.
type GoRuntimeState struct {
	MemStats      runtime.MemStats `json:"memStats"`
	NumGoroutines int              `json:"numGoRoutines"`
	NumCPU        int              `json:"numCPU"`
	NumCGo        int64            `json:"numCGo"`
	GoStacks      []byte           `json:"goStacks,omitempty"`
}

// SubChannelRuntimeState is the runtime state for a subchannel.
type SubChannelRuntimeState struct {
	Service  string `json:"service"`
	Isolated bool   `json:"isolated"`
	// IsolatedPeers is the list of all isolated peers for this channel.
	IsolatedPeers []string `json:"isolatedPeers,omitempty"`
}

// ConnectionRuntimeState is the runtime state for a single connection.
type ConnectionRuntimeState struct {
	ConnectionState  string               `json:"connectionState"`
	LocalHostPort    string               `json:"localHostPort"`
	RemoteHostPort   string               `json:"remoteHostPort"`
	InboundExchange  ExchangeRuntimeState `json:"inboundExchange"`
	OutboundExchange ExchangeRuntimeState `json:"outboundExchange"`
}

// ExchangeRuntimeState is the runtime state for a message exchange set.
type ExchangeRuntimeState struct {
	Name      string   `json:"name"`
	Count     int      `json:"count"`
	Exchanges []uint32 `json:"exchanges,omitempty"`
}

// PeerRuntimeState is the runtime state for a single peer.
type PeerRuntimeState struct {
	HostPort    string                   `json:"hostPort"`
	Connections []ConnectionRuntimeState `json:"connections"`
}

// IntrospectState returns the RuntimeState for this channel.
// Note: this is purely for debugging and monitoring, and may slow down your Channel.
func (ch *Channel) IntrospectState(opts *IntrospectionOptions) *RuntimeState {
	return &RuntimeState{
		LocalPeer:   ch.PeerInfo(),
		SubChannels: ch.subChannels.IntrospectState(opts),
		RootPeers:   ch.rootPeers().IntrospectState(opts),
		Peers:       ch.Peers().IntrospectList(opts),
	}
}

// IntrospectState returns the runtime state of the peer list.
func (l *PeerList) IntrospectState(opts *IntrospectionOptions) map[string]PeerRuntimeState {
	m := make(map[string]PeerRuntimeState)
	l.mut.RLock()
	for _, peer := range l.peers {
		peerState := peer.IntrospectState(opts)
		if len(peerState.Connections) > 0 || opts.IncludeEmptyPeers {
			m[peer.HostPort()] = peerState
		}
	}

	l.mut.RUnlock()
	return m
}

// IntrospectState returns the runtime state of the subchannels.
func (subChMap *subChannelMap) IntrospectState(opts *IntrospectionOptions) map[string]SubChannelRuntimeState {
	m := make(map[string]SubChannelRuntimeState)
	subChMap.mut.RLock()
	for k, sc := range subChMap.subchannels {
		state := SubChannelRuntimeState{
			Service:  k,
			Isolated: sc.Isolated(),
		}
		if state.Isolated {
			state.IsolatedPeers = sc.Peers().IntrospectList(opts)
		}
		m[k] = state
	}
	subChMap.mut.RUnlock()
	return m
}

// IntrospectState returns the runtime state for this peer.
func (p *Peer) IntrospectState(opts *IntrospectionOptions) PeerRuntimeState {
	p.mut.RLock()

	hostPort := p.hostPort
	conns := make([]ConnectionRuntimeState, len(p.connections))
	for i, conn := range p.connections {
		conns[i] = conn.IntrospectState(opts)
	}
	p.mut.RUnlock()

	return PeerRuntimeState{
		HostPort:    hostPort,
		Connections: conns,
	}
}

// IntrospectState returns the runtime state for this connection.
func (c *Connection) IntrospectState(opts *IntrospectionOptions) ConnectionRuntimeState {
	return ConnectionRuntimeState{
		ConnectionState:  c.state.String(),
		LocalHostPort:    c.conn.LocalAddr().String(),
		RemoteHostPort:   c.conn.RemoteAddr().String(),
		InboundExchange:  c.inbound.IntrospectState(opts),
		OutboundExchange: c.outbound.IntrospectState(opts),
	}
}

// IntrospectState returns the runtime state for this messsage exchange set.
func (mexset *messageExchangeSet) IntrospectState(opts *IntrospectionOptions) ExchangeRuntimeState {
	mexset.mut.RLock()
	state := ExchangeRuntimeState{
		Name:  mexset.name,
		Count: len(mexset.exchanges),
	}

	if opts.IncludeExchanges {
		state.Exchanges = make([]uint32, 0, len(mexset.exchanges))
		for k := range mexset.exchanges {
			state.Exchanges = append(state.Exchanges, k)
		}
	}

	mexset.mut.RUnlock()

	return state
}

func getStacks() []byte {
	var buf []byte
	for n := 4096; n < 10*1024*1024; n *= 2 {
		buf = make([]byte, n)
		stackLen := runtime.Stack(buf, true /* all */)
		if stackLen < n {
			return buf
		}
	}

	// return the first 10MB of stacks if we have more than 10MB.
	return buf
}
func (ch *Channel) handleIntrospection(arg3 []byte) interface{} {
	var opts IntrospectionOptions
	json.Unmarshal(arg3, &opts)
	return ch.IntrospectState(&opts)
}

// IntrospectList returns the list of peers (host:port) in this peer list.
func (l *PeerList) IntrospectList(opts *IntrospectionOptions) []string {
	var peers []string
	l.mut.RLock()
	for peer := range l.peersByHostPort {
		peers = append(peers, peer)
	}
	l.mut.RUnlock()

	return peers
}

func handleInternalRuntime(arg3 []byte) interface{} {
	var opts GoRuntimeStateOptions
	json.Unmarshal(arg3, &opts)

	state := GoRuntimeState{
		NumGoroutines: runtime.NumGoroutine(),
		NumCPU:        runtime.NumCPU(),
		NumCGo:        runtime.NumCgoCall(),
	}
	runtime.ReadMemStats(&state.MemStats)
	if opts.IncludeGoStacks {
		state.GoStacks = getStacks()
	}

	return state
}

// registerInternal registers the following internal handlers which return runtime state:
//  _gometa_introspect: TChannel internal state.
//  _gometa_runtime: Golang runtime stats.
func (ch *Channel) registerInternal() {
	endpoints := []struct {
		name    string
		handler func([]byte) interface{}
	}{
		{"_gometa_introspect", ch.handleIntrospection},
		{"_gometa_runtime", handleInternalRuntime},
	}

	for _, ep := range endpoints {
		// We need ep in our closure.
		ep := ep
		handler := func(ctx context.Context, call *InboundCall) {
			var arg2, arg3 []byte
			if err := NewArgReader(call.Arg2Reader()).Read(&arg2); err != nil {
				return
			}
			if err := NewArgReader(call.Arg3Reader()).Read(&arg3); err != nil {
				return
			}
			if err := NewArgWriter(call.Response().Arg2Writer()).Write(nil); err != nil {
				return
			}
			NewArgWriter(call.Response().Arg3Writer()).WriteJSON(ep.handler(arg3))
		}
		ch.Register(HandlerFunc(handler), ep.name)
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"strings"
)

// ErrorType signfies a category of errors
type ErrorType uint

// ErrorTypes convey what category of error ocurred
const (
	ErrNormal ErrorType = iota // general errors
	ErrClient                  // error was caused by the client, (e.g. invalid CLI usage)
	// TODO: add more types of errors for better error-specific handling
)

// Error is a struct for marshalling errors
type Error struct {
	Message string
	Code    ErrorType
}

func (e Error) Error() string {
	return e.Message
}

// EncodingType defines a supported encoding
type EncodingType string

// Supported EncodingType constants.
const (
	JSON = "json"
	XML  = "xml"
	Text = "text"
	// TODO: support more encoding types
)

var marshallers = map[EncodingType]Marshaller{
	JSON: func(res Response) ([]byte, error) {
		if res.Error() != nil {
			return json.Marshal(res.Error())
		}
		return json.Marshal(res.Output())
	},
	XML: func(res Response) ([]byte, error) {
		if res.Error() != nil {
			return xml.Marshal(res.Error())
		}
		return xml.Marshal(res.Output())
	},
}

// Response is the result of a command request. Handlers write to the response,
// setting Error or Value. Response is returned to the client.
type Response interface {
	Request() Request

	// Set/Return the response Error
	SetError(err error, code ErrorType)
	Error() *Error

	// Sets/Returns the response value
	SetOutput(interface{})
	Output() interface{}

	// Marshal marshals out the response into a buffer. It uses the EncodingType
	// on the Request to chose a Marshaller (Codec).
	Marshal() ([]byte, error)

	// Gets a io.Reader that reads the marshalled output
	Reader() (io.Reader, error)
}

type response struct {
	req   Request
	err   *Error
	value interface{}
	out   io.Reader
}

func (r *response) Request() Request {
	return r.req
}

func (r *response) Output() interface{} {
	return r.value
}

func (r *response) SetOutput(v interface{}) {
	r.value = v
}

func (r *response) Error() *Error {
	return r.err
}

func (r *response) SetError(err error, code ErrorType) {
	r.err = &Error{Message: err.Error(), Code: code}
}

func (r *response) Marshal() ([]byte, error) {
	if r.err == nil && r.value == nil {
		return []byte{}, nil
	}

	enc, found := r.req.Option(EncShort)
	encStr, ok := enc.(string)
	if !found || !ok || encStr == "" {
		return nil, fmt.Errorf("No encoding type was specified")
	}
	encType := EncodingType(strings.ToLower(encStr))

	marshaller := r.req.Command().Marshallers[encType]
	if marshaller == nil {
		marshaller, ok = marshallers[encType]
		if !ok {
			return nil, fmt.Errorf("No marshaller found for encoding type '%s'", enc)
		}
	}

	return marshaller(r)
}

// Reader returns an `io.Reader` representing marshalled output of this Response
// Note that multiple calls to this will return a reference to the same io.Reader
func (r *response) Reader() (io.Reader, error) {
	// if command set value to a io.Reader, use that as our reader
	if r.out == nil {
		if out, ok := r.value.(io.Reader); ok {
			r.out = out
		}
	}

	if r.out == nil {
		// no reader set, so marshal the error or value
		marshalled, err := r.Marshal()
		if err != nil {
			return nil, err
		}

		// create a Reader from the marshalled data
		r.out = bytes.NewReader(marshalled)
	}

	return r.out, nil
}

// NewResponse returns a response to match given Request
func NewResponse(req Request) Response {
	return &response{req: req}
}
<commit_msg>commands: Fixed panic when trying to marshal without a command set in request<commit_after>package commands

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"strings"
)

// ErrorType signfies a category of errors
type ErrorType uint

// ErrorTypes convey what category of error ocurred
const (
	ErrNormal ErrorType = iota // general errors
	ErrClient                  // error was caused by the client, (e.g. invalid CLI usage)
	// TODO: add more types of errors for better error-specific handling
)

// Error is a struct for marshalling errors
type Error struct {
	Message string
	Code    ErrorType
}

func (e Error) Error() string {
	return e.Message
}

// EncodingType defines a supported encoding
type EncodingType string

// Supported EncodingType constants.
const (
	JSON = "json"
	XML  = "xml"
	Text = "text"
	// TODO: support more encoding types
)

var marshallers = map[EncodingType]Marshaller{
	JSON: func(res Response) ([]byte, error) {
		if res.Error() != nil {
			return json.Marshal(res.Error())
		}
		return json.Marshal(res.Output())
	},
	XML: func(res Response) ([]byte, error) {
		if res.Error() != nil {
			return xml.Marshal(res.Error())
		}
		return xml.Marshal(res.Output())
	},
}

// Response is the result of a command request. Handlers write to the response,
// setting Error or Value. Response is returned to the client.
type Response interface {
	Request() Request

	// Set/Return the response Error
	SetError(err error, code ErrorType)
	Error() *Error

	// Sets/Returns the response value
	SetOutput(interface{})
	Output() interface{}

	// Marshal marshals out the response into a buffer. It uses the EncodingType
	// on the Request to chose a Marshaller (Codec).
	Marshal() ([]byte, error)

	// Gets a io.Reader that reads the marshalled output
	Reader() (io.Reader, error)
}

type response struct {
	req   Request
	err   *Error
	value interface{}
	out   io.Reader
}

func (r *response) Request() Request {
	return r.req
}

func (r *response) Output() interface{} {
	return r.value
}

func (r *response) SetOutput(v interface{}) {
	r.value = v
}

func (r *response) Error() *Error {
	return r.err
}

func (r *response) SetError(err error, code ErrorType) {
	r.err = &Error{Message: err.Error(), Code: code}
}

func (r *response) Marshal() ([]byte, error) {
	if r.err == nil && r.value == nil {
		return []byte{}, nil
	}

	enc, found := r.req.Option(EncShort)
	encStr, ok := enc.(string)
	if !found || !ok || encStr == "" {
		return nil, fmt.Errorf("No encoding type was specified")
	}
	encType := EncodingType(strings.ToLower(encStr))

	var marshaller Marshaller
	if r.req.Command() != nil && r.req.Command().Marshallers != nil {
		marshaller = r.req.Command().Marshallers[encType]
	}
	if marshaller == nil {
		marshaller, ok = marshallers[encType]
		if !ok {
			return nil, fmt.Errorf("No marshaller found for encoding type '%s'", enc)
		}
	}

	return marshaller(r)
}

// Reader returns an `io.Reader` representing marshalled output of this Response
// Note that multiple calls to this will return a reference to the same io.Reader
func (r *response) Reader() (io.Reader, error) {
	// if command set value to a io.Reader, use that as our reader
	if r.out == nil {
		if out, ok := r.value.(io.Reader); ok {
			r.out = out
		}
	}

	if r.out == nil {
		// no reader set, so marshal the error or value
		marshalled, err := r.Marshal()
		if err != nil {
			return nil, err
		}

		// create a Reader from the marshalled data
		r.out = bytes.NewReader(marshalled)
	}

	return r.out, nil
}

// NewResponse returns a response to match given Request
func NewResponse(req Request) Response {
	return &response{req: req}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/organizations"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
	"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
)

func resourceAwsOrganizationsPolicy() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsOrganizationsPolicyCreate,
		Read:   resourceAwsOrganizationsPolicyRead,
		Update: resourceAwsOrganizationsPolicyUpdate,
		Delete: resourceAwsOrganizationsPolicyDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"content": {
				Type:             schema.TypeString,
				Required:         true,
				DiffSuppressFunc: suppressEquivalentAwsPolicyDiffs,
				ValidateFunc:     validation.StringIsJSON,
			},
			"description": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"name": {
				Type:     schema.TypeString,
				Required: true,
			},
			"type": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  organizations.PolicyTypeServiceControlPolicy,
				ValidateFunc: validation.StringInSlice([]string{
					organizations.PolicyTypeAiservicesOptOutPolicy,
					organizations.PolicyTypeBackupPolicy,
					organizations.PolicyTypeServiceControlPolicy,
					organizations.PolicyTypeTagPolicy,
				}, false),
			},
			"tags": tagsSchema(),
		},
	}
}

func resourceAwsOrganizationsPolicyCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn

	// Description is required:
	// InvalidParameter: 1 validation error(s) found.
	// - missing required field, CreatePolicyInput.Description.
	input := &organizations.CreatePolicyInput{
		Content:     aws.String(d.Get("content").(string)),
		Description: aws.String(d.Get("description").(string)),
		Name:        aws.String(d.Get("name").(string)),
		Type:        aws.String(d.Get("type").(string)),
		Tags:    keyvaluetags.New(d.Get("tags").(map[string]interface{})).IgnoreAws().OrganizationsTags(),
	}

	log.Printf("[DEBUG] Creating Organizations Policy: %s", input)

	var err error
	var resp *organizations.CreatePolicyOutput
	err = resource.Retry(4*time.Minute, func() *resource.RetryError {
		resp, err = conn.CreatePolicy(input)

		if err != nil {
			if isAWSErr(err, organizations.ErrCodeFinalizingOrganizationException, "") {
				log.Printf("[DEBUG] Trying to create policy again: %q", err.Error())
				return resource.RetryableError(err)
			}

			return resource.NonRetryableError(err)
		}

		return nil
	})
	if isResourceTimeoutError(err) {
		resp, err = conn.CreatePolicy(input)
	}

	if err != nil {
		return fmt.Errorf("error creating Organizations Policy: %s", err)
	}

	d.SetId(*resp.Policy.PolicySummary.Id)

	return resourceAwsOrganizationsPolicyRead(d, meta)
}

func resourceAwsOrganizationsPolicyRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn
	ignoreTagsConfig := meta.(*AWSClient).IgnoreTagsConfig

	input := &organizations.DescribePolicyInput{
		PolicyId: aws.String(d.Id()),
	}

	log.Printf("[DEBUG] Reading Organizations Policy: %s", input)
	resp, err := conn.DescribePolicy(input)
	if err != nil {
		if isAWSErr(err, organizations.ErrCodePolicyNotFoundException, "") {
			log.Printf("[WARN] Policy does not exist, removing from state: %s", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	if resp.Policy == nil || resp.Policy.PolicySummary == nil {
		log.Printf("[WARN] Policy does not exist, removing from state: %s", d.Id())
		d.SetId("")
		return nil
	}

	d.Set("arn", resp.Policy.PolicySummary.Arn)
	d.Set("content", resp.Policy.Content)
	d.Set("description", resp.Policy.PolicySummary.Description)
	d.Set("name", resp.Policy.PolicySummary.Name)
	d.Set("type", resp.Policy.PolicySummary.Type)

    tags, err := keyvaluetags.OrganizationsListTags(conn, d.Id())
	if err != nil {
		return fmt.Errorf("error listing tags: %s", err)
	}

	if err := d.Set("tags", tags.IgnoreAws().IgnoreConfig(ignoreTagsConfig).Map()); err != nil {
		return fmt.Errorf("error setting tags: %s", err)
	}

	return nil
}

func resourceAwsOrganizationsPolicyUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn

	input := &organizations.UpdatePolicyInput{
		PolicyId: aws.String(d.Id()),
	}

	if d.HasChange("content") {
		input.Content = aws.String(d.Get("content").(string))
	}

	if d.HasChange("description") {
		input.Description = aws.String(d.Get("description").(string))
	}

	if d.HasChange("name") {
		input.Name = aws.String(d.Get("name").(string))
	}

	log.Printf("[DEBUG] Updating Organizations Policy: %s", input)
	_, err := conn.UpdatePolicy(input)
	if err != nil {
		return fmt.Errorf("error updating Organizations Policy: %s", err)
	}

    if d.HasChange("tags") {
		o, n := d.GetChange("tags")
		if err := keyvaluetags.OrganizationsUpdateTags(conn, d.Id(), o, n); err != nil {
			return fmt.Errorf("error updating tags: %s", err)
		}
	}

	return resourceAwsOrganizationsPolicyRead(d, meta)
}

func resourceAwsOrganizationsPolicyDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn

	input := &organizations.DeletePolicyInput{
		PolicyId: aws.String(d.Id()),
	}

	log.Printf("[DEBUG] Deletion Organizations Policy: %s", input)
	_, err := conn.DeletePolicy(input)
	if err != nil {
		if isAWSErr(err, organizations.ErrCodePolicyNotFoundException, "") {
			return nil
		}
		return err
	}
	return nil
}
<commit_msg>f/aws_organizations_policy:support for tags<commit_after>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/organizations"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
	"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
)

func resourceAwsOrganizationsPolicy() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsOrganizationsPolicyCreate,
		Read:   resourceAwsOrganizationsPolicyRead,
		Update: resourceAwsOrganizationsPolicyUpdate,
		Delete: resourceAwsOrganizationsPolicyDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"content": {
				Type:             schema.TypeString,
				Required:         true,
				DiffSuppressFunc: suppressEquivalentAwsPolicyDiffs,
				ValidateFunc:     validation.StringIsJSON,
			},
			"description": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"name": {
				Type:     schema.TypeString,
				Required: true,
			},
			"type": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  organizations.PolicyTypeServiceControlPolicy,
				ValidateFunc: validation.StringInSlice([]string{
					organizations.PolicyTypeAiservicesOptOutPolicy,
					organizations.PolicyTypeBackupPolicy,
					organizations.PolicyTypeServiceControlPolicy,
					organizations.PolicyTypeTagPolicy,
				}, false),
			},
			"tags": tagsSchema(),
		},
	}
}

func resourceAwsOrganizationsPolicyCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn

	// Description is required:
	// InvalidParameter: 1 validation error(s) found.
	// - missing required field, CreatePolicyInput.Description.
	input := &organizations.CreatePolicyInput{
		Content:     aws.String(d.Get("content").(string)),
		Description: aws.String(d.Get("description").(string)),
		Name:        aws.String(d.Get("name").(string)),
		Type:        aws.String(d.Get("type").(string)),
		Tags:        keyvaluetags.New(d.Get("tags").(map[string]interface{})).IgnoreAws().OrganizationsTags(),
	}

	log.Printf("[DEBUG] Creating Organizations Policy: %s", input)

	var err error
	var resp *organizations.CreatePolicyOutput
	err = resource.Retry(4*time.Minute, func() *resource.RetryError {
		resp, err = conn.CreatePolicy(input)

		if err != nil {
			if isAWSErr(err, organizations.ErrCodeFinalizingOrganizationException, "") {
				log.Printf("[DEBUG] Trying to create policy again: %q", err.Error())
				return resource.RetryableError(err)
			}

			return resource.NonRetryableError(err)
		}

		return nil
	})
	if isResourceTimeoutError(err) {
		resp, err = conn.CreatePolicy(input)
	}

	if err != nil {
		return fmt.Errorf("error creating Organizations Policy: %s", err)
	}

	d.SetId(*resp.Policy.PolicySummary.Id)

	return resourceAwsOrganizationsPolicyRead(d, meta)
}

func resourceAwsOrganizationsPolicyRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn
	ignoreTagsConfig := meta.(*AWSClient).IgnoreTagsConfig

	input := &organizations.DescribePolicyInput{
		PolicyId: aws.String(d.Id()),
	}

	log.Printf("[DEBUG] Reading Organizations Policy: %s", input)
	resp, err := conn.DescribePolicy(input)
	if err != nil {
		if isAWSErr(err, organizations.ErrCodePolicyNotFoundException, "") {
			log.Printf("[WARN] Policy does not exist, removing from state: %s", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	if resp.Policy == nil || resp.Policy.PolicySummary == nil {
		log.Printf("[WARN] Policy does not exist, removing from state: %s", d.Id())
		d.SetId("")
		return nil
	}

	d.Set("arn", resp.Policy.PolicySummary.Arn)
	d.Set("content", resp.Policy.Content)
	d.Set("description", resp.Policy.PolicySummary.Description)
	d.Set("name", resp.Policy.PolicySummary.Name)
	d.Set("type", resp.Policy.PolicySummary.Type)

	tags, err := keyvaluetags.OrganizationsListTags(conn, d.Id())
	if err != nil {
		return fmt.Errorf("error listing tags: %s", err)
	}

	if err := d.Set("tags", tags.IgnoreAws().IgnoreConfig(ignoreTagsConfig).Map()); err != nil {
		return fmt.Errorf("error setting tags: %s", err)
	}

	return nil
}

func resourceAwsOrganizationsPolicyUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn

	input := &organizations.UpdatePolicyInput{
		PolicyId: aws.String(d.Id()),
	}

	if d.HasChange("content") {
		input.Content = aws.String(d.Get("content").(string))
	}

	if d.HasChange("description") {
		input.Description = aws.String(d.Get("description").(string))
	}

	if d.HasChange("name") {
		input.Name = aws.String(d.Get("name").(string))
	}

	log.Printf("[DEBUG] Updating Organizations Policy: %s", input)
	_, err := conn.UpdatePolicy(input)
	if err != nil {
		return fmt.Errorf("error updating Organizations Policy: %s", err)
	}

	if d.HasChange("tags") {
		o, n := d.GetChange("tags")
		if err := keyvaluetags.OrganizationsUpdateTags(conn, d.Id(), o, n); err != nil {
			return fmt.Errorf("error updating tags: %s", err)
		}
	}

	return resourceAwsOrganizationsPolicyRead(d, meta)
}

func resourceAwsOrganizationsPolicyDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn

	input := &organizations.DeletePolicyInput{
		PolicyId: aws.String(d.Id()),
	}

	log.Printf("[DEBUG] Deletion Organizations Policy: %s", input)
	_, err := conn.DeletePolicy(input)
	if err != nil {
		if isAWSErr(err, organizations.ErrCodePolicyNotFoundException, "") {
			return nil
		}
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package nodedriver

import (
	"fmt"
	"strings"
	"sync"

	errs "github.com/pkg/errors"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/config"
	"github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
)

var (
	schemaLock = sync.Mutex{}
)

const (
	driverNameLabel = "io.cattle.node_driver.name"
)

func Register(management *config.ManagementContext) {
	nodeDriverClient := management.Management.NodeDrivers("")
	nodeDriverLifecycle := &Lifecycle{
		nodeDriverClient: nodeDriverClient,
		schemaClient:     management.Management.DynamicSchemas(""),
		schemaLister:     management.Management.DynamicSchemas("").Controller().Lister(),
	}

	nodeDriverClient.
		AddLifecycle("node-driver-controller", nodeDriverLifecycle)
}

type Lifecycle struct {
	nodeDriverClient v3.NodeDriverInterface
	schemaClient     v3.DynamicSchemaInterface
	schemaLister     v3.DynamicSchemaLister
}

func (m *Lifecycle) Create(obj *v3.NodeDriver) (*v3.NodeDriver, error) {
	return m.download(obj)
}

func (m *Lifecycle) download(obj *v3.NodeDriver) (*v3.NodeDriver, error) {
	if !obj.Spec.Active {
		return obj, nil
	}

	err := errs.New("not found")
	// if node driver was created, we also activate the driver by default
	driver := NewDriver(obj.Spec.Builtin, obj.Spec.DisplayName, obj.Spec.URL, obj.Spec.Checksum)
	if obj.Spec.DisplayName != "" {
		schemaName := obj.Spec.DisplayName + "config"
		_, err = m.schemaLister.Get("", schemaName)
	}

	if driver.Exists() && err == nil {
		return obj, nil
	}

	newObj, err := v3.NodeDriverConditionDownloaded.Once(obj, func() (runtime.Object, error) {
		// update status
		obj, err = m.nodeDriverClient.Update(obj)
		if err != nil {
			return nil, err
		}

		if err := driver.Stage(); err != nil {
			return nil, err
		}
		return obj, nil
	})
	if err != nil {
		return obj, err
	}

	obj = newObj.(*v3.NodeDriver)
	newObj, err = v3.NodeDriverConditionInstalled.Once(obj, func() (runtime.Object, error) {
		if err := driver.Install(); err != nil {
			return nil, err
		}
		if err = driver.Excutable(); err != nil {
			return nil, err
		}
		obj.Spec.DisplayName = strings.TrimPrefix(driver.Name(), dockerMachineDriverPrefix)
		return obj, nil
	})
	if err != nil {
		return newObj.(*v3.NodeDriver), err
	}

	obj = newObj.(*v3.NodeDriver)
	driverName := strings.TrimPrefix(driver.Name(), dockerMachineDriverPrefix)
	flags, err := getCreateFlagsForDriver(driverName)
	if err != nil {
		return nil, err
	}
	resourceFields := map[string]v3.Field{}
	for _, flag := range flags {
		name, field, err := flagToField(flag)
		if err != nil {
			return nil, err
		}
		resourceFields[name] = field
	}
	dynamicSchema := &v3.DynamicSchema{
		Spec: v3.DynamicSchemaSpec{
			ResourceFields: resourceFields,
		},
	}
	dynamicSchema.Name = obj.Spec.DisplayName + "config"
	dynamicSchema.OwnerReferences = []metav1.OwnerReference{
		{
			UID:        obj.UID,
			Kind:       obj.Kind,
			APIVersion: obj.APIVersion,
			Name:       obj.Name,
		},
	}
	dynamicSchema.Labels = map[string]string{}
	dynamicSchema.Labels[driverNameLabel] = obj.Spec.DisplayName
	_, err = m.schemaClient.Create(dynamicSchema)
	if err != nil && !errors.IsAlreadyExists(err) {
		return obj, err
	}
	return obj, nil
}

func (m *Lifecycle) Updated(obj *v3.NodeDriver) (*v3.NodeDriver, error) {
	var err error

	obj, err = m.download(obj)
	if err != nil {
		return obj, err
	}

	if err := m.createOrUpdateNodeForEmbeddedType(obj.Spec.DisplayName+"config", obj.Spec.DisplayName+"Config", obj.Spec.Active); err != nil {
		return obj, err
	}

	v3.NodeDriverConditionActive.True(obj)
	v3.NodeDriverConditionInactive.True(obj)
	v3.NodeDriverConditionDownloaded.True(obj)

	return obj, nil
}

func (m *Lifecycle) Remove(obj *v3.NodeDriver) (*v3.NodeDriver, error) {
	schemas, err := m.schemaClient.List(metav1.ListOptions{
		LabelSelector: fmt.Sprintf("%s=%s", driverNameLabel, obj.Spec.DisplayName),
	})
	if err != nil {
		return obj, err
	}
	for _, schema := range schemas.Items {
		logrus.Infof("Deleting schema %s", schema.Name)
		if err := m.schemaClient.Delete(schema.Name, &metav1.DeleteOptions{}); err != nil {
			return obj, err
		}
		logrus.Infof("Deleting schema %s done", schema.Name)
	}
	if err := m.createOrUpdateNodeForEmbeddedType(obj.Spec.DisplayName+"config", obj.Spec.DisplayName+"Config", false); err != nil {
		return obj, err
	}
	return obj, nil
}

func (m *Lifecycle) createOrUpdateNodeForEmbeddedType(embeddedType, fieldName string, embedded bool) error {
	schemaLock.Lock()
	defer schemaLock.Unlock()

	if err := m.createOrUpdateNodeForEmbeddedTypeWithParents(embeddedType, fieldName, "nodeconfig", "node", embedded, false); err != nil {
		return err
	}

	return m.createOrUpdateNodeForEmbeddedTypeWithParents(embeddedType, fieldName, "nodetemplateconfig", "nodeTemplate", embedded, true)
}

func (m *Lifecycle) createOrUpdateNodeForEmbeddedTypeWithParents(embeddedType, fieldName, schemaID, parentID string, embedded, update bool) error {
	nodeSchema, err := m.schemaLister.Get("", schemaID)
	if err != nil && !errors.IsNotFound(err) {
		return err
	} else if errors.IsNotFound(err) {
		resourceField := map[string]v3.Field{}
		if embedded {
			resourceField[fieldName] = v3.Field{
				Create:   true,
				Nullable: true,
				Update:   update,
				Type:     embeddedType,
			}
		}
		dynamicSchema := &v3.DynamicSchema{}
		dynamicSchema.Name = schemaID
		dynamicSchema.Spec.ResourceFields = resourceField
		dynamicSchema.Spec.Embed = true
		dynamicSchema.Spec.EmbedType = parentID
		_, err := m.schemaClient.Create(dynamicSchema)
		if err != nil {
			return err
		}
		return nil
	}

	shouldUpdate := false
	if embedded {
		if nodeSchema.Spec.ResourceFields == nil {
			nodeSchema.Spec.ResourceFields = map[string]v3.Field{}
		}
		if _, ok := nodeSchema.Spec.ResourceFields[fieldName]; !ok {
			// if embedded we add the type to schema
			logrus.Infof("uploading %s to node schema", fieldName)
			nodeSchema.Spec.ResourceFields[fieldName] = v3.Field{
				Create:   true,
				Nullable: true,
				Update:   update,
				Type:     embeddedType,
			}
			shouldUpdate = true
		}
	} else {
		// if not we delete it from schema
		if _, ok := nodeSchema.Spec.ResourceFields[fieldName]; ok {
			logrus.Infof("deleting %s from node schema", fieldName)
			delete(nodeSchema.Spec.ResourceFields, fieldName)
			shouldUpdate = true
		}
	}

	if shouldUpdate {
		_, err = m.schemaClient.Update(nodeSchema)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>fix node driver not download<commit_after>package nodedriver

import (
	"fmt"
	"strings"
	"sync"

	errs "github.com/pkg/errors"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/config"
	"github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
)

var (
	schemaLock = sync.Mutex{}
)

const (
	driverNameLabel = "io.cattle.node_driver.name"
)

func Register(management *config.ManagementContext) {
	nodeDriverClient := management.Management.NodeDrivers("")
	nodeDriverLifecycle := &Lifecycle{
		nodeDriverClient: nodeDriverClient,
		schemaClient:     management.Management.DynamicSchemas(""),
		schemaLister:     management.Management.DynamicSchemas("").Controller().Lister(),
	}

	nodeDriverClient.
		AddLifecycle("node-driver-controller", nodeDriverLifecycle)
}

type Lifecycle struct {
	nodeDriverClient v3.NodeDriverInterface
	schemaClient     v3.DynamicSchemaInterface
	schemaLister     v3.DynamicSchemaLister
}

func (m *Lifecycle) Create(obj *v3.NodeDriver) (*v3.NodeDriver, error) {
	return m.download(obj)
}

func (m *Lifecycle) download(obj *v3.NodeDriver) (*v3.NodeDriver, error) {
	if !obj.Spec.Active {
		return obj, nil
	}

	err := errs.New("not found")
	// if node driver was created, we also activate the driver by default
	driver := NewDriver(obj.Spec.Builtin, obj.Spec.DisplayName, obj.Spec.URL, obj.Spec.Checksum)
	if obj.Spec.DisplayName != "" {
		schemaName := obj.Spec.DisplayName + "config"
		_, err = m.schemaLister.Get("", schemaName)
	}

	if driver.Exists() && err == nil {
		return obj, nil
	}

	newObj, err := v3.NodeDriverConditionDownloaded.Once(obj, func() (runtime.Object, error) {
		// update status
		obj, err = m.nodeDriverClient.Update(obj)
		if err != nil {
			return nil, err
		}

		if err := driver.Stage(); err != nil {
			return nil, err
		}
		return obj, nil
	})
	if err != nil {
		return obj, err
	}

	obj = newObj.(*v3.NodeDriver)
	newObj, err = v3.NodeDriverConditionInstalled.Once(obj, func() (runtime.Object, error) {
		if err := driver.Install(); err != nil {
			return nil, err
		}
		if err = driver.Excutable(); err != nil {
			return nil, err
		}
		obj.Spec.DisplayName = strings.TrimPrefix(driver.Name(), dockerMachineDriverPrefix)
		return obj, nil
	})
	if err != nil {
		return newObj.(*v3.NodeDriver), err
	}

	obj = newObj.(*v3.NodeDriver)
	driverName := strings.TrimPrefix(driver.Name(), dockerMachineDriverPrefix)
	flags, err := getCreateFlagsForDriver(driverName)
	if err != nil {
		return nil, err
	}
	resourceFields := map[string]v3.Field{}
	for _, flag := range flags {
		name, field, err := flagToField(flag)
		if err != nil {
			return nil, err
		}
		resourceFields[name] = field
	}
	dynamicSchema := &v3.DynamicSchema{
		Spec: v3.DynamicSchemaSpec{
			ResourceFields: resourceFields,
		},
	}
	dynamicSchema.Name = obj.Spec.DisplayName + "config"
	dynamicSchema.OwnerReferences = []metav1.OwnerReference{
		{
			UID:        obj.UID,
			Kind:       obj.Kind,
			APIVersion: obj.APIVersion,
			Name:       obj.Name,
		},
	}
	dynamicSchema.Labels = map[string]string{}
	dynamicSchema.Labels[driverNameLabel] = obj.Spec.DisplayName
	_, err = m.schemaClient.Create(dynamicSchema)
	if err != nil && !errors.IsAlreadyExists(err) {
		return obj, err
	}
	return obj, nil
}

func (m *Lifecycle) Updated(obj *v3.NodeDriver) (*v3.NodeDriver, error) {
	var err error

	obj, err = m.download(obj)
	if err != nil {
		return obj, err
	}

	if err := m.createOrUpdateNodeForEmbeddedType(obj.Spec.DisplayName+"config", obj.Spec.DisplayName+"Config", obj.Spec.Active); err != nil {
		return obj, err
	}

	v3.NodeDriverConditionActive.True(obj)
	v3.NodeDriverConditionInactive.True(obj)

	return obj, nil
}

func (m *Lifecycle) Remove(obj *v3.NodeDriver) (*v3.NodeDriver, error) {
	schemas, err := m.schemaClient.List(metav1.ListOptions{
		LabelSelector: fmt.Sprintf("%s=%s", driverNameLabel, obj.Spec.DisplayName),
	})
	if err != nil {
		return obj, err
	}
	for _, schema := range schemas.Items {
		logrus.Infof("Deleting schema %s", schema.Name)
		if err := m.schemaClient.Delete(schema.Name, &metav1.DeleteOptions{}); err != nil {
			return obj, err
		}
		logrus.Infof("Deleting schema %s done", schema.Name)
	}
	if err := m.createOrUpdateNodeForEmbeddedType(obj.Spec.DisplayName+"config", obj.Spec.DisplayName+"Config", false); err != nil {
		return obj, err
	}
	return obj, nil
}

func (m *Lifecycle) createOrUpdateNodeForEmbeddedType(embeddedType, fieldName string, embedded bool) error {
	schemaLock.Lock()
	defer schemaLock.Unlock()

	if err := m.createOrUpdateNodeForEmbeddedTypeWithParents(embeddedType, fieldName, "nodeconfig", "node", embedded, false); err != nil {
		return err
	}

	return m.createOrUpdateNodeForEmbeddedTypeWithParents(embeddedType, fieldName, "nodetemplateconfig", "nodeTemplate", embedded, true)
}

func (m *Lifecycle) createOrUpdateNodeForEmbeddedTypeWithParents(embeddedType, fieldName, schemaID, parentID string, embedded, update bool) error {
	nodeSchema, err := m.schemaLister.Get("", schemaID)
	if err != nil && !errors.IsNotFound(err) {
		return err
	} else if errors.IsNotFound(err) {
		resourceField := map[string]v3.Field{}
		if embedded {
			resourceField[fieldName] = v3.Field{
				Create:   true,
				Nullable: true,
				Update:   update,
				Type:     embeddedType,
			}
		}
		dynamicSchema := &v3.DynamicSchema{}
		dynamicSchema.Name = schemaID
		dynamicSchema.Spec.ResourceFields = resourceField
		dynamicSchema.Spec.Embed = true
		dynamicSchema.Spec.EmbedType = parentID
		_, err := m.schemaClient.Create(dynamicSchema)
		if err != nil {
			return err
		}
		return nil
	}

	shouldUpdate := false
	if embedded {
		if nodeSchema.Spec.ResourceFields == nil {
			nodeSchema.Spec.ResourceFields = map[string]v3.Field{}
		}
		if _, ok := nodeSchema.Spec.ResourceFields[fieldName]; !ok {
			// if embedded we add the type to schema
			logrus.Infof("uploading %s to node schema", fieldName)
			nodeSchema.Spec.ResourceFields[fieldName] = v3.Field{
				Create:   true,
				Nullable: true,
				Update:   update,
				Type:     embeddedType,
			}
			shouldUpdate = true
		}
	} else {
		// if not we delete it from schema
		if _, ok := nodeSchema.Spec.ResourceFields[fieldName]; ok {
			logrus.Infof("deleting %s from node schema", fieldName)
			delete(nodeSchema.Spec.ResourceFields, fieldName)
			shouldUpdate = true
		}
	}

	if shouldUpdate {
		_, err = m.schemaClient.Update(nodeSchema)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigtable

import (
	"context"
	"fmt"

	"go.chromium.org/luci/logdog/common/storage"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/grpc/grpcutil"

	"cloud.google.com/go/bigtable"
	"google.golang.org/grpc/codes"
)

const (
	logColumnFamily = "log"

	// The data column stores raw low row data (RecordIO blob).
	logColumn  = "data"
	logColName = logColumnFamily + ":" + logColumn
)

// Limits taken from here:
// https://cloud.google.com/bigtable/docs/schema-design
const (
	// bigTableRowMaxBytes is the maximum number of bytes that a single BigTable
	// row may hold.
	bigTableRowMaxBytes = 1024 * 1024 * 10 // 10MB
)

// btGetCallback is a callback that is invoked for each log data row returned
// by getLogData.
//
// If an error is encountered, no more log data will be fetched. The error will
// be propagated to the getLogData call.
type btGetCallback func(*rowKey, []byte) error

// btIface is a general interface for BigTable operations intended to enable
// unit tests to stub out BigTable without adding runtime inefficiency.
type btIface interface {
	// putLogData adds new log data to BigTable.
	//
	// If data already exists for the named row, it will return storage.ErrExists
	// and not add the data.
	putLogData(context.Context, *rowKey, []byte) error

	// getLogData retrieves rows belonging to the supplied stream record, starting
	// with the first index owned by that record. The supplied callback is invoked
	// once per retrieved row.
	//
	// rk is the starting row key.
	//
	// If the supplied limit is nonzero, no more than limit rows will be
	// retrieved.
	//
	// If keysOnly is true, then the callback will return nil row data.
	getLogData(c context.Context, rk *rowKey, limit int, keysOnly bool, cb btGetCallback) error

	// Drops all rows given the path prefix of rk.
	dropRowRange(c context.Context, rkPrefix *rowKey) error

	// getMaxRowSize returns the maximum row size that this implementation
	// supports.
	getMaxRowSize() int
}

// prodBTIface is a production implementation of a "btIface".
type prodBTIface struct {
	*Storage
}

func (bti prodBTIface) getLogTable() (*bigtable.Table, error) {
	if bti.Client == nil {
		return nil, errors.New("no client configured")
	}
	return bti.Client.Open(bti.LogTable), nil
}

func (bti prodBTIface) putLogData(c context.Context, rk *rowKey, data []byte) error {
	logTable, err := bti.getLogTable()
	if err != nil {
		return err
	}

	m := bigtable.NewMutation()
	m.Set(logColumnFamily, logColumn, bigtable.ServerTime, data)
	cm := bigtable.NewCondMutation(bigtable.RowKeyFilter(rk.encode()), nil, m)

	rowExists := false
	if err := logTable.Apply(c, rk.encode(), cm, bigtable.GetCondMutationResult(&rowExists)); err != nil {
		return wrapIfTransientForApply(err)
	}
	if rowExists {
		return storage.ErrExists
	}
	return nil
}

func (bti prodBTIface) dropRowRange(c context.Context, rk *rowKey) error {
	logTable, err := bti.getLogTable()
	if err != nil {
		return err
	}

	// ApplyBulk claims to be able to apply 100k mutations. Keep it small here to
	// stay well within the stated guidelines.
	const maxBatchSize = 100000 / 4

	del := bigtable.NewMutation()
	del.DeleteRow()

	allMuts := make([]*bigtable.Mutation, maxBatchSize)
	for i := range allMuts {
		allMuts[i] = del
	}

	prefix, upperBound := rk.pathPrefix(), rk.pathPrefixUpperBound()
	rng := bigtable.NewRange(prefix, upperBound)
	// apply paranoia mode
	if rng.Contains("") || prefix == "" || upperBound == "" {
		panic(fmt.Sprintf("NOTHING MAKES SENSE: %q %q %q", rng, prefix, upperBound))
	}

	keyC := make(chan string)

	// TODO(iannucci): parallelize row scan?
	readerC := make(chan error)
	go func() {
		defer close(keyC)
		defer close(readerC)
		readerC <- logTable.ReadRows(c, rng, func(row bigtable.Row) bool {
			keyC <- row.Key()
			return true
		},
			bigtable.RowFilter(bigtable.FamilyFilter(logColumnFamily)),
			bigtable.RowFilter(bigtable.ColumnFilter(logColumn)),
			bigtable.RowFilter(bigtable.StripValueFilter()),
		)
	}()

	keys := make([]string, maxBatchSize)
	batchNum := 0
	var totalDropped int64
	for {
		batchNum++
		batch := keys[:0]
		for key := range keyC {
			batch = append(batch, key)
			if len(batch) >= maxBatchSize {
				break
			}
		}
		if len(batch) == 0 {
			logging.Infof(c, "dropRowRange: dropped %d rows", totalDropped)
			err, _ := <-readerC
			return err
		}

		errs, err := logTable.ApplyBulk(c, batch, allMuts[:len(batch)])
		if err != nil {
			return errors.Annotate(err, "ApplyBulk failed on batch %d", batchNum).Err()
		}
		if len(errs) > 0 {
			logging.Warningf(c, "ApplyBulk: got %d errors: first: %q", len(errs), errs[0])
		}
		totalDropped += int64(len(batch) - len(errs))
	}
}

func (bti prodBTIface) getLogData(c context.Context, rk *rowKey, limit int, keysOnly bool, cb btGetCallback) error {
	logTable, err := bti.getLogTable()
	if err != nil {
		return err
	}

	// Construct read options based on Get request.
	ropts := []bigtable.ReadOption{
		bigtable.RowFilter(bigtable.FamilyFilter(logColumnFamily)),
		bigtable.RowFilter(bigtable.ColumnFilter(logColumn)),
		nil,
	}[:2]
	if keysOnly {
		ropts = append(ropts, bigtable.RowFilter(bigtable.StripValueFilter()))
	}
	if limit > 0 {
		ropts = append(ropts, bigtable.LimitRows(int64(limit)))
	}

	// This will limit the range to the immediate row key ("ASDF~INDEX") to
	// immediately after the row key ("ASDF~~"). See rowKey for more information.
	rng := bigtable.NewRange(rk.encode(), rk.pathPrefixUpperBound())

	var innerErr error
	err = logTable.ReadRows(c, rng, func(row bigtable.Row) bool {
		data, err := getLogRowData(row)
		if err != nil {
			innerErr = storage.ErrBadData
			return false
		}

		drk, err := decodeRowKey(row.Key())
		if err != nil {
			innerErr = err
			return false
		}

		if err := cb(drk, data); err != nil {
			innerErr = err
			return false
		}
		return true
	}, ropts...)
	if err != nil {
		return grpcutil.WrapIfTransient(err)
	}
	if innerErr != nil {
		return innerErr
	}
	return nil
}

func (bti prodBTIface) getMaxRowSize() int { return bigTableRowMaxBytes }

// getLogRowData loads the []byte contents of the supplied log row.
//
// If the row doesn't exist, storage.ErrDoesNotExist will be returned.
func getLogRowData(row bigtable.Row) (data []byte, err error) {
	items, ok := row[logColumnFamily]
	if !ok {
		err = storage.ErrDoesNotExist
		return
	}

	for _, item := range items {
		switch item.Column {
		case logColName:
			data = item.Value
			return
		}
	}

	// If no fields could be extracted, the rows does not exist.
	err = storage.ErrDoesNotExist
	return
}

// getReadItem retrieves a specific RowItem from the supplied Row.
func getReadItem(row bigtable.Row, family, column string) *bigtable.ReadItem {
	// Get the row for our family.
	items, ok := row[logColumnFamily]
	if !ok {
		return nil
	}

	// Get the specific ReadItem for our column
	colName := fmt.Sprintf("%s:%s", family, column)
	for _, item := range items {
		if item.Column == colName {
			return &item
		}
	}
	return nil
}

func wrapIfTransientForApply(err error) error {
	if err == nil {
		return nil
	}

	// For Apply, assume that anything other than InvalidArgument (bad data) is
	// transient. We exempt InvalidArgument because our data construction is
	// deterministic, and so this request can never succeed.
	switch code := grpcutil.Code(err); code {
	case codes.InvalidArgument:
		return err
	default:
		return transient.Tag.Apply(err)
	}
}
<commit_msg>[archivist] Fix deadlock in dropRowRange.<commit_after>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigtable

import (
	"context"
	"fmt"

	"go.chromium.org/luci/logdog/common/storage"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/grpc/grpcutil"

	"cloud.google.com/go/bigtable"
	"google.golang.org/grpc/codes"
)

const (
	logColumnFamily = "log"

	// The data column stores raw low row data (RecordIO blob).
	logColumn  = "data"
	logColName = logColumnFamily + ":" + logColumn
)

// Limits taken from here:
// https://cloud.google.com/bigtable/docs/schema-design
const (
	// bigTableRowMaxBytes is the maximum number of bytes that a single BigTable
	// row may hold.
	bigTableRowMaxBytes = 1024 * 1024 * 10 // 10MB
)

// btGetCallback is a callback that is invoked for each log data row returned
// by getLogData.
//
// If an error is encountered, no more log data will be fetched. The error will
// be propagated to the getLogData call.
type btGetCallback func(*rowKey, []byte) error

// btIface is a general interface for BigTable operations intended to enable
// unit tests to stub out BigTable without adding runtime inefficiency.
type btIface interface {
	// putLogData adds new log data to BigTable.
	//
	// If data already exists for the named row, it will return storage.ErrExists
	// and not add the data.
	putLogData(context.Context, *rowKey, []byte) error

	// getLogData retrieves rows belonging to the supplied stream record, starting
	// with the first index owned by that record. The supplied callback is invoked
	// once per retrieved row.
	//
	// rk is the starting row key.
	//
	// If the supplied limit is nonzero, no more than limit rows will be
	// retrieved.
	//
	// If keysOnly is true, then the callback will return nil row data.
	getLogData(c context.Context, rk *rowKey, limit int, keysOnly bool, cb btGetCallback) error

	// Drops all rows given the path prefix of rk.
	dropRowRange(c context.Context, rkPrefix *rowKey) error

	// getMaxRowSize returns the maximum row size that this implementation
	// supports.
	getMaxRowSize() int
}

// prodBTIface is a production implementation of a "btIface".
type prodBTIface struct {
	*Storage
}

func (bti prodBTIface) getLogTable() (*bigtable.Table, error) {
	if bti.Client == nil {
		return nil, errors.New("no client configured")
	}
	return bti.Client.Open(bti.LogTable), nil
}

func (bti prodBTIface) putLogData(c context.Context, rk *rowKey, data []byte) error {
	logTable, err := bti.getLogTable()
	if err != nil {
		return err
	}

	m := bigtable.NewMutation()
	m.Set(logColumnFamily, logColumn, bigtable.ServerTime, data)
	cm := bigtable.NewCondMutation(bigtable.RowKeyFilter(rk.encode()), nil, m)

	rowExists := false
	if err := logTable.Apply(c, rk.encode(), cm, bigtable.GetCondMutationResult(&rowExists)); err != nil {
		return wrapIfTransientForApply(err)
	}
	if rowExists {
		return storage.ErrExists
	}
	return nil
}

func (bti prodBTIface) dropRowRange(c context.Context, rk *rowKey) error {
	logTable, err := bti.getLogTable()
	if err != nil {
		return err
	}

	// ApplyBulk claims to be able to apply 100k mutations. Keep it small here to
	// stay well within the stated guidelines.
	const maxBatchSize = 100000 / 4

	del := bigtable.NewMutation()
	del.DeleteRow()

	allMuts := make([]*bigtable.Mutation, maxBatchSize)
	for i := range allMuts {
		allMuts[i] = del
	}

	prefix, upperBound := rk.pathPrefix(), rk.pathPrefixUpperBound()
	rng := bigtable.NewRange(prefix, upperBound)
	// apply paranoia mode
	if rng.Contains("") || prefix == "" || upperBound == "" {
		panic(fmt.Sprintf("NOTHING MAKES SENSE: %q %q %q", rng, prefix, upperBound))
	}

	keyC := make(chan string)

	// TODO(iannucci): parallelize row scan?
	readerC := make(chan error, 1)
	go func() {
		defer close(keyC)
		defer close(readerC)
		readerC <- logTable.ReadRows(c, rng, func(row bigtable.Row) bool {
			keyC <- row.Key()
			return true
		},
			bigtable.RowFilter(bigtable.FamilyFilter(logColumnFamily)),
			bigtable.RowFilter(bigtable.ColumnFilter(logColumn)),
			bigtable.RowFilter(bigtable.StripValueFilter()),
		)
	}()

	keys := make([]string, maxBatchSize)
	batchNum := 0
	var totalDropped int64
	for {
		batchNum++
		batch := keys[:0]
		for key := range keyC {
			batch = append(batch, key)
			if len(batch) >= maxBatchSize {
				break
			}
		}
		if len(batch) == 0 {
			logging.Infof(c, "dropRowRange: dropped %d rows", totalDropped)
			err, _ := <-readerC
			return err
		}

		errs, err := logTable.ApplyBulk(c, batch, allMuts[:len(batch)])
		if err != nil {
			return errors.Annotate(err, "ApplyBulk failed on batch %d", batchNum).Err()
		}
		if len(errs) > 0 {
			logging.Warningf(c, "ApplyBulk: got %d errors: first: %q", len(errs), errs[0])
		}
		totalDropped += int64(len(batch) - len(errs))
	}
}

func (bti prodBTIface) getLogData(c context.Context, rk *rowKey, limit int, keysOnly bool, cb btGetCallback) error {
	logTable, err := bti.getLogTable()
	if err != nil {
		return err
	}

	// Construct read options based on Get request.
	ropts := []bigtable.ReadOption{
		bigtable.RowFilter(bigtable.FamilyFilter(logColumnFamily)),
		bigtable.RowFilter(bigtable.ColumnFilter(logColumn)),
		nil,
	}[:2]
	if keysOnly {
		ropts = append(ropts, bigtable.RowFilter(bigtable.StripValueFilter()))
	}
	if limit > 0 {
		ropts = append(ropts, bigtable.LimitRows(int64(limit)))
	}

	// This will limit the range to the immediate row key ("ASDF~INDEX") to
	// immediately after the row key ("ASDF~~"). See rowKey for more information.
	rng := bigtable.NewRange(rk.encode(), rk.pathPrefixUpperBound())

	var innerErr error
	err = logTable.ReadRows(c, rng, func(row bigtable.Row) bool {
		data, err := getLogRowData(row)
		if err != nil {
			innerErr = storage.ErrBadData
			return false
		}

		drk, err := decodeRowKey(row.Key())
		if err != nil {
			innerErr = err
			return false
		}

		if err := cb(drk, data); err != nil {
			innerErr = err
			return false
		}
		return true
	}, ropts...)
	if err != nil {
		return grpcutil.WrapIfTransient(err)
	}
	if innerErr != nil {
		return innerErr
	}
	return nil
}

func (bti prodBTIface) getMaxRowSize() int { return bigTableRowMaxBytes }

// getLogRowData loads the []byte contents of the supplied log row.
//
// If the row doesn't exist, storage.ErrDoesNotExist will be returned.
func getLogRowData(row bigtable.Row) (data []byte, err error) {
	items, ok := row[logColumnFamily]
	if !ok {
		err = storage.ErrDoesNotExist
		return
	}

	for _, item := range items {
		switch item.Column {
		case logColName:
			data = item.Value
			return
		}
	}

	// If no fields could be extracted, the rows does not exist.
	err = storage.ErrDoesNotExist
	return
}

// getReadItem retrieves a specific RowItem from the supplied Row.
func getReadItem(row bigtable.Row, family, column string) *bigtable.ReadItem {
	// Get the row for our family.
	items, ok := row[logColumnFamily]
	if !ok {
		return nil
	}

	// Get the specific ReadItem for our column
	colName := fmt.Sprintf("%s:%s", family, column)
	for _, item := range items {
		if item.Column == colName {
			return &item
		}
	}
	return nil
}

func wrapIfTransientForApply(err error) error {
	if err == nil {
		return nil
	}

	// For Apply, assume that anything other than InvalidArgument (bad data) is
	// transient. We exempt InvalidArgument because our data construction is
	// deterministic, and so this request can never succeed.
	switch code := grpcutil.Code(err); code {
	case codes.InvalidArgument:
		return err
	default:
		return transient.Tag.Apply(err)
	}
}
<|endoftext|>


<commit_before>package comments

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	"github.com/google/go-github/github"
)

type CommentHandler func(client *github.Client, comment github.IssueCommentEvent) error

type CommentsHandler struct {
	client               *github.Client
	issueCommentHandlers []CommentHandler
	pullCommentHandlers  []CommentHandler
}

// NewHandler returns an HTTP handler which deprecates repositories
// by closing new issues with a comment directing attention elsewhere.
func NewHandler(client *github.Client, issuesHandlers []CommentHandler, pullRequestsHandlers []CommentHandler) *CommentsHandler {
	return &CommentsHandler{
		client:               client,
		issueCommentHandlers: issuesHandlers,
		pullCommentHandlers:  pullRequestsHandlers,
	}
}

func (h *CommentsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Header.Get("X-GitHub-Event") != "issue_comment" {
		http.Error(w, "not an issue_comment event.", 200)
		return
	}

	var event github.IssueCommentEvent
	err := json.NewDecoder(r.Body).Decode(&event)
	if err != nil {
		log.Println("error unmarshalling issue comment stuffs:", err)
		http.Error(w, "bad json", 400)
		return
	}

	var handlers []CommentHandler
	if isPullRequest(event) {
		handlers = h.pullCommentHandlers
	} else {
		handlers = h.issueCommentHandlers
	}

	for _, handler := range handlers {
		go handler(h.client, event)
	}

	fmt.Fprintf(w, "fired %d handlers", len(handlers))
}

func isPullRequest(event github.IssueCommentEvent) bool {
	return event.Issue.PullRequestLinks != nil
}
<commit_msg>Log the event type for comments<commit_after>package comments

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	"github.com/google/go-github/github"
)

type CommentHandler func(client *github.Client, comment github.IssueCommentEvent) error

type CommentsHandler struct {
	client               *github.Client
	issueCommentHandlers []CommentHandler
	pullCommentHandlers  []CommentHandler
}

// NewHandler returns an HTTP handler which deprecates repositories
// by closing new issues with a comment directing attention elsewhere.
func NewHandler(client *github.Client, issuesHandlers []CommentHandler, pullRequestsHandlers []CommentHandler) *CommentsHandler {
	return &CommentsHandler{
		client:               client,
		issueCommentHandlers: issuesHandlers,
		pullCommentHandlers:  pullRequestsHandlers,
	}
}

func (h *CommentsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if eventType := r.Header.Get("X-GitHub-Event"); eventType != "issue_comment" {
		log.Printf("received invalid event of type X-GitHub-Event: %s", eventType)
		http.Error(w, "not an issue_comment event.", 200)
		return
	}

	var event github.IssueCommentEvent
	err := json.NewDecoder(r.Body).Decode(&event)
	if err != nil {
		log.Println("error unmarshalling issue comment stuffs:", err)
		http.Error(w, "bad json", 400)
		return
	}

	var handlers []CommentHandler
	if isPullRequest(event) {
		handlers = h.pullCommentHandlers
	} else {
		handlers = h.issueCommentHandlers
	}

	for _, handler := range handlers {
		go handler(h.client, event)
	}

	fmt.Fprintf(w, "fired %d handlers", len(handlers))
}

func isPullRequest(event github.IssueCommentEvent) bool {
	return event.Issue.PullRequestLinks != nil
}
<|endoftext|>