<commit_before>package resolver

import (
	"sync"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/patrickmn/go-cache"

	"nimona.io/internal/rand"
	"nimona.io/pkg/context"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/hyperspace"
	"nimona.io/pkg/hyperspace/peerstore"
	"nimona.io/pkg/localpeer"
	"nimona.io/pkg/log"
	"nimona.io/pkg/network"
	"nimona.io/pkg/object"
	"nimona.io/pkg/peer"
)

var (
	hyperspaceAnnouncementType   = new(hyperspace.Announcement).Type()
	hyperspaceLookupResponseType = new(hyperspace.LookupResponse).Type()

	peerCacheTTL = 1 * time.Minute
)

const (
	ErrNoPeersToAsk = errors.Error("no peers to ask")
)

//go:generate mockgen -destination=../resolvermock/resolvermock_generated.go -package=resolvermock -source=resolver.go

type (
	Resolver interface {
		Lookup(
			ctx context.Context,
			opts ...LookupOption,
		) ([]*peer.ConnectionInfo, error)
	}
	resolver struct {
		context                        context.Context
		network                        network.Network
		localpeer                      localpeer.LocalPeer
		peerCache                      *peerstore.PeerCache
		localPeerAnnouncementCache     *hyperspace.Announcement
		localPeerAnnouncementCacheLock sync.RWMutex
		bootstrapPeers                 []*peer.ConnectionInfo
		blocklist                      *cache.Cache
	}
	// Option for customizing a new resolver
	Option func(*resolver)
)

// New returns a new resolver
func New(
	ctx context.Context,
	netw network.Network,
	opts ...Option,
) Resolver {
	r := &resolver{
		context: ctx,
		network: netw,
		peerCache: peerstore.NewPeerCache(
			time.Minute,
			"nimona_hyperspace_resolver",
		),
		localPeerAnnouncementCacheLock: sync.RWMutex{},
		bootstrapPeers:                 []*peer.ConnectionInfo{},
		blocklist:                      cache.New(time.Second*5, time.Second*60),
	}

	for _, opt := range opts {
		opt(r)
	}

	r.localpeer = r.network.LocalPeer()

	// we are listening for all incoming object types in order to learn about
	// new peers that are talking to us so we can announce ourselves to them
	go network.HandleEnvelopeSubscription(
		r.network.Subscribe(),
		r.handleObject,
	)

	for _, p := range r.bootstrapPeers {
		r.peerCache.Put(&hyperspace.Announcement{
			ConnectionInfo: p,
		}, 0)
	}

	go func() {
		r.announceSelf()
		announceOnUpdate, cf := r.localpeer.ListenForUpdates()
		defer cf()
		announceTimer := time.NewTicker(30 * time.Second)
		select {
		case <-announceTimer.C:
			r.announceSelf()
		case <-announceOnUpdate:
			r.announceSelf()
		}
	}()

	return r
}

// Lookup finds and returns peer infos from a fingerprint
// TODO consider returning peers synchronously
func (r *resolver) Lookup(
	ctx context.Context,
	opts ...LookupOption,
) ([]*peer.ConnectionInfo, error) {
	if len(r.bootstrapPeers) == 0 {
		return nil, errors.New("no peers to ask")
	}

	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.Lookup"),
	)
	logger.Debug("looking up")

	opt := ParseLookupOptions(opts...)
	bl := hyperspace.New(opt.Lookups...)

	// send content requests to recipients
	req := &hyperspace.LookupRequest{
		Metadata: object.Metadata{
			Owner: r.localpeer.GetPrimaryPeerKey().PublicKey(),
		},
		Nonce:       rand.String(12),
		QueryVector: bl,
	}
	reqObject := req.ToObject()

	// listen for lookup responses
	resSub := r.network.Subscribe(
		network.FilterByObjectType(hyperspaceLookupResponseType),
		func(e *network.Envelope) bool {
			v := e.Payload.Data["nonce:s"]
			rn, ok := v.(string)
			return ok && rn == req.Nonce
		},
	)

	go func() {
		for _, bp := range r.bootstrapPeers {
			err := r.network.Send(
				ctx,
				reqObject,
				bp,
			)
			if err != nil {
				logger.Debug("could send request to peer", log.Error(err))
				continue
			}
			logger.Debug(
				"asked peer",
				log.String("peer", bp.PublicKey.String()),
			)
		}
	}()

	// create channel to keep peers we find
	peers := []*peer.ConnectionInfo{}
	done := make(chan struct{})

	go func() {
		for {
			e, err := resSub.Next()
			if err != nil {
				break
			}
			r := &hyperspace.LookupResponse{}
			if err := r.FromObject(e.Payload); err != nil {
				continue
			}
			// TODO verify peer?
			for _, ann := range r.Announcements {
				peers = append(peers, ann.ConnectionInfo)
			}
			close(done)
			break
		}
	}()

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case <-done:
		return peers, nil
	}
}

func (r *resolver) handleObject(
	e *network.Envelope,
) error {
	// attempt to recover correlation id from request id
	ctx := r.context

	// handle payload
	o := e.Payload
	if o.Type == hyperspaceAnnouncementType {
		v := &hyperspace.Announcement{}
		if err := v.FromObject(o); err != nil {
			return err
		}
		r.handleAnnouncement(ctx, v)
	}
	return nil
}

func (r *resolver) handleAnnouncement(
	ctx context.Context,
	p *hyperspace.Announcement,
) {
	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.handleAnnouncement"),
		log.String("peer.publicKey", p.ConnectionInfo.PublicKey.String()),
		log.Strings("peer.addresses", p.ConnectionInfo.Addresses),
	)
	logger.Debug("adding peer to cache")
	r.peerCache.Put(p, peerCacheTTL)
}

func (r *resolver) announceSelf() {
	ctx := context.New(
		context.WithParent(r.context),
	)
	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.announceSelf"),
	)
	n := 0
	for _, p := range r.bootstrapPeers {
		if err := r.network.Send(
			context.New(
				context.WithParent(ctx),
				context.WithTimeout(time.Second*3),
			),
			r.getLocalPeerAnnouncement().ToObject(),
			p,
		); err != nil {
			logger.Error(
				"error announcing self to bootstrap",
				log.String("peer", p.PublicKey.String()),
				log.Error(err),
			)
			continue
		}
		n++
	}
	logger.Info(
		"announced self to bootstrap peers",
		log.Int("bootstrapPeers", n),
	)
}

func (r *resolver) getLocalPeerAnnouncement() *hyperspace.Announcement {
	r.localPeerAnnouncementCacheLock.RLock()
	lastAnnouncement := r.localPeerAnnouncementCache
	r.localPeerAnnouncementCacheLock.RUnlock()

	peerKey := r.localpeer.GetPrimaryPeerKey().PublicKey()
	certificates := r.localpeer.GetCertificates()
	contentHashes := r.localpeer.GetContentHashes()
	contentTypes := r.localpeer.GetContentTypes()
	addresses := r.localpeer.GetAddresses()
	relays := r.localpeer.GetRelays()

	// gather up peer key, certificates, content ids and types
	hs := contentTypes
	hs = append(hs, peerKey.String())
	for _, c := range contentHashes {
		hs = append(hs, c.String())
	}
	for _, c := range certificates {
		if !c.Metadata.Signature.IsEmpty() {
			hs = append(hs, c.Metadata.Signature.Signer.String())
		}
	}
	vec := hyperspace.New(hs...)

	if lastAnnouncement != nil &&
		cmp.Equal(lastAnnouncement.ConnectionInfo.Addresses, addresses) &&
		cmp.Equal(lastAnnouncement.PeerVector, vec) {
		return lastAnnouncement
	}

	localPeerAnnouncementCache := &hyperspace.Announcement{
		Metadata: object.Metadata{
			Owner: peerKey,
		},
		Version: time.Now().Unix(),
		ConnectionInfo: &peer.ConnectionInfo{
			PublicKey: peerKey,
			Addresses: addresses,
			Relays:    relays,
		},
		PeerVector:       vec,
		PeerCapabilities: contentTypes,
	}

	r.localPeerAnnouncementCacheLock.Lock()
	r.localPeerAnnouncementCache = localPeerAnnouncementCache
	r.localPeerAnnouncementCacheLock.Unlock()

	return localPeerAnnouncementCache
}
<commit_msg>fix(resolver): fix not announcing self forever<commit_after>package resolver

import (
	"sync"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/patrickmn/go-cache"

	"nimona.io/internal/rand"
	"nimona.io/pkg/context"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/hyperspace"
	"nimona.io/pkg/hyperspace/peerstore"
	"nimona.io/pkg/localpeer"
	"nimona.io/pkg/log"
	"nimona.io/pkg/network"
	"nimona.io/pkg/object"
	"nimona.io/pkg/peer"
)

var (
	hyperspaceAnnouncementType   = new(hyperspace.Announcement).Type()
	hyperspaceLookupResponseType = new(hyperspace.LookupResponse).Type()

	peerCacheTTL = 1 * time.Minute
)

const (
	ErrNoPeersToAsk = errors.Error("no peers to ask")
)

//go:generate mockgen -destination=../resolvermock/resolvermock_generated.go -package=resolvermock -source=resolver.go

type (
	Resolver interface {
		Lookup(
			ctx context.Context,
			opts ...LookupOption,
		) ([]*peer.ConnectionInfo, error)
	}
	resolver struct {
		context                        context.Context
		network                        network.Network
		localpeer                      localpeer.LocalPeer
		peerCache                      *peerstore.PeerCache
		localPeerAnnouncementCache     *hyperspace.Announcement
		localPeerAnnouncementCacheLock sync.RWMutex
		bootstrapPeers                 []*peer.ConnectionInfo
		blocklist                      *cache.Cache
	}
	// Option for customizing a new resolver
	Option func(*resolver)
)

// New returns a new resolver
func New(
	ctx context.Context,
	netw network.Network,
	opts ...Option,
) Resolver {
	r := &resolver{
		context: ctx,
		network: netw,
		peerCache: peerstore.NewPeerCache(
			time.Minute,
			"nimona_hyperspace_resolver",
		),
		localPeerAnnouncementCacheLock: sync.RWMutex{},
		bootstrapPeers:                 []*peer.ConnectionInfo{},
		blocklist:                      cache.New(time.Second*5, time.Second*60),
	}

	for _, opt := range opts {
		opt(r)
	}

	r.localpeer = r.network.LocalPeer()

	// we are listening for all incoming object types in order to learn about
	// new peers that are talking to us so we can announce ourselves to them
	go network.HandleEnvelopeSubscription(
		r.network.Subscribe(),
		r.handleObject,
	)

	for _, p := range r.bootstrapPeers {
		r.peerCache.Put(&hyperspace.Announcement{
			ConnectionInfo: p,
		}, 0)
	}

	go func() {
		r.announceSelf()
		announceOnUpdate, cf := r.localpeer.ListenForUpdates()
		defer cf()
		announceTicker := time.NewTicker(30 * time.Second)
		for {
			select {
			case <-announceTicker.C:
				r.announceSelf()
			case <-announceOnUpdate:
				r.announceSelf()
			}
		}
	}()

	return r
}

// Lookup finds and returns peer infos from a fingerprint
// TODO consider returning peers synchronously
func (r *resolver) Lookup(
	ctx context.Context,
	opts ...LookupOption,
) ([]*peer.ConnectionInfo, error) {
	if len(r.bootstrapPeers) == 0 {
		return nil, errors.New("no peers to ask")
	}

	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.Lookup"),
	)
	logger.Debug("looking up")

	opt := ParseLookupOptions(opts...)
	bl := hyperspace.New(opt.Lookups...)

	// send content requests to recipients
	req := &hyperspace.LookupRequest{
		Metadata: object.Metadata{
			Owner: r.localpeer.GetPrimaryPeerKey().PublicKey(),
		},
		Nonce:       rand.String(12),
		QueryVector: bl,
	}
	reqObject := req.ToObject()

	// listen for lookup responses
	resSub := r.network.Subscribe(
		network.FilterByObjectType(hyperspaceLookupResponseType),
		func(e *network.Envelope) bool {
			v := e.Payload.Data["nonce:s"]
			rn, ok := v.(string)
			return ok && rn == req.Nonce
		},
	)

	go func() {
		for _, bp := range r.bootstrapPeers {
			err := r.network.Send(
				ctx,
				reqObject,
				bp,
			)
			if err != nil {
				logger.Debug("could send request to peer", log.Error(err))
				continue
			}
			logger.Debug(
				"asked peer",
				log.String("peer", bp.PublicKey.String()),
			)
		}
	}()

	// create channel to keep peers we find
	peers := []*peer.ConnectionInfo{}
	done := make(chan struct{})

	go func() {
		for {
			e, err := resSub.Next()
			if err != nil {
				break
			}
			r := &hyperspace.LookupResponse{}
			if err := r.FromObject(e.Payload); err != nil {
				continue
			}
			// TODO verify peer?
			for _, ann := range r.Announcements {
				peers = append(peers, ann.ConnectionInfo)
			}
			close(done)
			break
		}
	}()

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case <-done:
		return peers, nil
	}
}

func (r *resolver) handleObject(
	e *network.Envelope,
) error {
	// attempt to recover correlation id from request id
	ctx := r.context

	// handle payload
	o := e.Payload
	if o.Type == hyperspaceAnnouncementType {
		v := &hyperspace.Announcement{}
		if err := v.FromObject(o); err != nil {
			return err
		}
		r.handleAnnouncement(ctx, v)
	}
	return nil
}

func (r *resolver) handleAnnouncement(
	ctx context.Context,
	p *hyperspace.Announcement,
) {
	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.handleAnnouncement"),
		log.String("peer.publicKey", p.ConnectionInfo.PublicKey.String()),
		log.Strings("peer.addresses", p.ConnectionInfo.Addresses),
	)
	logger.Debug("adding peer to cache")
	r.peerCache.Put(p, peerCacheTTL)
}

func (r *resolver) announceSelf() {
	ctx := context.New(
		context.WithParent(r.context),
	)
	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.announceSelf"),
	)
	n := 0
	for _, p := range r.bootstrapPeers {
		if err := r.network.Send(
			context.New(
				context.WithParent(ctx),
				context.WithTimeout(time.Second*3),
			),
			r.getLocalPeerAnnouncement().ToObject(),
			p,
		); err != nil {
			logger.Error(
				"error announcing self to bootstrap",
				log.String("peer", p.PublicKey.String()),
				log.Error(err),
			)
			continue
		}
		n++
	}
	logger.Info(
		"announced self to bootstrap peers",
		log.Int("bootstrapPeers", n),
	)
}

func (r *resolver) getLocalPeerAnnouncement() *hyperspace.Announcement {
	r.localPeerAnnouncementCacheLock.RLock()
	lastAnnouncement := r.localPeerAnnouncementCache
	r.localPeerAnnouncementCacheLock.RUnlock()

	peerKey := r.localpeer.GetPrimaryPeerKey().PublicKey()
	certificates := r.localpeer.GetCertificates()
	contentHashes := r.localpeer.GetContentHashes()
	contentTypes := r.localpeer.GetContentTypes()
	addresses := r.localpeer.GetAddresses()
	relays := r.localpeer.GetRelays()

	// gather up peer key, certificates, content ids and types
	hs := contentTypes
	hs = append(hs, peerKey.String())
	for _, c := range contentHashes {
		hs = append(hs, c.String())
	}
	for _, c := range certificates {
		if !c.Metadata.Signature.IsEmpty() {
			hs = append(hs, c.Metadata.Signature.Signer.String())
		}
	}
	vec := hyperspace.New(hs...)

	if lastAnnouncement != nil &&
		cmp.Equal(lastAnnouncement.ConnectionInfo.Addresses, addresses) &&
		cmp.Equal(lastAnnouncement.PeerVector, vec) {
		return lastAnnouncement
	}

	localPeerAnnouncementCache := &hyperspace.Announcement{
		Metadata: object.Metadata{
			Owner: peerKey,
		},
		Version: time.Now().Unix(),
		ConnectionInfo: &peer.ConnectionInfo{
			PublicKey: peerKey,
			Addresses: addresses,
			Relays:    relays,
		},
		PeerVector:       vec,
		PeerCapabilities: contentTypes,
	}

	r.localPeerAnnouncementCacheLock.Lock()
	r.localPeerAnnouncementCache = localPeerAnnouncementCache
	r.localPeerAnnouncementCacheLock.Unlock()

	return localPeerAnnouncementCache
}
<|endoftext|>


<commit_before>package v5

import (
	"fmt"
	"net/http"

	"gopkg.in/olivere/elastic.v5"
	"k8s.io/client-go/tools/cache"

	"github.com/jetstack-experimental/navigator/pkg/client/clientset_generated/clientset"
	listersv1alpha1 "github.com/jetstack-experimental/navigator/pkg/client/listers_generated/navigator/v1alpha1"
	"github.com/jetstack-experimental/navigator/pkg/pilot/genericpilot/hook"
)

const (
	localESClientURL = "http://127.0.0.1:9200"
)

type Pilot struct {
	Options *PilotOptions

	navigatorClient     clientset.Interface
	pilotLister         listersv1alpha1.PilotLister
	pilotInformerSynced cache.InformerSynced

	esClusterLister         listersv1alpha1.ElasticsearchClusterLister
	esClusterInformerSynced cache.InformerSynced

	localESClient *elastic.Client
}

func NewPilot(opts *PilotOptions) (*Pilot, error) {
	pilotInformer := opts.sharedInformerFactory.Navigator().V1alpha1().Pilots()
	esClusterInformer := opts.sharedInformerFactory.Navigator().V1alpha1().ElasticsearchClusters()

	cl, err := elastic.NewClient(elastic.SetHttpClient(http.DefaultClient), elastic.SetURL(localESClientURL))
	if err != nil {
		return nil, nil
	}

	p := &Pilot{
		Options:                 opts,
		navigatorClient:         opts.navigatorClientset,
		pilotLister:             pilotInformer.Lister(),
		pilotInformerSynced:     pilotInformer.Informer().HasSynced,
		esClusterLister:         esClusterInformer.Lister(),
		esClusterInformerSynced: esClusterInformer.Informer().HasSynced,
		localESClient:           cl,
	}

	return p, nil
}

func (p *Pilot) WaitForCacheSync(stopCh <-chan struct{}) error {
	if !cache.WaitForCacheSync(stopCh, p.pilotInformerSynced, p.esClusterInformerSynced) {
		return fmt.Errorf("timed out waiting for caches to sync")
	}
	return nil
}

func (p *Pilot) Hooks() *hook.Hooks {
	return &hook.Hooks{
		PreStart: []hook.Interface{
			hook.New("WriteConfig", p.WriteConfig),
			hook.New("InstallPlugins", p.InstallPlugins),
		},
	}
}
<commit_msg>Retry creating elastic client<commit_after>package v5

import (
	"fmt"
	"net/http"

	"github.com/golang/glog"
	"gopkg.in/olivere/elastic.v5"
	"k8s.io/client-go/tools/cache"

	"github.com/jetstack-experimental/navigator/pkg/client/clientset_generated/clientset"
	listersv1alpha1 "github.com/jetstack-experimental/navigator/pkg/client/listers_generated/navigator/v1alpha1"
	"github.com/jetstack-experimental/navigator/pkg/pilot/genericpilot/hook"
)

const (
	localESClientURL = "http://127.0.0.1:9200"
)

type Pilot struct {
	Options *PilotOptions

	navigatorClient     clientset.Interface
	pilotLister         listersv1alpha1.PilotLister
	pilotInformerSynced cache.InformerSynced

	esClusterLister         listersv1alpha1.ElasticsearchClusterLister
	esClusterInformerSynced cache.InformerSynced

	localESClient *elastic.Client
}

func NewPilot(opts *PilotOptions) (*Pilot, error) {
	pilotInformer := opts.sharedInformerFactory.Navigator().V1alpha1().Pilots()
	esClusterInformer := opts.sharedInformerFactory.Navigator().V1alpha1().ElasticsearchClusters()

	p := &Pilot{
		Options:                 opts,
		navigatorClient:         opts.navigatorClientset,
		pilotLister:             pilotInformer.Lister(),
		pilotInformerSynced:     pilotInformer.Informer().HasSynced,
		esClusterLister:         esClusterInformer.Lister(),
		esClusterInformerSynced: esClusterInformer.Informer().HasSynced,
		localESClient:           &elastic.Client{},
	}

	// Setup a gofunc to keep attempting to create an API client
	go func() {
		for {
			cl, err := elastic.NewClient(elastic.SetHttpClient(http.DefaultClient), elastic.SetURL(localESClientURL))
			if err == nil {
				p.localESClient = cl
				break
			}
			glog.Errorf("Error creating elasticsearch api client: %s", err.Error())
		}
	}()

	return p, nil
}

func (p *Pilot) WaitForCacheSync(stopCh <-chan struct{}) error {
	if !cache.WaitForCacheSync(stopCh, p.pilotInformerSynced, p.esClusterInformerSynced) {
		return fmt.Errorf("timed out waiting for caches to sync")
	}
	return nil
}

func (p *Pilot) Hooks() *hook.Hooks {
	return &hook.Hooks{
		PreStart: []hook.Interface{
			hook.New("WriteConfig", p.WriteConfig),
			hook.New("InstallPlugins", p.InstallPlugins),
		},
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package systemlogmonitor

import (
	"encoding/json"
	"io/ioutil"
	"time"

	"github.com/golang/glog"

	"k8s.io/node-problem-detector/pkg/problemdaemon"
	"k8s.io/node-problem-detector/pkg/problemmetrics"
	"k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers"
	watchertypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers/types"
	logtypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/types"
	systemlogtypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/types"
	"k8s.io/node-problem-detector/pkg/types"
	"k8s.io/node-problem-detector/pkg/util"
	"k8s.io/node-problem-detector/pkg/util/tomb"
)

const SystemLogMonitorName = "system-log-monitor"

func init() {
	problemdaemon.Register(
		SystemLogMonitorName,
		types.ProblemDaemonHandler{
			CreateProblemDaemonOrDie: NewLogMonitorOrDie,
			CmdOptionDescription:     "Set to config file paths."})
}

type logMonitor struct {
	configPath string
	watcher    watchertypes.LogWatcher
	buffer     LogBuffer
	config     MonitorConfig
	conditions []types.Condition
	logCh      <-chan *logtypes.Log
	output     chan *types.Status
	tomb       *tomb.Tomb
}

// NewLogMonitorOrDie create a new LogMonitor, panic if error occurs.
func NewLogMonitorOrDie(configPath string) types.Monitor {
	l := &logMonitor{
		configPath: configPath,
		tomb:       tomb.NewTomb(),
	}

	f, err := ioutil.ReadFile(configPath)
	if err != nil {
		glog.Fatalf("Failed to read configuration file %q: %v", configPath, err)
	}
	err = json.Unmarshal(f, &l.config)
	if err != nil {
		glog.Fatalf("Failed to unmarshal configuration file %q: %v", configPath, err)
	}
	// Apply default configurations
	(&l.config).ApplyDefaultConfiguration()
	err = l.config.ValidateRules()
	if err != nil {
		glog.Fatalf("Failed to validate %s matching rules %+v: %v", l.configPath, l.config.Rules, err)
	}
	glog.Infof("Finish parsing log monitor config file %s: %+v", l.configPath, l.config)

	l.watcher = logwatchers.GetLogWatcherOrDie(l.config.WatcherConfig)
	l.buffer = NewLogBuffer(l.config.BufferSize)
	// A 1000 size channel should be big enough.
	l.output = make(chan *types.Status, 1000)

	if *l.config.EnableMetricsReporting {
		initializeProblemMetricsOrDie(l.config.Rules)
	}
	return l
}

// initializeProblemMetricsOrDie creates problem metrics for all problems and set the value to 0,
// panic if error occurs.
func initializeProblemMetricsOrDie(rules []systemlogtypes.Rule) {
	for _, rule := range rules {
		if rule.Type == types.Perm {
			err := problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(rule.Condition, rule.Reason, false)
			if err != nil {
				glog.Fatalf("Failed to initialize problem gauge metrics for problem %q, reason %q: %v",
					rule.Condition, rule.Reason, err)
			}
		}
		err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(rule.Reason, 0)
		if err != nil {
			glog.Fatalf("Failed to initialize problem counter metrics for %q: %v", rule.Reason, err)
		}
	}
}

func (l *logMonitor) Start() (<-chan *types.Status, error) {
	glog.Infof("Start log monitor %s", l.configPath)
	var err error
	l.logCh, err = l.watcher.Watch()
	if err != nil {
		return nil, err
	}
	go l.monitorLoop()
	return l.output, nil
}

func (l *logMonitor) Stop() {
	glog.Infof("Stop log monitor %s", l.configPath)
	l.tomb.Stop()
}

// monitorLoop is the main loop of log monitor.
func (l *logMonitor) monitorLoop() {
	defer l.tomb.Done()
	l.initializeStatus()
	for {
		select {
		case log := <-l.logCh:
			l.parseLog(log)
		case <-l.tomb.Stopping():
			l.watcher.Stop()
			glog.Infof("Log monitor stopped: %s", l.configPath)
			return
		}
	}
}

// parseLog parses one log line.
func (l *logMonitor) parseLog(log *logtypes.Log) {
	// Once there is new log, log monitor will push it into the log buffer and try
	// to match each rule. If any rule is matched, log monitor will report a status.
	l.buffer.Push(log)
	for _, rule := range l.config.Rules {
		matched := l.buffer.Match(rule.Pattern)
		if len(matched) == 0 {
			continue
		}
		status := l.generateStatus(matched, rule)
		glog.Infof("New status generated: %+v", status)
		l.output <- status
	}
}

// generateStatus generates status from the logs.
func (l *logMonitor) generateStatus(logs []*logtypes.Log, rule systemlogtypes.Rule) *types.Status {
	// We use the timestamp of the first log line as the timestamp of the status.
	timestamp := logs[0].Timestamp
	message := generateMessage(logs)
	var events []types.Event
	var changedConditions []*types.Condition
	if rule.Type == types.Temp {
		// For temporary error only generate event
		events = append(events, types.Event{
			Severity:  types.Warn,
			Timestamp: timestamp,
			Reason:    rule.Reason,
			Message:   message,
		})
	} else {
		// For permanent error changes the condition
		for i := range l.conditions {
			condition := &l.conditions[i]
			if condition.Type == rule.Condition {
				// Update transition timestamp and message when the condition
				// changes. Condition is considered to be changed only when
				// status or reason changes.
				if condition.Status == types.False || condition.Reason != rule.Reason {
					condition.Transition = timestamp
					condition.Message = message
					events = append(events, util.GenerateConditionChangeEvent(
						condition.Type,
						types.True,
						rule.Reason,
						timestamp,
					))
				}
				condition.Status = types.True
				condition.Reason = rule.Reason
				changedConditions = append(changedConditions, condition)
				break
			}
		}
	}

	if *l.config.EnableMetricsReporting {
		for _, event := range events {
			err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(event.Reason, 1)
			if err != nil {
				glog.Errorf("Failed to update problem counter metrics for %q: %v", event.Reason, err)
			}
		}
		for _, condition := range changedConditions {
			err := problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(
				condition.Type, condition.Reason, condition.Status == types.True)
			if err != nil {
				glog.Errorf("Failed to update problem gauge metrics for problem %q, reason %q: %v",
					condition.Type, condition.Reason, err)
			}
		}
	}

	return &types.Status{
		Source: l.config.Source,
		// TODO(random-liu): Aggregate events and conditions and then do periodically report.
		Events:     events,
		Conditions: l.conditions,
	}
}

// initializeStatus initializes the internal condition and also reports it to the node problem detector.
func (l *logMonitor) initializeStatus() {
	// Initialize the default node conditions
	l.conditions = initialConditions(l.config.DefaultConditions)
	glog.Infof("Initialize condition generated: %+v", l.conditions)
	// Update the initial status
	l.output <- &types.Status{
		Source:     l.config.Source,
		Conditions: l.conditions,
	}
}

func initialConditions(defaults []types.Condition) []types.Condition {
	conditions := make([]types.Condition, len(defaults))
	copy(conditions, defaults)
	for i := range conditions {
		conditions[i].Status = types.False
		conditions[i].Transition = time.Now()
	}
	return conditions
}

func generateMessage(logs []*logtypes.Log) string {
	messages := []string{}
	for _, log := range logs {
		messages = append(messages, log.Message)
	}
	return concatLogs(messages)
}
<commit_msg>avoid log channel closed caused endless loop<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package systemlogmonitor

import (
	"encoding/json"
	"io/ioutil"
	"time"

	"github.com/golang/glog"

	"k8s.io/node-problem-detector/pkg/problemdaemon"
	"k8s.io/node-problem-detector/pkg/problemmetrics"
	"k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers"
	watchertypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers/types"
	logtypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/types"
	systemlogtypes "k8s.io/node-problem-detector/pkg/systemlogmonitor/types"
	"k8s.io/node-problem-detector/pkg/types"
	"k8s.io/node-problem-detector/pkg/util"
	"k8s.io/node-problem-detector/pkg/util/tomb"
)

const SystemLogMonitorName = "system-log-monitor"

func init() {
	problemdaemon.Register(
		SystemLogMonitorName,
		types.ProblemDaemonHandler{
			CreateProblemDaemonOrDie: NewLogMonitorOrDie,
			CmdOptionDescription:     "Set to config file paths."})
}

type logMonitor struct {
	configPath string
	watcher    watchertypes.LogWatcher
	buffer     LogBuffer
	config     MonitorConfig
	conditions []types.Condition
	logCh      <-chan *logtypes.Log
	output     chan *types.Status
	tomb       *tomb.Tomb
}

// NewLogMonitorOrDie create a new LogMonitor, panic if error occurs.
func NewLogMonitorOrDie(configPath string) types.Monitor {
	l := &logMonitor{
		configPath: configPath,
		tomb:       tomb.NewTomb(),
	}

	f, err := ioutil.ReadFile(configPath)
	if err != nil {
		glog.Fatalf("Failed to read configuration file %q: %v", configPath, err)
	}
	err = json.Unmarshal(f, &l.config)
	if err != nil {
		glog.Fatalf("Failed to unmarshal configuration file %q: %v", configPath, err)
	}
	// Apply default configurations
	(&l.config).ApplyDefaultConfiguration()
	err = l.config.ValidateRules()
	if err != nil {
		glog.Fatalf("Failed to validate %s matching rules %+v: %v", l.configPath, l.config.Rules, err)
	}
	glog.Infof("Finish parsing log monitor config file %s: %+v", l.configPath, l.config)

	l.watcher = logwatchers.GetLogWatcherOrDie(l.config.WatcherConfig)
	l.buffer = NewLogBuffer(l.config.BufferSize)
	// A 1000 size channel should be big enough.
	l.output = make(chan *types.Status, 1000)

	if *l.config.EnableMetricsReporting {
		initializeProblemMetricsOrDie(l.config.Rules)
	}
	return l
}

// initializeProblemMetricsOrDie creates problem metrics for all problems and set the value to 0,
// panic if error occurs.
func initializeProblemMetricsOrDie(rules []systemlogtypes.Rule) {
	for _, rule := range rules {
		if rule.Type == types.Perm {
			err := problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(rule.Condition, rule.Reason, false)
			if err != nil {
				glog.Fatalf("Failed to initialize problem gauge metrics for problem %q, reason %q: %v",
					rule.Condition, rule.Reason, err)
			}
		}
		err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(rule.Reason, 0)
		if err != nil {
			glog.Fatalf("Failed to initialize problem counter metrics for %q: %v", rule.Reason, err)
		}
	}
}

func (l *logMonitor) Start() (<-chan *types.Status, error) {
	glog.Infof("Start log monitor %s", l.configPath)
	var err error
	l.logCh, err = l.watcher.Watch()
	if err != nil {
		return nil, err
	}
	go l.monitorLoop()
	return l.output, nil
}

func (l *logMonitor) Stop() {
	glog.Infof("Stop log monitor %s", l.configPath)
	l.tomb.Stop()
}

// monitorLoop is the main loop of log monitor.
func (l *logMonitor) monitorLoop() {
	defer l.tomb.Done()
	l.initializeStatus()
	for {
		select {
		case log, ok := <-l.logCh:
			if !ok {
				glog.Errorf("Log channel closed")
				return
			}
			l.parseLog(log)
		case <-l.tomb.Stopping():
			l.watcher.Stop()
			glog.Infof("Log monitor stopped: %s", l.configPath)
			return
		}
	}
}

// parseLog parses one log line.
func (l *logMonitor) parseLog(log *logtypes.Log) {
	// Once there is new log, log monitor will push it into the log buffer and try
	// to match each rule. If any rule is matched, log monitor will report a status.
	l.buffer.Push(log)
	for _, rule := range l.config.Rules {
		matched := l.buffer.Match(rule.Pattern)
		if len(matched) == 0 {
			continue
		}
		status := l.generateStatus(matched, rule)
		glog.Infof("New status generated: %+v", status)
		l.output <- status
	}
}

// generateStatus generates status from the logs.
func (l *logMonitor) generateStatus(logs []*logtypes.Log, rule systemlogtypes.Rule) *types.Status {
	// We use the timestamp of the first log line as the timestamp of the status.
	timestamp := logs[0].Timestamp
	message := generateMessage(logs)
	var events []types.Event
	var changedConditions []*types.Condition
	if rule.Type == types.Temp {
		// For temporary error only generate event
		events = append(events, types.Event{
			Severity:  types.Warn,
			Timestamp: timestamp,
			Reason:    rule.Reason,
			Message:   message,
		})
	} else {
		// For permanent error changes the condition
		for i := range l.conditions {
			condition := &l.conditions[i]
			if condition.Type == rule.Condition {
				// Update transition timestamp and message when the condition
				// changes. Condition is considered to be changed only when
				// status or reason changes.
				if condition.Status == types.False || condition.Reason != rule.Reason {
					condition.Transition = timestamp
					condition.Message = message
					events = append(events, util.GenerateConditionChangeEvent(
						condition.Type,
						types.True,
						rule.Reason,
						timestamp,
					))
				}
				condition.Status = types.True
				condition.Reason = rule.Reason
				changedConditions = append(changedConditions, condition)
				break
			}
		}
	}

	if *l.config.EnableMetricsReporting {
		for _, event := range events {
			err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(event.Reason, 1)
			if err != nil {
				glog.Errorf("Failed to update problem counter metrics for %q: %v", event.Reason, err)
			}
		}
		for _, condition := range changedConditions {
			err := problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(
				condition.Type, condition.Reason, condition.Status == types.True)
			if err != nil {
				glog.Errorf("Failed to update problem gauge metrics for problem %q, reason %q: %v",
					condition.Type, condition.Reason, err)
			}
		}
	}

	return &types.Status{
		Source: l.config.Source,
		// TODO(random-liu): Aggregate events and conditions and then do periodically report.
		Events:     events,
		Conditions: l.conditions,
	}
}

// initializeStatus initializes the internal condition and also reports it to the node problem detector.
func (l *logMonitor) initializeStatus() {
	// Initialize the default node conditions
	l.conditions = initialConditions(l.config.DefaultConditions)
	glog.Infof("Initialize condition generated: %+v", l.conditions)
	// Update the initial status
	l.output <- &types.Status{
		Source:     l.config.Source,
		Conditions: l.conditions,
	}
}

func initialConditions(defaults []types.Condition) []types.Condition {
	conditions := make([]types.Condition, len(defaults))
	copy(conditions, defaults)
	for i := range conditions {
		conditions[i].Status = types.False
		conditions[i].Transition = time.Now()
	}
	return conditions
}

func generateMessage(logs []*logtypes.Log) string {
	messages := []string{}
	for _, log := range logs {
		messages = append(messages, log.Message)
	}
	return concatLogs(messages)
}
<|endoftext|>


<commit_before>// +build !linux

/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mount

type Mounter struct{}

func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
	return nil
}

func (mounter *Mounter) Unmount(target string) error {
	return nil
}

func (mounter *Mounter) List() ([]MountPoint, error) {
	return []MountPoint{}, nil
}

func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
	return true, nil
}

func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
	return "", nil
}

func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
	return false, nil
}

func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
	return true, nil
}

func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
	return nil
}

func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) {
	return true, nil
}
<commit_msg>add IsNotMountPoint() to mount_unsupported.go<commit_after>// +build !linux

/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mount

type Mounter struct{}

func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
	return nil
}

func (mounter *Mounter) Unmount(target string) error {
	return nil
}

func (mounter *Mounter) List() ([]MountPoint, error) {
	return []MountPoint{}, nil
}

func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
	return true, nil
}

func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
	return "", nil
}

func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
	return false, nil
}

func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
	return true, nil
}

func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
	return nil
}

func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) {
	return true, nil
}

func IsNotMountPoint(file string) (bool, error) {
	return true, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2010 AFP Authors
// This source code is released under the terms of the
// MIT license. Please see the file LICENSE for license details.

package distort

import (
	"afp"
	"afp/flags"
	"math"
)

type DistortFilter struct {
	ctx *afp.Context
    gain, clip Float32
    clipper func(*DistortFilter)
}

var clipTypes = map[string]func(*DistortFilter) {
    "hard" : hardCutoff,
	"soft" : nil,
	"overflow" : nil,
	"foldback" : nil,

}

func (self *DistortFilter) Init(ctx *afp.Context, args []string) os.Error {
	self.ctx = ctx

    fParse := flags.FlagParser(args)
	gain64 := fParse.Float64("g", 1.0,
		"Signal gain to apply before clipping. Must be greater than 0.")
    clipLevel := fParse.Float64("c", 1.0,
        "The amplitude at which to clip the signal. Must be between 0 and 1.")
    clipType := fParse.String("t",
		"soft", "The type of clipping used: hard, soft, overflow, or foldback.")

	fParse.Parse()

	if gain64 <= 0 {
		return os.NewError("Gain must be greater than 0.")
	}
	self.gain = float32(gain64)

	if clip64 > 1 || clip64 < 0{
		return os.NewError("Clipping level must be between 0 and 1")
	}
	self.clip = float32(clip64)
	self.clipper, ok := clipTypes[clipType]

	if !ok {
		return os.NewError("Clipping type must be one of: hard, soft, overflow, or foldback")
	}

	return nil
}

func (self *DistortFilter) Stop() os.Error {
	return nil
}

func (self *DistortFilter) GetType() int {
	return afp.PIPE_LINK
}

func (self *DistortFilter) Start() {
}

//Original C version by Alexander Kritov
//http://www.musicdsp.org/archive.php?classid=1#68
func _DSF(x, a, N, fi float32) {
    var (
        s1 = pow(a, N-1.0) * sin((N - 1.0) * x + fi)
        s2 = pow(a, N) * sin(N * x + fi)
        s3 = a * sin(x + fi)
        s4 = 1.0 - (2 * a * cos(x)) + (a * a)
    )

    if s4 == 0 {
        return 0;
    } else {
        return (sin(fi) - s3 - s2 +s1) / s4;
    }
}
<commit_msg>distort: implement hard clipping<commit_after>// Copyright (c) 2010 AFP Authors
// This source code is released under the terms of the
// MIT license. Please see the file LICENSE for license details.

package distort

import (
	"afp"
	"afp/flags"
	"math"
)

type DistortFilter struct {
	ctx *afp.Context
    gain, clip Float32
    clipper func(*DistortFilter)
}

var clipTypes = map[string]func(*DistortFilter) {
    "hard" : hard,
	"soft" : nil,
	"overflow" : nil,
	"foldback" : nil,

}

func (self *DistortFilter) Init(ctx *afp.Context, args []string) os.Error {
	self.ctx = ctx

    fParse := flags.FlagParser(args)
	fParse.Float32Var("g", 1.0,
		"Signal gain to apply before clipping. Must be greater than 0.")
    clipLevel := fParse.Float64("c", 1.0,
        "The amplitude at which to clip the signal. Must be between 0 and 1.")
    clipType := fParse.String("t",
		"soft", "The type of clipping used: hard, soft, overflow, or foldback.")

	fParse.Parse()

	if gain64 <= 0 {
		return os.NewError("Gain must be greater than 0.")
	}
	self.gain = float32(gain64)

	if clip64 > 1 || clip64 < 0{
		return os.NewError("Clipping level must be between 0 and 1")
	}
	self.clip = float32(clip64)
	self.clipper, ok := clipTypes[clipType]

	if !ok {
		return os.NewError("Clipping type must be one of: hard, soft, overflow, or foldback")
	}

	return nil
}

func (self *DistortFilter) Stop() os.Error {
	return nil
}

func (self *DistortFilter) GetType() int {
	return afp.PIPE_LINK
}

func (self *DistortFilter) Start() {
	self.ctx.HeaderSink <- (<-self.ctx.HeaderSource)
	self.clipper(self)
}

func hard(f *DistortFilter) {
	for frame := range f.ctx.Source {
		for slice := range frame {
			for ch, sample := range slice {
				frame[slice][ch] = hardMin(f.clip, sample * f.gain)
			}
		}
		self.ctx.Sink <- frame
	}
}

//Min function which knows about hard().
//specifically that clip will always be positive
func hardMin(clip, sprime float32) {
	var t float32

	if sprime < 0 {
		t = -sprime
	} else {
		t = sprime
	}

	if t > clip {
		return clip
	}

	return sprime
}

//Original C version by Alexander Kritov
//http://www.musicdsp.org/archive.php?classid=1#68
func soft(x, a, N, fi float32) {
    var (
        s1 = pow(a, N-1.0) * sin((N - 1.0) * x + fi)
        s2 = pow(a, N) * sin(N * x + fi)
        s3 = a * sin(x + fi)
        s4 = 1.0 - (2 * a * cos(x)) + (a * a)
    )

    if s4 == 0 {
        return 0;
    } else {
        return (sin(fi) - s3 - s2 +s1) / s4;
    }
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package azure_file

import (
	"fmt"
	"io/ioutil"
	"os"
	"runtime"

	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/kubernetes/pkg/cloudprovider"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
	"k8s.io/kubernetes/pkg/util/mount"
	kstrings "k8s.io/kubernetes/pkg/util/strings"
	"k8s.io/kubernetes/pkg/volume"
	volutil "k8s.io/kubernetes/pkg/volume/util"
)

// ProbeVolumePlugins is the primary endpoint for volume plugins
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&azureFilePlugin{nil}}
}

type azureFilePlugin struct {
	host volume.VolumeHost
}

var _ volume.VolumePlugin = &azureFilePlugin{}
var _ volume.PersistentVolumePlugin = &azureFilePlugin{}
var _ volume.ExpandableVolumePlugin = &azureFilePlugin{}

const (
	azureFilePluginName = "kubernetes.io/azure-file"
)

func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, kstrings.EscapeQualifiedNameForDisk(azureFilePluginName), volName)
}

func (plugin *azureFilePlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}

func (plugin *azureFilePlugin) GetPluginName() string {
	return azureFilePluginName
}

func (plugin *azureFilePlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	share, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return share, nil
}

func (plugin *azureFilePlugin) CanSupport(spec *volume.Spec) bool {
	//TODO: check if mount.cifs is there
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.AzureFile != nil) ||
		(spec.Volume != nil && spec.Volume.AzureFile != nil)
}

func (plugin *azureFilePlugin) RequiresRemount() bool {
	return false
}

func (plugin *azureFilePlugin) SupportsMountOption() bool {
	return true
}

func (plugin *azureFilePlugin) SupportsBulkVolumeVerification() bool {
	return false
}

func (plugin *azureFilePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
		v1.ReadOnlyMany,
		v1.ReadWriteMany,
	}
}

func (plugin *azureFilePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	return plugin.newMounterInternal(spec, pod, &azureSvc{}, plugin.host.GetMounter(plugin.GetPluginName()))
}

func (plugin *azureFilePlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, util azureUtil, mounter mount.Interface) (volume.Mounter, error) {
	share, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}
	secretName, secretNamespace, err := getSecretNameAndNamespace(spec, pod.Namespace)
	return &azureFileMounter{
		azureFile: &azureFile{
			volName:         spec.Name(),
			mounter:         mounter,
			pod:             pod,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(pod.UID, spec.Name(), plugin.host)),
		},
		util:            util,
		secretNamespace: secretNamespace,
		secretName:      secretName,
		shareName:       share,
		readOnly:        readOnly,
		mountOptions:    volutil.MountOptionFromSpec(spec),
	}, nil
}

func (plugin *azureFilePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName()))
}

func (plugin *azureFilePlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Unmounter, error) {
	return &azureFileUnmounter{&azureFile{
		volName:         volName,
		mounter:         mounter,
		pod:             &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: podUID}},
		plugin:          plugin,
		MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
	}}, nil
}

func (plugin *azureFilePlugin) RequiresFSResize() bool {
	return false
}

func (plugin *azureFilePlugin) ExpandVolumeDevice(
	spec *volume.Spec,
	newSize resource.Quantity,
	oldSize resource.Quantity) (resource.Quantity, error) {

	if spec.PersistentVolume != nil || spec.PersistentVolume.Spec.AzureFile == nil {
		return oldSize, fmt.Errorf("invalid PV spec")
	}
	shareName := spec.PersistentVolume.Spec.AzureFile.ShareName
	azure, err := getAzureCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return oldSize, err
	}

	secretName, secretNamespace, err := getSecretNameAndNamespace(spec, spec.PersistentVolume.Spec.ClaimRef.Namespace)
	if err != nil {
		return oldSize, err
	}

	accountName, accountKey, err := (&azureSvc{}).GetAzureCredentials(plugin.host, secretNamespace, secretName)
	if err != nil {
		return oldSize, err
	}

	if err := azure.ResizeFileShare(accountName, accountKey, shareName, int(volutil.RoundUpToGiB(newSize))); err != nil {
		return oldSize, err
	}

	return newSize, nil
}

func (plugin *azureFilePlugin) ConstructVolumeSpec(volName, mountPath string) (*volume.Spec, error) {
	azureVolume := &v1.Volume{
		Name: volName,
		VolumeSource: v1.VolumeSource{
			AzureFile: &v1.AzureFileVolumeSource{
				SecretName: volName,
				ShareName:  volName,
			},
		},
	}
	return volume.NewSpecFromVolume(azureVolume), nil
}

// azureFile volumes represent mount of an AzureFile share.
type azureFile struct {
	volName string
	podUID  types.UID
	pod     *v1.Pod
	mounter mount.Interface
	plugin  *azureFilePlugin
	volume.MetricsProvider
}

func (azureFileVolume *azureFile) GetPath() string {
	return getPath(azureFileVolume.pod.UID, azureFileVolume.volName, azureFileVolume.plugin.host)
}

type azureFileMounter struct {
	*azureFile
	util            azureUtil
	secretName      string
	secretNamespace string
	shareName       string
	readOnly        bool
	mountOptions    []string
}

var _ volume.Mounter = &azureFileMounter{}

func (b *azureFileMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:        b.readOnly,
		Managed:         !b.readOnly,
		SupportsSELinux: false,
	}
}

// Checks prior to mount operations to verify that the required components (binaries, etc.)
// to mount the volume are available on the underlying node.
// If not, it returns an error
func (b *azureFileMounter) CanMount() error {
	return nil
}

// SetUp attaches the disk and bind mounts to the volume path.
func (b *azureFileMounter) SetUp(fsGroup *int64) error {
	return b.SetUpAt(b.GetPath(), fsGroup)
}

func (b *azureFileMounter) SetUpAt(dir string, fsGroup *int64) error {
	notMnt, err := b.mounter.IsLikelyNotMountPoint(dir)
	glog.V(4).Infof("AzureFile mount set up: %s %v %v", dir, !notMnt, err)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if !notMnt {
		// testing original mount point, make sure the mount link is valid
		if _, err := ioutil.ReadDir(dir); err == nil {
			glog.V(4).Infof("azureFile - already mounted to target %s", dir)
			return nil
		}
		// mount link is invalid, now unmount and remount later
		glog.Warningf("azureFile - ReadDir %s failed with %v, unmount this directory", dir, err)
		if err := b.mounter.Unmount(dir); err != nil {
			glog.Errorf("azureFile - Unmount directory %s failed with %v", dir, err)
			return err
		}
		notMnt = true
	}

	var accountKey, accountName string
	if accountName, accountKey, err = b.util.GetAzureCredentials(b.plugin.host, b.secretNamespace, b.secretName); err != nil {
		return err
	}

	mountOptions := []string{}
	source := ""
	osSeparator := string(os.PathSeparator)
	source = fmt.Sprintf("%s%s%s.file.%s%s%s", osSeparator, osSeparator, accountName, getStorageEndpointSuffix(b.plugin.host.GetCloudProvider()), osSeparator, b.shareName)

	if runtime.GOOS == "windows" {
		mountOptions = []string{fmt.Sprintf("AZURE\\%s", accountName), accountKey}
	} else {
		os.MkdirAll(dir, 0700)
		// parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/
		options := []string{fmt.Sprintf("username=%s,password=%s", accountName, accountKey)}
		if b.readOnly {
			options = append(options, "ro")
		}
		mountOptions = volutil.JoinMountOptions(b.mountOptions, options)
		mountOptions = appendDefaultMountOptions(mountOptions, fsGroup)
	}

	err = b.mounter.Mount(source, dir, "cifs", mountOptions)
	if err != nil {
		notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
		if mntErr != nil {
			glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
			return err
		}
		if !notMnt {
			if mntErr = b.mounter.Unmount(dir); mntErr != nil {
				glog.Errorf("Failed to unmount: %v", mntErr)
				return err
			}
			notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
			if mntErr != nil {
				glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
				return err
			}
			if !notMnt {
				// This is very odd, we don't expect it.  We'll try again next sync loop.
				glog.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop.", dir)
				return err
			}
		}
		os.Remove(dir)
		return err
	}
	return nil
}

var _ volume.Unmounter = &azureFileUnmounter{}

type azureFileUnmounter struct {
	*azureFile
}

func (c *azureFileUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}

func (c *azureFileUnmounter) TearDownAt(dir string) error {
	return volutil.UnmountPath(dir, c.mounter)
}

func getVolumeSource(spec *volume.Spec) (string, bool, error) {
	if spec.Volume != nil && spec.Volume.AzureFile != nil {
		share := spec.Volume.AzureFile.ShareName
		readOnly := spec.Volume.AzureFile.ReadOnly
		return share, readOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.AzureFile != nil {
		share := spec.PersistentVolume.Spec.AzureFile.ShareName
		readOnly := spec.ReadOnly
		return share, readOnly, nil
	}
	return "", false, fmt.Errorf("Spec does not reference an AzureFile volume type")
}

func getSecretNameAndNamespace(spec *volume.Spec, defaultNamespace string) (string, string, error) {
	secretName := ""
	secretNamespace := ""
	if spec.Volume != nil && spec.Volume.AzureFile != nil {
		secretName = spec.Volume.AzureFile.SecretName
		secretNamespace = defaultNamespace

	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.AzureFile != nil {
		secretNamespace = defaultNamespace
		if spec.PersistentVolume.Spec.AzureFile.SecretNamespace != nil {
			secretNamespace = *spec.PersistentVolume.Spec.AzureFile.SecretNamespace
		}
		secretName = spec.PersistentVolume.Spec.AzureFile.SecretName
	} else {
		return "", "", fmt.Errorf("Spec does not reference an AzureFile volume type")
	}

	if len(secretNamespace) == 0 {
		return "", "", fmt.Errorf("invalid Azure volume: nil namespace")
	}
	return secretName, secretNamespace, nil

}

func getAzureCloud(cloudProvider cloudprovider.Interface) (*azure.Cloud, error) {
	azure, ok := cloudProvider.(*azure.Cloud)
	if !ok || azure == nil {
		return nil, fmt.Errorf("Failed to get Azure Cloud Provider. GetCloudProvider returned %v instead", cloudProvider)
	}

	return azure, nil
}

func getStorageEndpointSuffix(cloudprovider cloudprovider.Interface) string {
	const publicCloudStorageEndpointSuffix = "core.windows.net"
	azure, err := getAzureCloud(cloudprovider)
	if err != nil {
		glog.Warningf("No Azure cloud provider found. Using the Azure public cloud endpoint: %s", publicCloudStorageEndpointSuffix)
		return publicCloudStorageEndpointSuffix
	}
	return azure.Environment.StorageEndpointSuffix
}
<commit_msg>fix azure file size grow issue<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package azure_file

import (
	"fmt"
	"io/ioutil"
	"os"
	"runtime"

	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/kubernetes/pkg/cloudprovider"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
	"k8s.io/kubernetes/pkg/util/mount"
	kstrings "k8s.io/kubernetes/pkg/util/strings"
	"k8s.io/kubernetes/pkg/volume"
	volutil "k8s.io/kubernetes/pkg/volume/util"
)

// ProbeVolumePlugins is the primary endpoint for volume plugins
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&azureFilePlugin{nil}}
}

type azureFilePlugin struct {
	host volume.VolumeHost
}

var _ volume.VolumePlugin = &azureFilePlugin{}
var _ volume.PersistentVolumePlugin = &azureFilePlugin{}
var _ volume.ExpandableVolumePlugin = &azureFilePlugin{}

const (
	azureFilePluginName = "kubernetes.io/azure-file"
)

func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, kstrings.EscapeQualifiedNameForDisk(azureFilePluginName), volName)
}

func (plugin *azureFilePlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}

func (plugin *azureFilePlugin) GetPluginName() string {
	return azureFilePluginName
}

func (plugin *azureFilePlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	share, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return share, nil
}

func (plugin *azureFilePlugin) CanSupport(spec *volume.Spec) bool {
	//TODO: check if mount.cifs is there
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.AzureFile != nil) ||
		(spec.Volume != nil && spec.Volume.AzureFile != nil)
}

func (plugin *azureFilePlugin) RequiresRemount() bool {
	return false
}

func (plugin *azureFilePlugin) SupportsMountOption() bool {
	return true
}

func (plugin *azureFilePlugin) SupportsBulkVolumeVerification() bool {
	return false
}

func (plugin *azureFilePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
		v1.ReadOnlyMany,
		v1.ReadWriteMany,
	}
}

func (plugin *azureFilePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	return plugin.newMounterInternal(spec, pod, &azureSvc{}, plugin.host.GetMounter(plugin.GetPluginName()))
}

func (plugin *azureFilePlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, util azureUtil, mounter mount.Interface) (volume.Mounter, error) {
	share, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}
	secretName, secretNamespace, err := getSecretNameAndNamespace(spec, pod.Namespace)
	return &azureFileMounter{
		azureFile: &azureFile{
			volName:         spec.Name(),
			mounter:         mounter,
			pod:             pod,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(pod.UID, spec.Name(), plugin.host)),
		},
		util:            util,
		secretNamespace: secretNamespace,
		secretName:      secretName,
		shareName:       share,
		readOnly:        readOnly,
		mountOptions:    volutil.MountOptionFromSpec(spec),
	}, nil
}

func (plugin *azureFilePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName()))
}

func (plugin *azureFilePlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Unmounter, error) {
	return &azureFileUnmounter{&azureFile{
		volName:         volName,
		mounter:         mounter,
		pod:             &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: podUID}},
		plugin:          plugin,
		MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
	}}, nil
}

func (plugin *azureFilePlugin) RequiresFSResize() bool {
	return false
}

func (plugin *azureFilePlugin) ExpandVolumeDevice(
	spec *volume.Spec,
	newSize resource.Quantity,
	oldSize resource.Quantity) (resource.Quantity, error) {

	if spec.PersistentVolume == nil || spec.PersistentVolume.Spec.AzureFile == nil {
		return oldSize, fmt.Errorf("invalid PV spec")
	}
	shareName := spec.PersistentVolume.Spec.AzureFile.ShareName
	azure, err := getAzureCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return oldSize, err
	}

	secretName, secretNamespace, err := getSecretNameAndNamespace(spec, spec.PersistentVolume.Spec.ClaimRef.Namespace)
	if err != nil {
		return oldSize, err
	}

	accountName, accountKey, err := (&azureSvc{}).GetAzureCredentials(plugin.host, secretNamespace, secretName)
	if err != nil {
		return oldSize, err
	}

	if err := azure.ResizeFileShare(accountName, accountKey, shareName, int(volutil.RoundUpToGiB(newSize))); err != nil {
		return oldSize, err
	}

	return newSize, nil
}

func (plugin *azureFilePlugin) ConstructVolumeSpec(volName, mountPath string) (*volume.Spec, error) {
	azureVolume := &v1.Volume{
		Name: volName,
		VolumeSource: v1.VolumeSource{
			AzureFile: &v1.AzureFileVolumeSource{
				SecretName: volName,
				ShareName:  volName,
			},
		},
	}
	return volume.NewSpecFromVolume(azureVolume), nil
}

// azureFile volumes represent mount of an AzureFile share.
type azureFile struct {
	volName string
	podUID  types.UID
	pod     *v1.Pod
	mounter mount.Interface
	plugin  *azureFilePlugin
	volume.MetricsProvider
}

func (azureFileVolume *azureFile) GetPath() string {
	return getPath(azureFileVolume.pod.UID, azureFileVolume.volName, azureFileVolume.plugin.host)
}

type azureFileMounter struct {
	*azureFile
	util            azureUtil
	secretName      string
	secretNamespace string
	shareName       string
	readOnly        bool
	mountOptions    []string
}

var _ volume.Mounter = &azureFileMounter{}

func (b *azureFileMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:        b.readOnly,
		Managed:         !b.readOnly,
		SupportsSELinux: false,
	}
}

// Checks prior to mount operations to verify that the required components (binaries, etc.)
// to mount the volume are available on the underlying node.
// If not, it returns an error
func (b *azureFileMounter) CanMount() error {
	return nil
}

// SetUp attaches the disk and bind mounts to the volume path.
func (b *azureFileMounter) SetUp(fsGroup *int64) error {
	return b.SetUpAt(b.GetPath(), fsGroup)
}

func (b *azureFileMounter) SetUpAt(dir string, fsGroup *int64) error {
	notMnt, err := b.mounter.IsLikelyNotMountPoint(dir)
	glog.V(4).Infof("AzureFile mount set up: %s %v %v", dir, !notMnt, err)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if !notMnt {
		// testing original mount point, make sure the mount link is valid
		if _, err := ioutil.ReadDir(dir); err == nil {
			glog.V(4).Infof("azureFile - already mounted to target %s", dir)
			return nil
		}
		// mount link is invalid, now unmount and remount later
		glog.Warningf("azureFile - ReadDir %s failed with %v, unmount this directory", dir, err)
		if err := b.mounter.Unmount(dir); err != nil {
			glog.Errorf("azureFile - Unmount directory %s failed with %v", dir, err)
			return err
		}
		notMnt = true
	}

	var accountKey, accountName string
	if accountName, accountKey, err = b.util.GetAzureCredentials(b.plugin.host, b.secretNamespace, b.secretName); err != nil {
		return err
	}

	mountOptions := []string{}
	source := ""
	osSeparator := string(os.PathSeparator)
	source = fmt.Sprintf("%s%s%s.file.%s%s%s", osSeparator, osSeparator, accountName, getStorageEndpointSuffix(b.plugin.host.GetCloudProvider()), osSeparator, b.shareName)

	if runtime.GOOS == "windows" {
		mountOptions = []string{fmt.Sprintf("AZURE\\%s", accountName), accountKey}
	} else {
		os.MkdirAll(dir, 0700)
		// parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/
		options := []string{fmt.Sprintf("username=%s,password=%s", accountName, accountKey)}
		if b.readOnly {
			options = append(options, "ro")
		}
		mountOptions = volutil.JoinMountOptions(b.mountOptions, options)
		mountOptions = appendDefaultMountOptions(mountOptions, fsGroup)
	}

	err = b.mounter.Mount(source, dir, "cifs", mountOptions)
	if err != nil {
		notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
		if mntErr != nil {
			glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
			return err
		}
		if !notMnt {
			if mntErr = b.mounter.Unmount(dir); mntErr != nil {
				glog.Errorf("Failed to unmount: %v", mntErr)
				return err
			}
			notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
			if mntErr != nil {
				glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
				return err
			}
			if !notMnt {
				// This is very odd, we don't expect it.  We'll try again next sync loop.
				glog.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop.", dir)
				return err
			}
		}
		os.Remove(dir)
		return err
	}
	return nil
}

var _ volume.Unmounter = &azureFileUnmounter{}

type azureFileUnmounter struct {
	*azureFile
}

func (c *azureFileUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}

func (c *azureFileUnmounter) TearDownAt(dir string) error {
	return volutil.UnmountPath(dir, c.mounter)
}

func getVolumeSource(spec *volume.Spec) (string, bool, error) {
	if spec.Volume != nil && spec.Volume.AzureFile != nil {
		share := spec.Volume.AzureFile.ShareName
		readOnly := spec.Volume.AzureFile.ReadOnly
		return share, readOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.AzureFile != nil {
		share := spec.PersistentVolume.Spec.AzureFile.ShareName
		readOnly := spec.ReadOnly
		return share, readOnly, nil
	}
	return "", false, fmt.Errorf("Spec does not reference an AzureFile volume type")
}

func getSecretNameAndNamespace(spec *volume.Spec, defaultNamespace string) (string, string, error) {
	secretName := ""
	secretNamespace := ""
	if spec.Volume != nil && spec.Volume.AzureFile != nil {
		secretName = spec.Volume.AzureFile.SecretName
		secretNamespace = defaultNamespace

	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.AzureFile != nil {
		secretNamespace = defaultNamespace
		if spec.PersistentVolume.Spec.AzureFile.SecretNamespace != nil {
			secretNamespace = *spec.PersistentVolume.Spec.AzureFile.SecretNamespace
		}
		secretName = spec.PersistentVolume.Spec.AzureFile.SecretName
	} else {
		return "", "", fmt.Errorf("Spec does not reference an AzureFile volume type")
	}

	if len(secretNamespace) == 0 {
		return "", "", fmt.Errorf("invalid Azure volume: nil namespace")
	}
	return secretName, secretNamespace, nil

}

func getAzureCloud(cloudProvider cloudprovider.Interface) (*azure.Cloud, error) {
	azure, ok := cloudProvider.(*azure.Cloud)
	if !ok || azure == nil {
		return nil, fmt.Errorf("Failed to get Azure Cloud Provider. GetCloudProvider returned %v instead", cloudProvider)
	}

	return azure, nil
}

func getStorageEndpointSuffix(cloudprovider cloudprovider.Interface) string {
	const publicCloudStorageEndpointSuffix = "core.windows.net"
	azure, err := getAzureCloud(cloudprovider)
	if err != nil {
		glog.Warningf("No Azure cloud provider found. Using the Azure public cloud endpoint: %s", publicCloudStorageEndpointSuffix)
		return publicCloudStorageEndpointSuffix
	}
	return azure.Environment.StorageEndpointSuffix
}
<|endoftext|>


<commit_before>/*
 * Copyright 2012 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package server

import (
	"fmt"
	"github.com/uniqush/uniqush-conn/msgcache"
	"github.com/uniqush/uniqush-conn/proto"
	"github.com/uniqush/uniqush-conn/rpc"
	"io"
	"math/rand"
	"net"
	"sync"
	"sync/atomic"
	"time"
)

// SendMessage() and ForwardMessage() are goroutine-safe.
// SendMessage() and ForwardMessage() will send a message ditest,
// instead of the message itself, if the message is too large.
// ReceiveMessage() should nevery be called concurrently.
type Conn interface {
	RemoteAddr() net.Addr
	Service() string
	Username() string
	UniqId() string
	Close() error

	// If the message is generated from the server, then use SendMessage()
	// to send it to the client.
	SendMessage(msg *rpc.Message, id string, extra map[string]string) error

	// If the message is generated from another client, then
	// use ForwardMessage() to send it to the client.
	ForwardMessage(sender, senderService string, msg *rpc.Message, id string) error

	// ReceiveMessage() will keep receiving Commands from the client
	// until it receives a Command with type CMD_DATA.
	ReceiveMessage() (msg *rpc.Message, err error)

	SetMessageCache(cache msgcache.Cache)
	SetForwardRequestChannel(fwdChan chan<- *rpc.ForwardRequest)
	SetSubscribeRequestChan(subChan chan<- *rpc.SubscribeRequest)
	Visible() bool
}

type serverConn struct {
	cmdio             *proto.CommandIO
	conn              net.Conn
	compressThreshold int32
	digestThreshold   int32
	service           string
	username          string
	connId            string
	digestFielsLock   sync.Mutex
	digestFields      []string
	cmdProcs          []CommandProcessor
	visible           int32
}

type CommandProcessor interface {
	ProcessCommand(cmd *proto.Command) (msg *rpc.Message, err error)
}

func (self *serverConn) Visible() bool {
	v := atomic.LoadInt32(&self.visible)
	return v > 0
}

func (self *serverConn) RemoteAddr() net.Addr {
	return self.conn.RemoteAddr()
}

func (self *serverConn) Close() error {
	if self == nil {
		return nil
	}
	return self.conn.Close()
}

func (self *serverConn) Service() string {
	if self == nil {
		return ""
	}
	return self.service
}

func (self *serverConn) Username() string {
	if self == nil {
		return ""
	}
	return self.username
}

func (self *serverConn) UniqId() string {
	if self == nil {
		return ""
	}
	return self.connId
}

func (self *serverConn) shouldCompress(size int) bool {
	t := int(atomic.LoadInt32(&self.compressThreshold))
	if t > 0 && t < size {
		return true
	}
	return false
}

func (self *serverConn) shouldDigest(sz int) bool {
	d := atomic.LoadInt32(&self.digestThreshold)
	if d >= 0 && d < int32(sz) {
		return true
	}
	return false
}

func (self *serverConn) writeDigest(mc *rpc.MessageContainer, extra map[string]string, sz int) error {
	digest := &proto.Command{
		Type: proto.CMD_DIGEST,
	}
	params := [4]string{fmt.Sprintf("%v", sz), mc.Id}

	if mc.FromUser() {
		params[2] = mc.Sender
		params[3] = mc.SenderService
		digest.Params = params[:4]
	} else {
		digest.Params = params[:2]
	}

	msg := mc.Message
	header := make(map[string]string, len(extra)+len(msg.Header))
	self.digestFielsLock.Lock()
	defer self.digestFielsLock.Unlock()

	for _, f := range self.digestFields {
		if len(msg.Header) > 0 {
			if v, ok := msg.Header[f]; ok {
				header[f] = v
			}
		}
		if len(extra) > 0 {
			if v, ok := extra[f]; ok {
				header[f] = v
			}
		}
	}
	if len(header) > 0 {
		digest.Message = &rpc.Message{
			Header: header,
		}
	}

	compress := self.shouldCompress(digest.Message.Size())
	return self.cmdio.WriteCommand(digest, compress)
}

func (self *serverConn) SendMessage(msg *rpc.Message, id string, extra map[string]string) error {
	return self.send(msg, id, extra, true)
}

func (self *serverConn) send(msg *rpc.Message, id string, extra map[string]string, tryDigest bool) error {
	if msg == nil {
		cmd := &proto.Command{
			Type: proto.CMD_EMPTY,
		}
		if len(id) > 0 {
			cmd.Params = []string{id}
		}
		return self.cmdio.WriteCommand(cmd, false)
	}
	sz := msg.Size()
	if tryDigest && self.shouldDigest(sz) {
		container := &rpc.MessageContainer{
			Id:      id,
			Message: msg,
		}
		return self.writeDigest(container, extra, sz)
	}
	cmd := &proto.Command{
		Type:    proto.CMD_DATA,
		Message: msg,
	}
	cmd.Params = []string{id}
	return self.cmdio.WriteCommand(cmd, self.shouldCompress(sz))
}

func (self *serverConn) ForwardMessage(sender, senderService string, msg *rpc.Message, id string) error {
	return self.forward(sender, senderService, msg, id, true)
}

func (self *serverConn) forward(sender, senderService string, msg *rpc.Message, id string, tryDigest bool) error {
	sz := msg.Size()
	if sz == 0 {
		return nil
	}
	if tryDigest && self.shouldDigest(sz) {
		container := &rpc.MessageContainer{
			Id:            id,
			Sender:        sender,
			SenderService: senderService,
			Message:       msg,
		}
		return self.writeDigest(container, nil, sz)
	}
	cmd := &proto.Command{
		Type:    proto.CMD_FWD,
		Message: msg,
	}
	cmd.Params = []string{sender, senderService, id}
	return self.cmdio.WriteCommand(cmd, self.shouldCompress(sz))
}

func (self *serverConn) processCommand(cmd *proto.Command) (msg *rpc.Message, err error) {
	if cmd == nil {
		return
	}

	t := int(cmd.Type)
	if t > len(self.cmdProcs) {
		return
	}
	proc := self.cmdProcs[t]
	if proc != nil {
		msg, err = proc.ProcessCommand(cmd)
	}
	return
}

func (self *serverConn) ReceiveMessage() (msg *rpc.Message, err error) {
	var cmd *proto.Command
	for {
		cmd, err = self.cmdio.ReadCommand()
		if err != nil {
			if err == io.ErrUnexpectedEOF || err == io.EOF {
				err = io.EOF
			}
			return
		}
		switch cmd.Type {
		case proto.CMD_DATA:
			msg = cmd.Message
			return
		case proto.CMD_BYE:
			err = io.EOF
			return
		default:
			msg, err = self.processCommand(cmd)
			if err != nil || msg != nil {
				return
			}
		}
	}
}

func (self *serverConn) SetMessageCache(cache msgcache.Cache) {
	if cache == nil {
		return
	}
	proc := new(messageRetriever)
	proc.cache = cache
	proc.conn = self
	self.setCommandProcessor(proto.CMD_MSG_RETRIEVE, proc)

	p2 := new(retriaveAllMessages)
	p2.cache = cache
	p2.conn = self
	self.setCommandProcessor(proto.CMD_REQ_ALL_CACHED, p2)
}

func (self *serverConn) SetForwardRequestChannel(fwdChan chan<- *rpc.ForwardRequest) {
	if fwdChan == nil {
		return
	}
	proc := new(forwardProcessor)
	proc.conn = self
	proc.fwdChan = fwdChan
	self.setCommandProcessor(proto.CMD_FWD_REQ, proc)
}

func (self *serverConn) SetSubscribeRequestChan(subChan chan<- *rpc.SubscribeRequest) {
	if subChan == nil {
		return
	}
	proc := new(subscribeProcessor)
	proc.conn = self
	proc.subChan = subChan
	self.setCommandProcessor(proto.CMD_SUBSCRIPTION, proc)
}

func (self *serverConn) setCommandProcessor(cmdType uint8, proc CommandProcessor) {
	if cmdType >= proto.CMD_NR_CMDS {
		return
	}
	if len(self.cmdProcs) <= int(cmdType) {
		self.cmdProcs = make([]CommandProcessor, proto.CMD_NR_CMDS)
	}
	self.cmdProcs[cmdType] = proc
}

func NewConn(cmdio *proto.CommandIO, service, username string, conn net.Conn) Conn {
	ret := new(serverConn)
	ret.conn = conn
	ret.cmdio = cmdio
	ret.service = service
	ret.username = username
	ret.connId = fmt.Sprintf("%x-%x", time.Now().UnixNano(), rand.Int63())
	ret.digestThreshold = 1024
	ret.compressThreshold = 1024

	settingproc := new(settingProcessor)
	settingproc.conn = ret
	ret.setCommandProcessor(proto.CMD_SETTING, settingproc)

	visproc := new(visibilityProcessor)
	visproc.conn = ret
	ret.setCommandProcessor(proto.CMD_SET_VISIBILITY, visproc)

	ret.visible = 1
	return ret
}
<commit_msg>added redirect call on server's connection. fixing #9<commit_after>/*
 * Copyright 2012 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package server

import (
	"fmt"
	"github.com/uniqush/uniqush-conn/msgcache"
	"github.com/uniqush/uniqush-conn/proto"
	"github.com/uniqush/uniqush-conn/rpc"
	"io"
	"math/rand"
	"net"
	"sync"
	"sync/atomic"
	"time"
)

// SendMessage() and ForwardMessage() are goroutine-safe.
// SendMessage() and ForwardMessage() will send a message ditest,
// instead of the message itself, if the message is too large.
// ReceiveMessage() should nevery be called concurrently.
type Conn interface {
	RemoteAddr() net.Addr
	Service() string
	Username() string
	UniqId() string
	Close() error

	// If the message is generated from the server, then use SendMessage()
	// to send it to the client.
	SendMessage(msg *rpc.Message, id string, extra map[string]string) error

	// If the message is generated from another client, then
	// use ForwardMessage() to send it to the client.
	ForwardMessage(sender, senderService string, msg *rpc.Message, id string) error

	// ReceiveMessage() will keep receiving Commands from the client
	// until it receives a Command with type CMD_DATA.
	ReceiveMessage() (msg *rpc.Message, err error)

	// Ask the client to connect to other servers.
	// Redirect() will not close the connection. The user should call Close()
	// seprately to close the connection.
	Redirect(addrs ...string) error

	SetMessageCache(cache msgcache.Cache)
	SetForwardRequestChannel(fwdChan chan<- *rpc.ForwardRequest)
	SetSubscribeRequestChan(subChan chan<- *rpc.SubscribeRequest)
	Visible() bool
}

type serverConn struct {
	cmdio             *proto.CommandIO
	conn              net.Conn
	compressThreshold int32
	digestThreshold   int32
	service           string
	username          string
	connId            string
	digestFielsLock   sync.Mutex
	digestFields      []string
	cmdProcs          []CommandProcessor
	visible           int32
}

type CommandProcessor interface {
	ProcessCommand(cmd *proto.Command) (msg *rpc.Message, err error)
}

func (self *serverConn) Visible() bool {
	v := atomic.LoadInt32(&self.visible)
	return v > 0
}

func (self *serverConn) RemoteAddr() net.Addr {
	return self.conn.RemoteAddr()
}

func (self *serverConn) Close() error {
	if self == nil {
		return nil
	}
	return self.conn.Close()
}

func (self *serverConn) Service() string {
	if self == nil {
		return ""
	}
	return self.service
}

func (self *serverConn) Username() string {
	if self == nil {
		return ""
	}
	return self.username
}

func (self *serverConn) UniqId() string {
	if self == nil {
		return ""
	}
	return self.connId
}

func (self *serverConn) shouldCompress(size int) bool {
	t := int(atomic.LoadInt32(&self.compressThreshold))
	if t > 0 && t < size {
		return true
	}
	return false
}

func (self *serverConn) shouldDigest(sz int) bool {
	d := atomic.LoadInt32(&self.digestThreshold)
	if d >= 0 && d < int32(sz) {
		return true
	}
	return false
}

func (self *serverConn) writeDigest(mc *rpc.MessageContainer, extra map[string]string, sz int) error {
	digest := &proto.Command{
		Type: proto.CMD_DIGEST,
	}
	params := [4]string{fmt.Sprintf("%v", sz), mc.Id}

	if mc.FromUser() {
		params[2] = mc.Sender
		params[3] = mc.SenderService
		digest.Params = params[:4]
	} else {
		digest.Params = params[:2]
	}

	msg := mc.Message
	header := make(map[string]string, len(extra)+len(msg.Header))
	self.digestFielsLock.Lock()
	defer self.digestFielsLock.Unlock()

	for _, f := range self.digestFields {
		if len(msg.Header) > 0 {
			if v, ok := msg.Header[f]; ok {
				header[f] = v
			}
		}
		if len(extra) > 0 {
			if v, ok := extra[f]; ok {
				header[f] = v
			}
		}
	}
	if len(header) > 0 {
		digest.Message = &rpc.Message{
			Header: header,
		}
	}

	compress := self.shouldCompress(digest.Message.Size())
	return self.cmdio.WriteCommand(digest, compress)
}

func (self *serverConn) Redirect(addrs ...string) error {
	if len(addrs) == 0 {
		return nil
	}
	cmd := &proto.Command{
		Type:   proto.CMD_REDIRECT,
		Params: addrs,
	}
	return self.cmdio.WriteCommand(cmd, false)
}

func (self *serverConn) SendMessage(msg *rpc.Message, id string, extra map[string]string) error {
	return self.send(msg, id, extra, true)
}

func (self *serverConn) send(msg *rpc.Message, id string, extra map[string]string, tryDigest bool) error {
	if msg == nil {
		cmd := &proto.Command{
			Type: proto.CMD_EMPTY,
		}
		if len(id) > 0 {
			cmd.Params = []string{id}
		}
		return self.cmdio.WriteCommand(cmd, false)
	}
	sz := msg.Size()
	if tryDigest && self.shouldDigest(sz) {
		container := &rpc.MessageContainer{
			Id:      id,
			Message: msg,
		}
		return self.writeDigest(container, extra, sz)
	}
	cmd := &proto.Command{
		Type:    proto.CMD_DATA,
		Message: msg,
	}
	cmd.Params = []string{id}
	return self.cmdio.WriteCommand(cmd, self.shouldCompress(sz))
}

func (self *serverConn) ForwardMessage(sender, senderService string, msg *rpc.Message, id string) error {
	return self.forward(sender, senderService, msg, id, true)
}

func (self *serverConn) forward(sender, senderService string, msg *rpc.Message, id string, tryDigest bool) error {
	sz := msg.Size()
	if sz == 0 {
		return nil
	}
	if tryDigest && self.shouldDigest(sz) {
		container := &rpc.MessageContainer{
			Id:            id,
			Sender:        sender,
			SenderService: senderService,
			Message:       msg,
		}
		return self.writeDigest(container, nil, sz)
	}
	cmd := &proto.Command{
		Type:    proto.CMD_FWD,
		Message: msg,
	}
	cmd.Params = []string{sender, senderService, id}
	return self.cmdio.WriteCommand(cmd, self.shouldCompress(sz))
}

func (self *serverConn) processCommand(cmd *proto.Command) (msg *rpc.Message, err error) {
	if cmd == nil {
		return
	}

	t := int(cmd.Type)
	if t > len(self.cmdProcs) {
		return
	}
	proc := self.cmdProcs[t]
	if proc != nil {
		msg, err = proc.ProcessCommand(cmd)
	}
	return
}

func (self *serverConn) ReceiveMessage() (msg *rpc.Message, err error) {
	var cmd *proto.Command
	for {
		cmd, err = self.cmdio.ReadCommand()
		if err != nil {
			if err == io.ErrUnexpectedEOF || err == io.EOF {
				err = io.EOF
			}
			return
		}
		switch cmd.Type {
		case proto.CMD_DATA:
			msg = cmd.Message
			return
		case proto.CMD_BYE:
			err = io.EOF
			return
		default:
			msg, err = self.processCommand(cmd)
			if err != nil || msg != nil {
				return
			}
		}
	}
}

func (self *serverConn) SetMessageCache(cache msgcache.Cache) {
	if cache == nil {
		return
	}
	proc := new(messageRetriever)
	proc.cache = cache
	proc.conn = self
	self.setCommandProcessor(proto.CMD_MSG_RETRIEVE, proc)

	p2 := new(retriaveAllMessages)
	p2.cache = cache
	p2.conn = self
	self.setCommandProcessor(proto.CMD_REQ_ALL_CACHED, p2)
}

func (self *serverConn) SetForwardRequestChannel(fwdChan chan<- *rpc.ForwardRequest) {
	if fwdChan == nil {
		return
	}
	proc := new(forwardProcessor)
	proc.conn = self
	proc.fwdChan = fwdChan
	self.setCommandProcessor(proto.CMD_FWD_REQ, proc)
}

func (self *serverConn) SetSubscribeRequestChan(subChan chan<- *rpc.SubscribeRequest) {
	if subChan == nil {
		return
	}
	proc := new(subscribeProcessor)
	proc.conn = self
	proc.subChan = subChan
	self.setCommandProcessor(proto.CMD_SUBSCRIPTION, proc)
}

func (self *serverConn) setCommandProcessor(cmdType uint8, proc CommandProcessor) {
	if cmdType >= proto.CMD_NR_CMDS {
		return
	}
	if len(self.cmdProcs) <= int(cmdType) {
		self.cmdProcs = make([]CommandProcessor, proto.CMD_NR_CMDS)
	}
	self.cmdProcs[cmdType] = proc
}

func NewConn(cmdio *proto.CommandIO, service, username string, conn net.Conn) Conn {
	ret := new(serverConn)
	ret.conn = conn
	ret.cmdio = cmdio
	ret.service = service
	ret.username = username
	ret.connId = fmt.Sprintf("%x-%x", time.Now().UnixNano(), rand.Int63())
	ret.digestThreshold = 1024
	ret.compressThreshold = 1024

	settingproc := new(settingProcessor)
	settingproc.conn = ret
	ret.setCommandProcessor(proto.CMD_SETTING, settingproc)

	visproc := new(visibilityProcessor)
	visproc.conn = ret
	ret.setCommandProcessor(proto.CMD_SET_VISIBILITY, visproc)

	ret.visible = 1
	return ret
}
<|endoftext|>


<commit_before>package depth

import (
	"bufio"
	"io"
	"log"

	"github.com/biogo/store/interval"
	"github.com/brentp/xopen"
)

// Integer-specific intervals
type irange struct {
	Start, End int
	UID        uintptr
}

func (i irange) Overlap(b interval.IntRange) bool {
	// Half-open interval indexing.
	return i.End > b.Start && i.Start < b.End
}
func (i irange) ID() uintptr              { return i.UID }
func (i irange) Range() interval.IntRange { return interval.IntRange{i.Start, i.End} }

// Overlaps checks for overlaps without pulling intervals from the tree.
func Overlaps(tree *interval.IntTree, start, end int) bool {
	if tree == nil {
		return false
	}

	q := irange{Start: start, End: end, UID: uintptr(tree.Len())}

	overlaps := false
	tree.DoMatching(func(iv interval.IntInterface) bool {
		overlaps = true
		return true
	}, q)
	return overlaps

}

// ReadTree takes a bed file and returns map of trees.
func ReadTree(p string) map[string]*interval.IntTree {
	if p == "" {
		return nil
	}
	r, err := xopen.Ropen(p)
	if err != nil {
		panic(err)
	}
	tree := make(map[string]*interval.IntTree, 10)
	br := bufio.NewReader(r)
	k := 0

	for {
		line, err := br.ReadBytes('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			panic(err)
		}

		chrom, start, end := chromStartEndFromLine(line)
		if _, ok := tree[chrom]; !ok {
			tree[chrom] = &interval.IntTree{}
		}
		tree[chrom].Insert(irange{start, end, uintptr(k)}, false)
		k++

	}
	log.Printf("read %d intervals into interval tree", k)
	return tree
}
<commit_msg>no logging and allow reading from multiple files<commit_after>package depth

import (
	"bufio"
	"io"

	"github.com/biogo/store/interval"
	"github.com/brentp/xopen"
)

// Integer-specific intervals
type irange struct {
	Start, End int
	UID        uintptr
}

func (i irange) Overlap(b interval.IntRange) bool {
	// Half-open interval indexing.
	return i.End > b.Start && i.Start < b.End
}
func (i irange) ID() uintptr              { return i.UID }
func (i irange) Range() interval.IntRange { return interval.IntRange{i.Start, i.End} }

// Overlaps checks for overlaps without pulling intervals from the tree.
func Overlaps(tree *interval.IntTree, start, end int) bool {
	if tree == nil {
		return false
	}

	q := irange{Start: start, End: end, UID: uintptr(tree.Len())}

	overlaps := false
	tree.DoMatching(func(iv interval.IntInterface) bool {
		overlaps = true
		return true
	}, q)
	return overlaps

}

// ReadTree takes a bed file and returns map of trees.
func ReadTree(ps ...string) map[string]*interval.IntTree {
	tree := make(map[string]*interval.IntTree, 10)
	k := 0
	for _, p := range ps {
		if p == "" {
			continue
		}
		r, err := xopen.Ropen(p)
		if err != nil {
			panic(err)
		}
		defer r.Close()
		br := bufio.NewReader(r)

		for {
			line, err := br.ReadBytes('\n')
			if err == io.EOF {
				break
			}
			if err != nil {
				panic(err)
			}

			chrom, start, end := chromStartEndFromLine(line)
			if _, ok := tree[chrom]; !ok {
				tree[chrom] = &interval.IntTree{}
			}
			tree[chrom].Insert(irange{start, end, uintptr(k)}, false)
			k += 1
		}
	}
	return tree
}
<|endoftext|>


<commit_before>package tx

import (
	"fmt"
	"io"
	"reflect"

	"chain/crypto/sha3pool"
	"chain/encoding/blockchain"
	"chain/errors"
	"chain/protocol/bc"
)

type entry interface {
	Type() string
	Body() interface{}

	// When an entry is created from a bc.TxInput or a bc.TxOutput, this
	// reports the position of that antecedent object within its
	// transaction. Both inputs (spends and issuances) and outputs
	// (including retirements) are numbered beginning at zero. Entries
	// not originating in this way report -1.
	Ordinal() int
}

var errInvalidValue = errors.New("invalid value")

func entryID(e entry) (hash bc.Hash) {
	if e == nil {
		return hash
	}

	hasher := sha3pool.Get256()
	defer sha3pool.Put256(hasher)

	hasher.Write([]byte("entryid:"))
	hasher.Write([]byte(e.Type()))
	hasher.Write([]byte{':'})

	bh := sha3pool.Get256()
	defer sha3pool.Put256(bh)
	err := writeForHash(bh, e.Body())
	if err != nil {
		panic(err)
	}
	var innerHash bc.Hash
	bh.Read(innerHash[:])
	hasher.Write(innerHash[:])

	hasher.Read(hash[:])
	return hash
}

func writeForHash(w io.Writer, c interface{}) error {
	switch v := c.(type) {
	case byte:
		_, err := w.Write([]byte{v})
		return errors.Wrap(err, "writing byte for hash")
	case uint64:
		_, err := blockchain.WriteVarint63(w, v)
		return errors.Wrapf(err, "writing uint64 (%d) for hash", v)
	case []byte:
		_, err := blockchain.WriteVarstr31(w, v)
		return errors.Wrapf(err, "writing []byte (len %d) for hash", len(v))
	case string:
		_, err := blockchain.WriteVarstr31(w, []byte(v))
		return errors.Wrapf(err, "writing string (len %d) for hash", len(v))

		// TODO: The rest of these are all aliases for [32]byte. Do we
		// really need them all?

	case bc.Hash:
		_, err := w.Write(v[:])
		return errors.Wrap(err, "writing bc.Hash for hash")
	case bc.AssetID:
		_, err := w.Write(v[:])
		return errors.Wrap(err, "writing bc.AssetID for hash")
	}

	// The two container types in the spec (List and Struct)
	// correspond to slices and structs in Go. They can't be
	// handled with type assertions, so we must use reflect.
	switch v := reflect.ValueOf(c); v.Kind() {
	case reflect.Slice:
		l := v.Len()
		_, err := blockchain.WriteVarint31(w, uint64(l))
		if err != nil {
			return errors.Wrapf(err, "writing slice (len %d) for hash", l)
		}
		for i := 0; i < l; i++ {
			c := v.Index(i)
			if !c.CanInterface() {
				return errInvalidValue
			}
			err := writeForHash(w, c.Interface())
			if err != nil {
				return errors.Wrapf(err, "writing slice element %d for hash", i)
			}
		}
		return nil

	case reflect.Struct:
		for i := 0; i < v.NumField(); i++ {
			c := v.Field(i)
			if !c.CanInterface() {
				return errInvalidValue
			}
			err := writeForHash(w, c.Interface())
			if err != nil {
				t := v.Type()
				f := t.Field(i)
				return errors.Wrapf(err, "writing struct field %d (%s.%s) for hash", i, t.Name(), f.Name)
			}
		}
		return nil
	}

	return errors.Wrap(fmt.Errorf("bad type %T", c))
}
<commit_msg>protocol/tx: catch more nils in entryID<commit_after>package tx

import (
	"fmt"
	"io"
	"reflect"

	"chain/crypto/sha3pool"
	"chain/encoding/blockchain"
	"chain/errors"
	"chain/protocol/bc"
)

type entry interface {
	Type() string
	Body() interface{}

	// When an entry is created from a bc.TxInput or a bc.TxOutput, this
	// reports the position of that antecedent object within its
	// transaction. Both inputs (spends and issuances) and outputs
	// (including retirements) are numbered beginning at zero. Entries
	// not originating in this way report -1.
	Ordinal() int
}

var errInvalidValue = errors.New("invalid value")

func entryID(e entry) (hash bc.Hash) {
	if e == nil {
		return hash
	}

	// Nil pointer; not the same as nil interface above. (See
	// https://golang.org/doc/faq#nil_error.)
	if v := reflect.ValueOf(e); v.Kind() == reflect.Ptr && v.IsNil() {
		return hash
	}

	hasher := sha3pool.Get256()
	defer sha3pool.Put256(hasher)

	hasher.Write([]byte("entryid:"))
	hasher.Write([]byte(e.Type()))
	hasher.Write([]byte{':'})

	bh := sha3pool.Get256()
	defer sha3pool.Put256(bh)
	err := writeForHash(bh, e.Body())
	if err != nil {
		panic(err)
	}
	var innerHash bc.Hash
	bh.Read(innerHash[:])
	hasher.Write(innerHash[:])

	hasher.Read(hash[:])
	return hash
}

func writeForHash(w io.Writer, c interface{}) error {
	switch v := c.(type) {
	case byte:
		_, err := w.Write([]byte{v})
		return errors.Wrap(err, "writing byte for hash")
	case uint64:
		_, err := blockchain.WriteVarint63(w, v)
		return errors.Wrapf(err, "writing uint64 (%d) for hash", v)
	case []byte:
		_, err := blockchain.WriteVarstr31(w, v)
		return errors.Wrapf(err, "writing []byte (len %d) for hash", len(v))
	case string:
		_, err := blockchain.WriteVarstr31(w, []byte(v))
		return errors.Wrapf(err, "writing string (len %d) for hash", len(v))

		// TODO: The rest of these are all aliases for [32]byte. Do we
		// really need them all?

	case bc.Hash:
		_, err := w.Write(v[:])
		return errors.Wrap(err, "writing bc.Hash for hash")
	case bc.AssetID:
		_, err := w.Write(v[:])
		return errors.Wrap(err, "writing bc.AssetID for hash")
	}

	// The two container types in the spec (List and Struct)
	// correspond to slices and structs in Go. They can't be
	// handled with type assertions, so we must use reflect.
	switch v := reflect.ValueOf(c); v.Kind() {
	case reflect.Slice:
		l := v.Len()
		_, err := blockchain.WriteVarint31(w, uint64(l))
		if err != nil {
			return errors.Wrapf(err, "writing slice (len %d) for hash", l)
		}
		for i := 0; i < l; i++ {
			c := v.Index(i)
			if !c.CanInterface() {
				return errInvalidValue
			}
			err := writeForHash(w, c.Interface())
			if err != nil {
				return errors.Wrapf(err, "writing slice element %d for hash", i)
			}
		}
		return nil

	case reflect.Struct:
		for i := 0; i < v.NumField(); i++ {
			c := v.Field(i)
			if !c.CanInterface() {
				return errInvalidValue
			}
			err := writeForHash(w, c.Interface())
			if err != nil {
				t := v.Type()
				f := t.Field(i)
				return errors.Wrapf(err, "writing struct field %d (%s.%s) for hash", i, t.Name(), f.Name)
			}
		}
		return nil
	}

	return errors.Wrap(fmt.Errorf("bad type %T", c))
}
<|endoftext|>