<commit_before>package logrus_sentry

import (
	"encoding/json"
	"fmt"
	"runtime"
	"sync"
	"time"

	"github.com/getsentry/raven-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

var (
	severityMap = map[logrus.Level]raven.Severity{
		logrus.DebugLevel: raven.DEBUG,
		logrus.InfoLevel:  raven.INFO,
		logrus.WarnLevel:  raven.WARNING,
		logrus.ErrorLevel: raven.ERROR,
		logrus.FatalLevel: raven.FATAL,
		logrus.PanicLevel: raven.FATAL,
	}
)

// SentryHook delivers logs to a sentry server.
type SentryHook struct {
	// Timeout sets the time to wait for a delivery error from the sentry server.
	// If this is set to zero the server will not wait for any response and will
	// consider the message correctly sent.
	//
	// This is ignored for asynchronous hooks. If you want to set a timeout when
	// using an async hook (to bound the length of time that hook.Flush can take),
	// you probably want to create your own raven.Client and set
	// ravenClient.Transport.(*raven.HTTPTransport).Client.Timeout to set a
	// timeout on the underlying HTTP request instead.
	Timeout                 time.Duration
	StacktraceConfiguration StackTraceConfiguration

	client *raven.Client
	levels []logrus.Level

	serverName   string
	ignoreFields map[string]struct{}
	extraFilters map[string]func(interface{}) interface{}

	asynchronous bool

	mu sync.RWMutex
	wg sync.WaitGroup
}

// The Stacktracer interface allows an error type to return a raven.Stacktrace.
type Stacktracer interface {
	GetStacktrace() *raven.Stacktrace
}

type causer interface {
	Cause() error
}

type pkgErrorStackTracer interface {
	StackTrace() errors.StackTrace
}

// StackTraceConfiguration allows for configuring stacktraces
type StackTraceConfiguration struct {
	// whether stacktraces should be enabled
	Enable bool
	// the level at which to start capturing stacktraces
	Level logrus.Level
	// how many stack frames to skip before stacktrace starts recording
	Skip int
	// the number of lines to include around a stack frame for context
	Context int
	// the prefixes that will be matched against the stack frame.
	// if the stack frame's package matches one of these prefixes
	// sentry will identify the stack frame as "in_app"
	InAppPrefixes []string
	// whether sending exception type should be enabled.
	SendExceptionType bool
	// whether the exception type and message should be switched.
	SwitchExceptionTypeAndMessage bool
}

// NewSentryHook creates a hook to be added to an instance of logger
// and initializes the raven client.
// This method sets the timeout to 100 milliseconds.
func NewSentryHook(DSN string, levels []logrus.Level) (*SentryHook, error) {
	client, err := raven.New(DSN)
	if err != nil {
		return nil, err
	}
	return NewWithClientSentryHook(client, levels)
}

// NewWithTagsSentryHook creates a hook with tags to be added to an instance
// of logger and initializes the raven client. This method sets the timeout to
// 100 milliseconds.
func NewWithTagsSentryHook(DSN string, tags map[string]string, levels []logrus.Level) (*SentryHook, error) {
	client, err := raven.NewWithTags(DSN, tags)
	if err != nil {
		return nil, err
	}
	return NewWithClientSentryHook(client, levels)
}

// NewWithClientSentryHook creates a hook using an initialized raven client.
// This method sets the timeout to 100 milliseconds.
func NewWithClientSentryHook(client *raven.Client, levels []logrus.Level) (*SentryHook, error) {
	return &SentryHook{
		Timeout: 100 * time.Millisecond,
		StacktraceConfiguration: StackTraceConfiguration{
			Enable:            false,
			Level:             logrus.ErrorLevel,
			Skip:              5,
			Context:           0,
			InAppPrefixes:     nil,
			SendExceptionType: true,
		},
		client:       client,
		levels:       levels,
		ignoreFields: make(map[string]struct{}),
		extraFilters: make(map[string]func(interface{}) interface{}),
	}, nil
}

// NewAsyncSentryHook creates a hook same as NewSentryHook, but in asynchronous
// mode.
func NewAsyncSentryHook(DSN string, levels []logrus.Level) (*SentryHook, error) {
	hook, err := NewSentryHook(DSN, levels)
	return setAsync(hook), err
}

// NewAsyncWithTagsSentryHook creates a hook same as NewWithTagsSentryHook, but
// in asynchronous mode.
func NewAsyncWithTagsSentryHook(DSN string, tags map[string]string, levels []logrus.Level) (*SentryHook, error) {
	hook, err := NewWithTagsSentryHook(DSN, tags, levels)
	return setAsync(hook), err
}

// NewAsyncWithClientSentryHook creates a hook same as NewWithClientSentryHook,
// but in asynchronous mode.
func NewAsyncWithClientSentryHook(client *raven.Client, levels []logrus.Level) (*SentryHook, error) {
	hook, err := NewWithClientSentryHook(client, levels)
	return setAsync(hook), err
}

func setAsync(hook *SentryHook) *SentryHook {
	if hook == nil {
		return nil
	}
	hook.asynchronous = true
	return hook
}

// Fire is called when an event should be sent to sentry
// Special fields that sentry uses to give more information to the server
// are extracted from entry.Data (if they are found)
// These fields are: error, logger, server_name, http_request, tags
func (hook *SentryHook) Fire(entry *logrus.Entry) error {
	hook.mu.RLock() // Allow multiple go routines to log simultaneously
	defer hook.mu.RUnlock()
	packet := raven.NewPacket(entry.Message)
	packet.Timestamp = raven.Timestamp(entry.Time)
	packet.Level = severityMap[entry.Level]
	packet.Platform = "go"

	df := newDataField(entry.Data)

	// set special fields
	if hook.serverName != "" {
		packet.ServerName = hook.serverName
	}
	if logger, ok := df.getLogger(); ok {
		packet.Logger = logger
	}
	if serverName, ok := df.getServerName(); ok {
		packet.ServerName = serverName
	}
	if eventID, ok := df.getEventID(); ok {
		packet.EventID = eventID
	}
	if tags, ok := df.getTags(); ok {
		packet.Tags = tags
	}
	if fingerprint, ok := df.getFingerprint(); ok {
		packet.Fingerprint = fingerprint
	}
	if req, ok := df.getHTTPRequest(); ok {
		packet.Interfaces = append(packet.Interfaces, req)
	}
	if user, ok := df.getUser(); ok {
		packet.Interfaces = append(packet.Interfaces, user)
	}

	// set stacktrace data
	stConfig := &hook.StacktraceConfiguration
	if stConfig.Enable && entry.Level <= stConfig.Level {
		if err, ok := df.getError(); ok {
			var currentStacktrace *raven.Stacktrace
			currentStacktrace = hook.findStacktrace(err)
			if currentStacktrace == nil {
				currentStacktrace = raven.NewStacktrace(stConfig.Skip, stConfig.Context, stConfig.InAppPrefixes)
			}
			cause := errors.Cause(err)
			if cause == nil {
				cause = err
			}
			exc := raven.NewException(cause, currentStacktrace)
			if !stConfig.SendExceptionType {
				exc.Type = ""
			}
			if stConfig.SwitchExceptionTypeAndMessage {
				packet.Interfaces = append(packet.Interfaces, currentStacktrace)
				packet.Culprit = exc.Type + ": " + currentStacktrace.Culprit()
			} else {
				packet.Interfaces = append(packet.Interfaces, exc)
				packet.Culprit = err.Error()
			}
		} else {
			currentStacktrace := raven.NewStacktrace(stConfig.Skip, stConfig.Context, stConfig.InAppPrefixes)
			if currentStacktrace != nil {
				packet.Interfaces = append(packet.Interfaces, currentStacktrace)
			}
		}
	} else {
		// set the culprit even when the stack trace is disabled, as long as we have an error
		if err, ok := df.getError(); ok {
			packet.Culprit = err.Error()
		}
	}

	// set other fields
	dataExtra := hook.formatExtraData(df)
	if packet.Extra == nil {
		packet.Extra = dataExtra
	} else {
		for k, v := range dataExtra {
			packet.Extra[k] = v
		}
	}

	_, errCh := hook.client.Capture(packet, nil)

	if hook.asynchronous {
		// Our use of hook.mu guarantees that we are following the WaitGroup rule of
		// not calling Add in parallel with Wait.
		hook.wg.Add(1)
		go func() {
			if err := <-errCh; err != nil {
				fmt.Println(err)
			}
			hook.wg.Done()
		}()
		return nil
	} else if timeout := hook.Timeout; timeout == 0 {
		return nil
	} else {
		timeoutCh := time.After(timeout)
		select {
		case err := <-errCh:
			return err
		case <-timeoutCh:
			return fmt.Errorf("no response from sentry server in %s", timeout)
		}
	}
}

// Flush waits for the log queue to empty. This function only does anything in
// asynchronous mode.
func (hook *SentryHook) Flush() {
	if !hook.asynchronous {
		return
	}
	hook.mu.Lock() // Claim exclusive access; any logging goroutines will block until the flush completes
	defer hook.mu.Unlock()

	hook.wg.Wait()
}

func (hook *SentryHook) findStacktrace(err error) *raven.Stacktrace {
	var stacktrace *raven.Stacktrace
	var stackErr errors.StackTrace
	for err != nil {
		// Find the earliest *raven.Stacktrace, or error.StackTrace
		if tracer, ok := err.(Stacktracer); ok {
			stacktrace = tracer.GetStacktrace()
			stackErr = nil
		} else if tracer, ok := err.(pkgErrorStackTracer); ok {
			stacktrace = nil
			stackErr = tracer.StackTrace()
		}
		if cause, ok := err.(causer); ok {
			err = cause.Cause()
		} else {
			break
		}
	}
	if stackErr != nil {
		stacktrace = hook.convertStackTrace(stackErr)
	}
	return stacktrace
}

// convertStackTrace converts an errors.StackTrace into a natively consumable
// *raven.Stacktrace
func (hook *SentryHook) convertStackTrace(st errors.StackTrace) *raven.Stacktrace {
	stConfig := &hook.StacktraceConfiguration
	stFrames := []errors.Frame(st)
	frames := make([]*raven.StacktraceFrame, 0, len(stFrames))
	for i := range stFrames {
		pc := uintptr(stFrames[i])
		fn := runtime.FuncForPC(pc)
		file, line := fn.FileLine(pc)
		frame := raven.NewStacktraceFrame(pc, file, line, stConfig.Context, stConfig.InAppPrefixes)
		if frame != nil {
			frames = append(frames, frame)
		}
	}

	// Sentry wants the frames with the oldest first, so reverse them
	for i, j := 0, len(frames)-1; i < j; i, j = i+1, j-1 {
		frames[i], frames[j] = frames[j], frames[i]
	}
	return &raven.Stacktrace{Frames: frames}
}

// Levels returns the available logging levels.
func (hook *SentryHook) Levels() []logrus.Level {
	return hook.levels
}

// SetRelease sets release tag.
func (hook *SentryHook) SetRelease(release string) {
	hook.client.SetRelease(release)
}

// SetEnvironment sets environment tag.
func (hook *SentryHook) SetEnvironment(environment string) {
	hook.client.SetEnvironment(environment)
}

// SetServerName sets server_name tag.
func (hook *SentryHook) SetServerName(serverName string) {
	hook.serverName = serverName
}

// AddIgnore adds field name to ignore.
func (hook *SentryHook) AddIgnore(name string) {
	hook.ignoreFields[name] = struct{}{}
}

// AddExtraFilter adds a custom filter function.
func (hook *SentryHook) AddExtraFilter(name string, fn func(interface{}) interface{}) {
	hook.extraFilters[name] = fn
}

func (hook *SentryHook) formatExtraData(df *dataField) (result map[string]interface{}) {
	// create a map for passing to Sentry's extra data
	result = make(map[string]interface{}, df.len())
	for k, v := range df.data {
		if df.isOmit(k) {
			continue // skip already used special fields
		}
		if _, ok := hook.ignoreFields[k]; ok {
			continue
		}

		if fn, ok := hook.extraFilters[k]; ok {
			v = fn(v) // apply custom filter
		} else {
			v = formatData(v) // use default formatter
		}
		result[k] = v
	}
	return result
}

// formatData returns value as a suitable format.
func formatData(value interface{}) (formatted interface{}) {
	switch value := value.(type) {
	case json.Marshaler:
		return value
	case error:
		return value.Error()
	case fmt.Stringer:
		return value.String()
	default:
		return value
	}
}
<commit_msg>Add function name to NewStackTraceFrame call for updating to raven-go v0.1.1 (#66)<commit_after>package logrus_sentry

import (
	"encoding/json"
	"fmt"
	"runtime"
	"sync"
	"time"

	"github.com/getsentry/raven-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

var (
	severityMap = map[logrus.Level]raven.Severity{
		logrus.DebugLevel: raven.DEBUG,
		logrus.InfoLevel:  raven.INFO,
		logrus.WarnLevel:  raven.WARNING,
		logrus.ErrorLevel: raven.ERROR,
		logrus.FatalLevel: raven.FATAL,
		logrus.PanicLevel: raven.FATAL,
	}
)

// SentryHook delivers logs to a sentry server.
type SentryHook struct {
	// Timeout sets the time to wait for a delivery error from the sentry server.
	// If this is set to zero the server will not wait for any response and will
	// consider the message correctly sent.
	//
	// This is ignored for asynchronous hooks. If you want to set a timeout when
	// using an async hook (to bound the length of time that hook.Flush can take),
	// you probably want to create your own raven.Client and set
	// ravenClient.Transport.(*raven.HTTPTransport).Client.Timeout to set a
	// timeout on the underlying HTTP request instead.
	Timeout                 time.Duration
	StacktraceConfiguration StackTraceConfiguration

	client *raven.Client
	levels []logrus.Level

	serverName   string
	ignoreFields map[string]struct{}
	extraFilters map[string]func(interface{}) interface{}

	asynchronous bool

	mu sync.RWMutex
	wg sync.WaitGroup
}

// The Stacktracer interface allows an error type to return a raven.Stacktrace.
type Stacktracer interface {
	GetStacktrace() *raven.Stacktrace
}

type causer interface {
	Cause() error
}

type pkgErrorStackTracer interface {
	StackTrace() errors.StackTrace
}

// StackTraceConfiguration allows for configuring stacktraces
type StackTraceConfiguration struct {
	// whether stacktraces should be enabled
	Enable bool
	// the level at which to start capturing stacktraces
	Level logrus.Level
	// how many stack frames to skip before stacktrace starts recording
	Skip int
	// the number of lines to include around a stack frame for context
	Context int
	// the prefixes that will be matched against the stack frame.
	// if the stack frame's package matches one of these prefixes
	// sentry will identify the stack frame as "in_app"
	InAppPrefixes []string
	// whether sending exception type should be enabled.
	SendExceptionType bool
	// whether the exception type and message should be switched.
	SwitchExceptionTypeAndMessage bool
}

// NewSentryHook creates a hook to be added to an instance of logger
// and initializes the raven client.
// This method sets the timeout to 100 milliseconds.
func NewSentryHook(DSN string, levels []logrus.Level) (*SentryHook, error) {
	client, err := raven.New(DSN)
	if err != nil {
		return nil, err
	}
	return NewWithClientSentryHook(client, levels)
}

// NewWithTagsSentryHook creates a hook with tags to be added to an instance
// of logger and initializes the raven client. This method sets the timeout to
// 100 milliseconds.
func NewWithTagsSentryHook(DSN string, tags map[string]string, levels []logrus.Level) (*SentryHook, error) {
	client, err := raven.NewWithTags(DSN, tags)
	if err != nil {
		return nil, err
	}
	return NewWithClientSentryHook(client, levels)
}

// NewWithClientSentryHook creates a hook using an initialized raven client.
// This method sets the timeout to 100 milliseconds.
func NewWithClientSentryHook(client *raven.Client, levels []logrus.Level) (*SentryHook, error) {
	return &SentryHook{
		Timeout: 100 * time.Millisecond,
		StacktraceConfiguration: StackTraceConfiguration{
			Enable:            false,
			Level:             logrus.ErrorLevel,
			Skip:              5,
			Context:           0,
			InAppPrefixes:     nil,
			SendExceptionType: true,
		},
		client:       client,
		levels:       levels,
		ignoreFields: make(map[string]struct{}),
		extraFilters: make(map[string]func(interface{}) interface{}),
	}, nil
}

// NewAsyncSentryHook creates a hook same as NewSentryHook, but in asynchronous
// mode.
func NewAsyncSentryHook(DSN string, levels []logrus.Level) (*SentryHook, error) {
	hook, err := NewSentryHook(DSN, levels)
	return setAsync(hook), err
}

// NewAsyncWithTagsSentryHook creates a hook same as NewWithTagsSentryHook, but
// in asynchronous mode.
func NewAsyncWithTagsSentryHook(DSN string, tags map[string]string, levels []logrus.Level) (*SentryHook, error) {
	hook, err := NewWithTagsSentryHook(DSN, tags, levels)
	return setAsync(hook), err
}

// NewAsyncWithClientSentryHook creates a hook same as NewWithClientSentryHook,
// but in asynchronous mode.
func NewAsyncWithClientSentryHook(client *raven.Client, levels []logrus.Level) (*SentryHook, error) {
	hook, err := NewWithClientSentryHook(client, levels)
	return setAsync(hook), err
}

func setAsync(hook *SentryHook) *SentryHook {
	if hook == nil {
		return nil
	}
	hook.asynchronous = true
	return hook
}

// Fire is called when an event should be sent to sentry
// Special fields that sentry uses to give more information to the server
// are extracted from entry.Data (if they are found)
// These fields are: error, logger, server_name, http_request, tags
func (hook *SentryHook) Fire(entry *logrus.Entry) error {
	hook.mu.RLock() // Allow multiple go routines to log simultaneously
	defer hook.mu.RUnlock()
	packet := raven.NewPacket(entry.Message)
	packet.Timestamp = raven.Timestamp(entry.Time)
	packet.Level = severityMap[entry.Level]
	packet.Platform = "go"

	df := newDataField(entry.Data)

	// set special fields
	if hook.serverName != "" {
		packet.ServerName = hook.serverName
	}
	if logger, ok := df.getLogger(); ok {
		packet.Logger = logger
	}
	if serverName, ok := df.getServerName(); ok {
		packet.ServerName = serverName
	}
	if eventID, ok := df.getEventID(); ok {
		packet.EventID = eventID
	}
	if tags, ok := df.getTags(); ok {
		packet.Tags = tags
	}
	if fingerprint, ok := df.getFingerprint(); ok {
		packet.Fingerprint = fingerprint
	}
	if req, ok := df.getHTTPRequest(); ok {
		packet.Interfaces = append(packet.Interfaces, req)
	}
	if user, ok := df.getUser(); ok {
		packet.Interfaces = append(packet.Interfaces, user)
	}

	// set stacktrace data
	stConfig := &hook.StacktraceConfiguration
	if stConfig.Enable && entry.Level <= stConfig.Level {
		if err, ok := df.getError(); ok {
			var currentStacktrace *raven.Stacktrace
			currentStacktrace = hook.findStacktrace(err)
			if currentStacktrace == nil {
				currentStacktrace = raven.NewStacktrace(stConfig.Skip, stConfig.Context, stConfig.InAppPrefixes)
			}
			cause := errors.Cause(err)
			if cause == nil {
				cause = err
			}
			exc := raven.NewException(cause, currentStacktrace)
			if !stConfig.SendExceptionType {
				exc.Type = ""
			}
			if stConfig.SwitchExceptionTypeAndMessage {
				packet.Interfaces = append(packet.Interfaces, currentStacktrace)
				packet.Culprit = exc.Type + ": " + currentStacktrace.Culprit()
			} else {
				packet.Interfaces = append(packet.Interfaces, exc)
				packet.Culprit = err.Error()
			}
		} else {
			currentStacktrace := raven.NewStacktrace(stConfig.Skip, stConfig.Context, stConfig.InAppPrefixes)
			if currentStacktrace != nil {
				packet.Interfaces = append(packet.Interfaces, currentStacktrace)
			}
		}
	} else {
		// set the culprit even when the stack trace is disabled, as long as we have an error
		if err, ok := df.getError(); ok {
			packet.Culprit = err.Error()
		}
	}

	// set other fields
	dataExtra := hook.formatExtraData(df)
	if packet.Extra == nil {
		packet.Extra = dataExtra
	} else {
		for k, v := range dataExtra {
			packet.Extra[k] = v
		}
	}

	_, errCh := hook.client.Capture(packet, nil)

	if hook.asynchronous {
		// Our use of hook.mu guarantees that we are following the WaitGroup rule of
		// not calling Add in parallel with Wait.
		hook.wg.Add(1)
		go func() {
			if err := <-errCh; err != nil {
				fmt.Println(err)
			}
			hook.wg.Done()
		}()
		return nil
	} else if timeout := hook.Timeout; timeout == 0 {
		return nil
	} else {
		timeoutCh := time.After(timeout)
		select {
		case err := <-errCh:
			return err
		case <-timeoutCh:
			return fmt.Errorf("no response from sentry server in %s", timeout)
		}
	}
}

// Flush waits for the log queue to empty. This function only does anything in
// asynchronous mode.
func (hook *SentryHook) Flush() {
	if !hook.asynchronous {
		return
	}
	hook.mu.Lock() // Claim exclusive access; any logging goroutines will block until the flush completes
	defer hook.mu.Unlock()

	hook.wg.Wait()
}

func (hook *SentryHook) findStacktrace(err error) *raven.Stacktrace {
	var stacktrace *raven.Stacktrace
	var stackErr errors.StackTrace
	for err != nil {
		// Find the earliest *raven.Stacktrace, or error.StackTrace
		if tracer, ok := err.(Stacktracer); ok {
			stacktrace = tracer.GetStacktrace()
			stackErr = nil
		} else if tracer, ok := err.(pkgErrorStackTracer); ok {
			stacktrace = nil
			stackErr = tracer.StackTrace()
		}
		if cause, ok := err.(causer); ok {
			err = cause.Cause()
		} else {
			break
		}
	}
	if stackErr != nil {
		stacktrace = hook.convertStackTrace(stackErr)
	}
	return stacktrace
}

// convertStackTrace converts an errors.StackTrace into a natively consumable
// *raven.Stacktrace
func (hook *SentryHook) convertStackTrace(st errors.StackTrace) *raven.Stacktrace {
	stConfig := &hook.StacktraceConfiguration
	stFrames := []errors.Frame(st)
	frames := make([]*raven.StacktraceFrame, 0, len(stFrames))
	for i := range stFrames {
		pc := uintptr(stFrames[i])
		fn := runtime.FuncForPC(pc)
		file, line := fn.FileLine(pc)
		frame := raven.NewStacktraceFrame(pc, fn.Name(), file, line, stConfig.Context, stConfig.InAppPrefixes)
		if frame != nil {
			frames = append(frames, frame)
		}
	}

	// Sentry wants the frames with the oldest first, so reverse them
	for i, j := 0, len(frames)-1; i < j; i, j = i+1, j-1 {
		frames[i], frames[j] = frames[j], frames[i]
	}
	return &raven.Stacktrace{Frames: frames}
}

// Levels returns the available logging levels.
func (hook *SentryHook) Levels() []logrus.Level {
	return hook.levels
}

// SetRelease sets release tag.
func (hook *SentryHook) SetRelease(release string) {
	hook.client.SetRelease(release)
}

// SetEnvironment sets environment tag.
func (hook *SentryHook) SetEnvironment(environment string) {
	hook.client.SetEnvironment(environment)
}

// SetServerName sets server_name tag.
func (hook *SentryHook) SetServerName(serverName string) {
	hook.serverName = serverName
}

// AddIgnore adds field name to ignore.
func (hook *SentryHook) AddIgnore(name string) {
	hook.ignoreFields[name] = struct{}{}
}

// AddExtraFilter adds a custom filter function.
func (hook *SentryHook) AddExtraFilter(name string, fn func(interface{}) interface{}) {
	hook.extraFilters[name] = fn
}

func (hook *SentryHook) formatExtraData(df *dataField) (result map[string]interface{}) {
	// create a map for passing to Sentry's extra data
	result = make(map[string]interface{}, df.len())
	for k, v := range df.data {
		if df.isOmit(k) {
			continue // skip already used special fields
		}
		if _, ok := hook.ignoreFields[k]; ok {
			continue
		}

		if fn, ok := hook.extraFilters[k]; ok {
			v = fn(v) // apply custom filter
		} else {
			v = formatData(v) // use default formatter
		}
		result[k] = v
	}
	return result
}

// formatData returns value as a suitable format.
func formatData(value interface{}) (formatted interface{}) {
	switch value := value.(type) {
	case json.Marshaler:
		return value
	case error:
		return value.Error()
	case fmt.Stringer:
		return value.String()
	default:
		return value
	}
}
<|endoftext|>


<commit_before>// Supports Windows, Linux, Mac, BeagleBone Black, and Raspberry Pi

package main

import (
	"encoding/json"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/arduino/arduino-create-agent/upload"
)

type writeRequest struct {
	p      *serport
	d      string
	buffer string
}

type serialhub struct {
	// Opened serial ports.
	ports map[*serport]bool

	//write chan *serport, chan []byte
	write chan writeRequest

	// Register requests from the connections.
	register chan *serport

	// Unregister requests from connections.
	unregister chan *serport

	mu sync.Mutex
}

type SpPortList struct {
	Ports   []SpPortItem
	Network bool
	mu      sync.Mutex `json:"-"`
}

type SpPortItem struct {
	Name            string
	SerialNumber    string
	DeviceClass     string
	IsOpen          bool
	IsPrimary       bool
	Baud            int
	BufferAlgorithm string
	Ver             string
	NetworkPort     bool
	VendorID        string
	ProductID       string
}

// SerialPorts contains the ports attached to the machine
var SerialPorts SpPortList

// NetworkPorts contains the ports on the network
var NetworkPorts SpPortList

var sh = serialhub{
	//write:   	make(chan *serport, chan []byte),
	write:      make(chan writeRequest),
	register:   make(chan *serport),
	unregister: make(chan *serport),
	ports:      make(map[*serport]bool),
}

func (sh *serialhub) run() {

	//log.Print("Inside run of serialhub")
	//cmdIdCtr := 0

	for {
		select {
		case p := <-sh.register:
			sh.mu.Lock()
			//log.Print("Registering a port: ", p.portConf.Name)
			h.broadcastSys <- []byte("{\"Cmd\":\"Open\",\"Desc\":\"Got register/open on port.\",\"Port\":\"" + p.portConf.Name + "\",\"Baud\":" + strconv.Itoa(p.portConf.Baud) + ",\"BufferType\":\"" + p.BufferType + "\"}")
			sh.ports[p] = true
			sh.mu.Unlock()
		case p := <-sh.unregister:
			sh.mu.Lock()
			//log.Print("Unregistering a port: ", p.portConf.Name)
			h.broadcastSys <- []byte("{\"Cmd\":\"Close\",\"Desc\":\"Got unregister/close on port.\",\"Port\":\"" + p.portConf.Name + "\",\"Baud\":" + strconv.Itoa(p.portConf.Baud) + "}")
			delete(sh.ports, p)
			close(p.sendBuffered)
			close(p.sendNoBuf)
			sh.mu.Unlock()
		case wr := <-sh.write:
			// if user sent in the commands as one text mode line
			write(wr)
		}
	}
}

func write(wr writeRequest) {
	switch wr.buffer {
	case "send":
		wr.p.sendBuffered <- wr.d
	case "sendnobuf":
		wr.p.sendNoBuf <- []byte(wr.d)
	case "sendraw":
		wr.p.sendRaw <- wr.d
	}
	// no default since we alredy verified in spWrite()
}

// spList broadcasts a Json representation of the ports found
func spList(network bool) {
	var ls []byte
	var err error
	if network {
		NetworkPorts.mu.Lock()
		ls, err = json.MarshalIndent(NetworkPorts, "", "\t")
		NetworkPorts.mu.Unlock()
	} else {
		SerialPorts.mu.Lock()
		ls, err = json.MarshalIndent(SerialPorts, "", "\t")
		SerialPorts.mu.Unlock()
	}
	if err != nil {
		//log.Println(err)
		h.broadcastSys <- []byte("Error creating json on port list " +
			err.Error())
	} else {
		h.broadcastSys <- ls
	}
}

// discoverLoop periodically update the list of ports found
func discoverLoop() {
	SerialPorts.mu.Lock()
	SerialPorts.Network = false
	SerialPorts.Ports = make([]SpPortItem, 0)
	SerialPorts.mu.Unlock()
	NetworkPorts.mu.Lock()
	NetworkPorts.Network = true
	NetworkPorts.Ports = make([]SpPortItem, 0)
	NetworkPorts.mu.Unlock()

	go func() {
		for {
			if !upload.Busy {
				spListDual(false)
			}
			time.Sleep(2 * time.Second)
		}
	}()
	go func() {
		for {
			spListDual(true)
			time.Sleep(2 * time.Second)
		}
	}()
}

func spListDual(network bool) {

	// call our os specific implementation of getting the serial list
	list, err := GetList(network)

	//log.Println(list)
	//log.Println(err)

	if err != nil {
		// avoid reporting dummy data if an error occurred
		return
	}

	// do a quick loop to see if any of our open ports
	// did not end up in the list port list. this can
	// happen on windows in a fallback scenario where an
	// open port can't be identified because it is locked,
	// so just solve that by manually inserting
	// if network {
	// 	for port := range sh.ports {

	// 		isFound := false
	// 		for _, item := range list {
	// 			if strings.ToLower(port.portConf.Name) == strings.ToLower(item.Name) {
	// 				isFound = true
	// 			}
	// 		}

	// 		if !isFound {
	// 			// artificially push to front of port list
	// 			log.Println(fmt.Sprintf("Did not find an open port in the serial port list. We are going to artificially push it onto the list. port:%v", port.portConf.Name))
	// 			var ossp OsSerialPort
	// 			ossp.Name = port.portConf.Name
	// 			ossp.FriendlyName = port.portConf.Name
	// 			list = append([]OsSerialPort{ossp}, list...)
	// 		}
	// 	}
	// }

	// we have a full clean list of ports now. iterate thru them
	// to append the open/close state, baud rates, etc to make
	// a super clean nice list to send back to browser
	n := len(list)
	spl := make([]SpPortItem, n)

	ctr := 0

	for _, item := range list {

		spl[ctr] = SpPortItem{
			Name:            item.Name,
			SerialNumber:    item.ISerial,
			DeviceClass:     item.DeviceClass,
			IsOpen:          false,
			IsPrimary:       false,
			Baud:            0,
			BufferAlgorithm: "",
			Ver:             version,
			NetworkPort:     item.NetworkPort,
			VendorID:        item.IdVendor,
			ProductID:       item.IdProduct,
		}

		// figure out if port is open
		myport, isFound := findPortByName(item.Name)

		if isFound {
			// we found our port
			spl[ctr].IsOpen = true
			spl[ctr].Baud = myport.portConf.Baud
			spl[ctr].BufferAlgorithm = myport.BufferType
		}
		ctr++
	}

	if network {
		NetworkPorts.mu.Lock()
		NetworkPorts.Ports = spl
		NetworkPorts.mu.Unlock()
	} else {
		SerialPorts.mu.Lock()
		SerialPorts.Ports = spl
		SerialPorts.mu.Unlock()
	}
}

func spErr(err string) {
	//log.Println("Sending err back: ", err)
	//h.broadcastSys <- []byte(err)
	h.broadcastSys <- []byte("{\"Error\" : \"" + err + "\"}")
}

func spClose(portname string) {
	// look up the registered port by name
	// then call the close method inside serialport
	// that should cause an unregister channel call back
	// to myself

	myport, isFound := findPortByName(portname)

	if isFound {
		// we found our port
		spHandlerClose(myport)
	} else {
		// we couldn't find the port, so send err
		spErr("We could not find the serial port " + portname + " that you were trying to close.")
	}
}

func spWrite(arg string) {
	// we will get a string of comXX asdf asdf asdf
	//log.Println("Inside spWrite arg: " + arg)
	arg = strings.TrimPrefix(arg, " ")
	//log.Println("arg after trim: " + arg)
	args := strings.SplitN(arg, " ", 3)
	if len(args) != 3 {
		errstr := "Could not parse send command: " + arg
		//log.Println(errstr)
		spErr(errstr)
		return
	}
	portname := strings.Trim(args[1], " ")
	//log.Println("The port to write to is:" + portname + "---")
	//log.Println("The data is:" + args[2] + "---")

	// see if we have this port open
	myport, isFound := findPortByName(portname)

	if !isFound {
		// we couldn't find the port, so send err
		spErr("We could not find the serial port " + portname + " that you were trying to write to.")
		return
	}

	// we found our port
	// create our write request
	var wr writeRequest
	wr.p = myport

	// see if args[0] is send or sendnobuf or sendraw
	switch args[0] {
	case "send", "sendnobuf", "sendraw":
		wr.buffer = args[0]
	default:
		spErr("Unsupported send command:" + args[0] + ". Please specify a valid one")
		return
	}

	// include newline or not in the write? that is the question.
	// for now lets skip the newline
	//wr.d = []byte(args[2] + "\n")
	wr.d = args[2] //[]byte(args[2])

	// send it to the write channel
	sh.write <- wr
}
<commit_msg>fix `go vet` failing, followup of #668 (#693)<commit_after>// Supports Windows, Linux, Mac, BeagleBone Black, and Raspberry Pi

package main

import (
	"encoding/json"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/arduino/arduino-create-agent/upload"
)

type writeRequest struct {
	p      *serport
	d      string
	buffer string
}

type serialhub struct {
	// Opened serial ports.
	ports map[*serport]bool

	//write chan *serport, chan []byte
	write chan writeRequest

	// Register requests from the connections.
	register chan *serport

	// Unregister requests from connections.
	unregister chan *serport

	mu sync.Mutex
}

type SpPortList struct {
	Ports   []SpPortItem
	Network bool
	mu      sync.Mutex `json:"-"`
}

type SpPortItem struct {
	Name            string
	SerialNumber    string
	DeviceClass     string
	IsOpen          bool
	IsPrimary       bool
	Baud            int
	BufferAlgorithm string
	Ver             string
	NetworkPort     bool
	VendorID        string
	ProductID       string
}

// SerialPorts contains the ports attached to the machine
var SerialPorts SpPortList

// NetworkPorts contains the ports on the network
var NetworkPorts SpPortList

var sh = serialhub{
	//write:   	make(chan *serport, chan []byte),
	write:      make(chan writeRequest),
	register:   make(chan *serport),
	unregister: make(chan *serport),
	ports:      make(map[*serport]bool),
}

func (sh *serialhub) run() {

	//log.Print("Inside run of serialhub")
	//cmdIdCtr := 0

	for {
		select {
		case p := <-sh.register:
			sh.mu.Lock()
			//log.Print("Registering a port: ", p.portConf.Name)
			h.broadcastSys <- []byte("{\"Cmd\":\"Open\",\"Desc\":\"Got register/open on port.\",\"Port\":\"" + p.portConf.Name + "\",\"Baud\":" + strconv.Itoa(p.portConf.Baud) + ",\"BufferType\":\"" + p.BufferType + "\"}")
			sh.ports[p] = true
			sh.mu.Unlock()
		case p := <-sh.unregister:
			sh.mu.Lock()
			//log.Print("Unregistering a port: ", p.portConf.Name)
			h.broadcastSys <- []byte("{\"Cmd\":\"Close\",\"Desc\":\"Got unregister/close on port.\",\"Port\":\"" + p.portConf.Name + "\",\"Baud\":" + strconv.Itoa(p.portConf.Baud) + "}")
			delete(sh.ports, p)
			close(p.sendBuffered)
			close(p.sendNoBuf)
			sh.mu.Unlock()
		case wr := <-sh.write:
			// if user sent in the commands as one text mode line
			write(wr)
		}
	}
}

func write(wr writeRequest) {
	switch wr.buffer {
	case "send":
		wr.p.sendBuffered <- wr.d
	case "sendnobuf":
		wr.p.sendNoBuf <- []byte(wr.d)
	case "sendraw":
		wr.p.sendRaw <- wr.d
	}
	// no default since we alredy verified in spWrite()
}

// spList broadcasts a Json representation of the ports found
func spList(network bool) {
	var ls []byte
	var err error
	if network {
		NetworkPorts.mu.Lock()
		ls, err = json.MarshalIndent(&NetworkPorts, "", "\t")
		NetworkPorts.mu.Unlock()
	} else {
		SerialPorts.mu.Lock()
		ls, err = json.MarshalIndent(&SerialPorts, "", "\t")
		SerialPorts.mu.Unlock()
	}
	if err != nil {
		//log.Println(err)
		h.broadcastSys <- []byte("Error creating json on port list " +
			err.Error())
	} else {
		h.broadcastSys <- ls
	}
}

// discoverLoop periodically update the list of ports found
func discoverLoop() {
	SerialPorts.mu.Lock()
	SerialPorts.Network = false
	SerialPorts.Ports = make([]SpPortItem, 0)
	SerialPorts.mu.Unlock()
	NetworkPorts.mu.Lock()
	NetworkPorts.Network = true
	NetworkPorts.Ports = make([]SpPortItem, 0)
	NetworkPorts.mu.Unlock()

	go func() {
		for {
			if !upload.Busy {
				spListDual(false)
			}
			time.Sleep(2 * time.Second)
		}
	}()
	go func() {
		for {
			spListDual(true)
			time.Sleep(2 * time.Second)
		}
	}()
}

func spListDual(network bool) {

	// call our os specific implementation of getting the serial list
	list, err := GetList(network)

	//log.Println(list)
	//log.Println(err)

	if err != nil {
		// avoid reporting dummy data if an error occurred
		return
	}

	// do a quick loop to see if any of our open ports
	// did not end up in the list port list. this can
	// happen on windows in a fallback scenario where an
	// open port can't be identified because it is locked,
	// so just solve that by manually inserting
	// if network {
	// 	for port := range sh.ports {

	// 		isFound := false
	// 		for _, item := range list {
	// 			if strings.ToLower(port.portConf.Name) == strings.ToLower(item.Name) {
	// 				isFound = true
	// 			}
	// 		}

	// 		if !isFound {
	// 			// artificially push to front of port list
	// 			log.Println(fmt.Sprintf("Did not find an open port in the serial port list. We are going to artificially push it onto the list. port:%v", port.portConf.Name))
	// 			var ossp OsSerialPort
	// 			ossp.Name = port.portConf.Name
	// 			ossp.FriendlyName = port.portConf.Name
	// 			list = append([]OsSerialPort{ossp}, list...)
	// 		}
	// 	}
	// }

	// we have a full clean list of ports now. iterate thru them
	// to append the open/close state, baud rates, etc to make
	// a super clean nice list to send back to browser
	n := len(list)
	spl := make([]SpPortItem, n)

	ctr := 0

	for _, item := range list {

		spl[ctr] = SpPortItem{
			Name:            item.Name,
			SerialNumber:    item.ISerial,
			DeviceClass:     item.DeviceClass,
			IsOpen:          false,
			IsPrimary:       false,
			Baud:            0,
			BufferAlgorithm: "",
			Ver:             version,
			NetworkPort:     item.NetworkPort,
			VendorID:        item.IdVendor,
			ProductID:       item.IdProduct,
		}

		// figure out if port is open
		myport, isFound := findPortByName(item.Name)

		if isFound {
			// we found our port
			spl[ctr].IsOpen = true
			spl[ctr].Baud = myport.portConf.Baud
			spl[ctr].BufferAlgorithm = myport.BufferType
		}
		ctr++
	}

	if network {
		NetworkPorts.mu.Lock()
		NetworkPorts.Ports = spl
		NetworkPorts.mu.Unlock()
	} else {
		SerialPorts.mu.Lock()
		SerialPorts.Ports = spl
		SerialPorts.mu.Unlock()
	}
}

func spErr(err string) {
	//log.Println("Sending err back: ", err)
	//h.broadcastSys <- []byte(err)
	h.broadcastSys <- []byte("{\"Error\" : \"" + err + "\"}")
}

func spClose(portname string) {
	// look up the registered port by name
	// then call the close method inside serialport
	// that should cause an unregister channel call back
	// to myself

	myport, isFound := findPortByName(portname)

	if isFound {
		// we found our port
		spHandlerClose(myport)
	} else {
		// we couldn't find the port, so send err
		spErr("We could not find the serial port " + portname + " that you were trying to close.")
	}
}

func spWrite(arg string) {
	// we will get a string of comXX asdf asdf asdf
	//log.Println("Inside spWrite arg: " + arg)
	arg = strings.TrimPrefix(arg, " ")
	//log.Println("arg after trim: " + arg)
	args := strings.SplitN(arg, " ", 3)
	if len(args) != 3 {
		errstr := "Could not parse send command: " + arg
		//log.Println(errstr)
		spErr(errstr)
		return
	}
	portname := strings.Trim(args[1], " ")
	//log.Println("The port to write to is:" + portname + "---")
	//log.Println("The data is:" + args[2] + "---")

	// see if we have this port open
	myport, isFound := findPortByName(portname)

	if !isFound {
		// we couldn't find the port, so send err
		spErr("We could not find the serial port " + portname + " that you were trying to write to.")
		return
	}

	// we found our port
	// create our write request
	var wr writeRequest
	wr.p = myport

	// see if args[0] is send or sendnobuf or sendraw
	switch args[0] {
	case "send", "sendnobuf", "sendraw":
		wr.buffer = args[0]
	default:
		spErr("Unsupported send command:" + args[0] + ". Please specify a valid one")
		return
	}

	// include newline or not in the write? that is the question.
	// for now lets skip the newline
	//wr.d = []byte(args[2] + "\n")
	wr.d = args[2] //[]byte(args[2])

	// send it to the write channel
	sh.write <- wr
}
<|endoftext|>


<commit_before>package orchestrators

import (
	"bytes"
	"fmt"
	"sort"
	"time"
	"unicode/utf8"

	"github.com/camptocamp/bivac/handler"
	"github.com/camptocamp/bivac/volume"

	log "github.com/Sirupsen/logrus"
	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/remotecommand"
)

// KubernetesOrchestrator implements a container orchestrator for Kubernetes
type KubernetesOrchestrator struct {
	Handler *handler.Bivac
	Client  *kubernetes.Clientset
}

// NewKubernetesOrchestrator creates a Kubernetes client
func NewKubernetesOrchestrator(c *handler.Bivac) (o *KubernetesOrchestrator) {
	var err error
	o = &KubernetesOrchestrator{
		Handler: c,
	}

	config, err := o.getConfig()
	if err != nil {
		log.Fatalf("failed to retrieve Kubernetes config: %s", err)
	}

	o.Client, err = kubernetes.NewForConfig(config)
	if err != nil {
		log.Errorf("failed to create a Kubernetes client: %v", err)
	}
	return
}

// GetName returns the orchestrator name
func (*KubernetesOrchestrator) GetName() string {
	return "Kubernetes"
}

// GetHandler returns the Orchestrator's handler
func (o *KubernetesOrchestrator) GetHandler() *handler.Bivac {
	return o.Handler
}

// GetVolumes returns the Kubernetes persistent volume claims, inspected and filtered
func (o *KubernetesOrchestrator) GetVolumes() (volumes []*volume.Volume, err error) {
	c := o.Handler

	pvcs, err := o.Client.CoreV1().PersistentVolumeClaims(o.Handler.Config.Kubernetes.Namespace).List(metav1.ListOptions{})
	if err != nil {
		log.Errorf("failed to retrieve the list of PVCs: %v", err)
	}

	containers, err := o.GetMountedVolumes()
	mountedVolumes := make(map[string]string)
	bindHostVolume := make(map[string]string)
	for _, container := range containers {
		for volName, volMountpath := range container.Volumes {
			mountedVolumes[volName] = volMountpath
			bindHostVolume[volName] = container.HostID
		}
	}
	var mountpoint string
	for _, pvc := range pvcs.Items {
		if value, ok := mountedVolumes[pvc.Name]; ok {
			mountpoint = value
		} else {
			mountpoint = "/data"
		}
		nv := &volume.Volume{
			Config:     &volume.Config{},
			Mountpoint: mountpoint,
			Name:       pvc.Name,
			HostBind:   bindHostVolume[pvc.Name],
			Hostname:   bindHostVolume[pvc.Name],
		}

		v := volume.NewVolume(nv, c.Config, c.Hostname)
		if b, r, s := o.blacklistedVolume(v); b {
			log.WithFields(log.Fields{
				"volume": pvc.Name,
				"reason": r,
				"source": s,
			}).Info("Ignoring volume")
			continue
		}
		volumes = append(volumes, v)
		log.Infof("%+v", v)
	}
	return
}

// LaunchContainer starts a container using the Kubernetes orchestrator
func (o *KubernetesOrchestrator) LaunchContainer(image string, env map[string]string, cmd []string, volumes []*volume.Volume) (state int, stdout string, err error) {

	var envVars []apiv1.EnvVar
	for envName, envValue := range env {
		ev := apiv1.EnvVar{
			Name:  envName,
			Value: envValue,
		}
		envVars = append(envVars, ev)
	}

	kvs := []apiv1.Volume{}
	kvms := []apiv1.VolumeMount{}
	var node string

	for _, v := range volumes {
		pvc, err := o.Client.CoreV1().PersistentVolumeClaims(o.Handler.Config.Kubernetes.Namespace).Get(v.Name, metav1.GetOptions{})
		if err != nil {
			log.Errorf("failed to retrieve PersistentVolumeClaim \""+v.Name+"\": %s", err)
			continue
		}

		for _, am := range pvc.Spec.AccessModes {
			if am == apiv1.ReadWriteOnce {
				node = v.HostBind
			}
		}

		kv := apiv1.Volume{
			Name: v.Name,
			VolumeSource: apiv1.VolumeSource{
				PersistentVolumeClaim: &apiv1.PersistentVolumeClaimVolumeSource{
					ClaimName: v.Name,
					ReadOnly:  false,
				},
			},
		}
		kvs = append(kvs, kv)

		kvm := apiv1.VolumeMount{
			Name:      v.Name,
			ReadOnly:  v.ReadOnly,
			MountPath: v.Mountpoint,
		}
		kvms = append(kvms, kvm)
	}

	pod, err := o.Client.CoreV1().Pods(o.Handler.Config.Kubernetes.Namespace).Create(&apiv1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "bivac-worker-",
		},
		Spec: apiv1.PodSpec{
			NodeName:      node,
			RestartPolicy: "Never",
			Volumes:       kvs,
			Containers: []apiv1.Container{
				{
					Name:            "bivac-worker",
					Image:           image,
					Args:            cmd,
					Env:             envVars,
					VolumeMounts:    kvms,
					ImagePullPolicy: apiv1.PullAlways,
				},
			},
		},
	})
	if err != nil {
		log.Errorf("failed to create worker: %s", err)
	}

	workerName := pod.ObjectMeta.Name

	defer o.DeleteWorker(workerName)

	timeout := time.After(60 * time.Second)
	terminated := false
	for !terminated {
		pod, err := o.Client.CoreV1().Pods(o.Handler.Config.Kubernetes.Namespace).Get(workerName, metav1.GetOptions{})
		if err != nil {
			log.Errorf("failed to get pod: %s", err)
		}

		if pod.Status.Phase == apiv1.PodSucceeded || pod.Status.Phase == apiv1.PodFailed {
			state = int(pod.Status.ContainerStatuses[0].State.Terminated.ExitCode)
			terminated = true
		} else if pod.Status.Phase != apiv1.PodRunning {
			select {
			case <-timeout:
				err = fmt.Errorf("failed to start worker: timeout")
				return -1, "", err
			default:
				continue
			}
		}
	}

	req := o.Client.CoreV1().Pods(o.Handler.Config.Kubernetes.Namespace).GetLogs(workerName, &apiv1.PodLogOptions{})

	readCloser, err := req.Stream()
	if err != nil {
		log.Errorf("failed to read logs: %s", err)
	}

	defer readCloser.Close()
	buf := new(bytes.Buffer)
	buf.ReadFrom(readCloser)
	stdout = buf.String()

	return
}

// DeleteWorker deletes a worker
func (o *KubernetesOrchestrator) DeleteWorker(name string) {
	err := o.Client.CoreV1().Pods(o.Handler.Config.Kubernetes.Namespace).Delete(name, &metav1.DeleteOptions{})
	if err != nil {
		log.Errorf("failed to delete worker: %s", err)
	}
	return
}

// GetMountedVolumes returns mounted volumes
func (o *KubernetesOrchestrator) GetMountedVolumes() (containers []*volume.MountedVolumes, err error) {

	pods, err := o.Client.CoreV1().Pods(o.Handler.Config.Kubernetes.Namespace).List(metav1.ListOptions{})
	if err != nil {
		log.Errorf("failed to get pods: %s", err)
	}

	mapVolClaim := make(map[string]string)

	for _, pod := range pods.Items {
		for _, volume := range pod.Spec.Volumes {
			if volume.PersistentVolumeClaim != nil {
				mapVolClaim[volume.Name] = volume.PersistentVolumeClaim.ClaimName
			}
		}

		for _, container := range pod.Spec.Containers {
			mv := &volume.MountedVolumes{
				PodID:       pod.Name,
				ContainerID: container.Name,
				HostID:      pod.Spec.NodeName,
				Volumes:     make(map[string]string),
			}
			for _, volumeMount := range container.VolumeMounts {
				if c, ok := mapVolClaim[volumeMount.Name]; ok {
					mv.Volumes[c] = volumeMount.MountPath
				}
			}
			containers = append(containers, mv)
		}
	}

	return
}

// ContainerExec executes a command in a container
func (o *KubernetesOrchestrator) ContainerExec(mountedVolumes *volume.MountedVolumes, command []string) (err error) {
	var stdout, stderr bytes.Buffer

	config, err := o.getConfig()
	if err != nil {
		log.Fatalf("failed to retrieve Kubernetes config: %s", err)
	}

	req := o.Client.Core().RESTClient().Post().
		Resource("pods").
		Name(mountedVolumes.PodID).
		Namespace(o.Handler.Config.Kubernetes.Namespace).
		SubResource("exec").
		Param("container", mountedVolumes.ContainerID)
	req.VersionedParams(&apiv1.PodExecOptions{
		Container: mountedVolumes.ContainerID,
		Command:   command,
		Stdin:     false,
		Stdout:    true,
		Stderr:    true,
		TTY:       false,
	}, scheme.ParameterCodec)

	exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL())
	if err != nil {
		log.Errorf("failed to call the API: %s", err)
		return err
	}
	err = exec.Stream(remotecommand.StreamOptions{
		Stdin:  nil,
		Stdout: &stdout,
		Stderr: &stderr,
		Tty:    false,
	})

	return
}

func (o *KubernetesOrchestrator) blacklistedVolume(vol *volume.Volume) (bool, string, string) {

	defaultBlacklistedVolumes := []string{
		"duplicity_cache",
		"restic_cache",
		"duplicity-cache",
		"restic-cache",
		"lost+found",
	}

	if utf8.RuneCountInString(vol.Name) == 64 {
		return true, "unnamed", ""
	}

	list := o.Handler.Config.VolumesBlacklist
	list = append(list, defaultBlacklistedVolumes...)
	sort.Strings(list)
	i := sort.SearchStrings(list, vol.Name)
	if i < len(list) && list[i] == vol.Name {
		return true, "blacklisted", "blacklist config"
	}

	if vol.Config.Ignore {
		return true, "blacklisted", "volume config"
	}

	return false, "", ""
}

func (o *KubernetesOrchestrator) getConfig() (config *rest.Config, err error) {
	if o.Handler.Config.Kubernetes.KubeConfig != "" {
		config, err = clientcmd.BuildConfigFromFlags("", o.Handler.Config.Kubernetes.KubeConfig)
	} else {
		kubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
			clientcmd.NewDefaultClientConfigLoadingRules(),
			&clientcmd.ConfigOverrides{},
		)
		o.Handler.Config.Kubernetes.Namespace, _, err = kubeconfig.Namespace()
		if err != nil {
			log.Errorf("Failed to retrieve the namespace from the cluster config: %v", err)
		}
		config, err = rest.InClusterConfig()
	}
	return
}

func detectKubernetes() bool {
	_, err := rest.InClusterConfig()
	if err != nil {
		return false
	}
	return true
}
<commit_msg>Add warning if k8s namespace provided without kubeconfig file<commit_after>package orchestrators

import (
	"bytes"
	"fmt"
	"sort"
	"time"
	"unicode/utf8"

	"github.com/camptocamp/bivac/handler"
	"github.com/camptocamp/bivac/volume"

	log "github.com/Sirupsen/logrus"
	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/remotecommand"
)

// KubernetesOrchestrator implements a container orchestrator for Kubernetes
type KubernetesOrchestrator struct {
	Handler *handler.Bivac
	Client  *kubernetes.Clientset
}

// NewKubernetesOrchestrator creates a Kubernetes client
func NewKubernetesOrchestrator(c *handler.Bivac) (o *KubernetesOrchestrator) {
	var err error
	o = &KubernetesOrchestrator{
		Handler: c,
	}

	config, err := o.getConfig()
	if err != nil {
		log.Fatalf("failed to retrieve Kubernetes config: %s", err)
	}

	o.Client, err = kubernetes.NewForConfig(config)
	if err != nil {
		log.Errorf("failed to create a Kubernetes client: %v", err)
	}
	return
}

// GetName returns the orchestrator name
func (*KubernetesOrchestrator) GetName() string {
	return "Kubernetes"
}

// GetHandler returns the Orchestrator's handler
func (o *KubernetesOrchestrator) GetHandler() *handler.Bivac {
	return o.Handler
}

// GetVolumes returns the Kubernetes persistent volume claims, inspected and filtered
func (o *KubernetesOrchestrator) GetVolumes() (volumes []*volume.Volume, err error) {
	c := o.Handler

	pvcs, err := o.Client.CoreV1().PersistentVolumeClaims(o.Handler.Config.Kubernetes.Namespace).List(metav1.ListOptions{})
	if err != nil {
		log.Errorf("failed to retrieve the list of PVCs: %v", err)
	}

	containers, err := o.GetMountedVolumes()
	mountedVolumes := make(map[string]string)
	bindHostVolume := make(map[string]string)
	for _, container := range containers {
		for volName, volMountpath := range container.Volumes {
			mountedVolumes[volName] = volMountpath
			bindHostVolume[volName] = container.HostID
		}
	}
	var mountpoint string
	for _, pvc := range pvcs.Items {
		if value, ok := mountedVolumes[pvc.Name]; ok {
			mountpoint = value
		} else {
			mountpoint = "/data"
		}
		nv := &volume.Volume{
			Config:     &volume.Config{},
			Mountpoint: mountpoint,
			Name:       pvc.Name,
			HostBind:   bindHostVolume[pvc.Name],
			Hostname:   bindHostVolume[pvc.Name],
		}

		v := volume.NewVolume(nv, c.Config, c.Hostname)
		if b, r, s := o.blacklistedVolume(v); b {
			log.WithFields(log.Fields{
				"volume": pvc.Name,
				"reason": r,
				"source": s,
			}).Info("Ignoring volume")
			continue
		}
		volumes = append(volumes, v)
		log.Infof("%+v", v)
	}
	return
}

// LaunchContainer starts a container using the Kubernetes orchestrator
func (o *KubernetesOrchestrator) LaunchContainer(image string, env map[string]string, cmd []string, volumes []*volume.Volume) (state int, stdout string, err error) {

	var envVars []apiv1.EnvVar
	for envName, envValue := range env {
		ev := apiv1.EnvVar{
			Name:  envName,
			Value: envValue,
		}
		envVars = append(envVars, ev)
	}

	kvs := []apiv1.Volume{}
	kvms := []apiv1.VolumeMount{}
	var node string

	for _, v := range volumes {
		pvc, err := o.Client.CoreV1().PersistentVolumeClaims(o.Handler.Config.Kubernetes.Namespace).Get(v.Name, metav1.GetOptions{})
		if err != nil {
			log.Errorf("failed to retrieve PersistentVolumeClaim \""+v.Name+"\": %s", err)
			continue
		}

		for _, am := range pvc.Spec.AccessModes {
			if am == apiv1.ReadWriteOnce {
				node = v.HostBind
			}
		}

		kv := apiv1.Volume{
			Name: v.Name,
			VolumeSource: apiv1.VolumeSource{
				PersistentVolumeClaim: &apiv1.PersistentVolumeClaimVolumeSource{
					ClaimName: v.Name,
					ReadOnly:  false,
				},
			},
		}
		kvs = append(kvs, kv)

		kvm := apiv1.VolumeMount{
			Name:      v.Name,
			ReadOnly:  v.ReadOnly,
			MountPath: v.Mountpoint,
		}
		kvms = append(kvms, kvm)
	}

	pod, err := o.Client.CoreV1().Pods(o.Handler.Config.Kubernetes.Namespace).Create(&apiv1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "bivac-worker-",
		},
		Spec: apiv1.PodSpec{
			NodeName:      node,
			RestartPolicy: "Never",
			Volumes:       kvs,
			Containers: []apiv1.Container{
				{
					Name:            "bivac-worker",
					Image:           image,
					Args:            cmd,
					Env:             envVars,
					VolumeMounts:    kvms,
					ImagePullPolicy: apiv1.PullAlways,
				},
			},
		},
	})
	if err != nil {
		log.Errorf("failed to create worker: %s", err)
	}

	workerName := pod.ObjectMeta.Name

	defer o.DeleteWorker(workerName)

	timeout := time.After(60 * time.Second)
	terminated := false
	for !terminated {
		pod, err := o.Client.CoreV1().Pods(o.Handler.Config.Kubernetes.Namespace).Get(workerName, metav1.GetOptions{})
		if err != nil {
			log.Errorf("failed to get pod: %s", err)
		}

		if pod.Status.Phase == apiv1.PodSucceeded || pod.Status.Phase == apiv1.PodFailed {
			state = int(pod.Status.ContainerStatuses[0].State.Terminated.ExitCode)
			terminated = true
		} else if pod.Status.Phase != apiv1.PodRunning {
			select {
			case <-timeout:
				err = fmt.Errorf("failed to start worker: timeout")
				return -1, "", err
			default:
				continue
			}
		}
	}

	req := o.Client.CoreV1().Pods(o.Handler.Config.Kubernetes.Namespace).GetLogs(workerName, &apiv1.PodLogOptions{})

	readCloser, err := req.Stream()
	if err != nil {
		log.Errorf("failed to read logs: %s", err)
	}

	defer readCloser.Close()
	buf := new(bytes.Buffer)
	buf.ReadFrom(readCloser)
	stdout = buf.String()

	return
}

// DeleteWorker deletes a worker
func (o *KubernetesOrchestrator) DeleteWorker(name string) {
	err := o.Client.CoreV1().Pods(o.Handler.Config.Kubernetes.Namespace).Delete(name, &metav1.DeleteOptions{})
	if err != nil {
		log.Errorf("failed to delete worker: %s", err)
	}
	return
}

// GetMountedVolumes returns mounted volumes
func (o *KubernetesOrchestrator) GetMountedVolumes() (containers []*volume.MountedVolumes, err error) {

	pods, err := o.Client.CoreV1().Pods(o.Handler.Config.Kubernetes.Namespace).List(metav1.ListOptions{})
	if err != nil {
		log.Errorf("failed to get pods: %s", err)
	}

	mapVolClaim := make(map[string]string)

	for _, pod := range pods.Items {
		for _, volume := range pod.Spec.Volumes {
			if volume.PersistentVolumeClaim != nil {
				mapVolClaim[volume.Name] = volume.PersistentVolumeClaim.ClaimName
			}
		}

		for _, container := range pod.Spec.Containers {
			mv := &volume.MountedVolumes{
				PodID:       pod.Name,
				ContainerID: container.Name,
				HostID:      pod.Spec.NodeName,
				Volumes:     make(map[string]string),
			}
			for _, volumeMount := range container.VolumeMounts {
				if c, ok := mapVolClaim[volumeMount.Name]; ok {
					mv.Volumes[c] = volumeMount.MountPath
				}
			}
			containers = append(containers, mv)
		}
	}

	return
}

// ContainerExec executes a command in a container
func (o *KubernetesOrchestrator) ContainerExec(mountedVolumes *volume.MountedVolumes, command []string) (err error) {
	var stdout, stderr bytes.Buffer

	config, err := o.getConfig()
	if err != nil {
		log.Fatalf("failed to retrieve Kubernetes config: %s", err)
	}

	req := o.Client.Core().RESTClient().Post().
		Resource("pods").
		Name(mountedVolumes.PodID).
		Namespace(o.Handler.Config.Kubernetes.Namespace).
		SubResource("exec").
		Param("container", mountedVolumes.ContainerID)
	req.VersionedParams(&apiv1.PodExecOptions{
		Container: mountedVolumes.ContainerID,
		Command:   command,
		Stdin:     false,
		Stdout:    true,
		Stderr:    true,
		TTY:       false,
	}, scheme.ParameterCodec)

	exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL())
	if err != nil {
		log.Errorf("failed to call the API: %s", err)
		return err
	}
	err = exec.Stream(remotecommand.StreamOptions{
		Stdin:  nil,
		Stdout: &stdout,
		Stderr: &stderr,
		Tty:    false,
	})

	return
}

func (o *KubernetesOrchestrator) blacklistedVolume(vol *volume.Volume) (bool, string, string) {

	defaultBlacklistedVolumes := []string{
		"duplicity_cache",
		"restic_cache",
		"duplicity-cache",
		"restic-cache",
		"lost+found",
	}

	if utf8.RuneCountInString(vol.Name) == 64 {
		return true, "unnamed", ""
	}

	list := o.Handler.Config.VolumesBlacklist
	list = append(list, defaultBlacklistedVolumes...)
	sort.Strings(list)
	i := sort.SearchStrings(list, vol.Name)
	if i < len(list) && list[i] == vol.Name {
		return true, "blacklisted", "blacklist config"
	}

	if vol.Config.Ignore {
		return true, "blacklisted", "volume config"
	}

	return false, "", ""
}

func (o *KubernetesOrchestrator) getConfig() (config *rest.Config, err error) {
	if o.Handler.Config.Kubernetes.KubeConfig != "" {
		config, err = clientcmd.BuildConfigFromFlags("", o.Handler.Config.Kubernetes.KubeConfig)
	} else {
		kubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
			clientcmd.NewDefaultClientConfigLoadingRules(),
			&clientcmd.ConfigOverrides{},
		)

		if o.Handler.Config.Kubernetes.Namespace != "" {
			log.Warningf("Using provided Kubernetes namespace.")
		} else {
			o.Handler.Config.Kubernetes.Namespace, _, err = kubeconfig.Namespace()
		}

		if err != nil {
			log.Errorf("Failed to retrieve the namespace from the cluster config: %v", err)
		}
		config, err = rest.InClusterConfig()
	}
	return
}

func detectKubernetes() bool {
	_, err := rest.InClusterConfig()
	if err != nil {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2016 Jacob Taylor jacob@ablox.io
// License: Apache2 - http://www.apache.org/licenses/LICENSE-2.0
package main

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"sort"
	"sync"
	"time"
)

// Event stores the relevant information on events or updates to the storage layer.
type Event struct {
	Source        string
	Name          string
	Message       string
	Time          time.Time
	IsDirectory   bool
	NetworkSource string
}

var events = make([]Event, 0, 100)

// SendEvent gets events that have happened off to the peer servers so they can replicate the same change
func SendEvent(event Event, fullPath string) {
	// look back through the events for a similar event in the recent path.
	// Set the event source  (server name)
	event.Source = globalSettings.Name
	event.Time = time.Now()

	// Get the current owner of this entry if any
	ownershipLock.RLock()
	originalEntry, exists := ownership[event.Message]
	ownershipLock.RUnlock()

	if exists {
		log.Printf("Original ownership: %#v\n", originalEntry)
		timeDelta := time.Since(originalEntry.Time)
		if timeDelta > OWNERSHIP_EXPIRATION_TIMEOUT {
			log.Printf("Owndership expired. Delta is: %v\n", timeDelta)
		} else {
			// At this point, someone owns this item.
			// If we are not the owner, we are done.
			if originalEntry.Source != globalSettings.Name {
				log.Println("We do not own this. Do not send")
				return
			}
		}
	}

	// At this point, it is our change and our event. Store our ownership
	ownershipLock.Lock()
	ownership[event.Message] = event
	ownershipLock.Unlock()

	// sendEvent to manager
	sendEvent(&event, fullPath, globalSettings.ManagerAddress, globalSettings.ManagerCredentials)

	// SendEvent to all peers
	for k, v := range serverMap {
		fmt.Printf("Considering sending to: %s\n", k)
		if k != globalSettings.Name {
			fmt.Printf("sending to peer %s at %s\n", k, v.Address)
			sendEvent(&event, fullPath, v.Address, globalSettings.ManagerCredentials)
		}
	}
}

func sendEvent(event *Event, fullPath string, address string, credentials string) {
	url := "http://" + address + "/event/"
	fmt.Printf("target url: %s\n", url)

	jsonStr, _ := json.Marshal(event)
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
	req.Header.Set("Content-Type", "application/json")

	data := []byte(credentials)
	authHash := base64.StdEncoding.EncodeToString(data)
	req.Header.Add("Authorization", "Basic "+authHash)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
		return
	}

	defer resp.Body.Close()

	fmt.Println("response Status:", resp.Status)
	fmt.Println("response Headers:", resp.Header)
	body, _ := ioutil.ReadAll(resp.Body)
	fmt.Println("response Body:", string(body))

	if event.Name == "notify.Write" {
		fmt.Printf("source: notify.Write => %s\n", fullPath)
		url := "http://" + address + "/upload/"
		postFile(event.Message, fullPath, url, credentials)
	}
}

var ownership = make(map[string]Event, 100)
var ownershipLock = sync.RWMutex{}

const (
	OWNERSHIP_EXPIRATION_TIMEOUT = 5 * time.Second
)

func eventHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		json.NewEncoder(w).Encode(events)
	case "POST":
		log.Println("Got POST: ", r.Body)
		decoder := json.NewDecoder(r.Body)
		var event Event
		err := decoder.Decode(&event)
		if err != nil {
			panic("bad json body")
		}

		log.Println(event.Name + ", path: " + event.Message)
		log.Printf("Event info: %#v\n", event)

		// At this point, the other side should have ownership of this path.
		ownershipLock.Lock()
		originalEntry, exists := ownership[event.Message]
		if exists {
			log.Printf("Original ownership: %#v\n", originalEntry)
		} else {
			log.Println("No original ownership found")
		}
		ownership[event.Message] = event
		ownershipLock.Unlock()

		pathName := globalSettings.Directory + "/" + event.Message
		relativePath := event.Message

		//todo remember these events and skip the logging on the other side. Possibly use nodeID?

		switch event.Name {
		case "notify.Create":
			fmt.Printf("notify.Create: %s\n", pathName)

			server, exists := serverMap[globalSettings.Name]
			if !exists {
				panic("Unable to find server definition")
			}

			fmt.Println("eventHandler->CreatePath")
			server.storage.CreatePath(relativePath, event.IsDirectory)
			fmt.Println("eventHandler->/CreatePath")
		case "notify.Remove":
			err = os.Remove(pathName)
			if err != nil && !os.IsNotExist(err) {
				panic(fmt.Sprintf("Error deleting folder %s: %v\n", pathName, err))
			}
			fmt.Printf("notify.Remove: %s\n", pathName)
		// todo fix this to handle the two rename events to be one event
		case "notify.Rename":
			fmt.Printf("notify.Rename: %s\n", pathName)

			server, exists := serverMap[globalSettings.Name]
			if !exists {
				panic("Unable to find server definition")
			}

			fmt.Println("eventHandler->CreatePath")
			server.storage.CreatePath(relativePath, event.IsDirectory)
			fmt.Println("eventHandler->/CreatePath")

		default:
			fmt.Printf("Unknown event found, doing nothing. Event: %v\n", event)
		}

		events = append([]Event{event}, events...)
	}
}

/*
Send the folder tree from this node to another node for comparison
*/
func sendFolderTree(initialTree DirTreeMap) {
	var tree = make(DirTreeMap)
	prefixLength := 1 // the slash prefix
	for key, value := range initialTree {
		if key == "" {
			continue
		}
		fmt.Printf("key is: '%s', length is: %d\n", key, prefixLength)
		key = key[prefixLength:]
		tree[key] = value
	}

	// package up the tree
	jsonStr, err := json.Marshal(tree)
	if err != nil {
		panic(err)
	}
	buffer := bytes.NewBuffer(jsonStr)
	client := &http.Client{}

	data := []byte(globalSettings.ManagerCredentials)
	authHash := base64.StdEncoding.EncodeToString(data)

	for k, v := range serverMap {
		// don't send an update to ourselves
		if k == globalSettings.Name {
			continue
		}

		url := "http://" + v.Address + "/tree/"
		fmt.Printf("Posting folder tree to node: %s at URL: %s\n", k, url)

		req, err := http.NewRequest("POST", url, buffer)
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Authorization", "Basic "+authHash)

		resp, err := client.Do(req)
		if err != nil {
			fmt.Printf("we encountered an error!\n%s\n", err)
			//panic(err)
			continue
		}

		//fmt.Println("response Status:", resp.Status)
		//fmt.Println("response Headers:", resp.Header)
		_, _ = ioutil.ReadAll(resp.Body)
		//body, _ := ioutil.ReadAll(resp.Body)
		//fmt.Println("response Body:", string(body))
		resp.Body.Close()
	}
}

func getListOfFolders(w http.ResponseWriter) {
	listOfFileInfo, err := scanDirectoryContents()
	if err != nil {
		log.Fatal(err)
	}
	json.NewEncoder(w).Encode(listOfFileInfo)
	fmt.Printf("Sending tree of size %d back to client\n", len(listOfFileInfo))
}

func deletePaths(deletedPaths []string) {
	if globalSettings.Directory == "" {
		panic("globalSettings.Directory is not configured correctly. Aborting")
	}

	fmt.Println("Paths were deleted on the other side. Delete them here")

	fmt.Printf("Paths to delete\nBefore sort:\n%v\n", deletedPaths)
	// Reverse sort the paths so the most specific is first. This allows us to get away without a recursive delete
	sort.Sort(sort.Reverse(sort.StringSlice(deletedPaths)))
	fmt.Printf("Paths to delete after sort\nAfter sort:\n%v\n", deletedPaths)
	for _, relativePath := range deletedPaths {
		if relativePath == "" || relativePath == "/" {
			fmt.Printf("We had a request to delete the base path. Skipping: %s\n", relativePath)
			continue
		}
		fullPath := globalSettings.Directory + relativePath
		fmt.Printf("Full path is: %s\n", fullPath)

		fmt.Printf("%s: about to remove\n", fullPath)

		// stop on any error except for not exist. We are trying to delete it anyway (or rather, it should have been deleted already)
		err := os.Remove(fullPath)
		if err != nil && !os.IsNotExist(err) {
			panic(err)
		}
		fmt.Printf("%s: done removing (err = %v)\n", fullPath, err)
	}
}

func addPaths(newPaths []string) {
	fmt.Println("Paths were added on the other side. Create them here")
	for _, newPathName := range newPaths {
		fmt.Printf("pathname is: %s\n", newPathName)
		err := os.Mkdir(newPathName, os.ModeDir+os.ModePerm)
		if err != nil && !os.IsExist(err) {
			panic(err)
		}
	}
}

func folderTreeHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		getListOfFolders(w)
	case "POST":
		decoder := json.NewDecoder(r.Body)

		var remoteTreePreTranslate DirTreeMap
		err := decoder.Decode(&remoteTreePreTranslate)
		if err != nil {
			panic("bad json body")
		}

		var remoteTree = make(DirTreeMap)
		for key, value := range remoteTreePreTranslate {
			key = fmt.Sprintf("%s/%s", globalSettings.Directory, key)
			remoteTree[key] = value
		}

		// todo get the node name in here
		fmt.Printf("Received a tree map from: %s\n%s\n", r.RemoteAddr, remoteTree)

		// let's compare to the current one we have.
		_, _, newPaths, deletedPaths, _ := checkForChanges(remoteTree, nil)

		// update the paths to be consistent. Remember, everything is backwards since this is comparing the other side to us instead of two successive states
		// deletion will show up as a new path.

		fmt.Printf("about to check for deletions: %s\n", newPaths)
		// delete folders that were deleted. We delete first, then add to make sure that the old ones will not be in the way
		if len(newPaths) > 0 {
			deletePaths(newPaths)
		}

		// add folders that were created
		fmt.Printf("about to check for new folders: %s\n", deletedPaths)
		// add folders that were added.
		if len(deletedPaths) > 0 {
			addPaths(deletedPaths)
		}
	}
}

func postFile(filename string, fullPath string, targetUrl string, credentials string) error {
	body := &bytes.Buffer{}
	bodyWriter := multipart.NewWriter(body)
	fileWriter, err := bodyWriter.CreateFormFile("uploadfile", filename)
	if err != nil {
		fmt.Println("error writing to buffer")
		return err
	}

	file, err := os.Open(fullPath)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = io.Copy(fileWriter, file)
	if err != nil {
		fmt.Println("error copying file")
		return err
	}

	myHash, err := fileMd5Hash(fullPath)
	if err != nil {
		fmt.Printf("failed to calculate MD5 Hash for %s\n", fullPath)
		return err
	}

	bodyWriter.WriteField("HASH", myHash)
	contentType := bodyWriter.FormDataContentType()
	err = bodyWriter.Close()
	if err != nil {
		return err
	}

	req, err := http.NewRequest("POST", targetUrl, body)
	req.Header.Set("Content-Type", contentType)

	data := []byte(credentials)
	authHash := base64.StdEncoding.EncodeToString(data)
	req.Header.Add("Authorization", "Basic "+authHash)

	client := &http.Client{}
	resp, err := client.Do(req)
	defer resp.Body.Close()

	if err != nil {
		return err
	}

	return nil
}

func fileMd5Hash(filePath string) (string, error) {
	var returnMD5String string
	file, err := os.Open(filePath)
	if err != nil {
		return returnMD5String, err
	}
	defer file.Close()

	hash := md5.New()
	_, err = io.Copy(hash, file)
	if err != nil {
		return returnMD5String, err
	}

	hashInBytes := hash.Sum(nil)[:16]

	returnMD5String = hex.EncodeToString(hashInBytes)

	return returnMD5String, nil

}
<commit_msg>adding more logging<commit_after>// Copyright 2016 Jacob Taylor jacob@ablox.io
// License: Apache2 - http://www.apache.org/licenses/LICENSE-2.0
package main

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"sort"
	"sync"
	"time"
)

// Event stores the relevant information on events or updates to the storage layer.
type Event struct {
	Source        string
	Name          string
	Message       string
	Time          time.Time
	IsDirectory   bool
	NetworkSource string
}

var events = make([]Event, 0, 100)

// SendEvent gets events that have happened off to the peer servers so they can replicate the same change
func SendEvent(event Event, fullPath string) {
	// look back through the events for a similar event in the recent path.
	// Set the event source  (server name)
	event.Source = globalSettings.Name
	event.Time = time.Now()

	// Get the current owner of this entry if any
	ownershipLock.RLock()
	originalEntry, exists := ownership[event.Message]
	ownershipLock.RUnlock()

	if exists {
		log.Printf("Original ownership: %#v\n", originalEntry)
		timeDelta := time.Since(originalEntry.Time)
		if timeDelta > OWNERSHIP_EXPIRATION_TIMEOUT {
			log.Printf("Owndership expired. Delta is: %v\n", timeDelta)
		} else {
			// At this point, someone owns this item.
			// If we are not the owner, we are done.
			if originalEntry.Source != globalSettings.Name {
				log.Println("We do not own this. Do not send")
				return
			}
		}
	} else {
		log.Printf("Send event called. No prior owndership: %s\n", fullPath)
	}

	// At this point, it is our change and our event. Store our ownership
	ownershipLock.Lock()
	ownership[event.Message] = event
	ownershipLock.Unlock()

	// sendEvent to manager
	sendEvent(&event, fullPath, globalSettings.ManagerAddress, globalSettings.ManagerCredentials)

	// SendEvent to all peers
	for k, v := range serverMap {
		fmt.Printf("Considering sending to: %s\n", k)
		if k != globalSettings.Name {
			fmt.Printf("sending to peer %s at %s\n", k, v.Address)
			sendEvent(&event, fullPath, v.Address, globalSettings.ManagerCredentials)
		}
	}
}

func sendEvent(event *Event, fullPath string, address string, credentials string) {
	url := "http://" + address + "/event/"
	fmt.Printf("target url: %s\n", url)

	jsonStr, _ := json.Marshal(event)
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
	req.Header.Set("Content-Type", "application/json")

	data := []byte(credentials)
	authHash := base64.StdEncoding.EncodeToString(data)
	req.Header.Add("Authorization", "Basic "+authHash)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
		return
	}

	defer resp.Body.Close()

	fmt.Println("response Status:", resp.Status)
	fmt.Println("response Headers:", resp.Header)
	body, _ := ioutil.ReadAll(resp.Body)
	fmt.Println("response Body:", string(body))

	if event.Name == "notify.Write" {
		fmt.Printf("source: notify.Write => %s\n", fullPath)
		url := "http://" + address + "/upload/"
		postFile(event.Message, fullPath, url, credentials)
	}
}

var ownership = make(map[string]Event, 100)
var ownershipLock = sync.RWMutex{}

const (
	OWNERSHIP_EXPIRATION_TIMEOUT = 5 * time.Second
)

func eventHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		json.NewEncoder(w).Encode(events)
	case "POST":
		log.Println("Got POST: ", r.Body)
		decoder := json.NewDecoder(r.Body)
		var event Event
		err := decoder.Decode(&event)
		if err != nil {
			panic("bad json body")
		}

		log.Println(event.Name + ", path: " + event.Message)
		log.Printf("Event info: %#v\n", event)

		// At this point, the other side should have ownership of this path.
		ownershipLock.Lock()
		originalEntry, exists := ownership[event.Message]
		if exists {
			log.Printf("Original ownership: %#v\n", originalEntry)
		} else {
			log.Println("No original ownership found")
		}
		ownership[event.Message] = event
		ownershipLock.Unlock()

		pathName := globalSettings.Directory + "/" + event.Message
		relativePath := event.Message

		//todo remember these events and skip the logging on the other side. Possibly use nodeID?

		switch event.Name {
		case "notify.Create":
			fmt.Printf("notify.Create: %s\n", pathName)

			server, exists := serverMap[globalSettings.Name]
			if !exists {
				panic("Unable to find server definition")
			}

			fmt.Println("eventHandler->CreatePath")
			server.storage.CreatePath(relativePath, event.IsDirectory)
			fmt.Println("eventHandler->/CreatePath")
		case "notify.Remove":
			err = os.Remove(pathName)
			if err != nil && !os.IsNotExist(err) {
				panic(fmt.Sprintf("Error deleting folder %s: %v\n", pathName, err))
			}
			fmt.Printf("notify.Remove: %s\n", pathName)
		// todo fix this to handle the two rename events to be one event
		case "notify.Rename":
			fmt.Printf("notify.Rename: %s\n", pathName)

			server, exists := serverMap[globalSettings.Name]
			if !exists {
				panic("Unable to find server definition")
			}

			fmt.Println("eventHandler->CreatePath")
			server.storage.CreatePath(relativePath, event.IsDirectory)
			fmt.Println("eventHandler->/CreatePath")

		default:
			fmt.Printf("Unknown event found, doing nothing. Event: %v\n", event)
		}

		events = append([]Event{event}, events...)
	}
}

/*
Send the folder tree from this node to another node for comparison
*/
func sendFolderTree(initialTree DirTreeMap) {
	var tree = make(DirTreeMap)
	prefixLength := 1 // the slash prefix
	for key, value := range initialTree {
		if key == "" {
			continue
		}
		fmt.Printf("key is: '%s', length is: %d\n", key, prefixLength)
		key = key[prefixLength:]
		tree[key] = value
	}

	// package up the tree
	jsonStr, err := json.Marshal(tree)
	if err != nil {
		panic(err)
	}
	buffer := bytes.NewBuffer(jsonStr)
	client := &http.Client{}

	data := []byte(globalSettings.ManagerCredentials)
	authHash := base64.StdEncoding.EncodeToString(data)

	for k, v := range serverMap {
		// don't send an update to ourselves
		if k == globalSettings.Name {
			continue
		}

		url := "http://" + v.Address + "/tree/"
		fmt.Printf("Posting folder tree to node: %s at URL: %s\n", k, url)

		req, err := http.NewRequest("POST", url, buffer)
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Authorization", "Basic "+authHash)

		resp, err := client.Do(req)
		if err != nil {
			fmt.Printf("we encountered an error!\n%s\n", err)
			//panic(err)
			continue
		}

		//fmt.Println("response Status:", resp.Status)
		//fmt.Println("response Headers:", resp.Header)
		_, _ = ioutil.ReadAll(resp.Body)
		//body, _ := ioutil.ReadAll(resp.Body)
		//fmt.Println("response Body:", string(body))
		resp.Body.Close()
	}
}

func getListOfFolders(w http.ResponseWriter) {
	listOfFileInfo, err := scanDirectoryContents()
	if err != nil {
		log.Fatal(err)
	}
	json.NewEncoder(w).Encode(listOfFileInfo)
	fmt.Printf("Sending tree of size %d back to client\n", len(listOfFileInfo))
}

func deletePaths(deletedPaths []string) {
	if globalSettings.Directory == "" {
		panic("globalSettings.Directory is not configured correctly. Aborting")
	}

	fmt.Println("Paths were deleted on the other side. Delete them here")

	fmt.Printf("Paths to delete\nBefore sort:\n%v\n", deletedPaths)
	// Reverse sort the paths so the most specific is first. This allows us to get away without a recursive delete
	sort.Sort(sort.Reverse(sort.StringSlice(deletedPaths)))
	fmt.Printf("Paths to delete after sort\nAfter sort:\n%v\n", deletedPaths)
	for _, relativePath := range deletedPaths {
		if relativePath == "" || relativePath == "/" {
			fmt.Printf("We had a request to delete the base path. Skipping: %s\n", relativePath)
			continue
		}
		fullPath := globalSettings.Directory + relativePath
		fmt.Printf("Full path is: %s\n", fullPath)

		fmt.Printf("%s: about to remove\n", fullPath)

		// stop on any error except for not exist. We are trying to delete it anyway (or rather, it should have been deleted already)
		err := os.Remove(fullPath)
		if err != nil && !os.IsNotExist(err) {
			panic(err)
		}
		fmt.Printf("%s: done removing (err = %v)\n", fullPath, err)
	}
}

func addPaths(newPaths []string) {
	fmt.Println("Paths were added on the other side. Create them here")
	for _, newPathName := range newPaths {
		fmt.Printf("pathname is: %s\n", newPathName)
		err := os.Mkdir(newPathName, os.ModeDir+os.ModePerm)
		if err != nil && !os.IsExist(err) {
			panic(err)
		}
	}
}

func folderTreeHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		getListOfFolders(w)
	case "POST":
		decoder := json.NewDecoder(r.Body)

		var remoteTreePreTranslate DirTreeMap
		err := decoder.Decode(&remoteTreePreTranslate)
		if err != nil {
			panic("bad json body")
		}

		var remoteTree = make(DirTreeMap)
		for key, value := range remoteTreePreTranslate {
			key = fmt.Sprintf("%s/%s", globalSettings.Directory, key)
			remoteTree[key] = value
		}

		// todo get the node name in here
		fmt.Printf("Received a tree map from: %s\n%s\n", r.RemoteAddr, remoteTree)

		// let's compare to the current one we have.
		_, _, newPaths, deletedPaths, _ := checkForChanges(remoteTree, nil)

		// update the paths to be consistent. Remember, everything is backwards since this is comparing the other side to us instead of two successive states
		// deletion will show up as a new path.

		fmt.Printf("about to check for deletions: %s\n", newPaths)
		// delete folders that were deleted. We delete first, then add to make sure that the old ones will not be in the way
		if len(newPaths) > 0 {
			deletePaths(newPaths)
		}

		// add folders that were created
		fmt.Printf("about to check for new folders: %s\n", deletedPaths)
		// add folders that were added.
		if len(deletedPaths) > 0 {
			addPaths(deletedPaths)
		}
	}
}

func postFile(filename string, fullPath string, targetUrl string, credentials string) error {
	body := &bytes.Buffer{}
	bodyWriter := multipart.NewWriter(body)
	fileWriter, err := bodyWriter.CreateFormFile("uploadfile", filename)
	if err != nil {
		fmt.Println("error writing to buffer")
		return err
	}

	file, err := os.Open(fullPath)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = io.Copy(fileWriter, file)
	if err != nil {
		fmt.Println("error copying file")
		return err
	}

	myHash, err := fileMd5Hash(fullPath)
	if err != nil {
		fmt.Printf("failed to calculate MD5 Hash for %s\n", fullPath)
		return err
	}

	bodyWriter.WriteField("HASH", myHash)
	contentType := bodyWriter.FormDataContentType()
	err = bodyWriter.Close()
	if err != nil {
		return err
	}

	req, err := http.NewRequest("POST", targetUrl, body)
	req.Header.Set("Content-Type", contentType)

	data := []byte(credentials)
	authHash := base64.StdEncoding.EncodeToString(data)
	req.Header.Add("Authorization", "Basic "+authHash)

	client := &http.Client{}
	resp, err := client.Do(req)
	defer resp.Body.Close()

	if err != nil {
		return err
	}

	return nil
}

func fileMd5Hash(filePath string) (string, error) {
	var returnMD5String string
	file, err := os.Open(filePath)
	if err != nil {
		return returnMD5String, err
	}
	defer file.Close()

	hash := md5.New()
	_, err = io.Copy(hash, file)
	if err != nil {
		return returnMD5String, err
	}

	hashInBytes := hash.Sum(nil)[:16]

	returnMD5String = hex.EncodeToString(hashInBytes)

	return returnMD5String, nil

}
<|endoftext|>