<commit_before>package agent

import (
	"errors"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/log"
	"github.com/docker/swarmkit/protobuf/ptypes"
	"github.com/docker/swarmkit/remotes"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

const dispatcherRPCTimeout = 5 * time.Second

var (
	errSessionDisconnect = errors.New("agent: session disconnect") // instructed to disconnect
	errSessionClosed     = errors.New("agent: session closed")
)

// session encapsulates one round of registration with the manager. session
// starts the registration and heartbeat control cycle. Any failure will result
// in a complete shutdown of the session and it must be reestablished.
//
// All communication with the master is done through session.  Changes that
// flow into the agent, such as task assignment, are called back into the
// agent through errs, messages and tasks.
type session struct {
	conn *grpc.ClientConn
	addr string

	agent         *Agent
	sessionID     string
	session       api.Dispatcher_SessionClient
	errs          chan error
	messages      chan *api.SessionMessage
	assignments   chan *api.AssignmentsMessage
	subscriptions chan *api.SubscriptionMessage

	registered chan struct{} // closed registration
	closed     chan struct{}
	closeOnce  sync.Once
}

func newSession(ctx context.Context, agent *Agent, delay time.Duration, sessionID string, description *api.NodeDescription) *session {
	s := &session{
		agent:         agent,
		sessionID:     sessionID,
		errs:          make(chan error, 1),
		messages:      make(chan *api.SessionMessage),
		assignments:   make(chan *api.AssignmentsMessage),
		subscriptions: make(chan *api.SubscriptionMessage),
		registered:    make(chan struct{}),
		closed:        make(chan struct{}),
	}

	// TODO(stevvooe): Need to move connection management up a level or create
	// independent connection for log broker client.

	peer, err := agent.config.Managers.Select()
	if err != nil {
		s.errs <- err
		return s
	}
	cc, err := grpc.Dial(peer.Addr,
		grpc.WithTransportCredentials(agent.config.Credentials),
		grpc.WithTimeout(dispatcherRPCTimeout),
	)
	if err != nil {
		s.errs <- err
		return s
	}
	s.addr = peer.Addr
	s.conn = cc

	go s.run(ctx, delay, description)
	return s
}

func (s *session) run(ctx context.Context, delay time.Duration, description *api.NodeDescription) {
	timer := time.NewTimer(delay) // delay before registering.
	defer timer.Stop()
	select {
	case <-timer.C:
	case <-ctx.Done():
		return
	}

	if err := s.start(ctx, description); err != nil {
		select {
		case s.errs <- err:
		case <-s.closed:
		case <-ctx.Done():
		}
		return
	}

	ctx = log.WithLogger(ctx, log.G(ctx).WithField("session.id", s.sessionID))

	go runctx(ctx, s.closed, s.errs, s.heartbeat)
	go runctx(ctx, s.closed, s.errs, s.watch)
	go runctx(ctx, s.closed, s.errs, s.listen)
	go runctx(ctx, s.closed, s.errs, s.logSubscriptions)

	close(s.registered)
}

// start begins the session and returns the first SessionMessage.
func (s *session) start(ctx context.Context, description *api.NodeDescription) error {
	log.G(ctx).Debugf("(*session).start")

	errChan := make(chan error, 1)
	var (
		msg    *api.SessionMessage
		stream api.Dispatcher_SessionClient
		err    error
	)
	// Note: we don't defer cancellation of this context, because the
	// streaming RPC is used after this function returned. We only cancel
	// it in the timeout case to make sure the goroutine completes.
	sessionCtx, cancelSession := context.WithCancel(ctx)

	// Need to run Session in a goroutine since there's no way to set a
	// timeout for an individual Recv call in a stream.
	go func() {
		client := api.NewDispatcherClient(s.conn)

		stream, err = client.Session(sessionCtx, &api.SessionRequest{
			Description: description,
			SessionID:   s.sessionID,
		})
		if err != nil {
			errChan <- err
			return
		}

		msg, err = stream.Recv()
		errChan <- err
	}()

	select {
	case err := <-errChan:
		if err != nil {
			return err
		}
	case <-time.After(dispatcherRPCTimeout):
		cancelSession()
		return errors.New("session initiation timed out")
	}

	s.sessionID = msg.SessionID
	s.session = stream

	return s.handleSessionMessage(ctx, msg)
}

func (s *session) heartbeat(ctx context.Context) error {
	log.G(ctx).Debugf("(*session).heartbeat")
	client := api.NewDispatcherClient(s.conn)
	heartbeat := time.NewTimer(1) // send out a heartbeat right away
	defer heartbeat.Stop()

	for {
		select {
		case <-heartbeat.C:
			heartbeatCtx, cancel := context.WithTimeout(ctx, dispatcherRPCTimeout)
			resp, err := client.Heartbeat(heartbeatCtx, &api.HeartbeatRequest{
				SessionID: s.sessionID,
			})
			cancel()
			if err != nil {
				if grpc.Code(err) == codes.NotFound {
					err = errNodeNotRegistered
				}

				return err
			}

			period, err := ptypes.Duration(&resp.Period)
			if err != nil {
				return err
			}

			heartbeat.Reset(period)
		case <-s.closed:
			return errSessionClosed
		case <-ctx.Done():
			return ctx.Err()
		}
	}
}

func (s *session) listen(ctx context.Context) error {
	defer s.session.CloseSend()
	log.G(ctx).Debugf("(*session).listen")
	for {
		msg, err := s.session.Recv()
		if err != nil {
			return err
		}

		if err := s.handleSessionMessage(ctx, msg); err != nil {
			return err
		}
	}
}

func (s *session) handleSessionMessage(ctx context.Context, msg *api.SessionMessage) error {
	select {
	case s.messages <- msg:
		return nil
	case <-s.closed:
		return errSessionClosed
	case <-ctx.Done():
		return ctx.Err()
	}
}

func (s *session) logSubscriptions(ctx context.Context) error {
	log := log.G(ctx).WithFields(logrus.Fields{"method": "(*session).logSubscriptions"})
	log.Debugf("")

	client := api.NewLogBrokerClient(s.conn)
	subscriptions, err := client.ListenSubscriptions(ctx, &api.ListenSubscriptionsRequest{})
	if grpc.Code(err) == codes.Unimplemented {
		log.Warning("manager does not support log subscriptions")
		// Don't return, because returning would bounce the session
		select {
		case <-s.closed:
			return errSessionClosed
		case <-ctx.Done():
			return ctx.Err()
		}
	}
	if err != nil {
		return err
	}
	defer subscriptions.CloseSend()

	for {
		resp, err := subscriptions.Recv()
		if err != nil {
			return err
		}

		select {
		case s.subscriptions <- resp:
		case <-s.closed:
			return errSessionClosed
		case <-ctx.Done():
			return ctx.Err()
		}
	}
}

func (s *session) watch(ctx context.Context) error {
	log := log.G(ctx).WithFields(logrus.Fields{"method": "(*session).watch"})
	log.Debugf("")
	var (
		resp            *api.AssignmentsMessage
		assignmentWatch api.Dispatcher_AssignmentsClient
		tasksWatch      api.Dispatcher_TasksClient
		streamReference string
		tasksFallback   bool
		err             error
	)

	client := api.NewDispatcherClient(s.conn)
	for {
		// If this is the first time we're running the loop, or there was a reference mismatch
		// attempt to get the assignmentWatch
		if assignmentWatch == nil && !tasksFallback {
			assignmentWatch, err = client.Assignments(ctx, &api.AssignmentsRequest{SessionID: s.sessionID})
			if err != nil {
				return err
			}
		}
		// We have an assignmentWatch, let's try to receive an AssignmentMessage
		if assignmentWatch != nil {
			// If we get a code = 12 desc = unknown method Assignments, try to use tasks
			resp, err = assignmentWatch.Recv()
			if err != nil {
				if grpc.Code(err) != codes.Unimplemented {
					return err
				}
				tasksFallback = true
				assignmentWatch = nil
				log.WithError(err).Infof("falling back to Tasks")
			}
		}

		// This code is here for backwards compatibility (so that newer clients can use the
		// older method Tasks)
		if tasksWatch == nil && tasksFallback {
			tasksWatch, err = client.Tasks(ctx, &api.TasksRequest{SessionID: s.sessionID})
			if err != nil {
				return err
			}
		}
		if tasksWatch != nil {
			// When falling back to Tasks because of an old managers, we wrap the tasks in assignments.
			var taskResp *api.TasksMessage
			var assignmentChanges []*api.AssignmentChange
			taskResp, err = tasksWatch.Recv()
			if err != nil {
				return err
			}
			for _, t := range taskResp.Tasks {
				taskChange := &api.AssignmentChange{
					Assignment: &api.Assignment{
						Item: &api.Assignment_Task{
							Task: t,
						},
					},
					Action: api.AssignmentChange_AssignmentActionUpdate,
				}

				assignmentChanges = append(assignmentChanges, taskChange)
			}
			resp = &api.AssignmentsMessage{Type: api.AssignmentsMessage_COMPLETE, Changes: assignmentChanges}
		}

		// If there seems to be a gap in the stream, let's break out of the inner for and
		// re-sync (by calling Assignments again).
		if streamReference != "" && streamReference != resp.AppliesTo {
			assignmentWatch = nil
		} else {
			streamReference = resp.ResultsIn
		}

		select {
		case s.assignments <- resp:
		case <-s.closed:
			return errSessionClosed
		case <-ctx.Done():
			return ctx.Err()
		}
	}
}

// sendTaskStatus uses the current session to send the status of a single task.
func (s *session) sendTaskStatus(ctx context.Context, taskID string, status *api.TaskStatus) error {
	client := api.NewDispatcherClient(s.conn)
	if _, err := client.UpdateTaskStatus(ctx, &api.UpdateTaskStatusRequest{
		SessionID: s.sessionID,
		Updates: []*api.UpdateTaskStatusRequest_TaskStatusUpdate{
			{
				TaskID: taskID,
				Status: status,
			},
		},
	}); err != nil {
		// TODO(stevvooe): Dispatcher should not return this error. Status
		// reports for unknown tasks should be ignored.
		if grpc.Code(err) == codes.NotFound {
			return errTaskUnknown
		}

		return err
	}

	return nil
}

func (s *session) sendTaskStatuses(ctx context.Context, updates ...*api.UpdateTaskStatusRequest_TaskStatusUpdate) ([]*api.UpdateTaskStatusRequest_TaskStatusUpdate, error) {
	if len(updates) < 1 {
		return nil, nil
	}

	const batchSize = 1024
	select {
	case <-s.registered:
		select {
		case <-s.closed:
			return updates, ErrClosed
		default:
		}
	case <-s.closed:
		return updates, ErrClosed
	case <-ctx.Done():
		return updates, ctx.Err()
	}

	client := api.NewDispatcherClient(s.conn)
	n := batchSize

	if len(updates) < n {
		n = len(updates)
	}

	if _, err := client.UpdateTaskStatus(ctx, &api.UpdateTaskStatusRequest{
		SessionID: s.sessionID,
		Updates:   updates[:n],
	}); err != nil {
		log.G(ctx).WithError(err).Errorf("failed sending task status batch size of %d", len(updates[:n]))
		return updates, err
	}

	return updates[n:], nil
}

// sendError is used to send errors to errs channel and trigger session recreation
func (s *session) sendError(err error) {
	select {
	case s.errs <- err:
	case <-s.closed:
	}
}

// close closing session. It should be called only in <-session.errs branch
// of event loop.
func (s *session) close() error {
	s.closeOnce.Do(func() {
		if s.conn != nil {
			s.agent.config.Managers.ObserveIfExists(api.Peer{Addr: s.addr}, -remotes.DefaultObservationWeight)
			s.conn.Close()
		}

		close(s.closed)
	})

	return nil
}
<commit_msg>agent: Correct fix for pre-LogBroker managers<commit_after>package agent

import (
	"errors"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/log"
	"github.com/docker/swarmkit/protobuf/ptypes"
	"github.com/docker/swarmkit/remotes"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

const dispatcherRPCTimeout = 5 * time.Second

var (
	errSessionDisconnect = errors.New("agent: session disconnect") // instructed to disconnect
	errSessionClosed     = errors.New("agent: session closed")
)

// session encapsulates one round of registration with the manager. session
// starts the registration and heartbeat control cycle. Any failure will result
// in a complete shutdown of the session and it must be reestablished.
//
// All communication with the master is done through session.  Changes that
// flow into the agent, such as task assignment, are called back into the
// agent through errs, messages and tasks.
type session struct {
	conn *grpc.ClientConn
	addr string

	agent         *Agent
	sessionID     string
	session       api.Dispatcher_SessionClient
	errs          chan error
	messages      chan *api.SessionMessage
	assignments   chan *api.AssignmentsMessage
	subscriptions chan *api.SubscriptionMessage

	registered chan struct{} // closed registration
	closed     chan struct{}
	closeOnce  sync.Once
}

func newSession(ctx context.Context, agent *Agent, delay time.Duration, sessionID string, description *api.NodeDescription) *session {
	s := &session{
		agent:         agent,
		sessionID:     sessionID,
		errs:          make(chan error, 1),
		messages:      make(chan *api.SessionMessage),
		assignments:   make(chan *api.AssignmentsMessage),
		subscriptions: make(chan *api.SubscriptionMessage),
		registered:    make(chan struct{}),
		closed:        make(chan struct{}),
	}

	// TODO(stevvooe): Need to move connection management up a level or create
	// independent connection for log broker client.

	peer, err := agent.config.Managers.Select()
	if err != nil {
		s.errs <- err
		return s
	}
	cc, err := grpc.Dial(peer.Addr,
		grpc.WithTransportCredentials(agent.config.Credentials),
		grpc.WithTimeout(dispatcherRPCTimeout),
	)
	if err != nil {
		s.errs <- err
		return s
	}
	s.addr = peer.Addr
	s.conn = cc

	go s.run(ctx, delay, description)
	return s
}

func (s *session) run(ctx context.Context, delay time.Duration, description *api.NodeDescription) {
	timer := time.NewTimer(delay) // delay before registering.
	defer timer.Stop()
	select {
	case <-timer.C:
	case <-ctx.Done():
		return
	}

	if err := s.start(ctx, description); err != nil {
		select {
		case s.errs <- err:
		case <-s.closed:
		case <-ctx.Done():
		}
		return
	}

	ctx = log.WithLogger(ctx, log.G(ctx).WithField("session.id", s.sessionID))

	go runctx(ctx, s.closed, s.errs, s.heartbeat)
	go runctx(ctx, s.closed, s.errs, s.watch)
	go runctx(ctx, s.closed, s.errs, s.listen)
	go runctx(ctx, s.closed, s.errs, s.logSubscriptions)

	close(s.registered)
}

// start begins the session and returns the first SessionMessage.
func (s *session) start(ctx context.Context, description *api.NodeDescription) error {
	log.G(ctx).Debugf("(*session).start")

	errChan := make(chan error, 1)
	var (
		msg    *api.SessionMessage
		stream api.Dispatcher_SessionClient
		err    error
	)
	// Note: we don't defer cancellation of this context, because the
	// streaming RPC is used after this function returned. We only cancel
	// it in the timeout case to make sure the goroutine completes.
	sessionCtx, cancelSession := context.WithCancel(ctx)

	// Need to run Session in a goroutine since there's no way to set a
	// timeout for an individual Recv call in a stream.
	go func() {
		client := api.NewDispatcherClient(s.conn)

		stream, err = client.Session(sessionCtx, &api.SessionRequest{
			Description: description,
			SessionID:   s.sessionID,
		})
		if err != nil {
			errChan <- err
			return
		}

		msg, err = stream.Recv()
		errChan <- err
	}()

	select {
	case err := <-errChan:
		if err != nil {
			return err
		}
	case <-time.After(dispatcherRPCTimeout):
		cancelSession()
		return errors.New("session initiation timed out")
	}

	s.sessionID = msg.SessionID
	s.session = stream

	return s.handleSessionMessage(ctx, msg)
}

func (s *session) heartbeat(ctx context.Context) error {
	log.G(ctx).Debugf("(*session).heartbeat")
	client := api.NewDispatcherClient(s.conn)
	heartbeat := time.NewTimer(1) // send out a heartbeat right away
	defer heartbeat.Stop()

	for {
		select {
		case <-heartbeat.C:
			heartbeatCtx, cancel := context.WithTimeout(ctx, dispatcherRPCTimeout)
			resp, err := client.Heartbeat(heartbeatCtx, &api.HeartbeatRequest{
				SessionID: s.sessionID,
			})
			cancel()
			if err != nil {
				if grpc.Code(err) == codes.NotFound {
					err = errNodeNotRegistered
				}

				return err
			}

			period, err := ptypes.Duration(&resp.Period)
			if err != nil {
				return err
			}

			heartbeat.Reset(period)
		case <-s.closed:
			return errSessionClosed
		case <-ctx.Done():
			return ctx.Err()
		}
	}
}

func (s *session) listen(ctx context.Context) error {
	defer s.session.CloseSend()
	log.G(ctx).Debugf("(*session).listen")
	for {
		msg, err := s.session.Recv()
		if err != nil {
			return err
		}

		if err := s.handleSessionMessage(ctx, msg); err != nil {
			return err
		}
	}
}

func (s *session) handleSessionMessage(ctx context.Context, msg *api.SessionMessage) error {
	select {
	case s.messages <- msg:
		return nil
	case <-s.closed:
		return errSessionClosed
	case <-ctx.Done():
		return ctx.Err()
	}
}

func (s *session) logSubscriptions(ctx context.Context) error {
	log := log.G(ctx).WithFields(logrus.Fields{"method": "(*session).logSubscriptions"})
	log.Debugf("")

	client := api.NewLogBrokerClient(s.conn)
	subscriptions, err := client.ListenSubscriptions(ctx, &api.ListenSubscriptionsRequest{})
	if err != nil {
		return err
	}
	defer subscriptions.CloseSend()

	for {
		resp, err := subscriptions.Recv()
		if grpc.Code(err) == codes.Unimplemented {
			log.Warning("manager does not support log subscriptions")
			// Don't return, because returning would bounce the session
			select {
			case <-s.closed:
				return errSessionClosed
			case <-ctx.Done():
				return ctx.Err()
			}
		}
		if err != nil {
			return err
		}

		select {
		case s.subscriptions <- resp:
		case <-s.closed:
			return errSessionClosed
		case <-ctx.Done():
			return ctx.Err()
		}
	}
}

func (s *session) watch(ctx context.Context) error {
	log := log.G(ctx).WithFields(logrus.Fields{"method": "(*session).watch"})
	log.Debugf("")
	var (
		resp            *api.AssignmentsMessage
		assignmentWatch api.Dispatcher_AssignmentsClient
		tasksWatch      api.Dispatcher_TasksClient
		streamReference string
		tasksFallback   bool
		err             error
	)

	client := api.NewDispatcherClient(s.conn)
	for {
		// If this is the first time we're running the loop, or there was a reference mismatch
		// attempt to get the assignmentWatch
		if assignmentWatch == nil && !tasksFallback {
			assignmentWatch, err = client.Assignments(ctx, &api.AssignmentsRequest{SessionID: s.sessionID})
			if err != nil {
				return err
			}
		}
		// We have an assignmentWatch, let's try to receive an AssignmentMessage
		if assignmentWatch != nil {
			// If we get a code = 12 desc = unknown method Assignments, try to use tasks
			resp, err = assignmentWatch.Recv()
			if err != nil {
				if grpc.Code(err) != codes.Unimplemented {
					return err
				}
				tasksFallback = true
				assignmentWatch = nil
				log.WithError(err).Infof("falling back to Tasks")
			}
		}

		// This code is here for backwards compatibility (so that newer clients can use the
		// older method Tasks)
		if tasksWatch == nil && tasksFallback {
			tasksWatch, err = client.Tasks(ctx, &api.TasksRequest{SessionID: s.sessionID})
			if err != nil {
				return err
			}
		}
		if tasksWatch != nil {
			// When falling back to Tasks because of an old managers, we wrap the tasks in assignments.
			var taskResp *api.TasksMessage
			var assignmentChanges []*api.AssignmentChange
			taskResp, err = tasksWatch.Recv()
			if err != nil {
				return err
			}
			for _, t := range taskResp.Tasks {
				taskChange := &api.AssignmentChange{
					Assignment: &api.Assignment{
						Item: &api.Assignment_Task{
							Task: t,
						},
					},
					Action: api.AssignmentChange_AssignmentActionUpdate,
				}

				assignmentChanges = append(assignmentChanges, taskChange)
			}
			resp = &api.AssignmentsMessage{Type: api.AssignmentsMessage_COMPLETE, Changes: assignmentChanges}
		}

		// If there seems to be a gap in the stream, let's break out of the inner for and
		// re-sync (by calling Assignments again).
		if streamReference != "" && streamReference != resp.AppliesTo {
			assignmentWatch = nil
		} else {
			streamReference = resp.ResultsIn
		}

		select {
		case s.assignments <- resp:
		case <-s.closed:
			return errSessionClosed
		case <-ctx.Done():
			return ctx.Err()
		}
	}
}

// sendTaskStatus uses the current session to send the status of a single task.
func (s *session) sendTaskStatus(ctx context.Context, taskID string, status *api.TaskStatus) error {
	client := api.NewDispatcherClient(s.conn)
	if _, err := client.UpdateTaskStatus(ctx, &api.UpdateTaskStatusRequest{
		SessionID: s.sessionID,
		Updates: []*api.UpdateTaskStatusRequest_TaskStatusUpdate{
			{
				TaskID: taskID,
				Status: status,
			},
		},
	}); err != nil {
		// TODO(stevvooe): Dispatcher should not return this error. Status
		// reports for unknown tasks should be ignored.
		if grpc.Code(err) == codes.NotFound {
			return errTaskUnknown
		}

		return err
	}

	return nil
}

func (s *session) sendTaskStatuses(ctx context.Context, updates ...*api.UpdateTaskStatusRequest_TaskStatusUpdate) ([]*api.UpdateTaskStatusRequest_TaskStatusUpdate, error) {
	if len(updates) < 1 {
		return nil, nil
	}

	const batchSize = 1024
	select {
	case <-s.registered:
		select {
		case <-s.closed:
			return updates, ErrClosed
		default:
		}
	case <-s.closed:
		return updates, ErrClosed
	case <-ctx.Done():
		return updates, ctx.Err()
	}

	client := api.NewDispatcherClient(s.conn)
	n := batchSize

	if len(updates) < n {
		n = len(updates)
	}

	if _, err := client.UpdateTaskStatus(ctx, &api.UpdateTaskStatusRequest{
		SessionID: s.sessionID,
		Updates:   updates[:n],
	}); err != nil {
		log.G(ctx).WithError(err).Errorf("failed sending task status batch size of %d", len(updates[:n]))
		return updates, err
	}

	return updates[n:], nil
}

// sendError is used to send errors to errs channel and trigger session recreation
func (s *session) sendError(err error) {
	select {
	case s.errs <- err:
	case <-s.closed:
	}
}

// close closing session. It should be called only in <-session.errs branch
// of event loop.
func (s *session) close() error {
	s.closeOnce.Do(func() {
		if s.conn != nil {
			s.agent.config.Managers.ObserveIfExists(api.Peer{Addr: s.addr}, -remotes.DefaultObservationWeight)
			s.conn.Close()
		}

		close(s.closed)
	})

	return nil
}
<|endoftext|>


<commit_before>package agent

// You can overridden buildVersion at compile time by using:
//
//  go run -ldflags "-X github.com/buildkite/agent/agent.buildVersion abc" *.go --version
//
// On CI, the binaries are always build with the buildVersion variable set.

var baseVersion string = "2.1"
var buildVersion string = ""

func Version() string {
	return baseVersion
}

func BuildVersion() string {
	if buildVersion != "" {
		return buildVersion
	} else {
		return "x"
	}
}
<commit_msg>Bumped agent to 2.1.1<commit_after>package agent

// You can overridden buildVersion at compile time by using:
//
//  go run -ldflags "-X github.com/buildkite/agent/agent.buildVersion abc" *.go --version
//
// On CI, the binaries are always build with the buildVersion variable set.

var baseVersion string = "2.1.1"
var buildVersion string = ""

func Version() string {
	return baseVersion
}

func BuildVersion() string {
	if buildVersion != "" {
		return buildVersion
	} else {
		return "x"
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/Zenika/MARCEL/backend/agenda"
	"github.com/Zenika/MARCEL/backend/weather"
	"github.com/Zenika/MARCEL/backend/twitter"
	"github.com/Zenika/MARCEL/backend/medias"
	"github.com/gorilla/mux"
	"github.com/rs/cors"
	"net/http"
	"log"
	"os"
	"strings"
)

//todo : service to create a new media
//todo : service to log with jwt (or at least a token based system)
//todo : service to delete a media
//todo : service to save an existing media

var logFile string = os.Getenv("MARCEL_LOG_FILE")
//current version of the API
const MARCEL_API_VERSION = "1"

func main() {

	if len(logFile) == 0 {
		logFile = "marcel.log"
	}
	f, err := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	log.SetOutput(f)

	//Load plugins list from DB
	//plugins.LoadPluginsCatalog()

	//Load Medias configuration from DB
	medias.LoadMedias()

	//Set API services
	c := cors.New(cors.Options{
		AllowedOrigins: []string{"*"},
		// AllowedOrigins:   []string{"http://localhost:*"},
		AllowedMethods:   []string{"GET", "POST", "DELETE", "OPTION", "PUT"},
		AllowCredentials: true,
	})

	r := mux.NewRouter()
	s := r.PathPrefix("/api/v" + MARCEL_API_VERSION).Subrouter()
	s.HandleFunc("/weather/forecast/{nbForecasts:[0-9]+}", weather.GetForecastWeatherHandler).Methods("GET")
	s.HandleFunc("/agenda/incoming/{nbEvents:[0-9]*}", agenda.GetNextEvents).Methods("GET")
	s.HandleFunc("/twitter/timeline/{nbTweets:[0-9]*}", twitter.GetTimeline).Methods("GET")
	s.HandleFunc("/medias/{idMedia:[0-9]*}", medias.HandleGetMedia).Methods("GET")
	s.HandleFunc("/medias", medias.HandleGetAll).Methods("GET")

	handler := c.Handler(r)

	ExposeAPI(r)

	http.ListenAndServe(":8090", handler)
	log.Printf("Server is started and listening on port %v", 8090)
}

func ExposeAPI(r *mux.Router) {
	//export API to the log
	r.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
		t, err := route.GetPathTemplate()
		if err != nil {
			return err
		}
		// p will contain regular expression is compatible with regular expression in Perl, Python, and other languages.
		// for instance the regular expression for path '/articles/{id}' will be '^/articles/(?P<v0>[^/]+)$'
		p, err := route.GetPathRegexp()
		if err != nil {
			return err
		}
		m, err := route.GetMethods()
		if err != nil {
			return err
		}
		log.Println(strings.Join(m, ","), t, p)
		return nil
	})
}
<commit_msg>[Add] Endpoint to create a new media<commit_after>package main

import (
	"github.com/Zenika/MARCEL/backend/agenda"
	"github.com/Zenika/MARCEL/backend/weather"
	"github.com/Zenika/MARCEL/backend/twitter"
	"github.com/Zenika/MARCEL/backend/medias"
	"github.com/gorilla/mux"
	"github.com/rs/cors"
	"net/http"
	"log"
	"os"
	"strings"
)

//todo : service to create a new media
//todo : service to log with jwt (or at least a token based system)
//todo : service to delete a media
//todo : service to save an existing media

var logFile string = os.Getenv("MARCEL_LOG_FILE")
//current version of the API
const MARCEL_API_VERSION = "1"

func main() {

	if len(logFile) == 0 {
		logFile = "marcel.log"
	}
	f, err := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	log.SetOutput(f)

	//Load plugins list from DB
	//plugins.LoadPluginsCatalog()

	//Load Medias configuration from DB
	medias.LoadMedias()

	//Set API services
	c := cors.New(cors.Options{
		AllowedOrigins: []string{"*"},
		// AllowedOrigins:   []string{"http://localhost:*"},
		AllowedMethods:   []string{"GET", "POST", "DELETE", "OPTION", "PUT"},
		AllowCredentials: true,
	})

	r := mux.NewRouter()
	s := r.PathPrefix("/api/v" + MARCEL_API_VERSION).Subrouter()
	s.HandleFunc("/weather/forecast/{nbForecasts:[0-9]+}", weather.GetForecastWeatherHandler).Methods("GET")
	s.HandleFunc("/agenda/incoming/{nbEvents:[0-9]*}", agenda.GetNextEvents).Methods("GET")
	s.HandleFunc("/twitter/timeline/{nbTweets:[0-9]*}", twitter.GetTimeline).Methods("GET")
	s.HandleFunc("/medias/{idMedia:[0-9]*}", medias.HandleGetMedia).Methods("GET")
	s.HandleFunc("/medias", medias.HandleGetAll).Methods("GET")
	s.HandleFunc("/medias/create", medias.HandleCreate).Methods("GET")

	handler := c.Handler(r)

	ExposeAPI(r)

	http.ListenAndServe(":8090", handler)
	log.Printf("Server is started and listening on port %v", 8090)
}

func ExposeAPI(r *mux.Router) {
	//export API to the log
	r.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
		t, err := route.GetPathTemplate()
		if err != nil {
			return err
		}
		// p will contain regular expression is compatible with regular expression in Perl, Python, and other languages.
		// for instance the regular expression for path '/articles/{id}' will be '^/articles/(?P<v0>[^/]+)$'
		p, err := route.GetPathRegexp()
		if err != nil {
			return err
		}
		m, err := route.GetMethods()
		if err != nil {
			return err
		}
		log.Println(strings.Join(m, ","), t, p)
		return nil
	})
}
<|endoftext|>


<commit_before>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package backend

import (
	//	"bytes"
	"code.google.com/p/log4go"
	"fmt"
	//	"github.com/quarnster/parser"
	//	"io/ioutil"
	//	"lime/backend/loaders"
	. "github.com/quarnster/util/text"
	"lime/backend/parser"
	"lime/backend/render"
	"lime/backend/textmate"
	. "lime/backend/util"
	"reflect"
	"runtime/debug"
	//	"sort"
	"strings"
	"sync"
)

type (
	// A View provides a view into a specific underlying buffer
	// with its own set of selections, settings, viewport, etc.
	// Multiple Views can share the same underlying data buffer.
	View struct {
		HasSettings
		HasId
		name        string
		window      *Window
		buffer      Buffer
		selection   RegionSet
		undoStack   UndoStack
		scratch     bool
		overwrite   bool
		cursyntax   string
		syntax      parser.SyntaxHighlighter
		regions     render.ViewRegionMap
		editstack   []*Edit
		lock        sync.Mutex
		modified    bool
		reparseChan chan parseReq
	}
	parseReq struct {
		forced bool
	}

	// The Edit object is an internal type passed as an argument
	// to a TextCommand. All text operations need to be associated
	// with a valid Edit object.
	Edit struct {
		invalid    bool
		composite  CompositeAction
		savedSel   RegionSet
		savedCount int
		command    string
		args       Args
		v          *View
		bypassUndo bool
	}
)

func newView(w *Window) *View {
	ret := &View{window: w, regions: make(render.ViewRegionMap)}
	ret.Settings().AddOnChange("lime.view.syntax", func() {
		ret.lock.Lock()
		defer ret.lock.Unlock()
		syn, _ := ret.Settings().Get("syntax", "").(string)
		if syn != ret.cursyntax {
			ret.cursyntax = syn
			defer ret.reparse(true)
		}
	})

	ret.reparseChan = make(chan parseReq, 32)
	go ret.parsethread()
	ret.Settings().Set("is_widget", false)
	return ret
}

func newEdit(v *View) *Edit {
	ret := &Edit{
		v:          v,
		savedCount: v.buffer.ChangeCount(),
	}
	for _, r := range v.Sel().Regions() {
		ret.savedSel.Add(r)
	}
	return ret
}

func (e *Edit) String() string {
	return fmt.Sprintf("%s: %v, %v, %v", e.command, e.args, e.bypassUndo, e.composite)
}

func (e *Edit) Apply() {
	e.composite.Apply()
}

func (e *Edit) Undo() {
	e.composite.Undo()
	e.v.Sel().Clear()
	for _, r := range e.savedSel.Regions() {
		e.v.Sel().Add(r)
	}
}

func (v *View) setBuffer(b Buffer) error {
	if v.buffer != nil {
		return fmt.Errorf("There is already a buffer set")
	}
	v.buffer = b
	// TODO(q): Dynamically load the correct syntax file
	b.AddCallback(func(_ Buffer, position, delta int) {
		v.flush(position, delta)
	})
	return nil
}

func (v *View) flush(position, delta int) {
	func() {
		v.lock.Lock()
		defer v.lock.Unlock()

		e := Prof.Enter("view.flush")
		defer e.Exit()
		v.selection.Adjust(position, delta)
		if v.syntax != nil {
			v.syntax.Adjust(position, delta)
		}
		for k, v2 := range v.regions {
			v2.Regions.Adjust(position, delta)
			v.regions[k] = v2
		}
	}()
	OnModified.Call(v)
	v.reparse(false)
}

func (v *View) parsethread() {
	pc := 0
	lastParse := -1
	doparse := func() {
		p := Prof.Enter("syntax.parse")
		defer p.Exit()
		defer func() {
			if r := recover(); r != nil {
				log4go.Error("Panic in parse thread: %v\n%s", r, string(debug.Stack()))
				if pc > 0 {
					panic(r)
				}
				pc++
			}
		}()
		b := v.Buffer()
		b.Lock()
		sub := b.Substr(Region{0, b.Size()})
		b.Unlock()
		source, _ := v.Settings().Get("syntax", "").(string)
		if len(source) == 0 {
			return
		}
		// TODO
		pr, err := textmate.NewLanguageParser(source, sub)
		if err != nil {
			log4go.Error("Couldn't parse: %v", err)
			return
		}
		syn, err := parser.NewSyntaxHighlighter(pr)
		if err != nil {
			log4go.Error("Couldn't create syntaxhighlighter: %v", err)
			return
		}
		v.lock.Lock()
		defer v.lock.Unlock()
		// Only set if it isn't invalid already, otherwise the
		// current syntax highlighting will be more accurate
		// as it will have had incremental adjustments done to it
		if v.buffer.ChangeCount() != lastParse {
			return
		}
		v.syntax = syn
		for k := range v.regions {
			if strings.HasPrefix(k, "lime.syntax") {
				delete(v.regions, k)
			}
		}
		for k, v2 := range syn.Flatten() {
			if v2.Regions.HasNonEmpty() {
				v.regions[k] = v2
			}
		}
	}
	for pr := range v.reparseChan {
		if cc := v.buffer.ChangeCount(); lastParse != cc || pr.forced {
			lastParse = cc
			doparse()
			v.Settings().Set("lime.syntax.updated", lastParse)
		}
	}
}

func (v *View) reparse(forced bool) {
	if len(v.reparseChan) < cap(v.reparseChan) || forced {
		v.reparseChan <- parseReq{forced}
	}
}

func (v *View) ScopeName(point int) string {
	v.lock.Lock()
	defer v.lock.Unlock()
	if v.syntax != nil {
		return v.syntax.ScopeName(point)
	}
	return ""
}

func (v *View) ExtractScope(point int) Region {
	v.lock.Lock()
	defer v.lock.Unlock()
	if v.syntax != nil {
		return v.syntax.ScopeExtent(point)
	}
	return Region{}
}

func (v *View) ScoreSelector(point int, selector string) int {
	if sn := v.syntax.ScopeName(point); len(sn) > 0 {
		return 1 + strings.Index(sn, selector)
	}
	return 0
}

func (v *View) Sel() *RegionSet {
	return &v.selection
}

func (v *View) Window() *Window {
	return v.window
}

func (v *View) Buffer() Buffer {
	return v.buffer
}

func (v *View) Insert(edit *Edit, point int, value string) int {
	if t, ok := v.Settings().Get("translate_tabs_to_spaces", false).(bool); ok && t && strings.Contains(value, "\t") {
		tab_size, ok := v.Settings().Get("tab_size", 4).(int)
		if !ok {
			tab_size = 4
		}

		lines := strings.Split(value, "\n")
		for i, li := range lines {
			for {
				idx := strings.Index(li, "\t")
				if idx == -1 {
					break
				}
				ai := idx
				if i == 0 {
					_, col := v.buffer.RowCol(point)
					ai = col + 1
				}
				add := 1 + ((ai + (tab_size - 1)) &^ (tab_size - 1))
				spaces := ""
				for j := ai; j < add; j++ {
					spaces += " "
				}
				li = li[:idx] + spaces + li[idx+1:]
			}
			lines[i] = li
		}
		value = strings.Join(lines, "\n")
	}
	edit.composite.AddExec(NewInsertAction(v.buffer, point, value))
	return len(value)
}

func (v *View) Erase(edit *Edit, r Region) {
	edit.composite.AddExec(NewEraseAction(v.buffer, r))
}

func (v *View) Replace(edit *Edit, r Region, value string) {
	edit.composite.AddExec(NewReplaceAction(v.buffer, r, value))
}

func (v *View) BeginEdit() *Edit {
	e := newEdit(v)
	v.editstack = append(v.editstack, e)
	return e
}

func (v *View) EndEdit(e *Edit) {
	if e.invalid {
		log4go.Fine("This edit has already been invalidated: %v, %v", e, v.editstack)
		return
	}
	i := len(v.editstack) - 1
	for i := len(v.editstack) - 1; i >= 0; i-- {
		if v.editstack[i] == e {
			break
		}
	}
	if i == -1 {
		log4go.Error("This edit isn't even in the stack... where did it come from? %v, %v", e, v.editstack)
		return
	}

	var selmod bool

	if l := len(v.editstack) - 1; i != l {
		log4go.Error("This edit wasn't last in the stack... %d !=  %d: %v, %v", i, l, e, v.editstack)
	}
	for j := len(v.editstack) - 1; j >= i; j-- {
		ce := v.editstack[j]
		ce.invalid = true
		is := reflect.DeepEqual(*v.Sel(), ce.savedSel)
		ib := v.buffer.ChangeCount() == ce.savedCount
		eq := (is && ib && ce.composite.Len() == 0)
		if !eq && is {
			selmod = true
		}
		if v.scratch || ce.bypassUndo || eq {
			continue
		}
		if i == 0 || j != i {
			// Presume someone forgot to add it in the j != i case
			v.undoStack.Add(e)
		} else {
			// This edit belongs to another edit
			v.editstack[i-1].composite.Add(ce)
		}
	}
	v.editstack = v.editstack[:i]
	if selmod {
		OnSelectionModified.Call(v)
	}
}

func (v *View) SetScratch(s bool) {
	v.scratch = s
}

func (v *View) IsScratch() bool {
	return v.scratch
}

func (v *View) OverwriteStatus() bool {
	return v.overwrite
}

func (v *View) SetOverwriteStatus(s bool) {
	v.overwrite = s
}

func (v *View) CommandHistory(idx int, modifying_only bool) (name string, args Args, count int) {
	// TODO: merge history when possible
	if i := v.undoStack.index(idx, modifying_only); i != -1 {
		e := v.undoStack.actions[i]
		return e.command, e.args, 1
	}
	return "", nil, 0
}

func (v *View) runCommand(cmd TextCommand, name string) error {
	e := v.BeginEdit()
	e.command = name
	//	e.args = args
	e.bypassUndo = cmd.BypassUndo()

	defer func() {
		v.EndEdit(e)
		if r := recover(); r != nil {
			log4go.Error("Paniced while running text command %s %v: %v\n%s", name, cmd, r, string(debug.Stack()))
		}
	}()
	p := Prof.Enter("view.cmd." + name)
	defer p.Exit()
	return cmd.Run(v, e)
}

func (v *View) AddRegions(key string, regions []Region, scope, icon string, flags render.ViewRegionFlags) {
	vr := render.ViewRegions{Scope: scope, Icon: icon, Flags: flags}
	vr.Regions.AddAll(regions)

	v.lock.Lock()
	defer v.lock.Unlock()
	v.regions[key] = vr
}

func (v *View) GetRegions(key string) (ret []Region) {
	v.lock.Lock()
	defer v.lock.Unlock()
	vr := v.regions[key]
	rs := vr.Regions.Regions()
	ret = make([]Region, len(rs))
	copy(ret, rs)
	return
}

func (v *View) EraseRegions(key string) {
	v.lock.Lock()
	defer v.lock.Unlock()
	delete(v.regions, key)
}

func (v *View) UndoStack() *UndoStack {
	return &v.undoStack
}

func (v *View) Transform(scheme render.ColourScheme, viewport Region) render.Recipe {
	v.lock.Lock()
	defer v.lock.Unlock()
	if v.syntax == nil {
		return nil
	}
	rr := make(render.ViewRegionMap)
	for k, v := range v.regions {
		rr[k] = v.Clone()
	}
	rs := render.ViewRegions{Flags: render.SELECTION}
	rs.Regions.AddAll(v.selection.Regions())
	rr["lime.selection"] = rs
	return render.Transform(scheme, rr, viewport)
}
<commit_msg>Modify EndEdit to use a more descriptive var name<commit_after>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package backend

import (
	//	"bytes"
	"code.google.com/p/log4go"
	"fmt"
	//	"github.com/quarnster/parser"
	//	"io/ioutil"
	//	"lime/backend/loaders"
	. "github.com/quarnster/util/text"
	"lime/backend/parser"
	"lime/backend/render"
	"lime/backend/textmate"
	. "lime/backend/util"
	"reflect"
	"runtime/debug"
	//	"sort"
	"strings"
	"sync"
)

type (
	// A View provides a view into a specific underlying buffer
	// with its own set of selections, settings, viewport, etc.
	// Multiple Views can share the same underlying data buffer.
	View struct {
		HasSettings
		HasId
		name        string
		window      *Window
		buffer      Buffer
		selection   RegionSet
		undoStack   UndoStack
		scratch     bool
		overwrite   bool
		cursyntax   string
		syntax      parser.SyntaxHighlighter
		regions     render.ViewRegionMap
		editstack   []*Edit
		lock        sync.Mutex
		modified    bool
		reparseChan chan parseReq
	}
	parseReq struct {
		forced bool
	}

	// The Edit object is an internal type passed as an argument
	// to a TextCommand. All text operations need to be associated
	// with a valid Edit object.
	Edit struct {
		invalid    bool
		composite  CompositeAction
		savedSel   RegionSet
		savedCount int
		command    string
		args       Args
		v          *View
		bypassUndo bool
	}
)

func newView(w *Window) *View {
	ret := &View{window: w, regions: make(render.ViewRegionMap)}
	ret.Settings().AddOnChange("lime.view.syntax", func() {
		ret.lock.Lock()
		defer ret.lock.Unlock()
		syn, _ := ret.Settings().Get("syntax", "").(string)
		if syn != ret.cursyntax {
			ret.cursyntax = syn
			defer ret.reparse(true)
		}
	})

	ret.reparseChan = make(chan parseReq, 32)
	go ret.parsethread()
	ret.Settings().Set("is_widget", false)
	return ret
}

func newEdit(v *View) *Edit {
	ret := &Edit{
		v:          v,
		savedCount: v.buffer.ChangeCount(),
	}
	for _, r := range v.Sel().Regions() {
		ret.savedSel.Add(r)
	}
	return ret
}

func (e *Edit) String() string {
	return fmt.Sprintf("%s: %v, %v, %v", e.command, e.args, e.bypassUndo, e.composite)
}

func (e *Edit) Apply() {
	e.composite.Apply()
}

func (e *Edit) Undo() {
	e.composite.Undo()
	e.v.Sel().Clear()
	for _, r := range e.savedSel.Regions() {
		e.v.Sel().Add(r)
	}
}

func (v *View) setBuffer(b Buffer) error {
	if v.buffer != nil {
		return fmt.Errorf("There is already a buffer set")
	}
	v.buffer = b
	// TODO(q): Dynamically load the correct syntax file
	b.AddCallback(func(_ Buffer, position, delta int) {
		v.flush(position, delta)
	})
	return nil
}

func (v *View) flush(position, delta int) {
	func() {
		v.lock.Lock()
		defer v.lock.Unlock()

		e := Prof.Enter("view.flush")
		defer e.Exit()
		v.selection.Adjust(position, delta)
		if v.syntax != nil {
			v.syntax.Adjust(position, delta)
		}
		for k, v2 := range v.regions {
			v2.Regions.Adjust(position, delta)
			v.regions[k] = v2
		}
	}()
	OnModified.Call(v)
	v.reparse(false)
}

func (v *View) parsethread() {
	pc := 0
	lastParse := -1
	doparse := func() {
		p := Prof.Enter("syntax.parse")
		defer p.Exit()
		defer func() {
			if r := recover(); r != nil {
				log4go.Error("Panic in parse thread: %v\n%s", r, string(debug.Stack()))
				if pc > 0 {
					panic(r)
				}
				pc++
			}
		}()
		b := v.Buffer()
		b.Lock()
		sub := b.Substr(Region{0, b.Size()})
		b.Unlock()
		source, _ := v.Settings().Get("syntax", "").(string)
		if len(source) == 0 {
			return
		}
		// TODO
		pr, err := textmate.NewLanguageParser(source, sub)
		if err != nil {
			log4go.Error("Couldn't parse: %v", err)
			return
		}
		syn, err := parser.NewSyntaxHighlighter(pr)
		if err != nil {
			log4go.Error("Couldn't create syntaxhighlighter: %v", err)
			return
		}
		v.lock.Lock()
		defer v.lock.Unlock()
		// Only set if it isn't invalid already, otherwise the
		// current syntax highlighting will be more accurate
		// as it will have had incremental adjustments done to it
		if v.buffer.ChangeCount() != lastParse {
			return
		}
		v.syntax = syn
		for k := range v.regions {
			if strings.HasPrefix(k, "lime.syntax") {
				delete(v.regions, k)
			}
		}
		for k, v2 := range syn.Flatten() {
			if v2.Regions.HasNonEmpty() {
				v.regions[k] = v2
			}
		}
	}
	for pr := range v.reparseChan {
		if cc := v.buffer.ChangeCount(); lastParse != cc || pr.forced {
			lastParse = cc
			doparse()
			v.Settings().Set("lime.syntax.updated", lastParse)
		}
	}
}

func (v *View) reparse(forced bool) {
	if len(v.reparseChan) < cap(v.reparseChan) || forced {
		v.reparseChan <- parseReq{forced}
	}
}

func (v *View) ScopeName(point int) string {
	v.lock.Lock()
	defer v.lock.Unlock()
	if v.syntax != nil {
		return v.syntax.ScopeName(point)
	}
	return ""
}

func (v *View) ExtractScope(point int) Region {
	v.lock.Lock()
	defer v.lock.Unlock()
	if v.syntax != nil {
		return v.syntax.ScopeExtent(point)
	}
	return Region{}
}

func (v *View) ScoreSelector(point int, selector string) int {
	if sn := v.syntax.ScopeName(point); len(sn) > 0 {
		return 1 + strings.Index(sn, selector)
	}
	return 0
}

func (v *View) Sel() *RegionSet {
	return &v.selection
}

func (v *View) Window() *Window {
	return v.window
}

func (v *View) Buffer() Buffer {
	return v.buffer
}

func (v *View) Insert(edit *Edit, point int, value string) int {
	if t, ok := v.Settings().Get("translate_tabs_to_spaces", false).(bool); ok && t && strings.Contains(value, "\t") {
		tab_size, ok := v.Settings().Get("tab_size", 4).(int)
		if !ok {
			tab_size = 4
		}

		lines := strings.Split(value, "\n")
		for i, li := range lines {
			for {
				idx := strings.Index(li, "\t")
				if idx == -1 {
					break
				}
				ai := idx
				if i == 0 {
					_, col := v.buffer.RowCol(point)
					ai = col + 1
				}
				add := 1 + ((ai + (tab_size - 1)) &^ (tab_size - 1))
				spaces := ""
				for j := ai; j < add; j++ {
					spaces += " "
				}
				li = li[:idx] + spaces + li[idx+1:]
			}
			lines[i] = li
		}
		value = strings.Join(lines, "\n")
	}
	edit.composite.AddExec(NewInsertAction(v.buffer, point, value))
	return len(value)
}

func (v *View) Erase(edit *Edit, r Region) {
	edit.composite.AddExec(NewEraseAction(v.buffer, r))
}

func (v *View) Replace(edit *Edit, r Region, value string) {
	edit.composite.AddExec(NewReplaceAction(v.buffer, r, value))
}

func (v *View) BeginEdit() *Edit {
	e := newEdit(v)
	v.editstack = append(v.editstack, e)
	return e
}

func (v *View) EndEdit(edit *Edit) {
	if edit.invalid {
		log4go.Fine("This edit has already been invalidated: %v, %v", edit, v.editstack)
		return
	}
	i := len(v.editstack) - 1
	for i := len(v.editstack) - 1; i >= 0; i-- {
		if v.editstack[i] == edit {
			break
		}
	}
	if i == -1 {
		log4go.Error("This edit isn't even in the stack... where did it come from? %v, %v", edit, v.editstack)
		return
	}

	var selmod bool

	if l := len(v.editstack) - 1; i != l {
		log4go.Error("This edit wasn't last in the stack... %d !=  %d: %v, %v", i, l, edit, v.editstack)
	}
	for j := len(v.editstack) - 1; j >= i; j-- {
		ce := v.editstack[j]
		ce.invalid = true
		is := reflect.DeepEqual(*v.Sel(), ce.savedSel)
		ib := v.buffer.ChangeCount() == ce.savedCount
		eq := (is && ib && ce.composite.Len() == 0)
		if !eq && is {
			selmod = true
		}
		if v.scratch || ce.bypassUndo || eq {
			continue
		}
		if i == 0 || j != i {
			// Presume someone forgot to add it in the j != i case
			v.undoStack.Add(edit)
		} else {
			// This edit belongs to another edit
			v.editstack[i-1].composite.Add(ce)
		}
	}
	v.editstack = v.editstack[:i]
	if selmod {
		OnSelectionModified.Call(v)
	}
}

func (v *View) SetScratch(s bool) {
	v.scratch = s
}

func (v *View) IsScratch() bool {
	return v.scratch
}

func (v *View) OverwriteStatus() bool {
	return v.overwrite
}

func (v *View) SetOverwriteStatus(s bool) {
	v.overwrite = s
}

func (v *View) CommandHistory(idx int, modifying_only bool) (name string, args Args, count int) {
	// TODO: merge history when possible
	if i := v.undoStack.index(idx, modifying_only); i != -1 {
		e := v.undoStack.actions[i]
		return e.command, e.args, 1
	}
	return "", nil, 0
}

func (v *View) runCommand(cmd TextCommand, name string) error {
	e := v.BeginEdit()
	e.command = name
	//	e.args = args
	e.bypassUndo = cmd.BypassUndo()

	defer func() {
		v.EndEdit(e)
		if r := recover(); r != nil {
			log4go.Error("Paniced while running text command %s %v: %v\n%s", name, cmd, r, string(debug.Stack()))
		}
	}()
	p := Prof.Enter("view.cmd." + name)
	defer p.Exit()
	return cmd.Run(v, e)
}

func (v *View) AddRegions(key string, regions []Region, scope, icon string, flags render.ViewRegionFlags) {
	vr := render.ViewRegions{Scope: scope, Icon: icon, Flags: flags}
	vr.Regions.AddAll(regions)

	v.lock.Lock()
	defer v.lock.Unlock()
	v.regions[key] = vr
}

func (v *View) GetRegions(key string) (ret []Region) {
	v.lock.Lock()
	defer v.lock.Unlock()
	vr := v.regions[key]
	rs := vr.Regions.Regions()
	ret = make([]Region, len(rs))
	copy(ret, rs)
	return
}

func (v *View) EraseRegions(key string) {
	v.lock.Lock()
	defer v.lock.Unlock()
	delete(v.regions, key)
}

func (v *View) UndoStack() *UndoStack {
	return &v.undoStack
}

func (v *View) Transform(scheme render.ColourScheme, viewport Region) render.Recipe {
	v.lock.Lock()
	defer v.lock.Unlock()
	if v.syntax == nil {
		return nil
	}
	rr := make(render.ViewRegionMap)
	for k, v := range v.regions {
		rr[k] = v.Clone()
	}
	rs := render.ViewRegions{Flags: render.SELECTION}
	rs.Regions.AddAll(v.selection.Regions())
	rr["lime.selection"] = rs
	return render.Transform(scheme, rr, viewport)
}
<|endoftext|>


<commit_before>package erasurecode

import "bytes"
import "math/rand"
import "testing"

var validParams = []ErasureCodeParams{
	{Name: "liberasurecode_rs_vand", K: 2, M: 1},
	{Name: "liberasurecode_rs_vand", K: 10, M: 4},
	{Name: "liberasurecode_rs_vand", K: 4, M: 3},
	{Name: "liberasurecode_rs_vand", K: 8, M: 4},
	{Name: "liberasurecode_rs_vand", K: 15, M: 4},
	{Name: "isa_l_rs_vand", K: 2, M: 1},
	{Name: "isa_l_rs_vand", K: 10, M: 4},
	{Name: "isa_l_rs_vand", K: 4, M: 3},
	{Name: "isa_l_rs_vand", K: 8, M: 4},
	{Name: "isa_l_rs_vand", K: 15, M: 4},
	{Name: "isa_l_rs_cauchy", K: 2, M: 1},
	{Name: "isa_l_rs_cauchy", K: 10, M: 4},
	{Name: "isa_l_rs_cauchy", K: 4, M: 3},
	{Name: "isa_l_rs_cauchy", K: 8, M: 4},
	{Name: "isa_l_rs_cauchy", K: 15, M: 4},
	{Name: "jerasure_rs_vand", K: 2, M: 1},
	{Name: "jerasure_rs_vand", K: 10, M: 4},
	{Name: "jerasure_rs_vand", K: 4, M: 3},
	{Name: "jerasure_rs_vand", K: 8, M: 4},
	{Name: "jerasure_rs_vand", K: 15, M: 4},
	{Name: "jerasure_rs_cauchy", K: 2, M: 1},
	{Name: "jerasure_rs_cauchy", K: 10, M: 4},
	{Name: "jerasure_rs_cauchy", K: 4, M: 3},
	{Name: "jerasure_rs_cauchy", K: 8, M: 4},
	{Name: "jerasure_rs_cauchy", K: 15, M: 4, W: 5},
}

var testPatterns = [][]byte{
	bytes.Repeat([]byte{0x00}, 1),
	bytes.Repeat([]byte{0xff}, 1),
	bytes.Repeat([]byte{0x00}, 1<<10),
	bytes.Repeat([]byte{0xff}, 1<<10),
	bytes.Repeat([]byte{0x00}, 1<<20),
	bytes.Repeat([]byte{0xff}, 1<<20),
	bytes.Repeat([]byte{0xf0, 0x0f}, 512),
	bytes.Repeat([]byte{0xde, 0xad, 0xbe, 0xef}, 256),
	bytes.Repeat([]byte{0xaa}, 1024),
	bytes.Repeat([]byte{0x55}, 1024),
}

func shuf(src [][]byte) [][]byte {
	dest := make([][]byte, len(src))
	perm := rand.Perm(len(src))
	for i, v := range perm {
		dest[v] = src[i]
	}
	return dest
}

func TestInitBackend(t *testing.T) {
	for _, params := range validParams {
		backend, err := InitBackend(params)
		if err != nil {
			t.Errorf("%q", err)
			continue
		}
		if backend.libec_desc <= 0 {
			t.Errorf("Expected backend descriptor > 0, got %d", backend.libec_desc)
		}

		if err = backend.Close(); err != nil {
			t.Errorf("%q", err)
		}
		if err = backend.Close(); err == nil {
			t.Errorf("Expected error when closing an already-closed backend.")
		}
	}
}

func TestInitBackendFailure(t *testing.T) {
	cases := []struct {
		params ErasureCodeParams
		want   string
	}{
		{ErasureCodeParams{Name: "liberasurecode_rs_vand", K: -1, M: 1},
			"instance_create() returned EINVALIDPARAMS"},
		{ErasureCodeParams{Name: "liberasurecode_rs_vand", K: 10, M: -1},
			"instance_create() returned EINVALIDPARAMS"},
		{ErasureCodeParams{Name: "non-existent-backend", K: 10, M: 4},
			"unsupported backend \"non-existent-backend\""},
		{ErasureCodeParams{Name: "", K: 10, M: 4},
			"unsupported backend \"\""},
		{ErasureCodeParams{Name: "liberasurecode_rs_vand", K: 20, M: 20},
			"instance_create() returned EINVALIDPARAMS"},
	}
	for _, args := range cases {
		backend, err := InitBackend(args.params)
		if err == nil {
			t.Errorf("Expected error when calling InitBackend(%v)",
				args.params)
			backend.Close()
			continue
		}
		if err.Error() != args.want {
			t.Errorf("InitBackend(%v) produced error %q, want %q",
				args.params, err, args.want)
		}
		if backend.libec_desc != 0 {
			t.Errorf("InitBackend(%v) produced backend with descriptor %v, want 0",
				args.params, backend.libec_desc)
			backend.Close()
		}
	}
}

func TestEncodeDecode(t *testing.T) {
	for _, params := range validParams {
		if !BackendIsAvailable(params.Name) {
			continue
		}
		backend, err := InitBackend(params)
		if err != nil {
			t.Errorf("Error creating backend %v: %q", params, err)
			continue
		}
		for patternIndex, pattern := range testPatterns {
			frags, err := backend.Encode(pattern)
			if err != nil {
				t.Errorf("Error encoding %v: %q", params, err)
				break
			}

			decode := func(frags [][]byte, description string) bool {
				data, err := backend.Decode(frags)
				if err != nil {
					t.Errorf("%v: %v: %q for pattern %d", description, backend, err, patternIndex)
					return false
				} else if bytes.Compare(data, pattern) != 0 {
					t.Errorf("%v: Expected %v to roundtrip pattern %d, got %q", description, backend, patternIndex, data)
					return false
				}
				return true
			}

			var good bool
			good = decode(frags, "all frags")
			good = good && decode(shuf(frags), "all frags, shuffled")
			good = good && decode(frags[:params.K], "data frags")
			good = good && decode(shuf(frags[:params.K]), "shuffled data frags")
			good = good && decode(frags[params.M:], "with parity frags")
			good = good && decode(shuf(frags[params.M:]), "shuffled parity frags")

			if !good {
				break
			}
		}

		if _, err := backend.Decode([][]byte{}); err == nil {
			t.Errorf("Expected error when decoding from empty fragment array")
		}

		err = backend.Close()
		if err != nil {
			t.Errorf("Error closing backend %v: %q", backend, err)
		}
	}
}

func TestReconstruct(t *testing.T) {
	for _, params := range validParams {
		if !BackendIsAvailable(params.Name) {
			continue
		}
		backend, err := InitBackend(params)
		if err != nil {
			t.Errorf("Error creating backend %v: %q", params, err)
			backend.Close()
			continue
		}
		for patternIndex, pattern := range testPatterns {
			frags, err := backend.Encode(pattern)
			if err != nil {
				t.Errorf("Error encoding %v: %q", params, err)
			}

			reconstruct := func(recon_frags [][]byte, frag_index int, description string) bool {
				data, err := backend.Reconstruct(recon_frags, frag_index)
				if err != nil {
					t.Errorf("%v: %v: %q for pattern %d", description, backend, err, patternIndex)
					return false
				} else if bytes.Compare(data, frags[frag_index]) != 0 {
					t.Errorf("%v: Expected %v to roundtrip pattern %d, got %q", description, backend, patternIndex, data)
					return false
				}
				return true
			}

			var good bool
			good = reconstruct(shuf(frags[:params.K]), params.K+params.M-1, "last frag from data frags")
			good = good && reconstruct(shuf(frags[params.M:]), 0, "first frag with parity frags")
			if !good {
				break
			}
		}

		if _, err := backend.Reconstruct([][]byte{}, 0); err == nil {
			t.Errorf("Expected error when reconstructing from empty fragment array")
		}

		err = backend.Close()
		if err != nil {
			t.Errorf("Error closing backend %v: %q", backend, err)
		}
	}
}

func TestIsInvalidFragment(t *testing.T) {
	for _, params := range validParams {
		if !BackendIsAvailable(params.Name) {
			continue
		}
		backend, err := InitBackend(params)
		if err != nil {
			t.Errorf("Error creating backend %v: %q", params, err)
			backend.Close()
			continue
		}
		for patternIndex, pattern := range testPatterns {
			frags, err := backend.Encode(pattern)
			if err != nil {
				t.Errorf("Error encoding %v: %q", params, err)
				continue
			}
			for index, frag := range frags {
				if backend.IsInvalidFragment(frag) {
					t.Errorf("%v: frag %v unexpectedly invalid for pattern %d", backend, index, patternIndex)
				}

				// corrupt the frag
				corruptedByte := rand.Intn(len(frag))
				for 71 <= corruptedByte && corruptedByte < 80 {
					// in the alignment padding -- try again
					corruptedByte = rand.Intn(len(frag))
				}
				frag[corruptedByte] ^= 0xff
				if !backend.IsInvalidFragment(frag) {
					t.Errorf("%v: frag %v unexpectedly valid after inverting byte %d for pattern %d", backend, index, corruptedByte, patternIndex)
				}
				frag[corruptedByte] ^= 0xff
				frag[corruptedByte] += 1
				if !backend.IsInvalidFragment(frag) {
					t.Errorf("%v: frag %v unexpectedly valid after incrementing byte %d for pattern %d", backend, index, corruptedByte, patternIndex)
				}
				frag[corruptedByte] -= 2
				if corruptedByte >= 63 && corruptedByte < 67 && frag[corruptedByte] != 0xff {
					if backend.IsInvalidFragment(frag) {
						t.Errorf("%v: frag %v unexpectedly invalid after decrementing version byte %d for pattern %d", backend, index, corruptedByte, patternIndex)
					}
				} else {
					if !backend.IsInvalidFragment(frag) {
						t.Errorf("%v: frag %v unexpectedly valid after decrementing byte %d for pattern %d", backend, index, corruptedByte, patternIndex)
					}
				}
			}
		}
		err = backend.Close()
		if err != nil {
			t.Errorf("Error closing backend %v: %q", backend, err)
		}
	}
}

func TestBackendIsAvailable(t *testing.T) {
	required_backends := []string{
		"null",
		"flat_xor_hd",
		"liberasurecode_rs_vand",
	}
	optional_backends := []string{
		"isa_l_rs_vand",
		"isa_l_rs_cauchy",
		"jerasure_rs_vand",
		"jerasure_rs_cauchy",
		"shss",
		"libphazr",
	}
	for _, name := range required_backends {
		if !BackendIsAvailable(name) {
			t.Fatalf("%v is not available", name)
		}
	}
	for _, name := range optional_backends {
		if !BackendIsAvailable(name) {
			t.Logf("%v is not available", name)
		}
	}
}
<commit_msg>Expiriment more with what types of corruptions are detected<commit_after>package erasurecode

import "bytes"
import "math/rand"
import "testing"

var validParams = []ErasureCodeParams{
	{Name: "liberasurecode_rs_vand", K: 2, M: 1},
	{Name: "liberasurecode_rs_vand", K: 10, M: 4},
	{Name: "liberasurecode_rs_vand", K: 4, M: 3},
	{Name: "liberasurecode_rs_vand", K: 8, M: 4},
	{Name: "liberasurecode_rs_vand", K: 15, M: 4},
	{Name: "isa_l_rs_vand", K: 2, M: 1},
	{Name: "isa_l_rs_vand", K: 10, M: 4},
	{Name: "isa_l_rs_vand", K: 4, M: 3},
	{Name: "isa_l_rs_vand", K: 8, M: 4},
	{Name: "isa_l_rs_vand", K: 15, M: 4},
	{Name: "isa_l_rs_cauchy", K: 2, M: 1},
	{Name: "isa_l_rs_cauchy", K: 10, M: 4},
	{Name: "isa_l_rs_cauchy", K: 4, M: 3},
	{Name: "isa_l_rs_cauchy", K: 8, M: 4},
	{Name: "isa_l_rs_cauchy", K: 15, M: 4},
	{Name: "jerasure_rs_vand", K: 2, M: 1},
	{Name: "jerasure_rs_vand", K: 10, M: 4},
	{Name: "jerasure_rs_vand", K: 4, M: 3},
	{Name: "jerasure_rs_vand", K: 8, M: 4},
	{Name: "jerasure_rs_vand", K: 15, M: 4},
	{Name: "jerasure_rs_cauchy", K: 2, M: 1},
	{Name: "jerasure_rs_cauchy", K: 10, M: 4},
	{Name: "jerasure_rs_cauchy", K: 4, M: 3},
	{Name: "jerasure_rs_cauchy", K: 8, M: 4},
	{Name: "jerasure_rs_cauchy", K: 15, M: 4, W: 5},
}

var testPatterns = [][]byte{
	bytes.Repeat([]byte{0x00}, 1),
	bytes.Repeat([]byte{0xff}, 1),
	bytes.Repeat([]byte{0x00}, 1<<10),
	bytes.Repeat([]byte{0xff}, 1<<10),
	bytes.Repeat([]byte{0x00}, 1<<20),
	bytes.Repeat([]byte{0xff}, 1<<20),
	bytes.Repeat([]byte{0xf0, 0x0f}, 512),
	bytes.Repeat([]byte{0xde, 0xad, 0xbe, 0xef}, 256),
	bytes.Repeat([]byte{0xaa}, 1024),
	bytes.Repeat([]byte{0x55}, 1024),
}

func shuf(src [][]byte) [][]byte {
	dest := make([][]byte, len(src))
	perm := rand.Perm(len(src))
	for i, v := range perm {
		dest[v] = src[i]
	}
	return dest
}

func TestInitBackend(t *testing.T) {
	for _, params := range validParams {
		backend, err := InitBackend(params)
		if err != nil {
			t.Errorf("%q", err)
			continue
		}
		if backend.libec_desc <= 0 {
			t.Errorf("Expected backend descriptor > 0, got %d", backend.libec_desc)
		}

		if err = backend.Close(); err != nil {
			t.Errorf("%q", err)
		}
		if err = backend.Close(); err == nil {
			t.Errorf("Expected error when closing an already-closed backend.")
		}
	}
}

func TestInitBackendFailure(t *testing.T) {
	cases := []struct {
		params ErasureCodeParams
		want   string
	}{
		{ErasureCodeParams{Name: "liberasurecode_rs_vand", K: -1, M: 1},
			"instance_create() returned EINVALIDPARAMS"},
		{ErasureCodeParams{Name: "liberasurecode_rs_vand", K: 10, M: -1},
			"instance_create() returned EINVALIDPARAMS"},
		{ErasureCodeParams{Name: "non-existent-backend", K: 10, M: 4},
			"unsupported backend \"non-existent-backend\""},
		{ErasureCodeParams{Name: "", K: 10, M: 4},
			"unsupported backend \"\""},
		{ErasureCodeParams{Name: "liberasurecode_rs_vand", K: 20, M: 20},
			"instance_create() returned EINVALIDPARAMS"},
	}
	for _, args := range cases {
		backend, err := InitBackend(args.params)
		if err == nil {
			t.Errorf("Expected error when calling InitBackend(%v)",
				args.params)
			backend.Close()
			continue
		}
		if err.Error() != args.want {
			t.Errorf("InitBackend(%v) produced error %q, want %q",
				args.params, err, args.want)
		}
		if backend.libec_desc != 0 {
			t.Errorf("InitBackend(%v) produced backend with descriptor %v, want 0",
				args.params, backend.libec_desc)
			backend.Close()
		}
	}
}

func TestEncodeDecode(t *testing.T) {
	for _, params := range validParams {
		if !BackendIsAvailable(params.Name) {
			continue
		}
		backend, err := InitBackend(params)
		if err != nil {
			t.Errorf("Error creating backend %v: %q", params, err)
			continue
		}
		for patternIndex, pattern := range testPatterns {
			frags, err := backend.Encode(pattern)
			if err != nil {
				t.Errorf("Error encoding %v: %q", params, err)
				break
			}

			decode := func(frags [][]byte, description string) bool {
				data, err := backend.Decode(frags)
				if err != nil {
					t.Errorf("%v: %v: %q for pattern %d", description, backend, err, patternIndex)
					return false
				} else if bytes.Compare(data, pattern) != 0 {
					t.Errorf("%v: Expected %v to roundtrip pattern %d, got %q", description, backend, patternIndex, data)
					return false
				}
				return true
			}

			var good bool
			good = decode(frags, "all frags")
			good = good && decode(shuf(frags), "all frags, shuffled")
			good = good && decode(frags[:params.K], "data frags")
			good = good && decode(shuf(frags[:params.K]), "shuffled data frags")
			good = good && decode(frags[params.M:], "with parity frags")
			good = good && decode(shuf(frags[params.M:]), "shuffled parity frags")

			if !good {
				break
			}
		}

		if _, err := backend.Decode([][]byte{}); err == nil {
			t.Errorf("Expected error when decoding from empty fragment array")
		}

		err = backend.Close()
		if err != nil {
			t.Errorf("Error closing backend %v: %q", backend, err)
		}
	}
}

func TestReconstruct(t *testing.T) {
	for _, params := range validParams {
		if !BackendIsAvailable(params.Name) {
			continue
		}
		backend, err := InitBackend(params)
		if err != nil {
			t.Errorf("Error creating backend %v: %q", params, err)
			backend.Close()
			continue
		}
		for patternIndex, pattern := range testPatterns {
			frags, err := backend.Encode(pattern)
			if err != nil {
				t.Errorf("Error encoding %v: %q", params, err)
			}

			reconstruct := func(recon_frags [][]byte, frag_index int, description string) bool {
				data, err := backend.Reconstruct(recon_frags, frag_index)
				if err != nil {
					t.Errorf("%v: %v: %q for pattern %d", description, backend, err, patternIndex)
					return false
				} else if bytes.Compare(data, frags[frag_index]) != 0 {
					t.Errorf("%v: Expected %v to roundtrip pattern %d, got %q", description, backend, patternIndex, data)
					return false
				}
				return true
			}

			var good bool
			good = reconstruct(shuf(frags[:params.K]), params.K+params.M-1, "last frag from data frags")
			good = good && reconstruct(shuf(frags[params.M:]), 0, "first frag with parity frags")
			if !good {
				break
			}
		}

		if _, err := backend.Reconstruct([][]byte{}, 0); err == nil {
			t.Errorf("Expected error when reconstructing from empty fragment array")
		}

		err = backend.Close()
		if err != nil {
			t.Errorf("Error closing backend %v: %q", backend, err)
		}
	}
}

func TestIsInvalidFragment(t *testing.T) {
	for _, params := range validParams {
		if !BackendIsAvailable(params.Name) {
			continue
		}
		backend, err := InitBackend(params)
		if err != nil {
			t.Errorf("Error creating backend %v: %q", params, err)
			backend.Close()
			continue
		}
		for patternIndex, pattern := range testPatterns {
			frags, err := backend.Encode(pattern)
			if err != nil {
				t.Errorf("Error encoding %v: %q", params, err)
				continue
			}
			for index, frag := range frags {
				if backend.IsInvalidFragment(frag) {
					t.Errorf("%v: frag %v unexpectedly invalid for pattern %d", backend, index, patternIndex)
				}
				fragCopy := make([]byte, len(frag))
				copy(fragCopy, frag)

				// corrupt the frag
				corruptedByte := rand.Intn(len(frag))
				for 71 <= corruptedByte && corruptedByte < 80 {
					// in the alignment padding -- try again
					corruptedByte = rand.Intn(len(frag))
				}
				frag[corruptedByte] ^= 0xff
				if !backend.IsInvalidFragment(frag) {
					t.Errorf("%v: frag %v unexpectedly valid after inverting byte %d for pattern %d", backend, index, corruptedByte, patternIndex)
				}
				if corruptedByte < 4 || 8 <= corruptedByte && corruptedByte <= 59 {
					/** corruption is in metadata; claim we were created by a version of
					 *  libec that predates metadata checksums. Note that
					 *  Note that a corrupted fragment size (bytes 4-7) will lead to a
					 *  segfault when we try to verify the fragment -- there's a reason
					 *  we added metadata checksums!
					 */
					copy(frag[63:67], []byte{9, 1, 1, 0})
					if 20 <= corruptedByte && corruptedByte <= 53 {
						/** Corrupted data checksum type or data checksum
						 *  We may or may not detect this type of error; in particular,
						 *      - if data checksum type is not in ec_checksum_type_t,
						 *        it is ignored
						 *      - if data checksum is mangled, we may still be valid
						 *        under the "alternative" CRC32; this seems more likely
						 *        with the byte inversion when the data is short
						 *  Either way, though, clearing the checksum type should make
						 *  it pass.
						 */
						frag[20] = 0
						if backend.IsInvalidFragment(frag) {
							t.Errorf("%v: frag %v unexpectedly invalid after clearing metadata crc and disabling data crc", backend, index)
						}
					} else if corruptedByte >= 54 || 0 <= corruptedByte && corruptedByte < 4 {
						/** Some corruptions of some bytes are still detectable. Since we're
						 *  inverting the byte, we can detect:
						 *      - frag index -- bytes 0-3
						 *      - data checksum type -- byte 20
						 *      - data checksum mismatch -- byte 54
						 *      - backend id -- byte 55
						 *      - backend version -- bytes 56-59
						 */
						if !backend.IsInvalidFragment(frag) {
							t.Errorf("%v: frag %v unexpectedly still valid after clearing metadata crc", backend, index)
						}
					} else {
						if backend.IsInvalidFragment(frag) {
							t.Errorf("%v: frag %v unexpectedly invalid after clearing metadata crc", backend, index)
						}
					}
				} else if corruptedByte >= 67 {
					copy(frag[20:25], []byte{1, 0, 0, 0, 0})
					// And since we've changed the metadata, roll back version as above...
					copy(frag[63:67], []byte{9, 1, 1, 0})
					if backend.IsInvalidFragment(frag) {
						t.Errorf("%v: frag %v unexpectedly invalid after clearing data crc", backend, index)
						t.FailNow()
					}
				}
				frag[corruptedByte] ^= 0xff
				copy(frag[63:67], fragCopy[63:67])
				copy(frag[20:25], fragCopy[20:25])

				if bytes.Compare(frag, fragCopy) != 0 {
					for i, orig := range fragCopy {
						if frag[i] != orig {
							t.Logf("%v != %v at index %v", frag[i], orig, i)
						}
					}
					t.Fatal(corruptedByte, frag, fragCopy)
				}

				frag[corruptedByte] += 1
				if !backend.IsInvalidFragment(frag) {
					t.Errorf("%v: frag %v unexpectedly valid after incrementing byte %d for pattern %d", backend, index, corruptedByte, patternIndex)
				}
				frag[corruptedByte] -= 2
				if corruptedByte >= 63 && corruptedByte < 67 && frag[corruptedByte] != 0xff {
					if backend.IsInvalidFragment(frag) {
						t.Errorf("%v: frag %v unexpectedly invalid after decrementing version byte %d for pattern %d", backend, index, corruptedByte, patternIndex)
					}
				} else {
					if !backend.IsInvalidFragment(frag) {
						t.Errorf("%v: frag %v unexpectedly valid after decrementing byte %d for pattern %d", backend, index, corruptedByte, patternIndex)
					}
				}
			}
		}
		err = backend.Close()
		if err != nil {
			t.Errorf("Error closing backend %v: %q", backend, err)
		}
	}
}

func TestBackendIsAvailable(t *testing.T) {
	required_backends := []string{
		"null",
		"flat_xor_hd",
		"liberasurecode_rs_vand",
	}
	optional_backends := []string{
		"isa_l_rs_vand",
		"isa_l_rs_cauchy",
		"jerasure_rs_vand",
		"jerasure_rs_cauchy",
		"shss",
		"libphazr",
	}
	for _, name := range required_backends {
		if !BackendIsAvailable(name) {
			t.Fatalf("%v is not available", name)
		}
	}
	for _, name := range optional_backends {
		if !BackendIsAvailable(name) {
			t.Logf("%v is not available", name)
		}
	}
}
<|endoftext|>


<commit_before>package rpc

import (
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"sync"
	"time"
)

// MuxConn is a connection that can be used bi-directionally for RPC. Normally,
// Go RPC only allows client-to-server connections. This allows the client
// to actually act as a server as well.
//
// MuxConn works using a fairly dumb multiplexing technique of simply
// framing every piece of data sent into a prefix + data format. Streams
// are established using a subset of the TCP protocol. Only a subset is
// necessary since we assume ordering on the underlying RWC.
type MuxConn struct {
	rwc     io.ReadWriteCloser
	streams map[byte]*Stream
	mu      sync.RWMutex
	wlock   sync.Mutex
}

type muxPacketType byte

const (
	muxPacketSyn muxPacketType = iota
	muxPacketAck
	muxPacketFin
	muxPacketData
)

func NewMuxConn(rwc io.ReadWriteCloser) *MuxConn {
	m := &MuxConn{
		rwc:     rwc,
		streams: make(map[byte]*Stream),
	}

	go m.loop()

	return m
}

// Close closes the underlying io.ReadWriteCloser. This will also close
// all streams that are open.
func (m *MuxConn) Close() error {
	m.mu.Lock()
	defer m.mu.Unlock()

	// Close all the streams
	for _, w := range m.streams {
		w.Close()
	}
	m.streams = make(map[byte]*Stream)

	return m.rwc.Close()
}

// Accept accepts a multiplexed connection with the given ID. This
// will block until a request is made to connect.
func (m *MuxConn) Accept(id byte) (io.ReadWriteCloser, error) {
	stream, err := m.openStream(id)
	if err != nil {
		return nil, err
	}

	// If the stream isn't closed, then it is already open somehow
	stream.mu.Lock()
	if stream.state != streamStateSynRecv && stream.state != streamStateClosed {
		stream.mu.Unlock()
		return nil, fmt.Errorf("Stream already open in bad state: %d", stream.state)
	}

	if stream.state == streamStateSynRecv {
		// Fast track establishing since we already got the syn
		stream.setState(streamStateEstablished)
		stream.mu.Unlock()
	}

	if stream.state != streamStateEstablished {
		// Go into the listening state
		stream.setState(streamStateListen)
		stream.mu.Unlock()

		// Wait for the connection to establish
	ACCEPT_ESTABLISH_LOOP:
		for {
			time.Sleep(50 * time.Millisecond)
			stream.mu.Lock()
			switch stream.state {
			case streamStateListen:
				stream.mu.Unlock()
			case streamStateEstablished:
				stream.mu.Unlock()
				break ACCEPT_ESTABLISH_LOOP
			default:
				defer stream.mu.Unlock()
				return nil, fmt.Errorf("Stream went to bad state: %d", stream.state)
			}
		}
	}

	// Send the ack down
	if _, err := m.write(stream.id, muxPacketAck, nil); err != nil {
		return nil, err
	}

	return stream, nil
}

// Dial opens a connection to the remote end using the given stream ID.
// An Accept on the remote end will only work with if the IDs match.
func (m *MuxConn) Dial(id byte) (io.ReadWriteCloser, error) {
	stream, err := m.openStream(id)
	if err != nil {
		return nil, err
	}

	// If the stream isn't closed, then it is already open somehow
	stream.mu.Lock()
	if stream.state != streamStateClosed {
		stream.mu.Unlock()
		return nil, fmt.Errorf("Stream already open in bad state: %d", stream.state)
	}

	// Open a connection
	if _, err := m.write(stream.id, muxPacketSyn, nil); err != nil {
		return nil, err
	}
	stream.setState(streamStateSynSent)
	stream.mu.Unlock()

	for {
		time.Sleep(50 * time.Millisecond)
		stream.mu.Lock()
		switch stream.state {
		case streamStateSynSent:
			stream.mu.Unlock()
		case streamStateEstablished:
			stream.mu.Unlock()
			return stream, nil
		default:
			defer stream.mu.Unlock()
			return nil, fmt.Errorf("Stream went to bad state: %d", stream.state)
		}
	}
}

func (m *MuxConn) openStream(id byte) (*Stream, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	if stream, ok := m.streams[id]; ok {
		return stream, nil
	}

	// Create the stream object and channel where data will be sent to
	dataR, dataW := io.Pipe()

	// Set the data channel so we can write to it.
	stream := &Stream{
		id:     id,
		mux:    m,
		reader: dataR,
		writer: dataW,
	}
	stream.setState(streamStateClosed)

	m.streams[id] = stream
	return m.streams[id], nil
}

func (m *MuxConn) loop() {
	defer m.Close()

	var id byte
	var packetType muxPacketType
	var length int32
	for {
		if err := binary.Read(m.rwc, binary.BigEndian, &id); err != nil {
			log.Printf("[ERR] Error reading stream ID: %s", err)
			return
		}
		if err := binary.Read(m.rwc, binary.BigEndian, &packetType); err != nil {
			log.Printf("[ERR] Error reading packet type: %s", err)
			return
		}
		if err := binary.Read(m.rwc, binary.BigEndian, &length); err != nil {
			log.Printf("[ERR] Error reading length: %s", err)
			return
		}

		// TODO(mitchellh): probably would be better to re-use a buffer...
		data := make([]byte, length)
		if length > 0 {
			if _, err := m.rwc.Read(data); err != nil {
				log.Printf("[ERR] Error reading data: %s", err)
				return
			}
		}

		stream, err := m.openStream(id)
		if err != nil {
			log.Printf("[ERR] Error opening stream %d: %s", id, err)
			return
		}

		switch packetType {
		case muxPacketAck:
			stream.mu.Lock()
			if stream.state == streamStateSynSent {
				stream.setState(streamStateEstablished)
			} else {
				log.Printf("[ERR] Ack received for stream in state: %d", stream.state)
			}
			stream.mu.Unlock()
		case muxPacketSyn:
			stream.mu.Lock()
			switch stream.state {
			case streamStateClosed:
				stream.setState(streamStateSynRecv)
			case streamStateListen:
				stream.setState(streamStateEstablished)
			default:
				log.Printf("[ERR] Syn received for stream in state: %d", stream.state)
			}
			stream.mu.Unlock()
		case muxPacketFin:
			stream.mu.Lock()
			stream.setState(streamStateClosed)
			stream.writer.Close()
			stream.mu.Unlock()

			m.mu.Lock()
			delete(m.streams, stream.id)
			m.mu.Unlock()
		case muxPacketData:
			stream.mu.Lock()
			if stream.state == streamStateEstablished {
				stream.writer.Write(data)
			} else {
				log.Printf("[ERR] Data received for stream in state: %d", stream.state)
			}
			stream.mu.Unlock()
		}
	}
}

func (m *MuxConn) write(id byte, dataType muxPacketType, p []byte) (int, error) {
	m.wlock.Lock()
	defer m.wlock.Unlock()

	if err := binary.Write(m.rwc, binary.BigEndian, id); err != nil {
		return 0, err
	}
	if err := binary.Write(m.rwc, binary.BigEndian, byte(dataType)); err != nil {
		return 0, err
	}
	if err := binary.Write(m.rwc, binary.BigEndian, int32(len(p))); err != nil {
		return 0, err
	}
	if len(p) == 0 {
		return 0, nil
	}
	return m.rwc.Write(p)
}

// Stream is a single stream of data and implements io.ReadWriteCloser
type Stream struct {
	id           byte
	mux          *MuxConn
	reader       io.Reader
	writer       io.WriteCloser
	state        streamState
	stateUpdated time.Time
	mu           sync.Mutex
}

type streamState byte

const (
	streamStateClosed streamState = iota
	streamStateListen
	streamStateSynRecv
	streamStateSynSent
	streamStateEstablished
	streamStateFinWait
)

func (s *Stream) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	if s.state != streamStateEstablished {
		return fmt.Errorf("Stream in bad state: %d", s.state)
	}

	if _, err := s.mux.write(s.id, muxPacketFin, nil); err != nil {
		return err
	}

	s.setState(streamStateClosed)
	s.writer.Close()
	return nil
}

func (s *Stream) Read(p []byte) (int, error) {
	return s.reader.Read(p)
}

func (s *Stream) Write(p []byte) (int, error) {
	return s.mux.write(s.id, muxPacketData, p)
}

func (s *Stream) setState(state streamState) {
	s.state = state
	s.stateUpdated = time.Now().UTC()
}
<commit_msg>packer/rpc: MuxConn can return next available stream ID<commit_after>package rpc

import (
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"sync"
	"time"
)

// MuxConn is a connection that can be used bi-directionally for RPC. Normally,
// Go RPC only allows client-to-server connections. This allows the client
// to actually act as a server as well.
//
// MuxConn works using a fairly dumb multiplexing technique of simply
// framing every piece of data sent into a prefix + data format. Streams
// are established using a subset of the TCP protocol. Only a subset is
// necessary since we assume ordering on the underlying RWC.
type MuxConn struct {
	curId   uint32
	rwc     io.ReadWriteCloser
	streams map[uint32]*Stream
	mu      sync.RWMutex
	wlock   sync.Mutex
}

type muxPacketType byte

const (
	muxPacketSyn muxPacketType = iota
	muxPacketAck
	muxPacketFin
	muxPacketData
)

func NewMuxConn(rwc io.ReadWriteCloser) *MuxConn {
	m := &MuxConn{
		rwc:     rwc,
		streams: make(map[uint32]*Stream),
	}

	go m.loop()

	return m
}

// Close closes the underlying io.ReadWriteCloser. This will also close
// all streams that are open.
func (m *MuxConn) Close() error {
	m.mu.Lock()
	defer m.mu.Unlock()

	// Close all the streams
	for _, w := range m.streams {
		w.Close()
	}
	m.streams = make(map[uint32]*Stream)

	return m.rwc.Close()
}

// Accept accepts a multiplexed connection with the given ID. This
// will block until a request is made to connect.
func (m *MuxConn) Accept(id uint32) (io.ReadWriteCloser, error) {
	stream, err := m.openStream(id)
	if err != nil {
		return nil, err
	}

	// If the stream isn't closed, then it is already open somehow
	stream.mu.Lock()
	if stream.state != streamStateSynRecv && stream.state != streamStateClosed {
		stream.mu.Unlock()
		return nil, fmt.Errorf("Stream already open in bad state: %d", stream.state)
	}

	if stream.state == streamStateSynRecv {
		// Fast track establishing since we already got the syn
		stream.setState(streamStateEstablished)
		stream.mu.Unlock()
	}

	if stream.state != streamStateEstablished {
		// Go into the listening state
		stream.setState(streamStateListen)
		stream.mu.Unlock()

		// Wait for the connection to establish
	ACCEPT_ESTABLISH_LOOP:
		for {
			time.Sleep(50 * time.Millisecond)
			stream.mu.Lock()
			switch stream.state {
			case streamStateListen:
				stream.mu.Unlock()
			case streamStateEstablished:
				stream.mu.Unlock()
				break ACCEPT_ESTABLISH_LOOP
			default:
				defer stream.mu.Unlock()
				return nil, fmt.Errorf("Stream went to bad state: %d", stream.state)
			}
		}
	}

	// Send the ack down
	if _, err := m.write(stream.id, muxPacketAck, nil); err != nil {
		return nil, err
	}

	return stream, nil
}

// Dial opens a connection to the remote end using the given stream ID.
// An Accept on the remote end will only work with if the IDs match.
func (m *MuxConn) Dial(id uint32) (io.ReadWriteCloser, error) {
	stream, err := m.openStream(id)
	if err != nil {
		return nil, err
	}

	// If the stream isn't closed, then it is already open somehow
	stream.mu.Lock()
	if stream.state != streamStateClosed {
		stream.mu.Unlock()
		return nil, fmt.Errorf("Stream already open in bad state: %d", stream.state)
	}

	// Open a connection
	if _, err := m.write(stream.id, muxPacketSyn, nil); err != nil {
		return nil, err
	}
	stream.setState(streamStateSynSent)
	stream.mu.Unlock()

	for {
		time.Sleep(50 * time.Millisecond)
		stream.mu.Lock()
		switch stream.state {
		case streamStateSynSent:
			stream.mu.Unlock()
		case streamStateEstablished:
			stream.mu.Unlock()
			return stream, nil
		default:
			defer stream.mu.Unlock()
			return nil, fmt.Errorf("Stream went to bad state: %d", stream.state)
		}
	}
}

// NextId returns the next available stream ID that isn't currently
// taken.
func (m *MuxConn) NextId() uint32 {
	m.mu.Lock()
	defer m.mu.Unlock()

	for {
		if _, ok := m.streams[m.curId]; !ok {
			return m.curId
		}

		m.curId++
	}
}

func (m *MuxConn) openStream(id uint32) (*Stream, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	if stream, ok := m.streams[id]; ok {
		return stream, nil
	}

	// Create the stream object and channel where data will be sent to
	dataR, dataW := io.Pipe()

	// Set the data channel so we can write to it.
	stream := &Stream{
		id:     id,
		mux:    m,
		reader: dataR,
		writer: dataW,
	}
	stream.setState(streamStateClosed)

	m.streams[id] = stream
	return m.streams[id], nil
}

func (m *MuxConn) loop() {
	defer m.Close()

	var id uint32
	var packetType muxPacketType
	var length int32
	for {
		if err := binary.Read(m.rwc, binary.BigEndian, &id); err != nil {
			log.Printf("[ERR] Error reading stream ID: %s", err)
			return
		}
		if err := binary.Read(m.rwc, binary.BigEndian, &packetType); err != nil {
			log.Printf("[ERR] Error reading packet type: %s", err)
			return
		}
		if err := binary.Read(m.rwc, binary.BigEndian, &length); err != nil {
			log.Printf("[ERR] Error reading length: %s", err)
			return
		}

		// TODO(mitchellh): probably would be better to re-use a buffer...
		data := make([]byte, length)
		if length > 0 {
			if _, err := m.rwc.Read(data); err != nil {
				log.Printf("[ERR] Error reading data: %s", err)
				return
			}
		}

		stream, err := m.openStream(id)
		if err != nil {
			log.Printf("[ERR] Error opening stream %d: %s", id, err)
			return
		}

		switch packetType {
		case muxPacketAck:
			stream.mu.Lock()
			if stream.state == streamStateSynSent {
				stream.setState(streamStateEstablished)
			} else {
				log.Printf("[ERR] Ack received for stream in state: %d", stream.state)
			}
			stream.mu.Unlock()
		case muxPacketSyn:
			stream.mu.Lock()
			switch stream.state {
			case streamStateClosed:
				stream.setState(streamStateSynRecv)
			case streamStateListen:
				stream.setState(streamStateEstablished)
			default:
				log.Printf("[ERR] Syn received for stream in state: %d", stream.state)
			}
			stream.mu.Unlock()
		case muxPacketFin:
			stream.mu.Lock()
			stream.setState(streamStateClosed)
			stream.writer.Close()
			stream.mu.Unlock()

			m.mu.Lock()
			delete(m.streams, stream.id)
			m.mu.Unlock()
		case muxPacketData:
			stream.mu.Lock()
			if stream.state == streamStateEstablished {
				stream.writer.Write(data)
			} else {
				log.Printf("[ERR] Data received for stream in state: %d", stream.state)
			}
			stream.mu.Unlock()
		}
	}
}

func (m *MuxConn) write(id uint32, dataType muxPacketType, p []byte) (int, error) {
	m.wlock.Lock()
	defer m.wlock.Unlock()

	if err := binary.Write(m.rwc, binary.BigEndian, id); err != nil {
		return 0, err
	}
	if err := binary.Write(m.rwc, binary.BigEndian, byte(dataType)); err != nil {
		return 0, err
	}
	if err := binary.Write(m.rwc, binary.BigEndian, int32(len(p))); err != nil {
		return 0, err
	}
	if len(p) == 0 {
		return 0, nil
	}
	return m.rwc.Write(p)
}

// Stream is a single stream of data and implements io.ReadWriteCloser
type Stream struct {
	id           uint32
	mux          *MuxConn
	reader       io.Reader
	writer       io.WriteCloser
	state        streamState
	stateUpdated time.Time
	mu           sync.Mutex
}

type streamState byte

const (
	streamStateClosed streamState = iota
	streamStateListen
	streamStateSynRecv
	streamStateSynSent
	streamStateEstablished
	streamStateFinWait
)

func (s *Stream) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	if s.state != streamStateEstablished {
		return fmt.Errorf("Stream in bad state: %d", s.state)
	}

	if _, err := s.mux.write(s.id, muxPacketFin, nil); err != nil {
		return err
	}

	s.setState(streamStateClosed)
	s.writer.Close()
	return nil
}

func (s *Stream) Read(p []byte) (int, error) {
	return s.reader.Read(p)
}

func (s *Stream) Write(p []byte) (int, error) {
	return s.mux.write(s.id, muxPacketData, p)
}

func (s *Stream) setState(state streamState) {
	s.state = state
	s.stateUpdated = time.Now().UTC()
}
<|endoftext|>


<commit_before>package anonuuid

import "fmt"

func ExampleNew() {
	anonuuid := New()
	fmt.Println(anonuuid)
	// Output:
	// &{map[]}
}

func ExampleSanitize() {
	anonuuid := New()
	input := `
VOLUMES_0_SERVER_ID=15573749-c89d-41dd-a655-16e79bed52e0
VOLUMES_0_SERVER_NAME=hello
VOLUMES_0_ID=c245c3cb-3336-4567-ada1-70cb1fe4eefe
VOLUMES_0_SIZE=50000000000
ORGANIZATION=fe1e54e8-d69d-4f7c-a9f1-42069e03da31
TEST=15573749-c89d-41dd-a655-16e79bed52e0
`
	fmt.Println(anonuuid.Sanitize(input))
	// Output:
	// VOLUMES_0_SERVER_ID=00000000-0000-0000-0000-000000000000
	// VOLUMES_0_SERVER_NAME=hello
	// VOLUMES_0_ID=11111111-1111-1111-1111-111111111111
	// VOLUMES_0_SIZE=50000000000
	// ORGANIZATION=22222222-2222-2222-2222-222222222222
	// TEST=00000000-0000-0000-0000-000000000000
}
<commit_msg>Added benchs<commit_after>package anonuuid

import (
	"fmt"
	"testing"
)

var (
	EXAMPLE_INPUT = `
VOLUMES_0_SERVER_ID=15573749-c89d-41dd-a655-16e79bed52e0
VOLUMES_0_SERVER_NAME=hello
VOLUMES_0_ID=c245c3cb-3336-4567-ada1-70cb1fe4eefe
VOLUMES_0_SIZE=50000000000
ORGANIZATION=fe1e54e8-d69d-4f7c-a9f1-42069e03da31
TEST=15573749-c89d-41dd-a655-16e79bed52e0
`
)

func ExampleNew() {
	anonuuid := New()
	fmt.Println(anonuuid)
	// Output:
	// &{map[]}
}

func ExampleSanitize() {
	anonuuid := New()
	fmt.Println(anonuuid.Sanitize(EXAMPLE_INPUT))
	// Output:
	// VOLUMES_0_SERVER_ID=00000000-0000-0000-0000-000000000000
	// VOLUMES_0_SERVER_NAME=hello
	// VOLUMES_0_ID=11111111-1111-1111-1111-111111111111
	// VOLUMES_0_SIZE=50000000000
	// ORGANIZATION=22222222-2222-2222-2222-222222222222
	// TEST=00000000-0000-0000-0000-000000000000
}

func BenchmarkNew(b *testing.B) {
	for i := 0; i < b.N; i++ {
		New()
	}
}

func BenchmarkSanitize(b *testing.B) {
	for i := 0; i < b.N; i++ {
		anonuuid := New()
		anonuuid.Sanitize(EXAMPLE_INPUT)
	}
}
<|endoftext|>


<commit_before>/* batch-rename.go

A small program to handle batch renames and copies.

Syntax:

	batch-rename [opts]

    batch-rename will construct a list of all files that match a given regex,
    or all files in the directory, and rename/copy them to a matching file that is
    modified according to the specified prefix or suffix.

    For example, 'batch-rename' --prefix to_sort_ --regex "/.png$/"' will rename all
    files matching .png in the current  directory to 'to_sort_<oldname>.png'.

    Arguments:

      --regex|-x <regex>      A regular expression for matching files.  You can use
                             "/<regex>/" or "<regex>", but the double-quotes are
                             necessary.
      --prefix|-p <prefix>    Renames matching files to have the specified prefix
      --suffix|-s <suffix>    Renames matching files to have the specified suffix
      --enumerate|-e <name>   Rename matching files to <name>_<num>, where <num>
                             is incremented from 000.
      --target-dir|-t <path>  The directory within which we rename/copy.  Default
                             is the current working directory.
      --copy|-c               Copy instead of rename.
      --recurse|-r            Search for matching files in subdirectories
      --lowercase|-l          lowercase the final rename
      --uppercase|-u          uppercase the final rename
      --dry-run|-n            List files, but don't copy/rename

Author: Ysgard (Jan Van Uytven) 2013
*/

package main

import (
	"fmt"
	"os"
	//"ioutil" // For ReadDir
	"path/filepath"
	"regexp"
)

// Why do I have to use "MustCompile"?  Why does a simple Compile raise an error?
var defaultFileMatch = regexp.MustCompile("^([a-zA-Z0-9\\s\\._-]+)$")

// Just return the concatenation of the prefix and the filename.
func prefixName(name, prefix string) string {
	return prefix + name
}

// Add a suffix to a filename, being careful to remove and re-add the extension
// on it (if it exists).
func suffixName(name, suffix string) string {

}

func main() {
	if defaultFileMatch == nil {
		fmt.Fprintf(os.Stderr, "Invalid regexp object!\n")
		os.Exit(1)
	}

}
<commit_msg>Added initial flag parsing<commit_after>/*
batch-rename.go

A small program to handle batch renames and copies.

Author: Ysgard (Jan Van Uytven) 2013
*/

package main

import (
	"flag"
	"fmt"
	"os"
	//"ioutil" // For ReadDir
	//"path/filepath"
	"regexp"
)

// Why do I have to use "MustCompile"?  Why does a simple Compile raise an error?
var defaultFileMatch = regexp.MustCompile(`^([a-zA-Z0-9\s\._-]+)$`)

var usageString = `
  batch-rename [opts]

    batch-rename will construct a list of all files that match a given regex,
    or all files in the directory, and rename/copy them to a matching file
    that is modified according to the specified prefix or suffix.

    For example, 'batch-rename' --prefix to_sort_ --regex "/.png$/"' will
    rename all files matching .png in the current  directory to
    'to_sort_<oldname>.png'.

    Arguments:

      --regex|-x <regex>
        A regular expression for matching files.  You can use "/<regex>/" or
        "<regex>", but the double-quotes are necessary.

      --prefix|-p <prefix>
        Renames matching files to have the specified prefix.

      --suffix|-s <suffix>
        Renames matching files to have the specified suffix.

      --enumerate|-e <name>
        Rename matching files to <name>_<num>, where <num> is
        incremented from 000.

      --target-dir|-t <path>
        The directory within which we rename/copy.  Default is the current
        working directory.

      --copy|-c
        Copy instead of rename.

      --recurse|-r
        Search for matching files in subdirectories.

      --lowercase|-l
        Lowercase the final rename.

      --uppercase|-u
        Uppercase the final rename.

      --dry-run|-n
        List files, but don't copy/rename
`

var regexArg string
var prefixArg string
var suffixArg string
var enumerateArg string
var targetArg string
var copyArg bool
var recurseArg bool
var lowerArg bool
var upperArg bool
var dryrunArg bool

// Just return the concatenation of the prefix and the filename.
func prefixName(name, prefix string) string {
	return prefix + name
}

// Add a suffix to a filename, being careful to remove and re-add the extension
// on it (if it exists).
// func suffixName(name, suffix string) string {

// }

// Call before parsing flags
func flagInit() {
	// redefine flag's Usage function
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usageString)
	}

	const (
		regex_default     = ""
		prefix_default    = ""
		suffix_default    = ""
		enumerate_default = ""
		target_default    = ""
		copy_default      = false
		recurse_default   = false
		lowercase_default = false
		uppercase_default = false
		dryrun_default    = false
	)

	flag.StringVar(&regexArg, "regex", regex_default, "")
	flag.StringVar(&regexArg, "x", regex_default, "")
	flag.StringVar(&prefixArg, "prefix", prefix_default, "")
	flag.StringVar(&prefixArg, "p", prefix_default, "")
	flag.StringVar(&suffixArg, "suffix", suffix_default, "")
	flag.StringVar(&suffixArg, "s", suffix_default, "")
	flag.StringVar(&enumerateArg, "enumerate", enumerate_default, "")
	flag.StringVar(&enumerateArg, "e", enumerate_default, "")
	flag.StringVar(&targetArg, "target-dir", target_default, "")
	flag.StringVar(&targetArg, "t", target_default, "")
	flag.BoolVar(&copyArg, "copy", copy_default, "")
	flag.BoolVar(&copyArg, "c", copy_default, "")
	flag.BoolVar(&recurseArg, "recurse", recurse_default, "")
	flag.BoolVar(&recurseArg, "r", recurse_default, "")
	flag.BoolVar(&lowerArg, "lowercase", lowercase_default, "")
	flag.BoolVar(&lowerArg, "l", lowercase_default, "")
	flag.BoolVar(&upperArg, "uppercase", uppercase_default, "")
	flag.BoolVar(&upperArg, "u", uppercase_default, "")
	flag.BoolVar(&dryrunArg, "dry-run", dryrun_default, "")
	flag.BoolVar(&dryrunArg, "n", dryrun_default, "")

	flag.Parse()

}

func main() {

	// Get the command-line arguments
	flagInit()

	// Print out the values of each argument
	fmt.Fprintf(os.Stdout, "regexArg: "+regexArg+"\n")
	fmt.Fprintf(os.Stdout, "prefixArg: "+prefixArg+"\n")
	fmt.Fprintf(os.Stdout, "suffixArg: "+suffixArg+"\n")
	fmt.Fprintf(os.Stdout, "enumerateArg: "+enumerateArg+"\n")
	fmt.Fprintf(os.Stdout, "targetArg: "+targetArg+"\n")
	fmt.Fprintf(os.Stdout, "copyArg: %b\n", copyArg)
	fmt.Fprintf(os.Stdout, "recurseArg: %b\n", recurseArg)
	fmt.Fprintf(os.Stdout, "lowerArg: %b\n", lowerArg)
	fmt.Fprintf(os.Stdout, "upperArg: %b\n", upperArg)
	fmt.Fprintf(os.Stdout, "dryrunArg: %b\n", dryrunArg)
}
<|endoftext|>


<commit_before>package bench

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	hummingbird "hummingbird/common"
)

type DirectObject struct {
	Url  string
	Data []byte
}

func (obj *DirectObject) Put() bool {
	req, _ := http.NewRequest("PUT", obj.Url, bytes.NewReader(obj.Data))
	req.Header.Set("Content-Length", strconv.FormatInt(int64(len(obj.Data)), 10))
	req.Header.Set("X-Timestamp", hummingbird.GetTimestamp())
	req.Header.Set("Content-Type", "application/octet-stream")
	req.ContentLength = int64(len(obj.Data))
	resp, err := client.Do(req)
	if resp != nil {
		resp.Body.Close()
	}
	return err == nil && resp.StatusCode/100 == 2
}

func (obj *DirectObject) Get() bool {
	req, _ := http.NewRequest("GET", obj.Url, nil)
	resp, err := client.Do(req)
	if resp != nil {
		io.Copy(ioutil.Discard, resp.Body)
	}
	return err == nil && resp.StatusCode/100 == 2
}

func (obj *DirectObject) Delete() bool {
	req, _ := http.NewRequest("DELETE", obj.Url, nil)
	req.Header.Set("X-Timestamp", hummingbird.GetTimestamp())
	resp, err := client.Do(req)
	if resp != nil {
		resp.Body.Close()
	}
	return err == nil && resp.StatusCode/100 == 2
}

func GetDevices(address string) []string {
	deviceUrl := fmt.Sprintf("%srecon/devices", address)
	req, err := http.NewRequest("GET", deviceUrl, nil)
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("ERROR GETTING DEVICES")
		os.Exit(1)
	}
	body, _ := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	var rdata interface{}
	json.Unmarshal(body, &rdata)
	for _, v := range rdata.(map[string]interface{}) {
		retvals := []string{}
		for _, val := range v.([]interface{}) {
			retvals = append(retvals, val.(string))
		}
		return retvals
	}
	return []string{}
}

func RunDBench(args []string) {
	if len(args) < 1 {
		fmt.Println("Usage: [configuration file]")
		fmt.Println("The configuration file should look something like:")
		fmt.Println("    [dbench]")
		fmt.Println("    address = http://localhost:6010/")
		fmt.Println("    concurrency = 15")
		fmt.Println("    object_size = 131072")
		fmt.Println("    num_objects = 5000")
		fmt.Println("    num_gets = 30000")
		fmt.Println("    delete = yes")
		os.Exit(1)
	}

	benchconf, err := hummingbird.LoadIniFile(args[0])
	if err != nil {
		fmt.Println("Error parsing ini file:", err)
		os.Exit(1)
	}

	address := benchconf.GetDefault("dbench", "address", "http://localhost:6010/")
	if !strings.HasSuffix(address, "/") {
		address = address + "/"
	}
	concurrency := int(benchconf.GetInt("dbench", "concurrency", 16))
	objectSize := benchconf.GetInt("dbench", "object_size", 131072)
	numObjects := benchconf.GetInt("dbench", "num_objects", 5000)
	numGets := benchconf.GetInt("dbench", "num_gets", 30000)
	delete := benchconf.GetBool("dbench", "delete", true)

	deviceList := GetDevices(address)

	data := make([]byte, objectSize)
	objects := make([]DirectObject, numObjects)
	for i, _ := range objects {
		objects[i].Url = fmt.Sprintf("%s%s/%d/%s/%s/%d", address, deviceList[i%len(deviceList)], rand.Int63()%100, "a", "c", rand.Int63())
		objects[i].Data = data
	}

	work := make([]func() bool, len(objects))
	for i, _ := range objects {
		work[i] = objects[i].Put
	}
	DoJobs("PUT", work, concurrency)

	time.Sleep(time.Second * 2)

	work = make([]func() bool, numGets)
	for i := int64(0); i < numGets; i++ {
		work[i] = objects[int(rand.Int63()%int64(len(objects)))].Get
	}
	DoJobs("GET", work, concurrency)

	if delete {
		work = make([]func() bool, len(objects))
		for i, _ := range objects {
			work[i] = objects[i].Delete
		}
		DoJobs("DELETE", work, concurrency)
	}
}
<commit_msg>use invalid partitions for dbench<commit_after>package bench

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	hummingbird "hummingbird/common"
)

type DirectObject struct {
	Url  string
	Data []byte
}

func (obj *DirectObject) Put() bool {
	req, _ := http.NewRequest("PUT", obj.Url, bytes.NewReader(obj.Data))
	req.Header.Set("Content-Length", strconv.FormatInt(int64(len(obj.Data)), 10))
	req.Header.Set("X-Timestamp", hummingbird.GetTimestamp())
	req.Header.Set("Content-Type", "application/octet-stream")
	req.ContentLength = int64(len(obj.Data))
	resp, err := client.Do(req)
	if resp != nil {
		resp.Body.Close()
	}
	return err == nil && resp.StatusCode/100 == 2
}

func (obj *DirectObject) Get() bool {
	req, _ := http.NewRequest("GET", obj.Url, nil)
	resp, err := client.Do(req)
	if resp != nil {
		io.Copy(ioutil.Discard, resp.Body)
	}
	return err == nil && resp.StatusCode/100 == 2
}

func (obj *DirectObject) Delete() bool {
	req, _ := http.NewRequest("DELETE", obj.Url, nil)
	req.Header.Set("X-Timestamp", hummingbird.GetTimestamp())
	resp, err := client.Do(req)
	if resp != nil {
		resp.Body.Close()
	}
	return err == nil && resp.StatusCode/100 == 2
}

func GetDevices(address string) []string {
	deviceUrl := fmt.Sprintf("%srecon/devices", address)
	req, err := http.NewRequest("GET", deviceUrl, nil)
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("ERROR GETTING DEVICES")
		os.Exit(1)
	}
	body, _ := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	var rdata interface{}
	json.Unmarshal(body, &rdata)
	for _, v := range rdata.(map[string]interface{}) {
		retvals := []string{}
		for _, val := range v.([]interface{}) {
			retvals = append(retvals, val.(string))
		}
		return retvals
	}
	return []string{}
}

func RunDBench(args []string) {
	if len(args) < 1 {
		fmt.Println("Usage: [configuration file]")
		fmt.Println("The configuration file should look something like:")
		fmt.Println("    [dbench]")
		fmt.Println("    address = http://localhost:6010/")
		fmt.Println("    concurrency = 15")
		fmt.Println("    object_size = 131072")
		fmt.Println("    num_objects = 5000")
		fmt.Println("    num_gets = 30000")
		fmt.Println("    delete = yes")
		os.Exit(1)
	}

	benchconf, err := hummingbird.LoadIniFile(args[0])
	if err != nil {
		fmt.Println("Error parsing ini file:", err)
		os.Exit(1)
	}

	address := benchconf.GetDefault("dbench", "address", "http://localhost:6010/")
	if !strings.HasSuffix(address, "/") {
		address = address + "/"
	}
	concurrency := int(benchconf.GetInt("dbench", "concurrency", 16))
	objectSize := benchconf.GetInt("dbench", "object_size", 131072)
	numObjects := benchconf.GetInt("dbench", "num_objects", 5000)
	numGets := benchconf.GetInt("dbench", "num_gets", 30000)
	minPartition := benchconf.GetInt("dbench", "minimum_partition_number", 1000000000)
	delete := benchconf.GetBool("dbench", "delete", true)

	deviceList := GetDevices(address)

	data := make([]byte, objectSize)
	objects := make([]DirectObject, numObjects)
	for i, _ := range objects {
		objects[i].Url = fmt.Sprintf("%s%s/%d/%s/%s/%d", address, deviceList[i%len(deviceList)], rand.Int63()%100 + minPartition, "a", "c", rand.Int63())
		objects[i].Data = data
	}

	work := make([]func() bool, len(objects))
	for i, _ := range objects {
		work[i] = objects[i].Put
	}
	DoJobs("PUT", work, concurrency)

	time.Sleep(time.Second * 2)

	work = make([]func() bool, numGets)
	for i := int64(0); i < numGets; i++ {
		work[i] = objects[int(rand.Int63()%int64(len(objects)))].Get
	}
	DoJobs("GET", work, concurrency)

	if delete {
		work = make([]func() bool, len(objects))
		for i, _ := range objects {
			work[i] = objects[i].Delete
		}
		DoJobs("DELETE", work, concurrency)
	}
}
<|endoftext|>