<commit_before>package protocol

import (
	"github.com/jliang00/gometa/src/common"
	"sync"
	"log"
	"time"
	"runtime/debug"
)

/////////////////////////////////////////////////////////////////////////////
// Type Declaration
/////////////////////////////////////////////////////////////////////////////

type LeaderServer struct {
	leader       	*Leader
	listener     	*common.PeerListener
	consentState 	*ConsentState
	state        	*LeaderState
	handler 	 	 ActionHandler
	factory 	 	 MsgFactory
}

type LeaderState struct {
	requestMgr 		RequestMgr

	// mutex protected variable
	mutex   		sync.Mutex
	ready   		bool
	readych         chan bool
	proxies         map[string](chan bool)
}

type ListenerState struct {
	donech			chan bool
	killch			chan bool
}

/////////////////////////////////////////////////////////////////////////////
// LeaderServer - Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Create a new LeaderServer.  This is a blocking call until the LeaderServer
// termintates.
//
// killch should be unbuffered to ensure the sender won't block
//
func RunLeaderServer(naddr string,
	listener *common.PeerListener,
	ss RequestMgr,
	handler ActionHandler,
	factory MsgFactory,
	killch <- chan bool) (err error) {

	log.Printf("LeaderServer.RunLeaderServer(): start leader server %s", naddr)

	// Catch panic at the main entry point for LeaderServer
	defer func() {
		if r := recover(); r != nil {
			log.Printf("panic in RunLeaderServer() : %s\n", r)
			err = r.(error)
		}

		log.Printf("RunLeaderServer terminates : Diagnostic Stack ...")
		log.Printf("%s", debug.Stack())
	}()

	// create a leader
	leader, err := NewLeader(naddr, handler, factory)
	if err != nil {
		return err
	}
	defer leader.Terminate()

	// create a ConsentState
	epoch, err := handler.GetAcceptedEpoch()
	if err != nil {
		return err
	}
	ensembleSize := handler.GetEnsembleSize()
	consentState := NewConsentState(naddr, epoch, ensembleSize)
	defer consentState.Terminate()

	// create the leader state
	state := newLeaderState(ss)

	// create the server
	server := &LeaderServer{leader: leader,
		listener:     listener,
		consentState: consentState,
		state:        state,
		handler:      handler,
		factory:	  factory}

	// start the listener.  This goroutine would continue to new follower even while
	// it is processing request.
	listenerState := newListenerState()
	go server.listenFollower(listenerState)

	// start the main loop for processing incoming request.  The leader will
	// process request only after it has received quorum of followers to
	// synchronized with it.
	err = server.processRequest(killch, listenerState)

	log.Printf("LeaderServer.RunLeaderServer(): leader server %s terminate", naddr)

	return err
}

/////////////////////////////////////////////////////////////////////////////
// LeaderServer - Private Function : Discovery Phase (synchronize with follower)
/////////////////////////////////////////////////////////////////////////////

//
// Listen to new connection request from the follower/peer.
// Start a new LeaderSyncProxy to synchronize the state
// between the leader and the peer.
//
func (l *LeaderServer) listenFollower(listenerState *ListenerState) {

	defer func() {
		if r := recover(); r != nil {
			log.Printf("panic in LeaderServer.listenFollower() : %s\n", r)
		}

		log.Printf("LeaderServer.listenFollower() terminates : Diagnostic Stack ...")
		log.Printf("%s", debug.Stack())

		common.SafeRun("LeaderServer.listenFollower()",
			func() {
				l.terminateAllOutstandingProxies()
			})

		common.SafeRun("LeaderServer.listenFollower()",
			func() {
				listenerState.donech <- true
			})
	}()

	connCh := l.listener.ConnChannel()
	if connCh == nil {
		// It should not happen unless the listener is closed
		return
	}

	// if there is a single server, then we don't need to wait for follower
	// for the server to be ready to process request.
	if l.handler.GetEnsembleSize() == 1 {
		if err := l.incrementEpoch(); err != nil {
			log.Printf("LeaderServer.listenFollower(): Error when boostraping leader with ensembleSize=1. Error = %s", err)
			return
		}

		l.notifyReady()
	}

	for {
		select {
		case conn := <-connCh:
			{
				// There is a new peer connection request from the follower.  Start a proxy to synchronize with the follower.
				// The leader does not proactively connect to follower:
				// 1) The ensemble is stable, but a follower may just reboot and needs to connect to the leader
				// 2) Even if the leader receives votes from the leader, the leader cannot tell for sure that the follower does
				//    not change its vote.  Only if the follower connects, the leader can confirm the follower's alliance.
				//
				log.Printf("LeaderServer.listenFollower(): Receive connection request from follower %s", conn.RemoteAddr())
				if l.registerOutstandingProxy(conn.RemoteAddr().String()) {
					pipe := common.NewPeerPipe(conn)
					go l.startProxy(pipe)
				} else {
					log.Printf("LeaderServer.listenFollower(): Sync Proxy already running for %s. Ignore new request.", conn.RemoteAddr())
					conn.Close()
				}
			}
		case <- listenerState.killch:
			log.Printf("LeaderServer.listenFollower(): Receive kill signal. Terminate.")
			return
		}
	}
}

//
// Start a LeaderSyncProxy to synchornize the leader
// and follower state.
//
func (l *LeaderServer) startProxy(peer *common.PeerPipe) {

	defer func() {
		if r := recover(); r != nil {
			log.Printf("panic in LeaderServer.startProxy() : %s\n", r)
			log.Printf("LeaderServer.startProxy() : Diagnostic Stack ...")
			log.Printf("%s", debug.Stack())
		}

		// deregister the proxy with the leader Server upon exit
		l.deregisterOutstandingProxy(peer.GetAddr())
	}()

	// create a proxy that will sycnhronize with the peer.
	log.Printf("LeaderServer.startProxy(): Start synchronization with follower. Peer TCP connection (%s)", peer.GetAddr())
	proxy := NewLeaderSyncProxy(l.leader, l.consentState, peer, l.handler, l.factory)
	donech := proxy.GetDoneChannel()

	// Create an observer for the leader.  The leader will put on-going proposal msg and commit msg
	// onto the observer queue.  This ensure that we can won't miss those mutations as the leader is
	// sync'ign withe follower.  The messages in observer queue will eventually route to follower.
	o := NewObserver()
	l.leader.AddObserver(peer.GetAddr(), o)
	defer l.leader.RemoveObserver(peer.GetAddr())

	// start the proxy
	go proxy.Start(o)
	defer proxy.Terminate()

	// Get the killch for this go-routine
	killch := l.getProxyKillChan(peer.GetAddr())
	if killch == nil {
   		log.Printf("LeaderServer.startProxy(): Cannot find killch for proxy (TCP connection = %s).", peer.GetAddr())
   		log.Printf("LeaderServer.startProxy(): Cannot start follower sync.")
   		return
	}

	// this go-routine will be blocked until handshake is completed between the
	// leader and the follower.  By then, the leader will also get majority
	// confirmation that it is a leader.
	select {
		case success := <-donech:
			if success {
				// tell the leader to add this follower for processing request.  If there is a follower running already,
				// AddFollower() will terminate the existing follower instance, and then create a new one.
	    		fid := proxy.GetFid()
	    		if proxy.CanFollowerVote() {
					l.leader.AddFollower(fid, peer, o)
	    			log.Printf("LeaderServer.startProxy(): Synchronization with follower %s done (TCP conn = %s).  Add follower.",
	    				fid, peer.GetAddr())

					// At this point, the follower has voted this server as the leader.
					// Notify the request processor to start processing new request for this host
					l.notifyReady()
				} else {
					l.leader.AddWatcher(fid, peer, o)
	    			log.Printf("LeaderServer.startProxy(): Sync with watcher done.  Add Watcher %s (TCP conn = %s)",
	    				fid, peer.GetAddr())
				}
			} else {
				log.Printf("LeaderServer:startProxy(): Leader Fail to synchronization with follower (TCP conn = %s)", peer.GetAddr())
			}
		case <- killch:
			log.Printf("LeaderServer:startProxy(): Sync proxy is killed while synchronizing with follower (TCP conn == %s)",
					peer.GetAddr())
	}
}

//
// Create a new LeaderState
//
func newLeaderState(ss RequestMgr) *LeaderState {
	state := &LeaderState{requestMgr: ss,
							ready: false,
							readych : make(chan bool, 1), // buffered so sender won't wait
							proxies : make(map[string](chan bool))}

	return state
}

//
// Increment the epoch.  Only call this method if the ensemble size is 1 (single server).
// This function can panic if the epoch reaches its limit.
//
func (l *LeaderServer) incrementEpoch() error {

	epoch, err := l.handler.GetCurrentEpoch()
	if err != nil {
		return err
	}

	epoch = common.CompareAndIncrementEpoch(epoch, epoch)

	if err := l.handler.NotifyNewAcceptedEpoch(epoch); err != nil {
		return err
	}

	if err := l.handler.NotifyNewCurrentEpoch(epoch); err != nil {
		return err
	}

	return nil
}

/////////////////////////////////////////////////////////////////////////////
// LeaderServer - Private Function : Broadcast phase (handle request)
/////////////////////////////////////////////////////////////////////////////

//
// Goroutine for processing each request one-by-one
//
func (s *LeaderServer) processRequest(killch <-chan bool,
									  listenerState *ListenerState) (err error) {

	defer func() {
		if r := recover(); r != nil {
			log.Printf("panic in LeaderServer.processRequest() : %s\n", r)
			err = r.(error)

			log.Printf("LeaderServer.processRequest() : Diagnostic Stack ...")
			log.Printf("%s", debug.Stack())
		}

		common.SafeRun("LeaderServer.processRequest()",
			func() {
				listenerState.killch <- true
			})
	}()

	// start processing loop after I am being confirmed as a leader (there
	// is a quorum of followers that have sync'ed with me)
	if !s.waitTillReady() {
		return common.NewError(common.ELECTION_ERROR,
			"LeaderServer.processRequest(): Leader times out waiting for quorum of followers. Terminate")
	}

	// At this point, the leader has gotten a majority of followers to follow, so it
	// can proceed.  It is possible that it may loose quorum of followers. But in that
	// case, the leader will not be able to process any request.
	log.Printf("LeaderServer.processRequest(): Leader Server is ready to proces request")

	// Leader is ready at this time.  This implies that there is a quorum of follower has
	// followed this leader.  Get the change channel to keep track of  number of followers.
	// If the leader no longer has quorum, it needs to let go of its leadership.
	leaderchangech := s.leader.GetEnsembleChangeChannel()
	ensembleSize := s.handler.GetEnsembleSize()

	// notify the request processor to start processing new request
	incomings := s.state.requestMgr.GetRequestChannel()
	for {
		select {
		case handle, ok := <- incomings:
			if ok {
				// de-queue the request
				s.state.requestMgr.AddPendingRequest(handle)

				// create the proposal and forward to the leader
				s.leader.CreateProposal(s.leader.GetFollowerId(), handle.Request)
			} else {
				// server shutdown.
				log.Printf("LeaderServer.processRequest(): channel for receiving client request is closed. Terminate.")
				return nil
			}
		case <-killch:
			// server shutdown
			log.Printf("LeaderServer.processRequest(): receive kill signal. Stop Client request processing.")
			return nil
		case <-listenerState.donech:
			// listener is down.  Terminate this request processing loop as well.
			log.Printf("LeaderServer.processRequest(): follower listener terminates. Stop client request processing.")
			return nil
		case <-leaderchangech:
			// Listen to any change to the leader's active ensemble, and to ensure that the leader maintain majority.
			// The active ensemble is the set of running followers connected to the leader.
			numFollowers := s.leader.GetActiveEnsembleSize()
			if numFollowers <= int(ensembleSize/2) {
				// leader looses majority of follower.
				log.Printf("LeaderServer.processRequest(): leader looses majority of follower. Stop client request processing.")
				return nil
			}
		}
	}

	return nil
}

/////////////////////////////////////////////////////////////////////////////
// Private Function for protecting shared state
/////////////////////////////////////////////////////////////////////////////

//
// Notify when server is ready
//
func (s *LeaderServer) notifyReady() {

	s.state.mutex.Lock()
	defer s.state.mutex.Unlock()

	if !s.state.ready {
		s.state.ready = true
		s.state.readych <- true
	}
}

//
// Wait for the ready flag to be set.  This is when the leader has gotten
// the quorum of followers to join/sync.
//
func (s *LeaderServer) waitTillReady() bool {

	timeout := time.After(common.LEADER_TIMEOUT * time.Millisecond)

	select {
		case <- s.state.readych :
			return true

		case <- timeout:
			log.Printf("LeaderServer.waitTillReady(): Leader cannot get quorum of followers to follow before timing out. Termiate.")
			return false
	}
}

//
// Tell if the server is ready
//
func (s *LeaderServer) isReady() bool {
	s.state.mutex.Lock()
	defer s.state.mutex.Unlock()

	return s.state.ready
}

/////////////////////////////////////////////////////////////////////////////
// Private Function for mananaging proxies
/////////////////////////////////////////////////////////////////////////////

// Add Proxy
func (s *LeaderServer) registerOutstandingProxy(key string) bool {
	s.state.mutex.Lock()
	defer s.state.mutex.Unlock()

	_, ok := s.state.proxies[key]
	if !ok {
		killch := make(chan bool, 1) // make it buffered
		s.state.proxies[key] = killch
		return true
	}

	return false
}

//
// Get proxy kill channel
//
func (s *LeaderServer) getProxyKillChan(key string) <-chan bool {
	s.state.mutex.Lock()
	defer s.state.mutex.Unlock()

	killch, ok := s.state.proxies[key]
	if ok {
		return killch
	}

	return nil
}

//
// Get proxy kill channel
//
func (s *LeaderServer) deregisterOutstandingProxy(key string) {
	s.state.mutex.Lock()
	defer s.state.mutex.Unlock()

	delete(s.state.proxies, key)
}

//
// Terminate all proxies
//
func (s *LeaderServer) terminateAllOutstandingProxies() {
	// TODO: Should copy the proxies and release the mutex
	// before sending to the channels
	s.state.mutex.Lock()
	defer s.state.mutex.Unlock()

	for _, killch := range s.state.proxies {
		killch <- true
	}
}

//
// Create the listener state
//
func newListenerState() *ListenerState {
	return &ListenerState{killch : make(chan bool, 1),   // buffered so sender won't block
					      donech : make(chan bool, 1)}   // buffered so sender won't block
}
<commit_msg>add logging statement<commit_after>package protocol

import (
	"github.com/jliang00/gometa/src/common"
	"sync"
	"log"
	"time"
	"runtime/debug"
)

/////////////////////////////////////////////////////////////////////////////
// Type Declaration
/////////////////////////////////////////////////////////////////////////////

type LeaderServer struct {
	leader       	*Leader
	listener     	*common.PeerListener
	consentState 	*ConsentState
	state        	*LeaderState
	handler 	 	 ActionHandler
	factory 	 	 MsgFactory
}

type LeaderState struct {
	requestMgr 		RequestMgr

	// mutex protected variable
	mutex   		sync.Mutex
	ready   		bool
	readych         chan bool
	proxies         map[string](chan bool)
}

type ListenerState struct {
	donech			chan bool
	killch			chan bool
}

/////////////////////////////////////////////////////////////////////////////
// LeaderServer - Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Create a new LeaderServer.  This is a blocking call until the LeaderServer
// termintates.
//
// killch should be unbuffered to ensure the sender won't block
//
func RunLeaderServer(naddr string,
	listener *common.PeerListener,
	ss RequestMgr,
	handler ActionHandler,
	factory MsgFactory,
	killch <- chan bool) (err error) {

	log.Printf("LeaderServer.RunLeaderServer(): start leader server %s", naddr)

	// Catch panic at the main entry point for LeaderServer
	defer func() {
		if r := recover(); r != nil {
			log.Printf("panic in RunLeaderServer() : %s\n", r)
			err = r.(error)
		}

		log.Printf("RunLeaderServer terminates : Diagnostic Stack ...")
		log.Printf("%s", debug.Stack())
	}()

	// create a leader
	leader, err := NewLeader(naddr, handler, factory)
	if err != nil {
		return err
	}
	defer leader.Terminate()

	// create a ConsentState
	epoch, err := handler.GetAcceptedEpoch()
	if err != nil {
		return err
	}
	ensembleSize := handler.GetEnsembleSize()
	consentState := NewConsentState(naddr, epoch, ensembleSize)
	defer consentState.Terminate()

	// create the leader state
	state := newLeaderState(ss)

	// create the server
	server := &LeaderServer{leader: leader,
		listener:     listener,
		consentState: consentState,
		state:        state,
		handler:      handler,
		factory:	  factory}

	// start the listener.  This goroutine would continue to new follower even while
	// it is processing request.
	listenerState := newListenerState()
	go server.listenFollower(listenerState)

	// start the main loop for processing incoming request.  The leader will
	// process request only after it has received quorum of followers to
	// synchronized with it.
	err = server.processRequest(killch, listenerState)

	log.Printf("LeaderServer.RunLeaderServer(): leader server %s terminate", naddr)

	return err
}

/////////////////////////////////////////////////////////////////////////////
// LeaderServer - Private Function : Discovery Phase (synchronize with follower)
/////////////////////////////////////////////////////////////////////////////

//
// Listen to new connection request from the follower/peer.
// Start a new LeaderSyncProxy to synchronize the state
// between the leader and the peer.
//
func (l *LeaderServer) listenFollower(listenerState *ListenerState) {

	defer func() {
		if r := recover(); r != nil {
			log.Printf("panic in LeaderServer.listenFollower() : %s\n", r)
		}

		log.Printf("LeaderServer.listenFollower() terminates : Diagnostic Stack ...")
		log.Printf("%s", debug.Stack())

		common.SafeRun("LeaderServer.listenFollower()",
			func() {
				l.terminateAllOutstandingProxies()
			})

		common.SafeRun("LeaderServer.listenFollower()",
			func() {
				listenerState.donech <- true
			})
	}()

	connCh := l.listener.ConnChannel()
	if connCh == nil {
		// It should not happen unless the listener is closed
		return
	}

	// if there is a single server, then we don't need to wait for follower
	// for the server to be ready to process request.
	if l.handler.GetEnsembleSize() == 1 {
		if err := l.incrementEpoch(); err != nil {
			log.Printf("LeaderServer.listenFollower(): Error when boostraping leader with ensembleSize=1. Error = %s", err)
			return
		}

		l.notifyReady()
	}

	for {
		select {
		case conn := <-connCh:
			{
				// There is a new peer connection request from the follower.  Start a proxy to synchronize with the follower.
				// The leader does not proactively connect to follower:
				// 1) The ensemble is stable, but a follower may just reboot and needs to connect to the leader
				// 2) Even if the leader receives votes from the leader, the leader cannot tell for sure that the follower does
				//    not change its vote.  Only if the follower connects, the leader can confirm the follower's alliance.
				//
				log.Printf("LeaderServer.listenFollower(): Receive connection request from follower %s", conn.RemoteAddr())
				if l.registerOutstandingProxy(conn.RemoteAddr().String()) {
					pipe := common.NewPeerPipe(conn)
					go l.startProxy(pipe)
				} else {
					log.Printf("LeaderServer.listenFollower(): Sync Proxy already running for %s. Ignore new request.", conn.RemoteAddr())
					conn.Close()
				}
			}
		case <- listenerState.killch:
			log.Printf("LeaderServer.listenFollower(): Receive kill signal. Terminate.")
			return
		}
	}
}

//
// Start a LeaderSyncProxy to synchornize the leader
// and follower state.
//
func (l *LeaderServer) startProxy(peer *common.PeerPipe) {

	defer func() {
		if r := recover(); r != nil {
			log.Printf("panic in LeaderServer.startProxy() : %s\n", r)
			log.Printf("LeaderServer.startProxy() : Diagnostic Stack ...")
			log.Printf("%s", debug.Stack())
		}

		// deregister the proxy with the leader Server upon exit
		l.deregisterOutstandingProxy(peer.GetAddr())
	}()

	// create a proxy that will sycnhronize with the peer.
	log.Printf("LeaderServer.startProxy(): Start synchronization with follower. Peer TCP connection (%s)", peer.GetAddr())
	proxy := NewLeaderSyncProxy(l.leader, l.consentState, peer, l.handler, l.factory)
	donech := proxy.GetDoneChannel()

	// Create an observer for the leader.  The leader will put on-going proposal msg and commit msg
	// onto the observer queue.  This ensure that we can won't miss those mutations as the leader is
	// sync'ign withe follower.  The messages in observer queue will eventually route to follower.
	o := NewObserver()
	l.leader.AddObserver(peer.GetAddr(), o)
	defer l.leader.RemoveObserver(peer.GetAddr())

	// start the proxy
	go proxy.Start(o)
	defer proxy.Terminate()

	// Get the killch for this go-routine
	killch := l.getProxyKillChan(peer.GetAddr())
	if killch == nil {
   		log.Printf("LeaderServer.startProxy(): Cannot find killch for proxy (TCP connection = %s).", peer.GetAddr())
   		log.Printf("LeaderServer.startProxy(): Cannot start follower sync.")
   		return
	}

	// this go-routine will be blocked until handshake is completed between the
	// leader and the follower.  By then, the leader will also get majority
	// confirmation that it is a leader.
	select {
		case success := <-donech:
			if success {
				// tell the leader to add this follower for processing request.  If there is a follower running already,
				// AddFollower() will terminate the existing follower instance, and then create a new one.
	    		fid := proxy.GetFid()
	    		if proxy.CanFollowerVote() {
					l.leader.AddFollower(fid, peer, o)
	    			log.Printf("LeaderServer.startProxy(): Synchronization with follower %s done (TCP conn = %s).  Add follower.",
	    				fid, peer.GetAddr())

					// At this point, the follower has voted this server as the leader.
					// Notify the request processor to start processing new request for this host
					l.notifyReady()
				} else {
					l.leader.AddWatcher(fid, peer, o)
	    			log.Printf("LeaderServer.startProxy(): Sync with watcher done.  Add Watcher %s (TCP conn = %s)",
	    				fid, peer.GetAddr())
				}
			} else {
				log.Printf("LeaderServer:startProxy(): Leader Fail to synchronization with follower (TCP conn = %s)", peer.GetAddr())
			}
		case <- killch:
			log.Printf("LeaderServer:startProxy(): Sync proxy is killed while synchronizing with follower (TCP conn == %s)",
					peer.GetAddr())
	}
}

//
// Create a new LeaderState
//
func newLeaderState(ss RequestMgr) *LeaderState {
	state := &LeaderState{requestMgr: ss,
							ready: false,
							readych : make(chan bool, 1), // buffered so sender won't wait
							proxies : make(map[string](chan bool))}

	return state
}

//
// Increment the epoch.  Only call this method if the ensemble size is 1 (single server).
// This function can panic if the epoch reaches its limit.
//
func (l *LeaderServer) incrementEpoch() error {

	epoch, err := l.handler.GetCurrentEpoch()
	if err != nil {
		return err
	}

	epoch = common.CompareAndIncrementEpoch(epoch, epoch)

	log.Printf("LeaderServer.incrementEpoch(): new epoch %d", epoch)

	if err := l.handler.NotifyNewAcceptedEpoch(epoch); err != nil {
		return err
	}

	if err := l.handler.NotifyNewCurrentEpoch(epoch); err != nil {
		return err
	}

	return nil
}

/////////////////////////////////////////////////////////////////////////////
// LeaderServer - Private Function : Broadcast phase (handle request)
/////////////////////////////////////////////////////////////////////////////

//
// Goroutine for processing each request one-by-one
//
func (s *LeaderServer) processRequest(killch <-chan bool,
									  listenerState *ListenerState) (err error) {

	defer func() {
		if r := recover(); r != nil {
			log.Printf("panic in LeaderServer.processRequest() : %s\n", r)
			err = r.(error)

			log.Printf("LeaderServer.processRequest() : Diagnostic Stack ...")
			log.Printf("%s", debug.Stack())
		}

		common.SafeRun("LeaderServer.processRequest()",
			func() {
				listenerState.killch <- true
			})
	}()

	// start processing loop after I am being confirmed as a leader (there
	// is a quorum of followers that have sync'ed with me)
	if !s.waitTillReady() {
		return common.NewError(common.ELECTION_ERROR,
			"LeaderServer.processRequest(): Leader times out waiting for quorum of followers. Terminate")
	}

	// At this point, the leader has gotten a majority of followers to follow, so it
	// can proceed.  It is possible that it may loose quorum of followers. But in that
	// case, the leader will not be able to process any request.
	log.Printf("LeaderServer.processRequest(): Leader Server is ready to proces request")

	// Leader is ready at this time.  This implies that there is a quorum of follower has
	// followed this leader.  Get the change channel to keep track of  number of followers.
	// If the leader no longer has quorum, it needs to let go of its leadership.
	leaderchangech := s.leader.GetEnsembleChangeChannel()
	ensembleSize := s.handler.GetEnsembleSize()

	// notify the request processor to start processing new request
	incomings := s.state.requestMgr.GetRequestChannel()
	for {
		select {
		case handle, ok := <- incomings:
			if ok {
				// de-queue the request
				s.state.requestMgr.AddPendingRequest(handle)

				// create the proposal and forward to the leader
				s.leader.CreateProposal(s.leader.GetFollowerId(), handle.Request)
			} else {
				// server shutdown.
				log.Printf("LeaderServer.processRequest(): channel for receiving client request is closed. Terminate.")
				return nil
			}
		case <-killch:
			// server shutdown
			log.Printf("LeaderServer.processRequest(): receive kill signal. Stop Client request processing.")
			return nil
		case <-listenerState.donech:
			// listener is down.  Terminate this request processing loop as well.
			log.Printf("LeaderServer.processRequest(): follower listener terminates. Stop client request processing.")
			return nil
		case <-leaderchangech:
			// Listen to any change to the leader's active ensemble, and to ensure that the leader maintain majority.
			// The active ensemble is the set of running followers connected to the leader.
			numFollowers := s.leader.GetActiveEnsembleSize()
			if numFollowers <= int(ensembleSize/2) {
				// leader looses majority of follower.
				log.Printf("LeaderServer.processRequest(): leader looses majority of follower. Stop client request processing.")
				return nil
			}
		}
	}

	return nil
}

/////////////////////////////////////////////////////////////////////////////
// Private Function for protecting shared state
/////////////////////////////////////////////////////////////////////////////

//
// Notify when server is ready
//
func (s *LeaderServer) notifyReady() {

	s.state.mutex.Lock()
	defer s.state.mutex.Unlock()

	if !s.state.ready {
		s.state.ready = true
		s.state.readych <- true
	}
}

//
// Wait for the ready flag to be set.  This is when the leader has gotten
// the quorum of followers to join/sync.
//
func (s *LeaderServer) waitTillReady() bool {

	timeout := time.After(common.LEADER_TIMEOUT * time.Millisecond)

	select {
		case <- s.state.readych :
			return true

		case <- timeout:
			log.Printf("LeaderServer.waitTillReady(): Leader cannot get quorum of followers to follow before timing out. Termiate.")
			return false
	}
}

//
// Tell if the server is ready
//
func (s *LeaderServer) isReady() bool {
	s.state.mutex.Lock()
	defer s.state.mutex.Unlock()

	return s.state.ready
}

/////////////////////////////////////////////////////////////////////////////
// Private Function for mananaging proxies
/////////////////////////////////////////////////////////////////////////////

// Add Proxy
func (s *LeaderServer) registerOutstandingProxy(key string) bool {
	s.state.mutex.Lock()
	defer s.state.mutex.Unlock()

	_, ok := s.state.proxies[key]
	if !ok {
		killch := make(chan bool, 1) // make it buffered
		s.state.proxies[key] = killch
		return true
	}

	return false
}

//
// Get proxy kill channel
//
func (s *LeaderServer) getProxyKillChan(key string) <-chan bool {
	s.state.mutex.Lock()
	defer s.state.mutex.Unlock()

	killch, ok := s.state.proxies[key]
	if ok {
		return killch
	}

	return nil
}

//
// Get proxy kill channel
//
func (s *LeaderServer) deregisterOutstandingProxy(key string) {
	s.state.mutex.Lock()
	defer s.state.mutex.Unlock()

	delete(s.state.proxies, key)
}

//
// Terminate all proxies
//
func (s *LeaderServer) terminateAllOutstandingProxies() {
	// TODO: Should copy the proxies and release the mutex
	// before sending to the channels
	s.state.mutex.Lock()
	defer s.state.mutex.Unlock()

	for _, killch := range s.state.proxies {
		killch <- true
	}
}

//
// Create the listener state
//
func newListenerState() *ListenerState {
	return &ListenerState{killch : make(chan bool, 1),   // buffered so sender won't block
					      donech : make(chan bool, 1)}   // buffered so sender won't block
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"

	"github.com/pachyderm/pachyderm/src/client"
	healthclient "github.com/pachyderm/pachyderm/src/client/health"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/discovery"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pkg/shard"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/health"
	pfs_persist "github.com/pachyderm/pachyderm/src/server/pfs/db"
	"github.com/pachyderm/pachyderm/src/server/pfs/drive"
	pfs_server "github.com/pachyderm/pachyderm/src/server/pfs/server"
	cache_pb "github.com/pachyderm/pachyderm/src/server/pkg/cache/groupcachepb"
	cache_server "github.com/pachyderm/pachyderm/src/server/pkg/cache/server"
	"github.com/pachyderm/pachyderm/src/server/pkg/metrics"
	"github.com/pachyderm/pachyderm/src/server/pkg/netutil"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps"
	"github.com/pachyderm/pachyderm/src/server/pps/persist"
	persist_server "github.com/pachyderm/pachyderm/src/server/pps/persist/server"
	pps_server "github.com/pachyderm/pachyderm/src/server/pps/server"

	flag "github.com/spf13/pflag"
	"go.pedge.io/env"
	"go.pedge.io/lion"
	"go.pedge.io/lion/proto"
	"go.pedge.io/proto/server"
	"google.golang.org/grpc"
	"k8s.io/kubernetes/pkg/api"
	kube_client "k8s.io/kubernetes/pkg/client/restclient"
	kube "k8s.io/kubernetes/pkg/client/unversioned"
)

var readinessCheck bool
var migrate string

func init() {
	flag.BoolVar(&readinessCheck, "readiness-check", false, "Set to true when checking if local pod is ready")
	flag.StringVar(&migrate, "migrate", "", "Use the format FROM_VERSION-TO_VERSION; e.g. 1.2.4-1.3.0")
	flag.Parse()
}

type appEnv struct {
	Port               uint16 `env:"PORT,default=650"`
	NumShards          uint64 `env:"NUM_SHARDS,default=32"`
	StorageRoot        string `env:"PACH_ROOT,default=/pach"`
	StorageBackend     string `env:"STORAGE_BACKEND,default="`
	DatabaseAddress    string `env:"RETHINK_PORT_28015_TCP_ADDR,required"`
	PPSDatabaseName    string `env:"DATABASE_NAME,default=pachyderm_pps"`
	PFSDatabaseName    string `env:"DATABASE_NAME,default=pachyderm_pfs"`
	KubeAddress        string `env:"KUBERNETES_PORT_443_TCP_ADDR,required"`
	EtcdAddress        string `env:"ETCD_PORT_2379_TCP_ADDR,required"`
	Namespace          string `env:"NAMESPACE,default=default"`
	Metrics            bool   `env:"METRICS,default=true"`
	Init               bool   `env:"INIT,default=false"`
	BlockCacheBytes    int64  `env:"BLOCK_CACHE_BYTES,default=1073741824"` //default = 1 gigabyte
	JobShimImage       string `env:"JOB_SHIM_IMAGE,default="`
	JobImagePullPolicy string `env:"JOB_IMAGE_PULL_POLICY,default="`
	LogLevel           string `env:"LOG_LEVEL,default=info"`
}

func main() {
	env.Main(do, &appEnv{})
}

func do(appEnvObj interface{}) error {
	go func() {
		lion.Println(http.ListenAndServe(":651", nil))
	}()
	appEnv := appEnvObj.(*appEnv)
	switch appEnv.LogLevel {
	case "debug":
		lion.SetLevel(lion.LevelDebug)
	case "info":
		lion.SetLevel(lion.LevelInfo)
	case "error":
		lion.SetLevel(lion.LevelError)
	default:
		lion.Errorf("Unrecognized log level %s, falling back to default of \"info\"", appEnv.LogLevel)
		lion.SetLevel(lion.LevelInfo)
	}
	etcdClient := getEtcdClient(appEnv)
	rethinkAddress := fmt.Sprintf("%s:28015", appEnv.DatabaseAddress)
	if appEnv.Init {
		if err := setClusterID(etcdClient); err != nil {
			return fmt.Errorf("error connecting to etcd, if this error persists it likely indicates that kubernetes services are not working correctly. See https://github.com/pachyderm/pachyderm/blob/master/SETUP.md#pachd-or-pachd-init-crash-loop-with-error-connecting-to-etcd for more info")
		}
		if err := persist_server.InitDBs(rethinkAddress, appEnv.PPSDatabaseName); err != nil {
			return err
		}
		return pfs_persist.InitDB(rethinkAddress, appEnv.PFSDatabaseName)
	}
	if readinessCheck {
		c, err := client.NewFromAddress("127.0.0.1:650")
		if err != nil {
			return err
		}

		// We want to use a PPS API instead of a PFS API because PFS APIs
		// typically talk to every node, but the point of the readiness probe
		// is that it checks to see if this particular node is functioning,
		// and removing it from the service if it's not.  So if we use a PFS
		// API such as ListRepo for readiness probe, then the failure of any
		// node will result in the failures of all readiness probes, causing
		// all nodes to be removed from the pachd service.
		_, err = c.ListPipeline()
		if err != nil {
			return err
		}

		os.Exit(0)

		return nil
	}
	if migrate != "" {
		return persist_server.Migrate(rethinkAddress, appEnv.PPSDatabaseName, migrate)
	}

	clusterID, err := getClusterID(etcdClient)
	if err != nil {
		return err
	}
	kubeClient, err := getKubeClient(appEnv)
	if err != nil {
		return err
	}
	var reporter *metrics.Reporter
	if appEnv.Metrics {
		reporter = metrics.NewReporter(clusterID, kubeClient, rethinkAddress, appEnv.PFSDatabaseName, appEnv.PPSDatabaseName)
	}
	rethinkAPIServer, err := getRethinkAPIServer(appEnv)
	if err != nil {
		return err
	}
	address, err := netutil.ExternalIP()
	if err != nil {
		return err
	}
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	sharder := shard.NewSharder(
		etcdClient,
		appEnv.NumShards,
		appEnv.Namespace,
	)
	go func() {
		if err := sharder.AssignRoles(address, nil); err != nil {
			protolion.Printf("error from sharder.AssignRoles: %s", sanitizeErr(err))
		}
	}()
	driver, err := getPFSDriver(address, appEnv)
	//	driver, err := drive.NewDriver(address)
	if err != nil {
		return err
	}
	router := shard.NewRouter(
		sharder,
		grpcutil.NewDialer(
			grpc.WithInsecure(),
		),
		address,
	)
	cacheServer := cache_server.NewCacheServer(router, appEnv.NumShards)
	go func() {
		if err := sharder.RegisterFrontends(nil, address, []shard.Frontend{cacheServer}); err != nil {
			protolion.Printf("error from sharder.RegisterFrontend %s", sanitizeErr(err))
		}
	}()
	apiServer := pfs_server.NewAPIServer(driver, reporter)
	ppsAPIServer := pps_server.NewAPIServer(
		ppsserver.NewHasher(appEnv.NumShards, appEnv.NumShards),
		address,
		kubeClient,
		getNamespace(),
		appEnv.JobShimImage,
		appEnv.JobImagePullPolicy,
		reporter,
	)
	go func() {
		if err := sharder.Register(nil, address, []shard.Server{ppsAPIServer, cacheServer}); err != nil {
			protolion.Printf("error from sharder.Register %s", sanitizeErr(err))
		}
	}()
	blockAPIServer, err := pfs_server.NewBlockAPIServer(appEnv.StorageRoot, appEnv.BlockCacheBytes, appEnv.StorageBackend)
	if err != nil {
		return err
	}
	healthServer := health.NewHealthServer()
	return protoserver.Serve(
		func(s *grpc.Server) {
			pfsclient.RegisterAPIServer(s, apiServer)
			pfsclient.RegisterBlockAPIServer(s, blockAPIServer)
			ppsclient.RegisterAPIServer(s, ppsAPIServer)
			ppsserver.RegisterInternalPodAPIServer(s, ppsAPIServer)
			persist.RegisterAPIServer(s, rethinkAPIServer)
			cache_pb.RegisterGroupCacheServer(s, cacheServer)
			healthclient.RegisterHealthServer(s, healthServer)
		},
		protoserver.ServeOptions{
			Version: version.Version,
		},
		protoserver.ServeEnv{
			GRPCPort: appEnv.Port,
		},
	)
}

func getEtcdClient(env *appEnv) discovery.Client {
	return discovery.NewEtcdClient(fmt.Sprintf("http://%s:2379", env.EtcdAddress))
}

const clusterIDKey = "cluster-id"

func setClusterID(client discovery.Client) error {
	return client.Set(clusterIDKey, uuid.NewWithoutDashes(), 0)
}

func getClusterID(client discovery.Client) (string, error) {
	id, err := client.Get(clusterIDKey)
	if err != nil {
		return "", err
	}
	if id == "" {
		return "", fmt.Errorf("clusterID not yet set")
	}
	return id, nil
}

func getKubeClient(env *appEnv) (*kube.Client, error) {
	kubeClient, err := kube.NewInCluster()
	if err != nil {
		protolion.Errorf("falling back to insecure kube client due to error from NewInCluster: %s", sanitizeErr(err))
	} else {
		return kubeClient, err
	}
	config := &kube_client.Config{
		Host:     fmt.Sprintf("%s:443", env.KubeAddress),
		Insecure: true,
	}
	return kube.New(config)
}

func getPFSDriver(address string, env *appEnv) (drive.Driver, error) {
	rethinkAddress := fmt.Sprintf("%s:28015", env.DatabaseAddress)
	return pfs_persist.NewDriver(address, rethinkAddress, env.PFSDatabaseName)
}

func getRethinkAPIServer(env *appEnv) (persist.APIServer, error) {
	if err := persist_server.CheckDBs(fmt.Sprintf("%s:28015", env.DatabaseAddress), env.PPSDatabaseName); err != nil {
		return nil, err
	}
	return persist_server.NewRethinkAPIServer(fmt.Sprintf("%s:28015", env.DatabaseAddress), env.PPSDatabaseName)
}

// getNamespace returns the kubernetes namespace that this pachd pod runs in
func getNamespace() string {
	namespace := os.Getenv("PACHD_POD_NAMESPACE")
	if namespace != "" {
		return namespace
	}
	return api.NamespaceDefault
}

func sanitizeErr(err error) error {
	if err == nil {
		return nil
	}

	return errors.New(grpc.ErrorDesc(err))
}
<commit_msg>Makes pachd set the cluster-id if it's unset.<commit_after>package main

import (
	"errors"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strings"

	"github.com/pachyderm/pachyderm/src/client"
	healthclient "github.com/pachyderm/pachyderm/src/client/health"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/discovery"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pkg/shard"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/health"
	pfs_persist "github.com/pachyderm/pachyderm/src/server/pfs/db"
	"github.com/pachyderm/pachyderm/src/server/pfs/drive"
	pfs_server "github.com/pachyderm/pachyderm/src/server/pfs/server"
	cache_pb "github.com/pachyderm/pachyderm/src/server/pkg/cache/groupcachepb"
	cache_server "github.com/pachyderm/pachyderm/src/server/pkg/cache/server"
	"github.com/pachyderm/pachyderm/src/server/pkg/metrics"
	"github.com/pachyderm/pachyderm/src/server/pkg/netutil"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps"
	"github.com/pachyderm/pachyderm/src/server/pps/persist"
	persist_server "github.com/pachyderm/pachyderm/src/server/pps/persist/server"
	pps_server "github.com/pachyderm/pachyderm/src/server/pps/server"

	flag "github.com/spf13/pflag"
	"go.pedge.io/env"
	"go.pedge.io/lion"
	"go.pedge.io/lion/proto"
	"go.pedge.io/proto/server"
	"google.golang.org/grpc"
	"k8s.io/kubernetes/pkg/api"
	kube_client "k8s.io/kubernetes/pkg/client/restclient"
	kube "k8s.io/kubernetes/pkg/client/unversioned"
)

var readinessCheck bool
var migrate string

func init() {
	flag.BoolVar(&readinessCheck, "readiness-check", false, "Set to true when checking if local pod is ready")
	flag.StringVar(&migrate, "migrate", "", "Use the format FROM_VERSION-TO_VERSION; e.g. 1.2.4-1.3.0")
	flag.Parse()
}

type appEnv struct {
	Port               uint16 `env:"PORT,default=650"`
	NumShards          uint64 `env:"NUM_SHARDS,default=32"`
	StorageRoot        string `env:"PACH_ROOT,default=/pach"`
	StorageBackend     string `env:"STORAGE_BACKEND,default="`
	DatabaseAddress    string `env:"RETHINK_PORT_28015_TCP_ADDR,required"`
	PPSDatabaseName    string `env:"DATABASE_NAME,default=pachyderm_pps"`
	PFSDatabaseName    string `env:"DATABASE_NAME,default=pachyderm_pfs"`
	KubeAddress        string `env:"KUBERNETES_PORT_443_TCP_ADDR,required"`
	EtcdAddress        string `env:"ETCD_PORT_2379_TCP_ADDR,required"`
	Namespace          string `env:"NAMESPACE,default=default"`
	Metrics            bool   `env:"METRICS,default=true"`
	Init               bool   `env:"INIT,default=false"`
	BlockCacheBytes    int64  `env:"BLOCK_CACHE_BYTES,default=1073741824"` //default = 1 gigabyte
	JobShimImage       string `env:"JOB_SHIM_IMAGE,default="`
	JobImagePullPolicy string `env:"JOB_IMAGE_PULL_POLICY,default="`
	LogLevel           string `env:"LOG_LEVEL,default=info"`
}

func main() {
	env.Main(do, &appEnv{})
}

func do(appEnvObj interface{}) error {
	go func() {
		lion.Println(http.ListenAndServe(":651", nil))
	}()
	appEnv := appEnvObj.(*appEnv)
	switch appEnv.LogLevel {
	case "debug":
		lion.SetLevel(lion.LevelDebug)
	case "info":
		lion.SetLevel(lion.LevelInfo)
	case "error":
		lion.SetLevel(lion.LevelError)
	default:
		lion.Errorf("Unrecognized log level %s, falling back to default of \"info\"", appEnv.LogLevel)
		lion.SetLevel(lion.LevelInfo)
	}
	etcdClient := getEtcdClient(appEnv)
	rethinkAddress := fmt.Sprintf("%s:28015", appEnv.DatabaseAddress)
	if appEnv.Init {
		if err := persist_server.InitDBs(rethinkAddress, appEnv.PPSDatabaseName); err != nil {
			return err
		}
		return pfs_persist.InitDB(rethinkAddress, appEnv.PFSDatabaseName)
	}
	if readinessCheck {
		c, err := client.NewFromAddress("127.0.0.1:650")
		if err != nil {
			return err
		}

		// We want to use a PPS API instead of a PFS API because PFS APIs
		// typically talk to every node, but the point of the readiness probe
		// is that it checks to see if this particular node is functioning,
		// and removing it from the service if it's not.  So if we use a PFS
		// API such as ListRepo for readiness probe, then the failure of any
		// node will result in the failures of all readiness probes, causing
		// all nodes to be removed from the pachd service.
		_, err = c.ListPipeline()
		if err != nil {
			return err
		}

		os.Exit(0)

		return nil
	}
	if migrate != "" {
		return persist_server.Migrate(rethinkAddress, appEnv.PPSDatabaseName, migrate)
	}

	clusterID, err := getClusterID(etcdClient)
	if err != nil {
		return err
	}
	kubeClient, err := getKubeClient(appEnv)
	if err != nil {
		return err
	}
	var reporter *metrics.Reporter
	if appEnv.Metrics {
		reporter = metrics.NewReporter(clusterID, kubeClient, rethinkAddress, appEnv.PFSDatabaseName, appEnv.PPSDatabaseName)
	}
	rethinkAPIServer, err := getRethinkAPIServer(appEnv)
	if err != nil {
		return err
	}
	address, err := netutil.ExternalIP()
	if err != nil {
		return err
	}
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	sharder := shard.NewSharder(
		etcdClient,
		appEnv.NumShards,
		appEnv.Namespace,
	)
	go func() {
		if err := sharder.AssignRoles(address, nil); err != nil {
			protolion.Printf("error from sharder.AssignRoles: %s", sanitizeErr(err))
		}
	}()
	driver, err := getPFSDriver(address, appEnv)
	//	driver, err := drive.NewDriver(address)
	if err != nil {
		return err
	}
	router := shard.NewRouter(
		sharder,
		grpcutil.NewDialer(
			grpc.WithInsecure(),
		),
		address,
	)
	cacheServer := cache_server.NewCacheServer(router, appEnv.NumShards)
	go func() {
		if err := sharder.RegisterFrontends(nil, address, []shard.Frontend{cacheServer}); err != nil {
			protolion.Printf("error from sharder.RegisterFrontend %s", sanitizeErr(err))
		}
	}()
	apiServer := pfs_server.NewAPIServer(driver, reporter)
	ppsAPIServer := pps_server.NewAPIServer(
		ppsserver.NewHasher(appEnv.NumShards, appEnv.NumShards),
		address,
		kubeClient,
		getNamespace(),
		appEnv.JobShimImage,
		appEnv.JobImagePullPolicy,
		reporter,
	)
	go func() {
		if err := sharder.Register(nil, address, []shard.Server{ppsAPIServer, cacheServer}); err != nil {
			protolion.Printf("error from sharder.Register %s", sanitizeErr(err))
		}
	}()
	blockAPIServer, err := pfs_server.NewBlockAPIServer(appEnv.StorageRoot, appEnv.BlockCacheBytes, appEnv.StorageBackend)
	if err != nil {
		return err
	}
	healthServer := health.NewHealthServer()
	return protoserver.Serve(
		func(s *grpc.Server) {
			pfsclient.RegisterAPIServer(s, apiServer)
			pfsclient.RegisterBlockAPIServer(s, blockAPIServer)
			ppsclient.RegisterAPIServer(s, ppsAPIServer)
			ppsserver.RegisterInternalPodAPIServer(s, ppsAPIServer)
			persist.RegisterAPIServer(s, rethinkAPIServer)
			cache_pb.RegisterGroupCacheServer(s, cacheServer)
			healthclient.RegisterHealthServer(s, healthServer)
		},
		protoserver.ServeOptions{
			Version: version.Version,
		},
		protoserver.ServeEnv{
			GRPCPort: appEnv.Port,
		},
	)
}

func getEtcdClient(env *appEnv) discovery.Client {
	return discovery.NewEtcdClient(fmt.Sprintf("http://%s:2379", env.EtcdAddress))
}

const clusterIDKey = "cluster-id"

func getClusterID(client discovery.Client) (string, error) {
	id, err := client.Get(clusterIDKey)
	// if it's a key not found error then we create the key
	if err != nil && strings.HasPrefix(err.Error(), "100:") {
		// This might error if it races with another pachd trying to set the
		// cluster id so we ignore the error.
		client.Create(clusterIDKey, uuid.NewWithoutDashes(), 0)
	} else if err != nil {
		return "", err
	} else {
		return id, nil
	}
	return client.Get(clusterIDKey)
}

func getKubeClient(env *appEnv) (*kube.Client, error) {
	kubeClient, err := kube.NewInCluster()
	if err != nil {
		protolion.Errorf("falling back to insecure kube client due to error from NewInCluster: %s", sanitizeErr(err))
	} else {
		return kubeClient, err
	}
	config := &kube_client.Config{
		Host:     fmt.Sprintf("%s:443", env.KubeAddress),
		Insecure: true,
	}
	return kube.New(config)
}

func getPFSDriver(address string, env *appEnv) (drive.Driver, error) {
	rethinkAddress := fmt.Sprintf("%s:28015", env.DatabaseAddress)
	return pfs_persist.NewDriver(address, rethinkAddress, env.PFSDatabaseName)
}

func getRethinkAPIServer(env *appEnv) (persist.APIServer, error) {
	if err := persist_server.CheckDBs(fmt.Sprintf("%s:28015", env.DatabaseAddress), env.PPSDatabaseName); err != nil {
		return nil, err
	}
	return persist_server.NewRethinkAPIServer(fmt.Sprintf("%s:28015", env.DatabaseAddress), env.PPSDatabaseName)
}

// getNamespace returns the kubernetes namespace that this pachd pod runs in
func getNamespace() string {
	namespace := os.Getenv("PACHD_POD_NAMESPACE")
	if namespace != "" {
		return namespace
	}
	return api.NamespaceDefault
}

func sanitizeErr(err error) error {
	if err == nil {
		return nil
	}

	return errors.New(grpc.ErrorDesc(err))
}
<|endoftext|>


<commit_before>package mm

import (
	"fmt"
	"github.com/freeformz/shh/config"
	"strings"
	"time"
)

type Measurement struct {
	When   time.Time
	Poller string
	What   []string
	Value  interface{}
}

func (m *Measurement) String() string {
	msg := fmt.Sprintf("when=%s sample#%s=%s", m.Timestamp(), m.Measured(), m.SValue())
	if config.Source != "" {
		return fmt.Sprintf("%s source=%s", msg, config.Source)
	}
	return msg
}

func (m *Measurement) SValue() string {
	switch m.Value.(type) {
	case float64:
		return fmt.Sprintf("%f", m.Value.(float64))
	case uint64:
		return fmt.Sprintf("%d", m.Value.(uint64))
	}
	return ""
}

func (m *Measurement) Measured() string {
	v := fmt.Sprintf("%s.%s", m.Poller, strings.Join(m.What, "."))
	if config.Prefix != "" {
		v = fmt.Sprintf("%s.%s", config.Prefix, v)
	}
	return v
}

func (m *Measurement) Source() string {
	return config.Source
}

func (current *Measurement) Difference(last *Measurement) uint64 {
	// This is a crappy way to handle wraps and resets when we don't know
	// what the max value is (32, 64 or 128 bit)
	// Leads to a little, loss, but should be minimal overall
	cv := current.Value.(uint64)
	lv := last.Value.(uint64)
	if cv < lv {
		return cv
	}
	return cv - lv
}

func (m *Measurement) Timestamp() string {
	return m.When.Format(time.RFC3339)
}

func (m *Measurement) UnixNano() int64 {
	return m.When.UnixNano()
}

func (m *Measurement) Unix() int64 {
	return m.When.Unix()
}
<commit_msg>Normalize metric names<commit_after>package mm

import (
	"fmt"
	"github.com/freeformz/shh/config"
	"strings"
	"time"
)

var (
	MetricNameNormalizer = strings.NewReplacer("#","_","-","_")
)

type Measurement struct {
	When   time.Time
	Poller string
	What   []string
	Value  interface{}
}

func (m *Measurement) String() string {
	msg := fmt.Sprintf("when=%s sample#%s=%s", m.Timestamp(), m.Measured(), m.SValue())
	if config.Source != "" {
		return fmt.Sprintf("%s source=%s", msg, config.Source)
	}
	return msg
}

func (m *Measurement) SValue() string {
	switch m.Value.(type) {
	case float64:
		return fmt.Sprintf("%f", m.Value.(float64))
	case uint64:
		return fmt.Sprintf("%d", m.Value.(uint64))
	}
	return ""
}

func (m *Measurement) Measured() string {
	v := fmt.Sprintf("%s.%s", m.Poller, strings.Join(m.What, "."))
	if config.Prefix != "" {
		v = fmt.Sprintf("%s.%s", config.Prefix, v)
	}
	return MetricNameNormalizer.Replace(v)
}

func (m *Measurement) Source() string {
	return config.Source
}

func (current *Measurement) Difference(last *Measurement) uint64 {
	// This is a crappy way to handle wraps and resets when we don't know
	// what the max value is (32, 64 or 128 bit)
	// Leads to a little, loss, but should be minimal overall
	cv := current.Value.(uint64)
	lv := last.Value.(uint64)
	if cv < lv {
		return cv
	}
	return cv - lv
}

func (m *Measurement) Timestamp() string {
	return m.When.Format(time.RFC3339)
}

func (m *Measurement) UnixNano() int64 {
	return m.When.UnixNano()
}

func (m *Measurement) Unix() int64 {
	return m.When.Unix()
}
<|endoftext|>


<commit_before>package soracom

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"strings"
	"testing"
)

func TestTextPlainApiError(t *testing.T) {
	r := ioutil.NopCloser(strings.NewReader("test"))
	h := http.Header{}
	h.Set("Content-Type", "text/plain")
	res := &http.Response{
		StatusCode: 404,
		Body:       r,
		Header:     h,
	}

	apiError := NewAPIError(res)
	if apiError.HTTPStatusCode != 404 {
		t.Fatalf("wrong http status code: %v", apiError.HTTPStatusCode)
	}

	if apiError.ErrorCode != "UNK0001" {
		t.Fatalf("wrong error code: %v", apiError.ErrorCode)
	}
}

func TestJsonClientApiError(t *testing.T) {
	json := `{"code":"SEM0095"}`
	r := ioutil.NopCloser(bytes.NewReader([]byte(json)))
	h := http.Header{}
	h.Set("Content-Type", "application/json")
	res := &http.Response{
		StatusCode: 404,
		Body:       r,
		Header:     h,
	}

	apiError := NewAPIError(res)
	if apiError.HTTPStatusCode != 404 {
		t.Fatalf("wrong http status code: %v", apiError.HTTPStatusCode)
	}

	if apiError.ErrorCode != "SEM0095" {
		t.Fatalf("wrong error code: %v", apiError.ErrorCode)
	}
}

func TestJsonServerApiError(t *testing.T) {
	json := `{}`
	r := ioutil.NopCloser(bytes.NewReader([]byte(json)))
	h := http.Header{}
	h.Set("Content-Type", "application/json")
	res := &http.Response{
		StatusCode: 500,
		Body:       r,
		Header:     h,
	}

	apiError := NewAPIError(res)
	if apiError.HTTPStatusCode != 500 {
		t.Fatalf("wrong http status code: %v", apiError.HTTPStatusCode)
	}

	if apiError.ErrorCode != "" {
		t.Fatalf("wrong error code: %v", apiError.ErrorCode)
	}
}

func TestNotSupportedContentTypeApiError(t *testing.T) {
	h := http.Header{}
	h.Set("Content-Type", "application/pdf")
	res := &http.Response{
		StatusCode: 415,
		Header:     h,
	}

	apiError := NewAPIError(res)
	if apiError.HTTPStatusCode != 415 {
		t.Fatalf("wrong http status code: %v", apiError.HTTPStatusCode)
	}

	if apiError.ErrorCode != "INT0001" {
		t.Fatalf("wrong error code: %v", apiError.ErrorCode)
	}
}
<commit_msg>replace deprecated ioutil package with io package<commit_after>package soracom

import (
	"bytes"
	"io"
	"net/http"
	"strings"
	"testing"
)

func TestTextPlainApiError(t *testing.T) {
	r := io.NopCloser(strings.NewReader("test"))
	h := http.Header{}
	h.Set("Content-Type", "text/plain")
	res := &http.Response{
		StatusCode: 404,
		Body:       r,
		Header:     h,
	}

	apiError := NewAPIError(res)
	if apiError.HTTPStatusCode != 404 {
		t.Fatalf("wrong http status code: %v", apiError.HTTPStatusCode)
	}

	if apiError.ErrorCode != "UNK0001" {
		t.Fatalf("wrong error code: %v", apiError.ErrorCode)
	}
}

func TestJsonClientApiError(t *testing.T) {
	json := `{"code":"SEM0095"}`
	r := io.NopCloser(bytes.NewReader([]byte(json)))
	h := http.Header{}
	h.Set("Content-Type", "application/json")
	res := &http.Response{
		StatusCode: 404,
		Body:       r,
		Header:     h,
	}

	apiError := NewAPIError(res)
	if apiError.HTTPStatusCode != 404 {
		t.Fatalf("wrong http status code: %v", apiError.HTTPStatusCode)
	}

	if apiError.ErrorCode != "SEM0095" {
		t.Fatalf("wrong error code: %v", apiError.ErrorCode)
	}
}

func TestJsonServerApiError(t *testing.T) {
	json := `{}`
	r := io.NopCloser(bytes.NewReader([]byte(json)))
	h := http.Header{}
	h.Set("Content-Type", "application/json")
	res := &http.Response{
		StatusCode: 500,
		Body:       r,
		Header:     h,
	}

	apiError := NewAPIError(res)
	if apiError.HTTPStatusCode != 500 {
		t.Fatalf("wrong http status code: %v", apiError.HTTPStatusCode)
	}

	if apiError.ErrorCode != "" {
		t.Fatalf("wrong error code: %v", apiError.ErrorCode)
	}
}

func TestNotSupportedContentTypeApiError(t *testing.T) {
	h := http.Header{}
	h.Set("Content-Type", "application/pdf")
	res := &http.Response{
		StatusCode: 415,
		Header:     h,
	}

	apiError := NewAPIError(res)
	if apiError.HTTPStatusCode != 415 {
		t.Fatalf("wrong http status code: %v", apiError.HTTPStatusCode)
	}

	if apiError.ErrorCode != "INT0001" {
		t.Fatalf("wrong error code: %v", apiError.ErrorCode)
	}
}
<|endoftext|>


<commit_before>package core

import (
	"log"
	"os"
	"path/filepath"
)

/*Model TODO

TODO tracked has bad performance once very large - replace with struct? Size
argument in make seems not to make a difference.
*/
type Model struct {
	root       string
	tracked    map[string]bool
	objinfo    map[string]staticinfo
	updatechan chan UpdateMessage
}

/*Objectinfo todo*/
type Objectinfo struct {
	directory      bool
	Identification string
	Name           string
	Path           string
	Shadow         bool
	Version        map[string]int
	// Objects        []*Objectinfo `json:",omitempty"`
	Content string `json:",omitempty"`
}

/*
staticinfo stores all information that Tinzenite must keep between calls to
m.Update(). This includes the object ID and version for reapplication, plus
the content hash if required for file content changes detection.
*/
type staticinfo struct {
	Identification string
	Version        map[string]int
	Directory      bool
	Content        string
}

/*LoadModel todo*/
func LoadModel(path string) (*Model, error) {
	if !IsTinzenite(path) {
		return nil, ErrNotTinzenite
	}
	/*TODO load if model available*/
	// what follows is the code for a NEW model
	m := &Model{
		root:    path,
		tracked: make(map[string]bool),
		objinfo: make(map[string]staticinfo)}
	// build first version (note that updatechan can't possibly be set already, so we won't spam UpdateMessages)
	err := m.Update()
	if err != nil {
		return nil, err
	}
	return m, nil
}

/*
Update the complete model state.
TODO: check concurrency allowances?
*/
func (m *Model) Update() error {
	current, err := m.populate()
	var removed, created []string
	if err != nil {
		return err
	}
	for path := range m.tracked {
		_, ok := current[path]
		if ok {
			// paths that still exist must only be checked for MODIFY
			delete(current, path)
			m.apply(Modify, path)
		} else {
			// REMOVED - paths that don't exist anymore have been removed
			removed = append(removed, path)
		}
	}
	// CREATED - any remaining paths are yet untracked in m.tracked
	for path := range current {
		created = append(created, path)
	}
	// update m.tracked
	for _, path := range removed {
		delete(m.tracked, path)
		m.apply(Remove, path)
	}
	for _, path := range created {
		m.tracked[path] = true
		m.apply(Create, path)
	}
	return nil
}

/*
Register the channel over which UpdateMessage can be received. Tinzenite will
only ever write to this channel, never read.
*/
func (m *Model) Register(v chan UpdateMessage) {
	m.updatechan = v
}

/*Read todo*/
func (m *Model) Read() (*Objectinfo, error) {
	/*TODO*/
	return nil, ErrUnsupported
}

func (m *Model) store() error {
	return ErrUnsupported
}

/*
getInfo creates the Objectinfo for the given path, so long as the path is
contained in m.tracked. Directories are NOT traversed!
*/
func (m *Model) getInfo(path string) (*Objectinfo, error) {
	_, exists := m.tracked[path]
	if !exists {
		return nil, ErrUntracked
	}
	stat, err := os.Lstat(path)
	if err != nil {
		return nil, err
	}
	// TODO lots still to do here!
	object := &Objectinfo{Path: path}
	if stat.IsDir() {
		object.directory = true
	}
	// TODO apply staticinfo!
	return object, ErrUnsupported
}

/*
populate a map[path] for the m.root path. Applies the root Matcher if provided.
*/
func (m *Model) populate() (map[string]bool, error) {
	match, err := CreateMatcher(m.root)
	if err != nil {
		return nil, err
	}
	tracked := make(map[string]bool)
	filepath.Walk(m.root, func(subpath string, stat os.FileInfo, inerr error) error {
		// ignore on match
		if match.Ignore(subpath) {
			// SkipDir is okay even if file
			if stat.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}
		tracked[subpath] = true
		return nil
	})
	// doesn't directly assign to m.tracked on purpose so that we can reuse this
	// method elsewhere
	return tracked, nil
}

/*
Apply changes to the internal model state. This method does the true logic on the
model, not touching m.tracked. NEVER call this method outside of m.Update()!
*/
func (m *Model) apply(op Operation, path string) {
	notify := false
	var infoToNotify staticinfo
	switch op {
	case Create:
		notify = true
		// fetch all values we'll need to store
		id, err := newIdentifier()
		if err != nil {
			log.Println(err.Error())
			return
		}
		stat, err := os.Lstat(path)
		if err != nil {
			log.Println(err.Error())
			return
		}
		hash := ""
		if !stat.IsDir() {
			hash, err = contentHash(path)
			if err != nil {
				log.Println(err.Error())
				return
			}
		}
		stin := staticinfo{
			Identification: id,
			Version:        make(map[string]int),
			Directory:      stat.IsDir(),
			Content:        hash}
		m.objinfo[path] = stin
		infoToNotify = stin
	case Modify:
		stin, ok := m.objinfo[path]
		if !ok {
			log.Println("staticinfo lookup failed!")
			return
		}
		// no need for further work here
		if stin.Directory {
			return
		}
		hash, err := contentHash(path)
		if err != nil {
			log.Println(err.Error())
			return
		}
		// if same --> no changes, so done
		if hash == stin.Content {
			return
		}
		// otherwise a change has happened
		notify = true
		// update
		stin.Content = hash
		m.objinfo[path] = stin
		// TODO update version
		infoToNotify = stin
	case Remove:
		notify = true
		var ok bool
		infoToNotify, ok = m.objinfo[path]
		if !ok {
			log.Println("staticinfo lookup failed!")
			notify = false
		}
		delete(m.objinfo, path)
	default:
		log.Printf("Unimplemented %s for now!\n", op)
	}
	// send the update message
	if notify && m.updatechan != nil {
		/*TODO select with default --> lost message? but we loose every update
		after the first... hm*/
		m.updatechan <- UpdateMessage{Operation: op, Object: infoToNotify}
	}
}

/*TODO for now only lists all tracked files*/
func (m *Model) String() string {
	var list string
	for path := range m.tracked {
		list += path + "\n"
	}
	return list
}
<commit_msg>model is now stored on every update to .tinzenite/local/model.json<commit_after>package core

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

/*Model TODO

TODO Tracked has bad performance once very large - replace with struct? Size
argument in make seems not to make a difference.
*/
type Model struct {
	Root       string
	Tracked    map[string]bool
	Objinfo    map[string]staticinfo
	updatechan chan UpdateMessage
}

/*Objectinfo todo*/
type Objectinfo struct {
	directory      bool
	Identification string
	Name           string
	Path           string
	Shadow         bool
	Version        map[string]int
	// Objects        []*Objectinfo `json:",omitempty"`
	Content string `json:",omitempty"`
}

/*
staticinfo stores all information that Tinzenite must keep between calls to
m.Update(). This includes the object ID and version for reapplication, plus
the content hash if required for file content changes detection.
*/
type staticinfo struct {
	Identification string
	Version        map[string]int
	Directory      bool
	Content        string
}

/*LoadModel todo*/
func LoadModel(path string) (*Model, error) {
	if !IsTinzenite(path) {
		return nil, ErrNotTinzenite
	}
	/*TODO load if model available*/
	// what follows is the code for a NEW model
	m := &Model{
		Root:    path,
		Tracked: make(map[string]bool),
		Objinfo: make(map[string]staticinfo)}
	// build first version (note that updatechan can't possibly be set already, so we won't spam UpdateMessages)
	err := m.Update()
	if err != nil {
		return nil, err
	}
	return m, nil
}

/*
Update the complete model state.
TODO: check concurrency allowances?
*/
func (m *Model) Update() error {
	current, err := m.populate()
	var removed, created []string
	if err != nil {
		return err
	}
	for path := range m.Tracked {
		_, ok := current[path]
		if ok {
			// paths that still exist must only be checked for MODIFY
			delete(current, path)
			m.apply(Modify, path)
		} else {
			// REMOVED - paths that don't exist anymore have been removed
			removed = append(removed, path)
		}
	}
	// CREATED - any remaining paths are yet untracked in m.tracked
	for path := range current {
		created = append(created, path)
	}
	// update m.Tracked
	for _, path := range removed {
		delete(m.Tracked, path)
		m.apply(Remove, path)
	}
	for _, path := range created {
		m.Tracked[path] = true
		m.apply(Create, path)
	}
	// finally also store the model for future loads.
	return m.store()
}

/*
Register the channel over which UpdateMessage can be received. Tinzenite will
only ever write to this channel, never read.
*/
func (m *Model) Register(v chan UpdateMessage) {
	m.updatechan = v
}

/*Read todo*/
func (m *Model) Read() (*Objectinfo, error) {
	/*TODO*/
	return nil, ErrUnsupported
}

func (m *Model) store() error {
	dir := m.Root + "/" + TINZENITEDIR + "/" + "local"
	err := makeDirectory(dir)
	if err != nil {
		return err
	}
	jsonBinary, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(dir+"/model.json", jsonBinary, FILEPERMISSIONMODE)
}

/*
getInfo creates the Objectinfo for the given path, so long as the path is
contained in m.Tracked. Directories are NOT traversed!
*/
func (m *Model) getInfo(path string) (*Objectinfo, error) {
	_, exists := m.Tracked[path]
	if !exists {
		return nil, ErrUntracked
	}
	stat, err := os.Lstat(path)
	if err != nil {
		return nil, err
	}
	// TODO lots still to do here!
	object := &Objectinfo{Path: path}
	if stat.IsDir() {
		object.directory = true
	}
	// TODO apply staticinfo!
	return object, ErrUnsupported
}

/*
populate a map[path] for the m.root path. Applies the root Matcher if provided.
*/
func (m *Model) populate() (map[string]bool, error) {
	match, err := CreateMatcher(m.Root)
	if err != nil {
		return nil, err
	}
	tracked := make(map[string]bool)
	filepath.Walk(m.Root, func(subpath string, stat os.FileInfo, inerr error) error {
		// ignore on match
		if match.Ignore(subpath) {
			// SkipDir is okay even if file
			if stat.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}
		tracked[subpath] = true
		return nil
	})
	// doesn't directly assign to m.tracked on purpose so that we can reuse this
	// method elsewhere
	return tracked, nil
}

/*
Apply changes to the internal model state. This method does the true logic on the
model, not touching m.Tracked. NEVER call this method outside of m.Update()!
*/
func (m *Model) apply(op Operation, path string) {
	notify := false
	var infoToNotify staticinfo
	switch op {
	case Create:
		notify = true
		// fetch all values we'll need to store
		id, err := newIdentifier()
		if err != nil {
			log.Println(err.Error())
			return
		}
		stat, err := os.Lstat(path)
		if err != nil {
			log.Println(err.Error())
			return
		}
		hash := ""
		if !stat.IsDir() {
			hash, err = contentHash(path)
			if err != nil {
				log.Println(err.Error())
				return
			}
		}
		stin := staticinfo{
			Identification: id,
			Version:        make(map[string]int),
			Directory:      stat.IsDir(),
			Content:        hash}
		m.Objinfo[path] = stin
		infoToNotify = stin
	case Modify:
		stin, ok := m.Objinfo[path]
		if !ok {
			log.Println("staticinfo lookup failed!")
			return
		}
		// no need for further work here
		if stin.Directory {
			return
		}
		hash, err := contentHash(path)
		if err != nil {
			log.Println(err.Error())
			return
		}
		// if same --> no changes, so done
		if hash == stin.Content {
			return
		}
		// otherwise a change has happened
		notify = true
		// update
		stin.Content = hash
		m.Objinfo[path] = stin
		// TODO update version
		infoToNotify = stin
	case Remove:
		/*TODO: delete logic for multiple peers required!*/
		notify = true
		var ok bool
		infoToNotify, ok = m.Objinfo[path]
		if !ok {
			log.Println("staticinfo lookup failed!")
			notify = false
		}
		delete(m.Objinfo, path)
	default:
		log.Printf("Unimplemented %s for now!\n", op)
	}
	// send the update message
	if notify && m.updatechan != nil {
		/*TODO select with default --> lost message? but we loose every update
		after the first... hm*/
		m.updatechan <- UpdateMessage{Operation: op, Object: infoToNotify}
	}
}

/*TODO for now only lists all tracked files*/
func (m *Model) String() string {
	var list string
	for path := range m.Tracked {
		list += path + "\n"
	}
	return list
}
<|endoftext|>


<commit_before>package fstab

import (
	"fmt"
	"strconv"
	"strings"
)

// Mount represetnts the filesystem info
type Mount struct {
	// The block special device or remote filesystem to be mounted
	Spec string

	// The mount point for the filesystem
	File string

	// The type of the filesystem
	VfsType string

	// Mount options associated with the filesystem
	MntOps map[string]string

	// Used by dump to determine which filesystems need to be dumped.
	Freq int

	// Used by the fsck(8) program to determine the order in which filesystem checks are done at reboot time
	PassNo int
}

const (
	Path = 0
	Label
	UUID
	PartUUID
	PartLabel
)

type DeviceIdentifierType int

// parseOptions parses the options field into an array of strings
func parseOptions(optionsString string) (options map[string]string) {
	options = make(map[string]string)
	for _, option := range strings.Split(optionsString, ",") {
		bits := strings.Split(strings.TrimSpace(option), "=")
		if len(bits) > 1 {
			options[bits[0]] = bits[1]
		} else {
			options[bits[0]] = ""
		}
	}
	return
}

func (mount Mount) MntOpsString() (opsstring string) {
	first := true
	for key, value := range mount.MntOps {
		if first {
			first = false
		} else {
			opsstring += ","
		}

		opsstring += key

		if "" != value {
			opsstring += "=" + value
		}
	}
	return
}

// String serializes the object into fstab format
func (mount Mount) String() string {
	return fmt.Sprintf("%-21s %-21s %-21s %-21s %2d %2d", mount.Spec, mount.File, mount.VfsType, mount.MntOpsString(), mount.Freq, mount.PassNo)
}

func (mount *Mount) IsSwap() bool {
	return "swap" == mount.VfsType
}

func (mount *Mount) IsNFS() bool {
	return "nfs" == mount.VfsType
}

// SpecType returns the device identifier type
func (mount *Mount) SpecType() (spectype DeviceIdentifierType) {
	bits := strings.Split(mount.Spec, "=")
	switch strings.ToUpper(bits[0]) {
	case "UUID":
		spectype = UUID

	case "LABEL":
		spectype = Label

	case "PARTUUID":
		spectype = PartUUID

	case "PARTLABEL":
		spectype = PartLabel

	default:
		spectype = Path
	}
	return
}

// SpecType returns the device identifier value; that is if Spec is
// "UUID=vogons-ate-my-sandwich", it will return "vogons-ate-my-sandwich"
func (mount *Mount) SpecValue() string {
	bits := strings.Split(mount.Spec, "=")
	if 1 == len(bits) {
		return mount.Spec
	} else {
		return bits[1]
	}
}

// ParseLine parses a single line (of an fstab).
// It will most frequently return a Mount; however,
// If a parsing error occurs, `err` will be non-nil and provide an error message.
// If the line is either empy or a comment line, `mount` will also be nil.
func ParseLine(line string) (mount *Mount, err error) {
	line = strings.TrimSpace(line)

	// Lines starting with a pound sign (#) are comments, and are ignored. So are empty lines.
	if ("" == line) || (line[0] == '#') {
		return nil, nil
	}

	fields := strings.Fields(line)
	if len(fields) < 4 {
		return nil, fmt.Errorf("too few fields (%i), at least 4 are expected", len(fields))
	} else {
		mount = new(Mount)
		mount.Spec = fields[0]
		mount.File = fields[1]
		mount.VfsType = fields[2]
		mount.MntOps = parseOptions(fields[3])

		var convErr error

		if len(fields) > 4 {
			mount.Freq, convErr = strconv.Atoi(fields[4])
			if nil != convErr {
				return nil, fmt.Errorf("%s is not a number", fields[4])
			}
		}

		if len(fields) > 5 {
			mount.PassNo, convErr = strconv.Atoi(fields[5])
			if nil != convErr {
				return nil, fmt.Errorf("%s it not a number", fields[5])
			}
		}
	}

	return
}
<commit_msg>Added equality check<commit_after>package fstab

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

// Mount represetnts the filesystem info
type Mount struct {
	// The block special device or remote filesystem to be mounted
	Spec string

	// The mount point for the filesystem
	File string

	// The type of the filesystem
	VfsType string

	// Mount options associated with the filesystem
	MntOps map[string]string

	// Used by dump to determine which filesystems need to be dumped.
	Freq int

	// Used by the fsck(8) program to determine the order in which filesystem checks are done at reboot time
	PassNo int
}

const (
	Path = 0
	Label
	UUID
	PartUUID
	PartLabel
)

type DeviceIdentifierType int

// parseOptions parses the options field into an array of strings
func parseOptions(optionsString string) (options map[string]string) {
	options = make(map[string]string)
	for _, option := range strings.Split(optionsString, ",") {
		bits := strings.Split(strings.TrimSpace(option), "=")
		if len(bits) > 1 {
			options[bits[0]] = bits[1]
		} else {
			options[bits[0]] = ""
		}
	}
	return
}

func (mount Mount) MntOpsString() (opsstring string) {
	first := true
	for key, value := range mount.MntOps {
		if first {
			first = false
		} else {
			opsstring += ","
		}

		opsstring += key

		if "" != value {
			opsstring += "=" + value
		}
	}
	return
}

// String serializes the object into fstab format
func (mount Mount) String() string {
	return fmt.Sprintf("%-21s %-21s %-21s %-21s %2d %2d", mount.Spec, mount.File, mount.VfsType, mount.MntOpsString(), mount.Freq, mount.PassNo)
}

func (mount *Mount) IsSwap() bool {
	return "swap" == mount.VfsType
}

func (mount *Mount) IsNFS() bool {
	return "nfs" == mount.VfsType
}

func (mount Mount) Equals(other Mount) bool {
	return reflect.DeepEqual(mount, other)
}

// SpecType returns the device identifier type
func (mount *Mount) SpecType() (spectype DeviceIdentifierType) {
	bits := strings.Split(mount.Spec, "=")
	switch strings.ToUpper(bits[0]) {
	case "UUID":
		spectype = UUID

	case "LABEL":
		spectype = Label

	case "PARTUUID":
		spectype = PartUUID

	case "PARTLABEL":
		spectype = PartLabel

	default:
		spectype = Path
	}
	return
}

// SpecType returns the device identifier value; that is if Spec is
// "UUID=vogons-ate-my-sandwich", it will return "vogons-ate-my-sandwich"
func (mount *Mount) SpecValue() string {
	bits := strings.Split(mount.Spec, "=")
	if 1 == len(bits) {
		return mount.Spec
	} else {
		return bits[1]
	}
}

// ParseLine parses a single line (of an fstab).
// It will most frequently return a Mount; however,
// If a parsing error occurs, `err` will be non-nil and provide an error message.
// If the line is either empy or a comment line, `mount` will also be nil.
func ParseLine(line string) (mount *Mount, err error) {
	line = strings.TrimSpace(line)

	// Lines starting with a pound sign (#) are comments, and are ignored. So are empty lines.
	if ("" == line) || (line[0] == '#') {
		return nil, nil
	}

	fields := strings.Fields(line)
	if len(fields) < 4 {
		return nil, fmt.Errorf("too few fields (%i), at least 4 are expected", len(fields))
	} else {
		mount = new(Mount)
		mount.Spec = fields[0]
		mount.File = fields[1]
		mount.VfsType = fields[2]
		mount.MntOps = parseOptions(fields[3])

		var convErr error

		if len(fields) > 4 {
			mount.Freq, convErr = strconv.Atoi(fields[4])
			if nil != convErr {
				return nil, fmt.Errorf("%s is not a number", fields[4])
			}
		}

		if len(fields) > 5 {
			mount.PassNo, convErr = strconv.Atoi(fields[5])
			if nil != convErr {
				return nil, fmt.Errorf("%s it not a number", fields[5])
			}
		}
	}

	return
}
<|endoftext|>


<commit_before><commit_msg>Fix incorrect core_freq for Raspberry Pi 4, Raspberry Pi 400 and CM4<commit_after><|endoftext|>


<commit_before>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package id3v2

import (
	"bufio"
	"io"
	"os"

	"github.com/bogem/id3v2/bwpool"
	"github.com/bogem/id3v2/util"
)

// Tag stores all frames of opened file.
type Tag struct {
	frames    map[string]Framer
	sequences map[string]sequencer

	file         *os.File
	originalSize int64
	version      byte
}

// AddFrame adds f to tag with appropriate id. If id is "" or f is nil,
// AddFrame will not add f to tag.
//
// If you want to add attached picture, comment or unsynchronised lyrics/text
// transcription frames, better use AddAttachedPicture, AddCommentFrame
// or AddUnsynchronisedLyricsFrame methods respectively.
func (t *Tag) AddFrame(id string, f Framer) {
	if id == "" || f == nil {
		return
	}

	if isFrameShouldBeInSequence(id) {
		if t.sequences[id] == nil { // if there is no sequence for id, ...
			constructor := sequenceConstructors[id] // ... find the constructor ...
			t.sequences[id] = constructor()         // ... and make the sequence.
		}
		t.sequences[id].AddFrame(f)
	} else {
		t.frames[id] = f
	}
}

// AddAttachedPicture adds a picture frame to tag.
//
// Example:
//
//	artwork, err := ioutil.ReadFile("artwork.jpg")
//	if err != nil {
//		log.Fatal("Error while reading artwork file", err)
//	}
//
//	pic := id3v2.PictureFrame{
//		Encoding:    id3v2.ENUTF8,
//		MimeType:    "image/jpeg",
//		PictureType: id3v2.PTFrontCover,
//		Description: "Front cover",
//		Picture:     artwork,
//	}
//	tag.AddAttachedPicture(pic)
//
// Available picture types you can see in constants.
func (t *Tag) AddAttachedPicture(pf PictureFrame) {
	id := t.CommonID("Attached picture")
	t.AddFrame(id, pf)
}

// AddCommentFrame adds a comment frame to tag.
//
// Example:
//
//	comment := id3v2.CommentFrame{
//		Encoding:   id3v2.ENUTF8,
//		Language:   "eng",
//		Desciption: "My opinion",
//		Text:       "Very good song",
//	}
//	tag.AddCommentFrame(comment)
//
// You should choose a language code from
// ISO 639-2 code list: https://www.loc.gov/standards/iso639-2/php/code_list.php
func (t *Tag) AddCommentFrame(cf CommentFrame) {
	id := t.CommonID("Comments")
	t.AddFrame(id, cf)
}

// AddUnsynchronisedLyricsFrame adds an unsynchronised lyrics/text frame
// to tag.
//
// Example:
//
//	uslt := id3v2.UnsynchronisedLyricsFrame{
//		Encoding:          id3v2.ENUTF8,
//		Language:          "ger",
//		ContentDescriptor: "Deutsche Nationalhymne",
//		Lyrics:            "Einigkeit und Recht und Freiheit...",
//	}
//	tag.AddUnsynchronisedLyricsFrame(uslt)
func (t *Tag) AddUnsynchronisedLyricsFrame(uslf UnsynchronisedLyricsFrame) {
	id := t.CommonID("Unsynchronised lyrics/text transcription")
	t.AddFrame(id, uslf)
}

// CommonID returns ID3v2.3 or ID3v2.4 (in appropriate to version of Tag) frame ID
// from given description.
// For example, CommonID("Language") will return "TLAN".
// All descriptions you can find in file common_ids.go or in id3 documentation (for fourth version: http://id3.org/id3v2.4.0-frames; for third version: http://id3.org/id3v2.3.0#Declared_ID3v2_frames).
func (t Tag) CommonID(description string) string {
	if t.version == 3 {
		return V23CommonIDs[description]
	}
	return V24CommonIDs[description]
}

// AllFrames returns map, that contains all frames in tag, that could be parsed.
// The key of this map is an ID of frame and value is an array of frames.
func (t *Tag) AllFrames() map[string][]Framer {
	frames := make(map[string][]Framer)

	for id, f := range t.frames {
		frames[id] = []Framer{f}
	}
	for id, sequence := range t.sequences {
		frames[id] = sequence.Frames()
	}

	return frames
}

// DeleteAllFrames deletes all frames in tag.
func (t *Tag) DeleteAllFrames() {
	t.frames = make(map[string]Framer)
	t.sequences = make(map[string]sequencer)
}

// DeleteFrames deletes frames in tag with given id.
func (t *Tag) DeleteFrames(id string) {
	delete(t.frames, id)
	delete(t.sequences, id)
}

// GetFrames returns frames with corresponding id.
// It returns nil if there is no frames with given id.
//
// Example of usage:
//	pictures := tag.GetFrames(tag.CommonID("Attached picture"))
//	if pictures != nil {
//		for _, f := range pictures {
//			pic, ok := f.(id3v2.PictureFrame)
//			if !ok {
//				log.Fatal("Couldn't assert picture frame")
//			}
//
//			// Do some operations with picture frame:
//			fmt.Println(pic.Description) // For example, print description of picture frame
//		}
//	}
func (t *Tag) GetFrames(id string) []Framer {
	if f, exists := t.frames[id]; exists {
		return []Framer{f}
	} else if s, exists := t.sequences[id]; exists {
		return s.Frames()
	}
	return nil
}

// GetLastFrame returns last frame from slice, that is returned from GetFrames function.
// GetLastFrame is suitable for frames, that can be only one in whole tag.
// For example, for text frames.
//
// Example of usage:
//	bpmFramer := tag.GetLastFrame(tag.CommonID("BPM"))
//	if bpmFramer != nil {
//		bpm, ok := bpmFramer.(id3v2.TextFrame)
//		if !ok {
//			log.Fatal("Couldn't assert bpm frame")
//		}
//		fmt.Println(bpm.Text)
//	}
func (t *Tag) GetLastFrame(id string) Framer {
	// Avoid an allocation of slice in GetFrames,
	// if there is anyway one frame.
	if f, exists := t.frames[id]; exists {
		return f
	}

	fs := t.GetFrames(id)
	if len(fs) == 0 {
		return nil
	}
	return fs[len(fs)-1]
}

// GetTextFrame returns text frame with corresponding id.
func (t Tag) GetTextFrame(id string) TextFrame {
	f := t.GetLastFrame(id)
	if f == nil {
		return TextFrame{}
	}
	tf := f.(TextFrame)
	return tf
}

// Count returns the number of frames in tag.
func (t Tag) Count() int {
	n := len(t.frames)
	for _, s := range t.sequences {
		n += s.Count()
	}
	return n
}

// HasAnyFrames checks if there is at least one frame in tag.
// It's much faster than tag.Count() > 0.
func (t Tag) HasAnyFrames() bool {
	return len(t.frames) > 0 || len(t.sequences) > 0
}

func (t Tag) Title() string {
	f := t.GetTextFrame(t.CommonID("Title/Songname/Content description"))
	return f.Text
}

func (t *Tag) SetTitle(title string) {
	t.AddFrame(t.CommonID("Title/Songname/Content description"), TextFrame{Encoding: ENUTF8, Text: title})
}

func (t Tag) Artist() string {
	f := t.GetTextFrame(t.CommonID("Lead artist/Lead performer/Soloist/Performing group"))
	return f.Text
}

func (t *Tag) SetArtist(artist string) {
	t.AddFrame(t.CommonID("Lead artist/Lead performer/Soloist/Performing group"), TextFrame{Encoding: ENUTF8, Text: artist})
}

func (t Tag) Album() string {
	f := t.GetTextFrame(t.CommonID("Album/Movie/Show title"))
	return f.Text
}

func (t *Tag) SetAlbum(album string) {
	t.AddFrame(t.CommonID("Album/Movie/Show title"), TextFrame{Encoding: ENUTF8, Text: album})
}

func (t Tag) Year() string {
	f := t.GetTextFrame(t.CommonID("Year"))
	return f.Text
}

func (t *Tag) SetYear(year string) {
	t.AddFrame(t.CommonID("Year"), TextFrame{Encoding: ENUTF8, Text: year})
}

func (t Tag) Genre() string {
	f := t.GetTextFrame(t.CommonID("Content type"))
	return f.Text
}

func (t *Tag) SetGenre(genre string) {
	t.AddFrame(t.CommonID("Content type"), TextFrame{Encoding: ENUTF8, Text: genre})
}

// iterateOverAllFrames iterates over every single frame in tag and call
// f for them. It consumps no memory at all, unlike the tag.AllFrames().
// It returns error only if f returns error.
func (t Tag) iterateOverAllFrames(f func(id string, frame Framer) error) error {
	for id, frame := range t.frames {
		if err := f(id, frame); err != nil {
			return err
		}
	}
	for id, sequence := range t.sequences {
		for _, frame := range sequence.Frames() {
			if err := f(id, frame); err != nil {
				return err
			}
		}
	}
	return nil
}

// Size returns the size of all ID3 tag in bytes.
func (t Tag) Size() int {
	if !t.HasAnyFrames() {
		return 0
	}

	var n int
	n += tagHeaderSize // Add the size of tag header
	t.iterateOverAllFrames(func(id string, f Framer) error {
		n += frameHeaderSize + f.Size() // Add the whole frame size
		return nil
	})

	return n
}

// Version returns current ID3v2 version of tag.
func (t Tag) Version() byte {
	return t.version
}

// SetVersion sets given ID3v2 version to tag.
// If version is less than 3 or more than 4, then this method will do nothing.
// If tag has some frames, which are deprecated or changed in given version,
// then to your notice you can delete, change or just stay them.
func (t *Tag) SetVersion(version byte) {
	if version < 3 || version > 4 {
		return
	}
	t.version = version
}

// Save writes tag to the file. If there are no frames in tag, Save will write
// only music part without any ID3v2 information.
func (t *Tag) Save() error {
	// Get original file mode
	originalFile := t.file
	originalStat, err := originalFile.Stat()
	if err != nil {
		return err
	}

	// Create a temp file for mp3 file, which will contain new tag
	name := t.file.Name() + "-id3v2"
	newFile, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, originalStat.Mode())
	if err != nil {
		return err
	}

	// Make sure we clean up the temp file if it's still around
	defer os.Remove(newFile.Name())

	// If there is at least one frame, write whole tag in new file
	var framesSize int64
	if t.HasAnyFrames() {
		tagSize, err := t.WriteTo(newFile)
		if err != nil {
			return err
		}
		framesSize = tagSize - tagHeaderSize
	}

	// Seek to a music part of original file
	if _, err = originalFile.Seek(t.originalSize, os.SEEK_SET); err != nil {
		return err
	}

	// Write to new file the music part
	if _, err = io.Copy(newFile, originalFile); err != nil {
		return err
	}

	// Set t.originalSize to new frames size
	t.originalSize = framesSize

	// Close files to allow replacing
	newFile.Close()
	originalFile.Close()

	// Replace original file with new file
	if err = os.Rename(newFile.Name(), originalFile.Name()); err != nil {
		return err
	}

	// Set t.file to new file with original name
	t.file, err = os.Open(originalFile.Name())
	if err != nil {
		return err
	}

	return nil
}

// WriteTo writes whole tag in w.
// It returns the number of bytes written and error during the write.
// It returns nil as error if the write was successful.
func (t Tag) WriteTo(w io.Writer) (n int64, err error) {
	// Form size of frames
	framesSize := t.Size() - tagHeaderSize
	byteFramesSize, err := util.FormSize(framesSize)
	if err != nil {
		return 0, err
	}

	// Write tag header
	if _, err = w.Write(formTagHeader(byteFramesSize, t.version)); err != nil {
		return 0, err
	}
	n += tagHeaderSize

	// Write frames
	nn, err := t.writeAllFrames(w)
	if err != nil {
		return 0, err
	}
	n += nn

	return n, err
}

// writeAllFrames writes all frames to w and returns
// the number of bytes written and error during the write.
// It returns nil as error if the write was successful.
func (t Tag) writeAllFrames(w io.Writer) (int64, error) {
	bw := bwpool.Get(w)
	defer bwpool.Put(bw)

	err := t.iterateOverAllFrames(func(id string, f Framer) error {
		return writeFrame(bw, id, f)
	})
	if err != nil {
		return 0, err
	}

	return int64(bw.Buffered()), bw.Flush()
}

func writeFrame(bw *bufio.Writer, id string, frame Framer) error {
	if err := writeFrameHeader(bw, id, frame.Size()); err != nil {
		return err
	}

	if _, err := frame.WriteTo(bw); err != nil {
		return err
	}

	return nil
}

func writeFrameHeader(bw *bufio.Writer, id string, frameSize int) error {
	size, err := util.FormSize(frameSize)
	if err != nil {
		return err
	}

	bw.WriteString(id)
	bw.Write(size)
	bw.Write([]byte{0, 0})
	return nil
}

// Close closes the tag's file, rendering it unusable for I/O.
// It returns an error, if any.
func (t *Tag) Close() error {
	return t.file.Close()
}
<commit_msg>Prevent tag.WriteTo writes tag if there are 0 frames in tag<commit_after>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package id3v2

import (
	"bufio"
	"io"
	"os"

	"github.com/bogem/id3v2/bwpool"
	"github.com/bogem/id3v2/util"
)

// Tag stores all frames of opened file.
type Tag struct {
	frames    map[string]Framer
	sequences map[string]sequencer

	file         *os.File
	originalSize int64
	version      byte
}

// AddFrame adds f to tag with appropriate id. If id is "" or f is nil,
// AddFrame will not add f to tag.
//
// If you want to add attached picture, comment or unsynchronised lyrics/text
// transcription frames, better use AddAttachedPicture, AddCommentFrame
// or AddUnsynchronisedLyricsFrame methods respectively.
func (t *Tag) AddFrame(id string, f Framer) {
	if id == "" || f == nil {
		return
	}

	if isFrameShouldBeInSequence(id) {
		if t.sequences[id] == nil { // if there is no sequence for id, ...
			constructor := sequenceConstructors[id] // ... find the constructor ...
			t.sequences[id] = constructor()         // ... and make the sequence.
		}
		t.sequences[id].AddFrame(f)
	} else {
		t.frames[id] = f
	}
}

// AddAttachedPicture adds a picture frame to tag.
//
// Example:
//
//	artwork, err := ioutil.ReadFile("artwork.jpg")
//	if err != nil {
//		log.Fatal("Error while reading artwork file", err)
//	}
//
//	pic := id3v2.PictureFrame{
//		Encoding:    id3v2.ENUTF8,
//		MimeType:    "image/jpeg",
//		PictureType: id3v2.PTFrontCover,
//		Description: "Front cover",
//		Picture:     artwork,
//	}
//	tag.AddAttachedPicture(pic)
//
// Available picture types you can see in constants.
func (t *Tag) AddAttachedPicture(pf PictureFrame) {
	id := t.CommonID("Attached picture")
	t.AddFrame(id, pf)
}

// AddCommentFrame adds a comment frame to tag.
//
// Example:
//
//	comment := id3v2.CommentFrame{
//		Encoding:   id3v2.ENUTF8,
//		Language:   "eng",
//		Desciption: "My opinion",
//		Text:       "Very good song",
//	}
//	tag.AddCommentFrame(comment)
//
// You should choose a language code from
// ISO 639-2 code list: https://www.loc.gov/standards/iso639-2/php/code_list.php
func (t *Tag) AddCommentFrame(cf CommentFrame) {
	id := t.CommonID("Comments")
	t.AddFrame(id, cf)
}

// AddUnsynchronisedLyricsFrame adds an unsynchronised lyrics/text frame
// to tag.
//
// Example:
//
//	uslt := id3v2.UnsynchronisedLyricsFrame{
//		Encoding:          id3v2.ENUTF8,
//		Language:          "ger",
//		ContentDescriptor: "Deutsche Nationalhymne",
//		Lyrics:            "Einigkeit und Recht und Freiheit...",
//	}
//	tag.AddUnsynchronisedLyricsFrame(uslt)
func (t *Tag) AddUnsynchronisedLyricsFrame(uslf UnsynchronisedLyricsFrame) {
	id := t.CommonID("Unsynchronised lyrics/text transcription")
	t.AddFrame(id, uslf)
}

// CommonID returns ID3v2.3 or ID3v2.4 (in appropriate to version of Tag) frame ID
// from given description.
// For example, CommonID("Language") will return "TLAN".
// All descriptions you can find in file common_ids.go or in id3 documentation (for fourth version: http://id3.org/id3v2.4.0-frames; for third version: http://id3.org/id3v2.3.0#Declared_ID3v2_frames).
func (t Tag) CommonID(description string) string {
	if t.version == 3 {
		return V23CommonIDs[description]
	}
	return V24CommonIDs[description]
}

// AllFrames returns map, that contains all frames in tag, that could be parsed.
// The key of this map is an ID of frame and value is an array of frames.
func (t *Tag) AllFrames() map[string][]Framer {
	frames := make(map[string][]Framer)

	for id, f := range t.frames {
		frames[id] = []Framer{f}
	}
	for id, sequence := range t.sequences {
		frames[id] = sequence.Frames()
	}

	return frames
}

// DeleteAllFrames deletes all frames in tag.
func (t *Tag) DeleteAllFrames() {
	t.frames = make(map[string]Framer)
	t.sequences = make(map[string]sequencer)
}

// DeleteFrames deletes frames in tag with given id.
func (t *Tag) DeleteFrames(id string) {
	delete(t.frames, id)
	delete(t.sequences, id)
}

// GetFrames returns frames with corresponding id.
// It returns nil if there is no frames with given id.
//
// Example of usage:
//	pictures := tag.GetFrames(tag.CommonID("Attached picture"))
//	if pictures != nil {
//		for _, f := range pictures {
//			pic, ok := f.(id3v2.PictureFrame)
//			if !ok {
//				log.Fatal("Couldn't assert picture frame")
//			}
//
//			// Do some operations with picture frame:
//			fmt.Println(pic.Description) // For example, print description of picture frame
//		}
//	}
func (t *Tag) GetFrames(id string) []Framer {
	if f, exists := t.frames[id]; exists {
		return []Framer{f}
	} else if s, exists := t.sequences[id]; exists {
		return s.Frames()
	}
	return nil
}

// GetLastFrame returns last frame from slice, that is returned from GetFrames function.
// GetLastFrame is suitable for frames, that can be only one in whole tag.
// For example, for text frames.
//
// Example of usage:
//	bpmFramer := tag.GetLastFrame(tag.CommonID("BPM"))
//	if bpmFramer != nil {
//		bpm, ok := bpmFramer.(id3v2.TextFrame)
//		if !ok {
//			log.Fatal("Couldn't assert bpm frame")
//		}
//		fmt.Println(bpm.Text)
//	}
func (t *Tag) GetLastFrame(id string) Framer {
	// Avoid an allocation of slice in GetFrames,
	// if there is anyway one frame.
	if f, exists := t.frames[id]; exists {
		return f
	}

	fs := t.GetFrames(id)
	if len(fs) == 0 {
		return nil
	}
	return fs[len(fs)-1]
}

// GetTextFrame returns text frame with corresponding id.
func (t Tag) GetTextFrame(id string) TextFrame {
	f := t.GetLastFrame(id)
	if f == nil {
		return TextFrame{}
	}
	tf := f.(TextFrame)
	return tf
}

// Count returns the number of frames in tag.
func (t Tag) Count() int {
	n := len(t.frames)
	for _, s := range t.sequences {
		n += s.Count()
	}
	return n
}

// HasAnyFrames checks if there is at least one frame in tag.
// It's much faster than tag.Count() > 0.
func (t Tag) HasAnyFrames() bool {
	return len(t.frames) > 0 || len(t.sequences) > 0
}

func (t Tag) Title() string {
	f := t.GetTextFrame(t.CommonID("Title/Songname/Content description"))
	return f.Text
}

func (t *Tag) SetTitle(title string) {
	t.AddFrame(t.CommonID("Title/Songname/Content description"), TextFrame{Encoding: ENUTF8, Text: title})
}

func (t Tag) Artist() string {
	f := t.GetTextFrame(t.CommonID("Lead artist/Lead performer/Soloist/Performing group"))
	return f.Text
}

func (t *Tag) SetArtist(artist string) {
	t.AddFrame(t.CommonID("Lead artist/Lead performer/Soloist/Performing group"), TextFrame{Encoding: ENUTF8, Text: artist})
}

func (t Tag) Album() string {
	f := t.GetTextFrame(t.CommonID("Album/Movie/Show title"))
	return f.Text
}

func (t *Tag) SetAlbum(album string) {
	t.AddFrame(t.CommonID("Album/Movie/Show title"), TextFrame{Encoding: ENUTF8, Text: album})
}

func (t Tag) Year() string {
	f := t.GetTextFrame(t.CommonID("Year"))
	return f.Text
}

func (t *Tag) SetYear(year string) {
	t.AddFrame(t.CommonID("Year"), TextFrame{Encoding: ENUTF8, Text: year})
}

func (t Tag) Genre() string {
	f := t.GetTextFrame(t.CommonID("Content type"))
	return f.Text
}

func (t *Tag) SetGenre(genre string) {
	t.AddFrame(t.CommonID("Content type"), TextFrame{Encoding: ENUTF8, Text: genre})
}

// iterateOverAllFrames iterates over every single frame in tag and call
// f for them. It consumps no memory at all, unlike the tag.AllFrames().
// It returns error only if f returns error.
func (t Tag) iterateOverAllFrames(f func(id string, frame Framer) error) error {
	for id, frame := range t.frames {
		if err := f(id, frame); err != nil {
			return err
		}
	}
	for id, sequence := range t.sequences {
		for _, frame := range sequence.Frames() {
			if err := f(id, frame); err != nil {
				return err
			}
		}
	}
	return nil
}

// Size returns the size of all ID3 tag in bytes.
func (t Tag) Size() int {
	if !t.HasAnyFrames() {
		return 0
	}

	var n int
	n += tagHeaderSize // Add the size of tag header
	t.iterateOverAllFrames(func(id string, f Framer) error {
		n += frameHeaderSize + f.Size() // Add the whole frame size
		return nil
	})

	return n
}

// Version returns current ID3v2 version of tag.
func (t Tag) Version() byte {
	return t.version
}

// SetVersion sets given ID3v2 version to tag.
// If version is less than 3 or more than 4, then this method will do nothing.
// If tag has some frames, which are deprecated or changed in given version,
// then to your notice you can delete, change or just stay them.
func (t *Tag) SetVersion(version byte) {
	if version < 3 || version > 4 {
		return
	}
	t.version = version
}

// Save writes tag to the file. If there are no frames in tag, Save will write
// only music part without any ID3v2 information.
func (t *Tag) Save() error {
	// Get original file mode
	originalFile := t.file
	originalStat, err := originalFile.Stat()
	if err != nil {
		return err
	}

	// Create a temp file for mp3 file, which will contain new tag
	name := t.file.Name() + "-id3v2"
	newFile, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, originalStat.Mode())
	if err != nil {
		return err
	}

	// Make sure we clean up the temp file if it's still around
	defer os.Remove(newFile.Name())

	// If there is at least one frame, write whole tag in new file
	var framesSize int64
	if t.HasAnyFrames() {
		tagSize, err := t.WriteTo(newFile)
		if err != nil {
			return err
		}
		framesSize = tagSize - tagHeaderSize
	}

	// Seek to a music part of original file
	if _, err = originalFile.Seek(t.originalSize, os.SEEK_SET); err != nil {
		return err
	}

	// Write to new file the music part
	if _, err = io.Copy(newFile, originalFile); err != nil {
		return err
	}

	// Set t.originalSize to new frames size
	t.originalSize = framesSize

	// Close files to allow replacing
	newFile.Close()
	originalFile.Close()

	// Replace original file with new file
	if err = os.Rename(newFile.Name(), originalFile.Name()); err != nil {
		return err
	}

	// Set t.file to new file with original name
	t.file, err = os.Open(originalFile.Name())
	if err != nil {
		return err
	}

	return nil
}

// WriteTo writes whole tag in w.
// It returns the number of bytes written and error during the write.
// It returns nil as error if the write was successful.
func (t Tag) WriteTo(w io.Writer) (n int64, err error) {
	// Form size of frames
	framesSize := t.Size() - tagHeaderSize
	if framesSize == 0 {
		return 0, nil
	}

	byteFramesSize, err := util.FormSize(framesSize)
	if err != nil {
		return 0, err
	}

	// Write tag header
	if _, err = w.Write(formTagHeader(byteFramesSize, t.version)); err != nil {
		return 0, err
	}
	n += tagHeaderSize

	// Write frames
	nn, err := t.writeAllFrames(w)
	if err != nil {
		return 0, err
	}
	n += nn

	return n, err
}

// writeAllFrames writes all frames to w and returns
// the number of bytes written and error during the write.
// It returns nil as error if the write was successful.
func (t Tag) writeAllFrames(w io.Writer) (int64, error) {
	bw := bwpool.Get(w)
	defer bwpool.Put(bw)

	err := t.iterateOverAllFrames(func(id string, f Framer) error {
		return writeFrame(bw, id, f)
	})
	if err != nil {
		return 0, err
	}

	return int64(bw.Buffered()), bw.Flush()
}

func writeFrame(bw *bufio.Writer, id string, frame Framer) error {
	if err := writeFrameHeader(bw, id, frame.Size()); err != nil {
		return err
	}

	if _, err := frame.WriteTo(bw); err != nil {
		return err
	}

	return nil
}

func writeFrameHeader(bw *bufio.Writer, id string, frameSize int) error {
	size, err := util.FormSize(frameSize)
	if err != nil {
		return err
	}

	bw.WriteString(id)
	bw.Write(size)
	bw.Write([]byte{0, 0})
	return nil
}

// Close closes the tag's file, rendering it unusable for I/O.
// It returns an error, if any.
func (t *Tag) Close() error {
	return t.file.Close()
}
<|endoftext|>


<commit_before><commit_msg>Fix example in godoc<commit_after><|endoftext|>


<commit_before><commit_msg>Add usage<commit_after><|endoftext|>


<commit_before><commit_msg>add spaces around "press esc or space to continue"<commit_after><|endoftext|>


<commit_before><commit_msg>Add initial drip client<commit_after><|endoftext|>


<commit_before><commit_msg>add verbose option<commit_after><|endoftext|>


<commit_before><commit_msg>Add space-trimming to processed words<commit_after><|endoftext|>


<commit_before><commit_msg>Refactor variable name<commit_after><|endoftext|>


<commit_before><commit_msg>Adding documentation.<commit_after><|endoftext|>


<commit_before><commit_msg>Remove some debugging noise<commit_after><|endoftext|>


<commit_before><commit_msg>Help prints with no args given. -h after a command prints help for that command. (#79)<commit_after><|endoftext|>


<commit_before><commit_msg>Improve usage message (previously misleading).<commit_after><|endoftext|>


<commit_before><commit_msg>fix orgalorg docopt about -p<commit_after><|endoftext|>


<commit_before><commit_msg>only create stick n find device if tag is close<commit_after><|endoftext|>


<commit_before><commit_msg>mongo+rabbit<commit_after><|endoftext|>


<commit_before><commit_msg>Get fabfile path<commit_after><|endoftext|>


<commit_before><commit_msg>adding tile<commit_after><|endoftext|>