<commit_before>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package optimize

import (
	"fmt"
	"math"
	"time"

	"github.com/gonum/floats"
	"github.com/gonum/matrix/mat64"
)

// Local finds a local minimum of a function using a sequential algorithm.
// In order to maximize a function, multiply the output by -1.
//
// The first argument is of Function type representing the function to be minimized.
// Type switching is used to see if the function implements Gradient, FunctionGradient
// and Statuser.
//
// The second argument is the initial location at which to start the minimization.
// The initial location must be supplied, and must have a length equal to the
// problem dimension.
//
// The third argument contains the settings for the minimization. It is here that
// gradient tolerance, etc. are specified. The DefaultSettings() function
// can be called for a Settings struct with the default values initialized.
// If settings == nil, the default settings are used. Please see the documentation
// for the Settings structure for more information. The optimization Method used
// may also contain settings, see documentation for the appropriate optimizer.
//
// The final argument is the optimization method to use. If method == nil, then
// an appropriate default is chosen based on the properties of the other arguments
// (dimension, gradient-free or gradient-based, etc.). The optimization
// methods in this package are designed such that reasonable defaults occur
// if options are not specified explicitly. For example, the code
//  method := &Bfgs{}
// creates a pointer to a new Bfgs struct. When minimize is called, the settings
// in the method will be populated with default values. The methods are also
// designed such that they can be reused in future calls to method.
//
// Local returns a Result struct and any error that occurred. Please see the
// documentation of Result for more information.
//
// Please be aware that the default behavior of Local is to find the minimum.
// For certain functions and optimization methods, this process can take many
// function evaluations. If you would like to put limits on this, for example
// maximum runtime or maximum function evaluations, please modify the Settings
// input struct.
func Local(f Function, initX []float64, settings *Settings, method Method) (*Result, error) {
	if len(initX) == 0 {
		panic("local: initial X has zero length")
	}

	startTime := time.Now()
	funcInfo := newFunctionInfo(f)
	if method == nil {
		method = getDefaultMethod(funcInfo)
	}
	if err := funcInfo.satisfies(method); err != nil {
		return nil, err
	}

	if funcInfo.IsStatuser {
		_, err := funcInfo.statuser.Status()
		if err != nil {
			return nil, err
		}
	}

	if settings == nil {
		settings = DefaultSettings()
	}

	if settings.Recorder != nil {
		// Initialize Recorder first. If it fails, we avoid the (possibly
		// time-consuming) evaluation of F and DF at the starting location.
		err := settings.Recorder.Init(&funcInfo.FunctionInfo)
		if err != nil {
			return nil, err
		}
	}

	stats := &Stats{}
	optLoc, evalType, err := getStartingLocation(funcInfo, method, initX, stats, settings)
	if err != nil {
		return nil, err
	}

	// Runtime is the only Stats field that needs to be updated here.
	stats.Runtime = time.Since(startTime)
	// Send optLoc to Recorder before checking it for convergence.
	if settings.Recorder != nil {
		err = settings.Recorder.Record(optLoc, evalType, InitIteration, stats)
	}

	// Check if the starting location satisfies the convergence criteria.
	status := checkConvergence(optLoc, InitIteration, stats, settings)
	if status == NotTerminated && err == nil {
		// The starting location is not good enough, we need to perform a
		// minimization. The optimal location will be stored in-place in
		// optLoc.
		status, err = minimize(settings, method, funcInfo, stats, optLoc, startTime)
	}

	if settings.Recorder != nil && err == nil {
		// Send the optimal location to Recorder.
		err = settings.Recorder.Record(optLoc, NoEvaluation, PostIteration, stats)
	}
	stats.Runtime = time.Since(startTime)
	return &Result{
		Location: *optLoc,
		Stats:    *stats,
		Status:   status,
	}, err
}

func minimize(settings *Settings, method Method, funcInfo *functionInfo, stats *Stats, optLoc *Location, startTime time.Time) (status Status, err error) {
	loc := &Location{}
	copyLocation(loc, optLoc)
	xNext := make([]float64, len(loc.X))

	methodStatus, methodIsStatuser := method.(Statuser)

	evalType, iterType, err := method.Init(loc, &funcInfo.FunctionInfo, xNext)
	if err != nil {
		return Failure, err
	}

	for {
		if funcInfo.IsStatuser {
			// Check the function status before evaluating.
			status, err = funcInfo.statuser.Status()
			if err != nil || status != NotTerminated {
				return
			}
		}

		// Perform evalType evaluation of the function at xNext and store the
		// result in location.
		evaluate(funcInfo, evalType, xNext, loc, stats)
		// Update the stats and optLoc.
		update(loc, optLoc, stats, iterType, startTime)
		// Get the convergence status before recording the new location.
		status = checkConvergence(optLoc, iterType, stats, settings)

		if settings.Recorder != nil {
			err = settings.Recorder.Record(loc, evalType, iterType, stats)
			if err != nil {
				if status == NotTerminated {
					status = Failure
				}
				return
			}
		}

		if status != NotTerminated {
			return
		}

		if methodIsStatuser {
			status, err = methodStatus.Status()
			if err != nil || status != NotTerminated {
				return
			}
		}

		// Find the next location (stored in-place into xNext).
		evalType, iterType, err = method.Iterate(loc, xNext)
		if err != nil {
			status = Failure
			return
		}
	}
	panic("unreachable")
}

func copyLocation(dst, src *Location) {
	dst.X = resize(dst.X, len(src.X))
	copy(dst.X, src.X)

	dst.F = src.F

	dst.Gradient = resize(dst.Gradient, len(src.Gradient))
	copy(dst.Gradient, src.Gradient)

	if src.Hessian != nil {
		if dst.Hessian == nil || dst.Hessian.Symmetric() != len(src.X) {
			dst.Hessian = mat64.NewSymDense(len(src.X), nil)
		}
		dst.Hessian.CopySym(src.Hessian)
	}
}

func getDefaultMethod(funcInfo *functionInfo) Method {
	if funcInfo.IsGradient || funcInfo.IsFunctionGradient {
		return &BFGS{}
	}
	// TODO: Implement a gradient-free method
	panic("optimize: gradient-free methods not yet coded")
}

// getStartingLocation allocates and initializes the starting location for the minimization.
func getStartingLocation(funcInfo *functionInfo, method Method, initX []float64, stats *Stats, settings *Settings) (*Location, EvaluationType, error) {
	dim := len(initX)
	loc := &Location{
		X: make([]float64, dim),
	}
	copy(loc.X, initX)
	if method.Needs().Gradient {
		loc.Gradient = make([]float64, dim)
	}
	if method.Needs().Hessian {
		loc.Hessian = mat64.NewSymDense(dim, nil)
	}

	evalType := NoEvaluation
	if settings.UseInitialData {
		loc.F = settings.InitialFunctionValue
		if loc.Gradient != nil {
			initG := settings.InitialGradient
			if len(initG) != dim {
				panic("local: initial location size mismatch")
			}
			copy(loc.Gradient, initG)
		}
	} else {
		evalType = FuncEvaluation
		if loc.Gradient != nil {
			evalType = FuncGradEvaluation
		}
		evaluate(funcInfo, evalType, loc.X, loc, stats)
	}

	if math.IsNaN(loc.F) {
		return loc, evalType, ErrNaN
	}
	if math.IsInf(loc.F, 1) {
		return loc, evalType, ErrInf
	}
	for _, v := range loc.Gradient {
		if math.IsInf(v, 0) {
			return loc, evalType, ErrGradInf
		}
		if math.IsNaN(v) {
			return loc, evalType, ErrGradNaN
		}
	}

	return loc, evalType, nil
}

func checkConvergence(loc *Location, iterType IterationType, stats *Stats, settings *Settings) Status {
	if iterType == MajorIteration || iterType == InitIteration {
		if loc.Gradient != nil {
			norm := floats.Norm(loc.Gradient, math.Inf(1))
			if norm < settings.GradientAbsTol {
				return GradientAbsoluteConvergence
			}
		}
		if loc.F < settings.FunctionAbsTol {
			return FunctionAbsoluteConvergence
		}
	}

	// Check every step for negative infinity because it could break the
	// linesearches and -inf is the best you can do anyway.
	if math.IsInf(loc.F, -1) {
		return FunctionNegativeInfinity
	}

	if settings.FuncEvaluations > 0 {
		totalFun := stats.FuncEvaluations + stats.FuncGradEvaluations + stats.FuncGradHessEvaluations
		if totalFun >= settings.FuncEvaluations {
			return FunctionEvaluationLimit
		}
	}

	if settings.GradEvaluations > 0 {
		totalGrad := stats.GradEvaluations + stats.FuncGradEvaluations + stats.FuncGradHessEvaluations
		if totalGrad >= settings.GradEvaluations {
			return GradientEvaluationLimit
		}
	}

	if settings.HessEvaluations > 0 {
		totalHess := stats.HessEvaluations + stats.FuncGradHessEvaluations
		if totalHess >= settings.HessEvaluations {
			return HessianEvaluationLimit
		}
	}

	if settings.Runtime > 0 {
		// TODO(vladimir-ch): It would be nice to update Runtime here.
		if stats.Runtime >= settings.Runtime {
			return RuntimeLimit
		}
	}

	if iterType == MajorIteration && settings.MajorIterations > 0 {
		if stats.MajorIterations >= settings.MajorIterations {
			return IterationLimit
		}
	}
	return NotTerminated
}

func invalidate(loc *Location, f, grad, hess bool) {
	if f {
		loc.F = math.NaN()
	}
	if grad {
		for i := range loc.Gradient {
			loc.Gradient[i] = math.NaN()
		}
	}
	if hess && loc.Hessian != nil {
		n := loc.Hessian.Symmetric()
		for i := 0; i < n; i++ {
			for j := i; j < n; j++ {
				loc.Hessian.SetSym(i, j, math.NaN())
			}
		}
	}
}

// evaluate evaluates the function given by funcInfo at xNext, stores the
// answer into loc and updates stats. If loc.X is not equal to xNext, then
// unused fields of loc are set to NaN.
// evaluate panics if the function does not support the requested evalType.
func evaluate(funcInfo *functionInfo, evalType EvaluationType, xNext []float64, loc *Location, stats *Stats) {
	different := !floats.Equal(loc.X, xNext)
	if different {
		copy(loc.X, xNext)
	}
	switch evalType {
	case FuncEvaluation:
		if different {
			invalidate(loc, false, true, true)
		}
		loc.F = funcInfo.function.Func(loc.X)
		stats.FuncEvaluations++
		return
	case GradEvaluation:
		if funcInfo.IsGradient {
			if different {
				invalidate(loc, true, false, true)
			}
			funcInfo.gradient.Grad(loc.X, loc.Gradient)
			stats.GradEvaluations++
			return
		}
		if funcInfo.IsFunctionGradient {
			if different {
				invalidate(loc, false, false, true)
			}
			loc.F = funcInfo.functionGradient.FuncGrad(loc.X, loc.Gradient)
			stats.FuncGradEvaluations++
			return
		}
		if funcInfo.IsFunctionGradientHessian {
			if loc.Hessian == nil {
				loc.Hessian = mat64.NewSymDense(len(loc.X), nil)
			}
			loc.F = funcInfo.functionGradientHessian.FuncGradHess(loc.X, loc.Gradient, loc.Hessian)
			stats.FuncGradHessEvaluations++
			return
		}
	case HessEvaluation:
		if funcInfo.IsHessian {
			if different {
				invalidate(loc, true, true, false)
			}
			funcInfo.hessian.Hess(loc.X, loc.Hessian)
			stats.HessEvaluations++
			return
		}
		if funcInfo.IsFunctionGradientHessian {
			if loc.Gradient == nil {
				loc.Gradient = make([]float64, len(loc.X))
			}
			loc.F = funcInfo.functionGradientHessian.FuncGradHess(loc.X, loc.Gradient, loc.Hessian)
			stats.FuncGradHessEvaluations++
			return
		}
	case FuncGradEvaluation:
		if funcInfo.IsFunctionGradient {
			if different {
				invalidate(loc, false, false, true)
			}
			loc.F = funcInfo.functionGradient.FuncGrad(loc.X, loc.Gradient)
			stats.FuncGradEvaluations++
			return
		}
		if funcInfo.IsGradient {
			if different {
				invalidate(loc, false, false, true)
			}
			loc.F = funcInfo.function.Func(loc.X)
			stats.FuncEvaluations++
			funcInfo.gradient.Grad(loc.X, loc.Gradient)
			stats.GradEvaluations++
			return
		}
		if funcInfo.IsFunctionGradientHessian {
			if loc.Hessian == nil {
				loc.Hessian = mat64.NewSymDense(len(loc.X), nil)
			}
			loc.F = funcInfo.functionGradientHessian.FuncGradHess(loc.X, loc.Gradient, loc.Hessian)
			stats.FuncGradHessEvaluations++
			return
		}
	case FuncGradHessEvaluation:
		if funcInfo.IsFunctionGradientHessian {
			loc.F = funcInfo.functionGradientHessian.FuncGradHess(loc.X, loc.Gradient, loc.Hessian)
			stats.FuncGradHessEvaluations++
			return
		}
		if funcInfo.IsGradient && funcInfo.IsHessian {
			loc.F = funcInfo.function.Func(loc.X)
			stats.FuncEvaluations++
			funcInfo.gradient.Grad(loc.X, loc.Gradient)
			stats.GradEvaluations++
			funcInfo.hessian.Hess(loc.X, loc.Hessian)
			stats.HessEvaluations++
			return
		}
	case NoEvaluation:
		if different {
			// Optimizers should not request NoEvaluation at a new location.
			// The intent and therefore an appropriate action are both unclear.
			panic("no evaluation requested at new location")
		}
		return
	default:
		panic(fmt.Sprintf("unknown evaluation type %v", evalType))
	}
	panic(fmt.Sprintf("objective function does not support %v", evalType))
}

// update updates the stats given the new evaluation
func update(loc *Location, optLoc *Location, stats *Stats, iterType IterationType, startTime time.Time) {
	if iterType == MajorIteration {
		stats.MajorIterations++
	}
	if loc.F <= optLoc.F {
		copyLocation(optLoc, loc)
	}
	stats.Runtime = time.Since(startTime)
}
<commit_msg>Simplify invalidate()<commit_after>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package optimize

import (
	"fmt"
	"math"
	"time"

	"github.com/gonum/floats"
	"github.com/gonum/matrix/mat64"
)

// Local finds a local minimum of a function using a sequential algorithm.
// In order to maximize a function, multiply the output by -1.
//
// The first argument is of Function type representing the function to be minimized.
// Type switching is used to see if the function implements Gradient, FunctionGradient
// and Statuser.
//
// The second argument is the initial location at which to start the minimization.
// The initial location must be supplied, and must have a length equal to the
// problem dimension.
//
// The third argument contains the settings for the minimization. It is here that
// gradient tolerance, etc. are specified. The DefaultSettings() function
// can be called for a Settings struct with the default values initialized.
// If settings == nil, the default settings are used. Please see the documentation
// for the Settings structure for more information. The optimization Method used
// may also contain settings, see documentation for the appropriate optimizer.
//
// The final argument is the optimization method to use. If method == nil, then
// an appropriate default is chosen based on the properties of the other arguments
// (dimension, gradient-free or gradient-based, etc.). The optimization
// methods in this package are designed such that reasonable defaults occur
// if options are not specified explicitly. For example, the code
//  method := &Bfgs{}
// creates a pointer to a new Bfgs struct. When minimize is called, the settings
// in the method will be populated with default values. The methods are also
// designed such that they can be reused in future calls to method.
//
// Local returns a Result struct and any error that occurred. Please see the
// documentation of Result for more information.
//
// Please be aware that the default behavior of Local is to find the minimum.
// For certain functions and optimization methods, this process can take many
// function evaluations. If you would like to put limits on this, for example
// maximum runtime or maximum function evaluations, please modify the Settings
// input struct.
func Local(f Function, initX []float64, settings *Settings, method Method) (*Result, error) {
	if len(initX) == 0 {
		panic("local: initial X has zero length")
	}

	startTime := time.Now()
	funcInfo := newFunctionInfo(f)
	if method == nil {
		method = getDefaultMethod(funcInfo)
	}
	if err := funcInfo.satisfies(method); err != nil {
		return nil, err
	}

	if funcInfo.IsStatuser {
		_, err := funcInfo.statuser.Status()
		if err != nil {
			return nil, err
		}
	}

	if settings == nil {
		settings = DefaultSettings()
	}

	if settings.Recorder != nil {
		// Initialize Recorder first. If it fails, we avoid the (possibly
		// time-consuming) evaluation of F and DF at the starting location.
		err := settings.Recorder.Init(&funcInfo.FunctionInfo)
		if err != nil {
			return nil, err
		}
	}

	stats := &Stats{}
	optLoc, evalType, err := getStartingLocation(funcInfo, method, initX, stats, settings)
	if err != nil {
		return nil, err
	}

	// Runtime is the only Stats field that needs to be updated here.
	stats.Runtime = time.Since(startTime)
	// Send optLoc to Recorder before checking it for convergence.
	if settings.Recorder != nil {
		err = settings.Recorder.Record(optLoc, evalType, InitIteration, stats)
	}

	// Check if the starting location satisfies the convergence criteria.
	status := checkConvergence(optLoc, InitIteration, stats, settings)
	if status == NotTerminated && err == nil {
		// The starting location is not good enough, we need to perform a
		// minimization. The optimal location will be stored in-place in
		// optLoc.
		status, err = minimize(settings, method, funcInfo, stats, optLoc, startTime)
	}

	if settings.Recorder != nil && err == nil {
		// Send the optimal location to Recorder.
		err = settings.Recorder.Record(optLoc, NoEvaluation, PostIteration, stats)
	}
	stats.Runtime = time.Since(startTime)
	return &Result{
		Location: *optLoc,
		Stats:    *stats,
		Status:   status,
	}, err
}

func minimize(settings *Settings, method Method, funcInfo *functionInfo, stats *Stats, optLoc *Location, startTime time.Time) (status Status, err error) {
	loc := &Location{}
	copyLocation(loc, optLoc)
	xNext := make([]float64, len(loc.X))

	methodStatus, methodIsStatuser := method.(Statuser)

	evalType, iterType, err := method.Init(loc, &funcInfo.FunctionInfo, xNext)
	if err != nil {
		return Failure, err
	}

	for {
		if funcInfo.IsStatuser {
			// Check the function status before evaluating.
			status, err = funcInfo.statuser.Status()
			if err != nil || status != NotTerminated {
				return
			}
		}

		// Perform evalType evaluation of the function at xNext and store the
		// result in location.
		evaluate(funcInfo, evalType, xNext, loc, stats)
		// Update the stats and optLoc.
		update(loc, optLoc, stats, iterType, startTime)
		// Get the convergence status before recording the new location.
		status = checkConvergence(optLoc, iterType, stats, settings)

		if settings.Recorder != nil {
			err = settings.Recorder.Record(loc, evalType, iterType, stats)
			if err != nil {
				if status == NotTerminated {
					status = Failure
				}
				return
			}
		}

		if status != NotTerminated {
			return
		}

		if methodIsStatuser {
			status, err = methodStatus.Status()
			if err != nil || status != NotTerminated {
				return
			}
		}

		// Find the next location (stored in-place into xNext).
		evalType, iterType, err = method.Iterate(loc, xNext)
		if err != nil {
			status = Failure
			return
		}
	}
	panic("unreachable")
}

func copyLocation(dst, src *Location) {
	dst.X = resize(dst.X, len(src.X))
	copy(dst.X, src.X)

	dst.F = src.F

	dst.Gradient = resize(dst.Gradient, len(src.Gradient))
	copy(dst.Gradient, src.Gradient)

	if src.Hessian != nil {
		if dst.Hessian == nil || dst.Hessian.Symmetric() != len(src.X) {
			dst.Hessian = mat64.NewSymDense(len(src.X), nil)
		}
		dst.Hessian.CopySym(src.Hessian)
	}
}

func getDefaultMethod(funcInfo *functionInfo) Method {
	if funcInfo.IsGradient || funcInfo.IsFunctionGradient {
		return &BFGS{}
	}
	// TODO: Implement a gradient-free method
	panic("optimize: gradient-free methods not yet coded")
}

// getStartingLocation allocates and initializes the starting location for the minimization.
func getStartingLocation(funcInfo *functionInfo, method Method, initX []float64, stats *Stats, settings *Settings) (*Location, EvaluationType, error) {
	dim := len(initX)
	loc := &Location{
		X: make([]float64, dim),
	}
	copy(loc.X, initX)
	if method.Needs().Gradient {
		loc.Gradient = make([]float64, dim)
	}
	if method.Needs().Hessian {
		loc.Hessian = mat64.NewSymDense(dim, nil)
	}

	evalType := NoEvaluation
	if settings.UseInitialData {
		loc.F = settings.InitialFunctionValue
		if loc.Gradient != nil {
			initG := settings.InitialGradient
			if len(initG) != dim {
				panic("local: initial location size mismatch")
			}
			copy(loc.Gradient, initG)
		}
	} else {
		evalType = FuncEvaluation
		if loc.Gradient != nil {
			evalType = FuncGradEvaluation
		}
		evaluate(funcInfo, evalType, loc.X, loc, stats)
	}

	if math.IsNaN(loc.F) {
		return loc, evalType, ErrNaN
	}
	if math.IsInf(loc.F, 1) {
		return loc, evalType, ErrInf
	}
	for _, v := range loc.Gradient {
		if math.IsInf(v, 0) {
			return loc, evalType, ErrGradInf
		}
		if math.IsNaN(v) {
			return loc, evalType, ErrGradNaN
		}
	}

	return loc, evalType, nil
}

func checkConvergence(loc *Location, iterType IterationType, stats *Stats, settings *Settings) Status {
	if iterType == MajorIteration || iterType == InitIteration {
		if loc.Gradient != nil {
			norm := floats.Norm(loc.Gradient, math.Inf(1))
			if norm < settings.GradientAbsTol {
				return GradientAbsoluteConvergence
			}
		}
		if loc.F < settings.FunctionAbsTol {
			return FunctionAbsoluteConvergence
		}
	}

	// Check every step for negative infinity because it could break the
	// linesearches and -inf is the best you can do anyway.
	if math.IsInf(loc.F, -1) {
		return FunctionNegativeInfinity
	}

	if settings.FuncEvaluations > 0 {
		totalFun := stats.FuncEvaluations + stats.FuncGradEvaluations + stats.FuncGradHessEvaluations
		if totalFun >= settings.FuncEvaluations {
			return FunctionEvaluationLimit
		}
	}

	if settings.GradEvaluations > 0 {
		totalGrad := stats.GradEvaluations + stats.FuncGradEvaluations + stats.FuncGradHessEvaluations
		if totalGrad >= settings.GradEvaluations {
			return GradientEvaluationLimit
		}
	}

	if settings.HessEvaluations > 0 {
		totalHess := stats.HessEvaluations + stats.FuncGradHessEvaluations
		if totalHess >= settings.HessEvaluations {
			return HessianEvaluationLimit
		}
	}

	if settings.Runtime > 0 {
		// TODO(vladimir-ch): It would be nice to update Runtime here.
		if stats.Runtime >= settings.Runtime {
			return RuntimeLimit
		}
	}

	if iterType == MajorIteration && settings.MajorIterations > 0 {
		if stats.MajorIterations >= settings.MajorIterations {
			return IterationLimit
		}
	}
	return NotTerminated
}

func invalidate(loc *Location, f, grad, hess bool) {
	if f {
		loc.F = math.NaN()
	}
	if grad && loc.Gradient != nil {
		loc.Gradient[0] = math.NaN()
	}
	if hess && loc.Hessian != nil {
		loc.Hessian.SetSym(0, 0, math.NaN())
	}
}

// evaluate evaluates the function given by funcInfo at xNext, stores the
// answer into loc and updates stats. If loc.X is not equal to xNext, then
// unused fields of loc are set to NaN.
// evaluate panics if the function does not support the requested evalType.
func evaluate(funcInfo *functionInfo, evalType EvaluationType, xNext []float64, loc *Location, stats *Stats) {
	different := !floats.Equal(loc.X, xNext)
	if different {
		copy(loc.X, xNext)
	}
	switch evalType {
	case FuncEvaluation:
		if different {
			invalidate(loc, false, true, true)
		}
		loc.F = funcInfo.function.Func(loc.X)
		stats.FuncEvaluations++
		return
	case GradEvaluation:
		if funcInfo.IsGradient {
			if different {
				invalidate(loc, true, false, true)
			}
			funcInfo.gradient.Grad(loc.X, loc.Gradient)
			stats.GradEvaluations++
			return
		}
		if funcInfo.IsFunctionGradient {
			if different {
				invalidate(loc, false, false, true)
			}
			loc.F = funcInfo.functionGradient.FuncGrad(loc.X, loc.Gradient)
			stats.FuncGradEvaluations++
			return
		}
		if funcInfo.IsFunctionGradientHessian {
			if loc.Hessian == nil {
				loc.Hessian = mat64.NewSymDense(len(loc.X), nil)
			}
			loc.F = funcInfo.functionGradientHessian.FuncGradHess(loc.X, loc.Gradient, loc.Hessian)
			stats.FuncGradHessEvaluations++
			return
		}
	case HessEvaluation:
		if funcInfo.IsHessian {
			if different {
				invalidate(loc, true, true, false)
			}
			funcInfo.hessian.Hess(loc.X, loc.Hessian)
			stats.HessEvaluations++
			return
		}
		if funcInfo.IsFunctionGradientHessian {
			if loc.Gradient == nil {
				loc.Gradient = make([]float64, len(loc.X))
			}
			loc.F = funcInfo.functionGradientHessian.FuncGradHess(loc.X, loc.Gradient, loc.Hessian)
			stats.FuncGradHessEvaluations++
			return
		}
	case FuncGradEvaluation:
		if funcInfo.IsFunctionGradient {
			if different {
				invalidate(loc, false, false, true)
			}
			loc.F = funcInfo.functionGradient.FuncGrad(loc.X, loc.Gradient)
			stats.FuncGradEvaluations++
			return
		}
		if funcInfo.IsGradient {
			if different {
				invalidate(loc, false, false, true)
			}
			loc.F = funcInfo.function.Func(loc.X)
			stats.FuncEvaluations++
			funcInfo.gradient.Grad(loc.X, loc.Gradient)
			stats.GradEvaluations++
			return
		}
		if funcInfo.IsFunctionGradientHessian {
			if loc.Hessian == nil {
				loc.Hessian = mat64.NewSymDense(len(loc.X), nil)
			}
			loc.F = funcInfo.functionGradientHessian.FuncGradHess(loc.X, loc.Gradient, loc.Hessian)
			stats.FuncGradHessEvaluations++
			return
		}
	case FuncGradHessEvaluation:
		if funcInfo.IsFunctionGradientHessian {
			loc.F = funcInfo.functionGradientHessian.FuncGradHess(loc.X, loc.Gradient, loc.Hessian)
			stats.FuncGradHessEvaluations++
			return
		}
		if funcInfo.IsGradient && funcInfo.IsHessian {
			loc.F = funcInfo.function.Func(loc.X)
			stats.FuncEvaluations++
			funcInfo.gradient.Grad(loc.X, loc.Gradient)
			stats.GradEvaluations++
			funcInfo.hessian.Hess(loc.X, loc.Hessian)
			stats.HessEvaluations++
			return
		}
	case NoEvaluation:
		if different {
			// Optimizers should not request NoEvaluation at a new location.
			// The intent and therefore an appropriate action are both unclear.
			panic("no evaluation requested at new location")
		}
		return
	default:
		panic(fmt.Sprintf("unknown evaluation type %v", evalType))
	}
	panic(fmt.Sprintf("objective function does not support %v", evalType))
}

// update updates the stats given the new evaluation
func update(loc *Location, optLoc *Location, stats *Stats, iterType IterationType, startTime time.Time) {
	if iterType == MajorIteration {
		stats.MajorIterations++
	}
	if loc.F <= optLoc.F {
		copyLocation(optLoc, loc)
	}
	stats.Runtime = time.Since(startTime)
}
<|endoftext|>


<commit_before>package main

import (
	"image/color"

	"github.com/leafo/weeklyloops/loops"
)

func main() {
	loop := loops.NewLoopWindow()

	loop.Title = "loop2"
	loop.Speed = 0.2

	circle := loops.NewCircle(1.0, 10)
	rect := loops.NewRectangle(0.5, 0.5)

	loop.Draw = func(t float64, g *loops.Graphics) {
		g.SetColor(color.RGBA{200, 200, 200, 255})
		g.DrawShape(circle)
		g.DrawShape(rect)
	}

	loop.Run()
}
<commit_msg>finish loop3<commit_after>package main

import (
	"image/color"
	"math"

	"github.com/leafo/weeklyloops/loops"
)

func main() {
	loop := loops.NewLoopWindow()

	loop.Title = "loop2"
	loop.Speed = 0.2

	circle := loops.NewCircle(1.0, 5)

	pal := loops.SmoothPalette{
		[]color.RGBA{
			color.RGBA{0, 0, 0, 255},
			color.RGBA{94, 92, 170, 255},
			color.RGBA{174, 111, 184, 255},
			color.RGBA{198, 137, 155, 255},
			color.RGBA{210, 187, 160, 255},
			color.RGBA{209, 224, 183, 255},
			color.RGBA{215, 241, 220, 255},
			color.RGBA{242, 250, 250, 255},
		},
	}

	loop.Draw = func(t float64, g *loops.Graphics) {
		//waver := math.Sin(t * math.Pi)
		t = t + math.Cos(t*math.Pi*40)/50.0

		g.SetColor(color.RGBA{200, 200, 200, 255})
		layers := 12
		for i := 0; i < layers-1; i += 1 {
			phase := (math.Sin(t*math.Pi*2+float64(i))+1.0)/6.0 + 2.0/3.0
			scale := float64(layers-i)/float64(layers)/2.0 + 0.5
			rot := t*math.Pi*2 + float64(i)

			m := loops.NewIdentityMat4().
				Scale(float32(phase), float32(phase), 1).
				Rotate(float32(rot)).
				Scale(float32(scale), float32(scale), 1)

			g.SetMat(m)
			g.SetColor(pal.Color(float64(i) / float64(layers)))
			g.DrawShape(circle)
		}
	}

	loop.Run()
}
<|endoftext|>


<commit_before>package sftp

// bufPool provides a pool of byte-slices to be reused in various parts of the package.
// It is safe to use concurrently through a pointer.
type bufPool struct {
	ch   chan []byte
	blen int
}

func newBufPool(depth, bufLen int) *bufPool {
	return &bufPool{
		ch:   make(chan []byte, depth),
		blen: bufLen,
	}
}

func (p *bufPool) Get() []byte {
	for {
		select {
		case b := <-p.ch:
			if cap(b) < p.blen {
				// just in case: throw away any buffer with insufficient capacity.
				continue
			}

			return b[:p.blen]

		default:
			return make([]byte, p.blen)
		}
	}
}

func (p *bufPool) Put(b []byte) {
	if p == nil {
		// functional default: no reuse.
		return
	}

	if cap(b) < p.blen || cap(b) > p.blen*2 {
		// DO NOT reuse buffers with insufficient capacity.
		// This could cause panics when resizing to p.blen.

		// DO NOT reuse buffers with excessive capacity.
		// This could cause memory leaks.
		return
	}

	select {
	case p.ch <- b:
	default:
	}
}

type resChanPool chan chan result

func newResChanPool(depth int) resChanPool {
	return make(chan chan result, depth)
}

func (p resChanPool) Get() chan result {
	select {
	case ch := <-p:
		return ch
	default:
		return make(chan result, 1)
	}
}

func (p resChanPool) Put(ch chan result) {
	select {
	case p <- ch:
	default:
	}
}
<commit_msg>actually, why not a better safe-guard<commit_after>package sftp

// bufPool provides a pool of byte-slices to be reused in various parts of the package.
// It is safe to use concurrently through a pointer.
type bufPool struct {
	ch   chan []byte
	blen int
}

func newBufPool(depth, bufLen int) *bufPool {
	return &bufPool{
		ch:   make(chan []byte, depth),
		blen: bufLen,
	}
}

func (p *bufPool) Get() []byte {
	if p.blen <= 0 {
		panic("bufPool: new buffer creation length must be greater than zero")
	}

	for {
		select {
		case b := <-p.ch:
			if cap(b) < p.blen {
				// just in case: throw away any buffer with insufficient capacity.
				continue
			}

			return b[:p.blen]

		default:
			return make([]byte, p.blen)
		}
	}
}

func (p *bufPool) Put(b []byte) {
	if p == nil {
		// functional default: no reuse.
		return
	}

	if cap(b) < p.blen || cap(b) > p.blen*2 {
		// DO NOT reuse buffers with insufficient capacity.
		// This could cause panics when resizing to p.blen.

		// DO NOT reuse buffers with excessive capacity.
		// This could cause memory leaks.
		return
	}

	select {
	case p.ch <- b:
	default:
	}
}

type resChanPool chan chan result

func newResChanPool(depth int) resChanPool {
	return make(chan chan result, depth)
}

func (p resChanPool) Get() chan result {
	select {
	case ch := <-p:
		return ch
	default:
		return make(chan result, 1)
	}
}

func (p resChanPool) Put(ch chan result) {
	select {
	case p <- ch:
	default:
	}
}
<|endoftext|>


<commit_before>package utils

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"
)

type PoolElemInterface interface {
	Recycle() //回收
	Close()
	Err() error
	SetErr(error)
	Active()         //激活
	Heartbeat()      //心跳
	Timeout()        //设置超时，激活心跳
	IsTimeout() bool //是否超时
}

type PoolElem struct {
	Error  error
	Mux    sync.Mutex
	status int32
}

func (this *PoolElem) Err() error {
	this.Mux.Lock()
	err := this.Error
	this.Mux.Unlock()
	return err
}

func (this *PoolElem) SetErr(err error) {
	this.Mux.Lock()
	this.Error = err
	this.Mux.Unlock()
}

func (this *PoolElem) Active() {
	atomic.StoreInt32(&this.status, 0)
}

func (this *PoolElem) Heartbeat() {
	//ping
}

func (this *PoolElem) Timeout() {
	atomic.AddInt32(&this.status, 1)
}

func (this *PoolElem) IsTimeout() bool {
	return atomic.LoadInt32(&this.status) > 1
}

type Pool struct {
	callback    func(*Pool) (PoolElemInterface, error)
	elems       chan PoolElemInterface
	activeElems chan PoolElemInterface
	maxIdle     int32
	maxActive   int32
	curActive   int32
	timer       int32
	status      int32 //0正常
}

func NewPool(callback func(*Pool) (PoolElemInterface, error), maxIdle, maxActive, timer int32) *Pool {
	pool := &Pool{
		callback:    callback,
		elems:       make(chan PoolElemInterface, maxIdle),
		activeElems: make(chan PoolElemInterface, maxActive),
		maxIdle:     maxIdle,
		maxActive:   maxActive,
		timer:       timer,
	}
	if timer > 0 {
		go pool.timerEvent()
	}
	return pool
}

func (this *Pool) Put(elem PoolElemInterface) {
	if atomic.LoadInt32(&this.status) != 0 {
		elem.Close()
		return
	}
	atomic.AddInt32(&this.curActive, -1)
	if elem.Err() != nil {
		elem.Close()
		return
	}

	select {
	case this.elems <- elem:
		break
	default:
		select {
		case this.activeElems <- elem:
		default:
			elem.Close()
		}
	}
}
func (this *Pool) Get() (PoolElemInterface, error) {
	var (
		elem PoolElemInterface
		err  error
	)
	for {
		elem, err = this.get()
		if elem != nil && elem.Err() != nil {
			atomic.AddInt32(&this.curActive, -1)
			elem.Close()
			continue
		}
		break
	}
	return elem, err
}
func (this *Pool) get() (PoolElemInterface, error) {
	var (
		conn PoolElemInterface
		err  error
	)
	if atomic.LoadInt32(&this.status) != 0 {
		return conn, err
	}
	select {
	case e := <-this.elems:
		conn = e
	default:
		select {
		case e := <-this.activeElems:
			conn = e
		default:
			ca := atomic.LoadInt32(&this.curActive)
			if ca < this.maxActive {
				conn, err = this.callback(this)
			} else {
				fmt.Println("Error 0001 : too many active conn, maxActive=", this.maxActive)
				conn = <-this.elems
				fmt.Println("return e")
			}
		}
	}
	if conn != nil {
		atomic.AddInt32(&this.curActive, 1)
		conn.Active()
	}
	return conn, err
}

func (this *Pool) Close() {
	atomic.StoreInt32(&this.status, 1)
	for {
		select {
		case e := <-this.elems:
			e.Close()
		case e := <-this.activeElems:
			e.Close()
		default:
			return
		}
	}
}

func (this *Pool) timerEvent() {
	timer := time.NewTicker(time.Second * time.Duration(this.timer))
	defer timer.Stop()
	for atomic.LoadInt32(&this.status) == 0 {
		select {
		case <-timer.C:
			select {
			case e := <-this.elems:
				if e.IsTimeout() {
					e.Heartbeat()
					e.Active()
				} else {
					e.Timeout()
				}
				e.Recycle()
			default:
				break
			}
			select {
			case e := <-this.activeElems:
				if e.IsTimeout() { //回收多余的空闲的链接
					e.Close()
				} else {
					//e.Do("PING")
					e.Timeout()
					e.Recycle()
				}
			}
		}
	}
}
<commit_msg>timerEvent<commit_after>package utils

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"
)

type PoolElemInterface interface {
	Recycle() //回收
	Close()
	Err() error
	SetErr(error)
	Active()         //激活
	Heartbeat()      //心跳
	Timeout()        //设置超时，激活心跳
	IsTimeout() bool //是否超时
}

type PoolElem struct {
	Error  error
	Mux    sync.Mutex
	status int32
}

func (this *PoolElem) Err() error {
	this.Mux.Lock()
	err := this.Error
	this.Mux.Unlock()
	return err
}

func (this *PoolElem) SetErr(err error) {
	this.Mux.Lock()
	this.Error = err
	this.Mux.Unlock()
}

func (this *PoolElem) Active() {
	atomic.StoreInt32(&this.status, 0)
}

func (this *PoolElem) Heartbeat() {
	//ping
}

func (this *PoolElem) Timeout() {
	atomic.AddInt32(&this.status, 1)
}

func (this *PoolElem) IsTimeout() bool {
	return atomic.LoadInt32(&this.status) > 1
}

type Pool struct {
	callback    func(*Pool) (PoolElemInterface, error)
	elems       chan PoolElemInterface
	activeElems chan PoolElemInterface
	maxIdle     int32
	maxActive   int32
	curActive   int32
	timer       int32
	status      int32 //0正常
}

func NewPool(callback func(*Pool) (PoolElemInterface, error), maxIdle, maxActive, timer int32) *Pool {
	pool := &Pool{
		callback:    callback,
		elems:       make(chan PoolElemInterface, maxIdle),
		activeElems: make(chan PoolElemInterface, maxActive),
		maxIdle:     maxIdle,
		maxActive:   maxActive,
		timer:       timer,
	}
	if timer > 0 {
		go pool.timerEvent()
	}
	return pool
}

func (this *Pool) Put(elem PoolElemInterface) {
	if atomic.LoadInt32(&this.status) != 0 {
		elem.Close()
		return
	}
	atomic.AddInt32(&this.curActive, -1)
	if elem.Err() != nil {
		elem.Close()
		return
	}

	select {
	case this.elems <- elem:
		break
	default:
		select {
		case this.activeElems <- elem:
		default:
			elem.Close()
		}
	}
}
func (this *Pool) Get() (PoolElemInterface, error) {
	var (
		elem PoolElemInterface
		err  error
	)
	for {
		elem, err = this.get()
		if elem != nil && elem.Err() != nil {
			atomic.AddInt32(&this.curActive, -1)
			elem.Close()
			continue
		}
		break
	}
	return elem, err
}
func (this *Pool) get() (PoolElemInterface, error) {
	var (
		conn PoolElemInterface
		err  error
	)
	if atomic.LoadInt32(&this.status) != 0 {
		return conn, err
	}
	select {
	case e := <-this.elems:
		conn = e
	default:
		select {
		case e := <-this.activeElems:
			conn = e
		default:
			ca := atomic.LoadInt32(&this.curActive)
			if ca < this.maxActive {
				conn, err = this.callback(this)
			} else {
				fmt.Println("Error 0001 : too many active conn, maxActive=", this.maxActive)
				conn = <-this.elems
				fmt.Println("return e")
			}
		}
	}
	if conn != nil {
		atomic.AddInt32(&this.curActive, 1)
		conn.Active()
	}
	return conn, err
}

func (this *Pool) Close() {
	atomic.StoreInt32(&this.status, 1)
	for {
		select {
		case e := <-this.elems:
			e.Close()
		case e := <-this.activeElems:
			e.Close()
		default:
			return
		}
	}
}

func (this *Pool) timerEvent() {
	timer := time.NewTicker(time.Second * time.Duration(this.timer))
	defer timer.Stop()
	for atomic.LoadInt32(&this.status) == 0 {
		select {
		case <-timer.C:
			select {
			case e := <-this.elems:
				e.Heartbeat()
				e.Recycle()
			default:
				break
			}
			select {
			case e := <-this.activeElems:
				if e.IsTimeout() { //回收多余的空闲的链接
					e.Close()
				} else {
					//e.Do("PING")
					e.Timeout()
					e.Recycle()
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package scipipe

import (
	"sync"
)

// ------------------------------------------------------------------------
// InPort
// ------------------------------------------------------------------------

// InPort represents a pluggable connection to multiple out-ports from other
// processes, from its own process, and with which it is communicating via
// channels under the hood
type InPort struct {
	Chan        chan *FileIP
	name        string
	process     WorkflowProcess
	RemotePorts map[string]*OutPort
	ready       bool
	closeLock   sync.Mutex
}

// NewInPort returns a new InPort struct
func NewInPort(name string) *InPort {
	inp := &InPort{
		name:        name,
		RemotePorts: map[string]*OutPort{},
		Chan:        make(chan *FileIP, BUFSIZE), // This one will contain merged inputs from inChans
		ready:       false,
	}
	return inp
}

// Name returns the name of the InPort
func (pt *InPort) Name() string {
	return pt.Process().Name() + "." + pt.name
}

// Process returns the process connected to the port
func (pt *InPort) Process() WorkflowProcess {
	if pt.process == nil {
		Failf("In-port %s has no connected process", pt.name)
	}
	return pt.process
}

// SetProcess sets the process of the port to p
func (pt *InPort) SetProcess(p WorkflowProcess) {
	pt.process = p
}

// AddRemotePort adds a remote OutPort to the InPort
func (pt *InPort) AddRemotePort(rpt *OutPort) {
	if pt.RemotePorts[rpt.Name()] != nil {
		Failf("[Process:%s]: A remote port with name %s already exists, for param-port %s connected to process %s\n", pt.Process().Name(), rpt.Name(), pt.Name(), rpt.Process().Name())
	}
	pt.RemotePorts[rpt.Name()] = rpt
}

// From connects an OutPort to the InPort
func (pt *InPort) From(rpt *OutPort) {
	pt.AddRemotePort(rpt)
	rpt.AddRemotePort(pt)

	pt.SetReady(true)
	rpt.SetReady(true)
}

// Disconnect disconnects the (out-)port with name rptName, from the InPort
func (pt *InPort) Disconnect(rptName string) {
	pt.removeRemotePort(rptName)
	if len(pt.RemotePorts) == 0 {
		pt.SetReady(false)
	}
}

// removeRemotePort removes the (out-)port with name rptName, from the InPort
func (pt *InPort) removeRemotePort(rptName string) {
	delete(pt.RemotePorts, rptName)
}

// SetReady sets the ready status of the InPort
func (pt *InPort) SetReady(ready bool) {
	pt.ready = ready
}

// Ready tells whether the port is ready or not
func (pt *InPort) Ready() bool {
	return pt.ready
}

// Send sends IPs to the in-port, and is supposed to be called from the remote
// (out-) port, to send to this in-port
func (pt *InPort) Send(ip *FileIP) {
	pt.Chan <- ip
}

// Recv receives IPs from the port
func (pt *InPort) Recv() *FileIP {
	return <-pt.Chan
}

// CloseConnection closes the connection to the remote out-port with name
// rptName, on the InPort
func (pt *InPort) CloseConnection(rptName string) {
	pt.closeLock.Lock()
	delete(pt.RemotePorts, rptName)
	if len(pt.RemotePorts) == 0 {
		close(pt.Chan)
	}
	pt.closeLock.Unlock()
}

// ------------------------------------------------------------------------
// OutPort
// ------------------------------------------------------------------------

// OutPort represents a pluggable connection to multiple in-ports from other
// processes, from its own process, and with which it is communicating via
// channels under the hood
type OutPort struct {
	name        string
	process     WorkflowProcess
	RemotePorts map[string]*InPort
	ready       bool
}

// NewOutPort returns a new OutPort struct
func NewOutPort(name string) *OutPort {
	outp := &OutPort{
		name:        name,
		RemotePorts: map[string]*InPort{},
		ready:       false,
	}
	return outp
}

// Name returns the name of the OutPort
func (pt *OutPort) Name() string {
	return pt.Process().Name() + "." + pt.name
}

// Process returns the process connected to the port
func (pt *OutPort) Process() WorkflowProcess {
	if pt.process == nil {
		Failf("Out-port %s has no connected process", pt.name)
	}
	return pt.process
}

// SetProcess sets the process of the port to p
func (pt *OutPort) SetProcess(p WorkflowProcess) {
	pt.process = p
}

// AddRemotePort adds a remote InPort to the OutPort
func (pt *OutPort) AddRemotePort(rpt *InPort) {
	if _, ok := pt.RemotePorts[rpt.Name()]; ok {
		Failf("[Process:%s]: A remote port with name %s already exists, for param-port %s connected to process %s\n", pt.Process().Name(), rpt.Name(), pt.Name(), rpt.Process().Name())
	}
	pt.RemotePorts[rpt.Name()] = rpt
}

// removeRemotePort removes the (in-)port with name rptName, from the OutPort
func (pt *OutPort) removeRemotePort(rptName string) {
	if _, ok := pt.RemotePorts[rptName]; !ok {
		Failf("[Process:%s]: No remote port with name %s exists, for param-port %s connected to process %s\n", pt.Process().Name(), rptName, pt.Name(), pt.Process().Name())
	}
	delete(pt.RemotePorts, rptName)
}

// To connects an InPort to the OutPort
func (pt *OutPort) To(rpt *InPort) {
	pt.AddRemotePort(rpt)
	rpt.AddRemotePort(pt)

	pt.SetReady(true)
	rpt.SetReady(true)
}

// Disconnect disconnects the (in-)port with name rptName, from the OutPort
func (pt *OutPort) Disconnect(rptName string) {
	pt.removeRemotePort(rptName)
	if len(pt.RemotePorts) == 0 {
		pt.SetReady(false)
	}
}

// SetReady sets the ready status of the OutPort
func (pt *OutPort) SetReady(ready bool) {
	pt.ready = ready
}

// Ready tells whether the port is ready or not
func (pt *OutPort) Ready() bool {
	return pt.ready
}

// Send sends an FileIP to all the in-ports connected to the OutPort
func (pt *OutPort) Send(ip *FileIP) {
	for _, rpt := range pt.RemotePorts {
		Debug.Printf("Sending on out-port %s connected to in-port %s", pt.Name(), rpt.Name())
		rpt.Send(ip)
	}
}

// Close closes the connection between this port and all the ports it is
// connected to. If this port is the last connected port to an in-port, that
// in-ports channel will also be closed.
func (pt *OutPort) Close() {
	for _, rpt := range pt.RemotePorts {
		Debug.Printf("Closing out-port %s connected to in-port %s", pt.Name(), rpt.Name())
		rpt.CloseConnection(pt.Name())
		pt.removeRemotePort(rpt.Name())
	}
}

// ------------------------------------------------------------------------
// InParamPort
// ------------------------------------------------------------------------

// InParamPort is an in-port for parameter values of string type
type InParamPort struct {
	Chan        chan string
	name        string
	process     WorkflowProcess
	RemotePorts map[string]*OutParamPort
	ready       bool
	closeLock   sync.Mutex
}

// NewInParamPort returns a new InParamPort
func NewInParamPort(name string) *InParamPort {
	return &InParamPort{
		name:        name,
		Chan:        make(chan string, BUFSIZE),
		RemotePorts: map[string]*OutParamPort{},
	}
}

// Name returns the name of the InParamPort
func (pip *InParamPort) Name() string {
	return pip.Process().Name() + "." + pip.name
}

// Process returns the process that is connected to the port
func (pip *InParamPort) Process() WorkflowProcess {
	if pip.process == nil {
		Failf("Parameter in-port %s has no connected process", pip.name)
	}
	return pip.process
}

// SetProcess sets the process of the port to p
func (pip *InParamPort) SetProcess(p WorkflowProcess) {
	pip.process = p
}

// AddRemotePort adds a remote OutParamPort to the InParamPort
func (pip *InParamPort) AddRemotePort(pop *OutParamPort) {
	if pip.RemotePorts[pop.Name()] != nil {
		Failf("[Process:%s]: A remote param port with name %s already exists, for in-param-port %s connected to process %s\n", pip.Process().Name(), pop.Name(), pip.Name(), pop.Process().Name())
	}
	pip.RemotePorts[pop.Name()] = pop
}

// From connects one parameter port with another one
func (pip *InParamPort) From(pop *OutParamPort) {
	pip.AddRemotePort(pop)
	pop.AddRemotePort(pip)

	pip.SetReady(true)
	pop.SetReady(true)
}

// FromStr connects a parameter port with a new go-routine feeding the
// strings in strings, on the fly, to the parameter port
func (pip *InParamPort) FromStr(strings ...string) {
	pop := NewOutParamPort("string_feeder")
	pop.process = pip.Process()
	pip.From(pop)
	go func() {
		defer pop.Close()
		for _, str := range strings {
			pop.Send(str)
		}
	}()
}

// SetReady sets the ready status of the InParamPort
func (pip *InParamPort) SetReady(ready bool) {
	pip.ready = ready
}

// Ready tells whether the port is ready or not
func (pip *InParamPort) Ready() bool {
	return pip.ready
}

// Send sends IPs to the in-port, and is supposed to be called from the remote
// (out-) port, to send to this in-port
func (pip *InParamPort) Send(param string) {
	pip.Chan <- param
}

// Recv receiveds a param value over the ports connection
func (pip *InParamPort) Recv() string {
	return <-pip.Chan
}

// CloseConnection closes the connection to the remote out-port with name
// popName, on the InParamPort
func (pip *InParamPort) CloseConnection(popName string) {
	pip.closeLock.Lock()
	delete(pip.RemotePorts, popName)
	if len(pip.RemotePorts) == 0 {
		close(pip.Chan)
	}
	pip.closeLock.Unlock()
}

// ------------------------------------------------------------------------
// OutParamPort
// ------------------------------------------------------------------------

// OutParamPort is an out-port for parameter values of string type
type OutParamPort struct {
	name        string
	process     WorkflowProcess
	RemotePorts map[string]*InParamPort
	ready       bool
}

// NewOutParamPort returns a new OutParamPort
func NewOutParamPort(name string) *OutParamPort {
	return &OutParamPort{
		name:        name,
		RemotePorts: map[string]*InParamPort{},
	}
}

// Name returns the name of the OutParamPort
func (pop *OutParamPort) Name() string {
	return pop.Process().Name() + "." + pop.name
}

// Process returns the process that is connected to the port
func (pop *OutParamPort) Process() WorkflowProcess {
	if pop.process == nil {
		Failf("Parameter out-port %s has no connected process", pop.name)
	}
	return pop.process
}

// SetProcess sets the process of the port to p
func (pop *OutParamPort) SetProcess(p WorkflowProcess) {
	pop.process = p
}

// AddRemotePort adds a remote InParamPort to the OutParamPort
func (pop *OutParamPort) AddRemotePort(pip *InParamPort) {
	if pop.RemotePorts[pip.Name()] != nil {
		Failf("[Process:%s]: A remote param port with name %s already exists, for in-param-port %s connected to process %s\n", pop.Process().Name(), pip.Name(), pop.Name(), pip.Process().Name())
	}
	pop.RemotePorts[pip.Name()] = pip
}

// To connects an InParamPort to the OutParamPort
func (pop *OutParamPort) To(pip *InParamPort) {
	pop.AddRemotePort(pip)
	pip.AddRemotePort(pop)

	pop.SetReady(true)
	pip.SetReady(true)
}

// Disconnect disonnects the (in-)port with name rptName, from the OutParamPort
func (pop *OutParamPort) Disconnect(pipName string) {
	pop.removeRemotePort(pipName)
	if len(pop.RemotePorts) == 0 {
		pop.SetReady(false)
	}
}

// removeRemotePort removes the (in-)port with name rptName, from the OutParamPort
func (pop *OutParamPort) removeRemotePort(pipName string) {
	delete(pop.RemotePorts, pipName)
}

// SetReady sets the ready status of the OutParamPort
func (pop *OutParamPort) SetReady(ready bool) {
	pop.ready = ready
}

// Ready tells whether the port is ready or not
func (pop *OutParamPort) Ready() bool {
	return pop.ready
}

// Send sends an FileIP to all the in-ports connected to the OutParamPort
func (pop *OutParamPort) Send(param string) {
	for _, pip := range pop.RemotePorts {
		Debug.Printf("Sending on out-param-port %s connected to in-param-port %s", pop.Name(), pip.Name())
		pip.Send(param)
	}
}

// Close closes the connection between this port and all the ports it is
// connected to. If this port is the last connected port to an in-port, that
// in-ports channel will also be closed.
func (pop *OutParamPort) Close() {
	for _, pip := range pop.RemotePorts {
		Debug.Printf("Closing out-param-port %s connected to in-param-port %s", pop.Name(), pip.Name())
		pip.CloseConnection(pop.Name())
		pop.removeRemotePort(pip.Name())
	}
}
<commit_msg>Add FromInt() method on param inport<commit_after>package scipipe

import (
	"strconv"
	"sync"
)

// ------------------------------------------------------------------------
// InPort
// ------------------------------------------------------------------------

// InPort represents a pluggable connection to multiple out-ports from other
// processes, from its own process, and with which it is communicating via
// channels under the hood
type InPort struct {
	Chan        chan *FileIP
	name        string
	process     WorkflowProcess
	RemotePorts map[string]*OutPort
	ready       bool
	closeLock   sync.Mutex
}

// NewInPort returns a new InPort struct
func NewInPort(name string) *InPort {
	inp := &InPort{
		name:        name,
		RemotePorts: map[string]*OutPort{},
		Chan:        make(chan *FileIP, BUFSIZE), // This one will contain merged inputs from inChans
		ready:       false,
	}
	return inp
}

// Name returns the name of the InPort
func (pt *InPort) Name() string {
	return pt.Process().Name() + "." + pt.name
}

// Process returns the process connected to the port
func (pt *InPort) Process() WorkflowProcess {
	if pt.process == nil {
		Failf("In-port %s has no connected process", pt.name)
	}
	return pt.process
}

// SetProcess sets the process of the port to p
func (pt *InPort) SetProcess(p WorkflowProcess) {
	pt.process = p
}

// AddRemotePort adds a remote OutPort to the InPort
func (pt *InPort) AddRemotePort(rpt *OutPort) {
	if pt.RemotePorts[rpt.Name()] != nil {
		Failf("[Process:%s]: A remote port with name %s already exists, for param-port %s connected to process %s\n", pt.Process().Name(), rpt.Name(), pt.Name(), rpt.Process().Name())
	}
	pt.RemotePorts[rpt.Name()] = rpt
}

// From connects an OutPort to the InPort
func (pt *InPort) From(rpt *OutPort) {
	pt.AddRemotePort(rpt)
	rpt.AddRemotePort(pt)

	pt.SetReady(true)
	rpt.SetReady(true)
}

// Disconnect disconnects the (out-)port with name rptName, from the InPort
func (pt *InPort) Disconnect(rptName string) {
	pt.removeRemotePort(rptName)
	if len(pt.RemotePorts) == 0 {
		pt.SetReady(false)
	}
}

// removeRemotePort removes the (out-)port with name rptName, from the InPort
func (pt *InPort) removeRemotePort(rptName string) {
	delete(pt.RemotePorts, rptName)
}

// SetReady sets the ready status of the InPort
func (pt *InPort) SetReady(ready bool) {
	pt.ready = ready
}

// Ready tells whether the port is ready or not
func (pt *InPort) Ready() bool {
	return pt.ready
}

// Send sends IPs to the in-port, and is supposed to be called from the remote
// (out-) port, to send to this in-port
func (pt *InPort) Send(ip *FileIP) {
	pt.Chan <- ip
}

// Recv receives IPs from the port
func (pt *InPort) Recv() *FileIP {
	return <-pt.Chan
}

// CloseConnection closes the connection to the remote out-port with name
// rptName, on the InPort
func (pt *InPort) CloseConnection(rptName string) {
	pt.closeLock.Lock()
	delete(pt.RemotePorts, rptName)
	if len(pt.RemotePorts) == 0 {
		close(pt.Chan)
	}
	pt.closeLock.Unlock()
}

// ------------------------------------------------------------------------
// OutPort
// ------------------------------------------------------------------------

// OutPort represents a pluggable connection to multiple in-ports from other
// processes, from its own process, and with which it is communicating via
// channels under the hood
type OutPort struct {
	name        string
	process     WorkflowProcess
	RemotePorts map[string]*InPort
	ready       bool
}

// NewOutPort returns a new OutPort struct
func NewOutPort(name string) *OutPort {
	outp := &OutPort{
		name:        name,
		RemotePorts: map[string]*InPort{},
		ready:       false,
	}
	return outp
}

// Name returns the name of the OutPort
func (pt *OutPort) Name() string {
	return pt.Process().Name() + "." + pt.name
}

// Process returns the process connected to the port
func (pt *OutPort) Process() WorkflowProcess {
	if pt.process == nil {
		Failf("Out-port %s has no connected process", pt.name)
	}
	return pt.process
}

// SetProcess sets the process of the port to p
func (pt *OutPort) SetProcess(p WorkflowProcess) {
	pt.process = p
}

// AddRemotePort adds a remote InPort to the OutPort
func (pt *OutPort) AddRemotePort(rpt *InPort) {
	if _, ok := pt.RemotePorts[rpt.Name()]; ok {
		Failf("[Process:%s]: A remote port with name %s already exists, for param-port %s connected to process %s\n", pt.Process().Name(), rpt.Name(), pt.Name(), rpt.Process().Name())
	}
	pt.RemotePorts[rpt.Name()] = rpt
}

// removeRemotePort removes the (in-)port with name rptName, from the OutPort
func (pt *OutPort) removeRemotePort(rptName string) {
	if _, ok := pt.RemotePorts[rptName]; !ok {
		Failf("[Process:%s]: No remote port with name %s exists, for param-port %s connected to process %s\n", pt.Process().Name(), rptName, pt.Name(), pt.Process().Name())
	}
	delete(pt.RemotePorts, rptName)
}

// To connects an InPort to the OutPort
func (pt *OutPort) To(rpt *InPort) {
	pt.AddRemotePort(rpt)
	rpt.AddRemotePort(pt)

	pt.SetReady(true)
	rpt.SetReady(true)
}

// Disconnect disconnects the (in-)port with name rptName, from the OutPort
func (pt *OutPort) Disconnect(rptName string) {
	pt.removeRemotePort(rptName)
	if len(pt.RemotePorts) == 0 {
		pt.SetReady(false)
	}
}

// SetReady sets the ready status of the OutPort
func (pt *OutPort) SetReady(ready bool) {
	pt.ready = ready
}

// Ready tells whether the port is ready or not
func (pt *OutPort) Ready() bool {
	return pt.ready
}

// Send sends an FileIP to all the in-ports connected to the OutPort
func (pt *OutPort) Send(ip *FileIP) {
	for _, rpt := range pt.RemotePorts {
		Debug.Printf("Sending on out-port %s connected to in-port %s", pt.Name(), rpt.Name())
		rpt.Send(ip)
	}
}

// Close closes the connection between this port and all the ports it is
// connected to. If this port is the last connected port to an in-port, that
// in-ports channel will also be closed.
func (pt *OutPort) Close() {
	for _, rpt := range pt.RemotePorts {
		Debug.Printf("Closing out-port %s connected to in-port %s", pt.Name(), rpt.Name())
		rpt.CloseConnection(pt.Name())
		pt.removeRemotePort(rpt.Name())
	}
}

// ------------------------------------------------------------------------
// InParamPort
// ------------------------------------------------------------------------

// InParamPort is an in-port for parameter values of string type
type InParamPort struct {
	Chan        chan string
	name        string
	process     WorkflowProcess
	RemotePorts map[string]*OutParamPort
	ready       bool
	closeLock   sync.Mutex
}

// NewInParamPort returns a new InParamPort
func NewInParamPort(name string) *InParamPort {
	return &InParamPort{
		name:        name,
		Chan:        make(chan string, BUFSIZE),
		RemotePorts: map[string]*OutParamPort{},
	}
}

// Name returns the name of the InParamPort
func (pip *InParamPort) Name() string {
	return pip.Process().Name() + "." + pip.name
}

// Process returns the process that is connected to the port
func (pip *InParamPort) Process() WorkflowProcess {
	if pip.process == nil {
		Failf("Parameter in-port %s has no connected process", pip.name)
	}
	return pip.process
}

// SetProcess sets the process of the port to p
func (pip *InParamPort) SetProcess(p WorkflowProcess) {
	pip.process = p
}

// AddRemotePort adds a remote OutParamPort to the InParamPort
func (pip *InParamPort) AddRemotePort(pop *OutParamPort) {
	if pip.RemotePorts[pop.Name()] != nil {
		Failf("[Process:%s]: A remote param port with name %s already exists, for in-param-port %s connected to process %s\n", pip.Process().Name(), pop.Name(), pip.Name(), pop.Process().Name())
	}
	pip.RemotePorts[pop.Name()] = pop
}

// From connects one parameter port with another one
func (pip *InParamPort) From(pop *OutParamPort) {
	pip.AddRemotePort(pop)
	pop.AddRemotePort(pip)

	pip.SetReady(true)
	pop.SetReady(true)
}

// FromStr feeds one or more parameters of type string to a port
func (pip *InParamPort) FromStr(strings ...string) {
	pop := NewOutParamPort("string_feeder")
	pop.process = pip.Process()
	pip.From(pop)
	go func() {
		defer pop.Close()
		for _, str := range strings {
			pop.Send(str)
		}
	}()
}

// FromInt feeds one or more parameters of type int to a port
func (pip *InParamPort) FromInt(ints ...int) {
	params := []string{}
	for _, i := range ints {
		params = append(params, strconv.Itoa(i))
	}
	pip.FromStr(params...)
}

// SetReady sets the ready status of the InParamPort
func (pip *InParamPort) SetReady(ready bool) {
	pip.ready = ready
}

// Ready tells whether the port is ready or not
func (pip *InParamPort) Ready() bool {
	return pip.ready
}

// Send sends IPs to the in-port, and is supposed to be called from the remote
// (out-) port, to send to this in-port
func (pip *InParamPort) Send(param string) {
	pip.Chan <- param
}

// Recv receiveds a param value over the ports connection
func (pip *InParamPort) Recv() string {
	return <-pip.Chan
}

// CloseConnection closes the connection to the remote out-port with name
// popName, on the InParamPort
func (pip *InParamPort) CloseConnection(popName string) {
	pip.closeLock.Lock()
	delete(pip.RemotePorts, popName)
	if len(pip.RemotePorts) == 0 {
		close(pip.Chan)
	}
	pip.closeLock.Unlock()
}

// ------------------------------------------------------------------------
// OutParamPort
// ------------------------------------------------------------------------

// OutParamPort is an out-port for parameter values of string type
type OutParamPort struct {
	name        string
	process     WorkflowProcess
	RemotePorts map[string]*InParamPort
	ready       bool
}

// NewOutParamPort returns a new OutParamPort
func NewOutParamPort(name string) *OutParamPort {
	return &OutParamPort{
		name:        name,
		RemotePorts: map[string]*InParamPort{},
	}
}

// Name returns the name of the OutParamPort
func (pop *OutParamPort) Name() string {
	return pop.Process().Name() + "." + pop.name
}

// Process returns the process that is connected to the port
func (pop *OutParamPort) Process() WorkflowProcess {
	if pop.process == nil {
		Failf("Parameter out-port %s has no connected process", pop.name)
	}
	return pop.process
}

// SetProcess sets the process of the port to p
func (pop *OutParamPort) SetProcess(p WorkflowProcess) {
	pop.process = p
}

// AddRemotePort adds a remote InParamPort to the OutParamPort
func (pop *OutParamPort) AddRemotePort(pip *InParamPort) {
	if pop.RemotePorts[pip.Name()] != nil {
		Failf("[Process:%s]: A remote param port with name %s already exists, for in-param-port %s connected to process %s\n", pop.Process().Name(), pip.Name(), pop.Name(), pip.Process().Name())
	}
	pop.RemotePorts[pip.Name()] = pip
}

// To connects an InParamPort to the OutParamPort
func (pop *OutParamPort) To(pip *InParamPort) {
	pop.AddRemotePort(pip)
	pip.AddRemotePort(pop)

	pop.SetReady(true)
	pip.SetReady(true)
}

// Disconnect disonnects the (in-)port with name rptName, from the OutParamPort
func (pop *OutParamPort) Disconnect(pipName string) {
	pop.removeRemotePort(pipName)
	if len(pop.RemotePorts) == 0 {
		pop.SetReady(false)
	}
}

// removeRemotePort removes the (in-)port with name rptName, from the OutParamPort
func (pop *OutParamPort) removeRemotePort(pipName string) {
	delete(pop.RemotePorts, pipName)
}

// SetReady sets the ready status of the OutParamPort
func (pop *OutParamPort) SetReady(ready bool) {
	pop.ready = ready
}

// Ready tells whether the port is ready or not
func (pop *OutParamPort) Ready() bool {
	return pop.ready
}

// Send sends an FileIP to all the in-ports connected to the OutParamPort
func (pop *OutParamPort) Send(param string) {
	for _, pip := range pop.RemotePorts {
		Debug.Printf("Sending on out-param-port %s connected to in-param-port %s", pop.Name(), pip.Name())
		pip.Send(param)
	}
}

// Close closes the connection between this port and all the ports it is
// connected to. If this port is the last connected port to an in-port, that
// in-ports channel will also be closed.
func (pop *OutParamPort) Close() {
	for _, pip := range pop.RemotePorts {
		Debug.Printf("Closing out-param-port %s connected to in-param-port %s", pop.Name(), pip.Name())
		pip.CloseConnection(pop.Name())
		pop.removeRemotePort(pip.Name())
	}
}
<|endoftext|>


<commit_before>//
// Blackblog
// Copyright 2012 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"errors"
	"io"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// A Post contains the metadata about a post.
type Post struct {
	// The full path to the Markdown file containing this post.
	Filename string

	// The title of the post, from the metadata.
	Title string

	// The URL fragment of the post, from the metadata.
	URLFragment string

	// The date the post was published, from the metadata.
	Date string

	// The MD5 checksum of the file's contents.
	checksum []byte
}

// GetPostsInDirectory recursively examines the directory at the path and finds
// any Markdown (.md) files and returns the corresponding Post objects.
func GetPostsInDirectory(dirPath string) (posts PostList, err error) {
	err = filepath.Walk(dirPath, func(file string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && strings.HasSuffix(file, ".md") {
			if post, err := NewPostFromPath(file); post != nil && err == nil {
				posts = append(posts, post)
			} else {
				return err
			}
		}
		return nil
	})
	return
}

// NewPostFromPath creates a new Post object from the file at the given path
// with the metadata updated.
func NewPostFromPath(path string) (*Post, error) {
	p := &Post{Filename: path}
	p.UpdateMetadata()
	if len(p.checksum) < 1 {
		return nil, errors.New("Could not checksum blog post")
	}
	return p, nil
}

// Open returns an opened file handle for the Post.
func (p *Post) Open() (*os.File, error) {
	return os.Open(p.Filename)
}

// IsUpToDate checks that the in-memory data about the Post matches that of the
// file on disk.
func (p *Post) IsUpToDate() bool {
	file, err := p.Open()
	if err != nil {
		return false
	}
	defer file.Close()
	return p.isUpToDateInternal(file)
}

func (p *Post) isUpToDateInternal(file *os.File) bool {
	fd, err := p.Open()
	if err != nil {
		return false
	}
	defer fd.Close()

	return bytes.Equal(p.checksum, computeChecksum(fd))
}

func computeChecksum(file *os.File) []byte {
	digest := md5.New()
	io.Copy(digest, file)
	return digest.Sum(nil)
}

// GetContents returns the Markdown content of a post, excluding metadata.
func (p *Post) GetContents() ([]byte, error) {
	file, err := p.Open()
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var result []byte
	reader := bufio.NewReader(file)
	for {
		line, err := reader.ReadString('\n')

		// If an error occurred, return the error except for EOF.
		if err != nil && err != io.EOF {
			return nil, err
		}

		// Skip lines that are for metadata.
		if len(line) >= 2 && string(line[0:2]) == "~~" {
			continue
		}

		// Store the rest of the file.
		result = append(result, line...)

		// If this is the end of file, exit the loop to return the result.
		if err == io.EOF {
			break
		}
	}
	return result, nil
}

// UpdateMetadata re-reads the file on disk and updates the in-memory metadata.
func (p *Post) UpdateMetadata() {
	file, err := p.Open()
	if err != nil || p.isUpToDateInternal(file) {
		return
	}
	defer file.Close()
	p.checksum = computeChecksum(file)

	file.Seek(0, 0)
	reader := bufio.NewReader(file)
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF || len(line) < 2 || line[0:2] != "~~" {
			break
		}
		p.parseMetadataLine(line)
	}
}

func (p *Post) parseMetadataLine(line string) {
	if len(line) < 2 || line[0:2] != "~~" {
		return
	}
	line = line[2:]

	pieces := strings.SplitN(line, ":", 2)
	if len(pieces) != 2 {
		return
	}

	val := strings.TrimSpace(pieces[1])

	switch strings.ToLower(strings.TrimSpace(pieces[0])) {
	case "title":
		p.Title = val
	case "url":
		p.URLFragment = val
	case "date":
		p.Date = val
	}
}

// CreateURL constructs the URL of a post based on its metadata.
func (p *Post) CreateURL() string {
	// First, create the file's basename.
	basename := p.URLFragment
	if basename == "" && p.Title != "" {
		basename = regexp.MustCompile("[^A-Za-z0-9_]").ReplaceAllString(p.Title, "_")
		basename = regexp.MustCompile("_{1,}").ReplaceAllString(basename, "_")
	} else if basename == "" {
		basename = path.Base(p.Filename)
		ext := path.Ext(basename)
		basename = basename[:len(basename)-len(ext)]
	}

	basename = strings.ToLower(basename)
	if strings.HasSuffix(basename, "_") {
		basename = basename[:len(basename)-1]
	}
	url := basename + ".html"

	// Next, try and get the date of the post to include subdirectories.
	if t := parseDate(p.Date); !t.IsZero() {
		year, month, _ := t.Date()
		url = path.Join(strconv.FormatInt(int64(year), 10), strconv.Itoa(int(month)), url)
	}

	return url
}

func parseDate(input string) time.Time {
	if input == "" {
		return time.Time{}
	}

	t, err := time.Parse("_2 January 2006", input)
	if err == nil {
		return t
	}

	t, err = time.Parse("January _2, 2006", input)
	if err == nil {
		return t
	}

	t, err = time.Parse("January _2 2006", input)
	if err == nil {
		return t
	}

	return time.Time{}
}

// sort.Interface implementation:

type PostList []*Post

func (pl PostList) Len() int {
	return len(pl)
}

func (pl PostList) Less(i, j int) bool {
	return pl[i].CreateURL() < pl[j].CreateURL()
}

func (pl PostList) Swap(i, j int) {
	pl[i], pl[j] = pl[j], pl[i]
}
<commit_msg>Do not re-open the file in isUpToDateInternal.<commit_after>//
// Blackblog
// Copyright 2012 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"errors"
	"io"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// A Post contains the metadata about a post.
type Post struct {
	// The full path to the Markdown file containing this post.
	Filename string

	// The title of the post, from the metadata.
	Title string

	// The URL fragment of the post, from the metadata.
	URLFragment string

	// The date the post was published, from the metadata.
	Date string

	// The MD5 checksum of the file's contents.
	checksum []byte
}

// GetPostsInDirectory recursively examines the directory at the path and finds
// any Markdown (.md) files and returns the corresponding Post objects.
func GetPostsInDirectory(dirPath string) (posts PostList, err error) {
	err = filepath.Walk(dirPath, func(file string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && strings.HasSuffix(file, ".md") {
			if post, err := NewPostFromPath(file); post != nil && err == nil {
				posts = append(posts, post)
			} else {
				return err
			}
		}
		return nil
	})
	return
}

// NewPostFromPath creates a new Post object from the file at the given path
// with the metadata updated.
func NewPostFromPath(path string) (*Post, error) {
	p := &Post{Filename: path}
	p.UpdateMetadata()
	if len(p.checksum) < 1 {
		return nil, errors.New("Could not checksum blog post")
	}
	return p, nil
}

// Open returns an opened file handle for the Post.
func (p *Post) Open() (*os.File, error) {
	return os.Open(p.Filename)
}

// IsUpToDate checks that the in-memory data about the Post matches that of the
// file on disk.
func (p *Post) IsUpToDate() bool {
	file, err := p.Open()
	if err != nil {
		return false
	}
	defer file.Close()
	return p.isUpToDateInternal(file)
}

func (p *Post) isUpToDateInternal(file *os.File) bool {
	if p.checksum == nil {
		return false
	}
	return bytes.Equal(p.checksum, computeChecksum(file))
}

func computeChecksum(file *os.File) []byte {
	digest := md5.New()
	io.Copy(digest, file)
	return digest.Sum(nil)
}

// GetContents returns the Markdown content of a post, excluding metadata.
func (p *Post) GetContents() ([]byte, error) {
	file, err := p.Open()
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var result []byte
	reader := bufio.NewReader(file)
	for {
		line, err := reader.ReadString('\n')

		// If an error occurred, return the error except for EOF.
		if err != nil && err != io.EOF {
			return nil, err
		}

		// Skip lines that are for metadata.
		if len(line) >= 2 && string(line[0:2]) == "~~" {
			continue
		}

		// Store the rest of the file.
		result = append(result, line...)

		// If this is the end of file, exit the loop to return the result.
		if err == io.EOF {
			break
		}
	}
	return result, nil
}

// UpdateMetadata re-reads the file on disk and updates the in-memory metadata.
func (p *Post) UpdateMetadata() {
	file, err := p.Open()
	if err != nil || p.isUpToDateInternal(file) {
		return
	}
	defer file.Close()
	p.checksum = computeChecksum(file)

	file.Seek(0, 0)
	reader := bufio.NewReader(file)
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF || len(line) < 2 || line[0:2] != "~~" {
			break
		}
		p.parseMetadataLine(line)
	}
}

func (p *Post) parseMetadataLine(line string) {
	if len(line) < 2 || line[0:2] != "~~" {
		return
	}
	line = line[2:]

	pieces := strings.SplitN(line, ":", 2)
	if len(pieces) != 2 {
		return
	}

	val := strings.TrimSpace(pieces[1])

	switch strings.ToLower(strings.TrimSpace(pieces[0])) {
	case "title":
		p.Title = val
	case "url":
		p.URLFragment = val
	case "date":
		p.Date = val
	}
}

// CreateURL constructs the URL of a post based on its metadata.
func (p *Post) CreateURL() string {
	// First, create the file's basename.
	basename := p.URLFragment
	if basename == "" && p.Title != "" {
		basename = regexp.MustCompile("[^A-Za-z0-9_]").ReplaceAllString(p.Title, "_")
		basename = regexp.MustCompile("_{1,}").ReplaceAllString(basename, "_")
	} else if basename == "" {
		basename = path.Base(p.Filename)
		ext := path.Ext(basename)
		basename = basename[:len(basename)-len(ext)]
	}

	basename = strings.ToLower(basename)
	if strings.HasSuffix(basename, "_") {
		basename = basename[:len(basename)-1]
	}
	url := basename + ".html"

	// Next, try and get the date of the post to include subdirectories.
	if t := parseDate(p.Date); !t.IsZero() {
		year, month, _ := t.Date()
		url = path.Join(strconv.FormatInt(int64(year), 10), strconv.Itoa(int(month)), url)
	}

	return url
}

func parseDate(input string) time.Time {
	if input == "" {
		return time.Time{}
	}

	t, err := time.Parse("_2 January 2006", input)
	if err == nil {
		return t
	}

	t, err = time.Parse("January _2, 2006", input)
	if err == nil {
		return t
	}

	t, err = time.Parse("January _2 2006", input)
	if err == nil {
		return t
	}

	return time.Time{}
}

// sort.Interface implementation:

type PostList []*Post

func (pl PostList) Len() int {
	return len(pl)
}

func (pl PostList) Less(i, j int) bool {
	return pl[i].CreateURL() < pl[j].CreateURL()
}

func (pl PostList) Swap(i, j int) {
	pl[i], pl[j] = pl[j], pl[i]
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
)

// process table generation

var validProcessNumber = regexp.MustCompile(`^[0-9/]+$`)

func IsNumberString(s string) bool {
	match := validProcessNumber.FindStringSubmatch(s)
	if match == nil {
		return false
	}
	return true
}

func init() {
	detectSystem()
}

var Sysname string

func detectSystem() {
	c := exec.Command("uname", "-s")
	o, err := c.Output()

	if err == nil {
		Sysname = strings.Trim(string(o), " \n\t")
	}
}

func ProcessTable() *map[int]bool {
	switch Sysname {
	case "Linux":
		return LinuxPs()

	case "Darwin":
		return DarwinPs()
	}
	m := make(map[int]bool)
	return &m
}

var regexDarwinPs = regexp.MustCompile(`^[^0-9]*[0-9]+[^0-9]+([0-9]+)`)

func DarwinPs() *map[int]bool {

	res := make(map[int]bool)
	o, err := exec.Command("/bin/ps", "-ef").Output()
	if err != nil {
		panic(err)
	}
	s := string(o)

	lines := strings.Split(s, "\n")
	for i := range lines {
		match := regexDarwinPs.FindStringSubmatch(lines[i])
		if match != nil && len(match) == 2 {
			//fmt.Printf("match = %#v\n", match[1])
			num, err := strconv.Atoi(match[1])
			if err == nil {
				res[num] = true
			}
		}
	}
	return &res
}

func LinuxPs() *map[int]bool {

	res := make(map[int]bool)

	fileInfoSlice, err := ioutil.ReadDir("/proc")
	if err != nil {
		panic(err)
	}

	for i := range fileInfoSlice {
		if fileInfoSlice[i].IsDir() {
			if IsNumberString(fileInfoSlice[i].Name()) {
				num, err := strconv.Atoi(fileInfoSlice[i].Name())
				if err != nil {
					panic("IsNumberString regex failed to filter down to only numbers")
				}
				res[num] = true
			}
		}
	}
	return &res
}
<commit_msg>latest ptab<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// process table generation

var validProcessNumber = regexp.MustCompile(`^[0-9/]+$`)

func IsNumberString(s string) bool {
	match := validProcessNumber.FindStringSubmatch(s)
	if match == nil {
		return false
	}
	return true
}

func init() {
	detectSystem()
}

var Sysname string

func detectSystem() {
	c := exec.Command("uname", "-s")
	o, err := c.Output()

	if err == nil {
		Sysname = strings.Trim(string(o), " \n\t")
	}
}

func ProcessTable() *map[int]bool {
	switch Sysname {
	case "Linux":
		return LinuxPs()

	case "Darwin":
		return DarwinPs()
	}
	m := make(map[int]bool)
	return &m
}

var regexDarwinPs = regexp.MustCompile(`^[^0-9]*[0-9]+[^0-9]+([0-9]+)`)

func DarwinPs() *map[int]bool {

	res := make(map[int]bool)
	o, err := exec.Command("/bin/ps", "-ef").Output()
	if err != nil {
		panic(err)
	}
	s := string(o)

	lines := strings.Split(s, "\n")
	for i := range lines {
		match := regexDarwinPs.FindStringSubmatch(lines[i])
		if match != nil && len(match) == 2 {
			//fmt.Printf("match = %#v\n", match[1])
			num, err := strconv.Atoi(match[1])
			if err == nil {
				res[num] = true
			}
		}
	}
	return &res
}

func LinuxPs() *map[int]bool {

	res := make(map[int]bool)

	fileInfoSlice, err := ioutil.ReadDir("/proc")
	if err != nil {
		panic(err)
	}

	for i := range fileInfoSlice {
		if fileInfoSlice[i].IsDir() {
			if IsNumberString(fileInfoSlice[i].Name()) {
				num, err := strconv.Atoi(fileInfoSlice[i].Name())
				if err != nil {
					panic("IsNumberString regex failed to filter down to only numbers")
				}
				res[num] = true
			}
		}
	}
	return &res
}

func OpenFiles(pid int) []string {
	switch Sysname {
	case "Linux":
		return LinuxOpenFiles(pid)

	case "Darwin":
		return DarwinOpenFiles(pid)
	}
	return []string{}
}

func LinuxOpenFiles(jobservPid int) []string {

	// wait until process shows up (in /proc or ps)
	waited := 0
	for {
		pt := *ProcessTable()
		_, jsAlive := pt[jobservPid]
		if jsAlive {
			break
		}
		time.Sleep(50 * time.Millisecond)
		waited++
		if waited > 10 {
			panic(fmt.Sprintf("jobserv with expected pid %d did not show up in /proc after 10 waits", jobservPid))
		}
	}
	fmt.Printf("\njobserv with expected pid %d was *found* in /proc after %d waits of 50msec\n", jobservPid, waited)

	// read all the open files
	fileInfoSlice, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", jobservPid))
	if err != nil {
		panic(err)
	}

	res := make([]string, len(fileInfoSlice))
	for i := range fileInfoSlice {
		res[i] = fileInfoSlice[i].Name()
	}

	return res
}

func DarwinOpenFiles(pid int) []string {
	// wait until process shows up (in /proc or ps)
	waited := 0
	for {
		pt := *ProcessTable()
		_, jsAlive := pt[pid]
		if jsAlive {
			break
		}
		time.Sleep(50 * time.Millisecond)
		waited++
		if waited > 10 {
			panic(fmt.Sprintf("jobserv with expected pid %d did not show up in /proc after 10 waits", pid))
		}
	}
	fmt.Printf("\njobserv with expected pid %d was *found* in /proc after %d waits of 50msec\n", pid, waited)

	spid := fmt.Sprintf("%d", pid)
	//fmt.Printf("about to do: lsof -p %s -Fnt\n", spid)
	o, err := exec.Command("/usr/sbin/lsof", "-p", spid, "-Fnt").Output()
	if err != nil {
		panic(err)
	}
	s := strings.Split(string(o), "\n")

	// join the name and type together on one line
	// skip the header line
	s = s[1:]
	r := make([]string, 0)
	n := len(s) - 1 // empty string at end
	//fmt.Printf("len(s) = %d; s='%#v'\n", len(s), s)
	for i := 0; i < n; i += 2 {
		//fmt.Printf("on i = %d, s[i]='%s', s[i+i]='%s'\n", i, s[i], s[i+1])
		r = append(r, s[i]+":"+s[i+1])
	}

	//fmt.Printf("DarwinOpenFiles got back from lsof:\n")
	//ShowStrings(r)

	return r
}

func SetDiff(a, b []string) []string {
	res := make([]string, 0)
	m := make(map[string]bool)
	for i := range a {
		m[a[i]] = true
	}
	for i := range b {
		delete(m, b[i])
	}
	for s := range m {
		res = append(res, s)
	}
	return res
}

func ShowStrings(s []string) {
	for _, line := range s {
		fmt.Printf("%s\n", line)
	}
}
<|endoftext|>


<commit_before>package raft

import (
	"fmt"
	"log"
	"sync"
)

type RaftState uint8

const (
	Follower RaftState = iota
	Candidate
	Leader
)

var (
	keyCurrentTerm  = []byte("CurrentTerm")
	keyLastVoteTerm = []byte("LastVoteTerm")
	keyLastVoteCand = []byte("LastVoteCand")
	keyCandidateId  = []byte("CandidateId")
)

type Raft struct {
	// Configuration
	conf *Config

	// Current state
	state RaftState

	// stable is a StableStore implementation for durable state
	stable StableStore

	// Cache the current term, write through to StableStore
	currentTerm uint64

	// logs is a LogStore implementation to keep our logs
	logs LogStore

	// Cache the latest log, though we can get from LogStore
	lastLog uint64

	// Highest commited log entry
	commitIndex uint64

	// Last applied log to the FSM
	lastApplied uint64

	// FSM is the state machine that can handle the logs
	fsm FSM

	// The transport layer we use
	trans Transport

	// If we are the leader, we have extra state
	leader *LeaderState

	// Shutdown channel to exit
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// NewRaft is used to construct a new Raft node
func NewRaft(conf *Config, stable StableStore, logs LogStore, fsm FSM, trans Transport) (*Raft, error) {
	// Read the last log value
	lastLog, err := logs.LastIndex()
	if err != nil {
		return nil, fmt.Errorf("Failed to find last log: %v", err)
	}

	// Create Raft struct
	r := &Raft{
		conf:        conf,
		state:       Follower,
		stable:      stable,
		logs:        logs,
		lastLog:     lastLog,
		commitIndex: 0,
		lastApplied: 0,
		fsm:         fsm,
		trans:       trans,
		shutdownCh:  make(chan struct{}),
	}

	// Start the background work
	go r.run()
	return r, nil
}

// run is a long running goroutine that runs the Raft FSM
func (r *Raft) run() {
	for {
		// Check if we are doing a shutdown
		select {
		case <-r.shutdownCh:
			return
		default:
		}

		// Enter into a sub-FSM
		switch r.state {
		case Follower:
			r.runFollower()
		case Candidate:
			r.runCandidate()
		case Leader:
			r.runLeader()
		}
	}
}

// runFollower runs the FSM for a follower
func (r *Raft) runFollower() {
	ch := r.trans.Consumer()
	for {
		select {
		case rpc := <-ch:
			switch cmd := rpc.Command.(type) {
			case *AppendEntriesRequest:
				r.appendEntries(rpc, cmd)
			case *RequestVoteRequest:
				r.requestVote(rpc, cmd)
			default:
				log.Printf("[ERR] Follower state, got unexpected command: %#v",
					rpc.Command)
				rpc.Respond(nil, fmt.Errorf("Unexpected command"))
			}

		case <-randomTimeout(r.conf.HeartbeatTimeout):
			// Heartbeat failed! Go to the candidate state
			r.state = Candidate
			return

		case <-r.shutdownCh:
			return
		}
	}
}

// runCandidate runs the FSM for a candidate
func (r *Raft) runCandidate() {
	ch := r.trans.Consumer()
	transition := false
	for !transition {
		select {
		case rpc := <-ch:
			switch cmd := rpc.Command.(type) {
			case *AppendEntriesRequest:
				transition = r.appendEntries(rpc, cmd)
			case *RequestVoteRequest:
				transition = r.requestVote(rpc, cmd)
			default:
				log.Printf("[ERR] Candidate state, got unexpected command: %#v",
					rpc.Command)
				rpc.Respond(nil, fmt.Errorf("Unexpected command"))
			}

		case <-randomTimeout(r.conf.ElectionTimeout):
			// Election failed! Restart the elction. We simply return,
			// which will kick us back into runCandidate
			return

		case <-r.shutdownCh:
			return
		}
	}
}

// runLeader runs the FSM for a leader
func (r *Raft) runLeader() {
	for {
		select {
		case <-r.shutdownCh:
			return
		}
	}
}

// Shutdown is used to stop the Raft background routines.
// This is not a graceful operation.
func (r *Raft) Shutdown() {
	r.shutdownLock.Lock()
	defer r.shutdownLock.Lock()

	if r.shutdownCh != nil {
		close(r.shutdownCh)
		r.shutdownCh = nil
	}
}

// appendEntries is invoked when we get an append entries RPC call
// Returns true if we transition to a Follower
func (r *Raft) appendEntries(rpc RPC, a *AppendEntriesRequest) (transition bool) {
	// Setup a response
	resp := &AppendEntriesResponse{
		Term:    r.currentTerm,
		Success: false,
	}
	var err error
	defer rpc.Respond(resp, err)

	// Ignore an older term
	if a.Term < r.currentTerm {
		return
	}

	// Increase the term if we see a newer one, also transition to follower
	// if we ever get an appendEntries call
	if a.Term > r.currentTerm || r.state != Follower {
		r.currentTerm = a.Term
		resp.Term = a.Term

		// Ensure transition to follower
		transition = true
		r.state = Follower
	}

	// Verify the last log entry
	var prevLog Log
	if err := r.logs.GetLog(a.PrevLogEntry, &prevLog); err != nil {
		log.Printf("[WARN] Failed to get previous log: %d %v",
			a.PrevLogEntry, err)
		return
	}
	if a.PrevLogTerm != prevLog.Term {
		log.Printf("[WARN] Previous log term mis-match: ours: %d remote: %d",
			prevLog.Term, a.PrevLogTerm)
		return
	}

	// Add all the entries
	for _, entry := range a.Entries {
		// Delete any conflicting entries
		if entry.Index <= r.lastLog {
			log.Printf("[WARN] Clearing log suffix from %d to %d", entry.Index, r.lastLog)
			if err := r.logs.DeleteRange(entry.Index, r.lastLog); err != nil {
				log.Printf("[ERR] Failed to clear log suffix: %v", err)
				return
			}
		}

		// Append the entry
		if err := r.logs.StoreLog(entry); err != nil {
			log.Printf("[ERR] Failed to append to log: %v", err)
			return
		}

		// Update the lastLog
		r.lastLog = entry.Index
	}

	// Update the commit index
	if a.LeaderCommitIndex > r.commitIndex {
		r.commitIndex = min(a.LeaderCommitIndex, r.lastLog)

		// TODO: Trigger applying logs locally!
	}

	// Everything went well, set success
	resp.Success = true
	return
}

// requestVote is invoked when we get an request vote RPC call
// Returns true if we transition to a Follower
func (r *Raft) requestVote(rpc RPC, req *RequestVoteRequest) (transition bool) {
	// Setup a response
	resp := &RequestVoteResponse{
		Term:    r.currentTerm,
		Granted: false,
	}
	var err error
	defer rpc.Respond(resp, err)

	// Ignore an older term
	if req.Term < r.currentTerm {
		return
	}

	// Increase the term if we see a newer one
	if req.Term > r.currentTerm {
		r.currentTerm = req.Term
		resp.Term = req.Term

		// Ensure transition to follower
		transition = true
		r.state = Follower
	}

	// Check if we have voted yet
	lastVoteTerm, err := r.stable.GetUint64(keyLastVoteTerm)
	if err != nil && err.Error() != "not found" {
		log.Printf("[ERR] Failed to get last vote term: %v", err)
		return
	}
	lastVoteCandBytes, err := r.stable.Get(keyLastVoteCand)
	if err != nil && err.Error() != "not found" {
		log.Printf("[ERR] Failed to get last vote candidate: %v", err)
		return
	}

	// Check if we've voted in this election before
	if lastVoteTerm == req.Term && lastVoteCandBytes != nil {
		log.Printf("[INFO] Duplicate RequestVote for same term: %d", req.Term)
		if string(lastVoteCandBytes) == req.CandidateId {
			log.Printf("[WARN] Duplicate RequestVote from candidate: %s", req.CandidateId)
			resp.Granted = true
		}
		return
	}

	// Reject if their term is older
	if r.lastLog > 0 {
		var lastLog Log
		if err := r.logs.GetLog(r.lastLog, &lastLog); err != nil {
			log.Printf("[ERR] Failed to get last log: %d %v",
				r.lastLog, err)
			return
		}
		if lastLog.Term > req.LastLogTerm {
			log.Printf("[WARN] Rejecting vote since our last term is greater")
			return
		}

		if lastLog.Index > req.LastLogIndex {
			log.Printf("[WARN] Rejecting vote since our last index is greater")
			return
		}
	}

	// Seems we should grant a vote
	if err := r.stable.SetUint64(keyLastVoteTerm, req.Term); err != nil {
		log.Printf("[ERR] Failed to persist last vote term: %v", err)
		return
	}
	if err := r.stable.Set(keyLastVoteCand, []byte(req.CandidateId)); err != nil {
		log.Printf("[ERR] Failed to persist last vote candidate: %v", err)
		return
	}

	resp.Granted = true
	return
}
<commit_msg>Adding some useful logging messages<commit_after>package raft

import (
	"fmt"
	"log"
	"sync"
)

type RaftState uint8

const (
	Follower RaftState = iota
	Candidate
	Leader
)

var (
	keyCurrentTerm  = []byte("CurrentTerm")
	keyLastVoteTerm = []byte("LastVoteTerm")
	keyLastVoteCand = []byte("LastVoteCand")
	keyCandidateId  = []byte("CandidateId")
)

type Raft struct {
	// Configuration
	conf *Config

	// Current state
	state RaftState

	// stable is a StableStore implementation for durable state
	stable StableStore

	// Cache the current term, write through to StableStore
	currentTerm uint64

	// logs is a LogStore implementation to keep our logs
	logs LogStore

	// Cache the latest log, though we can get from LogStore
	lastLog uint64

	// Highest commited log entry
	commitIndex uint64

	// Last applied log to the FSM
	lastApplied uint64

	// FSM is the state machine that can handle the logs
	fsm FSM

	// The transport layer we use
	trans Transport

	// If we are the leader, we have extra state
	leader *LeaderState

	// Shutdown channel to exit
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// NewRaft is used to construct a new Raft node
func NewRaft(conf *Config, stable StableStore, logs LogStore, fsm FSM, trans Transport) (*Raft, error) {
	// Read the last log value
	lastLog, err := logs.LastIndex()
	if err != nil {
		return nil, fmt.Errorf("Failed to find last log: %v", err)
	}

	// Create Raft struct
	r := &Raft{
		conf:        conf,
		state:       Follower,
		stable:      stable,
		logs:        logs,
		lastLog:     lastLog,
		commitIndex: 0,
		lastApplied: 0,
		fsm:         fsm,
		trans:       trans,
		shutdownCh:  make(chan struct{}),
	}

	// Start the background work
	go r.run()
	return r, nil
}

// run is a long running goroutine that runs the Raft FSM
func (r *Raft) run() {
	for {
		// Check if we are doing a shutdown
		select {
		case <-r.shutdownCh:
			return
		default:
		}

		// Enter into a sub-FSM
		switch r.state {
		case Follower:
			r.runFollower()
		case Candidate:
			r.runCandidate()
		case Leader:
			r.runLeader()
		}
	}
}

// runFollower runs the FSM for a follower
func (r *Raft) runFollower() {
	log.Printf("[INFO] Entering Follower state")
	ch := r.trans.Consumer()
	for {
		select {
		case rpc := <-ch:
			switch cmd := rpc.Command.(type) {
			case *AppendEntriesRequest:
				r.appendEntries(rpc, cmd)
			case *RequestVoteRequest:
				r.requestVote(rpc, cmd)
			default:
				log.Printf("[ERR] Follower state, got unexpected command: %#v",
					rpc.Command)
				rpc.Respond(nil, fmt.Errorf("Unexpected command"))
			}

		case <-randomTimeout(r.conf.HeartbeatTimeout):
			// Heartbeat failed! Go to the candidate state
			log.Printf("[WARN] Heartbeat timeout reached, starting election")
			r.state = Candidate
			return

		case <-r.shutdownCh:
			return
		}
	}
}

// runCandidate runs the FSM for a candidate
func (r *Raft) runCandidate() {
	log.Printf("[INFO] Entering Candidate state")
	ch := r.trans.Consumer()
	transition := false
	for !transition {
		select {
		case rpc := <-ch:
			switch cmd := rpc.Command.(type) {
			case *AppendEntriesRequest:
				transition = r.appendEntries(rpc, cmd)
			case *RequestVoteRequest:
				transition = r.requestVote(rpc, cmd)
			default:
				log.Printf("[ERR] Candidate state, got unexpected command: %#v",
					rpc.Command)
				rpc.Respond(nil, fmt.Errorf("Unexpected command"))
			}

		case <-randomTimeout(r.conf.ElectionTimeout):
			// Election failed! Restart the elction. We simply return,
			// which will kick us back into runCandidate
			log.Printf("[WARN] Election timeout reached, restarting election")
			return

		case <-r.shutdownCh:
			return
		}
	}
}

// runLeader runs the FSM for a leader
func (r *Raft) runLeader() {
	log.Printf("[INFO] Entering Leader state")
	for {
		select {
		case <-r.shutdownCh:
			return
		}
	}
}

// Shutdown is used to stop the Raft background routines.
// This is not a graceful operation.
func (r *Raft) Shutdown() {
	r.shutdownLock.Lock()
	defer r.shutdownLock.Lock()

	if r.shutdownCh != nil {
		close(r.shutdownCh)
		r.shutdownCh = nil
	}
}

// appendEntries is invoked when we get an append entries RPC call
// Returns true if we transition to a Follower
func (r *Raft) appendEntries(rpc RPC, a *AppendEntriesRequest) (transition bool) {
	// Setup a response
	resp := &AppendEntriesResponse{
		Term:    r.currentTerm,
		Success: false,
	}
	var err error
	defer rpc.Respond(resp, err)

	// Ignore an older term
	if a.Term < r.currentTerm {
		return
	}

	// Increase the term if we see a newer one, also transition to follower
	// if we ever get an appendEntries call
	if a.Term > r.currentTerm || r.state != Follower {
		r.currentTerm = a.Term
		resp.Term = a.Term

		// Ensure transition to follower
		transition = true
		r.state = Follower
	}

	// Verify the last log entry
	var prevLog Log
	if err := r.logs.GetLog(a.PrevLogEntry, &prevLog); err != nil {
		log.Printf("[WARN] Failed to get previous log: %d %v",
			a.PrevLogEntry, err)
		return
	}
	if a.PrevLogTerm != prevLog.Term {
		log.Printf("[WARN] Previous log term mis-match: ours: %d remote: %d",
			prevLog.Term, a.PrevLogTerm)
		return
	}

	// Add all the entries
	for _, entry := range a.Entries {
		// Delete any conflicting entries
		if entry.Index <= r.lastLog {
			log.Printf("[WARN] Clearing log suffix from %d to %d", entry.Index, r.lastLog)
			if err := r.logs.DeleteRange(entry.Index, r.lastLog); err != nil {
				log.Printf("[ERR] Failed to clear log suffix: %v", err)
				return
			}
		}

		// Append the entry
		if err := r.logs.StoreLog(entry); err != nil {
			log.Printf("[ERR] Failed to append to log: %v", err)
			return
		}

		// Update the lastLog
		r.lastLog = entry.Index
	}

	// Update the commit index
	if a.LeaderCommitIndex > r.commitIndex {
		r.commitIndex = min(a.LeaderCommitIndex, r.lastLog)

		// TODO: Trigger applying logs locally!
	}

	// Everything went well, set success
	resp.Success = true
	return
}

// requestVote is invoked when we get an request vote RPC call
// Returns true if we transition to a Follower
func (r *Raft) requestVote(rpc RPC, req *RequestVoteRequest) (transition bool) {
	// Setup a response
	resp := &RequestVoteResponse{
		Term:    r.currentTerm,
		Granted: false,
	}
	var err error
	defer rpc.Respond(resp, err)

	// Ignore an older term
	if req.Term < r.currentTerm {
		return
	}

	// Increase the term if we see a newer one
	if req.Term > r.currentTerm {
		r.currentTerm = req.Term
		resp.Term = req.Term

		// Ensure transition to follower
		transition = true
		r.state = Follower
	}

	// Check if we have voted yet
	lastVoteTerm, err := r.stable.GetUint64(keyLastVoteTerm)
	if err != nil && err.Error() != "not found" {
		log.Printf("[ERR] Failed to get last vote term: %v", err)
		return
	}
	lastVoteCandBytes, err := r.stable.Get(keyLastVoteCand)
	if err != nil && err.Error() != "not found" {
		log.Printf("[ERR] Failed to get last vote candidate: %v", err)
		return
	}

	// Check if we've voted in this election before
	if lastVoteTerm == req.Term && lastVoteCandBytes != nil {
		log.Printf("[INFO] Duplicate RequestVote for same term: %d", req.Term)
		if string(lastVoteCandBytes) == req.CandidateId {
			log.Printf("[WARN] Duplicate RequestVote from candidate: %s", req.CandidateId)
			resp.Granted = true
		}
		return
	}

	// Reject if their term is older
	if r.lastLog > 0 {
		var lastLog Log
		if err := r.logs.GetLog(r.lastLog, &lastLog); err != nil {
			log.Printf("[ERR] Failed to get last log: %d %v",
				r.lastLog, err)
			return
		}
		if lastLog.Term > req.LastLogTerm {
			log.Printf("[WARN] Rejecting vote since our last term is greater")
			return
		}

		if lastLog.Index > req.LastLogIndex {
			log.Printf("[WARN] Rejecting vote since our last index is greater")
			return
		}
	}

	// Seems we should grant a vote
	if err := r.stable.SetUint64(keyLastVoteTerm, req.Term); err != nil {
		log.Printf("[ERR] Failed to persist last vote term: %v", err)
		return
	}
	if err := r.stable.Set(keyLastVoteCand, []byte(req.CandidateId)); err != nil {
		log.Printf("[ERR] Failed to persist last vote candidate: %v", err)
		return
	}

	resp.Granted = true
	return
}
<|endoftext|>