<commit_before>//auto-generated by difficulty-convert.py DO NOT EDIT

package sudoku

func init() {
	difficultySignalWeights = map[string]float64{
		"Block Block Interactions Count" : 0.1274,
		"Block Block Interactions Percentage" : 0.005,
		"Constant" : 0.1689,
		"Forcing Chain (1 steps) Count" : 0.0,
		"Forcing Chain (1 steps) Percentage" : 0.0,
		"Forcing Chain (2 steps) Count" : 0.0,
		"Forcing Chain (2 steps) Percentage" : 0.0,
		"Forcing Chain (3 steps) Count" : 0.1262,
		"Forcing Chain (3 steps) Percentage" : 0.002,
		"Forcing Chain (4 steps) Count" : 0.0773,
		"Forcing Chain (4 steps) Percentage" : 0.0005,
		"Forcing Chain (5 steps) Count" : 0.1613,
		"Forcing Chain (5 steps) Percentage" : 0.0023,
		"Forcing Chain (6 steps) Count" : -0.0115,
		"Forcing Chain (6 steps) Percentage" : 0.003,
		"Guess Count" : -0.0117,
		"Guess Percentage" : -0.0021,
		"Hidden Pair Block Count" : -0.08,
		"Hidden Pair Block Percentage" : 0.003,
		"Hidden Pair Col Count" : -0.0254,
		"Hidden Pair Col Percentage" : 0.0039,
		"Hidden Pair Row Count" : -0.0081,
		"Hidden Pair Row Percentage" : 0.0018,
		"Hidden Quad Block Count" : 0.1444,
		"Hidden Quad Block Percentage" : 0.0022,
		"Hidden Quad Col Count" : 0.1724,
		"Hidden Quad Col Percentage" : 0.0028,
		"Hidden Quad Row Count" : -0.1,
		"Hidden Quad Row Percentage" : -0.0014,
		"Hidden Triple Block Count" : 0.0674,
		"Hidden Triple Block Percentage" : -0.0002,
		"Hidden Triple Col Count" : 0.1395,
		"Hidden Triple Col Percentage" : 0.0032,
		"Hidden Triple Row Count" : 0.0896,
		"Hidden Triple Row Percentage" : 0.0018,
		"Naked Pair Block Count" : -0.0739,
		"Naked Pair Block Percentage" : -0.0113,
		"Naked Pair Col Count" : -0.0651,
		"Naked Pair Col Percentage" : 0.0041,
		"Naked Pair Row Count" : -0.0502,
		"Naked Pair Row Percentage" : 0.0322,
		"Naked Quad Block Count" : -0.0284,
		"Naked Quad Block Percentage" : 0.0196,
		"Naked Quad Col Count" : -0.0186,
		"Naked Quad Col Percentage" : 0.0067,
		"Naked Quad Row Count" : -0.0697,
		"Naked Quad Row Percentage" : -0.0023,
		"Naked Triple Block Count" : -0.0728,
		"Naked Triple Block Percentage" : 0.0008,
		"Naked Triple Col Count" : -0.0382,
		"Naked Triple Col Percentage" : 0.0158,
		"Naked Triple Row Count" : -0.0159,
		"Naked Triple Row Percentage" : 0.0165,
		"Necessary In Block Count" : -0.0584,
		"Necessary In Block Percentage" : 0.0189,
		"Necessary In Col Count" : -0.037,
		"Necessary In Col Percentage" : -0.0473,
		"Necessary In Row Count" : -0.0439,
		"Necessary In Row Percentage" : 0.0311,
		"Number Unfilled Cells" : 0.0018,
		"Number of Steps" : 0.0546,
		"Obvious In Block Count" : -0.062,
		"Obvious In Block Percentage" : -0.0021,
		"Obvious In Col Count" : -0.0423,
		"Obvious In Col Percentage" : -0.0413,
		"Obvious In Row Count" : -0.0544,
		"Obvious In Row Percentage" : -0.0347,
		"Only Legal Number Count" : -0.0418,
		"Only Legal Number Percentage" : -0.0616,
		"Percentage Fill Steps" : -0.1313,
		"Pointing Pair Col Count" : 0.019,
		"Pointing Pair Col Percentage" : 0.0108,
		"Pointing Pair Row Count" : -0.021,
		"Pointing Pair Row Percentage" : 0.0015,
		"Steps Until Nonfill" : -0.0104,
		"XWing Col Count" : -0.0013,
		"XWing Col Percentage" : 0.0082,
		"XWing Row Count" : -0.0382,
		"XWing Row Percentage" : 0.0067,
	}
}
<commit_msg>New model with r2 of 0.7889 (new record)<commit_after>//auto-generated by difficulty-convert.py DO NOT EDIT

package sudoku

func init() {
	difficultySignalWeights = map[string]float64{
		"Block Block Interactions Count" : 0.0367,
		"Block Block Interactions Percentage" : 0.0023,
		"Constant" : -0.0245,
		"Forcing Chain (1 steps) Count" : 0.0,
		"Forcing Chain (1 steps) Percentage" : 0.0,
		"Forcing Chain (2 steps) Count" : 0.0,
		"Forcing Chain (2 steps) Percentage" : 0.0,
		"Forcing Chain (3 steps) Count" : -0.0859,
		"Forcing Chain (3 steps) Percentage" : -0.001,
		"Forcing Chain (4 steps) Count" : 0.1589,
		"Forcing Chain (4 steps) Percentage" : 0.0043,
		"Forcing Chain (5 steps) Count" : 0.1636,
		"Forcing Chain (5 steps) Percentage" : 0.0051,
		"Forcing Chain (6 steps) Count" : 0.0588,
		"Forcing Chain (6 steps) Percentage" : 0.007,
		"Guess Count" : -0.1167,
		"Guess Percentage" : -0.0042,
		"Hidden Pair Block Count" : -0.0197,
		"Hidden Pair Block Percentage" : 0.0017,
		"Hidden Pair Col Count" : -0.0248,
		"Hidden Pair Col Percentage" : 0.0012,
		"Hidden Pair Row Count" : -0.0034,
		"Hidden Pair Row Percentage" : -0.002,
		"Hidden Quad Block Count" : 0.1991,
		"Hidden Quad Block Percentage" : 0.0019,
		"Hidden Quad Col Count" : -0.0073,
		"Hidden Quad Col Percentage" : 0.0004,
		"Hidden Quad Row Count" : -0.1,
		"Hidden Quad Row Percentage" : -0.0013,
		"Hidden Triple Block Count" : 0.1135,
		"Hidden Triple Block Percentage" : 0.0006,
		"Hidden Triple Col Count" : -0.035,
		"Hidden Triple Col Percentage" : -0.0009,
		"Hidden Triple Row Count" : 0.1327,
		"Hidden Triple Row Percentage" : 0.0022,
		"Naked Pair Block Count" : -0.0499,
		"Naked Pair Block Percentage" : -0.0167,
		"Naked Pair Col Count" : -0.0442,
		"Naked Pair Col Percentage" : -0.0071,
		"Naked Pair Row Count" : -0.0424,
		"Naked Pair Row Percentage" : 0.0036,
		"Naked Quad Block Count" : -0.0181,
		"Naked Quad Block Percentage" : -0.0042,
		"Naked Quad Col Count" : 0.0335,
		"Naked Quad Col Percentage" : 0.0004,
		"Naked Quad Row Count" : -0.0309,
		"Naked Quad Row Percentage" : 0.0,
		"Naked Triple Block Count" : -0.0587,
		"Naked Triple Block Percentage" : -0.0012,
		"Naked Triple Col Count" : -0.0246,
		"Naked Triple Col Percentage" : -0.001,
		"Naked Triple Row Count" : -0.024,
		"Naked Triple Row Percentage" : -0.0052,
		"Necessary In Block Count" : -0.0384,
		"Necessary In Block Percentage" : 0.082,
		"Necessary In Col Count" : -0.0205,
		"Necessary In Col Percentage" : -0.0554,
		"Necessary In Row Count" : -0.02,
		"Necessary In Row Percentage" : -0.0073,
		"Number Unfilled Cells" : -0.0049,
		"Number of Steps" : 0.0416,
		"Obvious In Block Count" : -0.0159,
		"Obvious In Block Percentage" : -0.0365,
		"Obvious In Col Count" : -0.0317,
		"Obvious In Col Percentage" : 0.0611,
		"Obvious In Row Count" : -0.0314,
		"Obvious In Row Percentage" : 0.0208,
		"Only Legal Number Count" : -0.0257,
		"Only Legal Number Percentage" : -0.0561,
		"Percentage Fill Steps" : 0.0199,
		"Pointing Pair Col Count" : 0.0148,
		"Pointing Pair Col Percentage" : 0.0072,
		"Pointing Pair Row Count" : 0.0096,
		"Pointing Pair Row Percentage" : -0.0026,
		"Steps Until Nonfill" : -0.0109,
		"XWing Col Count" : -0.0041,
		"XWing Col Percentage" : 0.0004,
		"XWing Row Count" : -0.0063,
		"XWing Row Percentage" : 0.0006,
	}
}
<|endoftext|>


<commit_before>/*----------------------------------------------------------------
 *  Copyright (c) ThoughtWorks, Inc.
 *  Licensed under the Apache License, Version 2.0
 *  See LICENSE in the project root for license information.
 *----------------------------------------------------------------*/

package execution

import (
	"fmt"
	"time"

	"github.com/getgauge/gauge/execution/event"
	"github.com/getgauge/gauge/execution/result"
	"github.com/getgauge/gauge/gauge"
	"github.com/getgauge/gauge/gauge_messages"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/manifest"
	"github.com/getgauge/gauge/plugin"
	"github.com/getgauge/gauge/runner"
)

// ExecuteTags holds the tags to filter the execution by
var ExecuteTags = ""
var tableRowsIndexes []int

// SetTableRows is used to limit data driven execution to specific rows
func SetTableRows(tableRows string) {
	tableRowsIndexes = getDataTableRows(tableRows)
}

type simpleExecution struct {
	manifest             *manifest.Manifest
	runner               runner.Runner
	specCollection       *gauge.SpecCollection
	pluginHandler        plugin.Handler
	currentExecutionInfo *gauge_messages.ExecutionInfo
	suiteResult          *result.SuiteResult
	errMaps              *gauge.BuildErrors
	startTime            time.Time
	stream               int
	skipSuiteEvents      bool
}

func newSimpleExecution(executionInfo *executionInfo, combineDataTableSpecs, skipSuiteEvents bool) *simpleExecution {
	if combineDataTableSpecs {
		executionInfo.specs = gauge.NewSpecCollection(executionInfo.specs.Specs(), true)
	}
	return &simpleExecution{
		manifest:        executionInfo.manifest,
		specCollection:  executionInfo.specs,
		runner:          executionInfo.runner,
		pluginHandler:   executionInfo.pluginHandler,
		errMaps:         executionInfo.errMaps,
		stream:          executionInfo.stream,
		skipSuiteEvents: skipSuiteEvents,
	}
}

func (e *simpleExecution) run() *result.SuiteResult {
	e.start()
	e.execute()
	e.finish()
	return e.suiteResult
}

func (e *simpleExecution) execute() {
	e.suiteResult = result.NewSuiteResult(ExecuteTags, e.startTime)
	defer func() {
		e.suiteResult.UpdateExecTime(e.startTime)
		e.suiteResult.SetSpecsSkippedCount()
	}()
	if !e.skipSuiteEvents {
		logger.Debug(true, "Initialising suite data store.")
		initSuiteDataStoreResult := e.initSuiteDataStore()
		if initSuiteDataStoreResult.GetFailed() {
			e.suiteResult.AddUnhandledError(fmt.Errorf("failed to initialize suite datastore. Error: %s", initSuiteDataStoreResult.GetErrorMessage()))
			return
		}
		e.notifyBeforeSuite()
	}

	if !e.suiteResult.GetFailed() {
		results := e.executeSpecs(e.specCollection)
		e.suiteResult.AddSpecResults(results)
	}

	if !e.skipSuiteEvents {
		e.notifyAfterSuite()
	}
}

func (e *simpleExecution) start() {
	e.startTime = time.Now()
	event.Notify(event.NewExecutionEvent(event.SuiteStart, nil, nil, 0, gauge_messages.ExecutionInfo{}))
	e.pluginHandler = plugin.StartPlugins(e.manifest)
}

func (e *simpleExecution) finish() {
	e.suiteResult = mergeDataTableSpecResults(e.suiteResult)
	event.Notify(event.NewExecutionEvent(event.SuiteEnd, nil, e.suiteResult, 0, gauge_messages.ExecutionInfo{}))
	e.notifyExecutionResult()
	e.stopAllPlugins()
}

func (e *simpleExecution) stopAllPlugins() {
	e.notifyExecutionStop()
	if err := e.runner.Kill(); err != nil {
		logger.Errorf(true, "Failed to kill Runner: %s", err.Error())
	}
}

func (e *simpleExecution) executeSpecs(sc *gauge.SpecCollection) (results []*result.SpecResult) {
	for sc.HasNext() {
		specs := sc.Next()
		var preHookFailures, postHookFailures []*gauge_messages.ProtoHookFailure
		var specResults []*result.SpecResult
		var before, after = true, false
		for i, spec := range specs {
			if i == len(specs)-1 {
				after = true
			}
			res := newSpecExecutor(spec, e.runner, e.pluginHandler, e.errMaps, e.stream).execute(before, preHookFailures == nil, after)
			before = false
			specResults = append(specResults, res)
			preHookFailures = append(preHookFailures, res.GetPreHook()...)
			postHookFailures = append(postHookFailures, res.GetPostHook()...)
			res.ProtoSpec.PreHookFailures, res.ProtoSpec.PostHookFailures = []*gauge_messages.ProtoHookFailure{}, []*gauge_messages.ProtoHookFailure{}
		}
		for _, res := range specResults {
			for _, preHook := range preHookFailures {
				res.AddPreHook(&gauge_messages.ProtoHookFailure{
					StackTrace:            preHook.StackTrace,
					ErrorMessage:          preHook.ErrorMessage,
					FailureScreenshot:     preHook.FailureScreenshot,
					FailureScreenshotFile: preHook.FailureScreenshotFile,
					TableRowIndex:         preHook.TableRowIndex,
				})
			}
			for _, postHook := range postHookFailures {
				res.AddPostHook(&gauge_messages.ProtoHookFailure{
					StackTrace:            postHook.StackTrace,
					ErrorMessage:          postHook.ErrorMessage,
					FailureScreenshot:     postHook.FailureScreenshot,
					FailureScreenshotFile: postHook.FailureScreenshotFile,
					TableRowIndex:         postHook.TableRowIndex,
				})
			}
			results = append(results, res)
		}
	}
	return results
}

func (e *simpleExecution) notifyBeforeSuite() {
	m := &gauge_messages.Message{MessageType: gauge_messages.Message_ExecutionStarting,
		ExecutionStartingRequest: &gauge_messages.ExecutionStartingRequest{CurrentExecutionInfo: e.currentExecutionInfo, Stream: int32(e.stream)}}
	res := e.executeHook(m)
	e.suiteResult.PreHookMessages = res.Message
	e.suiteResult.PreHookScreenshotFiles = res.ScreenshotFiles
	e.suiteResult.PreHookScreenshots = res.Screenshots
	if res.GetFailed() {
		handleHookFailure(e.suiteResult, res, result.AddPreHook)
	}
	m.ExecutionStartingRequest.SuiteResult = gauge.ConvertToProtoSuiteResult(e.suiteResult)
	e.pluginHandler.NotifyPlugins(m)
}

func (e *simpleExecution) notifyAfterSuite() {
	m := &gauge_messages.Message{MessageType: gauge_messages.Message_ExecutionEnding,
		ExecutionEndingRequest: &gauge_messages.ExecutionEndingRequest{CurrentExecutionInfo: e.currentExecutionInfo, Stream: int32(e.stream)}}
	res := e.executeHook(m)
	e.suiteResult.PostHookMessages = res.Message
	e.suiteResult.PostHookScreenshotFiles = res.ScreenshotFiles
	e.suiteResult.PostHookScreenshots = res.Screenshots
	if res.GetFailed() {
		handleHookFailure(e.suiteResult, res, result.AddPostHook)
	}
	m.ExecutionEndingRequest.SuiteResult = gauge.ConvertToProtoSuiteResult(e.suiteResult)
	e.pluginHandler.NotifyPlugins(m)
}

func (e *simpleExecution) initSuiteDataStore() *(gauge_messages.ProtoExecutionResult) {
	m := &gauge_messages.Message{MessageType: gauge_messages.Message_SuiteDataStoreInit,
		SuiteDataStoreInitRequest: &gauge_messages.SuiteDataStoreInitRequest{Stream: int32(e.stream)}}
	return e.runner.ExecuteAndGetStatus(m)
}

func (e *simpleExecution) executeHook(m *gauge_messages.Message) *(gauge_messages.ProtoExecutionResult) {
	e.pluginHandler.NotifyPlugins(m)
	return e.runner.ExecuteAndGetStatus(m)
}

func (e *simpleExecution) notifyExecutionResult() {
	m := &gauge_messages.Message{MessageType: gauge_messages.Message_SuiteExecutionResult,
		SuiteExecutionResult: &gauge_messages.SuiteExecutionResult{SuiteResult: gauge.ConvertToProtoSuiteResult(e.suiteResult)}}
	e.pluginHandler.NotifyPlugins(m)
}

func (e *simpleExecution) notifyExecutionStop() {
	m := &gauge_messages.Message{MessageType: gauge_messages.Message_KillProcessRequest,
		KillProcessRequest: &gauge_messages.KillProcessRequest{}}
	e.pluginHandler.NotifyPlugins(m)
	e.pluginHandler.GracefullyKillPlugins()
}

func handleHookFailure(hookResult result.Result, execResult *gauge_messages.ProtoExecutionResult, f func(result.Result, *gauge_messages.ProtoExecutionResult)) {
	f(hookResult, execResult)
}
<commit_msg>add current execution info in simple execution (#1775)<commit_after>/*----------------------------------------------------------------
 *  Copyright (c) ThoughtWorks, Inc.
 *  Licensed under the Apache License, Version 2.0
 *  See LICENSE in the project root for license information.
 *----------------------------------------------------------------*/

package execution

import (
	"fmt"
	"time"
	"path/filepath"

	"github.com/getgauge/gauge/config"
	"github.com/getgauge/gauge/execution/event"
	"github.com/getgauge/gauge/execution/result"
	"github.com/getgauge/gauge/gauge"
	"github.com/getgauge/gauge/gauge_messages"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/manifest"
	"github.com/getgauge/gauge/plugin"
	"github.com/getgauge/gauge/runner"
)

// ExecuteTags holds the tags to filter the execution by
var ExecuteTags = ""
var tableRowsIndexes []int

// SetTableRows is used to limit data driven execution to specific rows
func SetTableRows(tableRows string) {
	tableRowsIndexes = getDataTableRows(tableRows)
}

type simpleExecution struct {
	manifest             *manifest.Manifest
	runner               runner.Runner
	specCollection       *gauge.SpecCollection
	pluginHandler        plugin.Handler
	currentExecutionInfo *gauge_messages.ExecutionInfo
	suiteResult          *result.SuiteResult
	errMaps              *gauge.BuildErrors
	startTime            time.Time
	stream               int
	skipSuiteEvents      bool
}

func newSimpleExecution(executionInfo *executionInfo, combineDataTableSpecs, skipSuiteEvents bool) *simpleExecution {
	if combineDataTableSpecs {
		executionInfo.specs = gauge.NewSpecCollection(executionInfo.specs.Specs(), true)
	}
	ei := &gauge_messages.ExecutionInfo{
		ProjectName:              filepath.Base(config.ProjectRoot),
		NumberOfExecutionStreams: int32(NumberOfExecutionStreams),
		RunnerId:                 int32(executionInfo.stream),
		ExecutionArgs:            gauge.ConvertToProtoExecutionArg(ExecutionArgs),
	}

	return &simpleExecution{
		manifest:        executionInfo.manifest,
		specCollection:  executionInfo.specs,
		runner:          executionInfo.runner,
		pluginHandler:   executionInfo.pluginHandler,
		errMaps:         executionInfo.errMaps,
		stream:          executionInfo.stream,
		skipSuiteEvents: skipSuiteEvents,
		currentExecutionInfo: ei,
	}
}

func (e *simpleExecution) run() *result.SuiteResult {
	e.start()
	e.execute()
	e.finish()
	return e.suiteResult
}

func (e *simpleExecution) execute() {
	e.suiteResult = result.NewSuiteResult(ExecuteTags, e.startTime)
	defer func() {
		e.suiteResult.UpdateExecTime(e.startTime)
		e.suiteResult.SetSpecsSkippedCount()
	}()
	if !e.skipSuiteEvents {
		logger.Debug(true, "Initialising suite data store.")
		initSuiteDataStoreResult := e.initSuiteDataStore()
		if initSuiteDataStoreResult.GetFailed() {
			e.suiteResult.AddUnhandledError(fmt.Errorf("failed to initialize suite datastore. Error: %s", initSuiteDataStoreResult.GetErrorMessage()))
			return
		}
		e.notifyBeforeSuite()
	}

	if !e.suiteResult.GetFailed() {
		results := e.executeSpecs(e.specCollection)
		e.suiteResult.AddSpecResults(results)
	}

	if !e.skipSuiteEvents {
		e.notifyAfterSuite()
	}
}

func (e *simpleExecution) start() {
	e.startTime = time.Now()
	event.Notify(event.NewExecutionEvent(event.SuiteStart, nil, nil, 0, gauge_messages.ExecutionInfo{}))
	e.pluginHandler = plugin.StartPlugins(e.manifest)
}

func (e *simpleExecution) finish() {
	e.suiteResult = mergeDataTableSpecResults(e.suiteResult)
	event.Notify(event.NewExecutionEvent(event.SuiteEnd, nil, e.suiteResult, 0, gauge_messages.ExecutionInfo{}))
	e.notifyExecutionResult()
	e.stopAllPlugins()
}

func (e *simpleExecution) stopAllPlugins() {
	e.notifyExecutionStop()
	if err := e.runner.Kill(); err != nil {
		logger.Errorf(true, "Failed to kill Runner: %s", err.Error())
	}
}

func (e *simpleExecution) executeSpecs(sc *gauge.SpecCollection) (results []*result.SpecResult) {
	for sc.HasNext() {
		specs := sc.Next()
		var preHookFailures, postHookFailures []*gauge_messages.ProtoHookFailure
		var specResults []*result.SpecResult
		var before, after = true, false
		for i, spec := range specs {
			if i == len(specs)-1 {
				after = true
			}
			res := newSpecExecutor(spec, e.runner, e.pluginHandler, e.errMaps, e.stream).execute(before, preHookFailures == nil, after)
			before = false
			specResults = append(specResults, res)
			preHookFailures = append(preHookFailures, res.GetPreHook()...)
			postHookFailures = append(postHookFailures, res.GetPostHook()...)
			res.ProtoSpec.PreHookFailures, res.ProtoSpec.PostHookFailures = []*gauge_messages.ProtoHookFailure{}, []*gauge_messages.ProtoHookFailure{}
		}
		for _, res := range specResults {
			for _, preHook := range preHookFailures {
				res.AddPreHook(&gauge_messages.ProtoHookFailure{
					StackTrace:            preHook.StackTrace,
					ErrorMessage:          preHook.ErrorMessage,
					FailureScreenshot:     preHook.FailureScreenshot,
					FailureScreenshotFile: preHook.FailureScreenshotFile,
					TableRowIndex:         preHook.TableRowIndex,
				})
			}
			for _, postHook := range postHookFailures {
				res.AddPostHook(&gauge_messages.ProtoHookFailure{
					StackTrace:            postHook.StackTrace,
					ErrorMessage:          postHook.ErrorMessage,
					FailureScreenshot:     postHook.FailureScreenshot,
					FailureScreenshotFile: postHook.FailureScreenshotFile,
					TableRowIndex:         postHook.TableRowIndex,
				})
			}
			results = append(results, res)
		}
	}
	return results
}

func (e *simpleExecution) notifyBeforeSuite() {
	m := &gauge_messages.Message{MessageType: gauge_messages.Message_ExecutionStarting,
		ExecutionStartingRequest: &gauge_messages.ExecutionStartingRequest{CurrentExecutionInfo: e.currentExecutionInfo, Stream: int32(e.stream)}}
	res := e.executeHook(m)
	e.suiteResult.PreHookMessages = res.Message
	e.suiteResult.PreHookScreenshotFiles = res.ScreenshotFiles
	e.suiteResult.PreHookScreenshots = res.Screenshots
	if res.GetFailed() {
		handleHookFailure(e.suiteResult, res, result.AddPreHook)
	}
	m.ExecutionStartingRequest.SuiteResult = gauge.ConvertToProtoSuiteResult(e.suiteResult)
	e.pluginHandler.NotifyPlugins(m)
}

func (e *simpleExecution) notifyAfterSuite() {
	m := &gauge_messages.Message{MessageType: gauge_messages.Message_ExecutionEnding,
		ExecutionEndingRequest: &gauge_messages.ExecutionEndingRequest{CurrentExecutionInfo: e.currentExecutionInfo, Stream: int32(e.stream)}}
	res := e.executeHook(m)
	e.suiteResult.PostHookMessages = res.Message
	e.suiteResult.PostHookScreenshotFiles = res.ScreenshotFiles
	e.suiteResult.PostHookScreenshots = res.Screenshots
	if res.GetFailed() {
		handleHookFailure(e.suiteResult, res, result.AddPostHook)
	}
	m.ExecutionEndingRequest.SuiteResult = gauge.ConvertToProtoSuiteResult(e.suiteResult)
	e.pluginHandler.NotifyPlugins(m)
}

func (e *simpleExecution) initSuiteDataStore() *(gauge_messages.ProtoExecutionResult) {
	m := &gauge_messages.Message{MessageType: gauge_messages.Message_SuiteDataStoreInit,
		SuiteDataStoreInitRequest: &gauge_messages.SuiteDataStoreInitRequest{Stream: int32(e.stream)}}
	return e.runner.ExecuteAndGetStatus(m)
}

func (e *simpleExecution) executeHook(m *gauge_messages.Message) *(gauge_messages.ProtoExecutionResult) {
	e.pluginHandler.NotifyPlugins(m)
	return e.runner.ExecuteAndGetStatus(m)
}

func (e *simpleExecution) notifyExecutionResult() {
	m := &gauge_messages.Message{MessageType: gauge_messages.Message_SuiteExecutionResult,
		SuiteExecutionResult: &gauge_messages.SuiteExecutionResult{SuiteResult: gauge.ConvertToProtoSuiteResult(e.suiteResult)}}
	e.pluginHandler.NotifyPlugins(m)
}

func (e *simpleExecution) notifyExecutionStop() {
	m := &gauge_messages.Message{MessageType: gauge_messages.Message_KillProcessRequest,
		KillProcessRequest: &gauge_messages.KillProcessRequest{}}
	e.pluginHandler.NotifyPlugins(m)
	e.pluginHandler.GracefullyKillPlugins()
}

func handleHookFailure(hookResult result.Result, execResult *gauge_messages.ProtoExecutionResult, f func(result.Result, *gauge_messages.ProtoExecutionResult)) {
	f(hookResult, execResult)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	exif "github.com/garyhouston/exif44"
	jseg "github.com/garyhouston/jpegsegs"
	tiff "github.com/garyhouston/tiff66"
	"io"
	"io/ioutil"
	"log"
	"os"
)

// Process a TIFF file.
func processTIFF(outfile io.Writer, infile io.Reader) error {
	buf, err := ioutil.ReadAll(infile)
	if err != nil {
		return err
	}
	tree, err := exif.GetExifTree(buf)
	if err != nil {
		return err
	}
	tree.TIFF.Fix()
	if err = tree.CheckMakerNote(); err != nil {
		return err
	}
	if err = tree.MakerNoteComplexities(); err != nil {
		return err
	}
	fileSize := tiff.HeaderSize + tree.TreeSize()
	out := make([]byte, fileSize)
	tiff.PutHeader(out, tree.TIFF.Order, tiff.HeaderSize)
	_, err = tree.TIFF.PutIFDTree(out, tiff.HeaderSize)
	if err != nil {
		return err
	}
	_, err = outfile.Write(out)
	return err
}

// Process a single image in a JPEG file. A file using Multi-Picture
// Format will contain multiple images.
func processImage(writer io.WriteSeeker, reader io.ReadSeeker, mpfProcessor jseg.MPFProcessor) error {
	scanner, err := jseg.NewScanner(reader)
	if err != nil {
		return err
	}
	dumper, err := jseg.NewDumper(writer)
	if err != nil {
		return err
	}
	for {
		marker, buf, err := scanner.Scan()
		if err != nil {
			return err
		}
		if marker == jseg.APP0+1 {
			isExif, next := exif.GetHeader(buf)
			if isExif {
				tree, err := exif.GetExifTree(buf[next:])
				if err != nil {
					return err
				}
				tree.TIFF.Fix()
				if err = tree.CheckMakerNote(); err != nil {
					return err
				}
				if err = tree.MakerNoteComplexities(); err != nil {
					return err
				}
				app1 := make([]byte, exif.HeaderSize+tree.TreeSize())
				next := exif.PutHeader(app1)
				next, err = tree.Put(app1[next:])
				if err != nil {
					return err
				}
				buf = app1
			}

		}
		if marker == jseg.APP0+2 {
			_, buf, err = mpfProcessor.ProcessAPP2(writer, reader, buf)
			if err != nil {
				return err
			}
		}
		if err := dumper.Dump(marker, buf); err != nil {
			return err
		}
		if marker == jseg.EOI {
			return nil
		}
	}
}

// State for MPF image iterator.
type iterData struct {
	writer     io.WriteSeeker
	newOffsets []uint32
}

// Function to be applied to each MPF image.
func (iter *iterData) MPFApply(reader io.ReadSeeker, index uint32, length uint32) error {
	if index > 0 {
		pos, err := iter.writer.Seek(0, io.SeekCurrent)
		if err != nil {
			return err
		}
		iter.newOffsets[index] = uint32(pos)
		return processImage(iter.writer, reader, &jseg.MPFCheck{})
	}
	return nil
}

// Process additional images found in the MPF index.
func processMPFImages(writer io.WriteSeeker, reader io.ReadSeeker, index *jseg.MPFIndex) ([]uint32, error) {
	var iter iterData
	iter.writer = writer
	iter.newOffsets = make([]uint32, len(index.ImageOffsets))
	index.ImageIterate(reader, &iter)
	return iter.newOffsets, nil
}

// Process a JPEG file.
func processJPEG(writer io.WriteSeeker, reader io.ReadSeeker) error {
	var mpfIndex jseg.MPFIndexRewriter
	if err := processImage(writer, reader, &mpfIndex); err != nil {
		return err
	}
	if mpfIndex.Tree != nil {
		newOffsets, err := processMPFImages(writer, reader, mpfIndex.Index)
		if err != nil {
			return err
		}
		end, err := writer.Seek(0, io.SeekCurrent)
		if err != nil {
			return err
		}
		if err = jseg.RewriteMPF(writer, mpfIndex.Tree, mpfIndex.APP2WritePos, newOffsets, uint32(end)); err != nil {
			return err
		}
	}
	return nil
}

const (
	TIFFFile = 1
	JPEGFile = 2
)

// Determine if file is TIFF, JPEG or neither (error)
func fileType(file io.Reader) (int, error) {
	buf := make([]byte, tiff.HeaderSize)
	if _, err := io.ReadFull(file, buf); err != nil {
		return 0, err
	}
	if jseg.IsJPEGHeader(buf) {
		return JPEGFile, nil
	}
	if validTIFF, _, _ := tiff.GetHeader(buf); validTIFF {
		return TIFFFile, nil
	}
	return 0, errors.New("File doesn't have a TIFF or JPEG header")
}

// Decode a TIFF file, or the Exif segment in a JPEG file, then re-encode
// it and write to a new file.
func main() {
	if len(os.Args) != 3 {
		fmt.Printf("Usage: %s file outfile\n", os.Args[0])
		return
	}
	infile, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	defer infile.Close()
	fileType, err := fileType(infile)
	if err != nil {
		log.Fatal(err)
	}
	if _, err := infile.Seek(0, 0); err != nil {
		log.Fatal(err)
	}
	outfile, err := os.Create(os.Args[2])
	if err != nil {
		log.Fatal(err)
	}
	defer outfile.Close()
	if fileType == TIFFFile {
		err = processTIFF(outfile, infile)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		err = processJPEG(outfile, infile)
		if err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>Simpify using read/write scanner.<commit_after>package main

import (
	"fmt"
	exif "github.com/garyhouston/exif44"
	//	tiff "github.com/garyhouston/tiff66"
	"log"
	"os"
)

// Exif handler.
type readWriteExif struct {
}

func (readWriteExif readWriteExif) ReadWriteExif(imageIdx uint32, xif exif.Exif) (exif.Exif, error) {
	/*
	           // For the first image in the file, delete any "Software"
	           // field from the TIFF IFD0, and if there's an Exif IFD add a
	           // LensMake field.
	   	if imageIdx == 0 {
	   		xif.TIFF.DeleteFields([]tiff.Tag{tiff.Software})
	   		if xif.Exif != nil {
	   			xif.Exif.DeleteFields([]tiff.Tag{exif.LensMake})
	   			lens := "Dog Nose Lens"
	   			lensField := tiff.Field{
	   				Tag: exif.LensMake,
	   				Type: tiff.ASCII,
	   				Count: uint32(len(lens)),
	   				Data: make([]byte, len(lens) + 1)}
	   			lensField.PutASCII(lens)
	   			xif.Exif.AddFields([]tiff.Field{lensField})
	   		}
	   	}
	*/
	return xif, nil
}

// Decode a TIFF file, or the Exif segment(s) in a JPEG file, then re-encode
// it and write to a new file.
func main() {
	if len(os.Args) != 3 {
		fmt.Printf("Usage: %s file outfile\n", os.Args[0])
		return
	}
	var control exif.ReadWriteControl
	control.ReadWriteExif = readWriteExif{}
	if err := exif.ReadWriteFile(os.Args[1], os.Args[2], control); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Benchnet
//
// Copyright 2012 Vadim Vygonets
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package sched implements a simple scheduler.
package sched

import (
	"time"
)

/*
const (
	minPeriod = time.Minute
	maxPeriod = time.Hour * 24 * 14
)
*/

type Sched struct {
	headShot chan bool
}

// Stop stops the scheduler.  If f is currently running, Stop will
// not return until it's finished.
func (s *Sched) Stop() {
	s.headShot <- true
}

func (s *Sched) thread(period time.Duration, start time.Duration, f func()) {
	timer := time.NewTimer(start)
	select {
	case <-s.headShot:
		timer.Stop()
		return
	case <-timer.C:
	}
	f()
	ticker := time.NewTicker(period)
	for {
		select {
		case <-s.headShot:
			ticker.Stop()
			return
		case <-ticker.C:
		}
		f()
	}
}

// New starts a new scheduler running f each period, at Unix time
// N*period+offset where N is natural.  No more than one instance
// of f will run at any given moment.
func New(period time.Duration, offset time.Duration, f func()) *Sched {
	// This will break after Fri Apr 11 23:47:16 +0000 UTC 2262
	nanonow := time.Duration(time.Now().UnixNano())
	start := period - (nanonow-offset)%period
	if start < time.Millisecond {
		start += period
	}
	s := Sched{headShot: make(chan bool)}
	go s.thread(period, start, f)
	return &s
}
<commit_msg>Sched: cleanup<commit_after>// Benchnet
//
// Copyright 2012 Vadim Vygonets
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package sched implements a simple scheduler.
package sched

import "time"

// Sched represets a scheduler instance.
type Sched struct {
	headShot chan bool
}

// Stop stops the scheduler s.  If f is currently running, Stop
// will not return until it's finished.  If called twice, Stop
// will hang forever.
func (s *Sched) Stop() {
	s.headShot <- true
}

func (s *Sched) thread(period time.Duration, start time.Duration, f func()) {
	select {
	case <-s.headShot:
		return
	case <-time.After(start):
	}
	f()
	ticker := time.NewTicker(period)
	for {
		select {
		case <-s.headShot:
			ticker.Stop()
			return
		case <-ticker.C:
		}
		f()
	}
}

// New starts a new scheduler running f each period, at Unix time
// N*period+offset where N is natural.  No more than one instance
// of f will run at any given moment.
func New(period time.Duration, offset time.Duration, f func()) *Sched {
	// This will break after Fri Apr 11 23:47:16 +0000 UTC 2262
	nanonow := time.Duration(time.Now().UnixNano())
	start := period - (nanonow-offset)%period
	if start < time.Millisecond {
		start += period
	}
	s := Sched{headShot: make(chan bool)}
	go s.thread(period, start, f)
	return &s
}
<|endoftext|>


<commit_before>package template

import (
	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/pufferd/http"
	"github.com/pufferpanel/pufferd/httphandlers"
	"github.com/pufferpanel/pufferd/programs"
	"os"
)

func RegisterRoutes(e *gin.Engine) {
	l := e.Group("_templates")
	{
		l.GET("", httphandlers.OAuth2Handler("node.templates", false), GetTemplates)
		l.GET("/:id", httphandlers.OAuth2Handler("node.templates", false), GetTemplate)
	}
}

func GetTemplates(c *gin.Context) {
	http.Respond(c).Data(programs.GetPlugins()).Send()
}

func GetTemplate(c *gin.Context) {
	name, exists := c.GetQuery("name")
	if !exists || name == "" {
		http.Respond(c).Fail().Code(400).MessageCode(http.INVALIDREQUEST).Message("no template name provided").Send()
		return
	}
	data, err := programs.GetPlugin(name)
	if err != nil {
		if os.IsNotExist(err) {
			http.Respond(c).Fail().Code(404).MessageCode(http.NOFILE).Message("no template with provided name").Send()
		} else {
			http.Respond(c).Fail().Code(500).MessageCode(http.UNKNOWN).Message("error reading template").Send()
		}
	} else {
		http.Respond(c).Code(200).Data(data).Send()
	}
}<commit_msg>Fix template routing<commit_after>package template

import (
	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/pufferd/http"
	"github.com/pufferpanel/pufferd/httphandlers"
	"github.com/pufferpanel/pufferd/programs"
	"os"
)

func RegisterRoutes(e *gin.Engine) {
	l := e.Group("_templates")
	{
		l.GET("", httphandlers.OAuth2Handler("node.templates", false), GetTemplates)
		l.GET("/:id", httphandlers.OAuth2Handler("node.templates", false), GetTemplate)
		l.POST("/:id", httphandlers.OAuth2Handler("node.templates.edit", false), EditTemplate)
	}
}

func GetTemplates(c *gin.Context) {
	http.Respond(c).Data(programs.GetPlugins()).Send()
}

func GetTemplate(c *gin.Context) {
	name, exists := c.GetQuery("id")
	if !exists || name == "" {
		http.Respond(c).Fail().Code(400).MessageCode(http.INVALIDREQUEST).Message("no template name provided").Send()
		return
	}
	data, err := programs.GetPlugin(name)
	if err != nil {
		if os.IsNotExist(err) {
			http.Respond(c).Fail().Code(404).MessageCode(http.NOFILE).Message("no template with provided name").Send()
		} else {
			http.Respond(c).Fail().Code(500).MessageCode(http.UNKNOWN).Message("error reading template").Send()
		}
	} else {
		http.Respond(c).Code(200).Data(data).Send()
	}
}

func EditTemplate(c *gin.Context) {
	name, exists := c.GetQuery("id")
	if !exists || name == "" {
		http.Respond(c).Fail().Code(400).MessageCode(http.INVALIDREQUEST).Message("no template name provided").Send()
		return
	}
	data, err := programs.GetPlugin(name)
	if err != nil {
		if os.IsNotExist(err) {
			http.Respond(c).Fail().Code(404).MessageCode(http.NOFILE).Message("no template with provided name").Send()
		} else {
			http.Respond(c).Fail().Code(500).MessageCode(http.UNKNOWN).Message("error reading template").Send()
		}
	} else {
		http.Respond(c).Code(200).Data(data).Send()
	}
}<|endoftext|>


<commit_before>package main
import(
    "fmt"
    "net/http"
    "log"
    "time"
    "bufio"
    "os"
    "strings"
    "encoding/json"
    "strconv"
    "io/ioutil"
    "sort"
    "crystal/service"
)

type WeightLabel struct {
    weight float64
    label string
}

type WeightLabelList []WeightLabel

func (self *WeightLabelList) MaxWeight() float64 {
    ret := (*self)[0].weight
    for _, wl := range *self {
        if ret < wl.weight {
            ret = wl.weight
        }
    }
    return ret
}

func (self *WeightLabelList) MinWeight() float64 {
    ret := (*self)[0].weight
    for _, wl := range *self {
        if ret > wl.weight {
            ret = wl.weight
        }
    }
    return ret
}

type StatResult struct {
    discrete_stat map[string]map[string]map[string]float64
    labels map[string]bool
    features map[string]bool
    continuous_stat map[string]WeightLabelList
}

func (self *StatResult) Init() {
    self.discrete_stat = make(map[string]map[string]map[string]float64)
    self.labels = make(map[string]bool)
    self.features = make(map[string]bool)
    self.continuous_stat = make(map[string]WeightLabelList)
}

func StatData(path string) *StatResult {
    file, err := os.Open(path)
    if err != nil {
        return nil
    }
    defer file.Close()

    stat := StatResult{}
    stat.Init()
    scanner := bufio.NewScanner(file)
    head := []string{}
    n := 0
    for scanner.Scan() {
        n += 1
        line := scanner.Text()
        tks := strings.Split(line, "\t")
        if n == 1{
            head = tks
        } else {
            label := tks[0]
            if len(label) == 0{
                continue
            }
            stat.labels[label] = true
            for i, tk := range tks {
                if i == 0 {
                    continue
                } else {
                    key := head[i]
                    if key[0] == '['{
                        stat.features[key] = true
                        value := tk
                        _, ok := stat.discrete_stat[key]
                        if !ok {
                            stat.discrete_stat[key] = make(map[string]map[string]float64)
                        }
                        _, ok = stat.discrete_stat[key][value]
                        if !ok {
                            stat.discrete_stat[key][value] = make(map[string]float64)
                        }
                        _, ok = stat.discrete_stat[key][value][label]
                        if !ok {
                            stat.discrete_stat[key][value][label] = 1.0
                        } else {
                            stat.discrete_stat[key][value][label] += 1.0
                        }
                    } else {
                        stat.features[key] = true
                        _, ok := stat.continuous_stat[key]
                        if !ok {
                            stat.continuous_stat[key] = WeightLabelList{}
                        }
                        value, _ := strconv.ParseFloat(tk, 64)
                        stat.continuous_stat[key] = append(stat.continuous_stat[key], WeightLabel{weight: value, label: label})
                    }
                }
            }
        }
    }
    return &stat
}

func StatAllData() map[string]*StatResult {
    ret := make(map[string]*StatResult)
    files, _ := ioutil.ReadDir("./data/")
    for _, f := range files {
        if f.IsDir(){
            stat := StatData("./data/" + f.Name() + "/data.tsv")
            ret[f.Name()] = stat
        }
    }
    return ret
}

var global_stat map[string]*StatResult

func SingleContinuousFeatureStat(w http.ResponseWriter, r *http.Request) {
    w.Header().Set("Content-Type", "application/json")
    params := service.GetParameters(w, r)
    is_100percent := r.FormValue("is_100percent")
    weight_labels, _ := global_stat[params.Dataset].continuous_stat[params.Feature]

    max_weight := weight_labels.MaxWeight()
    min_weight := weight_labels.MinWeight()

    bin_stat := make(map[int]map[string]float64)
    bin_sum := make(map[int]float64)
    for _, wl := range weight_labels {
        bin := int(20.0 * (wl.weight - min_weight) / (max_weight - min_weight))
        _, ok := bin_stat[bin]
        if !ok {
            bin_stat[bin] = make(map[string]float64)
            bin_sum[bin] = 0.0
        }
        bin_sum[bin] += 1.0
        _, ok = bin_stat[bin][wl.label]
        if !ok {
            bin_stat[bin][wl.label] = 1.0
        } else {
            bin_stat[bin][wl.label] += 1.0
        }
    }
    ret := []interface{}{}

    bins := []int{}
    for bin, _ := range bin_sum {
        bins = append(bins, bin)
    }
    sort.Ints(bins)
    for label, _ := range global_stat[params.Dataset].labels {
        record := make(map[string]interface{})
        record["key"] = params.Feature + ": " + label
        values := []map[string]interface{}{}
        for _, bin := range bins {
            label_dis, _ := bin_stat[bin]
            count, ok := label_dis[label]
            if !ok {
                count = 0.0
            }
            point := make(map[string]interface{})
            point["x"] = min_weight + float64(bin) * (max_weight - min_weight) / 20.0
            if is_100percent == "true" {
                point["y"] = count / bin_sum[bin]
            } else {
                point["y"] = count
            }
            values = append(values, point)
        }
        record["values"] = values
        ret = append(ret, record)
    }
    b, _ := json.Marshal(ret)
    fmt.Fprint(w, string(b))
}

func SingleFeatureStat(w http.ResponseWriter, r *http.Request){
    feature := r.FormValue("feature")
    if feature[0] == '[' {
        SingleDiscreteFeatureStat(w, r)
    } else {
        SingleContinuousFeatureStat(w, r)
    }
}

type StringDoublePair struct {
    Key string
    Value float64
}

type StringDoublePairList struct {
    data []*StringDoublePair
}

func (self *StringDoublePairList) Less(i, j int) bool{
    return self.data[i].Value > self.data[j].Value
}

func (self *StringDoublePairList) Len() int{
    return len(self.data)
}

func (self *StringDoublePairList) Swap(i, j int) {
    self.data[i], self.data[j] = self.data[j], self.data[i]
}

func SingleDiscreteFeatureStat(w http.ResponseWriter,r *http.Request){
    w.Header().Set("Content-Type", "application/json")
    ret := []interface{}{}
    feature := r.FormValue("feature")
    dataset := r.FormValue("dataset")
    is_100percent := r.FormValue("is_100percent")
    value_label_dis, ok := global_stat[dataset].discrete_stat[feature]

    valuestr := []string{}
    for value, _ := range value_label_dis {
        valuestr = append(valuestr, value)
    }

    if !ok {
        b, _ := json.Marshal(ret)
        fmt.Fprint(w, string(b))
        return
    }

    value_sum := make(map[string]float64)

    for value, label_dis := range value_label_dis {
        value_sum[value] = 0.0
        for _, count := range label_dis {
            value_sum[value] += count
        }
    }

    value_sum_array := StringDoublePairList{}
    for value, sum := range value_sum {
        value_sum_array.data = append(value_sum_array.data, &(StringDoublePair{Key: value, Value: sum}))
    }
    sort.Sort(&value_sum_array)

    for label, _ := range global_stat[dataset].labels {
        record := make(map[string]interface{})
        record["key"] = feature + ": " + label
        values := []map[string]interface{}{}
        for i, vs := range value_sum_array.data{
            if i > 64 {
                break
            }
            value := vs.Key
            label_dis, _ := value_label_dis[value]
            count, ok := label_dis[label]
            if !ok {
                count = 0.0
            }
            point := make(map[string]interface{})
            point["x"] = value
            if is_100percent == "true" {
                point["y"] = count / value_sum[value]
            } else {
                point["y"] = count
            }
            values = append(values, point)
        }
        record["values"] = values
        ret = append(ret, record)
    }
    b, _ := json.Marshal(ret)
    fmt.Fprint(w, string(b))
}

func FeatureList( w http.ResponseWriter,r *http.Request ){
    w.Header().Set("Content-Type", "application/json")
    dataset := r.FormValue("dataset")
    ret := []string{}

    for key, _ := range global_stat[dataset].discrete_stat {
        ret = append(ret, key)
    }
    for key, _ := range global_stat[dataset].continuous_stat {
        ret = append(ret, key)
    }
    b, _ := json.Marshal(ret)
    fmt.Fprint(w, string(b))
}

func DataSetList(w http.ResponseWriter, r *http.Request) {
    w.Header().Set("Content-Type", "application/json")
    files, _ := ioutil.ReadDir("./data/")
    ret := []string{}
    for _, f := range files {
        if f.IsDir(){
            ret = append(ret, f.Name())
        }
    }
    b, _ := json.Marshal(ret)
    fmt.Fprint(w, string(b))
}

func main(){
    global_stat = StatAllData()
    http.Handle("/", http.FileServer(http.Dir(".")))
    http.HandleFunc( "/single_feature",SingleFeatureStat)
    http.HandleFunc("/feature_list", FeatureList)
    http.HandleFunc("/dataset_list", DataSetList)
    s := &http.Server{
        Addr:           ":2014",
        ReadTimeout:    30 * time.Second,
        WriteTimeout:   30 * time.Second,
        MaxHeaderBytes: 1 << 20,
    }
    log.Fatal(s.ListenAndServe())
}<commit_msg>remove feature with value < 20<commit_after>package main
import(
    "fmt"
    "net/http"
    "log"
    "time"
    "bufio"
    "os"
    "strings"
    "encoding/json"
    "strconv"
    "io/ioutil"
    "sort"
    "crystal/service"
)

type WeightLabel struct {
    weight float64
    label string
}

type WeightLabelList []WeightLabel

func (self *WeightLabelList) MaxWeight() float64 {
    ret := (*self)[0].weight
    for _, wl := range *self {
        if ret < wl.weight {
            ret = wl.weight
        }
    }
    return ret
}

func (self *WeightLabelList) MinWeight() float64 {
    ret := (*self)[0].weight
    for _, wl := range *self {
        if ret > wl.weight {
            ret = wl.weight
        }
    }
    return ret
}

type StatResult struct {
    discrete_stat map[string]map[string]map[string]float64
    labels map[string]bool
    features map[string]bool
    continuous_stat map[string]WeightLabelList
}

func (self *StatResult) Init() {
    self.discrete_stat = make(map[string]map[string]map[string]float64)
    self.labels = make(map[string]bool)
    self.features = make(map[string]bool)
    self.continuous_stat = make(map[string]WeightLabelList)
}

func StatData(path string) *StatResult {
    file, err := os.Open(path)
    if err != nil {
        return nil
    }
    defer file.Close()

    stat := StatResult{}
    stat.Init()
    scanner := bufio.NewScanner(file)
    head := []string{}
    n := 0
    for scanner.Scan() {
        n += 1
        line := scanner.Text()
        tks := strings.Split(line, "\t")
        if n == 1{
            head = tks
        } else {
            label := tks[0]
            if len(label) == 0{
                continue
            }
            stat.labels[label] = true
            for i, tk := range tks {
                if i == 0 {
                    continue
                } else {
                    key := head[i]
                    if key[0] == '['{
                        stat.features[key] = true
                        value := tk
                        _, ok := stat.discrete_stat[key]
                        if !ok {
                            stat.discrete_stat[key] = make(map[string]map[string]float64)
                        }
                        _, ok = stat.discrete_stat[key][value]
                        if !ok {
                            stat.discrete_stat[key][value] = make(map[string]float64)
                        }
                        _, ok = stat.discrete_stat[key][value][label]
                        if !ok {
                            stat.discrete_stat[key][value][label] = 1.0
                        } else {
                            stat.discrete_stat[key][value][label] += 1.0
                        }
                    } else {
                        stat.features[key] = true
                        _, ok := stat.continuous_stat[key]
                        if !ok {
                            stat.continuous_stat[key] = WeightLabelList{}
                        }
                        value, _ := strconv.ParseFloat(tk, 64)
                        stat.continuous_stat[key] = append(stat.continuous_stat[key], WeightLabel{weight: value, label: label})
                    }
                }
            }
        }
    }
    return &stat
}

func StatAllData() map[string]*StatResult {
    ret := make(map[string]*StatResult)
    files, _ := ioutil.ReadDir("./data/")
    for _, f := range files {
        if f.IsDir(){
            stat := StatData("./data/" + f.Name() + "/data.tsv")
            ret[f.Name()] = stat
        }
    }
    return ret
}

var global_stat map[string]*StatResult

func SingleContinuousFeatureStat(w http.ResponseWriter, r *http.Request) {
    w.Header().Set("Content-Type", "application/json")
    params := service.GetParameters(w, r)
    is_100percent := r.FormValue("is_100percent")
    weight_labels, _ := global_stat[params.Dataset].continuous_stat[params.Feature]

    max_weight := weight_labels.MaxWeight()
    min_weight := weight_labels.MinWeight()

    bin_stat := make(map[int]map[string]float64)
    bin_sum := make(map[int]float64)
    for _, wl := range weight_labels {
        bin := int(20.0 * (wl.weight - min_weight) / (max_weight - min_weight))
        _, ok := bin_stat[bin]
        if !ok {
            bin_stat[bin] = make(map[string]float64)
            bin_sum[bin] = 0.0
        }
        bin_sum[bin] += 1.0
        _, ok = bin_stat[bin][wl.label]
        if !ok {
            bin_stat[bin][wl.label] = 1.0
        } else {
            bin_stat[bin][wl.label] += 1.0
        }
    }
    ret := []interface{}{}

    bins := []int{}
    for bin, _ := range bin_sum {
        bins = append(bins, bin)
    }
    sort.Ints(bins)
    for label, _ := range global_stat[params.Dataset].labels {
        record := make(map[string]interface{})
        record["key"] = params.Feature + ": " + label
        values := []map[string]interface{}{}
        for _, bin := range bins {
            label_dis, _ := bin_stat[bin]
            count, ok := label_dis[label]
            if !ok {
                count = 0.0
            }
            point := make(map[string]interface{})
            point["x"] = min_weight + float64(bin) * (max_weight - min_weight) / 20.0
            if is_100percent == "true" {
                point["y"] = count / bin_sum[bin]
            } else {
                point["y"] = count
            }
            values = append(values, point)
        }
        record["values"] = values
        ret = append(ret, record)
    }
    b, _ := json.Marshal(ret)
    fmt.Fprint(w, string(b))
}

func SingleFeatureStat(w http.ResponseWriter, r *http.Request){
    feature := r.FormValue("feature")
    if feature[0] == '[' {
        SingleDiscreteFeatureStat(w, r)
    } else {
        SingleContinuousFeatureStat(w, r)
    }
}

type StringDoublePair struct {
    Key string
    Value float64
}

type StringDoublePairList struct {
    data []*StringDoublePair
}

func (self *StringDoublePairList) Less(i, j int) bool{
    return self.data[i].Value > self.data[j].Value
}

func (self *StringDoublePairList) Len() int{
    return len(self.data)
}

func (self *StringDoublePairList) Swap(i, j int) {
    self.data[i], self.data[j] = self.data[j], self.data[i]
}

func SingleDiscreteFeatureStat(w http.ResponseWriter,r *http.Request){
    w.Header().Set("Content-Type", "application/json")
    ret := []interface{}{}
    feature := r.FormValue("feature")
    dataset := r.FormValue("dataset")
    is_100percent := r.FormValue("is_100percent")
    value_label_dis, ok := global_stat[dataset].discrete_stat[feature]

    valuestr := []string{}
    for value, _ := range value_label_dis {
        valuestr = append(valuestr, value)
    }

    if !ok {
        b, _ := json.Marshal(ret)
        fmt.Fprint(w, string(b))
        return
    }

    value_sum := make(map[string]float64)

    for value, label_dis := range value_label_dis {
        value_sum[value] = 0.0
        for _, count := range label_dis {
            value_sum[value] += count
        }
    }

    value_sum_array := StringDoublePairList{}
    for value, sum := range value_sum {
        value_sum_array.data = append(value_sum_array.data, &(StringDoublePair{Key: value, Value: sum}))
    }
    sort.Sort(&value_sum_array)

    for label, _ := range global_stat[dataset].labels {
        record := make(map[string]interface{})
        record["key"] = feature + ": " + label
        values := []map[string]interface{}{}
        for i, vs := range value_sum_array.data{
            if i > 64 || vs.Value < 20.0 {
                break
            }
            value := vs.Key
            label_dis, _ := value_label_dis[value]
            count, ok := label_dis[label]
            if !ok {
                count = 0.0
            }
            point := make(map[string]interface{})
            point["x"] = value
            if is_100percent == "true" {
                point["y"] = count / value_sum[value]
            } else {
                point["y"] = count
            }
            values = append(values, point)
        }
        record["values"] = values
        ret = append(ret, record)
    }
    b, _ := json.Marshal(ret)
    fmt.Fprint(w, string(b))
}

func FeatureList( w http.ResponseWriter,r *http.Request ){
    w.Header().Set("Content-Type", "application/json")
    dataset := r.FormValue("dataset")
    ret := []string{}

    for key, _ := range global_stat[dataset].discrete_stat {
        ret = append(ret, key)
    }
    for key, _ := range global_stat[dataset].continuous_stat {
        ret = append(ret, key)
    }
    b, _ := json.Marshal(ret)
    fmt.Fprint(w, string(b))
}

func DataSetList(w http.ResponseWriter, r *http.Request) {
    w.Header().Set("Content-Type", "application/json")
    files, _ := ioutil.ReadDir("./data/")
    ret := []string{}
    for _, f := range files {
        if f.IsDir(){
            ret = append(ret, f.Name())
        }
    }
    b, _ := json.Marshal(ret)
    fmt.Fprint(w, string(b))
}

func main(){
    global_stat = StatAllData()
    http.Handle("/", http.FileServer(http.Dir(".")))
    http.HandleFunc( "/single_feature",SingleFeatureStat)
    http.HandleFunc("/feature_list", FeatureList)
    http.HandleFunc("/dataset_list", DataSetList)
    s := &http.Server{
        Addr:           ":2014",
        ReadTimeout:    30 * time.Second,
        WriteTimeout:   30 * time.Second,
        MaxHeaderBytes: 1 << 20,
    }
    log.Fatal(s.ListenAndServe())
}<|endoftext|>


<commit_before>// +build windows

package ansicolor

import (
	"bytes"
	"io"
	"strings"
	"syscall"
	"unsafe"
)

type csiState int

const (
	TEXT csiState = iota
	ESC1
	ESC2
)

type ansiColorWriter struct {
	w        io.Writer
	state    csiState
	paramBuf bytes.Buffer
	textBuf  bytes.Buffer
}

const (
	CSI1  byte = '\x1b'
	CSI2  byte = '['
	SEP   byte = ';'
	COLOR byte = 'm'
)

const (
	foreground_blue      = uint16(0x0001)
	foreground_green     = uint16(0x0002)
	foreground_red       = uint16(0x0004)
	foreground_intensity = uint16(0x0008)
	background_blue      = uint16(0x0010)
	background_green     = uint16(0x0020)
	background_red       = uint16(0x0040)
	background_intensity = uint16(0x0080)

	foreground_mask = foreground_blue | foreground_green | foreground_red | foreground_intensity
	background_mask = background_blue | background_green | background_red | background_intensity
)

const (
	ansi_reset         = "0"
	ansi_intensity_on  = "1"
	ansi_intensity_off = "22"

	ansi_foreground_black   = "30"
	ansi_foreground_red     = "31"
	ansi_foreground_green   = "32"
	ansi_foreground_yellow  = "33"
	ansi_foreground_blue    = "34"
	ansi_foreground_magenta = "35"
	ansi_foreground_cyan    = "36"
	ansi_foreground_white   = "37"
	ansi_foreground_default = "39"

	ansi_background_black   = "40"
	ansi_background_red     = "41"
	ansi_background_green   = "42"
	ansi_background_yellow  = "43"
	ansi_background_blue    = "44"
	ansi_background_magenta = "45"
	ansi_background_cyan    = "46"
	ansi_background_white   = "47"
	ansi_background_default = "49"
)

type drawType int

const (
	foreground drawType = iota
	background
)

type winColor struct {
	code     uint16
	drawType drawType
}

var colorMap = map[string]winColor{
	ansi_foreground_black:   {0, foreground},
	ansi_foreground_red:     {foreground_red, foreground},
	ansi_foreground_green:   {foreground_green, foreground},
	ansi_foreground_yellow:  {foreground_red | foreground_green, foreground},
	ansi_foreground_blue:    {foreground_blue, foreground},
	ansi_foreground_magenta: {foreground_red | foreground_blue, foreground},
	ansi_foreground_cyan:    {foreground_green | foreground_blue, foreground},
	ansi_foreground_white:   {foreground_red | foreground_green | foreground_blue, foreground},
	ansi_foreground_default: {foreground_red | foreground_green | foreground_blue, foreground},

	ansi_background_black:   {0, background},
	ansi_background_red:     {background_red, background},
	ansi_background_green:   {background_green, background},
	ansi_background_yellow:  {background_red | background_green, background},
	ansi_background_blue:    {background_blue, background},
	ansi_background_magenta: {background_red | background_blue, background},
	ansi_background_cyan:    {background_green | background_blue, background},
	ansi_background_white:   {background_red | background_green | background_blue, background},
	ansi_background_default: {0, background},
}

var (
	kernel32                       = syscall.NewLazyDLL("kernel32.dll")
	procSetConsoleTextAttribute    = kernel32.NewProc("SetConsoleTextAttribute")
	procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo")
)

type coord struct {
	X, Y int16
}

type small_rect struct {
	Left, Top, Right, Bottom int16
}

type console_screen_buffer_info struct {
	DwSize              coord
	DwCursorPosition    coord
	WAttributes         uint16
	SrWindow            small_rect
	DwMaximumWindowSize coord
}

func getConsoleScreenBufferInfo(hConsoleOutput uintptr) *console_screen_buffer_info {
	var csbi console_screen_buffer_info
	ret, _, _ := procGetConsoleScreenBufferInfo.Call(
		hConsoleOutput,
		uintptr(unsafe.Pointer(&csbi)))
	if ret == 0 {
		return nil
	}
	return &csbi
}

func setConsoleTextAttribute(hConsoleOutput uintptr, wAttributes uint16) bool {
	ret, _, _ := procSetConsoleTextAttribute.Call(
		hConsoleOutput,
		uintptr(wAttributes))
	return ret != 0
}

func changeColor(param []byte) {
	screenInfo := getConsoleScreenBufferInfo(uintptr(syscall.Stdout))
	if screenInfo == nil {
		return
	}

	wAttributes := screenInfo.WAttributes
	winForeColor := wAttributes & (foreground_red | foreground_green | foreground_blue)
	winBackColor := wAttributes & (background_red | background_green | background_blue)
	winIntensity := (wAttributes & foreground_intensity) != 0
	param_line := strings.Split(string(param), string(SEP))
	for _, p := range param_line {
		c, ok := colorMap[p]
		switch {
		case !ok:
			switch p {
			case ansi_reset:
				winForeColor = foreground_red | foreground_green | foreground_blue
				winBackColor = 0
				winIntensity = false
			case ansi_intensity_on:
				winIntensity = true
			case ansi_intensity_off:
				winIntensity = false
			default:
				// unknown code
			}
		case c.drawType == foreground:
			winForeColor = c.code
		case c.drawType == background:
			winBackColor = c.code
		}
	}
	if winIntensity {
		winForeColor |= foreground_intensity
	}
	setConsoleTextAttribute(uintptr(syscall.Stdout), winForeColor|winBackColor)
}

func parseEscapeSequence(command byte, param []byte) {
	switch command {
	case COLOR:
		changeColor(param)
	}
}

func isParam(b byte) bool {
	return ('0' <= b && b <= '9') || b == SEP
}

func (cw *ansiColorWriter) pushBuffer(ch byte) {
	cw.textBuf.WriteByte(ch)
}

func (cw *ansiColorWriter) flushBuffer() (int, error) {
	text := cw.textBuf.Bytes()
	cw.textBuf.Reset()
	return cw.w.Write(text)
}

func (cw *ansiColorWriter) Write(p []byte) (int, error) {
	r := 0
	for _, ch := range p {
		switch cw.state {
		case TEXT:
			if ch == CSI1 {
				cw.state = ESC1
			} else {
				cw.pushBuffer(ch)
			}
		case ESC1:
			switch ch {
			case CSI1:
				cw.pushBuffer(ch)
			case CSI2:
				cw.state = ESC2
			default:
				cw.pushBuffer(CSI1)
				cw.pushBuffer(ch)
				cw.state = TEXT
			}
		case ESC2:
			if isParam(ch) {
				cw.paramBuf.WriteByte(ch)
			} else {
				nw, err := cw.flushBuffer()
				r += nw
				if err != nil {
					return r, err
				}
				param := cw.paramBuf.Bytes()
				cw.paramBuf.Reset()
				parseEscapeSequence(ch, param)
				cw.state = TEXT
			}
		default:
			cw.pushBuffer(ch)
			cw.state = TEXT
		}
	}

	nw, err := cw.flushBuffer()
	return r + nw, err
}
<commit_msg>Unify name by mixedCaps<commit_after>// +build windows

package ansicolor

import (
	"bytes"
	"io"
	"strings"
	"syscall"
	"unsafe"
)

type csiState int

const (
	TEXT csiState = iota
	ESC1
	ESC2
)

type ansiColorWriter struct {
	w        io.Writer
	state    csiState
	paramBuf bytes.Buffer
	textBuf  bytes.Buffer
}

const (
	CSI1  byte = '\x1b'
	CSI2  byte = '['
	SEP   byte = ';'
	COLOR byte = 'm'
)

const (
	foregroundBlue      = uint16(0x0001)
	foregroundGreen     = uint16(0x0002)
	foregroundRed       = uint16(0x0004)
	foregroundIntensity = uint16(0x0008)
	backgroundBlue      = uint16(0x0010)
	backgroundGreen     = uint16(0x0020)
	backgroundRed       = uint16(0x0040)
	backgroundIntensity = uint16(0x0080)

	foregroundMask = foregroundBlue | foregroundGreen | foregroundRed | foregroundIntensity
	backgroundMask = backgroundBlue | backgroundGreen | backgroundRed | backgroundIntensity
)

const (
	ansiReset        = "0"
	ansiIntensityOn  = "1"
	ansiIntensityOff = "22"

	ansiForegroundBlack   = "30"
	ansiForegroundRed     = "31"
	ansiForegroundGreen   = "32"
	ansiForegroundYellow  = "33"
	ansiForegroundBlue    = "34"
	ansiForegroundMagenta = "35"
	ansiForegroundCyan    = "36"
	ansiForegroundWhite   = "37"
	ansiForegroundDefault = "39"

	ansiBackgroundBlack   = "40"
	ansiBackgroundRed     = "41"
	ansiBackgroundGreen   = "42"
	ansiBackgroundYellow  = "43"
	ansiBackgroundBlue    = "44"
	ansiBackgroundMagenta = "45"
	ansiBackgroundCyan    = "46"
	ansiBackgroundWhite   = "47"
	ansiBackgroundDefault = "49"
)

type drawType int

const (
	foreground drawType = iota
	background
)

type winColor struct {
	code     uint16
	drawType drawType
}

var colorMap = map[string]winColor{
	ansiForegroundBlack:   {0, foreground},
	ansiForegroundRed:     {foregroundRed, foreground},
	ansiForegroundGreen:   {foregroundGreen, foreground},
	ansiForegroundYellow:  {foregroundRed | foregroundGreen, foreground},
	ansiForegroundBlue:    {foregroundBlue, foreground},
	ansiForegroundMagenta: {foregroundRed | foregroundBlue, foreground},
	ansiForegroundCyan:    {foregroundGreen | foregroundBlue, foreground},
	ansiForegroundWhite:   {foregroundRed | foregroundGreen | foregroundBlue, foreground},
	ansiForegroundDefault: {foregroundRed | foregroundGreen | foregroundBlue, foreground},

	ansiBackgroundBlack:   {0, background},
	ansiBackgroundRed:     {backgroundRed, background},
	ansiBackgroundGreen:   {backgroundGreen, background},
	ansiBackgroundYellow:  {backgroundRed | backgroundGreen, background},
	ansiBackgroundBlue:    {backgroundBlue, background},
	ansiBackgroundMagenta: {backgroundRed | backgroundBlue, background},
	ansiBackgroundCyan:    {backgroundGreen | backgroundBlue, background},
	ansiBackgroundWhite:   {backgroundRed | backgroundGreen | backgroundBlue, background},
	ansiBackgroundDefault: {0, background},
}

var (
	kernel32                       = syscall.NewLazyDLL("kernel32.dll")
	procSetConsoleTextAttribute    = kernel32.NewProc("SetConsoleTextAttribute")
	procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo")
)

type coord struct {
	X, Y int16
}

type smallRect struct {
	Left, Top, Right, Bottom int16
}

type consoleScreenBufferInfo struct {
	DwSize              coord
	DwCursorPosition    coord
	WAttributes         uint16
	SrWindow            smallRect
	DwMaximumWindowSize coord
}

func getConsoleScreenBufferInfo(hConsoleOutput uintptr) *consoleScreenBufferInfo {
	var csbi consoleScreenBufferInfo
	ret, _, _ := procGetConsoleScreenBufferInfo.Call(
		hConsoleOutput,
		uintptr(unsafe.Pointer(&csbi)))
	if ret == 0 {
		return nil
	}
	return &csbi
}

func setConsoleTextAttribute(hConsoleOutput uintptr, wAttributes uint16) bool {
	ret, _, _ := procSetConsoleTextAttribute.Call(
		hConsoleOutput,
		uintptr(wAttributes))
	return ret != 0
}

func changeColor(param []byte) {
	screenInfo := getConsoleScreenBufferInfo(uintptr(syscall.Stdout))
	if screenInfo == nil {
		return
	}

	wAttributes := screenInfo.WAttributes
	winForeColor := wAttributes & (foregroundRed | foregroundGreen | foregroundBlue)
	winBackColor := wAttributes & (backgroundRed | backgroundGreen | backgroundBlue)
	winIntensity := (wAttributes & foregroundIntensity) != 0
	paramLine := strings.Split(string(param), string(SEP))
	for _, p := range paramLine {
		c, ok := colorMap[p]
		switch {
		case !ok:
			switch p {
			case ansiReset:
				winForeColor = foregroundRed | foregroundGreen | foregroundBlue
				winBackColor = 0
				winIntensity = false
			case ansiIntensityOn:
				winIntensity = true
			case ansiIntensityOff:
				winIntensity = false
			default:
				// unknown code
			}
		case c.drawType == foreground:
			winForeColor = c.code
		case c.drawType == background:
			winBackColor = c.code
		}
	}
	if winIntensity {
		winForeColor |= foregroundIntensity
	}
	setConsoleTextAttribute(uintptr(syscall.Stdout), winForeColor|winBackColor)
}

func parseEscapeSequence(command byte, param []byte) {
	switch command {
	case COLOR:
		changeColor(param)
	}
}

func isParam(b byte) bool {
	return ('0' <= b && b <= '9') || b == SEP
}

func (cw *ansiColorWriter) pushBuffer(ch byte) {
	cw.textBuf.WriteByte(ch)
}

func (cw *ansiColorWriter) flushBuffer() (int, error) {
	text := cw.textBuf.Bytes()
	cw.textBuf.Reset()
	return cw.w.Write(text)
}

func (cw *ansiColorWriter) Write(p []byte) (int, error) {
	r := 0
	for _, ch := range p {
		switch cw.state {
		case TEXT:
			if ch == CSI1 {
				cw.state = ESC1
			} else {
				cw.pushBuffer(ch)
			}
		case ESC1:
			switch ch {
			case CSI1:
				cw.pushBuffer(ch)
			case CSI2:
				cw.state = ESC2
			default:
				cw.pushBuffer(CSI1)
				cw.pushBuffer(ch)
				cw.state = TEXT
			}
		case ESC2:
			if isParam(ch) {
				cw.paramBuf.WriteByte(ch)
			} else {
				nw, err := cw.flushBuffer()
				r += nw
				if err != nil {
					return r, err
				}
				param := cw.paramBuf.Bytes()
				cw.paramBuf.Reset()
				parseEscapeSequence(ch, param)
				cw.state = TEXT
			}
		default:
			cw.pushBuffer(ch)
			cw.state = TEXT
		}
	}

	nw, err := cw.flushBuffer()
	return r + nw, err
}
<|endoftext|>


<commit_before>// (C) 1990-2016.  Robert W Solomon.  All rights reserved.
// rpn.go
package main

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"fmt"
	//	"io/ioutil"
	"os"
	"runtime"
	"strconv"
	"strings"
	//
	"src/getcommandline"
	hpcalc "src/hpcalc2"
	"src/makesubst"
)

const LastCompiled = "21 June 2021"

/*
REVISION HISTORY
----------------
 1 Dec 89 -- Changed prompt.
24 Dec 91 -- Converted to M-2 V 4.00.  Changed params to GETRESULT.
25 Jul 93 -- Output result without trailing insignificant zeros,
	                 imported UL2, and changed prompt again.
 3 Mar 96 -- Fixed bug in string display if real2str fails because
	                 number is too large (ie, Avogadro's Number).
18 May 03 -- First Win32 version.  And changed name.
 1 Apr 13 -- Back to console mode pgm that will read from the cmdline.  Intended to be a quick and useful little utility.
	                 And will save/restore the stack to/from a file.
 2 May 13 -- Will use console mode flag for HPCALC, so it will write to console instead of the terminal module routines.
	                 And I now have the skipline included in MiscStdInOut so it is removed from here.
15 Oct 13 -- Now writing for gm2 under linux.
22 Jul 14 -- Converting to Ada.
 6 Dec 14 -- Converting to cpp.
20 Dec 14 -- Added macros for date and time last compiled.
31 Dec 14 -- Started coding HOL command.
 1 Jan 15 -- After getting HOL command to work, I did more fiddling to further understand c-strings and c++ string class.
10 Jan 15 -- Playing with string conversions and number formatting.
 5 Nov 15 -- Added the RECIP, CURT, VOL commands to hpcalc.cpp
22 Nov 15 -- Noticed that T1 and T2 stack operations are not correct.  This effects HP2cursed and rpnc.
13 Apr 16 -- Adding undo and redo commands, which operate on the entire stack not just X register.
 2 Jul 16 -- Fixed help to include PI command, and changed pivot for JUL command.  See hpcalcc.cpp
 7 Jul 16 -- Added UP command to hpcalcc.cpp
 8 Jul 16 -- Added display of stack dump to always happen, and a start up message.
22 Aug 16 -- Started conversion to Go.
28 Aug 16 -- added makesubst capability for substitutions = -> + and ; -> *
28 Nov 16 -- Backported stringslice return of hpcalc so can use the new, improved hpcalc.
23 Feb 17 -- Made "?" equivalent to "help"
26 Mar 17 -- Changed startup message to include written in Go.
22 Aug 18 -- Learning about code folding
 2 Oct 18 -- Now using code folding.  za normal mode command toggles the fold mode where cursor is.
 8 Feb 20 -- Added PopX to hpcalc.go, and will test it here.
 9 Apr 20 -- Will add the suppressdump map I've been using for a while in rpng.
 8 Nov 20 -- Now will use hpcalc2.  I'm adding toclip, fromclip (based on code from "Go Standard Library Cookbook") to hpcalc2.
	                And finally removed code that was commented out long ago.
13 Jun 21 -- Converted code to use modules
19 Jun 21 -- Now uses the new MAP code written in hpcalc2 that does not require calling MapClose, which was never done here anyway.
21 Jun 21 -- As the ioutil package is depracated, I'm replacing it with the os package calls.
*/

var suppressDump map[string]bool

func main() {
	var R float64
	var INBUF, ans string
	const StackFileName = "RPNStack.sav"
	var stringslice []string

	var Stk hpcalc.StackType // used when time to write out the stack upon exit.

	var err error

	suppressDump = make(map[string]bool)
	suppressDump["PRIME"] = true
	suppressDump["HEX"] = true
	suppressDump["DOW"] = true
	suppressDump["HOL"] = true
	suppressDump["ABOUT"] = true
	suppressDump["HELP"] = true
	suppressDump["TOCLIP"] = true
	suppressDump["DUMP"] = true
	suppressDump["DUMPFIX"] = true
	suppressDump["DUMPFIXED"] = true
	suppressDump["DUMPFLOAT"] = true
	suppressDump["?"] = true
	//        suppressDump[""] = true

	allowDumpFlag := true

	StackFileExists := true
	InputByteSlice := make([]byte, 8*hpcalc.StackSize) // I hope this is a slice of 64 bytes, ie, 8*8.

	if InputByteSlice, err = os.ReadFile(StackFileName); err != nil {
		fmt.Printf(" Error from os.ReadFile.  Probably because no Stack File found: %v\n", err)
		StackFileExists = false
	}
	if StackFileExists { // This code is ackward to me now in 2021.  I wrote it in 2016.  I've learned a thing or 2 since then.
		for i := 0; i < hpcalc.StackSize*8; i = i + 8 {
			buf := bytes.NewReader(InputByteSlice[i : i+8])
			err := binary.Read(buf, binary.LittleEndian, &R)
			if err != nil {
				fmt.Printf(" binary.Read failed with error of %v \n", err)
				StackFileExists = false
			}
			hpcalc.PUSHX(R)
		} // loop to extract each 8 byte chunk to convert to a longreal (float64) and push onto the hpcalc stack.
	} // stackfileexists

	hpcalc.PushMatrixStacks()

	fmt.Println(" HP-type RPN calculator written in Go.  Last compiled ", LastCompiled, "by", runtime.Version())
	fmt.Println()

	scanner := bufio.NewScanner(os.Stdin)
	if len(os.Args) > 1 {
		INBUF = getcommandline.GetCommandLineString()
		INBUF = makesubst.MakeSubst(INBUF)
	} else {
		fmt.Print(" Enter calculation, HELP or Enter to exit: ")
		scanner.Scan()
		INBUF = scanner.Text()
		if err := scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, "reading standard input:", err)
			os.Exit(1)
		}
		if len(INBUF) == 0 {
			os.Exit(0)
		}
		INBUF = makesubst.MakeSubst(INBUF)
	} // if command tail exists

	hpcalc.PushMatrixStacks()

	for len(INBUF) > 0 { // main reading loop
		R, stringslice = hpcalc.GetResult(INBUF)
		ans = strconv.FormatFloat(R, 'g', -1, 64)
		ans = hpcalc.CropNStr(ans)
		if R > 10000 {
			ans = hpcalc.AddCommas(ans)
		}
		fmt.Println()
		fmt.Println()
		for _, ss := range stringslice {
			fmt.Println(ss)
			allowDumpFlag = false // Don't update stack if any strings were returned from hpcalc.GetResult()
		}

		if strings.ToLower(INBUF) == "about" {
			fmt.Println(" Last compiled rpn.go ", LastCompiled)
			allowDumpFlag = false
		}

		INBUF = strings.ToUpper(INBUF)
		if suppressDump[INBUF] {
			allowDumpFlag = false
		}

		if allowDumpFlag {
			_, stringslice = hpcalc.GetResult("DUMP") // discard result.  Only need stack dump general executed.
			for _, ss := range stringslice {
				fmt.Println(ss)
			}
		}

		fmt.Println()
		fmt.Print("                                            Result = ")
		hpcalc.OutputFixedOrFloat(R)
		fmt.Println("         |    ", ans)
		fmt.Print(" Enter calculation, HELP or Enter to exit: ")
		scanner.Scan()
		INBUF = scanner.Text()
		if err := scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, "reading standard input:", err)
			os.Exit(1)
		}
		INBUF = makesubst.MakeSubst(INBUF)
		allowDumpFlag = true
	}

	// Now that I've got this working, I'm taking notes.  The binary.Write appends to the buf after each
	// call, since I'm not doing anthing to the bytes.Buffer to reset it.  I don't need a separate slice of
	// bytes to accumulate the stack for output.  I just have to reverse the order I write them out so that
	// they are read in correctly, without reversing the stack after each write.  I could reset the
	// buf.Bytes each time if I wanted.  I tested that and it works.  But it is unnecessary for my needs so
	// I commented it out.

	Stk = hpcalc.GETSTACK()
	//  OutputByteSlice := make([]byte,0,8*hpcalc.StackSize);  // each stack element is a float64, and there are StackSize of these (now StackSize=8), so this is a slice of 64 bytes
	buf := new(bytes.Buffer)

	for i := hpcalc.T1; i >= hpcalc.X; i-- { // in reverse.  for range cannot go in reverse.
		r := Stk[i]
		err := binary.Write(buf, binary.LittleEndian, r)
		if err != nil {
			fmt.Printf(" binary.write into buf failed with error %v \n", err)
			os.Exit(1)
		}
	}
	err = os.WriteFile(StackFileName, buf.Bytes(), os.ModePerm) // os.ModePerm = 0777
	if err != nil {
		fmt.Printf(" os.WriteFile failed with error %v \n", err)
	}
} // main in rpn.go
<commit_msg>06/22/2021 05:48:57 PM rpn/rpn.go<commit_after>// (C) 1990-2016.  Robert W Solomon.  All rights reserved.
// rpn.go
package main

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"fmt"
	//	"io/ioutil"
	"os"
	"runtime"
	"strconv"
	"strings"
	//
	"src/getcommandline"
	hpcalc "src/hpcalc2"
	"src/makesubst"
)

const LastCompiled = "22 June 2021"

/*
REVISION HISTORY
----------------
 1 Dec 89 -- Changed prompt.
24 Dec 91 -- Converted to M-2 V 4.00.  Changed params to GETRESULT.
25 Jul 93 -- Output result without trailing insignificant zeros,
	                 imported UL2, and changed prompt again.
 3 Mar 96 -- Fixed bug in string display if real2str fails because
	                 number is too large (ie, Avogadro's Number).
18 May 03 -- First Win32 version.  And changed name.
 1 Apr 13 -- Back to console mode pgm that will read from the cmdline.  Intended to be a quick and useful little utility.
	                 And will save/restore the stack to/from a file.
 2 May 13 -- Will use console mode flag for HPCALC, so it will write to console instead of the terminal module routines.
	                 And I now have the skipline included in MiscStdInOut so it is removed from here.
15 Oct 13 -- Now writing for gm2 under linux.
22 Jul 14 -- Converting to Ada.
 6 Dec 14 -- Converting to cpp.
20 Dec 14 -- Added macros for date and time last compiled.
31 Dec 14 -- Started coding HOL command.
 1 Jan 15 -- After getting HOL command to work, I did more fiddling to further understand c-strings and c++ string class.
10 Jan 15 -- Playing with string conversions and number formatting.
 5 Nov 15 -- Added the RECIP, CURT, VOL commands to hpcalc.cpp
22 Nov 15 -- Noticed that T1 and T2 stack operations are not correct.  This effects HP2cursed and rpnc.
13 Apr 16 -- Adding undo and redo commands, which operate on the entire stack not just X register.
 2 Jul 16 -- Fixed help to include PI command, and changed pivot for JUL command.  See hpcalcc.cpp
 7 Jul 16 -- Added UP command to hpcalcc.cpp
 8 Jul 16 -- Added display of stack dump to always happen, and a start up message.
22 Aug 16 -- Started conversion to Go.
28 Aug 16 -- added makesubst capability for substitutions = -> + and ; -> *
28 Nov 16 -- Backported stringslice return of hpcalc so can use the new, improved hpcalc.
23 Feb 17 -- Made "?" equivalent to "help"
26 Mar 17 -- Changed startup message to include written in Go.
22 Aug 18 -- Learning about code folding
 2 Oct 18 -- Now using code folding.  za normal mode command toggles the fold mode where cursor is.
 8 Feb 20 -- Added PopX to hpcalc.go, and will test it here.
 9 Apr 20 -- Will add the suppressdump map I've been using for a while in rpng.
 8 Nov 20 -- Now will use hpcalc2.  I'm adding toclip, fromclip (based on code from "Go Standard Library Cookbook") to hpcalc2.
	                And finally removed code that was commented out long ago.
13 Jun 21 -- Converted code to use modules
19 Jun 21 -- Now uses the new MAP code written in hpcalc2 that does not require calling MapClose, which was never done here anyway.
21 Jun 21 -- As the ioutil package is depracated, I'm replacing it with the os package calls.
22 Jun 21 -- I'm rewriting the file reading code.  I wrote that 5 yrs ago.  It looks painful to me now.
*/

var suppressDump map[string]bool

func main() {
	var R float64
	var INBUF, ans string
	const StackFileName = "RPNStack.sav"
	var stringslice []string

	var Stk hpcalc.StackType // used when time to write out the stack upon exit.

	var err error

	suppressDump = make(map[string]bool)
	suppressDump["PRIME"] = true
	suppressDump["HEX"] = true
	suppressDump["DOW"] = true
	suppressDump["HOL"] = true
	suppressDump["ABOUT"] = true
	suppressDump["HELP"] = true
	suppressDump["TOCLIP"] = true
	suppressDump["DUMP"] = true
	suppressDump["DUMPFIX"] = true
	suppressDump["DUMPFIXED"] = true
	suppressDump["DUMPFLOAT"] = true
	suppressDump["?"] = true
	//        suppressDump[""] = true

	allowDumpFlag := true

	StackFileExists := true
	//InputByteSlice := make([]byte, 8*hpcalc.StackSize)

	InputByteSlice, err := os.ReadFile(StackFileName);
	if err != nil {
		fmt.Printf(" Error from os.ReadFile.  Probably because no Stack File found: %v\n", err)
		StackFileExists = false
	}
	if StackFileExists { // This code is ackward to me now in 2021.  I wrote it in 2016.  I've learned a thing or 2 since then.
		buf := bytes.NewReader(InputByteSlice)
		for i := 0; i < hpcalc.StackSize; i++ { // loop to extract each 8 byte chunk to convert to a float64 longreal and push onto the hpcalc stack.
			err := binary.Read(buf, binary.LittleEndian, &R)
			if err != nil {
				fmt.Printf(" binary.Read failed with error of %v \n", err)
				StackFileExists = false
			}
			hpcalc.PUSHX(R)
		}
	} // stackfileexists

	hpcalc.PushMatrixStacks()

	fmt.Println(" HP-type RPN calculator written in Go.  Last compiled ", LastCompiled, "by", runtime.Version())
	fmt.Println()

	scanner := bufio.NewScanner(os.Stdin)
	if len(os.Args) > 1 {
		INBUF = getcommandline.GetCommandLineString()
		INBUF = makesubst.MakeSubst(INBUF)
	} else {
		fmt.Print(" Enter calculation, HELP or Enter to exit: ")
		scanner.Scan()
		INBUF = scanner.Text()
		if err := scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, "reading standard input:", err)
			os.Exit(1)
		}
		if len(INBUF) == 0 {
			os.Exit(0)
		}
		INBUF = makesubst.MakeSubst(INBUF)
	} // if command tail exists

	hpcalc.PushMatrixStacks()

	for len(INBUF) > 0 { // main reading loop
		R, stringslice = hpcalc.GetResult(INBUF)
		ans = strconv.FormatFloat(R, 'g', -1, 64)
		ans = hpcalc.CropNStr(ans)
		if R > 10000 {
			ans = hpcalc.AddCommas(ans)
		}
		fmt.Println()
		fmt.Println()
		for _, ss := range stringslice {
			fmt.Println(ss)
			allowDumpFlag = false // Don't update stack if any strings were returned from hpcalc.GetResult()
		}

		if strings.ToLower(INBUF) == "about" {
			fmt.Println(" Last compiled rpn.go ", LastCompiled)
			allowDumpFlag = false
		}

		INBUF = strings.ToUpper(INBUF)
		if suppressDump[INBUF] {
			allowDumpFlag = false
		}

		if allowDumpFlag {
			_, stringslice = hpcalc.GetResult("DUMP") // discard result.  Only need stack dump general executed.
			for _, ss := range stringslice {
				fmt.Println(ss)
			}
		}

		fmt.Println()
		fmt.Print("                                            Result = ")
		hpcalc.OutputFixedOrFloat(R)
		fmt.Println("         |    ", ans)
		fmt.Print(" Enter calculation, HELP or Enter to exit: ")
		scanner.Scan()
		INBUF = scanner.Text()
		if err := scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, "reading standard input:", err)
			os.Exit(1)
		}
		INBUF = makesubst.MakeSubst(INBUF)
		allowDumpFlag = true
	}

	// Now that I've got this working, I'm taking notes.  The binary.Write appends to the buf after each
	// call, since I'm not doing anthing to the bytes.Buffer to reset it.  I don't need a separate slice of
	// bytes to accumulate the stack for output.  I just have to reverse the order I write them out so that
	// they are read in correctly, without reversing the stack after each write.  I could reset the
	// buf.Bytes each time if I wanted.  I tested that and it works.  But it is unnecessary for my needs so
	// I commented it out.

	Stk = hpcalc.GETSTACK()
	//  OutputByteSlice := make([]byte,0,8*hpcalc.StackSize);  // each stack element is a float64, and there are StackSize of these (now StackSize=8), so this is a slice of 64 bytes
	buf := new(bytes.Buffer)

	for i := hpcalc.T1; i >= hpcalc.X; i-- { // in reverse.  for range cannot go in reverse.
		r := Stk[i]
		err := binary.Write(buf, binary.LittleEndian, r)
		if err != nil {
			fmt.Printf(" binary.write into buf failed with error %v \n", err)
			os.Exit(1)
		}
	}
	err = os.WriteFile(StackFileName, buf.Bytes(), os.ModePerm) // os.ModePerm = 0777
	if err != nil {
		fmt.Printf(" os.WriteFile failed with error %v \n", err)
	}
} // main in rpn.go
<|endoftext|>


<commit_before>package rss

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"sort"
	"strings"
	"time"
)

func parseRSS2(data []byte, read *db) (*Feed, error) {
	warnings := false
	feed := rss2_0Feed{}
	p := xml.NewDecoder(bytes.NewReader(data))
	p.CharsetReader = charsetReader
	err := p.Decode(&feed)
	if err != nil {
		return nil, err
	}
	if feed.Channel == nil {
		return nil, fmt.Errorf("Error: no channel found in %q.", string(data))
	}

	channel := feed.Channel

	out := new(Feed)
	out.Title = channel.Title
	out.Description = channel.Description
	for _, link := range channel.Link {
		if link != "" {
			out.Link = link
		}
	}
	out.Image = channel.Image.Image()
	if channel.MinsToLive != 0 {
		sort.Ints(channel.SkipHours)
		next := time.Now().Add(time.Duration(channel.MinsToLive) * time.Minute)
		for _, hour := range channel.SkipHours {
			if hour == next.Hour() {
				next.Add(time.Duration(60-next.Minute()) * time.Minute)
			}
		}
		trying := true
		for trying {
			trying = false
			for _, day := range channel.SkipDays {
				if strings.Title(day) == next.Weekday().String() {
					next.Add(time.Duration(24-next.Hour()) * time.Hour)
					trying = true
					break
				}
			}
		}

		out.Refresh = next
	}

	if out.Refresh.IsZero() {
		out.Refresh = time.Now().Add(10 * time.Minute)
	}

	if channel.Items == nil {
		return nil, fmt.Errorf("Error: no feeds found in %q.", string(data))
	}

	out.Items = make([]*Item, 0, len(channel.Items))
	out.ItemMap = make(map[string]struct{})

	// Process items.
	for _, item := range channel.Items {

		if item.ID == "" {
			if item.Link == "" {
				if debug {
					fmt.Printf("[w] Item %q has no ID or link and will be ignored.\n", item.Title)
					fmt.Printf("[w] %#v\n", item)
				}
				warnings = true
				continue
			}
			item.ID = item.Link
		}

		// Skip items already known.
		if read.req <- item.ID; <-read.res {
			continue
		}

		next := new(Item)
		next.Title = item.Title
		next.Content = item.Content
		next.Link = item.Link
		if item.Date != "" {
			next.Date, err = parseTime(item.Date)
			if err != nil {
				return nil, err
			}
		} else if item.PubDate != "" {
			next.Date, err = parseTime(item.PubDate)
			if err != nil {
				return nil, err
			}
		}
		next.ID = item.ID
		if len(item.Enclosures) > 0 {
			next.Enclosures = make([]*Enclosure, len(item.Enclosures))
			for i := range item.Enclosures {
				next.Enclosures[i] = item.Enclosures[i].Enclosure()
			}
		}
		next.Read = false

		if _, ok := out.ItemMap[next.ID]; ok {
			if debug {
				fmt.Printf("[w] Item %q has duplicate ID.\n", next.Title)
				fmt.Printf("[w] %#v\n", next)
			}
			warnings = true
			continue
		}

		out.Items = append(out.Items, next)
		out.ItemMap[next.ID] = struct{}{}
		out.Unread++
	}

	if warnings && debug {
		fmt.Printf("[i] Encountered warnings:\n%s\n", data)
	}

	return out, nil
}

type rss2_0Feed struct {
	XMLName xml.Name       `xml:"rss"`
	Channel *rss2_0Channel `xml:"channel"`
}

type rss2_0Channel struct {
	XMLName     xml.Name     `xml:"channel"`
	Title       string       `xml:"title"`
	Description string       `xml:"description"`
	Link        []string     `xml:"link"`
	Image       rss2_0Image  `xml:"image"`
	Items       []rss2_0Item `xml:"item"`
	MinsToLive  int          `xml:"ttl"`
	SkipHours   []int        `xml:"skipHours>hour"`
	SkipDays    []string     `xml:"skipDays>day"`
}

type rss2_0Item struct {
	XMLName    xml.Name          `xml:"item"`
	Title      string            `xml:"title"`
	Content    string            `xml:"description"`
	Link       string            `xml:"link"`
	PubDate    string            `xml:"pubDate"`
	Date       string            `xml:"date"`
	ID         string            `xml:"guid"`
	Enclosures []rss2_0Enclosure `xml:"enclosure"`
}

type rss2_0Enclosure struct {
	XMLName xml.Name `xml:"enclosure"`
	Url     string   `xml:"url"`
	Type    string   `xml:"type"`
	Length  int      `xml:"length"`
}

func (r *rss2_0Enclosure) Enclosure() *Enclosure {
	out := new(Enclosure)
	out.Url = r.Url
	out.Type = r.Type
	out.Length = r.Length
	return out
}

type rss2_0Image struct {
	XMLName xml.Name `xml:"image"`
	Title   string   `xml:"title"`
	Url     string   `xml:"url"`
	Height  int      `xml:"height"`
	Width   int      `xml:"width"`
}

func (i *rss2_0Image) Image() *Image {
	out := new(Image)
	out.Title = i.Title
	out.Url = i.Url
	out.Height = uint32(i.Height)
	out.Width = uint32(i.Width)
	return out
}
<commit_msg>Updated rss_2.0.go to support multiple Link<commit_after>package rss

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"sort"
	"strings"
	"time"
)

func parseRSS2(data []byte, read *db) (*Feed, error) {
	warnings := false
	feed := rss2_0Feed{}
	p := xml.NewDecoder(bytes.NewReader(data))
	p.CharsetReader = charsetReader
	err := p.Decode(&feed)
	if err != nil {
		return nil, err
	}
	if feed.Channel == nil {
		return nil, fmt.Errorf("Error: no channel found in %q.", string(data))
	}

	channel := feed.Channel

	out := new(Feed)
	out.Title = channel.Title
	out.Description = channel.Description
	for _, link := range channel.Link {
		if link.Rel == "" && link.Type == "" && link.Href == "" && link.Chardata != "" {
			out.Link = link.Chardata
			break
		}
	}
	out.Image = channel.Image.Image()
	if channel.MinsToLive != 0 {
		sort.Ints(channel.SkipHours)
		next := time.Now().Add(time.Duration(channel.MinsToLive) * time.Minute)
		for _, hour := range channel.SkipHours {
			if hour == next.Hour() {
				next.Add(time.Duration(60-next.Minute()) * time.Minute)
			}
		}
		trying := true
		for trying {
			trying = false
			for _, day := range channel.SkipDays {
				if strings.Title(day) == next.Weekday().String() {
					next.Add(time.Duration(24-next.Hour()) * time.Hour)
					trying = true
					break
				}
			}
		}

		out.Refresh = next
	}

	if out.Refresh.IsZero() {
		out.Refresh = time.Now().Add(10 * time.Minute)
	}

	if channel.Items == nil {
		return nil, fmt.Errorf("Error: no feeds found in %q.", string(data))
	}

	out.Items = make([]*Item, 0, len(channel.Items))
	out.ItemMap = make(map[string]struct{})

	// Process items.
	for _, item := range channel.Items {

		if item.ID == "" {
			if item.Link == "" {
				if debug {
					fmt.Printf("[w] Item %q has no ID or link and will be ignored.\n", item.Title)
					fmt.Printf("[w] %#v\n", item)
				}
				warnings = true
				continue
			}
			item.ID = item.Link
		}

		// Skip items already known.
		if read.req <- item.ID; <-read.res {
			continue
		}

		next := new(Item)
		next.Title = item.Title
		next.Content = item.Content
		next.Link = item.Link
		if item.Date != "" {
			next.Date, err = parseTime(item.Date)
			if err != nil {
				return nil, err
			}
		} else if item.PubDate != "" {
			next.Date, err = parseTime(item.PubDate)
			if err != nil {
				return nil, err
			}
		}
		next.ID = item.ID
		if len(item.Enclosures) > 0 {
			next.Enclosures = make([]*Enclosure, len(item.Enclosures))
			for i := range item.Enclosures {
				next.Enclosures[i] = item.Enclosures[i].Enclosure()
			}
		}
		next.Read = false

		if _, ok := out.ItemMap[next.ID]; ok {
			if debug {
				fmt.Printf("[w] Item %q has duplicate ID.\n", next.Title)
				fmt.Printf("[w] %#v\n", next)
			}
			warnings = true
			continue
		}

		out.Items = append(out.Items, next)
		out.ItemMap[next.ID] = struct{}{}
		out.Unread++
	}

	if warnings && debug {
		fmt.Printf("[i] Encountered warnings:\n%s\n", data)
	}

	return out, nil
}

type rss2_0Feed struct {
	XMLName xml.Name       `xml:"rss"`
	Channel *rss2_0Channel `xml:"channel"`
}

type rss2_0Channel struct {
	XMLName     xml.Name     `xml:"channel"`
	Title       string       `xml:"title"`
	Description string       `xml:"description"`
	Link        []rss2_0Link `xml:"link"`
	Image       rss2_0Image  `xml:"image"`
	Items       []rss2_0Item `xml:"item"`
	MinsToLive  int          `xml:"ttl"`
	SkipHours   []int        `xml:"skipHours>hour"`
	SkipDays    []string     `xml:"skipDays>day"`
}

type rss2_0Link struct {
	Rel      string `xml:"rel,attr"`
	Href     string `xml:"href,attr"`
	Type     string `xml:"type,attr"`
	Chardata string `xml:",chardata"`
}

type rss2_0Item struct {
	XMLName    xml.Name          `xml:"item"`
	Title      string            `xml:"title"`
	Content    string            `xml:"description"`
	Link       string            `xml:"link"`
	PubDate    string            `xml:"pubDate"`
	Date       string            `xml:"date"`
	ID         string            `xml:"guid"`
	Enclosures []rss2_0Enclosure `xml:"enclosure"`
}

type rss2_0Enclosure struct {
	XMLName xml.Name `xml:"enclosure"`
	Url     string   `xml:"url"`
	Type    string   `xml:"type"`
	Length  int      `xml:"length"`
}

func (r *rss2_0Enclosure) Enclosure() *Enclosure {
	out := new(Enclosure)
	out.Url = r.Url
	out.Type = r.Type
	out.Length = r.Length
	return out
}

type rss2_0Image struct {
	XMLName xml.Name `xml:"image"`
	Title   string   `xml:"title"`
	Url     string   `xml:"url"`
	Height  int      `xml:"height"`
	Width   int      `xml:"width"`
}

func (i *rss2_0Image) Image() *Image {
	out := new(Image)
	out.Title = i.Title
	out.Url = i.Url
	out.Height = uint32(i.Height)
	out.Width = uint32(i.Width)
	return out
}
<|endoftext|>


<commit_before>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package retrieval

import (
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/extraction"
	"github.com/prometheus/prometheus/utility"
)

const (
	targetAddQueueSize     = 100
	targetReplaceQueueSize = 1
)

// TargetPool is a pool of targets for the same job.
type TargetPool struct {
	sync.RWMutex

	manager        TargetManager
	targetsByURL   map[string]Target
	interval       time.Duration
	ingester       extraction.Ingester
	addTargetQueue chan Target

	targetProvider TargetProvider

	stopping, stopped chan struct{}
}

// NewTargetPool creates a TargetPool, ready to be started by calling Run.
func NewTargetPool(m TargetManager, p TargetProvider, ing extraction.Ingester, i time.Duration) *TargetPool {
	return &TargetPool{
		manager:        m,
		interval:       i,
		ingester:       ing,
		targetsByURL:   make(map[string]Target),
		addTargetQueue: make(chan Target, targetAddQueueSize),
		targetProvider: p,
		stopping:       make(chan struct{}),
		stopped:        make(chan struct{}),
	}
}

// Run starts the target pool. It returns when the target pool has stopped
// (after calling Stop). Run is usually called as a goroutine.
func (p *TargetPool) Run() {
	ticker := time.NewTicker(p.interval)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			if p.targetProvider != nil {
				targets, err := p.targetProvider.Targets()
				if err != nil {
					glog.Warningf("Error looking up targets, keeping old list: %s", err)
				} else {
					p.ReplaceTargets(targets)
				}
			}
		case newTarget := <-p.addTargetQueue:
			p.addTarget(newTarget)
		case <-p.stopping:
			p.ReplaceTargets([]Target{})
			close(p.stopped)
			return
		}
	}
}

// Stop stops the target pool and returns once the shutdown is complete.
func (p *TargetPool) Stop() {
	close(p.stopping)
	<-p.stopped
}

// AddTarget adds a target by queuing it in the target queue.
func (p *TargetPool) AddTarget(target Target) {
	p.addTargetQueue <- target
}

func (p *TargetPool) addTarget(target Target) {
	p.Lock()
	defer p.Unlock()

	p.targetsByURL[target.URL()] = target
	go target.RunScraper(p.ingester, p.interval)
}

// ReplaceTargets replaces the old targets by the provided new ones but reuses
// old targets that are also present in newTargets to preserve scheduling and
// health state. Targets no longer present are stopped.
func (p *TargetPool) ReplaceTargets(newTargets []Target) {
	p.Lock()
	defer p.Unlock()

	newTargetURLs := make(utility.Set)
	for _, newTarget := range newTargets {
		newTargetURLs.Add(newTarget.URL())
		oldTarget, ok := p.targetsByURL[newTarget.URL()]
		if ok {
			oldTarget.SetBaseLabelsFrom(newTarget)
		} else {
			p.targetsByURL[newTarget.URL()] = newTarget
			go newTarget.RunScraper(p.ingester, p.interval)
		}
	}

	var wg sync.WaitGroup
	for k, oldTarget := range p.targetsByURL {
		if !newTargetURLs.Has(k) {
			wg.Add(1)
			go func(k string, oldTarget Target) {
				defer wg.Done()
				glog.V(1).Infof("Stopping scraper for target %s...", k)
				oldTarget.StopScraper()
				glog.V(1).Infof("Scraper for target %s stopped.", k)
			}(k, oldTarget)
			delete(p.targetsByURL, k)
		}
	}
	wg.Wait()
}

// Targets returns a copy of the current target list.
func (p *TargetPool) Targets() []Target {
	p.RLock()
	defer p.RUnlock()

	targets := make([]Target, 0, len(p.targetsByURL))
	for _, v := range p.targetsByURL {
		targets = append(targets, v)
	}
	return targets
}
<commit_msg>Sort targets on the status page.<commit_after>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package retrieval

import (
	"sort"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/extraction"
	"github.com/prometheus/prometheus/utility"
)

const (
	targetAddQueueSize     = 100
	targetReplaceQueueSize = 1
)

// TargetPool is a pool of targets for the same job.
type TargetPool struct {
	sync.RWMutex

	manager        TargetManager
	targetsByURL   map[string]Target
	interval       time.Duration
	ingester       extraction.Ingester
	addTargetQueue chan Target

	targetProvider TargetProvider

	stopping, stopped chan struct{}
}

// NewTargetPool creates a TargetPool, ready to be started by calling Run.
func NewTargetPool(m TargetManager, p TargetProvider, ing extraction.Ingester, i time.Duration) *TargetPool {
	return &TargetPool{
		manager:        m,
		interval:       i,
		ingester:       ing,
		targetsByURL:   make(map[string]Target),
		addTargetQueue: make(chan Target, targetAddQueueSize),
		targetProvider: p,
		stopping:       make(chan struct{}),
		stopped:        make(chan struct{}),
	}
}

// Run starts the target pool. It returns when the target pool has stopped
// (after calling Stop). Run is usually called as a goroutine.
func (p *TargetPool) Run() {
	ticker := time.NewTicker(p.interval)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			if p.targetProvider != nil {
				targets, err := p.targetProvider.Targets()
				if err != nil {
					glog.Warningf("Error looking up targets, keeping old list: %s", err)
				} else {
					p.ReplaceTargets(targets)
				}
			}
		case newTarget := <-p.addTargetQueue:
			p.addTarget(newTarget)
		case <-p.stopping:
			p.ReplaceTargets([]Target{})
			close(p.stopped)
			return
		}
	}
}

// Stop stops the target pool and returns once the shutdown is complete.
func (p *TargetPool) Stop() {
	close(p.stopping)
	<-p.stopped
}

// AddTarget adds a target by queuing it in the target queue.
func (p *TargetPool) AddTarget(target Target) {
	p.addTargetQueue <- target
}

func (p *TargetPool) addTarget(target Target) {
	p.Lock()
	defer p.Unlock()

	p.targetsByURL[target.URL()] = target
	go target.RunScraper(p.ingester, p.interval)
}

// ReplaceTargets replaces the old targets by the provided new ones but reuses
// old targets that are also present in newTargets to preserve scheduling and
// health state. Targets no longer present are stopped.
func (p *TargetPool) ReplaceTargets(newTargets []Target) {
	p.Lock()
	defer p.Unlock()

	newTargetURLs := make(utility.Set)
	for _, newTarget := range newTargets {
		newTargetURLs.Add(newTarget.URL())
		oldTarget, ok := p.targetsByURL[newTarget.URL()]
		if ok {
			oldTarget.SetBaseLabelsFrom(newTarget)
		} else {
			p.targetsByURL[newTarget.URL()] = newTarget
			go newTarget.RunScraper(p.ingester, p.interval)
		}
	}

	var wg sync.WaitGroup
	for k, oldTarget := range p.targetsByURL {
		if !newTargetURLs.Has(k) {
			wg.Add(1)
			go func(k string, oldTarget Target) {
				defer wg.Done()
				glog.V(1).Infof("Stopping scraper for target %s...", k)
				oldTarget.StopScraper()
				glog.V(1).Infof("Scraper for target %s stopped.", k)
			}(k, oldTarget)
			delete(p.targetsByURL, k)
		}
	}
	wg.Wait()
}

type targetsByURL []Target

func (s targetsByURL) Len() int {
	return len(s)
}
func (s targetsByURL) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s targetsByURL) Less(i, j int) bool {
	return s[i].URL() < s[j].URL()
}

// Targets returns a sorted copy of the current target list.
func (p *TargetPool) Targets() []Target {
	p.RLock()
	defer p.RUnlock()

	targets := make(targetsByURL, 0, len(p.targetsByURL))
	for _, v := range p.targetsByURL {
		targets = append(targets, v)
	}
	sort.Sort(targets)
	return targets
}
<|endoftext|>