<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor:
// - Aaron Meihm ameihm@mozilla.com

package main

import (
	"encoding/json"
	"fmt"
	"github.com/ameihm0912/gozdef"
	elastigo "github.com/mattbaird/elastigo/lib"
	"strings"
	"time"
)

// Query the database for a list of hosts we need to update vulnerability
// scores for; we want to return at most the batch size specified in the
// configuration file
func scoreVulnGetHosts() (ret map[int]string, err error) {
	ret = make(map[int]string)
	// Just use the API opContext to access the database
	op := opContext{}
	op.newContext(dbconn, false, "scorevuln")

	dur, err := time.ParseDuration(cfg.Vulnerabilities.ScoreEvery)
	if err != nil {
		return
	}
	cutoff := time.Now().UTC().Add(-1 * dur)

	// Grab a list of candidate hosts
	rows, err := op.Query(`SELECT assetid, hostname FROM asset
		WHERE lastvulnscore < $1 AND assettype = 'host'
		ORDER BY lastvulnscore LIMIT $2`, cutoff, cfg.Vulnerabilities.ScoringBatchSize)
	if err != nil {
		return
	}
	for rows.Next() {
		var (
			h   string
			hid int
		)
		err = rows.Scan(&hid, &h)
		if err != nil {
			rows.Close()
			return
		}
		ret[hid] = h
	}

	return
}

// Request vulnerability for hostname h from ES and update the vuln
// scoring table with the result
func scoreVulnScoreHost(hid int, h string) (err error) {
	conn := elastigo.NewConn()
	defer conn.Close()
	conn.Domain = cfg.Compliance.ESHost

	op := opContext{}
	op.newContext(dbconn, false, "scorevuln")

	defer func() {
		_, e := op.Exec(`UPDATE asset SET
		lastvulnscore = now()
		WHERE lower(hostname) = lower($1)
		AND assettype = 'host'
		AND assetid = $2`, h, hid)
		if e != nil {
			// Only change the error message if we haven't already
			// encountered an error
			if err != nil {
				err = e
			}
		}
	}()

	logf("scorevuln: processing %v", h)
	template := `{
		"size": 10000,
		"query": {
			"bool": {
				"must": [
				{
					"query_string": {
						"query": "asset.hostname: \"%v\""
					}
				},
				{
					"term": {
						"status": "open"
					}
				},
				{
					"range": {
						"utctimestamp": {
							"gt": "now-2d"
						}
					}
				}
				]
			}
		}
	}`
	tempbuf := fmt.Sprintf(template, h)
	res, err := conn.Search(cfg.Vulnerabilities.Index, "vulnerability_state", nil, tempbuf)
	if err != nil {
		return err
	}

	var maxcnt, highcnt, medcnt, lowcnt int
	var vi []gozdef.VulnEvent

	highlike := 1

	if res.Hits.Len() == 0 {
		// If we didn't get any results, it could be because the host is not being
		// assessed, or there are no known issues. We want to insert an entry indicating
		// no issues. This should not provide invalid information for hosts that are
		// not assessed, as vulnstatus should note the status of coverage for the host
		// when we perform risk calculation.
		goto skipanalyze
	}
	for _, x := range res.Hits.Hits {
		var nvi gozdef.VulnEvent
		err = json.Unmarshal(*x.Source, &nvi)
		vi = append(vi, nvi)
	}

	// Count issues on the host
	for _, x := range vi {
		var tl int
		// We want the highest likelihood indicator value from the set
		if x.Vuln.LikelihoodIndicator != "" {
			switch strings.ToLower(x.Vuln.LikelihoodIndicator) {
			case "maximum":
				tl = 4
			case "high":
				tl = 3
			case "medium":
				tl = 2
			case "low":
				tl = 1
			default:
				logf("warning: vulnscore: entry for %v (%v) has invalid likelihood indicator",
					h, x.Vuln.VulnID)
			}
		}
		if tl > highlike {
			highlike = tl
		}
		// First try the impact label, if that is not successful try the CVSS score
		if x.Vuln.ImpactLabel != "" {
			switch strings.ToLower(x.Vuln.ImpactLabel) {
			case "maximum":
				maxcnt++
			case "high":
				highcnt++
			case "mediumlow":
				medcnt++
			case "medium":
				medcnt++
			case "low":
				lowcnt++
			default:
				logf("warning: vulnscore: entry for %v (%v) has invalid impact label",
					h, x.Vuln.VulnID)
			}
		} else if x.Vuln.CVSS > 0 {
			if x.Vuln.CVSS > 9.0 {
				maxcnt++
			} else if x.Vuln.CVSS > 7.0 {
				highcnt++
			} else if x.Vuln.CVSS > 5.0 {
				medcnt++
			} else {
				lowcnt++
			}
		} else {
			logf("warning: vulnscore: entry for %v (%v) has no usable information",
				h, x.Vuln.VulnID)
		}
	}

skipanalyze:
	_, err = op.Exec(`INSERT INTO vulnscore
		(timestamp, assetid, maxcount, highcount, mediumcount, lowcount,
		likelihoodindicator)
		VALUES
		(now(),
		(SELECT assetid FROM asset
		WHERE lower(hostname) = lower($1)
		AND assettype = 'host'
		AND assetid = $2),
		$3, $4, $5, $6, $7)`, h, hid, maxcnt, highcnt, medcnt, lowcnt, highlike)
	if err != nil {
		return err
	}

	return nil
}

// Entry point for compliance scoring routine
func scoreVuln() {
	defer func() {
		if e := recover(); e != nil {
			logf("error in vulnerability scoring routine: %v", e)
		}
	}()
	hl, err := scoreVulnGetHosts()
	if err != nil {
		panic(err)
	}
	for hid, hname := range hl {
		err = scoreVulnScoreHost(hid, hname)
		if err != nil {
			panic(err)
		}
	}
}
<commit_msg>update vuln scoring to use v2 vuln format<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor:
// - Aaron Meihm ameihm@mozilla.com

package main

import (
	"encoding/json"
	"fmt"
	elastigo "github.com/mattbaird/elastigo/lib"
	slib "servicelib"
	"strconv"
	"strings"
	"time"
)

// Query the database for a list of hosts we need to update vulnerability
// scores for; we want to return at most the batch size specified in the
// configuration file
func scoreVulnGetHosts() (ret map[int]string, err error) {
	ret = make(map[int]string)
	// Just use the API opContext to access the database
	op := opContext{}
	op.newContext(dbconn, false, "scorevuln")

	dur, err := time.ParseDuration(cfg.Vulnerabilities.ScoreEvery)
	if err != nil {
		return
	}
	cutoff := time.Now().UTC().Add(-1 * dur)

	// Grab a list of candidate hosts
	rows, err := op.Query(`SELECT assetid, hostname FROM asset
		WHERE lastvulnscore < $1 AND assettype = 'host'
		ORDER BY lastvulnscore LIMIT $2`, cutoff, cfg.Vulnerabilities.ScoringBatchSize)
	if err != nil {
		return
	}
	for rows.Next() {
		var (
			h   string
			hid int
		)
		err = rows.Scan(&hid, &h)
		if err != nil {
			rows.Close()
			return
		}
		ret[hid] = h
	}

	return
}

// Request vulnerability for hostname h from ES and update the vuln
// scoring table with the result
func scoreVulnScoreHost(hid int, h string) (err error) {
	conn := elastigo.NewConn()
	defer conn.Close()
	conn.Domain = cfg.Compliance.ESHost

	op := opContext{}
	op.newContext(dbconn, false, "scorevuln")

	defer func() {
		_, e := op.Exec(`UPDATE asset SET
		lastvulnscore = now()
		WHERE lower(hostname) = lower($1)
		AND assettype = 'host'
		AND assetid = $2`, h, hid)
		if e != nil {
			// Only change the error message if we haven't already
			// encountered an error
			if err != nil {
				err = e
			}
		}
	}()

	logf("scorevuln: processing %v", h)
	template := `{
		"size": 10000,
		"query": {
			"bool": {
				"must": [
				{
					"query_string": {
						"query": "asset.hostname: \"%v\""
					}
				},
				{
					"range": {
						"utctimestamp": {
							"gt": "now-2d"
						}
					}
				}
				]
			}
		}
	}`
	tempbuf := fmt.Sprintf(template, h)
	res, err := conn.Search(cfg.Vulnerabilities.Index, "vulnerability_state", nil, tempbuf)
	if err != nil {
		return err
	}

	var maxcnt, highcnt, medcnt, lowcnt int
	var vi slib.Vuln

	highlike := 1

	if res.Hits.Len() == 0 {
		// If we didn't get any results, it could be because the host is not being
		// assessed, or there are no known issues. We want to insert an entry indicating
		// no issues. This should not provide invalid information for hosts that are
		// not assessed, as vulnstatus should note the status of coverage for the host
		// when we perform risk calculation.
		goto skipanalyze
	}
	err = json.Unmarshal(*res.Hits.Hits[0].Source, &vi)
	if err != nil {
		return err
	}
	for i := range vi.Vulns {
		err = vi.Vulns[i].Normalize()
		if err != nil {
			return err
		}
	}

	// Count issues on the host
	for _, x := range vi.Vulns {
		var tl int
		// We want the highest likelihood indicator value from the set
		if x.LikelihoodIndicator != "" {
			switch strings.ToLower(x.LikelihoodIndicator) {
			case "maximum":
				tl = 4
			case "high":
				tl = 3
			case "medium":
				tl = 2
			case "low":
				tl = 1
			default:
				logf("warning: vulnscore: entry for %v has invalid likelihood indicator",
					h)
			}
		}
		if tl > highlike {
			highlike = tl
		}
		// First try the impact label, if that is not successful try the CVSS score
		if x.Risk != "" {
			switch strings.ToLower(x.Risk) {
			case "maximum":
				maxcnt++
			case "high":
				highcnt++
			case "mediumlow":
				medcnt++
			case "medium":
				medcnt++
			case "low":
				lowcnt++
			default:
				logf("warning: vulnscore: entry for %v has invalid impact label",
					h)
			}
		} else if x.CVSS != "" {
			cvssf, err := strconv.ParseFloat(x.CVSS, 64)
			if err != nil {
				if cvssf > 9.0 {
					maxcnt++
				} else if cvssf > 7.0 {
					highcnt++
				} else if cvssf > 5.0 {
					medcnt++
				} else {
					lowcnt++
				}
			}
		} else {
			logf("warning: vulnscore: entry for %v has no usable information",
				h)
		}
	}

skipanalyze:
	_, err = op.Exec(`INSERT INTO vulnscore
		(timestamp, assetid, maxcount, highcount, mediumcount, lowcount,
		likelihoodindicator)
		VALUES
		(now(),
		(SELECT assetid FROM asset
		WHERE lower(hostname) = lower($1)
		AND assettype = 'host'
		AND assetid = $2),
		$3, $4, $5, $6, $7)`, h, hid, maxcnt, highcnt, medcnt, lowcnt, highlike)
	if err != nil {
		return err
	}

	return nil
}

// Entry point for compliance scoring routine
func scoreVuln() {
	defer func() {
		if e := recover(); e != nil {
			logf("error in vulnerability scoring routine: %v", e)
		}
	}()
	hl, err := scoreVulnGetHosts()
	if err != nil {
		panic(err)
	}
	for hid, hname := range hl {
		err = scoreVulnScoreHost(hid, hname)
		if err != nil {
			panic(err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package tsbridge deals with Time Series Bridge configuration files and metric representations.
// This file defines metric stats collector and contains metric definitions.
package tsbridge

import (
	"context"
	"fmt"
	"os"
	"sync"

	sdexporter "contrib.go.opencensus.io/exporter/stackdriver"
	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
	"go.opencensus.io/tag"
	"google.golang.org/appengine/log"
)

// Manually configured buckets for a distribution metric measuring latency in milliseconds.
var latencyDistribution = view.Distribution(100, 250, 500, 1000, 2000, 3000, 4000, 5000, 7500, 10000, 15000, 20000, 40000, 60000, 90000, 120000, 300000, 600000)

// statsMu makes sure there's only a single active StatsCollector, since OpenCensus metrics need to be globally defined for the whole process.
// This lock is taken when a new collector is created, and released when it's closed.
var statsMu = &sync.Mutex{}

// statsExporter is implemented by Stackdriver exporter and by fakeExporter we use in tests.
type statsExporter interface {
	view.Exporter
	Flush()
}

// StatsCollector has all metrics, tags, and the exporter used to publish them.
type StatsCollector struct {
	Exporter            statsExporter
	MetricImportLatency *stats.Int64Measure
	TotalImportLatency  *stats.Int64Measure
	OldestMetricAge     *stats.Int64Measure
	MetricKey           tag.Key
	views               []*view.View
	ctx                 context.Context
}

// NewCollector creates a new StatsCollector.
// Users need to call StatsCollector.Close() when it's no longer needed. Only a single collector can be active per process.
func NewCollector(ctx context.Context) (*StatsCollector, error) {
	var err error
	c := &StatsCollector{ctx: ctx}

	project := os.Getenv("SD_PROJECT_FOR_INTERNAL_METRICS")
	if project == "" {
		project = projectID(ctx)
	}
	if project == "" {
		return nil, fmt.Errorf("Please define SD_PROJECT_FOR_INTERNAL_METRICS if not running on App Engine")
	}

	c.Exporter, err = sdexporter.NewExporter(sdexporter.Options{
		ProjectID: project,
		OnError:   c.logError,
		Context:   ctx,
	})
	if err != nil {
		return nil, err
	}
	if err = c.registerAndCreateMetrics(); err != nil {
		// Clean up after registerAndCreateMetrics. Don't delete this!
		c.Close()
		return nil, err
	}
	return c, nil
}

// logError is configured as the error handler for Stackdriver exporter.
func (c *StatsCollector) logError(err error) {
	log.Errorf(c.ctx, "StatsCollector: %v", err)
}

// Close unregisters all metrics and Stackdriver exporter and flushes accumulated
// metric points to Stackdriver.
func (c *StatsCollector) Close() {
	view.Unregister(c.views...)
	view.UnregisterExporter(c.Exporter)
	c.Exporter.Flush()
	statsMu.Unlock()
}

// registerAndCreateMetrics registers Stackdriver exporter and metric views.
func (c *StatsCollector) registerAndCreateMetrics() error {
	statsMu.Lock()
	var err error
	view.RegisterExporter(c.Exporter)

	c.MetricKey, err = tag.NewKey("metric_name")
	if err != nil {
		return err
	}

	c.MetricImportLatency = stats.Int64("ts_bridge/metric_import_latencies", "time since last successful import for a metric", stats.UnitMilliseconds)
	c.TotalImportLatency = stats.Int64("ts_bridge/import_latencies", "total time it took to import all metrics", stats.UnitMilliseconds)
	c.OldestMetricAge = stats.Int64("ts_bridge/oldest_metric_age", "oldest time since last successful import across all metrics", stats.UnitMilliseconds)
	c.views = []*view.View{
		&view.View{
			Name:        c.MetricImportLatency.Name(),
			Description: c.MetricImportLatency.Description(),
			Measure:     c.MetricImportLatency,
			Aggregation: latencyDistribution,
			TagKeys:     []tag.Key{c.MetricKey},
		},
		&view.View{
			Name:        c.TotalImportLatency.Name(),
			Description: c.TotalImportLatency.Description(),
			Measure:     c.TotalImportLatency,
			Aggregation: latencyDistribution,
		},
		&view.View{
			Name:        c.OldestMetricAge.Name(),
			Description: c.OldestMetricAge.Description(),
			Measure:     c.OldestMetricAge,
			Aggregation: view.LastValue(),
		},
	}
	if err := view.Register(c.views...); err != nil {
		return err
	}
	return nil
}
<commit_msg>Disable regular flushing of internal SD metrics (fix #11).<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package tsbridge deals with Time Series Bridge configuration files and metric representations.
// This file defines metric stats collector and contains metric definitions.
package tsbridge

import (
	"context"
	"fmt"
	"os"
	"sync"
	"time"

	sdexporter "contrib.go.opencensus.io/exporter/stackdriver"
	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
	"go.opencensus.io/tag"
	"google.golang.org/appengine/log"
)

// Manually configured buckets for a distribution metric measuring latency in milliseconds.
var latencyDistribution = view.Distribution(100, 250, 500, 1000, 2000, 3000, 4000, 5000, 7500, 10000, 15000, 20000, 40000, 60000, 90000, 120000, 300000, 600000)

// statsMu makes sure there's only a single active StatsCollector, since OpenCensus metrics need to be globally defined for the whole process.
// This lock is taken when a new collector is created, and released when it's closed.
var statsMu = &sync.Mutex{}

// statsExporter is implemented by Stackdriver exporter and by fakeExporter we use in tests.
type statsExporter interface {
	view.Exporter
	Flush()
}

// StatsCollector has all metrics, tags, and the exporter used to publish them.
type StatsCollector struct {
	Exporter            statsExporter
	MetricImportLatency *stats.Int64Measure
	TotalImportLatency  *stats.Int64Measure
	OldestMetricAge     *stats.Int64Measure
	MetricKey           tag.Key
	views               []*view.View
	ctx                 context.Context
}

// NewCollector creates a new StatsCollector.
// Users need to call StatsCollector.Close() when it's no longer needed. Only a single collector can be active per process.
func NewCollector(ctx context.Context) (*StatsCollector, error) {
	var err error
	c := &StatsCollector{ctx: ctx}

	project := os.Getenv("SD_PROJECT_FOR_INTERNAL_METRICS")
	if project == "" {
		project = projectID(ctx)
	}
	if project == "" {
		return nil, fmt.Errorf("Please define SD_PROJECT_FOR_INTERNAL_METRICS if not running on App Engine")
	}

	c.Exporter, err = sdexporter.NewExporter(sdexporter.Options{
		ProjectID: project,
		OnError:   c.logError,
		Context:   ctx,
	})
	if err != nil {
		return nil, err
	}
	if err = c.registerAndCreateMetrics(); err != nil {
		// Clean up after registerAndCreateMetrics. Don't delete this!
		c.Close()
		return nil, err
	}
	return c, nil
}

// logError is configured as the error handler for Stackdriver exporter.
func (c *StatsCollector) logError(err error) {
	log.Errorf(c.ctx, "StatsCollector: %v", err)
}

// Close unregisters all metrics and Stackdriver exporter and flushes accumulated
// metric points to Stackdriver.
func (c *StatsCollector) Close() {
	view.Unregister(c.views...)
	view.UnregisterExporter(c.Exporter)
	c.Exporter.Flush()
	statsMu.Unlock()
}

// registerAndCreateMetrics registers Stackdriver exporter and metric views.
func (c *StatsCollector) registerAndCreateMetrics() error {
	statsMu.Lock()
	var err error
	view.RegisterExporter(c.Exporter)

	// Reporting period is set very high here to effectively disable regular flushing of metrics by OpenCensus
	// view worker. Since stats collector is relatively short-lived, we rely on metric flushing that happens when
	// the Stackdriver exporter is closed by the Close function above (at the end of each sync operation).
	view.SetReportingPeriod(time.Hour)

	c.MetricKey, err = tag.NewKey("metric_name")
	if err != nil {
		return err
	}

	c.MetricImportLatency = stats.Int64("ts_bridge/metric_import_latencies", "time since last successful import for a metric", stats.UnitMilliseconds)
	c.TotalImportLatency = stats.Int64("ts_bridge/import_latencies", "total time it took to import all metrics", stats.UnitMilliseconds)
	c.OldestMetricAge = stats.Int64("ts_bridge/oldest_metric_age", "oldest time since last successful import across all metrics", stats.UnitMilliseconds)
	c.views = []*view.View{
		&view.View{
			Name:        c.MetricImportLatency.Name(),
			Description: c.MetricImportLatency.Description(),
			Measure:     c.MetricImportLatency,
			Aggregation: latencyDistribution,
			TagKeys:     []tag.Key{c.MetricKey},
		},
		&view.View{
			Name:        c.TotalImportLatency.Name(),
			Description: c.TotalImportLatency.Description(),
			Measure:     c.TotalImportLatency,
			Aggregation: latencyDistribution,
		},
		&view.View{
			Name:        c.OldestMetricAge.Name(),
			Description: c.OldestMetricAge.Description(),
			Measure:     c.OldestMetricAge,
			Aggregation: view.LastValue(),
		},
	}
	if err := view.Register(c.views...); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013 gopm authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// gopm(Go Package Manager) is a Go package manage tool for search, install, update and share packages in Go.
package main

import (
	"os"
	"runtime"

	"github.com/codegangsta/cli"

	"github.com/gpmgo/gopm/cmd"
)

// +build go1.1

// Test that go1.1 tag above is included in builds. main.go refers to this definition.
const go11tag = true

const APP_VER = "0.5.5.1111"

// 	//cmd.CmdSearch,
// 		cmdClean,
// 		cmdDoc,
// 		cmdEnv,
// 		cmdFix,
// 		cmdList,
// 		cmdTest,
// 		cmdTool,
// 		cmdVet,
// }

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func main() {
	app := cli.NewApp()
	app.Name = "gopm"
	app.Usage = "Go Package Manager"
	app.Version = APP_VER
	app.Commands = []cli.Command{
		cmd.CmdGet,
		cmd.CmdGen,
		cmd.CmdRun,
		cmd.CmdBuild,
		cmd.CmdInstall,
	}
	app.Run(os.Args)
}
<commit_msg>get bug fixed<commit_after>// Copyright 2013 gopm authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// gopm(Go Package Manager) is a Go package manage tool for search, install, update and share packages in Go.
package main

import (
	"os"
	"runtime"

	"github.com/codegangsta/cli"

	"github.com/gpmgo/gopm/cmd"
)

// +build go1.1

// Test that go1.1 tag above is included in builds. main.go refers to this definition.
const go11tag = true

const APP_VER = "0.5.5.1129"

// 	//cmd.CmdSearch,
// 		cmdClean,
// 		cmdDoc,
// 		cmdEnv,
// 		cmdFix,
// 		cmdList,
// 		cmdTest,
// 		cmdTool,
// 		cmdVet,
// }

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func main() {
	app := cli.NewApp()
	app.Name = "gopm"
	app.Usage = "Go Package Manager"
	app.Version = APP_VER
	app.Commands = []cli.Command{
		cmd.CmdGet,
		cmd.CmdGen,
		cmd.CmdRun,
		cmd.CmdBuild,
		cmd.CmdInstall,
		//cmd.CmdTest,
	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/golang/groupcache/lru"
	"github.com/zyedidia/tcell"
	"github.com/zyedidia/tcell/encoding"
)

var topRow = 0
var leftColumn = 0
var height = 0
var width = 0
var offset = 0
var s tcell.Screen

func createDisplay() {
	loadDark()
	os.Setenv("TERM", "xterm-truecolor")
	sc, e := tcell.NewScreen()
	s = sc
	if e != nil {
		fmt.Fprintf(os.Stderr, "%v\n", e)
		os.Exit(1)
	}
	encoding.Register()

	if e = s.Init(); e != nil {
		fmt.Fprintf(os.Stderr, "%v\n", e)
		os.Exit(1)
	}

	s.SetStyle(tcell.StyleDefault.
		Foreground(tcell.ColorWhite).
		Background(tcell.ColorDefault))
	s.Clear()
	s.Show()
	w, h := s.Size()
	width = w
	height = h
}

//Display renders the editor
func Display(buffer *Buffer) {
	createDisplay()
	quit := make(chan struct{})
	b := make(chan [][]Backing)
	evName := ""
	go func() {
		for {
			t := time.Now()
			offset = drawRuler(topRow, height-1, s)
			var w = width - offset + 1
			lines := buffer.GetLines(topRow, leftColumn, height-1, w)
			go drawBufferHighlight(b, topRow, lines)

			puts(s, tcell.StyleDefault.
				Foreground(tcell.ColorWhite).
				Background(tcell.ColorDefault), 0, height-1, "                                                                                ")
			puts(s, tcell.StyleDefault.
				Foreground(tcell.ColorWhite).
				Background(tcell.ColorDefault), 0, height-1, buffer.filename)

			var text = strconv.Itoa(GetCursor().loc.row+1) + ":" + strconv.Itoa(GetCursor().loc.column+1) +
				"/" + strconv.Itoa(buffer.Len()) + ":" + strconv.Itoa(len(buffer.GetLine(GetCursor().loc.row)))

			puts(s, tcell.StyleDefault.
				Foreground(tcell.ColorWhite).
				Background(tcell.ColorDefault), width-len(text), height-1, text)

			puts(s, tcell.StyleDefault.
				Foreground(tcell.ColorWhite).
				Background(tcell.ColorDefault), width-len(text)-2-len(evName), height-1, evName)

			drawBack(<-b)
			time2 := time.Now().Sub(t).String()
			puts(s, tcell.StyleDefault.
				Foreground(tcell.ColorWhite).
				Background(tcell.ColorDefault), len(buffer.filename)+1, height-1, time2)

			s.Show()
			ev := s.PollEvent()
			switch ev := ev.(type) {
			case *tcell.EventKey:
				switch ev.Key() {
				case tcell.KeyDown:
					c := GetCursor()
					c.MoveDown()
				case tcell.KeyUp:
					c := GetCursor()
					c.MoveUp()
				case tcell.KeyLeft:
					c := GetCursor()
					c.MoveLeft()
				case tcell.KeyRight:
					c := GetCursor()
					c.MoveRight()
				case tcell.KeyPgDn:
					c := GetCursor()
					c.MovePageDown()
				case tcell.KeyPgUp:
					c := GetCursor()
					c.MovePageUp()
				case tcell.KeyEscape:
					close(quit)
					return
				case tcell.KeyCtrlD:
					c := GetCursor()
					buffer.Insert(c.loc.row, 0, buffer.GetLine(c.loc.row))
					c.MoveDown()
				case tcell.KeyCtrlY:
					c := GetCursor()
					buffer.RemoveRow(c.loc.row)
				case tcell.KeyDelete:
					c := GetCursor()
					buffer.Delete(c.loc.row, c.loc.column)
				case tcell.KeyBackspace2:
					c := GetCursor()
					l := buffer.Backspace(c.loc.row, c.loc.column)
					if c.loc.column == 0 && c.loc.row > 0 {
						c.MoveUp()
						c.MoveToEndOfLine()
						for index := 0; index < l; index++ {
							c.MoveLeft()
						}
					} else {
						c.MoveLeft()
					}
				case tcell.KeyCtrlS:
					buffer.Save()
				case tcell.KeyTab:
					c := GetCursor()
					buffer.Insert(c.loc.row, c.loc.column, []rune(string('\t')))
					c.MoveRight()
				case tcell.KeyRune:
					evName = ev.Name()
					c := GetCursor()
					buffer.Insert(c.loc.row, c.loc.column, []rune(string(ev.Rune())))
					c.MoveRight()
				case tcell.KeyEnter:
					c := GetCursor()
					//buffer.Insert(c.loc.row, c.loc.column, []rune(string(ev.Rune())))
					buffer.InsertEnter(c.loc.row, c.loc.column)
					c.MoveDown()
					c.MoveStartOfLine()
				default:
					evName = ev.Name()
				}
			case *tcell.EventPaste:
				clip := ev.Text()
				c := GetCursor()
				buffer.Insert(c.loc.row, c.loc.column, []rune(clip))
				for range clip {
					c.MoveRight()
				}

			case *tcell.EventResize:
				s.Sync()
			}
		}
	}()

	<-quit

	s.Fini()
}

// Backing backs the display so we don't overwrite runes unnessecary
type Backing struct {
	value rune
	style tcell.Style
}

var cache = lru.New(1000)

func drawBufferHighlight(b chan [][]Backing, topRow int, lines [][]rune) {
	backing := highlightLines(lines)
	backing = paintCursor(backing, topRow)
	b <- backing
}
func drawBuffer(b chan [][]Backing, topRow int, lines [][]rune) {
	backing := drawLines(lines)
	backing = paintCursor(backing, topRow)
	b <- backing
}
func paintCursor(backing [][]Backing, topRow int) [][]Backing {
	c := GetCursor()
	if len(backing) > c.loc.row-topRow && len(backing[c.loc.row-topRow]) > c.loc.column-leftColumn {
		for i := range backing[c.loc.row-topRow] {
			backing[c.loc.row-topRow][i].style = backing[c.loc.row-topRow][i].style.Background(tcell.GetColor("#282828"))
		}
		// No cursor on enter
		if c.loc.column != -1 {
			if backing[c.loc.row-topRow][c.loc.column-leftColumn].value == '\n' {
				backing[c.loc.row-topRow][c.loc.column-leftColumn+1].style = tcell.StyleDefault.Reverse(true)
			} else {
				backing[c.loc.row-topRow][c.loc.column-leftColumn].style = tcell.StyleDefault.Reverse(true)
			}
		}
	}
	return backing
}
func highlightLines(lines [][]rune) [][]Backing {
	w, h := s.Size()
	backing := make([][]Backing, h-1, h-1)
	for i := range backing {
		backing[i] = make([]Backing, w, w)
	}

	var style = getThemeColor("editor")
	var syn = Syntax{}
	for _, s := range syntaxDef {
		for _, f := range s.FileTypes {
			if strings.HasSuffix(buffer.filename, f) {
				syn = s
				break
			}
		}
	}
	for i := range backing {
		if len(lines)-1 < i {
			continue
		}
		jj := 0
		tokens := []Token{}
		key := string(lines[i])
		if val, ok := cache.Get(key); !ok {
			val = highlightLine(lines[i], syn)
			cache.Add(key, val)
			tokens = val.([]Token)
		} else {
			tokens = val.([]Token)
		}
		for _, r := range lines[i] {
			found := false
			for _, token := range tokens {
				if jj >= token.Loc[0] && jj < token.Loc[1] {
					backing[i][jj].style = getThemeColor(token.Scope...)
					found = true
					break
				}
			}

			backing[i][jj].value = r
			if !found {
				backing[i][jj].style = style
			}
			jj++
		}
		for index := len(lines[i]); index < width; index++ {
			backing[i][index].value = ' '
			backing[i][index].style = style
		}
	}
	return backing
}
func drawLines(lines [][]rune) [][]Backing {
	w, h := s.Size()
	backing := make([][]Backing, h-1, h-1)
	for i := range backing {
		backing[i] = make([]Backing, w, w)
	}

	var style = getThemeColor("editor")

	for i := range backing {
		if len(lines)-1 < i {
			continue
		}
		jj := 0
		for _, r := range lines[i] {
			backing[i][jj].value = r
			backing[i][jj].style = style
			jj++
		}
		for index := len(lines[i]); index < width; index++ {
			backing[i][index].value = ' '
			backing[i][index].style = style
		}
	}
	return backing
}
func drawBack(backing [][]Backing) {
	for ir, row := range backing {
		x := offset
		for _, column := range row {
			if column.value == '\t' {
				s.SetContent(x, ir, ' ', nil, column.style)
				s.SetContent(x+1, ir, ' ', nil, column.style)
				s.SetContent(x+2, ir, ' ', nil, column.style)
				s.SetContent(x+3, ir, ' ', nil, column.style)
				x += 4
				continue
			}
			if column.value == '\n' {
				continue
			}
			s.SetContent(x, ir, column.value, nil, column.style)
			x++
		}
	}
}

func drawRuler(topRow, h int, s tcell.Screen) int {
	length := strconv.Itoa(len(fmt.Sprintf("%d", topRow+h)))
	ret := len(fmt.Sprintf("%d", topRow+h))
	style := getThemeColor("editor.ruler")
	for index := 0; index < h; index++ {
		puts(s, style, 0, index, fmt.Sprintf("%"+length+"d ", topRow+index+1))
	}
	return ret + 1
}
<commit_msg>Unused code<commit_after>package main

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/golang/groupcache/lru"
	"github.com/zyedidia/tcell"
	"github.com/zyedidia/tcell/encoding"
)

var topRow = 0
var leftColumn = 0
var height = 0
var width = 0
var offset = 0
var s tcell.Screen

func createDisplay() {
	loadDark()
	os.Setenv("TERM", "xterm-truecolor")
	sc, e := tcell.NewScreen()
	s = sc
	if e != nil {
		fmt.Fprintf(os.Stderr, "%v\n", e)
		os.Exit(1)
	}
	encoding.Register()

	if e = s.Init(); e != nil {
		fmt.Fprintf(os.Stderr, "%v\n", e)
		os.Exit(1)
	}

	s.SetStyle(tcell.StyleDefault.
		Foreground(tcell.ColorWhite).
		Background(tcell.ColorDefault))
	s.Clear()
	s.Show()
	w, h := s.Size()
	width = w
	height = h
}

//Display renders the editor
func Display(buffer *Buffer) {
	createDisplay()
	quit := make(chan struct{})
	b := make(chan [][]Backing)
	evName := ""
	go func() {
		for {
			t := time.Now()
			offset = drawRuler(topRow, height-1, s)
			var w = width - offset + 1
			lines := buffer.GetLines(topRow, leftColumn, height-1, w)
			go drawBufferHighlight(b, topRow, lines)

			puts(s, tcell.StyleDefault.
				Foreground(tcell.ColorWhite).
				Background(tcell.ColorDefault), 0, height-1, "                                                                                ")
			puts(s, tcell.StyleDefault.
				Foreground(tcell.ColorWhite).
				Background(tcell.ColorDefault), 0, height-1, buffer.filename)

			var text = strconv.Itoa(GetCursor().loc.row+1) + ":" + strconv.Itoa(GetCursor().loc.column+1) +
				"/" + strconv.Itoa(buffer.Len()) + ":" + strconv.Itoa(len(buffer.GetLine(GetCursor().loc.row)))

			puts(s, tcell.StyleDefault.
				Foreground(tcell.ColorWhite).
				Background(tcell.ColorDefault), width-len(text), height-1, text)

			puts(s, tcell.StyleDefault.
				Foreground(tcell.ColorWhite).
				Background(tcell.ColorDefault), width-len(text)-2-len(evName), height-1, evName)

			drawBack(<-b)
			time2 := time.Now().Sub(t).String()
			puts(s, tcell.StyleDefault.
				Foreground(tcell.ColorWhite).
				Background(tcell.ColorDefault), len(buffer.filename)+1, height-1, time2)

			s.Show()
			ev := s.PollEvent()
			switch ev := ev.(type) {
			case *tcell.EventKey:
				switch ev.Key() {
				case tcell.KeyDown:
					c := GetCursor()
					c.MoveDown()
				case tcell.KeyUp:
					c := GetCursor()
					c.MoveUp()
				case tcell.KeyLeft:
					c := GetCursor()
					c.MoveLeft()
				case tcell.KeyRight:
					c := GetCursor()
					c.MoveRight()
				case tcell.KeyPgDn:
					c := GetCursor()
					c.MovePageDown()
				case tcell.KeyPgUp:
					c := GetCursor()
					c.MovePageUp()
				case tcell.KeyEscape:
					close(quit)
					return
				case tcell.KeyCtrlD:
					c := GetCursor()
					buffer.Insert(c.loc.row, 0, buffer.GetLine(c.loc.row))
					c.MoveDown()
				case tcell.KeyCtrlY:
					c := GetCursor()
					buffer.RemoveRow(c.loc.row)
				case tcell.KeyDelete:
					c := GetCursor()
					buffer.Delete(c.loc.row, c.loc.column)
				case tcell.KeyBackspace2:
					c := GetCursor()
					l := buffer.Backspace(c.loc.row, c.loc.column)
					if c.loc.column == 0 && c.loc.row > 0 {
						c.MoveUp()
						c.MoveToEndOfLine()
						for index := 0; index < l; index++ {
							c.MoveLeft()
						}
					} else {
						c.MoveLeft()
					}
				case tcell.KeyCtrlS:
					buffer.Save()
				case tcell.KeyTab:
					c := GetCursor()
					buffer.Insert(c.loc.row, c.loc.column, []rune(string('\t')))
					c.MoveRight()
				case tcell.KeyRune:
					evName = ev.Name()
					c := GetCursor()
					buffer.Insert(c.loc.row, c.loc.column, []rune(string(ev.Rune())))
					c.MoveRight()
				case tcell.KeyEnter:
					c := GetCursor()
					buffer.InsertEnter(c.loc.row, c.loc.column)
					c.MoveDown()
					c.MoveStartOfLine()
				default:
					evName = ev.Name()
				}
			case *tcell.EventPaste:
				clip := ev.Text()
				c := GetCursor()
				buffer.Insert(c.loc.row, c.loc.column, []rune(clip))
				for range clip {
					c.MoveRight()
				}

			case *tcell.EventResize:
				s.Sync()
			}
		}
	}()

	<-quit

	s.Fini()
}

// Backing backs the display so we don't overwrite runes unnessecary
type Backing struct {
	value rune
	style tcell.Style
}

var cache = lru.New(1000)

func drawBufferHighlight(b chan [][]Backing, topRow int, lines [][]rune) {
	backing := highlightLines(lines)
	backing = paintCursor(backing, topRow)
	b <- backing
}
func drawBuffer(b chan [][]Backing, topRow int, lines [][]rune) {
	backing := drawLines(lines)
	backing = paintCursor(backing, topRow)
	b <- backing
}
func paintCursor(backing [][]Backing, topRow int) [][]Backing {
	c := GetCursor()
	if len(backing) > c.loc.row-topRow && len(backing[c.loc.row-topRow]) > c.loc.column-leftColumn {
		for i := range backing[c.loc.row-topRow] {
			backing[c.loc.row-topRow][i].style = backing[c.loc.row-topRow][i].style.Background(tcell.GetColor("#282828"))
		}
		// No cursor on enter
		if c.loc.column != -1 {
			backing[c.loc.row-topRow][c.loc.column-leftColumn].style = tcell.StyleDefault.Reverse(true)
		}
	}
	return backing
}
func highlightLines(lines [][]rune) [][]Backing {
	w, h := s.Size()
	backing := make([][]Backing, h-1, h-1)
	for i := range backing {
		backing[i] = make([]Backing, w, w)
	}

	var style = getThemeColor("editor")
	var syn = Syntax{}
	for _, s := range syntaxDef {
		for _, f := range s.FileTypes {
			if strings.HasSuffix(buffer.filename, f) {
				syn = s
				break
			}
		}
	}
	for i := range backing {
		if len(lines)-1 < i {
			continue
		}
		jj := 0
		tokens := []Token{}
		key := string(lines[i])
		if val, ok := cache.Get(key); !ok {
			val = highlightLine(lines[i], syn)
			cache.Add(key, val)
			tokens = val.([]Token)
		} else {
			tokens = val.([]Token)
		}
		for _, r := range lines[i] {
			found := false
			for _, token := range tokens {
				if jj >= token.Loc[0] && jj < token.Loc[1] {
					backing[i][jj].style = getThemeColor(token.Scope...)
					found = true
					break
				}
			}

			backing[i][jj].value = r
			if !found {
				backing[i][jj].style = style
			}
			jj++
		}
		for index := len(lines[i]); index < width; index++ {
			backing[i][index].value = ' '
			backing[i][index].style = style
		}
	}
	return backing
}
func drawLines(lines [][]rune) [][]Backing {
	w, h := s.Size()
	backing := make([][]Backing, h-1, h-1)
	for i := range backing {
		backing[i] = make([]Backing, w, w)
	}

	var style = getThemeColor("editor")

	for i := range backing {
		if len(lines)-1 < i {
			continue
		}
		jj := 0
		for _, r := range lines[i] {
			backing[i][jj].value = r
			backing[i][jj].style = style
			jj++
		}
		for index := len(lines[i]); index < width; index++ {
			backing[i][index].value = ' '
			backing[i][index].style = style
		}
	}
	return backing
}
func drawBack(backing [][]Backing) {
	for ir, row := range backing {
		x := offset
		for _, column := range row {
			if column.value == '\t' {
				s.SetContent(x, ir, ' ', nil, column.style)
				s.SetContent(x+1, ir, ' ', nil, column.style)
				s.SetContent(x+2, ir, ' ', nil, column.style)
				s.SetContent(x+3, ir, ' ', nil, column.style)
				x += 4
				continue
			}
			if column.value == '\n' {
				continue
			}
			s.SetContent(x, ir, column.value, nil, column.style)
			x++
		}
	}
}

func drawRuler(topRow, h int, s tcell.Screen) int {
	length := strconv.Itoa(len(fmt.Sprintf("%d", topRow+h)))
	ret := len(fmt.Sprintf("%d", topRow+h))
	style := getThemeColor("editor.ruler")
	for index := 0; index < h; index++ {
		puts(s, style, 0, index, fmt.Sprintf("%"+length+"d ", topRow+index+1))
	}
	return ret + 1
}
<|endoftext|>


<commit_before>/*
 * Copyright 2014 Xuyuan Pang <xuyuanp # gmail.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package glogger

// LogLevel type
type LogLevel uint8

// AutoRoot is a switcher that controlls GetLogger result
var AutoRoot = true

// LogLevel values
const (
	DebugLevel LogLevel = iota
	InfoLevel
	WarnLevel
	ErrorLevel
	CriticalLevel
)

// LevelToString is a map to translate LogLevel to a level name string
var LevelToString = map[LogLevel]string{
	DebugLevel:    "DBUG",
	InfoLevel:     "INFO",
	WarnLevel:     "WARN",
	ErrorLevel:    "ERRO",
	CriticalLevel: "CRIT",
}

// StringToLevel is a map to translate level name to LogLevel type
var StringToLevel = map[string]LogLevel{
	"DEBUG":    DebugLevel,
	"INFO":     InfoLevel,
	"WARNING":  WarnLevel,
	"ERROR":    ErrorLevel,
	"CRITICAL": CriticalLevel,
}

// Leveler is an interface provided set/get LogLevel method
type Leveler interface {
	Level() LogLevel
	SetLevel(level LogLevel)
}

var loggerRegister = NewRegister()

// GetLogger return a Logger registered with the name,
// or the root Logger if AutoRoot is true and name is not root, or nil
func GetLogger(name string) *Logger {
	if v := loggerRegister.Get(name); v != nil {
		return v.(*Logger)
	}
	if name != "root" {
		return loggerRegister.Get("root").(*Logger)
	}
	return nil
}

// UnregisterLogger unregister the logger from global manager, this will make the logger
// unreachable for others and return the logger, this's the last chance getting it.
// If this Logger hasn't been registered, nothing will happen and return nil
func UnregisterLogger(name string) *Logger {
	if l := loggerRegister.Unregister(name); l != nil {
		return l.(*Logger)
	}
	return nil
}

// RegisterLogger will register the logger to global manager. The logger registered can be
// accessed by GetLogger() method with logger's name.
func RegisterLogger(name string, l *Logger) {
	loggerRegister.Register(name, l)
	l.Name = name
}
<commit_msg>enable autoroot<commit_after>/*
 * Copyright 2014 Xuyuan Pang <xuyuanp # gmail.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package glogger

// LogLevel type
type LogLevel uint8

// AutoRoot is a switcher that controlls GetLogger result
var AutoRoot = true

// LogLevel values
const (
	DebugLevel LogLevel = iota
	InfoLevel
	WarnLevel
	ErrorLevel
	CriticalLevel
)

// LevelToString is a map to translate LogLevel to a level name string
var LevelToString = map[LogLevel]string{
	DebugLevel:    "DBUG",
	InfoLevel:     "INFO",
	WarnLevel:     "WARN",
	ErrorLevel:    "ERRO",
	CriticalLevel: "CRIT",
}

// StringToLevel is a map to translate level name to LogLevel type
var StringToLevel = map[string]LogLevel{
	"DEBUG":    DebugLevel,
	"INFO":     InfoLevel,
	"WARNING":  WarnLevel,
	"ERROR":    ErrorLevel,
	"CRITICAL": CriticalLevel,
}

// Leveler is an interface provided set/get LogLevel method
type Leveler interface {
	Level() LogLevel
	SetLevel(level LogLevel)
}

var loggerRegister = NewRegister()

// GetLogger return a Logger registered with the name,
// or the root Logger if AutoRoot is true and name is not root, or nil
func GetLogger(name string) *Logger {
	if v := loggerRegister.Get(name); v != nil {
		return v.(*Logger)
	}
	if AutoRoot && name != "root" {
		return loggerRegister.Get("root").(*Logger)
	}
	return nil
}

// UnregisterLogger unregister the logger from global manager, this will make the logger
// unreachable for others and return the logger, this's the last chance getting it.
// If this Logger hasn't been registered, nothing will happen and return nil
func UnregisterLogger(name string) *Logger {
	if l := loggerRegister.Unregister(name); l != nil {
		return l.(*Logger)
	}
	return nil
}

// RegisterLogger will register the logger to global manager. The logger registered can be
// accessed by GetLogger() method with logger's name.
func RegisterLogger(name string, l *Logger) {
	loggerRegister.Register(name, l)
	l.Name = name
}
<|endoftext|>


<commit_before>package gnuplot

import (
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	// "os/exec"
)

type Plotter struct {
	configures map[string]string
}

func (p *Plotter) Init() {
	p.configures = map[string]string{}
}

func (p *Plotter) Configure(key, val string) {
	p.configures[key] = val
}

func (p *Plotter) GetC(key string) string {
	return p.configures[key]
}

var DefaultFunction2dSplitNum int = 1000

type Function2d struct {
	plotter  Plotter
	splitNum int
	f        func(float64) float64
}

func (fun *Function2d) Init() {
	fun.splitNum = DefaultFunction2dSplitNum
	fun.plotter.configures = map[string]string{
		"_xMin": "-10.0",
		"_xMax": "10.0"}
}

func (fun *Function2d) UpdatePlotter(plotter *Plotter) {
	for key, val := range plotter.configures {
		fun.plotter.configures[key] = val
	}
}

func (fun *Function2d) GetData() [][2]float64 { // TODO: テスト書く
	xMin, _ := strconv.ParseFloat(fun.plotter.configures["_xMin"], 32)
	xMax, _ := strconv.ParseFloat(fun.plotter.configures["_xMax"], 32)
	var sep = float64(xMax-xMin) / float64(fun.splitNum-1)

	var a [][2]float64
	for j := 0; j < fun.splitNum; j++ {
		t := xMin + float64(j)*sep
		y := fun.f(t)
		a = append(a, [2]float64{t, y})
	}
	return a
}

func (fun *Function2d) getGnuData() string {
	var s string
	for _, xs := range fun.GetData() {
		s += fmt.Sprintf("%f %f\n", xs[0], xs[1])
	}
	return s
}

func (fun *Function2d) SetF(_f func(float64) float64) {
	fun.f = _f
}

func (fun Function2d) gnuplot(filename string) string {
	var s = fmt.Sprintf("plot \"%v\"", filename)
	for key, val := range fun.plotter.configures {
		if !strings.HasPrefix(key, "_") {
			s += fmt.Sprintf(" %v %v", key, val)
		}
	}
	return s + ";\n"
}

func (fun *Function2d) writeIntoGnufile(f os.File) {
	f.WriteString(fun.getGnuData())
}

var DefaultCurve2dSplitNum int = 100

type Curve2d struct {
	plotter  Plotter
	splitNum int
	c        func(float64) [2]float64
}

func (c *Curve2d) Init() {
	c.splitNum = DefaultCurve2dSplitNum
	c.plotter.configures = map[string]string{
		"_tMin": "-10.0",
		"_tMax": "10.0"}
}

func (c *Curve2d) UpdatePlotter(plotter *Plotter) {
	for key, val := range plotter.configures {
		c.plotter.Configure(key, val)
	}
}

func (c *Curve2d) GetData() [][2]float64 { // TODO: test
	tMin, _ := strconv.ParseFloat(c.plotter.configures["_tMin"], 32)
	tMax, _ := strconv.ParseFloat(c.plotter.configures["_tMax"], 32)
	var sep = float64(tMax-tMin) / float64(c.splitNum-1)

	var a [][2]float64
	for j := 0; j < c.splitNum; j++ {
		var t float64 = tMin + float64(j)*sep
		cs := c.c(tMin + t*float64(j))
		a = append(a, [2]float64{cs[0], cs[1]})
	}
	return a
}

func (c Curve2d) gnuplot(fileName string) string {
	return fmt.Sprintf("plot %v\n;", fileName)
}

// Graph
type Graph2d struct {
	plotter   Plotter
	functions []Function2d
	curves    []Curve2d
}

func (g *Graph2d) AppendFunc(f Function2d) {
	fmt.Println("before of AppendFunc")
	g.functions = append(g.functions, f)
	fmt.Println("after of AppendFunc")
}

func (g Graph2d) writeIntoFile(data string, f *os.File) {
	f.WriteString(data)
}

func (g Graph2d) exec_gnuplot() {
	// until
}

func (g Graph2d) gnuplot(funcFilenames []string, curveFilenames []string) string {
	var s string
	for j, _ := range g.functions {
		s += g.functions[j].gnuplot(funcFilenames[j])
	}
	s += "\n"
	for j, _ := range g.curves {
		s += g.curves[j].gnuplot(curveFilenames[j])
	}
	s += "pause -1;"
	s += "\n"
	return s
}

func (g *Graph2d) Run() {
	tmpDir := os.TempDir()

	// それぞれのfunctionのdataをtempファイルに書き込む
	// また, それらのファイルの名前を func_filenames []string に格納する
	var funcFilenames []string
	for _, fun := range g.functions {
		file, _ := ioutil.TempFile(tmpDir, "")
		defer func() {
			file.Close()
		}()
		g.writeIntoFile(fun.getGnuData(), file)
		funcFilenames = append(funcFilenames, file.Name())
	}

	// それぞれのcurveのdataをtempファイルに書き込む
	// また, それらのファイルの名前を curve_filenames []stringに格納する

	// 実行するgnuplotの実行ファイルをtempファイルに書き込む
	execFile, _ := os.OpenFile("exec_gnu.gnu", os.O_CREATE|os.O_WRONLY, 0666)
	defer func() {
		execFile.Close()
	}()
	fmt.Println(funcFilenames)
	execFile.WriteString(g.gnuplot(funcFilenames, []string{}))
}
<commit_msg>Curve2d.configure<commit_after>package gnuplot

import (
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	// "os/exec"
)

type Plotter struct {
	configures map[string]string
}

func (p *Plotter) Init() {
	p.configures = map[string]string{}
}

func (p *Plotter) Configure(key, val string) {
	p.configures[key] = val
}

func (p *Plotter) GetC(key string) string {
	return p.configures[key]
}

var DefaultFunction2dSplitNum int = 1000

type Function2d struct {
	plotter  Plotter
	splitNum int
	f        func(float64) float64
}

func (fun *Function2d) Init() {
	fun.splitNum = DefaultFunction2dSplitNum
	fun.plotter.configures = map[string]string{
		"_xMin": "-10.0",
		"_xMax": "10.0"}
}

func (fun *Function2d) UpdatePlotter(plotter *Plotter) {
	for key, val := range plotter.configures {
		fun.plotter.configures[key] = val
	}
}

func (fun *Function2d) GetData() [][2]float64 { // TODO: テスト書く
	xMin, _ := strconv.ParseFloat(fun.plotter.configures["_xMin"], 32)
	xMax, _ := strconv.ParseFloat(fun.plotter.configures["_xMax"], 32)
	var sep = float64(xMax-xMin) / float64(fun.splitNum-1)

	var a [][2]float64
	for j := 0; j < fun.splitNum; j++ {
		t := xMin + float64(j)*sep
		y := fun.f(t)
		a = append(a, [2]float64{t, y})
	}
	return a
}

func (fun *Function2d) getGnuData() string {
	var s string
	for _, xs := range fun.GetData() {
		s += fmt.Sprintf("%f %f\n", xs[0], xs[1])
	}
	return s
}

func (fun *Function2d) SetF(_f func(float64) float64) {
	fun.f = _f
}

func (fun Function2d) gnuplot(filename string) string {
	var s = fmt.Sprintf("plot \"%v\"", filename)
	for key, val := range fun.plotter.configures {
		if !strings.HasPrefix(key, "_") {
			s += fmt.Sprintf(" %v %v", key, val)
		}
	}
	return s + ";\n"
}

func (fun *Function2d) writeIntoGnufile(f os.File) {
	f.WriteString(fun.getGnuData())
}

var DefaultCurve2dSplitNum int = 100

type Curve2d struct {
	plotter  Plotter
	splitNum int
	c        func(float64) [2]float64
}

func (c *Curve2d) Init() {
	c.splitNum = DefaultCurve2dSplitNum
	c.plotter.configures = map[string]string{
		"_tMin": "-10.0",
		"_tMax": "10.0"}
}

func (c *Curve2d) UpdatePlotter(plotter *Plotter) {
	for key, val := range plotter.configures {
		c.plotter.Configure(key, val)
	}
}

func (c *Curve2d) GetData() [][2]float64 { // TODO: test
	tMin, _ := strconv.ParseFloat(c.plotter.configures["_tMin"], 32)
	tMax, _ := strconv.ParseFloat(c.plotter.configures["_tMax"], 32)
	var sep = float64(tMax-tMin) / float64(c.splitNum-1)

	var a [][2]float64
	for j := 0; j < c.splitNum; j++ {
		var t float64 = tMin + float64(j)*sep
		cs := c.c(tMin + t*float64(j))
		a = append(a, [2]float64{cs[0], cs[1]})
	}
	return a
}

func (c Curve2d) gnuplot(fileName string) string {
	var s = fmt.Sprintf("plot %v\n;", fileName)
	for key, val := range fun.plotter.configures {
		if !strings.HasPrefix(key, "_") {
			s += fmt.Sprintf(" %v %v", key, val)
		}
	}
	return s + ";\n"
}

// Graph
type Graph2d struct {
	plotter   Plotter
	functions []Function2d
	curves    []Curve2d
}

func (g *Graph2d) AppendFunc(f Function2d) {
	fmt.Println("before of AppendFunc")
	g.functions = append(g.functions, f)
	fmt.Println("after of AppendFunc")
}

func (g Graph2d) writeIntoFile(data string, f *os.File) {
	f.WriteString(data)
}

func (g Graph2d) exec_gnuplot() {
	// until
}

func (g Graph2d) gnuplot(funcFilenames []string, curveFilenames []string) string {
	var s string
	for j, _ := range g.functions {
		s += g.functions[j].gnuplot(funcFilenames[j])
	}
	s += "\n"
	for j, _ := range g.curves {
		s += g.curves[j].gnuplot(curveFilenames[j])
	}
	s += "pause -1;"
	s += "\n"
	return s
}

func (g *Graph2d) Run() {
	tmpDir := os.TempDir()

	// それぞれのfunctionのdataをtempファイルに書き込む
	// また, それらのファイルの名前を func_filenames []string に格納する
	var funcFilenames []string
	for _, fun := range g.functions {
		file, _ := ioutil.TempFile(tmpDir, "")
		defer func() {
			file.Close()
		}()
		g.writeIntoFile(fun.getGnuData(), file)
		funcFilenames = append(funcFilenames, file.Name())
	}

	// それぞれのcurveのdataをtempファイルに書き込む
	// また, それらのファイルの名前を curve_filenames []stringに格納する

	// 実行するgnuplotの実行ファイルをtempファイルに書き込む
	execFile, _ := os.OpenFile("exec_gnu.gnu", os.O_CREATE|os.O_WRONLY, 0666)
	defer func() {
		execFile.Close()
	}()
	fmt.Println(funcFilenames)
	execFile.WriteString(g.gnuplot(funcFilenames, []string{}))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"strings"
)

const (
	C_TAB       = "\t"
	C_SPACE     = " "
	C_SEMICOLON = ";"
)

func main() {

	var (
		// for flag parsing
		//fromDelimit string
		//toDelimit   string
		// for conversion parsing
		tFromDelimit string
		tToDelimit   string
	)

	filenamePtr := flag.String("f", "", "Filename to convert the delimiter")
	verbosePtr := flag.Bool("v", false, "Verbose output")
	fromdelimitPtr := flag.String("from", "TAB", "From delimiter (TAB,SEMICOLON,SPACE)")
	todelimitPtr := flag.String("to", "SEMICOLON", "To delimiter (TAB,SEMICOLON,SPACE)")

	// Parsing flags
	flag.Parse()

	read, err := ioutil.ReadFile(*filenamePtr)
	if err != nil {
		panic(err)
	}

	if *verbosePtr {
		fmt.Println("From:")
		fmt.Println(string(read))
	}

	switch strings.ToUpper(*fromdelimitPtr) {
	default:
		tFromDelimit = C_TAB
	case "SEMICOLON":
		tFromDelimit = C_SEMICOLON
	case "SPACE":
		tFromDelimit = C_SPACE
	}

	switch strings.ToUpper(*todelimitPtr) {
	default:
		tToDelimit = C_SEMICOLON
	case "TAB":
		tToDelimit = C_TAB
	case "SPACE":
		tToDelimit = C_SPACE
	}

	newContent := strings.Replace(string(read), tFromDelimit, tToDelimit, -1)

	if *verbosePtr {
		fmt.Println("\nTo:")
		fmt.Println(newContent)
	}

	err = ioutil.WriteFile(*filenamePtr, []byte(newContent), 0)
	if err != nil {
		panic(err)
	}
}
<commit_msg>- add more option for conversion<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"strings"
)

const (
	C_TAB       = "\t"
	C_SPACE     = " "
	C_SEMICOLON = ";"
	C_COMMA     = ","
	C_PERIOD    = "."
)

func main() {

	var (
		// for flag parsing
		//fromDelimit string
		//toDelimit   string
		// for conversion parsing
		tFromDelimit string
		tToDelimit   string
	)

	filenamePtr := flag.String("f", "", "Filename to convert the delimiter")
	verbosePtr := flag.Bool("v", false, "Verbose output")
	fromdelimitPtr := flag.String("from", "TAB", "From delimiter (TAB,SEMICOLON,SPACE)")
	todelimitPtr := flag.String("to", "SEMICOLON", "To delimiter (TAB,SEMICOLON,SPACE)")

	// Parsing flags
	flag.Parse()

	read, err := ioutil.ReadFile(*filenamePtr)
	if err != nil {
		panic(err)
	}

	if *verbosePtr {
		fmt.Println("From:")
		fmt.Println(string(read))
	}

	switch strings.ToUpper(*fromdelimitPtr) {
	default:
		tFromDelimit = C_TAB
	case "SEMICOLON":
		tFromDelimit = C_SEMICOLON
	case "SPACE":
		tFromDelimit = C_SPACE
	case "COMMA":
		tFromDelimit = C_COMMA
	case "PERIOD":
		tFromDelimit = C_PERIOD
	}

	switch strings.ToUpper(*todelimitPtr) {
	default:
		tToDelimit = C_SEMICOLON
	case "TAB":
		tToDelimit = C_TAB
	case "SPACE":
		tToDelimit = C_SPACE
	case "COMMA":
		tToDelimit = C_COMMA
	case "PERIOD":
		tToDelimit = C_PERIOD
	}

	newContent := strings.Replace(string(read), tFromDelimit, tToDelimit, -1)

	if *verbosePtr {
		fmt.Println("\nTo:")
		fmt.Println(newContent)
	}

	err = ioutil.WriteFile(*filenamePtr, []byte(newContent), 0)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// 一个简单的Go语言热编译工具
package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/issue9/term/colors"
	"gopkg.in/fsnotify.v1"
)

// 当前程序的版本号
const version = "0.1.1.150605"

const usage = `gobuild 用于热编译Go程序。

用法:
 gobuild [options] [dependents]

 options:
  -h 显示当前帮助信息；
  -v 显示gobuild和go程序的版本信息；
  -o 执行编译后的可执行文件名。
  -main 指定需要编译的文件，默认为""。

 dependents
  指定其它依赖的目录，只能出现在命令的尾部。

常见用法:

 - gobuild
   监视当前目录，若有变动，则重新编译当前目录下的*.go文件；

 - gobuild -main=main.go
   监视当前目录，若有变动，则重新编译当前目录下的main.go文件；

 - gobuild dir1 dir2
   监视当前目录及dir1和dir2，若有变动，则重新编译当前目录下的*.go文件；
`

var (
	showHelp    = false
	showVersion = false
	mainFiles   = ""
	outputName  = ""

	watcher *fsnotify.Watcher

	wd string // 当前工作目录

	// outputName的命令
	cmd *exec.Cmd
)

func init() {
	// 基本环境检测
	gopath := os.Getenv("GOPATH")
	if len(gopath) == 0 {
		log(erro, "未设置环境变量GOPATH")
		os.Exit(2)
	}

	// 获取所有被监视的路径
	var err error
	wd, err = os.Getwd()
	if err != nil {
		log(erro, "获取当前工作目录时，发生以下错误:", err)
		os.Exit(2)
	}

	// 初始化flag相关设置
	flag.BoolVar(&showHelp, "h", false, "显示帮助信息")
	flag.BoolVar(&showVersion, "v", false, "显示版本号")
	flag.StringVar(&outputName, "o", "", "指定输出名称")
	flag.StringVar(&mainFiles, "main", "", "指定需要编译的文件")

	flag.Usage = func() {
		fmt.Println(usage)
	}

	// 初始化监视器
	if watcher, err = fsnotify.NewWatcher(); err != nil {
		log(erro, err)
	}
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				log(info, "watcher.Events:", event)
				if event.Name == outputName {
					continue
				}

				autoBuild()
			case err := <-watcher.Errors:
				log(erro, "watcher.Errors", err)
			}
		}
	}()
}

func main() {
	flag.Parse()

	if showHelp {
		flag.Usage()
		return
	}

	if showVersion {
		colors.Print(colors.Stdout, colors.Green, colors.Default, "gobuild: ")
		colors.Println(colors.Stdout, colors.Default, colors.Default, version)

		colors.Print(colors.Stdout, colors.Green, colors.Default, "Go: ")
		goVersion := runtime.Version() + " " + runtime.GOOS + "/" + runtime.GOARCH
		colors.Println(colors.Stdout, colors.Default, colors.Default, goVersion)

		return
	}

	// 确定编译后的文件名
	if len(outputName) == 0 {
		outputName = filepath.Base(wd)
	}
	if strings.IndexByte(outputName, '/') < 0 || strings.IndexByte(outputName, filepath.Separator) < 0 {
		outputName = wd + string(filepath.Separator) + outputName
	}
	if runtime.GOOS == "windows" {
		outputName += ".exe"
	}
	cmd = exec.Command(outputName)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout

	// 监视的路径，必定包含当前工作目录
	paths := append(flag.Args(), wd)
	log(info, "初始化监视器...")
	log(info, "以下路径或是文件将被监视:", paths)
	for _, path := range paths {
		if err := watcher.Add(path); err != nil {
			log(erro, "watcher.Add:", err)
			os.Exit(2)
		}
	}

	autoBuild()
}

func autoBuild() {
	log(info, "编译代码...")

	args := []string{"build", "-o", outputName}
	if len(mainFiles) > 0 {
		args = append(args, mainFiles)
	}

	goCmd := exec.Command("go", args...)
	goCmd.Stderr = os.Stderr
	goCmd.Stdout = os.Stdout

	if err := goCmd.Run(); err != nil {
		log(erro, "编译失败:", err)
		return
	}

	log(succ, "编译成功!")

	restart()
}

func kill() {
	defer func() {
		if err := recover(); err != nil {
			log(erro, "kill.defer:", err)
		}
	}()

	if cmd != nil && cmd.Process != nil {
		log(info, "中止旧进程...")
		if err := cmd.Process.Kill(); err != nil {
			log(erro, "kill:", err)
		}
	}
}

func restart() {
	kill()

	if err := cmd.Run(); err != nil {
		log(erro, "启动进程时出错:", err)
	}
}
<commit_msg>添加扩展名支持<commit_after>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// 一个简单的Go语言热编译工具
package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/issue9/term/colors"
	"gopkg.in/fsnotify.v1"
)

// 当前程序的版本号
const version = "0.1.1.150606"

const usage = `gobuild 用于热编译Go程序。

用法:
 gobuild [options] [dependents]

 options:
  -h 显示当前帮助信息；
  -v 显示gobuild和go程序的版本信息；
  -o 执行编译后的可执行文件名；
  -ext 监视的扩展名，默认值为"go"；
  -main 指定需要编译的文件，默认为""。

 dependents
  指定其它依赖的目录，只能出现在命令的尾部。

常见用法:

 - gobuild
   监视当前目录，若有变动，则重新编译当前目录下的*.go文件；

 - gobuild -main=main.go
   监视当前目录，若有变动，则重新编译当前目录下的main.go文件；

 - gobuild dir1 dir2
   监视当前目录及dir1和dir2，若有变动，则重新编译当前目录下的*.go文件；
`

var (
	showHelp    = false
	showVersion = false
	mainFiles   = ""
	outputName  = ""
	extString   = "go"
	exts        []string

	watcher *fsnotify.Watcher // 监视器
	wd      string            // 当前工作目录
	cmd     *exec.Cmd         // outputName的命令
)

func init() {
	// 基本环境检测
	gopath := os.Getenv("GOPATH")
	if len(gopath) == 0 {
		log(erro, "未设置环境变量GOPATH")
		os.Exit(2)
	}

	// 获取所有被监视的路径
	var err error
	wd, err = os.Getwd()
	if err != nil {
		log(erro, "获取当前工作目录时，发生以下错误:", err)
		os.Exit(2)
	}

	// 初始化flag相关设置
	flag.BoolVar(&showHelp, "h", false, "显示帮助信息")
	flag.BoolVar(&showVersion, "v", false, "显示版本号")
	flag.StringVar(&outputName, "o", "", "指定输出名称")
	flag.StringVar(&extString, "ext", "go", "指定监视的文件扩展名")
	flag.StringVar(&mainFiles, "main", "", "指定需要编译的文件")

	flag.Usage = func() {
		fmt.Println(usage)
	}

	// 初始化监视器
	if watcher, err = fsnotify.NewWatcher(); err != nil {
		log(erro, err)
	}
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if event.Name == outputName { // 过滤程序本身
					continue
				}

				if !isEnabled(event.Name) { // 不需要监视的扩展名
					continue
				}

				log(info, "watcher.Events:", event)
				autoBuild()
			case err := <-watcher.Errors:
				log(erro, "watcher.Errors", err)
			}
		}
	}()
}

func main() {
	flag.Parse()

	if showHelp {
		flag.Usage()
		return
	}

	if showVersion {
		colors.Print(colors.Stdout, colors.Green, colors.Default, "gobuild: ")
		colors.Println(colors.Stdout, colors.Default, colors.Default, version)

		colors.Print(colors.Stdout, colors.Green, colors.Default, "Go: ")
		goVersion := runtime.Version() + " " + runtime.GOOS + "/" + runtime.GOARCH
		colors.Println(colors.Stdout, colors.Default, colors.Default, goVersion)

		return
	}

	if len(extString) > 0 {
		exts = strings.Split(extString, ",")
	}

	// 确定编译后的文件名
	if len(outputName) == 0 {
		outputName = filepath.Base(wd)
	}
	if strings.IndexByte(outputName, '/') < 0 || strings.IndexByte(outputName, filepath.Separator) < 0 {
		outputName = wd + string(filepath.Separator) + outputName
	}

	cmd = exec.Command(outputName)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout

	// 监视的路径，必定包含当前工作目录
	paths := append(flag.Args(), wd)
	log(info, "初始化监视器...")
	log(info, "以下路径或是文件将被监视:", paths)
	for _, path := range paths {
		if err := watcher.Add(path); err != nil {
			log(erro, "watcher.Add:", err)
			os.Exit(2)
		}
	}

	autoBuild()

	done := make(chan bool)
	<-done
}

func autoBuild() {
	log(info, "编译代码...")

	args := []string{"build", "-o", outputName}
	if len(mainFiles) > 0 {
		args = append(args, mainFiles)
	}

	goCmd := exec.Command("go", args...)
	goCmd.Stderr = os.Stderr
	goCmd.Stdout = os.Stdout

	if err := goCmd.Run(); err != nil {
		log(erro, "编译失败:", err)
		return
	}

	log(succ, "编译成功!")

	restart()
}

func kill() {
	defer func() {
		if err := recover(); err != nil {
			log(erro, "kill.defer:", err)
		}
	}()

	if cmd != nil && cmd.Process != nil {
		log(info, "中止旧进程...")
		if err := cmd.Process.Kill(); err != nil {
			log(erro, "kill:", err)
		}
	}
}

func restart() {
	kill()

	if err := cmd.Run(); err != nil {
		log(erro, "启动进程时出错:", err)
	}
}

func isEnabled(path string) bool {
	for _, ext := range exts {
		if strings.HasSuffix(path, ext) {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>// Package api is to start the api to be listening on different endpoints
// The API will listen on the port specified in the PulseConfig.toml.
// There are 2 endpoints:
// POST /log/file this will read the file line by line passing in each line to the algorithm
// POST /log/message (in development) this will take a string and pass it directly to the algorithm
package api

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/davecgh/go-spew/spew"

	"github.com/gophergala2016/Pulse/LogPulse/config"
	"github.com/gophergala2016/Pulse/LogPulse/email"
	"github.com/gophergala2016/Pulse/LogPulse/file"
	"github.com/gophergala2016/Pulse/pulse"
)

// Result is used for ResponseWriter in handlers
type Result struct {
	Status  int    `json:"status"`
	Message string `json:"message"`
}

var buffStrings []string
var port int

func init() {
	defer func() {
		if r := recover(); r != nil {
			fmt.Println(r)
			os.Exit(0)
		}
	}()
	val, err := config.Load()
	if err != nil {
		panic(fmt.Errorf("API: %s", err))
	}
	port = val.Port
}

// Start will run the REST API.
func Start() {
	http.HandleFunc("/", HelloWorld)
	http.HandleFunc("/log/message", StreamLog)
	http.HandleFunc("/log/file", SendFile)

	fmt.Printf("Listening on localhost:%d\n", port)
	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)

}

// HelloWorld ... testdummy handler for ec2 instance
func HelloWorld(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	result, _ := json.Marshal(Result{200, "hello world"})
	io.WriteString(w, string(result))
}

// StreamLog listens for post request for a string value.
// This string is then passed to the algorithm for analyzing.
func StreamLog(w http.ResponseWriter, r *http.Request) {

	// Checking to see if the request was a post.
	// If not return a 400: bad request
	if r.Method != "POST" {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{400, "bad request"})
		io.WriteString(w, string(result))
		return
	}

	// Decoding the body of the response.
	// If we could not parse it as json then respond with a 400: bad request
	decoder := json.NewDecoder(r.Body)
	var body struct {
		Message string `json:"message"`
	}
	err := decoder.Decode(&body)
	if err != nil {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{400, "bad request"})
		io.WriteString(w, string(result))
		return
	}

	//TODO: post the string to the algorthm for analyzing

	// If we were able to decode and send string to algorithm return a 200: success
	w.Header().Set("Content-Type", "application/json")
	result, _ := json.Marshal(Result{200, "success"})
	io.WriteString(w, string(result))

}

// SendFile listens for a POST that has a form field named file and email in the body.
// Using the file field we will download the specified file to the server.
// The email field is used to email the user the results once algorithm is done.
func SendFile(w http.ResponseWriter, r *http.Request) {

	// Checking to see if the request was a post.
	// If not return a 400: bad request
	if r.Method != "POST" {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{400, "bad request"})
		io.WriteString(w, string(result))
		return
	}
	compressed := false
	// Get the file field from the form in the response.
	// If we cannot parse it a 400 bad request is returned
	f, header, err := r.FormFile("file")
	fmt.Println("Form File")
	if err != nil {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{400, "bad request"})
		io.WriteString(w, string(result))
		return
	}

	defer f.Close()

	var body struct {
		Email string `json:"email"`
	}
	// Parse the Form in the response.
	// Check if the email field is a valid email.
	// If not return an 400: bad request
	r.ParseForm()
	body.Email = r.Form["email"][0]
	if !email.IsValid(body.Email) {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{400, "email not valid"})
		io.WriteString(w, string(result))
		return
	}
	if err != nil {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{400, "bad request"})
		io.WriteString(w, string(result))
		return
	}
	fmt.Println("Received bodys", body.Email)
	extension := filepath.Ext(header.Filename)
	filename := header.Filename[0 : len(header.Filename)-len(extension)]

	if extension == ".gz" {
		// Load compressed file on disk
		out, err := os.Create(fmt.Sprintf("%s.gz", filename))
		if err != nil {
			w.Header().Set("Content-Type", "application/json")
			result, _ := json.Marshal(Result{400, "bad request"})
			io.WriteString(w, string(result))
			return
		}

		defer out.Close()

		// Write the content from POST to the file
		_, err = io.Copy(out, f)
		if err != nil {
			w.Header().Set("Content-Type", "application/json")
			result, _ := json.Marshal(Result{400, "gzip copy failed"})
			io.WriteString(w, string(result))
			return
		}

		// Uncompress file
		err = file.UnGZip(fmt.Sprintf("%s.gz", filename))
		if err != nil {
			spew.Dump(err)
			w.Header().Set("Content-Type", "application/json")
			result, _ := json.Marshal(Result{400, "gzip uncompressed failed"})
			io.WriteString(w, string(result))
			return
		}

		compressed = true
	}

	stdIn := make(chan string)
	email.ByPassMail = true // Needs to bypass emails and store in JSON
	email.OutputFile = fmt.Sprintf("%s-%s.json", filename, body.Email)
	email.EmailList = []string{body.Email}

	if _, err := os.Stat(email.OutputFile); err == nil {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{406, "file is being processed"})
		io.WriteString(w, string(result))
		return
	}

	// Run on separat go routine so that we can give users a response on page first.
	go func() {
		start := time.Now()
		// Start the pulse algorithm
		pulse.Run(stdIn, email.SaveToCache)
		line := make(chan string)

		if compressed {
			file.Read(filename, line)
		} else {
			file.StreamRead(f, line)
		}

		for l := range line {
			if l == "EOF" {
				email.ByPassMail = false
				// Once EOF, time to send email from cache JSON storage
				email.DumpBuffer() // Must clear buffer first before sending
				email.SendFromCache(email.OutputFile)
				break
			}
			stdIn <- l
		}
		close(stdIn)

		elapsed := time.Since(start)
		log.Printf("Pulse Algorithm took %s", elapsed)

		// Clean up
		defer func() {
			if compressed {
				err := os.Remove(filename)
				if err != nil {
					fmt.Println("Failed to delete uncompressed file, please delete")
				}

				err = os.Remove(fmt.Sprintf("%s.gz", filename))
				if err != nil {
					fmt.Println("Failed to delete uncompressed file, please delete")
				}
			}

			err = os.Remove(email.OutputFile)
			if err != nil {
				fmt.Println("Failed to delete output file, please delete")
			}
		}()
	}()

	// Return a 200 success even if algorithm is still going.
	w.Header().Set("Content-Type", "application/json")
	result, _ := json.Marshal(Result{200, "success"})
	io.WriteString(w, string(result))
}
<commit_msg>api updated: can't request again only after complete<commit_after>// Package api is to start the api to be listening on different endpoints
// The API will listen on the port specified in the PulseConfig.toml.
// There are 2 endpoints:
// POST /log/file this will read the file line by line passing in each line to the algorithm
// POST /log/message (in development) this will take a string and pass it directly to the algorithm
package api

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/davecgh/go-spew/spew"

	"github.com/gophergala2016/Pulse/LogPulse/config"
	"github.com/gophergala2016/Pulse/LogPulse/email"
	"github.com/gophergala2016/Pulse/LogPulse/file"
	"github.com/gophergala2016/Pulse/pulse"
)

// Result is used for ResponseWriter in handlers
type Result struct {
	Status  int    `json:"status"`
	Message string `json:"message"`
}

var buffStrings []string
var port int

func init() {
	defer func() {
		if r := recover(); r != nil {
			fmt.Println(r)
			os.Exit(0)
		}
	}()
	val, err := config.Load()
	if err != nil {
		panic(fmt.Errorf("API: %s", err))
	}
	port = val.Port
}

// Start will run the REST API.
func Start() {
	http.HandleFunc("/", HelloWorld)
	http.HandleFunc("/log/message", StreamLog)
	http.HandleFunc("/log/file", SendFile)

	fmt.Printf("Listening on localhost:%d\n", port)
	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)

}

// HelloWorld ... testdummy handler for ec2 instance
func HelloWorld(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	result, _ := json.Marshal(Result{200, "hello world"})
	io.WriteString(w, string(result))
}

// StreamLog listens for post request for a string value.
// This string is then passed to the algorithm for analyzing.
func StreamLog(w http.ResponseWriter, r *http.Request) {

	// Checking to see if the request was a post.
	// If not return a 400: bad request
	if r.Method != "POST" {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{400, "bad request"})
		io.WriteString(w, string(result))
		return
	}

	// Decoding the body of the response.
	// If we could not parse it as json then respond with a 400: bad request
	decoder := json.NewDecoder(r.Body)
	var body struct {
		Message string `json:"message"`
	}
	err := decoder.Decode(&body)
	if err != nil {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{400, "bad request"})
		io.WriteString(w, string(result))
		return
	}

	//TODO: post the string to the algorthm for analyzing

	// If we were able to decode and send string to algorithm return a 200: success
	w.Header().Set("Content-Type", "application/json")
	result, _ := json.Marshal(Result{200, "success"})
	io.WriteString(w, string(result))

}

// SendFile listens for a POST that has a form field named file and email in the body.
// Using the file field we will download the specified file to the server.
// The email field is used to email the user the results once algorithm is done.
func SendFile(w http.ResponseWriter, r *http.Request) {

	// Checking to see if the request was a post.
	// If not return a 400: bad request
	if r.Method != "POST" {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{400, "bad request"})
		io.WriteString(w, string(result))
		return
	}
	compressed := false
	// Get the file field from the form in the response.
	// If we cannot parse it a 400 bad request is returned
	f, header, err := r.FormFile("file")
	fmt.Println("Form File")
	if err != nil {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{400, "bad request"})
		io.WriteString(w, string(result))
		return
	}

	defer f.Close()

	var body struct {
		Email string `json:"email"`
	}
	// Parse the Form in the response.
	// Check if the email field is a valid email.
	// If not return an 400: bad request
	r.ParseForm()
	body.Email = r.Form["email"][0]
	if !email.IsValid(body.Email) {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{400, "email not valid"})
		io.WriteString(w, string(result))
		return
	}
	if err != nil {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{400, "bad request"})
		io.WriteString(w, string(result))
		return
	}
	fmt.Println("Received bodys", body.Email)
	extension := filepath.Ext(header.Filename)
	filename := header.Filename[0 : len(header.Filename)-len(extension)]

	stdIn := make(chan string)
	email.ByPassMail = true // Needs to bypass emails and store in JSON
	email.OutputFile = fmt.Sprintf("%s-%s.json", filename, body.Email)
	email.EmailList = []string{body.Email}

	if _, err := os.Stat(email.OutputFile); err == nil {
		w.Header().Set("Content-Type", "application/json")
		result, _ := json.Marshal(Result{406, "file is being processed"})
		io.WriteString(w, string(result))
		return
	}

	if extension == ".gz" {
		// Load compressed file on disk
		out, err := os.Create(fmt.Sprintf("%s.gz", filename))
		if err != nil {
			w.Header().Set("Content-Type", "application/json")
			result, _ := json.Marshal(Result{400, "bad request"})
			io.WriteString(w, string(result))
			return
		}

		defer out.Close()

		// Write the content from POST to the file
		_, err = io.Copy(out, f)
		if err != nil {
			w.Header().Set("Content-Type", "application/json")
			result, _ := json.Marshal(Result{400, "gzip copy failed"})
			io.WriteString(w, string(result))
			return
		}

		// Uncompress file
		err = file.UnGZip(fmt.Sprintf("%s.gz", filename))
		if err != nil {
			spew.Dump(err)
			w.Header().Set("Content-Type", "application/json")
			result, _ := json.Marshal(Result{400, "gzip uncompressed failed"})
			io.WriteString(w, string(result))
			return
		}

		compressed = true
	}

	// Run on separat go routine so that we can give users a response on page first.
	go func() {
		start := time.Now()
		// Start the pulse algorithm
		pulse.Run(stdIn, email.SaveToCache)
		line := make(chan string)

		if compressed {
			file.Read(filename, line)
		} else {
			file.StreamRead(f, line)
		}

		for l := range line {
			if l == "EOF" {
				email.ByPassMail = false
				// Once EOF, time to send email from cache JSON storage
				email.DumpBuffer() // Must clear buffer first before sending
				email.SendFromCache(email.OutputFile)
				break
			}
			stdIn <- l
		}
		close(stdIn)

		elapsed := time.Since(start)
		log.Printf("Pulse Algorithm took %s", elapsed)

		// Clean up
		defer func() {
			if compressed {
				err := os.Remove(filename)
				if err != nil {
					fmt.Println("Failed to delete uncompressed file, please delete")
				}

				err = os.Remove(fmt.Sprintf("%s.gz", filename))
				if err != nil {
					fmt.Println("Failed to delete uncompressed file, please delete")
				}
			}

			err = os.Remove(email.OutputFile)
			if err != nil {
				fmt.Println("Failed to delete output file, please delete")
			}
		}()
	}()

	// Return a 200 success even if algorithm is still going.
	w.Header().Set("Content-Type", "application/json")
	result, _ := json.Marshal(Result{200, "success"})
	io.WriteString(w, string(result))
}
<|endoftext|>


<commit_before>package gordian

import (
	"code.google.com/p/go.net/websocket"

	"fmt"
	"io"
)

// Control types.
const (
	CONNECT = iota
	REGISTER
	CLOSE
)

// ClientId is a user-defined client identifier, which can be of any hashable type.
type ClientId interface{}

// MessageData is a user-defined message payload.
type MessageData interface{}

// Message is the internal message format
type Message struct {
	From ClientId    // From is the originating client.
	To   ClientId    // To is the destination client, if any.
	Data MessageData // Data is the message payload.
}

// Client stores state and control information for a client.
type Client struct {
	Id      ClientId        // Id is a unique identifier.
	Ctrl    int             // Ctrl is the current control type.
	Conn    *websocket.Conn // Conn is the connection info provided by the websocket package.
	message chan *Message
}

// Gordian processes and distributes messages and manages clients.
type Gordian struct {
	Control chan *Client  // Control is used to pass client control information within Gordian.
	Message chan *Message // Message is used to pass messages within Gordian.
	manage  chan *Client
	clients map[ClientId]*Client
}

// New constructs an initialized Gordian instance.
func New() *Gordian {
	g := &Gordian{
		Control: make(chan *Client),
		Message: make(chan *Message),
		manage:  make(chan *Client),
		clients: make(map[ClientId]*Client),
	}
	return g
}

// Run starts Gordian's event loop.
func (g *Gordian) Run() {
	go func() {
		for {
			select {
			case msg := <-g.Message:
				if client, ok := g.clients[msg.To]; ok {
					client.message <- msg
				}
			case client := <-g.manage:
				switch client.Ctrl {
				case REGISTER:
					g.clients[client.Id] = client
				case CLOSE:
					close(client.message)
					delete(g.clients, client.Id)
				}
			}
		}
	}()
}

// WSHandler returns a websocket.Handler compatible function to handle connections.
func (g *Gordian) WSHandler() func(conn *websocket.Conn) {
	return func(conn *websocket.Conn) {
		g.Control <- &Client{Ctrl: CONNECT, Conn: conn}
		client := <-g.Control
		if client.Id == nil || client.Ctrl != REGISTER {
			return
		}
		client.message = make(chan *Message)
		g.manage <- client
		go g.writeToWS(client)
		g.readFromWS(client)
		client.Ctrl = CLOSE
		g.Control <- client
		g.manage <- client
	}
}

func (g *Gordian) readFromWS(client *Client) {
	for {
		var data MessageData
		err := websocket.JSON.Receive(client.Conn, &data)
		switch err {
		case nil:
			g.Message <- &Message{From: client.Id, Data: data}
		case io.EOF:
			return
		default:
			fmt.Println(err)
		}
	}
}

func (g *Gordian) writeToWS(client *Client) {
	for {
		msg, ok := <-client.message
		if !ok {
			return
		}
		if err := websocket.JSON.Send(client.Conn, msg.Data); err != nil {
			fmt.Println(err)
		}
	}
}
<commit_msg>Removed unnecessary qualifier from comment.<commit_after>package gordian

import (
	"code.google.com/p/go.net/websocket"

	"fmt"
	"io"
)

// Control types.
const (
	CONNECT = iota
	REGISTER
	CLOSE
)

// ClientId is a user-defined client identifier, which can be of any hashable type.
type ClientId interface{}

// MessageData is a user-defined message payload.
type MessageData interface{}

// Message is the internal message format
type Message struct {
	From ClientId    // From is the originating client.
	To   ClientId    // To is the destination client.
	Data MessageData // Data is the message payload.
}

// Client stores state and control information for a client.
type Client struct {
	Id      ClientId        // Id is a unique identifier.
	Ctrl    int             // Ctrl is the current control type.
	Conn    *websocket.Conn // Conn is the connection info provided by the websocket package.
	message chan *Message
}

// Gordian processes and distributes messages and manages clients.
type Gordian struct {
	Control chan *Client  // Control is used to pass client control information within Gordian.
	Message chan *Message // Message is used to pass messages within Gordian.
	manage  chan *Client
	clients map[ClientId]*Client
}

// New constructs an initialized Gordian instance.
func New() *Gordian {
	g := &Gordian{
		Control: make(chan *Client),
		Message: make(chan *Message),
		manage:  make(chan *Client),
		clients: make(map[ClientId]*Client),
	}
	return g
}

// Run starts Gordian's event loop.
func (g *Gordian) Run() {
	go func() {
		for {
			select {
			case msg := <-g.Message:
				if client, ok := g.clients[msg.To]; ok {
					client.message <- msg
				}
			case client := <-g.manage:
				switch client.Ctrl {
				case REGISTER:
					g.clients[client.Id] = client
				case CLOSE:
					close(client.message)
					delete(g.clients, client.Id)
				}
			}
		}
	}()
}

// WSHandler returns a websocket.Handler compatible function to handle connections.
func (g *Gordian) WSHandler() func(conn *websocket.Conn) {
	return func(conn *websocket.Conn) {
		g.Control <- &Client{Ctrl: CONNECT, Conn: conn}
		client := <-g.Control
		if client.Id == nil || client.Ctrl != REGISTER {
			return
		}
		client.message = make(chan *Message)
		g.manage <- client
		go g.writeToWS(client)
		g.readFromWS(client)
		client.Ctrl = CLOSE
		g.Control <- client
		g.manage <- client
	}
}

func (g *Gordian) readFromWS(client *Client) {
	for {
		var data MessageData
		err := websocket.JSON.Receive(client.Conn, &data)
		switch err {
		case nil:
			g.Message <- &Message{From: client.Id, Data: data}
		case io.EOF:
			return
		default:
			fmt.Println(err)
		}
	}
}

func (g *Gordian) writeToWS(client *Client) {
	for {
		msg, ok := <-client.message
		if !ok {
			return
		}
		if err := websocket.JSON.Send(client.Conn, msg.Data); err != nil {
			fmt.Println(err)
		}
	}
}
<|endoftext|>


<commit_before>// Package kube provides helper utilities common for kubernetes

package kube

import (
	"encoding/json"
	"fmt"

	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/api/equality"
	apierr "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/discovery"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

// TestConfig tests to make sure the REST config is usable
func TestConfig(config *rest.Config) error {
	kubeclientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		return fmt.Errorf("REST config invalid: %s", err)
	}
	_, err = kubeclientset.ServerVersion()
	if err != nil {
		return fmt.Errorf("REST config invalid: %s", err)
	}
	return nil
}

// ToUnstructured converts a concrete K8s API type to a un unstructured object
func ToUnstructured(obj interface{}) (*unstructured.Unstructured, error) {
	uObj, err := runtime.NewTestUnstructuredConverter(equality.Semantic).ToUnstructured(obj)
	if err != nil {
		return nil, err
	}
	return &unstructured.Unstructured{Object: uObj}, nil
}

// MustToUnstructured converts a concrete K8s API type to a un unstructured object and panics if not successful
func MustToUnstructured(obj interface{}) *unstructured.Unstructured {
	uObj, err := ToUnstructured(obj)
	if err != nil {
		panic(err)
	}
	return uObj
}

// ListAPIResources discovers all API resources supported by the Kube API sererver
func ListAPIResources(disco discovery.DiscoveryInterface) ([]metav1.APIResource, error) {
	apiResources := make([]metav1.APIResource, 0)
	resList, err := disco.ServerResources()
	if err != nil {
		return nil, errors.WithStack(err)
	}
	for _, resGroup := range resList {
		for _, apiRes := range resGroup.APIResources {
			apiResources = append(apiResources, apiRes)
		}
	}
	return apiResources, nil
}

// GetLiveResource returns the corresponding live resource from a unstructured object
func GetLiveResource(dclient dynamic.Interface, obj *unstructured.Unstructured, apiResource *metav1.APIResource, namespace string) (*unstructured.Unstructured, error) {
	resourceName := obj.GetName()
	if resourceName == "" {
		return nil, fmt.Errorf("resource was supplied without a name")
	}
	reIf := dclient.Resource(apiResource, namespace)
	liveObj, err := reIf.Get(resourceName, metav1.GetOptions{})
	if err != nil {
		if apierr.IsNotFound(err) {
			log.Infof("No live counterpart to %s/%s/%s/%s in namespace: '%s'", apiResource.Group, apiResource.Version, apiResource.Name, resourceName, namespace)
			return nil, nil
		}
		return nil, errors.WithStack(err)
	}
	return liveObj, nil
}

// DeleteResourceWithLabel delete all resources which match to specified label selector
func DeleteResourceWithLabel(config *rest.Config, namespace string, labelSelector string) error {
	dynClientPool := dynamic.NewDynamicClientPool(config)
	disco, err := discovery.NewDiscoveryClientForConfig(config)
	if err != nil {
		return err
	}
	resources, err := disco.ServerResources()
	if err != nil {
		return err
	}
	for _, apiResourcesList := range resources {
		for _, apiResource := range apiResourcesList.APIResources {
			deleteSupported := false
			for _, verb := range apiResource.Verbs {
				if verb == "deletecollection" {
					deleteSupported = true
					break
				}
			}
			if deleteSupported {
				dclient, err := dynClientPool.ClientForGroupVersionKind(schema.FromAPIVersionAndKind(apiResourcesList.GroupVersion, apiResource.Kind))
				if err != nil {
					return err
				}
				propagationPolicy := metav1.DeletePropagationForeground
				err = dclient.Resource(&apiResource, namespace).DeleteCollection(&metav1.DeleteOptions{
					PropagationPolicy: &propagationPolicy,
				}, metav1.ListOptions{LabelSelector: labelSelector})
				if err != nil && !apierr.IsNotFound(err) {
					return err
				}
			}
		}
	}
	return nil
}

// GetLiveResources returns the corresponding live resource from a list of resources
func GetLiveResources(config *rest.Config, objs []*unstructured.Unstructured, namespace string) ([]*unstructured.Unstructured, error) {
	liveObjs := make([]*unstructured.Unstructured, len(objs))
	dynClientPool := dynamic.NewDynamicClientPool(config)
	disco, err := discovery.NewDiscoveryClientForConfig(config)
	if err != nil {
		return nil, err
	}
	for i, obj := range objs {
		gvk := obj.GroupVersionKind()
		dclient, err := dynClientPool.ClientForGroupVersionKind(gvk)
		if err != nil {
			return nil, err
		}
		apiResource, err := ServerResourceForGroupVersionKind(disco, gvk)
		if err != nil {
			return nil, err
		}
		liveObj, err := GetLiveResource(dclient, obj, apiResource, namespace)
		if err != nil {
			return nil, err
		}
		liveObjs[i] = liveObj
	}
	return liveObjs, nil
}

// See: https://github.com/ksonnet/ksonnet/blob/master/utils/client.go
func ServerResourceForGroupVersionKind(disco discovery.DiscoveryInterface, gvk schema.GroupVersionKind) (*metav1.APIResource, error) {
	resources, err := disco.ServerResourcesForGroupVersion(gvk.GroupVersion().String())
	if err != nil {
		return nil, err
	}
	for _, r := range resources.APIResources {
		if r.Kind == gvk.Kind {
			log.Debugf("Chose API '%s' for %s", r.Name, gvk)
			return &r, nil
		}
	}
	return nil, fmt.Errorf("Server is unable to handle %s", gvk)
}

type listResult struct {
	Items []*unstructured.Unstructured `json:"items"`
}

// ListResources returns a list of resources of a particular API type using the dynamic client
func ListResources(dclient dynamic.Interface, apiResource metav1.APIResource, namespace string, listOpts metav1.ListOptions) ([]*unstructured.Unstructured, error) {
	reIf := dclient.Resource(&apiResource, namespace)
	liveObjs, err := reIf.List(listOpts)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	liveObjsBytes, err := json.Marshal(liveObjs)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	var objList listResult
	err = json.Unmarshal(liveObjsBytes, &objList)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	return objList.Items, nil
}

// ListAllResources iterates the list of API resources, and returns all resources with the given filters
func ListAllResources(config *rest.Config, apiResources []metav1.APIResource, namespace string, listOpts metav1.ListOptions) ([]*unstructured.Unstructured, error) {
	// itemMap dedups items when there is duplication of a resource in multiple API types
	// e.g. extensions/v1beta1/namespaces/default/deployments and apps/v1/namespaces/default/deployments
	itemMap := make(map[string]*unstructured.Unstructured)

	for _, apiResource := range apiResources {
		dynConfig := *config
		dynConfig.GroupVersion = &schema.GroupVersion{
			Group:   apiResource.Group,
			Version: apiResource.Kind,
		}
		dclient, err := dynamic.NewClient(&dynConfig)
		if err != nil {
			return nil, errors.WithStack(err)
		}
		resList, err := ListResources(dclient, apiResource, namespace, listOpts)
		if err != nil {
			return nil, errors.WithStack(err)
		}
		for _, liveObj := range resList {
			itemMap[string(liveObj.GetUID())] = liveObj
		}

	}
	resources := make([]*unstructured.Unstructured, len(itemMap))
	i := 0
	for _, obj := range itemMap {
		resources[i] = obj
		i++
	}
	return resources, nil
}

// ApplyResource performs an apply of a unstructured resource
func ApplyResource(config *rest.Config, obj *unstructured.Unstructured, namespace string) (*unstructured.Unstructured, error) {
	dynClientPool := dynamic.NewDynamicClientPool(config)
	disco, err := discovery.NewDiscoveryClientForConfig(config)
	if err != nil {
		return nil, err
	}
	gvk := obj.GroupVersionKind()
	dclient, err := dynClientPool.ClientForGroupVersionKind(gvk)
	if err != nil {
		return nil, err
	}
	apiResource, err := ServerResourceForGroupVersionKind(disco, gvk)
	if err != nil {
		return nil, err
	}
	reIf := dclient.Resource(apiResource, namespace)
	liveObj, err := reIf.Update(obj)
	if err != nil {
		if !apierr.IsNotFound(err) {
			return nil, errors.WithStack(err)
		}
		liveObj, err = reIf.Create(obj)
		if err != nil {
			return nil, errors.WithStack(err)
		}
	}
	return liveObj, nil
}
<commit_msg>Fix deleting resources which do not support 'deletecollection' method but support 'delete' (#56)<commit_after>// Package kube provides helper utilities common for kubernetes

package kube

import (
	"encoding/json"
	"fmt"

	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/api/equality"
	apierr "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/discovery"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

// TestConfig tests to make sure the REST config is usable
func TestConfig(config *rest.Config) error {
	kubeclientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		return fmt.Errorf("REST config invalid: %s", err)
	}
	_, err = kubeclientset.ServerVersion()
	if err != nil {
		return fmt.Errorf("REST config invalid: %s", err)
	}
	return nil
}

// ToUnstructured converts a concrete K8s API type to a un unstructured object
func ToUnstructured(obj interface{}) (*unstructured.Unstructured, error) {
	uObj, err := runtime.NewTestUnstructuredConverter(equality.Semantic).ToUnstructured(obj)
	if err != nil {
		return nil, err
	}
	return &unstructured.Unstructured{Object: uObj}, nil
}

// MustToUnstructured converts a concrete K8s API type to a un unstructured object and panics if not successful
func MustToUnstructured(obj interface{}) *unstructured.Unstructured {
	uObj, err := ToUnstructured(obj)
	if err != nil {
		panic(err)
	}
	return uObj
}

// ListAPIResources discovers all API resources supported by the Kube API sererver
func ListAPIResources(disco discovery.DiscoveryInterface) ([]metav1.APIResource, error) {
	apiResources := make([]metav1.APIResource, 0)
	resList, err := disco.ServerResources()
	if err != nil {
		return nil, errors.WithStack(err)
	}
	for _, resGroup := range resList {
		for _, apiRes := range resGroup.APIResources {
			apiResources = append(apiResources, apiRes)
		}
	}
	return apiResources, nil
}

// GetLiveResource returns the corresponding live resource from a unstructured object
func GetLiveResource(dclient dynamic.Interface, obj *unstructured.Unstructured, apiResource *metav1.APIResource, namespace string) (*unstructured.Unstructured, error) {
	resourceName := obj.GetName()
	if resourceName == "" {
		return nil, fmt.Errorf("resource was supplied without a name")
	}
	reIf := dclient.Resource(apiResource, namespace)
	liveObj, err := reIf.Get(resourceName, metav1.GetOptions{})
	if err != nil {
		if apierr.IsNotFound(err) {
			log.Infof("No live counterpart to %s/%s/%s/%s in namespace: '%s'", apiResource.Group, apiResource.Version, apiResource.Name, resourceName, namespace)
			return nil, nil
		}
		return nil, errors.WithStack(err)
	}
	return liveObj, nil
}

// DeleteResourceWithLabel delete all resources which match to specified label selector
func DeleteResourceWithLabel(config *rest.Config, namespace string, labelSelector string) error {
	dynClientPool := dynamic.NewDynamicClientPool(config)
	disco, err := discovery.NewDiscoveryClientForConfig(config)
	if err != nil {
		return err
	}
	resources, err := disco.ServerResources()
	if err != nil {
		return err
	}
	for _, apiResourcesList := range resources {
		for _, apiResource := range apiResourcesList.APIResources {
			deleteCollectionSupported := false
			deleteSupported := false
			for _, verb := range apiResource.Verbs {
				if verb == "deletecollection" {
					deleteCollectionSupported = true
				} else if verb == "delete" {
					deleteSupported = true
				}
			}
			dclient, err := dynClientPool.ClientForGroupVersionKind(schema.FromAPIVersionAndKind(apiResourcesList.GroupVersion, apiResource.Kind))
			if err != nil {
				return err
			}
			propagationPolicy := metav1.DeletePropagationForeground
			if deleteCollectionSupported {
				err = dclient.Resource(&apiResource, namespace).DeleteCollection(&metav1.DeleteOptions{
					PropagationPolicy: &propagationPolicy,
				}, metav1.ListOptions{LabelSelector: labelSelector})
				if err != nil && !apierr.IsNotFound(err) {
					return err
				}
			} else if deleteSupported {
				items, err := dclient.Resource(&apiResource, namespace).List(metav1.ListOptions{LabelSelector: labelSelector})
				if err != nil {
					return err
				}
				for _, item := range items.(*unstructured.UnstructuredList).Items {
					err = dclient.Resource(&apiResource, namespace).Delete(item.GetName(), &metav1.DeleteOptions{
						PropagationPolicy: &propagationPolicy,
					})
					if err != nil {
						return err
					}
				}
			}
		}
	}
	return nil
}

// GetLiveResources returns the corresponding live resource from a list of resources
func GetLiveResources(config *rest.Config, objs []*unstructured.Unstructured, namespace string) ([]*unstructured.Unstructured, error) {
	liveObjs := make([]*unstructured.Unstructured, len(objs))
	dynClientPool := dynamic.NewDynamicClientPool(config)
	disco, err := discovery.NewDiscoveryClientForConfig(config)
	if err != nil {
		return nil, err
	}
	for i, obj := range objs {
		gvk := obj.GroupVersionKind()
		dclient, err := dynClientPool.ClientForGroupVersionKind(gvk)
		if err != nil {
			return nil, err
		}
		apiResource, err := ServerResourceForGroupVersionKind(disco, gvk)
		if err != nil {
			return nil, err
		}
		liveObj, err := GetLiveResource(dclient, obj, apiResource, namespace)
		if err != nil {
			return nil, err
		}
		liveObjs[i] = liveObj
	}
	return liveObjs, nil
}

// See: https://github.com/ksonnet/ksonnet/blob/master/utils/client.go
func ServerResourceForGroupVersionKind(disco discovery.DiscoveryInterface, gvk schema.GroupVersionKind) (*metav1.APIResource, error) {
	resources, err := disco.ServerResourcesForGroupVersion(gvk.GroupVersion().String())
	if err != nil {
		return nil, err
	}
	for _, r := range resources.APIResources {
		if r.Kind == gvk.Kind {
			log.Debugf("Chose API '%s' for %s", r.Name, gvk)
			return &r, nil
		}
	}
	return nil, fmt.Errorf("Server is unable to handle %s", gvk)
}

type listResult struct {
	Items []*unstructured.Unstructured `json:"items"`
}

// ListResources returns a list of resources of a particular API type using the dynamic client
func ListResources(dclient dynamic.Interface, apiResource metav1.APIResource, namespace string, listOpts metav1.ListOptions) ([]*unstructured.Unstructured, error) {
	reIf := dclient.Resource(&apiResource, namespace)
	liveObjs, err := reIf.List(listOpts)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	liveObjsBytes, err := json.Marshal(liveObjs)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	var objList listResult
	err = json.Unmarshal(liveObjsBytes, &objList)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	return objList.Items, nil
}

// ListAllResources iterates the list of API resources, and returns all resources with the given filters
func ListAllResources(config *rest.Config, apiResources []metav1.APIResource, namespace string, listOpts metav1.ListOptions) ([]*unstructured.Unstructured, error) {
	// itemMap dedups items when there is duplication of a resource in multiple API types
	// e.g. extensions/v1beta1/namespaces/default/deployments and apps/v1/namespaces/default/deployments
	itemMap := make(map[string]*unstructured.Unstructured)

	for _, apiResource := range apiResources {
		dynConfig := *config
		dynConfig.GroupVersion = &schema.GroupVersion{
			Group:   apiResource.Group,
			Version: apiResource.Kind,
		}
		dclient, err := dynamic.NewClient(&dynConfig)
		if err != nil {
			return nil, errors.WithStack(err)
		}
		resList, err := ListResources(dclient, apiResource, namespace, listOpts)
		if err != nil {
			return nil, errors.WithStack(err)
		}
		for _, liveObj := range resList {
			itemMap[string(liveObj.GetUID())] = liveObj
		}

	}
	resources := make([]*unstructured.Unstructured, len(itemMap))
	i := 0
	for _, obj := range itemMap {
		resources[i] = obj
		i++
	}
	return resources, nil
}

// ApplyResource performs an apply of a unstructured resource
func ApplyResource(config *rest.Config, obj *unstructured.Unstructured, namespace string) (*unstructured.Unstructured, error) {
	dynClientPool := dynamic.NewDynamicClientPool(config)
	disco, err := discovery.NewDiscoveryClientForConfig(config)
	if err != nil {
		return nil, err
	}
	gvk := obj.GroupVersionKind()
	dclient, err := dynClientPool.ClientForGroupVersionKind(gvk)
	if err != nil {
		return nil, err
	}
	apiResource, err := ServerResourceForGroupVersionKind(disco, gvk)
	if err != nil {
		return nil, err
	}
	reIf := dclient.Resource(apiResource, namespace)
	liveObj, err := reIf.Update(obj)
	if err != nil {
		if !apierr.IsNotFound(err) {
			return nil, errors.WithStack(err)
		}
		liveObj, err = reIf.Create(obj)
		if err != nil {
			return nil, errors.WithStack(err)
		}
	}
	return liveObj, nil
}
<|endoftext|>