<commit_before>package collectors

import (
	"github.com/bosun-monitor/scollector/_third_party/github.com/StackExchange/wmi"
	"github.com/bosun-monitor/scollector/_third_party/github.com/bosun-monitor/metadata"
	"github.com/bosun-monitor/scollector/_third_party/github.com/bosun-monitor/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_physical_disk_windows})
	collectors = append(collectors, &IntervalCollector{F: c_diskspace_windows})
}

const (
	//Converts 100nS samples to 1S samples
	winDisk100nS_1S = 10000000

	//Converts 100nS samples to 1mS samples
	winDisk100nS_1mS = 1000000

	//Converts 100nS samples to 0-100 Percent samples
	winDisk100nS_Pct = 100000
)

func c_diskspace_windows() (opentsdb.MultiDataPoint, error) {
	var dst []Win32_LogicalDisk
	var q = wmi.CreateQuery(&dst, "WHERE DriveType = 3")
	err := queryWmi(q, &dst)
	if err != nil {
		return nil, err
	}
	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		tags := opentsdb.TagSet{"disk": v.Name}
		space_used := v.Size - v.FreeSpace
		Add(&md, "win.disk.fs.space_free", v.FreeSpace, tags, metadata.Gauge, metadata.Bytes, osDiskFreeDesc)
		Add(&md, "win.disk.fs.space_total", v.Size, tags, metadata.Gauge, metadata.Bytes, osDiskTotalDesc)
		Add(&md, "win.disk.fs.space_used", space_used, tags, metadata.Gauge, metadata.Bytes, osDiskUsedDesc)
		Add(&md, osDiskFree, v.FreeSpace, tags, metadata.Gauge, metadata.Bytes, osDiskFreeDesc)
		Add(&md, osDiskTotal, v.Size, tags, metadata.Gauge, metadata.Bytes, osDiskTotalDesc)
		Add(&md, osDiskUsed, space_used, tags, metadata.Gauge, metadata.Bytes, osDiskUsedDesc)
		if v.Size != 0 {
			percent_free := float64(v.FreeSpace / v.Size * 100)
			Add(&md, "win.disk.fs.percent_free", percent_free, tags, metadata.Gauge, metadata.Pct, osDiskPctFreeDesc)
			Add(&md, osDiskPctFree, percent_free, tags, metadata.Gauge, metadata.Pct, osDiskPctFreeDesc)
		}
	}
	return md, nil
}

type Win32_LogicalDisk struct {
	FreeSpace uint64
	Name      string
	Size      uint64
}

func c_physical_disk_windows() (opentsdb.MultiDataPoint, error) {
	var dst []Win32_PerfRawData_PerfDisk_PhysicalDisk
	var q = wmi.CreateQuery(&dst, `WHERE Name <> '_Total'`)
	err := queryWmi(q, &dst)
	if err != nil {
		return nil, err
	}
	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		Add(&md, "win.disk.duration", v.AvgDiskSecPerRead/winDisk100nS_1mS, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.MilliSecond, "Time, in milliseconds, of a read from the disk.")
		Add(&md, "win.disk.duration", v.AvgDiskSecPerWrite/winDisk100nS_1mS, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.MilliSecond, "Time, in milliseconds, of a write to the disk.")
		Add(&md, "win.disk.queue", v.AvgDiskReadQueueLength/winDisk100nS_1S, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.Operation, "Number of read requests that were queued for the disk.")
		Add(&md, "win.disk.queue", v.AvgDiskWriteQueueLength/winDisk100nS_1S, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.Operation, "Number of write requests that were queued for the disk.")
		Add(&md, "win.disk.ops", v.DiskReadsPerSec, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.PerSecond, "Number of read operations on the disk.")
		Add(&md, "win.disk.ops", v.DiskWritesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.PerSecond, "Number of write operations on the disk.")
		Add(&md, "win.disk.bytes", v.DiskReadBytesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.BytesPerSecond, "Number of bytes read from the disk.")
		Add(&md, "win.disk.bytes", v.DiskWriteBytesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.BytesPerSecond, "Number of bytes written to the disk.")
		Add(&md, "win.disk.percent_time", v.PercentDiskReadTime/winDisk100nS_Pct, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.Pct, "Percentage of time that the disk was busy servicing read requests.")
		Add(&md, "win.disk.percent_time", v.PercentDiskWriteTime/winDisk100nS_Pct, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.Pct, "Percentage of time that the disk was busy servicing write requests.")
		Add(&md, "win.disk.spltio", v.SplitIOPerSec, opentsdb.TagSet{"disk": v.Name}, metadata.Counter, metadata.PerSecond, "Number of requests to the disk that were split into multiple requests due to size or fragmentation.")
	}
	return md, nil
}

//See msdn for counter types http://msdn.microsoft.com/en-us/library/ms804035.aspx
type Win32_PerfRawData_PerfDisk_PhysicalDisk struct {
	AvgDiskReadQueueLength  uint64
	AvgDiskSecPerRead       uint32
	AvgDiskSecPerWrite      uint32
	AvgDiskWriteQueueLength uint64
	DiskReadBytesPerSec     uint64
	DiskReadsPerSec         uint32
	DiskWriteBytesPerSec    uint64
	DiskWritesPerSec        uint32
	Name                    string
	PercentDiskReadTime     uint64
	PercentDiskWriteTime    uint64
	SplitIOPerSec           uint32
}
<commit_msg>cmd/scollector: Fix win.disk.fs.percent_free<commit_after>package collectors

import (
	"github.com/bosun-monitor/scollector/_third_party/github.com/StackExchange/wmi"
	"github.com/bosun-monitor/scollector/_third_party/github.com/bosun-monitor/metadata"
	"github.com/bosun-monitor/scollector/_third_party/github.com/bosun-monitor/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_physical_disk_windows})
	collectors = append(collectors, &IntervalCollector{F: c_diskspace_windows})
}

const (
	//Converts 100nS samples to 1S samples
	winDisk100nS_1S = 10000000

	//Converts 100nS samples to 1mS samples
	winDisk100nS_1mS = 1000000

	//Converts 100nS samples to 0-100 Percent samples
	winDisk100nS_Pct = 100000
)

func c_diskspace_windows() (opentsdb.MultiDataPoint, error) {
	var dst []Win32_LogicalDisk
	var q = wmi.CreateQuery(&dst, "WHERE DriveType = 3")
	err := queryWmi(q, &dst)
	if err != nil {
		return nil, err
	}
	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		tags := opentsdb.TagSet{"disk": v.Name}
		space_used := v.Size - v.FreeSpace
		Add(&md, "win.disk.fs.space_free", v.FreeSpace, tags, metadata.Gauge, metadata.Bytes, osDiskFreeDesc)
		Add(&md, "win.disk.fs.space_total", v.Size, tags, metadata.Gauge, metadata.Bytes, osDiskTotalDesc)
		Add(&md, "win.disk.fs.space_used", space_used, tags, metadata.Gauge, metadata.Bytes, osDiskUsedDesc)
		Add(&md, osDiskFree, v.FreeSpace, tags, metadata.Gauge, metadata.Bytes, osDiskFreeDesc)
		Add(&md, osDiskTotal, v.Size, tags, metadata.Gauge, metadata.Bytes, osDiskTotalDesc)
		Add(&md, osDiskUsed, space_used, tags, metadata.Gauge, metadata.Bytes, osDiskUsedDesc)
		if v.Size != 0 {
			percent_free := float64(v.FreeSpace) / float64(v.Size) * 100
			Add(&md, "win.disk.fs.percent_free", percent_free, tags, metadata.Gauge, metadata.Pct, osDiskPctFreeDesc)
			Add(&md, osDiskPctFree, percent_free, tags, metadata.Gauge, metadata.Pct, osDiskPctFreeDesc)
		}
	}
	return md, nil
}

type Win32_LogicalDisk struct {
	FreeSpace uint64
	Name      string
	Size      uint64
}

func c_physical_disk_windows() (opentsdb.MultiDataPoint, error) {
	var dst []Win32_PerfRawData_PerfDisk_PhysicalDisk
	var q = wmi.CreateQuery(&dst, `WHERE Name <> '_Total'`)
	err := queryWmi(q, &dst)
	if err != nil {
		return nil, err
	}
	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		Add(&md, "win.disk.duration", v.AvgDiskSecPerRead/winDisk100nS_1mS, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.MilliSecond, "Time, in milliseconds, of a read from the disk.")
		Add(&md, "win.disk.duration", v.AvgDiskSecPerWrite/winDisk100nS_1mS, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.MilliSecond, "Time, in milliseconds, of a write to the disk.")
		Add(&md, "win.disk.queue", v.AvgDiskReadQueueLength/winDisk100nS_1S, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.Operation, "Number of read requests that were queued for the disk.")
		Add(&md, "win.disk.queue", v.AvgDiskWriteQueueLength/winDisk100nS_1S, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.Operation, "Number of write requests that were queued for the disk.")
		Add(&md, "win.disk.ops", v.DiskReadsPerSec, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.PerSecond, "Number of read operations on the disk.")
		Add(&md, "win.disk.ops", v.DiskWritesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.PerSecond, "Number of write operations on the disk.")
		Add(&md, "win.disk.bytes", v.DiskReadBytesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.BytesPerSecond, "Number of bytes read from the disk.")
		Add(&md, "win.disk.bytes", v.DiskWriteBytesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.BytesPerSecond, "Number of bytes written to the disk.")
		Add(&md, "win.disk.percent_time", v.PercentDiskReadTime/winDisk100nS_Pct, opentsdb.TagSet{"disk": v.Name, "type": "read"}, metadata.Counter, metadata.Pct, "Percentage of time that the disk was busy servicing read requests.")
		Add(&md, "win.disk.percent_time", v.PercentDiskWriteTime/winDisk100nS_Pct, opentsdb.TagSet{"disk": v.Name, "type": "write"}, metadata.Counter, metadata.Pct, "Percentage of time that the disk was busy servicing write requests.")
		Add(&md, "win.disk.spltio", v.SplitIOPerSec, opentsdb.TagSet{"disk": v.Name}, metadata.Counter, metadata.PerSecond, "Number of requests to the disk that were split into multiple requests due to size or fragmentation.")
	}
	return md, nil
}

//See msdn for counter types http://msdn.microsoft.com/en-us/library/ms804035.aspx
type Win32_PerfRawData_PerfDisk_PhysicalDisk struct {
	AvgDiskReadQueueLength  uint64
	AvgDiskSecPerRead       uint32
	AvgDiskSecPerWrite      uint32
	AvgDiskWriteQueueLength uint64
	DiskReadBytesPerSec     uint64
	DiskReadsPerSec         uint32
	DiskWriteBytesPerSec    uint64
	DiskWritesPerSec        uint32
	Name                    string
	PercentDiskReadTime     uint64
	PercentDiskWriteTime    uint64
	SplitIOPerSec           uint32
}
<|endoftext|>


<commit_before>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"fmt"
	"github.com/getgauge/gauge/api"
	"github.com/getgauge/gauge/config"
	"github.com/getgauge/gauge/env"
	"github.com/getgauge/gauge/execution"
	"github.com/getgauge/gauge/filter"
	"github.com/getgauge/gauge/formatter"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/logger/execLogger"
	"github.com/getgauge/gauge/plugin/install"
	"github.com/getgauge/gauge/project_init"
	"github.com/getgauge/gauge/refactor"
	"github.com/getgauge/gauge/util"
	"github.com/getgauge/gauge/version"
	flag "github.com/getgauge/mflag"
	"os"
	"time"
)

// Command line flags
var daemonize = flag.Bool([]string{"-daemonize"}, false, "Run as a daemon")
var gaugeVersion = flag.Bool([]string{"v", "-version", "version"}, false, "Print the current version and exit. Eg: gauge --version")
var verbosity = flag.Bool([]string{"-verbose"}, false, "Enable verbose logging for debugging")
var logLevel = flag.String([]string{"-log-level"}, "", "Set level of logging to debug, info, warning, error or critical")
var simpleConsoleOutput = flag.Bool([]string{"-simple-console"}, false, "Removes colouring and simplifies from the console output")
var initialize = flag.String([]string{"-init"}, "", "Initializes project structure in the current directory. Eg: gauge --init java")
var installPlugin = flag.String([]string{"-install"}, "", "Downloads and installs a plugin. Eg: gauge --install java")
var uninstallPlugin = flag.String([]string{"-uninstall"}, "", "Uninstalls a plugin. Eg: gauge --uninstall java")
var installAll = flag.Bool([]string{"-install-all"}, false, "Installs all the plugins specified in project manifest, if not installed. Eg: gauge --install-all")
var update = flag.String([]string{"-update"}, "", "Updates a plugin. Eg: gauge --update java")
var installVersion = flag.String([]string{"-plugin-version"}, "", "Version of plugin to be installed. This is used with --install")
var installZip = flag.String([]string{"-file", "f"}, "", "Installs the plugin from zip file. This is used with --install. Eg: gauge --install java -f ZIP_FILE")
var currentEnv = flag.String([]string{"-env"}, "default", "Specifies the environment. If not specified, default will be used")
var addPlugin = flag.String([]string{"-add-plugin"}, "", "Adds the specified non-language plugin to the current project")
var pluginArgs = flag.String([]string{"-plugin-args"}, "", "Specified additional arguments to the plugin. This is used together with --add-plugin")
var specFilesToFormat = flag.String([]string{"-format"}, "", "Formats the specified spec files")
var executeTags = flag.String([]string{"-tags"}, "", "Executes the specs and scenarios tagged with given tags. Eg: gauge --tags tag1,tag2 specs")
var tableRows = flag.String([]string{"-table-rows"}, "", "Executes the specs and scenarios only for the selected rows. Eg: gauge --table-rows \"1-3\" specs/hello.spec")
var apiPort = flag.String([]string{"-api-port"}, "", "Specifies the api port to be used. Eg: gauge --daemonize --api-port 7777")
var refactorSteps = flag.String([]string{"-refactor"}, "", "Refactor steps")
var parallel = flag.Bool([]string{"-parallel", "p"}, false, "Execute specs in parallel")
var numberOfExecutionStreams = flag.Int([]string{"n"}, util.NumberOfCores(), "Specify number of parallel execution streams")
var distribute = flag.Int([]string{"g", "-group"}, -1, "Specify which group of specification to execute based on -n flag")
var workingDir = flag.String([]string{"-dir"}, ".", "Set the working directory for the current command, accepts a path relative to current directory.")
var doNotRandomize = flag.Bool([]string{"-sort", "s"}, false, "run specs in Alphabetical Order. Eg: gauge -s specs")

func main() {
	flag.Parse()
	project_init.SetWorkingDir(*workingDir)
	initPackageFlags()
	validGaugeProject := true
	err := config.SetProjectRoot(flag.Args())
	if err != nil {
		validGaugeProject = false
	}
	env.LoadEnv(true)
	logger.Initialize(*verbosity, *logLevel)
	if *gaugeVersion {
		version.PrintVersion()
	} else if *daemonize {
		if validGaugeProject {
			api.RunInBackground(*apiPort)
		} else {
			logger.Log.Error(err.Error())
		}
	} else if *specFilesToFormat != "" {
		if validGaugeProject {
			formatter.FormatSpecFilesIn(*specFilesToFormat)
		} else {
			logger.Log.Error(err.Error())
		}
	} else if *initialize != "" {
		project_init.InitializeProject(*initialize)
	} else if *installZip != "" && *installPlugin != "" {
		install.InstallPluginZip(*installZip, *installPlugin)
	} else if *installPlugin != "" {
		install.DownloadAndInstallPlugin(*installPlugin, *installVersion)
	} else if *uninstallPlugin != "" {
		install.UninstallPlugin(*uninstallPlugin)
	} else if *installAll {
		install.InstallAllPlugins()
	} else if *update != "" {
		install.UpdatePlugin(*update)
	} else if *addPlugin != "" {
		install.AddPluginToProject(*addPlugin, *pluginArgs)
	} else if *refactorSteps != "" {
		if validGaugeProject {
			startChan := api.StartAPI()
			refactor.RefactorSteps(*refactorSteps, newStepName(), startChan)
		} else {
			logger.Log.Error(err.Error())
		}
	} else {
		if len(flag.Args()) == 0 {
			printUsage()
		} else if validGaugeProject {
			if *distribute != -1 {
				*doNotRandomize = true
			}
			execution.ExecuteSpecs(*parallel, flag.Args())
		} else {
			logger.Log.Error("Could not set project root: %s", err.Error())
			os.Exit(1)
		}
	}
}

func printUsage() {
	fmt.Printf("gauge - version %s\n", version.CurrentGaugeVersion.String())
	fmt.Printf("Copyright %d Thoughtworks\n\n", time.Now().Year())
	fmt.Println("Usage:")
	fmt.Println("\tgauge specs/")
	fmt.Println("\tgauge specs/spec_name.spec")
	fmt.Println("\nOptions:")
	flag.PrintDefaults()
	os.Exit(2)
}

func newStepName() string {
	if len(flag.Args()) != 1 {
		printUsage()
	}
	return flag.Args()[0]
}

func initPackageFlags() {
	if util.IsWindows() {
		*simpleConsoleOutput = true
	}
	execLogger.SimpleConsoleOutput = *simpleConsoleOutput
	env.ProjectEnv = *currentEnv
	execution.ExecuteTags = *executeTags
	execution.TableRows = *tableRows
	execution.NumberOfExecutionStreams = *numberOfExecutionStreams
	filter.ExecuteTags = *executeTags
	filter.DoNotRandomize = *doNotRandomize
	filter.Distribute = *distribute
	filter.NumberOfExecutionStreams = *numberOfExecutionStreams
}
<commit_msg>fixing typo<commit_after>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"fmt"
	"github.com/getgauge/gauge/api"
	"github.com/getgauge/gauge/config"
	"github.com/getgauge/gauge/env"
	"github.com/getgauge/gauge/execution"
	"github.com/getgauge/gauge/filter"
	"github.com/getgauge/gauge/formatter"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/logger/execLogger"
	"github.com/getgauge/gauge/plugin/install"
	"github.com/getgauge/gauge/project_init"
	"github.com/getgauge/gauge/refactor"
	"github.com/getgauge/gauge/util"
	"github.com/getgauge/gauge/version"
	flag "github.com/getgauge/mflag"
	"os"
	"time"
)

// Command line flags
var daemonize = flag.Bool([]string{"-daemonize"}, false, "Run as a daemon")
var gaugeVersion = flag.Bool([]string{"v", "-version", "version"}, false, "Print the current version and exit. Eg: gauge --version")
var verbosity = flag.Bool([]string{"-verbose"}, false, "Enable verbose logging for debugging")
var logLevel = flag.String([]string{"-log-level"}, "", "Set level of logging to debug, info, warning, error or critical")
var simpleConsoleOutput = flag.Bool([]string{"-simple-console"}, false, "Removes colouring and simplifies from the console output")
var initialize = flag.String([]string{"-init"}, "", "Initializes project structure in the current directory. Eg: gauge --init java")
var installPlugin = flag.String([]string{"-install"}, "", "Downloads and installs a plugin. Eg: gauge --install java")
var uninstallPlugin = flag.String([]string{"-uninstall"}, "", "Uninstalls a plugin. Eg: gauge --uninstall java")
var installAll = flag.Bool([]string{"-install-all"}, false, "Installs all the plugins specified in project manifest, if not installed. Eg: gauge --install-all")
var update = flag.String([]string{"-update"}, "", "Updates a plugin. Eg: gauge --update java")
var installVersion = flag.String([]string{"-plugin-version"}, "", "Version of plugin to be installed. This is used with --install")
var installZip = flag.String([]string{"-file", "f"}, "", "Installs the plugin from zip file. This is used with --install. Eg: gauge --install java -f ZIP_FILE")
var currentEnv = flag.String([]string{"-env"}, "default", "Specifies the environment. If not specified, default will be used")
var addPlugin = flag.String([]string{"-add-plugin"}, "", "Adds the specified non-language plugin to the current project")
var pluginArgs = flag.String([]string{"-plugin-args"}, "", "Specified additional arguments to the plugin. This is used together with --add-plugin")
var specFilesToFormat = flag.String([]string{"-format"}, "", "Formats the specified spec files")
var executeTags = flag.String([]string{"-tags"}, "", "Executes the specs and scenarios tagged with given tags. Eg: gauge --tags tag1,tag2 specs")
var tableRows = flag.String([]string{"-table-rows"}, "", "Executes the specs and scenarios only for the selected rows. Eg: gauge --table-rows \"1-3\" specs/hello.spec")
var apiPort = flag.String([]string{"-api-port"}, "", "Specifies the api port to be used. Eg: gauge --daemonize --api-port 7777")
var refactorSteps = flag.String([]string{"-refactor"}, "", "Refactor steps")
var parallel = flag.Bool([]string{"-parallel", "p"}, false, "Execute specs in parallel")
var numberOfExecutionStreams = flag.Int([]string{"n"}, util.NumberOfCores(), "Specify number of parallel execution streams")
var distribute = flag.Int([]string{"g", "-group"}, -1, "Specify which group of specification to execute based on -n flag")
var workingDir = flag.String([]string{"-dir"}, ".", "Set the working directory for the current command, accepts a path relative to current directory.")
var doNotRandomize = flag.Bool([]string{"-sort", "s"}, false, "run specs in Alphabetical Order. Eg: gauge -s specs")

func main() {
	flag.Parse()
	project_init.SetWorkingDir(*workingDir)
	initPackageFlags()
	validGaugeProject := true
	err := config.SetProjectRoot(flag.Args())
	if err != nil {
		validGaugeProject = false
	}
	env.LoadEnv(true)
	logger.Initialize(*verbosity, *logLevel)
	if *gaugeVersion {
		version.PrintVersion()
	} else if *daemonize {
		if validGaugeProject {
			api.RunInBackground(*apiPort)
		} else {
			logger.Log.Error(err.Error())
		}
	} else if *specFilesToFormat != "" {
		if validGaugeProject {
			formatter.FormatSpecFilesIn(*specFilesToFormat)
		} else {
			logger.Log.Error(err.Error())
		}
	} else if *initialize != "" {
		project_init.InitializeProject(*initialize)
	} else if *installZip != "" && *installPlugin != "" {
		install.InstallPluginZip(*installZip, *installPlugin)
	} else if *installPlugin != "" {
		install.DownloadAndInstallPlugin(*installPlugin, *installVersion)
	} else if *uninstallPlugin != "" {
		install.UninstallPlugin(*uninstallPlugin)
	} else if *installAll {
		install.InstallAllPlugins()
	} else if *update != "" {
		install.UpdatePlugin(*update)
	} else if *addPlugin != "" {
		install.AddPluginToProject(*addPlugin, *pluginArgs)
	} else if *refactorSteps != "" {
		if validGaugeProject {
			startChan := api.StartAPI()
			refactor.RefactorSteps(*refactorSteps, newStepName(), startChan)
		} else {
			logger.Log.Error(err.Error())
		}
	} else {
		if len(flag.Args()) == 0 {
			printUsage()
		} else if validGaugeProject {
			if *distribute != -1 {
				*doNotRandomize = true
			}
			execution.ExecuteSpecs(*parallel, flag.Args())
		} else {
			logger.Log.Error("Could not set project root: %s", err.Error())
			os.Exit(1)
		}
	}
}

func printUsage() {
	fmt.Printf("gauge -version %s\n", version.CurrentGaugeVersion.String())
	fmt.Printf("Copyright %d Thoughtworks\n\n", time.Now().Year())
	fmt.Println("Usage:")
	fmt.Println("\tgauge specs/")
	fmt.Println("\tgauge specs/spec_name.spec")
	fmt.Println("\nOptions:")
	flag.PrintDefaults()
	os.Exit(2)
}

func newStepName() string {
	if len(flag.Args()) != 1 {
		printUsage()
	}
	return flag.Args()[0]
}

func initPackageFlags() {
	if util.IsWindows() {
		*simpleConsoleOutput = true
	}
	execLogger.SimpleConsoleOutput = *simpleConsoleOutput
	env.ProjectEnv = *currentEnv
	execution.ExecuteTags = *executeTags
	execution.TableRows = *tableRows
	execution.NumberOfExecutionStreams = *numberOfExecutionStreams
	filter.ExecuteTags = *executeTags
	filter.DoNotRandomize = *doNotRandomize
	filter.Distribute = *distribute
	filter.NumberOfExecutionStreams = *numberOfExecutionStreams
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"expvar"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"

	"github.com/NYTimes/gziphandler"
	"github.com/mat/besticon/besticon"
	"github.com/mat/besticon/besticon/iconserver/assets"
	"github.com/mat/besticon/lettericon"
)

func indexHandler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "" || r.URL.Path == "/" {
		renderHTMLTemplate(w, 200, indexHTML, nil)
	} else {
		renderHTMLTemplate(w, 404, notFoundHTML, nil)
	}
}

func iconsHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue(urlParam)
	if len(url) == 0 {
		http.Redirect(w, r, "/", 302)
		return
	}

	finder := besticon.IconFinder{}

	formats := r.FormValue("formats")
	if formats != "" {
		finder.FormatsAllowed = strings.Split(r.FormValue("formats"), ",")
	}

	icons, e := finder.FetchIcons(url)
	switch {
	case e != nil:
		renderHTMLTemplate(w, 404, iconsHTML, pageInfo{URL: url, Error: e})
	case len(icons) == 0:
		errNoIcons := errors.New("this poor site has no icons at all :-(")
		renderHTMLTemplate(w, 404, iconsHTML, pageInfo{URL: url, Error: errNoIcons})
	default:
		renderHTMLTemplate(w, 200, iconsHTML, pageInfo{Icons: icons, URL: url})
	}
}

func iconHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue("url")
	if len(url) == 0 {
		writeAPIError(w, 400, errors.New("need url parameter"))
		return
	}

	size := r.FormValue("size")
	if size == "" {
		writeAPIError(w, 400, errors.New("need size parameter"))
		return
	}
	minSize, err := strconv.Atoi(size)
	if err != nil || minSize < besticon.MinIconSize || minSize > besticon.MaxIconSize {
		writeAPIError(w, 400, errors.New("bad size parameter"))
		return
	}

	finder := besticon.IconFinder{}
	formats := r.FormValue("formats")
	if formats != "" {
		finder.FormatsAllowed = strings.Split(r.FormValue("formats"), ",")
	}

	finder.FetchIcons(url)

	icon := finder.IconWithMinSize(minSize)
	if icon != nil {
		redirectWithCacheControl(w, r, icon.URL)
		return
	}

	fallbackIconURL := r.FormValue("fallback_icon_url")
	if fallbackIconURL != "" {
		redirectWithCacheControl(w, r, fallbackIconURL)
		return
	}

	iconColor := finder.MainColorForIcons()
	letter := lettericon.MainLetterFromURL(url)
	redirectPath := lettericon.IconPath(letter, size, iconColor)
	redirectWithCacheControl(w, r, redirectPath)
}

func popularHandler(w http.ResponseWriter, r *http.Request) {
	iconSize, err := strconv.Atoi(r.FormValue("iconsize"))
	if iconSize > besticon.MaxIconSize || iconSize < besticon.MinIconSize || err != nil {
		iconSize = 120
	}

	pageInfo := struct {
		URLs        []string
		IconSize    int
		DisplaySize int
	}{
		besticon.PopularSites,
		iconSize,
		iconSize / 2,
	}
	renderHTMLTemplate(w, 200, popularHTML, pageInfo)
}

const (
	urlParam = "url"
)

func alliconsHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue(urlParam)
	if len(url) == 0 {
		errMissingURL := errors.New("need url query parameter")
		writeAPIError(w, 400, errMissingURL)
		return
	}

	finder := besticon.IconFinder{}
	formats := r.FormValue("formats")
	if formats != "" {
		finder.FormatsAllowed = strings.Split(r.FormValue("formats"), ",")
	}

	icons, e := finder.FetchIcons(url)
	if e != nil {
		writeAPIError(w, 404, e)
		return
	}

	writeAPIIcons(w, url, icons)
}

func lettericonHandler(w http.ResponseWriter, r *http.Request) {
	charParam, col, size := lettericon.ParseIconPath(r.URL.Path)
	if charParam == "" {
		writeAPIError(w, 400, errors.New("wrong format for lettericons/ path, must look like lettericons/M-144-EFC25D.png"))
	}

	w.Header().Add(contentType, imagePNG)
	w.Header().Add(cacheControl, fmt.Sprintf("max-age=%d", oneYear))
	lettericon.Render(charParam, col, size, w)
}

func obsoleteAPIHandler(w http.ResponseWriter, r *http.Request) {
	if r.FormValue("i_am_feeling_lucky") == "yes" {
		http.Redirect(w, r, fmt.Sprintf("/icon?size=120&%s", r.URL.RawQuery), 302)
	} else {
		http.Redirect(w, r, fmt.Sprintf("/allicons.json?%s", r.URL.RawQuery), 302)
	}
}

func writeAPIError(w http.ResponseWriter, httpStatus int, e error) {
	data := struct {
		Error string `json:"error"`
	}{
		e.Error(),
	}
	renderJSONResponse(w, httpStatus, data)
}

func writeAPIIcons(w http.ResponseWriter, url string, icons []besticon.Icon) {
	// Don't return whole image data
	newIcons := []besticon.Icon{}
	for _, ico := range icons {
		newIcon := ico
		newIcon.ImageData = nil
		newIcons = append(newIcons, newIcon)
	}

	data := &struct {
		URL   string          `json:"url"`
		Icons []besticon.Icon `json:"icons"`
	}{
		url,
		newIcons,
	}
	renderJSONResponse(w, 200, data)
}

const (
	contentType     = "Content-Type"
	applicationJSON = "application/json"
	imagePNG        = "image/png"
	cacheControl    = "Cache-Control"
)

func renderJSONResponse(w http.ResponseWriter, httpStatus int, data interface{}) {
	w.Header().Add(contentType, applicationJSON)
	w.WriteHeader(httpStatus)
	enc := json.NewEncoder(w)
	enc.Encode(data)
}

type pageInfo struct {
	URL   string
	Icons []besticon.Icon
	Error error
}

func (pi pageInfo) Host() string {
	u := pi.URL
	url, _ := url.Parse(u)
	if url != nil && url.Host != "" {
		return url.Host
	}
	return pi.URL
}

func (pi pageInfo) Best() string {
	if len(pi.Icons) > 0 {
		best := pi.Icons[0]
		return best.URL
	}
	return ""
}

func renderHTMLTemplate(w http.ResponseWriter, httpStatus int, templ *template.Template, data interface{}) {
	w.Header().Add(contentType, "text/html; charset=utf-8")
	w.WriteHeader(httpStatus)

	err := templ.Execute(w, data)
	if err != nil {
		err = fmt.Errorf("server: could not generate output: %s", err)
		logger.Print(err)
		w.Write([]byte(err.Error()))
	}
}

func startServer(port string) {
	registerGzipHandler("/", indexHandler)
	registerGzipHandler("/icons", iconsHandler)
	registerHandler("/icon", iconHandler)
	registerGzipHandler("/popular", popularHandler)
	registerGzipHandler("/allicons.json", alliconsHandler)
	registerHandler("/lettericons/", lettericonHandler)
	registerHandler("/api/icons", obsoleteAPIHandler)

	serveAsset("/pure-0.5.0-min.css", "besticon/iconserver/assets/pure-0.5.0-min.css", oneYear)
	serveAsset("/grids-responsive-0.5.0-min.css", "besticon/iconserver/assets/grids-responsive-0.5.0-min.css", oneYear)
	serveAsset("/main-min.css", "besticon/iconserver/assets/main-min.css", oneYear)

	serveAsset("/icon.svg", "besticon/iconserver/assets/icon.svg", oneYear)
	serveAsset("/favicon.ico", "besticon/iconserver/assets/favicon.ico", oneYear)
	serveAsset("/apple-touch-icon.png", "besticon/iconserver/assets/apple-touch-icon.png", oneYear)

	addr := "0.0.0.0:" + port
	logger.Print("Starting server on ", addr, "...")
	e := http.ListenAndServe(addr, newLoggingMux())
	if e != nil {
		logger.Fatalf("cannot start server: %s\n", e)
	}
}

const (
	oneYear = 365 * 24 * 3600
	oneDay  = 24 * 3600
)

func redirectWithCacheControl(w http.ResponseWriter, r *http.Request, redirectURL string) {
	w.Header().Add(cacheControl, fmt.Sprintf("max-age=%d", oneDay))
	http.Redirect(w, r, redirectURL, 302)
}

func serveAsset(path string, assetPath string, maxAgeSeconds int) {
	registerGzipHandler(path, func(w http.ResponseWriter, r *http.Request) {
		assetInfo, err := assets.AssetInfo(assetPath)
		if err != nil {
			panic(err)
		}

		w.Header().Add(cacheControl, fmt.Sprintf("max-age=%d", maxAgeSeconds))

		http.ServeContent(w, r, assetInfo.Name(), assetInfo.ModTime(),
			bytes.NewReader(assets.MustAsset(assetPath)))
	})
}

func registerHandler(path string, f http.HandlerFunc) {
	http.Handle(path, newExpvarHandler(path, f))
}

func registerGzipHandler(path string, f http.HandlerFunc) {
	http.Handle(path, gziphandler.GzipHandler(newExpvarHandler(path, f)))
}

func main() {
	fmt.Printf("iconserver %s (%s) - https://icons.better-idea.org\n", besticon.VersionString, runtime.Version())
	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}
	startServer(port)
}

func init() {
	indexHTML = templateFromAsset("besticon/iconserver/assets/index.html", "index.html")
	iconsHTML = templateFromAsset("besticon/iconserver/assets/icons.html", "icons.html")
	popularHTML = templateFromAsset("besticon/iconserver/assets/popular.html", "popular.html")
	notFoundHTML = templateFromAsset("besticon/iconserver/assets/not_found.html", "not_found.html")
}

func templateFromAsset(assetPath, templateName string) *template.Template {
	bytes := assets.MustAsset(assetPath)
	return template.Must(template.New(templateName).Funcs(funcMap).Parse(string(bytes)))
}

var indexHTML *template.Template
var iconsHTML *template.Template
var popularHTML *template.Template
var notFoundHTML *template.Template

var funcMap = template.FuncMap{
	"ImgWidth": imgWidth,
}

func imgWidth(i *besticon.Icon) int {
	return i.Width / 2.0
}

func init() {
	cacheSize := os.Getenv("CACHE_SIZE_MB")
	if cacheSize == "" {
		besticon.SetCacheMaxSize(32)
	} else {
		n, _ := strconv.Atoi(cacheSize)
		besticon.SetCacheMaxSize(int64(n))
	}

	if besticon.CacheEnabled() {
		expvar.Publish("cacheBytes", expvar.Func(func() interface{} { return besticon.GetCacheStats().Bytes }))
		expvar.Publish("cacheItems", expvar.Func(func() interface{} { return besticon.GetCacheStats().Items }))
		expvar.Publish("cacheGets", expvar.Func(func() interface{} { return besticon.GetCacheStats().Gets }))
		expvar.Publish("cacheHits", expvar.Func(func() interface{} { return besticon.GetCacheStats().Hits }))
		expvar.Publish("cacheEvictions", expvar.Func(func() interface{} { return besticon.GetCacheStats().Evictions }))
	}
}
<commit_msg>Enable pprof runtime profiling<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"expvar"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"

	"github.com/NYTimes/gziphandler"
	"github.com/mat/besticon/besticon"
	"github.com/mat/besticon/besticon/iconserver/assets"
	"github.com/mat/besticon/lettericon"

	// Enable runtime profiling at /debug/pprof
	_ "net/http/pprof"
)

func indexHandler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "" || r.URL.Path == "/" {
		renderHTMLTemplate(w, 200, indexHTML, nil)
	} else {
		renderHTMLTemplate(w, 404, notFoundHTML, nil)
	}
}

func iconsHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue(urlParam)
	if len(url) == 0 {
		http.Redirect(w, r, "/", 302)
		return
	}

	finder := besticon.IconFinder{}

	formats := r.FormValue("formats")
	if formats != "" {
		finder.FormatsAllowed = strings.Split(r.FormValue("formats"), ",")
	}

	icons, e := finder.FetchIcons(url)
	switch {
	case e != nil:
		renderHTMLTemplate(w, 404, iconsHTML, pageInfo{URL: url, Error: e})
	case len(icons) == 0:
		errNoIcons := errors.New("this poor site has no icons at all :-(")
		renderHTMLTemplate(w, 404, iconsHTML, pageInfo{URL: url, Error: errNoIcons})
	default:
		renderHTMLTemplate(w, 200, iconsHTML, pageInfo{Icons: icons, URL: url})
	}
}

func iconHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue("url")
	if len(url) == 0 {
		writeAPIError(w, 400, errors.New("need url parameter"))
		return
	}

	size := r.FormValue("size")
	if size == "" {
		writeAPIError(w, 400, errors.New("need size parameter"))
		return
	}
	minSize, err := strconv.Atoi(size)
	if err != nil || minSize < besticon.MinIconSize || minSize > besticon.MaxIconSize {
		writeAPIError(w, 400, errors.New("bad size parameter"))
		return
	}

	finder := besticon.IconFinder{}
	formats := r.FormValue("formats")
	if formats != "" {
		finder.FormatsAllowed = strings.Split(r.FormValue("formats"), ",")
	}

	finder.FetchIcons(url)

	icon := finder.IconWithMinSize(minSize)
	if icon != nil {
		redirectWithCacheControl(w, r, icon.URL)
		return
	}

	fallbackIconURL := r.FormValue("fallback_icon_url")
	if fallbackIconURL != "" {
		redirectWithCacheControl(w, r, fallbackIconURL)
		return
	}

	iconColor := finder.MainColorForIcons()
	letter := lettericon.MainLetterFromURL(url)
	redirectPath := lettericon.IconPath(letter, size, iconColor)
	redirectWithCacheControl(w, r, redirectPath)
}

func popularHandler(w http.ResponseWriter, r *http.Request) {
	iconSize, err := strconv.Atoi(r.FormValue("iconsize"))
	if iconSize > besticon.MaxIconSize || iconSize < besticon.MinIconSize || err != nil {
		iconSize = 120
	}

	pageInfo := struct {
		URLs        []string
		IconSize    int
		DisplaySize int
	}{
		besticon.PopularSites,
		iconSize,
		iconSize / 2,
	}
	renderHTMLTemplate(w, 200, popularHTML, pageInfo)
}

const (
	urlParam = "url"
)

func alliconsHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue(urlParam)
	if len(url) == 0 {
		errMissingURL := errors.New("need url query parameter")
		writeAPIError(w, 400, errMissingURL)
		return
	}

	finder := besticon.IconFinder{}
	formats := r.FormValue("formats")
	if formats != "" {
		finder.FormatsAllowed = strings.Split(r.FormValue("formats"), ",")
	}

	icons, e := finder.FetchIcons(url)
	if e != nil {
		writeAPIError(w, 404, e)
		return
	}

	writeAPIIcons(w, url, icons)
}

func lettericonHandler(w http.ResponseWriter, r *http.Request) {
	charParam, col, size := lettericon.ParseIconPath(r.URL.Path)
	if charParam == "" {
		writeAPIError(w, 400, errors.New("wrong format for lettericons/ path, must look like lettericons/M-144-EFC25D.png"))
	}

	w.Header().Add(contentType, imagePNG)
	w.Header().Add(cacheControl, fmt.Sprintf("max-age=%d", oneYear))
	lettericon.Render(charParam, col, size, w)
}

func obsoleteAPIHandler(w http.ResponseWriter, r *http.Request) {
	if r.FormValue("i_am_feeling_lucky") == "yes" {
		http.Redirect(w, r, fmt.Sprintf("/icon?size=120&%s", r.URL.RawQuery), 302)
	} else {
		http.Redirect(w, r, fmt.Sprintf("/allicons.json?%s", r.URL.RawQuery), 302)
	}
}

func writeAPIError(w http.ResponseWriter, httpStatus int, e error) {
	data := struct {
		Error string `json:"error"`
	}{
		e.Error(),
	}
	renderJSONResponse(w, httpStatus, data)
}

func writeAPIIcons(w http.ResponseWriter, url string, icons []besticon.Icon) {
	// Don't return whole image data
	newIcons := []besticon.Icon{}
	for _, ico := range icons {
		newIcon := ico
		newIcon.ImageData = nil
		newIcons = append(newIcons, newIcon)
	}

	data := &struct {
		URL   string          `json:"url"`
		Icons []besticon.Icon `json:"icons"`
	}{
		url,
		newIcons,
	}
	renderJSONResponse(w, 200, data)
}

const (
	contentType     = "Content-Type"
	applicationJSON = "application/json"
	imagePNG        = "image/png"
	cacheControl    = "Cache-Control"
)

func renderJSONResponse(w http.ResponseWriter, httpStatus int, data interface{}) {
	w.Header().Add(contentType, applicationJSON)
	w.WriteHeader(httpStatus)
	enc := json.NewEncoder(w)
	enc.Encode(data)
}

type pageInfo struct {
	URL   string
	Icons []besticon.Icon
	Error error
}

func (pi pageInfo) Host() string {
	u := pi.URL
	url, _ := url.Parse(u)
	if url != nil && url.Host != "" {
		return url.Host
	}
	return pi.URL
}

func (pi pageInfo) Best() string {
	if len(pi.Icons) > 0 {
		best := pi.Icons[0]
		return best.URL
	}
	return ""
}

func renderHTMLTemplate(w http.ResponseWriter, httpStatus int, templ *template.Template, data interface{}) {
	w.Header().Add(contentType, "text/html; charset=utf-8")
	w.WriteHeader(httpStatus)

	err := templ.Execute(w, data)
	if err != nil {
		err = fmt.Errorf("server: could not generate output: %s", err)
		logger.Print(err)
		w.Write([]byte(err.Error()))
	}
}

func startServer(port string) {
	registerGzipHandler("/", indexHandler)
	registerGzipHandler("/icons", iconsHandler)
	registerHandler("/icon", iconHandler)
	registerGzipHandler("/popular", popularHandler)
	registerGzipHandler("/allicons.json", alliconsHandler)
	registerHandler("/lettericons/", lettericonHandler)
	registerHandler("/api/icons", obsoleteAPIHandler)

	serveAsset("/pure-0.5.0-min.css", "besticon/iconserver/assets/pure-0.5.0-min.css", oneYear)
	serveAsset("/grids-responsive-0.5.0-min.css", "besticon/iconserver/assets/grids-responsive-0.5.0-min.css", oneYear)
	serveAsset("/main-min.css", "besticon/iconserver/assets/main-min.css", oneYear)

	serveAsset("/icon.svg", "besticon/iconserver/assets/icon.svg", oneYear)
	serveAsset("/favicon.ico", "besticon/iconserver/assets/favicon.ico", oneYear)
	serveAsset("/apple-touch-icon.png", "besticon/iconserver/assets/apple-touch-icon.png", oneYear)

	addr := "0.0.0.0:" + port
	logger.Print("Starting server on ", addr, "...")
	e := http.ListenAndServe(addr, newLoggingMux())
	if e != nil {
		logger.Fatalf("cannot start server: %s\n", e)
	}
}

const (
	oneYear = 365 * 24 * 3600
	oneDay  = 24 * 3600
)

func redirectWithCacheControl(w http.ResponseWriter, r *http.Request, redirectURL string) {
	w.Header().Add(cacheControl, fmt.Sprintf("max-age=%d", oneDay))
	http.Redirect(w, r, redirectURL, 302)
}

func serveAsset(path string, assetPath string, maxAgeSeconds int) {
	registerGzipHandler(path, func(w http.ResponseWriter, r *http.Request) {
		assetInfo, err := assets.AssetInfo(assetPath)
		if err != nil {
			panic(err)
		}

		w.Header().Add(cacheControl, fmt.Sprintf("max-age=%d", maxAgeSeconds))

		http.ServeContent(w, r, assetInfo.Name(), assetInfo.ModTime(),
			bytes.NewReader(assets.MustAsset(assetPath)))
	})
}

func registerHandler(path string, f http.HandlerFunc) {
	http.Handle(path, newExpvarHandler(path, f))
}

func registerGzipHandler(path string, f http.HandlerFunc) {
	http.Handle(path, gziphandler.GzipHandler(newExpvarHandler(path, f)))
}

func main() {
	fmt.Printf("iconserver %s (%s) - https://icons.better-idea.org\n", besticon.VersionString, runtime.Version())
	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}
	startServer(port)
}

func init() {
	indexHTML = templateFromAsset("besticon/iconserver/assets/index.html", "index.html")
	iconsHTML = templateFromAsset("besticon/iconserver/assets/icons.html", "icons.html")
	popularHTML = templateFromAsset("besticon/iconserver/assets/popular.html", "popular.html")
	notFoundHTML = templateFromAsset("besticon/iconserver/assets/not_found.html", "not_found.html")
}

func templateFromAsset(assetPath, templateName string) *template.Template {
	bytes := assets.MustAsset(assetPath)
	return template.Must(template.New(templateName).Funcs(funcMap).Parse(string(bytes)))
}

var indexHTML *template.Template
var iconsHTML *template.Template
var popularHTML *template.Template
var notFoundHTML *template.Template

var funcMap = template.FuncMap{
	"ImgWidth": imgWidth,
}

func imgWidth(i *besticon.Icon) int {
	return i.Width / 2.0
}

func init() {
	cacheSize := os.Getenv("CACHE_SIZE_MB")
	if cacheSize == "" {
		besticon.SetCacheMaxSize(32)
	} else {
		n, _ := strconv.Atoi(cacheSize)
		besticon.SetCacheMaxSize(int64(n))
	}

	if besticon.CacheEnabled() {
		expvar.Publish("cacheBytes", expvar.Func(func() interface{} { return besticon.GetCacheStats().Bytes }))
		expvar.Publish("cacheItems", expvar.Func(func() interface{} { return besticon.GetCacheStats().Items }))
		expvar.Publish("cacheGets", expvar.Func(func() interface{} { return besticon.GetCacheStats().Gets }))
		expvar.Publish("cacheHits", expvar.Func(func() interface{} { return besticon.GetCacheStats().Hits }))
		expvar.Publish("cacheEvictions", expvar.Func(func() interface{} { return besticon.GetCacheStats().Evictions }))
	}
}
<|endoftext|>


<commit_before>package tasks

import (
	"../../proto"
	"errors"
	"os/exec"
	"path/filepath"
	"strings"
)

type Config struct {
	Root string
}

type Task struct {
	Semaphore chan int
	Config    *Config
}

func (t *Task) CompileC(args *proto.Args, resp *proto.Response) error {
	t.Semaphore <- 1
	defer func() { <-t.Semaphore }()

	var outfilename string
	if len(args.Inputs) != 1 {
		return errors.New("Compile requires one file to be compiled!")
	}

	if !strings.HasSuffix(args.Inputs[0].Filename, ".c") {
		outfilename = args.Inputs[0].Filename + ".o"
	} else {
		outfilename = strings.TrimSuffix(args.Inputs[0].Filename, ".c") + ".o"
	}

	outfile_path := filepath.Join(t.Config.Root, "BINFILES", outfilename)
	infile_path := filepath.Join(t.Config.Root, args.Inputs[0].Filename)

	output, err := exec.Command("gcc", "-std=c99", "-c", "-o",
		outfile_path, infile_path).CombinedOutput()

	if err != nil {
		return err
	}

	err = prepareResponse(outfile_path, output, args.SendContent, resp)
	return err
}

func (t *Task) ArLink(args *proto.Args, resp *proto.Response) error {
	t.Semaphore <- 1
	defer func() { <-t.Semaphore }()

	var outfilename string
	if len(args.Inputs) < 1 {
		return errors.New("Library linking requires at least one file!")
	}

	outfilename = args.Name + ".a"
	outdir := filepath.Join(t.Config.Root, "BINFILES")

	outfile_path := filepath.Join(outdir, outfilename)

	ar_args := processInputs(args.Inputs, outdir, []string{"rcs", outfile_path})

	output, err := exec.Command("ar", ar_args...).CombinedOutput()
	if err != nil {
		return err
	}

	err = prepareResponse(outfile_path, output, args.SendContent, resp)
	return err
}

func (t *Task) LdLink(args *proto.Args, resp *proto.Response) error {
	t.Semaphore <- 1
	defer func() { <-t.Semaphore }()

	var outfilename string
	if len(args.Inputs) < 1 {
		return errors.New("Application linking required at least one file!")
	}

	outfilename = args.Name
	outdir := filepath.Join(t.Config.Root, "BINFILES")

	outfile_path := filepath.Join(outdir, outfilename)

	ld_args := processInputs(args.Inputs, outdir, []string{"-o", outfile_path})

	output, err := exec.Command("gcc", ld_args...).CombinedOutput()
	if err != nil {
		return err
	}

	err = prepareResponse(outfile_path, output, args.SendContent, resp)
	return err
}
<commit_msg>worker: include .h files in CompileC inputs<commit_after>package tasks

import (
	"../../proto"
	"errors"
	"os/exec"
	"path/filepath"
	"strings"
)

type Config struct {
	Root string
}

type Task struct {
	Semaphore chan int
	Config    *Config
}

func (t *Task) CompileC(args *proto.Args, resp *proto.Response) error {
	t.Semaphore <- 1
	defer func() { <-t.Semaphore }()

	var outfilename string
	if len(args.Inputs) < 1 {
		return errors.New("compile requires at least one file to be compiled")
	}

	if !strings.HasSuffix(args.Inputs[0].Filename, ".c") {
		outfilename = args.Inputs[0].Filename + ".o"
	} else {
		outfilename = strings.TrimSuffix(args.Inputs[0].Filename, ".c") + ".o"
	}

	outfile_path := filepath.Join(t.Config.Root, outfilename)
	infile_path := filepath.Join(t.Config.Root, args.Inputs[0].Filename)

	output, err := exec.Command("gcc", "-std=c99", "-c", "-o",
		outfile_path, infile_path).CombinedOutput()

	if err != nil {
		return err
	}

	err = prepareResponse(outfile_path, output, args.SendContent, resp)
	return err
}

func (t *Task) ArLink(args *proto.Args, resp *proto.Response) error {
	t.Semaphore <- 1
	defer func() { <-t.Semaphore }()

	var outfilename string
	if len(args.Inputs) < 1 {
		return errors.New("library linking requires at least one file")
	}

	outfilename = args.Name + ".a"
	outdir := t.Config.Root

	outfile_path := filepath.Join(outdir, outfilename)

	ar_args := processInputs(args.Inputs, outdir, []string{"rcs", outfile_path})

	output, err := exec.Command("ar", ar_args...).CombinedOutput()
	if err != nil {
		return err
	}

	err = prepareResponse(outfile_path, output, args.SendContent, resp)
	return err
}

func (t *Task) LdLink(args *proto.Args, resp *proto.Response) error {
	t.Semaphore <- 1
	defer func() { <-t.Semaphore }()

	var outfilename string
	if len(args.Inputs) < 1 {
		return errors.New("application linking requires at least one file")
	}

	outfilename = args.Name
	outdir := t.Config.Root

	outfile_path := filepath.Join(outdir, outfilename)

	ld_args := processInputs(args.Inputs, outdir, []string{"-o", outfile_path})

	output, err := exec.Command("gcc", ld_args...).CombinedOutput()
	if err != nil {
		return err
	}

	err = prepareResponse(outfile_path, output, args.SendContent, resp)
	return err
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type TriggerDB struct {
	*ResourceDB
	mapping string
}

func NewTriggerDB(service *WorkflowService, esi elasticsearch.IIndex) (*TriggerDB, error) {

	rdb, err := NewResourceDB(service, esi, TriggerIndexSettings)
	if err != nil {
		return nil, err
	}
	ardb := TriggerDB{ResourceDB: rdb, mapping: TriggerDBMapping}
	return &ardb, nil
}

func (db *TriggerDB) PostTrigger(trigger *Trigger, id piazza.Ident) (piazza.Ident, error) {

	{ //CHECK SERVICE EXISTS
		jobData := trigger.Job.JobType.Data
		serviceId := jobData["serviceId"]
		strServiceId, ok := serviceId.(string)
		if !ok {
			return piazza.NoIdent, LoggedError("TriggerDB.PostData faile: serviceId field not of type string")
		}
		serviceControllerURL, err := db.service.sys.GetAddress("pz-servicecontroller")
		if err != nil {
			return piazza.NoIdent, LoggedError("TriggerDB.PostData failed to find ServiceController: %s", err)
		}
		serviceControllerURL = "http://" + serviceControllerURL + "/service/" + strServiceId
		req, err := http.NewRequest("GET", serviceControllerURL, nil)
		if err != nil {
			return piazza.NoIdent, LoggedError("TriggerDB.PostData failed to make request to ServiceController: %s", err)
		}
		client := &http.Client{}
		response, err := client.Do(req)
		if err != nil {
			return piazza.NoIdent, LoggedError("TriggerDB.PostData failed to reach ServiceController: %s", err)
		}
		if response.StatusCode != 200 {
			return piazza.NoIdent, LoggedError("TriggerDB.PostData: serviceId %s does not exist", strServiceId)
		}
	}

	ifaceObj := trigger.Condition.Query
	//log.Printf("Query: %v", ifaceObj)
	body, err := json.Marshal(ifaceObj)
	if err != nil {
		return piazza.NoIdent, err
	}

	json := string(body)
	//log.Printf("Current json: %s", json)
	// Remove trailing }
	json = json[:len(json)-1]
	json += ",\"type\":["
	// Add the types that the percolation query can match
	for _, id := range trigger.Condition.EventTypeIds {
		json += fmt.Sprintf("\"%s\",", id)
	}
	json = json[:len(json)-1]
	// Add back trailing } and ] to close array
	json += "]}"

	//log.Printf("Posting percolation query: %s", body)
	indexResult, err := db.service.eventDB.Esi.AddPercolationQuery(string(trigger.TriggerId), piazza.JsonString(body))
	if err != nil {
		return piazza.NoIdent, LoggedError("TriggerDB.PostData addpercquery failed: %s", err)
	}
	if indexResult == nil {
		return piazza.NoIdent, LoggedError("TriggerDB.PostData addpercquery failed: no indexResult")
	}
	if !indexResult.Created {
		return piazza.NoIdent, LoggedError("TriggerDB.PostData addpercquery failed: not created")
	}

	//log.Printf("percolation query added: ID: %s, Type: %s, Index: %s", indexResult.Id, indexResult.Type, indexResult.Index)
	//log.Printf("percolation id: %s", indexResult.Id)
	trigger.PercolationId = piazza.Ident(indexResult.Id)

	indexResult2, err := db.Esi.PostData(db.mapping, id.String(), trigger)
	if err != nil {
		db.service.eventDB.Esi.DeletePercolationQuery(string(trigger.TriggerId))
		return piazza.NoIdent, LoggedError("TriggerDB.PostData failed: %s", err)
	}
	if !indexResult2.Created {
		db.service.eventDB.Esi.DeletePercolationQuery(string(trigger.TriggerId))
		return piazza.NoIdent, LoggedError("TriggerDB.PostData failed: not created")
	}

	return id, nil
}

func (db *TriggerDB) GetAll(format *piazza.JsonPagination) ([]Trigger, int64, error) {
	triggers := []Trigger{}

	exists := db.Esi.TypeExists(db.mapping)
	if !exists {
		return triggers, 0, nil
	}

	searchResult, err := db.Esi.FilterByMatchAll(db.mapping, format)
	if err != nil {
		return nil, 0, LoggedError("TriggerDB.GetAll failed: %s", err)
	}
	if searchResult == nil {
		return nil, 0, LoggedError("TriggerDB.GetAll failed: no searchResult")
	}

	if searchResult != nil && searchResult.GetHits() != nil {

		for _, hit := range *searchResult.GetHits() {
			var trigger Trigger
			err := json.Unmarshal(*hit.Source, &trigger)
			if err != nil {
				return nil, 0, err
			}
			triggers = append(triggers, trigger)
		}
	}
	return triggers, searchResult.TotalHits(), nil
}

func (db *TriggerDB) GetOne(id piazza.Ident) (*Trigger, error) {

	getResult, err := db.Esi.GetByID(db.mapping, id.String())
	if err != nil {
		return nil, LoggedError("TriggerDB.GetOne failed: %s", err)
	}
	if getResult == nil {
		return nil, LoggedError("TriggerDB.GetOne failed: no getResult")
	}

	if !getResult.Found {
		return nil, nil
	}

	src := getResult.Source
	var obj Trigger
	err = json.Unmarshal(*src, &obj)
	if err != nil {
		return nil, err
	}

	return &obj, nil
}

func (db *TriggerDB) DeleteTrigger(id piazza.Ident) (bool, error) {

	trigger, err := db.GetOne(id)
	if err != nil {
		return false, err
	}
	if trigger == nil {
		return false, nil
	}

	deleteResult, err := db.Esi.DeleteByID(db.mapping, string(id))
	if err != nil {
		return deleteResult.Found, LoggedError("TriggerDB.DeleteById failed: %s", err)
	}
	if deleteResult == nil {
		return false, LoggedError("TriggerDB.DeleteById failed: no deleteResult")
	}
	if !deleteResult.Found {
		return false, nil
	}

	deleteResult2, err := db.service.eventDB.Esi.DeletePercolationQuery(string(trigger.PercolationId))
	if err != nil {
		return deleteResult2.Found, LoggedError("TriggerDB.DeleteById percquery failed: %s", err)
	}
	if deleteResult2 == nil {
		return false, LoggedError("TriggerDB.DeleteById percquery failed: no deleteResult")
	}

	return deleteResult2.Found, nil
}
<commit_msg>quick fixes to shorten code<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type TriggerDB struct {
	*ResourceDB
	mapping string
}

func NewTriggerDB(service *WorkflowService, esi elasticsearch.IIndex) (*TriggerDB, error) {

	rdb, err := NewResourceDB(service, esi, TriggerIndexSettings)
	if err != nil {
		return nil, err
	}
	ardb := TriggerDB{ResourceDB: rdb, mapping: TriggerDBMapping}
	return &ardb, nil
}

func (db *TriggerDB) PostTrigger(trigger *Trigger, id piazza.Ident) (piazza.Ident, error) {

	{ //CHECK SERVICE EXISTS
		jobData := trigger.Job.JobType.Data
		serviceId := jobData["serviceId"]
		strServiceId, ok := serviceId.(string)
		if !ok {
			return piazza.NoIdent, LoggedError("TriggerDB.PostData faile: serviceId field not of type string")
		}
		serviceControllerURL, err := db.service.sys.GetURL("pz-servicecontroller")
		if err != nil {
			return piazza.NoIdent, LoggedError("TriggerDB.PostData failed to find ServiceController: %s", err)
		}
		response, err := http.Get(serviceControllerURL)
		if err != nil {
			return piazza.NoIdent, LoggedError("TriggerDB.PostData failed to make request to ServiceController: %s", err)
		}
		if response.StatusCode != 200 {
			return piazza.NoIdent, LoggedError("TriggerDB.PostData: serviceId %s does not exist", strServiceId)
		}
	}

	ifaceObj := trigger.Condition.Query
	//log.Printf("Query: %v", ifaceObj)
	body, err := json.Marshal(ifaceObj)
	if err != nil {
		return piazza.NoIdent, err
	}

	json := string(body)
	//log.Printf("Current json: %s", json)
	// Remove trailing }
	json = json[:len(json)-1]
	json += ",\"type\":["
	// Add the types that the percolation query can match
	for _, id := range trigger.Condition.EventTypeIds {
		json += fmt.Sprintf("\"%s\",", id)
	}
	json = json[:len(json)-1]
	// Add back trailing } and ] to close array
	json += "]}"

	//log.Printf("Posting percolation query: %s", body)
	indexResult, err := db.service.eventDB.Esi.AddPercolationQuery(string(trigger.TriggerId), piazza.JsonString(body))
	if err != nil {
		return piazza.NoIdent, LoggedError("TriggerDB.PostData addpercquery failed: %s", err)
	}
	if indexResult == nil {
		return piazza.NoIdent, LoggedError("TriggerDB.PostData addpercquery failed: no indexResult")
	}
	if !indexResult.Created {
		return piazza.NoIdent, LoggedError("TriggerDB.PostData addpercquery failed: not created")
	}

	//log.Printf("percolation query added: ID: %s, Type: %s, Index: %s", indexResult.Id, indexResult.Type, indexResult.Index)
	//log.Printf("percolation id: %s", indexResult.Id)
	trigger.PercolationId = piazza.Ident(indexResult.Id)

	indexResult2, err := db.Esi.PostData(db.mapping, id.String(), trigger)
	if err != nil {
		db.service.eventDB.Esi.DeletePercolationQuery(string(trigger.TriggerId))
		return piazza.NoIdent, LoggedError("TriggerDB.PostData failed: %s", err)
	}
	if !indexResult2.Created {
		db.service.eventDB.Esi.DeletePercolationQuery(string(trigger.TriggerId))
		return piazza.NoIdent, LoggedError("TriggerDB.PostData failed: not created")
	}

	return id, nil
}

func (db *TriggerDB) GetAll(format *piazza.JsonPagination) ([]Trigger, int64, error) {
	triggers := []Trigger{}

	exists := db.Esi.TypeExists(db.mapping)
	if !exists {
		return triggers, 0, nil
	}

	searchResult, err := db.Esi.FilterByMatchAll(db.mapping, format)
	if err != nil {
		return nil, 0, LoggedError("TriggerDB.GetAll failed: %s", err)
	}
	if searchResult == nil {
		return nil, 0, LoggedError("TriggerDB.GetAll failed: no searchResult")
	}

	if searchResult != nil && searchResult.GetHits() != nil {

		for _, hit := range *searchResult.GetHits() {
			var trigger Trigger
			err := json.Unmarshal(*hit.Source, &trigger)
			if err != nil {
				return nil, 0, err
			}
			triggers = append(triggers, trigger)
		}
	}
	return triggers, searchResult.TotalHits(), nil
}

func (db *TriggerDB) GetOne(id piazza.Ident) (*Trigger, error) {

	getResult, err := db.Esi.GetByID(db.mapping, id.String())
	if err != nil {
		return nil, LoggedError("TriggerDB.GetOne failed: %s", err)
	}
	if getResult == nil {
		return nil, LoggedError("TriggerDB.GetOne failed: no getResult")
	}

	if !getResult.Found {
		return nil, nil
	}

	src := getResult.Source
	var obj Trigger
	err = json.Unmarshal(*src, &obj)
	if err != nil {
		return nil, err
	}

	return &obj, nil
}

func (db *TriggerDB) DeleteTrigger(id piazza.Ident) (bool, error) {

	trigger, err := db.GetOne(id)
	if err != nil {
		return false, err
	}
	if trigger == nil {
		return false, nil
	}

	deleteResult, err := db.Esi.DeleteByID(db.mapping, string(id))
	if err != nil {
		return deleteResult.Found, LoggedError("TriggerDB.DeleteById failed: %s", err)
	}
	if deleteResult == nil {
		return false, LoggedError("TriggerDB.DeleteById failed: no deleteResult")
	}
	if !deleteResult.Found {
		return false, nil
	}

	deleteResult2, err := db.service.eventDB.Esi.DeletePercolationQuery(string(trigger.PercolationId))
	if err != nil {
		return deleteResult2.Found, LoggedError("TriggerDB.DeleteById percquery failed: %s", err)
	}
	if deleteResult2 == nil {
		return false, LoggedError("TriggerDB.DeleteById percquery failed: no deleteResult")
	}

	return deleteResult2.Found, nil
}
<|endoftext|>


<commit_before>package godbg

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"regexp"
	"runtime"
	"strings"
)

// http://stackoverflow.com/a/23554672/6309 https://vividcortex.com/blog/2013/12/03/go-idiom-package-and-object/
// you design a type with methods as usual, and then you also place matching functions at the package level itself.
// These functions simply delegate to a default instance of the type that’s a private package-level variable, created in an init() function.

// Pdbg allows to print debug message with indent and function name added
type Pdbg struct {
	bout     *bytes.Buffer
	berr     *bytes.Buffer
	sout     *bufio.Writer
	serr     *bufio.Writer
	breaks   []string
	excludes []string
	skips    []string
}

// Out returns a writer for normal messages.
// By default, os.StdOut
func Out() io.Writer {
	return pdbg.Out()
}

// Out returns a writer for normal messages for a given pdbg instance.
// By default, os.StdOut
func (pdbg *Pdbg) Out() io.Writer {
	if pdbg.sout == nil {
		return os.Stdout
	}
	return pdbg.sout
}

// Err returns a writer for error messages.
// By default, os.StdErr
func Err() io.Writer {
	return pdbg.Err()
}

// Err returns a writer for error messages for a given pdbg instance.
// By default, os.StdErr
func (pdbg *Pdbg) Err() io.Writer {
	if pdbg.serr == nil {
		return os.Stderr
	}
	return pdbg.serr
}

// global pdbg used for printing
var pdbg = NewPdbg()

// Option set an option for a Pdbg
// http://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis
type Option func(*Pdbg)

// SetBuffers is an option for replacing stdout and stderr by
// bytes buffers (in a bufio.Writer).
// If apdbg is nil, set for the global pdbg instance
func SetBuffers(apdbg *Pdbg) {
	if apdbg == nil {
		apdbg = pdbg
	}
	apdbg.bout = bytes.NewBuffer(nil)
	apdbg.sout = bufio.NewWriter(apdbg.bout)
	apdbg.berr = bytes.NewBuffer(nil)
	apdbg.serr = bufio.NewWriter(apdbg.berr)
}

// SetExcludes set excludes on a pdbg (nil for global pdbg)
func (pdbg *Pdbg) SetExcludes(excludes []string) {
	pdbg.excludes = excludes
}

// OptExcludes is an option to set excludes at the creation of a pdbg
func OptExcludes(excludes []string) Option {
	return func(apdbg *Pdbg) {
		apdbg.SetExcludes(excludes)
	}
}

// SetExcludes set skips on a pdbg (nil for global pdbg)
func (pdbg *Pdbg) SetSkips(skips []string) {
	sk := []string{"/godbg.go'"}
	sk = append(sk, skips...)
	pdbg.skips = sk
}

// OptExcludes is an option to set excludes at the creation of a pdbg
func OptSkips(skips []string) Option {
	return func(apdbg *Pdbg) {
		apdbg.SetSkips(skips)
	}
}

// NewPdbg creates a PDbg instance, with options
func NewPdbg(options ...Option) *Pdbg {
	newpdbg := &Pdbg{}
	for _, option := range options {
		option(newpdbg)
	}
	newpdbg.breaks = append(newpdbg.breaks, "smartystreets")
	//newpdbg.breaks = append(newpdbg.breaks, "(*Pdbg).Pdbgf")
	newpdbg.skips = append(newpdbg.skips, "/godbg.go'")
	return newpdbg
}

// ResetIOs reset the out and err buffer of global pdbg instance
func ResetIOs() {
	pdbg.ResetIOs()
}

// ResetIOs reset the out and err buffer
// (unless they were the default stdout and stderr,
// in which case it does nothing)
func (pdbg *Pdbg) ResetIOs() {
	if pdbg.sout != nil {
		pdbg.bout = bytes.NewBuffer(nil)
		pdbg.sout.Reset(pdbg.bout)
		pdbg.berr = bytes.NewBuffer(nil)
		pdbg.serr.Reset(pdbg.berr)
	}
}

// OutString returns the string for out messages for the global pdbg instance.
// It flushes the out buffer.
// If out is set to os.Stdout, returns an empty string
func OutString() string {
	return pdbg.OutString()
}

// OutString returns the string for out messages for a given pdbg instance.
// It flushes the out buffer.
// If out is set to os.Stdout, returns an empty string
func (pdbg *Pdbg) OutString() string {
	if pdbg.sout == nil {
		return ""
	}
	pdbg.sout.Flush()
	return pdbg.bout.String()
}

// ErrString returns the string for error messages for the global pdbg instance.
// It flushes the err buffer.
// If err is set to os.StdErr, returns an empty string
func ErrString() string {
	return pdbg.ErrString()
}

// ErrString returns the string for error messages for a given pdbg instance.
// It flushes the err buffer.
// If err is set to os.StdErr, returns an empty string
func (pdbg *Pdbg) ErrString() string {
	if pdbg.serr == nil {
		return ""
	}
	pdbg.serr.Flush()
	return pdbg.berr.String()
}

func (pdbg *Pdbg) pdbgExcluded(dbg string) bool {
	for _, e := range pdbg.excludes {
		if strings.Contains(dbg, e) {
			// fmt.Printf("EXCLUDE over '%v' including '%v'\n", dbg, e) // DBG
			return true
		}
	}
	return false
}

func (pdbg *Pdbg) pdbgBreak(dbg string) bool {
	for _, b := range pdbg.breaks {
		if strings.Contains(dbg, b) {
			// fmt.Printf("BREAK over '%v' including '%v'\n", dbg, b) // DBG
			return true
		}
	}
	return false
}

func (pdbg *Pdbg) pdbgSkip(dbg string) (bool, int) {
	depthToAdd := 0
	for i, s := range pdbg.skips {
		if strings.Contains(dbg, s) {
			if i > 0 {
				depthToAdd = 1
			}
			// fmt.Printf("SKIP over '%v' including '%v'\n", dbg, s) // DBG
			return true, depthToAdd
		}
	}
	return false, depthToAdd
}

// Pdbgf uses global Pdbg variable for printing strings, with indent and function name
func Pdbgf(format string, args ...interface{}) string {
	return pdbg.Pdbgf(format, args...)
}

type caller func(skip int) (pc uintptr, file string, line int, ok bool)

var mycaller = runtime.Caller

// Pdbgf uses custom Pdbg variable for printing strings, with indent and function name
func (pdbg *Pdbg) Pdbgf(format string, args ...interface{}) string {
	msg := fmt.Sprintf(format+"\n", args...)
	msg = strings.TrimSpace(msg)

	pmsg := ""
	depth := 0
	nbskip := 0
	nbInitialSkips := 0
	first := true
	addOneForSkip := 0
	// fmt.Printf("~~~~~~~~~~~~~~~~~~~~~~\n") // DBG
	for ok := true; ok; {
		pc, file, line, ok := mycaller(depth)
		if !ok {
			break
		}
		fname := runtime.FuncForPC(pc).Name()
		fline := fmt.Sprintf("Name of function: '%v': '%+x' (line %v): file '%v'\n", fname, fname, line, file)
		// fmt.Println(fline) // DBG
		if pdbg.pdbgExcluded(fline) {
			depth = depth + 1
			if first {
				return ""
			}
			continue
		}
		if pdbg.pdbgBreak(fline) {
			break
		}
		if isSkipped, depthToAdd := pdbg.pdbgSkip(fline); isSkipped {
			depth = depth + 1
			nbskip = nbskip + 1
			addOneForSkip = addOneForSkip + depthToAdd
			continue
		}
		fnamerx1 := regexp.MustCompile(`.*\.func[^a-zA-Z0-9]`)
		fname = fnamerx1.ReplaceAllString(fname, "func.")
		fnamerx2 := regexp.MustCompile(`.*/`)
		fname = fnamerx2.ReplaceAllString(fname, "")
		if !strings.HasPrefix(fname, "func.") {
			fnamerx3 := regexp.MustCompile(`^.*?\.`)
			// fmt.Printf("fname before: '%v'", fname)
			fname = fnamerx3.ReplaceAllString(fname, "")
			// fmt.Printf(" => fname after: '%v'\n", fname)
			fnamerx4 := regexp.MustCompile(`[\(\)]`)
			fname = fnamerx4.ReplaceAllString(fname, "")
		}
		dbg := fname + ":" + fmt.Sprintf("%d", line)
		if first {
			// fmt.Printf(" => nbskip '%v'; addOneForSkip '%v'\n", nbskip, addOneForSkip) // DBG
			nbInitialSkips = nbskip - addOneForSkip
			pmsg = "[" + dbg + "]"
		} else {
			pmsg = pmsg + " (" + dbg + ")"
		}
		first = false
		depth = depth + 1
	}
	finalDepth := depth
	depth = finalDepth - nbInitialSkips - 1

	spaces := ""
	if depth >= 0 {
		spaces = strings.Repeat(" ", depth*2)
	}
	// fmt.Printf("spaces '%s', finalDepth '%d', depth '%d', nbInitialSkips '%d', addOneForSkip='%d'\n", spaces, finalDepth, depth, nbInitialSkips, addOneForSkip) // DBG
	res := pmsg
	if pmsg != "" {
		pmsg = spaces + pmsg + "\n"
	}
	msg = pmsg + spaces + "  " + msg + "\n"
	// fmt.Printf("==> MSG '%v'\n", msg) // DBG
	fmt.Fprint(pdbg.Err(), fmt.Sprint(msg))
	return res
}
<commit_msg>Fix comment on OptSkips<commit_after>package godbg

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"regexp"
	"runtime"
	"strings"
)

// http://stackoverflow.com/a/23554672/6309 https://vividcortex.com/blog/2013/12/03/go-idiom-package-and-object/
// you design a type with methods as usual, and then you also place matching functions at the package level itself.
// These functions simply delegate to a default instance of the type that’s a private package-level variable, created in an init() function.

// Pdbg allows to print debug message with indent and function name added
type Pdbg struct {
	bout     *bytes.Buffer
	berr     *bytes.Buffer
	sout     *bufio.Writer
	serr     *bufio.Writer
	breaks   []string
	excludes []string
	skips    []string
}

// Out returns a writer for normal messages.
// By default, os.StdOut
func Out() io.Writer {
	return pdbg.Out()
}

// Out returns a writer for normal messages for a given pdbg instance.
// By default, os.StdOut
func (pdbg *Pdbg) Out() io.Writer {
	if pdbg.sout == nil {
		return os.Stdout
	}
	return pdbg.sout
}

// Err returns a writer for error messages.
// By default, os.StdErr
func Err() io.Writer {
	return pdbg.Err()
}

// Err returns a writer for error messages for a given pdbg instance.
// By default, os.StdErr
func (pdbg *Pdbg) Err() io.Writer {
	if pdbg.serr == nil {
		return os.Stderr
	}
	return pdbg.serr
}

// global pdbg used for printing
var pdbg = NewPdbg()

// Option set an option for a Pdbg
// http://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis
type Option func(*Pdbg)

// SetBuffers is an option for replacing stdout and stderr by
// bytes buffers (in a bufio.Writer).
// If apdbg is nil, set for the global pdbg instance
func SetBuffers(apdbg *Pdbg) {
	if apdbg == nil {
		apdbg = pdbg
	}
	apdbg.bout = bytes.NewBuffer(nil)
	apdbg.sout = bufio.NewWriter(apdbg.bout)
	apdbg.berr = bytes.NewBuffer(nil)
	apdbg.serr = bufio.NewWriter(apdbg.berr)
}

// SetExcludes set excludes on a pdbg (nil for global pdbg)
func (pdbg *Pdbg) SetExcludes(excludes []string) {
	pdbg.excludes = excludes
}

// OptExcludes is an option to set excludes at the creation of a pdbg
func OptExcludes(excludes []string) Option {
	return func(apdbg *Pdbg) {
		apdbg.SetExcludes(excludes)
	}
}

// SetExcludes set skips on a pdbg (nil for global pdbg)
func (pdbg *Pdbg) SetSkips(skips []string) {
	sk := []string{"/godbg.go'"}
	sk = append(sk, skips...)
	pdbg.skips = sk
}

// OptSkips is an option to set excludes at the creation of a pdbg
func OptSkips(skips []string) Option {
	return func(apdbg *Pdbg) {
		apdbg.SetSkips(skips)
	}
}

// NewPdbg creates a PDbg instance, with options
func NewPdbg(options ...Option) *Pdbg {
	newpdbg := &Pdbg{}
	for _, option := range options {
		option(newpdbg)
	}
	newpdbg.breaks = append(newpdbg.breaks, "smartystreets")
	//newpdbg.breaks = append(newpdbg.breaks, "(*Pdbg).Pdbgf")
	newpdbg.skips = append(newpdbg.skips, "/godbg.go'")
	return newpdbg
}

// ResetIOs reset the out and err buffer of global pdbg instance
func ResetIOs() {
	pdbg.ResetIOs()
}

// ResetIOs reset the out and err buffer
// (unless they were the default stdout and stderr,
// in which case it does nothing)
func (pdbg *Pdbg) ResetIOs() {
	if pdbg.sout != nil {
		pdbg.bout = bytes.NewBuffer(nil)
		pdbg.sout.Reset(pdbg.bout)
		pdbg.berr = bytes.NewBuffer(nil)
		pdbg.serr.Reset(pdbg.berr)
	}
}

// OutString returns the string for out messages for the global pdbg instance.
// It flushes the out buffer.
// If out is set to os.Stdout, returns an empty string
func OutString() string {
	return pdbg.OutString()
}

// OutString returns the string for out messages for a given pdbg instance.
// It flushes the out buffer.
// If out is set to os.Stdout, returns an empty string
func (pdbg *Pdbg) OutString() string {
	if pdbg.sout == nil {
		return ""
	}
	pdbg.sout.Flush()
	return pdbg.bout.String()
}

// ErrString returns the string for error messages for the global pdbg instance.
// It flushes the err buffer.
// If err is set to os.StdErr, returns an empty string
func ErrString() string {
	return pdbg.ErrString()
}

// ErrString returns the string for error messages for a given pdbg instance.
// It flushes the err buffer.
// If err is set to os.StdErr, returns an empty string
func (pdbg *Pdbg) ErrString() string {
	if pdbg.serr == nil {
		return ""
	}
	pdbg.serr.Flush()
	return pdbg.berr.String()
}

func (pdbg *Pdbg) pdbgExcluded(dbg string) bool {
	for _, e := range pdbg.excludes {
		if strings.Contains(dbg, e) {
			// fmt.Printf("EXCLUDE over '%v' including '%v'\n", dbg, e) // DBG
			return true
		}
	}
	return false
}

func (pdbg *Pdbg) pdbgBreak(dbg string) bool {
	for _, b := range pdbg.breaks {
		if strings.Contains(dbg, b) {
			// fmt.Printf("BREAK over '%v' including '%v'\n", dbg, b) // DBG
			return true
		}
	}
	return false
}

func (pdbg *Pdbg) pdbgSkip(dbg string) (bool, int) {
	depthToAdd := 0
	for i, s := range pdbg.skips {
		if strings.Contains(dbg, s) {
			if i > 0 {
				depthToAdd = 1
			}
			// fmt.Printf("SKIP over '%v' including '%v'\n", dbg, s) // DBG
			return true, depthToAdd
		}
	}
	return false, depthToAdd
}

// Pdbgf uses global Pdbg variable for printing strings, with indent and function name
func Pdbgf(format string, args ...interface{}) string {
	return pdbg.Pdbgf(format, args...)
}

type caller func(skip int) (pc uintptr, file string, line int, ok bool)

var mycaller = runtime.Caller

// Pdbgf uses custom Pdbg variable for printing strings, with indent and function name
func (pdbg *Pdbg) Pdbgf(format string, args ...interface{}) string {
	msg := fmt.Sprintf(format+"\n", args...)
	msg = strings.TrimSpace(msg)

	pmsg := ""
	depth := 0
	nbskip := 0
	nbInitialSkips := 0
	first := true
	addOneForSkip := 0
	// fmt.Printf("~~~~~~~~~~~~~~~~~~~~~~\n") // DBG
	for ok := true; ok; {
		pc, file, line, ok := mycaller(depth)
		if !ok {
			break
		}
		fname := runtime.FuncForPC(pc).Name()
		fline := fmt.Sprintf("Name of function: '%v': '%+x' (line %v): file '%v'\n", fname, fname, line, file)
		// fmt.Println(fline) // DBG
		if pdbg.pdbgExcluded(fline) {
			depth = depth + 1
			if first {
				return ""
			}
			continue
		}
		if pdbg.pdbgBreak(fline) {
			break
		}
		if isSkipped, depthToAdd := pdbg.pdbgSkip(fline); isSkipped {
			depth = depth + 1
			nbskip = nbskip + 1
			addOneForSkip = addOneForSkip + depthToAdd
			continue
		}
		fnamerx1 := regexp.MustCompile(`.*\.func[^a-zA-Z0-9]`)
		fname = fnamerx1.ReplaceAllString(fname, "func.")
		fnamerx2 := regexp.MustCompile(`.*/`)
		fname = fnamerx2.ReplaceAllString(fname, "")
		if !strings.HasPrefix(fname, "func.") {
			fnamerx3 := regexp.MustCompile(`^.*?\.`)
			// fmt.Printf("fname before: '%v'", fname)
			fname = fnamerx3.ReplaceAllString(fname, "")
			// fmt.Printf(" => fname after: '%v'\n", fname)
			fnamerx4 := regexp.MustCompile(`[\(\)]`)
			fname = fnamerx4.ReplaceAllString(fname, "")
		}
		dbg := fname + ":" + fmt.Sprintf("%d", line)
		if first {
			// fmt.Printf(" => nbskip '%v'; addOneForSkip '%v'\n", nbskip, addOneForSkip) // DBG
			nbInitialSkips = nbskip - addOneForSkip
			pmsg = "[" + dbg + "]"
		} else {
			pmsg = pmsg + " (" + dbg + ")"
		}
		first = false
		depth = depth + 1
	}
	finalDepth := depth
	depth = finalDepth - nbInitialSkips - 1

	spaces := ""
	if depth >= 0 {
		spaces = strings.Repeat(" ", depth*2)
	}
	// fmt.Printf("spaces '%s', finalDepth '%d', depth '%d', nbInitialSkips '%d', addOneForSkip='%d'\n", spaces, finalDepth, depth, nbInitialSkips, addOneForSkip) // DBG
	res := pmsg
	if pmsg != "" {
		pmsg = spaces + pmsg + "\n"
	}
	msg = pmsg + spaces + "  " + msg + "\n"
	// fmt.Printf("==> MSG '%v'\n", msg) // DBG
	fmt.Fprint(pdbg.Err(), fmt.Sprint(msg))
	return res
}
<|endoftext|>


<commit_before>package golds

const version = "0.0.0"

// Container basic interface for all collections
type Container interface {
	// Size returns amount of elements inside a collection
	Size() int

	// IsEmpty returns true if collection is empty
	IsEmpty() bool

	// Clear removes all elements from the collection
	Clear()
}

// Comparable is an interface for comparable items
// contains only one method Less, which return true
// if an element is less than given
type Comparable interface {
	Less(Comparable) bool
}
<commit_msg>add CmpFunc<commit_after>package golds

// Container basic interface for all collections
type Container interface {
	// Size returns amount of elements inside a collection
	Size() int

	// IsEmpty returns true if collection is empty
	IsEmpty() bool

	// Clear removes all elements from the collection
	Clear()
}

// CmpFunc returns true if a is greater then b.
type CmpFunc func(a, b interface{}) bool
<|endoftext|>


<commit_before>package golog

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
)

type GoLogger struct {
	Mutex         sync.Mutex
	InfoLogger    *log.Logger
	WarningLogger *log.Logger
	ErrorLogger   *log.Logger
	FatalLogger   *log.Logger
	DebugLogger   *log.Logger
}

func getStack(all bool) []string {
	buf := make([]byte, 1<<8)
	for {
		n := runtime.Stack(buf, all)
		if n < len(buf) {
			break
		}
		buf = make([]byte, len(buf)*2)
	}
	return strings.Split(string(buf), "\n")
}

func NewGoLogger(output *os.File) *GoLogger {
	logger := new(GoLogger)

	logger.InfoLogger = log.New(output, "[ INFO  ]", log.LstdFlags)
	logger.WarningLogger = log.New(output, "[WARNING]", log.LstdFlags)
	logger.ErrorLogger = log.New(output, "[ ERROR ]", log.LstdFlags)
	logger.FatalLogger = log.New(output, "[ FATAL ]", log.LstdFlags)
	logger.DebugLogger = log.New(output, "[ DEBUG ]", log.LstdFlags)

	return logger
}

func (l *GoLogger) Infoln(v ...interface{}) {
	l.Mutex.Lock()
	l.InfoLogger.Println(v...)
	l.Mutex.Unlock()
}

func (l *GoLogger) Infof(format string, v ...interface{}) {
	l.Infoln(fmt.Sprintf(format, v...))
}

func (l *GoLogger) Warningln(v ...interface{}) {
	l.Mutex.Lock()
	l.WarningLogger.Println(v...)
	l.Mutex.Unlock()
}

func (l *GoLogger) Warningf(format string, v ...interface{}) {
	l.Warningln(fmt.Sprintf(format, v...))
}

func (l *GoLogger) errorln(v ...interface{}) {
	_, file, line, _ := runtime.Caller(2)
	caller := fmt.Sprintf("%s:%d", filepath.Base(file), line)
	v = append([]interface{}{caller}, v...)
	l.Mutex.Lock()
	l.ErrorLogger.Println(v...)
	l.Mutex.Unlock()
}

func (l *GoLogger) Errorln(v ...interface{}) {
	l.errorln(v...)
}

func (l *GoLogger) Errorf(format string, v ...interface{}) {
	l.errorln(fmt.Sprintf(format, v...))
}

func (l *GoLogger) Fatalln(v ...interface{}) {
	l.Mutex.Lock()
	l.FatalLogger.Println(v...)
	for _, v := range getStack(false) {
		l.FatalLogger.Println(v)
	}
	l.Mutex.Unlock()
	os.Exit(1)
}

func (l *GoLogger) Fatalf(format string, v ...interface{}) {
	l.Fatalln(fmt.Sprintf(format, v...))
}

func (l *GoLogger) Debugln(debug bool, v ...interface{}) {
	if debug {
		l.Mutex.Lock()
		l.DebugLogger.Println(v...)
		l.Mutex.Unlock()
	}
}

func (l *GoLogger) Debugf(debug bool, format string, v ...interface{}) {
	l.Debugln(debug, fmt.Sprintf(format, v...))
}
<commit_msg>show file:line when calling debug<commit_after>package golog

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
)

type GoLogger struct {
	Mutex         sync.Mutex
	InfoLogger    *log.Logger
	WarningLogger *log.Logger
	ErrorLogger   *log.Logger
	FatalLogger   *log.Logger
	DebugLogger   *log.Logger
}

func getStack(all bool) []string {
	buf := make([]byte, 1<<8)
	for {
		n := runtime.Stack(buf, all)
		if n < len(buf) {
			break
		}
		buf = make([]byte, len(buf)*2)
	}
	return strings.Split(string(buf), "\n")
}

func NewGoLogger(output *os.File) *GoLogger {
	logger := new(GoLogger)

	logger.InfoLogger = log.New(output, "[ INFO  ]", log.LstdFlags)
	logger.WarningLogger = log.New(output, "[WARNING]", log.LstdFlags)
	logger.ErrorLogger = log.New(output, "[ ERROR ]", log.LstdFlags)
	logger.FatalLogger = log.New(output, "[ FATAL ]", log.LstdFlags)
	logger.DebugLogger = log.New(output, "[ DEBUG ]", log.LstdFlags)

	return logger
}

func (l *GoLogger) Infoln(v ...interface{}) {
	l.Mutex.Lock()
	l.InfoLogger.Println(v...)
	l.Mutex.Unlock()
}

func (l *GoLogger) Infof(format string, v ...interface{}) {
	l.Infoln(fmt.Sprintf(format, v...))
}

func (l *GoLogger) Warningln(v ...interface{}) {
	l.Mutex.Lock()
	l.WarningLogger.Println(v...)
	l.Mutex.Unlock()
}

func (l *GoLogger) Warningf(format string, v ...interface{}) {
	l.Warningln(fmt.Sprintf(format, v...))
}

func (l *GoLogger) errorln(v ...interface{}) {
	_, file, line, _ := runtime.Caller(2)
	caller := fmt.Sprintf("%s:%d", filepath.Base(file), line)
	v = append([]interface{}{caller}, v...)
	l.Mutex.Lock()
	l.ErrorLogger.Println(v...)
	l.Mutex.Unlock()
}

func (l *GoLogger) Errorln(v ...interface{}) {
	l.errorln(v...)
}

func (l *GoLogger) Errorf(format string, v ...interface{}) {
	l.errorln(fmt.Sprintf(format, v...))
}

func (l *GoLogger) Fatalln(v ...interface{}) {
	l.Mutex.Lock()
	l.FatalLogger.Println(v...)
	for _, v := range getStack(false) {
		l.FatalLogger.Println(v)
	}
	l.Mutex.Unlock()
	os.Exit(1)
}

func (l *GoLogger) Fatalf(format string, v ...interface{}) {
	l.Fatalln(fmt.Sprintf(format, v...))
}

func (l *GoLogger) debugln(v ...interface{}) {
	_, file, line, _ := runtime.Caller(2)
	caller := fmt.Sprintf("%s:%d", filepath.Base(file), line)
	v = append([]interface{}{caller}, v...)
	l.Mutex.Lock()
	l.DebugLogger.Println(v...)
	l.Mutex.Unlock()
}

func (l *GoLogger) Debugln(debug bool, v ...interface{}) {
	if !debug {
		return
	}
	l.debugln(v...)
}

func (l *GoLogger) Debugf(debug bool, format string, v ...interface{}) {
	if !debug {
		return
	}
	l.debugln(fmt.Sprintf(format, v...))
}
<|endoftext|>


<commit_before>package gopla

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"

	"github.com/PuerkitoBio/goquery"
	"github.com/headzoo/surf"
	"github.com/tidwall/gjson"
)

func init() {

}

//VideoStream has actual video file data
type VideoStream struct {
	URL     string
	Bitrate string
	Format  string
	Size    string
	Quality string
}

//VideoData has all information about video page
type VideoData struct {
	Title       string
	Description string
	Thumbnails  []string
	Duration    string
	Hash        string
	Videos      VideoStreams
}

//SearchData has all information from search page
type SearchData struct {
	Title       string
	Description string
	ID          string
	Date        string
	Image       string
	URL         string
	Hash        string
	Type        string
}

//SearchDatas ...
type SearchDatas []SearchData

//VideoDatas ..
type VideoDatas []VideoData

//VideoStreams ..
type VideoStreams []VideoStream

func (slice VideoStreams) Len() int {
	return len(slice)
}

func (slice VideoStreams) Less(i, j int) bool {
	var s1, _ = strconv.Atoi(slice[i].Size)
	var s2, _ = strconv.Atoi(slice[j].Size)
	return s1 < s2
}

func (slice VideoStreams) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

func getJSON(url string) gjson.Result {
	client := &http.Client{}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		panic(err)
	}
	req.Header.Set("User-Agent", "Mozilla")
	res, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()
	json, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err)
	}
	var jsonR = gjson.Parse(string(json))
	return jsonR
}

//GetAllHashes returns all video hashes from given search
func GetAllHashes(id string, name string) []string {
	var link = fmt.Sprintf("https://seeker.redefine.pl/ipla/multi.json?free=1&page=1&platform=www_ipla_tv&portal_id=ipla&query=%s&category_ids%%5B%%5D=%s&size=5000&sort=recent&sound=&top_category=", url.QueryEscape(name), url.QueryEscape(id))
	fmt.Println(link)
	var jsonR = getJSON(link)
	var data []string
	for _, v := range jsonR.Array() {
		data = append(data, v.Get("media_id").String())
	}
	return data
}

//FindVideo finds the material and returns informations about it
func FindVideo(name string, category string) SearchDatas {
	var link = fmt.Sprintf("https://seeker.redefine.pl/ipla/multi.json?free=1&page=1&platform=www_ipla_tv&portal_id=ipla&query=%s&size=150&sort=recent&sound=&top_category=%s", url.QueryEscape(name), category)
	var jsonR = getJSON(link)
	//fmt.Println(link)

	if len(jsonR.Array()) == 0 {
		return nil
	}
	var data SearchDatas
	for _, v := range jsonR.Array() {

		data = append(data, SearchData{
			ID:          v.Get("id").String(),
			Description: v.Get("description").String(),
			Title:       v.Get("title").String(),
			Date:        v.Get("created_date").String(),
			Image:       v.Get("image").String(),
			URL:         fmt.Sprintf("http://www.ipla.tv/kategoria/%s", v.Get("id").String()),
			Hash:        v.Get("media_id").String(),
			Type:        v.Get("_type").String(),
		})

	}
	return data
}

//GetHash returns playvod| link from http link
func GetHash(link string) string {
	bow := surf.NewBrowser()
	bow.AddRequestHeader("Accept", "text/html")
	bow.AddRequestHeader("Accept-Charset", "utf8")
	bow.AddRequestHeader("User-Agent", "Mozilla")
	err := bow.Open(link)
	if err != nil {
		panic(err)
	}
	bow.Find("a.start-watch").Each(func(_ int, s *goquery.Selection) {
		link, _ = s.Attr("href")

		re := regexp.MustCompile("^ipla://playvod-1\\|([a-f0-9]{16})")
		link = re.ReplaceAllString(link, "$1")
	})
	return link
}

//GetVideo returns VideoStream object with basic link informations
func GetVideo(hash string) VideoData {

	var jsonR = getJSON("http://getmedia.redefine.pl/vods/get_vod/?cpid=1&ua=mipla/23&media_id=" + hash)
	var data = VideoData{
		Hash:        hash,
		Title:       jsonR.Get("vod.title").String(),
		Description: jsonR.Get("vod.text").String(),
		Duration:    jsonR.Get("vod.duration").String(),
		Videos:      VideoStreams{},
	}

	jsonR.Get("vod.copies").ForEach(func(key gjson.Result, value gjson.Result) bool {

		data.Videos = append(data.Videos, VideoStream{
			URL:     value.Get("url").String(),
			Bitrate: value.Get("bitrate").String(),
			Format:  value.Get("format").String(),
			Quality: value.Get("quality_p").String(),
			Size:    value.Get("size").String(),
		})
		return true
	})

	return data
}
<commit_msg>Remove debug print<commit_after>package gopla

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"

	"github.com/PuerkitoBio/goquery"
	"github.com/headzoo/surf"
	"github.com/tidwall/gjson"
)

func init() {

}

//VideoStream has actual video file data
type VideoStream struct {
	URL     string
	Bitrate string
	Format  string
	Size    string
	Quality string
}

//VideoData has all information about video page
type VideoData struct {
	Title       string
	Description string
	Thumbnails  []string
	Duration    string
	Hash        string
	Videos      VideoStreams
}

//SearchData has all information from search page
type SearchData struct {
	Title       string
	Description string
	ID          string
	Date        string
	Image       string
	URL         string
	Hash        string
	Type        string
}

//SearchDatas ...
type SearchDatas []SearchData

//VideoDatas ..
type VideoDatas []VideoData

//VideoStreams ..
type VideoStreams []VideoStream

func (slice VideoStreams) Len() int {
	return len(slice)
}

func (slice VideoStreams) Less(i, j int) bool {
	var s1, _ = strconv.Atoi(slice[i].Size)
	var s2, _ = strconv.Atoi(slice[j].Size)
	return s1 < s2
}

func (slice VideoStreams) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

func getJSON(url string) gjson.Result {
	client := &http.Client{}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		panic(err)
	}
	req.Header.Set("User-Agent", "Mozilla")
	res, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()
	json, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err)
	}
	var jsonR = gjson.Parse(string(json))
	return jsonR
}

//GetAllHashes returns all video hashes from given search
func GetAllHashes(id string, name string) []string {
	var link = fmt.Sprintf("https://seeker.redefine.pl/ipla/multi.json?free=1&page=1&platform=www_ipla_tv&portal_id=ipla&query=%s&category_ids%%5B%%5D=%s&size=5000&sort=recent&sound=&top_category=", url.QueryEscape(name), url.QueryEscape(id))
	//fmt.Println(link)
	var jsonR = getJSON(link)
	var data []string
	for _, v := range jsonR.Array() {
		data = append(data, v.Get("media_id").String())
	}
	return data
}

//FindVideo finds the material and returns informations about it
func FindVideo(name string, category string) SearchDatas {
	var link = fmt.Sprintf("https://seeker.redefine.pl/ipla/multi.json?free=1&page=1&platform=www_ipla_tv&portal_id=ipla&query=%s&size=150&sort=recent&sound=&top_category=%s", url.QueryEscape(name), category)
	var jsonR = getJSON(link)
	//fmt.Println(link)

	if len(jsonR.Array()) == 0 {
		return nil
	}
	var data SearchDatas
	for _, v := range jsonR.Array() {

		data = append(data, SearchData{
			ID:          v.Get("id").String(),
			Description: v.Get("description").String(),
			Title:       v.Get("title").String(),
			Date:        v.Get("created_date").String(),
			Image:       v.Get("image").String(),
			URL:         fmt.Sprintf("http://www.ipla.tv/kategoria/%s", v.Get("id").String()),
			Hash:        v.Get("media_id").String(),
			Type:        v.Get("_type").String(),
		})

	}
	return data
}

//GetHash returns playvod| link from http link
func GetHash(link string) string {
	bow := surf.NewBrowser()
	bow.AddRequestHeader("Accept", "text/html")
	bow.AddRequestHeader("Accept-Charset", "utf8")
	bow.AddRequestHeader("User-Agent", "Mozilla")
	err := bow.Open(link)
	if err != nil {
		panic(err)
	}
	bow.Find("a.start-watch").Each(func(_ int, s *goquery.Selection) {
		link, _ = s.Attr("href")

		re := regexp.MustCompile("^ipla://playvod-1\\|([a-f0-9]{16})")
		link = re.ReplaceAllString(link, "$1")
	})
	return link
}

//GetVideo returns VideoStream object with basic link informations
func GetVideo(hash string) VideoData {

	var jsonR = getJSON("http://getmedia.redefine.pl/vods/get_vod/?cpid=1&ua=mipla/23&media_id=" + hash)
	var data = VideoData{
		Hash:        hash,
		Title:       jsonR.Get("vod.title").String(),
		Description: jsonR.Get("vod.text").String(),
		Duration:    jsonR.Get("vod.duration").String(),
		Videos:      VideoStreams{},
	}

	jsonR.Get("vod.copies").ForEach(func(key gjson.Result, value gjson.Result) bool {

		data.Videos = append(data.Videos, VideoStream{
			URL:     value.Get("url").String(),
			Bitrate: value.Get("bitrate").String(),
			Format:  value.Get("format").String(),
			Quality: value.Get("quality_p").String(),
			Size:    value.Get("size").String(),
		})
		return true
	})

	return data
}
<|endoftext|>