<commit_before>// Package editorconfig can be used to parse and generate editorconfig files.
// For more information about editorconfig, see http://editorconfig.org/
package editorconfig

import (
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"gopkg.in/ini.v1"
)

const (
	// ConfigNameDefault represents the name of the configuration file
	ConfigNameDefault = ".editorconfig"
)

// IndentStyle possible values
const (
	IndentStyleTab    = "tab"
	IndentStyleSpaces = "space"
)

// EndOfLine possible values
const (
	EndOfLineLf   = "lf"
	EndOfLineCr   = "cr"
	EndOfLineCrLf = "crlf"
)

// Charset possible values
const (
	CharsetLatin1  = "latin1"
	CharsetUTF8    = "utf-8"
	CharsetUTF16BE = "utf-16be"
	CharsetUTF16LE = "utf-16le"
	CharsetUTF8BOM = "utf-8 bom"
)

// Definition represents a definition inside the .editorconfig file.
// E.g. a section of the file.
// The definition is composed of the selector ("*", "*.go", "*.{js.css}", etc),
// plus the properties of the selected files.
type Definition struct {
	Selector string `ini:"-" json:"-"`

	Charset                string `ini:"charset" json:"charset,omitempty"`
	IndentStyle            string `ini:"indent_style" json:"indent_style,omitempty"`
	IndentSize             string `ini:"indent_size" json:"indent_size,omitempty"`
	TabWidth               int    `ini:"tab_width" json:"tab_width,omitempty"`
	EndOfLine              string `ini:"end_of_line" json:"end_of_line,omitempty"`
	TrimTrailingWhitespace bool   `ini:"trim_trailing_whitespace" json:"trim_trailing_whitespace,omitempty"`
	InsertFinalNewline     bool   `ini:"insert_final_newline" json:"insert_final_newline,omitempty"`

	Raw map[string]string `ini:"-" json:"-"`
}

// Editorconfig represents a .editorconfig file.
// It is composed by a "root" property, plus the definitions defined in the
// file.
type Editorconfig struct {
	Root        bool
	Definitions []*Definition
}

// ParseBytes parses from a slice of bytes.
func ParseBytes(data []byte) (*Editorconfig, error) {
	iniFile, err := ini.Load(data)
	if err != nil {
		return nil, err
	}

	editorConfig := &Editorconfig{}
	editorConfig.Root = iniFile.Section(ini.DEFAULT_SECTION).Key("root").MustBool(false)
	for _, sectionStr := range iniFile.SectionStrings() {
		if sectionStr == ini.DEFAULT_SECTION {
			continue
		}
		var (
			iniSection = iniFile.Section(sectionStr)
			definition = &Definition{}
			raw        = make(map[string]string)
		)
		err := iniSection.MapTo(&definition)
		if err != nil {
			return nil, err
		}

		// Shallow copy all properties
		for k, v := range iniSection.KeysHash() {
			raw[strings.ToLower(k)] = v
		}

		definition.Selector = sectionStr
		definition.Raw = raw
		definition.normalize()
		editorConfig.Definitions = append(editorConfig.Definitions, definition)
	}
	return editorConfig, nil
}

// ParseFile parses from a file.
func ParseFile(f string) (*Editorconfig, error) {
	data, err := ioutil.ReadFile(f)
	if err != nil {
		return nil, err
	}

	return ParseBytes(data)
}

var (
	regexpBraces = regexp.MustCompile("{.*}")
)

// normalize fixes some values to their lowercaes value
func (d *Definition) normalize() {
	d.Charset = strings.ToLower(d.Charset)
	d.EndOfLine = strings.ToLower(d.EndOfLine)
	d.IndentStyle = strings.ToLower(d.IndentStyle)

	// tab_width defaults to indent_size:
	// https://github.com/editorconfig/editorconfig/wiki/EditorConfig-Properties#tab_width
	num, err := strconv.Atoi(d.IndentSize)

	if err == nil && d.TabWidth <= 0 {
		d.TabWidth = num
	}
}

func (d *Definition) merge(md *Definition) {
	if len(d.Charset) == 0 {
		d.Charset = md.Charset
	}
	if len(d.IndentStyle) == 0 {
		d.IndentStyle = md.IndentStyle
	}
	if len(d.IndentSize) == 0 {
		d.IndentSize = md.IndentSize
	}
	if d.TabWidth <= 0 {
		d.TabWidth = md.TabWidth
	}
	if len(d.EndOfLine) == 0 {
		d.EndOfLine = md.EndOfLine
	}
	if !d.TrimTrailingWhitespace {
		d.TrimTrailingWhitespace = md.TrimTrailingWhitespace
	}
	if !d.InsertFinalNewline {
		d.InsertFinalNewline = md.InsertFinalNewline
	}

	for k, v := range md.Raw {
		if _, ok := d.Raw[k]; !ok {
			d.Raw[k] = v
		}
	}
}

// InsertToIniFile ... TODO
func (d *Definition) InsertToIniFile(iniFile *ini.File) {
	iniSec := iniFile.Section(d.Selector)
	for k, v := range d.Raw {
		if k == "insert_final_newline" {
			iniSec.Key(k).SetValue(strconv.FormatBool(d.InsertFinalNewline))
		} else if k == "trim_trailing_whitespace" {
			iniSec.Key(k).SetValue(strconv.FormatBool(d.TrimTrailingWhitespace))
		} else if k == "charset" {
			iniSec.Key(k).SetValue(d.Charset)
		} else if k == "end_of_line" {
			iniSec.Key(k).SetValue(d.EndOfLine)
		} else if k == "indent_style" {
			iniSec.Key(k).SetValue(d.IndentStyle)
		} else if k == "tab_width" {
			iniSec.Key(k).SetValue(strconv.Itoa(d.TabWidth))
		} else if k == "indent_size" {
			iniSec.Key(k).SetValue(d.IndentSize)
		} else {
			iniSec.Key(k).SetValue(v)
		}
	}
	if _, ok := d.Raw["indent_size"]; !ok {
		if d.TabWidth > 0 {
			iniSec.Key("indent_size").SetValue(strconv.Itoa(d.TabWidth))
		} else if d.IndentStyle == IndentStyleTab {
			iniSec.Key("indent_size").SetValue(IndentStyleTab)
		}
	}

	if _, ok := d.Raw["tab_width"]; !ok && len(d.IndentSize) > 0 {
		if _, err := strconv.Atoi(d.IndentSize); err == nil {
			iniSec.Key("tab_width").SetValue(d.IndentSize)
		}
	}
}

// GetDefinitionForFilename returns a definition for the given filename.
// The result is a merge of the selectors that matched the file.
// The last section has preference over the priors.
func (e *Editorconfig) GetDefinitionForFilename(name string) *Definition {
	def := &Definition{}
	def.Raw = make(map[string]string)
	for i := len(e.Definitions) - 1; i >= 0; i-- {
		actualDef := e.Definitions[i]
		selector := actualDef.Selector
		if !strings.HasPrefix(selector, "/") {
			if strings.ContainsRune(selector, '/') {
				selector = "/" + selector
			} else {
				selector = "/**/" + selector
			}
		}
		if !strings.HasPrefix(name, "/") {
			name = "/" + name
		}
		if FnmatchCase(selector, name) {
			def.merge(actualDef)
		}
	}
	return def
}

func boolToString(b bool) string {
	if b {
		return "true"
	}
	return "false"
}

// Serialize converts the Editorconfig to a slice of bytes, containing the
// content of the file in the INI format.
func (e *Editorconfig) Serialize() ([]byte, error) {
	var (
		iniFile = ini.Empty()
		buffer  = bytes.NewBuffer(nil)
	)
	iniFile.Section(ini.DEFAULT_SECTION).Comment = "http://editorconfig.org"
	if e.Root {
		iniFile.Section(ini.DEFAULT_SECTION).Key("root").SetValue(boolToString(e.Root))
	}
	for _, d := range e.Definitions {
		d.InsertToIniFile(iniFile)
	}
	_, err := iniFile.WriteTo(buffer)
	if err != nil {
		return nil, err
	}
	return buffer.Bytes(), nil
}

// Save saves the Editorconfig to a compatible INI file.
func (e *Editorconfig) Save(filename string) error {
	data, err := e.Serialize()
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filename, data, 0666)
}

// GetDefinitionForFilename given a filename, searches
// for .editorconfig files, starting from the file folder,
// walking through the previous folders, until it reaches a
// folder with `root = true`, and returns the right editorconfig
// definition for the given file.
func GetDefinitionForFilename(filename string) (*Definition, error) {
	return GetDefinitionForFilenameWithConfigname(filename, ConfigNameDefault)
}

func GetDefinitionForFilenameWithConfigname(filename string, configname string) (*Definition, error) {
	abs, err := filepath.Abs(filename)
	if err != nil {
		return nil, err
	}
	definition := &Definition{}
	definition.Raw = make(map[string]string)

	dir := abs
	for dir != filepath.Dir(dir) {
		dir = filepath.Dir(dir)
		ecFile := filepath.Join(dir, configname)
		if _, err := os.Stat(ecFile); os.IsNotExist(err) {
			continue
		}
		ec, err := ParseFile(ecFile)
		if err != nil {
			return nil, err
		}
		relativeFilename := filename
		if len(dir) < len(filename) {
			relativeFilename = filename[len(dir):]
		}
		definition.merge(ec.GetDefinitionForFilename(relativeFilename))
		if ec.Root {
			break
		}
	}

	return definition, nil
}
<commit_msg>Update editorconfig.go<commit_after>// Package editorconfig can be used to parse and generate editorconfig files.
// For more information about editorconfig, see http://editorconfig.org/
package editorconfig

import (
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"gopkg.in/ini.v1"
)

const (
	// ConfigNameDefault represents the name of the configuration file
	ConfigNameDefault = ".editorconfig"
)

// IndentStyle possible values
const (
	IndentStyleTab    = "tab"
	IndentStyleSpaces = "space"
)

// EndOfLine possible values
const (
	EndOfLineLf   = "lf"
	EndOfLineCr   = "cr"
	EndOfLineCrLf = "crlf"
)

// Charset possible values
const (
	CharsetLatin1  = "latin1"
	CharsetUTF8    = "utf-8"
	CharsetUTF16BE = "utf-16be"
	CharsetUTF16LE = "utf-16le"
	CharsetUTF8BOM = "utf-8 bom"
)

// Definition represents a definition inside the .editorconfig file.
// E.g. a section of the file.
// The definition is composed of the selector ("*", "*.go", "*.{js.css}", etc),
// plus the properties of the selected files.
type Definition struct {
	Selector string `ini:"-" json:"-"`

	Charset                string `ini:"charset" json:"charset,omitempty"`
	IndentStyle            string `ini:"indent_style" json:"indent_style,omitempty"`
	IndentSize             string `ini:"indent_size" json:"indent_size,omitempty"`
	TabWidth               int    `ini:"tab_width" json:"tab_width,omitempty"`
	EndOfLine              string `ini:"end_of_line" json:"end_of_line,omitempty"`
	TrimTrailingWhitespace bool   `ini:"trim_trailing_whitespace" json:"trim_trailing_whitespace,omitempty"`
	InsertFinalNewline     bool   `ini:"insert_final_newline" json:"insert_final_newline,omitempty"`

	Raw map[string]string `ini:"-" json:"-"`
}

// Editorconfig represents a .editorconfig file.
// It is composed by a "root" property, plus the definitions defined in the
// file.
type Editorconfig struct {
	Root        bool
	Definitions []*Definition
}

// ParseBytes parses from a slice of bytes.
func ParseBytes(data []byte) (*Editorconfig, error) {
	iniFile, err := ini.Load(data)
	if err != nil {
		return nil, err
	}

	editorConfig := &Editorconfig{}
	editorConfig.Root = iniFile.Section(ini.DEFAULT_SECTION).Key("root").MustBool(false)
	for _, sectionStr := range iniFile.SectionStrings() {
		if sectionStr == ini.DEFAULT_SECTION {
			continue
		}
		var (
			iniSection = iniFile.Section(sectionStr)
			definition = &Definition{}
			raw        = make(map[string]string)
		)
		err := iniSection.MapTo(&definition)
		if err != nil {
			return nil, err
		}

		// Shallow copy all properties
		for k, v := range iniSection.KeysHash() {
			raw[strings.ToLower(k)] = v
		}

		definition.Selector = sectionStr
		definition.Raw = raw
		definition.normalize()
		editorConfig.Definitions = append(editorConfig.Definitions, definition)
	}
	return editorConfig, nil
}

// ParseFile parses from a file.
func ParseFile(f string) (*Editorconfig, error) {
	data, err := ioutil.ReadFile(f)
	if err != nil {
		return nil, err
	}

	return ParseBytes(data)
}

var (
	regexpBraces = regexp.MustCompile("{.*}")
)

// normalize fixes some values to their lowercaes value
func (d *Definition) normalize() {
	d.Charset = strings.ToLower(d.Charset)
	d.EndOfLine = strings.ToLower(d.EndOfLine)
	d.IndentStyle = strings.ToLower(d.IndentStyle)

	// tab_width defaults to indent_size:
	// https://github.com/editorconfig/editorconfig/wiki/EditorConfig-Properties#tab_width
	num, err := strconv.Atoi(d.IndentSize)
	if err == nil && d.TabWidth <= 0 {
		d.TabWidth = num
	}
}

func (d *Definition) merge(md *Definition) {
	if len(d.Charset) == 0 {
		d.Charset = md.Charset
	}
	if len(d.IndentStyle) == 0 {
		d.IndentStyle = md.IndentStyle
	}
	if len(d.IndentSize) == 0 {
		d.IndentSize = md.IndentSize
	}
	if d.TabWidth <= 0 {
		d.TabWidth = md.TabWidth
	}
	if len(d.EndOfLine) == 0 {
		d.EndOfLine = md.EndOfLine
	}
	if !d.TrimTrailingWhitespace {
		d.TrimTrailingWhitespace = md.TrimTrailingWhitespace
	}
	if !d.InsertFinalNewline {
		d.InsertFinalNewline = md.InsertFinalNewline
	}

	for k, v := range md.Raw {
		if _, ok := d.Raw[k]; !ok {
			d.Raw[k] = v
		}
	}
}

// InsertToIniFile ... TODO
func (d *Definition) InsertToIniFile(iniFile *ini.File) {
	iniSec := iniFile.Section(d.Selector)
	for k, v := range d.Raw {
		if k == "insert_final_newline" {
			iniSec.Key(k).SetValue(strconv.FormatBool(d.InsertFinalNewline))
		} else if k == "trim_trailing_whitespace" {
			iniSec.Key(k).SetValue(strconv.FormatBool(d.TrimTrailingWhitespace))
		} else if k == "charset" {
			iniSec.Key(k).SetValue(d.Charset)
		} else if k == "end_of_line" {
			iniSec.Key(k).SetValue(d.EndOfLine)
		} else if k == "indent_style" {
			iniSec.Key(k).SetValue(d.IndentStyle)
		} else if k == "tab_width" {
			iniSec.Key(k).SetValue(strconv.Itoa(d.TabWidth))
		} else if k == "indent_size" {
			iniSec.Key(k).SetValue(d.IndentSize)
		} else {
			iniSec.Key(k).SetValue(v)
		}
	}
	if _, ok := d.Raw["indent_size"]; !ok {
		if d.TabWidth > 0 {
			iniSec.Key("indent_size").SetValue(strconv.Itoa(d.TabWidth))
		} else if d.IndentStyle == IndentStyleTab {
			iniSec.Key("indent_size").SetValue(IndentStyleTab)
		}
	}

	if _, ok := d.Raw["tab_width"]; !ok && len(d.IndentSize) > 0 {
		if _, err := strconv.Atoi(d.IndentSize); err == nil {
			iniSec.Key("tab_width").SetValue(d.IndentSize)
		}
	}
}

// GetDefinitionForFilename returns a definition for the given filename.
// The result is a merge of the selectors that matched the file.
// The last section has preference over the priors.
func (e *Editorconfig) GetDefinitionForFilename(name string) *Definition {
	def := &Definition{}
	def.Raw = make(map[string]string)
	for i := len(e.Definitions) - 1; i >= 0; i-- {
		actualDef := e.Definitions[i]
		selector := actualDef.Selector
		if !strings.HasPrefix(selector, "/") {
			if strings.ContainsRune(selector, '/') {
				selector = "/" + selector
			} else {
				selector = "/**/" + selector
			}
		}
		if !strings.HasPrefix(name, "/") {
			name = "/" + name
		}
		if FnmatchCase(selector, name) {
			def.merge(actualDef)
		}
	}
	return def
}

func boolToString(b bool) string {
	if b {
		return "true"
	}
	return "false"
}

// Serialize converts the Editorconfig to a slice of bytes, containing the
// content of the file in the INI format.
func (e *Editorconfig) Serialize() ([]byte, error) {
	var (
		iniFile = ini.Empty()
		buffer  = bytes.NewBuffer(nil)
	)
	iniFile.Section(ini.DEFAULT_SECTION).Comment = "http://editorconfig.org"
	if e.Root {
		iniFile.Section(ini.DEFAULT_SECTION).Key("root").SetValue(boolToString(e.Root))
	}
	for _, d := range e.Definitions {
		d.InsertToIniFile(iniFile)
	}
	_, err := iniFile.WriteTo(buffer)
	if err != nil {
		return nil, err
	}
	return buffer.Bytes(), nil
}

// Save saves the Editorconfig to a compatible INI file.
func (e *Editorconfig) Save(filename string) error {
	data, err := e.Serialize()
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filename, data, 0666)
}

// GetDefinitionForFilename given a filename, searches
// for .editorconfig files, starting from the file folder,
// walking through the previous folders, until it reaches a
// folder with `root = true`, and returns the right editorconfig
// definition for the given file.
func GetDefinitionForFilename(filename string) (*Definition, error) {
	return GetDefinitionForFilenameWithConfigname(filename, ConfigNameDefault)
}

func GetDefinitionForFilenameWithConfigname(filename string, configname string) (*Definition, error) {
	abs, err := filepath.Abs(filename)
	if err != nil {
		return nil, err
	}
	definition := &Definition{}
	definition.Raw = make(map[string]string)

	dir := abs
	for dir != filepath.Dir(dir) {
		dir = filepath.Dir(dir)
		ecFile := filepath.Join(dir, configname)
		if _, err := os.Stat(ecFile); os.IsNotExist(err) {
			continue
		}
		ec, err := ParseFile(ecFile)
		if err != nil {
			return nil, err
		}
		relativeFilename := filename
		if len(dir) < len(filename) {
			relativeFilename = filename[len(dir):]
		}
		definition.merge(ec.GetDefinitionForFilename(relativeFilename))
		if ec.Root {
			break
		}
	}

	return definition, nil
}
<|endoftext|>


<commit_before>package meta

// Picture contains the image data of an embedded picture.
//
// ref: https://www.xiph.org/flac/format.html#metadata_block_picture
type Picture struct{}
<commit_msg>meta: Add Picture definition. This was the last piece of the new meta API. Updates issue #4.<commit_after>package meta

// Picture contains the image data of an embedded picture.
//
// ref: https://www.xiph.org/flac/format.html#metadata_block_picture
type Picture struct {
	// Picture type according to the ID3v2 APIC frame:
	//
	//     0: Other
	//     1: 32x32 pixels 'file icon' (PNG only)
	//     2: Other file icon
	//     3: Cover (front)
	//     4: Cover (back)
	//     5: Leaflet page
	//     6: Media (e.g. label side of CD)
	//     7: Lead artist/lead performer/soloist
	//     8: Artist/performer
	//     9: Conductor
	//    10: Band/Orchestra
	//    11: Composer
	//    12: Lyricist/text writer
	//    13: Recording Location
	//    14: During recording
	//    15: During performance
	//    16: Movie/video screen capture
	//    17: A bright coloured fish
	//    18: Illustration
	//    19: Band/artist logotype
	//    20: Publisher/Studio logotype
	//
	// ref: http://id3.org/id3v2.4.0-frames
	Type uint32
	// MIME type string. The MIME type "-->" specifies that the picture data is
	// to be interpreted as an URL instead of image data.
	MIMEType string
	// Description of the picture.
	Desc string
	// Image dimensions.
	Width, Height uint32
	// Color depth in bits-per-pixel.
	Depth uint32
	// Number of colors in palette; 0 for non-indexed images.
	NPalColors uint32
	// Image data.
	Data []byte
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/fweikert/continuous-integration/metrics/data"

	"github.com/fweikert/continuous-integration/metrics/clients"
	"github.com/fweikert/continuous-integration/metrics/metrics"
	"github.com/fweikert/continuous-integration/metrics/publishers"
	"github.com/fweikert/continuous-integration/metrics/service"
	"google.golang.org/appengine"
)

var (
	projectID             = flag.String("project_id", "bazel-untrusted", "ID of the GCP project.")
	datastoreSettingsName = flag.String("datastore_settings_name", "MetricSettings", "Name of the settings entity in Datastore.")
	testMode              = flag.Bool("test", false, "If true, the service will collect and publish all metrics immediately and only once.")
)

const megaByte = 1024 * 1024

func handleError(metricName string, err error) {
	fmt.Printf("[%s] %v\n", metricName, err)
}

func handleRequest(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "You should try https://bazel.build")
}

func main() {
	flag.Parse()

	settings, err := ReadSettingsFromDatastore(*projectID, *datastoreSettingsName)
	if err != nil {
		log.Fatalf("Could not read settings from Datastore: %v", err)
	}
	pipelines, err := settings.GetPipelineIDs()
	if err != nil {
		log.Fatalf("Could not get Buildkite pipeline IDs from Datastore: %v", err)
	}
	if len(pipelines) == 0 {
		log.Fatalf("No pipelines were specified.")
	}

	bk, err := clients.CreateBuildkiteClient(settings.BuildkiteApiToken, settings.BuildkiteDebug)
	if err != nil {
		log.Fatalf("Cannot create Buildkite client: %v", err)
	}

	gcs, err := clients.CreateGcsClient()
	if err != nil {
		log.Fatalf("Cannot create GCS client: %v", err)
	}

	stackdriverClient, err := clients.CreateStackdriverClient()
	if err != nil {
		log.Fatalf("Cannot create Stackdriver client: %v", err)
	}

	/*
		cloudSql, err := publishers.CreateCloudSqlPublisher(settings.CloudSqlUser, settings.CloudSqlPassword, settings.CloudSqlInstance, settings.CloudSqlDatabase, settings.CloudSqlLocalPort)
		if err != nil {
			log.Fatalf("Failed to set up Cloud SQL publisher: %v", err)
		}
	*/

	pid := &data.PipelineID{Org: "bazel", Slug: "google-bazel-presubmit"}

	stackdriver := publishers.CreateStackdriverPublisher(stackdriverClient, *projectID)

	stdout := publishers.CreateStdoutPublisher(publishers.Csv)

	srv := service.CreateService(handleError)

	// TODO: support multiple organizations
	platformLoad := metrics.CreatePlatformLoad(bk, 100, settings.BuildkiteOrgs...)
	srv.AddMetric(platformLoad, 60, stackdriver)

	buildsPerChange := metrics.CreateBuildsPerChange(bk, 500, pipelines...)
	srv.AddMetric(buildsPerChange, 60, stdout)

	buildSuccess := metrics.CreateBuildSuccess(bk, 200, pipelines...)
	srv.AddMetric(buildSuccess, 60, stdout)

	// TODO(fweikert): use real settings instead of hardcoded values
	flakiness := metrics.CreateFlakiness(gcs, "bazel-buildkite-stats", "flaky-tests-bep", pid) // TODO: pipelines...)
	srv.AddMetric(flakiness, 60, stdout)

	macPerformance := metrics.CreateMacPerformance(bk, 20, pid) // TODO: pipelines...)
	srv.AddMetric(macPerformance, 60, stdout)

	pipelinePerformance := metrics.CreatePipelinePerformance(bk, 20, pipelines...)
	srv.AddMetric(pipelinePerformance, 60, stdout)

	platformSignificance := metrics.CreatePlatformSignificance(bk, 100, pipelines...)
	srv.AddMetric(platformSignificance, 24*60, stdout)

	platformUsage := metrics.CreatePlatformUsage(bk, 100, settings.BuildkiteOrgs...)
	srv.AddMetric(platformUsage, 60, stdout)

	releaseDownloads := metrics.CreateReleaseDownloads(settings.GitHubOrg,
		settings.GitHubRepo,
		settings.GitHubApiToken, megaByte)
	srv.AddMetric(releaseDownloads, 12*60, stdout)

	workerAvailability := metrics.CreateWorkerAvailability(bk, settings.BuildkiteOrgs...)
	srv.AddMetric(workerAvailability, 60, stdout)

	if *testMode {
		log.Println("[Test mode] Running all jobs exactly once...")
		srv.RunJobsOnce()
		os.Exit(0)
	}

	srv.Start()
	http.HandleFunc("/", handleRequest)
	appengine.Main()
}
<commit_msg>Disable all metrics except for flakiness<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/fweikert/continuous-integration/metrics/data"

	"github.com/fweikert/continuous-integration/metrics/clients"
	"github.com/fweikert/continuous-integration/metrics/metrics"
	"github.com/fweikert/continuous-integration/metrics/publishers"
	"github.com/fweikert/continuous-integration/metrics/service"
	"google.golang.org/appengine"
)

var (
	projectID             = flag.String("project_id", "bazel-untrusted", "ID of the GCP project.")
	datastoreSettingsName = flag.String("datastore_settings_name", "MetricSettings", "Name of the settings entity in Datastore.")
	testMode              = flag.Bool("test", false, "If true, the service will collect and publish all metrics immediately and only once.")
)

const megaByte = 1024 * 1024

func handleError(metricName string, err error) {
	fmt.Printf("[%s] %v\n", metricName, err)
}

func handleRequest(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "You should try https://bazel.build")
}

func main() {
	flag.Parse()

	settings, err := ReadSettingsFromDatastore(*projectID, *datastoreSettingsName)
	if err != nil {
		log.Fatalf("Could not read settings from Datastore: %v", err)
	}
	pipelines, err := settings.GetPipelineIDs()
	if err != nil {
		log.Fatalf("Could not get Buildkite pipeline IDs from Datastore: %v", err)
	}
	if len(pipelines) == 0 {
		log.Fatalf("No pipelines were specified.")
	}

	/*
		bk, err := clients.CreateBuildkiteClient(settings.BuildkiteApiToken, settings.BuildkiteDebug)
		if err != nil {
			log.Fatalf("Cannot create Buildkite client: %v", err)
		}
	*/

	gcs, err := clients.CreateGcsClient()
	if err != nil {
		log.Fatalf("Cannot create GCS client: %v", err)
	}

	/*
		stackdriverClient, err := clients.CreateStackdriverClient()
		if err != nil {
			log.Fatalf("Cannot create Stackdriver client: %v", err)
		}

		cloudSql, err := publishers.CreateCloudSqlPublisher(settings.CloudSqlUser, settings.CloudSqlPassword, settings.CloudSqlInstance, settings.CloudSqlDatabase, settings.CloudSqlLocalPort)
		if err != nil {
			log.Fatalf("Failed to set up Cloud SQL publisher: %v", err)
		}
	*/

	pid := &data.PipelineID{Org: "bazel", Slug: "google-bazel-presubmit"}

	/*
		stackdriver := publishers.CreateStackdriverPublisher(stackdriverClient, *projectID)
	*/

	stdout := publishers.CreateStdoutPublisher(publishers.Csv)

	srv := service.CreateService(handleError)

	/*
		// TODO: support multiple organizations
		platformLoad := metrics.CreatePlatformLoad(bk, 100, settings.BuildkiteOrgs...)
		srv.AddMetric(platformLoad, 60, stackdriver)

		buildsPerChange := metrics.CreateBuildsPerChange(bk, 500, pipelines...)
		srv.AddMetric(buildsPerChange, 60, stdout)

		buildSuccess := metrics.CreateBuildSuccess(bk, 200, pipelines...)
		srv.AddMetric(buildSuccess, 60, stdout)
	*/

	// TODO(fweikert): use real settings instead of hardcoded values
	flakiness := metrics.CreateFlakiness(gcs, "bazel-buildkite-stats", "flaky-tests-bep", pid) // TODO: pipelines...)
	srv.AddMetric(flakiness, 60, stdout)

	/*
		macPerformance := metrics.CreateMacPerformance(bk, 20, pid) // TODO: pipelines...)
		srv.AddMetric(macPerformance, 60, stdout)

		pipelinePerformance := metrics.CreatePipelinePerformance(bk, 20, pipelines...)
		srv.AddMetric(pipelinePerformance, 60, stdout)

		platformSignificance := metrics.CreatePlatformSignificance(bk, 100, pipelines...)
		srv.AddMetric(platformSignificance, 24*60, stdout)

		platformUsage := metrics.CreatePlatformUsage(bk, 100, settings.BuildkiteOrgs...)
		srv.AddMetric(platformUsage, 60, stdout)

		releaseDownloads := metrics.CreateReleaseDownloads(settings.GitHubOrg,
			settings.GitHubRepo,
			settings.GitHubApiToken, megaByte)
		srv.AddMetric(releaseDownloads, 12*60, stdout)

		workerAvailability := metrics.CreateWorkerAvailability(bk, settings.BuildkiteOrgs...)
		srv.AddMetric(workerAvailability, 60, stdout)
	*/

	if *testMode {
		log.Println("[Test mode] Running all jobs exactly once...")
		srv.RunJobsOnce()
		os.Exit(0)
	}

	srv.Start()
	http.HandleFunc("/", handleRequest)
	appengine.Main()
}
<|endoftext|>


<commit_before>//  Copyright 2016 The goscope Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package gui

import (
	"fmt"
	"image"
	"image/color"
	"image/png"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"testing"

	"github.com/zagrodzki/goscope/dummy"
	"github.com/zagrodzki/goscope/scope"
)

// TODO: more tests for specific functions

// isOn returns true if pixel x,y is of a different color than white
func isOn(img image.Image, x, y int) bool {
	return img.At(x, y) != colorWhite
}

// evaluatePlot checks whether the tested plot:
// 1) has the same bounds as the true plot
// 2) is entirely contained in the true plot
// 3) contains at least one pixel in every column
// 4) contains at least minPointCount pixels
func evaluatePlot(truePlot, testPlot image.Image, minPointCount int) (bool, string) {
	if truePlot.Bounds() != testPlot.Bounds() {
		return false, fmt.Sprintf("plot bounds: got %v, expected %v", testPlot.Bounds(), truePlot.Bounds())
	}
	b := truePlot.Bounds()
	pointCount := 0
	for x := b.Min.X; x < b.Max.X; x++ {
		col := false
		for y := b.Min.Y; y < b.Max.Y; y++ {
			testOn := isOn(testPlot, x, y)
			col = col || testOn
			if testOn {
				pointCount++
			}
			if testOn && !isOn(truePlot, x, y) {
				return false, "test plot is not contained in true plot"
			}
		}
		if !col {
			return false, fmt.Sprintf("image column %v does not contain any point", x)
		}
	}
	if pointCount < minPointCount {
		return false, fmt.Sprintf("too few plot points: got %v, expected at least %v", pointCount, minPointCount)
	}
	return true, ""
}

func TestPlot(t *testing.T) {
	for _, tc := range []struct {
		desc          string
		numSamples    int
		gen           func(int) scope.Sample
		minPointCount int
		plotFile      string
	}{
		{
			desc:       "sin",
			numSamples: 1000,
			gen: func(i int) scope.Sample {
				return scope.Sample(math.Sin(float64(i) * 4 * math.Pi / 999))
			},
			minPointCount: 2000,
			plotFile:      "sin-gp.png",
		},
		{
			desc:       "zero",
			numSamples: 1000,
			gen: func(i int) scope.Sample {
				return 0
			},
			minPointCount: 800,
			plotFile:      "zero-gp.png",
		},
		{
			desc:       "square",
			numSamples: 1000,
			gen: func(i int) scope.Sample {
				return scope.Sample(-2*(i/250%2) + 1)
			},
			minPointCount: 2000,
			plotFile:      "square-gp.png",
		},
		{
			desc:       "triangle",
			numSamples: 999,
			gen: func(i int) scope.Sample {
				sign := 2*(i/333%2) - 1
				return scope.Sample(float64(sign) * (1.0 - float64(i%333)*2.0/332.0))
			},
			minPointCount: 1000,
			plotFile:      "triangle-gp.png",
		},
	} {
		samples := make([]scope.Sample, tc.numSamples)
		for i := 0; i < tc.numSamples; i++ {
			samples[i] = tc.gen(i)
		}
		file, err := os.Open(tc.plotFile)
		if err != nil {
			t.Fatalf("Cannot open file: %v", err)
		}
		img, err := png.Decode(file)
		if err != nil {
			t.Fatalf("Cannot decode file: %v", err)
		}

		plot := Plot{image.NewRGBA(image.Rect(0, 0, 800, 600))}
		plot.Fill(colorWhite)
		plotBounds := plot.Bounds()
		plot.DrawSamples(samples, TracePos{0.5, 0.25}, plotBounds.Min, plotBounds.Max, colorBlack)
		eval, msg := evaluatePlot(img, plot, tc.minPointCount)
		if !eval {
			t.Errorf(fmt.Sprintf("error in evaluating plot %v: %v", tc.desc, msg))
		}
	}
}

func TestPlotToPng(t *testing.T) {
	dev, err := dummy.Open("")
	if err != nil {
		t.Fatalf("Cannot open the device: %v", err)
	}
	dir, err := ioutil.TempDir("", "TestPlotToPng")
	if err != nil {
		t.Fatalf("Cannot create temp dir: %v", err)
	}
	defer os.RemoveAll(dir)
	err = PlotToPng(dev, 800, 600,
		make(map[scope.ChanID]TracePos),
		make(map[scope.ChanID]color.RGBA),
		filepath.Join(dir, "plot.png"))
	if err != nil {
		t.Fatalf("Cannot plot to file: %v", err)
	}
}

func TestPlotToPngWithCustomParameters(t *testing.T) {
	dev, err := dummy.Open("")
	if err != nil {
		t.Fatalf("Cannot open the device: %v", err)
	}
	dir, err := ioutil.TempDir("", "TestPlotToPngWithCustomScales")
	if err != nil {
		t.Fatalf("Cannot create temp dir: %v", err)
	}
	defer os.RemoveAll(dir)
	tracePos := map[scope.ChanID]TracePos{
		"square":   TracePos{0.1, 5},
		"triangle": TracePos{0.8, 2},
	}
	cols := map[scope.ChanID]color.RGBA{
		"random":   color.RGBA{255, 0, 0, 255},
		"sin":      color.RGBA{255, 0, 255, 255},
		"square":   color.RGBA{0, 255, 0, 255},
		"triangle": color.RGBA{0, 0, 255, 255},
	}
	err = PlotToPng(dev, 800, 600, tracePos, cols, filepath.Join(dir, "plot.png"))
	if err != nil {
		t.Fatalf("Cannot plot to file: %v", err)
	}
}

func BenchmarkCreatePlot(b *testing.B) {
	dev, err := dummy.Open("")
	if err != nil {
		b.Fatalf("Cannot open the device: %v", err)
	}
	plot := Plot{image.NewRGBA(image.Rect(0, 0, 800, 600))}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		err = plot.DrawFromDevice(dev,
			make(map[scope.ChanID]TracePos),
			make(map[scope.ChanID]color.RGBA))
		if err != nil {
			b.Fatalf("Cannot create plot: %v", err)
		}
	}
}
<commit_msg>"true plot" changed into "reference plot"<commit_after>//  Copyright 2016 The goscope Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package gui

import (
	"fmt"
	"image"
	"image/color"
	"image/png"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"testing"

	"github.com/zagrodzki/goscope/dummy"
	"github.com/zagrodzki/goscope/scope"
)

// TODO: more tests for specific functions

// isOn returns true if pixel x,y is of a different color than white
func isOn(img image.Image, x, y int) bool {
	return img.At(x, y) != colorWhite
}

// evaluatePlot checks whether the tested plot:
// 1) has the same bounds as the reference plot
// 2) is entirely contained in the reference plot
// 3) contains at least one pixel in every column
// 4) contains at least minPointCount pixels
func evaluatePlot(refPlot, testPlot image.Image, minPointCount int) (bool, string) {
	if refPlot.Bounds() != testPlot.Bounds() {
		return false, fmt.Sprintf("plot bounds: got %v, expected %v", testPlot.Bounds(), refPlot.Bounds())
	}
	b := refPlot.Bounds()
	pointCount := 0
	for x := b.Min.X; x < b.Max.X; x++ {
		col := false
		for y := b.Min.Y; y < b.Max.Y; y++ {
			testOn := isOn(testPlot, x, y)
			col = col || testOn
			if testOn {
				pointCount++
			}
			if testOn && !isOn(refPlot, x, y) {
				return false, "test plot is not contained in reference plot"
			}
		}
		if !col {
			return false, fmt.Sprintf("image column %v does not contain any point", x)
		}
	}
	if pointCount < minPointCount {
		return false, fmt.Sprintf("too few plot points: got %v, expected at least %v", pointCount, minPointCount)
	}
	return true, ""
}

func TestPlot(t *testing.T) {
	for _, tc := range []struct {
		desc          string
		numSamples    int
		gen           func(int) scope.Sample
		minPointCount int
		refPlotFile   string
	}{
		{
			desc:       "sin",
			numSamples: 1000,
			gen: func(i int) scope.Sample {
				return scope.Sample(math.Sin(float64(i) * 4 * math.Pi / 999))
			},
			minPointCount: 2000,
			refPlotFile:   "sin-gp.png",
		},
		{
			desc:       "zero",
			numSamples: 1000,
			gen: func(i int) scope.Sample {
				return 0
			},
			minPointCount: 800,
			refPlotFile:   "zero-gp.png",
		},
		{
			desc:       "square",
			numSamples: 1000,
			gen: func(i int) scope.Sample {
				return scope.Sample(-2*(i/250%2) + 1)
			},
			minPointCount: 2000,
			refPlotFile:   "square-gp.png",
		},
		{
			desc:       "triangle",
			numSamples: 999,
			gen: func(i int) scope.Sample {
				sign := 2*(i/333%2) - 1
				return scope.Sample(float64(sign) * (1.0 - float64(i%333)*2.0/332.0))
			},
			minPointCount: 1000,
			refPlotFile:   "triangle-gp.png",
		},
	} {
		samples := make([]scope.Sample, tc.numSamples)
		for i := 0; i < tc.numSamples; i++ {
			samples[i] = tc.gen(i)
		}
		file, err := os.Open(tc.refPlotFile)
		if err != nil {
			t.Fatalf("Cannot open file: %v", err)
		}
		refPlot, err := png.Decode(file)
		if err != nil {
			t.Fatalf("Cannot decode file: %v", err)
		}

		testPlot := Plot{image.NewRGBA(image.Rect(0, 0, 800, 600))}
		testPlot.Fill(colorWhite)
		b := testPlot.Bounds()
		testPlot.DrawSamples(samples, TracePos{0.5, 0.25}, b.Min, b.Max, colorBlack)
		eval, msg := evaluatePlot(refPlot, testPlot, tc.minPointCount)
		if !eval {
			t.Errorf(fmt.Sprintf("error in evaluating plot %v: %v", tc.desc, msg))
		}
	}
}

func TestPlotToPng(t *testing.T) {
	dev, err := dummy.Open("")
	if err != nil {
		t.Fatalf("Cannot open the device: %v", err)
	}
	dir, err := ioutil.TempDir("", "TestPlotToPng")
	if err != nil {
		t.Fatalf("Cannot create temp dir: %v", err)
	}
	defer os.RemoveAll(dir)
	err = PlotToPng(dev, 800, 600,
		make(map[scope.ChanID]TracePos),
		make(map[scope.ChanID]color.RGBA),
		filepath.Join(dir, "plot.png"))
	if err != nil {
		t.Fatalf("Cannot plot to file: %v", err)
	}
}

func TestPlotToPngWithCustomParameters(t *testing.T) {
	dev, err := dummy.Open("")
	if err != nil {
		t.Fatalf("Cannot open the device: %v", err)
	}
	dir, err := ioutil.TempDir("", "TestPlotToPngWithCustomScales")
	if err != nil {
		t.Fatalf("Cannot create temp dir: %v", err)
	}
	defer os.RemoveAll(dir)
	tracePos := map[scope.ChanID]TracePos{
		"square":   TracePos{0.1, 5},
		"triangle": TracePos{0.8, 2},
	}
	cols := map[scope.ChanID]color.RGBA{
		"random":   color.RGBA{255, 0, 0, 255},
		"sin":      color.RGBA{255, 0, 255, 255},
		"square":   color.RGBA{0, 255, 0, 255},
		"triangle": color.RGBA{0, 0, 255, 255},
	}
	err = PlotToPng(dev, 800, 600, tracePos, cols, filepath.Join(dir, "plot.png"))
	if err != nil {
		t.Fatalf("Cannot plot to file: %v", err)
	}
}

func BenchmarkCreatePlot(b *testing.B) {
	dev, err := dummy.Open("")
	if err != nil {
		b.Fatalf("Cannot open the device: %v", err)
	}
	plot := Plot{image.NewRGBA(image.Rect(0, 0, 800, 600))}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		err = plot.DrawFromDevice(dev,
			make(map[scope.ChanID]TracePos),
			make(map[scope.ChanID]color.RGBA))
		if err != nil {
			b.Fatalf("Cannot create plot: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>package main

// $ curl localhost:9999/articles -H "Last-Event-ID: 33895"

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/donovanhide/eventsource"
	"github.com/golang/glog"
	"github.com/gorilla/mux"
	_ "github.com/lib/pq"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

var port = flag.Int("port", 9999, "port to run server on")
var dbstring = flag.String("db", "user=jl dbname=jl host=/var/run/postgresql sslmode=disable", "connection string for database")
var timeout = flag.Duration("timeout", 10*time.Second, "timeout for a client disconnection")
var interval = flag.Duration("interval", 5*time.Second, "delay between monitor polls")

// article encodes article data we want to stream out as events
type article struct {
	// internal article id
	id string
	// internal channel
	channel     string
	Permalink   string    `json:"permalink"`
	Title       string    `json:"title"`
	Content     string    `json:"text"`
	LastScraped time.Time `json:"lastscraped"`
	Published   time.Time `json:"published"`
	Source      string    `json:"source"`
	Urls        []string  `json:"urls"`
	Journalists []string  `json:"journalists"`
}

func (art *article) Id() string {
	return art.id
}

func (art *article) Event() string {
	return "article"
}

func (art *article) Data() string {
	out, _ := json.Marshal(art)
	return string(out)
}

type articleRepository struct {
	*sql.DB
}

func (repo *articleRepository) streamArticles(channel, lastEventId string, stream chan *article) (last *article) {
	tx, err := repo.Begin()
	if err != nil {
		glog.Error(err)
		return
	}
	defer tx.Rollback()
	glog.V(2).Infoln("Declaring cursor", lastEventId, channel)
	_, err = tx.Exec(`	DECLARE cur NO SCROLL CURSOR FOR
					  	SELECT id
					  	FROM article
						WHERE id > $1
        				AND ($2 = 'articles' OR srcorg=(SELECT id FROM organisation WHERE shortname=$2))
        				ORDER BY id ASC
						`, lastEventId, channel)
	if err != nil {
		glog.Fatalln(err)
		return nil
	}
	for {
		glog.V(2).Infoln("Fetching next 100.")
		rows, err := tx.Query(`FETCH FORWARD 100 FROM cur;`)
		if err != nil {
			glog.Error(err)
			return
		}
		defer rows.Close()
		ids := make([]string, 0)
		for rows.Next() {
			var id string
			if err := rows.Scan(&id); err != nil {
				glog.Error(err)
				return
			}
			ids = append(ids, id)
		}
		if len(ids) == 0 {
			break
		}
		articles, err := tx.Query(fmt.Sprintf(`SELECT a.id,
						MAX(a.permalink),
						MAX(a.title),
						MAX(a.pubdate),
						MAX(a.lastscraped) AS lastscraped,
						COALESCE(MAX(o.prettyname),''),
						COALESCE(string_agg(u.url,' '),''),
						COALESCE(string_agg(j.prettyname,','),''),
						COALESCE(MAX(c.content),''),
						MAX(o.shortname)
						FROM article a
						INNER JOIN article_content c
						ON a.id=c.article_id
						LEFT OUTER JOIN article_url u
						ON a.id=u.article_id
						LEFT OUTER JOIN journo_attr attr
						ON a.id = attr.article_id
						INNER JOIN journo j
						ON attr.journo_id=j.id
						INNER JOIN organisation o
						ON a.srcorg=o.id
						WHERE a.id IN (%s)
						GROUP BY a.id
        				ORDER BY a.id ASC`, strings.Join(ids, ",")))
		if err != nil {
			glog.Error(err)
			return
		}
		for articles.Next() {
			var art article
			var urls, journalists string
			if err := articles.Scan(&art.id, &art.Permalink, &art.Title, &art.Published, &art.LastScraped, &art.Source, &urls, &journalists, &art.Content, &art.channel); err != nil {
				glog.Error(err)
				return nil
			}
			art.Urls = strings.Split(urls, " ")
			art.Journalists = strings.Split(journalists, ",")
			glog.V(2).Infof("Got Channel: %s Id: %s", art.channel, art.id)
			select {
			case <-time.After(*timeout):
				glog.Warning("Timeout on cursor pump")
				return
			case stream <- &art:
				last = &art
			}
		}
	}
	return
}

func (repo *articleRepository) Replay(channel, lastEventId string) (events chan eventsource.Event) {
	stream := make(chan *article)
	events = make(chan eventsource.Event, 500)
	glog.Infof("Replaying Channel: %s from Last-Event-Id: %s", channel, lastEventId)
	go func() {
		for art := range stream {
			events <- art
		}
		close(events)
	}()
	go func() {
		defer close(stream)
		last := repo.streamArticles(channel, lastEventId, stream)
		glog.Infof("Finished Replaying Channel: %s from Last-Event-ID: %s To: %s", channel, lastEventId, last.id)
	}()
	return
}

func (repo *articleRepository) Monitor(srv *eventsource.Server) {
	stream := make(chan *article)
	defer close(stream)
	var lastEventId string
	if err := repo.QueryRow("SELECT MAX(id) FROM article").Scan(&lastEventId); err != nil {
		glog.Fatal(err)
	}
	glog.Infof("Monitoring from Id: %s onwards ", lastEventId)
	go func() {
		for art := range stream {
			glog.Infof("Publishing Id: %s Channel: %s", art.id, art.channel)
			srv.Publish([]string{"articles", art.channel}, art)
		}
	}()
	for {
		glog.V(2).Infof("Polling for anything after: %s", lastEventId)
		if last := repo.streamArticles("articles", lastEventId, stream); last != nil {
			lastEventId = last.id
		}
		time.Sleep(*interval)
	}

}

func main() {
	flag.Parse()
	db, err := sql.Open("postgres", *dbstring)
	if err != nil {
		glog.Fatal(err)
	}
	if err = db.Ping(); err != nil {
		glog.Fatal(err)
	}
	repo := &articleRepository{db}
	srv := eventsource.NewServer()
	srv.Register("articles", repo)
	channels := make(map[string]struct{})
	var lock sync.Mutex
	router := mux.NewRouter().StrictSlash(false)
	router.HandleFunc("/", srv.Handler("articles"))
	router.HandleFunc("/{channel}/", func(resp http.ResponseWriter, req *http.Request) {
		channel := mux.Vars(req)["channel"]
		lock.Lock()
		if _, ok := channels[channel]; !ok {
			channels[channel] = struct{}{}
			srv.Register(channel, repo)
		}
		lock.Unlock()
		srv.Handler(mux.Vars(req)["channel"])(resp, req)
	})
	http.Handle("/", router)
	l, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
	if err != nil {
		glog.Fatal(err)
	}
	defer l.Close()
	glog.Infof("Listening on port %d", *port)
	go repo.Monitor(srv)
	http.Serve(l, nil)
}
<commit_msg>Replay bugfix<commit_after>package main

// $ curl localhost:9999/articles -H "Last-Event-ID: 33895"

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/donovanhide/eventsource"
	"github.com/golang/glog"
	"github.com/gorilla/mux"
	_ "github.com/lib/pq"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

var port = flag.Int("port", 9999, "port to run server on")
var dbstring = flag.String("db", "user=jl dbname=jl host=/var/run/postgresql sslmode=disable", "connection string for database")
var timeout = flag.Duration("timeout", 10*time.Second, "timeout for a client disconnection")
var interval = flag.Duration("interval", 5*time.Second, "delay between monitor polls")

// article encodes article data we want to stream out as events
type article struct {
	// internal article id
	id string
	// internal channel
	channel     string
	Permalink   string    `json:"permalink"`
	Title       string    `json:"title"`
	Content     string    `json:"text"`
	LastScraped time.Time `json:"lastscraped"`
	Published   time.Time `json:"published"`
	Source      string    `json:"source"`
	Urls        []string  `json:"urls"`
	Journalists []string  `json:"journalists"`
}

func (art *article) Id() string {
	return art.id
}

func (art *article) Event() string {
	return "article"
}

func (art *article) Data() string {
	out, _ := json.Marshal(art)
	return string(out)
}

type articleRepository struct {
	*sql.DB
}

func (repo *articleRepository) streamArticles(channel, lastEventId string, stream chan *article) (last *article) {
	tx, err := repo.Begin()
	if err != nil {
		glog.Error(err)
		return
	}
	defer tx.Rollback()
	glog.V(2).Infoln("Declaring cursor", lastEventId, channel)
	_, err = tx.Exec(`	DECLARE cur NO SCROLL CURSOR FOR
					  	SELECT id
					  	FROM article
						WHERE id > $1
        				AND ($2 = 'articles' OR srcorg=(SELECT id FROM organisation WHERE shortname=$2))
        				ORDER BY id ASC
						`, lastEventId, channel)
	if err != nil {
		glog.Fatalln(err)
		return nil
	}
	for {
		glog.V(2).Infoln("Fetching next 100.")
		rows, err := tx.Query(`FETCH FORWARD 100 FROM cur;`)
		if err != nil {
			glog.Error(err)
			return
		}
		defer rows.Close()
		ids := make([]string, 0)
		for rows.Next() {
			var id string
			if err := rows.Scan(&id); err != nil {
				glog.Error(err)
				return
			}
			ids = append(ids, id)
		}
		if len(ids) == 0 {
			break
		}
		articles, err := tx.Query(fmt.Sprintf(`SELECT a.id,
						MAX(a.permalink),
						MAX(a.title),
						MAX(a.pubdate),
						MAX(a.lastscraped) AS lastscraped,
						COALESCE(MAX(o.prettyname),''),
						COALESCE(string_agg(u.url,' '),''),
						COALESCE(string_agg(j.prettyname,','),''),
						COALESCE(MAX(c.content),''),
						MAX(o.shortname)
						FROM article a
						INNER JOIN article_content c
						ON a.id=c.article_id
						LEFT OUTER JOIN article_url u
						ON a.id=u.article_id
						LEFT OUTER JOIN journo_attr attr
						ON a.id = attr.article_id
						INNER JOIN journo j
						ON attr.journo_id=j.id
						INNER JOIN organisation o
						ON a.srcorg=o.id
						WHERE a.id IN (%s)
						GROUP BY a.id
        				ORDER BY a.id ASC`, strings.Join(ids, ",")))
		if err != nil {
			glog.Error(err)
			return
		}
		for articles.Next() {
			var art article
			var urls, journalists string
			if err := articles.Scan(&art.id, &art.Permalink, &art.Title, &art.Published, &art.LastScraped, &art.Source, &urls, &journalists, &art.Content, &art.channel); err != nil {
				glog.Error(err)
				return nil
			}
			art.Urls = strings.Split(urls, " ")
			art.Journalists = strings.Split(journalists, ",")
			glog.V(2).Infof("Got Channel: %s Id: %s", art.channel, art.id)
			select {
			case <-time.After(*timeout):
				glog.Warning("Timeout on cursor pump")
				return
			case stream <- &art:
				last = &art
			}
		}
	}
	return
}

func (repo *articleRepository) Replay(channel, lastEventId string) (events chan eventsource.Event) {
	stream := make(chan *article)
	events = make(chan eventsource.Event, 500)
	glog.Infof("Replaying Channel: %s from Last-Event-Id: %s", channel, lastEventId)
	go func() {
		for art := range stream {
			events <- art
		}
		close(events)
	}()
	go func() {
		defer close(stream)
		if last := repo.streamArticles(channel, lastEventId, stream); last != nil {
			glog.Infof("Finished Replaying Channel: %s from Last-Event-ID: %s To: %s", channel, lastEventId, last.id)
		} else {
			glog.Infof("Nothing to replay")
		}
	}()
	return
}

func (repo *articleRepository) Monitor(srv *eventsource.Server) {
	stream := make(chan *article)
	defer close(stream)
	var lastEventId string
	if err := repo.QueryRow("SELECT MAX(id) FROM article").Scan(&lastEventId); err != nil {
		glog.Fatal(err)
	}
	glog.Infof("Monitoring from Id: %s onwards ", lastEventId)
	go func() {
		for art := range stream {
			glog.Infof("Publishing Id: %s Channel: %s", art.id, art.channel)
			srv.Publish([]string{"articles", art.channel}, art)
		}
	}()
	for {
		glog.V(2).Infof("Polling for anything after: %s", lastEventId)
		if last := repo.streamArticles("articles", lastEventId, stream); last != nil {
			lastEventId = last.id
		}
		time.Sleep(*interval)
	}

}

func main() {
	flag.Parse()
	db, err := sql.Open("postgres", *dbstring)
	if err != nil {
		glog.Fatal(err)
	}
	if err = db.Ping(); err != nil {
		glog.Fatal(err)
	}
	repo := &articleRepository{db}
	srv := eventsource.NewServer()
	srv.Register("articles", repo)
	channels := make(map[string]struct{})
	var lock sync.Mutex
	router := mux.NewRouter().StrictSlash(false)
	router.HandleFunc("/", srv.Handler("articles"))
	router.HandleFunc("/{channel}/", func(resp http.ResponseWriter, req *http.Request) {
		channel := mux.Vars(req)["channel"]
		lock.Lock()
		if _, ok := channels[channel]; !ok {
			channels[channel] = struct{}{}
			srv.Register(channel, repo)
		}
		lock.Unlock()
		srv.Handler(mux.Vars(req)["channel"])(resp, req)
	})
	http.Handle("/", router)
	l, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
	if err != nil {
		glog.Fatal(err)
	}
	defer l.Close()
	glog.Infof("Listening on port %d", *port)
	go repo.Monitor(srv)
	http.Serve(l, nil)
}
<|endoftext|>


<commit_before>package xml2json

import (
	"strconv"
	"strings"
)

// https://cswr.github.io/JsonSchema/spec/basic_types/
// JSType is a JavaScript extracted from a string
type JSType int

const (
	Bool JSType = iota
	Int
	Float
	String
)

// Str2JSType extract a JavaScript type from a string
func Str2JSType(s string) JSType {
	var (
		output JSType
	)
	s = strings.TrimSpace(s) // santize the given string
	switch {
	case isBool(s):
		output = Bool
	case isFloat(s):
		output = Float
	case isInt(s):
		output = Int
	default:
		output = String // if all alternatives have been eliminated, the input is a string
	}
	return output
}

func isBool(s string) bool {
	return s == "true" || s == "false"
}

func isFloat(s string) bool {
	var output = false
	if strings.Contains(s, ".") {
		_, err := strconv.ParseFloat(s, 64)
		if err == nil { // the string successfully converts to a decimal
			output = true
		}
	}
	return output
}

func isInt(s string) bool {
	var output = false
	if len(s) > 1 {
		_, err := strconv.Atoi(s)
		if err == nil { // the string successfully converts to an int
			if s[0] == '0' {
				// if the first rune is '0' and there is more than 1 rune, then the input is most likely a float or intended to be
				// a string value -- such as in the case of a guid, or an international phone number
			} else {
				output = true
			}
		}
	}
	return output
}
<commit_msg>[fix]  Change JS type `Int` parser to accept `0` and single digit numbers<commit_after>package xml2json

import (
	"strconv"
	"strings"
)

// https://cswr.github.io/JsonSchema/spec/basic_types/
// JSType is a JavaScript extracted from a string
type JSType int

const (
	Bool JSType = iota
	Int
	Float
	String
)

// Str2JSType extract a JavaScript type from a string
func Str2JSType(s string) JSType {
	var (
		output JSType
	)
	s = strings.TrimSpace(s) // santize the given string
	switch {
	case isBool(s):
		output = Bool
	case isFloat(s):
		output = Float
	case isInt(s):
		output = Int
	default:
		output = String // if all alternatives have been eliminated, the input is a string
	}
	return output
}

func isBool(s string) bool {
	return s == "true" || s == "false"
}

func isFloat(s string) bool {
	var output = false
	if strings.Contains(s, ".") {
		_, err := strconv.ParseFloat(s, 64)
		if err == nil { // the string successfully converts to a decimal
			output = true
		}
	}
	return output
}

func isInt(s string) bool {
	var output = false
	if len(s) >= 1 {
		_, err := strconv.Atoi(s)
		if err == nil { // the string successfully converts to an int
			if s != "0" && s[0] == '0' {
				// if the first rune is '0' and there is more than 1 rune, then the input is most likely a float or intended to be
				// a string value -- such as in the case of a guid, or an international phone number
			} else {
				output = true
			}
		}
	}
	return output
}
<|endoftext|>


<commit_before>/*
Package kenken provides functions to read KenKen puzzles
in the format used by http://app.kenkenpuzzle.com/kenken/puzzles/
*/
package kenken

type Puzzle struct {
	Answer    [][]int
	Clue      [][]int
	Operation [][]Operation

	// N rows of N-1 columns
	// Vertical[y][x-1] == true means there is a heavy vertical line
	// between (x-1, y) and (x, y)
	Vertical [][]bool

	// N rows of N-1 columns
	// Horizontal[x][y-1] == true means there is a heavy horizontal line
	// between (x, y-1) and (x, y) (note transpose)
	Horizontal [][]bool
}

type Operation byte

const (
	None Operation = iota
	Given
	Sum
	Difference
	Product
	Quotient
)

func (k *Puzzle) Size() int {
	return len(k.Answer)
}

var ops = []string{"?", "?", "+", "−", "×", "∕"}

func (op Operation) Symbol() string {
	return ops[op]
}
<commit_msg>Improve comments<commit_after>/*
Package kenken provides functions to read KenKen puzzles
in the format used by http://app.kenkenpuzzle.com/kenken/puzzles/
*/
package kenken

type Puzzle struct {
	Answer    [][]int
	Clue      [][]int
	Operation [][]Operation

	// N rows of N-1 columns
	// Vertical[y][x] is true if there is a heavy vertical line
	// between (x, y) and (x+1, y)
	Vertical [][]bool

	// N rows of N-1 columns
	// Horizontal[x][y] is true if there is a heavy horizontal line
	// between (x, y) and (x, y+1) (note transpose)
	Horizontal [][]bool
}

type Operation byte

const (
	None Operation = iota
	Given
	Sum
	Difference
	Product
	Quotient
)

func (k *Puzzle) Size() int {
	return len(k.Answer)
}

var ops = []string{"?", "?", "+", "−", "×", "∕"}

func (op Operation) Symbol() string {
	return ops[op]
}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2013, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"os"
	"fmt"
	"time"
	"strings"
	"reflect"
	"strconv"
	"syscall"
	"os/signal"
	"math/rand"
)

const (
	nadaStr = "" // This is internal to dlshared.
	commaStr = "," // This is internal to dlshared.

	injectLoggerName = "dlshared.Logger"
	injectLoggerFieldName = "Logger"

	injectConfigurationName = "*dlshared.Configuration"
	injectConfigurationFieldName = "Configuration"

	injectMongoDataSourceName = "dlshared.MongoDataSource"
)

type Kernel struct {
	Configuration *Configuration
	Components map[string]Component
	components []Component
	Id string
	Logger
	Pid int
}

type Component struct {
	componentId string
	singleton interface{}
	startMethodName string
	stopMethodName string
}

// Returns true if the component is present. This panics if the component id is empty.
func (self *Kernel) HasComponent(componentId string) (found bool) {
	panicIfComponentIdNotSet(componentId)
	_, found = self.Components[componentId]
	return
}

// Access another component. This method will panic if you attempt to reference a
// non-existent component. If the component id has a length of zero, it is also panics.
func (self *Kernel) GetComponent(componentId string) interface{} {

	panicIfComponentIdNotSet(componentId)

	if _, found := self.Components[componentId]; !found {
		panic(fmt.Sprintf("kernel.GetComponent called with an invalid component id: %s", componentId))
	}

	return self.Components[componentId].singleton.(interface{})
}

func panicIfComponentIdNotSet(componentId string) { if len(componentId) == 0 { panic("kernel.GetComponent called with an empty component id") } }

// Register a component with a start and stop methods. This method will panic if a nil component is passed.
func (self *Kernel) AddComponentWithStartStopMethods(componentId string, singleton interface{}, startMethodName, stopMethodName string) {

	if singleton == nil { panic(fmt.Sprintf("Nil component passed to kernel for id: %s", componentId)) }

	component := Component{ componentId : componentId, singleton : singleton, startMethodName : startMethodName, stopMethodName : stopMethodName }

	self.components = append(self.components , component)
	self.Components[componentId] = component
}

// Register a component with a start method.
func (self *Kernel) AddComponentWithStartMethod(componentId string, singleton interface{}, startMethodName string) {
	self.AddComponentWithStartStopMethods(componentId, singleton, startMethodName, "")
}

// Register a component with a stop method.
func (self *Kernel) AddComponentWithStopMethod(componentId string, singleton interface{}, stopMethodName string) {
	self.AddComponentWithStartStopMethods(componentId, singleton, "", stopMethodName)
}

// Register a component without a start or stop method.
func (self *Kernel) AddComponent(componentId string, singleton interface{}) {
	self.AddComponentWithStartStopMethods(componentId, singleton, "", "")
}

// Called by the kernel during Start/Stop.
func callStartStopMethod(methodTypeName, methodName string, singleton interface{}, kernel *Kernel) error {

	value := reflect.ValueOf(singleton)

	methodValue := value.MethodByName(methodName)

	if !methodValue.IsValid() {
		return fmt.Errorf("Start method: %s is NOT found on struct: %s", methodName, value.Type())
	}

	methodType := methodValue.Type()

	if methodType.NumOut() > 1 {
		panic(fmt.Sprintf("The %s method: %s on struct: %s has more than one return value - you can only return error or nothing", methodTypeName, methodName, value.Type()))
	}

	if methodType.NumIn() > 1 {
		return fmt.Errorf("The %s method: %s on struct: %s has more than one parameter - you can only pass the Kernel or nothing", methodTypeName, methodName, value.Type())
	}

	// Verify the return type is error
	if methodType.NumOut() == 1 && methodType.Out(0).Name() != "error" {
		return fmt.Errorf("The %s method: %s on struct: %s has an invalid return type - you can return nothing or error", methodTypeName, methodName, value.Type())
	}

	methodInputs := make([]reflect.Value, 0)
	if methodType.NumIn() == 1 { methodInputs = append(methodInputs, reflect.ValueOf(kernel)) }

	returnValues := methodValue.Call(methodInputs)

	// Check to see if there was an error
	if len(returnValues) == 1 {
		err := returnValues[0].Interface()
		if err != nil { return err.(error) }
	}

	return nil
}

// Call this after the kernel has been created and components registered.
func (self *Kernel) Start() error {

	rand.Seed(time.Now().UTC().UnixNano())

	self.Logf(Info, "Starting %s - version: %s - config file: %s", self.Id, self.Configuration.Version, self.Configuration.FileName)

	if err := self.injectComponents(); err != nil { return err }

	for i := range self.components {
		if len(self.components[i].startMethodName) > 0 {
			if err := callStartStopMethod("start", self.components[i].startMethodName, self.components[i].singleton, self); err != nil {
				return err
			}
		}
	}

	self.Logf(Info, "Started %s - version: %s - config file: %s ", self.Id, self.Configuration.Version, self.Configuration.FileName)

	return nil
}

func (self *Kernel) injectComponents() error {

	// Loop through the components, look at the variables for tags and automatically do the injection
	// if the tag is set on a field.
	for componentId, component := range self.Components {

		// Get the value of the component and cast.
		componentValue := reflect.ValueOf(component.singleton).Elem()
		componentType := reflect.TypeOf(componentValue.Interface())

		// Loop through the fields.
		fieldCount := componentType.NumField()
		for i := 0; i < fieldCount; i++ {

			structField := componentType.Field(i)
			fieldValue := componentValue.Field(i)

			var mongoDbName string
			var mongoCollectionName string

			// Check to see if the tag is set.
			injectComponentId := structField.Tag.Get("dlinject")

			if len(injectComponentId) == 0 {
				if structField.Type.String() == injectLoggerName && structField.Name == injectLoggerFieldName {
					fieldValue.Set(reflect.ValueOf(self.Logger))
					continue
				}

				if structField.Type.String() == injectConfigurationName && structField.Name == injectConfigurationFieldName {
					fieldValue.Set(reflect.ValueOf(self.Configuration))
					continue
				}

				continue
			}

			if structField.Type.String() == injectMongoDataSourceName {
				dataSourceConfig := strings.Split(injectComponentId, commaStr)

				if len(dataSourceConfig) != 3 {
					return NewStackError(	"Unable to inject component: %s - into component: %s - reason: config must be componentId,dbName,collectionName",
											injectComponentId,
											componentId)
				}

				injectComponentId = dataSourceConfig[0]
				mongoDbName = dataSourceConfig[1]
				mongoCollectionName = dataSourceConfig[2]
			}

			// Make sure the component is present.
			injectComponent, found := self.Components[injectComponentId]
			if !found {
				return NewStackError(	"Unable to inject component: %s - into component: %s - reason: %s not found",
										injectComponentId,
										componentId,
										injectComponentId)
			}

			if !fieldValue.CanSet() {
				return NewStackError(	"Unable to inject component: %s - into component: %s - on field: %s - reason: field not exported",
										injectComponentId,
										componentId,
										structField.Name)
			}

			// Check to see if this is a mongo data source component
			if structField.Type.String() == injectMongoDataSourceName {
				fieldValue.Set(reflect.ValueOf(MongoDataSource{	DbName: mongoDbName,
																CollectionName: mongoCollectionName,
																Mongo: injectComponent.singleton.(*Mongo),
																Logger: self.Logger,
				}))

			} else { fieldValue.Set(reflect.ValueOf(injectComponent.singleton)) }
		}
	}

	return nil
}

// Stop the kernel. Call this before exiting.
func (self *Kernel) Stop() error {

	self.Logf(Info, "Stopping %s - version: %s - config file %s", self.Id, self.Configuration.Version, self.Configuration.FileName)

	for i := len(self.components)-1 ; i >= 0 ; i-- {

		if len(self.components[i].stopMethodName) > 0 {
			if err := callStartStopMethod("stop", self.components[i].stopMethodName, self.components[i].singleton, self); err != nil {
				return err
			}
		}
	}

	self.Logf(Info, "Stopped %s - version: %s - config file: %s", self.Id, self.Configuration.Version, self.Configuration.FileName)

	return nil
}

func newKernel(id, configFileName string) (*Kernel, error) {

	// Init the application configuration
	conf, err := NewConfiguration(configFileName)

	if err != nil {
		return nil, err
	}

	// Init the logger
	logAppenders, err := configureLogger(id, conf)
	if err != nil {
		return nil, err
	}

	logger := Logger{ Prefix: id, Appenders: logAppenders }

	// Create the kernel
	kernel := &Kernel{ Components : make(map[string]Component), Configuration : conf }
	kernel.Logger = logger
	kernel.Id = id

	if err = writePidFile(kernel); err != nil {
		return nil, err
	}

	return kernel, nil
}

// TODO: Add a logging structure to the configuration file and configure. Make
// sure this supports configuring syslog.
func configureLogger(id string, conf *Configuration) ([]Appender, error) {

	var appenders []Appender
	appenders = append(appenders, LevelFilter(Debug, StdErrAppender()))

	if conf.EnvironmentIs("prod") {
		syslogAppender, err := NewSyslogAppender("", "", id)
		if err != nil {
			return nil, err
		}

		appenders = append(appenders, LevelFilter(Debug, syslogAppender))
	}

	return appenders, nil
}

func writePidFile(kernel *Kernel) error {
	kernel.Pid = os.Getpid()
	pidFile, err := os.Create(kernel.Configuration.PidFile)
	if err != nil {
		return NewStackError("Unable to start kernel - problem creating pid file %s - error: %v", kernel.Configuration.PidFile, err)
	}
	defer pidFile.Close()

	if _, err := pidFile.Write([]byte(strconv.Itoa(kernel.Pid))); err != nil {
		return NewStackError("Unable to start kernel - problem writing pid file %s - error: %v", kernel.Configuration.PidFile, err)
	}

	return nil
}

// Call this from your main to create the kernel. After init kernel is called you must add
// your components and then call kernel.Start()
func StartKernel(id string, configFileName string, addComponentsFunction func(kernel *Kernel)) (*Kernel, error) {

	kernel, err := newKernel(id, configFileName)
	if err != nil {
		return nil, err
	}

	addComponentsFunction(kernel)

    if err = kernel.Start(); err != nil {
		return nil, err
	}

	return kernel, nil
}

// ListenForInterrupt blocks until an interrupt signal is detected.
func (self *Kernel) ListenForInterrupt() error {
	quitChannel := make(chan bool)

	// Register the interrupt listener.
	interruptSignalChannel := make(chan os.Signal, 1)
	signal.Notify(interruptSignalChannel, os.Interrupt)
	go func() {
		for sig := range interruptSignalChannel {
			if sig == syscall.SIGINT {
				quitChannel <- true
			}
		}
	}()

	// Block until we receive the stop notification.
	select {
		case <- quitChannel: {
			return self.Stop()
		}
	}

	// Should never happen
	panic("How did we end up here?")
}

<commit_msg>added trim to mongo component injection<commit_after>/**
 * (C) Copyright 2013, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"os"
	"fmt"
	"time"
	"strings"
	"reflect"
	"strconv"
	"syscall"
	"os/signal"
	"math/rand"
)

const (
	nadaStr = "" // This is internal to dlshared.
	commaStr = "," // This is internal to dlshared.

	injectLoggerName = "dlshared.Logger"
	injectLoggerFieldName = "Logger"

	injectConfigurationName = "*dlshared.Configuration"
	injectConfigurationFieldName = "Configuration"

	injectMongoDataSourceName = "dlshared.MongoDataSource"
)

type Kernel struct {
	Configuration *Configuration
	Components map[string]Component
	components []Component
	Id string
	Logger
	Pid int
}

type Component struct {
	componentId string
	singleton interface{}
	startMethodName string
	stopMethodName string
}

// Returns true if the component is present. This panics if the component id is empty.
func (self *Kernel) HasComponent(componentId string) (found bool) {
	panicIfComponentIdNotSet(componentId)
	_, found = self.Components[componentId]
	return
}

// Access another component. This method will panic if you attempt to reference a
// non-existent component. If the component id has a length of zero, it is also panics.
func (self *Kernel) GetComponent(componentId string) interface{} {

	panicIfComponentIdNotSet(componentId)

	if _, found := self.Components[componentId]; !found {
		panic(fmt.Sprintf("kernel.GetComponent called with an invalid component id: %s", componentId))
	}

	return self.Components[componentId].singleton.(interface{})
}

func panicIfComponentIdNotSet(componentId string) { if len(componentId) == 0 { panic("kernel.GetComponent called with an empty component id") } }

// Register a component with a start and stop methods. This method will panic if a nil component is passed.
func (self *Kernel) AddComponentWithStartStopMethods(componentId string, singleton interface{}, startMethodName, stopMethodName string) {

	if singleton == nil { panic(fmt.Sprintf("Nil component passed to kernel for id: %s", componentId)) }

	component := Component{ componentId : componentId, singleton : singleton, startMethodName : startMethodName, stopMethodName : stopMethodName }

	self.components = append(self.components , component)
	self.Components[componentId] = component
}

// Register a component with a start method.
func (self *Kernel) AddComponentWithStartMethod(componentId string, singleton interface{}, startMethodName string) {
	self.AddComponentWithStartStopMethods(componentId, singleton, startMethodName, "")
}

// Register a component with a stop method.
func (self *Kernel) AddComponentWithStopMethod(componentId string, singleton interface{}, stopMethodName string) {
	self.AddComponentWithStartStopMethods(componentId, singleton, "", stopMethodName)
}

// Register a component without a start or stop method.
func (self *Kernel) AddComponent(componentId string, singleton interface{}) {
	self.AddComponentWithStartStopMethods(componentId, singleton, "", "")
}

// Called by the kernel during Start/Stop.
func callStartStopMethod(methodTypeName, methodName string, singleton interface{}, kernel *Kernel) error {

	value := reflect.ValueOf(singleton)

	methodValue := value.MethodByName(methodName)

	if !methodValue.IsValid() {
		return fmt.Errorf("Start method: %s is NOT found on struct: %s", methodName, value.Type())
	}

	methodType := methodValue.Type()

	if methodType.NumOut() > 1 {
		panic(fmt.Sprintf("The %s method: %s on struct: %s has more than one return value - you can only return error or nothing", methodTypeName, methodName, value.Type()))
	}

	if methodType.NumIn() > 1 {
		return fmt.Errorf("The %s method: %s on struct: %s has more than one parameter - you can only pass the Kernel or nothing", methodTypeName, methodName, value.Type())
	}

	// Verify the return type is error
	if methodType.NumOut() == 1 && methodType.Out(0).Name() != "error" {
		return fmt.Errorf("The %s method: %s on struct: %s has an invalid return type - you can return nothing or error", methodTypeName, methodName, value.Type())
	}

	methodInputs := make([]reflect.Value, 0)
	if methodType.NumIn() == 1 { methodInputs = append(methodInputs, reflect.ValueOf(kernel)) }

	returnValues := methodValue.Call(methodInputs)

	// Check to see if there was an error
	if len(returnValues) == 1 {
		err := returnValues[0].Interface()
		if err != nil { return err.(error) }
	}

	return nil
}

// Call this after the kernel has been created and components registered.
func (self *Kernel) Start() error {

	rand.Seed(time.Now().UTC().UnixNano())

	self.Logf(Info, "Starting %s - version: %s - config file: %s", self.Id, self.Configuration.Version, self.Configuration.FileName)

	if err := self.injectComponents(); err != nil { return err }

	for i := range self.components {
		if len(self.components[i].startMethodName) > 0 {
			if err := callStartStopMethod("start", self.components[i].startMethodName, self.components[i].singleton, self); err != nil {
				return err
			}
		}
	}

	self.Logf(Info, "Started %s - version: %s - config file: %s ", self.Id, self.Configuration.Version, self.Configuration.FileName)

	return nil
}

func (self *Kernel) injectComponents() error {

	// Loop through the components, look at the variables for tags and automatically do the injection
	// if the tag is set on a field.
	for componentId, component := range self.Components {

		// Get the value of the component and cast.
		componentValue := reflect.ValueOf(component.singleton).Elem()
		componentType := reflect.TypeOf(componentValue.Interface())

		// Loop through the fields.
		fieldCount := componentType.NumField()
		for i := 0; i < fieldCount; i++ {

			structField := componentType.Field(i)
			fieldValue := componentValue.Field(i)

			var mongoDbName string
			var mongoCollectionName string

			// Check to see if the tag is set.
			injectComponentId := structField.Tag.Get("dlinject")

			if len(injectComponentId) == 0 {
				if structField.Type.String() == injectLoggerName && structField.Name == injectLoggerFieldName {
					fieldValue.Set(reflect.ValueOf(self.Logger))
					continue
				}

				if structField.Type.String() == injectConfigurationName && structField.Name == injectConfigurationFieldName {
					fieldValue.Set(reflect.ValueOf(self.Configuration))
					continue
				}

				continue
			}

			if structField.Type.String() == injectMongoDataSourceName {
				dataSourceConfig := strings.Split(injectComponentId, commaStr)

				if len(dataSourceConfig) != 3 {
					return NewStackError(	"Unable to inject component: %s - into component: %s - reason: config must be componentId,dbName,collectionName",
											injectComponentId,
											componentId)
				}

				injectComponentId = strings.TrimSpace(dataSourceConfig[0])
				mongoDbName = strings.TrimSpace(dataSourceConfig[1])
				mongoCollectionName = strings.TrimSpace(dataSourceConfig[2])
			}

			// Make sure the component is present.
			injectComponent, found := self.Components[injectComponentId]
			if !found {
				return NewStackError(	"Unable to inject component: %s - into component: %s - reason: %s not found",
										injectComponentId,
										componentId,
										injectComponentId)
			}

			if !fieldValue.CanSet() {
				return NewStackError(	"Unable to inject component: %s - into component: %s - on field: %s - reason: field not exported",
										injectComponentId,
										componentId,
										structField.Name)
			}

			// Check to see if this is a mongo data source component
			if structField.Type.String() == injectMongoDataSourceName {
				fieldValue.Set(reflect.ValueOf(MongoDataSource{	DbName: mongoDbName,
																CollectionName: mongoCollectionName,
																Mongo: injectComponent.singleton.(*Mongo),
																Logger: self.Logger,
				}))

			} else { fieldValue.Set(reflect.ValueOf(injectComponent.singleton)) }
		}
	}

	return nil
}

// Stop the kernel. Call this before exiting.
func (self *Kernel) Stop() error {

	self.Logf(Info, "Stopping %s - version: %s - config file %s", self.Id, self.Configuration.Version, self.Configuration.FileName)

	for i := len(self.components)-1 ; i >= 0 ; i-- {

		if len(self.components[i].stopMethodName) > 0 {
			if err := callStartStopMethod("stop", self.components[i].stopMethodName, self.components[i].singleton, self); err != nil {
				return err
			}
		}
	}

	self.Logf(Info, "Stopped %s - version: %s - config file: %s", self.Id, self.Configuration.Version, self.Configuration.FileName)

	return nil
}

func newKernel(id, configFileName string) (*Kernel, error) {

	// Init the application configuration
	conf, err := NewConfiguration(configFileName)

	if err != nil {
		return nil, err
	}

	// Init the logger
	logAppenders, err := configureLogger(id, conf)
	if err != nil {
		return nil, err
	}

	logger := Logger{ Prefix: id, Appenders: logAppenders }

	// Create the kernel
	kernel := &Kernel{ Components : make(map[string]Component), Configuration : conf }
	kernel.Logger = logger
	kernel.Id = id

	if err = writePidFile(kernel); err != nil {
		return nil, err
	}

	return kernel, nil
}

// TODO: Add a logging structure to the configuration file and configure. Make
// sure this supports configuring syslog.
func configureLogger(id string, conf *Configuration) ([]Appender, error) {

	var appenders []Appender
	appenders = append(appenders, LevelFilter(Debug, StdErrAppender()))

	if conf.EnvironmentIs("prod") {
		syslogAppender, err := NewSyslogAppender("", "", id)
		if err != nil {
			return nil, err
		}

		appenders = append(appenders, LevelFilter(Debug, syslogAppender))
	}

	return appenders, nil
}

func writePidFile(kernel *Kernel) error {
	kernel.Pid = os.Getpid()
	pidFile, err := os.Create(kernel.Configuration.PidFile)
	if err != nil {
		return NewStackError("Unable to start kernel - problem creating pid file %s - error: %v", kernel.Configuration.PidFile, err)
	}
	defer pidFile.Close()

	if _, err := pidFile.Write([]byte(strconv.Itoa(kernel.Pid))); err != nil {
		return NewStackError("Unable to start kernel - problem writing pid file %s - error: %v", kernel.Configuration.PidFile, err)
	}

	return nil
}

// Call this from your main to create the kernel. After init kernel is called you must add
// your components and then call kernel.Start()
func StartKernel(id string, configFileName string, addComponentsFunction func(kernel *Kernel)) (*Kernel, error) {

	kernel, err := newKernel(id, configFileName)
	if err != nil {
		return nil, err
	}

	addComponentsFunction(kernel)

    if err = kernel.Start(); err != nil {
		return nil, err
	}

	return kernel, nil
}

// ListenForInterrupt blocks until an interrupt signal is detected.
func (self *Kernel) ListenForInterrupt() error {
	quitChannel := make(chan bool)

	// Register the interrupt listener.
	interruptSignalChannel := make(chan os.Signal, 1)
	signal.Notify(interruptSignalChannel, os.Interrupt)
	go func() {
		for sig := range interruptSignalChannel {
			if sig == syscall.SIGINT {
				quitChannel <- true
			}
		}
	}()

	// Block until we receive the stop notification.
	select {
		case <- quitChannel: {
			return self.Stop()
		}
	}

	// Should never happen
	panic("How did we end up here?")
}

<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"syscall"
)

func main() {

	initFlags()

	// check permissions
	if os.Getuid() != 0 && os.Geteuid() != 0 {
		fmt.Printf("npserver-daemon should be run as root, have uid=%d and euid=%d\n", os.Getuid(), os.Geteuid())
		os.Exit(1)
	}

	// check start || sto
	if flags.Start == flags.Stop {
		fmt.Println("need --start or --stop flag")
	}

	if flags.Start {
		startDaemon()
	}
	if flags.Stop {
		stopDaemon()
	}

	// all good :)
}

func startDaemon() {
	// setup args for daemon call
	args := []string{
		"--name=npserver",
		"--noconfig",
		"--errlog=/var/log/npserver-daemon.log",
		"--output=/var/log/npserver.log",
		fmt.Sprintf("--pidfile=%s", flags.PIDFile),
		"--unsafe",
		"--",
		"/usr/local/bin/npserver",
	}

	// append extra args to args
	args = append(args, extraArgs...)

	// start process
	proc, err := os.StartProcess("daemon", args, &os.ProcAttr{
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
		Sys: &syscall.SysProcAttr{
			Credential: &syscall.Credential{
				Uid: uint32(os.Geteuid()),
				Gid: uint32(os.Getegid()),
			},
		},
	})
	if err != nil {
		fmt.Printf("os/exec returned an error: '%s'\n", err)
		os.Exit(1)
	}

	// wait for daemon to be ready
	_, err = proc.Wait()
	if err != nil {
		fmt.Printf("proc.Wait() failed. %s\n", err)
		os.Exit(1)
	}
}

func stopDaemon() {
	pidFile, err := os.Open(flags.PIDFile)
	if err != nil {
		fmt.Printf("error on opening pidfile: %s", err)
		os.Exit(1)
	}

	pidFileContents, err := ioutil.ReadAll(pidFile)
	pidFile.Close()
	if err != nil {
		fmt.Printf("error reading pidfile contents: %s\n", err)
		os.Exit(1)
	}

	// convert pid string to pid int
	pid, err := strconv.Atoi(string(pidFileContents))
	if err != nil {
		fmt.Printf("error parsing pidfile contents: %s\n", err)
		os.Exit(1)
	}

	// lookup process
	proc, err := os.FindProcess(pid)
	if err != nil {
		fmt.Printf("error finding process with pid %d: %s\n", pid, err)
		os.Exit(1)
	}

	// signal process to stop
	err = proc.Signal(os.Interrupt)
	if err != nil {
		fmt.Printf("error sending interrupt signal to npserver: %s\n", err)
		os.Exit(1)
	}

	// wait until process is done
	state, err := proc.Wait()
	if err != nil {
		fmt.Printf("error waiting for process to stop: %s\n", err)
		os.Exit(1)
	}
	if !state.Exited() || !state.Success() {
		fmt.Printf("npserver process exited badly")
		os.Exit(1)
	}
}
<commit_msg>Don't exit with error when pid file is not found.<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"syscall"
)

func main() {

	initFlags()

	// check permissions
	if os.Getuid() != 0 && os.Geteuid() != 0 {
		fmt.Printf("npserver-daemon should be run as root, have uid=%d and euid=%d\n", os.Getuid(), os.Geteuid())
		os.Exit(1)
	}

	// check start || sto
	if flags.Start == flags.Stop {
		fmt.Println("need --start or --stop flag")
	}

	if flags.Start {
		startDaemon()
	}
	if flags.Stop {
		stopDaemon()
	}

	// all good :)
}

func startDaemon() {
	// setup args for daemon call
	args := []string{
		"--name=npserver",
		"--noconfig",
		"--errlog=/var/log/npserver-daemon.log",
		"--output=/var/log/npserver.log",
		fmt.Sprintf("--pidfile=%s", flags.PIDFile),
		"--unsafe",
		"--",
		"/usr/local/bin/npserver",
	}

	// append extra args to args
	args = append(args, extraArgs...)

	// start process
	proc, err := os.StartProcess("daemon", args, &os.ProcAttr{
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
		Sys: &syscall.SysProcAttr{
			Credential: &syscall.Credential{
				Uid: uint32(os.Geteuid()),
				Gid: uint32(os.Getegid()),
			},
		},
	})
	if err != nil {
		fmt.Printf("os/exec returned an error: '%s'\n", err)
		os.Exit(1)
	}

	// wait for daemon to be ready
	_, err = proc.Wait()
	if err != nil {
		fmt.Printf("proc.Wait() failed. %s\n", err)
		os.Exit(1)
	}
}

func stopDaemon() {
	pidFile, err := os.Open(flags.PIDFile)
	if err != nil {
		if err == os.ErrNotExist {
			fmt.Printf("it looks like npserver is not running")
			os.Exit(0)
		}
		fmt.Printf("error on opening pidfile: %s", err)
		os.Exit(1)
	}

	pidFileContents, err := ioutil.ReadAll(pidFile)
	pidFile.Close()
	if err != nil {
		fmt.Printf("error reading pidfile contents: %s\n", err)
		os.Exit(1)
	}

	// convert pid string to pid int
	pid, err := strconv.Atoi(string(pidFileContents))
	if err != nil {
		fmt.Printf("error parsing pidfile contents: %s\n", err)
		os.Exit(1)
	}

	// lookup process
	proc, err := os.FindProcess(pid)
	if err != nil {
		fmt.Printf("error finding process with pid %d: %s\n", pid, err)
		os.Exit(1)
	}

	// signal process to stop
	err = proc.Signal(os.Interrupt)
	if err != nil {
		fmt.Printf("error sending interrupt signal to npserver: %s\n", err)
		os.Exit(1)
	}

	// wait until process is done
	state, err := proc.Wait()
	if err != nil {
		fmt.Printf("error waiting for process to stop: %s\n", err)
		os.Exit(1)
	}
	if !state.Exited() || !state.Success() {
		fmt.Printf("npserver process exited badly")
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"

	"github.com/panjf2000/gnet"
)

var res string

type request struct {
	proto, method string
	path, query   string
	head, body    string
	remoteAddr    string
}

type httpServer struct {
	*gnet.EventServer
}

var errMsg = "Internal Server Error"
var errMsgBytes = []byte(errMsg)

type httpCodec struct {
	req request
}

func (hc *httpCodec) Encode(c gnet.Conn, buf []byte) (out []byte, err error) {
	if c.Context() == nil {
		return buf, nil
	}
	return appendResp(out, "500 Error", "", errMsg+"\n"), nil
}

func (hc *httpCodec) Decode(c gnet.Conn) (out []byte, err error) {
	buf := c.Read()
	c.ResetBuffer()

	// process the pipeline
	var leftover []byte
pipeline:
	leftover, err = parseReq(buf, &hc.req)
	// bad thing happened
	if err != nil {
		c.SetContext(err)
		return nil, err
	} else if len(leftover) == len(buf) {
		// request not ready, yet
		return
	}
	out = appendHandle(out, res)
	buf = leftover
	goto pipeline
}

func (hs *httpServer) OnInitComplete(srv gnet.Server) (action gnet.Action) {
	log.Printf("HTTP server is listening on %s (multi-cores: %t, loops: %d)\n",
		srv.Addr.String(), srv.Multicore, srv.NumLoops)
	return
}

func (hs *httpServer) React(frame []byte, c gnet.Conn) (out []byte, action gnet.Action) {
	if c.Context() != nil {
		// bad thing happened
		out = errMsgBytes
		action = gnet.Close
		return
	}
	// handle the request
	out = frame
	return
}

func main() {
	var port int
	var multicore bool

	// Example command: go run main.go --port 8080 --multicore true
	flag.IntVar(&port, "port", 8080, "server port")
	flag.BoolVar(&multicore, "multicore", true, "multicore")
	flag.Parse()

	res = "Hello, World!"

	http := new(httpServer)
	hc := new(httpCodec)

	// Start serving!
	log.Fatal(gnet.Serve(http, fmt.Sprintf("tcp://:%d", port), gnet.WithMulticore(multicore), gnet.WithCodec(hc)))
}

// appendHandle handles the incoming request and appends the response to
// the provided bytes, which is then returned to the caller.
func appendHandle(b []byte, res string) []byte {
	return appendResp(b, "200 OK", "", res)
}

// appendResp will append a valid http response to the provide bytes.
// The status param should be the code plus text such as "200 OK".
// The head parameter should be a series of lines ending with "\r\n" or empty.
func appendResp(b []byte, status, head, body string) []byte {
	b = append(b, "HTTP/1.1"...)
	b = append(b, ' ')
	b = append(b, status...)
	b = append(b, '\r', '\n')
	b = append(b, "Server: gnet\r\n"...)
	b = append(b, "Content-Type: text/plain\r\n"...)
	b = append(b, "Date: "...)
	b = time.Now().AppendFormat(b, "Mon, 02 Jan 2006 15:04:05 GMT")
	b = append(b, '\r', '\n')
	if len(body) > 0 {
		b = append(b, "Content-Length: "...)
		b = strconv.AppendInt(b, int64(len(body)), 10)
		b = append(b, '\r', '\n')
	}
	b = append(b, head...)
	b = append(b, '\r', '\n')
	if len(body) > 0 {
		b = append(b, body...)
	}
	return b
}

// parseReq is a very simple http request parser. This operation
// waits for the entire payload to be buffered before returning a
// valid request.
func parseReq(data []byte, req *request) (leftover []byte, err error) {
	sdata := string(data)
	var i, s int
	var head string
	var clen int
	var q = -1
	// method, path, proto line
	for ; i < len(sdata); i++ {
		if sdata[i] == ' ' {
			req.method = sdata[s:i]
			for i, s = i+1, i+1; i < len(sdata); i++ {
				if sdata[i] == '?' && q == -1 {
					q = i - s
				} else if sdata[i] == ' ' {
					if q != -1 {
						req.path = sdata[s:q]
						req.query = req.path[q+1 : i]
					} else {
						req.path = sdata[s:i]
					}
					for i, s = i+1, i+1; i < len(sdata); i++ {
						if sdata[i] == '\n' && sdata[i-1] == '\r' {
							req.proto = sdata[s:i]
							i, s = i+1, i+1
							break
						}
					}
					break
				}
			}
			break
		}
	}
	if req.proto == "" {
		return data, fmt.Errorf("malformed request")
	}
	head = sdata[:s]
	for ; i < len(sdata); i++ {
		if i > 1 && sdata[i] == '\n' && sdata[i-1] == '\r' {
			line := sdata[s : i-1]
			s = i + 1
			if line == "" {
				req.head = sdata[len(head)+2 : i+1]
				i++
				if clen > 0 {
					if len(sdata[i:]) < clen {
						break
					}
					req.body = sdata[i : i+clen]
					i += clen
				}
				return data[i:], nil
			}
			if strings.HasPrefix(line, "Content-Length:") {
				n, err := strconv.ParseInt(strings.TrimSpace(line[len("Content-Length:"):]), 10, 64)
				if err == nil {
					clen = int(n)
				}
			}
		}
	}
	// not enough data
	return data, nil
}
<commit_msg>Optimize http server of gnet (#5522)<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/panjf2000/gnet"
)

var res string

type request struct {
	proto, method string
	path, query   string
	head, body    string
	remoteAddr    string
}

type httpServer struct {
	*gnet.EventServer
}

var errMsg = "Internal Server Error"
var errMsgBytes = []byte(errMsg)

type httpCodec struct {
	req request
}

func (hc *httpCodec) Encode(c gnet.Conn, buf []byte) (out []byte, err error) {
	if c.Context() == nil {
		return buf, nil
	}
	return appendResp(out, "500 Error", "", errMsg+"\n"), nil
}

func (hc *httpCodec) Decode(c gnet.Conn) (out []byte, err error) {
	buf := c.Read()
	c.ResetBuffer()

	// process the pipeline
	var leftover []byte
pipeline:
	leftover, err = parseReq(buf, &hc.req)
	// bad thing happened
	if err != nil {
		c.SetContext(err)
		return nil, err
	} else if len(leftover) == len(buf) {
		// request not ready, yet
		return
	}
	out = appendHandle(out, res)
	buf = leftover
	goto pipeline
}

func (hs *httpServer) OnInitComplete(srv gnet.Server) (action gnet.Action) {
	log.Printf("HTTP server is listening on %s (multi-cores: %t, loops: %d)\n",
		srv.Addr.String(), srv.Multicore, srv.NumLoops)
	return
}

func (hs *httpServer) React(frame []byte, c gnet.Conn) (out []byte, action gnet.Action) {
	if c.Context() != nil {
		// bad thing happened
		out = errMsgBytes
		action = gnet.Close
		return
	}
	// handle the request
	out = frame
	return
}

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU() * 2)
}

func main() {
	var port int
	var multicore bool

	// Example command: go run main.go --port 8080 --multicore true
	flag.IntVar(&port, "port", 8080, "server port")
	flag.BoolVar(&multicore, "multicore", true, "multicore")
	flag.Parse()

	res = "Hello, World!"

	http := new(httpServer)
	hc := new(httpCodec)

	// Start serving!
	log.Fatal(gnet.Serve(http, fmt.Sprintf("tcp://:%d", port), gnet.WithMulticore(multicore), gnet.WithCodec(hc)))
}

// appendHandle handles the incoming request and appends the response to
// the provided bytes, which is then returned to the caller.
func appendHandle(b []byte, res string) []byte {
	return appendResp(b, "200 OK", "", res)
}

// appendResp will append a valid http response to the provide bytes.
// The status param should be the code plus text such as "200 OK".
// The head parameter should be a series of lines ending with "\r\n" or empty.
func appendResp(b []byte, status, head, body string) []byte {
	b = append(b, "HTTP/1.1"...)
	b = append(b, ' ')
	b = append(b, status...)
	b = append(b, '\r', '\n')
	b = append(b, "Server: gnet\r\n"...)
	b = append(b, "Content-Type: text/plain\r\n"...)
	b = append(b, "Date: "...)
	b = time.Now().AppendFormat(b, "Mon, 02 Jan 2006 15:04:05 GMT")
	b = append(b, '\r', '\n')
	if len(body) > 0 {
		b = append(b, "Content-Length: "...)
		b = strconv.AppendInt(b, int64(len(body)), 10)
		b = append(b, '\r', '\n')
	}
	b = append(b, head...)
	b = append(b, '\r', '\n')
	if len(body) > 0 {
		b = append(b, body...)
	}
	return b
}

// parseReq is a very simple http request parser. This operation
// waits for the entire payload to be buffered before returning a
// valid request.
func parseReq(data []byte, req *request) (leftover []byte, err error) {
	sdata := string(data)
	var i, s int
	var head string
	var clen int
	var q = -1
	// method, path, proto line
	for ; i < len(sdata); i++ {
		if sdata[i] == ' ' {
			req.method = sdata[s:i]
			for i, s = i+1, i+1; i < len(sdata); i++ {
				if sdata[i] == '?' && q == -1 {
					q = i - s
				} else if sdata[i] == ' ' {
					if q != -1 {
						req.path = sdata[s:q]
						req.query = req.path[q+1 : i]
					} else {
						req.path = sdata[s:i]
					}
					for i, s = i+1, i+1; i < len(sdata); i++ {
						if sdata[i] == '\n' && sdata[i-1] == '\r' {
							req.proto = sdata[s:i]
							i, s = i+1, i+1
							break
						}
					}
					break
				}
			}
			break
		}
	}
	if req.proto == "" {
		return data, fmt.Errorf("malformed request")
	}
	head = sdata[:s]
	for ; i < len(sdata); i++ {
		if i > 1 && sdata[i] == '\n' && sdata[i-1] == '\r' {
			line := sdata[s : i-1]
			s = i + 1
			if line == "" {
				req.head = sdata[len(head)+2 : i+1]
				i++
				if clen > 0 {
					if len(sdata[i:]) < clen {
						break
					}
					req.body = sdata[i : i+clen]
					i += clen
				}
				return data[i:], nil
			}
			if strings.HasPrefix(line, "Content-Length:") {
				n, err := strconv.ParseInt(strings.TrimSpace(line[len("Content-Length:"):]), 10, 64)
				if err == nil {
					clen = int(n)
				}
			}
		}
	}
	// not enough data
	return data, nil
}
<|endoftext|>


<commit_before>// Package ql implements support for parsing and running logsprays
// simple query expressions.
//
// Query expressions set of label and value matches. The ql does not currently
// support matching of the message text itself, this will be corrected in future
// Either label or value can be given as a quoted string using ",', or `
// quotes. Four match types are supported:
//
//   = : an exact match, or the single wild card "*" for any value
//   != : any value not equal to the value
//   ~ : A regular expression match, against the label value
//   !~ : A negated regular expression match against the label value
//
// Matches for the same label are or'd together. Matches for different labels
// are and'd together.
package ql
<commit_msg>ql: Add some example log queries<commit_after>// Package ql implements support for parsing and running logsprays
// simple query expressions.
//
// Query expressions set of label and value matches. The ql does not currently
// support matching of the message text itself, this will be corrected in future
// Either label or value can be given as a quoted string using ",', or `
// quotes. Four match types are supported:
//
//   = : an exact match, or the single wild card "*" for any value
//   != : any value not equal to the value
//   ~ : A regular expression match, against the label value
//   !~ : A negated regular expression match against the label value
//
// Matches for the same label are or'd together. Matches for different labels
// are and'd together.
//
// For example:
//
//   job=somejob : a single match for a single job
//   "job"=somejob job="otherjob with long name" : match two specific jobsl
//   job=somejob instance=myserver : logs for job on a specific instance
//   job~things-.* : all jobs matching things
//   job=billing customer=* : all illing logs with any customer label set
//   job=billing customer~acme-.* : all illing logs with any acme- customer label set
package ql
<|endoftext|>