<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package graph

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
)

// Graph ...
type Graph struct {
	Components []Component `json:"components"`
	Changes    []Component `json:"changes,omitempty"`
	Edges      []Edge      `json:"edges,omitempty"`
}

// New returns a new graph
func New() *Graph {
	return &Graph{
		Components: make([]Component, 0),
		Edges:      make([]Edge, 0),
	}
}

// Component returns a component given the name matches
func (g *Graph) Component(component string) Component {
	for i, v := range g.Components {
		if v.GetID() == component {
			return g.Components[i]
		}
	}
	return nil
}

// HasComponent finds if the specified component exists
func (g *Graph) HasComponent(componentID string) bool {
	for _, v := range g.Components {
		if v.GetID() == componentID {
			return true
		}
	}
	return false
}

// AddComponent adds a component to the graphs vertices if it does not already exist
func (g *Graph) AddComponent(component Component) error {
	if g.HasComponent(component.GetID()) {
		return errors.New("Component already exists: " + component.GetID())
	}
	g.Components = append(g.Components, component)

	return nil
}

// UpdateComponent updates the graph
func (g *Graph) UpdateComponent(component Component) {
	for i := 0; i < len(g.Components); i++ {
		if g.Components[i].GetID() == component.GetID() {
			g.Components[i] = component
			return
		}
	}
}

// DisconnectComponent removes a component from the graph. It will connect any neighbour/origin components together
func (g *Graph) DisconnectComponent(name string) error {
	origins := g.Origins(name)

	for i := len(g.Edges) - 1; i >= 0; i-- {
		// Remove any edges that connect to the disconnected component
		if g.Edges[i].Destination == name {
			g.Edges = append(g.Edges[:i], g.Edges[i+1:]...)
		}

		// Remove any neighbouring connections and reconnect them to origins
		if g.Edges[i].Source == name {
			for _, ov := range *origins {
				err := g.Connect(ov.GetID(), g.Edges[i].Destination)
				if err != nil {
					return err
				}
			}
			g.Edges = append(g.Edges[:i], g.Edges[i+1:]...)
		}
	}

	return nil
}

// connect is the internal method for connecting two verticies, it provides less checks than publicly exposed methods
func (g *Graph) connect(source, destination string) {
	if g.Connected(source, destination) != true {
		g.Edges = append(g.Edges, Edge{Source: source, Destination: destination, Length: 1})
	}
}

// Connect adds a dependency between two vertices
func (g *Graph) Connect(source, destination string) error {
	if !g.HasComponent(source) || !g.HasComponent(destination) {
		return errors.New("Could not connect Component, does not exist")
	}

	g.connect(source, destination)

	return nil
}

// ConnectMutually connects two vertices to eachother
func (g *Graph) ConnectMutually(source, destination string) error {
	err := g.Connect(source, destination)
	if err != nil {
		return err
	}
	return g.Connect(destination, source)
}

// ConnectSequential adds a dependency between two vertices. If the source has more than 1 neighbouring vertex, the destination vertex will be connected to that.
func (g *Graph) ConnectSequential(source, destination string) error {
	if !g.HasComponent(source) {
		source = "start"
	}

	if !g.HasComponent(destination) {
		return errors.New("Could not connect Component, does not exist")
	}

	c := g.Component(destination)
	gc := g.Neighbours(source).GetComponentGroup(c.GetGroup())

	for gc != nil {
		source = gc.GetID()
		gc = g.Neighbours(source).GetComponentGroup(c.GetGroup())
	}

	g.connect(source, destination)

	return nil
}

// Connected returns true if two components are connected
func (g *Graph) Connected(source, destination string) bool {
	for _, edge := range g.Edges {
		if edge.Source == source && edge.Destination == destination {
			return true
		}
	}

	return false
}

// ComponentByProviderID : returns a single component by matching provider id
func (g *Graph) ComponentByProviderID(id string) Component {
	for _, component := range g.Components {
		if component.GetProviderID() == id {
			return component
		}
	}

	return nil
}

// Neighbours returns all depencencies of a component
func (g *Graph) Neighbours(component string) *Neighbours {
	var n Neighbours

	for _, edge := range g.Edges {
		if edge.Source == component {
			n = append(n, g.Component(edge.Destination))
		}
	}

	return n.Unique()
}

// Origins returns all source components of a component
func (g *Graph) Origins(component string) *Neighbours {
	var n Neighbours

	for _, edge := range g.Edges {
		if edge.Destination == component {
			n = append(n, g.Component(edge.Source))
		}
	}

	return n.Unique()
}

// LengthBetween returns the length between two edges
func (g *Graph) LengthBetween(source, destination string) int {
	for _, e := range g.Edges {
		if e.Source == source && e.Destination == destination {
			return e.Length
		}
	}
	return -1
}

// Diff : diff two graphs, new, modified or removed components will be moved to Changes, and components will be
func (g *Graph) Diff(og *Graph) (*Graph, error) {
	// new temporary graph
	ng := New()

	for _, c := range g.Components {
		oc := og.Component(c.GetID())
		if oc != nil {
			if c.Diff(oc) {
				c.SetAction("update")
				c.SetState("waiting")
				ng.AddComponent(c)
			}
		} else {
			c.SetAction("create")
			c.SetState("waiting")
			ng.AddComponent(c)
		}
	}

	for _, oc := range og.Components {
		c := g.Component(oc.GetID())
		if c == nil {
			oc.SetAction("delete")
			oc.SetState("waiting")
			ng.AddComponent(oc)
		}
	}

	ng.SetDiffDependencies()

	// Set changes + edges on original graph
	og.Changes = ng.Components
	og.Edges = ng.Edges

	return og, nil
}

// Graphviz outputs the graph in graphviz format
func (g *Graph) Graphviz() string {
	var output []string

	output = append(output, "digraph G {")

	for _, edge := range g.Edges {
		output = append(output, fmt.Sprintf("  \"%s\" -> \"%s\"", edge.Source, edge.Destination))
	}

	output = append(output, "}")

	return strings.Join(output, "\n")
}

// SetDiffDependencies rebuilds the graph's dependencies when diffing
func (g *Graph) SetDiffDependencies() {
	// This needs improvement

	g.Edges = make([]Edge, 0)

	for _, c := range g.Components {
		for _, dep := range c.Dependencies() {
			switch c.GetAction() {
			case "delete":
				if c.IsStateful() {
					g.Connect(c.GetID(), dep)
				}
			case "update":
				g.ConnectSequential(dep, c.GetID())
			case "create", "find":
				g.Connect(dep, c.GetID())
			}
		}
	}

	g.SetStartFinish()
}

// SetStartFinish sets a start and finish point
func (g *Graph) SetStartFinish() {
	for _, c := range g.Components {
		o := g.Origins(c.GetID())
		n := g.Neighbours(c.GetID())

		if len(*o) < 1 {
			g.connect("start", c.GetID())
		}

		if len(*n) < 1 {
			g.connect(c.GetID(), "end")
		}
	}
}

// ToJSON serialises the graph as json
func (g *Graph) ToJSON() ([]byte, error) {
	return json.Marshal(g)
}

// func (g *Graph) DepthFirstSearch()

// func (g *Graph) CycleDetection()
<commit_msg>updated graphviz output to include labels for changes<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package graph

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
)

// Graph ...
type Graph struct {
	Components []Component `json:"components"`
	Changes    []Component `json:"changes,omitempty"`
	Edges      []Edge      `json:"edges,omitempty"`
}

// New returns a new graph
func New() *Graph {
	return &Graph{
		Components: make([]Component, 0),
		Edges:      make([]Edge, 0),
	}
}

// Component returns a component given the name matches
func (g *Graph) Component(component string) Component {
	for i, v := range g.Components {
		if v.GetID() == component {
			return g.Components[i]
		}
	}
	return nil
}

// ComponentAll returns a component from either changes or components given the name matches
func (g *Graph) ComponentAll(component string) Component {
	for i, v := range g.Changes {
		if v.GetID() == component {
			return g.Changes[i]
		}
	}
	for i, v := range g.Components {
		if v.GetID() == component {
			return g.Components[i]
		}
	}
	return nil
}

// HasComponent finds if the specified component exists
func (g *Graph) HasComponent(componentID string) bool {
	for _, v := range g.Components {
		if v.GetID() == componentID {
			return true
		}
	}
	return false
}

// AddComponent adds a component to the graphs vertices if it does not already exist
func (g *Graph) AddComponent(component Component) error {
	if g.HasComponent(component.GetID()) {
		return errors.New("Component already exists: " + component.GetID())
	}
	g.Components = append(g.Components, component)

	return nil
}

// UpdateComponent updates the graph
func (g *Graph) UpdateComponent(component Component) {
	for i := 0; i < len(g.Components); i++ {
		if g.Components[i].GetID() == component.GetID() {
			g.Components[i] = component
			return
		}
	}
}

// DisconnectComponent removes a component from the graph. It will connect any neighbour/origin components together
func (g *Graph) DisconnectComponent(name string) error {
	origins := g.Origins(name)

	for i := len(g.Edges) - 1; i >= 0; i-- {
		// Remove any edges that connect to the disconnected component
		if g.Edges[i].Destination == name {
			g.Edges = append(g.Edges[:i], g.Edges[i+1:]...)
		}

		// Remove any neighbouring connections and reconnect them to origins
		if g.Edges[i].Source == name {
			for _, ov := range *origins {
				err := g.Connect(ov.GetID(), g.Edges[i].Destination)
				if err != nil {
					return err
				}
			}
			g.Edges = append(g.Edges[:i], g.Edges[i+1:]...)
		}
	}

	return nil
}

// connect is the internal method for connecting two verticies, it provides less checks than publicly exposed methods
func (g *Graph) connect(source, destination string) {
	if g.Connected(source, destination) != true {
		g.Edges = append(g.Edges, Edge{Source: source, Destination: destination, Length: 1})
	}
}

// Connect adds a dependency between two vertices
func (g *Graph) Connect(source, destination string) error {
	if !g.HasComponent(source) || !g.HasComponent(destination) {
		return errors.New("Could not connect Component, does not exist")
	}

	g.connect(source, destination)

	return nil
}

// ConnectMutually connects two vertices to eachother
func (g *Graph) ConnectMutually(source, destination string) error {
	err := g.Connect(source, destination)
	if err != nil {
		return err
	}
	return g.Connect(destination, source)
}

// ConnectSequential adds a dependency between two vertices. If the source has more than 1 neighbouring vertex, the destination vertex will be connected to that.
func (g *Graph) ConnectSequential(source, destination string) error {
	if !g.HasComponent(source) {
		source = "start"
	}

	if !g.HasComponent(destination) {
		return errors.New("Could not connect Component, does not exist")
	}

	c := g.Component(destination)
	gc := g.Neighbours(source).GetComponentGroup(c.GetGroup())

	for gc != nil {
		source = gc.GetID()
		gc = g.Neighbours(source).GetComponentGroup(c.GetGroup())
	}

	g.connect(source, destination)

	return nil
}

// Connected returns true if two components are connected
func (g *Graph) Connected(source, destination string) bool {
	for _, edge := range g.Edges {
		if edge.Source == source && edge.Destination == destination {
			return true
		}
	}

	return false
}

// ComponentByProviderID : returns a single component by matching provider id
func (g *Graph) ComponentByProviderID(id string) Component {
	for _, component := range g.Components {
		if component.GetProviderID() == id {
			return component
		}
	}

	return nil
}

// Neighbours returns all depencencies of a component
func (g *Graph) Neighbours(component string) *Neighbours {
	var n Neighbours

	for _, edge := range g.Edges {
		if edge.Source == component {
			n = append(n, g.Component(edge.Destination))
		}
	}

	return n.Unique()
}

// Origins returns all source components of a component
func (g *Graph) Origins(component string) *Neighbours {
	var n Neighbours

	for _, edge := range g.Edges {
		if edge.Destination == component {
			n = append(n, g.Component(edge.Source))
		}
	}

	return n.Unique()
}

// LengthBetween returns the length between two edges
func (g *Graph) LengthBetween(source, destination string) int {
	for _, e := range g.Edges {
		if e.Source == source && e.Destination == destination {
			return e.Length
		}
	}
	return -1
}

// Diff : diff two graphs, new, modified or removed components will be moved to Changes, and components will be
func (g *Graph) Diff(og *Graph) (*Graph, error) {
	// new temporary graph
	ng := New()

	for _, c := range g.Components {
		oc := og.Component(c.GetID())
		if oc != nil {
			if c.Diff(oc) {
				c.SetAction("update")
				c.SetState("waiting")
				ng.AddComponent(c)
			}
		} else {
			c.SetAction("create")
			c.SetState("waiting")
			ng.AddComponent(c)
		}
	}

	for _, oc := range og.Components {
		c := g.Component(oc.GetID())
		if c == nil {
			oc.SetAction("delete")
			oc.SetState("waiting")
			ng.AddComponent(oc)
		}
	}

	ng.SetDiffDependencies()

	// Set changes + edges on original graph
	og.Changes = ng.Components
	og.Edges = ng.Edges

	return og, nil
}

// Graphviz outputs the graph in graphviz format
func (g *Graph) Graphviz() string {
	var output []string

	output = append(output, "digraph G {")

	for _, edge := range g.Edges {
		dest := g.ComponentAll(edge.Destination)
		if dest != nil {
			output = append(output, fmt.Sprintf("  \"%s\" -> \"%s\" [label=\"%s\"]", edge.Source, edge.Destination, dest.GetAction()))
		} else {
			output = append(output, fmt.Sprintf("  \"%s\" -> \"%s\"", edge.Source, edge.Destination))
		}
	}

	output = append(output, "}")

	return strings.Join(output, "\n")
}

// SetDiffDependencies rebuilds the graph's dependencies when diffing
func (g *Graph) SetDiffDependencies() {
	// This needs improvement

	g.Edges = make([]Edge, 0)

	for _, c := range g.Components {
		for _, dep := range c.Dependencies() {
			switch c.GetAction() {
			case "delete":
				if c.IsStateful() {
					g.Connect(c.GetID(), dep)
				}
			case "update":
				g.ConnectSequential(dep, c.GetID())
			case "create", "find":
				g.Connect(dep, c.GetID())
			}
		}
	}

	g.SetStartFinish()
}

// SetStartFinish sets a start and finish point
func (g *Graph) SetStartFinish() {
	for _, c := range g.Components {
		o := g.Origins(c.GetID())
		n := g.Neighbours(c.GetID())

		if len(*o) < 1 {
			g.connect("start", c.GetID())
		}

		if len(*n) < 1 {
			g.connect(c.GetID(), "end")
		}
	}
}

// ToJSON serialises the graph as json
func (g *Graph) ToJSON() ([]byte, error) {
	return json.Marshal(g)
}

// func (g *Graph) DepthFirstSearch()

// func (g *Graph) CycleDetection()
<|endoftext|>


<commit_before>package arn

import (
	"errors"

	"github.com/aerogo/aero"
	"github.com/aerogo/api"
)

// Force interface implementations
var (
	_ Publishable   = (*Company)(nil)
	_ api.Newable   = (*Company)(nil)
	_ api.Editable  = (*Company)(nil)
	_ api.Deletable = (*Company)(nil)
)

// Actions
func init() {
	API.RegisterActions("Company", []*api.Action{
		// Publish
		PublishAction(),

		// Unpublish
		UnpublishAction(),

		// Like
		LikeAction(),

		// Unlike
		UnlikeAction(),
	})
}

// Create sets the data for a new company with data we received from the API request.
func (company *Company) Create(ctx *aero.Context) error {
	user := GetUserFromContext(ctx)

	if user == nil {
		return errors.New("Not logged in")
	}

	company.ID = GenerateID("Company")
	company.Created = DateTimeUTC()
	company.CreatedBy = user.ID
	company.Mappings = []*Mapping{}
	company.Links = []*Link{}
	company.Tags = []string{}
	company.Likes = []string{}

	return company.Unpublish()
}

// AfterEdit updates the metadata.
func (company *Company) AfterEdit(ctx *aero.Context) error {
	company.Edited = DateTimeUTC()
	company.EditedBy = GetUserFromContext(ctx).ID
	return nil
}

// Save saves the company in the database.
func (company *Company) Save() {
	DB.Set("Company", company.ID, company)
}

// Delete deletes the object from the database.
func (company *Company) Delete() error {
	if company.IsDraft {
		draftIndex := company.Creator().DraftIndex()
		draftIndex.CompanyID = ""
		draftIndex.Save()
	}

	// Remove company ID from all anime
	for anime := range StreamAnime() {
		for index, id := range anime.StudioIDs {
			if id == company.ID {
				anime.StudioIDs = append(anime.StudioIDs[:index], anime.StudioIDs[index+1:]...)
				break
			}
		}

		for index, id := range anime.ProducerIDs {
			if id == company.ID {
				anime.ProducerIDs = append(anime.ProducerIDs[:index], anime.ProducerIDs[index+1:]...)
				break
			}
		}

		for index, id := range anime.LicensorIDs {
			if id == company.ID {
				anime.LicensorIDs = append(anime.LicensorIDs[:index], anime.LicensorIDs[index+1:]...)
				break
			}
		}
	}

	DB.Delete("Company", company.ID)
	return nil
}

// Authorize returns an error if the given API request is not authorized.
func (company *Company) Authorize(ctx *aero.Context, action string) error {
	user := GetUserFromContext(ctx)

	if user == nil {
		return errors.New("Not logged in")
	}

	if action == "delete" {
		if user.Role != "editor" && user.Role != "admin" {
			return errors.New("Insufficient permissions")
		}
	}

	return nil
}
<commit_msg>Only editors can edit companies (for now)<commit_after>package arn

import (
	"errors"

	"github.com/aerogo/aero"
	"github.com/aerogo/api"
)

// Force interface implementations
var (
	_ Publishable   = (*Company)(nil)
	_ api.Newable   = (*Company)(nil)
	_ api.Editable  = (*Company)(nil)
	_ api.Deletable = (*Company)(nil)
)

// Actions
func init() {
	API.RegisterActions("Company", []*api.Action{
		// Publish
		PublishAction(),

		// Unpublish
		UnpublishAction(),

		// Like
		LikeAction(),

		// Unlike
		UnlikeAction(),
	})
}

// Create sets the data for a new company with data we received from the API request.
func (company *Company) Create(ctx *aero.Context) error {
	user := GetUserFromContext(ctx)

	if user == nil {
		return errors.New("Not logged in")
	}

	company.ID = GenerateID("Company")
	company.Created = DateTimeUTC()
	company.CreatedBy = user.ID
	company.Mappings = []*Mapping{}
	company.Links = []*Link{}
	company.Tags = []string{}
	company.Likes = []string{}

	return company.Unpublish()
}

// AfterEdit updates the metadata.
func (company *Company) AfterEdit(ctx *aero.Context) error {
	company.Edited = DateTimeUTC()
	company.EditedBy = GetUserFromContext(ctx).ID
	return nil
}

// Save saves the company in the database.
func (company *Company) Save() {
	DB.Set("Company", company.ID, company)
}

// Delete deletes the object from the database.
func (company *Company) Delete() error {
	if company.IsDraft {
		draftIndex := company.Creator().DraftIndex()
		draftIndex.CompanyID = ""
		draftIndex.Save()
	}

	// Remove company ID from all anime
	for anime := range StreamAnime() {
		for index, id := range anime.StudioIDs {
			if id == company.ID {
				anime.StudioIDs = append(anime.StudioIDs[:index], anime.StudioIDs[index+1:]...)
				break
			}
		}

		for index, id := range anime.ProducerIDs {
			if id == company.ID {
				anime.ProducerIDs = append(anime.ProducerIDs[:index], anime.ProducerIDs[index+1:]...)
				break
			}
		}

		for index, id := range anime.LicensorIDs {
			if id == company.ID {
				anime.LicensorIDs = append(anime.LicensorIDs[:index], anime.LicensorIDs[index+1:]...)
				break
			}
		}
	}

	DB.Delete("Company", company.ID)
	return nil
}

// Authorize returns an error if the given API request is not authorized.
func (company *Company) Authorize(ctx *aero.Context, action string) error {
	user := GetUserFromContext(ctx)

	if user == nil {
		return errors.New("Not logged in")
	}

	if user.Role != "editor" && user.Role != "admin" {
		return errors.New("Insufficient permissions")
	}

	return nil
}
<|endoftext|>


<commit_before>package manager_test

import (
	"log"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/geolite2"
	"github.com/m-lab/annotation-service/geoloader"
	"github.com/m-lab/annotation-service/handler"
	"github.com/m-lab/annotation-service/manager"
)

func init() {
	// Always prepend the filename and line number.
	log.SetFlags(log.LstdFlags | log.Lshortfile | log.Lmicroseconds)
}

func fakeLoader(date string) (api.Annotator, error) {
	time.Sleep(10 * time.Millisecond)
	return &geolite2.GeoDataset{}, nil
}

func TestInitDataset(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping test that uses GCS")
	}
	// Make the dataset filters much more restrictive to prevent OOM and make test faster.
	geoloader.UseOnlyMarchForTest()
	// Load the small directory.
	manager.MustUpdateDirectory()

	tests := []struct {
		ip   string
		time string
		res  string
	}{
		// This request needs a legacy binary dataset
		{"1.4.128.0", "1199145600", `{"Geo":{"continent_code":"AS","country_code":"TH","country_code3":"THA","country_name":"Thailand","region":"40","city":"Bangkok","latitude":13.754,"longitude":100.501},"Network":null}`},
		// This request needs another legacy binary dataset
		// `{"Geo":{"continent_code":"AS","country_code":"TH","country_code3":"THA","country_name":"Thailand","region":"77","city":"Bung","postal_code":"37000","latitude":15.695,"longitude":104.648},"Network":null}`
		{"1.4.128.0", "1399145600", `{"Geo":{"continent_code":"AS","country_code":"TH","country_code3":"THA","country_name":"Thailand","region":"40","city":"Bangkok","latitude":13.754,"longitude":100.501},"Network":null}`},
		// This request needs a geolite2 dataset
		{"1.9.128.0", "1512086400", `{"Geo":{"continent_code":"AS","country_code":"MY","country_code3":"MYS","country_name":"Malaysia","region":"14","city":"Kuala Lumpur","postal_code":"50586","latitude":3.167,"longitude":101.7},"Network":null}`},
		// This request needs the latest dataset in the memory.
		{"1.22.128.0", "1544400000", `{"Geo":{"continent_code":"AS","country_code":"IN","country_name":"India","region":"HR","city":"Faridabad","latitude":28.4333,"longitude":77.3167},"Network":null}`},
		// This request used a loaded & removed legacy dataset.
		//{"1.4.128.0", "1199145600", `{"Geo":{"continent_code":"AS","country_code":"TH","country_code3":"THA","country_name":"Thailand","region":"40","city":"Bangkok","latitude":13.754,"longitude":100.501},"Network":null}`},
	}
	for n, test := range tests {
		w := httptest.NewRecorder()
		r := &http.Request{}
		r.URL, _ = url.Parse("/annotate?ip_addr=" + url.QueryEscape(test.ip) + "&since_epoch=" + url.QueryEscape(test.time))
		log.Println("Calling handler")
		handler.Annotate(w, r)
		if w.Result().StatusCode != http.StatusOK {
			t.Error("Failed annotation for", test.ip)
			continue
		}

		body := w.Body.String()
		log.Println(body)

		if string(body) != test.res {
			t.Errorf("%d:\nGot\n__%s__\nexpected\n__%s__\n", n, body, test.res)
		}
	}
}
<commit_msg>disable broken unittest<commit_after>package manager_test

import (
	"log"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/geolite2"
	"github.com/m-lab/annotation-service/geoloader"
	"github.com/m-lab/annotation-service/handler"
	"github.com/m-lab/annotation-service/manager"
)

func init() {
	// Always prepend the filename and line number.
	log.SetFlags(log.LstdFlags | log.Lshortfile | log.Lmicroseconds)
}

func fakeLoader(date string) (api.Annotator, error) {
	time.Sleep(10 * time.Millisecond)
	return &geolite2.GeoDataset{}, nil
}

func XTestInitDataset(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping test that uses GCS")
	}
	// Make the dataset filters much more restrictive to prevent OOM and make test faster.
	geoloader.UseOnlyMarchForTest()
	// Load the small directory.
	manager.MustUpdateDirectory()

	tests := []struct {
		ip   string
		time string
		res  string
	}{
		// This request needs a legacy binary dataset
		{"1.4.128.0", "1199145600", `{"Geo":{"continent_code":"AS","country_code":"TH","country_code3":"THA","country_name":"Thailand","region":"40","city":"Bangkok","latitude":13.754,"longitude":100.501},"Network":null}`},
		// This request needs another legacy binary dataset
		// `{"Geo":{"continent_code":"AS","country_code":"TH","country_code3":"THA","country_name":"Thailand","region":"77","city":"Bung","postal_code":"37000","latitude":15.695,"longitude":104.648},"Network":null}`
		{"1.4.128.0", "1399145600", `{"Geo":{"continent_code":"AS","country_code":"TH","country_code3":"THA","country_name":"Thailand","region":"40","city":"Bangkok","latitude":13.754,"longitude":100.501},"Network":null}`},
		// This request needs a geolite2 dataset
		{"1.9.128.0", "1512086400", `{"Geo":{"continent_code":"AS","country_code":"MY","country_code3":"MYS","country_name":"Malaysia","region":"14","city":"Kuala Lumpur","postal_code":"50586","latitude":3.167,"longitude":101.7},"Network":null}`},
		// This request needs the latest dataset in the memory.
		{"1.22.128.0", "1544400000", `{"Geo":{"continent_code":"AS","country_code":"IN","country_name":"India","region":"HR","city":"Faridabad","latitude":28.4333,"longitude":77.3167},"Network":null}`},
		// This request used a loaded & removed legacy dataset.
		//{"1.4.128.0", "1199145600", `{"Geo":{"continent_code":"AS","country_code":"TH","country_code3":"THA","country_name":"Thailand","region":"40","city":"Bangkok","latitude":13.754,"longitude":100.501},"Network":null}`},
	}
	for n, test := range tests {
		w := httptest.NewRecorder()
		r := &http.Request{}
		r.URL, _ = url.Parse("/annotate?ip_addr=" + url.QueryEscape(test.ip) + "&since_epoch=" + url.QueryEscape(test.time))
		log.Println("Calling handler")
		handler.Annotate(w, r)
		if w.Result().StatusCode != http.StatusOK {
			t.Error("Failed annotation for", test.ip)
			continue
		}

		body := w.Body.String()
		log.Println(body)

		if string(body) != test.res {
			t.Errorf("%d:\nGot\n__%s__\nexpected\n__%s__\n", n, body, test.res)
		}
	}
}
<|endoftext|>


<commit_before>package micro

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/netdata/go-orchestrator/module"

	"github.com/micro/go-micro/client"
	"github.com/micro/go-micro/registry"
	stats "github.com/micro/micro/debug/stats/proto"
)

// Config is the go-micro collector configuration
type Config struct{}

// New creates the micro module with default values
func New(c client.Client) *Micro {
	return &Micro{
		Config:   Config{},
		charts:   charts(),
		services: make(map[string][]*stats.Snapshot),
		metrics:  make(map[string]int64),
		client:   c,
	}
}

// Register Registers with the orchestrator
func (m *Micro) Register() {
	module.Register("go_micro_services", module.Creator{
		Defaults: module.Defaults{
			Disabled: false,
		},
		Create: func() module.Module { return m },
	})
}

// Micro is a netdata module that collects metrics from the go.micro.debug service
type Micro struct {
	// netdata
	module.Base
	Config  `yaml:",inline"`
	charts  Charts
	metrics map[string]int64

	// go-micro
	client client.Client

	// internal
	sync.RWMutex
	cached   []*registry.Service
	services map[string][]*stats.Snapshot
}

// Cleanup is a no-op, called by netdata's orchestrator before shutdown
func (m *Micro) Cleanup() {}

// Init ensures the client is working, then starts saving snapshots.
func (m *Micro) Init() bool {
	// do initial scrape
	if err := m.collect(context.Background()); err != nil {
		m.Logger.Error(err)
	}

	go func() {
		for {
			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
			if err := m.collect(ctx); err != nil {
				m.Logger.Error(err)
			}
			cancel()
			time.Sleep(time.Second)
		}
	}()

	return true
}

// Check can be called by netdata's orchestrator to ensure the collector is working
func (m *Micro) Check() bool {
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	err := m.collect(ctx)
	return err == nil
}

// Charts passes the module's charts to the orchestrator
func (m *Micro) Charts() *Charts {
	m.RLock()
	c := m.charts.Copy()
	m.RUnlock()
	return c
}

// Collect returns the current view of the metrics to the orchestrator
func (m *Micro) Collect() map[string]int64 {
	metrics := make(map[string]int64)
	m.RLock()
	for k, v := range m.metrics {
		metrics[k] = v
	}
	m.RUnlock()
	return metrics
}

func (m *Micro) updateCharts(snapshots []*stats.Snapshot) error {
	// Construct a new Micro.services map based on the list of incoming service snapshots.
	// The map is keyed on Name_Version and sorted by Node ID for consistent graphs
	newServices := make(map[string][]*stats.Snapshot)
	for _, snap := range snapshots {
		key := key(snap.Service)
		if srv, found := newServices[key]; found {
			newServices[key] = append(srv, snap)
		} else {
			newServices[key] = []*stats.Snapshot{snap}
		}
	}
	for _, services := range newServices {
		sort.Slice(services, func(i, j int) bool {
			return services[i].Service.Node.Id < services[j].Service.Node.Id
		})
	}

	// Create / remove chart dimensions based on the previous state.
	m.Lock()
	defer m.Unlock()
	processedServices := make(map[string]bool)
	for newKey, newServices := range newServices {
		if oldServices, found := m.services[newKey]; found {
			// existing service, make sure the dimensions match what we currently have
			oldDims, newDims := len(oldServices), len(m.services[newKey])
			if oldDims != newDims {
				for _, ch := range m.charts {
					for i, svc := range oldServices {
						switch ch.ID {
						case chartServiceGCRate, chartServiceRequests, chartServiceErrors:
							ch.MarkDimRemove(fmt.Sprintf("%s_%d_%s", key(svc.Service), i, ch.ID), true)
						default:
							ch.MarkDimRemove(fmt.Sprintf("%s_%d_%s", key(svc.Service), i, ch.ID), true)
						}
						ch.MarkNotCreated()
					}
				}

				// create as many dimensions as we have
				for _, ch := range m.charts {
					for i, svc := range newServices {
						prefix := strings.TrimPrefix(key(svc.Service), "go_micro_")
						switch ch.ID {
						case chartServiceGCRate, chartServiceRequests, chartServiceErrors:
							ch.AddDim(&module.Dim{
								Algo: module.Incremental,
								ID:   fmt.Sprintf("%s_%d_%s", key(svc.Service), i, ch.ID),
								Name: fmt.Sprintf("%s.%d", prefix, i),
							})
						default:
							ch.AddDim(&module.Dim{
								Algo: module.Absolute,
								ID:   fmt.Sprintf("%s_%d_%s", key(svc.Service), i, ch.ID),
								Name: fmt.Sprintf("%s.%d", prefix, i),
							})
						}
						ch.MarkNotCreated()
					}
				}
			}
		} else {
			// create as many dimensions as we have
			for _, ch := range m.charts {
				for i, svc := range newServices {
					prefix := strings.TrimPrefix(key(svc.Service), "go_micro_")
					switch ch.ID {
					case chartServiceGCRate, chartServiceRequests, chartServiceErrors:
						ch.AddDim(&module.Dim{
							Algo: module.Incremental,
							ID:   fmt.Sprintf("%s_%d_%s", key(svc.Service), i, ch.ID),
							Name: fmt.Sprintf("%s.%d", prefix, i),
						})
					default:
						ch.AddDim(&module.Dim{
							Algo: module.Absolute,
							ID:   fmt.Sprintf("%s_%d_%s", key(svc.Service), i, ch.ID),
							Name: fmt.Sprintf("%s.%d", prefix, i),
						})
					}
					ch.MarkNotCreated()
				}
			}
		}
		processedServices[newKey] = true
	}
	for k := range m.services {
		if _, processed := processedServices[k]; !processed {
			for _, ch := range m.charts {
				for i, svc := range m.services[k] {
					switch ch.ID {
					case chartServiceGCRate, chartServiceRequests, chartServiceErrors:
						ch.MarkDimRemove(fmt.Sprintf("%s_%d_%s", key(svc.Service), i, ch.ID), true)
					default:
						ch.MarkDimRemove(fmt.Sprintf("%s_%d_%s", key(svc.Service), i, ch.ID), true)
					}
					ch.MarkNotCreated()
				}
			}

		}
	}
	m.services = newServices
	return nil
}

// Collect contacts the Debug service to retrieve snapshots of stats
func (m *Micro) collect(ctx context.Context) error {
	// Grab snapshots from the Debug service
	req := &stats.ReadRequest{}
	rsp := &stats.ReadResponse{}
	err := m.client.Call(ctx, client.NewRequest("go.micro.debug", "Stats.Read", req), rsp)
	if err != nil {
		return err
	}

	// If we don't already have a dimension for the service, create it
	if err := m.updateCharts(rsp.Stats); err != nil {
		m.Logger.Error(err)
		return err
	}

	// Populate metrics map
	m.RLock()
	for name, services := range m.services {
		for i, s := range services {
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceStarted)] = int64(s.Started)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceUptime)] = int64(s.Uptime)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceMemory)] = int64(s.Memory)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceThreads)] = int64(s.Threads)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceGC)] = int64(s.Gc)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceGCRate)] = int64(s.Gc)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceRequests)] = int64(s.Requests)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceErrors)] = int64(s.Errors)
		}
	}
	m.RUnlock()
	return nil
}

func format(v string) string {
	return strings.ReplaceAll(v, ".", "_")
}

func key(s *stats.Service) string {
	return format(s.Name + "-" + s.Version)
}
<commit_msg>Try to remove existing services<commit_after>package micro

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/netdata/go-orchestrator/module"

	"github.com/micro/go-micro/client"
	"github.com/micro/go-micro/registry"
	stats "github.com/micro/micro/debug/stats/proto"
)

// Config is the go-micro collector configuration
type Config struct{}

// New creates the micro module with default values
func New(c client.Client) *Micro {
	return &Micro{
		Config:   Config{},
		charts:   charts(),
		services: make(map[string][]*stats.Snapshot),
		metrics:  make(map[string]int64),
		client:   c,
	}
}

// Register Registers with the orchestrator
func (m *Micro) Register() {
	module.Register("go_micro_services", module.Creator{
		Defaults: module.Defaults{
			Disabled: false,
		},
		Create: func() module.Module { return m },
	})
}

// Micro is a netdata module that collects metrics from the go.micro.debug service
type Micro struct {
	// netdata
	module.Base
	Config  `yaml:",inline"`
	charts  Charts
	metrics map[string]int64

	// go-micro
	client client.Client

	// internal
	sync.RWMutex
	cached   []*registry.Service
	services map[string][]*stats.Snapshot
}

// Cleanup is a no-op, called by netdata's orchestrator before shutdown
func (m *Micro) Cleanup() {}

// Init ensures the client is working, then starts saving snapshots.
func (m *Micro) Init() bool {
	// do initial scrape
	if err := m.collect(context.Background()); err != nil {
		m.Logger.Error(err)
	}

	go func() {
		for {
			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
			if err := m.collect(ctx); err != nil {
				m.Logger.Error(err)
			}
			cancel()
			time.Sleep(time.Second)
		}
	}()

	return true
}

// Check can be called by netdata's orchestrator to ensure the collector is working
func (m *Micro) Check() bool {
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	err := m.collect(ctx)
	return err == nil
}

// Charts passes the module's charts to the orchestrator
func (m *Micro) Charts() *Charts {
	m.RLock()
	c := m.charts.Copy()
	m.RUnlock()
	return c
}

// Collect returns the current view of the metrics to the orchestrator
func (m *Micro) Collect() map[string]int64 {
	metrics := make(map[string]int64)
	m.RLock()
	for k, v := range m.metrics {
		metrics[k] = v
	}
	m.RUnlock()
	return metrics
}

func (m *Micro) updateCharts(snapshots []*stats.Snapshot) error {
	// Construct a new Micro.services map based on the list of incoming service snapshots.
	// The map is keyed on Name_Version and sorted by Node ID for consistent graphs
	newServices := make(map[string][]*stats.Snapshot)
	for _, snap := range snapshots {
		key := key(snap.Service)
		if srv, found := newServices[key]; found {
			newServices[key] = append(srv, snap)
		} else {
			newServices[key] = []*stats.Snapshot{snap}
		}
	}
	for _, services := range newServices {
		sort.Slice(services, func(i, j int) bool {
			return services[i].Service.Node.Id < services[j].Service.Node.Id
		})
	}

	// Check for any services that we used to have that disappeared
	m.Lock()
	defer m.Unlock()
	for oldService := range m.services {
		if oldServices, found := newServices[oldService]; !found {
			// Service was in old map, isn't in new map, so remove the dimensions for it.
			for _, ch := range m.charts {
				for i, svc := range oldServices {
					id := fmt.Sprintf("%s_%d_%s", key(svc.Service), i, ch.ID)
					if ch.HasDim(id) {
						ch.MarkDimRemove(id, true)
						ch.MarkNotCreated()
					}
				}
			}
		}
	}

	// Create / remove chart dimensions based on the previous state.
	for newKey, newServices := range newServices {
		if oldServices, found := m.services[newKey]; found {
			// existing service, make sure the dimensions match what we currently have
			if len(newServices) < len(oldServices) {
				// Fewer Services, delete the trailing dimensions
				for len(newServices) < len(oldServices) {
					for _, ch := range m.charts {
						idx := len(oldServices) - 1
						id := fmt.Sprintf("%s_%d_%s", key(oldServices[idx].Service), idx, ch.ID)
						if ch.HasDim(id) {
							ch.MarkDimRemove(id, true)
						}
					}
					oldServices = oldServices[:len(oldServices)-1]
				}
			} else if len(newServices) > len(oldServices) {
				// More services, grow the dimensions
				for idx := len(oldServices); idx <= len(newServices); idx++ {
					for _, ch := range m.charts {
						name := strings.TrimPrefix(key(newServices[0].Service), "go_micro_")
						switch ch.ID {
						case chartServiceGCRate, chartServiceRequests, chartServiceErrors:
							ch.AddDim(&module.Dim{
								Algo: module.Incremental,
								ID:   fmt.Sprintf("%s_%d_%s", key(newServices[0].Service), idx, ch.ID),
								Name: fmt.Sprintf("%s.%d", name, idx),
							})
						default:
							ch.AddDim(&module.Dim{
								Algo: module.Absolute,
								ID:   fmt.Sprintf("%s_%d_%s", key(newServices[0].Service), idx, ch.ID),
								Name: fmt.Sprintf("%s.%d", name, idx),
							})
						}
						ch.MarkNotCreated()
					}
				}
			}
		} else {
			// create as many dimensions as we have
			for _, ch := range m.charts {
				for i, svc := range newServices {
					name := strings.TrimPrefix(key(svc.Service), "go_micro_")
					switch ch.ID {
					case chartServiceGCRate, chartServiceRequests, chartServiceErrors:
						ch.AddDim(&module.Dim{
							Algo: module.Incremental,
							ID:   fmt.Sprintf("%s_%d_%s", key(svc.Service), i, ch.ID),
							Name: fmt.Sprintf("%s.%d", name, i),
						})
					default:
						ch.AddDim(&module.Dim{
							Algo: module.Absolute,
							ID:   fmt.Sprintf("%s_%d_%s", key(svc.Service), i, ch.ID),
							Name: fmt.Sprintf("%s.%d", name, i),
						})
					}
					ch.MarkNotCreated()
				}
			}
		}
	}

	// swap in the new services, then return (m.Unlock was deferred)
	m.services = newServices
	return nil
}

// Collect contacts the Debug service to retrieve snapshots of stats
func (m *Micro) collect(ctx context.Context) error {
	// Grab snapshots from the Debug service
	req := &stats.ReadRequest{}
	rsp := &stats.ReadResponse{}
	err := m.client.Call(ctx, client.NewRequest("go.micro.debug", "Stats.Read", req), rsp)
	if err != nil {
		return err
	}

	// If we don't already have a dimension for the service, create it
	if err := m.updateCharts(rsp.Stats); err != nil {
		m.Logger.Error(err)
		return err
	}

	// Populate metrics map
	m.RLock()
	for name, services := range m.services {
		for i, s := range services {
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceStarted)] = int64(s.Started)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceUptime)] = int64(s.Uptime)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceMemory)] = int64(s.Memory)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceThreads)] = int64(s.Threads)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceGC)] = int64(s.Gc)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceGCRate)] = int64(s.Gc)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceRequests)] = int64(s.Requests)
			m.metrics[fmt.Sprintf("%s_%d_%s", name, i, chartServiceErrors)] = int64(s.Errors)
		}
	}
	m.RUnlock()
	return nil
}

func format(v string) string {
	return strings.ReplaceAll(v, ".", "_")
}

func key(s *stats.Service) string {
	return format(s.Name + "-" + s.Version)
}
<|endoftext|>


<commit_before>package errs

import (
	"fmt"
	"io"
)

// Group is a list of errors.
type Group []error

// Combine combines multiple errors into a single error.
func Combine(errs ...error) error {
	var group Group
	group.Add(errs...)
	return group.Err()
}

// Add adds non-empty errors to the Group.
func (group *Group) Add(errs ...error) {
	for _, err := range errs {
		if err != nil {
			*group = append(*group, err)
		}
	}
}

// Err returns an error containing all of the non-nil errors.
// If there was only one error, it will return it.
// If there were none, it returns nil.
func (group Group) Err() error {
	sanitized := group.sanitize()
	if len(sanitized) == 0 {
		return nil
	}
	if len(sanitized) == 1 {
		return sanitized[0]
	}
	return combinedError(sanitized)
}

// sanitize returns group that doesn't contain nil-s
func (group Group) sanitize() Group {
	// sanity check for non-nil errors
	for i, err := range group {
		if err == nil {
			sanitized := make(Group, 0, len(group)-1)
			sanitized = append(sanitized, group[:i]...)
			sanitized.Add(group[i+1:]...)
			return sanitized
		}
	}

	return group
}

// combinedError is a list of non-empty errors
type combinedError []error

// Cause returns the first error.
func (group combinedError) Cause() error {
	if len(group) > 0 {
		return group[0]
	}
	return nil
}

// Unwrap returns the first error.
func (group combinedError) Unwrap() error {
	return group.Cause()
}

// Error returns error string delimited by semicolons.
func (group combinedError) Error() string { return fmt.Sprintf("%v", group) }

// Format handles the formatting of the error. Using a "+" on the format
// string specifier will cause the errors to be formatted with "+" and
// delimited by newlines. They are delimited by semicolons otherwise.
func (group combinedError) Format(f fmt.State, c rune) {
	delim := "; "
	if f.Flag(int('+')) {
		io.WriteString(f, "group:\n--- ")
		delim = "\n--- "
	}

	for i, err := range group {
		if i != 0 {
			io.WriteString(f, delim)
		}
		if formatter, ok := err.(fmt.Formatter); ok {
			formatter.Format(f, c)
		} else {
			fmt.Fprintf(f, "%v", err)
		}
	}
}
<commit_msg>add subtle detail to Combine doc comment<commit_after>package errs

import (
	"fmt"
	"io"
)

// Group is a list of errors.
type Group []error

// Combine combines multiple non-empty errors into a single error.
func Combine(errs ...error) error {
	var group Group
	group.Add(errs...)
	return group.Err()
}

// Add adds non-empty errors to the Group.
func (group *Group) Add(errs ...error) {
	for _, err := range errs {
		if err != nil {
			*group = append(*group, err)
		}
	}
}

// Err returns an error containing all of the non-nil errors.
// If there was only one error, it will return it.
// If there were none, it returns nil.
func (group Group) Err() error {
	sanitized := group.sanitize()
	if len(sanitized) == 0 {
		return nil
	}
	if len(sanitized) == 1 {
		return sanitized[0]
	}
	return combinedError(sanitized)
}

// sanitize returns group that doesn't contain nil-s
func (group Group) sanitize() Group {
	// sanity check for non-nil errors
	for i, err := range group {
		if err == nil {
			sanitized := make(Group, 0, len(group)-1)
			sanitized = append(sanitized, group[:i]...)
			sanitized.Add(group[i+1:]...)
			return sanitized
		}
	}

	return group
}

// combinedError is a list of non-empty errors
type combinedError []error

// Cause returns the first error.
func (group combinedError) Cause() error {
	if len(group) > 0 {
		return group[0]
	}
	return nil
}

// Unwrap returns the first error.
func (group combinedError) Unwrap() error {
	return group.Cause()
}

// Error returns error string delimited by semicolons.
func (group combinedError) Error() string { return fmt.Sprintf("%v", group) }

// Format handles the formatting of the error. Using a "+" on the format
// string specifier will cause the errors to be formatted with "+" and
// delimited by newlines. They are delimited by semicolons otherwise.
func (group combinedError) Format(f fmt.State, c rune) {
	delim := "; "
	if f.Flag(int('+')) {
		io.WriteString(f, "group:\n--- ")
		delim = "\n--- "
	}

	for i, err := range group {
		if i != 0 {
			io.WriteString(f, delim)
		}
		if formatter, ok := err.(fmt.Formatter); ok {
			formatter.Format(f, c)
		} else {
			fmt.Fprintf(f, "%v", err)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	github_util "k8s.io/contrib/mungegithub/github"
	"k8s.io/contrib/mungegithub/mungers"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
)

var (
	_ = fmt.Print
)

type mungeConfig struct {
	github_util.Config
	MinIssueNumber int
	PRMungersList  []string
	Once           bool
	Period         time.Duration
}

func addMungeFlags(config *mungeConfig, cmd *cobra.Command) {
	cmd.Flags().BoolVar(&config.Once, "once", false, "If true, run one loop and exit")
	cmd.Flags().StringSliceVar(&config.PRMungersList, "pr-mungers", []string{"blunderbuss", "lgtm-after-commit", "needs-rebase", "ok-to-test", "path-label", "ping-ci", "size", "submit-queue"}, "A list of pull request mungers to run")
	cmd.Flags().DurationVar(&config.Period, "period", 10*time.Minute, "The period for running mungers")
}

func doMungers(config *mungeConfig) error {
	for {
		nextRunStartTime := time.Now().Add(config.Period)
		glog.Infof("Running mungers")
		config.NextExpectedUpdate(nextRunStartTime)

		mungers.EachLoop()

		if err := config.ForEachIssueDo(mungers.MungeIssue); err != nil {
			glog.Errorf("Error munging PRs: %v", err)
		}
		config.ResetAPICount()
		if config.Once {
			break
		}
		if nextRunStartTime.After(time.Now()) {
			sleepDuration := nextRunStartTime.Sub(time.Now())
			glog.Infof("Sleeping for %v\n", sleepDuration)
			time.Sleep(sleepDuration)
		} else {
			glog.Infof("Not sleeping as we took more than %v to complete one loop\n", config.Period)
		}
	}
	return nil
}

func main() {
	config := &mungeConfig{}
	root := &cobra.Command{
		Use:   filepath.Base(os.Args[0]),
		Short: "A program to add labels, check tests, and generally mess with outstanding PRs",
		RunE: func(_ *cobra.Command, _ []string) error {
			if err := config.PreExecute(); err != nil {
				return err
			}
			if len(config.PRMungersList) == 0 {
				glog.Fatalf("must include at least one --pr-mungers")
			}
			mungers.InitializeMungers(config.PRMungersList, &config.Config)
			return doMungers(config)
		},
	}
	config.AddRootFlags(root)
	addMungeFlags(config, root)

	allMungers := mungers.GetAllMungers()
	for _, m := range allMungers {
		m.AddFlags(root, &config.Config)
	}

	if err := root.Execute(); err != nil {
		glog.Fatalf("%v\n", err)
	}
}
<commit_msg>Use the kubernetes WordSepNormalizeFunc for consistent flag names<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	github_util "k8s.io/contrib/mungegithub/github"
	"k8s.io/contrib/mungegithub/mungers"
	"k8s.io/kubernetes/pkg/util"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
)

var (
	_ = fmt.Print
)

type mungeConfig struct {
	github_util.Config
	MinIssueNumber int
	PRMungersList  []string
	Once           bool
	Period         time.Duration
}

func addMungeFlags(config *mungeConfig, cmd *cobra.Command) {
	cmd.Flags().BoolVar(&config.Once, "once", false, "If true, run one loop and exit")
	cmd.Flags().StringSliceVar(&config.PRMungersList, "pr-mungers", []string{"blunderbuss", "lgtm-after-commit", "needs-rebase", "ok-to-test", "path-label", "ping-ci", "size", "submit-queue"}, "A list of pull request mungers to run")
	cmd.Flags().DurationVar(&config.Period, "period", 10*time.Minute, "The period for running mungers")
}

func doMungers(config *mungeConfig) error {
	for {
		nextRunStartTime := time.Now().Add(config.Period)
		glog.Infof("Running mungers")
		config.NextExpectedUpdate(nextRunStartTime)

		mungers.EachLoop()

		if err := config.ForEachIssueDo(mungers.MungeIssue); err != nil {
			glog.Errorf("Error munging PRs: %v", err)
		}
		config.ResetAPICount()
		if config.Once {
			break
		}
		if nextRunStartTime.After(time.Now()) {
			sleepDuration := nextRunStartTime.Sub(time.Now())
			glog.Infof("Sleeping for %v\n", sleepDuration)
			time.Sleep(sleepDuration)
		} else {
			glog.Infof("Not sleeping as we took more than %v to complete one loop\n", config.Period)
		}
	}
	return nil
}

func main() {
	config := &mungeConfig{}
	root := &cobra.Command{
		Use:   filepath.Base(os.Args[0]),
		Short: "A program to add labels, check tests, and generally mess with outstanding PRs",
		RunE: func(_ *cobra.Command, _ []string) error {
			if err := config.PreExecute(); err != nil {
				return err
			}
			if len(config.PRMungersList) == 0 {
				glog.Fatalf("must include at least one --pr-mungers")
			}
			mungers.InitializeMungers(config.PRMungersList, &config.Config)
			return doMungers(config)
		},
	}
	root.SetGlobalNormalizationFunc(util.WordSepNormalizeFunc)
	config.AddRootFlags(root)
	addMungeFlags(config, root)

	allMungers := mungers.GetAllMungers()
	for _, m := range allMungers {
		m.AddFlags(root, &config.Config)
	}

	if err := root.Execute(); err != nil {
		glog.Fatalf("%v\n", err)
	}
}
<|endoftext|>


<commit_before>package convert

import (
	"crypto/md5"
	"fmt"
	"strconv"
)

func Int(val interface{}) int64 {
	if val != nil {
		switch val.(type) {
		case float64:
			return int64(val.(float64))
		case int64:
			return val.(int64)
		case string:
			ret, err := strconv.Atoi(val.(string))
			if err != nil {
				return 0
			} else {
				return int64(ret)
			}
		}
	}
	return 0
}

func IntWith(val interface{}, defaultValue int64) int64 {
	if val != nil {
		switch val.(type) {
		case float64:
			return int64(val.(float64))
		case int64:
			return val.(int64)
		case string:
			ret, err := strconv.Atoi(val.(string))
			if err == nil {
				return int64(ret)
			}
		}
	}
	return defaultValue
}

func String(val interface{}) string {
	switch val.(type) {
	case string:
		return val.(string)
	case []byte:
		return string(val.([]byte))
	case nil:
		return ""
	default:
		return fmt.Sprintf("%v", val)
	}
}

func MD5(src string) string {
	return fmt.Sprintf("%x", md5.Sum([]byte(src)))
}
<commit_msg>add QueryString<commit_after>package convert

import (
	"crypto/md5"
	"fmt"
	"strconv"
)

func Int(val interface{}) int64 {
	if val != nil {
		switch val.(type) {
		case float64:
			return int64(val.(float64))
		case int64:
			return val.(int64)
		case string:
			ret, err := strconv.Atoi(val.(string))
			if err != nil {
				return 0
			} else {
				return int64(ret)
			}
		}
	}
	return 0
}

func IntWith(val interface{}, defaultValue int64) int64 {
	if val != nil {
		switch val.(type) {
		case float64:
			return int64(val.(float64))
		case int64:
			return val.(int64)
		case string:
			ret, err := strconv.Atoi(val.(string))
			if err == nil {
				return int64(ret)
			}
		}
	}
	return defaultValue
}

func String(val interface{}) string {
	switch val.(type) {
	case string:
		return val.(string)
	case []byte:
		return string(val.([]byte))
	case nil:
		return ""
	default:
		return fmt.Sprintf("%v", val)
	}
}

func QueryString(val string) string {
	return strings.Join(strings.Split(Queting, "'"), "''")
}

func MD5(src string) string {
	return fmt.Sprintf("%x", md5.Sum([]byte(src)))
}
<|endoftext|>


<commit_before>// Package power provides algorithms for simulating the power dissipation of
// concurrent applications running on multiprocessor platforms.
package power

// #include <string.h>
import "C"

import (
	"errors"
	"unsafe"

	"github.com/ready-steady/simulation/system"
	"github.com/ready-steady/simulation/time"
)

// Power represents a power simulator configured for a particular system.
type Power struct {
	platform    *system.Platform
	application *system.Application
	Δt          float64
}

// New returns a power distributor for the given platform, application, and
// sampling period.
func New(platform *system.Platform, application *system.Application, Δt float64) (*Power, error) {
	if Δt <= 0 {
		return nil, errors.New("the time step should be positive")
	}

	power := &Power{
		platform:    platform,
		application: application,
		Δt:          Δt,
	}

	return power, nil
}

// Compute constructs the power profile of the given schedule and stores it in a
// cc-by-sc matrix P where cc is the number of cores and sc is the maximal
// number of steps (samples) that the matrix can accommodate.
func (p *Power) Compute(schedule *time.Schedule, P []float64, sc uint) {
	const (
		sizeOfFloat64 = 8
	)

	cores, tasks := p.platform.Cores, p.application.Tasks
	cc, tc := uint(len(cores)), uint(len(tasks))
	Δt := p.Δt

	if count := uint(schedule.Span / Δt); count < sc {
		sc = count
	}

	// FIXME: Bad, bad, bad!
	C.memset(unsafe.Pointer(&P[0]), 0, C.size_t(sizeOfFloat64*cc*sc))

	for i := uint(0); i < tc; i++ {
		j := schedule.Mapping[i]
		p := cores[j].Power[tasks[i].Type]

		s := uint(schedule.Start[i] / Δt)
		f := uint(schedule.Finish[i] / Δt)
		if f > sc {
			f = sc
		}

		for ; s < f; s++ {
			P[s*cc+j] = p
		}
	}
}
<commit_msg>Fixed the buffer nullification in power.Compute<commit_after>// Package power provides algorithms for simulating the power dissipation of
// concurrent applications running on multiprocessor platforms.
package power

// #include <string.h>
import "C"

import (
	"errors"
	"unsafe"

	"github.com/ready-steady/simulation/system"
	"github.com/ready-steady/simulation/time"
)

// Power represents a power simulator configured for a particular system.
type Power struct {
	platform    *system.Platform
	application *system.Application
	Δt          float64
}

// New returns a power distributor for the given platform, application, and
// sampling period.
func New(platform *system.Platform, application *system.Application, Δt float64) (*Power, error) {
	if Δt <= 0 {
		return nil, errors.New("the time step should be positive")
	}

	power := &Power{
		platform:    platform,
		application: application,
		Δt:          Δt,
	}

	return power, nil
}

// Compute constructs the power profile of the given schedule and stores it in a
// cc-by-sc matrix P where cc is the number of cores and sc is the maximal
// number of steps (samples) that the matrix can accommodate.
func (p *Power) Compute(schedule *time.Schedule, P []float64, sc uint) {
	const (
		sizeOfFloat64 = 8
	)

	cores, tasks := p.platform.Cores, p.application.Tasks
	cc, tc := uint(len(cores)), uint(len(tasks))
	Δt := p.Δt

	// FIXME: Bad, bad, bad!
	C.memset(unsafe.Pointer(&P[0]), 0, C.size_t(sizeOfFloat64*cc*sc))

	if count := uint(schedule.Span / Δt); count < sc {
		sc = count
	}

	for i := uint(0); i < tc; i++ {
		j := schedule.Mapping[i]
		p := cores[j].Power[tasks[i].Type]

		s := uint(schedule.Start[i] / Δt)
		f := uint(schedule.Finish[i] / Δt)
		if f > sc {
			f = sc
		}

		for ; s < f; s++ {
			P[s*cc+j] = p
		}
	}
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
	engineCli "github.com/docker/docker/client"
	"github.com/patrickmn/go-cache"
	"github.com/pkg/errors"
	"github.com/rancher/agent/core/progress"
	"github.com/rancher/agent/model"
	"github.com/rancher/agent/utilities/constants"
	"github.com/rancher/agent/utilities/docker"
	"github.com/rancher/agent/utilities/utils"
	"golang.org/x/net/context"
	"os"
	"time"
)

func DoVolumeActivate(volume model.Volume, storagePool model.StoragePool, progress *progress.Progress, client *engineCli.Client) error {
	if !isManagedVolume(volume) {
		return nil
	}
	driver := volume.Data.Fields.Driver
	driverOpts := volume.Data.Fields.DriverOpts
	opts := map[string]string{}
	if driverOpts != nil {
		for k, v := range driverOpts {
			opts[k] = utils.InterfaceToString(v)
		}
	}

	// Rancher longhorn volumes indicate when they've been moved to a
	// different host. If so, we have to delete before we create
	// to cleanup the reference in docker.

	vol, err := client.VolumeInspect(context.Background(), volume.Name)
	if err != nil {
		if vol.Mountpoint == "moved" {
			logrus.Info(fmt.Sprintf("Removing moved volume %s so that it can be re-added.", volume.Name))
			if err := client.VolumeRemove(context.Background(), volume.Name, true); err != nil {
				return errors.Wrap(err, constants.DoVolumeActivateError+"failed to remove volume")
			}
		}
	}

	options := types.VolumeCreateRequest{
		Name:       volume.Name,
		Driver:     driver,
		DriverOpts: opts,
	}
	_, err1 := client.VolumeCreate(context.Background(), options)
	if err1 != nil {
		return errors.Wrap(err1, constants.DoVolumeActivateError+"failed to create volume")
	}
	return nil
}

func PullImage(image model.Image, progress *progress.Progress, client *engineCli.Client, imageUUID string) error {
	dockerCli := docker.GetClient(docker.DefaultVersion)
	return DoImageActivate(image, model.StoragePool{}, progress, dockerCli, imageUUID)
}

func DoImageActivate(image model.Image, storagePool model.StoragePool, progress *progress.Progress, client *engineCli.Client, imageUUID string) error {
	if utils.IsImageNoOp(image) {
		return nil
	}
	imageName := utils.ParseRepoTag(imageUUID)
	if isBuild(image) {
		return imageBuild(image, progress, client)
	}
	rc := image.RegistryCredential
	auth := types.AuthConfig{
		Username:      rc.PublicValue,
		Email:         rc.Data.Fields.Email,
		ServerAddress: rc.Data.Fields.ServerAddress,
		Password:      rc.SecretValue,
	}
	if auth.ServerAddress == "https://docker.io" {
		auth.ServerAddress = "https://index.docker.io"
	}
	registryAuth := wrapAuth(auth)
	pullOption := types.ImagePullOptions{
		RegistryAuth: registryAuth,
	}
	return pullImageWrap(client, imageName, pullOption, progress)
}

func DoVolumeRemove(volume model.Volume, storagePool model.StoragePool, progress *progress.Progress, dockerClient *engineCli.Client, ca *cache.Cache, resourceID string) error {
	if _, ok := ca.Get(resourceID); ok {
		ca.Delete(resourceID)
		return nil
	}
	if ok, err := IsVolumeRemoved(volume, storagePool, dockerClient); ok {
		return nil
	} else if err != nil {
		return errors.Wrap(err, constants.DoVolumeRemoveError+"failed to check whether volume is removed")
	}
	if volume.DeviceNumber == 0 {
		container, err := utils.GetContainer(dockerClient, volume.Instance, false)
		if err != nil {
			if !utils.IsContainerNotFoundError(err) {
				return errors.Wrap(err, constants.DoVolumeRemoveError+"faild to get container")
			}
		}
		if container.ID == "" {
			return nil
		}
		errorList := []error{}
		for i := 0; i < 3; i++ {
			if err := utils.RemoveContainer(dockerClient, container.ID); !engineCli.IsErrContainerNotFound(err) {
				errorList = append(errorList, err)
			} else {
				break
			}
			time.Sleep(time.Second * 1)
		}
		if len(errorList) == 3 {
			ca.Add(resourceID, true, cache.DefaultExpiration)
			logrus.Warnf("Failed to remove container id [%v]. Tried three times and failed. Error msg: %v", container.ID, errorList)
		}
	} else if isManagedVolume(volume) {
		errorList := []error{}
		for i := 0; i < 3; i++ {
			err := dockerClient.VolumeRemove(context.Background(), volume.Name, false)
			if err != nil {
				errorList = append(errorList, err)
			} else {
				break
			}
			time.Sleep(time.Second * 1)
		}
		if len(errorList) == 3 {
			ca.Add(resourceID, true, cache.DefaultExpiration)
			logrus.Warnf("Failed to remove volume name [%v]. Tried three times and failed. Error msg: %v", volume.Name, errorList)
		}
		return nil
	}
	path := pathToVolume(volume)
	if !volume.Data.Fields.IsHostPath {
		_, existErr := os.Stat(path)
		if existErr == nil {
			if err := os.RemoveAll(path); err != nil {
				return errors.Wrap(err, constants.DoVolumeRemoveError+"failed to remove directory")
			}
		}
		return errors.Wrap(existErr, constants.DoVolumeRemoveError+"failed to find the path")
	}
	return nil
}
<commit_msg>small fix<commit_after>package storage

import (
	"fmt"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
	engineCli "github.com/docker/docker/client"
	"github.com/patrickmn/go-cache"
	"github.com/pkg/errors"
	"github.com/rancher/agent/core/progress"
	"github.com/rancher/agent/model"
	"github.com/rancher/agent/utilities/constants"
	"github.com/rancher/agent/utilities/docker"
	"github.com/rancher/agent/utilities/utils"
	"golang.org/x/net/context"
	"os"
	"time"
)

func DoVolumeActivate(volume model.Volume, storagePool model.StoragePool, progress *progress.Progress, client *engineCli.Client) error {
	if !isManagedVolume(volume) {
		return nil
	}
	driver := volume.Data.Fields.Driver
	driverOpts := volume.Data.Fields.DriverOpts
	opts := map[string]string{}
	if driverOpts != nil {
		for k, v := range driverOpts {
			opts[k] = utils.InterfaceToString(v)
		}
	}

	// Rancher longhorn volumes indicate when they've been moved to a
	// different host. If so, we have to delete before we create
	// to cleanup the reference in docker.

	vol, err := client.VolumeInspect(context.Background(), volume.Name)
	if err != nil {
		if vol.Mountpoint == "moved" {
			logrus.Info(fmt.Sprintf("Removing moved volume %s so that it can be re-added.", volume.Name))
			if err := client.VolumeRemove(context.Background(), volume.Name, true); err != nil {
				return errors.Wrap(err, constants.DoVolumeActivateError+"failed to remove volume")
			}
		}
	}

	options := types.VolumeCreateRequest{
		Name:       volume.Name,
		Driver:     driver,
		DriverOpts: opts,
	}
	_, err1 := client.VolumeCreate(context.Background(), options)
	if err1 != nil {
		return errors.Wrap(err1, constants.DoVolumeActivateError+"failed to create volume")
	}
	return nil
}

func PullImage(image model.Image, progress *progress.Progress, client *engineCli.Client, imageUUID string) error {
	dockerCli := docker.GetClient(docker.DefaultVersion)
	return DoImageActivate(image, model.StoragePool{}, progress, dockerCli, imageUUID)
}

func DoImageActivate(image model.Image, storagePool model.StoragePool, progress *progress.Progress, client *engineCli.Client, imageUUID string) error {
	if utils.IsImageNoOp(image) {
		return nil
	}
	imageName := utils.ParseRepoTag(imageUUID)
	if isBuild(image) {
		return imageBuild(image, progress, client)
	}
	rc := image.RegistryCredential
	auth := types.AuthConfig{
		Username:      rc.PublicValue,
		Email:         rc.Data.Fields.Email,
		ServerAddress: rc.Data.Fields.ServerAddress,
		Password:      rc.SecretValue,
	}
	if auth.ServerAddress == "https://docker.io" {
		auth.ServerAddress = "https://index.docker.io"
	}
	registryAuth := wrapAuth(auth)
	pullOption := types.ImagePullOptions{
		RegistryAuth: registryAuth,
	}
	return pullImageWrap(client, imageName, pullOption, progress)
}

func DoVolumeRemove(volume model.Volume, storagePool model.StoragePool, progress *progress.Progress, dockerClient *engineCli.Client, ca *cache.Cache, resourceID string) error {
	if _, ok := ca.Get(resourceID); ok {
		ca.Delete(resourceID)
		return nil
	}
	if ok, err := IsVolumeRemoved(volume, storagePool, dockerClient); ok {
		return nil
	} else if err != nil {
		return errors.Wrap(err, constants.DoVolumeRemoveError+"failed to check whether volume is removed")
	}
	if volume.DeviceNumber == 0 {
		container, err := utils.GetContainer(dockerClient, volume.Instance, false)
		if err != nil {
			if !utils.IsContainerNotFoundError(err) {
				return errors.Wrap(err, constants.DoVolumeRemoveError+"faild to get container")
			}
		}
		if container.ID == "" {
			return nil
		}
		errorList := []error{}
		for i := 0; i < 3; i++ {
			if err := utils.RemoveContainer(dockerClient, container.ID); err != nil && !engineCli.IsErrContainerNotFound(err) {
				errorList = append(errorList, err)
			} else {
				break
			}
			time.Sleep(time.Second * 1)
		}
		if len(errorList) == 3 {
			ca.Add(resourceID, true, cache.DefaultExpiration)
			logrus.Warnf("Failed to remove container id [%v]. Tried three times and failed. Error msg: %v", container.ID, errorList)
		}
	} else if isManagedVolume(volume) {
		errorList := []error{}
		for i := 0; i < 3; i++ {
			err := dockerClient.VolumeRemove(context.Background(), volume.Name, false)
			if err != nil {
				errorList = append(errorList, err)
			} else {
				break
			}
			time.Sleep(time.Second * 1)
		}
		if len(errorList) == 3 {
			ca.Add(resourceID, true, cache.DefaultExpiration)
			logrus.Warnf("Failed to remove volume name [%v]. Tried three times and failed. Error msg: %v", volume.Name, errorList)
		}
		return nil
	}
	path := pathToVolume(volume)
	if !volume.Data.Fields.IsHostPath {
		_, existErr := os.Stat(path)
		if existErr == nil {
			if err := os.RemoveAll(path); err != nil {
				return errors.Wrap(err, constants.DoVolumeRemoveError+"failed to remove directory")
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Package natty provides a Go language wrapper to the natty NAT traversal
// utility.  See https://github.com/getlantern/t.
//
// See natty_test for an example of Natty in use, including debug logging
// showing the messages that are sent across the signaling channel.
package natty

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/byteexec"
	"github.com/getlantern/golog"
)

const (
	UDP = Protocol("udp")
	TCP = Protocol("tcp")
)

var (
	log = golog.LoggerFor("natty")

	reallyHighTimeout = 100000 * time.Hour
)

type Protocol string

// A FiveTuple is the result of a successful NAT traversal.
type FiveTuple struct {
	Proto  Protocol
	Local  string
	Remote string
}

// UDPAddrs returns a pair of UDPAddrs representing the Local and Remote
// addresses of this FiveTuple. If the FiveTuple's Proto is not UDP, this method
// returns an error.
func (ft *FiveTuple) UDPAddrs() (local *net.UDPAddr, remote *net.UDPAddr, err error) {
	if ft.Proto != UDP {
		err = fmt.Errorf("FiveTuple.Proto was not UDP!: %s", ft.Proto)
		return
	}
	local, err = net.ResolveUDPAddr("udp", ft.Local)
	if err != nil {
		err = fmt.Errorf("Unable to resolve local UDP address %s: %s", ft.Local)
		return
	}
	remote, err = net.ResolveUDPAddr("udp", ft.Remote)
	if err != nil {
		err = fmt.Errorf("Unable to resolve remote UDP address %s: %s", ft.Remote)
	}
	return
}

// Traversal represents a single NAT traversal using natty, whose result is
// available via the methods FiveTuple() and FiveTupleTimeout().
//
// Consumers should make sure to call Close() after finishing with this Natty
// in order to make sure the underlying natty process and associated resources
// are closed.
type Traversal struct {
	debugOut           io.Writer       // target for output from natty's stderr
	cmd                *exec.Cmd       // the natty command
	stdin              io.WriteCloser  // pipe to natty's stdin
	stdout             io.ReadCloser   // pipe from natty's stdout
	stdoutbuf          *bufio.Reader   // buffered stdout
	stderr             io.ReadCloser   // pipe from natty's stderr
	msgInCh            chan string     // channel for messages inbound to this Natty
	msgOutCh           chan string     // channel for messages outbound from this Natty
	peerGotFiveTupleCh chan bool       // channel to signal once we know that our peer received their own FiveTuple
	fiveTupleCh        chan *FiveTuple // intermediary channel for the FiveTuple emitted by the natty command
	errCh              chan error      // intermediary channel for any error encountered while running natty
	fiveTupleOutCh     chan *FiveTuple // channel for FiveTuple output
	errOutCh           chan error      // channel for error output
	fiveTupleOut       *FiveTuple      // the output FiveTuple
	errOut             error           // the output error
	outMutex           sync.Mutex      // mutex for synchronizing access to output variables
}

// Offer starts a Traversal as an Offerer, meaning that it will make an offer to
// initiate an ICE session. Call FiveTuple() or FiveTupleTimeout() to get the
// FiveTuple resulting from Traversal.
//
// debugOut (optional) is an optional io.Writer to which debug output from natty
// will be written.
//
func Offer(debugOut io.Writer) *Traversal {
	log.Trace("Offering")
	t := &Traversal{
		debugOut: debugOut,
	}
	t.run([]string{"-offer"})
	return t
}

// Answer starts a Traversal as an Answerer, meaning that it will accept offers
// to initiate an ICE session. Call FiveTuple() or FiveTupleTimeout() to get the
// FiveTuple resulting from Traversal.
//
// debugOut (optional) is an optional io.Writer to which debug output from Natty
// will be written.
//
func Answer(debugOut io.Writer) *Traversal {
	log.Trace("Answering")
	t := &Traversal{
		debugOut: debugOut,
	}
	t.run([]string{})
	return t
}

// MsgIn is used to pass this Traversal a message from the peer t. This method
// is buffered and will typically not block.
func (t *Traversal) MsgIn(msg string) {
	log.Tracef("Got message: %s", msg)
	t.msgInCh <- msg
}

// NextMsgOut gets the next message to pass to the peer.  If done is true, there
// are no more messages to be read, and the currently returned message should be
// ignored.
func (t *Traversal) NextMsgOut() (msg string, done bool) {
	m, ok := <-t.msgOutCh
	log.Tracef("Returning out message: %s", m)
	return m, !ok
}

// FiveTuple gets the FiveTuple from the Traversal, blocking until such is
// available.
func (t *Traversal) FiveTuple() (*FiveTuple, error) {
	log.Trace("Getting FiveTuple")
	return t.FiveTupleTimeout(reallyHighTimeout)
}

// FiveTupleTimeout gets the FiveTuple from the Traversal, blocking until such
// is available or the timeout is hit.
func (t *Traversal) FiveTupleTimeout(timeout time.Duration) (*FiveTuple, error) {
	log.Trace("Getting FiveTupleTimeout")
	t.outMutex.Lock()
	defer t.outMutex.Unlock()

	if t.fiveTupleOut == nil && t.errOut == nil {
		log.Trace("We don't have a result yet, wait for one")
		select {
		case ft := <-t.fiveTupleOutCh:
			log.Tracef("FiveTuple is: %s", ft)
			t.fiveTupleOut = ft
		case err := <-t.errOutCh:
			log.Tracef("Error is: %s", err)
			t.errOut = err
		case <-time.After(timeout):
			log.Trace("Return an error, but don't store it (lets caller try again)")
			return nil, fmt.Errorf("Timed out waiting for five-tuple")
		}
	}

	return t.fiveTupleOut, t.errOut
}

// Close closes this Traversal, terminating any outstanding natty process by
// sending SIGKILL. Close blocks until the natty process has terminated, at
// which point any ports that it bound should be available for use.
func (t *Traversal) Close() error {
	log.Trace("Closing pipes")
	t.closePipes()

	if t.cmd != nil && t.cmd.Process != nil {
		log.Trace("Killing natty process")
		t.cmd.Process.Kill()
		t.cmd.Process.Wait()
		log.Trace("Done killing natty process")
	}
	return nil
}

// run runs the natty command to obtain a FiveTuple. The actual running of
// natty happens on a goroutine so that run itself doesn't block.
func (t *Traversal) run(params []string) {
	t.msgInCh = make(chan string, 100)
	t.msgOutCh = make(chan string, 100)

	// Note - these channels are a buffered in order to prevent deadlocks
	t.peerGotFiveTupleCh = make(chan bool, 10)
	t.fiveTupleCh = make(chan *FiveTuple, 10)
	t.errCh = make(chan error, 10)
	t.fiveTupleOutCh = make(chan *FiveTuple, 10)
	t.errOutCh = make(chan error, 10)

	err := t.initCommand(params)

	go func() {
		if err != nil {
			t.errOutCh <- err
			return
		}

		ft, err := t.doRun(params)
		// Once doRun is finished, inform client of the FiveTuple
		if err != nil {
			log.Tracef("Returning error: %s", err)
			t.errOutCh <- err
		} else {
			log.Tracef("Returning FiveTuple: %s", ft)
			t.fiveTupleOutCh <- ft
		}
	}()
}

// doRun does the running, including resource cleanup.  doRun blocks until
// Close() has finished, meaning that natty is no longer running and whatever
// port it returned in the FiveTuple can now be used for other things.
func (t *Traversal) doRun(params []string) (*FiveTuple, error) {
	defer t.Close()

	go t.processStdout()
	go t.processStderr()

	// Start the natty command
	t.errCh <- t.cmd.Start()

	// Process incoming messages
	go func() {
		for {
			msg := <-t.msgInCh
			log.Tracef("Got incoming message: %s", msg)

			if IsFiveTuple(msg) {
				log.Trace("Incoming message was a FiveTuple!")
				t.peerGotFiveTupleCh <- true
				continue
			}

			log.Trace("Forward message to natty process")
			_, err := t.stdin.Write([]byte(msg))
			if err == nil {
				_, err = t.stdin.Write([]byte("\n"))
			}
			if err != nil {
				log.Tracef("Unable to forward message to natty process: %s: %s", msg, err)
				t.errCh <- err
			} else {
				log.Tracef("Forwarded message to natty process: %s", msg)
			}
		}
	}()

	for {
		select {
		case result := <-t.fiveTupleCh:
			// Wait for peer to get FiveTuple before returning.  If we didn't do
			// this, our natty instance might stop running before the peer
			// finishes its work to get its own FiveTuple.
			log.Trace("Got our own FiveTuple, waiting for peer to get FiveTuple")
			<-t.peerGotFiveTupleCh
			log.Trace("Peer got FiveTuple!")
			return result, nil
		case err := <-t.errCh:
			if err != nil && err != io.EOF {
				return nil, err
			}
		}
	}
}

// initCommand sets up the natty command
func (t *Traversal) initCommand(params []string) error {
	if t.debugOut == nil {
		// Discard stderr output by default
		t.debugOut = ioutil.Discard
	} else {
		// Tell natty to log debug output
		params = append(params, "-debug")
	}

	nattyBytes, err := Asset("natty")
	if err != nil {
		return err
	}
	be, err := byteexec.NewNamedByteExec(nattyBytes, "natty")
	if err != nil {
		return err
	}

	t.cmd = be.Command(params...)
	t.stdin, err = t.cmd.StdinPipe()
	if err != nil {
		return err
	}
	t.stdout, err = t.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	t.stderr, err = t.cmd.StderrPipe()
	if err != nil {
		return err
	}

	t.stdoutbuf = bufio.NewReader(t.stdout)

	return nil
}

// processStdout reads the output from natty and sends it to the msgOutCh. If
// it finds a FiveTuple, it records that.
func (t *Traversal) processStdout() {
	for {
		// Read next message from natty
		msg, err := t.stdoutbuf.ReadString('\n')
		if err != nil {
			t.errCh <- err
			return
		}

		// Request send of message to peer
		t.msgOutCh <- msg

		if IsFiveTuple(msg) {
			// We got a FiveTuple!
			fiveTuple := &FiveTuple{}
			err = json.Unmarshal([]byte(msg), fiveTuple)
			if err != nil {
				t.errCh <- err
				return
			}
			t.fiveTupleCh <- fiveTuple
		}
	}
}

// processStderr copies the output from natty's stderr to the configured
// DebugOut
func (t *Traversal) processStderr() {
	_, err := io.Copy(t.debugOut, t.stderr)
	t.errCh <- err
}

func (t *Traversal) closePipes() {
	if t.stdin != nil {
		t.stdin.Close()
	}
	if t.stdout != nil {
		t.stdout.Close()
	}
	if t.stderr != nil {
		t.stderr.Close()
	}
}

func IsFiveTuple(msg string) bool {
	return strings.Contains(msg, "\"type\":\"5-tuple\"")
}
<commit_msg>Closing pipes on goroutine<commit_after>// Package natty provides a Go language wrapper to the natty NAT traversal
// utility.  See https://github.com/getlantern/t.
//
// See natty_test for an example of Natty in use, including debug logging
// showing the messages that are sent across the signaling channel.
package natty

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/byteexec"
	"github.com/getlantern/golog"
)

const (
	UDP = Protocol("udp")
	TCP = Protocol("tcp")
)

var (
	log = golog.LoggerFor("natty")

	reallyHighTimeout = 100000 * time.Hour
)

type Protocol string

// A FiveTuple is the result of a successful NAT traversal.
type FiveTuple struct {
	Proto  Protocol
	Local  string
	Remote string
}

// UDPAddrs returns a pair of UDPAddrs representing the Local and Remote
// addresses of this FiveTuple. If the FiveTuple's Proto is not UDP, this method
// returns an error.
func (ft *FiveTuple) UDPAddrs() (local *net.UDPAddr, remote *net.UDPAddr, err error) {
	if ft.Proto != UDP {
		err = fmt.Errorf("FiveTuple.Proto was not UDP!: %s", ft.Proto)
		return
	}
	local, err = net.ResolveUDPAddr("udp", ft.Local)
	if err != nil {
		err = fmt.Errorf("Unable to resolve local UDP address %s: %s", ft.Local)
		return
	}
	remote, err = net.ResolveUDPAddr("udp", ft.Remote)
	if err != nil {
		err = fmt.Errorf("Unable to resolve remote UDP address %s: %s", ft.Remote)
	}
	return
}

// Traversal represents a single NAT traversal using natty, whose result is
// available via the methods FiveTuple() and FiveTupleTimeout().
//
// Consumers should make sure to call Close() after finishing with this Natty
// in order to make sure the underlying natty process and associated resources
// are closed.
type Traversal struct {
	debugOut           io.Writer       // target for output from natty's stderr
	cmd                *exec.Cmd       // the natty command
	stdin              io.WriteCloser  // pipe to natty's stdin
	stdout             io.ReadCloser   // pipe from natty's stdout
	stdoutbuf          *bufio.Reader   // buffered stdout
	stderr             io.ReadCloser   // pipe from natty's stderr
	msgInCh            chan string     // channel for messages inbound to this Natty
	msgOutCh           chan string     // channel for messages outbound from this Natty
	peerGotFiveTupleCh chan bool       // channel to signal once we know that our peer received their own FiveTuple
	fiveTupleCh        chan *FiveTuple // intermediary channel for the FiveTuple emitted by the natty command
	errCh              chan error      // intermediary channel for any error encountered while running natty
	fiveTupleOutCh     chan *FiveTuple // channel for FiveTuple output
	errOutCh           chan error      // channel for error output
	fiveTupleOut       *FiveTuple      // the output FiveTuple
	errOut             error           // the output error
	outMutex           sync.Mutex      // mutex for synchronizing access to output variables
}

// Offer starts a Traversal as an Offerer, meaning that it will make an offer to
// initiate an ICE session. Call FiveTuple() or FiveTupleTimeout() to get the
// FiveTuple resulting from Traversal.
//
// debugOut (optional) is an optional io.Writer to which debug output from natty
// will be written.
//
func Offer(debugOut io.Writer) *Traversal {
	log.Trace("Offering")
	t := &Traversal{
		debugOut: debugOut,
	}
	t.run([]string{"-offer"})
	return t
}

// Answer starts a Traversal as an Answerer, meaning that it will accept offers
// to initiate an ICE session. Call FiveTuple() or FiveTupleTimeout() to get the
// FiveTuple resulting from Traversal.
//
// debugOut (optional) is an optional io.Writer to which debug output from Natty
// will be written.
//
func Answer(debugOut io.Writer) *Traversal {
	log.Trace("Answering")
	t := &Traversal{
		debugOut: debugOut,
	}
	t.run([]string{})
	return t
}

// MsgIn is used to pass this Traversal a message from the peer t. This method
// is buffered and will typically not block.
func (t *Traversal) MsgIn(msg string) {
	log.Tracef("Got message: %s", msg)
	t.msgInCh <- msg
}

// NextMsgOut gets the next message to pass to the peer.  If done is true, there
// are no more messages to be read, and the currently returned message should be
// ignored.
func (t *Traversal) NextMsgOut() (msg string, done bool) {
	m, ok := <-t.msgOutCh
	log.Tracef("Returning out message: %s", m)
	return m, !ok
}

// FiveTuple gets the FiveTuple from the Traversal, blocking until such is
// available.
func (t *Traversal) FiveTuple() (*FiveTuple, error) {
	log.Trace("Getting FiveTuple")
	return t.FiveTupleTimeout(reallyHighTimeout)
}

// FiveTupleTimeout gets the FiveTuple from the Traversal, blocking until such
// is available or the timeout is hit.
func (t *Traversal) FiveTupleTimeout(timeout time.Duration) (*FiveTuple, error) {
	log.Trace("Getting FiveTupleTimeout")
	t.outMutex.Lock()
	defer t.outMutex.Unlock()

	if t.fiveTupleOut == nil && t.errOut == nil {
		log.Trace("We don't have a result yet, wait for one")
		select {
		case ft := <-t.fiveTupleOutCh:
			log.Tracef("FiveTuple is: %s", ft)
			t.fiveTupleOut = ft
		case err := <-t.errOutCh:
			log.Tracef("Error is: %s", err)
			t.errOut = err
		case <-time.After(timeout):
			log.Trace("Return an error, but don't store it (lets caller try again)")
			return nil, fmt.Errorf("Timed out waiting for five-tuple")
		}
	}

	return t.fiveTupleOut, t.errOut
}

// Close closes this Traversal, terminating any outstanding natty process by
// sending SIGKILL. Close blocks until the natty process has terminated, at
// which point any ports that it bound should be available for use.
func (t *Traversal) Close() error {
	go func() {
		// For some reason, we have to do this on a goroutine on Windows
		log.Trace("Closing pipes")
		t.closePipes()
	}()

	if t.cmd != nil && t.cmd.Process != nil {
		log.Trace("Killing natty process")
		t.cmd.Process.Kill()
		t.cmd.Process.Wait()
		log.Trace("Done killing natty process")
	}
	return nil
}

// run runs the natty command to obtain a FiveTuple. The actual running of
// natty happens on a goroutine so that run itself doesn't block.
func (t *Traversal) run(params []string) {
	t.msgInCh = make(chan string, 100)
	t.msgOutCh = make(chan string, 100)

	// Note - these channels are a buffered in order to prevent deadlocks
	t.peerGotFiveTupleCh = make(chan bool, 10)
	t.fiveTupleCh = make(chan *FiveTuple, 10)
	t.errCh = make(chan error, 10)
	t.fiveTupleOutCh = make(chan *FiveTuple, 10)
	t.errOutCh = make(chan error, 10)

	err := t.initCommand(params)

	go func() {
		if err != nil {
			t.errOutCh <- err
			return
		}

		ft, err := t.doRun(params)
		// Once doRun is finished, inform client of the FiveTuple
		if err != nil {
			log.Tracef("Returning error: %s", err)
			t.errOutCh <- err
		} else {
			log.Tracef("Returning FiveTuple: %s", ft)
			t.fiveTupleOutCh <- ft
		}
	}()
}

// doRun does the running, including resource cleanup.  doRun blocks until
// Close() has finished, meaning that natty is no longer running and whatever
// port it returned in the FiveTuple can now be used for other things.
func (t *Traversal) doRun(params []string) (*FiveTuple, error) {
	defer t.Close()

	go t.processStdout()
	go t.processStderr()

	// Start the natty command
	t.errCh <- t.cmd.Start()

	// Process incoming messages
	go func() {
		for {
			msg := <-t.msgInCh
			log.Tracef("Got incoming message: %s", msg)

			if IsFiveTuple(msg) {
				log.Trace("Incoming message was a FiveTuple!")
				t.peerGotFiveTupleCh <- true
				continue
			}

			log.Trace("Forward message to natty process")
			_, err := t.stdin.Write([]byte(msg))
			if err == nil {
				_, err = t.stdin.Write([]byte("\n"))
			}
			if err != nil {
				log.Tracef("Unable to forward message to natty process: %s: %s", msg, err)
				t.errCh <- err
			} else {
				log.Tracef("Forwarded message to natty process: %s", msg)
			}
		}
	}()

	for {
		select {
		case result := <-t.fiveTupleCh:
			// Wait for peer to get FiveTuple before returning.  If we didn't do
			// this, our natty instance might stop running before the peer
			// finishes its work to get its own FiveTuple.
			log.Trace("Got our own FiveTuple, waiting for peer to get FiveTuple")
			<-t.peerGotFiveTupleCh
			log.Trace("Peer got FiveTuple!")
			return result, nil
		case err := <-t.errCh:
			if err != nil && err != io.EOF {
				return nil, err
			}
		}
	}
}

// initCommand sets up the natty command
func (t *Traversal) initCommand(params []string) error {
	if t.debugOut == nil {
		// Discard stderr output by default
		t.debugOut = ioutil.Discard
	} else {
		// Tell natty to log debug output
		params = append(params, "-debug")
	}

	nattyBytes, err := Asset("natty")
	if err != nil {
		return err
	}
	be, err := byteexec.NewNamedByteExec(nattyBytes, "natty")
	if err != nil {
		return err
	}

	t.cmd = be.Command(params...)
	t.stdin, err = t.cmd.StdinPipe()
	if err != nil {
		return err
	}
	t.stdout, err = t.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	t.stderr, err = t.cmd.StderrPipe()
	if err != nil {
		return err
	}

	t.stdoutbuf = bufio.NewReader(t.stdout)

	return nil
}

// processStdout reads the output from natty and sends it to the msgOutCh. If
// it finds a FiveTuple, it records that.
func (t *Traversal) processStdout() {
	for {
		// Read next message from natty
		msg, err := t.stdoutbuf.ReadString('\n')
		if err != nil {
			t.errCh <- err
			return
		}

		// Request send of message to peer
		t.msgOutCh <- msg

		if IsFiveTuple(msg) {
			// We got a FiveTuple!
			fiveTuple := &FiveTuple{}
			err = json.Unmarshal([]byte(msg), fiveTuple)
			if err != nil {
				t.errCh <- err
				return
			}
			t.fiveTupleCh <- fiveTuple
		}
	}
}

// processStderr copies the output from natty's stderr to the configured
// DebugOut
func (t *Traversal) processStderr() {
	_, err := io.Copy(t.debugOut, t.stderr)
	t.errCh <- err
}

func (t *Traversal) closePipes() {
	if t.stdin != nil {
		log.Trace("Closing stdin")
		t.stdin.Close()
	}
	if t.stdout != nil {
		log.Trace("Closing stdout")
		t.stdout.Close()
	}
	if t.stderr != nil {
		log.Trace("Closing stderr")
		t.stderr.Close()
	}
}

func IsFiveTuple(msg string) bool {
	return strings.Contains(msg, "\"type\":\"5-tuple\"")
}
<|endoftext|>


<commit_before>package db

import(
    "github.com/brutella/hap"
    "github.com/brutella/hap/common"
)

type Database interface {
    ClientWithName(name string) Client
    SaveClient(client Client) error
    DeleteClient(client Client)
}

type database struct {
    storage hap.Storage
}

func NewDatabase(path string) (*database, error) {
    storage, err := common.NewFileStorage(path)
    if err != nil {
        return nil, err
    }

    return NewDatabaseWithStorage(storage), nil
}

func NewDatabaseWithStorage(storage hap.Storage) *database {
    c := database{storage: storage}

    return &c
}

// Returns the client for a specific name
//
// Loads the ltpk from disk and returns initialized client object
func (m *database) ClientWithName(name string) (Client) {
    data, err := m.storage.Get(name + ".ltpk")

    if len(data) > 0 && err == nil{
        client := NewClient(name, data)
        return client
    }

    return nil
}

// Stores the long-term public key of the client as {client-name}.ltpk
func (m *database) SaveClient(client Client) error {
    if len(client.PublicKey()) == 0 {
        return common.NewErrorf("No public key to save for client%s\n", client.Name())
    }

    return m.storage.Set(client.Name() + ".ltpk", client.PublicKey())
}

func (m *database) DeleteClient(client Client) {
    m.storage.Delete(client.Name() + ".ltpk")
}<commit_msg>Database refactoring<commit_after>package db

import(
    "github.com/brutella/hap"
    "github.com/brutella/hap/common"
)

type Database interface {
    ClientWithName(name string) Client
    SaveClient(client Client) error
    DeleteClient(client Client)
}

type database struct {
    storage hap.Storage
}

func NewDatabase(path string) (*database, error) {
    storage, err := common.NewFileStorage(path)
    if err != nil {
        return nil, err
    }

    return NewDatabaseWithStorage(storage), nil
}

func NewDatabaseWithStorage(storage hap.Storage) *database {
    c := database{storage: storage}

    return &c
}

// Returns the client for a specific name
//
// Loads the ltpk from disk and returns initialized client object
func (m *database) ClientWithName(name string) (Client) {
    data, err := m.storage.Get(keyForClientName(name))

    if len(data) > 0 && err == nil{
        client := NewClient(name, data)
        return client
    }

    return nil
}

// Stores the long-term public key of the client as {client-name}.ltpk
func (m *database) SaveClient(client Client) error {
    if len(client.PublicKey()) == 0 {
        return common.NewErrorf("No public key to save for client%s\n", client.Name())
    }

    return m.storage.Set(keyForClient(client), client.PublicKey())
}

// Returns they key for which the client is saved in the database
func (m *database) DeleteClient(client Client) {
    m.storage.Delete(keyForClient(client))
}

func keyForClientName(name string) string {
    return name + ".ltpk"
}

func keyForClient(client Client) string {
    return keyForClientName(client.Name())
}<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"github.com/open-falcon/common/model"
	"github.com/toolkits/container/set"
	"log"
	"strings"
	"time"
)

// 获取所有的Strategy列表
func QueryStrategies(tpls map[int]*model.Template) (map[int]*model.Strategy, error) {
	ret := make(map[int]*model.Strategy)

	if tpls == nil || len(tpls) == 0 {
		return ret, fmt.Errorf("illegal argument")
	}

	now := time.Now().Format("15:04")
	sql := fmt.Sprintf(
		"select %s from strategy as s where (s.run_begin='' and s.run_end='') or (s.run_begin <= '%s' and s.run_end > '%s')",
		"s.id, s.metric, s.tags, s.func, s.op, s.right_value, s.max_step, s.priority, s.note, s.tpl_id",
		now,
		now,
	)

	rows, err := DB.Query(sql)
	if err != nil {
		log.Println("ERROR:", err)
		return ret, err
	}

	defer rows.Close()
	for rows.Next() {
		s := model.Strategy{}
		var tags string
		var tid int
		err = rows.Scan(&s.Id, &s.Metric, &tags, &s.Func, &s.Operator, &s.RightValue, &s.MaxStep, &s.Priority, &s.Note, &tid)
		if err != nil {
			log.Println("ERROR:", err)
			continue
		}

		tt := make(map[string]string)

		if tags != "" {
			arr := strings.Split(tags, ",")
			for _, tag := range arr {
				kv := strings.Split(tag, "=")
				if len(kv) != 2 {
					continue
				}
				tt[kv[0]] = kv[1]
			}
		}

		s.Tags = tt
		s.Tpl = tpls[tid]
		if s.Tpl == nil {
			log.Printf("WARN: tpl is nil. strategy id=%d, tpl id=%d", s.Id, tid)
			// 如果Strategy没有对应的Tpl，那就没有action，就没法报警，无需往后传递了
			continue
		}

		ret[s.Id] = &s
	}

	return ret, nil
}

func QueryBuiltinMetrics(tids string) ([]*model.BuiltinMetric, error) {
	sql := fmt.Sprintf(
		"select metric, tags from strategy where tpl_id in (%s) and metric in ('net.port.listen', 'proc.num', 'du.bs', 'url.check.health')",
		tids,
	)

	ret := []*model.BuiltinMetric{}

	rows, err := DB.Query(sql)
	if err != nil {
		log.Println("ERROR:", err)
		return ret, err
	}

	metricTagsSet := set.NewStringSet()

	defer rows.Close()
	for rows.Next() {
		builtinMetric := model.BuiltinMetric{}
		err = rows.Scan(&builtinMetric.Metric, &builtinMetric.Tags)
		if err != nil {
			log.Println("WARN:", err)
			continue
		}

		k := fmt.Sprintf("%s%s", builtinMetric.Metric, builtinMetric.Tags)
		if metricTagsSet.Exists(k) {
			continue
		}

		ret = append(ret, &builtinMetric)
		metricTagsSet.Add(k)
	}

	return ret, nil
}
<commit_msg>fix bug:当模板中策略的tag包含大于1个等号时,误告警<commit_after>package db

import (
	"fmt"
	"github.com/open-falcon/common/model"
	"github.com/toolkits/container/set"
	"log"
	"strings"
	"time"
)

// 获取所有的Strategy列表
func QueryStrategies(tpls map[int]*model.Template) (map[int]*model.Strategy, error) {
	ret := make(map[int]*model.Strategy)

	if tpls == nil || len(tpls) == 0 {
		return ret, fmt.Errorf("illegal argument")
	}

	now := time.Now().Format("15:04")
	sql := fmt.Sprintf(
		"select %s from strategy as s where (s.run_begin='' and s.run_end='') or (s.run_begin <= '%s' and s.run_end > '%s')",
		"s.id, s.metric, s.tags, s.func, s.op, s.right_value, s.max_step, s.priority, s.note, s.tpl_id",
		now,
		now,
	)

	rows, err := DB.Query(sql)
	if err != nil {
		log.Println("ERROR:", err)
		return ret, err
	}

	defer rows.Close()
	for rows.Next() {
		s := model.Strategy{}
		var tags string
		var tid int
		err = rows.Scan(&s.Id, &s.Metric, &tags, &s.Func, &s.Operator, &s.RightValue, &s.MaxStep, &s.Priority, &s.Note, &tid)
		if err != nil {
			log.Println("ERROR:", err)
			continue
		}

		tt := make(map[string]string)

		if tags != "" {
			arr := strings.Split(tags, ",")
			for _, tag := range arr {
				kv := strings.SplitN(tag, "=", 2)
				if len(kv) != 2 {
					continue
				}
				tt[kv[0]] = kv[1]
			}
		}

		s.Tags = tt
		s.Tpl = tpls[tid]
		if s.Tpl == nil {
			log.Printf("WARN: tpl is nil. strategy id=%d, tpl id=%d", s.Id, tid)
			// 如果Strategy没有对应的Tpl，那就没有action，就没法报警，无需往后传递了
			continue
		}

		ret[s.Id] = &s
	}

	return ret, nil
}

func QueryBuiltinMetrics(tids string) ([]*model.BuiltinMetric, error) {
	sql := fmt.Sprintf(
		"select metric, tags from strategy where tpl_id in (%s) and metric in ('net.port.listen', 'proc.num', 'du.bs', 'url.check.health')",
		tids,
	)

	ret := []*model.BuiltinMetric{}

	rows, err := DB.Query(sql)
	if err != nil {
		log.Println("ERROR:", err)
		return ret, err
	}

	metricTagsSet := set.NewStringSet()

	defer rows.Close()
	for rows.Next() {
		builtinMetric := model.BuiltinMetric{}
		err = rows.Scan(&builtinMetric.Metric, &builtinMetric.Tags)
		if err != nil {
			log.Println("WARN:", err)
			continue
		}

		k := fmt.Sprintf("%s%s", builtinMetric.Metric, builtinMetric.Tags)
		if metricTagsSet.Exists(k) {
			continue
		}

		ret = append(ret, &builtinMetric)
		metricTagsSet.Add(k)
	}

	return ret, nil
}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {
	fmt.Println("Hello, 世界")
	fmt.Println("Hola!")
	fmt.Println("halla")
	fmt.Println("nils var her")
}

 "Mikael la til denne teksten"
"JK surret her"
<commit_msg>eirik oppdaterte hello.go<commit_after>package main

import "fmt"

func main() {
	fmt.Println("Hello, 世界")
	fmt.Println("Hola!")
	fmt.Println("halla")
	fmt.Println("nils var her")
	fmt.Println("Eirik sier hei") //überfin kommentar
}

 "Mikael la til denne teksten"
"JK surret her"
<|endoftext|>