<commit_before>package test

import (
	"reflect"
	"strings"
	"testing"

	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestResource_basic(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required = "yep"
	required_map = {
	    key = "value"
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

// Targeted test in TestContext2Apply_ignoreChangesCreate
func TestResource_ignoreChangesRequired(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
        required = "yep"
	required_map = {
	    key = "value"
	}
        lifecycle {
                ignore_changes = ["required"]
        }
}
                               `),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

func TestResource_ignoreChangesEmpty(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required = "yep"
	required_map = {
	    key = "value"
	}
	optional_force_new = "one"
	lifecycle {
		ignore_changes = []
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required = "yep"
	required_map = {
	    key = "value"
	}
	optional_force_new = "two"
	lifecycle {
		ignore_changes = []
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

func TestResource_ignoreChangesForceNew(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required           = "yep"
	required_map = {
	    key = "value"
	}
	optional_force_new = "one"
	lifecycle {
		ignore_changes = ["optional_force_new"]
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required           = "yep"
	required_map = {
	    key = "value"
	}
	optional_force_new = "two"
	lifecycle {
		ignore_changes = ["optional_force_new"]
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

// Covers specific scenario in #6005, handled by normalizing boolean strings in
// helper/schema
func TestResource_ignoreChangesForceNewBoolean(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
  required           = "yep"
  required_map = {
    key = "value"
  }
  optional_force_new = "one"
  optional_bool      = true
  lifecycle {
    ignore_changes = ["optional_force_new"]
  }
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
  required           = "yep"
  required_map = {
    key = "value"
  }
  optional_force_new = "two"
  optional_bool      = true
  lifecycle {
    ignore_changes = ["optional_force_new"]
  }
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

func TestResource_ignoreChangesMap(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required           = "yep"
	required_map = {
	  key = "value"
	}
	optional_computed_map = {
		foo = "bar"
	}
	lifecycle {
		ignore_changes = ["optional_computed_map"]
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required           = "yep"
	required_map = {
	  key = "value"
	}
	optional_computed_map = {
		foo = "bar"
		no  = "update"
	}
	lifecycle {
		ignore_changes = ["optional_computed_map"]
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

func TestResource_ignoreChangesDependent(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	count = 2
	required = "yep"
	required_map = {
		key = "value"
	}

	optional_force_new = "one"
	lifecycle {
		ignore_changes = ["optional_force_new"]
	}
}
resource "test_resource" "bar" {
	count = 2
	required = "yep"
	required_map = {
		key = "value"
	}
	optional = "${element(test_resource.foo.*.id, count.index)}"
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	count = 2
	required = "yep"
	required_map = {
		key = "value"
	}

	optional_force_new = "two"
	lifecycle {
		ignore_changes = ["optional_force_new"]
	}
}
resource "test_resource" "bar" {
	count = 2
	required = "yep"
	required_map = {
		key = "value"
	}
	optional = "${element(test_resource.foo.*.id, count.index)}"
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

func TestResource_ignoreChangesStillReplaced(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
  required     = "yep"
  required_map = {
    key = "value"
  }
  optional_force_new = "one"
  optional_bool      = true
  lifecycle {
    ignore_changes = ["optional_bool"]
  }
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
  required     = "yep"
  required_map = {
    key = "value"
  }
  optional_force_new = "two"
  optional_bool      = false
  lifecycle {
    ignore_changes = ["optional_bool"]
  }
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

// Reproduces plan-time panic when the wrong type is interpolated in a list of
// maps.
// TODO: this should return a type error, rather than silently setting an empty
//       list
func TestResource_dataSourceListMapPanic(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
  required = "val"
  required_map = {x = "y"}
  list_of_map = "${var.maplist}"
}

variable "maplist" {
  type = "list"

  default = [
    {a = "b"}
  ]
}
				`),
				ExpectError: nil,
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

func TestResource_dataSourceIndexMapList(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
  required = "val"

  required_map = {
    x = "y"
  }

  list_of_map = [
    {
      a = "1"
      b = "2"
    },
    {
      c = "3"
      d = "4"
    },
  ]
}

output "map_from_list" {
  value = "${test_resource.foo.list_of_map[0]}"
}

output "value_from_map_from_list" {
  value = "${lookup(test_resource.foo.list_of_map[1], "d")}"
}
				`),
				ExpectError: nil,
				Check: func(s *terraform.State) error {
					root := s.ModuleByPath(addrs.RootModuleInstance)
					mapOut := root.Outputs["map_from_list"].Value
					expectedMapOut := map[string]interface{}{
						"a": "1",
						"b": "2",
					}

					valueOut := root.Outputs["value_from_map_from_list"].Value
					expectedValueOut := "4"

					if !reflect.DeepEqual(mapOut, expectedMapOut) {
						t.Fatalf("Expected: %#v\nGot: %#v", expectedMapOut, mapOut)
					}
					if !reflect.DeepEqual(valueOut, expectedValueOut) {
						t.Fatalf("Expected: %#v\nGot: %#v", valueOut, expectedValueOut)
					}
					return nil
				},
			},
		},
	})
}

func testAccCheckResourceDestroy(s *terraform.State) error {
	return nil
}

func TestResource_removeForceNew(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required           = "yep"
	required_map = {
	  key = "value"
	}
	optional_force_new = "here"
}
				`),
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required           = "yep"
	required_map = {
	  key = "value"
	}
}
				`),
			},
		},
	})
}
<commit_msg>add test for computed map value<commit_after>package test

import (
	"reflect"
	"strings"
	"testing"

	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestResource_basic(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required = "yep"
	required_map = {
	    key = "value"
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

// Targeted test in TestContext2Apply_ignoreChangesCreate
func TestResource_ignoreChangesRequired(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
        required = "yep"
	required_map = {
	    key = "value"
	}
        lifecycle {
                ignore_changes = ["required"]
        }
}
                               `),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

func TestResource_ignoreChangesEmpty(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required = "yep"
	required_map = {
	    key = "value"
	}
	optional_force_new = "one"
	lifecycle {
		ignore_changes = []
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required = "yep"
	required_map = {
	    key = "value"
	}
	optional_force_new = "two"
	lifecycle {
		ignore_changes = []
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

func TestResource_ignoreChangesForceNew(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required           = "yep"
	required_map = {
	    key = "value"
	}
	optional_force_new = "one"
	lifecycle {
		ignore_changes = ["optional_force_new"]
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required           = "yep"
	required_map = {
	    key = "value"
	}
	optional_force_new = "two"
	lifecycle {
		ignore_changes = ["optional_force_new"]
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

// Covers specific scenario in #6005, handled by normalizing boolean strings in
// helper/schema
func TestResource_ignoreChangesForceNewBoolean(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
  required           = "yep"
  required_map = {
    key = "value"
  }
  optional_force_new = "one"
  optional_bool      = true
  lifecycle {
    ignore_changes = ["optional_force_new"]
  }
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
  required           = "yep"
  required_map = {
    key = "value"
  }
  optional_force_new = "two"
  optional_bool      = true
  lifecycle {
    ignore_changes = ["optional_force_new"]
  }
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

func TestResource_ignoreChangesMap(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required           = "yep"
	required_map = {
	  key = "value"
	}
	optional_computed_map = {
		foo = "bar"
	}
	lifecycle {
		ignore_changes = ["optional_computed_map"]
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required           = "yep"
	required_map = {
	  key = "value"
	}
	optional_computed_map = {
		foo = "bar"
		no  = "update"
	}
	lifecycle {
		ignore_changes = ["optional_computed_map"]
	}
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

func TestResource_ignoreChangesDependent(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	count = 2
	required = "yep"
	required_map = {
		key = "value"
	}

	optional_force_new = "one"
	lifecycle {
		ignore_changes = ["optional_force_new"]
	}
}
resource "test_resource" "bar" {
	count = 2
	required = "yep"
	required_map = {
		key = "value"
	}
	optional = "${element(test_resource.foo.*.id, count.index)}"
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	count = 2
	required = "yep"
	required_map = {
		key = "value"
	}

	optional_force_new = "two"
	lifecycle {
		ignore_changes = ["optional_force_new"]
	}
}
resource "test_resource" "bar" {
	count = 2
	required = "yep"
	required_map = {
		key = "value"
	}
	optional = "${element(test_resource.foo.*.id, count.index)}"
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

func TestResource_ignoreChangesStillReplaced(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
  required     = "yep"
  required_map = {
    key = "value"
  }
  optional_force_new = "one"
  optional_bool      = true
  lifecycle {
    ignore_changes = ["optional_bool"]
  }
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
  required     = "yep"
  required_map = {
    key = "value"
  }
  optional_force_new = "two"
  optional_bool      = false
  lifecycle {
    ignore_changes = ["optional_bool"]
  }
}
				`),
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

// Reproduces plan-time panic when the wrong type is interpolated in a list of
// maps.
// TODO: this should return a type error, rather than silently setting an empty
//       list
func TestResource_dataSourceListMapPanic(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
  required = "val"
  required_map = {x = "y"}
  list_of_map = "${var.maplist}"
}

variable "maplist" {
  type = "list"

  default = [
    {a = "b"}
  ]
}
				`),
				ExpectError: nil,
				Check: func(s *terraform.State) error {
					return nil
				},
			},
		},
	})
}

func TestResource_dataSourceIndexMapList(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
  required = "val"

  required_map = {
    x = "y"
  }

  list_of_map = [
    {
      a = "1"
      b = "2"
    },
    {
      c = "3"
      d = "4"
    },
  ]
}

output "map_from_list" {
  value = "${test_resource.foo.list_of_map[0]}"
}

output "value_from_map_from_list" {
  value = "${lookup(test_resource.foo.list_of_map[1], "d")}"
}
				`),
				ExpectError: nil,
				Check: func(s *terraform.State) error {
					root := s.ModuleByPath(addrs.RootModuleInstance)
					mapOut := root.Outputs["map_from_list"].Value
					expectedMapOut := map[string]interface{}{
						"a": "1",
						"b": "2",
					}

					valueOut := root.Outputs["value_from_map_from_list"].Value
					expectedValueOut := "4"

					if !reflect.DeepEqual(mapOut, expectedMapOut) {
						t.Fatalf("Expected: %#v\nGot: %#v", expectedMapOut, mapOut)
					}
					if !reflect.DeepEqual(valueOut, expectedValueOut) {
						t.Fatalf("Expected: %#v\nGot: %#v", valueOut, expectedValueOut)
					}
					return nil
				},
			},
		},
	})
}

func testAccCheckResourceDestroy(s *terraform.State) error {
	return nil
}

func TestResource_removeForceNew(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required           = "yep"
	required_map = {
	  key = "value"
	}
	optional_force_new = "here"
}
				`),
			},
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required           = "yep"
	required_map = {
	  key = "value"
	}
}
				`),
			},
		},
	})
}

func TestResource_unknownFuncInMap(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckResourceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: strings.TrimSpace(`
resource "test_resource" "foo" {
	required           = "ok"
	required_map = {
	  key = "${uuid()}"
	}
}
				`),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}
<|endoftext|>


<commit_before>package supervisor

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/mackerelio/mackerel-agent/logging"
)

var logger = logging.GetLogger("supervisor")

type supervisor struct {
	prog string
	argv []string

	cmd     *exec.Cmd
	startAt time.Time
	mu      sync.RWMutex

	signaled   bool
	signaledMu sync.RWMutex

	hupped   bool
	huppedMu sync.RWMutex
}

// Supervise starts a child mackerel-agent process and supervises it.
// 'c' can be nil and it's typically nil. When you pass signal channel to this
// method, you have to close the channel to stop internal goroutine.
func Supervise(agentProg string, argv []string, c chan os.Signal) error {
	return (&supervisor{
		prog: agentProg,
		argv: argv,
	}).supervise(c)
}

func (sv *supervisor) setSignaled(signaled bool) {
	sv.signaledMu.Lock()
	defer sv.signaledMu.Unlock()
	sv.signaled = signaled
}

func (sv *supervisor) getSignaled() bool {
	sv.signaledMu.RLock()
	defer sv.signaledMu.RUnlock()
	return sv.signaled
}

func (sv *supervisor) setHupped(hupped bool) {
	sv.huppedMu.Lock()
	defer sv.huppedMu.Unlock()
	sv.hupped = hupped
}

func (sv *supervisor) getHupped() bool {
	sv.huppedMu.RLock()
	defer sv.huppedMu.RUnlock()
	return sv.hupped
}

func (sv *supervisor) getCmd() *exec.Cmd {
	sv.mu.RLock()
	defer sv.mu.RUnlock()
	return sv.cmd
}

func (sv *supervisor) getStartAt() time.Time {
	sv.mu.RLock()
	defer sv.mu.RUnlock()
	return sv.startAt
}

// If the child process dies within 30 seconds, it is regarded as launching failure
// and terminate the process without crash recovery
var spawnInterval = 30 * time.Second

func (sv *supervisor) launched() bool {
	return sv.getCmd().Process != nil && time.Now().After(sv.getStartAt().Add(spawnInterval))
}

func (sv *supervisor) buildCmd() *exec.Cmd {
	argv := append(sv.argv, "-child")
	cmd := exec.Command(sv.prog, argv...)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd
}

func (sv *supervisor) start() error {
	sv.mu.Lock()
	sv.setHupped(false)
	defer sv.mu.Unlock()
	sv.cmd = sv.buildCmd()
	sv.startAt = time.Now()
	return sv.cmd.Start()
}

func (sv *supervisor) stop(sig os.Signal) error {
	sv.setSignaled(true)
	return sv.getCmd().Process.Signal(sig)
}

func (sv *supervisor) configtest() error {
	argv := append([]string{"configtest"}, sv.argv...)
	cmd := exec.Command(sv.prog, argv...)
	buf := &bytes.Buffer{}
	cmd.Stderr = buf
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("configtest failed: %s", buf.String())
	}
	return nil
}

func (sv *supervisor) reload() error {
	err := sv.configtest()
	if err != nil {
		return err
	}
	sv.setHupped(true)
	return sv.getCmd().Process.Signal(syscall.SIGTERM)
}

func (sv *supervisor) wait() (err error) {
	for {
		err = sv.cmd.Wait()
		if sv.getSignaled() || (!sv.getHupped() && !sv.launched()) {
			break
		}
		if err != nil {
			logger.Warningf("mackerel-agent abnormally finished with following error and try to restart it: %s", err.Error())
		}
		if err = sv.start(); err != nil {
			break
		}
	}
	return
}

func (sv *supervisor) handleSignal(ch <-chan os.Signal) {
	for sig := range ch {
		if sig == syscall.SIGHUP {
			logger.Infof("receiving HUP, spawning a new mackerel-agent")
			err := sv.reload()
			if err != nil {
				logger.Warningf("failed to reload: %s", err.Error())
			}
		} else {
			sv.stop(sig)
		}
	}
}

func (sv *supervisor) supervise(c chan os.Signal) error {
	if err := sv.start(); err != nil {
		return err
	}
	if c == nil {
		c = make(chan os.Signal, 1)
	}
	defer close(c)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP)
	go sv.handleSignal(c)
	return sv.wait()
}
<commit_msg>fix comment of supervisor<commit_after>package supervisor

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/mackerelio/mackerel-agent/logging"
)

var logger = logging.GetLogger("supervisor")

type supervisor struct {
	prog string
	argv []string

	cmd     *exec.Cmd
	startAt time.Time
	mu      sync.RWMutex

	signaled   bool
	signaledMu sync.RWMutex

	hupped   bool
	huppedMu sync.RWMutex
}

// Supervise starts a child mackerel-agent process and supervises it.
// 'c' can be nil and it's typically nil. When you pass signal channel to this
// method, the channel will be closed internally.
func Supervise(agentProg string, argv []string, c chan os.Signal) error {
	return (&supervisor{
		prog: agentProg,
		argv: argv,
	}).supervise(c)
}

func (sv *supervisor) setSignaled(signaled bool) {
	sv.signaledMu.Lock()
	defer sv.signaledMu.Unlock()
	sv.signaled = signaled
}

func (sv *supervisor) getSignaled() bool {
	sv.signaledMu.RLock()
	defer sv.signaledMu.RUnlock()
	return sv.signaled
}

func (sv *supervisor) setHupped(hupped bool) {
	sv.huppedMu.Lock()
	defer sv.huppedMu.Unlock()
	sv.hupped = hupped
}

func (sv *supervisor) getHupped() bool {
	sv.huppedMu.RLock()
	defer sv.huppedMu.RUnlock()
	return sv.hupped
}

func (sv *supervisor) getCmd() *exec.Cmd {
	sv.mu.RLock()
	defer sv.mu.RUnlock()
	return sv.cmd
}

func (sv *supervisor) getStartAt() time.Time {
	sv.mu.RLock()
	defer sv.mu.RUnlock()
	return sv.startAt
}

// If the child process dies within 30 seconds, it is regarded as launching failure
// and terminate the process without crash recovery
var spawnInterval = 30 * time.Second

func (sv *supervisor) launched() bool {
	return sv.getCmd().Process != nil && time.Now().After(sv.getStartAt().Add(spawnInterval))
}

func (sv *supervisor) buildCmd() *exec.Cmd {
	argv := append(sv.argv, "-child")
	cmd := exec.Command(sv.prog, argv...)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd
}

func (sv *supervisor) start() error {
	sv.mu.Lock()
	sv.setHupped(false)
	defer sv.mu.Unlock()
	sv.cmd = sv.buildCmd()
	sv.startAt = time.Now()
	return sv.cmd.Start()
}

func (sv *supervisor) stop(sig os.Signal) error {
	sv.setSignaled(true)
	return sv.getCmd().Process.Signal(sig)
}

func (sv *supervisor) configtest() error {
	argv := append([]string{"configtest"}, sv.argv...)
	cmd := exec.Command(sv.prog, argv...)
	buf := &bytes.Buffer{}
	cmd.Stderr = buf
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("configtest failed: %s", buf.String())
	}
	return nil
}

func (sv *supervisor) reload() error {
	err := sv.configtest()
	if err != nil {
		return err
	}
	sv.setHupped(true)
	return sv.getCmd().Process.Signal(syscall.SIGTERM)
}

func (sv *supervisor) wait() (err error) {
	for {
		err = sv.cmd.Wait()
		if sv.getSignaled() || (!sv.getHupped() && !sv.launched()) {
			break
		}
		if err != nil {
			logger.Warningf("mackerel-agent abnormally finished with following error and try to restart it: %s", err.Error())
		}
		if err = sv.start(); err != nil {
			break
		}
	}
	return
}

func (sv *supervisor) handleSignal(ch <-chan os.Signal) {
	for sig := range ch {
		if sig == syscall.SIGHUP {
			logger.Infof("receiving HUP, spawning a new mackerel-agent")
			err := sv.reload()
			if err != nil {
				logger.Warningf("failed to reload: %s", err.Error())
			}
		} else {
			sv.stop(sig)
		}
	}
}

func (sv *supervisor) supervise(c chan os.Signal) error {
	if err := sv.start(); err != nil {
		return err
	}
	if c == nil {
		c = make(chan os.Signal, 1)
	}
	defer close(c)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP)
	go sv.handleSignal(c)
	return sv.wait()
}
<|endoftext|>


<commit_before>package fuse

import (
	"bufio"
	"exec"
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"testing"
	"time"
)

var CheckSuccess = fuse.CheckSuccess

type StatFs struct {
	fuse.DefaultFileSystem
	entries map[string]*os.FileInfo
	dirs    map[string][]fuse.DirEntry
}

func (me *StatFs) add(name string, fi os.FileInfo) {
	name = strings.TrimRight(name, "/")
	_, ok := me.entries[name]
	if ok {
		return
	}

	me.entries[name] = &fi
	if name == "/" || name == "" {
		return
	}

	dir, base := filepath.Split(name)
	dir = strings.TrimRight(dir, "/")
	me.dirs[dir] = append(me.dirs[dir], fuse.DirEntry{Name: base, Mode: fi.Mode})
	me.add(dir, os.FileInfo{Mode: fuse.S_IFDIR | 0755})
}

func (me *StatFs) GetAttr(name string, context *fuse.Context) (*os.FileInfo, fuse.Status) {
	e := me.entries[name]
	if e == nil {
		return nil, fuse.ENOENT
	}
	return e, fuse.OK
}

func (me *StatFs) OpenDir(name string, context *fuse.Context) (stream chan fuse.DirEntry, status fuse.Status) {
	entries := me.dirs[name]
	if entries == nil {
		return nil, fuse.ENOENT
	}
	stream = make(chan fuse.DirEntry, len(entries))
	for _, e := range entries {
		stream <- e
	}
	close(stream)
	return stream, fuse.OK
}

func NewStatFs() *StatFs {
	return &StatFs{
		entries: make(map[string]*os.FileInfo),
		dirs:    make(map[string][]fuse.DirEntry),
	}
}

func setupFs(fs fuse.FileSystem, opts *fuse.FileSystemOptions) (string, func()) {
	mountPoint := fuse.MakeTempDir()
	state, _, err := fuse.MountPathFileSystem(mountPoint, fs, opts)
	if err != nil {
		panic(fmt.Sprintf("cannot mount %v", err)) // ugh - benchmark has no error methods.
	}
	// state.Debug = true
	go state.Loop(false)

	return mountPoint, func() {
		err := state.Unmount()
		if err != nil {
			log.Println("error during unmount", err)
		} else {
			os.RemoveAll(mountPoint)
		}
	}
}

func TestNewStatFs(t *testing.T) {
	fs := NewStatFs()
	for _, n := range []string{
		"file.txt", "sub/dir/foo.txt",
		"sub/dir/bar.txt", "sub/marine.txt"} {
		fs.add(n, os.FileInfo{Mode: fuse.S_IFREG | 0644})
	}

	wd, clean := setupFs(fs, nil)
	defer clean()

	names, err := ioutil.ReadDir(wd)
	CheckSuccess(err)
	if len(names) != 2 {
		t.Error("readdir /", names)
	}

	fi, err := os.Lstat(wd + "/sub")
	CheckSuccess(err)
	if !fi.IsDirectory() {
		t.Error("mode", fi)
	}
	names, err = ioutil.ReadDir(wd + "/sub")
	CheckSuccess(err)
	if len(names) != 2 {
		t.Error("readdir /sub", names)
	}
	names, err = ioutil.ReadDir(wd + "/sub/dir")
	CheckSuccess(err)
	if len(names) != 2 {
		t.Error("readdir /sub/dir", names)
	}

	fi, err = os.Lstat(wd + "/sub/marine.txt")
	CheckSuccess(err)
	if !fi.IsRegular() {
		t.Error("mode", fi)
	}
}

func GetTestLines() []string {
	wd, _ := os.Getwd()
	// Names from OpenJDK 1.6
	fn := wd + "/testpaths.txt"

	f, err := os.Open(fn)
	CheckSuccess(err)

	defer f.Close()
	r := bufio.NewReader(f)

	l := []string{}
	for {
		line, _, err := r.ReadLine()
		if line == nil || err != nil {
			break
		}

		fn := string(line)
		l = append(l, fn)
	}
	return l
}

func BenchmarkThreadedStat(b *testing.B) {
	b.StopTimer()
	fs := NewStatFs()
	files := GetTestLines()
	for _, fn := range files {
		fs.add(fn, os.FileInfo{Mode: fuse.S_IFREG | 0644})
	}
	if len(files) == 0 {
		log.Fatal("no files added")
	}

	log.Printf("Read %d file names", len(files))

	ttl := 0.1
	opts := fuse.FileSystemOptions{
		EntryTimeout:    ttl,
		AttrTimeout:     ttl,
		NegativeTimeout: 0.0,
	}
	wd, clean := setupFs(fs, &opts)
	defer clean()

	for i, l := range files {
		files[i] = filepath.Join(wd, l)
	}

	log.Println("N = ", b.N)
	threads := runtime.GOMAXPROCS(0)
	results := TestingBOnePass(b, threads, ttl*1.2, files)
	AnalyzeBenchmarkRuns(results)
}

func TestingBOnePass(b *testing.B, threads int, sleepTime float64, files []string) (results []float64) {
	runs := b.N + 1
	for j := 0; j < runs; j++ {
		if j > 0 {
			b.StartTimer()
		}
		result := BulkStat(threads, files)
		if j > 0 {
			b.StopTimer()
			results = append(results, result)
		} else {
			fmt.Println("Ignoring first run to preheat caches.")
		}

		if j < runs-1 {
			fmt.Printf("Sleeping %.2f seconds\n", sleepTime)
			time.Sleep(int64(sleepTime * 1e9))
		}
	}
	return results
}

func BenchmarkCFuseThreadedStat(b *testing.B) {
	log.Println("benchmarking CFuse")

	lines := GetTestLines()
	unique := map[string]int{}
	for _, l := range lines {
		unique[l] = 1
		dir, _ := filepath.Split(l)
		for dir != "/" && dir != "" {
			unique[dir] = 1
			dir = filepath.Clean(dir)
			dir, _ = filepath.Split(dir)
		}
	}

	out := []string{}
	for k, _ := range unique {
		out = append(out, k)
	}

	f, err := ioutil.TempFile("", "")
	CheckSuccess(err)
	sort.Strings(out)
	for _, k := range out {
		f.Write([]byte(fmt.Sprintf("/%s\n", k)))
	}
	f.Close()

	log.Println("Written:", f.Name())
	mountPoint := fuse.MakeTempDir()
	wd, _ := os.Getwd()
	cmd := exec.Command(wd + "/cstatfs", mountPoint)
	cmd.Env = append(os.Environ(), fmt.Sprintf("STATFS_INPUT=%s", f.Name()))
	cmd.Start()

	bin, err := exec.LookPath("fusermount")
	CheckSuccess(err)
	stop := exec.Command(bin, "-u", mountPoint)
	CheckSuccess(err)
	defer stop.Run()

	for i, l := range lines {
		lines[i] = filepath.Join(mountPoint, l)
	}

	// Wait for the daemon to mount.
	time.Sleep(0.2e9)
	ttl := 1.0
	log.Println("N = ", b.N)
	threads := runtime.GOMAXPROCS(0)
	results := TestingBOnePass(b, threads, ttl*1.2, lines)
	AnalyzeBenchmarkRuns(results)
}
<commit_msg>Rename benchmark.<commit_after>package fuse

import (
	"bufio"
	"exec"
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"testing"
	"time"
)

var CheckSuccess = fuse.CheckSuccess

type StatFs struct {
	fuse.DefaultFileSystem
	entries map[string]*os.FileInfo
	dirs    map[string][]fuse.DirEntry
}

func (me *StatFs) add(name string, fi os.FileInfo) {
	name = strings.TrimRight(name, "/")
	_, ok := me.entries[name]
	if ok {
		return
	}

	me.entries[name] = &fi
	if name == "/" || name == "" {
		return
	}

	dir, base := filepath.Split(name)
	dir = strings.TrimRight(dir, "/")
	me.dirs[dir] = append(me.dirs[dir], fuse.DirEntry{Name: base, Mode: fi.Mode})
	me.add(dir, os.FileInfo{Mode: fuse.S_IFDIR | 0755})
}

func (me *StatFs) GetAttr(name string, context *fuse.Context) (*os.FileInfo, fuse.Status) {
	e := me.entries[name]
	if e == nil {
		return nil, fuse.ENOENT
	}
	return e, fuse.OK
}

func (me *StatFs) OpenDir(name string, context *fuse.Context) (stream chan fuse.DirEntry, status fuse.Status) {
	entries := me.dirs[name]
	if entries == nil {
		return nil, fuse.ENOENT
	}
	stream = make(chan fuse.DirEntry, len(entries))
	for _, e := range entries {
		stream <- e
	}
	close(stream)
	return stream, fuse.OK
}

func NewStatFs() *StatFs {
	return &StatFs{
		entries: make(map[string]*os.FileInfo),
		dirs:    make(map[string][]fuse.DirEntry),
	}
}

func setupFs(fs fuse.FileSystem, opts *fuse.FileSystemOptions) (string, func()) {
	mountPoint := fuse.MakeTempDir()
	state, _, err := fuse.MountPathFileSystem(mountPoint, fs, opts)
	if err != nil {
		panic(fmt.Sprintf("cannot mount %v", err)) // ugh - benchmark has no error methods.
	}
	// state.Debug = true
	go state.Loop(false)

	return mountPoint, func() {
		err := state.Unmount()
		if err != nil {
			log.Println("error during unmount", err)
		} else {
			os.RemoveAll(mountPoint)
		}
	}
}

func TestNewStatFs(t *testing.T) {
	fs := NewStatFs()
	for _, n := range []string{
		"file.txt", "sub/dir/foo.txt",
		"sub/dir/bar.txt", "sub/marine.txt"} {
		fs.add(n, os.FileInfo{Mode: fuse.S_IFREG | 0644})
	}

	wd, clean := setupFs(fs, nil)
	defer clean()

	names, err := ioutil.ReadDir(wd)
	CheckSuccess(err)
	if len(names) != 2 {
		t.Error("readdir /", names)
	}

	fi, err := os.Lstat(wd + "/sub")
	CheckSuccess(err)
	if !fi.IsDirectory() {
		t.Error("mode", fi)
	}
	names, err = ioutil.ReadDir(wd + "/sub")
	CheckSuccess(err)
	if len(names) != 2 {
		t.Error("readdir /sub", names)
	}
	names, err = ioutil.ReadDir(wd + "/sub/dir")
	CheckSuccess(err)
	if len(names) != 2 {
		t.Error("readdir /sub/dir", names)
	}

	fi, err = os.Lstat(wd + "/sub/marine.txt")
	CheckSuccess(err)
	if !fi.IsRegular() {
		t.Error("mode", fi)
	}
}

func GetTestLines() []string {
	wd, _ := os.Getwd()
	// Names from OpenJDK 1.6
	fn := wd + "/testpaths.txt"

	f, err := os.Open(fn)
	CheckSuccess(err)

	defer f.Close()
	r := bufio.NewReader(f)

	l := []string{}
	for {
		line, _, err := r.ReadLine()
		if line == nil || err != nil {
			break
		}

		fn := string(line)
		l = append(l, fn)
	}
	return l
}

func BenchmarkGoFuseThreadedStat(b *testing.B) {
	b.StopTimer()
	fs := NewStatFs()
	files := GetTestLines()
	for _, fn := range files {
		fs.add(fn, os.FileInfo{Mode: fuse.S_IFREG | 0644})
	}
	if len(files) == 0 {
		log.Fatal("no files added")
	}

	log.Printf("Read %d file names", len(files))

	ttl := 0.1
	opts := fuse.FileSystemOptions{
		EntryTimeout:    ttl,
		AttrTimeout:     ttl,
		NegativeTimeout: 0.0,
	}
	wd, clean := setupFs(fs, &opts)
	defer clean()

	for i, l := range files {
		files[i] = filepath.Join(wd, l)
	}

	log.Println("N = ", b.N)
	threads := runtime.GOMAXPROCS(0)
	results := TestingBOnePass(b, threads, ttl*1.2, files)
	AnalyzeBenchmarkRuns(results)
}

func TestingBOnePass(b *testing.B, threads int, sleepTime float64, files []string) (results []float64) {
	runs := b.N + 1
	for j := 0; j < runs; j++ {
		if j > 0 {
			b.StartTimer()
		}
		result := BulkStat(threads, files)
		if j > 0 {
			b.StopTimer()
			results = append(results, result)
		} else {
			fmt.Println("Ignoring first run to preheat caches.")
		}

		if j < runs-1 {
			fmt.Printf("Sleeping %.2f seconds\n", sleepTime)
			time.Sleep(int64(sleepTime * 1e9))
		}
	}
	return results
}

func BenchmarkCFuseThreadedStat(b *testing.B) {
	log.Println("benchmarking CFuse")

	lines := GetTestLines()
	unique := map[string]int{}
	for _, l := range lines {
		unique[l] = 1
		dir, _ := filepath.Split(l)
		for dir != "/" && dir != "" {
			unique[dir] = 1
			dir = filepath.Clean(dir)
			dir, _ = filepath.Split(dir)
		}
	}

	out := []string{}
	for k, _ := range unique {
		out = append(out, k)
	}

	f, err := ioutil.TempFile("", "")
	CheckSuccess(err)
	sort.Strings(out)
	for _, k := range out {
		f.Write([]byte(fmt.Sprintf("/%s\n", k)))
	}
	f.Close()

	log.Println("Written:", f.Name())
	mountPoint := fuse.MakeTempDir()
	wd, _ := os.Getwd()
	cmd := exec.Command(wd + "/cstatfs", mountPoint)
	cmd.Env = append(os.Environ(), fmt.Sprintf("STATFS_INPUT=%s", f.Name()))
	cmd.Start()

	bin, err := exec.LookPath("fusermount")
	CheckSuccess(err)
	stop := exec.Command(bin, "-u", mountPoint)
	CheckSuccess(err)
	defer stop.Run()

	for i, l := range lines {
		lines[i] = filepath.Join(mountPoint, l)
	}

	// Wait for the daemon to mount.
	time.Sleep(0.2e9)
	ttl := 1.0
	log.Println("N = ", b.N)
	threads := runtime.GOMAXPROCS(0)
	results := TestingBOnePass(b, threads, ttl*1.2, lines)
	AnalyzeBenchmarkRuns(results)
}

<|endoftext|>


<commit_before>// Copyright 2015 Eleme Inc. All rights reserved.

// Package alerter implements an alerter to send sms/email messages
// on anomalies found.
package alerter

import (
	"encoding/json"
	"os/exec"
	"sync/atomic"
	"time"

	"github.com/eleme/banshee/config"
	"github.com/eleme/banshee/models"
	"github.com/eleme/banshee/storage"
	"github.com/eleme/banshee/util/log"
	"github.com/eleme/banshee/util/safemap"
)

// Limit for buffered detected metric results, further results will be dropped
// if this limit is reached.
const bufferedMetricResultsLimit = 10 * 1024

// Alerter alerts on anomalies detected.
type Alerter struct {
	// Storage
	db *storage.DB
	// Config
	cfg *config.Config
	// Input
	In chan *models.Metric
	// Alertings stamps
	m *safemap.SafeMap
	// Alertings counters
	c *safemap.SafeMap
}

// Alerting message.
type msg struct {
	Project *models.Project `json:"project"`
	Metric  *models.Metric  `json:"metric"`
	User    *models.User    `json:"user"`
}

// New creates a alerter.
func New(cfg *config.Config, db *storage.DB) *Alerter {
	al := new(Alerter)
	al.cfg = cfg
	al.db = db
	al.In = make(chan *models.Metric, bufferedMetricResultsLimit)
	al.m = safemap.New()
	al.c = safemap.New()
	return al
}

// Start several goroutines to wait for detected metrics, then check each
// metric with all the rules, the configured shell command will be executed
// once a rule is hit.
func (al *Alerter) Start() {
	log.Info("start %d alerter workers..", al.cfg.Alerter.Workers)
	for i := 0; i < al.cfg.Alerter.Workers; i++ {
		go al.work()
	}
	go func() {
		ticker := time.NewTicker(time.Hour * 24)
		for _ = range ticker.C {
			al.c.Clear()
		}
	}()
}

// work waits for detected metrics, then check each metric with all the
// rules, the configured shell command will be executed once a rule is hit.
func (al *Alerter) work() {
	for {
		metric := <-al.In
		// Check interval.
		v, ok := al.m.Get(metric.Name)
		if ok && metric.Stamp-v.(uint32) < al.cfg.Alerter.Interval {
			return
		}
		// Check alert times in one day
		v, ok = al.c.Get(metric.Name)
		if ok && atomic.LoadUint32(v.(*uint32)) > al.cfg.Alerter.OneDayLimit {
			return
		}
		if !ok {
			var newCounter uint32
			newCounter = 1
			al.c.Set(metric.Name, &newCounter)
		} else {
			atomic.AddUint32(v.(*uint32), 1)
		}
		// Universals
		var univs []models.User
		if err := al.db.Admin.DB().Where("universal = ?", true).Find(&univs).Error; err != nil {
			log.Error("get universal users: %v, skiping..", err)
			continue
		}
		for _, rule := range metric.TestedRules {
			// Project
			proj := &models.Project{}
			if err := al.db.Admin.DB().Model(rule).Related(proj).Error; err != nil {
				log.Error("project not found, %v, skiping..", err)
				continue
			}
			// Users
			var users []models.User
			if err := al.db.Admin.DB().Model(proj).Related(&users, "Users").Error; err != nil {
				log.Error("get users: %v, skiping..", err)
				continue
			}
			users = append(users, univs...)
			// Send
			for _, user := range users {
				d := &msg{
					Project: proj,
					Metric:  metric,
					User:    &user,
				}
				// Exec
				if len(al.cfg.Alerter.Command) == 0 {
					log.Warn("alert command not configured")
					continue
				}
				b, _ := json.Marshal(d)
				cmd := exec.Command(al.cfg.Alerter.Command, string(b))
				if err := cmd.Run(); err != nil {
					log.Error("exec %s: %v", al.cfg.Alerter.Command, err)
				}
			}
			if len(users) != 0 {
				al.m.Set(metric.Name, metric.Stamp)
			}
		}
	}
}
<commit_msg>Log warn if metric hit one day limit<commit_after>// Copyright 2015 Eleme Inc. All rights reserved.

// Package alerter implements an alerter to send sms/email messages
// on anomalies found.
package alerter

import (
	"encoding/json"
	"os/exec"
	"sync/atomic"
	"time"

	"github.com/eleme/banshee/config"
	"github.com/eleme/banshee/models"
	"github.com/eleme/banshee/storage"
	"github.com/eleme/banshee/util/log"
	"github.com/eleme/banshee/util/safemap"
)

// Limit for buffered detected metric results, further results will be dropped
// if this limit is reached.
const bufferedMetricResultsLimit = 10 * 1024

// Alerter alerts on anomalies detected.
type Alerter struct {
	// Storage
	db *storage.DB
	// Config
	cfg *config.Config
	// Input
	In chan *models.Metric
	// Alertings stamps
	m *safemap.SafeMap
	// Alertings counters
	c *safemap.SafeMap
}

// Alerting message.
type msg struct {
	Project *models.Project `json:"project"`
	Metric  *models.Metric  `json:"metric"`
	User    *models.User    `json:"user"`
}

// New creates a alerter.
func New(cfg *config.Config, db *storage.DB) *Alerter {
	al := new(Alerter)
	al.cfg = cfg
	al.db = db
	al.In = make(chan *models.Metric, bufferedMetricResultsLimit)
	al.m = safemap.New()
	al.c = safemap.New()
	return al
}

// Start several goroutines to wait for detected metrics, then check each
// metric with all the rules, the configured shell command will be executed
// once a rule is hit.
func (al *Alerter) Start() {
	log.Info("start %d alerter workers..", al.cfg.Alerter.Workers)
	for i := 0; i < al.cfg.Alerter.Workers; i++ {
		go al.work()
	}
	go func() {
		ticker := time.NewTicker(time.Hour * 24)
		for _ = range ticker.C {
			al.c.Clear()
		}
	}()
}

// work waits for detected metrics, then check each metric with all the
// rules, the configured shell command will be executed once a rule is hit.
func (al *Alerter) work() {
	for {
		metric := <-al.In
		// Check interval.
		v, ok := al.m.Get(metric.Name)
		if ok && metric.Stamp-v.(uint32) < al.cfg.Alerter.Interval {
			return
		}
		// Check alert times in one day
		v, ok = al.c.Get(metric.Name)
		if ok && atomic.LoadUint32(v.(*uint32)) > al.cfg.Alerter.OneDayLimit {
			log.Warn("%s hit alerting one day limit, skipping..", metric.Name)
			return
		}
		if !ok {
			var newCounter uint32
			newCounter = 1
			al.c.Set(metric.Name, &newCounter)
		} else {
			atomic.AddUint32(v.(*uint32), 1)
		}
		// Universals
		var univs []models.User
		if err := al.db.Admin.DB().Where("universal = ?", true).Find(&univs).Error; err != nil {
			log.Error("get universal users: %v, skiping..", err)
			continue
		}
		for _, rule := range metric.TestedRules {
			// Project
			proj := &models.Project{}
			if err := al.db.Admin.DB().Model(rule).Related(proj).Error; err != nil {
				log.Error("project not found, %v, skiping..", err)
				continue
			}
			// Users
			var users []models.User
			if err := al.db.Admin.DB().Model(proj).Related(&users, "Users").Error; err != nil {
				log.Error("get users: %v, skiping..", err)
				continue
			}
			users = append(users, univs...)
			// Send
			for _, user := range users {
				d := &msg{
					Project: proj,
					Metric:  metric,
					User:    &user,
				}
				// Exec
				if len(al.cfg.Alerter.Command) == 0 {
					log.Warn("alert command not configured")
					continue
				}
				b, _ := json.Marshal(d)
				cmd := exec.Command(al.cfg.Alerter.Command, string(b))
				if err := cmd.Run(); err != nil {
					log.Error("exec %s: %v", al.cfg.Alerter.Command, err)
				}
			}
			if len(users) != 0 {
				al.m.Set(metric.Name, metric.Stamp)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Google LLC.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package option contains options for Google API clients.
package option

import (
	"crypto/tls"
	"net/http"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/internal"
	"google.golang.org/api/internal/impersonate"
	"google.golang.org/grpc"
)

// A ClientOption is an option for a Google API client.
type ClientOption interface {
	Apply(*internal.DialSettings)
}

// WithTokenSource returns a ClientOption that specifies an OAuth2 token
// source to be used as the basis for authentication.
func WithTokenSource(s oauth2.TokenSource) ClientOption {
	return withTokenSource{s}
}

type withTokenSource struct{ ts oauth2.TokenSource }

func (w withTokenSource) Apply(o *internal.DialSettings) {
	o.TokenSource = w.ts
}

type withCredFile string

func (w withCredFile) Apply(o *internal.DialSettings) {
	o.CredentialsFile = string(w)
}

// WithCredentialsFile returns a ClientOption that authenticates
// API calls with the given service account or refresh token JSON
// credentials file.
func WithCredentialsFile(filename string) ClientOption {
	return withCredFile(filename)
}

// WithServiceAccountFile returns a ClientOption that uses a Google service
// account credentials file to authenticate.
//
// Deprecated: Use WithCredentialsFile instead.
func WithServiceAccountFile(filename string) ClientOption {
	return WithCredentialsFile(filename)
}

// WithCredentialsJSON returns a ClientOption that authenticates
// API calls with the given service account or refresh token JSON
// credentials.
func WithCredentialsJSON(p []byte) ClientOption {
	return withCredentialsJSON(p)
}

type withCredentialsJSON []byte

func (w withCredentialsJSON) Apply(o *internal.DialSettings) {
	o.CredentialsJSON = make([]byte, len(w))
	copy(o.CredentialsJSON, w)
}

// WithEndpoint returns a ClientOption that overrides the default endpoint
// to be used for a service.
func WithEndpoint(url string) ClientOption {
	return withEndpoint(url)
}

type withEndpoint string

func (w withEndpoint) Apply(o *internal.DialSettings) {
	o.Endpoint = string(w)
}

// WithScopes returns a ClientOption that overrides the default OAuth2 scopes
// to be used for a service.
//
// If both WithScopes and WithTokenSource are used, scope settings from the
// token source will be used instead.
func WithScopes(scope ...string) ClientOption {
	return withScopes(scope)
}

type withScopes []string

func (w withScopes) Apply(o *internal.DialSettings) {
	o.Scopes = make([]string, len(w))
	copy(o.Scopes, w)
}

// WithUserAgent returns a ClientOption that sets the User-Agent.
func WithUserAgent(ua string) ClientOption {
	return withUA(ua)
}

type withUA string

func (w withUA) Apply(o *internal.DialSettings) { o.UserAgent = string(w) }

// WithHTTPClient returns a ClientOption that specifies the HTTP client to use
// as the basis of communications. This option may only be used with services
// that support HTTP as their communication transport. When used, the
// WithHTTPClient option takes precedent over all other supplied options.
func WithHTTPClient(client *http.Client) ClientOption {
	return withHTTPClient{client}
}

type withHTTPClient struct{ client *http.Client }

func (w withHTTPClient) Apply(o *internal.DialSettings) {
	o.HTTPClient = w.client
}

// WithGRPCConn returns a ClientOption that specifies the gRPC client
// connection to use as the basis of communications. This option may only be
// used with services that support gRPC as their communication transport. When
// used, the WithGRPCConn option takes precedent over all other supplied
// options.
func WithGRPCConn(conn *grpc.ClientConn) ClientOption {
	return withGRPCConn{conn}
}

type withGRPCConn struct{ conn *grpc.ClientConn }

func (w withGRPCConn) Apply(o *internal.DialSettings) {
	o.GRPCConn = w.conn
}

// WithGRPCDialOption returns a ClientOption that appends a new grpc.DialOption
// to an underlying gRPC dial. It does not work with WithGRPCConn.
func WithGRPCDialOption(opt grpc.DialOption) ClientOption {
	return withGRPCDialOption{opt}
}

type withGRPCDialOption struct{ opt grpc.DialOption }

func (w withGRPCDialOption) Apply(o *internal.DialSettings) {
	o.GRPCDialOpts = append(o.GRPCDialOpts, w.opt)
}

// WithGRPCConnectionPool returns a ClientOption that creates a pool of gRPC
// connections that requests will be balanced between.
func WithGRPCConnectionPool(size int) ClientOption {
	return withGRPCConnectionPool(size)
}

type withGRPCConnectionPool int

func (w withGRPCConnectionPool) Apply(o *internal.DialSettings) {
	o.GRPCConnPoolSize = int(w)
}

// WithAPIKey returns a ClientOption that specifies an API key to be used
// as the basis for authentication.
//
// API Keys can only be used for JSON-over-HTTP APIs, including those under
// the import path google.golang.org/api/....
func WithAPIKey(apiKey string) ClientOption {
	return withAPIKey(apiKey)
}

type withAPIKey string

func (w withAPIKey) Apply(o *internal.DialSettings) { o.APIKey = string(w) }

// WithAudiences returns a ClientOption that specifies an audience to be used
// as the audience field ("aud") for the JWT token authentication.
func WithAudiences(audience ...string) ClientOption {
	return withAudiences(audience)
}

type withAudiences []string

func (w withAudiences) Apply(o *internal.DialSettings) {
	o.Audiences = make([]string, len(w))
	copy(o.Audiences, w)
}

// WithoutAuthentication returns a ClientOption that specifies that no
// authentication should be used. It is suitable only for testing and for
// accessing public resources, like public Google Cloud Storage buckets.
// It is an error to provide both WithoutAuthentication and any of WithAPIKey,
// WithTokenSource, WithCredentialsFile or WithServiceAccountFile.
func WithoutAuthentication() ClientOption {
	return withoutAuthentication{}
}

type withoutAuthentication struct{}

func (w withoutAuthentication) Apply(o *internal.DialSettings) { o.NoAuth = true }

// WithQuotaProject returns a ClientOption that specifies the project used
// for quota and billing purposes.
//
// For more information please read:
// https://cloud.google.com/apis/docs/system-parameters
func WithQuotaProject(quotaProject string) ClientOption {
	return withQuotaProject(quotaProject)
}

type withQuotaProject string

func (w withQuotaProject) Apply(o *internal.DialSettings) {
	o.QuotaProject = string(w)
}

// WithRequestReason returns a ClientOption that specifies a reason for
// making the request, which is intended to be recorded in audit logging.
// An example reason would be a support-case ticket number.
//
// For more information please read:
// https://cloud.google.com/apis/docs/system-parameters
func WithRequestReason(requestReason string) ClientOption {
	return withRequestReason(requestReason)
}

type withRequestReason string

func (w withRequestReason) Apply(o *internal.DialSettings) {
	o.RequestReason = string(w)
}

// WithTelemetryDisabled returns a ClientOption that disables default telemetry (OpenCensus)
// settings on gRPC and HTTP clients.
// An example reason would be to bind custom telemetry that overrides the defaults.
func WithTelemetryDisabled() ClientOption {
	return withTelemetryDisabled{}
}

type withTelemetryDisabled struct{}

func (w withTelemetryDisabled) Apply(o *internal.DialSettings) {
	o.TelemetryDisabled = true
}

// ClientCertSource is a function that returns a TLS client certificate to be used
// when opening TLS connections.
//
// It follows the same semantics as crypto/tls.Config.GetClientCertificate.
//
// This is an EXPERIMENTAL API and may be changed or removed in the future.
type ClientCertSource = func(*tls.CertificateRequestInfo) (*tls.Certificate, error)

// WithClientCertSource returns a ClientOption that specifies a
// callback function for obtaining a TLS client certificate.
//
// This option is used for supporting mTLS authentication, where the
// server validates the client certifcate when establishing a connection.
//
// The callback function will be invoked whenever the server requests a
// certificate from the client. Implementations of the callback function
// should try to ensure that a valid certificate can be repeatedly returned
// on demand for the entire life cycle of the transport client. If a nil
// Certificate is returned (i.e. no Certificate can be obtained), an error
// should be returned.
//
// This is an EXPERIMENTAL API and may be changed or removed in the future.
func WithClientCertSource(s ClientCertSource) ClientOption {
	return withClientCertSource{s}
}

type withClientCertSource struct{ s ClientCertSource }

func (w withClientCertSource) Apply(o *internal.DialSettings) {
	o.ClientCertSource = w.s
}

// ImpersonateCredentials returns a ClientOption that will impersonate the
// target service account.
//
// In order to impersonate the target service account
// the base service account must have the Service Account Token Creator role,
// roles/iam.serviceAccountTokenCreator, on the target service account.
// See https://cloud.google.com/iam/docs/understanding-service-accounts.
//
// Optionally, delegates can be used during impersonation if the base service
// account lacks the token creator role on the target. When using delegates,
// each service account must be granted roles/iam.serviceAccountTokenCreator
// on the next service account in the chain.
//
// For example, if a base service account of SA1 is trying to impersonate target
// service account SA2 while using delegate service accounts DSA1 and DSA2,
// the following must be true:
//
//  1. Base service account SA1 has roles/iam.serviceAccountTokenCreator on
//     DSA1.
//  2. DSA1 has roles/iam.serviceAccountTokenCreator on DSA2.
//  3. DSA2 has roles/iam.serviceAccountTokenCreator on target SA2.
//
// The resulting impersonated credential will either have the default scopes of
// the client being instantiating or the scopes from WithScopes if provided.
// Scopes are required for creating impersonated credentials, so if this option
// is used while not using a NewClient/NewService function, WithScopes must also
// be explicitly passed in as well.
//
// If the base credential is an authorized user and not a service account, or if
// the option WithQuotaProject is set, the target service account must have a
// role that grants the serviceusage.services.use permission such as
// roles/serviceusage.serviceUsageConsumer.
//
// This is an EXPERIMENTAL API and may be changed or removed in the future.
//
// Deprecated: This option has been replaced by `impersonate` package:
// `google.golang.org/api/impersonate`. Please use the `impersonate` package
// instead with the WithTokenSource option.
func ImpersonateCredentials(target string, delegates ...string) ClientOption {
	return impersonateServiceAccount{
		target:    target,
		delegates: delegates,
	}
}

type impersonateServiceAccount struct {
	target    string
	delegates []string
}

func (i impersonateServiceAccount) Apply(o *internal.DialSettings) {
	o.ImpersonationConfig = &impersonate.Config{
		Target: i.target,
	}
	o.ImpersonationConfig.Delegates = make([]string, len(i.delegates))
	copy(o.ImpersonationConfig.Delegates, i.delegates)
}

type withCreds google.Credentials

func (w *withCreds) Apply(o *internal.DialSettings) {
	o.Credentials = (*google.Credentials)(w)
}

// WithCredentials returns a ClientOption that authenticates API calls.
func WithCredentials(creds *google.Credentials) ClientOption {
	return (*withCreds)(creds)
}
<commit_msg>docs: document limitation of WithUserAgent (#1747)<commit_after>// Copyright 2017 Google LLC.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package option contains options for Google API clients.
package option

import (
	"crypto/tls"
	"net/http"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/internal"
	"google.golang.org/api/internal/impersonate"
	"google.golang.org/grpc"
)

// A ClientOption is an option for a Google API client.
type ClientOption interface {
	Apply(*internal.DialSettings)
}

// WithTokenSource returns a ClientOption that specifies an OAuth2 token
// source to be used as the basis for authentication.
func WithTokenSource(s oauth2.TokenSource) ClientOption {
	return withTokenSource{s}
}

type withTokenSource struct{ ts oauth2.TokenSource }

func (w withTokenSource) Apply(o *internal.DialSettings) {
	o.TokenSource = w.ts
}

type withCredFile string

func (w withCredFile) Apply(o *internal.DialSettings) {
	o.CredentialsFile = string(w)
}

// WithCredentialsFile returns a ClientOption that authenticates
// API calls with the given service account or refresh token JSON
// credentials file.
func WithCredentialsFile(filename string) ClientOption {
	return withCredFile(filename)
}

// WithServiceAccountFile returns a ClientOption that uses a Google service
// account credentials file to authenticate.
//
// Deprecated: Use WithCredentialsFile instead.
func WithServiceAccountFile(filename string) ClientOption {
	return WithCredentialsFile(filename)
}

// WithCredentialsJSON returns a ClientOption that authenticates
// API calls with the given service account or refresh token JSON
// credentials.
func WithCredentialsJSON(p []byte) ClientOption {
	return withCredentialsJSON(p)
}

type withCredentialsJSON []byte

func (w withCredentialsJSON) Apply(o *internal.DialSettings) {
	o.CredentialsJSON = make([]byte, len(w))
	copy(o.CredentialsJSON, w)
}

// WithEndpoint returns a ClientOption that overrides the default endpoint
// to be used for a service.
func WithEndpoint(url string) ClientOption {
	return withEndpoint(url)
}

type withEndpoint string

func (w withEndpoint) Apply(o *internal.DialSettings) {
	o.Endpoint = string(w)
}

// WithScopes returns a ClientOption that overrides the default OAuth2 scopes
// to be used for a service.
//
// If both WithScopes and WithTokenSource are used, scope settings from the
// token source will be used instead.
func WithScopes(scope ...string) ClientOption {
	return withScopes(scope)
}

type withScopes []string

func (w withScopes) Apply(o *internal.DialSettings) {
	o.Scopes = make([]string, len(w))
	copy(o.Scopes, w)
}

// WithUserAgent returns a ClientOption that sets the User-Agent. This option
// is incompatible with the [WithHTTPClient] option. If you wish to provide a
// custom client you will need to add this header via RoundTripper middleware.
func WithUserAgent(ua string) ClientOption {
	return withUA(ua)
}

type withUA string

func (w withUA) Apply(o *internal.DialSettings) { o.UserAgent = string(w) }

// WithHTTPClient returns a ClientOption that specifies the HTTP client to use
// as the basis of communications. This option may only be used with services
// that support HTTP as their communication transport. When used, the
// WithHTTPClient option takes precedent over all other supplied options.
func WithHTTPClient(client *http.Client) ClientOption {
	return withHTTPClient{client}
}

type withHTTPClient struct{ client *http.Client }

func (w withHTTPClient) Apply(o *internal.DialSettings) {
	o.HTTPClient = w.client
}

// WithGRPCConn returns a ClientOption that specifies the gRPC client
// connection to use as the basis of communications. This option may only be
// used with services that support gRPC as their communication transport. When
// used, the WithGRPCConn option takes precedent over all other supplied
// options.
func WithGRPCConn(conn *grpc.ClientConn) ClientOption {
	return withGRPCConn{conn}
}

type withGRPCConn struct{ conn *grpc.ClientConn }

func (w withGRPCConn) Apply(o *internal.DialSettings) {
	o.GRPCConn = w.conn
}

// WithGRPCDialOption returns a ClientOption that appends a new grpc.DialOption
// to an underlying gRPC dial. It does not work with WithGRPCConn.
func WithGRPCDialOption(opt grpc.DialOption) ClientOption {
	return withGRPCDialOption{opt}
}

type withGRPCDialOption struct{ opt grpc.DialOption }

func (w withGRPCDialOption) Apply(o *internal.DialSettings) {
	o.GRPCDialOpts = append(o.GRPCDialOpts, w.opt)
}

// WithGRPCConnectionPool returns a ClientOption that creates a pool of gRPC
// connections that requests will be balanced between.
func WithGRPCConnectionPool(size int) ClientOption {
	return withGRPCConnectionPool(size)
}

type withGRPCConnectionPool int

func (w withGRPCConnectionPool) Apply(o *internal.DialSettings) {
	o.GRPCConnPoolSize = int(w)
}

// WithAPIKey returns a ClientOption that specifies an API key to be used
// as the basis for authentication.
//
// API Keys can only be used for JSON-over-HTTP APIs, including those under
// the import path google.golang.org/api/....
func WithAPIKey(apiKey string) ClientOption {
	return withAPIKey(apiKey)
}

type withAPIKey string

func (w withAPIKey) Apply(o *internal.DialSettings) { o.APIKey = string(w) }

// WithAudiences returns a ClientOption that specifies an audience to be used
// as the audience field ("aud") for the JWT token authentication.
func WithAudiences(audience ...string) ClientOption {
	return withAudiences(audience)
}

type withAudiences []string

func (w withAudiences) Apply(o *internal.DialSettings) {
	o.Audiences = make([]string, len(w))
	copy(o.Audiences, w)
}

// WithoutAuthentication returns a ClientOption that specifies that no
// authentication should be used. It is suitable only for testing and for
// accessing public resources, like public Google Cloud Storage buckets.
// It is an error to provide both WithoutAuthentication and any of WithAPIKey,
// WithTokenSource, WithCredentialsFile or WithServiceAccountFile.
func WithoutAuthentication() ClientOption {
	return withoutAuthentication{}
}

type withoutAuthentication struct{}

func (w withoutAuthentication) Apply(o *internal.DialSettings) { o.NoAuth = true }

// WithQuotaProject returns a ClientOption that specifies the project used
// for quota and billing purposes.
//
// For more information please read:
// https://cloud.google.com/apis/docs/system-parameters
func WithQuotaProject(quotaProject string) ClientOption {
	return withQuotaProject(quotaProject)
}

type withQuotaProject string

func (w withQuotaProject) Apply(o *internal.DialSettings) {
	o.QuotaProject = string(w)
}

// WithRequestReason returns a ClientOption that specifies a reason for
// making the request, which is intended to be recorded in audit logging.
// An example reason would be a support-case ticket number.
//
// For more information please read:
// https://cloud.google.com/apis/docs/system-parameters
func WithRequestReason(requestReason string) ClientOption {
	return withRequestReason(requestReason)
}

type withRequestReason string

func (w withRequestReason) Apply(o *internal.DialSettings) {
	o.RequestReason = string(w)
}

// WithTelemetryDisabled returns a ClientOption that disables default telemetry (OpenCensus)
// settings on gRPC and HTTP clients.
// An example reason would be to bind custom telemetry that overrides the defaults.
func WithTelemetryDisabled() ClientOption {
	return withTelemetryDisabled{}
}

type withTelemetryDisabled struct{}

func (w withTelemetryDisabled) Apply(o *internal.DialSettings) {
	o.TelemetryDisabled = true
}

// ClientCertSource is a function that returns a TLS client certificate to be used
// when opening TLS connections.
//
// It follows the same semantics as crypto/tls.Config.GetClientCertificate.
//
// This is an EXPERIMENTAL API and may be changed or removed in the future.
type ClientCertSource = func(*tls.CertificateRequestInfo) (*tls.Certificate, error)

// WithClientCertSource returns a ClientOption that specifies a
// callback function for obtaining a TLS client certificate.
//
// This option is used for supporting mTLS authentication, where the
// server validates the client certifcate when establishing a connection.
//
// The callback function will be invoked whenever the server requests a
// certificate from the client. Implementations of the callback function
// should try to ensure that a valid certificate can be repeatedly returned
// on demand for the entire life cycle of the transport client. If a nil
// Certificate is returned (i.e. no Certificate can be obtained), an error
// should be returned.
//
// This is an EXPERIMENTAL API and may be changed or removed in the future.
func WithClientCertSource(s ClientCertSource) ClientOption {
	return withClientCertSource{s}
}

type withClientCertSource struct{ s ClientCertSource }

func (w withClientCertSource) Apply(o *internal.DialSettings) {
	o.ClientCertSource = w.s
}

// ImpersonateCredentials returns a ClientOption that will impersonate the
// target service account.
//
// In order to impersonate the target service account
// the base service account must have the Service Account Token Creator role,
// roles/iam.serviceAccountTokenCreator, on the target service account.
// See https://cloud.google.com/iam/docs/understanding-service-accounts.
//
// Optionally, delegates can be used during impersonation if the base service
// account lacks the token creator role on the target. When using delegates,
// each service account must be granted roles/iam.serviceAccountTokenCreator
// on the next service account in the chain.
//
// For example, if a base service account of SA1 is trying to impersonate target
// service account SA2 while using delegate service accounts DSA1 and DSA2,
// the following must be true:
//
//  1. Base service account SA1 has roles/iam.serviceAccountTokenCreator on
//     DSA1.
//  2. DSA1 has roles/iam.serviceAccountTokenCreator on DSA2.
//  3. DSA2 has roles/iam.serviceAccountTokenCreator on target SA2.
//
// The resulting impersonated credential will either have the default scopes of
// the client being instantiating or the scopes from WithScopes if provided.
// Scopes are required for creating impersonated credentials, so if this option
// is used while not using a NewClient/NewService function, WithScopes must also
// be explicitly passed in as well.
//
// If the base credential is an authorized user and not a service account, or if
// the option WithQuotaProject is set, the target service account must have a
// role that grants the serviceusage.services.use permission such as
// roles/serviceusage.serviceUsageConsumer.
//
// This is an EXPERIMENTAL API and may be changed or removed in the future.
//
// Deprecated: This option has been replaced by `impersonate` package:
// `google.golang.org/api/impersonate`. Please use the `impersonate` package
// instead with the WithTokenSource option.
func ImpersonateCredentials(target string, delegates ...string) ClientOption {
	return impersonateServiceAccount{
		target:    target,
		delegates: delegates,
	}
}

type impersonateServiceAccount struct {
	target    string
	delegates []string
}

func (i impersonateServiceAccount) Apply(o *internal.DialSettings) {
	o.ImpersonationConfig = &impersonate.Config{
		Target: i.target,
	}
	o.ImpersonationConfig.Delegates = make([]string, len(i.delegates))
	copy(o.ImpersonationConfig.Delegates, i.delegates)
}

type withCreds google.Credentials

func (w *withCreds) Apply(o *internal.DialSettings) {
	o.Credentials = (*google.Credentials)(w)
}

// WithCredentials returns a ClientOption that authenticates API calls.
func WithCredentials(creds *google.Credentials) ClientOption {
	return (*withCreds)(creds)
}
<|endoftext|>


<commit_before>package config

import (
	"flag"
	"strings"

	"github.com/wanelo/image-server/core"
	"github.com/wanelo/image-server/events"
	"github.com/wanelo/image-server/processor/cli"
	sm "github.com/wanelo/image-server/source_mapper/waneloS3"
	"github.com/wanelo/image-server/uploader/manta"
)

// ServerConfiguration initializes a ServerConfiguration from flags
func ServerConfiguration() (*core.ServerConfiguration, error) {
	sc := configurationFromFlags()

	sc.Events = &core.EventChannels{
		ImageProcessed:     make(chan *core.ImageConfiguration),
		OriginalDownloaded: make(chan *core.ImageConfiguration),
	}

	adapters := &core.Adapters{
		Processor:    &cli.Processor{sc},
		SourceMapper: &sm.SourceMapper{},
		Uploader:     manta.InitializeUploader(sc),
	}
	sc.Adapters = adapters

	go events.InitializeEventListeners(sc)

	return sc, nil
}

func configurationFromFlags() *core.ServerConfiguration {
	var (
		whitelistedExtensions = flag.String("extensions", "jpg,gif,webp", "Whitelisted extensions (separated by commas)")
		localBasePath         = flag.String("local_base_path", "public", "Directory where the images will be saved")
		sourceDomain          = flag.String("source_domain", "http://wanelo.s3.amazonaws.com", "Source domain for images")
		mantaBasePath         = flag.String("manta_base_path", "public/images/development", "base path for manta storage")
		graphiteHost          = flag.String("graphite_host", "127.0.0.1", "Graphite Host")
		graphitePort          = flag.Int("graphite_port", 8125, "Graphite port")
		maximumWidth          = flag.Int("maximum_width", 1000, "Maximum image width")
		defaultQuality        = flag.Uint("default_quality", 75, "Default image compression quality")
		uploaderConcurrency   = flag.Uint("uploader_concurrency", 10, "Uploader concurrency")
	)
	flag.Parse()

	sc := &core.ServerConfiguration{
		WhitelistedExtensions: strings.Split(*whitelistedExtensions, ","),
		LocalBasePath:         *localBasePath,
		GraphitePort:          *graphitePort,
		GraphiteHost:          *graphiteHost,
		MaximumWidth:          *maximumWidth,
		MantaBasePath:         *mantaBasePath,
		DefaultQuality:        *defaultQuality,
		SourceDomain:          *sourceDomain,
		UploaderConcurrency:   *uploaderConcurrency,
	}

	return sc
}
<commit_msg>Add missing channels for events<commit_after>package config

import (
	"flag"
	"strings"

	"github.com/wanelo/image-server/core"
	"github.com/wanelo/image-server/events"
	"github.com/wanelo/image-server/processor/cli"
	sm "github.com/wanelo/image-server/source_mapper/waneloS3"
	"github.com/wanelo/image-server/uploader/manta"
)

// ServerConfiguration initializes a ServerConfiguration from flags
func ServerConfiguration() (*core.ServerConfiguration, error) {
	sc := configurationFromFlags()

	sc.Events = &core.EventChannels{
		ImageProcessed:              make(chan *core.ImageConfiguration),
		ImageProcessedWithErrors:    make(chan *core.ImageConfiguration),
		OriginalDownloaded:          make(chan *core.ImageConfiguration),
		OriginalDownloadUnavailable: make(chan *core.ImageConfiguration),
	}

	adapters := &core.Adapters{
		Processor:    &cli.Processor{sc},
		SourceMapper: &sm.SourceMapper{},
		Uploader:     manta.InitializeUploader(sc),
	}
	sc.Adapters = adapters

	go events.InitializeEventListeners(sc)

	return sc, nil
}

func configurationFromFlags() *core.ServerConfiguration {
	var (
		whitelistedExtensions = flag.String("extensions", "jpg,gif,webp", "Whitelisted extensions (separated by commas)")
		localBasePath         = flag.String("local_base_path", "public", "Directory where the images will be saved")
		sourceDomain          = flag.String("source_domain", "http://wanelo.s3.amazonaws.com", "Source domain for images")
		mantaBasePath         = flag.String("manta_base_path", "public/images/development", "base path for manta storage")
		graphiteHost          = flag.String("graphite_host", "127.0.0.1", "Graphite Host")
		graphitePort          = flag.Int("graphite_port", 8125, "Graphite port")
		maximumWidth          = flag.Int("maximum_width", 1000, "Maximum image width")
		defaultQuality        = flag.Uint("default_quality", 75, "Default image compression quality")
		uploaderConcurrency   = flag.Uint("uploader_concurrency", 10, "Uploader concurrency")
	)
	flag.Parse()

	sc := &core.ServerConfiguration{
		WhitelistedExtensions: strings.Split(*whitelistedExtensions, ","),
		LocalBasePath:         *localBasePath,
		GraphitePort:          *graphitePort,
		GraphiteHost:          *graphiteHost,
		MaximumWidth:          *maximumWidth,
		MantaBasePath:         *mantaBasePath,
		DefaultQuality:        *defaultQuality,
		SourceDomain:          *sourceDomain,
		UploaderConcurrency:   *uploaderConcurrency,
	}

	return sc
}
<|endoftext|>


<commit_before>package converter

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"
)

// Converter from DAGMan log to PlantUML timeline
type Converter struct {
	timeRef int
	// Map job->last event timestamp
	jobs map[string]int
	conf Configuration
}

// Configuration of Converter
type Configuration struct {
	eventsToIgnore map[string]bool
	decoEvents     bool
}

type dagEvent struct {
	timestamp       int
	job             string
	id              string
	eventComplement string
}

// NewConfiguration yields a Converter configuration
func NewConfiguration() *Configuration {
	c := Configuration{eventsToIgnore: make(map[string]bool), decoEvents: true}
	return &c
}

// NewConverter yields a converter with the given configuration.
func NewConverter(config Configuration) *Converter {
	var c Converter
	c.jobs = make(map[string]int)
	c.conf = config
	return &c
}

// IgnoreEvents configures events to ignore
func (c *Configuration) IgnoreEvents(events string) {
	parts := strings.Split(events, ",")
	for _, e := range parts {
		c.eventsToIgnore[e] = true
	}
}

// DecorateEvents configures if events should be decorated
func (c *Configuration) DecorateEvents(d bool) {
	c.decoEvents = d
}

// Convert DAGMan log into PlantUML timeline
func (c *Converter) Convert(line string) (string, error) {
	event, err := parseDagLog(line)

	if err != nil {
		return "", err
	}

	if c.timeRef == 0 {
		c.timeRef = event.timestamp
	}

	job := event.job
	eventID := event.id
	if eventID == "***" {
		eventID = event.eventComplement
		job = "DAG"
	}

	if _, ok := c.conf.eventsToIgnore[eventID]; ok {
		return "' Ignoring: " + line, nil
	}

	result := fmt.Sprintf("@%d\n%s is %s", event.timestamp-c.timeRef, job, eventID)
	if t, ok := c.jobs[job]; !ok {
		result = fmt.Sprintf("concise \"%s\" as %s\n%s", job, job, result)
	} else if c.conf.decoEvents {
		result = fmt.Sprintf("%s@%d <-> @%d : %ds\n%s", job, t-c.timeRef, event.timestamp-c.timeRef, event.timestamp-t, result)
	}
	c.jobs[job] = event.timestamp

	return result, nil
}

func parseDagLog(logLine string) (dagEvent, error) {
	parts := strings.SplitN(logLine, " ", 5)

	if len(parts) < 5 {
		errmsg := fmt.Sprintf("' [WARN] skipping line: %s", logLine)
		log.Printf(errmsg)
		return dagEvent{}, errors.New(errmsg)
	}

	timestamp, err := strconv.Atoi(parts[0])
	if err != nil {
		errmsg := fmt.Sprintf("' [WARN] skipping line (unable to parse timestamp): %s", logLine)
		log.Printf(errmsg)
		return dagEvent{}, errors.New(errmsg)
	}

	return dagEvent{timestamp: timestamp, job: parts[1], id: parts[2], eventComplement: parts[3]}, nil
}
<commit_msg>do not decorates 0s duration events<commit_after>package converter

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"
)

// Converter from DAGMan log to PlantUML timeline
type Converter struct {
	timeRef int
	// Map job->last event timestamp
	jobs map[string]int
	conf Configuration
}

// Configuration of Converter
type Configuration struct {
	eventsToIgnore map[string]bool
	decoEvents     bool
}

type dagEvent struct {
	timestamp       int
	job             string
	id              string
	eventComplement string
}

// NewConfiguration yields a Converter configuration
func NewConfiguration() *Configuration {
	c := Configuration{eventsToIgnore: make(map[string]bool), decoEvents: true}
	return &c
}

// NewConverter yields a converter with the given configuration.
func NewConverter(config Configuration) *Converter {
	var c Converter
	c.jobs = make(map[string]int)
	c.conf = config
	return &c
}

// IgnoreEvents configures events to ignore
func (c *Configuration) IgnoreEvents(events string) {
	parts := strings.Split(events, ",")
	for _, e := range parts {
		c.eventsToIgnore[e] = true
	}
}

// DecorateEvents configures if events should be decorated
func (c *Configuration) DecorateEvents(d bool) {
	c.decoEvents = d
}

// Convert DAGMan log into PlantUML timeline
func (c *Converter) Convert(line string) (string, error) {
	event, err := parseDagLog(line)

	if err != nil {
		return "", err
	}

	if c.timeRef == 0 {
		c.timeRef = event.timestamp
	}

	job := event.job
	eventID := event.id
	if eventID == "***" {
		eventID = event.eventComplement
		job = "DAG"
	}

	if _, ok := c.conf.eventsToIgnore[eventID]; ok {
		return "' Ignoring: " + line, nil
	}

	result := fmt.Sprintf("@%d\n%s is %s", event.timestamp-c.timeRef, job, eventID)
	if t, ok := c.jobs[job]; !ok {
		result = fmt.Sprintf("concise \"%s\" as %s\n%s", job, job, result)
	} else if c.conf.decoEvents && event.timestamp-t > 0 {
		result = fmt.Sprintf("%s@%d <-> @%d : %ds\n%s", job, t-c.timeRef, event.timestamp-c.timeRef, event.timestamp-t, result)
	}
	c.jobs[job] = event.timestamp

	return result, nil
}

func parseDagLog(logLine string) (dagEvent, error) {
	parts := strings.SplitN(logLine, " ", 5)

	if len(parts) < 5 {
		errmsg := fmt.Sprintf("' [WARN] skipping line: %s", logLine)
		log.Printf(errmsg)
		return dagEvent{}, errors.New(errmsg)
	}

	timestamp, err := strconv.Atoi(parts[0])
	if err != nil {
		errmsg := fmt.Sprintf("' [WARN] skipping line (unable to parse timestamp): %s", logLine)
		log.Printf(errmsg)
		return dagEvent{}, errors.New(errmsg)
	}

	return dagEvent{timestamp: timestamp, job: parts[1], id: parts[2], eventComplement: parts[3]}, nil
}
<|endoftext|>


<commit_before>// Package intern implements fast, immutable string interning.
//
// The package is a cgo binding for libintern:
//
//	https://github.com/chriso/intern
//
// Interning is a way of storing distinct strings only once in memory:
//
//	https://en.wikipedia.org/wiki/String_interning
//
// Each string is assigned an ID of type uint32. IDs start at 1 and
// increment towards 2^32-1:
//
//	repository := intern.NewRepository()
//
// 	id := repository.intern("foo")
// 	fmt.Println(id) // => 1
//
// 	id := repository.intern("bar")
// 	fmt.Println(id) // => 2
//
// 	id := repository.intern("foo")
// 	fmt.Println(id) // => 1
//
// 	id := repository.intern("qux")
// 	fmt.Println(id) // => 3
//
// Two-way lookup is provided:
//
//  if id, ok := repository.Lookup("foo"); ok {
//    fmt.Printf("string 'foo' has ID: %v", id)
//  }
//
//  if str, ok := repository.LookupID(1); ok {
//    fmt.Printf("string with ID 1: %v", str)
//  }
//
// This package is *NOT* safe to use from multiple goroutines without
// locking, e.g. https://golang.org/pkg/sync/#Mutex
package intern

// #include <intern/strings.h>
// #include <intern/optimize.h>
// #cgo LDFLAGS: -lintern
import "C"

import (
	"fmt"
	"runtime"
)

// ErrInvalidSnapshot is returned by Repository.Restore when the
// repository and snapshot are incompatible
var ErrInvalidSnapshot = fmt.Errorf("invalid snapshot")

// Repository stores a collection of unique strings
type Repository struct {
	ptr *C.struct_strings
}

// NewRepository creates a new string repository
func NewRepository() *Repository {
	ptr := C.strings_new()
	return newRepositoryFromPtr(ptr)
}

func newRepositoryFromPtr(ptr *C.struct_strings) *Repository {
	if ptr == nil {
		outOfMemory()
	}
	repo := &Repository{ptr}
	runtime.SetFinalizer(repo, (*Repository).free)
	return repo
}

func outOfMemory() {
	panic("out of memory")
}

func (repo *Repository) free() {
	C.strings_free(repo.ptr)
}

// Count returns the total number of unique strings in the repository
func (repo *Repository) Count() uint32 {
	return uint32(C.strings_count(repo.ptr))
}

// Intern interns a string and returns its unique ID. Note that IDs increment
// from 1. This function will panic if the string does not fit in one page:
// len(string) < repo.PageSize()
func (repo *Repository) Intern(str string) uint32 {
	id := uint32(C.strings_intern(repo.ptr, C.CString(str)))
	if id == 0 {
		outOfMemory()
	}
	return id
}

// Lookup returns the ID associated with a string, or false if the ID
// does not exist in the repository
func (repo *Repository) Lookup(str string) (uint32, bool) {
	id := uint32(C.strings_lookup(repo.ptr, C.CString(str)))
	return id, id != 0
}

// LookupID returns the string associated with an ID, or false if the string
// does not exist in the repository
func (repo *Repository) LookupID(id uint32) (string, bool) {
	str := C.strings_lookup_id(repo.ptr, C.uint32_t(id))
	if str == nil {
		return "", false
	}
	return C.GoString(str), true
}

// AllocatedBytes returns the total number of bytes allocated by the string
// repository
func (repo *Repository) AllocatedBytes() uint64 {
	return uint64(C.strings_allocated_bytes(repo.ptr))
}

// Cursor creates a new cursor for iterating strings
func (repo *Repository) Cursor() *Cursor {
	cursor := _Ctype_struct_strings_cursor{}
	C.strings_cursor_init(&cursor, repo.ptr)
	return &Cursor{repo, &cursor}
}

// Optimize creates a new, optimized string repository which stores the most
// frequently seen strings together. The string with the lowest ID (1) is the
// most frequently seen string
func (repo *Repository) Optimize(freq *Frequency) *Repository {
	ptr := C.strings_optimize(repo.ptr, freq.ptr)
	return newRepositoryFromPtr(ptr)
}

// Snapshot creates a new snapshot of the repository. It can later be
// restored to this position
func (repo *Repository) Snapshot() *Snapshot {
	snapshot := _Ctype_struct_strings_snapshot{}
	C.strings_snapshot(repo.ptr, &snapshot)
	return &Snapshot{repo, &snapshot}
}

// Restore restores the string repository to a previous snapshot
func (repo *Repository) Restore(snapshot *Snapshot) error {
	if ok := C.strings_restore(repo.ptr, snapshot.ptr); !ok {
		return ErrInvalidSnapshot
	}
	return nil
}

// PageSize returns the compile-time page size setting
func (repo *Repository) PageSize() uint64 {
	return uint64(C.strings_page_size())
}

// Snapshot is a snapshot of a string repository
type Snapshot struct {
	repo *Repository
	ptr  *C.struct_strings_snapshot
}

// Cursor is used to iterate strings in a repository
type Cursor struct {
	repo *Repository
	ptr  *C.struct_strings_cursor
}

// ID returns the ID that the cursor currently points to
func (cursor *Cursor) ID() uint32 {
	return uint32(C.strings_cursor_id(cursor.ptr))
}

// String returns the string that the cursor currently points to
func (cursor *Cursor) String() string {
	str := C.strings_cursor_string(cursor.ptr)
	if str == nil {
		return ""
	}
	return C.GoString(str)
}

// Next advances the cursor. It returns true if there is another
// string, and false otherwise
func (cursor *Cursor) Next() bool {
	return bool(C.strings_cursor_next(cursor.ptr))
}

// Frequency is used to track string frequencies
type Frequency struct {
	ptr *C.struct_strings_frequency
}

// NewFrequency creates a new string frequency tracker
func NewFrequency() *Frequency {
	ptr := C.strings_frequency_new()
	if ptr == nil {
		outOfMemory()
	}
	freq := &Frequency{ptr}
	runtime.SetFinalizer(freq, (*Frequency).free)
	return freq
}

func (freq *Frequency) free() {
	C.strings_frequency_free(freq.ptr)
}

// Add adds a string ID. This should be called after interning a string and
// getting back the ID
func (freq *Frequency) Add(id uint32) {
	if ok := C.strings_frequency_add(freq.ptr, C.uint32_t(id)); !ok {
		outOfMemory()
	}
}

// AddAll adds all string IDs, to ensure that each string is present in the
// optimized repository
func (freq *Frequency) AddAll(repo *Repository) {
	if ok := C.strings_frequency_add_all(freq.ptr, repo.ptr); !ok {
		outOfMemory()
	}
}
<commit_msg>Make a note of the other features<commit_after>// Package intern implements fast, immutable string interning.
//
// The package is a cgo binding for libintern:
//
//	https://github.com/chriso/intern
//
// Interning is a way of storing distinct strings only once in memory:
//
//	https://en.wikipedia.org/wiki/String_interning
//
// Each string is assigned an ID of type uint32. IDs start at 1 and
// increment towards 2^32-1:
//
//	repository := intern.NewRepository()
//
// 	id := repository.intern("foo")
// 	fmt.Println(id) // => 1
//
// 	id := repository.intern("bar")
// 	fmt.Println(id) // => 2
//
// 	id := repository.intern("foo")
// 	fmt.Println(id) // => 1
//
// 	id := repository.intern("qux")
// 	fmt.Println(id) // => 3
//
// Two-way lookup is provided:
//
//  if id, ok := repository.Lookup("foo"); ok {
//    fmt.Printf("string 'foo' has ID: %v", id)
//  }
//
//  if str, ok := repository.LookupID(1); ok {
//    fmt.Printf("string with ID 1: %v", str)
//  }
//
// The package also provides a way to iterate unique strings in order
// of ID, optimize string repositories using frequency analysis, and
// restore string repositories to a previous snapshot.
//
// This package is *NOT* safe to use from multiple goroutines without
// locking, e.g. https://golang.org/pkg/sync/#Mutex
package intern

// #include <intern/strings.h>
// #include <intern/optimize.h>
// #cgo LDFLAGS: -lintern
import "C"

import (
	"fmt"
	"runtime"
)

// ErrInvalidSnapshot is returned by Repository.Restore when the
// repository and snapshot are incompatible
var ErrInvalidSnapshot = fmt.Errorf("invalid snapshot")

// Repository stores a collection of unique strings
type Repository struct {
	ptr *C.struct_strings
}

// NewRepository creates a new string repository
func NewRepository() *Repository {
	ptr := C.strings_new()
	return newRepositoryFromPtr(ptr)
}

func newRepositoryFromPtr(ptr *C.struct_strings) *Repository {
	if ptr == nil {
		outOfMemory()
	}
	repo := &Repository{ptr}
	runtime.SetFinalizer(repo, (*Repository).free)
	return repo
}

func outOfMemory() {
	panic("out of memory")
}

func (repo *Repository) free() {
	C.strings_free(repo.ptr)
}

// Count returns the total number of unique strings in the repository
func (repo *Repository) Count() uint32 {
	return uint32(C.strings_count(repo.ptr))
}

// Intern interns a string and returns its unique ID. Note that IDs increment
// from 1. This function will panic if the string does not fit in one page:
// len(string) < repo.PageSize()
func (repo *Repository) Intern(str string) uint32 {
	id := uint32(C.strings_intern(repo.ptr, C.CString(str)))
	if id == 0 {
		outOfMemory()
	}
	return id
}

// Lookup returns the ID associated with a string, or false if the ID
// does not exist in the repository
func (repo *Repository) Lookup(str string) (uint32, bool) {
	id := uint32(C.strings_lookup(repo.ptr, C.CString(str)))
	return id, id != 0
}

// LookupID returns the string associated with an ID, or false if the string
// does not exist in the repository
func (repo *Repository) LookupID(id uint32) (string, bool) {
	str := C.strings_lookup_id(repo.ptr, C.uint32_t(id))
	if str == nil {
		return "", false
	}
	return C.GoString(str), true
}

// AllocatedBytes returns the total number of bytes allocated by the string
// repository
func (repo *Repository) AllocatedBytes() uint64 {
	return uint64(C.strings_allocated_bytes(repo.ptr))
}

// Cursor creates a new cursor for iterating strings
func (repo *Repository) Cursor() *Cursor {
	cursor := _Ctype_struct_strings_cursor{}
	C.strings_cursor_init(&cursor, repo.ptr)
	return &Cursor{repo, &cursor}
}

// Optimize creates a new, optimized string repository which stores the most
// frequently seen strings together. The string with the lowest ID (1) is the
// most frequently seen string
func (repo *Repository) Optimize(freq *Frequency) *Repository {
	ptr := C.strings_optimize(repo.ptr, freq.ptr)
	return newRepositoryFromPtr(ptr)
}

// Snapshot creates a new snapshot of the repository. It can later be
// restored to this position
func (repo *Repository) Snapshot() *Snapshot {
	snapshot := _Ctype_struct_strings_snapshot{}
	C.strings_snapshot(repo.ptr, &snapshot)
	return &Snapshot{repo, &snapshot}
}

// Restore restores the string repository to a previous snapshot
func (repo *Repository) Restore(snapshot *Snapshot) error {
	if ok := C.strings_restore(repo.ptr, snapshot.ptr); !ok {
		return ErrInvalidSnapshot
	}
	return nil
}

// PageSize returns the compile-time page size setting
func (repo *Repository) PageSize() uint64 {
	return uint64(C.strings_page_size())
}

// Snapshot is a snapshot of a string repository
type Snapshot struct {
	repo *Repository
	ptr  *C.struct_strings_snapshot
}

// Cursor is used to iterate strings in a repository
type Cursor struct {
	repo *Repository
	ptr  *C.struct_strings_cursor
}

// ID returns the ID that the cursor currently points to
func (cursor *Cursor) ID() uint32 {
	return uint32(C.strings_cursor_id(cursor.ptr))
}

// String returns the string that the cursor currently points to
func (cursor *Cursor) String() string {
	str := C.strings_cursor_string(cursor.ptr)
	if str == nil {
		return ""
	}
	return C.GoString(str)
}

// Next advances the cursor. It returns true if there is another
// string, and false otherwise
func (cursor *Cursor) Next() bool {
	return bool(C.strings_cursor_next(cursor.ptr))
}

// Frequency is used to track string frequencies
type Frequency struct {
	ptr *C.struct_strings_frequency
}

// NewFrequency creates a new string frequency tracker
func NewFrequency() *Frequency {
	ptr := C.strings_frequency_new()
	if ptr == nil {
		outOfMemory()
	}
	freq := &Frequency{ptr}
	runtime.SetFinalizer(freq, (*Frequency).free)
	return freq
}

func (freq *Frequency) free() {
	C.strings_frequency_free(freq.ptr)
}

// Add adds a string ID. This should be called after interning a string and
// getting back the ID
func (freq *Frequency) Add(id uint32) {
	if ok := C.strings_frequency_add(freq.ptr, C.uint32_t(id)); !ok {
		outOfMemory()
	}
}

// AddAll adds all string IDs, to ensure that each string is present in the
// optimized repository
func (freq *Frequency) AddAll(repo *Repository) {
	if ok := C.strings_frequency_add_all(freq.ptr, repo.ptr); !ok {
		outOfMemory()
	}
}
<|endoftext|>