<commit_before>// +build linux,amd64

package devmapper

import (
	"fmt"
	"github.com/dotcloud/docker/graphdriver"
	"github.com/dotcloud/docker/utils"
	"io/ioutil"
	"path"
)

func init() {
	graphdriver.Register("devicemapper", Init)
}

// Placeholder interfaces, to be replaced
// at integration.

// End of placeholder interfaces.

type Driver struct {
	*DeviceSet
	home string
}

var Init = func(home string) (graphdriver.Driver, error) {
	deviceSet, err := NewDeviceSet(home, true)
	if err != nil {
		return nil, err
	}
	d := &Driver{
		DeviceSet: deviceSet,
		home:      home,
	}
	return d, nil
}

func (d *Driver) String() string {
	return "devicemapper"
}

func (d *Driver) Status() [][2]string {
	s := d.DeviceSet.Status()

	status := [][2]string{
		{"Pool Name", s.PoolName},
		{"Data file", s.DataLoopback},
		{"Metadata file", s.MetadataLoopback},
		{"Data Space Used", fmt.Sprintf("%.1f Mb", float64(s.Data.Used)/(1024*1024))},
		{"Data Space Total", fmt.Sprintf("%.1f Mb", float64(s.Data.Total)/(1024*1024))},
		{"Metadata Space Used", fmt.Sprintf("%.1f Mb", float64(s.Metadata.Used)/(1024*1024))},
		{"Metadata Space Total", fmt.Sprintf("%.1f Mb", float64(s.Metadata.Total)/(1024*1024))},
	}
	return status
}

func (d *Driver) Cleanup() error {
	return d.DeviceSet.Shutdown()
}

func (d *Driver) Create(id, parent string) error {
	if err := d.DeviceSet.AddDevice(id, parent); err != nil {
		return err
	}

	mp := path.Join(d.home, "mnt", id)
	if err := d.mount(id, mp); err != nil {
		return err
	}

	if err := osMkdirAll(path.Join(mp, "rootfs"), 0755); err != nil && !osIsExist(err) {
		return err
	}

	// Create an "id" file with the container/image id in it to help reconscruct this in case
	// of later problems
	if err := ioutil.WriteFile(path.Join(mp, "id"), []byte(id), 0600); err != nil {
		return err
	}

	// We float this reference so that the next Get call can
	// steal it, so we don't have to unmount
	if err := d.DeviceSet.UnmountDevice(id, UnmountFloat); err != nil {
		return err
	}

	return nil
}

func (d *Driver) Remove(id string) error {
	// Sink the float from create in case no Get() call was made
	if err := d.DeviceSet.UnmountDevice(id, UnmountSink); err != nil {
		return err
	}
	// This assumes the device has been properly Get/Put:ed and thus is unmounted
	return d.DeviceSet.DeleteDevice(id)
}

func (d *Driver) Get(id string) (string, error) {
	mp := path.Join(d.home, "mnt", id)
	if err := d.mount(id, mp); err != nil {
		return "", err
	}

	return path.Join(mp, "rootfs"), nil
}

func (d *Driver) Put(id string) {
	if err := d.DeviceSet.UnmountDevice(id, UnmountRegular); err != nil {
		utils.Errorf("Warning: error unmounting device %s: %s\n", id, err)
	}
}

func (d *Driver) mount(id, mountPoint string) error {
	// Create the target directories if they don't exist
	if err := osMkdirAll(mountPoint, 0755); err != nil && !osIsExist(err) {
		return err
	}
	// Mount the device
	return d.DeviceSet.MountDevice(id, mountPoint)
}

func (d *Driver) Exists(id string) bool {
	return d.Devices[id] != nil
}
<commit_msg>devmapper: Remove directory when removing devicemapper device<commit_after>// +build linux,amd64

package devmapper

import (
	"fmt"
	"github.com/dotcloud/docker/graphdriver"
	"github.com/dotcloud/docker/utils"
	"io/ioutil"
	"os"
	"path"
)

func init() {
	graphdriver.Register("devicemapper", Init)
}

// Placeholder interfaces, to be replaced
// at integration.

// End of placeholder interfaces.

type Driver struct {
	*DeviceSet
	home string
}

var Init = func(home string) (graphdriver.Driver, error) {
	deviceSet, err := NewDeviceSet(home, true)
	if err != nil {
		return nil, err
	}
	d := &Driver{
		DeviceSet: deviceSet,
		home:      home,
	}
	return d, nil
}

func (d *Driver) String() string {
	return "devicemapper"
}

func (d *Driver) Status() [][2]string {
	s := d.DeviceSet.Status()

	status := [][2]string{
		{"Pool Name", s.PoolName},
		{"Data file", s.DataLoopback},
		{"Metadata file", s.MetadataLoopback},
		{"Data Space Used", fmt.Sprintf("%.1f Mb", float64(s.Data.Used)/(1024*1024))},
		{"Data Space Total", fmt.Sprintf("%.1f Mb", float64(s.Data.Total)/(1024*1024))},
		{"Metadata Space Used", fmt.Sprintf("%.1f Mb", float64(s.Metadata.Used)/(1024*1024))},
		{"Metadata Space Total", fmt.Sprintf("%.1f Mb", float64(s.Metadata.Total)/(1024*1024))},
	}
	return status
}

func (d *Driver) Cleanup() error {
	return d.DeviceSet.Shutdown()
}

func (d *Driver) Create(id, parent string) error {
	if err := d.DeviceSet.AddDevice(id, parent); err != nil {
		return err
	}

	mp := path.Join(d.home, "mnt", id)
	if err := d.mount(id, mp); err != nil {
		return err
	}

	if err := osMkdirAll(path.Join(mp, "rootfs"), 0755); err != nil && !osIsExist(err) {
		return err
	}

	// Create an "id" file with the container/image id in it to help reconscruct this in case
	// of later problems
	if err := ioutil.WriteFile(path.Join(mp, "id"), []byte(id), 0600); err != nil {
		return err
	}

	// We float this reference so that the next Get call can
	// steal it, so we don't have to unmount
	if err := d.DeviceSet.UnmountDevice(id, UnmountFloat); err != nil {
		return err
	}

	return nil
}

func (d *Driver) Remove(id string) error {
	// Sink the float from create in case no Get() call was made
	if err := d.DeviceSet.UnmountDevice(id, UnmountSink); err != nil {
		return err
	}
	// This assumes the device has been properly Get/Put:ed and thus is unmounted
	if err := d.DeviceSet.DeleteDevice(id); err != nil {
		return err
	}

	mp := path.Join(d.home, "mnt", id)
	if err := os.RemoveAll(mp); err != nil && !os.IsNotExist(err) {
		return err
	}

	return nil
}

func (d *Driver) Get(id string) (string, error) {
	mp := path.Join(d.home, "mnt", id)
	if err := d.mount(id, mp); err != nil {
		return "", err
	}

	return path.Join(mp, "rootfs"), nil
}

func (d *Driver) Put(id string) {
	if err := d.DeviceSet.UnmountDevice(id, UnmountRegular); err != nil {
		utils.Errorf("Warning: error unmounting device %s: %s\n", id, err)
	}
}

func (d *Driver) mount(id, mountPoint string) error {
	// Create the target directories if they don't exist
	if err := osMkdirAll(mountPoint, 0755); err != nil && !osIsExist(err) {
		return err
	}
	// Mount the device
	return d.DeviceSet.MountDevice(id, mountPoint)
}

func (d *Driver) Exists(id string) bool {
	return d.Devices[id] != nil
}
<|endoftext|>


<commit_before>package verify

import (
  "os"

  "fmt"
  "strings"

  p "path"
  fp "path/filepath"

)

func doLocalVerify(cliExec CLIExecution) (err error) {
  var examples []SchemaExample
  if cliExec.SchemaPathIsDir {
    examples,err = allExamples(cliExec)
  } else {
    examples,err = examplesForSchema(cliExec.SchemaPath, cliExec)
  }

  if err != nil {
    return
  } else if len(examples) == 0 {
    err = fmt.Errorf("no test JSON files found. try `--help` for more information")
    return
  }

  fmt.Println(len(examples),"TESTS TO BE GENERATED")
  errorCount := 0
  for index,example := range examples {
    err = doVerify(cliExec, index, example)
    if err != nil {
        errorCount ++
    }
  }
  if errorCount > 0 {
      err = fmt.Errorf("\nLocal verify experienced %d errors", errorCount)
      return
  }

  return
}

func doVerify(cliExec CLIExecution, index int, testCase SchemaExample) (err error) {
  var thereWasAnError bool
  fmt.Println(HR)
  fmt.Printf("TEST %d (%s)", index+1, p.Base(testCase.ExamplePath))
  err = loadFilesAndMakeRequest(cliExec.BaseURL, testCase.SchemaPath, testCase.ExamplePath, cliExec.Verbose)
  if err != nil {
    fmt.Printf("\rTEST %d (%s) FAILED: %s\n", index+1, p.Base(testCase.ExamplePath), err.Error())
    if cliExec.StopOnError {
      os.Exit(1)
    } else {
      thereWasAnError = true
    }
  } else {
    if cliExec.Verbose {
      fmt.Println("")
    }
    fmt.Printf("\rTEST %d (%s) WAS A SUCCESS\n", index+1, p.Base(testCase.ExamplePath))
    if cliExec.Verbose {
      fmt.Println("")
    }
  }

  if thereWasAnError && cliExec.StopOnError {
    err = fmt.Errorf("AT LEAST ONE OF THE TEST CASES FAILED")
    return
  }

  return
}

func loadFilesAndMakeRequest(baseUrl, schemaPath, examplePath string, verbose bool) (err error) {
  f,err := os.Open(schemaPath)
  if err != nil {
    return
  }

  exampleF,err := os.Open(examplePath)
  if err != nil {
    return
  }

  err = Request(baseUrl, "RETAILOPS_SDK", f, exampleF, verbose)
  if err != nil {
    return
  }

  _,err = exampleF.Seek(0,0)
  if err != nil {
    return
  }
  _,err = f.Seek(0,0)
  if err != nil {
    return
  }
  err = Request(baseUrl, "KDS_SPOLIATER", f, exampleF, verbose)
  if err == nil {
    err = fmt.Errorf("failed to check integration_auth_token. expected HTTP 401")
    return
  }

  return
}

func examplesForSchema(schemaPath string, cliExec CLIExecution) (verifications []SchemaExample, err error) {
  dirname,filename := p.Split(schemaPath)
  exampleFilename := strings.Replace(filename, ".json", "", -1)
  exampleFilenameGlob := fmt.Sprintf("%s_ex_*.json", exampleFilename)

  if cliExec.ExamplesPathIsDir {
      dirname = cliExec.ExamplesPath
  }

  pathGlob := p.Join(dirname, exampleFilenameGlob)

  examplePaths,err := fp.Glob(pathGlob)
  if err != nil {
    return
  }

  verifications = make([]SchemaExample,0)
  for _,exPath := range examplePaths {
    verifications = append(verifications, SchemaExample{
      SchemaPath: cliExec.SchemaPath,
      ExamplePath: exPath,
    })
  }

  return
}

func allExamples(cliExec CLIExecution) (verifications []SchemaExample, err error) {
  verifications = make([]SchemaExample,0)
  dirname := cliExec.SchemaPath
  filter := cliExec.SchemaFilter
  var allSchemasGlob string
  if filter == "" {
    allSchemasGlob = p.Join(dirname, "*v1.json")
  } else {
    allSchemasGlob = p.Join(dirname, fmt.Sprintf("*%s*v1.json", filter))
  }
  allSchemaPaths,err := fp.Glob(allSchemasGlob)
  if err != nil {
    return
  }
  if len(allSchemaPaths) == 0 {
    err = fmt.Errorf("`%s` did not contain schemas", dirname)
    return
  }

  for _,schemaPath := range allSchemaPaths {
    exs,err := examplesForSchema(schemaPath, cliExec)
    if err != nil {
      return nil,err
    }

    for _,ex := range exs {
      verifications = append(verifications, ex)
    }
  }

  return
}
<commit_msg>added debug logging statements<commit_after>package verify

import (
  "os"

  "fmt"
  "strings"

  p "path"
  fp "path/filepath"

)

func doLocalVerify(cliExec CLIExecution) (err error) {
  var examples []SchemaExample
  if cliExec.SchemaPathIsDir {
    examples,err = allExamples(cliExec)
  } else {
    examples,err = examplesForSchema(cliExec.SchemaPath, cliExec)
  }

  if err != nil {
    return
  } else if len(examples) == 0 {
    err = fmt.Errorf("no test JSON files found. try `--help` for more information")
    return
  }

  fmt.Println(len(examples),"TESTS TO BE GENERATED")
  errorCount := 0
  for index,example := range examples {
    err = doVerify(cliExec, index, example)
    if err != nil {
        errorCount ++
    }
  }
  if errorCount > 0 {
      err = fmt.Errorf("\nLocal verify experienced %d errors", errorCount)
      return
  }

  return
}

func doVerify(cliExec CLIExecution, index int, testCase SchemaExample) (err error) {
  var thereWasAnError bool
  fmt.Println(HR)
  fmt.Printf("TEST %d (%s)", index+1, p.Base(testCase.ExamplePath))
  err = loadFilesAndMakeRequest(cliExec.BaseURL, testCase.SchemaPath, testCase.ExamplePath, cliExec.Verbose)
  if err != nil {
    fmt.Printf("\rTEST %d (%s) FAILED: %s\n", index+1, p.Base(testCase.ExamplePath), err.Error())
    if cliExec.StopOnError {
      os.Exit(1)
    } else {
      thereWasAnError = true
    }
  } else {
    if cliExec.Verbose {
      fmt.Println("")
    }
    fmt.Printf("\rTEST %d (%s) WAS A SUCCESS\n", index+1, p.Base(testCase.ExamplePath))
    if cliExec.Verbose {
      fmt.Println("")
    }
  }

  if thereWasAnError && cliExec.StopOnError {
    err = fmt.Errorf("AT LEAST ONE OF THE TEST CASES FAILED")
    return
  }

  return
}

func loadFilesAndMakeRequest(baseUrl, schemaPath, examplePath string, verbose bool) (err error) {
  f,err := os.Open(schemaPath)
  if err != nil {
    return
  }

  exampleF,err := os.Open(examplePath)
  if err != nil {
    return
  }

  err = Request(baseUrl, "RETAILOPS_SDK", f, exampleF, verbose)
  if err != nil {
    return
  }

  _,err = exampleF.Seek(0,0)
  if err != nil {
    return
  }
  _,err = f.Seek(0,0)
  if err != nil {
    return
  }
  err = Request(baseUrl, "KDS_SPOLIATER", f, exampleF, verbose)
  if err == nil {
    err = fmt.Errorf("failed to check integration_auth_token. expected HTTP 401")
    return
  }

  return
}

func examplesForSchema(schemaPath string, cliExec CLIExecution) (verifications []SchemaExample, err error) {
  dirname,filename := p.Split(schemaPath)
  fmt.Println("dirname: ", dirname)
  fmt.Println("filename: ", filename)
  exampleFilename := strings.Replace(filename, ".json", "", -1)
  exampleFilenameGlob := fmt.Sprintf("%s_ex_*.json", exampleFilename)

  if cliExec.ExamplesPathIsDir {
      fmt.Println("isDir")
      fmt.Println("cliExec.ExamplesPath: ",cliExec.ExamplesPath)
      dirname = cliExec.ExamplesPath
  }
fmt.Println("dirname(2): ", dirname)
  pathGlob := p.Join(dirname, exampleFilenameGlob)

  examplePaths,err := fp.Glob(pathGlob)
  if err != nil {
    return
  }
fmt.Println("examplePaths: ", examplePaths)
  verifications = make([]SchemaExample,0)
  for _,exPath := range examplePaths {
      fmt.Println("exPath: ", exPath)
    verifications = append(verifications, SchemaExample{
      SchemaPath: cliExec.SchemaPath,
      ExamplePath: exPath,
    })
  }

  return
}

func allExamples(cliExec CLIExecution) (verifications []SchemaExample, err error) {
  verifications = make([]SchemaExample,0)
  dirname := cliExec.SchemaPath
  filter := cliExec.SchemaFilter
  var allSchemasGlob string
  if filter == "" {
    allSchemasGlob = p.Join(dirname, "*v1.json")
  } else {
    allSchemasGlob = p.Join(dirname, fmt.Sprintf("*%s*v1.json", filter))
  }
  allSchemaPaths,err := fp.Glob(allSchemasGlob)
  if err != nil {
    return
  }
  if len(allSchemaPaths) == 0 {
    err = fmt.Errorf("`%s` did not contain schemas", dirname)
    return
  }

  for _,schemaPath := range allSchemaPaths {
      fmt.Println("schemaPath: ", schemaPath)
      fmt.Println("cliExec", cliExec)
    exs,err := examplesForSchema(schemaPath, cliExec)
    if err != nil {
      return nil,err
    }

    for _,ex := range exs {
        fmt.Println("ex: ", ex)
      verifications = append(verifications, ex)
    }
  }

  return
}
<|endoftext|>


<commit_before>package gubled

import (
	"github.com/smancke/guble/client"
	"github.com/smancke/guble/gcm"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/testutil"

	"github.com/stretchr/testify/assert"

	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"sync"
	"testing"
	"time"
)

func BenchmarkGCMConnector_BroadcastMessagesSingleWorker(b *testing.B) {
	throughput := throughputSend(b, 1, sendBroadcastSample)
	fmt.Printf("Broadcast throughput for single worker: %.2f msg/sec\n", throughput)
}

func BenchmarkGCMConnector_BroadcastMessagesMultipleWorkers(b *testing.B) {
	gomaxprocs := runtime.GOMAXPROCS(0)
	throughput := throughputSend(b, gomaxprocs, sendBroadcastSample)
	fmt.Printf("Broadcast throughput for GOMAXPROCS (%v): %.2f msg/sec\n", gomaxprocs, throughput)
}

func BenchmarkGCMConnector_SendMessagesSingleWorker(b *testing.B) {
	throughput := throughputSend(b, 1, sendMessageSample)
	fmt.Printf("Send throughput for single worker: %.2f msg/sec\n", throughput)
}

func BenchmarkGCMConnector_SendMessagesMultipleWorkers(b *testing.B) {
	gomaxprocs := runtime.GOMAXPROCS(0)
	throughput := throughputSend(b, gomaxprocs, sendMessageSample)
	fmt.Printf("Send throughput for GOMAXPROCS (%v): %.2f msg/sec\n", gomaxprocs, throughput)
}

func throughputSend(b *testing.B, nWorkers int, sampleSend func(c client.Client) error) float64 {
	//testutil.EnableDebugForMethod()
	defer testutil.ResetDefaultRegistryHealthCheck()
	protocol.Debug("b.N=%v\n", b.N)
	a := assert.New(b)

	dir, errTempDir := ioutil.TempDir("", "guble_benchmarking_gcm_test")
	defer func() {
		errRemove := os.RemoveAll(dir)
		if errRemove != nil {
			protocol.Err("Could not remove directory", errRemove)
		}
	}()
	a.NoError(errTempDir)

	args := Args{
		Listen:      "localhost:0",
		KVBackend:   "memory",
		MSBackend:   "file",
		StoragePath: dir,
		GcmEnable:   true,
		GcmApiKey:   "WILL BE OVERWRITTEN",
		GcmWorkers:  nWorkers}

	service := StartService(args)

	protocol.Debug("Overwriting the GCM Sender with a Mock")
	gcmConnector, ok := service.Modules()[2].(*gcm.GCMConnector)
	a.True(ok, "Modules[2] should be of type GCMConnector")
	gcmConnector.Sender = testutil.CreateGcmSender(
		testutil.CreateRoundTripperWithJsonResponse(http.StatusOK, testutil.CorrectGcmResponseMessageJSON, nil))

	service.Start()

	gomaxprocs := runtime.GOMAXPROCS(0)
	clients := make([]client.Client, 0, gomaxprocs)
	for clientID := 0; clientID < gomaxprocs; clientID++ {
		location := "ws://" + service.WebServer().GetAddr() + "/stream/user/" + strconv.Itoa(clientID)
		client, err := client.Open(location, "http://localhost/", 1000, true)
		a.NoError(err)
		clients = append(clients, client)
	}

	// create a topic
	url := fmt.Sprintf("http://%s/gcm/0/gcmId0/subscribe/topic", service.WebServer().GetAddr())
	response, errPost := http.Post(url, "text/plain", bytes.NewBufferString(""))
	a.NoError(errPost)
	a.Equal(response.StatusCode, 200)
	body, errReadAll := ioutil.ReadAll(response.Body)
	a.NoError(errReadAll)
	a.Equal("registered: /topic\n", string(body))

	protocol.Debug("Starting the benchmark - sending multiple messages from each client")
	start := time.Now()
	b.ResetTimer()

	var wg sync.WaitGroup
	wg.Add(gomaxprocs)
	for _, c := range clients {
		go func(c client.Client) {
			defer wg.Done()
			for i := 0; i < b.N; i++ {
				a.NoError(sampleSend(c))
			}
		}(c)
	}
	wg.Wait()

	// stop service (and wait for all the messages to be processed during the given grace period)
	service.StopGracePeriod = 10 * time.Second
	err := service.Stop()
	a.Nil(err)

	end := time.Now()
	b.StopTimer()

	return float64(b.N*gomaxprocs) / end.Sub(start).Seconds()
}

func sendBroadcastSample(c client.Client) error {
	return c.Send("/gcm/broadcast", "general offer", "{id:id}")
}

func sendMessageSample(c client.Client) error {
	return c.Send("topic", "personalized offer", "{id:id}")
}
<commit_msg>disabled topic creation in benchmark<commit_after>package gubled

import (
	"github.com/smancke/guble/client"
	"github.com/smancke/guble/gcm"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/testutil"

	"github.com/stretchr/testify/assert"

	//"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"sync"
	"testing"
	"time"
)

func BenchmarkGCMConnector_BroadcastMessagesSingleWorker(b *testing.B) {
	throughput := throughputSend(b, 1, sendBroadcastSample)
	fmt.Printf("Broadcast throughput for single worker: %.2f msg/sec\n", throughput)
}

func BenchmarkGCMConnector_BroadcastMessagesMultipleWorkers(b *testing.B) {
	gomaxprocs := runtime.GOMAXPROCS(0)
	throughput := throughputSend(b, gomaxprocs, sendBroadcastSample)
	fmt.Printf("Broadcast throughput for GOMAXPROCS (%v): %.2f msg/sec\n", gomaxprocs, throughput)
}

func BenchmarkGCMConnector_SendMessagesSingleWorker(b *testing.B) {
	throughput := throughputSend(b, 1, sendMessageSample)
	fmt.Printf("Send throughput for single worker: %.2f msg/sec\n", throughput)
}

func BenchmarkGCMConnector_SendMessagesMultipleWorkers(b *testing.B) {
	gomaxprocs := runtime.GOMAXPROCS(0)
	throughput := throughputSend(b, gomaxprocs, sendMessageSample)
	fmt.Printf("Send throughput for GOMAXPROCS (%v): %.2f msg/sec\n", gomaxprocs, throughput)
}

func throughputSend(b *testing.B, nWorkers int, sampleSend func(c client.Client) error) float64 {
	//testutil.EnableDebugForMethod()
	fmt.Printf("b.N=%v\n", b.N)
	defer testutil.ResetDefaultRegistryHealthCheck()
	a := assert.New(b)

	dir, errTempDir := ioutil.TempDir("", "guble_benchmarking_gcm_test")
	defer func() {
		errRemove := os.RemoveAll(dir)
		if errRemove != nil {
			protocol.Err("Could not remove directory", errRemove)
		}
	}()
	a.NoError(errTempDir)

	args := Args{
		Listen:      "localhost:0",
		KVBackend:   "memory",
		MSBackend:   "file",
		StoragePath: dir,
		GcmEnable:   true,
		GcmApiKey:   "WILL BE OVERWRITTEN",
		GcmWorkers:  nWorkers}

	service := StartService(args)

	protocol.Debug("Overwriting the GCM Sender with a Mock")
	gcmConnector, ok := service.Modules()[4].(*gcm.GCMConnector)
	a.True(ok, "Modules[2] should be of type GCMConnector")
	gcmConnector.Sender = testutil.CreateGcmSender(
		testutil.CreateRoundTripperWithJsonResponse(http.StatusOK, testutil.CorrectGcmResponseMessageJSON, nil))

	gomaxprocs := runtime.GOMAXPROCS(0)
	clients := make([]client.Client, 0, gomaxprocs)
	for clientID := 0; clientID < gomaxprocs; clientID++ {
		location := "ws://" + service.WebServer().GetAddr() + "/stream/user/" + strconv.Itoa(clientID)
		client, err := client.Open(location, "http://localhost/", 1000, true)
		a.NoError(err)
		clients = append(clients, client)
	}

	// create a topic
	//url := fmt.Sprintf("http://%s/gcm/0/gcmId0/subscribe/topic", service.WebServer().GetAddr())
	//response, errPost := http.Post(url, "text/plain", bytes.NewBufferString(""))
	//a.NoError(errPost)
	//a.Equal(response.StatusCode, 200)
	//body, errReadAll := ioutil.ReadAll(response.Body)
	//a.NoError(errReadAll)
	//a.Equal("registered: /topic\n", string(body))

	protocol.Debug("starting the benchmark timer")
	start := time.Now()
	b.ResetTimer()

	protocol.Debug("sending multiple messages from each client in separate goroutines")
	var wg sync.WaitGroup
	wg.Add(gomaxprocs)
	for _, c := range clients {
		go func(c client.Client) {
			defer wg.Done()
			for i := 0; i < b.N; i++ {
				a.NoError(sampleSend(c))
			}
		}(c)
	}
	wg.Wait()

	// stop service (and wait for all the messages to be processed during the given grace period)
	service.StopGracePeriod = 10 * time.Second
	err := service.Stop()
	a.Nil(err)

	end := time.Now()
	b.StopTimer()

	return float64(b.N*gomaxprocs) / end.Sub(start).Seconds()
}

func sendBroadcastSample(c client.Client) error {
	return c.Send("/gcm/broadcast", "general offer", "{id:id}")
}

func sendMessageSample(c client.Client) error {
	return c.Send("topic", "personalized offer", "{id:id}")
}
<|endoftext|>


<commit_before>package pollingLocation

import (
	"github.com/votinginfoproject/sms-worker/civic_api"
	"github.com/votinginfoproject/sms-worker/responses"
)

func BuildMessage(res *civicApi.Response, language string, newUser bool, firstContact bool, content *responses.Content) ([]string, bool) {
	if len(res.Error.Errors) == 0 && len(res.PollingLocations) > 0 {
		return success(res, language, content), true
	} else if len(res.Error.Errors) == 0 && len(res.PollingLocations) == 0 {
		return []string{content.Errors.Text[language]["noElectionInfo"]}, true
	} else {
		return failure(res, language, newUser, firstContact, content), false
	}
}

func success(res *civicApi.Response, language string, content *responses.Content) []string {
	pl := res.PollingLocations[0]
	response := content.PollingLocation.Text[language]["prefix"] + "\n"

	if len(pl.Address.LocationName) > 0 {
		response = response + pl.Address.LocationName + "\n"
	}

	if len(pl.Address.Line1) > 0 {
		response = response + pl.Address.Line1 + "\n"
		response = response + pl.Address.City + ", "
		response = response + pl.Address.State + " "
		response = response + pl.Address.Zip
	}

	if len(pl.PollingHours) > 0 {
		response = response + "\n" + content.PollingLocation.Text["en"]["hours"] + " " + pl.PollingHours
	}

	return []string{response + "\n\n" + content.Help.Text[language]["menu"] + " " + content.Help.Text[language]["languages"]}
}

func failure(res *civicApi.Response, language string, newUser bool, firstContact bool, content *responses.Content) []string {
	if len(res.Error.Errors) > 0 {
		if res.Error.Errors[0].Reason == "parseError" {
			if newUser == true {
				if firstContact == true {
					return []string{content.Intro.Text[language]["all"]}
				} else {
					return []string{content.Errors.Text[language]["addressParseNewUser"] + "\n\n" + content.Help.Text[language]["languages"]}
				}
			} else {
				return []string{content.Errors.Text[language]["addressParseExistingUser"]}
			}
		} else if res.Error.Errors[0].Reason == "notFound" {
			return []string{content.Errors.Text[language]["noElectionInfo"]}
		}
	}

	return []string{content.Errors.Text[language]["generalBackend"]}
}
<commit_msg>clean up polling location error response<commit_after>package pollingLocation

import (
	"github.com/votinginfoproject/sms-worker/civic_api"
	"github.com/votinginfoproject/sms-worker/responses"
)

func BuildMessage(res *civicApi.Response, language string, newUser bool, firstContact bool, content *responses.Content) ([]string, bool) {
	if len(res.PollingLocations) > 0 {
		return success(res, language, content), true
	} else if len(res.Error.Errors) == 0 && len(res.PollingLocations) == 0 {
		return []string{content.Errors.Text[language]["noElectionInfo"]}, true
	} else {
		return failure(res, language, newUser, firstContact, content), false
	}
}

func success(res *civicApi.Response, language string, content *responses.Content) []string {
	pl := res.PollingLocations[0]
	response := content.PollingLocation.Text[language]["prefix"] + "\n"

	if len(pl.Address.LocationName) > 0 {
		response = response + pl.Address.LocationName + "\n"
	}

	if len(pl.Address.Line1) > 0 {
		response = response + pl.Address.Line1 + "\n"
		response = response + pl.Address.City + ", "
		response = response + pl.Address.State + " "
		response = response + pl.Address.Zip
	}

	if len(pl.PollingHours) > 0 {
		response = response + "\n" + content.PollingLocation.Text["en"]["hours"] + " " + pl.PollingHours
	}

	return []string{response + "\n\n" + content.Help.Text[language]["menu"] + " " + content.Help.Text[language]["languages"]}
}

func failure(res *civicApi.Response, language string, newUser bool, firstContact bool, content *responses.Content) []string {
	var reason string
	if len(res.Error.Errors) > 0 {
		reason = res.Error.Errors[0].Reason
	}

	switch reason {
	case "parseError":
		if newUser == true && firstContact == true {
			return []string{content.Intro.Text[language]["all"]}
		} else if newUser == true && firstContact == false {
			return []string{content.Errors.Text[language]["addressParseNewUser"] + "\n\n" + content.Help.Text[language]["languages"]}
		} else {
			return []string{content.Errors.Text[language]["addressParseExistingUser"]}
		}
	case "notFound":
		return []string{content.Errors.Text[language]["noElectionInfo"]}
	default:
		return []string{content.Errors.Text[language]["generalBackend"]}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"
)

var stylesheet template.HTML = template.HTML(`

<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css" integrity="sha384-1q8mTJOASx8j1Au+a5WDVnPi2lkFfwwEAa8hDDdjZlpLegxhjVME1fgjWPGmkzs7" crossorigin="anonymous">

<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap-theme.min.css" integrity="sha384-fLW2N01lMqjakBkx3l/M9EahuwpSfeNvV63J5ezn3uZzapT0u7EYsXMjQV+0En5r" crossorigin="anonymous">

`)

type FormPage struct {
	Stylesheet  template.HTML
	Cachebuster int
}

var formPageTemplate string = `
<!DOCTYPE html>

		<title>Frontend</title>
		<meta charset="utf-8">
		<meta http-equiv="X-UA-Compatible" content="IE=edge">
		<meta name="viewport" content="width=device-width, initial-scale=1">
		{{.Stylesheet}}

Frontend Sample App

				<form action="/proxy/" method="get" class="form-inline">

								Backend URL
								<input type="text" name="url" class="form-control" placeholder="127.0.0.1:8080">

							<input type="hidden" name="cachebuster" value="{{.Cachebuster}}">
							<button type="submit" class="btn btn-default">Submit</button>

				</form>

`

type ProxyPage struct {
	Stylesheet template.HTML
	CatBody    template.HTML
}

var proxyPageTemplate string = `
<!DOCTYPE html>

		<title>Frontend</title>
		<meta charset="utf-8">
		<meta http-equiv="X-UA-Compatible" content="IE=edge">
		<meta name="viewport" content="width=device-width, initial-scale=1">
		{{.Stylesheet}}

Frontend Sample App

			{{.CatBody}}

`

type ErrorPage struct {
	Stylesheet template.HTML
	Error      error
}

var errorPageTemplate string = `
<!DOCTYPE html>

		<title>Frontend</title>
		<meta charset="utf-8">
		<meta http-equiv="X-UA-Compatible" content="IE=edge">
		<meta name="viewport" content="width=device-width, initial-scale=1">
		{{.Stylesheet}}

Frontend Sample App

request failed: {{.Error}}

`

type InfoHandler struct {
	Port int
}

func (h *InfoHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	template := template.Must(template.New("formPage").Parse(formPageTemplate))
	err := template.Execute(resp, FormPage{
		Stylesheet:  stylesheet,
		Cachebuster: rand.Int(),
	})
	if err != nil {
		panic(err)
	}
	return
}

type ProxyHandler struct{}

func (h *ProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	queryParams, err := url.ParseQuery(req.URL.RawQuery)
	if err != nil {
		panic(err)
	}
	destination := queryParams["url"][0]
	destination = "http://" + destination
	httpClient := http.DefaultClient
	httpClient.Timeout = 30 * time.Second
	getResp, err := httpClient.Get(destination)
	if err != nil {
		template := template.Must(template.New("formPage").Parse(errorPageTemplate))
		err = template.Execute(resp, ErrorPage{
			Stylesheet: stylesheet,
			Error:      err,
		})
		if err != nil {
			panic(err)
		}

		return
	}
	defer getResp.Body.Close()

	readBytes, err := ioutil.ReadAll(getResp.Body)
	if err != nil {
		resp.WriteHeader(http.StatusInternalServerError)
		resp.Write([]byte(fmt.Sprintf("read body failed: %s", err)))
		return
	}

	theTemplate := template.Must(template.New("proxyPage").Parse(proxyPageTemplate))
	catBody := template.HTML(string(readBytes))
	err = theTemplate.Execute(resp, ProxyPage{
		Stylesheet: stylesheet,
		CatBody:    catBody,
	})
	if err != nil {
		panic(err)
	}

	return
}

func launchHandler(port int, proxyHandler http.Handler) {
	mux := http.NewServeMux()
	mux.Handle("/proxy/", proxyHandler)
	mux.Handle("/", &InfoHandler{
		Port: port,
	})
	http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", port), mux)
}

func main() {
	systemPortString := os.Getenv("PORT")
	systemPort, err := strconv.Atoi(systemPortString)
	if err != nil {
		log.Fatal("invalid required env var PORT")
	}

	proxyHandler := &ProxyHandler{}

	launchHandler(systemPort, proxyHandler)
}
<commit_msg>Reduce frontend http client timeout to 5 seconds<commit_after>package main

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"
)

var stylesheet template.HTML = template.HTML(`

<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css" integrity="sha384-1q8mTJOASx8j1Au+a5WDVnPi2lkFfwwEAa8hDDdjZlpLegxhjVME1fgjWPGmkzs7" crossorigin="anonymous">

<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap-theme.min.css" integrity="sha384-fLW2N01lMqjakBkx3l/M9EahuwpSfeNvV63J5ezn3uZzapT0u7EYsXMjQV+0En5r" crossorigin="anonymous">

`)

type FormPage struct {
	Stylesheet  template.HTML
	Cachebuster int
}

var formPageTemplate string = `
<!DOCTYPE html>

		<title>Frontend</title>
		<meta charset="utf-8">
		<meta http-equiv="X-UA-Compatible" content="IE=edge">
		<meta name="viewport" content="width=device-width, initial-scale=1">
		{{.Stylesheet}}

Frontend Sample App

				<form action="/proxy/" method="get" class="form-inline">

								Backend URL
								<input type="text" name="url" class="form-control" placeholder="127.0.0.1:8080">

							<input type="hidden" name="cachebuster" value="{{.Cachebuster}}">
							<button type="submit" class="btn btn-default">Submit</button>

				</form>

`

type ProxyPage struct {
	Stylesheet template.HTML
	CatBody    template.HTML
}

var proxyPageTemplate string = `
<!DOCTYPE html>

		<title>Frontend</title>
		<meta charset="utf-8">
		<meta http-equiv="X-UA-Compatible" content="IE=edge">
		<meta name="viewport" content="width=device-width, initial-scale=1">
		{{.Stylesheet}}

Frontend Sample App

			{{.CatBody}}

`

type ErrorPage struct {
	Stylesheet template.HTML
	Error      error
}

var errorPageTemplate string = `
<!DOCTYPE html>

		<title>Frontend</title>
		<meta charset="utf-8">
		<meta http-equiv="X-UA-Compatible" content="IE=edge">
		<meta name="viewport" content="width=device-width, initial-scale=1">
		{{.Stylesheet}}

Frontend Sample App

request failed: {{.Error}}

`

type InfoHandler struct {
	Port int
}

func (h *InfoHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	template := template.Must(template.New("formPage").Parse(formPageTemplate))
	err := template.Execute(resp, FormPage{
		Stylesheet:  stylesheet,
		Cachebuster: rand.Int(),
	})
	if err != nil {
		panic(err)
	}
	return
}

type ProxyHandler struct{}

func (h *ProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	queryParams, err := url.ParseQuery(req.URL.RawQuery)
	if err != nil {
		panic(err)
	}
	destination := queryParams["url"][0]
	destination = "http://" + destination
	httpClient := http.DefaultClient
	httpClient.Timeout = 5 * time.Second
	getResp, err := httpClient.Get(destination)
	if err != nil {
		template := template.Must(template.New("formPage").Parse(errorPageTemplate))
		err = template.Execute(resp, ErrorPage{
			Stylesheet: stylesheet,
			Error:      err,
		})
		if err != nil {
			panic(err)
		}

		return
	}
	defer getResp.Body.Close()

	readBytes, err := ioutil.ReadAll(getResp.Body)
	if err != nil {
		resp.WriteHeader(http.StatusInternalServerError)
		resp.Write([]byte(fmt.Sprintf("read body failed: %s", err)))
		return
	}

	theTemplate := template.Must(template.New("proxyPage").Parse(proxyPageTemplate))
	catBody := template.HTML(string(readBytes))
	err = theTemplate.Execute(resp, ProxyPage{
		Stylesheet: stylesheet,
		CatBody:    catBody,
	})
	if err != nil {
		panic(err)
	}

	return
}

func launchHandler(port int, proxyHandler http.Handler) {
	mux := http.NewServeMux()
	mux.Handle("/proxy/", proxyHandler)
	mux.Handle("/", &InfoHandler{
		Port: port,
	})
	http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", port), mux)
}

func main() {
	systemPortString := os.Getenv("PORT")
	systemPort, err := strconv.Atoi(systemPortString)
	if err != nil {
		log.Fatal("invalid required env var PORT")
	}

	proxyHandler := &ProxyHandler{}

	launchHandler(systemPort, proxyHandler)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync/atomic"

	log "github.com/sirupsen/logrus"

	"github.com/calebamiles/keps/helpers/convert/internal/convert"
)

func main() {
	if len(os.Args) != 2 { // PROGNAME is the 0th argument
		log.Fatal("must only specifiy KEP location as argument")
	}

	startLocation := os.Args[1]

	outputDir, err := ioutil.TempDir("", "kep-conversion-helper")
	if err != nil {
		log.Fatalf("creating output directory")
	}

	var filesToSkip = map[string]bool{
		"0004-cloud-provider-template.md": true,
	}

	var possibleKEPs uint32 = 0
	var convertedKEPs uint32 = 0

	err = filepath.Walk(startLocation, func(path string, info os.FileInfo, err error) error {
		ext := filepath.Ext(path)
		if ext != ".md" {
			return nil
		}

		if filesToSkip[info.Name()] {
			log.Infof("skipping known file: %s", info.Name())
			return nil
		}

		content, err := ioutil.ReadFile(path)
		if err != nil {
			log.Fatalf("could not read content at: %s", path)
			return nil
		}

		if !bytes.Contains(content, []byte("---")) {
			// document unlikely to contain KEP metadata
			return nil
		}

		atomic.AddUint32(&possibleKEPs, 1)

		convertedKEPLocation, err := convert.ToCurrent(outputDir, path)
		if err != nil {
			log.Errorf("failed to convert possible KEP at: %s with error: %s", path, err)
			return nil // keep going and try to convert the next KEP
		}

		atomic.AddUint32(&convertedKEPs, 1)
		log.Infof("converted KEP saved at: %s", convertedKEPLocation)

		return nil
	})

	fmt.Println("converted KEP content located at: " + outputDir)
	fmt.Printf("converted %d out of %d KEPs: %f percent success rate\n", atomic.LoadUint32(&convertedKEPs), atomic.LoadUint32(&possibleKEPs), 100.0*float32(atomic.LoadUint32(&convertedKEPs))/float32(atomic.LoadUint32(&possibleKEPs)))
	os.Exit(0)
}
<commit_msg>helpers: add list of KEPs which could not be converted to output<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"

	log "github.com/sirupsen/logrus"

	"github.com/calebamiles/keps/helpers/convert/internal/convert"
)

func main() {
	if len(os.Args) != 2 { // PROGNAME is the 0th argument
		log.Fatal("must only specifiy KEP location as argument")
	}

	startLocation := os.Args[1]

	outputDir, err := ioutil.TempDir("", "kep-conversion-helper")
	if err != nil {
		log.Fatalf("creating output directory")
	}

	var filesToSkip = map[string]bool{
		"0000-kep-template.md":            true,
		"0004-cloud-provider-template.md": true,
	}

	var possibleKEPs uint32 = 0
	var convertedKEPs uint32 = 0

	var failedPaths []string

	err = filepath.Walk(startLocation, func(path string, info os.FileInfo, err error) error {
		ext := filepath.Ext(path)
		if ext != ".md" {
			return nil
		}

		if filesToSkip[info.Name()] {
			log.Infof("skipping known file: %s", info.Name())
			return nil
		}

		content, err := ioutil.ReadFile(path)
		if err != nil {
			log.Fatalf("could not read content at: %s", path)
			return nil
		}

		if !bytes.Contains(content, []byte("---")) {
			// document unlikely to contain KEP metadata
			return nil
		}

		atomic.AddUint32(&possibleKEPs, 1)

		convertedKEPLocation, err := convert.ToCurrent(outputDir, path)
		if err != nil {
			log.Errorf("failed to convert possible KEP at: %s with error: %s", path, err)
			failedPaths = append(failedPaths, path)
			return nil // keep going and try to convert the next KEP
		}

		atomic.AddUint32(&convertedKEPs, 1)
		log.Infof("converted KEP saved at: %s", convertedKEPLocation)

		return nil
	})

	fmt.Println("")
	fmt.Println("converted KEP content located at: " + outputDir)
	fmt.Printf("converted %d out of %d KEPs: %f percent success rate\n", atomic.LoadUint32(&convertedKEPs), atomic.LoadUint32(&possibleKEPs), 100.0*float32(atomic.LoadUint32(&convertedKEPs))/float32(atomic.LoadUint32(&possibleKEPs)))
	fmt.Printf("failed conversions: \n\n%s", strings.Join(failedPaths, " \n"))
	fmt.Println("")
	os.Exit(0)
}
<|endoftext|>


<commit_before>//  Copyright (c) 2019 Cisco and/or its affiliates.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at:
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package watcher

import (
	"context"
	"fmt"
	"strings"

	"go.ligato.io/cn-infra/v2/datasync"
	"go.ligato.io/cn-infra/v2/datasync/kvdbsync"
	"go.ligato.io/cn-infra/v2/datasync/kvdbsync/local"
	"go.ligato.io/cn-infra/v2/datasync/resync"
	"go.ligato.io/cn-infra/v2/datasync/syncbase"
	"go.ligato.io/cn-infra/v2/infra"
	"go.ligato.io/cn-infra/v2/logging"
	"go.ligato.io/cn-infra/v2/utils/safeclose"
	"go.ligato.io/vpp-agent/v3/plugins/orchestrator/localregistry"
)

// Option is a function that acts on a Plugin to inject Dependencies or configuration
type Option func(*Aggregator)

// UseWatchers returns option that sets watchers.
func UseWatchers(watchers ...datasync.KeyValProtoWatcher) Option {
	return func(p *Aggregator) {
		p.Watchers = watchers
	}
}

// Aggregator is an adapter that allows multiple
// watchers (KeyValProtoWatcher) to be aggregated in one.
// Watch request is delegated to all of them.
type Aggregator struct {
	infra.PluginDeps

	keyPrefixes []string
	localKVs    map[string]datasync.KeyVal

	Resync   *resync.Plugin
	Local    *syncbase.Registry
	Watchers []datasync.KeyValProtoWatcher
}

// NewPlugin creates a new Plugin with the provides Options
func NewPlugin(opts ...Option) *Aggregator {
	p := &Aggregator{}

	p.PluginName = "aggregator"
	p.Local = local.DefaultRegistry
	p.Resync = &resync.DefaultPlugin

	for _, o := range opts {
		o(p)
	}
	p.PluginDeps.SetupLog()

	return p
}

func (p *Aggregator) Init() error {
	p.localKVs = map[string]datasync.KeyVal{}
	return nil
}

// Watch subscribes to every transport available within transport aggregator
// and also subscribes to localclient (local.Registry).
// The function implements KeyValProtoWatcher.Watch().
func (p *Aggregator) Watch(
	resyncName string,
	changeChan chan datasync.ChangeEvent,
	resyncChan chan datasync.ResyncEvent,
	keyPrefixes ...string,
) (datasync.WatchRegistration, error) {

	p.keyPrefixes = keyPrefixes

	// prepare list of watchers
	var watchers []datasync.KeyValProtoWatcher
	for _, w := range p.Watchers {
		if l, ok := w.(*syncbase.Registry); ok && p.Local != nil && l == p.Local {
			p.Log.Warn("found local registry (localclient) in watchers, ignoring it..")
			continue
		}
		// ignoring watchers that have data sources that will be never used and
		// therefore never send configuration data to this aggregator
		if syncer, ok := w.(*kvdbsync.Plugin); ok {
			if syncer.KvPlugin != nil && syncer.KvPlugin.Disabled() {
				continue
			}
		}
		// TODO Handle kvdbsync.Plugin watchers that are not disabled, but won't transmit any resync data.
		//  This aggregator collects resyncs from all watchers, but if one or more resync from watcher don't happen,
		//  the agregator resyncing won't be triggered. This might happen when one of watchers is not registered
		//  to resync plugin and that is usually due to not connecting to data source as the registration to
		//  resync plugin happen in OnConnect callback function.
		//  To properly handle the situation, OnConnect callback must be used to distinguish what watched is
		//  reached by resync trigger. That might lead to delayed readiness of all watchers and hence the trigger
		//  for initial call of DoResync() to do initial Agent resync must be delayed as well SOMEHOW (can't use
		//  init or after init of plugins).

		// localregistry.InitFileRegistry can also be watcher that never sends anything (i.e. if misconfigured
		// or no default init file is present or loading of data fails or ... -> check for Empty())
		if initRegistry, ok := w.(*localregistry.InitFileRegistry); ok && initRegistry.Empty() {
			continue
		}

		watchers = append(watchers, w)
	}
	p.Watchers = watchers

	// start watch for all watchers
	p.Log.Infof("Watch for %v with %d prefixes", resyncName, len(keyPrefixes))

	aggrResync := make(chan datasync.ResyncEvent, len(watchers))

	go p.watchAggrResync(aggrResync, resyncChan)

	var registrations []datasync.WatchRegistration
	for i, adapter := range watchers {
		partChange := make(chan datasync.ChangeEvent)
		partResync := make(chan datasync.ResyncEvent)

		name := fmt.Sprint(adapter) + "/" + resyncName
		watcherReg, err := adapter.Watch(name, partChange, partResync, keyPrefixes...)
		if err != nil {
			return nil, err
		}

		go func(i int, chanChange chan datasync.ChangeEvent, chanResync chan datasync.ResyncEvent) {
			for {
				select {
				case e := <-chanChange:
					p.Log.Debugf("watcher %d got CHANGE PART, sending to aggregated", i)
					changeChan <- e

				case e := <-chanResync:
					p.Log.Debugf("watcher %d got RESYNC PART, sending to aggregated", i)
					aggrResync <- e
				}
			}
		}(i+1, partChange, partResync)

		if watcherReg != nil {
			registrations = append(registrations, watcherReg)
		}
	}

	// register and watch for localclient
	partResync := make(chan datasync.ResyncEvent)
	partChange := make(chan datasync.ChangeEvent)

	go p.watchLocalEvents(partChange, changeChan, partResync)

	name := "LOCAL" + "/" + resyncName
	localReg, err := p.Local.Watch(name, partChange, partResync, keyPrefixes...)
	if err != nil {
		return nil, err
	}

	p.Log.Debug("added localclient as aggregated watcher")

	registrations = append(registrations, localReg)

	return &WatchRegistration{
		Registrations: registrations,
	}, nil
}

func (p *Aggregator) watchAggrResync(aggrResync, resyncCh chan datasync.ResyncEvent) {
	aggregatedResync := func(allResyncs []datasync.ResyncEvent) {
		var prefixKeyVals = map[string]map[string]datasync.KeyVal{}

		kvToKeyVals := func(prefix string, kv datasync.KeyVal) {
			keyVals, ok := prefixKeyVals[prefix]
			if !ok {
				p.Log.Debugf(" - keyval prefix: %v", prefix)
				keyVals = map[string]datasync.KeyVal{}
				prefixKeyVals[prefix] = keyVals
			}
			key := kv.GetKey()
			if _, ok := keyVals[key]; ok {
				p.Log.Warnf("resync from watcher overwrites key: %v", key)
			}
			keyVals[key] = kv
		}

		// process resync events from all watchers
		p.Log.Debugf("preparing keyvals for aggregated resync from %d cached resyncs", len(allResyncs))
		for _, ev := range allResyncs {
			for prefix, iterator := range ev.GetValues() {
				for {
					kv, allReceived := iterator.GetNext()
					if allReceived {
						break
					}

					kvToKeyVals(prefix, kv)
				}
			}
		}

		// process keyvals from localclient
		p.Log.Debugf("preparing localclient keyvals for aggregated resync with %d keyvals", len(allResyncs))
		for key, kv := range p.localKVs {
			var kvprefix string
			for _, prefix := range p.keyPrefixes {
				if strings.HasPrefix(key, prefix) {
					kvprefix = prefix
					break
				}
			}
			if kvprefix == "" {
				p.Log.Warnf("not found registered prefix for keyval from localclient with key: %v", key)
			}
			kvToKeyVals(kvprefix, kv)
		}

		// prepare aggregated resync
		var vals = map[string]datasync.KeyValIterator{}
		for prefix, keyVals := range prefixKeyVals {
			var data []datasync.KeyVal
			for _, kv := range keyVals {
				data = append(data, kv)
			}
			vals[prefix] = syncbase.NewKVIterator(data)
		}
		resEv := syncbase.NewResyncEventDB(context.Background(), vals)

		p.Log.Debugf("sending aggregated resync event (%d prefixes) to original resync channel", len(vals))
		resyncCh <- resEv
		p.Log.Debugf("aggregated resync was accepted, waiting for done chan")
		resErr := <-resEv.DoneChan
		p.Log.Debugf("aggregated resync done (err=%v) watchers", resErr)

	}

	var cachedResyncs []datasync.ResyncEvent

	// process resync events from watchers
	for {
		select {
		case e, ok := <-aggrResync:
			if !ok {
				p.Log.Debugf("aggrResync channel was closed")
				return
			}

			cachedResyncs = append(cachedResyncs, e)
			p.Log.Debugf("watchers received resync event (%d/%d watchers done)", len(cachedResyncs), len(p.Watchers))

			e.Done(nil)
		}

		if len(cachedResyncs) == len(p.Watchers) {
			p.Log.Debug("resyncs from all watchers received, calling aggregated resync")
			aggregatedResync(cachedResyncs)
			// clear resyncs
			cachedResyncs = nil
		}
	}
}

func (p *Aggregator) watchLocalEvents(partChange, changeChan chan datasync.ChangeEvent, partResync chan datasync.ResyncEvent) {
	for {
		select {
		case e := <-partChange:
			p.Log.Debugf("LOCAL got CHANGE part, %d changes, sending to aggregated", len(e.GetChanges()))

			for _, change := range e.GetChanges() {
				key := change.GetKey()
				switch change.GetChangeType() {
				case datasync.Delete:
					p.Log.Debugf(" - DEL %s", key)
					delete(p.localKVs, key)
				case datasync.Put:
					p.Log.Debugf(" - PUT %s", key)
					p.localKVs[key] = change
				}
			}
			changeChan <- e

		case e := <-partResync:
			p.Log.Debugf("LOCAL watcher got RESYNC part, sending to aggregated")

			p.localKVs = map[string]datasync.KeyVal{}
			for _, iterator := range e.GetValues() {
				for {
					kv, allReceived := iterator.GetNext()
					if allReceived {
						break
					}

					key := kv.GetKey()
					p.localKVs[key] = kv
				}
			}
			p.Log.Debugf("LOCAL watcher resynced %d keyvals", len(p.localKVs))
			e.Done(nil)

			p.Log.Debug("LOCAL watcher calling RESYNC")
			p.Resync.DoResync() // execution will appear in p.watchAggrResync go routine where p.localKVs will handled
		}
	}
}

// WatchRegistration is adapter that allows multiple
// registrations (WatchRegistration) to be aggregated in one.
// Close operation is applied collectively to all included registration.
type WatchRegistration struct {
	Registrations []datasync.WatchRegistration
}

// Register new key for all available aggregator objects. Call Register(keyPrefix) on specific registration
// to add the key from that registration only
func (wa *WatchRegistration) Register(resyncName, keyPrefix string) error {
	for _, registration := range wa.Registrations {
		if err := registration.Register(resyncName, keyPrefix); err != nil {
			logging.DefaultLogger.Warnf("aggregated register failed: %v", err)
		}
	}

	return nil
}

// Unregister closed registration of specific key under all available aggregator objects.
// Call Unregister(keyPrefix) on specific registration to remove the key from that registration only
func (wa *WatchRegistration) Unregister(keyPrefix string) error {
	for _, registration := range wa.Registrations {
		if err := registration.Unregister(keyPrefix); err != nil {
			logging.DefaultLogger.Warnf("aggregated unregister failed: %v", err)
		}
	}

	return nil
}

// Close every registration under the aggregator.
// This function implements WatchRegistration.Close().
func (wa *WatchRegistration) Close() error {
	return safeclose.Close(wa.Registrations)
}
<commit_msg>fix: fixed delete by resync failure of init file data by using agentctl (#1782)<commit_after>//  Copyright (c) 2019 Cisco and/or its affiliates.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at:
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package watcher

import (
	"context"
	"fmt"
	"strings"

	"go.ligato.io/cn-infra/v2/config"
	"go.ligato.io/cn-infra/v2/datasync"
	"go.ligato.io/cn-infra/v2/datasync/kvdbsync"
	"go.ligato.io/cn-infra/v2/datasync/kvdbsync/local"
	"go.ligato.io/cn-infra/v2/datasync/resync"
	"go.ligato.io/cn-infra/v2/datasync/syncbase"
	"go.ligato.io/cn-infra/v2/infra"
	"go.ligato.io/cn-infra/v2/logging"
	"go.ligato.io/cn-infra/v2/utils/safeclose"
	"go.ligato.io/vpp-agent/v3/plugins/orchestrator/contextdecorator"
	"go.ligato.io/vpp-agent/v3/plugins/orchestrator/localregistry"
)

// Option is a function that acts on a Plugin to inject Dependencies or configuration
type Option func(*Aggregator)

// UseWatchers returns option that sets watchers.
func UseWatchers(watchers ...datasync.KeyValProtoWatcher) Option {
	return func(p *Aggregator) {
		p.Watchers = watchers
	}
}

// Aggregator is an adapter that allows multiple
// watchers (KeyValProtoWatcher) to be aggregated in one.
// Watch request is delegated to all of them.
type Aggregator struct {
	infra.PluginDeps

	keyPrefixes []string
	localKVs    map[string]datasync.KeyVal
	config      *Config

	Resync   *resync.Plugin
	Local    *syncbase.Registry
	Watchers []datasync.KeyValProtoWatcher
}

// Config holds the Aggregator configuration.
type Config struct {
	// ResyncDataSourceOverride overrides default data source (empty in aggregator and later elsewhere
	// set to "datasync") to support certain use cases where data sources must match otherwise resync doesn't
	// affect the same set of data (i.e. using only initfile watcher to fill initial data and agentctl resync
	// to clean them -> agentctl resync works only on 'grpc'-sourced data and default 'datasync'-sourced
	// initfile data couldn't be handled)
	// This is not a full solution covering all combinations of watchers and agentctl resync, but rather
	// a possible fix for some use cases. The full solution should handle multiple resyncs (one per data source)
	// and all its corner cases.
	ResyncDataSourceOverride string `json:"resync-data-source-override"`
}

// NewPlugin creates a new Plugin with the provides Options
func NewPlugin(opts ...Option) *Aggregator {
	p := &Aggregator{}

	p.PluginName = "aggregator"
	p.Local = local.DefaultRegistry
	p.Resync = &resync.DefaultPlugin

	for _, o := range opts {
		o(p)
	}
	if p.Cfg == nil {
		p.Cfg = config.ForPlugin(p.String(),
			config.WithCustomizedFlag(config.FlagName(p.String()), "aggregator.conf"),
		)
	}
	p.PluginDeps.SetupLog()

	return p
}

func (p *Aggregator) Init() error {
	p.localKVs = map[string]datasync.KeyVal{}

	// parse configuration file
	var err error
	p.config, err = p.retrieveConfig()
	if err != nil {
		return err
	}

	return nil
}

// Watch subscribes to every transport available within transport aggregator
// and also subscribes to localclient (local.Registry).
// The function implements KeyValProtoWatcher.Watch().
func (p *Aggregator) Watch(
	resyncName string,
	changeChan chan datasync.ChangeEvent,
	resyncChan chan datasync.ResyncEvent,
	keyPrefixes ...string,
) (datasync.WatchRegistration, error) {

	p.keyPrefixes = keyPrefixes

	// prepare list of watchers
	var watchers []datasync.KeyValProtoWatcher
	for _, w := range p.Watchers {
		if l, ok := w.(*syncbase.Registry); ok && p.Local != nil && l == p.Local {
			p.Log.Warn("found local registry (localclient) in watchers, ignoring it..")
			continue
		}
		// ignoring watchers that have data sources that will be never used and
		// therefore never send configuration data to this aggregator
		if syncer, ok := w.(*kvdbsync.Plugin); ok {
			if syncer.KvPlugin != nil && syncer.KvPlugin.Disabled() {
				continue
			}
		}
		// TODO Handle kvdbsync.Plugin watchers that are not disabled, but won't transmit any resync data.
		//  This aggregator collects resyncs from all watchers, but if one or more resync from watcher don't happen,
		//  the agregator resyncing won't be triggered. This might happen when one of watchers is not registered
		//  to resync plugin and that is usually due to not connecting to data source as the registration to
		//  resync plugin happen in OnConnect callback function.
		//  To properly handle the situation, OnConnect callback must be used to distinguish what watched is
		//  reached by resync trigger. That might lead to delayed readiness of all watchers and hence the trigger
		//  for initial call of DoResync() to do initial Agent resync must be delayed as well SOMEHOW (can't use
		//  init or after init of plugins).

		// localregistry.InitFileRegistry can also be watcher that never sends anything (i.e. if misconfigured
		// or no default init file is present or loading of data fails or ... -> check for Empty())
		if initRegistry, ok := w.(*localregistry.InitFileRegistry); ok && initRegistry.Empty() {
			continue
		}

		watchers = append(watchers, w)
	}
	p.Watchers = watchers

	// start watch for all watchers
	p.Log.Infof("Watch for %v with %d prefixes", resyncName, len(keyPrefixes))

	aggrResync := make(chan datasync.ResyncEvent, len(watchers))

	go p.watchAggrResync(aggrResync, resyncChan)

	var registrations []datasync.WatchRegistration
	for i, adapter := range watchers {
		partChange := make(chan datasync.ChangeEvent)
		partResync := make(chan datasync.ResyncEvent)

		name := fmt.Sprint(adapter) + "/" + resyncName
		watcherReg, err := adapter.Watch(name, partChange, partResync, keyPrefixes...)
		if err != nil {
			return nil, err
		}

		go func(i int, chanChange chan datasync.ChangeEvent, chanResync chan datasync.ResyncEvent) {
			for {
				select {
				case e := <-chanChange:
					p.Log.Debugf("watcher %d got CHANGE PART, sending to aggregated", i)
					changeChan <- e

				case e := <-chanResync:
					p.Log.Debugf("watcher %d got RESYNC PART, sending to aggregated", i)
					aggrResync <- e
				}
			}
		}(i+1, partChange, partResync)

		if watcherReg != nil {
			registrations = append(registrations, watcherReg)
		}
	}

	// register and watch for localclient
	partResync := make(chan datasync.ResyncEvent)
	partChange := make(chan datasync.ChangeEvent)

	go p.watchLocalEvents(partChange, changeChan, partResync)

	name := "LOCAL" + "/" + resyncName
	localReg, err := p.Local.Watch(name, partChange, partResync, keyPrefixes...)
	if err != nil {
		return nil, err
	}

	p.Log.Debug("added localclient as aggregated watcher")

	registrations = append(registrations, localReg)

	return &WatchRegistration{
		Registrations: registrations,
	}, nil
}

func (p *Aggregator) watchAggrResync(aggrResync, resyncCh chan datasync.ResyncEvent) {
	aggregatedResync := func(allResyncs []datasync.ResyncEvent) {
		var prefixKeyVals = map[string]map[string]datasync.KeyVal{}

		kvToKeyVals := func(prefix string, kv datasync.KeyVal) {
			keyVals, ok := prefixKeyVals[prefix]
			if !ok {
				p.Log.Debugf(" - keyval prefix: %v", prefix)
				keyVals = map[string]datasync.KeyVal{}
				prefixKeyVals[prefix] = keyVals
			}
			key := kv.GetKey()
			if _, ok := keyVals[key]; ok {
				p.Log.Warnf("resync from watcher overwrites key: %v", key)
			}
			keyVals[key] = kv
		}

		// process resync events from all watchers
		p.Log.Debugf("preparing keyvals for aggregated resync from %d cached resyncs", len(allResyncs))
		for _, ev := range allResyncs {
			for prefix, iterator := range ev.GetValues() {
				for {
					kv, allReceived := iterator.GetNext()
					if allReceived {
						break
					}

					kvToKeyVals(prefix, kv)
				}
			}
		}

		// process keyvals from localclient
		p.Log.Debugf("preparing localclient keyvals for aggregated resync with %d keyvals", len(allResyncs))
		for key, kv := range p.localKVs {
			var kvprefix string
			for _, prefix := range p.keyPrefixes {
				if strings.HasPrefix(key, prefix) {
					kvprefix = prefix
					break
				}
			}
			if kvprefix == "" {
				p.Log.Warnf("not found registered prefix for keyval from localclient with key: %v", key)
			}
			kvToKeyVals(kvprefix, kv)
		}

		// prepare aggregated resync
		var vals = map[string]datasync.KeyValIterator{}
		for prefix, keyVals := range prefixKeyVals {
			var data []datasync.KeyVal
			for _, kv := range keyVals {
				data = append(data, kv)
			}
			vals[prefix] = syncbase.NewKVIterator(data)
		}

		ctx := context.Background()
		if p.config.ResyncDataSourceOverride != "" {
			ctx = contextdecorator.DataSrcContext(ctx, p.config.ResyncDataSourceOverride)
		}
		resEv := syncbase.NewResyncEventDB(ctx, vals)

		p.Log.Debugf("sending aggregated resync event (%d prefixes) to original resync channel", len(vals))
		resyncCh <- resEv
		p.Log.Debugf("aggregated resync was accepted, waiting for done chan")
		resErr := <-resEv.DoneChan
		p.Log.Debugf("aggregated resync done (err=%v) watchers", resErr)

	}

	var cachedResyncs []datasync.ResyncEvent

	// process resync events from watchers
	for {
		select {
		case e, ok := <-aggrResync:
			if !ok {
				p.Log.Debugf("aggrResync channel was closed")
				return
			}

			cachedResyncs = append(cachedResyncs, e)
			p.Log.Debugf("watchers received resync event (%d/%d watchers done)", len(cachedResyncs), len(p.Watchers))

			e.Done(nil)
		}

		if len(cachedResyncs) == len(p.Watchers) {
			p.Log.Debug("resyncs from all watchers received, calling aggregated resync")
			aggregatedResync(cachedResyncs)
			// clear resyncs
			cachedResyncs = nil
		}
	}
}

func (p *Aggregator) watchLocalEvents(partChange, changeChan chan datasync.ChangeEvent, partResync chan datasync.ResyncEvent) {
	for {
		select {
		case e := <-partChange:
			p.Log.Debugf("LOCAL got CHANGE part, %d changes, sending to aggregated", len(e.GetChanges()))

			for _, change := range e.GetChanges() {
				key := change.GetKey()
				switch change.GetChangeType() {
				case datasync.Delete:
					p.Log.Debugf(" - DEL %s", key)
					delete(p.localKVs, key)
				case datasync.Put:
					p.Log.Debugf(" - PUT %s", key)
					p.localKVs[key] = change
				}
			}
			changeChan <- e

		case e := <-partResync:
			p.Log.Debugf("LOCAL watcher got RESYNC part, sending to aggregated")

			p.localKVs = map[string]datasync.KeyVal{}
			for _, iterator := range e.GetValues() {
				for {
					kv, allReceived := iterator.GetNext()
					if allReceived {
						break
					}

					key := kv.GetKey()
					p.localKVs[key] = kv
				}
			}
			p.Log.Debugf("LOCAL watcher resynced %d keyvals", len(p.localKVs))
			e.Done(nil)

			p.Log.Debug("LOCAL watcher calling RESYNC")
			p.Resync.DoResync() // execution will appear in p.watchAggrResync go routine where p.localKVs will handled
		}
	}
}

// retrieveConfig loads Aggregator plugin configuration file.
func (p *Aggregator) retrieveConfig() (*Config, error) {
	config := &Config{
		// default configuration
		ResyncDataSourceOverride: "", // don't override
	}
	found, err := p.Cfg.LoadValue(config)
	if !found {
		if err == nil {
			p.Log.Debug("Aggregator plugin config not found")
		} else {
			p.Log.Debugf("Aggregator plugin config can't be loaded due to: %v", err)
		}
		return config, err
	}
	if err != nil {
		return nil, err
	}
	return config, err
}

// WatchRegistration is adapter that allows multiple
// registrations (WatchRegistration) to be aggregated in one.
// Close operation is applied collectively to all included registration.
type WatchRegistration struct {
	Registrations []datasync.WatchRegistration
}

// Register new key for all available aggregator objects. Call Register(keyPrefix) on specific registration
// to add the key from that registration only
func (wa *WatchRegistration) Register(resyncName, keyPrefix string) error {
	for _, registration := range wa.Registrations {
		if err := registration.Register(resyncName, keyPrefix); err != nil {
			logging.DefaultLogger.Warnf("aggregated register failed: %v", err)
		}
	}

	return nil
}

// Unregister closed registration of specific key under all available aggregator objects.
// Call Unregister(keyPrefix) on specific registration to remove the key from that registration only
func (wa *WatchRegistration) Unregister(keyPrefix string) error {
	for _, registration := range wa.Registrations {
		if err := registration.Unregister(keyPrefix); err != nil {
			logging.DefaultLogger.Warnf("aggregated unregister failed: %v", err)
		}
	}

	return nil
}

// Close every registration under the aggregator.
// This function implements WatchRegistration.Close().
func (wa *WatchRegistration) Close() error {
	return safeclose.Close(wa.Registrations)
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"fmt"
	"net/http"
	"strings"
	"sync"
	"unicode"

	"contrib.go.opencensus.io/exporter/prometheus"
	"github.com/google/uuid"
	"go.opencensus.io/stats/view"
	otelprometheus "go.opentelemetry.io/otel/exporters/prometheus"
	"go.opentelemetry.io/otel/metric/global"
	export "go.opentelemetry.io/otel/sdk/export/metric"
	"go.opentelemetry.io/otel/sdk/metric/aggregator/histogram"
	controller "go.opentelemetry.io/otel/sdk/metric/controller/basic"
	processor "go.opentelemetry.io/otel/sdk/metric/processor/basic"
	selector "go.opentelemetry.io/otel/sdk/metric/selector/simple"
	"go.uber.org/zap"

	"go.opentelemetry.io/collector/config/configtelemetry"
	"go.opentelemetry.io/collector/internal/obsreportconfig"
	"go.opentelemetry.io/collector/internal/version"
	semconv "go.opentelemetry.io/collector/model/semconv/v1.5.0"
	"go.opentelemetry.io/collector/processor/batchprocessor"
	telemetry2 "go.opentelemetry.io/collector/service/internal/telemetry"
)

// collectorTelemetry is collector's own telemetry.
var collectorTelemetry collectorTelemetryExporter = &colTelemetry{}

// AddCollectorVersionTag indicates if the collector version tag should be added to all telemetry metrics
const AddCollectorVersionTag = true

type collectorTelemetryExporter interface {
	init(asyncErrorChannel chan<- error, ballastSizeBytes uint64, logger *zap.Logger) error
	shutdown() error
}

type colTelemetry struct {
	views      []*view.View
	server     *http.Server
	doInitOnce sync.Once
}

func (tel *colTelemetry) init(asyncErrorChannel chan<- error, ballastSizeBytes uint64, logger *zap.Logger) error {
	var err error
	tel.doInitOnce.Do(
		func() {
			err = tel.initOnce(asyncErrorChannel, ballastSizeBytes, logger)
		},
	)
	if err != nil {
		return fmt.Errorf("failed to initialize telemetry: %w", err)
	}
	return nil
}

func (tel *colTelemetry) initOnce(asyncErrorChannel chan<- error, ballastSizeBytes uint64, logger *zap.Logger) error {
	logger.Info("Setting up own telemetry...")

	level := configtelemetry.GetMetricsLevelFlagValue()
	metricsAddr := getMetricsAddr()

	if level == configtelemetry.LevelNone || metricsAddr == "" {
		return nil
	}

	var instanceID string

	if getAddInstanceID() {
		instanceUUID, _ := uuid.NewRandom()
		instanceID = instanceUUID.String()
	}

	var pe http.Handler
	if configtelemetry.UseOpenTelemetryForInternalMetrics {
		otelHandler, err := tel.initOpenTelemetry()
		if err != nil {
			return err
		}
		pe = otelHandler
	} else {
		ocHandler, err := tel.initOpenCensus(level, instanceID, ballastSizeBytes)
		if err != nil {
			return err
		}
		pe = ocHandler
	}

	logger.Info(
		"Serving Prometheus metrics",
		zap.String("address", metricsAddr),
		zap.Int8("level", int8(level)), // TODO: make it human friendly
		zap.String(semconv.AttributeServiceInstanceID, instanceID),
		zap.String(semconv.AttributeServiceVersion, version.Version),
	)

	mux := http.NewServeMux()
	mux.Handle("/metrics", pe)

	tel.server = &http.Server{
		Addr:    metricsAddr,
		Handler: mux,
	}

	go func() {
		serveErr := tel.server.ListenAndServe()
		if serveErr != nil && serveErr != http.ErrServerClosed {
			asyncErrorChannel <- serveErr
		}
	}()

	return nil
}

func (tel *colTelemetry) initOpenCensus(level configtelemetry.Level, instanceID string, ballastSizeBytes uint64) (http.Handler, error) {
	processMetricsViews, err := telemetry2.NewProcessMetricsViews(ballastSizeBytes)
	if err != nil {
		return nil, err
	}

	var views []*view.View
	obsMetrics := obsreportconfig.Configure(level)
	views = append(views, batchprocessor.MetricViews()...)
	views = append(views, obsMetrics.Views...)
	views = append(views, processMetricsViews.Views()...)

	tel.views = views
	if err = view.Register(views...); err != nil {
		return nil, err
	}

	processMetricsViews.StartCollection()

	// Until we can use a generic metrics exporter, default to Prometheus.
	opts := prometheus.Options{
		Namespace: getMetricsPrefix(),
	}

	opts.ConstLabels = make(map[string]string)

	if getAddInstanceID() {
		opts.ConstLabels[sanitizePrometheusKey(semconv.AttributeServiceInstanceID)] = instanceID
	}

	if AddCollectorVersionTag {
		opts.ConstLabels[sanitizePrometheusKey(semconv.AttributeServiceVersion)] = version.Version
	}

	pe, err := prometheus.NewExporter(opts)
	if err != nil {
		return nil, err
	}

	view.RegisterExporter(pe)
	return pe, nil
}

func (tel *colTelemetry) initOpenTelemetry() (http.Handler, error) {
	config := otelprometheus.Config{}
	c := controller.New(
		processor.NewFactory(
			selector.NewWithHistogramDistribution(
				histogram.WithExplicitBoundaries(config.DefaultHistogramBoundaries),
			),
			export.CumulativeExportKindSelector(),
			processor.WithMemory(true),
		),
	)

	pe, err := otelprometheus.New(config, c)
	if err != nil {
		return nil, err
	}

	global.SetMeterProvider(pe.MeterProvider())
	return pe, err
}

func (tel *colTelemetry) shutdown() error {
	view.Unregister(tel.views...)

	if tel.server != nil {
		return tel.server.Close()
	}

	return nil
}

func sanitizePrometheusKey(str string) string {
	runeFilterMap := func(r rune) rune {
		if unicode.IsDigit(r) || unicode.IsLetter(r) || r == '_' {
			return r
		}
		return '_'
	}
	return strings.Map(runeFilterMap, str)
}
<commit_msg>Improve logging on telemetry initialization (#4189)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"fmt"
	"net/http"
	"strings"
	"sync"
	"unicode"

	"contrib.go.opencensus.io/exporter/prometheus"
	"github.com/google/uuid"
	"go.opencensus.io/stats/view"
	otelprometheus "go.opentelemetry.io/otel/exporters/prometheus"
	"go.opentelemetry.io/otel/metric/global"
	export "go.opentelemetry.io/otel/sdk/export/metric"
	"go.opentelemetry.io/otel/sdk/metric/aggregator/histogram"
	controller "go.opentelemetry.io/otel/sdk/metric/controller/basic"
	processor "go.opentelemetry.io/otel/sdk/metric/processor/basic"
	selector "go.opentelemetry.io/otel/sdk/metric/selector/simple"
	"go.uber.org/zap"

	"go.opentelemetry.io/collector/config/configtelemetry"
	"go.opentelemetry.io/collector/internal/obsreportconfig"
	"go.opentelemetry.io/collector/internal/version"
	semconv "go.opentelemetry.io/collector/model/semconv/v1.5.0"
	"go.opentelemetry.io/collector/processor/batchprocessor"
	telemetry2 "go.opentelemetry.io/collector/service/internal/telemetry"
)

// collectorTelemetry is collector's own telemetry.
var collectorTelemetry collectorTelemetryExporter = &colTelemetry{}

// AddCollectorVersionTag indicates if the collector version tag should be added to all telemetry metrics
const AddCollectorVersionTag = true

const (
	zapKeyTelemetryAddress = "address"
	zapKeyTelemetryLevel   = "level"
)

type collectorTelemetryExporter interface {
	init(asyncErrorChannel chan<- error, ballastSizeBytes uint64, logger *zap.Logger) error
	shutdown() error
}

type colTelemetry struct {
	views      []*view.View
	server     *http.Server
	doInitOnce sync.Once
}

func (tel *colTelemetry) init(asyncErrorChannel chan<- error, ballastSizeBytes uint64, logger *zap.Logger) error {
	var err error
	tel.doInitOnce.Do(
		func() {
			err = tel.initOnce(asyncErrorChannel, ballastSizeBytes, logger)
		},
	)
	if err != nil {
		return fmt.Errorf("failed to initialize telemetry: %w", err)
	}
	return nil
}

func (tel *colTelemetry) initOnce(asyncErrorChannel chan<- error, ballastSizeBytes uint64, logger *zap.Logger) error {
	level := configtelemetry.GetMetricsLevelFlagValue()
	metricsAddr := getMetricsAddr()

	if level == configtelemetry.LevelNone || metricsAddr == "" {
		logger.Info(
			"Skipping telemetry setup.",
			zap.String(zapKeyTelemetryAddress, metricsAddr),
			zap.String(zapKeyTelemetryLevel, level.String()),
		)
		return nil
	}

	logger.Info("Setting up own telemetry...")

	var instanceID string

	if getAddInstanceID() {
		instanceUUID, _ := uuid.NewRandom()
		instanceID = instanceUUID.String()
	}

	var pe http.Handler
	if configtelemetry.UseOpenTelemetryForInternalMetrics {
		otelHandler, err := tel.initOpenTelemetry()
		if err != nil {
			return err
		}
		pe = otelHandler
	} else {
		ocHandler, err := tel.initOpenCensus(level, instanceID, ballastSizeBytes)
		if err != nil {
			return err
		}
		pe = ocHandler
	}

	logger.Info(
		"Serving Prometheus metrics",
		zap.String(zapKeyTelemetryAddress, metricsAddr),
		zap.String(zapKeyTelemetryLevel, level.String()),
		zap.String(semconv.AttributeServiceInstanceID, instanceID),
		zap.String(semconv.AttributeServiceVersion, version.Version),
	)

	mux := http.NewServeMux()
	mux.Handle("/metrics", pe)

	tel.server = &http.Server{
		Addr:    metricsAddr,
		Handler: mux,
	}

	go func() {
		serveErr := tel.server.ListenAndServe()
		if serveErr != nil && serveErr != http.ErrServerClosed {
			asyncErrorChannel <- serveErr
		}
	}()

	return nil
}

func (tel *colTelemetry) initOpenCensus(level configtelemetry.Level, instanceID string, ballastSizeBytes uint64) (http.Handler, error) {
	processMetricsViews, err := telemetry2.NewProcessMetricsViews(ballastSizeBytes)
	if err != nil {
		return nil, err
	}

	var views []*view.View
	obsMetrics := obsreportconfig.Configure(level)
	views = append(views, batchprocessor.MetricViews()...)
	views = append(views, obsMetrics.Views...)
	views = append(views, processMetricsViews.Views()...)

	tel.views = views
	if err = view.Register(views...); err != nil {
		return nil, err
	}

	processMetricsViews.StartCollection()

	// Until we can use a generic metrics exporter, default to Prometheus.
	opts := prometheus.Options{
		Namespace: getMetricsPrefix(),
	}

	opts.ConstLabels = make(map[string]string)

	if getAddInstanceID() {
		opts.ConstLabels[sanitizePrometheusKey(semconv.AttributeServiceInstanceID)] = instanceID
	}

	if AddCollectorVersionTag {
		opts.ConstLabels[sanitizePrometheusKey(semconv.AttributeServiceVersion)] = version.Version
	}

	pe, err := prometheus.NewExporter(opts)
	if err != nil {
		return nil, err
	}

	view.RegisterExporter(pe)
	return pe, nil
}

func (tel *colTelemetry) initOpenTelemetry() (http.Handler, error) {
	config := otelprometheus.Config{}
	c := controller.New(
		processor.NewFactory(
			selector.NewWithHistogramDistribution(
				histogram.WithExplicitBoundaries(config.DefaultHistogramBoundaries),
			),
			export.CumulativeExportKindSelector(),
			processor.WithMemory(true),
		),
	)

	pe, err := otelprometheus.New(config, c)
	if err != nil {
		return nil, err
	}

	global.SetMeterProvider(pe.MeterProvider())
	return pe, err
}

func (tel *colTelemetry) shutdown() error {
	view.Unregister(tel.views...)

	if tel.server != nil {
		return tel.server.Close()
	}

	return nil
}

func sanitizePrometheusKey(str string) string {
	runeFilterMap := func(r rune) rune {
		if unicode.IsDigit(r) || unicode.IsLetter(r) || r == '_' {
			return r
		}
		return '_'
	}
	return strings.Map(runeFilterMap, str)
}
<|endoftext|>


<commit_before>// Peano integers are represented by a linked
// list whose nodes contain no data
// (the nodes are the data).
// http://en.wikipedia.org/wiki/Peano_axioms

// This program demonstrates the power of Go's
// segmented stacks when doing massively
// recursive computations.

package main

import "fmt"

// Number is a pointer to a Number
type Number *Number

// The arithmetic value of a Number is the
// count of the nodes comprising the list.
// (See the count function below.)

// -------------------------------------
// Peano primitives

func zero() *Number {
	return nil
}

func isZero(x *Number) bool {
	return x == nil
}

func add1(x *Number) *Number {
	e := new(Number)
	*e = x
	return e
}

func sub1(x *Number) *Number {
	return *x
}

func add(x, y *Number) *Number {
	if isZero(y) {
		return x
	}
	return add(add1(x), sub1(y))
}

func mul(x, y *Number) *Number {
	if isZero(x) || isZero(y) {
		return zero()
	}
	return add(mul(x, sub1(y)), x)
}

func fact(n *Number) *Number {
	if isZero(n) {
		return add1(zero())
	}
	return mul(fact(sub1(n)), n)
}

// -------------------------------------
// Helpers to generate/count Peano integers

func gen(n int) *Number {
	if n > 0 {
		return add1(gen(n - 1))
	}
	return zero()
}

func count(x *Number) int {
	if isZero(x) {
		return 0
	}
	return count(sub1(x)) + 1
}

// -------------------------------------
// Print i! for i in [0,9]

func main() {
	for i := 0; i <= 9; i++ {
		f := count(fact(gen(i)))
		fmt.Println(i, "! =", f)
	}
}
<commit_msg>doc/play: update obsolete comment in peano.go.<commit_after>// Peano integers are represented by a linked
// list whose nodes contain no data
// (the nodes are the data).
// http://en.wikipedia.org/wiki/Peano_axioms

// This program demonstrates the effectiveness
// of the Go runtime's dynamically growing
// stacks for heavily recursive computations.

package main

import "fmt"

// Number is a pointer to a Number
type Number *Number

// The arithmetic value of a Number is the
// count of the nodes comprising the list.
// (See the count function below.)

// -------------------------------------
// Peano primitives

func zero() *Number {
	return nil
}

func isZero(x *Number) bool {
	return x == nil
}

func add1(x *Number) *Number {
	e := new(Number)
	*e = x
	return e
}

func sub1(x *Number) *Number {
	return *x
}

func add(x, y *Number) *Number {
	if isZero(y) {
		return x
	}
	return add(add1(x), sub1(y))
}

func mul(x, y *Number) *Number {
	if isZero(x) || isZero(y) {
		return zero()
	}
	return add(mul(x, sub1(y)), x)
}

func fact(n *Number) *Number {
	if isZero(n) {
		return add1(zero())
	}
	return mul(fact(sub1(n)), n)
}

// -------------------------------------
// Helpers to generate/count Peano integers

func gen(n int) *Number {
	if n > 0 {
		return add1(gen(n - 1))
	}
	return zero()
}

func count(x *Number) int {
	if isZero(x) {
		return 0
	}
	return count(sub1(x)) + 1
}

// -------------------------------------
// Print i! for i in [0,9]

func main() {
	for i := 0; i <= 9; i++ {
		f := count(fact(gen(i)))
		fmt.Println(i, "! =", f)
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The template uses the function "code" to inject program
// source into the output by extracting code from files and
// injecting them as HTML-escaped
 blocks.
//
// The syntax is simple: 1, 2, or 3 space-separated arguments:
//
// Whole file:
//	{{code "foo.go"}}
// One line (here the signature of main):
//	{{code "foo.go" `/^func.main/`}}
// Block of text, determined by start and end (here the body of main):
//	{{code "foo.go" `/^func.main/` `/^}/`
//
// Patterns can be `/regular expression/`, a decimal number, or "$"
// to signify the end of the file.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"
	"text/template"
)

func Usage() {
	fmt.Fprintf(os.Stderr, "usage: tmpltohtml file\n")
	os.Exit(2)
}

func main() {
	flag.Usage = Usage
	flag.Parse()
	if len(flag.Args()) != 1 {
		Usage()
	}

	// Read and parse the input.
	name := flag.Args()[0]
	tmpl := template.New(name).Funcs(template.FuncMap{"code": code})
	if _, err := tmpl.ParseFiles(name); err != nil {
		log.Fatal(err)
	}

	// Execute the template.
	if err := tmpl.Execute(os.Stdout, 0); err != nil {
		log.Fatal(err)
	}
}

// contents reads a file by name and returns its contents as a string.
func contents(name string) string {
	file, err := ioutil.ReadFile(name)
	if err != nil {
		log.Fatal(err)
	}
	return string(file)
}

// format returns a textual representation of the arg, formatted according to its nature.
func format(arg interface{}) string {
	switch arg := arg.(type) {
	case int:
		return fmt.Sprintf("%d", arg)
	case string:
		if len(arg) > 2 && arg[0] == '/' && arg[len(arg)-1] == '/' {
			return fmt.Sprintf("%#q", arg)
		}
		return fmt.Sprintf("%q", arg)
	default:
		log.Fatalf("unrecognized argument: %v type %T", arg, arg)
	}
	return ""
}

func code(file string, arg ...interface{}) (string, error) {
	text := contents(file)
	var command string
	switch len(arg) {
	case 0:
		// text is already whole file.
		command = fmt.Sprintf("code %q", file)
	case 1:
		command = fmt.Sprintf("code %q %s", file, format(arg[0]))
		text = oneLine(file, text, arg[0])
	case 2:
		command = fmt.Sprintf("code %q %s %s", file, format(arg[0]), format(arg[1]))
		text = multipleLines(file, text, arg[0], arg[1])
	default:
		return "", fmt.Errorf("incorrect code invocation: code %q %q", file, arg)
	}
	// Replace tabs by spaces, which work better in HTML.
	text = strings.Replace(text, "\t", "    ", -1)
	// Escape the program text for HTML.
	text = template.HTMLEscapeString(text)
	// Include the command as a comment.
	text = fmt.Sprintf("
%s
", command, text)
	return text, nil
}

// parseArg returns the integer or string value of the argument and tells which it is.
func parseArg(arg interface{}, file string, max int) (ival int, sval string, isInt bool) {
	switch n := arg.(type) {
	case int:
		if n <= 0 || n > max {
			log.Fatalf("%q:%d is out of range", file, n)
		}
		return n, "", true
	case string:
		return 0, n, false
	}
	log.Fatalf("unrecognized argument %v type %T", arg, arg)
	return
}

// oneLine returns the single line generated by a two-argument code invocation.
func oneLine(file, text string, arg interface{}) string {
	lines := strings.SplitAfter(contents(file), "\n")
	line, pattern, isInt := parseArg(arg, file, len(lines))
	if isInt {
		return lines[line-1]
	}
	return lines[match(file, 0, lines, pattern)-1]
}

// multipleLines returns the text generated by a three-argument code invocation.
func multipleLines(file, text string, arg1, arg2 interface{}) string {
	lines := strings.SplitAfter(contents(file), "\n")
	line1, pattern1, isInt1 := parseArg(arg1, file, len(lines))
	line2, pattern2, isInt2 := parseArg(arg2, file, len(lines))
	if !isInt1 {
		line1 = match(file, 0, lines, pattern1)
	}
	if !isInt2 {
		line2 = match(file, line1, lines, pattern2)
	} else if line2 < line1 {
		log.Fatal("lines out of order for %q: %d %d", line1, line2)
	}
	return strings.Join(lines[line1-1:line2], "")
}

// match identifies the input line that matches the pattern in a code invocation.
// If start>0, match lines starting there rather than at the beginning.
// The return value is 1-indexed.
func match(file string, start int, lines []string, pattern string) int {
	// $ matches the end of the file.
	if pattern == "$" {
		if len(lines) == 0 {
			log.Fatal("%q: empty file", file)
		}
		return len(lines)
	}
	// /regexp/ matches the line that matches the regexp.
	if len(pattern) > 2 && pattern[0] == '/' && pattern[len(pattern)-1] == '/' {
		re, err := regexp.Compile(pattern[1 : len(pattern)-1])
		if err != nil {
			log.Fatal(err)
		}
		for i := start; i < len(lines); i++ {
			if re.MatchString(lines[i]) {
				return i + 1
			}
		}
		log.Fatalf("%s: no match for %#q", file, pattern)
	}
	log.Fatalf("unrecognized pattern: %q", pattern)
	return 0
}
<commit_msg>doc/tmptohtml: output fix<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The template uses the function "code" to inject program
// source into the output by extracting code from files and
// injecting them as HTML-escaped
 blocks.
//
// The syntax is simple: 1, 2, or 3 space-separated arguments:
//
// Whole file:
//	{{code "foo.go"}}
// One line (here the signature of main):
//	{{code "foo.go" `/^func.main/`}}
// Block of text, determined by start and end (here the body of main):
//	{{code "foo.go" `/^func.main/` `/^}/`
//
// Patterns can be `/regular expression/`, a decimal number, or "$"
// to signify the end of the file.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"
	"text/template"
)

func Usage() {
	fmt.Fprintf(os.Stderr, "usage: tmpltohtml file\n")
	os.Exit(2)
}

func main() {
	flag.Usage = Usage
	flag.Parse()
	if len(flag.Args()) != 1 {
		Usage()
	}

	// Read and parse the input.
	name := flag.Args()[0]
	tmpl := template.New(name).Funcs(template.FuncMap{"code": code})
	if _, err := tmpl.ParseFiles(name); err != nil {
		log.Fatal(err)
	}

	// Execute the template.
	if err := tmpl.Execute(os.Stdout, 0); err != nil {
		log.Fatal(err)
	}
}

// contents reads a file by name and returns its contents as a string.
func contents(name string) string {
	file, err := ioutil.ReadFile(name)
	if err != nil {
		log.Fatal(err)
	}
	return string(file)
}

// format returns a textual representation of the arg, formatted according to its nature.
func format(arg interface{}) string {
	switch arg := arg.(type) {
	case int:
		return fmt.Sprintf("%d", arg)
	case string:
		if len(arg) > 2 && arg[0] == '/' && arg[len(arg)-1] == '/' {
			return fmt.Sprintf("%#q", arg)
		}
		return fmt.Sprintf("%q", arg)
	default:
		log.Fatalf("unrecognized argument: %v type %T", arg, arg)
	}
	return ""
}

func code(file string, arg ...interface{}) (string, error) {
	text := contents(file)
	var command string
	switch len(arg) {
	case 0:
		// text is already whole file.
		command = fmt.Sprintf("code %q", file)
	case 1:
		command = fmt.Sprintf("code %q %s", file, format(arg[0]))
		text = oneLine(file, text, arg[0])
	case 2:
		command = fmt.Sprintf("code %q %s %s", file, format(arg[0]), format(arg[1]))
		text = multipleLines(file, text, arg[0], arg[1])
	default:
		return "", fmt.Errorf("incorrect code invocation: code %q %q", file, arg)
	}
	// Replace tabs by spaces, which work better in HTML.
	text = strings.Replace(text, "\t", "    ", -1)
	// Escape the program text for HTML.
	text = template.HTMLEscapeString(text)
	// Include the command as a comment.
	text = fmt.Sprintf("
%s
", command, text)
	return text, nil
}

// parseArg returns the integer or string value of the argument and tells which it is.
func parseArg(arg interface{}, file string, max int) (ival int, sval string, isInt bool) {
	switch n := arg.(type) {
	case int:
		if n <= 0 || n > max {
			log.Fatalf("%q:%d is out of range", file, n)
		}
		return n, "", true
	case string:
		return 0, n, false
	}
	log.Fatalf("unrecognized argument %v type %T", arg, arg)
	return
}

// oneLine returns the single line generated by a two-argument code invocation.
func oneLine(file, text string, arg interface{}) string {
	lines := strings.SplitAfter(contents(file), "\n")
	line, pattern, isInt := parseArg(arg, file, len(lines))
	if isInt {
		return lines[line-1]
	}
	return lines[match(file, 0, lines, pattern)-1]
}

// multipleLines returns the text generated by a three-argument code invocation.
func multipleLines(file, text string, arg1, arg2 interface{}) string {
	lines := strings.SplitAfter(contents(file), "\n")
	line1, pattern1, isInt1 := parseArg(arg1, file, len(lines))
	line2, pattern2, isInt2 := parseArg(arg2, file, len(lines))
	if !isInt1 {
		line1 = match(file, 0, lines, pattern1)
	}
	if !isInt2 {
		line2 = match(file, line1, lines, pattern2)
	} else if line2 < line1 {
		log.Fatalf("lines out of order for %q: %d %d", text, line1, line2)
	}
	return strings.Join(lines[line1-1:line2], "")
}

// match identifies the input line that matches the pattern in a code invocation.
// If start>0, match lines starting there rather than at the beginning.
// The return value is 1-indexed.
func match(file string, start int, lines []string, pattern string) int {
	// $ matches the end of the file.
	if pattern == "$" {
		if len(lines) == 0 {
			log.Fatalf("%q: empty file", file)
		}
		return len(lines)
	}
	// /regexp/ matches the line that matches the regexp.
	if len(pattern) > 2 && pattern[0] == '/' && pattern[len(pattern)-1] == '/' {
		re, err := regexp.Compile(pattern[1 : len(pattern)-1])
		if err != nil {
			log.Fatal(err)
		}
		for i := start; i < len(lines); i++ {
			if re.MatchString(lines[i]) {
				return i + 1
			}
		}
		log.Fatalf("%s: no match for %#q", file, pattern)
	}
	log.Fatalf("unrecognized pattern: %q", pattern)
	return 0
}
<|endoftext|>