<commit_before>// Copyright © 2011-12 Qtrac Ltd.
//
// This program or package and any associated files are licensed under the
// Apache License, Version 2.0 (the "License"); you may not use these files
// except in compliance with the License. You can get a copy of the License
// at: http://www.apache.org/licenses/LICENSE-2.0.
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The approach taken here was inspired by an example on the gonuts mailing
// list by Roger Peppe.
//
// REVISION HISTORY
// ----------------
// 20 Mar 20 -- Made comparisons case insensitive.  And decided to make this cgrepi.go

package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"runtime"
	"strings"
	"time"
)

var workers = runtime.NumCPU()

type Result struct {
	filename string
	lino     int
	line     string
}

type Job struct {
	filename string
	results  chan<- Result
}

func (job Job) Do(lineRx *regexp.Regexp) {
	file, err := os.Open(job.filename)
	if err != nil {
		log.Printf("error: %s\n", err)
		return
	}
	defer file.Close()
	reader := bufio.NewReader(file)
	for lino := 1; ; lino++ {
		line, err := reader.ReadBytes('\n')
		line = bytes.TrimRight(line, "\n\r")

		// this is the change I made to make every comparison case insensitive
		linestr := string(line)
		linestr = strings.ToLower(linestr)
		line = []byte(linestr)

		if lineRx.Match(line) {
			job.results <- Result{job.filename, lino, string(line)}
		}
		if err != nil {
			if err != io.EOF {
				log.Printf("error:%d: %s\n", lino, err)
			}
			break
		}
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU()) // Use all the machine's cores
	log.SetFlags(0)
	var timeoutOpt *int64 = flag.Int64("timeout", 0, "seconds (0 means no timeout)")
	flag.Parse()
	if *timeoutOpt < 0 || *timeoutOpt > 240 {
		log.Fatalln("timeout must be in the range [0,240] seconds")
	}
	args := flag.Args()
	if len(args) < 1 {
		log.Fatalln("a regexp to match must be specified")
	}
	pattern := args[0]
	pattern = strings.ToLower(pattern) // this is the change for the pattern.
	files := args[1:]
	if len(files) < 1 {
		log.Fatalln("must provide at least one filename")
	}
	if lineRx, err := regexp.Compile(pattern); err != nil {
		log.Fatalf("invalid regexp: %s\n", err)
	} else {
		var timeout int64 = 1e9 * 60 * 10 // 10 minutes!
		if *timeoutOpt != 0 {
			timeout = *timeoutOpt * 1e9
		}
		grep(timeout, lineRx, commandLineFiles(files))  // this fails vet because it's in the platform specific code files.
	}
}

func grep(timeout int64, lineRx *regexp.Regexp, filenames []string) {
	jobs := make(chan Job, workers)
	results := make(chan Result, minimum(1000, len(filenames)))
	done := make(chan struct{}, workers)

	go addJobs(jobs, filenames, results)
	for i := 0; i < workers; i++ {
		go doJobs(done, lineRx, jobs)
	}
	waitAndProcessResults(timeout, done, results)
}

func addJobs(jobs chan<- Job, filenames []string, results chan<- Result) {
	for _, filename := range filenames {
		jobs <- Job{filename, results}
	}
	close(jobs)
}

func doJobs(done chan<- struct{}, lineRx *regexp.Regexp, jobs <-chan Job) {
	for job := range jobs {
		job.Do(lineRx)
	}
	done <- struct{}{}
}

func waitAndProcessResults(timeout int64, done <-chan struct{},
	results <-chan Result) {
	finish := time.After(time.Duration(timeout))
	for working := workers; working > 0; {
		select { // Blocking
		case result := <-results:
			fmt.Printf("%s:%d:%s\n", result.filename, result.lino,
				result.line)
		case <-finish:
			fmt.Println("timed out")
			return // Time's up so finish with what results there were
		case <-done:
			working--
		}
	}
	for {
		select { // Nonblocking
		case result := <-results:
			fmt.Printf("%s:%d:%s\n", result.filename, result.lino,
				result.line)
		case <-finish:
			fmt.Println("timed out")
			return // Time's up so finish with what results there were
		default:
			return
		}
	}
}

func minimum(x int, ys ...int) int {
	for _, y := range ys {
		if y < x {
			x = y
		}
	}
	return x
}
<commit_msg>03/20/2020 10:58:35 AM cgrepi/cgrep.go -- decided I could not make it any faster, so I stopped.<commit_after>// Copyright © 2011-12 Qtrac Ltd.
//
// This program or package and any associated files are licensed under the
// Apache License, Version 2.0 (the "License"); you may not use these files
// except in compliance with the License. You can get a copy of the License
// at: http://www.apache.org/licenses/LICENSE-2.0.
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The approach taken here was inspired by an example on the gonuts mailing
// list by Roger Peppe.
/*
  REVISION HISTORY
  ----------------
  20 Mar 20 -- Made comparisons case insensitive.  And decided to make this cgrepi.go.
                 And then I figured I could not improve performance by using more packages.

 */
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"runtime"
	"strings"
	"time"
)

var workers = runtime.NumCPU()

type Result struct {
	filename string
	lino     int
	line     string
}

type Job struct {
	filename string
	results  chan<- Result
}

func (job Job) Do(lineRx *regexp.Regexp) {
	file, err := os.Open(job.filename)
	if err != nil {
		log.Printf("error: %s\n", err)
		return
	}
	defer file.Close()
	reader := bufio.NewReader(file)
	for lino := 1; ; lino++ {
		line, err := reader.ReadBytes('\n')
		line = bytes.TrimRight(line, "\n\r")

		// this is the change I made to make every comparison case insensitive
		linestr := string(line)
		linestr = strings.ToLower(linestr)
		line = []byte(linestr)

		if lineRx.Match(line) {
			job.results <- Result{job.filename, lino, string(line)}
		}
		if err != nil {
			if err != io.EOF {
				log.Printf("error:%d: %s\n", lino, err)
			}
			break
		}
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU()) // Use all the machine's cores
	log.SetFlags(0)
	var timeoutOpt *int64 = flag.Int64("timeout", 0, "seconds (0 means no timeout)")
	flag.Parse()
	if *timeoutOpt < 0 || *timeoutOpt > 240 {
		log.Fatalln("timeout must be in the range [0,240] seconds")
	}
	args := flag.Args()
	if len(args) < 1 {
		log.Fatalln("a regexp to match must be specified")
	}
	pattern := args[0]
	pattern = strings.ToLower(pattern) // this is the change for the pattern.
	files := args[1:]
	if len(files) < 1 {
		log.Fatalln("must provide at least one filename")
	}
	if lineRx, err := regexp.Compile(pattern); err != nil {
		log.Fatalf("invalid regexp: %s\n", err)
	} else {
		var timeout int64 = 1e9 * 60 * 10 // 10 minutes!
		if *timeoutOpt != 0 {
			timeout = *timeoutOpt * 1e9
		}
		grep(timeout, lineRx, commandLineFiles(files))  // this fails vet because it's in the platform specific code files.
	}
}

func grep(timeout int64, lineRx *regexp.Regexp, filenames []string) {
	jobs := make(chan Job, workers)
	results := make(chan Result, minimum(1000, len(filenames)))
	done := make(chan struct{}, workers)

	go addJobs(jobs, filenames, results)
	for i := 0; i < workers; i++ {
		go doJobs(done, lineRx, jobs)
	}
	waitAndProcessResults(timeout, done, results)
}

func addJobs(jobs chan<- Job, filenames []string, results chan<- Result) {
	for _, filename := range filenames {
		jobs <- Job{filename, results}
	}
	close(jobs)
}

func doJobs(done chan<- struct{}, lineRx *regexp.Regexp, jobs <-chan Job) {
	for job := range jobs {
		job.Do(lineRx)
	}
	done <- struct{}{}
}

func waitAndProcessResults(timeout int64, done <-chan struct{},
	results <-chan Result) {
	finish := time.After(time.Duration(timeout))
	for working := workers; working > 0; {
		select { // Blocking
		case result := <-results:
			fmt.Printf("%s:%d:%s\n", result.filename, result.lino,
				result.line)
		case <-finish:
			fmt.Println("timed out")
			return // Time's up so finish with what results there were
		case <-done:
			working--
		}
	}
	for {
		select { // Nonblocking
		case result := <-results:
			fmt.Printf("%s:%d:%s\n", result.filename, result.lino,
				result.line)
		case <-finish:
			fmt.Println("timed out")
			return // Time's up so finish with what results there were
		default:
			return
		}
	}
}

func minimum(x int, ys ...int) int {
	for _, y := range ys {
		if y < x {
			x = y
		}
	}
	return x
}
<|endoftext|>


<commit_before>// +build linux

package hsup

import (
	"bufio"
	"compress/gzip"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/htcat/htcat"
	"gopkg.in/yaml.v2"
)

const (
	// currently available versions of Heroku stack images
	HEROKU_STACKS_MANIFEST_URL = "https://s3.amazonaws.com/heroku_stacks_production/manifest.yml"
)

// HerokuStackImage models stack images as they are distributed by Heroku:
// binary disk images, usually intended to be mounted on loopback devices.
// The common use is to mount those images read-only, so a single immutable
// image can be shared by multiple containers, saving disk space and memory
// due to Linux CoW page sharing.
//
// Support for Heroku stack images is currently only enabled when building for
// Linux, because these images are currently only used by the libcontainer
// driver.
type HerokuStackImage struct {
	Name    string
	Version string
	URL     string `yaml:"url"`
	Md5     string
	Primary bool

	basedir string
}

func HerokuStacksFromManifest(stacksDir string) ([]HerokuStackImage, error) {
	// TODO: cache the manifest
	resp, err := http.Get(HEROKU_STACKS_MANIFEST_URL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf(
			"invalid stacks manifest at %q",
			HEROKU_STACKS_MANIFEST_URL,
		)
	}
	manifest, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var stacks []HerokuStackImage
	if err := yaml.Unmarshal(manifest, &stacks); err != nil {
		return nil, err
	}
	for i := range stacks {
		stacks[i].basedir = stacksDir
	}
	return stacks, nil
}

func CurrentStackImagePath(stacksDir, name string) (string, error) {
	// TODO: check if it is really mounted
	// TODO: order by version numbers and pick the last
	names, err := filepath.Glob(filepath.Join(stacksDir, name+"-*"))
	if err != nil {
		return "", err
	}
	last := len(names) - 1
	for i := range names {
		n := names[last-i]
		if !strings.HasSuffix(n, ".img") {
			return n, nil
		}
	}

	return "", errors.New("no matching stack image found")
}

func (img *HerokuStackImage) Dir() string {
	return filepath.Join(img.basedir, img.Name+"-"+img.Version)
}

func (img *HerokuStackImage) Filename() string {
	return img.Dir() + ".img"
}

//TODO: avoid multiple processes trying to mount the same stack image
func (img *HerokuStackImage) mount() error {
	var (
		imgFile = img.Filename()
		imgDir  = img.Dir()
	)
	if _, err := os.Stat(imgFile); err != nil {
		if err := img.fetch(); err != nil {
			return err
		}
	}
	if err := os.MkdirAll(imgDir, 0755); err != nil {
		return err
	}
	if contents, err := ioutil.ReadDir(imgDir); err != nil {
		return err
	} else if len(contents) != 0 {
		return nil // already mounted
	}
	log.Printf("Mounting stack image %q onto %q", imgFile, imgDir)
	out, err := exec.Command(
		"/bin/mount", "-o", "loop,nosuid,nodev,noatime,nodiratime,rw",
		imgFile, imgDir,
	).CombinedOutput()
	if err != nil {
		log.Println(string(out))
		return err
	}
	if err := img.addMissingDirectories(); err != nil {
		return err
	}
	return syscall.Mount("", imgDir, "", syscall.MS_REMOUNT|syscall.MS_RDONLY, "")

}

// addMissingDirectories is required until https://github.com/heroku/stack-images/pull/13
// gets merged.
func (img *HerokuStackImage) addMissingDirectories() error {
	return os.MkdirAll(filepath.Join(img.Dir(), "sys"), 0755)
}

//TODO: avoid multiple processes trying to fetch the same stack image
func (img *HerokuStackImage) fetch() error {
	log.Printf("Downloading stack image %q. This may take a while...", img.Name)
	// TODO check md5
	pr, pw := io.Pipe()
	defer pr.Close()

	u, err := url.Parse(img.URL)
	if err != nil {
		return err
	}

	// buffered channel: avoid blocking the writer goroutine
	dlResult := make(chan error, 1)
	go func() {
		// close the pipe as soon as the download finishes so readers receive EOF
		defer pw.Close()
		client := *http.DefaultClient
		if u.Scheme == "https" {
			client.Transport = &http.Transport{
				TLSClientConfig: &tls.Config{},
			}
		}
		// buffer the download in case gunzip can't keep up
		w := bufio.NewWriter(pw)
		_, err := htcat.New(&client, u, 5).WriteTo(w)
		w.Flush()
		dlResult <- err
	}()

	r, err := gzip.NewReader(pr)
	if err != nil {
		return err
	}
	defer r.Close()

	imageFile, err := os.Create(img.Filename())
	if err != nil {
		return err
	}
	defer imageFile.Close()
	if err := imageFile.Chmod(0644); err != nil {
		return err
	}

	if _, err := io.Copy(imageFile, r); err != nil {
		return err
	}
	log.Println("Stack image download finished")
	return <-dlResult
}
<commit_msg>Use idiomatic CamelCase for constant<commit_after>// +build linux

package hsup

import (
	"bufio"
	"compress/gzip"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/htcat/htcat"
	"gopkg.in/yaml.v2"
)

const (
	// currently available versions of Heroku stack images
	HerokuStacksManifestURL = "https://s3.amazonaws.com/heroku_stacks_production/manifest.yml"
)

// HerokuStackImage models stack images as they are distributed by Heroku:
// binary disk images, usually intended to be mounted on loopback devices.
// The common use is to mount those images read-only, so a single immutable
// image can be shared by multiple containers, saving disk space and memory
// due to Linux CoW page sharing.
//
// Support for Heroku stack images is currently only enabled when building for
// Linux, because these images are currently only used by the libcontainer
// driver.
type HerokuStackImage struct {
	Name    string
	Version string
	URL     string `yaml:"url"`
	Md5     string
	Primary bool

	basedir string
}

func HerokuStacksFromManifest(stacksDir string) ([]HerokuStackImage, error) {
	// TODO: cache the manifest
	resp, err := http.Get(HerokuStacksManifestURL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf(
			"invalid stacks manifest at %q",
			HerokuStacksManifestURL,
		)
	}
	manifest, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var stacks []HerokuStackImage
	if err := yaml.Unmarshal(manifest, &stacks); err != nil {
		return nil, err
	}
	for i := range stacks {
		stacks[i].basedir = stacksDir
	}
	return stacks, nil
}

func CurrentStackImagePath(stacksDir, name string) (string, error) {
	// TODO: check if it is really mounted
	// TODO: order by version numbers and pick the last
	names, err := filepath.Glob(filepath.Join(stacksDir, name+"-*"))
	if err != nil {
		return "", err
	}
	last := len(names) - 1
	for i := range names {
		n := names[last-i]
		if !strings.HasSuffix(n, ".img") {
			return n, nil
		}
	}

	return "", errors.New("no matching stack image found")
}

func (img *HerokuStackImage) Dir() string {
	return filepath.Join(img.basedir, img.Name+"-"+img.Version)
}

func (img *HerokuStackImage) Filename() string {
	return img.Dir() + ".img"
}

//TODO: avoid multiple processes trying to mount the same stack image
func (img *HerokuStackImage) mount() error {
	var (
		imgFile = img.Filename()
		imgDir  = img.Dir()
	)
	if _, err := os.Stat(imgFile); err != nil {
		if err := img.fetch(); err != nil {
			return err
		}
	}
	if err := os.MkdirAll(imgDir, 0755); err != nil {
		return err
	}
	if contents, err := ioutil.ReadDir(imgDir); err != nil {
		return err
	} else if len(contents) != 0 {
		return nil // already mounted
	}
	log.Printf("Mounting stack image %q onto %q", imgFile, imgDir)
	out, err := exec.Command(
		"/bin/mount", "-o", "loop,nosuid,nodev,noatime,nodiratime,rw",
		imgFile, imgDir,
	).CombinedOutput()
	if err != nil {
		log.Println(string(out))
		return err
	}
	if err := img.addMissingDirectories(); err != nil {
		return err
	}
	return syscall.Mount("", imgDir, "", syscall.MS_REMOUNT|syscall.MS_RDONLY, "")

}

// addMissingDirectories is required until https://github.com/heroku/stack-images/pull/13
// gets merged.
func (img *HerokuStackImage) addMissingDirectories() error {
	return os.MkdirAll(filepath.Join(img.Dir(), "sys"), 0755)
}

//TODO: avoid multiple processes trying to fetch the same stack image
func (img *HerokuStackImage) fetch() error {
	log.Printf("Downloading stack image %q. This may take a while...", img.Name)
	// TODO check md5
	pr, pw := io.Pipe()
	defer pr.Close()

	u, err := url.Parse(img.URL)
	if err != nil {
		return err
	}

	// buffered channel: avoid blocking the writer goroutine
	dlResult := make(chan error, 1)
	go func() {
		// close the pipe as soon as the download finishes so readers receive EOF
		defer pw.Close()
		client := *http.DefaultClient
		if u.Scheme == "https" {
			client.Transport = &http.Transport{
				TLSClientConfig: &tls.Config{},
			}
		}
		// buffer the download in case gunzip can't keep up
		w := bufio.NewWriter(pw)
		_, err := htcat.New(&client, u, 5).WriteTo(w)
		w.Flush()
		dlResult <- err
	}()

	r, err := gzip.NewReader(pr)
	if err != nil {
		return err
	}
	defer r.Close()

	imageFile, err := os.Create(img.Filename())
	if err != nil {
		return err
	}
	defer imageFile.Close()
	if err := imageFile.Chmod(0644); err != nil {
		return err
	}

	if _, err := io.Copy(imageFile, r); err != nil {
		return err
	}
	log.Println("Stack image download finished")
	return <-dlResult
}
<|endoftext|>


<commit_before>package rabbithole_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/streadway/amqp"
	. "rabbithole"
)

func FindQueueByName(qs []QueueInfo, name string) QueueInfo {
	var q QueueInfo
	for _, i := range qs {
		if i.Name == name {
			q = i
		}
	}

	return q
}

var _ = Describe("Client", func() {
	var (
		rmqc *Client
	)

	BeforeEach(func() {
		rmqc, _ = NewClient("http://127.0.0.1:15672", "guest", "guest")
	})

	Context("GET /overview", func() {
		It("returns decoded response", func() {
			res, err := rmqc.Overview()

			Ω(err).Should(BeNil())

			Ω(res.Node).ShouldNot(BeNil())
			Ω(res.StatisticsDBNode).ShouldNot(BeNil())

			fanoutExchange := ExchangeType{Name: "fanout", Description: "AMQP fanout exchange, as per the AMQP specification", Enabled: true}
			Ω(res.ExchangeTypes).Should(ContainElement(fanoutExchange))

		})
	})

	Context("GET /nodes", func() {
		It("returns decoded response", func() {
			xs, err := rmqc.ListNodes()
			res := xs[0]

			Ω(err).Should(BeNil())

			Ω(res.Name).ShouldNot(BeNil())
			Ω(res.NodeType).Should(Equal("disc"))

			Ω(res.FdUsed).Should(BeNumerically(">=", 0))
			Ω(res.FdTotal).Should(BeNumerically(">", 64))

			Ω(res.MemUsed).Should(BeNumerically(">", 10*1024*1024))
			Ω(res.MemLimit).Should(BeNumerically(">", 64*1024*1024))
			Ω(res.MemAlarm).Should(Equal(false))

			Ω(res.IsRunning).Should(Equal(true))

			Ω(res.SocketsUsed).Should(BeNumerically(">=", 0))
			Ω(res.SocketsTotal).Should(BeNumerically(">=", 1))

		})
	})

	Context("GET /connections when there are active connections", func() {
		It("returns decoded response", func() {
			// this really should be tested with > 1 connection and channel. MK.
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
			Ω(err).Should(BeNil())
			defer conn.Close()

			conn2, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
			Ω(err).Should(BeNil())
			defer conn2.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			ch2, err := conn2.Channel()
			Ω(err).Should(BeNil())
			defer ch2.Close()

			ch3, err := conn2.Channel()
			Ω(err).Should(BeNil())
			defer ch3.Close()

			err = ch.Publish("",
				"",
				false,
				false,
				amqp.Publishing{Body: []byte("")})
			Ω(err).Should(BeNil())

			xs, err := rmqc.ListConnections()
			Ω(err).Should(BeNil())

			info := xs[0]
			Ω(info.Name).ShouldNot(BeNil())
			Ω(info.Host).Should(Equal("127.0.0.1"))
			Ω(info.UsesTLS).Should(Equal(false))
		})
	})

	Context("GET /channels when there are active connections with open channels", func() {
		It("returns decoded response", func() {
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
			Ω(err).Should(BeNil())
			defer conn.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			ch2, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch2.Close()

			ch3, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch3.Close()

			ch4, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch4.Close()

			err = ch.Publish("",
				"",
				false,
				false,
				amqp.Publishing{Body: []byte("")})
			Ω(err).Should(BeNil())

			err = ch2.Publish("",
				"",
				false,
				false,
				amqp.Publishing{Body: []byte("")})
			Ω(err).Should(BeNil())

			xs, err := rmqc.ListChannels()
			Ω(err).Should(BeNil())

			info := xs[0]
			Ω(info.Node).ShouldNot(BeNil())
			Ω(info.User).Should(Equal("guest"))
			Ω(info.Vhost).Should(Equal("/"))

			Ω(info.Transactional).Should(Equal(false))

			Ω(info.UnacknowledgedMessageCount).Should(Equal(0))
			Ω(info.UnconfirmedMessageCount).Should(Equal(0))
			Ω(info.UncommittedMessageCount).Should(Equal(0))
			Ω(info.UncommittedAckCount).Should(Equal(0))
		})
	})

	Context("GET /connections/{name] when connection exists", func() {
		It("returns decoded response", func() {
			// this really should be tested with > 1 connection and channel. MK.
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
			Ω(err).Should(BeNil())
			defer conn.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			err = ch.Publish("",
				"",
				false,
				false,
				amqp.Publishing{Body: []byte("")})
			Ω(err).Should(BeNil())

			xs, err := rmqc.ListConnections()
			Ω(err).Should(BeNil())

			c1 := xs[0]
			info, err := rmqc.GetConnection(c1.Name)
			Ω(err).Should(BeNil())
			Ω(info.Protocol).Should(Equal("AMQP 0-9-1"))
			Ω(info.User).Should(Equal("guest"))
		})
	})

	Context("GET /channels/{name} when channel exists", func() {
		It("returns decoded response", func() {
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
			Ω(err).Should(BeNil())
			defer conn.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			err = ch.Publish("",
				"",
				false,
				false,
				amqp.Publishing{Body: []byte("")})
			Ω(err).Should(BeNil())

			xs, err := rmqc.ListChannels()
			Ω(err).Should(BeNil())

			x := xs[0]
			info, err := rmqc.GetChannel(x.Name)

			Ω(info.Node).ShouldNot(BeNil())
			Ω(info.User).Should(Equal("guest"))
			Ω(info.Vhost).Should(Equal("/"))

			Ω(info.Transactional).Should(Equal(false))

			Ω(info.UnacknowledgedMessageCount).Should(Equal(0))
			Ω(info.UnconfirmedMessageCount).Should(Equal(0))
			Ω(info.UncommittedMessageCount).Should(Equal(0))
			Ω(info.UncommittedAckCount).Should(Equal(0))
		})
	})

	Context("GET /exchanges", func() {
		It("returns decoded response", func() {
			xs, err := rmqc.ListExchanges()
			Ω(err).Should(BeNil())

			x := xs[0]
			Ω(x.Name).Should(Equal(""))
			Ω(x.Durable).Should(Equal(true))
		})
	})

	Context("GET /exchanges/{vhost}", func() {
		It("returns decoded response", func() {
			xs, err := rmqc.ListExchangesIn("/")
			Ω(err).Should(BeNil())

			x := xs[0]
			Ω(x.Name).Should(Equal(""))
			Ω(x.Durable).Should(Equal(true))
		})
	})

	Context("GET /exchanges/{vhost}/{name}", func() {
		It("returns decoded response", func() {
			x, err := rmqc.GetExchange("rabbit/hole", "amq.fanout")
			Ω(err).Should(BeNil())
			Ω(x.Name).Should(Equal("amq.fanout"))
			Ω(x.Durable).Should(Equal(true))
			Ω(x.Type).Should(Equal("fanout"))
			Ω(x.Vhost).Should(Equal("rabbit/hole"))
		})
	})

	Context("GET /queues", func() {
		It("returns decoded response", func() {
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672")
			Ω(err).Should(BeNil())
			defer conn.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			ch.QueueDeclare(
				"q1",  // name
				false, // durable
				false, // delete when usused
				true,  // exclusive
				false,
				nil)

			qs, err := rmqc.ListQueues()
			Ω(err).Should(BeNil())

			q := qs[0]
			Ω(q.Name).ShouldNot(Equal(""))
			Ω(q.Node).ShouldNot(BeNil())
			Ω(q.Durable).ShouldNot(BeNil())
			Ω(q.Status).ShouldNot(BeNil())
		})
	})

	Context("GET /queues/{vhost}", func() {
		It("returns decoded response", func() {
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/rabbit%2Fhole")
			Ω(err).Should(BeNil())
			defer conn.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			ch.QueueDeclare(
				"q1",  // name
				false, // durable
				false, // delete when usused
				true,  // exclusive
				false,
				nil)

			qs, err := rmqc.ListQueuesIn("rabbit/hole")
			Ω(err).Should(BeNil())

			q := FindQueueByName(qs, "q1")
			Ω(q.Name).Should(Equal("q1"))
			Ω(q.Vhost).Should(Equal("rabbit/hole"))
			Ω(q.Durable).Should(Equal(false))
			Ω(q.Status).ShouldNot(BeNil())
		})
	})

	Context("GET /queues/{vhost}/{name}", func() {
		It("returns decoded response", func() {
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/rabbit%2Fhole")
			Ω(err).Should(BeNil())
			defer conn.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			ch.QueueDeclare(
				"q1",  // name
				false, // durable
				false, // delete when usused
				true,  // exclusive
				false,
				nil)

			q, err := rmqc.GetQueue("rabbit/hole", "q1")
			Ω(err).Should(BeNil())
			Ω(q).Should(BeNil())

			Ω(q.Name).Should(Equal("q1"))
			Ω(q.Vhost).Should(Equal("rabbit/hole"))
			Ω(q.Durable).Should(Equal(false))
			Ω(q.Status).ShouldNot(BeNil())
		})
	})
})
<commit_msg>Remove a debug line<commit_after>package rabbithole_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/streadway/amqp"
	. "rabbithole"
)

func FindQueueByName(qs []QueueInfo, name string) QueueInfo {
	var q QueueInfo
	for _, i := range qs {
		if i.Name == name {
			q = i
		}
	}

	return q
}

var _ = Describe("Client", func() {
	var (
		rmqc *Client
	)

	BeforeEach(func() {
		rmqc, _ = NewClient("http://127.0.0.1:15672", "guest", "guest")
	})

	Context("GET /overview", func() {
		It("returns decoded response", func() {
			res, err := rmqc.Overview()

			Ω(err).Should(BeNil())

			Ω(res.Node).ShouldNot(BeNil())
			Ω(res.StatisticsDBNode).ShouldNot(BeNil())

			fanoutExchange := ExchangeType{Name: "fanout", Description: "AMQP fanout exchange, as per the AMQP specification", Enabled: true}
			Ω(res.ExchangeTypes).Should(ContainElement(fanoutExchange))

		})
	})

	Context("GET /nodes", func() {
		It("returns decoded response", func() {
			xs, err := rmqc.ListNodes()
			res := xs[0]

			Ω(err).Should(BeNil())

			Ω(res.Name).ShouldNot(BeNil())
			Ω(res.NodeType).Should(Equal("disc"))

			Ω(res.FdUsed).Should(BeNumerically(">=", 0))
			Ω(res.FdTotal).Should(BeNumerically(">", 64))

			Ω(res.MemUsed).Should(BeNumerically(">", 10*1024*1024))
			Ω(res.MemLimit).Should(BeNumerically(">", 64*1024*1024))
			Ω(res.MemAlarm).Should(Equal(false))

			Ω(res.IsRunning).Should(Equal(true))

			Ω(res.SocketsUsed).Should(BeNumerically(">=", 0))
			Ω(res.SocketsTotal).Should(BeNumerically(">=", 1))

		})
	})

	Context("GET /connections when there are active connections", func() {
		It("returns decoded response", func() {
			// this really should be tested with > 1 connection and channel. MK.
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
			Ω(err).Should(BeNil())
			defer conn.Close()

			conn2, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
			Ω(err).Should(BeNil())
			defer conn2.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			ch2, err := conn2.Channel()
			Ω(err).Should(BeNil())
			defer ch2.Close()

			ch3, err := conn2.Channel()
			Ω(err).Should(BeNil())
			defer ch3.Close()

			err = ch.Publish("",
				"",
				false,
				false,
				amqp.Publishing{Body: []byte("")})
			Ω(err).Should(BeNil())

			xs, err := rmqc.ListConnections()
			Ω(err).Should(BeNil())

			info := xs[0]
			Ω(info.Name).ShouldNot(BeNil())
			Ω(info.Host).Should(Equal("127.0.0.1"))
			Ω(info.UsesTLS).Should(Equal(false))
		})
	})

	Context("GET /channels when there are active connections with open channels", func() {
		It("returns decoded response", func() {
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
			Ω(err).Should(BeNil())
			defer conn.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			ch2, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch2.Close()

			ch3, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch3.Close()

			ch4, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch4.Close()

			err = ch.Publish("",
				"",
				false,
				false,
				amqp.Publishing{Body: []byte("")})
			Ω(err).Should(BeNil())

			err = ch2.Publish("",
				"",
				false,
				false,
				amqp.Publishing{Body: []byte("")})
			Ω(err).Should(BeNil())

			xs, err := rmqc.ListChannels()
			Ω(err).Should(BeNil())

			info := xs[0]
			Ω(info.Node).ShouldNot(BeNil())
			Ω(info.User).Should(Equal("guest"))
			Ω(info.Vhost).Should(Equal("/"))

			Ω(info.Transactional).Should(Equal(false))

			Ω(info.UnacknowledgedMessageCount).Should(Equal(0))
			Ω(info.UnconfirmedMessageCount).Should(Equal(0))
			Ω(info.UncommittedMessageCount).Should(Equal(0))
			Ω(info.UncommittedAckCount).Should(Equal(0))
		})
	})

	Context("GET /connections/{name] when connection exists", func() {
		It("returns decoded response", func() {
			// this really should be tested with > 1 connection and channel. MK.
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
			Ω(err).Should(BeNil())
			defer conn.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			err = ch.Publish("",
				"",
				false,
				false,
				amqp.Publishing{Body: []byte("")})
			Ω(err).Should(BeNil())

			xs, err := rmqc.ListConnections()
			Ω(err).Should(BeNil())

			c1 := xs[0]
			info, err := rmqc.GetConnection(c1.Name)
			Ω(err).Should(BeNil())
			Ω(info.Protocol).Should(Equal("AMQP 0-9-1"))
			Ω(info.User).Should(Equal("guest"))
		})
	})

	Context("GET /channels/{name} when channel exists", func() {
		It("returns decoded response", func() {
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
			Ω(err).Should(BeNil())
			defer conn.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			err = ch.Publish("",
				"",
				false,
				false,
				amqp.Publishing{Body: []byte("")})
			Ω(err).Should(BeNil())

			xs, err := rmqc.ListChannels()
			Ω(err).Should(BeNil())

			x := xs[0]
			info, err := rmqc.GetChannel(x.Name)

			Ω(info.Node).ShouldNot(BeNil())
			Ω(info.User).Should(Equal("guest"))
			Ω(info.Vhost).Should(Equal("/"))

			Ω(info.Transactional).Should(Equal(false))

			Ω(info.UnacknowledgedMessageCount).Should(Equal(0))
			Ω(info.UnconfirmedMessageCount).Should(Equal(0))
			Ω(info.UncommittedMessageCount).Should(Equal(0))
			Ω(info.UncommittedAckCount).Should(Equal(0))
		})
	})

	Context("GET /exchanges", func() {
		It("returns decoded response", func() {
			xs, err := rmqc.ListExchanges()
			Ω(err).Should(BeNil())

			x := xs[0]
			Ω(x.Name).Should(Equal(""))
			Ω(x.Durable).Should(Equal(true))
		})
	})

	Context("GET /exchanges/{vhost}", func() {
		It("returns decoded response", func() {
			xs, err := rmqc.ListExchangesIn("/")
			Ω(err).Should(BeNil())

			x := xs[0]
			Ω(x.Name).Should(Equal(""))
			Ω(x.Durable).Should(Equal(true))
		})
	})

	Context("GET /exchanges/{vhost}/{name}", func() {
		It("returns decoded response", func() {
			x, err := rmqc.GetExchange("rabbit/hole", "amq.fanout")
			Ω(err).Should(BeNil())
			Ω(x.Name).Should(Equal("amq.fanout"))
			Ω(x.Durable).Should(Equal(true))
			Ω(x.Type).Should(Equal("fanout"))
			Ω(x.Vhost).Should(Equal("rabbit/hole"))
		})
	})

	Context("GET /queues", func() {
		It("returns decoded response", func() {
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672")
			Ω(err).Should(BeNil())
			defer conn.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			ch.QueueDeclare(
				"q1",  // name
				false, // durable
				false, // delete when usused
				true,  // exclusive
				false,
				nil)

			qs, err := rmqc.ListQueues()
			Ω(err).Should(BeNil())

			q := qs[0]
			Ω(q.Name).ShouldNot(Equal(""))
			Ω(q.Node).ShouldNot(BeNil())
			Ω(q.Durable).ShouldNot(BeNil())
			Ω(q.Status).ShouldNot(BeNil())
		})
	})

	Context("GET /queues/{vhost}", func() {
		It("returns decoded response", func() {
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/rabbit%2Fhole")
			Ω(err).Should(BeNil())
			defer conn.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			ch.QueueDeclare(
				"q1",  // name
				false, // durable
				false, // delete when usused
				true,  // exclusive
				false,
				nil)

			qs, err := rmqc.ListQueuesIn("rabbit/hole")
			Ω(err).Should(BeNil())

			q := FindQueueByName(qs, "q1")
			Ω(q.Name).Should(Equal("q1"))
			Ω(q.Vhost).Should(Equal("rabbit/hole"))
			Ω(q.Durable).Should(Equal(false))
			Ω(q.Status).ShouldNot(BeNil())
		})
	})

	Context("GET /queues/{vhost}/{name}", func() {
		It("returns decoded response", func() {
			conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/rabbit%2Fhole")
			Ω(err).Should(BeNil())
			defer conn.Close()

			ch, err := conn.Channel()
			Ω(err).Should(BeNil())
			defer ch.Close()

			ch.QueueDeclare(
				"q1",  // name
				false, // durable
				false, // delete when usused
				true,  // exclusive
				false,
				nil)

			q, err := rmqc.GetQueue("rabbit/hole", "q1")
			Ω(err).Should(BeNil())

			Ω(q.Name).Should(Equal("q1"))
			Ω(q.Vhost).Should(Equal("rabbit/hole"))
			Ω(q.Durable).Should(Equal(false))
			Ω(q.Status).ShouldNot(BeNil())
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/lib/pq"
	"io/ioutil"
	"regexp"
)

var dry bool

func init() {
	flag.BoolVar(&dry, "dry", false, "dry run: show what will be run but do not execute statements")
	flag.Parse()
}

func main() {
	// tool to migrate the database
	// This is very immature code.
	//
	// Usage:
	//
	// go run bin/migrate.go 		# apply migrations
	// go run bin/migrate.go --dry=true 	# show what will be run, do not execute statements.
	//
	// TODO:
	// - rollback
	// - transactional migrations
	// - show output of migration
	// - decent error handling

	// load all migrations and find version numbers
	file_list, err := ioutil.ReadDir("db/")
	if err != nil {
		fmt.Println("error loading directory list", err)
	}

	all_migrations := make(map[string]string)

	re := regexp.MustCompile(`\A(\d+)[^\.]+\.sql\z`)
	for _, item := range file_list {
		if matches := re.FindSubmatch([]byte(item.Name())); matches != nil {
			all_migrations[string(matches[1])] = item.Name()
		}
	}

	fmt.Println("found versions: ", all_migrations)

	// find migrations to apply
	db, err := sql.Open("postgres", "dbname=cwfy sslmode=disable")
	if err != nil {
		fmt.Println("Cannot open database connection", err)
	}

	rows, err := db.Query("SELECT * FROM schema_info ORDER BY version DESC;")
	if err != nil {
		fmt.Println("error loading applied migrations", err)
	}

	var applied_migrations []string
	for rows.Next() {
		var version string
		_ = rows.Scan(&version)
		applied_migrations = append(applied_migrations, version)
	}

	fmt.Println("migrations already applied: ", applied_migrations)

	for _, version := range applied_migrations {
		delete(all_migrations, version)
	}

	fmt.Println("migrations that will be applied: ", all_migrations)

	// apply migrations
	for version, migration := range all_migrations {
		// MIGRATE!
		raw_sql, err := ioutil.ReadFile("db/" + migration)
		if err != nil {
			fmt.Printf("error reading %s: %s\n", migration, err)
		}

		fmt.Printf("===== executing migration %s =====\n\n%s\n\n", migration, string(raw_sql))

		if !dry {
			_, err = db.Exec(string(raw_sql))
			if err != nil {
				fmt.Printf("error migrating %s: %s", migration, err)
				return
			}

			fmt.Println("===== completed migration =====", migration)

			_, err = db.Exec("INSERT INTO schema_info (version) VALUES ($1);", version)
			if err != nil {
				fmt.Printf("error migrating %s: %s", migration, err)
			}
		}
	}
}
<commit_msg>add env support to migrate tool<commit_after>package main

import (
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/lib/pq"
	"io/ioutil"
	"regexp"
	"github.com/kylelemons/go-gypsy/yaml"
	"log"
)

var (
        dry     =       flag.Bool("dry", false, "dry run: show what will be run but do not execute statements")
        environment = flag.String("environment", "", "Environment to run in, e.g. staging, production")
	config      = flag.String("config", "./config/database.yml", "database configuration file")
	db *sql.DB
)

func init() {
	flag.Parse()

	log.Printf("running in %s environment, configuration file %s", *environment, *config)
	settings := yaml.ConfigFile(*config)

	// setup database connection
	driver, err := settings.Get(fmt.Sprintf("%s.driver", *environment))
	if err != nil {
		log.Fatal("error loading db driver", err)
	}

	connstr, err := settings.Get(fmt.Sprintf("%s.connstr", *environment))
	if err != nil {
		log.Fatal("error loading db connstr", err)
	}

	_db, err := sql.Open(driver, connstr)
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}

	log.Printf("database connstr: %s", connstr)

	db = _db
}

func main() {
	// tool to migrate the database
	// This is very immature code.
	//
	// Usage:
	//
	// go run bin/migrate.go 		# apply migrations
	// go run bin/migrate.go --dry=true 	# show what will be run, do not execute statements.
	//
	// TODO:
	// - rollback
	// - transactional migrations
	// - show output of migration
	// - decent error handling

	// load all migrations and find version numbers
	file_list, err := ioutil.ReadDir("db/")
	if err != nil {
		fmt.Println("error loading directory list", err)
	}

	all_migrations := make(map[string]string)

	re := regexp.MustCompile(`\A(\d+)[^\.]+\.sql\z`)
	for _, item := range file_list {
		if matches := re.FindSubmatch([]byte(item.Name())); matches != nil {
			all_migrations[string(matches[1])] = item.Name()
		}
	}

	fmt.Println("found versions: ", all_migrations)

	// find migrations to apply
	rows, err := db.Query("SELECT * FROM schema_info ORDER BY version DESC;")
	if err != nil {
		fmt.Println("error loading applied migrations", err)
	}

	var applied_migrations []string
	for rows.Next() {
		var version string
		_ = rows.Scan(&version)
		applied_migrations = append(applied_migrations, version)
	}

	fmt.Println("migrations already applied: ", applied_migrations)

	for _, version := range applied_migrations {
		delete(all_migrations, version)
	}

	fmt.Println("migrations that will be applied: ", all_migrations)

	// apply migrations
	for version, migration := range all_migrations {
		// MIGRATE!
		raw_sql, err := ioutil.ReadFile("db/" + migration)
		if err != nil {
			fmt.Printf("error reading %s: %s\n", migration, err)
		}

		fmt.Printf("===== executing migration %s =====\n\n%s\n\n", migration, string(raw_sql))

		if !*dry {
			_, err = db.Exec(string(raw_sql))
			if err != nil {
				fmt.Printf("error migrating %s: %s", migration, err)
				return
			}

			fmt.Println("===== completed migration =====", migration)

			_, err = db.Exec("INSERT INTO schema_info (version) VALUES ($1);", version)
			if err != nil {
				fmt.Printf("error migrating %s: %s", migration, err)
			}
		}
	}
}
<|endoftext|>


<commit_before>package golog

const (
	INFO int = iota
	WARNING
	ERROR
	FATAL
)

type PackageLogger struct {
	LocationLogger
	MultiLogOuter
	failFunc func()
}

func newPackageLoggerCommon(outer MultiLogOuter, minloglevel_flag *int,
failFunc func(), locFunc func(skip int) *LogLocation) *PackageLogger {
	ret := &PackageLogger{failFunc: failFunc, MultiLogOuter: outer}

	ret.LocationLogger = NewLocationLogger(
		&loggerImpl{outer, minloglevel_flag, func() { ret.failFunc() }},
		FullLocation)

	return ret
}

func NewDefaultPackageLogger() *PackageLogger {
	return newPackageLoggerCommon(
		NewDefaultMultiLogOuter(),
		flag_minloglevel,
		ExitError,
		FullLocation)
}

func NewPackageLogger(outer MultiLogOuter, minloglevel int,
failFunc func(), locFunc func(skip int) *LogLocation) *PackageLogger {
	return newPackageLoggerCommon(outer, &minloglevel, failFunc, locFunc)
}

func (l *PackageLogger) StartTestLogging(t TestController) {
	l.MultiLogOuter.AddLogOuter("testing", NewTestLogOuter(t))
	// TODO(awreece) Safe old failFunc so we can restore it properly.
	l.failFunc = func() { t.FailNow() }
}

func (l *PackageLogger) StopTestLogging() {
	l.MultiLogOuter.RemoveLogOuter("testing")
	l.failFunc = ExitError
}
func (l *PackageLogger) Info(msg ...interface{}) {
	l.LogDepth(INFO, printClosure(msg...), 1)
}
func (l *PackageLogger) Infof(fmt string, vals ...interface{}) {
	l.LogDepth(INFO, printfClosure(fmt, vals...), 1)
}
func (l *PackageLogger) Infoc(closure func() string) {
	l.LogDepth(INFO, closure, 1)
}
func (l *PackageLogger) Warning(msg ...interface{}) {
	l.LogDepth(WARNING, printClosure(msg...), 1)
}
func (l *PackageLogger) Warningf(fmt string, vals ...interface{}) {
	l.LogDepth(WARNING, printfClosure(fmt, vals...), 1)
}
func (l *PackageLogger) Warningc(closure func() string) {
	l.LogDepth(WARNING, closure, 1)
}
func (l *PackageLogger) Error(msg ...interface{}) {
	l.LogDepth(ERROR, printClosure(msg...), 1)
}
func (l *PackageLogger) Errorf(fmt string, vals ...interface{}) {
	l.LogDepth(ERROR, printfClosure(fmt, vals...), 1)
}
func (l *PackageLogger) Errorc(closure func() string) {
	l.LogDepth(ERROR, closure, 1)
}
func (l *PackageLogger) Fatal(msg ...interface{}) {
	l.LogDepth(FATAL, printClosure(msg...), 1)
	l.FailNow()
}
func (l *PackageLogger) Fatalf(fmt string, vals ...interface{}) {
	l.LogDepth(FATAL, printfClosure(fmt, vals...), 1)
	l.FailNow()
}
func (l *PackageLogger) Fatalc(closure func() string) {
	l.LogDepth(FATAL, closure, 1)
	l.FailNow()
}
<commit_msg>Comment StartTestLogging<commit_after>package golog

const (
	INFO int = iota
	WARNING
	ERROR
	FATAL
)

type PackageLogger struct {
	LocationLogger
	MultiLogOuter
	failFunc func()
}

func newPackageLoggerCommon(outer MultiLogOuter, minloglevel_flag *int,
failFunc func(), locFunc func(skip int) *LogLocation) *PackageLogger {
	ret := &PackageLogger{failFunc: failFunc, MultiLogOuter: outer}

	ret.LocationLogger = NewLocationLogger(
		&loggerImpl{outer, minloglevel_flag, func() { ret.failFunc() }},
		FullLocation)

	return ret
}

func NewDefaultPackageLogger() *PackageLogger {
	return newPackageLoggerCommon(
		NewDefaultMultiLogOuter(),
		flag_minloglevel,
		ExitError,
		FullLocation)
}

func NewPackageLogger(outer MultiLogOuter, minloglevel int,
failFunc func(), locFunc func(skip int) *LogLocation) *PackageLogger {
	return newPackageLoggerCommon(outer, &minloglevel, failFunc, locFunc)
}

// Associates TestController with a the "testing LogOuter and updates
// l.FailNow() to call t.FailNow().
func (l *PackageLogger) StartTestLogging(t TestController) {
	l.MultiLogOuter.AddLogOuter("testing", NewTestLogOuter(t))
	// TODO(awreece) Save old failFunc so we can restore it properly.
	l.failFunc = func() { t.FailNow() }
}

// Removes the testing logger and restores l.FailNow() to its previous state.
func (l *PackageLogger) StopTestLogging() {
	l.MultiLogOuter.RemoveLogOuter("testing")
	// TODO(awreece) Restored to saved failFunc.
	l.failFunc = ExitError
}
func (l *PackageLogger) Info(msg ...interface{}) {
	l.LogDepth(INFO, printClosure(msg...), 1)
}
func (l *PackageLogger) Infof(fmt string, vals ...interface{}) {
	l.LogDepth(INFO, printfClosure(fmt, vals...), 1)
}
func (l *PackageLogger) Infoc(closure func() string) {
	l.LogDepth(INFO, closure, 1)
}
func (l *PackageLogger) Warning(msg ...interface{}) {
	l.LogDepth(WARNING, printClosure(msg...), 1)
}
func (l *PackageLogger) Warningf(fmt string, vals ...interface{}) {
	l.LogDepth(WARNING, printfClosure(fmt, vals...), 1)
}
func (l *PackageLogger) Warningc(closure func() string) {
	l.LogDepth(WARNING, closure, 1)
}
func (l *PackageLogger) Error(msg ...interface{}) {
	l.LogDepth(ERROR, printClosure(msg...), 1)
}
func (l *PackageLogger) Errorf(fmt string, vals ...interface{}) {
	l.LogDepth(ERROR, printfClosure(fmt, vals...), 1)
}
func (l *PackageLogger) Errorc(closure func() string) {
	l.LogDepth(ERROR, closure, 1)
}
func (l *PackageLogger) Fatal(msg ...interface{}) {
	l.LogDepth(FATAL, printClosure(msg...), 1)
	l.FailNow()
}
func (l *PackageLogger) Fatalf(fmt string, vals ...interface{}) {
	l.LogDepth(FATAL, printfClosure(fmt, vals...), 1)
	l.FailNow()
}
func (l *PackageLogger) Fatalc(closure func() string) {
	l.LogDepth(FATAL, closure, 1)
	l.FailNow()
}
<|endoftext|>


<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"math/rand"
	"net"
	"net/http"
	"sync"
	"time"

	clientV2 "github.com/coreos/etcd/client"
	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"golang.org/x/net/context"
	"golang.org/x/time/rate"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
	"google.golang.org/grpc/transport"
)

func init() {
	grpclog.SetLogger(plog)
}

type stressFunc func(ctx context.Context) error

type stressEntry struct {
	weight float32
	f      stressFunc
}

type stressTable struct {
	entries    []stressEntry
	sumWeights float32
}

func createStressTable(entries []stressEntry) *stressTable {
	st := stressTable{entries: entries}
	for _, entry := range st.entries {
		st.sumWeights += entry.weight
	}
	return &st
}

func (st *stressTable) choose() stressFunc {
	v := rand.Float32() * st.sumWeights
	var sum float32
	var idx int
	for i := range st.entries {
		sum += st.entries[i].weight
		if sum >= v {
			idx = i
			break
		}
	}
	return st.entries[idx].f
}

func newStressPut(kvc pb.KVClient, keySuffixRange, keySize int) stressFunc {
	return func(ctx context.Context) error {
		_, err := kvc.Put(ctx, &pb.PutRequest{
			Key:   []byte(fmt.Sprintf("foo%d", rand.Intn(keySuffixRange))),
			Value: randBytes(keySize),
		}, grpc.FailFast(false))
		return err
	}
}

func newStressRange(kvc pb.KVClient, keySuffixRange int) stressFunc {
	return func(ctx context.Context) error {
		_, err := kvc.Range(ctx, &pb.RangeRequest{
			Key: []byte(fmt.Sprintf("foo%d", rand.Intn(keySuffixRange))),
		}, grpc.FailFast(false))
		return err
	}
}

func newStressRangePrefix(kvc pb.KVClient, keySuffixRange int) stressFunc {
	return func(ctx context.Context) error {
		_, err := kvc.Range(ctx, &pb.RangeRequest{
			Key:      []byte("foo"),
			RangeEnd: []byte(fmt.Sprintf("foo%d", rand.Intn(keySuffixRange))),
		}, grpc.FailFast(false))
		return err
	}
}

func newStressDelete(kvc pb.KVClient, keySuffixRange int) stressFunc {
	return func(ctx context.Context) error {
		_, err := kvc.DeleteRange(ctx, &pb.DeleteRangeRequest{
			Key: []byte(fmt.Sprintf("foo%d", rand.Intn(keySuffixRange))),
		}, grpc.FailFast(false))
		return err
	}
}

func newStressDeletePrefix(kvc pb.KVClient, keySuffixRange int) stressFunc {
	return func(ctx context.Context) error {
		_, err := kvc.DeleteRange(ctx, &pb.DeleteRangeRequest{
			Key:      []byte("foo"),
			RangeEnd: []byte(fmt.Sprintf("foo%d", rand.Intn(keySuffixRange))),
		}, grpc.FailFast(false))
		return err
	}
}

type Stresser interface {
	// Stress starts to stress the etcd cluster
	Stress() error
	// Cancel cancels the stress test on the etcd cluster
	Cancel()
	// Report reports the success and failure of the stress test
	Report() (success int, failure int)
}

type stresser struct {
	Endpoint string

	KeySize        int
	KeySuffixRange int

	qps int
	N   int

	mu sync.Mutex
	wg *sync.WaitGroup

	rateLimiter *rate.Limiter

	cancel func()
	conn   *grpc.ClientConn

	success int

	stressTable *stressTable
}

func (s *stresser) Stress() error {
	// TODO: add backoff option
	conn, err := grpc.Dial(s.Endpoint, grpc.WithInsecure())
	if err != nil {
		return fmt.Errorf("%v (%s)", err, s.Endpoint)
	}
	ctx, cancel := context.WithCancel(context.Background())

	wg := &sync.WaitGroup{}
	wg.Add(s.N)

	s.mu.Lock()
	s.conn = conn
	s.cancel = cancel
	s.wg = wg
	s.rateLimiter = rate.NewLimiter(rate.Every(time.Second), s.qps)
	s.mu.Unlock()

	kvc := pb.NewKVClient(conn)

	var stressEntries = []stressEntry{
		{weight: 0.7, f: newStressPut(kvc, s.KeySuffixRange, s.KeySize)},
		{weight: 0.07, f: newStressRange(kvc, s.KeySuffixRange)},
		{weight: 0.07, f: newStressRangePrefix(kvc, s.KeySuffixRange)},
		{weight: 0.07, f: newStressDelete(kvc, s.KeySuffixRange)},
		{weight: 0.07, f: newStressDeletePrefix(kvc, s.KeySuffixRange)},
	}
	s.stressTable = createStressTable(stressEntries)

	for i := 0; i < s.N; i++ {
		go s.run(ctx, kvc)
	}

	plog.Printf("stresser %q is started", s.Endpoint)
	return nil
}

func (s *stresser) run(ctx context.Context, kvc pb.KVClient) {
	defer s.wg.Done()

	for {
		if err := s.rateLimiter.Wait(ctx); err == context.Canceled {
			return
		}

		// TODO: 10-second is enough timeout to cover leader failure
		// and immediate leader election. Find out what other cases this
		// could be timed out.
		sctx, scancel := context.WithTimeout(ctx, 10*time.Second)

		err := s.stressTable.choose()(sctx)

		scancel()

		if err != nil {
			shouldContinue := false
			switch grpc.ErrorDesc(err) {
			case context.DeadlineExceeded.Error():
				// This retries when request is triggered at the same time as
				// leader failure. When we terminate the leader, the request to
				// that leader cannot be processed, and times out. Also requests
				// to followers cannot be forwarded to the old leader, so timing out
				// as well. We want to keep stressing until the cluster elects a
				// new leader and start processing requests again.
				shouldContinue = true

			case etcdserver.ErrTimeoutDueToLeaderFail.Error(), etcdserver.ErrTimeout.Error():
				// This retries when request is triggered at the same time as
				// leader failure and follower nodes receive time out errors
				// from losing their leader. Followers should retry to connect
				// to the new leader.
				shouldContinue = true

			case etcdserver.ErrStopped.Error():
				// one of the etcd nodes stopped from failure injection
				shouldContinue = true

			case transport.ErrConnClosing.Desc:
				// server closed the transport (failure injected node)
				shouldContinue = true

			case rpctypes.ErrNotCapable.Error():
				// capability check has not been done (in the beginning)
				shouldContinue = true

				// default:
				// errors from stresser.Cancel method:
				// rpc error: code = 1 desc = context canceled (type grpc.rpcError)
				// rpc error: code = 2 desc = grpc: the client connection is closing (type grpc.rpcError)
			}
			if shouldContinue {
				continue
			}
			return
		}
		s.mu.Lock()
		s.success++
		s.mu.Unlock()
	}
}

func (s *stresser) Cancel() {
	s.mu.Lock()
	s.cancel()
	s.conn.Close()
	wg := s.wg
	s.mu.Unlock()

	wg.Wait()
	plog.Printf("stresser %q is canceled", s.Endpoint)
}

func (s *stresser) Report() (int, int) {
	s.mu.Lock()
	defer s.mu.Unlock()
	// TODO: find a better way to report v3 tests
	return s.success, -1
}

type stresserV2 struct {
	Endpoint string

	KeySize        int
	KeySuffixRange int

	N int

	mu      sync.Mutex
	failure int
	success int

	cancel func()
}

func (s *stresserV2) Stress() error {
	cfg := clientV2.Config{
		Endpoints: []string{s.Endpoint},
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout:   time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			MaxIdleConnsPerHost: s.N,
		},
	}
	c, err := clientV2.New(cfg)
	if err != nil {
		return err
	}

	kv := clientV2.NewKeysAPI(c)
	ctx, cancel := context.WithCancel(context.Background())
	s.cancel = cancel

	for i := 0; i < s.N; i++ {
		go func() {
			for {
				setctx, setcancel := context.WithTimeout(ctx, clientV2.DefaultRequestTimeout)
				key := fmt.Sprintf("foo%d", rand.Intn(s.KeySuffixRange))
				_, err := kv.Set(setctx, key, string(randBytes(s.KeySize)), nil)
				setcancel()
				if err == context.Canceled {
					return
				}
				s.mu.Lock()
				if err != nil {
					s.failure++
				} else {
					s.success++
				}
				s.mu.Unlock()
			}
		}()
	}

	<-ctx.Done()
	return nil
}

func (s *stresserV2) Cancel() {
	s.cancel()
}

func (s *stresserV2) Report() (success int, failure int) {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.success, s.failure
}

func randBytes(size int) []byte {
	data := make([]byte, size)
	for i := 0; i < size; i++ {
		data[i] = byte(int('a') + rand.Intn(26))
	}
	return data
}
<commit_msg>etcd-tester: remove unnecessary arg from stresser<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"math/rand"
	"net"
	"net/http"
	"sync"
	"time"

	clientV2 "github.com/coreos/etcd/client"
	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"golang.org/x/net/context"
	"golang.org/x/time/rate"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
	"google.golang.org/grpc/transport"
)

func init() {
	grpclog.SetLogger(plog)
}

type stressFunc func(ctx context.Context) error

type stressEntry struct {
	weight float32
	f      stressFunc
}

type stressTable struct {
	entries    []stressEntry
	sumWeights float32
}

func createStressTable(entries []stressEntry) *stressTable {
	st := stressTable{entries: entries}
	for _, entry := range st.entries {
		st.sumWeights += entry.weight
	}
	return &st
}

func (st *stressTable) choose() stressFunc {
	v := rand.Float32() * st.sumWeights
	var sum float32
	var idx int
	for i := range st.entries {
		sum += st.entries[i].weight
		if sum >= v {
			idx = i
			break
		}
	}
	return st.entries[idx].f
}

func newStressPut(kvc pb.KVClient, keySuffixRange, keySize int) stressFunc {
	return func(ctx context.Context) error {
		_, err := kvc.Put(ctx, &pb.PutRequest{
			Key:   []byte(fmt.Sprintf("foo%d", rand.Intn(keySuffixRange))),
			Value: randBytes(keySize),
		}, grpc.FailFast(false))
		return err
	}
}

func newStressRange(kvc pb.KVClient, keySuffixRange int) stressFunc {
	return func(ctx context.Context) error {
		_, err := kvc.Range(ctx, &pb.RangeRequest{
			Key: []byte(fmt.Sprintf("foo%d", rand.Intn(keySuffixRange))),
		}, grpc.FailFast(false))
		return err
	}
}

func newStressRangePrefix(kvc pb.KVClient, keySuffixRange int) stressFunc {
	return func(ctx context.Context) error {
		_, err := kvc.Range(ctx, &pb.RangeRequest{
			Key:      []byte("foo"),
			RangeEnd: []byte(fmt.Sprintf("foo%d", rand.Intn(keySuffixRange))),
		}, grpc.FailFast(false))
		return err
	}
}

func newStressDelete(kvc pb.KVClient, keySuffixRange int) stressFunc {
	return func(ctx context.Context) error {
		_, err := kvc.DeleteRange(ctx, &pb.DeleteRangeRequest{
			Key: []byte(fmt.Sprintf("foo%d", rand.Intn(keySuffixRange))),
		}, grpc.FailFast(false))
		return err
	}
}

func newStressDeletePrefix(kvc pb.KVClient, keySuffixRange int) stressFunc {
	return func(ctx context.Context) error {
		_, err := kvc.DeleteRange(ctx, &pb.DeleteRangeRequest{
			Key:      []byte("foo"),
			RangeEnd: []byte(fmt.Sprintf("foo%d", rand.Intn(keySuffixRange))),
		}, grpc.FailFast(false))
		return err
	}
}

type Stresser interface {
	// Stress starts to stress the etcd cluster
	Stress() error
	// Cancel cancels the stress test on the etcd cluster
	Cancel()
	// Report reports the success and failure of the stress test
	Report() (success int, failure int)
}

type stresser struct {
	Endpoint string

	KeySize        int
	KeySuffixRange int

	qps int
	N   int

	mu sync.Mutex
	wg *sync.WaitGroup

	rateLimiter *rate.Limiter

	cancel func()
	conn   *grpc.ClientConn

	success int

	stressTable *stressTable
}

func (s *stresser) Stress() error {
	// TODO: add backoff option
	conn, err := grpc.Dial(s.Endpoint, grpc.WithInsecure())
	if err != nil {
		return fmt.Errorf("%v (%s)", err, s.Endpoint)
	}
	ctx, cancel := context.WithCancel(context.Background())

	wg := &sync.WaitGroup{}
	wg.Add(s.N)

	s.mu.Lock()
	s.conn = conn
	s.cancel = cancel
	s.wg = wg
	s.rateLimiter = rate.NewLimiter(rate.Every(time.Second), s.qps)
	s.mu.Unlock()

	kvc := pb.NewKVClient(conn)

	var stressEntries = []stressEntry{
		{weight: 0.7, f: newStressPut(kvc, s.KeySuffixRange, s.KeySize)},
		{weight: 0.07, f: newStressRange(kvc, s.KeySuffixRange)},
		{weight: 0.07, f: newStressRangePrefix(kvc, s.KeySuffixRange)},
		{weight: 0.07, f: newStressDelete(kvc, s.KeySuffixRange)},
		{weight: 0.07, f: newStressDeletePrefix(kvc, s.KeySuffixRange)},
	}
	s.stressTable = createStressTable(stressEntries)

	for i := 0; i < s.N; i++ {
		go s.run(ctx)
	}

	plog.Printf("stresser %q is started", s.Endpoint)
	return nil
}

func (s *stresser) run(ctx context.Context) {
	defer s.wg.Done()

	for {
		if err := s.rateLimiter.Wait(ctx); err == context.Canceled {
			return
		}

		// TODO: 10-second is enough timeout to cover leader failure
		// and immediate leader election. Find out what other cases this
		// could be timed out.
		sctx, scancel := context.WithTimeout(ctx, 10*time.Second)

		err := s.stressTable.choose()(sctx)

		scancel()

		if err != nil {
			shouldContinue := false
			switch grpc.ErrorDesc(err) {
			case context.DeadlineExceeded.Error():
				// This retries when request is triggered at the same time as
				// leader failure. When we terminate the leader, the request to
				// that leader cannot be processed, and times out. Also requests
				// to followers cannot be forwarded to the old leader, so timing out
				// as well. We want to keep stressing until the cluster elects a
				// new leader and start processing requests again.
				shouldContinue = true

			case etcdserver.ErrTimeoutDueToLeaderFail.Error(), etcdserver.ErrTimeout.Error():
				// This retries when request is triggered at the same time as
				// leader failure and follower nodes receive time out errors
				// from losing their leader. Followers should retry to connect
				// to the new leader.
				shouldContinue = true

			case etcdserver.ErrStopped.Error():
				// one of the etcd nodes stopped from failure injection
				shouldContinue = true

			case transport.ErrConnClosing.Desc:
				// server closed the transport (failure injected node)
				shouldContinue = true

			case rpctypes.ErrNotCapable.Error():
				// capability check has not been done (in the beginning)
				shouldContinue = true

				// default:
				// errors from stresser.Cancel method:
				// rpc error: code = 1 desc = context canceled (type grpc.rpcError)
				// rpc error: code = 2 desc = grpc: the client connection is closing (type grpc.rpcError)
			}
			if shouldContinue {
				continue
			}
			return
		}
		s.mu.Lock()
		s.success++
		s.mu.Unlock()
	}
}

func (s *stresser) Cancel() {
	s.mu.Lock()
	s.cancel()
	s.conn.Close()
	wg := s.wg
	s.mu.Unlock()

	wg.Wait()
	plog.Printf("stresser %q is canceled", s.Endpoint)
}

func (s *stresser) Report() (int, int) {
	s.mu.Lock()
	defer s.mu.Unlock()
	// TODO: find a better way to report v3 tests
	return s.success, -1
}

type stresserV2 struct {
	Endpoint string

	KeySize        int
	KeySuffixRange int

	N int

	mu      sync.Mutex
	failure int
	success int

	cancel func()
}

func (s *stresserV2) Stress() error {
	cfg := clientV2.Config{
		Endpoints: []string{s.Endpoint},
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout:   time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			MaxIdleConnsPerHost: s.N,
		},
	}
	c, err := clientV2.New(cfg)
	if err != nil {
		return err
	}

	kv := clientV2.NewKeysAPI(c)
	ctx, cancel := context.WithCancel(context.Background())
	s.cancel = cancel

	for i := 0; i < s.N; i++ {
		go func() {
			for {
				setctx, setcancel := context.WithTimeout(ctx, clientV2.DefaultRequestTimeout)
				key := fmt.Sprintf("foo%d", rand.Intn(s.KeySuffixRange))
				_, err := kv.Set(setctx, key, string(randBytes(s.KeySize)), nil)
				setcancel()
				if err == context.Canceled {
					return
				}
				s.mu.Lock()
				if err != nil {
					s.failure++
				} else {
					s.success++
				}
				s.mu.Unlock()
			}
		}()
	}

	<-ctx.Done()
	return nil
}

func (s *stresserV2) Cancel() {
	s.cancel()
}

func (s *stresserV2) Report() (success int, failure int) {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.success, s.failure
}

func randBytes(size int) []byte {
	data := make([]byte, size)
	for i := 0; i < size; i++ {
		data[i] = byte(int('a') + rand.Intn(26))
	}
	return data
}
<|endoftext|>


<commit_before>package images // import "github.com/docker/docker/daemon/images"

import (
	"context"
	"io"
	"runtime"

	"github.com/containerd/containerd/platforms"
	"github.com/docker/distribution/reference"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/backend"
	"github.com/docker/docker/builder"
	"github.com/docker/docker/errdefs"
	"github.com/docker/docker/image"
	"github.com/docker/docker/layer"
	"github.com/docker/docker/pkg/containerfs"
	"github.com/docker/docker/pkg/progress"
	"github.com/docker/docker/pkg/streamformatter"
	"github.com/docker/docker/pkg/stringid"
	"github.com/docker/docker/pkg/system"
	"github.com/docker/docker/registry"
	specs "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type roLayer struct {
	released   bool
	layerStore layer.Store
	roLayer    layer.Layer
}

func (l *roLayer) DiffID() layer.DiffID {
	if l.roLayer == nil {
		return layer.DigestSHA256EmptyTar
	}
	return l.roLayer.DiffID()
}

func (l *roLayer) Release() error {
	if l.released {
		return nil
	}
	if l.roLayer != nil {
		metadata, err := l.layerStore.Release(l.roLayer)
		layer.LogReleaseMetadata(metadata)
		if err != nil {
			return errors.Wrap(err, "failed to release ROLayer")
		}
	}
	l.roLayer = nil
	l.released = true
	return nil
}

func (l *roLayer) NewRWLayer() (builder.RWLayer, error) {
	var chainID layer.ChainID
	if l.roLayer != nil {
		chainID = l.roLayer.ChainID()
	}

	mountID := stringid.GenerateRandomID()
	newLayer, err := l.layerStore.CreateRWLayer(mountID, chainID, nil)
	if err != nil {
		return nil, errors.Wrap(err, "failed to create rwlayer")
	}

	rwLayer := &rwLayer{layerStore: l.layerStore, rwLayer: newLayer}

	fs, err := newLayer.Mount("")
	if err != nil {
		rwLayer.Release()
		return nil, err
	}

	rwLayer.fs = fs

	return rwLayer, nil
}

type rwLayer struct {
	released   bool
	layerStore layer.Store
	rwLayer    layer.RWLayer
	fs         containerfs.ContainerFS
}

func (l *rwLayer) Root() containerfs.ContainerFS {
	return l.fs
}

func (l *rwLayer) Commit() (builder.ROLayer, error) {
	stream, err := l.rwLayer.TarStream()
	if err != nil {
		return nil, err
	}
	defer stream.Close()

	var chainID layer.ChainID
	if parent := l.rwLayer.Parent(); parent != nil {
		chainID = parent.ChainID()
	}

	newLayer, err := l.layerStore.Register(stream, chainID)
	if err != nil {
		return nil, err
	}
	// TODO: An optimization would be to handle empty layers before returning
	return &roLayer{layerStore: l.layerStore, roLayer: newLayer}, nil
}

func (l *rwLayer) Release() error {
	if l.released {
		return nil
	}

	if l.fs != nil {
		if err := l.rwLayer.Unmount(); err != nil {
			return errors.Wrap(err, "failed to unmount RWLayer")
		}
		l.fs = nil
	}

	metadata, err := l.layerStore.ReleaseRWLayer(l.rwLayer)
	layer.LogReleaseMetadata(metadata)
	if err != nil {
		return errors.Wrap(err, "failed to release RWLayer")
	}
	l.released = true
	return nil
}

func newROLayerForImage(img *image.Image, layerStore layer.Store) (builder.ROLayer, error) {
	if img == nil || img.RootFS.ChainID() == "" {
		return &roLayer{layerStore: layerStore}, nil
	}
	// Hold a reference to the image layer so that it can't be removed before
	// it is released
	layer, err := layerStore.Get(img.RootFS.ChainID())
	if err != nil {
		return nil, errors.Wrapf(err, "failed to get layer for image %s", img.ImageID())
	}
	return &roLayer{layerStore: layerStore, roLayer: layer}, nil
}

// TODO: could this use the regular daemon PullImage ?
func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, platform *specs.Platform) (*image.Image, error) {
	ref, err := reference.ParseNormalizedNamed(name)
	if err != nil {
		return nil, err
	}
	ref = reference.TagNameOnly(ref)

	pullRegistryAuth := &types.AuthConfig{}
	if len(authConfigs) > 0 {
		// The request came with a full auth config, use it
		repoInfo, err := i.registryService.ResolveRepository(ref)
		if err != nil {
			return nil, err
		}

		resolvedConfig := registry.ResolveAuthConfig(authConfigs, repoInfo.Index)
		pullRegistryAuth = &resolvedConfig
	}

	if err := i.pullImageWithReference(ctx, ref, platform, nil, pullRegistryAuth, output); err != nil {
		return nil, err
	}

	img, err := i.GetImage(name, platform)
	if errdefs.IsNotFound(err) && img != nil && platform != nil {
		imgPlat := specs.Platform{
			OS:           img.OS,
			Architecture: img.BaseImgArch(),
			Variant:      img.BaseImgVariant(),
		}

		p := *platform
		if !platforms.Only(p).Match(imgPlat) {
			po := streamformatter.NewJSONProgressOutput(output, false)
			progress.Messagef(po, "", `
WARNING: Pulled image with specified platform (%s), but the resulting image's configured platform (%s) does not match.
This is most likely caused by a bug in the build system that created the fetched image (%s).
Please notify the image author to correct the configuration.`,
				platforms.Format(p), platforms.Format(imgPlat), name,
			)
			logrus.WithError(err).WithField("image", name).Warn("Ignoring error about platform mismatch where the manifest list points to an image whose configuration does not match the platform in the manifest.")
			err = nil
		}
	}
	return img, err
}

// GetImageAndReleasableLayer returns an image and releaseable layer for a reference or ID.
// Every call to GetImageAndReleasableLayer MUST call releasableLayer.Release() to prevent
// leaking of layers.
func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) {
	if refOrID == "" { // ie FROM scratch
		os := runtime.GOOS
		if runtime.GOOS == "windows" {
			os = "linux"
		}
		if opts.Platform != nil {
			os = opts.Platform.OS
		}
		if !system.IsOSSupported(os) {
			return nil, nil, system.ErrNotSupportedOperatingSystem
		}
		layer, err := newROLayerForImage(nil, i.layerStore)
		return nil, layer, err
	}

	if opts.PullOption != backend.PullOptionForcePull {
		image, err := i.GetImage(refOrID, opts.Platform)
		if err != nil && opts.PullOption == backend.PullOptionNoPull {
			return nil, nil, err
		}
		// TODO: shouldn't we error out if error is different from "not found" ?
		if image != nil {
			if !system.IsOSSupported(image.OperatingSystem()) {
				return nil, nil, system.ErrNotSupportedOperatingSystem
			}
			layer, err := newROLayerForImage(image, i.layerStore)
			return image, layer, err
		}
	}

	image, err := i.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.Platform)
	if err != nil {
		return nil, nil, err
	}
	if !system.IsOSSupported(image.OperatingSystem()) {
		return nil, nil, system.ErrNotSupportedOperatingSystem
	}
	layer, err := newROLayerForImage(image, i.layerStore)
	return image, layer, err
}

// CreateImage creates a new image by adding a config and ID to the image store.
// This is similar to LoadImage() except that it receives JSON encoded bytes of
// an image instead of a tar archive.
func (i *ImageService) CreateImage(config []byte, parent string) (builder.Image, error) {
	id, err := i.imageStore.Create(config)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create image")
	}

	if parent != "" {
		if err := i.imageStore.SetParent(id, image.ID(parent)); err != nil {
			return nil, errors.Wrapf(err, "failed to set parent %s", parent)
		}
	}

	return i.imageStore.Get(id)
}
<commit_msg>daemon/images: rename variables that collided with imports<commit_after>package images // import "github.com/docker/docker/daemon/images"

import (
	"context"
	"io"
	"runtime"

	"github.com/containerd/containerd/platforms"
	"github.com/docker/distribution/reference"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/backend"
	"github.com/docker/docker/builder"
	"github.com/docker/docker/errdefs"
	"github.com/docker/docker/image"
	"github.com/docker/docker/layer"
	"github.com/docker/docker/pkg/containerfs"
	"github.com/docker/docker/pkg/progress"
	"github.com/docker/docker/pkg/streamformatter"
	"github.com/docker/docker/pkg/stringid"
	"github.com/docker/docker/pkg/system"
	"github.com/docker/docker/registry"
	specs "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type roLayer struct {
	released   bool
	layerStore layer.Store
	roLayer    layer.Layer
}

func (l *roLayer) DiffID() layer.DiffID {
	if l.roLayer == nil {
		return layer.DigestSHA256EmptyTar
	}
	return l.roLayer.DiffID()
}

func (l *roLayer) Release() error {
	if l.released {
		return nil
	}
	if l.roLayer != nil {
		metadata, err := l.layerStore.Release(l.roLayer)
		layer.LogReleaseMetadata(metadata)
		if err != nil {
			return errors.Wrap(err, "failed to release ROLayer")
		}
	}
	l.roLayer = nil
	l.released = true
	return nil
}

func (l *roLayer) NewRWLayer() (builder.RWLayer, error) {
	var chainID layer.ChainID
	if l.roLayer != nil {
		chainID = l.roLayer.ChainID()
	}

	mountID := stringid.GenerateRandomID()
	newLayer, err := l.layerStore.CreateRWLayer(mountID, chainID, nil)
	if err != nil {
		return nil, errors.Wrap(err, "failed to create rwlayer")
	}

	rwLayer := &rwLayer{layerStore: l.layerStore, rwLayer: newLayer}

	fs, err := newLayer.Mount("")
	if err != nil {
		rwLayer.Release()
		return nil, err
	}

	rwLayer.fs = fs

	return rwLayer, nil
}

type rwLayer struct {
	released   bool
	layerStore layer.Store
	rwLayer    layer.RWLayer
	fs         containerfs.ContainerFS
}

func (l *rwLayer) Root() containerfs.ContainerFS {
	return l.fs
}

func (l *rwLayer) Commit() (builder.ROLayer, error) {
	stream, err := l.rwLayer.TarStream()
	if err != nil {
		return nil, err
	}
	defer stream.Close()

	var chainID layer.ChainID
	if parent := l.rwLayer.Parent(); parent != nil {
		chainID = parent.ChainID()
	}

	newLayer, err := l.layerStore.Register(stream, chainID)
	if err != nil {
		return nil, err
	}
	// TODO: An optimization would be to handle empty layers before returning
	return &roLayer{layerStore: l.layerStore, roLayer: newLayer}, nil
}

func (l *rwLayer) Release() error {
	if l.released {
		return nil
	}

	if l.fs != nil {
		if err := l.rwLayer.Unmount(); err != nil {
			return errors.Wrap(err, "failed to unmount RWLayer")
		}
		l.fs = nil
	}

	metadata, err := l.layerStore.ReleaseRWLayer(l.rwLayer)
	layer.LogReleaseMetadata(metadata)
	if err != nil {
		return errors.Wrap(err, "failed to release RWLayer")
	}
	l.released = true
	return nil
}

func newROLayerForImage(img *image.Image, layerStore layer.Store) (builder.ROLayer, error) {
	if img == nil || img.RootFS.ChainID() == "" {
		return &roLayer{layerStore: layerStore}, nil
	}
	// Hold a reference to the image layer so that it can't be removed before
	// it is released
	lyr, err := layerStore.Get(img.RootFS.ChainID())
	if err != nil {
		return nil, errors.Wrapf(err, "failed to get layer for image %s", img.ImageID())
	}
	return &roLayer{layerStore: layerStore, roLayer: lyr}, nil
}

// TODO: could this use the regular daemon PullImage ?
func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, platform *specs.Platform) (*image.Image, error) {
	ref, err := reference.ParseNormalizedNamed(name)
	if err != nil {
		return nil, err
	}
	ref = reference.TagNameOnly(ref)

	pullRegistryAuth := &types.AuthConfig{}
	if len(authConfigs) > 0 {
		// The request came with a full auth config, use it
		repoInfo, err := i.registryService.ResolveRepository(ref)
		if err != nil {
			return nil, err
		}

		resolvedConfig := registry.ResolveAuthConfig(authConfigs, repoInfo.Index)
		pullRegistryAuth = &resolvedConfig
	}

	if err := i.pullImageWithReference(ctx, ref, platform, nil, pullRegistryAuth, output); err != nil {
		return nil, err
	}

	img, err := i.GetImage(name, platform)
	if errdefs.IsNotFound(err) && img != nil && platform != nil {
		imgPlat := specs.Platform{
			OS:           img.OS,
			Architecture: img.BaseImgArch(),
			Variant:      img.BaseImgVariant(),
		}

		p := *platform
		if !platforms.Only(p).Match(imgPlat) {
			po := streamformatter.NewJSONProgressOutput(output, false)
			progress.Messagef(po, "", `
WARNING: Pulled image with specified platform (%s), but the resulting image's configured platform (%s) does not match.
This is most likely caused by a bug in the build system that created the fetched image (%s).
Please notify the image author to correct the configuration.`,
				platforms.Format(p), platforms.Format(imgPlat), name,
			)
			logrus.WithError(err).WithField("image", name).Warn("Ignoring error about platform mismatch where the manifest list points to an image whose configuration does not match the platform in the manifest.")
			err = nil
		}
	}
	return img, err
}

// GetImageAndReleasableLayer returns an image and releaseable layer for a reference or ID.
// Every call to GetImageAndReleasableLayer MUST call releasableLayer.Release() to prevent
// leaking of layers.
func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) {
	if refOrID == "" { // ie FROM scratch
		os := runtime.GOOS
		if runtime.GOOS == "windows" {
			os = "linux"
		}
		if opts.Platform != nil {
			os = opts.Platform.OS
		}
		if !system.IsOSSupported(os) {
			return nil, nil, system.ErrNotSupportedOperatingSystem
		}
		lyr, err := newROLayerForImage(nil, i.layerStore)
		return nil, lyr, err
	}

	if opts.PullOption != backend.PullOptionForcePull {
		img, err := i.GetImage(refOrID, opts.Platform)
		if err != nil && opts.PullOption == backend.PullOptionNoPull {
			return nil, nil, err
		}
		// TODO: shouldn't we error out if error is different from "not found" ?
		if img != nil {
			if !system.IsOSSupported(img.OperatingSystem()) {
				return nil, nil, system.ErrNotSupportedOperatingSystem
			}
			lyr, err := newROLayerForImage(img, i.layerStore)
			return img, lyr, err
		}
	}

	img, err := i.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.Platform)
	if err != nil {
		return nil, nil, err
	}
	if !system.IsOSSupported(img.OperatingSystem()) {
		return nil, nil, system.ErrNotSupportedOperatingSystem
	}
	lyr, err := newROLayerForImage(img, i.layerStore)
	return img, lyr, err
}

// CreateImage creates a new image by adding a config and ID to the image store.
// This is similar to LoadImage() except that it receives JSON encoded bytes of
// an image instead of a tar archive.
func (i *ImageService) CreateImage(config []byte, parent string) (builder.Image, error) {
	id, err := i.imageStore.Create(config)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create image")
	}

	if parent != "" {
		if err := i.imageStore.SetParent(id, image.ID(parent)); err != nil {
			return nil, errors.Wrapf(err, "failed to set parent %s", parent)
		}
	}

	return i.imageStore.Get(id)
}
<|endoftext|>


<commit_before>package instana

import (
	"os"
	"sync"
	"time"

	"github.com/opentracing/basictracer-go"
	ext "github.com/opentracing/opentracing-go/ext"
)

type SpanRecorder struct {
	sync.RWMutex
	spans []Span
}

type Span struct {
	TraceID   uint64      `json:"t"`
	ParentID  *uint64     `json:"p,omitempty"`
	SpanID    uint64      `json:"s"`
	Timestamp uint64      `json:"ts"`
	Duration  uint64      `json:"d"`
	Name      string      `json:"n"`
	From      *FromS      `json:"f"`
	Data      interface{} `json:"data"`
}

func NewRecorder() *SpanRecorder {
	r := new(SpanRecorder)
	r.init()

	return r
}

func getTag(rawSpan basictracer.RawSpan, tag string) interface{} {
	return rawSpan.Tags[tag]
}

func getIntTag(rawSpan basictracer.RawSpan, tag string) int {
	d := rawSpan.Tags[tag]
	if d == nil {
		return -1
	}

	r, ok := d.(int)
	if !ok {
		return -1
	}

	return r
}

func getStringTag(rawSpan basictracer.RawSpan, tag string) string {
	d := getTag(rawSpan, tag)
	if d == nil {
		return ""
	}

	return d.(string)
}

func getHostName(rawSpan basictracer.RawSpan) string {
	hostTag := getStringTag(rawSpan, string(ext.PeerHostname))
	if hostTag != "" {
		return hostTag
	}

	h, err := os.Hostname()
	if err != nil {
		h = "localhost"
	}

	return h
}

func getServiceName(rawSpan basictracer.RawSpan) string {
	s := getStringTag(rawSpan, string(ext.Component))
	if s == "" {
		s = getStringTag(rawSpan, string(ext.PeerService))
		if s == "" {
			return sensor.serviceName
		}
	}

	return s
}

func getHTTPType(rawSpan basictracer.RawSpan) string {
	kind := getStringTag(rawSpan, string(ext.SpanKind))
	if kind == string(ext.SpanKindRPCServerEnum) {
		return HTTPServer
	}

	return HTTPClient
}

func collectLogs(rawSpan basictracer.RawSpan) map[uint64]map[string]interface{} {
	logs := make(map[uint64]map[string]interface{})
	for _, l := range rawSpan.Logs {
		if _, ok := logs[uint64(l.Timestamp.UnixNano())/uint64(time.Millisecond)]; !ok {
			logs[uint64(l.Timestamp.UnixNano())/uint64(time.Millisecond)] = make(map[string]interface{})
		}

		for _, f := range l.Fields {
			logs[uint64(l.Timestamp.UnixNano())/uint64(time.Millisecond)][f.Key()] = f.Value()
		}
	}

	return logs
}

func (r *SpanRecorder) init() {
	r.reset()
	ticker := time.NewTicker(1 * time.Second)
	go func() {
		for range ticker.C {
			log.debug("Sending spans to agent", len(r.spans))

			r.send()
		}
	}()
}

func (r *SpanRecorder) reset() {
	r.Lock()
	defer r.Unlock()
	r.spans = make([]Span, 0, sensor.options.MaxBufferedSpans)
}

func (r *SpanRecorder) RecordSpan(rawSpan basictracer.RawSpan) {
	var data = &Data{}
	var tp string
	h := getHostName(rawSpan)
	status := getIntTag(rawSpan, string(ext.HTTPStatusCode))
	if status >= 0 {
		tp = getHTTPType(rawSpan)
		data = &Data{HTTP: &HTTPData{
			Host:   h,
			URL:    getStringTag(rawSpan, string(ext.HTTPUrl)),
			Method: getStringTag(rawSpan, string(ext.HTTPMethod)),
			Status: status},
			SDK: &SDKData{Name: tp}}
	} else {
		log.debug("No HTTP status code provided or invalid status code, opting out to RPC")

		tp = RPC
		data = &Data{RPC: &RPCData{
			Host: h,
			Call: rawSpan.Operation},
			SDK: &SDKData{Name: tp}}
	}

	data.Custom = &CustomData{Tags: rawSpan.Tags, Logs: collectLogs(rawSpan)}

	baggage := make(map[string]string)
	rawSpan.Context.ForeachBaggageItem(func(k string, v string) bool {
		baggage[k] = v

		return true
	})

	if len(baggage) > 0 {
		data.Baggage = baggage
	}

	data.Service = getServiceName(rawSpan)

	var parentID *uint64
	if rawSpan.ParentSpanID == 0 {
		parentID = nil
	} else {
		parentID = &rawSpan.ParentSpanID
	}

	r.Lock()
	defer r.Unlock()

	if len(r.spans) == sensor.options.MaxBufferedSpans {
		r.spans = r.spans[1:]
	}

	r.spans = append(r.spans, Span{
		TraceID:   rawSpan.Context.TraceID,
		ParentID:  parentID,
		SpanID:    rawSpan.Context.SpanID,
		Timestamp: uint64(rawSpan.Start.UnixNano()) / uint64(time.Millisecond),
		Duration:  uint64(rawSpan.Duration) / uint64(time.Millisecond),
		Name:      "sdk",
		From:      sensor.agent.from,
		Data:      &data})

	if len(r.spans) == sensor.options.ForceTransmissionStartingAt {
		log.debug("Forcing spans to agent", len(r.spans))

		r.send()
	}
}

func (r *SpanRecorder) send() {
	if sensor.agent.canSend() {
		go func() {
			_, err := sensor.agent.request(sensor.agent.makeURL(AgentTracesURL), "POST", r.spans)

			r.reset()

			if err != nil {
				sensor.agent.reset()
			}
		}()
	}
}
<commit_msg>Add testMode support to recorder. Update to use SDK span type<commit_after>package instana

import (
	"os"
	"sync"
	"time"

	"github.com/opentracing/basictracer-go"
	ext "github.com/opentracing/opentracing-go/ext"
)

type SpanRecorder struct {
	sync.RWMutex
	spans    []Span
	testMode bool
}

type Span struct {
	TraceID   uint64      `json:"t"`
	ParentID  *uint64     `json:"p,omitempty"`
	SpanID    uint64      `json:"s"`
	Timestamp uint64      `json:"ts"`
	Duration  uint64      `json:"d"`
	Name      string      `json:"n"`
	From      *FromS      `json:"f"`
	Data      interface{} `json:"data"`
}

// NewRecorder Establish a new span recorder
func NewRecorder(testMode bool) *SpanRecorder {
	r := new(SpanRecorder)
	r.testMode = testMode
	r.init()

	return r
}

// GetSpans returns a copy of the array of spans accumulated so far.
func (r *SpanRecorder) GetSpans() []Span {
	r.RLock()
	defer r.RUnlock()
	spans := make([]Span, len(r.spans))
	copy(spans, r.spans)
	return spans
}

func getTag(rawSpan basictracer.RawSpan, tag string) interface{} {
	return rawSpan.Tags[tag]
}

func getIntTag(rawSpan basictracer.RawSpan, tag string) int {
	d := rawSpan.Tags[tag]
	if d == nil {
		return -1
	}

	r, ok := d.(int)
	if !ok {
		return -1
	}

	return r
}

func getStringTag(rawSpan basictracer.RawSpan, tag string) string {
	d := getTag(rawSpan, tag)
	if d == nil {
		return ""
	}

	return d.(string)
}

func getHostName(rawSpan basictracer.RawSpan) string {
	hostTag := getStringTag(rawSpan, string(ext.PeerHostname))
	if hostTag != "" {
		return hostTag
	}

	h, err := os.Hostname()
	if err != nil {
		h = "localhost"
	}

	return h
}

func getServiceName(rawSpan basictracer.RawSpan) string {
	// ServiceName can be determined from multiple sources and has
	// the following priority (preferred first):
	//   1. Added to the span via the OT component tag
	//   2. If span has http.url, then return ""
	//   2. Specified in the tracer instantiation
	component := getStringTag(rawSpan, string(ext.Component))

	if len(component) == 0 {
		httpURL := getStringTag(rawSpan, string(ext.HTTPUrl))

		if len(httpURL) > 0 {
			return httpURL
		}
		return component
	}
	return sensor.serviceName
}

func getSpanKind(rawSpan basictracer.RawSpan) string {
	kind := getStringTag(rawSpan, string(ext.SpanKind))
	if kind == string(ext.SpanKindRPCServerEnum) || kind == "consumer" {
		return "entry"
	} else if kind == string(ext.SpanKindRPCClientEnum) || kind == "producer" {
		return "exit"
	} else {
		return ""
	}
}

func collectLogs(rawSpan basictracer.RawSpan) map[uint64]map[string]interface{} {
	logs := make(map[uint64]map[string]interface{})
	for _, l := range rawSpan.Logs {
		if _, ok := logs[uint64(l.Timestamp.UnixNano())/uint64(time.Millisecond)]; !ok {
			logs[uint64(l.Timestamp.UnixNano())/uint64(time.Millisecond)] = make(map[string]interface{})
		}

		for _, f := range l.Fields {
			logs[uint64(l.Timestamp.UnixNano())/uint64(time.Millisecond)][f.Key()] = f.Value()
		}
	}

	return logs
}

func (r *SpanRecorder) init() {
	r.reset()

	if r.testMode {
		log.debug("Recorder in test mode.  Not reporting spans to the backend.")
	} else {
		ticker := time.NewTicker(1 * time.Second)
		go func() {
			for range ticker.C {
				log.debug("Sending spans to agent", len(r.spans))

				r.send()
			}
		}()
	}
}

func (r *SpanRecorder) reset() {
	r.Lock()
	defer r.Unlock()
	r.spans = make([]Span, 0, sensor.options.MaxBufferedSpans)
}

func (r *SpanRecorder) RecordSpan(rawSpan basictracer.RawSpan) {
	var data = &Data{}
	kind := getSpanKind(rawSpan)

	data.SDK = &SDKData{
		Name:   rawSpan.Operation,
		Type:   kind,
		Custom: &CustomData{Tags: rawSpan.Tags, Logs: collectLogs(rawSpan)}}

	baggage := make(map[string]string)
	rawSpan.Context.ForeachBaggageItem(func(k string, v string) bool {
		baggage[k] = v

		return true
	})

	if len(baggage) > 0 {
		data.SDK.Custom.Baggage = baggage
	}

	data.Service = getServiceName(rawSpan)

	var parentID *uint64
	if rawSpan.ParentSpanID == 0 {
		parentID = nil
	} else {
		parentID = &rawSpan.ParentSpanID
	}

	r.Lock()
	defer r.Unlock()

	if len(r.spans) == sensor.options.MaxBufferedSpans {
		r.spans = r.spans[1:]
	}

	r.spans = append(r.spans, Span{
		TraceID:   rawSpan.Context.TraceID,
		ParentID:  parentID,
		SpanID:    rawSpan.Context.SpanID,
		Timestamp: uint64(rawSpan.Start.UnixNano()) / uint64(time.Millisecond),
		Duration:  uint64(rawSpan.Duration) / uint64(time.Millisecond),
		Name:      "sdk",
		From:      sensor.agent.from,
		Data:      &data})

	if !r.testMode && (len(r.spans) == sensor.options.ForceTransmissionStartingAt) {
		log.debug("Forcing spans to agent", len(r.spans))

		r.send()
	}
}

func (r *SpanRecorder) send() {
	if sensor.agent.canSend() && !r.testMode {
		go func() {
			_, err := sensor.agent.request(sensor.agent.makeURL(AgentTracesURL), "POST", r.spans)

			r.reset()

			if err != nil {
				sensor.agent.reset()
			}
		}()
	}
}
<|endoftext|>


<commit_before>package sample

import (
	"bytes"
	"fmt"
	"os"
	"strings"

	"github.com/aliyun/aliyun-oss-go-sdk/oss"
)

// PutObjectSample illustrates two methods for uploading a file: simple upload and multipart upload.
func PutObjectSample() {
	// Create bucket
	bucket, err := GetTestBucket(bucketName)
	if err != nil {
		HandleError(err)
	}

	var val = "花间一壶酒，独酌无相亲。 举杯邀明月，对影成三人。"

	// Case 1: Upload an object from a string
	err = bucket.PutObject(objectKey, strings.NewReader(val))
	if err != nil {
		HandleError(err)
	}

	// Case 2: Upload an object whose value is a byte[]
	err = bucket.PutObject(objectKey, bytes.NewReader([]byte(val)))
	if err != nil {
		HandleError(err)
	}

	// Case 3: Upload the local file with file handle, user should open the file at first.
	fd, err := os.Open(localFile)
	if err != nil {
		HandleError(err)
	}
	defer fd.Close()

	err = bucket.PutObject(objectKey, fd)
	if err != nil {
		HandleError(err)
	}

	// Case 4: Upload an object with local file name, user need not open the file.
	err = bucket.PutObjectFromFile(objectKey, localFile)
	if err != nil {
		HandleError(err)
	}

	// Case 5: Upload an object with specified properties, PutObject/PutObjectFromFile/UploadFile also support this feature.
	options := []oss.Option{
		oss.Expires(futureDate),
		oss.ObjectACL(oss.ACLPublicRead),
		oss.Meta("myprop", "mypropval"),
	}
	err = bucket.PutObject(objectKey, strings.NewReader(val), options...)
	if err != nil {
		HandleError(err)
	}

	err = bucket.PutObject(objectKey, strings.NewReader(val), oss.ServerSideEncryption("KMS"), oss.ServerSideEncryptionKeyID(kmsID))
	if err != nil {
		HandleError(err)
	}

	props, err := bucket.GetObjectDetailedMeta(objectKey)
	if err != nil {
		HandleError(err)
	}
	fmt.Println("Object Meta:", props)

	// Case 6: Big file's multipart upload. It supports concurrent upload with resumable upload.
	// multipart upload with 100K as part size. By default 1 coroutine is used and no checkpoint is used.
	err = bucket.UploadFile(objectKey, localFile, 100*1024)
	if err != nil {
		HandleError(err)
	}

	// Part size is 100K and 3 coroutines are used
	err = bucket.UploadFile(objectKey, localFile, 100*1024, oss.Routines(3))
	if err != nil {
		HandleError(err)
	}

	// Part size is 100K and 3 coroutines with checkpoint
	err = bucket.UploadFile(objectKey, localFile, 100*1024, oss.Routines(3), oss.Checkpoint(true, ""))
	if err != nil {
		HandleError(err)
	}

	// Specify the local file path for checkpoint files.
	// the 2nd parameter of Checkpoint can specify the file path, when the file path is empty, it will upload the directory.
	err = bucket.UploadFile(objectKey, localFile, 100*1024, oss.Checkpoint(true, localFile+".cp"))
	if err != nil {
		HandleError(err)
	}

	// Delete object and bucket
	err = DeleteTestBucketAndObject(bucketName)
	if err != nil {
		HandleError(err)
	}

	fmt.Println("PutObjectSample completed")
}
<commit_msg>add sample for put object with callback<commit_after>package sample

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"os"
	"strings"

	"github.com/aliyun/aliyun-oss-go-sdk/oss"
)

// PutObjectSample illustrates two methods for uploading a file: simple upload and multipart upload.
func PutObjectSample() {
	// Create bucket
	bucket, err := GetTestBucket(bucketName)
	if err != nil {
		HandleError(err)
	}

	var val = "花间一壶酒，独酌无相亲。 举杯邀明月，对影成三人。"

	// Case 1: Upload an object from a string
	err = bucket.PutObject(objectKey, strings.NewReader(val))
	if err != nil {
		HandleError(err)
	}

	// Case 2: Upload an object whose value is a byte[]
	err = bucket.PutObject(objectKey, bytes.NewReader([]byte(val)))
	if err != nil {
		HandleError(err)
	}

	// Case 3: Upload the local file with file handle, user should open the file at first.
	fd, err := os.Open(localFile)
	if err != nil {
		HandleError(err)
	}
	defer fd.Close()

	err = bucket.PutObject(objectKey, fd)
	if err != nil {
		HandleError(err)
	}

	// Case 4: Upload an object with local file name, user need not open the file.
	err = bucket.PutObjectFromFile(objectKey, localFile)
	if err != nil {
		HandleError(err)
	}

	// Case 5: Upload an object with specified properties, PutObject/PutObjectFromFile/UploadFile also support this feature.
	options := []oss.Option{
		oss.Expires(futureDate),
		oss.ObjectACL(oss.ACLPublicRead),
		oss.Meta("myprop", "mypropval"),
	}
	err = bucket.PutObject(objectKey, strings.NewReader(val), options...)
	if err != nil {
		HandleError(err)
	}

	props, err := bucket.GetObjectDetailedMeta(objectKey)
	if err != nil {
		HandleError(err)
	}
	fmt.Println("Object Meta:", props)

	// Case 6: Upload an object with sever side encrpytion kms and kms id specified
	err = bucket.PutObject(objectKey, strings.NewReader(val), oss.ServerSideEncryption("KMS"), oss.ServerSideEncryptionKeyID(kmsID))
	if err != nil {
		HandleError(err)
	}

	// Case 7: Upload an object with callback
	callbackMap := map[string]string{}
	callbackMap["callbackUrl"] = "http://oss-demo.aliyuncs.com:23450"
	callbackMap["callbackHost"] = "oss-cn-hangzhou.aliyuncs.com"
	callbackMap["callbackBody"] = "filename=${object}&size=${size}&mimeType=${mimeType}"
	callbackMap["callbackBodyType"] = "application/x-www-form-urlencoded"

	callbackValJSON, err := json.Marshal(callbackMap)
	if err != nil {
		HandleError(err)
	}

	callbackVal := base64.StdEncoding.EncodeToString(callbackValJSON)
	err = bucket.PutObject(objectKey, strings.NewReader(val), oss.Callback(callbackVal))
	if err != nil {
		HandleError(err)
	}

	// Case 8: Big file's multipart upload. It supports concurrent upload with resumable upload.
	// multipart upload with 100K as part size. By default 1 coroutine is used and no checkpoint is used.
	err = bucket.UploadFile(objectKey, localFile, 100*1024)
	if err != nil {
		HandleError(err)
	}

	// Part size is 100K and 3 coroutines are used
	err = bucket.UploadFile(objectKey, localFile, 100*1024, oss.Routines(3))
	if err != nil {
		HandleError(err)
	}

	// Part size is 100K and 3 coroutines with checkpoint
	err = bucket.UploadFile(objectKey, localFile, 100*1024, oss.Routines(3), oss.Checkpoint(true, ""))
	if err != nil {
		HandleError(err)
	}

	// Specify the local file path for checkpoint files.
	// the 2nd parameter of Checkpoint can specify the file path, when the file path is empty, it will upload the directory.
	err = bucket.UploadFile(objectKey, localFile, 100*1024, oss.Checkpoint(true, localFile+".cp"))
	if err != nil {
		HandleError(err)
	}

	// Delete object and bucket
	err = DeleteTestBucketAndObject(bucketName)
	if err != nil {
		HandleError(err)
	}

	fmt.Println("PutObjectSample completed")
}
<|endoftext|>


<commit_before>package cmd

// Version defines the current Pop version.
const Version = "development"
<commit_msg>version bump: v4.9.0<commit_after>package cmd

// Version defines the current Pop version.
const Version = "v4.9.0"
<|endoftext|>