<commit_before>package tools

// In /etc/hummingbird/andrewd-server.conf:
// [ring-monitor]
// initial_delay = 1      # seconds to wait between ring checks for the first pass
// pass_time_target = 60  # seconds to try to make subsequent passes take
// report_interval = 600  # seconds between progress reports

import (
	"fmt"
	"sync/atomic"
	"time"

	"github.com/troubling/hummingbird/common/ring"
	"go.uber.org/zap"
)

type ringMonitor struct {
	aa *AutoAdmin
	// delay between each ring check; adjusted each pass to try to make passes last passTimeTarget
	delay          time.Duration
	passTimeTarget time.Duration
	reportInterval time.Duration
	prefix         string
	suffix         string
}

func newRingMonitor(aa *AutoAdmin) *ringMonitor {
	rm := &ringMonitor{
		aa:             aa,
		delay:          time.Duration(aa.serverconf.GetInt("ring-monitor", "initial_delay", 1)) * time.Second,
		passTimeTarget: time.Duration(aa.serverconf.GetInt("ring-monitor", "pass_time_target", 60)) * time.Second,
		reportInterval: time.Duration(aa.serverconf.GetInt("ring-monitor", "report_interval", 600)) * time.Second,
	}
	if rm.delay < 0 {
		rm.delay = time.Second
	}
	if rm.passTimeTarget < 0 {
		rm.passTimeTarget = time.Second
	}
	if rm.reportInterval < 0 {
		rm.reportInterval = time.Second
	}
	rm.prefix, rm.suffix = getAffixes()
	return rm
}

func (rm *ringMonitor) runForever() {
	for {
		sleepFor := rm.runOnce()
		if sleepFor < 0 {
			break
		}
		time.Sleep(sleepFor)
	}
}

func (rm *ringMonitor) runOnce() time.Duration {
	start := time.Now()
	logger := rm.aa.logger.With(zap.String("process", "ring monitor"))
	logger.Debug("starting pass")
	if err := rm.aa.db.startProcessPass("ring monitor", "", 0); err != nil {
		logger.Error("startProcessPass", zap.Error(err))
	}
	var delays int64
	var errors int64
	var partitionCopiesChanged int64
	type ringTask struct {
		typ         string
		policy      int
		ring        ring.RingMD5
		previousMD5 string
	}
	var ringTasks []*ringTask
	ryng, err := ring.GetRingMD5("account", rm.prefix, rm.suffix, 0)
	if err != nil {
		errors++
		logger.Error("could not load ring", zap.String("type", "account"), zap.Int("policy", 0), zap.Error(err))
	} else {
		ringTasks = append(ringTasks, &ringTask{
			typ:  "account",
			ring: ryng,
		})
	}
	ryng, err = ring.GetRingMD5("container", rm.prefix, rm.suffix, 0)
	if err != nil {
		errors++
		logger.Error("could not load ring", zap.String("type", "container"), zap.Int("policy", 0), zap.Error(err))
	} else {
		ringTasks = append(ringTasks, &ringTask{
			typ:  "container",
			ring: ryng,
		})
	}
	for _, policy := range rm.aa.policies {
		if !policy.Deprecated {
			ryng, err = ring.GetRingMD5("object", rm.prefix, rm.suffix, policy.Index)
			if err != nil {
				errors++
				logger.Error("could not load ring", zap.String("type", "object"), zap.Int("policy", policy.Index), zap.Error(err))
				continue
			}
			ringTasks = append(ringTasks, &ringTask{
				typ:    "object",
				policy: policy.Index,
				ring:   ryng,
			})
		}
	}
	cancel := make(chan struct{})
	progressDone := make(chan struct{})
	go func() {
		for {
			select {
			case <-cancel:
				close(progressDone)
				return
			case <-time.After(rm.reportInterval):
				d := atomic.LoadInt64(&delays)
				e := atomic.LoadInt64(&errors)
				p := atomic.LoadInt64(&partitionCopiesChanged)
				eta := time.Duration(int64(time.Since(start)) / d * (int64(len(ringTasks)) - d))
				logger.Debug("progress", zap.Int64("rings so far", d), zap.Int("total rings", len(ringTasks)), zap.Int64("errors", e), zap.Int64("partition copies changed", p), zap.String("eta", eta.String()))
				if err := rm.aa.db.progressProcessPass("ring monitor", "", 0, fmt.Sprintf("%d of %d rings, %d errors, %d partition copies changed, eta %s", d, len(ringTasks), e, p, eta)); err != nil {
					logger.Error("progressProcessPass", zap.Error(err))
				}
			}
		}
	}()
	for _, ringTask := range ringTasks {
		atomic.AddInt64(&delays, 1)
		time.Sleep(rm.delay)
		taskLogger := logger.With(zap.String("type", ringTask.typ), zap.Int("policy", ringTask.policy), zap.String("previous md5", ringTask.previousMD5))
		if err := ringTask.ring.Reload(); err != nil {
			atomic.AddInt64(&errors, 1)
			taskLogger.Error("couldn't not reload", zap.Error(err))
			continue
		}
		if ringTask.previousMD5 == "" {
			ringTask.previousMD5, err = rm.aa.db.ringHash(ringTask.typ, ringTask.policy)
			if err != nil {
				atomic.AddInt64(&errors, 1)
				taskLogger.Error("couldn't retreive previous hash", zap.Error(err))
				continue
			}
		}
		if ringTask.ring.MD5() != ringTask.previousMD5 {
			if ringTask.previousMD5 == "" {
				// First time seeing this ring
				rm.aa.db.setRingHash(ringTask.typ, ringTask.policy, ringTask.ring.MD5())
				continue
			}
			taskLogger.Debug("ring md5 changed", zap.String("current md5", ringTask.ring.MD5()))
			previousRing := ringTask.ring.RingMatching(ringTask.previousMD5)
			if previousRing == nil {
				atomic.AddInt64(&errors, 1)
				taskLogger.Error("can't find previous ring; assuming nothing changed")
				rm.aa.db.setRingHash(ringTask.typ, ringTask.policy, ringTask.ring.MD5())
				continue
			}
			partitionCount := previousRing.PartitionCount()
			if partitionCount != ringTask.ring.PartitionCount() {
				atomic.AddInt64(&errors, 1)
				taskLogger.Error(
					"cannot handle changing partition counts; assuming new ring as if no previous ring ever existed",
					zap.Uint64("previous partition count", previousRing.PartitionCount()),
					zap.Uint64("current partition count", ringTask.ring.PartitionCount()),
				)
				rm.aa.db.setRingHash(ringTask.typ, ringTask.policy, ringTask.ring.MD5())
				continue
			}
			replicaCount := previousRing.ReplicaCount()
			if replicaCount != ringTask.ring.ReplicaCount() {
				atomic.AddInt64(&errors, 1)
				taskLogger.Error(
					"cannot handle changing replica counts; assuming new ring as if no previous ring ever existed",
					zap.Uint64("previous replica count", previousRing.ReplicaCount()),
					zap.Uint64("current replica count", ringTask.ring.ReplicaCount()),
				)
				rm.aa.db.setRingHash(ringTask.typ, ringTask.policy, ringTask.ring.MD5())
				continue
			}
			failed := false
			for partition := uint64(0); partition < partitionCount && !failed; partition++ {
				previousDev := previousRing.GetNodes(partition)
				currentDev := ringTask.ring.GetNodes(partition)
				for replica := uint64(0); replica < replicaCount; replica++ {
					if previousDev[replica].Id != currentDev[replica].Id {
						if err := rm.aa.db.queuePartitionReplication(
							ringTask.typ,
							ringTask.policy,
							partition,
							"ring change",
							previousDev[replica].Id,
							currentDev[replica].Id,
						); err != nil {
							atomic.AddInt64(&errors, 1)
							taskLogger.Error(
								"could not queue ring change; will try again next pass",
								zap.Uint64("partition", partition),
								zap.Uint64("replica", replica),
								zap.Int("previous id", previousDev[replica].Id),
								zap.Int("current id", currentDev[replica].Id),
								zap.Error(err),
							)
							failed = true
							break
						} else {
							atomic.AddInt64(&partitionCopiesChanged, 1)
						}
					}
				}
			}
			if !failed {
				if err = rm.aa.db.setRingHash(ringTask.typ, ringTask.policy, ringTask.ring.MD5()); err != nil {
					atomic.AddInt64(&errors, 1)
				}
			}
		}
	}
	close(cancel)
	<-progressDone
	if delays > 0 {
		rm.delay = rm.passTimeTarget / time.Duration(delays)
	}
	sleepFor := time.Until(start.Add(rm.passTimeTarget))
	if sleepFor < 0 {
		sleepFor = 0
	}
	logger.Debug("pass complete", zap.Int("total rings", len(ringTasks)), zap.Int64("errors", errors), zap.Int64("partition copies changed", partitionCopiesChanged), zap.String("next delay", rm.delay.String()), zap.String("sleep for", sleepFor.String()))
	if err := rm.aa.db.progressProcessPass("ring monitor", "", 0, fmt.Sprintf("%d of %d urls, %d errors, %d partition copies changed", delays, len(ringTasks), errors, partitionCopiesChanged)); err != nil {
		logger.Error("progressProcessPass", zap.Error(err))
	}
	if err := rm.aa.db.completeProcessPass("ring monitor", "", 0); err != nil {
		logger.Error("completeProcessPass", zap.Error(err))
	}
	return sleepFor
}
<commit_msg>Updated logging<commit_after>package tools

// In /etc/hummingbird/andrewd-server.conf:
// [ring-monitor]
// initial_delay = 1      # seconds to wait between ring checks for the first pass
// pass_time_target = 60  # seconds to try to make subsequent passes take
// report_interval = 600  # seconds between progress reports

import (
	"fmt"
	"sync/atomic"
	"time"

	"github.com/troubling/hummingbird/common/ring"
	"go.uber.org/zap"
)

type ringMonitor struct {
	aa *AutoAdmin
	// delay between each ring check; adjusted each pass to try to make passes last passTimeTarget
	delay          time.Duration
	passTimeTarget time.Duration
	reportInterval time.Duration
	prefix         string
	suffix         string
}

func newRingMonitor(aa *AutoAdmin) *ringMonitor {
	rm := &ringMonitor{
		aa:             aa,
		delay:          time.Duration(aa.serverconf.GetInt("ring-monitor", "initial_delay", 1)) * time.Second,
		passTimeTarget: time.Duration(aa.serverconf.GetInt("ring-monitor", "pass_time_target", 60)) * time.Second,
		reportInterval: time.Duration(aa.serverconf.GetInt("ring-monitor", "report_interval", 600)) * time.Second,
	}
	if rm.delay < 0 {
		rm.delay = time.Second
	}
	if rm.passTimeTarget < 0 {
		rm.passTimeTarget = time.Second
	}
	if rm.reportInterval < 0 {
		rm.reportInterval = time.Second
	}
	rm.prefix, rm.suffix = getAffixes()
	return rm
}

func (rm *ringMonitor) runForever() {
	for {
		sleepFor := rm.runOnce()
		if sleepFor < 0 {
			break
		}
		time.Sleep(sleepFor)
	}
}

func (rm *ringMonitor) runOnce() time.Duration {
	start := time.Now()
	logger := rm.aa.logger.With(zap.String("process", "ring monitor"))
	logger.Debug("starting pass")
	if err := rm.aa.db.startProcessPass("ring monitor", "", 0); err != nil {
		logger.Error("startProcessPass", zap.Error(err))
	}
	var delays int64
	var errors int64
	var partitionCopiesChanged int64
	type ringTask struct {
		typ         string
		policy      int
		ring        ring.RingMD5
		previousMD5 string
	}
	var ringTasks []*ringTask
	ryng, err := ring.GetRingMD5("account", rm.prefix, rm.suffix, 0)
	if err != nil {
		errors++
		logger.Error("could not load ring", zap.String("type", "account"), zap.Int("policy", 0), zap.Error(err))
	} else {
		ringTasks = append(ringTasks, &ringTask{
			typ:  "account",
			ring: ryng,
		})
	}
	ryng, err = ring.GetRingMD5("container", rm.prefix, rm.suffix, 0)
	if err != nil {
		errors++
		logger.Error("could not load ring", zap.String("type", "container"), zap.Int("policy", 0), zap.Error(err))
	} else {
		ringTasks = append(ringTasks, &ringTask{
			typ:  "container",
			ring: ryng,
		})
	}
	for _, policy := range rm.aa.policies {
		if !policy.Deprecated {
			ryng, err = ring.GetRingMD5("object", rm.prefix, rm.suffix, policy.Index)
			if err != nil {
				errors++
				logger.Error("could not load ring", zap.String("type", "object"), zap.Int("policy", policy.Index), zap.Error(err))
				continue
			}
			ringTasks = append(ringTasks, &ringTask{
				typ:    "object",
				policy: policy.Index,
				ring:   ryng,
			})
		}
	}
	cancel := make(chan struct{})
	progressDone := make(chan struct{})
	go func() {
		for {
			select {
			case <-cancel:
				close(progressDone)
				return
			case <-time.After(rm.reportInterval):
				d := atomic.LoadInt64(&delays)
				e := atomic.LoadInt64(&errors)
				p := atomic.LoadInt64(&partitionCopiesChanged)
				eta := time.Duration(int64(time.Since(start)) / d * (int64(len(ringTasks)) - d))
				logger.Debug("progress", zap.Int64("rings so far", d), zap.Int("total rings", len(ringTasks)), zap.Int64("errors", e), zap.Int64("partition copies changed", p), zap.String("eta", eta.String()))
				if err := rm.aa.db.progressProcessPass("ring monitor", "", 0, fmt.Sprintf("%d of %d rings, %d errors, %d partition copies changed, eta %s", d, len(ringTasks), e, p, eta)); err != nil {
					logger.Error("progressProcessPass", zap.Error(err))
				}
			}
		}
	}()
	for _, ringTask := range ringTasks {
		atomic.AddInt64(&delays, 1)
		time.Sleep(rm.delay)
		taskLogger := logger.With(zap.String("type", ringTask.typ), zap.Int("policy", ringTask.policy))
		if err := ringTask.ring.Reload(); err != nil {
			atomic.AddInt64(&errors, 1)
			taskLogger.Error("couldn't not reload", zap.Error(err))
			continue
		}
		if ringTask.previousMD5 == "" {
			ringTask.previousMD5, err = rm.aa.db.ringHash(ringTask.typ, ringTask.policy)
			if err != nil {
				atomic.AddInt64(&errors, 1)
				taskLogger.Error("couldn't retreive previous hash", zap.Error(err))
				continue
			}
		}
		if ringTask.ring.MD5() != ringTask.previousMD5 {
			if ringTask.previousMD5 == "" {
				// First time seeing this ring
				rm.aa.db.setRingHash(ringTask.typ, ringTask.policy, ringTask.ring.MD5())
				continue
			}
			changeTaskLogger := taskLogger.With(zap.String("previous md5", ringTask.previousMD5), zap.String("current md5", ringTask.ring.MD5()))
			changeTaskLogger.Debug("ring md5 changed")
			previousRing := ringTask.ring.RingMatching(ringTask.previousMD5)
			if previousRing == nil {
				atomic.AddInt64(&errors, 1)
				changeTaskLogger.Error("can't find previous ring; assuming nothing changed")
				rm.aa.db.setRingHash(ringTask.typ, ringTask.policy, ringTask.ring.MD5())
				continue
			}
			partitionCount := previousRing.PartitionCount()
			if partitionCount != ringTask.ring.PartitionCount() {
				atomic.AddInt64(&errors, 1)
				changeTaskLogger.Error(
					"cannot handle changing partition counts; assuming new ring as if no previous ring ever existed",
					zap.Uint64("previous partition count", previousRing.PartitionCount()),
					zap.Uint64("current partition count", ringTask.ring.PartitionCount()),
				)
				rm.aa.db.setRingHash(ringTask.typ, ringTask.policy, ringTask.ring.MD5())
				continue
			}
			replicaCount := previousRing.ReplicaCount()
			if replicaCount != ringTask.ring.ReplicaCount() {
				atomic.AddInt64(&errors, 1)
				changeTaskLogger.Error(
					"cannot handle changing replica counts; assuming new ring as if no previous ring ever existed",
					zap.Uint64("previous replica count", previousRing.ReplicaCount()),
					zap.Uint64("current replica count", ringTask.ring.ReplicaCount()),
				)
				rm.aa.db.setRingHash(ringTask.typ, ringTask.policy, ringTask.ring.MD5())
				continue
			}
			failed := false
			for partition := uint64(0); partition < partitionCount && !failed; partition++ {
				previousDev := previousRing.GetNodes(partition)
				currentDev := ringTask.ring.GetNodes(partition)
				for replica := uint64(0); replica < replicaCount; replica++ {
					if previousDev[replica].Id != currentDev[replica].Id {
						if err := rm.aa.db.queuePartitionReplication(
							ringTask.typ,
							ringTask.policy,
							partition,
							"ring change",
							previousDev[replica].Id,
							currentDev[replica].Id,
						); err != nil {
							atomic.AddInt64(&errors, 1)
							changeTaskLogger.Error(
								"could not queue ring change; will try again next pass",
								zap.Uint64("partition", partition),
								zap.Uint64("replica", replica),
								zap.Int("previous id", previousDev[replica].Id),
								zap.Int("current id", currentDev[replica].Id),
								zap.Error(err),
							)
							failed = true
							break
						} else {
							atomic.AddInt64(&partitionCopiesChanged, 1)
						}
					}
				}
			}
			if !failed {
				if err = rm.aa.db.setRingHash(ringTask.typ, ringTask.policy, ringTask.ring.MD5()); err != nil {
					atomic.AddInt64(&errors, 1)
					changeTaskLogger.Error(
						"could not record the new ring hash; will try again next pass",
						zap.Error(err),
					)
				}
			}
		}
	}
	close(cancel)
	<-progressDone
	if delays > 0 {
		rm.delay = rm.passTimeTarget / time.Duration(delays)
	}
	sleepFor := time.Until(start.Add(rm.passTimeTarget))
	if sleepFor < 0 {
		sleepFor = 0
	}
	logger.Debug("pass complete", zap.Int("total rings", len(ringTasks)), zap.Int64("errors", errors), zap.Int64("partition copies changed", partitionCopiesChanged), zap.String("next delay", rm.delay.String()), zap.String("sleep for", sleepFor.String()))
	if err := rm.aa.db.progressProcessPass("ring monitor", "", 0, fmt.Sprintf("%d of %d urls, %d errors, %d partition copies changed", delays, len(ringTasks), errors, partitionCopiesChanged)); err != nil {
		logger.Error("progressProcessPass", zap.Error(err))
	}
	if err := rm.aa.db.completeProcessPass("ring monitor", "", 0); err != nil {
		logger.Error("completeProcessPass", zap.Error(err))
	}
	return sleepFor
}
<|endoftext|>


<commit_before>package gofetch_test

import (
	"fmt"
	"os"

	"github.com/c4milo/gofetch"
)

func Example() {
	gf := gofetch.New(
		gofetch.WithDestDir("/tmp"),
		gofetch.WithConcurrency(10),
		gofetch.WithETag(true),
	)

	progressCh := make(chan gofetch.ProgressReport)

	var myFile *os.File
	go func() {
		var err error
		myFile, err = gf.Fetch(
			"http://releases.ubuntu.com/15.10/ubuntu-15.10-server-amd64.iso",
			progressCh)
		if err != nil {
			panic(err)
		}
	}()

	// pogressCh is closed by gofetch once a download finishes.
	var totalWritten int64
	for p := range progressCh {
		// p.WrittenBytes does not accumulate, it represents the chunk size written
		// in the current operation.
		totalWritten += p.WrittenBytes
		fmt.Printf("\r%d of %d bytes", totalWritten, p.Total)
	}
	fmt.Printf("\nFile saved at %q\n", myFile.Name())
}
<commit_msg>Fixes example code<commit_after>package gofetch_test

import (
	"fmt"
	"os"

	"github.com/c4milo/gofetch"
)

func Example() {
	gf := gofetch.New(
		gofetch.WithDestDir("/tmp"),
		gofetch.WithConcurrency(10),
		gofetch.WithETag(),
	)

	progressCh := make(chan gofetch.ProgressReport)

	var myFile *os.File
	go func() {
		var err error
		myFile, err = gf.Fetch(
			"http://releases.ubuntu.com/15.10/ubuntu-15.10-server-amd64.iso",
			progressCh)
		if err != nil {
			panic(err)
		}
	}()

	// pogressCh is closed by gofetch once a download finishes.
	var totalWritten int64
	for p := range progressCh {
		// p.WrittenBytes does not accumulate, it represents the chunk size written
		// in the current operation.
		totalWritten += p.WrittenBytes
		fmt.Printf("\r%d of %d bytes", totalWritten, p.Total)
	}
	defer myFile.Close()

	fmt.Printf("\nFile saved at %q\n", myFile.Name())
}
<|endoftext|>


<commit_before>// +build go1.10

package mutagen

import (
	"encoding/binary"
	"fmt"
	"io"
)

const (
	// VersionMajor represents the current major version of Mutagen.
	VersionMajor = 0
	// VersionMinor represents the current minor version of Mutagen.
	VersionMinor = 5
	// VersionPatch represents the current patch version of Mutagen.
	VersionPatch = 0
)

var Version string

func init() {
	Version = fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch)
}

type versionBytes [12]byte

func SendVersion(writer io.Writer) error {
	// Compute the version bytes.
	var data versionBytes
	binary.BigEndian.PutUint32(data[:4], VersionMajor)
	binary.BigEndian.PutUint32(data[4:8], VersionMinor)
	binary.BigEndian.PutUint32(data[8:], VersionPatch)

	// Transmit the bytes.
	_, err := writer.Write(data[:])
	return err
}

func ReceiveVersion(reader io.Reader) (uint32, uint32, uint32, error) {
	// Read the bytes.
	var data versionBytes
	if _, err := io.ReadFull(reader, data[:]); err != nil {
		return 0, 0, 0, err
	}

	// Decode components.
	major := binary.BigEndian.Uint32(data[:4])
	minor := binary.BigEndian.Uint32(data[4:8])
	patch := binary.BigEndian.Uint32(data[8:])

	// Done.
	return major, minor, patch, nil
}

func ReceiveAndCompareVersion(reader io.Reader) (bool, error) {
	// Receive the version.
	major, minor, patch, err := ReceiveVersion(reader)
	if err != nil {
		return false, err
	}

	// Compare the version.
	return major == VersionMajor &&
		minor == VersionMinor &&
		patch == VersionPatch, nil
}
<commit_msg>Bumped development version to v0.5.1.<commit_after>// +build go1.10

package mutagen

import (
	"encoding/binary"
	"fmt"
	"io"
)

const (
	// VersionMajor represents the current major version of Mutagen.
	VersionMajor = 0
	// VersionMinor represents the current minor version of Mutagen.
	VersionMinor = 5
	// VersionPatch represents the current patch version of Mutagen.
	VersionPatch = 1
)

var Version string

func init() {
	Version = fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch)
}

type versionBytes [12]byte

func SendVersion(writer io.Writer) error {
	// Compute the version bytes.
	var data versionBytes
	binary.BigEndian.PutUint32(data[:4], VersionMajor)
	binary.BigEndian.PutUint32(data[4:8], VersionMinor)
	binary.BigEndian.PutUint32(data[8:], VersionPatch)

	// Transmit the bytes.
	_, err := writer.Write(data[:])
	return err
}

func ReceiveVersion(reader io.Reader) (uint32, uint32, uint32, error) {
	// Read the bytes.
	var data versionBytes
	if _, err := io.ReadFull(reader, data[:]); err != nil {
		return 0, 0, 0, err
	}

	// Decode components.
	major := binary.BigEndian.Uint32(data[:4])
	minor := binary.BigEndian.Uint32(data[4:8])
	patch := binary.BigEndian.Uint32(data[8:])

	// Done.
	return major, minor, patch, nil
}

func ReceiveAndCompareVersion(reader io.Reader) (bool, error) {
	// Receive the version.
	major, minor, patch, err := ReceiveVersion(reader)
	if err != nil {
		return false, err
	}

	// Compare the version.
	return major == VersionMajor &&
		minor == VersionMinor &&
		patch == VersionPatch, nil
}
<|endoftext|>


<commit_before>package chromedp_test

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"

	"github.com/chromedp/cdproto/page"
	cdpruntime "github.com/chromedp/cdproto/runtime"
	"github.com/chromedp/cdproto/target"
	"github.com/chromedp/chromedp"
)

func writeHTML(content string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html")
		io.WriteString(w, strings.TrimSpace(content))
	})
}

func ExampleTitle() {
	ctx, cancel := chromedp.NewContext(context.Background())
	defer cancel()

	ts := httptest.NewServer(writeHTML(`

	<title>fancy website title</title>

	`))
	defer ts.Close()

	var title string
	if err := chromedp.Run(ctx,
		chromedp.Navigate(ts.URL),
		chromedp.Title(&title),
	); err != nil {
		panic(err)
	}
	fmt.Println(title)

	// Output:
	// fancy website title
}

func ExampleExecAllocator() {
	dir, err := ioutil.TempDir("", "chromedp-example")
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(dir)

	opts := []chromedp.ExecAllocatorOption{
		chromedp.NoFirstRun,
		chromedp.NoDefaultBrowserCheck,
		chromedp.Headless,
		chromedp.DisableGPU,
		chromedp.UserDataDir(dir),
	}

	allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
	defer cancel()

	// also set up a custom logger
	taskCtx, cancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(log.Printf))
	defer cancel()

	// ensure that the browser process is started
	if err := chromedp.Run(taskCtx); err != nil {
		panic(err)
	}

	path := filepath.Join(dir, "DevToolsActivePort")
	bs, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	lines := bytes.Split(bs, []byte("\n"))
	fmt.Printf("DevToolsActivePort has %d lines\n", len(lines))

	// Output:
	// DevToolsActivePort has 2 lines
}

func ExampleNewContext_manyTabs() {
	// new browser, first tab
	ctx1, cancel := chromedp.NewContext(context.Background())
	defer cancel()

	// ensure the first tab is created
	if err := chromedp.Run(ctx1); err != nil {
		panic(err)
	}

	// same browser, second tab
	ctx2, _ := chromedp.NewContext(ctx1)

	// ensure the second tab is created
	if err := chromedp.Run(ctx2); err != nil {
		panic(err)
	}

	c1 := chromedp.FromContext(ctx1)
	c2 := chromedp.FromContext(ctx2)

	fmt.Printf("Same browser: %t\n", c1.Browser == c2.Browser)
	fmt.Printf("Same tab: %t\n", c1.Target == c2.Target)

	// Output:
	// Same browser: true
	// Same tab: false
}

func ExampleListenTarget_consoleLog() {
	ctx, cancel := chromedp.NewContext(context.Background())
	defer cancel()

	ts := httptest.NewServer(writeHTML(`

	`))
	defer ts.Close()

	chromedp.ListenTarget(ctx, func(ev interface{}) {
		switch ev := ev.(type) {
		case *cdpruntime.EventConsoleAPICalled:
			fmt.Printf("console.%s call:\n", ev.Type)
			for _, arg := range ev.Args {
				fmt.Printf("%s - %s\n", arg.Type, arg.Value)
			}
		}
	})

	if err := chromedp.Run(ctx,
		chromedp.Navigate(ts.URL),
		chromedp.WaitVisible("#jsFinished", chromedp.ByID),
	); err != nil {
		panic(err)
	}

	// Output:
	// console.log call:
	// string - "hello js world"
}

func ExampleClickNewTab() {
	ctx, cancel := chromedp.NewContext(context.Background())
	defer cancel()

	mux := http.NewServeMux()
	mux.Handle("/first", writeHTML(`
<input id='newtab' type='button' value='open' onclick='window.open("/second", "_blank");'/>
	`))
	mux.Handle("/second", writeHTML(``))
	ts := httptest.NewServer(mux)
	defer ts.Close()

	lctx, cancel := context.WithCancel(ctx)
	ch := make(chan target.ID, 1)
	chromedp.ListenTarget(lctx, func(ev interface{}) {
		if ev, ok := ev.(*target.EventTargetCreated); ok &&
			// if OpenerID == "", this is the first tab.
			ev.TargetInfo.OpenerID != "" {
			ch <- ev.TargetInfo.TargetID
			cancel()
		}
	})
	if err := chromedp.Run(ctx,
		chromedp.Navigate(ts.URL+"/first"),
		chromedp.Click("#newtab", chromedp.ByID),
	); err != nil {
		panic(err)
	}
	newCtx, cancel := chromedp.NewContext(ctx, chromedp.WithTargetID(<-ch))
	defer cancel()

	var urlstr string
	if err := chromedp.Run(newCtx, chromedp.Location(&urlstr)); err != nil {
		panic(err)
	}
	fmt.Println("new tab's path:", strings.TrimPrefix(urlstr, ts.URL))

	// Output:
	// new tab's path: /second
}

func ExampleAcceptAlert() {
	ctx, cancel := chromedp.NewContext(context.Background())
	defer cancel()

	mux := http.NewServeMux()
	mux.Handle("/second", writeHTML(``))
	ts := httptest.NewServer(writeHTML(`
<input id='alert' type='button' value='alert' onclick='alert("alert text");'/>
	`))
	defer ts.Close()

	chromedp.ListenTarget(ctx, func(ev interface{}) {
		if ev, ok := ev.(*page.EventJavascriptDialogOpening); ok {
			fmt.Println("closing alert:", ev.Message)
			go func() {
				if err := chromedp.Run(ctx,
					page.HandleJavaScriptDialog(true),
				); err != nil {
					panic(err)
				}
			}()
		}
	})

	if err := chromedp.Run(ctx,
		chromedp.Navigate(ts.URL),
		chromedp.Click("#alert", chromedp.ByID),
	); err != nil {
		panic(err)
	}

	// Output:
	// closing alert: alert text
}
<commit_msg>add example to retrieve HTML at various stages<commit_after>package chromedp_test

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"

	"github.com/chromedp/cdproto/network"
	"github.com/chromedp/cdproto/page"
	cdpruntime "github.com/chromedp/cdproto/runtime"
	"github.com/chromedp/cdproto/target"
	"github.com/chromedp/chromedp"
)

func writeHTML(content string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html")
		io.WriteString(w, strings.TrimSpace(content))
	})
}

func ExampleTitle() {
	ctx, cancel := chromedp.NewContext(context.Background())
	defer cancel()

	ts := httptest.NewServer(writeHTML(`

	<title>fancy website title</title>

	`))
	defer ts.Close()

	var title string
	if err := chromedp.Run(ctx,
		chromedp.Navigate(ts.URL),
		chromedp.Title(&title),
	); err != nil {
		panic(err)
	}
	fmt.Println(title)

	// Output:
	// fancy website title
}

func ExampleExecAllocator() {
	dir, err := ioutil.TempDir("", "chromedp-example")
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(dir)

	opts := []chromedp.ExecAllocatorOption{
		chromedp.NoFirstRun,
		chromedp.NoDefaultBrowserCheck,
		chromedp.Headless,
		chromedp.DisableGPU,
		chromedp.UserDataDir(dir),
	}

	allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
	defer cancel()

	// also set up a custom logger
	taskCtx, cancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(log.Printf))
	defer cancel()

	// ensure that the browser process is started
	if err := chromedp.Run(taskCtx); err != nil {
		panic(err)
	}

	path := filepath.Join(dir, "DevToolsActivePort")
	bs, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	lines := bytes.Split(bs, []byte("\n"))
	fmt.Printf("DevToolsActivePort has %d lines\n", len(lines))

	// Output:
	// DevToolsActivePort has 2 lines
}

func ExampleNewContext_manyTabs() {
	// new browser, first tab
	ctx1, cancel := chromedp.NewContext(context.Background())
	defer cancel()

	// ensure the first tab is created
	if err := chromedp.Run(ctx1); err != nil {
		panic(err)
	}

	// same browser, second tab
	ctx2, _ := chromedp.NewContext(ctx1)

	// ensure the second tab is created
	if err := chromedp.Run(ctx2); err != nil {
		panic(err)
	}

	c1 := chromedp.FromContext(ctx1)
	c2 := chromedp.FromContext(ctx2)

	fmt.Printf("Same browser: %t\n", c1.Browser == c2.Browser)
	fmt.Printf("Same tab: %t\n", c1.Target == c2.Target)

	// Output:
	// Same browser: true
	// Same tab: false
}

func ExampleListenTarget_consoleLog() {
	ctx, cancel := chromedp.NewContext(context.Background())
	defer cancel()

	ts := httptest.NewServer(writeHTML(`

	`))
	defer ts.Close()

	chromedp.ListenTarget(ctx, func(ev interface{}) {
		switch ev := ev.(type) {
		case *cdpruntime.EventConsoleAPICalled:
			fmt.Printf("console.%s call:\n", ev.Type)
			for _, arg := range ev.Args {
				fmt.Printf("%s - %s\n", arg.Type, arg.Value)
			}
		}
	})

	if err := chromedp.Run(ctx,
		chromedp.Navigate(ts.URL),
		chromedp.WaitVisible("#jsFinished", chromedp.ByID),
	); err != nil {
		panic(err)
	}

	// Output:
	// console.log call:
	// string - "hello js world"
}

func ExampleClickNewTab() {
	ctx, cancel := chromedp.NewContext(context.Background())
	defer cancel()

	mux := http.NewServeMux()
	mux.Handle("/first", writeHTML(`
<input id='newtab' type='button' value='open' onclick='window.open("/second", "_blank");'/>
	`))
	mux.Handle("/second", writeHTML(``))
	ts := httptest.NewServer(mux)
	defer ts.Close()

	lctx, cancel := context.WithCancel(ctx)
	ch := make(chan target.ID, 1)
	chromedp.ListenTarget(lctx, func(ev interface{}) {
		if ev, ok := ev.(*target.EventTargetCreated); ok &&
			// if OpenerID == "", this is the first tab.
			ev.TargetInfo.OpenerID != "" {
			ch <- ev.TargetInfo.TargetID
			cancel()
		}
	})
	if err := chromedp.Run(ctx,
		chromedp.Navigate(ts.URL+"/first"),
		chromedp.Click("#newtab", chromedp.ByID),
	); err != nil {
		panic(err)
	}
	newCtx, cancel := chromedp.NewContext(ctx, chromedp.WithTargetID(<-ch))
	defer cancel()

	var urlstr string
	if err := chromedp.Run(newCtx, chromedp.Location(&urlstr)); err != nil {
		panic(err)
	}
	fmt.Println("new tab's path:", strings.TrimPrefix(urlstr, ts.URL))

	// Output:
	// new tab's path: /second
}

func ExampleAcceptAlert() {
	ctx, cancel := chromedp.NewContext(context.Background())
	defer cancel()

	mux := http.NewServeMux()
	mux.Handle("/second", writeHTML(``))
	ts := httptest.NewServer(writeHTML(`
<input id='alert' type='button' value='alert' onclick='alert("alert text");'/>
	`))
	defer ts.Close()

	chromedp.ListenTarget(ctx, func(ev interface{}) {
		if ev, ok := ev.(*page.EventJavascriptDialogOpening); ok {
			fmt.Println("closing alert:", ev.Message)
			go func() {
				if err := chromedp.Run(ctx,
					page.HandleJavaScriptDialog(true),
				); err != nil {
					panic(err)
				}
			}()
		}
	})

	if err := chromedp.Run(ctx,
		chromedp.Navigate(ts.URL),
		chromedp.Click("#alert", chromedp.ByID),
	); err != nil {
		panic(err)
	}

	// Output:
	// closing alert: alert text
}

func ExampleRetrieveHTML() {
	ctx, cancel := chromedp.NewContext(context.Background())
	defer cancel()

	ts := httptest.NewServer(writeHTML(`

Original content.

	`))
	defer ts.Close()

	respBody := make(chan string)
	chromedp.ListenTarget(ctx, func(ev interface{}) {
		ev2, ok := ev.(*network.EventResponseReceived)
		if !ok {
			return
		}
		go func() {
			if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
				body, err := network.GetResponseBody(ev2.RequestID).Do(ctx)
				respBody <- string(body)
				return err
			})); err != nil {
				panic(err)
			}
		}()
	})

	var outerBefore, outerAfter string
	if err := chromedp.Run(ctx,
		network.Enable(),
		chromedp.Navigate(ts.URL),
		chromedp.OuterHTML("#content", &outerBefore),
		chromedp.Click("#content", chromedp.ByID),
		chromedp.OuterHTML("#content", &outerAfter),
	); err != nil {
		panic(err)
	}
	fmt.Println("Original response body:")
	fmt.Println(<-respBody)
	fmt.Println()
	fmt.Println("OuterHTML before clicking:")
	fmt.Println(outerBefore)
	fmt.Println("OuterHTML after clicking:")
	fmt.Println(outerAfter)

	// Output:
	// Original response body:
	//
	//
Original content.

	//
	//
	//
	// OuterHTML before clicking:
	//
Original content.

	// OuterHTML after clicking:
	//
New content!

}
<|endoftext|>


<commit_before>package oauth

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/hellofresh/janus/pkg/session"
	"github.com/hellofresh/janus/pkg/stats"
)

// AwareTransport is a RoundTripper that is aware of the access tokens that come back from the
// authentication server. After retrieving the token, we delagete the storage of it for the
// oauth manager
type AwareTransport struct {
	manager          Manager
	oAuthServersRepo *MongoRepository
	statsClient      *stats.StatsClient
}

// NewAwareTransport creates a new instance of AwareTransport
func NewAwareTransport(manager Manager, oAuthServersRepo *MongoRepository, statsClient *stats.StatsClient) *AwareTransport {
	return &AwareTransport{manager, oAuthServersRepo, statsClient}
}

// GetRoundTripper returns initialized RoundTripper insnace
func (at *AwareTransport) GetRoundTripper(roundTripper http.RoundTripper) http.RoundTripper {
	return &RoundTripper{roundTripper, at.manager, at.oAuthServersRepo, at.statsClient}
}

type RoundTripper struct {
	RoundTripper     http.RoundTripper
	manager          Manager
	oAuthServersRepo *MongoRepository
	statsClient      *stats.StatsClient
}

// RoundTrip executes a single HTTP transaction, returning
// a Response for the provided Request.
//
// RoundTrip should not attempt to interpret the response. In
// particular, RoundTrip must return err == nil if it obtained
// a response, regardless of the response's HTTP status code.
// A non-nil err should be reserved for failure to obtain a
// response. Similarly, RoundTrip should not attempt to
// handle higher-level protocol details such as redirects,
// authentication, or cookies.
//
// RoundTrip should not modify the request, except for
// consuming and closing the Request's Body.
//
// RoundTrip must always close the body, including on errors,
// but depending on the implementation may do so in a separate
// goroutine even after RoundTrip returns. This means that
// callers wanting to reuse the body for subsequent requests
// must arrange to wait for the Close call before doing so.
//
// The Request's URL and Header fields must be initialized.
func (t *RoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	timing := t.statsClient.BuildTimeTracker()
	timing.Start()
	resp, err = t.RoundTripper.RoundTrip(req)

	if nil != err {
		t.statsClient.TrackRoundTrip(req, timing, false)
		log.Error("Response from the server was an error", err)
		return resp, err
	}

	statusCodeSuccess := resp.StatusCode < http.StatusInternalServerError
	t.statsClient.TrackRoundTrip(req, timing, statusCodeSuccess)

	if resp.StatusCode < http.StatusMultipleChoices && resp.Body != nil {
		var newSession session.SessionState

		//This is useful for the middlewares
		var bodyBytes []byte

		defer func(body io.Closer) {
			err := body.Close()
			if err != nil {
				log.Error(err)
			}
		}(resp.Body)
		bodyBytes, _ = ioutil.ReadAll(resp.Body)

		if marshalErr := json.Unmarshal(bodyBytes, &newSession); marshalErr == nil {
			if newSession.AccessToken != "" {
				log.Debug("Setting body in the oauth storage")
				t.manager.Set(newSession.AccessToken, newSession, newSession.ExpiresIn)
			}
		}

		// Restore the io.ReadCloser to its original state
		resp.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
	}

	return resp, err
}
<commit_msg>RoundTrip now needs the manager<commit_after>package oauth

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"

	log "github.com/Sirupsen/logrus"
	"github.com/hellofresh/janus/pkg/session"
	"github.com/hellofresh/janus/pkg/stats"
	"github.com/hellofresh/janus/pkg/store"
)

// AwareTransport is a RoundTripper that is aware of the access tokens that come back from the
// authentication server. After retrieving the token, we delagete the storage of it for the
// oauth manager
type AwareTransport struct {
	statsClient *stats.StatsClient
	storage     store.Store
	repo        Repository
}

// NewAwareTransport creates a new instance of AwareTransport
func NewAwareTransport(statsClient *stats.StatsClient, storage store.Store, repo Repository) *AwareTransport {
	return &AwareTransport{statsClient, storage, repo}
}

// GetRoundTripper returns initialized RoundTripper insnace
func (at *AwareTransport) GetRoundTripper(roundTripper http.RoundTripper) http.RoundTripper {
	return &RoundTripper{roundTripper, at.statsClient, at.storage, at.repo}
}

type RoundTripper struct {
	RoundTripper http.RoundTripper
	statsClient  *stats.StatsClient
	storage      store.Store
	repo         Repository
}

// RoundTrip executes a single HTTP transaction, returning
// a Response for the provided Request.
//
// RoundTrip should not attempt to interpret the response. In
// particular, RoundTrip must return err == nil if it obtained
// a response, regardless of the response's HTTP status code.
// A non-nil err should be reserved for failure to obtain a
// response. Similarly, RoundTrip should not attempt to
// handle higher-level protocol details such as redirects,
// authentication, or cookies.
//
// RoundTrip should not modify the request, except for
// consuming and closing the Request's Body.
//
// RoundTrip must always close the body, including on errors,
// but depending on the implementation may do so in a separate
// goroutine even after RoundTrip returns. This means that
// callers wanting to reuse the body for subsequent requests
// must arrange to wait for the Close call before doing so.
//
// The Request's URL and Header fields must be initialized.
func (t *RoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	timing := t.statsClient.BuildTimeTracker()
	timing.Start()
	resp, err = t.RoundTripper.RoundTrip(req)

	if nil != err {
		t.statsClient.TrackRoundTrip(req, timing, false)
		log.Error("Response from the server was an error", err)
		return resp, err
	}

	statusCodeSuccess := resp.StatusCode < http.StatusInternalServerError
	t.statsClient.TrackRoundTrip(req, timing, statusCodeSuccess)

	if resp.StatusCode < http.StatusMultipleChoices && resp.Body != nil {
		var newSession session.SessionState

		//This is useful for the middlewares
		var bodyBytes []byte

		defer func(body io.Closer) {
			err := body.Close()
			if err != nil {
				log.Error(err)
			}
		}(resp.Body)
		bodyBytes, _ = ioutil.ReadAll(resp.Body)

		if marshalErr := json.Unmarshal(bodyBytes, &newSession); marshalErr == nil {
			if newSession.AccessToken != "" {
				tokenURL := url.URL{Scheme: req.URL.Scheme, Host: req.URL.Host, Path: req.URL.Path}
				log.WithField("token_url", tokenURL.String()).Debug("Looking for OAuth provider who issued the token")
				manager, oAuthServer, err := t.getManager(tokenURL)
				if err != nil {
					log.WithError(err).Error("Failed to find OAuth server by token URL", err)
				} else {
					newSession.OAuthServerID = oAuthServer.ID
					log.Debug("Setting body in the oauth storage")
					manager.Set(newSession.AccessToken, newSession, newSession.ExpiresIn)
				}
			}
		}

		// Restore the io.ReadCloser to its original state
		resp.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
	}

	return resp, err
}

func (t *RoundTripper) getManager(url url.URL) (Manager, *OAuth, error) {
	oauthServer, err := t.repo.FindByTokenURL(url)
	if nil != err {
		return nil, nil, err
	}

	managerType, err := ParseType(oauthServer.TokenStrategy)
	if nil != err {
		return nil, nil, err
	}

	manager, err := NewManagerFactory(t.storage, oauthServer.JWTMeta.Secret).Build(managerType)

	return manager, oauthServer, err
}
<|endoftext|>


<commit_before>// Copyright 2015 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.

package ipaddr_test

import (
	"fmt"
	"log"
	"net"

	"github.com/mikioh/ipaddr"
)

func ExampleCursor_traversal() {
	c, err := ipaddr.Parse("2001:db8::/126,192.168.1.128/30,192.168.0.0/29")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(c.Pos(), c.First(), c.Last(), c.List())
	for pos := c.Next(); pos != nil; pos = c.Next() {
		fmt.Println(pos)
	}
	fmt.Println(c.Pos(), c.First(), c.Last(), c.List())
	for pos := c.Prev(); pos != nil; pos = c.Prev() {
		fmt.Println(pos)
	}
	fmt.Println(c.Pos(), c.First(), c.Last(), c.List())
	// Output:
	// &{192.168.0.0 192.168.0.0/29} &{192.168.0.0 192.168.0.0/29} &{2001:db8::3 2001:db8::/126} [192.168.0.0/29 192.168.1.128/30 2001:db8::/126]
	// &{192.168.0.1 192.168.0.0/29}
	// &{192.168.0.2 192.168.0.0/29}
	// &{192.168.0.3 192.168.0.0/29}
	// &{192.168.0.4 192.168.0.0/29}
	// &{192.168.0.5 192.168.0.0/29}
	// &{192.168.0.6 192.168.0.0/29}
	// &{192.168.0.7 192.168.0.0/29}
	// &{192.168.1.128 192.168.1.128/30}
	// &{192.168.1.129 192.168.1.128/30}
	// &{192.168.1.130 192.168.1.128/30}
	// &{192.168.1.131 192.168.1.128/30}
	// &{2001:db8:: 2001:db8::/126}
	// &{2001:db8::1 2001:db8::/126}
	// &{2001:db8::2 2001:db8::/126}
	// &{2001:db8::3 2001:db8::/126}
	// &{2001:db8::3 2001:db8::/126} &{192.168.0.0 192.168.0.0/29} &{2001:db8::3 2001:db8::/126} [192.168.0.0/29 192.168.1.128/30 2001:db8::/126]
	// &{2001:db8::2 2001:db8::/126}
	// &{2001:db8::1 2001:db8::/126}
	// &{2001:db8:: 2001:db8::/126}
	// &{192.168.1.131 192.168.1.128/30}
	// &{192.168.1.130 192.168.1.128/30}
	// &{192.168.1.129 192.168.1.128/30}
	// &{192.168.1.128 192.168.1.128/30}
	// &{192.168.0.7 192.168.0.0/29}
	// &{192.168.0.6 192.168.0.0/29}
	// &{192.168.0.5 192.168.0.0/29}
	// &{192.168.0.4 192.168.0.0/29}
	// &{192.168.0.3 192.168.0.0/29}
	// &{192.168.0.2 192.168.0.0/29}
	// &{192.168.0.1 192.168.0.0/29}
	// &{192.168.0.0 192.168.0.0/29}
	// &{192.168.0.0 192.168.0.0/29} &{192.168.0.0 192.168.0.0/29} &{2001:db8::3 2001:db8::/126} [192.168.0.0/29 192.168.1.128/30 2001:db8::/126]
}

func ExamplePrefix_subnettingAndSupernetting() {
	_, n, err := net.ParseCIDR("172.16.0.0/16")
	if err != nil {
		log.Fatal(err)
	}
	p := ipaddr.NewPrefix(n)
	fmt.Println(p.IP, p.Last(), p.Len(), p.Mask, p.Hostmask())
	fmt.Println()
	ps := p.Subnets(3)
	for _, p := range ps {
		fmt.Println(p)
	}
	fmt.Println()
	fmt.Println(ipaddr.Supernet(ps[4:6]))
	// Output:
	// 172.16.0.0 172.16.255.255 16 ffff0000 0000ffff
	//
	// 172.16.0.0/19
	// 172.16.32.0/19
	// 172.16.64.0/19
	// 172.16.96.0/19
	// 172.16.128.0/19
	// 172.16.160.0/19
	// 172.16.192.0/19
	// 172.16.224.0/19
	//
	// 172.16.128.0/18
}

func ExamplePrefix_subnettingAndAggregation() {
	_, n, err := net.ParseCIDR("192.168.0.0/24")
	if err != nil {
		log.Fatal(err)
	}
	p := ipaddr.NewPrefix(n)
	fmt.Println(p.IP, p.Last(), p.Len(), p.Mask, p.Hostmask())
	fmt.Println()
	ps := p.Subnets(2)
	for _, p := range ps {
		fmt.Println(p)
	}
	fmt.Println()
	_, n, err = net.ParseCIDR("192.168.100.0/24")
	if err != nil {
		log.Fatal(err)
	}
	ps = append(ps, *ipaddr.NewPrefix(n))
	fmt.Println(ipaddr.Aggregate(ps[2:]))
	// Output:
	// 192.168.0.0 192.168.0.255 24 ffffff00 000000ff
	//
	// 192.168.0.0/26
	// 192.168.0.64/26
	// 192.168.0.128/26
	// 192.168.0.192/26
	//
	// [192.168.0.128/25 192.168.100.0/24]
}

func ExamplePrefix_addressRangeSummarization() {
	ps := ipaddr.Summarize(net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::8000"))
	for _, p := range ps {
		fmt.Println(p)
	}
	// Output:
	// 2001:db8::1/128
	// 2001:db8::2/127
	// 2001:db8::4/126
	// 2001:db8::8/125
	// 2001:db8::10/124
	// 2001:db8::20/123
	// 2001:db8::40/122
	// 2001:db8::80/121
	// 2001:db8::100/120
	// 2001:db8::200/119
	// 2001:db8::400/118
	// 2001:db8::800/117
	// 2001:db8::1000/116
	// 2001:db8::2000/115
	// 2001:db8::4000/114
	// 2001:db8::8000/128
}
<commit_msg>ipaddr: update examples<commit_after>// Copyright 2015 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.

package ipaddr_test

import (
	"fmt"
	"log"
	"net"

	"github.com/mikioh/ipaddr"
)

func ExampleCursor_traversal() {
	c, err := ipaddr.Parse("2001:db8::/126,192.168.1.128/30,192.168.0.0/29")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(c.Pos(), c.First(), c.Last(), c.List())
	for pos := c.Next(); pos != nil; pos = c.Next() {
		fmt.Println(pos)
	}
	fmt.Println(c.Pos(), c.First(), c.Last(), c.List())
	for pos := c.Prev(); pos != nil; pos = c.Prev() {
		fmt.Println(pos)
	}
	fmt.Println(c.Pos(), c.First(), c.Last(), c.List())
	// Output:
	// &{192.168.0.0 192.168.0.0/29} &{192.168.0.0 192.168.0.0/29} &{2001:db8::3 2001:db8::/126} [192.168.0.0/29 192.168.1.128/30 2001:db8::/126]
	// &{192.168.0.1 192.168.0.0/29}
	// &{192.168.0.2 192.168.0.0/29}
	// &{192.168.0.3 192.168.0.0/29}
	// &{192.168.0.4 192.168.0.0/29}
	// &{192.168.0.5 192.168.0.0/29}
	// &{192.168.0.6 192.168.0.0/29}
	// &{192.168.0.7 192.168.0.0/29}
	// &{192.168.1.128 192.168.1.128/30}
	// &{192.168.1.129 192.168.1.128/30}
	// &{192.168.1.130 192.168.1.128/30}
	// &{192.168.1.131 192.168.1.128/30}
	// &{2001:db8:: 2001:db8::/126}
	// &{2001:db8::1 2001:db8::/126}
	// &{2001:db8::2 2001:db8::/126}
	// &{2001:db8::3 2001:db8::/126}
	// &{2001:db8::3 2001:db8::/126} &{192.168.0.0 192.168.0.0/29} &{2001:db8::3 2001:db8::/126} [192.168.0.0/29 192.168.1.128/30 2001:db8::/126]
	// &{2001:db8::2 2001:db8::/126}
	// &{2001:db8::1 2001:db8::/126}
	// &{2001:db8:: 2001:db8::/126}
	// &{192.168.1.131 192.168.1.128/30}
	// &{192.168.1.130 192.168.1.128/30}
	// &{192.168.1.129 192.168.1.128/30}
	// &{192.168.1.128 192.168.1.128/30}
	// &{192.168.0.7 192.168.0.0/29}
	// &{192.168.0.6 192.168.0.0/29}
	// &{192.168.0.5 192.168.0.0/29}
	// &{192.168.0.4 192.168.0.0/29}
	// &{192.168.0.3 192.168.0.0/29}
	// &{192.168.0.2 192.168.0.0/29}
	// &{192.168.0.1 192.168.0.0/29}
	// &{192.168.0.0 192.168.0.0/29}
	// &{192.168.0.0 192.168.0.0/29} &{192.168.0.0 192.168.0.0/29} &{2001:db8::3 2001:db8::/126} [192.168.0.0/29 192.168.1.128/30 2001:db8::/126]
}

func ExamplePrefix_subnettingAndSupernetting() {
	_, n, err := net.ParseCIDR("172.16.0.0/16")
	if err != nil {
		log.Fatal(err)
	}
	p := ipaddr.NewPrefix(n)
	fmt.Println(p.IP, p.Last(), p.Len(), p.Mask, p.Hostmask())
	fmt.Println()
	ps := p.Subnets(3)
	for _, p := range ps {
		fmt.Println(p)
	}
	fmt.Println()
	fmt.Println(ipaddr.Supernet(ps))
	fmt.Println(ipaddr.Supernet(ps[:2]))
	fmt.Println(ipaddr.Supernet(ps[2:4]))
	fmt.Println(ipaddr.Supernet(ps[4:6]))
	fmt.Println(ipaddr.Supernet(ps[6:8]))
	// Output:
	// 172.16.0.0 172.16.255.255 16 ffff0000 0000ffff
	//
	// 172.16.0.0/19
	// 172.16.32.0/19
	// 172.16.64.0/19
	// 172.16.96.0/19
	// 172.16.128.0/19
	// 172.16.160.0/19
	// 172.16.192.0/19
	// 172.16.224.0/19
	//
	// 172.16.0.0/16
	// 172.16.0.0/18
	// 172.16.64.0/18
	// 172.16.128.0/18
	// 172.16.192.0/18
}

func ExamplePrefix_subnettingAndAggregation() {
	_, n, err := net.ParseCIDR("192.168.0.0/24")
	if err != nil {
		log.Fatal(err)
	}
	p := ipaddr.NewPrefix(n)
	fmt.Println(p.IP, p.Last(), p.Len(), p.Mask, p.Hostmask())
	fmt.Println()
	ps := p.Subnets(2)
	for _, p := range ps {
		fmt.Println(p)
	}
	fmt.Println()
	_, n, err = net.ParseCIDR("192.168.100.0/24")
	if err != nil {
		log.Fatal(err)
	}
	ps = append(ps, *ipaddr.NewPrefix(n))
	fmt.Println(ipaddr.Aggregate(ps))
	fmt.Println(ipaddr.Aggregate(ps[:2]))
	fmt.Println(ipaddr.Aggregate(ps[2:4]))
	// Output:
	// 192.168.0.0 192.168.0.255 24 ffffff00 000000ff
	//
	// 192.168.0.0/26
	// 192.168.0.64/26
	// 192.168.0.128/26
	// 192.168.0.192/26
	//
	// [192.168.0.0/24 192.168.100.0/24]
	// [192.168.0.0/25]
	// [192.168.0.128/25]
}

func ExamplePrefix_addressRangeSummarization() {
	ps := ipaddr.Summarize(net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::8000"))
	for _, p := range ps {
		fmt.Println(p)
	}
	// Output:
	// 2001:db8::1/128
	// 2001:db8::2/127
	// 2001:db8::4/126
	// 2001:db8::8/125
	// 2001:db8::10/124
	// 2001:db8::20/123
	// 2001:db8::40/122
	// 2001:db8::80/121
	// 2001:db8::100/120
	// 2001:db8::200/119
	// 2001:db8::400/118
	// 2001:db8::800/117
	// 2001:db8::1000/116
	// 2001:db8::2000/115
	// 2001:db8::4000/114
	// 2001:db8::8000/128
}
<|endoftext|>


<commit_before>package plugins

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"github.com/grafana/grafana/pkg/infra/log"
	"github.com/grafana/grafana/pkg/registry"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/util"
	"github.com/grafana/grafana/pkg/util/errutil"
	"golang.org/x/xerrors"
)

var (
	DataSources  map[string]*DataSourcePlugin
	Panels       map[string]*PanelPlugin
	StaticRoutes []*PluginStaticRoute
	Apps         map[string]*AppPlugin
	Plugins      map[string]*PluginBase
	PluginTypes  map[string]interface{}
	Renderer     *RendererPlugin

	GrafanaLatestVersion string
	GrafanaHasUpdate     bool
	plog                 log.Logger
)

type PluginScanner struct {
	pluginPath string
	errors     []error
}

type PluginManager struct {
	log log.Logger
}

func init() {
	registry.RegisterService(&PluginManager{})
}

func (pm *PluginManager) Init() error {
	pm.log = log.New("plugins")
	plog = log.New("plugins")

	DataSources = map[string]*DataSourcePlugin{}
	StaticRoutes = []*PluginStaticRoute{}
	Panels = map[string]*PanelPlugin{}
	Apps = map[string]*AppPlugin{}
	Plugins = map[string]*PluginBase{}
	PluginTypes = map[string]interface{}{
		"panel":      PanelPlugin{},
		"datasource": DataSourcePlugin{},
		"app":        AppPlugin{},
		"renderer":   RendererPlugin{},
	}

	pm.log.Info("Starting plugin search")
	plugDir := path.Join(setting.StaticRootPath, "app/plugins")
	if err := pm.scan(plugDir); err != nil {
		return errutil.Wrapf(err, "Failed to scan main plugin directory '%s'", plugDir)
	}

	// check if plugins dir exists
	if _, err := os.Stat(setting.PluginsPath); os.IsNotExist(err) {
		if err = os.MkdirAll(setting.PluginsPath, os.ModePerm); err != nil {
			plog.Error("Failed to create plugin dir", "dir", setting.PluginsPath, "error", err)
		} else {
			plog.Info("Plugin dir created", "dir", setting.PluginsPath)
			if err := pm.scan(setting.PluginsPath); err != nil {
				return errutil.Wrapf(err, "Failed to scan configured plugin directory '%s'",
					setting.PluginsPath)
			}
		}
	} else {
		if err := pm.scan(setting.PluginsPath); err != nil {
			return errutil.Wrapf(err, "Failed to scan configured plugin directory '%s'",
				setting.PluginsPath)
		}
	}

	// check plugin paths defined in config
	if err := pm.checkPluginPaths(); err != nil {
		return err
	}

	for _, panel := range Panels {
		panel.initFrontendPlugin()
	}

	for _, ds := range DataSources {
		ds.initFrontendPlugin()
	}

	for _, app := range Apps {
		app.initApp()
	}

	return nil
}

func (pm *PluginManager) startBackendPlugins(ctx context.Context) {
	for _, ds := range DataSources {
		if !ds.Backend {
			continue
		}

		if err := ds.startBackendPlugin(ctx, plog); err != nil {
			pm.log.Error("Failed to init plugin.", "error", err, "plugin", ds.Id)
		}
	}
}

func (pm *PluginManager) Run(ctx context.Context) error {
	pm.startBackendPlugins(ctx)
	pm.updateAppDashboards()
	pm.checkForUpdates()

	ticker := time.NewTicker(time.Minute * 10)
	run := true

	for run {
		select {
		case <-ticker.C:
			pm.checkForUpdates()
		case <-ctx.Done():
			run = false
		}
	}

	// kill backend plugins
	for _, p := range DataSources {
		p.Kill()
	}

	return ctx.Err()
}

func (pm *PluginManager) checkPluginPaths() error {
	for _, section := range setting.Raw.Sections() {
		if !strings.HasPrefix(section.Name(), "plugin.") {
			continue
		}

		path := section.Key("path").String()
		if path == "" {
			continue
		}

		if err := pm.scan(path); err != nil {
			return errutil.Wrapf(err, "Failed to scan directory configured for plugin '%s': '%s'",
				section.Name(), path)
		}
	}

	return nil
}

// scan a directory for plugins.
func (pm *PluginManager) scan(pluginDir string) error {
	scanner := &PluginScanner{
		pluginPath: pluginDir,
	}

	if err := util.Walk(pluginDir, true, true, scanner.walker); err != nil {
		if xerrors.Is(err, os.ErrNotExist) {
			pm.log.Debug("Couldn't scan dir '%s' since it doesn't exist")
			return nil
		}
		if xerrors.Is(err, os.ErrPermission) {
			pm.log.Debug("Couldn't scan dir '%s' due to lack of permissions")
			return nil
		}
		if pluginDir != "data/plugins" {
			pm.log.Warn("Could not scan dir", "pluginDir", pluginDir, "err", err)
		}
		return err
	}

	if len(scanner.errors) > 0 {
		return errutil.Wrapf(scanner.errors[0], "Some plugins failed to load")
	}

	return nil
}

func (scanner *PluginScanner) walker(currentPath string, f os.FileInfo, err error) error {
	// We scan all the subfolders for plugin.json (with some exceptions) so that we also load embedded plugins, for
	// example https://github.com/raintank/worldping-app/tree/master/dist/grafana-worldmap-panel worldmap panel plugin
	// is embedded in worldping app.
	if err != nil {
		return err
	}

	if f.Name() == "node_modules" || f.Name() == "Chromium.app" {
		return util.ErrWalkSkipDir
	}

	if f.IsDir() {
		return nil
	}

	if f.Name() == "plugin.json" {
		err := scanner.loadPluginJson(currentPath)
		if err != nil {
			log.Error(3, "Plugins: Failed to load plugin json file: %v,  err: %v", currentPath, err)
			scanner.errors = append(scanner.errors, err)
		}
	}
	return nil
}

func (scanner *PluginScanner) loadPluginJson(pluginJsonFilePath string) error {
	currentDir := filepath.Dir(pluginJsonFilePath)
	reader, err := os.Open(pluginJsonFilePath)
	if err != nil {
		return err
	}

	defer reader.Close()

	jsonParser := json.NewDecoder(reader)
	pluginCommon := PluginBase{}
	if err := jsonParser.Decode(&pluginCommon); err != nil {
		return err
	}

	if pluginCommon.Id == "" || pluginCommon.Type == "" {
		return errors.New("Did not find type and id property in plugin.json")
	}

	var loader PluginLoader
	pluginGoType, exists := PluginTypes[pluginCommon.Type]
	if !exists {
		return errors.New("Unknown plugin type " + pluginCommon.Type)
	}
	loader = reflect.New(reflect.TypeOf(pluginGoType)).Interface().(PluginLoader)

	// External plugins need a module.js file for SystemJS to load
	if !strings.HasPrefix(pluginJsonFilePath, setting.StaticRootPath) && !scanner.IsBackendOnlyPlugin(pluginCommon.Type) {
		module := filepath.Join(filepath.Dir(pluginJsonFilePath), "module.js")
		if _, err := os.Stat(module); os.IsNotExist(err) {
			plog.Warn("Plugin missing module.js",
				"name", pluginCommon.Name,
				"warning", "Missing module.js, If you loaded this plugin from git, make sure to compile it.",
				"path", module)
		}
	}

	if _, err := reader.Seek(0, 0); err != nil {
		return err
	}
	return loader.Load(jsonParser, currentDir)
}

func (scanner *PluginScanner) IsBackendOnlyPlugin(pluginType string) bool {
	return pluginType == "renderer"
}

func GetPluginMarkdown(pluginId string, name string) ([]byte, error) {
	plug, exists := Plugins[pluginId]
	if !exists {
		return nil, PluginNotFoundError{pluginId}
	}

	path := filepath.Join(plug.PluginDir, fmt.Sprintf("%s.md", strings.ToUpper(name)))
	if _, err := os.Stat(path); os.IsNotExist(err) {
		path = filepath.Join(plug.PluginDir, fmt.Sprintf("%s.md", strings.ToLower(name)))
	}

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return make([]byte, 0), nil
	}

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	return data, nil
}
<commit_msg>pkg/plugins: Only warn if plugins fail to load. Fixes #19846 (#19859)<commit_after>package plugins

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"github.com/grafana/grafana/pkg/infra/log"
	"github.com/grafana/grafana/pkg/registry"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/util"
	"github.com/grafana/grafana/pkg/util/errutil"
	"golang.org/x/xerrors"
)

var (
	DataSources  map[string]*DataSourcePlugin
	Panels       map[string]*PanelPlugin
	StaticRoutes []*PluginStaticRoute
	Apps         map[string]*AppPlugin
	Plugins      map[string]*PluginBase
	PluginTypes  map[string]interface{}
	Renderer     *RendererPlugin

	GrafanaLatestVersion string
	GrafanaHasUpdate     bool
	plog                 log.Logger
)

type PluginScanner struct {
	pluginPath string
	errors     []error
}

type PluginManager struct {
	log log.Logger
}

func init() {
	registry.RegisterService(&PluginManager{})
}

func (pm *PluginManager) Init() error {
	pm.log = log.New("plugins")
	plog = log.New("plugins")

	DataSources = map[string]*DataSourcePlugin{}
	StaticRoutes = []*PluginStaticRoute{}
	Panels = map[string]*PanelPlugin{}
	Apps = map[string]*AppPlugin{}
	Plugins = map[string]*PluginBase{}
	PluginTypes = map[string]interface{}{
		"panel":      PanelPlugin{},
		"datasource": DataSourcePlugin{},
		"app":        AppPlugin{},
		"renderer":   RendererPlugin{},
	}

	pm.log.Info("Starting plugin search")
	plugDir := path.Join(setting.StaticRootPath, "app/plugins")
	if err := pm.scan(plugDir); err != nil {
		return errutil.Wrapf(err, "Failed to scan main plugin directory '%s'", plugDir)
	}

	// check if plugins dir exists
	if _, err := os.Stat(setting.PluginsPath); os.IsNotExist(err) {
		if err = os.MkdirAll(setting.PluginsPath, os.ModePerm); err != nil {
			plog.Error("Failed to create plugin dir", "dir", setting.PluginsPath, "error", err)
		} else {
			plog.Info("Plugin dir created", "dir", setting.PluginsPath)
			if err := pm.scan(setting.PluginsPath); err != nil {
				return errutil.Wrapf(err, "Failed to scan configured plugin directory '%s'",
					setting.PluginsPath)
			}
		}
	} else {
		if err := pm.scan(setting.PluginsPath); err != nil {
			return errutil.Wrapf(err, "Failed to scan configured plugin directory '%s'",
				setting.PluginsPath)
		}
	}

	// check plugin paths defined in config
	if err := pm.checkPluginPaths(); err != nil {
		return err
	}

	for _, panel := range Panels {
		panel.initFrontendPlugin()
	}

	for _, ds := range DataSources {
		ds.initFrontendPlugin()
	}

	for _, app := range Apps {
		app.initApp()
	}

	return nil
}

func (pm *PluginManager) startBackendPlugins(ctx context.Context) {
	for _, ds := range DataSources {
		if !ds.Backend {
			continue
		}

		if err := ds.startBackendPlugin(ctx, plog); err != nil {
			pm.log.Error("Failed to init plugin.", "error", err, "plugin", ds.Id)
		}
	}
}

func (pm *PluginManager) Run(ctx context.Context) error {
	pm.startBackendPlugins(ctx)
	pm.updateAppDashboards()
	pm.checkForUpdates()

	ticker := time.NewTicker(time.Minute * 10)
	run := true

	for run {
		select {
		case <-ticker.C:
			pm.checkForUpdates()
		case <-ctx.Done():
			run = false
		}
	}

	// kill backend plugins
	for _, p := range DataSources {
		p.Kill()
	}

	return ctx.Err()
}

func (pm *PluginManager) checkPluginPaths() error {
	for _, section := range setting.Raw.Sections() {
		if !strings.HasPrefix(section.Name(), "plugin.") {
			continue
		}

		path := section.Key("path").String()
		if path == "" {
			continue
		}

		if err := pm.scan(path); err != nil {
			return errutil.Wrapf(err, "Failed to scan directory configured for plugin '%s': '%s'",
				section.Name(), path)
		}
	}

	return nil
}

// scan a directory for plugins.
func (pm *PluginManager) scan(pluginDir string) error {
	scanner := &PluginScanner{
		pluginPath: pluginDir,
	}

	if err := util.Walk(pluginDir, true, true, scanner.walker); err != nil {
		if xerrors.Is(err, os.ErrNotExist) {
			pm.log.Debug("Couldn't scan dir '%s' since it doesn't exist")
			return nil
		}
		if xerrors.Is(err, os.ErrPermission) {
			pm.log.Debug("Couldn't scan dir '%s' due to lack of permissions")
			return nil
		}
		if pluginDir != "data/plugins" {
			pm.log.Warn("Could not scan dir", "pluginDir", pluginDir, "err", err)
		}
		return err
	}

	if len(scanner.errors) > 0 {
		pm.log.Warn("Some plugins failed to load", "errors", scanner.errors)
	}

	return nil
}

func (scanner *PluginScanner) walker(currentPath string, f os.FileInfo, err error) error {
	// We scan all the subfolders for plugin.json (with some exceptions) so that we also load embedded plugins, for
	// example https://github.com/raintank/worldping-app/tree/master/dist/grafana-worldmap-panel worldmap panel plugin
	// is embedded in worldping app.
	if err != nil {
		return err
	}

	if f.Name() == "node_modules" || f.Name() == "Chromium.app" {
		return util.ErrWalkSkipDir
	}

	if f.IsDir() {
		return nil
	}

	if f.Name() == "plugin.json" {
		err := scanner.loadPluginJson(currentPath)
		if err != nil {
			log.Error(3, "Plugins: Failed to load plugin json file: %v,  err: %v", currentPath, err)
			scanner.errors = append(scanner.errors, err)
		}
	}
	return nil
}

func (scanner *PluginScanner) loadPluginJson(pluginJsonFilePath string) error {
	currentDir := filepath.Dir(pluginJsonFilePath)
	reader, err := os.Open(pluginJsonFilePath)
	if err != nil {
		return err
	}

	defer reader.Close()

	jsonParser := json.NewDecoder(reader)
	pluginCommon := PluginBase{}
	if err := jsonParser.Decode(&pluginCommon); err != nil {
		return err
	}

	if pluginCommon.Id == "" || pluginCommon.Type == "" {
		return errors.New("Did not find type and id property in plugin.json")
	}

	var loader PluginLoader
	pluginGoType, exists := PluginTypes[pluginCommon.Type]
	if !exists {
		return errors.New("Unknown plugin type " + pluginCommon.Type)
	}
	loader = reflect.New(reflect.TypeOf(pluginGoType)).Interface().(PluginLoader)

	// External plugins need a module.js file for SystemJS to load
	if !strings.HasPrefix(pluginJsonFilePath, setting.StaticRootPath) && !scanner.IsBackendOnlyPlugin(pluginCommon.Type) {
		module := filepath.Join(filepath.Dir(pluginJsonFilePath), "module.js")
		if _, err := os.Stat(module); os.IsNotExist(err) {
			plog.Warn("Plugin missing module.js",
				"name", pluginCommon.Name,
				"warning", "Missing module.js, If you loaded this plugin from git, make sure to compile it.",
				"path", module)
		}
	}

	if _, err := reader.Seek(0, 0); err != nil {
		return err
	}
	return loader.Load(jsonParser, currentDir)
}

func (scanner *PluginScanner) IsBackendOnlyPlugin(pluginType string) bool {
	return pluginType == "renderer"
}

func GetPluginMarkdown(pluginId string, name string) ([]byte, error) {
	plug, exists := Plugins[pluginId]
	if !exists {
		return nil, PluginNotFoundError{pluginId}
	}

	path := filepath.Join(plug.PluginDir, fmt.Sprintf("%s.md", strings.ToUpper(name)))
	if _, err := os.Stat(path); os.IsNotExist(err) {
		path = filepath.Join(plug.PluginDir, fmt.Sprintf("%s.md", strings.ToLower(name)))
	}

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return make([]byte, 0), nil
	}

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	return data, nil
}
<|endoftext|>


<commit_before>package project

var (
	description        = "The aws-operator manages Kubernetes clusters running on AWS."
	gitSHA             = "n/a"
	name        string = "aws-operator"
	source      string = "https://github.com/giantswarm/aws-operator"
	version            = "14.2.0"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<commit_msg>Bump version to 14.2.1-dev (#3449)<commit_after>package project

var (
	description        = "The aws-operator manages Kubernetes clusters running on AWS."
	gitSHA             = "n/a"
	name        string = "aws-operator"
	source      string = "https://github.com/giantswarm/aws-operator"
	version            = "14.2.1-dev"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<|endoftext|>


<commit_before>// +build !windows

package static

import (
	"fmt"
	"net/http"
	"path"
)

func ExampleBuild() {
	handler := http.NewServeMux()
	paths := []string{}

	handler.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Hello %s!", path.Base(r.URL.Path))
	})

	paths = append(paths, "/world")

	Build(DefaultOptions, handler, paths, func(e Event) {
		fmt.Println(e)
	})

	// Output:
	// Action: build, Path: /world, StatusCode: 200, OutputPath: build/world
}

func ExampleBuildSingle() {
	handler := http.NewServeMux()

	handler.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Hello %s!", path.Base(r.URL.Path))
	})

	statusCode, outputPath, err := BuildSingle(DefaultOptions, handler, "/world")
	fmt.Printf("Built: /world, StatusCode: %d, OutputPath: %v, Error: %v", statusCode, outputPath, err)

	// Output:
	// Built: /world, StatusCode: 200, OutputPath: build/world, Error: <nil>
}
<commit_msg>Move examples to the _test package.<commit_after>// +build !windows

package static_test

import (
	"fmt"
	"net/http"
	"path"

	"github.com/leighmcculloch/static"
)

func ExampleBuild() {
	handler := http.NewServeMux()
	paths := []string{}

	handler.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Hello %s!", path.Base(r.URL.Path))
	})

	paths = append(paths, "/world")

	static.Build(static.DefaultOptions, handler, paths, func(e static.Event) {
		fmt.Println(e)
	})

	// Output:
	// Action: build, Path: /world, StatusCode: 200, OutputPath: build/world
}

func ExampleBuildSingle() {
	handler := http.NewServeMux()

	handler.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Hello %s!", path.Base(r.URL.Path))
	})

	statusCode, outputPath, err := static.BuildSingle(static.DefaultOptions, handler, "/world")
	fmt.Printf("Built: /world, StatusCode: %d, OutputPath: %v, Error: %v", statusCode, outputPath, err)

	// Output:
	// Built: /world, StatusCode: 200, OutputPath: build/world, Error: <nil>
}
<|endoftext|>


<commit_before>package refmt_test

import (
	"bytes"
	"fmt"

	"github.com/polydawn/refmt"
	"github.com/polydawn/refmt/obj/atlas"
)

func ExampleJsonEncodeAtlasDefaults() {
	type MyType struct {
		X string
		Y int
	}

	MyType_AtlasEntry := atlas.BuildEntry(MyType{}).
		StructMap().Autogenerate().
		Complete()

	atl := atlas.MustBuild(
		MyType_AtlasEntry,
		// this is a vararg... stack more entries here!
	)

	var buf bytes.Buffer
	encoder := refmt.NewAtlasedJsonEncoder(&buf, atl)
	err := encoder.Marshal(MyType{"a", 1})
	fmt.Println(buf.String())
	fmt.Printf("%v\n", err)

	// Output:
	// {"x":"a","y":1}
	// <nil>
}

func ExampleJsonEncodeAtlasCustom() {
	type MyType struct {
		X string
		Y int
	}

	MyType_AtlasEntry := atlas.BuildEntry(MyType{}).
		StructMap().
		AddField("X", atlas.StructMapEntry{SerialName: "overrideName"}).
		// and no "Y" mapping at all!
		Complete()

	atl := atlas.MustBuild(
		MyType_AtlasEntry,
		// this is a vararg... stack more entries here!
	)

	var buf bytes.Buffer
	encoder := refmt.NewAtlasedJsonEncoder(&buf, atl)
	err := encoder.Marshal(MyType{"a", 1})
	fmt.Println(buf.String())
	fmt.Printf("%v\n", err)

	// Output:
	// {"overrideName":"a"}
	// <nil>
}
<commit_msg>Add example of building an atlas using kind transformers (one of our wilder features).<commit_after>package refmt_test

import (
	"bytes"
	"fmt"
	"strings"

	"github.com/polydawn/refmt"
	"github.com/polydawn/refmt/obj/atlas"
)

func ExampleJsonEncodeAtlasDefaults() {
	type MyType struct {
		X string
		Y int
	}

	MyType_AtlasEntry := atlas.BuildEntry(MyType{}).
		StructMap().Autogenerate().
		Complete()

	atl := atlas.MustBuild(
		MyType_AtlasEntry,
		// this is a vararg... stack more entries here!
	)

	var buf bytes.Buffer
	encoder := refmt.NewAtlasedJsonEncoder(&buf, atl)
	err := encoder.Marshal(MyType{"a", 1})
	fmt.Println(buf.String())
	fmt.Printf("%v\n", err)

	// Output:
	// {"x":"a","y":1}
	// <nil>
}

func ExampleJsonEncodeAtlasCustom() {
	type MyType struct {
		X string
		Y int
	}

	MyType_AtlasEntry := atlas.BuildEntry(MyType{}).
		StructMap().
		AddField("X", atlas.StructMapEntry{SerialName: "overrideName"}).
		// and no "Y" mapping at all!
		Complete()

	atl := atlas.MustBuild(
		MyType_AtlasEntry,
		// this is a vararg... stack more entries here!
	)

	var buf bytes.Buffer
	encoder := refmt.NewAtlasedJsonEncoder(&buf, atl)
	err := encoder.Marshal(MyType{"a", 1})
	fmt.Println(buf.String())
	fmt.Printf("%v\n", err)

	// Output:
	// {"overrideName":"a"}
	// <nil>
}

func ExampleJsonEncodeAtlas() {
	type MyType struct {
		X string
		Y string
		Z string
	}

	MyType_AtlasEntry := atlas.BuildEntry(MyType{}).
		Transform().
		TransformMarshal(atlas.MakeMarshalTransformFunc(
			func(x MyType) (string, error) {
				return string(x.X) + ":" + string(x.Y) + ":" + string(x.Z), nil
			})).
		TransformUnmarshal(atlas.MakeUnmarshalTransformFunc(
			func(x string) (MyType, error) {
				ss := strings.Split(x, ":")
				if len(ss) != 3 {
					return MyType{}, fmt.Errorf("parsing MyType: string must have 3 parts, separated by colon")
				}
				return MyType{ss[0], ss[1], ss[2]}, nil
			})).
		Complete()

	atl := atlas.MustBuild(
		MyType_AtlasEntry,
		// this is a vararg... stack more entries here!
	)

	var buf bytes.Buffer
	encoder := refmt.NewAtlasedJsonEncoder(&buf, atl)

	err := encoder.Marshal(MyType{"serializes", "as", "string!"})
	fmt.Println(buf.String())
	fmt.Printf("%v\n", err)

	// Output:
	// "serializes:as:string!"
	// <nil>
}
<|endoftext|>


<commit_before>package storage

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/opentracing/opentracing-go"
	"github.com/pkg/errors"
)

var (
	// ErrRepoNotValid is returned for invalid repositories
	ErrRepoNotValid = fmt.Errorf("not a valid repository")
)

type (
	// Storage TODO: is something that should be split up
	Storage interface {
		Create(ctx context.Context, id string) error
		GetRepository(ctx context.Context, id string) (Repository, error)
	}

	// LocalStorage implements Storage for Local disk-access
	LocalStorage struct {
		git  string
		root string
	}

	// Repository is the interface for manipulating repos
	Repository interface {
		GetID() string
		SetDescription(ctx context.Context, description string) error
		ListBranches(ctx context.Context) ([]Branch, error)
		GetCommit(ctx context.Context, ref string) (Commit, error)
		Tree(ctx context.Context, ref, path string) ([]TreeEntry, error)
		UploadPack(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) (int32, error)
		ReceivePack(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) (int32, error)
	}

	// LocalRepository implements Repository for Local disk-access
	LocalRepository struct {
		git  string
		path string
		id   string
	}
)

// NewLocalStorage returns a LocalStorage in the given `root`
func NewLocalStorage(root string) (*LocalStorage, error) {
	if err := os.MkdirAll(root, 0755); err != nil {
		return nil, fmt.Errorf("failed to create storage root: %s", root)
	}
	return &LocalStorage{
		git:  "/usr/bin/git",
		root: root,
	}, nil
}

func (s *LocalStorage) repoPath(id string) string {
	id = strings.Replace(id, "-", "", -1)
	return filepath.Join(s.root, id[0:2], id[2:4], id[4:])
}

// Create a new repository
func (s *LocalStorage) Create(ctx context.Context, id string) error {
	dir := s.repoPath(id)

	if err := os.MkdirAll(dir, 0755); err != nil {
		return fmt.Errorf("failed to repository directory: %s", dir)
	}

	cmd := exec.CommandContext(ctx, s.git, "init", "--bare")
	cmd.Dir = dir

	//TODO: Don't throw away stdout/err...
	return cmd.Run()
}

// GetRepository from Storage
// TODO: Cache these somehow?
func (s *LocalStorage) GetRepository(ctx context.Context, repoPath string) (Repository, error) {
	dir := s.repoPath(repoPath)

	cmd := exec.CommandContext(ctx, s.git, "config", "--null", "core.repositoryformatversion")
	cmd.Dir = dir

	output, err := cmd.CombinedOutput()
	if err != nil {
		return nil, ErrRepoNotValid
	}
	if strings.TrimSuffix(string(output), "\x00") != "0" {
		return nil, ErrRepoNotValid
	}

	return &LocalRepository{git: s.git, path: dir, id: repoPath}, nil
}

// GetID returns the repos ID
func (r *LocalRepository) GetID() string {
	return r.id
}

// Branch of a repository
type Branch struct {
	Name string
	Sha1 string
	Type string
}

// SetDescription of repository
func (r *LocalRepository) SetDescription(ctx context.Context, description string) error {
	file := filepath.Join(r.path, "description")
	return ioutil.WriteFile(file, []byte(description+"\n"), 0644)
}

// ListBranches returns all branches of a given repository
func (r *LocalRepository) ListBranches(ctx context.Context) ([]Branch, error) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	args := []string{"for-each-ref", "--format=%(objectname) %(objecttype) %(refname)", "refs/heads"}
	cmd := exec.CommandContext(ctx, r.git, args...)
	cmd.Dir = r.path
	out, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err = cmd.Start(); err != nil {
		return nil, err
	}

	var bs []Branch
	scanner := bufio.NewScanner(out)
	for scanner.Scan() {
		s := strings.Split(scanner.Text(), " ")

		bs = append(bs, Branch{
			Name: strings.TrimPrefix(s[2], "refs/heads/"),
			Sha1: s[0],
			Type: s[1],
		})
	}

	if err := cmd.Wait(); err != nil {
		return nil, err
	}

	return bs, nil
}

// Commit holds a commit
type Commit struct {
	Hash    string
	Tree    string
	Parent  string
	Message string
	Body    string

	Author Signature

	Committer Signature
}

// GetCommit returns a single Commit from a Repository
func (r *LocalRepository) GetCommit(ctx context.Context, ref string) (Commit, error) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	args := []string{"cat-file", "-p", ref}
	cmd := exec.CommandContext(ctx, r.git, args...)
	cmd.Dir = r.path
	out, err := cmd.StdoutPipe()
	if err != nil {
		return Commit{}, err
	}
	if err = cmd.Start(); err != nil {
		return Commit{}, err
	}

	commit, err := parseCommit(out, ref)
	if err != nil {
		return Commit{}, err
	}

	if err := cmd.Wait(); err != nil {
		return Commit{}, err
	}

	return commit, nil
}

func parseCommit(r io.Reader, hash string) (Commit, error) {
	scanner := bufio.NewScanner(r)

	c := Commit{
		Hash: hash,
	}

	// if true then the following lines are the subject
	hasHeader := false

	for scanner.Scan() {
		line := scanner.Text()

		if !hasHeader {
			var err error
			hasHeader, err = parseCommitHeader(&c, line)
			if err != nil {
				return c, err
			}
			continue
		}

		if c.Message == "" {
			c.Message = line
			continue
		}

		// TODO: use a string.Builder instead?
		// TODO: handle commit signatures
		c.Body = fmt.Sprintf("%s\n%s", c.Body, line)
	}

	// Trim excessive stringing new lines
	c.Body = strings.TrimLeft(c.Body, "\n")

	return c, nil
}

// returns true when it's passed the header
func parseCommitHeader(c *Commit, line string) (bool, error) {
	const (
		treePrefix      = "tree "
		parentPrefix    = "parent "
		authorPrefix    = "author "
		committerPrefix = "committer "
	)

	if line == "" {
		return true, nil
	}
	if strings.HasPrefix(line, treePrefix) {
		c.Tree = strings.TrimPrefix(line, treePrefix)
		return false, nil
	}

	if strings.HasPrefix(line, parentPrefix) {
		c.Parent = strings.TrimPrefix(line, parentPrefix)
		return false, nil
	}

	if strings.HasPrefix(line, authorPrefix) {
		var err error
		line := strings.TrimPrefix(line, authorPrefix)

		c.Author, err = parseSignature(line)
		if err != nil {
			// This should probably just error out, and not return a partial commit...
			return false, err
		}
		return false, nil
	}

	if strings.HasPrefix(line, committerPrefix) {
		var err error
		line := strings.TrimPrefix(line, committerPrefix)

		c.Committer, err = parseSignature(line)
		if err != nil {
			// This should probably just error out, and not return a partial commit...
			return false, err
		}

		return false, nil
	}

	// skip any excessive header-lines
	return false, nil
}

//TreeEntry is a file or folder at a given path in a repository
type TreeEntry struct {
	Mode   string
	Type   string
	Object string
	Path   string
}

//Tree returns the files and folders at a given ref at a path in a repository
func (r *LocalRepository) Tree(ctx context.Context, ref, path string) ([]TreeEntry, error) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	span, ctx := opentracing.StartSpanFromContext(ctx, "storage.Server.Tree")
	span.SetTag("ref", ref)
	span.SetTag("path", path)
	defer span.Finish()

	if !strings.HasSuffix(path, "/") {
		path += "/"
	}

	treeEntries, err := r.tree(ctx, ref, path)
	if err != nil {
		return nil, err
	}

	return treeEntries, nil
}

func (r *LocalRepository) tree(ctx context.Context, ref, path string) ([]TreeEntry, error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "storage.Server.tree")
	span.SetTag("ref", ref)
	span.SetTag("path", path)
	defer span.Finish()

	args := []string{"ls-tree", ref, path}
	cmd := exec.CommandContext(ctx, r.git, args...)
	cmd.Dir = r.path
	out, err := cmd.StdoutPipe()
	if err != nil {
		return nil, errors.Wrap(err, "failed to run git ls-tree")
	}
	if err = cmd.Start(); err != nil {
		return nil, errors.Wrap(err, "failed to run git ls-tree")
	}

	// TODO: The scanner takes unbounded inputs. This could cause OOMs

	var treeEntries []TreeEntry
	scanner := bufio.NewScanner(out)
	for scanner.Scan() {
		line := scanner.Text()
		te, err := parseTreeEntry(line)
		if err != nil {
			return treeEntries, errors.Wrap(err, "unable to parse tree entry line")
		}
		treeEntries = append(treeEntries, te)
	}

	if err := cmd.Wait(); err != nil {
		return nil, errors.Wrap(err, "failed to wait for command to finish")
	}

	return treeEntries, nil
}

func parseTreeEntry(s string) (TreeEntry, error) {
	tabs := strings.Split(s, "\t")
	if len(tabs) != 2 {
		return TreeEntry{}, errors.New("expected 2 tab separated inputs")
	}
	spaces := strings.Split(tabs[0], " ")
	if len(spaces) != 3 {
		return TreeEntry{}, errors.New("expected 3 space separated inputs")
	}

	return TreeEntry{
		Mode:   spaces[0],
		Type:   spaces[1],
		Object: spaces[2],
		Path:   tabs[1],
	}, nil
}

// UploadPack is a hack because we need r.path
//  int32 exitCode - The commands exit-code
//  error internalError - And internal error occured
func (r *LocalRepository) UploadPack(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) (int32, error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "storage.Repository.UploadPack")
	span.SetTag("repo_path", r.path)
	defer span.Finish()

	cmd := exec.CommandContext(ctx, "/usr/bin/git", "upload-pack", r.path)
	span.LogEvent(fmt.Sprintf("%v", cmd.Args))

	cmd.Stdin = stdin
	cmd.Stdout = stdout
	cmd.Stderr = stderr

	if err := cmd.Start(); err != nil {
		return 0, fmt.Errorf("cmd.Start: %v", err)
	}

	return exitStatus(cmd.Wait())
}

// ReceivePack is a hack because we need r.path
//  int32 exitCode - The commands exit-code
//  error internalError - And internal error occured
func (r *LocalRepository) ReceivePack(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) (int32, error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "storage.Repository.ReceivePack")
	span.SetTag("repo_path", r.path)
	defer span.Finish()

	cmd := exec.CommandContext(ctx, "/usr/bin/git", "receive-pack", r.path)
	span.LogEvent(fmt.Sprintf("%v", cmd.Args))

	cmd.Stdin = stdin
	cmd.Stdout = stdout
	cmd.Stderr = stderr

	if err := cmd.Start(); err != nil {
		return 0, fmt.Errorf("cmd.Start: %v", err)
	}

	return exitStatus(cmd.Wait())
}

// Thankfully borrowed from https://github.com/gliderlabs/sshfront/blob/ff9cab19386c1b3bcdf1d574c5cbaf8bd046fc12/handlers.go#L25-L37
func exitStatus(err error) (int32, error) {
	if err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			// There is no platform independent way to retrieve
			// the exit code, but the following will work on Unix
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				return int32(status.ExitStatus()), nil
			}
		}
		return 0, err
	}
	return 0, nil
}
<commit_msg>don't try <dir>/.git over ssh<commit_after>package storage

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/opentracing/opentracing-go"
	"github.com/pkg/errors"
)

var (
	// ErrRepoNotValid is returned for invalid repositories
	ErrRepoNotValid = fmt.Errorf("not a valid repository")
)

type (
	// Storage TODO: is something that should be split up
	Storage interface {
		Create(ctx context.Context, id string) error
		GetRepository(ctx context.Context, id string) (Repository, error)
	}

	// LocalStorage implements Storage for Local disk-access
	LocalStorage struct {
		git  string
		root string
	}

	// Repository is the interface for manipulating repos
	Repository interface {
		GetID() string
		SetDescription(ctx context.Context, description string) error
		ListBranches(ctx context.Context) ([]Branch, error)
		GetCommit(ctx context.Context, ref string) (Commit, error)
		Tree(ctx context.Context, ref, path string) ([]TreeEntry, error)
		UploadPack(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) (int32, error)
		ReceivePack(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) (int32, error)
	}

	// LocalRepository implements Repository for Local disk-access
	LocalRepository struct {
		git  string
		path string
		id   string
	}
)

// NewLocalStorage returns a LocalStorage in the given `root`
func NewLocalStorage(root string) (*LocalStorage, error) {
	if err := os.MkdirAll(root, 0755); err != nil {
		return nil, fmt.Errorf("failed to create storage root: %s", root)
	}
	return &LocalStorage{
		git:  "/usr/bin/git",
		root: root,
	}, nil
}

func (s *LocalStorage) repoPath(id string) string {
	id = strings.Replace(id, "-", "", -1)
	return filepath.Join(s.root, id[0:2], id[2:4], id[4:])
}

// Create a new repository
func (s *LocalStorage) Create(ctx context.Context, id string) error {
	dir := s.repoPath(id)

	if err := os.MkdirAll(dir, 0755); err != nil {
		return fmt.Errorf("failed to repository directory: %s", dir)
	}

	cmd := exec.CommandContext(ctx, s.git, "init", "--bare")
	cmd.Dir = dir

	//TODO: Don't throw away stdout/err...
	return cmd.Run()
}

// GetRepository from Storage
// TODO: Cache these somehow?
func (s *LocalStorage) GetRepository(ctx context.Context, repoPath string) (Repository, error) {
	dir := s.repoPath(repoPath)

	cmd := exec.CommandContext(ctx, s.git, "config", "--null", "core.repositoryformatversion")
	cmd.Dir = dir

	output, err := cmd.CombinedOutput()
	if err != nil {
		return nil, ErrRepoNotValid
	}
	if strings.TrimSuffix(string(output), "\x00") != "0" {
		return nil, ErrRepoNotValid
	}

	return &LocalRepository{git: s.git, path: dir, id: repoPath}, nil
}

// GetID returns the repos ID
func (r *LocalRepository) GetID() string {
	return r.id
}

// Branch of a repository
type Branch struct {
	Name string
	Sha1 string
	Type string
}

// SetDescription of repository
func (r *LocalRepository) SetDescription(ctx context.Context, description string) error {
	file := filepath.Join(r.path, "description")
	return ioutil.WriteFile(file, []byte(description+"\n"), 0644)
}

// ListBranches returns all branches of a given repository
func (r *LocalRepository) ListBranches(ctx context.Context) ([]Branch, error) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	args := []string{"for-each-ref", "--format=%(objectname) %(objecttype) %(refname)", "refs/heads"}
	cmd := exec.CommandContext(ctx, r.git, args...)
	cmd.Dir = r.path
	out, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err = cmd.Start(); err != nil {
		return nil, err
	}

	var bs []Branch
	scanner := bufio.NewScanner(out)
	for scanner.Scan() {
		s := strings.Split(scanner.Text(), " ")

		bs = append(bs, Branch{
			Name: strings.TrimPrefix(s[2], "refs/heads/"),
			Sha1: s[0],
			Type: s[1],
		})
	}

	if err := cmd.Wait(); err != nil {
		return nil, err
	}

	return bs, nil
}

// Commit holds a commit
type Commit struct {
	Hash    string
	Tree    string
	Parent  string
	Message string
	Body    string

	Author Signature

	Committer Signature
}

// GetCommit returns a single Commit from a Repository
func (r *LocalRepository) GetCommit(ctx context.Context, ref string) (Commit, error) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	args := []string{"cat-file", "-p", ref}
	cmd := exec.CommandContext(ctx, r.git, args...)
	cmd.Dir = r.path
	out, err := cmd.StdoutPipe()
	if err != nil {
		return Commit{}, err
	}
	if err = cmd.Start(); err != nil {
		return Commit{}, err
	}

	commit, err := parseCommit(out, ref)
	if err != nil {
		return Commit{}, err
	}

	if err := cmd.Wait(); err != nil {
		return Commit{}, err
	}

	return commit, nil
}

func parseCommit(r io.Reader, hash string) (Commit, error) {
	scanner := bufio.NewScanner(r)

	c := Commit{
		Hash: hash,
	}

	// if true then the following lines are the subject
	hasHeader := false

	for scanner.Scan() {
		line := scanner.Text()

		if !hasHeader {
			var err error
			hasHeader, err = parseCommitHeader(&c, line)
			if err != nil {
				return c, err
			}
			continue
		}

		if c.Message == "" {
			c.Message = line
			continue
		}

		// TODO: use a string.Builder instead?
		// TODO: handle commit signatures
		c.Body = fmt.Sprintf("%s\n%s", c.Body, line)
	}

	// Trim excessive stringing new lines
	c.Body = strings.TrimLeft(c.Body, "\n")

	return c, nil
}

// returns true when it's passed the header
func parseCommitHeader(c *Commit, line string) (bool, error) {
	const (
		treePrefix      = "tree "
		parentPrefix    = "parent "
		authorPrefix    = "author "
		committerPrefix = "committer "
	)

	if line == "" {
		return true, nil
	}
	if strings.HasPrefix(line, treePrefix) {
		c.Tree = strings.TrimPrefix(line, treePrefix)
		return false, nil
	}

	if strings.HasPrefix(line, parentPrefix) {
		c.Parent = strings.TrimPrefix(line, parentPrefix)
		return false, nil
	}

	if strings.HasPrefix(line, authorPrefix) {
		var err error
		line := strings.TrimPrefix(line, authorPrefix)

		c.Author, err = parseSignature(line)
		if err != nil {
			// This should probably just error out, and not return a partial commit...
			return false, err
		}
		return false, nil
	}

	if strings.HasPrefix(line, committerPrefix) {
		var err error
		line := strings.TrimPrefix(line, committerPrefix)

		c.Committer, err = parseSignature(line)
		if err != nil {
			// This should probably just error out, and not return a partial commit...
			return false, err
		}

		return false, nil
	}

	// skip any excessive header-lines
	return false, nil
}

//TreeEntry is a file or folder at a given path in a repository
type TreeEntry struct {
	Mode   string
	Type   string
	Object string
	Path   string
}

//Tree returns the files and folders at a given ref at a path in a repository
func (r *LocalRepository) Tree(ctx context.Context, ref, path string) ([]TreeEntry, error) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	span, ctx := opentracing.StartSpanFromContext(ctx, "storage.Server.Tree")
	span.SetTag("ref", ref)
	span.SetTag("path", path)
	defer span.Finish()

	if !strings.HasSuffix(path, "/") {
		path += "/"
	}

	treeEntries, err := r.tree(ctx, ref, path)
	if err != nil {
		return nil, err
	}

	return treeEntries, nil
}

func (r *LocalRepository) tree(ctx context.Context, ref, path string) ([]TreeEntry, error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "storage.Server.tree")
	span.SetTag("ref", ref)
	span.SetTag("path", path)
	defer span.Finish()

	args := []string{"ls-tree", ref, path}
	cmd := exec.CommandContext(ctx, r.git, args...)
	cmd.Dir = r.path
	out, err := cmd.StdoutPipe()
	if err != nil {
		return nil, errors.Wrap(err, "failed to run git ls-tree")
	}
	if err = cmd.Start(); err != nil {
		return nil, errors.Wrap(err, "failed to run git ls-tree")
	}

	// TODO: The scanner takes unbounded inputs. This could cause OOMs

	var treeEntries []TreeEntry
	scanner := bufio.NewScanner(out)
	for scanner.Scan() {
		line := scanner.Text()
		te, err := parseTreeEntry(line)
		if err != nil {
			return treeEntries, errors.Wrap(err, "unable to parse tree entry line")
		}
		treeEntries = append(treeEntries, te)
	}

	if err := cmd.Wait(); err != nil {
		return nil, errors.Wrap(err, "failed to wait for command to finish")
	}

	return treeEntries, nil
}

func parseTreeEntry(s string) (TreeEntry, error) {
	tabs := strings.Split(s, "\t")
	if len(tabs) != 2 {
		return TreeEntry{}, errors.New("expected 2 tab separated inputs")
	}
	spaces := strings.Split(tabs[0], " ")
	if len(spaces) != 3 {
		return TreeEntry{}, errors.New("expected 3 space separated inputs")
	}

	return TreeEntry{
		Mode:   spaces[0],
		Type:   spaces[1],
		Object: spaces[2],
		Path:   tabs[1],
	}, nil
}

// UploadPack is a hack because we need r.path
//  int32 exitCode - The commands exit-code
//  error internalError - And internal error occured
func (r *LocalRepository) UploadPack(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) (int32, error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "storage.Repository.UploadPack")
	span.SetTag("repo_path", r.path)
	defer span.Finish()

	cmd := exec.CommandContext(ctx, "/usr/bin/git", "upload-pack", "--strict", r.path)
	span.LogEvent(fmt.Sprintf("%v", cmd.Args))

	cmd.Stdin = stdin
	cmd.Stdout = stdout
	cmd.Stderr = stderr

	if err := cmd.Start(); err != nil {
		return 0, fmt.Errorf("cmd.Start: %v", err)
	}

	return exitStatus(cmd.Wait())
}

// ReceivePack is a hack because we need r.path
//  int32 exitCode - The commands exit-code
//  error internalError - And internal error occured
func (r *LocalRepository) ReceivePack(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) (int32, error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "storage.Repository.ReceivePack")
	span.SetTag("repo_path", r.path)
	defer span.Finish()

	cmd := exec.CommandContext(ctx, "/usr/bin/git", "receive-pack", "--strict", r.path)
	span.LogEvent(fmt.Sprintf("%v", cmd.Args))

	cmd.Stdin = stdin
	cmd.Stdout = stdout
	cmd.Stderr = stderr

	if err := cmd.Start(); err != nil {
		return 0, fmt.Errorf("cmd.Start: %v", err)
	}

	return exitStatus(cmd.Wait())
}

// Thankfully borrowed from https://github.com/gliderlabs/sshfront/blob/ff9cab19386c1b3bcdf1d574c5cbaf8bd046fc12/handlers.go#L25-L37
func exitStatus(err error) (int32, error) {
	if err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			// There is no platform independent way to retrieve
			// the exit code, but the following will work on Unix
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				return int32(status.ExitStatus()), nil
			}
		}
		return 0, err
	}
	return 0, nil
}
<|endoftext|>