<commit_before>package hostingde

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/StackExchange/dnscontrol/v3/pkg/diff"
	"golang.org/x/net/idna"
)

const endpoint = "%s/api/%s/v1/json/%s"

type hostingdeProvider struct {
	authToken      string
	ownerAccountID string
	baseURL        string
}

func (hp *hostingdeProvider) getDomainConfig(domain string) (*domainConfig, error) {
	zc, err := hp.getZoneConfig(domain)
	if err != nil {
		return nil, fmt.Errorf("error getting zone config: %w", err)
	}

	params := request{
		Filter: filter{
			Field: "domainName",
			Value: zc.Name,
		},
	}

	resp, err := hp.get("domain", "domainsFind", params)
	if err != nil {
		return nil, fmt.Errorf("error getting domain info: %w", err)
	}

	domainConf := []*domainConfig{}
	if err := json.Unmarshal(resp.Data, &domainConf); err != nil {
		return nil, fmt.Errorf("error parsing response: %w", err)
	}

	if len(domainConf) == 0 {
		return nil, fmt.Errorf("could not get domain config: %s", domain)
	}

	return domainConf[0], nil
}

func (hp *hostingdeProvider) createZone(domain string) error {
	t, err := idna.ToASCII(domain)
	if err != nil {
		return err
	}

	records := []*record{}
	for _, ns := range defaultNameservers {
		records = append(records, &record{
			Name:    domain,
			Type:    "NS",
			Content: ns,
			TTL:     86400,
		})
	}

	params := request{
		ZoneConfig: &zoneConfig{
			Name: t,
			Type: "NATIVE",
		},
		Records: records,
	}

	_, err = hp.get("dns", "zoneCreate", params)
	if err != nil {
		return fmt.Errorf("error creating zone: %w", err)
	}
	return nil
}

func (hp *hostingdeProvider) getNameservers(domain string) ([]string, error) {
	t, err := idna.ToASCII(domain)
	if err != nil {
		return nil, err
	}

	domainConf, err := hp.getDomainConfig(t)
	if err != nil {
		return nil, fmt.Errorf("error getting domain config: %w", err)
	}

	nss := []string{}
	for _, ns := range domainConf.Nameservers {
		// Currently does not support glued IP addresses
		if len(ns.IPs) > 0 {
			return nil, fmt.Errorf("domain %s has glued IP addresses which are not supported", domain)
		}

		nss = append(nss, ns.Name)
	}

	return nss, nil
}

func (hp *hostingdeProvider) updateNameservers(nss []string, domain string) func() error {
	return func() error {
		domainConf, err := hp.getDomainConfig(domain)
		if err != nil {
			return err
		}

		nameservers := []nameserver{}
		for _, ns := range nss {
			nameservers = append(nameservers, nameserver{Name: ns})
		}

		domainConf.Nameservers = nameservers

		params := request{
			Domain: domainConf,
		}

		if _, err := hp.get("domain", "domainUpdate", params); err != nil {
			return err
		}
		return nil
	}
}

func (hp *hostingdeProvider) getRecords(domain string) ([]*record, error) {
	zc, err := hp.getZoneConfig(domain)
	if err != nil {
		return nil, err
	}

	records := []*record{}
	page := uint(1)
	for {
		params := request{
			Filter: filter{
				Field: "ZoneConfigId",
				Value: zc.ID,
			},
			Limit: 1000,
			Page:  page,
		}

		resp, err := hp.get("dns", "recordsFind", params)
		if err != nil {
			return nil, err
		}

		newRecords := []*record{}
		if err := json.Unmarshal(resp.Data, &newRecords); err != nil {
			return nil, err
		}

		records = append(records, newRecords...)

		if page >= resp.TotalPages {
			break
		}
		page++
	}
	return records, nil
}

func (hp *hostingdeProvider) updateRecords(domain string, create, del, mod diff.Changeset) error {
	zc, err := hp.getZoneConfig(domain)
	if err != nil {
		return err
	}

	toAdd := []*record{}
	for _, c := range create {
		r := recordToNative(c.Desired)
		toAdd = append(toAdd, r)
	}

	toDelete := []*record{}
	for _, d := range del {
		r := recordToNative(d.Existing)
		r.ID = d.Existing.Original.(*record).ID
		toDelete = append(toDelete, r)
	}

	toModify := []*record{}
	for _, m := range mod {
		r := recordToNative(m.Desired)
		r.ID = m.Existing.Original.(*record).ID
		toModify = append(toModify, r)
	}

	params := request{
		ZoneConfig:      zc,
		RecordsToAdd:    toAdd,
		RecordsToDelete: toDelete,
		RecordsToModify: toModify,
	}

	_, err = hp.get("dns", "zoneUpdate", params)
	if err != nil {
		return err
	}
	return nil
}

func (hp *hostingdeProvider) getZoneConfig(domain string) (*zoneConfig, error) {
	t, err := idna.ToASCII(domain)
	if err != nil {
		return nil, err
	}

	params := request{
		Filter: filter{
			Field: "ZoneName",
			Value: t,
		},
	}

	resp, err := hp.get("dns", "zoneConfigsFind", params)
	if err != nil {
		return nil, fmt.Errorf("could not get zone config: %w", err)
	}

	zc := []*zoneConfig{}
	if err := json.Unmarshal(resp.Data, &zc); err != nil {
		return nil, fmt.Errorf("could not parse response: %w", err)
	}

	if len(zc) == 0 {
		return nil, errZoneNotFound
	}

	return zc[0], nil
}

func (hp *hostingdeProvider) get(service, method string, params request) (*responseData, error) {
	params.AuthToken = hp.authToken
	params.OwnerAccountID = hp.ownerAccountID
	reqBody, err := json.Marshal(params)
	if err != nil {
		return nil, fmt.Errorf("could not marshal request body: %w", err)
	}

	url := fmt.Sprintf(endpoint, hp.baseURL, service, method)
	resp, err := http.Post(url, "application/json", bytes.NewBuffer(reqBody))
	if err != nil {
		return nil, fmt.Errorf("could not carry out request: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode >= 400 {
		return nil, fmt.Errorf("error occurred: %s", resp.Status)
	}

	bodyBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("could not read response body: %w", err)
	}

	respData := &response{}
	if err := json.Unmarshal(bodyBytes, &respData); err != nil {
		return nil, fmt.Errorf("could not unmarshal response body: %w", err)
	}
	if len(respData.Errors) > 0 && respData.Status == "error" {
		return nil, fmt.Errorf("%+v", respData.Errors)
	}

	return respData.Response, nil
}
<commit_msg>Hosting.de: Allow using as registrar only (#1307)<commit_after>package hostingde

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/StackExchange/dnscontrol/v3/pkg/diff"
	"golang.org/x/net/idna"
)

const endpoint = "%s/api/%s/v1/json/%s"

type hostingdeProvider struct {
	authToken      string
	ownerAccountID string
	baseURL        string
}

func (hp *hostingdeProvider) getDomainConfig(domain string) (*domainConfig, error) {
	params := request{
		Filter: filter{
			Field: "domainName",
			Value: domain,
		},
	}

	resp, err := hp.get("domain", "domainsFind", params)
	if err != nil {
		return nil, fmt.Errorf("error getting domain info: %w", err)
	}

	domainConf := []*domainConfig{}
	if err := json.Unmarshal(resp.Data, &domainConf); err != nil {
		return nil, fmt.Errorf("error parsing response: %w", err)
	}

	if len(domainConf) == 0 {
		return nil, fmt.Errorf("could not get domain config: %s", domain)
	}

	return domainConf[0], nil
}

func (hp *hostingdeProvider) createZone(domain string) error {
	t, err := idna.ToASCII(domain)
	if err != nil {
		return err
	}

	records := []*record{}
	for _, ns := range defaultNameservers {
		records = append(records, &record{
			Name:    domain,
			Type:    "NS",
			Content: ns,
			TTL:     86400,
		})
	}

	params := request{
		ZoneConfig: &zoneConfig{
			Name: t,
			Type: "NATIVE",
		},
		Records: records,
	}

	_, err = hp.get("dns", "zoneCreate", params)
	if err != nil {
		return fmt.Errorf("error creating zone: %w", err)
	}
	return nil
}

func (hp *hostingdeProvider) getNameservers(domain string) ([]string, error) {
	t, err := idna.ToASCII(domain)
	if err != nil {
		return nil, err
	}

	domainConf, err := hp.getDomainConfig(t)
	if err != nil {
		return nil, fmt.Errorf("error getting domain config: %w", err)
	}

	nss := []string{}
	for _, ns := range domainConf.Nameservers {
		// Currently does not support glued IP addresses
		if len(ns.IPs) > 0 {
			return nil, fmt.Errorf("domain %s has glued IP addresses which are not supported", domain)
		}

		nss = append(nss, ns.Name)
	}

	return nss, nil
}

func (hp *hostingdeProvider) updateNameservers(nss []string, domain string) func() error {
	return func() error {
		domainConf, err := hp.getDomainConfig(domain)
		if err != nil {
			return err
		}

		nameservers := []nameserver{}
		for _, ns := range nss {
			nameservers = append(nameservers, nameserver{Name: ns})
		}

		domainConf.Nameservers = nameservers

		params := request{
			Domain: domainConf,
		}

		if _, err := hp.get("domain", "domainUpdate", params); err != nil {
			return err
		}
		return nil
	}
}

func (hp *hostingdeProvider) getRecords(domain string) ([]*record, error) {
	zc, err := hp.getZoneConfig(domain)
	if err != nil {
		return nil, err
	}

	records := []*record{}
	page := uint(1)
	for {
		params := request{
			Filter: filter{
				Field: "ZoneConfigId",
				Value: zc.ID,
			},
			Limit: 1000,
			Page:  page,
		}

		resp, err := hp.get("dns", "recordsFind", params)
		if err != nil {
			return nil, err
		}

		newRecords := []*record{}
		if err := json.Unmarshal(resp.Data, &newRecords); err != nil {
			return nil, err
		}

		records = append(records, newRecords...)

		if page >= resp.TotalPages {
			break
		}
		page++
	}
	return records, nil
}

func (hp *hostingdeProvider) updateRecords(domain string, create, del, mod diff.Changeset) error {
	zc, err := hp.getZoneConfig(domain)
	if err != nil {
		return err
	}

	toAdd := []*record{}
	for _, c := range create {
		r := recordToNative(c.Desired)
		toAdd = append(toAdd, r)
	}

	toDelete := []*record{}
	for _, d := range del {
		r := recordToNative(d.Existing)
		r.ID = d.Existing.Original.(*record).ID
		toDelete = append(toDelete, r)
	}

	toModify := []*record{}
	for _, m := range mod {
		r := recordToNative(m.Desired)
		r.ID = m.Existing.Original.(*record).ID
		toModify = append(toModify, r)
	}

	params := request{
		ZoneConfig:      zc,
		RecordsToAdd:    toAdd,
		RecordsToDelete: toDelete,
		RecordsToModify: toModify,
	}

	_, err = hp.get("dns", "zoneUpdate", params)
	if err != nil {
		return err
	}
	return nil
}

func (hp *hostingdeProvider) getZoneConfig(domain string) (*zoneConfig, error) {
	t, err := idna.ToASCII(domain)
	if err != nil {
		return nil, err
	}

	params := request{
		Filter: filter{
			Field: "ZoneName",
			Value: t,
		},
	}

	resp, err := hp.get("dns", "zoneConfigsFind", params)
	if err != nil {
		return nil, fmt.Errorf("could not get zone config: %w", err)
	}

	zc := []*zoneConfig{}
	if err := json.Unmarshal(resp.Data, &zc); err != nil {
		return nil, fmt.Errorf("could not parse response: %w", err)
	}

	if len(zc) == 0 {
		return nil, errZoneNotFound
	}

	return zc[0], nil
}

func (hp *hostingdeProvider) get(service, method string, params request) (*responseData, error) {
	params.AuthToken = hp.authToken
	params.OwnerAccountID = hp.ownerAccountID
	reqBody, err := json.Marshal(params)
	if err != nil {
		return nil, fmt.Errorf("could not marshal request body: %w", err)
	}

	url := fmt.Sprintf(endpoint, hp.baseURL, service, method)
	resp, err := http.Post(url, "application/json", bytes.NewBuffer(reqBody))
	if err != nil {
		return nil, fmt.Errorf("could not carry out request: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode >= 400 {
		return nil, fmt.Errorf("error occurred: %s", resp.Status)
	}

	bodyBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("could not read response body: %w", err)
	}

	respData := &response{}
	if err := json.Unmarshal(bodyBytes, &respData); err != nil {
		return nil, fmt.Errorf("could not unmarshal response body: %w", err)
	}
	if len(respData.Errors) > 0 && respData.Status == "error" {
		return nil, fmt.Errorf("%+v", respData.Errors)
	}

	return respData.Response, nil
}
<|endoftext|>


<commit_before>// Copyright 2017, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package prometheus contains the Prometheus exporters for
// Stackdriver Monitoring.
//
// Please note that this exporter is currently work in progress and not complete.
package prometheus

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
	"sync"

	"go.opencensus.io/internal"
	"go.opencensus.io/stats"
	"go.opencensus.io/tag"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

const (
	defaultNamespace = "opencensus"
)

// Exporter exports stats to Prometheus, users need
// to register the exporter as an http.Handler to be
// able to export.
type Exporter struct {
	opts    Options
	g       prometheus.Gatherer
	c       *collector
	handler http.Handler
}

// Options contains options for configuring the exporter.
type Options struct {
	Namespace string
	OnError   func(err error)
}

// NewExporter returns an exporter that exports stats to Prometheus.
func NewExporter(o Options) (*Exporter, error) {
	if o.Namespace == "" {
		o.Namespace = defaultNamespace
	}
	reg := prometheus.NewRegistry()
	collector := newCollector(o, reg)
	e := &Exporter{
		opts:    o,
		g:       reg,
		c:       collector,
		handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}),
	}
	return e, nil
}

var _ http.Handler = (*Exporter)(nil)
var _ stats.Exporter = (*Exporter)(nil)

func (c *collector) registerViews(views ...*stats.View) {
	count := 0
	for _, view := range views {
		sig := viewSignature(c.opts.Namespace, view)
		c.registeredViewsMu.Lock()
		_, ok := c.registeredViews[sig]
		c.registeredViewsMu.Unlock()

		if !ok {
			desc := prometheus.NewDesc(
				viewName(c.opts.Namespace, view),
				view.Description(),
				tagKeysToLabels(view.TagKeys()),
				nil,
			)
			c.registeredViewsMu.Lock()
			c.registeredViews[sig] = desc
			c.registeredViewsMu.Unlock()
			count++
		}
	}
	if count == 0 {
		return
	}

	c.reg.Unregister(c)
	if err := c.reg.Register(c); err != nil {
		c.opts.onError(fmt.Errorf("cannot register the collector: %v", err))
	}
}

func (o *Options) onError(err error) {
	if o.OnError != nil {
		o.OnError(err)
	} else {
		log.Printf("Failed to export to Prometheus: %v", err)
	}
}

// Export exports to the Prometheus if view data has one or more rows.
func (e *Exporter) Export(vd *stats.ViewData) {
	if len(vd.Rows) == 0 {
		return
	}
	e.c.addViewData(vd)
}

// ServeHTTP serves the Prometheus endpoint.
func (e *Exporter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	e.handler.ServeHTTP(w, r)
}

// collector implements prometheus.Collector
type collector struct {
	opts Options
	mu   sync.Mutex // mu guards all the fields.

	// reg helps collector register views dyanmically.
	reg *prometheus.Registry

	// viewData are accumulated and atomically
	// appended to on every Export invocation, from
	// stats. These views are cleared out when
	// Collect is invoked and the cycle is repeated.
	viewData map[string]*stats.ViewData

	registeredViewsMu sync.Mutex
	// registeredViews maps a view to a prometheus desc.
	registeredViews map[string]*prometheus.Desc

	// seenMetrics maps from the metric's rawType to the actual Metric.
	// It is an interface to interface mapping
	// but the key is the zero value while the value is the instance.
	seenMetrics map[stats.AggregationData]prometheus.Metric
}

func (c *collector) addViewData(vd *stats.ViewData) {
	c.registerViews(vd.View)
	sig := viewSignature(c.opts.Namespace, vd.View)

	c.mu.Lock()
	c.viewData[sig] = vd
	c.mu.Unlock()
}

func (c *collector) Describe(ch chan<- *prometheus.Desc) {
	c.registeredViewsMu.Lock()
	registered := make(map[string]*prometheus.Desc)
	for k, desc := range c.registeredViews {
		registered[k] = desc
	}
	c.registeredViewsMu.Unlock()

	for _, desc := range registered {
		ch <- desc
	}
}

func (c *collector) lookupMetric(key stats.AggregationData) (prometheus.Metric, bool) {
	c.mu.Lock()
	value, ok := c.seenMetrics[key]
	c.mu.Unlock()
	return value, ok
}

func (c *collector) memoizeMetric(key stats.AggregationData, value prometheus.Metric) {
	c.mu.Lock()
	c.seenMetrics[key] = value
	c.mu.Unlock()
}

// Collect fetches the statistics from OpenCensus
// and delivers them as Prometheus Metrics.
// Collect is invoked everytime a prometheus.Gatherer is run
// for example when the HTTP endpoint is invoked by Prometheus.
func (c *collector) Collect(ch chan<- prometheus.Metric) {
	c.mu.Lock()
	views := make(map[string]*stats.ViewData, len(c.viewData))
	for i, vd := range c.viewData {
		views[i] = vd
	}
	c.mu.Unlock()

	for _, vd := range views {
		for _, row := range vd.Rows {
			metric, err := c.toMetric(vd.View, row)
			if err != nil {
				c.opts.onError(err)
			} else {
				ch <- metric
			}
		}
	}

}

func (c *collector) toMetric(view *stats.View, row *stats.Row) (prometheus.Metric, error) {
	switch agg := view.Aggregation().(type) {
	case stats.CountAggregation:
		data := row.Data.(*stats.CountData)
		var key *stats.CountData
		sc, ok := c.lookupMetric(key)
		if !ok {
			sc = prometheus.NewCounter(prometheus.CounterOpts{
				Name:      internal.Sanitize(view.Name()),
				Help:      view.Description(),
				Namespace: c.opts.Namespace,
			})
			c.memoizeMetric(key, sc)
		}
		counter := sc.(prometheus.Counter)
		counter.Add(float64(*data))
		return counter, nil

	case stats.DistributionAggregation:
		data := row.Data.(*stats.DistributionData)
		sig := viewSignature(c.opts.Namespace, view)

		c.registeredViewsMu.Lock()
		desc := c.registeredViews[sig]
		c.registeredViewsMu.Unlock()

		var tagValues []string
		for _, t := range row.Tags {
			tagValues = append(tagValues, t.Value)
		}

		points := make(map[float64]uint64)
		for i, b := range agg {
			points[b] = uint64(data.CountPerBucket[i])
		}
		hist, err := prometheus.NewConstHistogram(desc, uint64(data.Count), data.Sum(), points, tagValues...)
		if err != nil {
			return nil, err
		}
		return hist, nil

	case stats.SumAggregation:
		panic("stats.SumData not supported yet")

	case *stats.MeanAggregation:
		panic("stats.MeanData ont supported yet")

	default:
		return nil, fmt.Errorf("aggregation %T is not yet supported", view.Aggregation())
	}
}

func tagKeysToLabels(keys []tag.Key) (labels []string) {
	for _, key := range keys {
		labels = append(labels, internal.Sanitize(key.Name()))
	}
	return labels
}

func tagsToLabels(tags []tag.Tag) []string {
	var names []string
	for _, tag := range tags {
		names = append(names, internal.Sanitize(tag.Key.Name()))
	}
	return names
}

func newCollector(opts Options, registrar *prometheus.Registry) *collector {
	return &collector{
		reg:             registrar,
		opts:            opts,
		registeredViews: make(map[string]*prometheus.Desc),
		seenMetrics:     make(map[stats.AggregationData]prometheus.Metric),
		viewData:        make(map[string]*stats.ViewData),
	}
}

func viewName(namespace string, v *stats.View) string {
	return namespace + "_" + internal.Sanitize(v.Name())
}

func viewSignature(namespace string, v *stats.View) string {
	var buf bytes.Buffer
	buf.WriteString(viewName(namespace, v))
	for _, k := range v.TagKeys() {
		buf.WriteString("-" + k.Name())
	}
	return buf.String()
}
<commit_msg>Report the count data correctly from the Prometheus exporter (#193)<commit_after>// Copyright 2017, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package prometheus contains the Prometheus exporters for
// Stackdriver Monitoring.
//
// Please note that this exporter is currently work in progress and not complete.
package prometheus

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
	"sync"

	"go.opencensus.io/internal"
	"go.opencensus.io/stats"
	"go.opencensus.io/tag"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

const (
	defaultNamespace = "opencensus"
)

// Exporter exports stats to Prometheus, users need
// to register the exporter as an http.Handler to be
// able to export.
type Exporter struct {
	opts    Options
	g       prometheus.Gatherer
	c       *collector
	handler http.Handler
}

// Options contains options for configuring the exporter.
type Options struct {
	Namespace string
	OnError   func(err error)
}

// NewExporter returns an exporter that exports stats to Prometheus.
func NewExporter(o Options) (*Exporter, error) {
	if o.Namespace == "" {
		o.Namespace = defaultNamespace
	}
	reg := prometheus.NewRegistry()
	collector := newCollector(o, reg)
	e := &Exporter{
		opts:    o,
		g:       reg,
		c:       collector,
		handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}),
	}
	return e, nil
}

var _ http.Handler = (*Exporter)(nil)
var _ stats.Exporter = (*Exporter)(nil)

func (c *collector) registerViews(views ...*stats.View) {
	count := 0
	for _, view := range views {
		sig := viewSignature(c.opts.Namespace, view)
		c.registeredViewsMu.Lock()
		_, ok := c.registeredViews[sig]
		c.registeredViewsMu.Unlock()

		if !ok {
			desc := prometheus.NewDesc(
				viewName(c.opts.Namespace, view),
				view.Description(),
				tagKeysToLabels(view.TagKeys()),
				nil,
			)
			c.registeredViewsMu.Lock()
			c.registeredViews[sig] = desc
			c.registeredViewsMu.Unlock()
			count++
		}
	}
	if count == 0 {
		return
	}

	c.reg.Unregister(c)
	if err := c.reg.Register(c); err != nil {
		c.opts.onError(fmt.Errorf("cannot register the collector: %v", err))
	}
}

func (o *Options) onError(err error) {
	if o.OnError != nil {
		o.OnError(err)
	} else {
		log.Printf("Failed to export to Prometheus: %v", err)
	}
}

// Export exports to the Prometheus if view data has one or more rows.
func (e *Exporter) Export(vd *stats.ViewData) {
	if len(vd.Rows) == 0 {
		return
	}
	e.c.addViewData(vd)
}

// ServeHTTP serves the Prometheus endpoint.
func (e *Exporter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	e.handler.ServeHTTP(w, r)
}

// collector implements prometheus.Collector
type collector struct {
	opts Options
	mu   sync.Mutex // mu guards all the fields.

	// reg helps collector register views dyanmically.
	reg *prometheus.Registry

	// viewData are accumulated and atomically
	// appended to on every Export invocation, from
	// stats. These views are cleared out when
	// Collect is invoked and the cycle is repeated.
	viewData map[string]*stats.ViewData

	registeredViewsMu sync.Mutex
	// registeredViews maps a view to a prometheus desc.
	registeredViews map[string]*prometheus.Desc
}

func (c *collector) addViewData(vd *stats.ViewData) {
	c.registerViews(vd.View)
	sig := viewSignature(c.opts.Namespace, vd.View)

	c.mu.Lock()
	c.viewData[sig] = vd
	c.mu.Unlock()
}

func (c *collector) Describe(ch chan<- *prometheus.Desc) {
	c.registeredViewsMu.Lock()
	registered := make(map[string]*prometheus.Desc)
	for k, desc := range c.registeredViews {
		registered[k] = desc
	}
	c.registeredViewsMu.Unlock()

	for _, desc := range registered {
		ch <- desc
	}
}

// Collect fetches the statistics from OpenCensus
// and delivers them as Prometheus Metrics.
// Collect is invoked everytime a prometheus.Gatherer is run
// for example when the HTTP endpoint is invoked by Prometheus.
func (c *collector) Collect(ch chan<- prometheus.Metric) {
	c.mu.Lock()
	views := make(map[string]*stats.ViewData, len(c.viewData))
	for i, vd := range c.viewData {
		views[i] = vd
	}
	c.mu.Unlock()

	for _, vd := range c.viewData {
		sig := viewSignature(c.opts.Namespace, vd.View)
		c.registeredViewsMu.Lock()
		desc := c.registeredViews[sig]
		c.registeredViewsMu.Unlock()

		for _, row := range vd.Rows {
			metric, err := c.toMetric(desc, vd.View, row)
			if err != nil {
				c.opts.onError(err)
			} else {
				ch <- metric
			}
		}
	}

}

func (c *collector) toMetric(desc *prometheus.Desc, view *stats.View, row *stats.Row) (prometheus.Metric, error) {
	switch agg := view.Aggregation().(type) {
	case stats.CountAggregation:
		data := row.Data.(*stats.CountData)
		return prometheus.NewConstMetric(desc, prometheus.CounterValue, float64(*data), tagValues(row.Tags)...)

	case stats.DistributionAggregation:
		data := row.Data.(*stats.DistributionData)
		points := make(map[float64]uint64)
		for i, b := range agg {
			points[b] = uint64(data.CountPerBucket[i])
		}
		hist, err := prometheus.NewConstHistogram(desc, uint64(data.Count), data.Sum(), points, tagValues(row.Tags)...)
		if err != nil {
			return nil, err
		}
		return hist, nil

	case stats.SumAggregation:
		panic("stats.SumData not supported yet")

	case *stats.MeanAggregation:
		panic("stats.MeanData ont supported yet")

	default:
		return nil, fmt.Errorf("aggregation %T is not yet supported", view.Aggregation())
	}
}

func tagKeysToLabels(keys []tag.Key) (labels []string) {
	for _, key := range keys {
		labels = append(labels, internal.Sanitize(key.Name()))
	}
	return labels
}

func tagsToLabels(tags []tag.Tag) []string {
	var names []string
	for _, tag := range tags {
		names = append(names, internal.Sanitize(tag.Key.Name()))
	}
	return names
}

func newCollector(opts Options, registrar *prometheus.Registry) *collector {
	return &collector{
		reg:             registrar,
		opts:            opts,
		registeredViews: make(map[string]*prometheus.Desc),
		viewData:        make(map[string]*stats.ViewData),
	}
}

func tagValues(t []tag.Tag) []string {
	var values []string
	for _, t := range t {
		values = append(values, t.Value)
	}
	return values
}

func viewName(namespace string, v *stats.View) string {
	return namespace + "_" + internal.Sanitize(v.Name())
}

func viewSignature(namespace string, v *stats.View) string {
	var buf bytes.Buffer
	buf.WriteString(viewName(namespace, v))
	for _, k := range v.TagKeys() {
		buf.WriteString("-" + k.Name())
	}
	return buf.String()
}
<|endoftext|>


<commit_before>// This is a Go implementation to calculate Levenshtein Distance
// Implementation taken from
// http://en.wikipedia.org/wiki/Levenshtein_distance#Iterative_with_two_matrix_rows
package levenshtein

import (
  //"log"
  "math"
  )

// ComputeDistance computes the levenshtein distance between the two
// strings passed as an argument. The return value is the levenshtein distance
// and error if any
func ComputeDistance(s1, s2 string) (n int, err error) {

  if s1 == s2 {
    return 0, nil
  }
  if len(s1) == 0 {
    return len(s2), nil
  }
  if len(s2) == 0 {
    return len(s1), nil
  }

  x := make([]int, len(s2)+1)
  y := make([]int, len(s2)+1)
  for i, _ := range(x) {
    x[i] = i
  }
  for i, _ := range(s1) {
    y[0] = i+1

    for j, _ := range(s2) {
      var cost int
      if s1[i] == s2[j] {
        cost = 0
      } else {
        cost = 1
      }
      y[j+1] = int(math.Min(float64(y[j]+1),
                        math.Min(float64(x[j+1]+1), float64(x[j]+cost))))
    }
    copy(x, y)
  }
  return y[len(s2)], nil
}
<commit_msg>Updating comment<commit_after>// This is a Go implementation to calculate Levenshtein Distance.
// Implementation taken from
// http://en.wikipedia.org/wiki/Levenshtein_distance#Iterative_with_two_matrix_rows
package levenshtein

import (
  //"log"
  "math"
  )

// ComputeDistance computes the levenshtein distance between the two
// strings passed as an argument. The return value is the levenshtein distance
// and error if any
func ComputeDistance(s1, s2 string) (n int, err error) {

  if s1 == s2 {
    return 0, nil
  }
  if len(s1) == 0 {
    return len(s2), nil
  }
  if len(s2) == 0 {
    return len(s1), nil
  }

  x := make([]int, len(s2)+1)
  y := make([]int, len(s2)+1)
  for i, _ := range(x) {
    x[i] = i
  }
  for i, _ := range(s1) {
    y[0] = i+1

    for j, _ := range(s2) {
      var cost int
      if s1[i] == s2[j] {
        cost = 0
      } else {
        cost = 1
      }
      y[j+1] = int(math.Min(float64(y[j]+1),
                        math.Min(float64(x[j+1]+1), float64(x[j]+cost))))
    }
    copy(x, y)
  }
  return y[len(s2)], nil
}
<|endoftext|>


<commit_before>package jsonnet

import (
	"bytes"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/drone/drone-yaml/yaml"
	"github.com/drone/drone-yaml/yaml/pretty"
	"github.com/fatih/color"
	"github.com/google/go-jsonnet"
	"github.com/urfave/cli"
)

// Command exports the jsonnet command.
var Command = cli.Command{
	Name:      "jsonnet",
	Usage:     "generate .drone.yml from jsonnet",
	ArgsUsage: "[path/to/.drone.jsonnet]",
	Action: func(c *cli.Context) {
		if err := generate(c); err != nil {
			log.Fatalln(err)
		}
	},
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "source",
			Usage: "Source file",
			Value: ".drone.jsonnet",
		},
		cli.StringFlag{
			Name:  "target",
			Usage: "target file",
			Value: ".drone.yml",
		},
		cli.BoolFlag{
			Name:  "stream",
			Usage: "Write output as a YAML stream.",
		},
		cli.BoolTFlag{
			Name:  "format",
			Usage: "Write output as formatted YAML",
		},
		cli.BoolFlag{
			Name:  "stdout",
			Usage: "Write output to stdout",
		},
		cli.BoolFlag{
			Name:  "string",
			Usage: "Expect a string, manifest as plain text",
		},
	},
}

func generate(c *cli.Context) error {
	source := c.String("source")
	target := c.String("target")

	data, err := ioutil.ReadFile(source)
	if err != nil {
		return err
	}

	vm := jsonnet.MakeVM()
	vm.MaxStack = 500
	vm.StringOutput = c.Bool("string")
	vm.ErrorFormatter.SetMaxStackTraceSize(20)
	vm.ErrorFormatter.SetColorFormatter(
		color.New(color.FgRed).Fprintf,
	)

	// register native functions
	RegisterNativeFuncs(vm)

	buf := new(bytes.Buffer)
	if c.Bool("stream") {
		docs, err := vm.EvaluateSnippetStream(source, string(data))
		if err != nil {
			return err
		}
		for _, doc := range docs {
			buf.WriteString("---")
			buf.WriteString("\n")
			buf.WriteString(doc)
		}
	} else {
		result, err := vm.EvaluateSnippet(source, string(data))
		if err != nil {
			return err
		}
		buf.WriteString(result)
	}

	// the yaml file is parsed and formatted by default. This
	// can be disabled for --format=false.
	if c.BoolT("format") {
		manifest, err := yaml.Parse(buf)
		if err != nil {
			return err
		}
		buf.Reset()
		pretty.Print(buf, manifest)
	}

	// the user can optionally write the yaml to stdout. This
	// is useful for debugging purposes without mutating an
	// existing file.
	if c.Bool("stdout") {
		io.Copy(os.Stdout, buf)
		return nil
	}

	return ioutil.WriteFile(target, buf.Bytes(), 0644)
}
<commit_msg>feat(jsonnet): support setting extVars<commit_after>package jsonnet

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/drone/drone-yaml/yaml"
	"github.com/drone/drone-yaml/yaml/pretty"
	"github.com/fatih/color"
	"github.com/google/go-jsonnet"
	"github.com/urfave/cli"
)

// Command exports the jsonnet command.
var Command = cli.Command{
	Name:      "jsonnet",
	Usage:     "generate .drone.yml from jsonnet",
	ArgsUsage: "[path/to/.drone.jsonnet]",
	Action: func(c *cli.Context) {
		if err := generate(c); err != nil {
			log.Fatalln(err)
		}
	},
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "source",
			Usage: "Source file",
			Value: ".drone.jsonnet",
		},
		cli.StringFlag{
			Name:  "target",
			Usage: "target file",
			Value: ".drone.yml",
		},
		cli.BoolFlag{
			Name:  "stream",
			Usage: "Write output as a YAML stream.",
		},
		cli.BoolTFlag{
			Name:  "format",
			Usage: "Write output as formatted YAML",
		},
		cli.BoolFlag{
			Name:  "stdout",
			Usage: "Write output to stdout",
		},
		cli.BoolFlag{
			Name:  "string",
			Usage: "Expect a string, manifest as plain text",
		},
		cli.StringSliceFlag{
			Name:  "extVar, V",
			Usage: "Pass extVars to Jsonnet (can be specified multiple times)",
		},
	},
}

func generate(c *cli.Context) error {
	source := c.String("source")
	target := c.String("target")

	data, err := ioutil.ReadFile(source)
	if err != nil {
		return err
	}

	vm := jsonnet.MakeVM()
	vm.MaxStack = 500
	vm.StringOutput = c.Bool("string")
	vm.ErrorFormatter.SetMaxStackTraceSize(20)
	vm.ErrorFormatter.SetColorFormatter(
		color.New(color.FgRed).Fprintf,
	)

	// register native functions
	RegisterNativeFuncs(vm)

	// extVars
	vars := c.StringSlice("extVar")
	for _, v := range vars {
		name, value, err := getVarVal(v)
		if err != nil {
			return err
		}
		vm.ExtVar(name, value)
	}

	buf := new(bytes.Buffer)
	if c.Bool("stream") {
		docs, err := vm.EvaluateSnippetStream(source, string(data))
		if err != nil {
			return err
		}
		for _, doc := range docs {
			buf.WriteString("---")
			buf.WriteString("\n")
			buf.WriteString(doc)
		}
	} else {
		result, err := vm.EvaluateSnippet(source, string(data))
		if err != nil {
			return err
		}
		buf.WriteString(result)
	}

	// the yaml file is parsed and formatted by default. This
	// can be disabled for --format=false.
	if c.BoolT("format") {
		manifest, err := yaml.Parse(buf)
		if err != nil {
			return err
		}
		buf.Reset()
		pretty.Print(buf, manifest)
	}

	// the user can optionally write the yaml to stdout. This
	// is useful for debugging purposes without mutating an
	// existing file.
	if c.Bool("stdout") {
		io.Copy(os.Stdout, buf)
		return nil
	}

	return ioutil.WriteFile(target, buf.Bytes(), 0644)
}

// https://github.com/google/go-jsonnet/blob/master/cmd/jsonnet/cmd.go#L149
func getVarVal(s string) (string, string, error) {
	parts := strings.SplitN(s, "=", 2)
	name := parts[0]
	if len(parts) == 1 {
		content, exists := os.LookupEnv(name)
		if exists {
			return name, content, nil
		}
		return "", "", fmt.Errorf("environment variable %v was undefined", name)
	}
	return name, parts[1], nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"text/template"

	"appengine"
	"appengine/datastore"
	"appengine/delay"
	"appengine/mail"
	"appengine/urlfetch"
)

const (
	mailFrom   = "builder@golang.org" // use this for sending any mail
	failMailTo = "golang-dev@googlegroups.com"
	domain     = "build.golang.org"
	gobotBase  = "http://research.swtch.com/gobot_codereview"
)

// ignoreFailure is a set of builders that we don't email about because
// they are not yet production-ready.
var ignoreFailure = map[string]bool{
	"dragonfly-386":           true,
	"dragonfly-amd64":         true,
	"netbsd-arm-rpi":          true,
	"solaris-amd64-smartos":   true,
	"solaris-amd64-solaris11": true,
}

// notifyOnFailure checks whether the supplied Commit or the subsequent
// Commit (if present) breaks the build for this builder.
// If either of those commits break the build an email notification is sent
// from a delayed task. (We use a task because this way the mail won't be
// sent if the enclosing datastore transaction fails.)
//
// This must be run in a datastore transaction, and the provided *Commit must
// have been retrieved from the datastore within that transaction.
func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
	if ignoreFailure[builder] {
		return nil
	}

	// TODO(adg): implement notifications for packages
	if com.PackagePath != "" {
		return nil
	}

	p := &Package{Path: com.PackagePath}
	var broken *Commit
	cr := com.Result(builder, "")
	if cr == nil {
		return fmt.Errorf("no result for %s/%s", com.Hash, builder)
	}
	q := datastore.NewQuery("Commit").Ancestor(p.Key(c))
	if cr.OK {
		// This commit is OK. Notify if next Commit is broken.
		next := new(Commit)
		q = q.Filter("ParentHash=", com.Hash)
		if err := firstMatch(c, q, next); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// OK at tip, no notification necessary.
				return nil
			}
			return err
		}
		if nr := next.Result(builder, ""); nr != nil && !nr.OK {
			c.Debugf("commit ok: %#v\nresult: %#v", com, cr)
			c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr)
			broken = next
		}
	} else {
		// This commit is broken. Notify if the previous Commit is OK.
		prev := new(Commit)
		q = q.Filter("Hash=", com.ParentHash)
		if err := firstMatch(c, q, prev); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// No previous result, let the backfill of
				// this result trigger the notification.
				return nil
			}
			return err
		}
		if pr := prev.Result(builder, ""); pr != nil && pr.OK {
			c.Debugf("commit broken: %#v\nresult: %#v", com, cr)
			c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr)
			broken = com
		}
	}
	var err error
	if broken != nil && !broken.FailNotificationSent {
		c.Infof("%s is broken commit; notifying", broken.Hash)
		notifyLater.Call(c, broken, builder) // add task to queue
		broken.FailNotificationSent = true
		_, err = datastore.Put(c, broken.Key(c), broken)
	}
	return err
}

// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
	t := q.Limit(1).Run(c)
	_, err := t.Next(v)
	if err == datastore.Done {
		err = datastore.ErrNoSuchEntity
	}
	return err
}

var notifyLater = delay.Func("notify", notify)

// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder string) {
	if !updateCL(c, com, builder) {
		// Send a mail notification if the CL can't be found.
		sendFailMail(c, com, builder)
	}
}

// updateCL updates the CL for the given Commit with a failure message
// for the given builder.
func updateCL(c appengine.Context, com *Commit, builder string) bool {
	cl, err := lookupCL(c, com)
	if err != nil {
		c.Errorf("could not find CL for %v: %v", com.Hash, err)
		return false
	}
	url := fmt.Sprintf("%v?cl=%v&brokebuild=%v", gobotBase, cl, builder)
	r, err := urlfetch.Client(c).Post(url, "text/plain", nil)
	if err != nil {
		c.Errorf("could not update CL %v: %v", cl, err)
		return false
	}
	r.Body.Close()
	if r.StatusCode != http.StatusOK {
		c.Errorf("could not update CL %v: %v", cl, r.Status)
		return false
	}
	return true
}

var clURL = regexp.MustCompile(`https://codereview.appspot.com/([0-9]+)`)

// lookupCL consults code.google.com for the full change description for the
// provided Commit, and returns the relevant CL number.
func lookupCL(c appengine.Context, com *Commit) (string, error) {
	url := "https://code.google.com/p/go/source/detail?r=" + com.Hash
	r, err := urlfetch.Client(c).Get(url)
	if err != nil {
		return "", err
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return "", fmt.Errorf("retrieving %v: %v", url, r.Status)
	}
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return "", err
	}
	m := clURL.FindAllSubmatch(b, -1)
	if m == nil {
		return "", errors.New("no CL URL found on changeset page")
	}
	// Return the last visible codereview URL on the page,
	// in case the change description refers to another CL.
	return string(m[len(m)-1][1]), nil
}

var sendFailMailTmpl = template.Must(template.New("notify.txt").
	Funcs(template.FuncMap(tmplFuncs)).
	ParseFiles("build/notify.txt"))

func init() {
	gob.Register(&Commit{}) // for delay
}

// sendFailMail sends a mail notification that the build failed on the
// provided commit and builder.
func sendFailMail(c appengine.Context, com *Commit, builder string) {
	// TODO(adg): handle packages

	// get Result
	r := com.Result(builder, "")
	if r == nil {
		c.Errorf("finding result for %q: %+v", builder, com)
		return
	}

	// get Log
	k := datastore.NewKey(c, "Log", r.LogHash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, k, l); err != nil {
		c.Errorf("finding Log record %v: %v", r.LogHash, err)
		return
	}

	// prepare mail message
	var body bytes.Buffer
	err := sendFailMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "Result": r, "Log": l,
		"Hostname": domain,
	})
	if err != nil {
		c.Errorf("rendering mail template: %v", err)
		return
	}
	subject := fmt.Sprintf("%s broken by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}
<commit_msg>go.tools/dashboard/app: send log hash to gobot on build failure<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"text/template"

	"appengine"
	"appengine/datastore"
	"appengine/delay"
	"appengine/mail"
	"appengine/urlfetch"
)

const (
	mailFrom   = "builder@golang.org" // use this for sending any mail
	failMailTo = "golang-dev@googlegroups.com"
	domain     = "build.golang.org"
	gobotBase  = "http://research.swtch.com/gobot_codereview"
)

// ignoreFailure is a set of builders that we don't email about because
// they are not yet production-ready.
var ignoreFailure = map[string]bool{
	"dragonfly-386":           true,
	"dragonfly-amd64":         true,
	"netbsd-arm-rpi":          true,
	"solaris-amd64-smartos":   true,
	"solaris-amd64-solaris11": true,
}

// notifyOnFailure checks whether the supplied Commit or the subsequent
// Commit (if present) breaks the build for this builder.
// If either of those commits break the build an email notification is sent
// from a delayed task. (We use a task because this way the mail won't be
// sent if the enclosing datastore transaction fails.)
//
// This must be run in a datastore transaction, and the provided *Commit must
// have been retrieved from the datastore within that transaction.
func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
	if ignoreFailure[builder] {
		return nil
	}

	// TODO(adg): implement notifications for packages
	if com.PackagePath != "" {
		return nil
	}

	p := &Package{Path: com.PackagePath}
	var broken *Commit
	cr := com.Result(builder, "")
	if cr == nil {
		return fmt.Errorf("no result for %s/%s", com.Hash, builder)
	}
	q := datastore.NewQuery("Commit").Ancestor(p.Key(c))
	if cr.OK {
		// This commit is OK. Notify if next Commit is broken.
		next := new(Commit)
		q = q.Filter("ParentHash=", com.Hash)
		if err := firstMatch(c, q, next); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// OK at tip, no notification necessary.
				return nil
			}
			return err
		}
		if nr := next.Result(builder, ""); nr != nil && !nr.OK {
			c.Debugf("commit ok: %#v\nresult: %#v", com, cr)
			c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr)
			broken = next
		}
	} else {
		// This commit is broken. Notify if the previous Commit is OK.
		prev := new(Commit)
		q = q.Filter("Hash=", com.ParentHash)
		if err := firstMatch(c, q, prev); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// No previous result, let the backfill of
				// this result trigger the notification.
				return nil
			}
			return err
		}
		if pr := prev.Result(builder, ""); pr != nil && pr.OK {
			c.Debugf("commit broken: %#v\nresult: %#v", com, cr)
			c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr)
			broken = com
		}
	}
	var err error
	if broken != nil && !broken.FailNotificationSent {
		c.Infof("%s is broken commit; notifying", broken.Hash)
		notifyLater.Call(c, broken, builder) // add task to queue
		broken.FailNotificationSent = true
		_, err = datastore.Put(c, broken.Key(c), broken)
	}
	return err
}

// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
	t := q.Limit(1).Run(c)
	_, err := t.Next(v)
	if err == datastore.Done {
		err = datastore.ErrNoSuchEntity
	}
	return err
}

var notifyLater = delay.Func("notify", notify)

// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder string) {
	if !updateCL(c, com, builder) {
		// Send a mail notification if the CL can't be found.
		sendFailMail(c, com, builder)
	}
}

// updateCL updates the CL for the given Commit with a failure message
// for the given builder.
func updateCL(c appengine.Context, com *Commit, builder string) bool {
	cl, err := lookupCL(c, com)
	if err != nil {
		c.Errorf("could not find CL for %v: %v", com.Hash, err)
		return false
	}
	res := com.Result(builder, "")
	if res == nil {
		c.Errorf("finding result for %q: %+v", builder, com)
		return false
	}
	url := fmt.Sprintf("%v?cl=%v&brokebuild=%v&log=%v", gobotBase, cl, builder, res.LogHash)
	r, err := urlfetch.Client(c).Post(url, "text/plain", nil)
	if err != nil {
		c.Errorf("could not update CL %v: %v", cl, err)
		return false
	}
	r.Body.Close()
	if r.StatusCode != http.StatusOK {
		c.Errorf("could not update CL %v: %v", cl, r.Status)
		return false
	}
	return true
}

var clURL = regexp.MustCompile(`https://codereview.appspot.com/([0-9]+)`)

// lookupCL consults code.google.com for the full change description for the
// provided Commit, and returns the relevant CL number.
func lookupCL(c appengine.Context, com *Commit) (string, error) {
	url := "https://code.google.com/p/go/source/detail?r=" + com.Hash
	r, err := urlfetch.Client(c).Get(url)
	if err != nil {
		return "", err
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return "", fmt.Errorf("retrieving %v: %v", url, r.Status)
	}
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return "", err
	}
	m := clURL.FindAllSubmatch(b, -1)
	if m == nil {
		return "", errors.New("no CL URL found on changeset page")
	}
	// Return the last visible codereview URL on the page,
	// in case the change description refers to another CL.
	return string(m[len(m)-1][1]), nil
}

var sendFailMailTmpl = template.Must(template.New("notify.txt").
	Funcs(template.FuncMap(tmplFuncs)).
	ParseFiles("build/notify.txt"))

func init() {
	gob.Register(&Commit{}) // for delay
}

// sendFailMail sends a mail notification that the build failed on the
// provided commit and builder.
func sendFailMail(c appengine.Context, com *Commit, builder string) {
	// TODO(adg): handle packages

	// get Result
	r := com.Result(builder, "")
	if r == nil {
		c.Errorf("finding result for %q: %+v", builder, com)
		return
	}

	// get Log
	k := datastore.NewKey(c, "Log", r.LogHash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, k, l); err != nil {
		c.Errorf("finding Log record %v: %v", r.LogHash, err)
		return
	}

	// prepare mail message
	var body bytes.Buffer
	err := sendFailMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "Result": r, "Log": l,
		"Hostname": domain,
	})
	if err != nil {
		c.Errorf("rendering mail template: %v", err)
		return
	}
	subject := fmt.Sprintf("%s broken by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}
<|endoftext|>


<commit_before><commit_msg>fix typo in transer.sh-web<commit_after><|endoftext|>


<commit_before>package changelog

import (
	"os"
	"testing"
)

func TestParseChangelog(t *testing.T) {
	r, err := os.Open("fixtures/changelog.Debian")
	if err != nil {
		t.Fatal("Cannot load fixture, reason", err)
		return
	}
	defer r.Close()

	cl, err := Parse(r, "")
	if err != nil {
		t.Error("unexpected error", err)
		return
	}
	if len(cl) == 0 {
		t.Error("empty changelog")
	}
}

func TestParseChangelogUntil(t *testing.T) {
	r, err := os.Open("fixtures/changelog.Debian")
	if err != nil {
		t.Fatal("Cannot load fixture, reason", err)
		return
	}
	defer r.Close()

	cl, err := Parse(r, "2:1.1.2-2")
	if err != nil {
		t.Fatal("unexpected error", err)
		return
	}
	if len(cl) == 0 {
		t.Fatal("empty changelog")
	}
	want := 1
	got := len(cl)

	if got != want {
		t.Errorf("got %d changelog entries, want %d", got, want)
	}
}
<commit_msg>Add more tests to prepare for TDD bugfix<commit_after>package changelog

import (
	"os"
	"reflect"
	"testing"
)

func TestParseChangelog(t *testing.T) {
	r, err := os.Open("fixtures/changelog.Debian")
	if err != nil {
		t.Fatal("Cannot load fixture, reason", err)
		return
	}
	defer r.Close()

	cl, err := Parse(r, "")
	if err != nil {
		t.Error("unexpected error", err)
		return
	}
	if len(cl) == 0 {
		t.Error("empty changelog")
	}
}

func TestParseChangelogUntil(t *testing.T) {
	r, err := os.Open("fixtures/changelog.Debian")
	if err != nil {
		t.Fatal("Cannot load fixture, reason", err)
		return
	}
	defer r.Close()

	cl, err := Parse(r, "2:1.1.2-2")
	if err != nil {
		t.Fatal("unexpected error", err)
		return
	}
	if len(cl) == 0 {
		t.Fatal("empty changelog")
	}
	want := 1
	got := len(cl)

	if got != want {
		t.Errorf("got %d changelog entries, want %d", got, want)
	}
}

func TestParseVersionLine(t *testing.T) {
	for i, test := range versionLines {
		change := Change{}
		ok := change.parseVersionLine([]byte(test.line))
		if ok && test.bad {
			t.Errorf("%d: did parse %q, but shouldn't", i, test.line)
		} else if !ok && !test.bad {
			t.Errorf("%d: cannot parse %q", i, test.line)
		}
		if ok && !reflect.DeepEqual(test.want, change) {
			t.Errorf("%d: want:\n%v\n, got:\n%v\n", i, test.want, change)
		}
	}
}

var versionLines = []struct {
	line string
	bad  bool
	want Change
}{
	{

		line: "golang (2:1.1.2-2ubuntu1) saucy; urgency=low",
		want: Change{
			Source:  "golang",
			Version: "2:1.1.2-2ubuntu1",
			Dist:    "saucy",
			Urgency: "low",
		},
	},
	{

		line: "golang (2:1.1.2-2ubuntu1) saucy; urgency=low",
		want: Change{
			Source:  "golang",
			Version: "2:1.1.2-2ubuntu1",
			Dist:    "saucy",
			Urgency: "low",
		},
	},
	{

		line: "golang (2:1.1.2-2ubuntu1) ; urgency=low",
		bad:  true,
	},
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/dancannon/gorethink"
)

type Table string

const (
	diffTable   Table = "Diffs"
	commitTable Table = "Commits"

	commitClockIndex = "CommitClockIndex"

	databaseName      = "pachyderm_pfs"
	rethinkAddressEnv = "RETHINK_PORT_28015_TCP_ADDR"
)

func main() {
	dbAddress := fmt.Sprintf("%s:28015", os.Getenv(rethinkAddressEnv))

	session, err := gorethink.Connect(gorethink.ConnectOpts{
		Address: dbAddress,
	})
	if err != nil {
		panic(err)
	}

	getTerm := func(table Table) gorethink.Term {
		return gorethink.DB(databaseName).Table(table)
	}

	if _, err = getTerm(diffTable).ForEach(func(diff gorethink.Term) gorethink.Term {
		clock := diff.Field("Clock")
		fullClock := getTerm(commitTable).GetAllByIndex(commitClockIndex, []interface{}{
			diff.Field("Repo"),
			clock.Field("Branch"),
			clock.Field("Clock"),
		}).Field("FullClock").Nth(0)
		return getTerm(diffTable).Get(diff.Field("ID")).Update(map[string]interface{}{
			"Clock": fullClock,
		}, gorethink.UpdateOpts{
			NonAtomic: true,
		})
	}).RunWrite(session); err != nil {
		panic(err)
	}

	fmt.Println("The migration has completed successfully.")
}
<commit_msg>Make the script safe to run on partially migrated DB<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/dancannon/gorethink"
)

type Table string

const (
	diffTable   Table = "Diffs"
	commitTable Table = "Commits"

	commitClockIndex = "CommitClockIndex"

	databaseName      = "pachyderm_pfs"
	rethinkAddressEnv = "RETHINK_PORT_28015_TCP_ADDR"
)

func main() {
	dbAddress := fmt.Sprintf("%s:28015", os.Getenv(rethinkAddressEnv))

	session, err := gorethink.Connect(gorethink.ConnectOpts{
		Address: dbAddress,
	})
	if err != nil {
		panic(err)
	}

	getTerm := func(table Table) gorethink.Term {
		return gorethink.DB(databaseName).Table(table)
	}

	if _, err = getTerm(diffTable).ForEach(func(diff gorethink.Term) gorethink.Term {
		clock := diff.Field("Clock")
		return gorethink.Branch(
			clock.TypeOf().Eq("OBJECT"),
			getTerm(diffTable).Get(diff.Field("ID")).Update(map[string]interface{}{
				"Clock": getTerm(commitTable).GetAllByIndex(commitClockIndex, []interface{}{
					diff.Field("Repo"),
					clock.Field("Branch"),
					clock.Field("Clock"),
				}).Field("FullClock").Nth(0),
			}, gorethink.UpdateOpts{
				NonAtomic: true,
			}),
			nil,
		)
	}).RunWrite(session); err != nil {
		panic(err)
	}

	fmt.Println("The migration has completed successfully.")
}
<|endoftext|>


<commit_before>package api

import (
	"math/big"

	"gopkg.in/fatih/set.v0"

	"github.com/bytom/blockchain/query"
	"github.com/bytom/consensus/difficulty"
	chainjson "github.com/bytom/encoding/json"
	"github.com/bytom/errors"
	"github.com/bytom/protocol/bc"
	"github.com/bytom/protocol/bc/types"
)

// return best block hash
func (a *API) getBestBlockHash() Response {
	blockHash := map[string]string{"block_hash": a.chain.BestBlockHash().String()}
	return NewSuccessResponse(blockHash)
}

// return current block count
func (a *API) getBlockCount() Response {
	blockHeight := map[string]uint64{"block_count": a.chain.BestBlockHeight()}
	return NewSuccessResponse(blockHeight)
}

// BlockTx is the tx struct for getBlock func
type BlockTx struct {
	ID         bc.Hash                  `json:"id"`
	Version    uint64                   `json:"version"`
	Size       uint64                   `json:"size"`
	TimeRange  uint64                   `json:"time_range"`
	Inputs     []*query.AnnotatedInput  `json:"inputs"`
	Outputs    []*query.AnnotatedOutput `json:"outputs"`
	StatusFail bool                     `json:"status_fail"`
	MuxID      bc.Hash                  `json:"mux_id"`
}

// BlockReq is used to handle getBlock req
type BlockReq struct {
	BlockHeight uint64             `json:"block_height"`
	BlockHash   chainjson.HexBytes `json:"block_hash"`
}

// GetBlockResp is the resp for getBlock api
type GetBlockResp struct {
	Hash                   *bc.Hash   `json:"hash"`
	Size                   uint64     `json:"size"`
	Version                uint64     `json:"version"`
	Height                 uint64     `json:"height"`
	PreviousBlockHash      *bc.Hash   `json:"previous_block_hash"`
	Timestamp              uint64     `json:"timestamp"`
	Nonce                  uint64     `json:"nonce"`
	Bits                   uint64     `json:"bits"`
	Difficulty             string     `json:"difficulty"`
	TransactionsMerkleRoot *bc.Hash   `json:"transaction_merkle_root"`
	TransactionStatusHash  *bc.Hash   `json:"transaction_status_hash"`
	Transactions           []*BlockTx `json:"transactions"`
}

// return block by hash/height
func (a *API) getBlock(ins BlockReq) Response {
	block, err := a.getBlockHelper(ins)
	if err != nil {
		return NewErrorResponse(err)
	}

	blockHash := block.Hash()
	txStatus, err := a.chain.GetTransactionStatus(&blockHash)
	rawBlock, err := block.MarshalText()
	if err != nil {
		return NewErrorResponse(err)
	}

	resp := &GetBlockResp{
		Hash:                   &blockHash,
		Size:                   uint64(len(rawBlock)),
		Version:                block.Version,
		Height:                 block.Height,
		PreviousBlockHash:      &block.PreviousBlockHash,
		Timestamp:              block.Timestamp,
		Nonce:                  block.Nonce,
		Bits:                   block.Bits,
		Difficulty:             difficulty.CalcWork(block.Bits).String(),
		TransactionsMerkleRoot: &block.TransactionsMerkleRoot,
		TransactionStatusHash:  &block.TransactionStatusHash,
		Transactions:           []*BlockTx{},
	}

	for i, orig := range block.Transactions {
		tx := &BlockTx{
			ID:        orig.ID,
			Version:   orig.Version,
			Size:      orig.SerializedSize,
			TimeRange: orig.TimeRange,
			Inputs:    []*query.AnnotatedInput{},
			Outputs:   []*query.AnnotatedOutput{},
		}
		tx.StatusFail, err = txStatus.GetStatus(i)
		if err != nil {
			return NewSuccessResponse(resp)
		}

		resOutID := orig.ResultIds[0]
		resOut, ok := orig.Entries[*resOutID].(*bc.Output)
		if ok {
			tx.MuxID = *resOut.Source.Ref
		} else {
			resRetire, _ := orig.Entries[*resOutID].(*bc.Retirement)
			tx.MuxID = *resRetire.Source.Ref
		}

		for i := range orig.Inputs {
			tx.Inputs = append(tx.Inputs, a.wallet.BuildAnnotatedInput(orig, uint32(i)))
		}
		for i := range orig.Outputs {
			tx.Outputs = append(tx.Outputs, a.wallet.BuildAnnotatedOutput(orig, i))
		}
		resp.Transactions = append(resp.Transactions, tx)
	}
	return NewSuccessResponse(resp)
}

// GetRawBlockResp is resp struct for getRawBlock API
type GetRawBlockResp struct {
	RawBlock *types.Block `json:"raw_block"`
}

func (a *API) getRawBlock(ins BlockReq) Response {
	block, err := a.getBlockHelper(ins)
	if err != nil {
		return NewErrorResponse(err)
	}

	resp := GetRawBlockResp{RawBlock: block}
	return NewSuccessResponse(resp)
}

// GetBlockHeaderResp is resp struct for getBlockHeader API
type GetBlockHeaderResp struct {
	BlockHeader *types.BlockHeader `json:"block_header"`
	Reward      uint64             `json:"reward"`
}

func (a *API) getBlockHeader(ins BlockReq) Response {
	block, err := a.getBlockHelper(ins)
	if err != nil {
		return NewErrorResponse(err)
	}

	resp := &GetBlockHeaderResp{
		BlockHeader: &block.BlockHeader,
		Reward:      block.Transactions[0].Outputs[0].Amount,
	}
	return NewSuccessResponse(resp)
}

func (a *API) getBlockHelper(ins BlockReq) (*types.Block, error) {
	if len(ins.BlockHash) == 32 {
		hash := hexBytesToHash(ins.BlockHash)
		return a.chain.GetBlockByHash(&hash)
	} else {
		return a.chain.GetBlockByHeight(ins.BlockHeight)
	}
}

func hexBytesToHash(hexBytes chainjson.HexBytes) bc.Hash {
	b32 := [32]byte{}
	copy(b32[:], hexBytes)
	return bc.NewHash(b32)
}

// GetDifficultyResp is resp struct for getDifficulty API
type GetDifficultyResp struct {
	BlockHash   *bc.Hash `json:"hash"`
	BlockHeight uint64   `json:"height"`
	Bits        uint64   `json:"bits"`
	Difficulty  string   `json:"difficulty"`
}

func (a *API) getDifficulty(ins BlockReq) Response {
	block, err := a.getBlockHelper(ins)
	if err != nil {
		return NewErrorResponse(err)
	}

	blockHash := block.Hash()
	resp := &GetDifficultyResp{
		BlockHash:   &blockHash,
		BlockHeight: block.Height,
		Bits:        block.Bits,
		Difficulty:  difficulty.CalcWork(block.Bits).String(),
	}
	return NewSuccessResponse(resp)
}

// getHashRateResp is resp struct for getHashRate API
type getHashRateResp struct {
	BlockHash   *bc.Hash `json:"hash"`
	BlockHeight uint64   `json:"height"`
	HashRate    uint64   `json:"hash_rate"`
}

func (a *API) getHashRate(ins BlockReq) Response {
	if len(ins.BlockHash) != 32 && len(ins.BlockHash) != 0 {
		err := errors.New("Block hash format error.")
		return NewErrorResponse(err)
	}
	if ins.BlockHeight == 0 {
		ins.BlockHeight = a.chain.BestBlockHeight()
	}

	block, err := a.getBlockHelper(ins)
	if err != nil {
		return NewErrorResponse(err)
	}

	preBlock, err := a.chain.GetBlockByHash(&block.PreviousBlockHash)
	if err != nil {
		return NewErrorResponse(err)
	}

	diffTime := block.Timestamp - preBlock.Timestamp
	if preBlock.Timestamp >= block.Timestamp {
		diffTime = 1
	}
	hashCount := difficulty.CalcWork(block.Bits)
	hashRate := new(big.Int).Div(hashCount, big.NewInt(int64(diffTime)))

	blockHash := block.Hash()
	resp := &getHashRateResp{
		BlockHash:   &blockHash,
		BlockHeight: block.Height,
		HashRate:    hashRate.Uint64(),
	}
	return NewSuccessResponse(resp)
}

// MerkleBlockReq is used to handle getTxOutProof req
type MerkleBlockReq struct {
	TxIDs     []chainjson.HexBytes `json:"tx_ids"`
	BlockHash chainjson.HexBytes   `json:"block_hash"`
}

// GetMerkleBlockResp is resp struct for GetTxOutProof API
type GetMerkleBlockResp struct {
	BlockHeader  types.BlockHeader `json:"block_header"`
	TxHashes     []*bc.Hash        `json:"tx_hashes"`
	StatusHashes []*bc.Hash        `json:"status_hashes"`
	Flags        []uint32          `json:"flags"`
	MatchedTxIDs []*bc.Hash        `json:"matched_tx_ids"`
}

func (a *API) getMerkleProof(ins MerkleBlockReq) Response {
	blockReq := BlockReq{BlockHash: ins.BlockHash}
	block, err := a.getBlockHelper(blockReq)
	if err != nil {
		return NewErrorResponse(err)
	}

	matchedTxs := getMatchedTx(block.Transactions, ins.TxIDs)
	var matchedTxIDs []*bc.Hash
	for _, tx := range matchedTxs {
		matchedTxIDs = append(matchedTxIDs, &tx.ID)
	}

	hashes, compactFlags := types.GetTxMerkleTreeProof(block.Transactions, matchedTxs)
	flags := make([]uint32, len(compactFlags))
	for i, flag := range compactFlags {
		flags[i] = uint32(flag)
	}

	blockHash := block.Hash()
	statuses, err := a.chain.GetTransactionStatus(&blockHash)
	if err != nil {
		return NewErrorResponse(err)
	}

	statusHashes := types.GetStatusMerkleTreeProof(statuses.VerifyStatus, compactFlags)

	resp := &GetMerkleBlockResp{
		BlockHeader:  block.BlockHeader,
		TxHashes:     hashes,
		StatusHashes: statusHashes,
		Flags:        flags,
		MatchedTxIDs: matchedTxIDs,
	}
	return NewSuccessResponse(resp)
}

func getMatchedTx(txs []*types.Tx, filterTxIDs []chainjson.HexBytes) []*types.Tx {
	txIDSet := set.New()
	for _, txID := range filterTxIDs {
		hash := hexBytesToHash(txID)
		txIDSet.Add(hash.String())
	}

	var matchedTxs []*types.Tx
	for _, tx := range txs {
		hashStr := tx.ID.String()
		if txIDSet.Has(hashStr) {
			matchedTxs = append(matchedTxs, tx)
		}
	}
	return matchedTxs
}
<commit_msg>Edit api (#1544)<commit_after>package api

import (
	"math/big"

	"gopkg.in/fatih/set.v0"

	"github.com/bytom/blockchain/query"
	"github.com/bytom/consensus/difficulty"
	chainjson "github.com/bytom/encoding/json"
	"github.com/bytom/errors"
	"github.com/bytom/protocol/bc"
	"github.com/bytom/protocol/bc/types"
)

// return best block hash
func (a *API) getBestBlockHash() Response {
	blockHash := map[string]string{"block_hash": a.chain.BestBlockHash().String()}
	return NewSuccessResponse(blockHash)
}

// return current block count
func (a *API) getBlockCount() Response {
	blockHeight := map[string]uint64{"block_count": a.chain.BestBlockHeight()}
	return NewSuccessResponse(blockHeight)
}

// BlockTx is the tx struct for getBlock func
type BlockTx struct {
	ID         bc.Hash                  `json:"id"`
	Version    uint64                   `json:"version"`
	Size       uint64                   `json:"size"`
	TimeRange  uint64                   `json:"time_range"`
	Inputs     []*query.AnnotatedInput  `json:"inputs"`
	Outputs    []*query.AnnotatedOutput `json:"outputs"`
	StatusFail bool                     `json:"status_fail"`
	MuxID      bc.Hash                  `json:"mux_id"`
}

// BlockReq is used to handle getBlock req
type BlockReq struct {
	BlockHeight uint64             `json:"block_height"`
	BlockHash   chainjson.HexBytes `json:"block_hash"`
}

// GetBlockResp is the resp for getBlock api
type GetBlockResp struct {
	Hash                   *bc.Hash   `json:"hash"`
	Size                   uint64     `json:"size"`
	Version                uint64     `json:"version"`
	Height                 uint64     `json:"height"`
	PreviousBlockHash      *bc.Hash   `json:"previous_block_hash"`
	Timestamp              uint64     `json:"timestamp"`
	Nonce                  uint64     `json:"nonce"`
	Bits                   uint64     `json:"bits"`
	Difficulty             string     `json:"difficulty"`
	TransactionsMerkleRoot *bc.Hash   `json:"transaction_merkle_root"`
	TransactionStatusHash  *bc.Hash   `json:"transaction_status_hash"`
	Transactions           []*BlockTx `json:"transactions"`
}

// return block by hash/height
func (a *API) getBlock(ins BlockReq) Response {
	block, err := a.getBlockHelper(ins)
	if err != nil {
		return NewErrorResponse(err)
	}

	blockHash := block.Hash()
	txStatus, err := a.chain.GetTransactionStatus(&blockHash)
	rawBlock, err := block.MarshalText()
	if err != nil {
		return NewErrorResponse(err)
	}

	resp := &GetBlockResp{
		Hash:                   &blockHash,
		Size:                   uint64(len(rawBlock)),
		Version:                block.Version,
		Height:                 block.Height,
		PreviousBlockHash:      &block.PreviousBlockHash,
		Timestamp:              block.Timestamp,
		Nonce:                  block.Nonce,
		Bits:                   block.Bits,
		Difficulty:             difficulty.CalcWork(block.Bits).String(),
		TransactionsMerkleRoot: &block.TransactionsMerkleRoot,
		TransactionStatusHash:  &block.TransactionStatusHash,
		Transactions:           []*BlockTx{},
	}

	for i, orig := range block.Transactions {
		tx := &BlockTx{
			ID:        orig.ID,
			Version:   orig.Version,
			Size:      orig.SerializedSize,
			TimeRange: orig.TimeRange,
			Inputs:    []*query.AnnotatedInput{},
			Outputs:   []*query.AnnotatedOutput{},
		}
		tx.StatusFail, err = txStatus.GetStatus(i)
		if err != nil {
			return NewSuccessResponse(resp)
		}

		resOutID := orig.ResultIds[0]
		resOut, ok := orig.Entries[*resOutID].(*bc.Output)
		if ok {
			tx.MuxID = *resOut.Source.Ref
		} else {
			resRetire, _ := orig.Entries[*resOutID].(*bc.Retirement)
			tx.MuxID = *resRetire.Source.Ref
		}

		for i := range orig.Inputs {
			tx.Inputs = append(tx.Inputs, a.wallet.BuildAnnotatedInput(orig, uint32(i)))
		}
		for i := range orig.Outputs {
			tx.Outputs = append(tx.Outputs, a.wallet.BuildAnnotatedOutput(orig, i))
		}
		resp.Transactions = append(resp.Transactions, tx)
	}
	return NewSuccessResponse(resp)
}

// GetRawBlockResp is resp struct for getRawBlock API
type GetRawBlockResp struct {
	RawBlock          *types.Block          `json:"raw_block"`
	TransactionStatus *bc.TransactionStatus `json:"transaction_status"`
}

func (a *API) getRawBlock(ins BlockReq) Response {
	block, err := a.getBlockHelper(ins)
	if err != nil {
		return NewErrorResponse(err)
	}

	blockHash := block.Hash()
	txStatus, err := a.chain.GetTransactionStatus(&blockHash)
	if err != nil {
		return NewErrorResponse(err)
	}

	resp := GetRawBlockResp{
		RawBlock:          block,
		TransactionStatus: txStatus,
	}
	return NewSuccessResponse(resp)
}

// GetBlockHeaderResp is resp struct for getBlockHeader API
type GetBlockHeaderResp struct {
	BlockHeader *types.BlockHeader `json:"block_header"`
	Reward      uint64             `json:"reward"`
}

func (a *API) getBlockHeader(ins BlockReq) Response {
	block, err := a.getBlockHelper(ins)
	if err != nil {
		return NewErrorResponse(err)
	}

	resp := &GetBlockHeaderResp{
		BlockHeader: &block.BlockHeader,
		Reward:      block.Transactions[0].Outputs[0].Amount,
	}
	return NewSuccessResponse(resp)
}

func (a *API) getBlockHelper(ins BlockReq) (*types.Block, error) {
	if len(ins.BlockHash) == 32 {
		hash := hexBytesToHash(ins.BlockHash)
		return a.chain.GetBlockByHash(&hash)
	} else {
		return a.chain.GetBlockByHeight(ins.BlockHeight)
	}
}

func hexBytesToHash(hexBytes chainjson.HexBytes) bc.Hash {
	b32 := [32]byte{}
	copy(b32[:], hexBytes)
	return bc.NewHash(b32)
}

// GetDifficultyResp is resp struct for getDifficulty API
type GetDifficultyResp struct {
	BlockHash   *bc.Hash `json:"hash"`
	BlockHeight uint64   `json:"height"`
	Bits        uint64   `json:"bits"`
	Difficulty  string   `json:"difficulty"`
}

func (a *API) getDifficulty(ins BlockReq) Response {
	block, err := a.getBlockHelper(ins)
	if err != nil {
		return NewErrorResponse(err)
	}

	blockHash := block.Hash()
	resp := &GetDifficultyResp{
		BlockHash:   &blockHash,
		BlockHeight: block.Height,
		Bits:        block.Bits,
		Difficulty:  difficulty.CalcWork(block.Bits).String(),
	}
	return NewSuccessResponse(resp)
}

// getHashRateResp is resp struct for getHashRate API
type getHashRateResp struct {
	BlockHash   *bc.Hash `json:"hash"`
	BlockHeight uint64   `json:"height"`
	HashRate    uint64   `json:"hash_rate"`
}

func (a *API) getHashRate(ins BlockReq) Response {
	if len(ins.BlockHash) != 32 && len(ins.BlockHash) != 0 {
		err := errors.New("Block hash format error.")
		return NewErrorResponse(err)
	}
	if ins.BlockHeight == 0 {
		ins.BlockHeight = a.chain.BestBlockHeight()
	}

	block, err := a.getBlockHelper(ins)
	if err != nil {
		return NewErrorResponse(err)
	}

	preBlock, err := a.chain.GetBlockByHash(&block.PreviousBlockHash)
	if err != nil {
		return NewErrorResponse(err)
	}

	diffTime := block.Timestamp - preBlock.Timestamp
	if preBlock.Timestamp >= block.Timestamp {
		diffTime = 1
	}
	hashCount := difficulty.CalcWork(block.Bits)
	hashRate := new(big.Int).Div(hashCount, big.NewInt(int64(diffTime)))

	blockHash := block.Hash()
	resp := &getHashRateResp{
		BlockHash:   &blockHash,
		BlockHeight: block.Height,
		HashRate:    hashRate.Uint64(),
	}
	return NewSuccessResponse(resp)
}

// MerkleBlockReq is used to handle getTxOutProof req
type MerkleBlockReq struct {
	TxIDs     []chainjson.HexBytes `json:"tx_ids"`
	BlockHash chainjson.HexBytes   `json:"block_hash"`
}

// GetMerkleBlockResp is resp struct for GetTxOutProof API
type GetMerkleBlockResp struct {
	BlockHeader  types.BlockHeader `json:"block_header"`
	TxHashes     []*bc.Hash        `json:"tx_hashes"`
	StatusHashes []*bc.Hash        `json:"status_hashes"`
	Flags        []uint32          `json:"flags"`
	MatchedTxIDs []*bc.Hash        `json:"matched_tx_ids"`
}

func (a *API) getMerkleProof(ins MerkleBlockReq) Response {
	blockReq := BlockReq{BlockHash: ins.BlockHash}
	block, err := a.getBlockHelper(blockReq)
	if err != nil {
		return NewErrorResponse(err)
	}

	matchedTxs := getMatchedTx(block.Transactions, ins.TxIDs)
	var matchedTxIDs []*bc.Hash
	for _, tx := range matchedTxs {
		matchedTxIDs = append(matchedTxIDs, &tx.ID)
	}

	hashes, compactFlags := types.GetTxMerkleTreeProof(block.Transactions, matchedTxs)
	flags := make([]uint32, len(compactFlags))
	for i, flag := range compactFlags {
		flags[i] = uint32(flag)
	}

	blockHash := block.Hash()
	statuses, err := a.chain.GetTransactionStatus(&blockHash)
	if err != nil {
		return NewErrorResponse(err)
	}

	statusHashes := types.GetStatusMerkleTreeProof(statuses.VerifyStatus, compactFlags)

	resp := &GetMerkleBlockResp{
		BlockHeader:  block.BlockHeader,
		TxHashes:     hashes,
		StatusHashes: statusHashes,
		Flags:        flags,
		MatchedTxIDs: matchedTxIDs,
	}
	return NewSuccessResponse(resp)
}

func getMatchedTx(txs []*types.Tx, filterTxIDs []chainjson.HexBytes) []*types.Tx {
	txIDSet := set.New()
	for _, txID := range filterTxIDs {
		hash := hexBytesToHash(txID)
		txIDSet.Add(hash.String())
	}

	var matchedTxs []*types.Tx
	for _, tx := range txs {
		hashStr := tx.ID.String()
		if txIDSet.Has(hashStr) {
			matchedTxs = append(matchedTxs, tx)
		}
	}
	return matchedTxs
}
<|endoftext|>


<commit_before>package linkedlist

import (
	"bytes"
	"fmt"
	"testing"
)

func TestNew(t *testing.T) {
	for _, tc := range newListTestCases {
		t.Run(tc.name, func(t *testing.T) {
			actual := NewList(tc.in...)

			checkDoublyLinkedList(t, actual, tc.expected)
		})
	}
}

func TestReverse(t *testing.T) {
	for _, tc := range reverseTestCases {
		t.Run(tc.name, func(t *testing.T) {
			actual := NewList(tc.in...)
			actual.Reverse()

			checkDoublyLinkedList(t, actual, tc.expected)
		})
	}
}

func TestPushPop(t *testing.T) {
	for _, tc := range pushPopTestCases {
		t.Run(tc.name, func(t *testing.T) {
			actual := NewList(tc.in...)
			for _, ac := range tc.actions {
				ac(t, actual)
			}

			checkDoublyLinkedList(t, actual, tc.expected)
		})
	}
}

// checkDoublyLinkedList checks that the linked list is constructed correctly.
func checkDoublyLinkedList(t *testing.T, ll *List, expected []interface{}) {
	// check that length and elements are correct (scan once from begin -> end)
	elem, count, idx := ll.First(), 0, 0
	for ; elem != nil && idx < len(expected); elem, count, idx = elem.Next(), count+1, idx+1 {
		if elem.Val != expected[idx] {
			t.Errorf("wrong value from %d-th element, expected= %v, got= %v", idx, expected[idx], elem.Val)
		}
	}
	if !(elem == nil && idx == len(expected)) {
		t.Errorf("expected %d elements, got= %d", len(expected), count)
	}

	// if elements are the same, we also need to examine the links (next & prev)
	switch {
	case ll.First() == nil && ll.Last() == nil: // empty list
		return
	case ll.First() != nil && ll.Last() != nil && ll.First().Next() == nil: // 1 element
		valid := ll.First() == ll.Last() &&
			ll.First().Next() == nil &&
			ll.First().Prev() == nil &&
			ll.Last().Next() == nil &&
			ll.Last().Prev() == nil

		if !valid {
			t.Errorf("expected to only have 1 element and no links, got= %v", ll.debugString())
		}
	}

	// >1 element
	if ll.First().Prev() != nil {
		t.Errorf("expected Head.Prev() == nil, got= %v", ll.First().Prev())
	}

	prev := ll.First()
	cur := ll.First().Next()
	for idx := 0; cur != nil; idx++ {
		if !(prev.Next() == cur && cur.Prev() == prev) {
			t.Errorf("%d-th element's links is wrong", idx)
		}

		prev = cur
		cur = cur.Next()
	}

	if ll.Last().Next() != nil {
		t.Errorf("expected Last().Next() == nil, got= %v", ll.Last().Next())
	}
}

// debugString prints the linked list with both node's Val, next & prev pointers.
func (ll *List) debugString() string {
	buf := bytes.NewBuffer([]byte{'{'})
	buf.WriteString(fmt.Sprintf("First()= %p; ", ll.First()))

	for cur := ll.First(); cur != nil; cur = cur.Next() {
		buf.WriteString(fmt.Sprintf("[Prev()= %p, Val= %p (%v), Next()= %p] <-> ", cur.Prev(), cur, cur.Val, cur.Next()))
	}

	buf.WriteString(fmt.Sprintf("; Last()= %p; ", ll.Last()))
	buf.WriteByte('}')

	return buf.String()
}
<commit_msg>linked-list: added example API covered by tests (#1234)<commit_after>// API:
//
// type Element struct
// type List struct
// var ErrEmptyList
//
// func (e *Element) Next() *Element
// func (e *Element) Prev() *Element
// func NewList(args ...interface{}) *List
// func (l *List) PushFront(v interface{})
// func (l *List) PushBack(v interface{})
// func (l *List) PopFront() (interface{}, error)
// func (l *List) PopBack() (interface{}, error)
// func (l *List) Reverse() *List
// func (l *List) First() *Element
// func (l *List) Last() *Element
package linkedlist

import (
	"bytes"
	"fmt"
	"testing"
)

func TestNew(t *testing.T) {
	for _, tc := range newListTestCases {
		t.Run(tc.name, func(t *testing.T) {
			actual := NewList(tc.in...)

			checkDoublyLinkedList(t, actual, tc.expected)
		})
	}
}

func TestReverse(t *testing.T) {
	for _, tc := range reverseTestCases {
		t.Run(tc.name, func(t *testing.T) {
			actual := NewList(tc.in...)
			actual.Reverse()

			checkDoublyLinkedList(t, actual, tc.expected)
		})
	}
}

func TestPushPop(t *testing.T) {
	for _, tc := range pushPopTestCases {
		t.Run(tc.name, func(t *testing.T) {
			actual := NewList(tc.in...)
			for _, ac := range tc.actions {
				ac(t, actual)
			}

			checkDoublyLinkedList(t, actual, tc.expected)
		})
	}
}

// checkDoublyLinkedList checks that the linked list is constructed correctly.
func checkDoublyLinkedList(t *testing.T, ll *List, expected []interface{}) {
	// check that length and elements are correct (scan once from begin -> end)
	elem, count, idx := ll.First(), 0, 0
	for ; elem != nil && idx < len(expected); elem, count, idx = elem.Next(), count+1, idx+1 {
		if elem.Val != expected[idx] {
			t.Errorf("wrong value from %d-th element, expected= %v, got= %v", idx, expected[idx], elem.Val)
		}
	}
	if !(elem == nil && idx == len(expected)) {
		t.Errorf("expected %d elements, got= %d", len(expected), count)
	}

	// if elements are the same, we also need to examine the links (next & prev)
	switch {
	case ll.First() == nil && ll.Last() == nil: // empty list
		return
	case ll.First() != nil && ll.Last() != nil && ll.First().Next() == nil: // 1 element
		valid := ll.First() == ll.Last() &&
			ll.First().Next() == nil &&
			ll.First().Prev() == nil &&
			ll.Last().Next() == nil &&
			ll.Last().Prev() == nil

		if !valid {
			t.Errorf("expected to only have 1 element and no links, got= %v", ll.debugString())
		}
	}

	// >1 element
	if ll.First().Prev() != nil {
		t.Errorf("expected Head.Prev() == nil, got= %v", ll.First().Prev())
	}

	prev := ll.First()
	cur := ll.First().Next()
	for idx := 0; cur != nil; idx++ {
		if !(prev.Next() == cur && cur.Prev() == prev) {
			t.Errorf("%d-th element's links is wrong", idx)
		}

		prev = cur
		cur = cur.Next()
	}

	if ll.Last().Next() != nil {
		t.Errorf("expected Last().Next() == nil, got= %v", ll.Last().Next())
	}
}

// debugString prints the linked list with both node's Val, next & prev pointers.
func (ll *List) debugString() string {
	buf := bytes.NewBuffer([]byte{'{'})
	buf.WriteString(fmt.Sprintf("First()= %p; ", ll.First()))

	for cur := ll.First(); cur != nil; cur = cur.Next() {
		buf.WriteString(fmt.Sprintf("[Prev()= %p, Val= %p (%v), Next()= %p] <-> ", cur.Prev(), cur, cur.Val, cur.Next()))
	}

	buf.WriteString(fmt.Sprintf("; Last()= %p; ", ll.Last()))
	buf.WriteByte('}')

	return buf.String()
}
<|endoftext|>


<commit_before>package paxos

type network interface {
	send(msg message)
}

type proposal struct {
	seq  int
	data []byte
}

func (p proposal) lessThan(a proposal) bool { return p.seq < a.seq }

func (p proposal) isEmpty() bool { return p.seq == 0 }

type fsm struct {
	id    int
	peers []int
	nt    network

	propSeq  int
	promises int
	reported proposal
	accepts  int
	chosen   proposal

	// these two fields need to be persisted to stable storage
	promiseSeq int
	accepted   proposal
}

func (sm *fsm) recv(msg message) {
	switch msg.typ {
	case msgPrepare:
		sm.handlePrepare(msg)
	case msgPromise:
		sm.handlePromise(msg)
	case msgAcceptRequest:
		sm.handleAcceptRequest(msg)
	case msgAccepted:
		sm.handleAccepted(msg)
	}
}

func (sm *fsm) propose(seq int, data []byte) {
	if sm.propSeq >= seq {
		panic("expect higher seq")
	}
	sm.propSeq = seq
	sm.promises = 0
	sm.reported = proposal{data: data}
	sm.bcast(message{
		typ: msgPrepare,
		seq: sm.propSeq,
	})
}

func (sm *fsm) chosenValue() ([]byte, bool) {
	if sm.chosen.isEmpty() {
		return nil, false
	} else {
		return sm.chosen.data, true
	}
}

func (sm *fsm) handlePrepare(msg message) {
	if msg.typ != msgPrepare {
		panic("invalid message type")
	}
	if sm.promiseSeq >= msg.seq {
		return
	}
	sm.promiseSeq = msg.seq
	sm.send(msg.from, message{
		typ:  msgPromise,
		seq:  msg.seq,
		prop: sm.accepted,
	})
}

func (sm *fsm) handlePromise(msg message) {
	if msg.typ != msgPromise {
		panic("invalid message type")
	}
	if sm.propSeq != msg.seq {
		return
	}
	sm.promises++
	if sm.reported.lessThan(msg.prop) {
		sm.reported = msg.prop
	}

	if sm.promises != sm.quorum() {
		return
	}
	sm.reported.seq = sm.propSeq
	sm.bcast(message{
		typ:  msgAcceptRequest,
		seq:  sm.propSeq,
		prop: sm.reported,
	})
}

func (sm *fsm) handleAcceptRequest(msg message) {
	if msg.typ != msgAcceptRequest {
		panic("invalid message type")
	}
	if sm.promiseSeq > msg.seq {
		return
	}
	sm.accepted = msg.prop
	sm.send(msg.from, message{
		typ: msgAccepted,
		seq: msg.seq,
	})
}

func (sm *fsm) handleAccepted(msg message) {
	if msg.typ != msgAccepted {
		panic("invalid message type")
	}
	if sm.propSeq != msg.seq {
		return
	}
	sm.accepts++
	if sm.accepts != sm.quorum() {
		return
	}
	sm.chosen = sm.reported
}

func (sm *fsm) bcast(msg message) {
	for _, to := range sm.peers {
		sm.send(to, msg)
	}
}

func (sm *fsm) send(to int, msg message) {
	msg.from = sm.id
	msg.to = to
	sm.nt.send(msg)
}

func (sm *fsm) quorum() int { return len(sm.peers)/2 + 1 }
<commit_msg>add reboot and treat everyone as learner<commit_after>package paxos

import (
	"log"
	"reflect"
)

type network interface {
	send(msg message)
}

type proposal struct {
	seq  int
	data []byte
}

func (p proposal) lessThan(a proposal) bool { return p.seq < a.seq }

func (p proposal) isEmpty() bool { return p.seq == 0 }

type fsm struct {
	id    int
	peers []int
	nt    network

	// proposer
	propSeq  int
	promises int
	reported proposal

	// learner
	acceptSeq int
	accepts   int
	learning  proposal
	chosen    proposal

	// acceptor
	// these two fields need to be persisted to stable storage
	promiseSeq int
	accepted   proposal
}

func (sm *fsm) recv(msg message) {
	switch msg.typ {
	case msgPrepare:
		sm.handlePrepare(msg)
	case msgPromise:
		sm.handlePromise(msg)
	case msgAcceptRequest:
		sm.handleAcceptRequest(msg)
	case msgAccepted:
		sm.handleAccepted(msg)
	}
}

func (sm *fsm) propose(seq int, data []byte) {
	if sm.propSeq >= seq {
		panic("expect higher seq")
	}
	sm.propSeq = seq
	sm.promises = 0
	sm.reported = proposal{data: data}
	sm.bcast(message{
		typ: msgPrepare,
		seq: sm.propSeq,
	})
}

func (sm *fsm) chosenValue() ([]byte, bool) {
	if sm.chosen.isEmpty() {
		return nil, false
	} else {
		return sm.chosen.data, true
	}
}

func (sm *fsm) reboot() *fsm {
	return &fsm{
		id:         sm.id,
		peers:      sm.peers,
		nt:         sm.nt,
		promiseSeq: sm.promiseSeq,
		accepted:   sm.accepted,
	}
}

func (sm *fsm) handlePrepare(msg message) {
	if msg.typ != msgPrepare {
		panic("invalid message type")
	}
	if sm.promiseSeq >= msg.seq {
		return
	}
	sm.promiseSeq = msg.seq
	sm.send(msg.from, message{
		typ:  msgPromise,
		seq:  msg.seq,
		prop: sm.accepted,
	})
}

func (sm *fsm) handlePromise(msg message) {
	if msg.typ != msgPromise {
		panic("invalid message type")
	}
	if sm.propSeq != msg.seq {
		return
	}
	sm.promises++
	if sm.reported.lessThan(msg.prop) {
		sm.reported = msg.prop
	}

	if sm.promises != sm.quorum() {
		return
	}
	sm.reported.seq = sm.propSeq
	sm.bcast(message{
		typ:  msgAcceptRequest,
		seq:  sm.propSeq,
		prop: sm.reported,
	})
}

func (sm *fsm) handleAcceptRequest(msg message) {
	if msg.typ != msgAcceptRequest {
		panic("invalid message type")
	}
	if sm.promiseSeq > msg.seq {
		return
	}
	sm.accepted = msg.prop
	// TODO: distinguished learner
	sm.bcast(message{
		typ:  msgAccepted,
		seq:  msg.seq,
		prop: sm.accepted,
	})
}

func (sm *fsm) handleAccepted(msg message) {
	if msg.typ != msgAccepted {
		panic("invalid message type")
	}
	if sm.acceptSeq > msg.seq {
		return
	} else if sm.acceptSeq < msg.seq {
		sm.acceptSeq = msg.seq
		sm.accepts = 1
		sm.learning = msg.prop
	} else {
		sm.accepts++
		if !reflect.DeepEqual(sm.learning.data, msg.prop.data) {
			log.Panicf("different acceptor accepts different value (%q, %q) at seq %d", sm.learning.data, msg.prop.data, sm.acceptSeq)
		}
	}
	if sm.accepts != sm.quorum() {
		return
	}
	if !sm.chosen.isEmpty() && !reflect.DeepEqual(sm.chosen.data, sm.learning.data) {
		log.Panicf("have chosen value %q while new value %q is accepted", sm.chosen.data, sm.learning.data)
	}
	sm.chosen = sm.learning
}

func (sm *fsm) bcast(msg message) {
	for _, to := range sm.peers {
		sm.send(to, msg)
	}
}

func (sm *fsm) send(to int, msg message) {
	msg.from = sm.id
	msg.to = to
	sm.nt.send(msg)
}

func (sm *fsm) quorum() int { return len(sm.peers)/2 + 1 }
<|endoftext|>