<commit_before>// Copyright 2014 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"errors"
	"fmt"
	"math"
	"sync/atomic"

	dto "github.com/prometheus/client_model/go"
)

// Counter is a Metric that represents a single numerical value that only ever
// goes up. That implies that it cannot be used to count items whose number can
// also go down, e.g. the number of currently running goroutines. Those
// "counters" are represented by Gauges.
//
// A Counter is typically used to count requests served, tasks completed, errors
// occurred, etc.
//
// To create Counter instances, use NewCounter.
type Counter interface {
	Metric
	Collector

	// Inc increments the counter by 1. Use Add to increment it by arbitrary
	// non-negative values.
	Inc()
	// Add adds the given value to the counter. It panics if the value is <
	// 0.
	Add(float64)
}

// CounterOpts is an alias for Opts. See there for doc comments.
type CounterOpts Opts

// NewCounter creates a new Counter based on the provided CounterOpts.
//
// The returned implementation tracks the counter value in two separate
// variables, a float64 and a uint64. The latter is used to track calls of the
// Inc method and calls of the Add method with a value that can be represented
// as a uint64. This allows atomic increments of the counter with optimal
// performance. (It is common to have an Inc call in very hot execution paths.)
// Both internal tracking values are added up in the Write method. This has to
// be taken into account when it comes to precision and overflow behavior.
func NewCounter(opts CounterOpts) Counter {
	desc := NewDesc(
		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
		opts.Help,
		nil,
		opts.ConstLabels,
	)
	result := &counter{desc: desc, labelPairs: desc.constLabelPairs}
	result.init(result) // Init self-collection.
	return result
}

type counter struct {
	// valBits contains the bits of the represented float64 value, while
	// valInt stores values that are exact integers. Both have to go first
	// in the struct to guarantee alignment for atomic operations.
	// http://golang.org/pkg/sync/atomic/#pkg-note-BUG
	valBits uint64
	valInt  uint64

	selfCollector
	desc *Desc

	labelPairs []*dto.LabelPair
}

func (c *counter) Desc() *Desc {
	return c.desc
}

func (c *counter) Add(v float64) {
	if v < 0 {
		panic(errors.New("counter cannot decrease in value"))
	}
	ival := uint64(v)
	if float64(ival) == v {
		atomic.AddUint64(&c.valInt, ival)
		return
	}

	for {
		oldBits := atomic.LoadUint64(&c.valBits)
		newBits := math.Float64bits(math.Float64frombits(oldBits) + v)
		if atomic.CompareAndSwapUint64(&c.valBits, oldBits, newBits) {
			return
		}
	}
}

func (c *counter) Inc() {
	atomic.AddUint64(&c.valInt, 1)
}

func (c *counter) Write(out *dto.Metric) error {
	fval := math.Float64frombits(atomic.LoadUint64(&c.valBits))
	ival := atomic.LoadUint64(&c.valInt)
	val := fval + float64(ival)

	return populateMetric(CounterValue, val, c.labelPairs, out)
}

// CounterVec is a Collector that bundles a set of Counters that all share the
// same Desc, but have different values for their variable labels. This is used
// if you want to count the same thing partitioned by various dimensions
// (e.g. number of HTTP requests, partitioned by response code and
// method). Create instances with NewCounterVec.
type CounterVec struct {
	*metricVec
}

// NewCounterVec creates a new CounterVec based on the provided CounterOpts and
// partitioned by the given label names.
func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec {
	desc := NewDesc(
		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
		opts.Help,
		labelNames,
		opts.ConstLabels,
	)
	return &CounterVec{
		metricVec: newMetricVec(desc, func(lvs ...string) Metric {
			if len(lvs) != len(desc.variableLabels) {
				panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, lvs))
			}
			result := &counter{desc: desc, labelPairs: makeLabelPairs(desc, lvs)}
			result.init(result) // Init self-collection.
			return result
		}),
	}
}

// GetMetricWithLabelValues returns the Counter for the given slice of label
// values (same order as the VariableLabels in Desc). If that combination of
// label values is accessed for the first time, a new Counter is created.
//
// It is possible to call this method without using the returned Counter to only
// create the new Counter but leave it at its starting value 0. See also the
// SummaryVec example.
//
// Keeping the Counter for later use is possible (and should be considered if
// performance is critical), but keep in mind that Reset, DeleteLabelValues and
// Delete can be used to delete the Counter from the CounterVec. In that case,
// the Counter will still exist, but it will not be exported anymore, even if a
// Counter with the same label values is created later.
//
// An error is returned if the number of label values is not the same as the
// number of VariableLabels in Desc (minus any curried labels).
//
// Note that for more than one label value, this method is prone to mistakes
// caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as
// an alternative to avoid that type of mistake. For higher label numbers, the
// latter has a much more readable (albeit more verbose) syntax, but it comes
// with a performance overhead (for creating and processing the Labels map).
// See also the GaugeVec example.
func (v *CounterVec) GetMetricWithLabelValues(lvs ...string) (Counter, error) {
	metric, err := v.metricVec.getMetricWithLabelValues(lvs...)
	if metric != nil {
		return metric.(Counter), err
	}
	return nil, err
}

// GetMetricWith returns the Counter for the given Labels map (the label names
// must match those of the VariableLabels in Desc). If that label map is
// accessed for the first time, a new Counter is created. Implications of
// creating a Counter without using it and keeping the Counter for later use are
// the same as for GetMetricWithLabelValues.
//
// An error is returned if the number and names of the Labels are inconsistent
// with those of the VariableLabels in Desc (minus any curried labels).
//
// This method is used for the same purpose as
// GetMetricWithLabelValues(...string). See there for pros and cons of the two
// methods.
func (v *CounterVec) GetMetricWith(labels Labels) (Counter, error) {
	metric, err := v.metricVec.getMetricWith(labels)
	if metric != nil {
		return metric.(Counter), err
	}
	return nil, err
}

// WithLabelValues works as GetMetricWithLabelValues, but panics where
// GetMetricWithLabelValues would have returned an error. Not returning an
// error allows shortcuts like
//     myVec.WithLabelValues("404", "GET").Add(42)
func (v *CounterVec) WithLabelValues(lvs ...string) Counter {
	c, err := v.GetMetricWithLabelValues(lvs...)
	if err != nil {
		panic(err)
	}
	return c
}

// With works as GetMetricWith, but panics where GetMetricWithLabels would have
// returned an error. Not returning an error allows shortcuts like
//     myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Add(42)
func (v *CounterVec) With(labels Labels) Counter {
	c, err := v.GetMetricWith(labels)
	if err != nil {
		panic(err)
	}
	return c
}

// CurryWith returns a vector curried with the provided labels, i.e. the
// returned vector has those labels pre-set for all labeled operations performed
// on it. The cardinality of the curried vector is reduced accordingly. The
// order of the remaining labels stays the same (just with the curried labels
// taken out of the sequence – which is relevant for the
// (GetMetric)WithLabelValues methods). It is possible to curry a curried
// vector, but only with labels not yet used for currying before.
//
// The metrics contained in the CounterVec are shared between the curried and
// uncurried vectors. They are just accessed differently. Curried and uncurried
// vectors behave identically in terms of collection. Only one must be
// registered with a given registry (usually the uncurried version). The Reset
// method deletes all metrics, even if called on a curried vector.
func (v *CounterVec) CurryWith(labels Labels) (*CounterVec, error) {
	vec, err := v.curryWith(labels)
	if vec != nil {
		return &CounterVec{vec}, err
	}
	return nil, err
}

// MustCurryWith works as CurryWith but panics where CurryWith would have
// returned an error.
func (v *CounterVec) MustCurryWith(labels Labels) *CounterVec {
	vec, err := v.CurryWith(labels)
	if err != nil {
		panic(err)
	}
	return vec
}

// CounterFunc is a Counter whose value is determined at collect time by calling a
// provided function.
//
// To create CounterFunc instances, use NewCounterFunc.
type CounterFunc interface {
	Metric
	Collector
}

// NewCounterFunc creates a new CounterFunc based on the provided
// CounterOpts. The value reported is determined by calling the given function
// from within the Write method. Take into account that metric collection may
// happen concurrently. If that results in concurrent calls to Write, like in
// the case where a CounterFunc is directly registered with Prometheus, the
// provided function must be concurrency-safe. The function should also honor
// the contract for a Counter (values only go up, not down), but compliance will
// not be checked.
func NewCounterFunc(opts CounterOpts, function func() float64) CounterFunc {
	return newValueFunc(NewDesc(
		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
		opts.Help,
		nil,
		opts.ConstLabels,
	), CounterValue, function)
}
<commit_msg>Remove fmt from import<commit_after>// Copyright 2014 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"errors"
	"math"
	"sync/atomic"

	dto "github.com/prometheus/client_model/go"
)

// Counter is a Metric that represents a single numerical value that only ever
// goes up. That implies that it cannot be used to count items whose number can
// also go down, e.g. the number of currently running goroutines. Those
// "counters" are represented by Gauges.
//
// A Counter is typically used to count requests served, tasks completed, errors
// occurred, etc.
//
// To create Counter instances, use NewCounter.
type Counter interface {
	Metric
	Collector

	// Inc increments the counter by 1. Use Add to increment it by arbitrary
	// non-negative values.
	Inc()
	// Add adds the given value to the counter. It panics if the value is <
	// 0.
	Add(float64)
}

// CounterOpts is an alias for Opts. See there for doc comments.
type CounterOpts Opts

// NewCounter creates a new Counter based on the provided CounterOpts.
//
// The returned implementation tracks the counter value in two separate
// variables, a float64 and a uint64. The latter is used to track calls of the
// Inc method and calls of the Add method with a value that can be represented
// as a uint64. This allows atomic increments of the counter with optimal
// performance. (It is common to have an Inc call in very hot execution paths.)
// Both internal tracking values are added up in the Write method. This has to
// be taken into account when it comes to precision and overflow behavior.
func NewCounter(opts CounterOpts) Counter {
	desc := NewDesc(
		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
		opts.Help,
		nil,
		opts.ConstLabels,
	)
	result := &counter{desc: desc, labelPairs: desc.constLabelPairs}
	result.init(result) // Init self-collection.
	return result
}

type counter struct {
	// valBits contains the bits of the represented float64 value, while
	// valInt stores values that are exact integers. Both have to go first
	// in the struct to guarantee alignment for atomic operations.
	// http://golang.org/pkg/sync/atomic/#pkg-note-BUG
	valBits uint64
	valInt  uint64

	selfCollector
	desc *Desc

	labelPairs []*dto.LabelPair
}

func (c *counter) Desc() *Desc {
	return c.desc
}

func (c *counter) Add(v float64) {
	if v < 0 {
		panic(errors.New("counter cannot decrease in value"))
	}
	ival := uint64(v)
	if float64(ival) == v {
		atomic.AddUint64(&c.valInt, ival)
		return
	}

	for {
		oldBits := atomic.LoadUint64(&c.valBits)
		newBits := math.Float64bits(math.Float64frombits(oldBits) + v)
		if atomic.CompareAndSwapUint64(&c.valBits, oldBits, newBits) {
			return
		}
	}
}

func (c *counter) Inc() {
	atomic.AddUint64(&c.valInt, 1)
}

func (c *counter) Write(out *dto.Metric) error {
	fval := math.Float64frombits(atomic.LoadUint64(&c.valBits))
	ival := atomic.LoadUint64(&c.valInt)
	val := fval + float64(ival)

	return populateMetric(CounterValue, val, c.labelPairs, out)
}

// CounterVec is a Collector that bundles a set of Counters that all share the
// same Desc, but have different values for their variable labels. This is used
// if you want to count the same thing partitioned by various dimensions
// (e.g. number of HTTP requests, partitioned by response code and
// method). Create instances with NewCounterVec.
type CounterVec struct {
	*metricVec
}

// NewCounterVec creates a new CounterVec based on the provided CounterOpts and
// partitioned by the given label names.
func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec {
	desc := NewDesc(
		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
		opts.Help,
		labelNames,
		opts.ConstLabels,
	)
	return &CounterVec{
		metricVec: newMetricVec(desc, func(lvs ...string) Metric {
			if len(lvs) != len(desc.variableLabels) {
				panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, lvs))
			}
			result := &counter{desc: desc, labelPairs: makeLabelPairs(desc, lvs)}
			result.init(result) // Init self-collection.
			return result
		}),
	}
}

// GetMetricWithLabelValues returns the Counter for the given slice of label
// values (same order as the VariableLabels in Desc). If that combination of
// label values is accessed for the first time, a new Counter is created.
//
// It is possible to call this method without using the returned Counter to only
// create the new Counter but leave it at its starting value 0. See also the
// SummaryVec example.
//
// Keeping the Counter for later use is possible (and should be considered if
// performance is critical), but keep in mind that Reset, DeleteLabelValues and
// Delete can be used to delete the Counter from the CounterVec. In that case,
// the Counter will still exist, but it will not be exported anymore, even if a
// Counter with the same label values is created later.
//
// An error is returned if the number of label values is not the same as the
// number of VariableLabels in Desc (minus any curried labels).
//
// Note that for more than one label value, this method is prone to mistakes
// caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as
// an alternative to avoid that type of mistake. For higher label numbers, the
// latter has a much more readable (albeit more verbose) syntax, but it comes
// with a performance overhead (for creating and processing the Labels map).
// See also the GaugeVec example.
func (v *CounterVec) GetMetricWithLabelValues(lvs ...string) (Counter, error) {
	metric, err := v.metricVec.getMetricWithLabelValues(lvs...)
	if metric != nil {
		return metric.(Counter), err
	}
	return nil, err
}

// GetMetricWith returns the Counter for the given Labels map (the label names
// must match those of the VariableLabels in Desc). If that label map is
// accessed for the first time, a new Counter is created. Implications of
// creating a Counter without using it and keeping the Counter for later use are
// the same as for GetMetricWithLabelValues.
//
// An error is returned if the number and names of the Labels are inconsistent
// with those of the VariableLabels in Desc (minus any curried labels).
//
// This method is used for the same purpose as
// GetMetricWithLabelValues(...string). See there for pros and cons of the two
// methods.
func (v *CounterVec) GetMetricWith(labels Labels) (Counter, error) {
	metric, err := v.metricVec.getMetricWith(labels)
	if metric != nil {
		return metric.(Counter), err
	}
	return nil, err
}

// WithLabelValues works as GetMetricWithLabelValues, but panics where
// GetMetricWithLabelValues would have returned an error. Not returning an
// error allows shortcuts like
//     myVec.WithLabelValues("404", "GET").Add(42)
func (v *CounterVec) WithLabelValues(lvs ...string) Counter {
	c, err := v.GetMetricWithLabelValues(lvs...)
	if err != nil {
		panic(err)
	}
	return c
}

// With works as GetMetricWith, but panics where GetMetricWithLabels would have
// returned an error. Not returning an error allows shortcuts like
//     myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Add(42)
func (v *CounterVec) With(labels Labels) Counter {
	c, err := v.GetMetricWith(labels)
	if err != nil {
		panic(err)
	}
	return c
}

// CurryWith returns a vector curried with the provided labels, i.e. the
// returned vector has those labels pre-set for all labeled operations performed
// on it. The cardinality of the curried vector is reduced accordingly. The
// order of the remaining labels stays the same (just with the curried labels
// taken out of the sequence – which is relevant for the
// (GetMetric)WithLabelValues methods). It is possible to curry a curried
// vector, but only with labels not yet used for currying before.
//
// The metrics contained in the CounterVec are shared between the curried and
// uncurried vectors. They are just accessed differently. Curried and uncurried
// vectors behave identically in terms of collection. Only one must be
// registered with a given registry (usually the uncurried version). The Reset
// method deletes all metrics, even if called on a curried vector.
func (v *CounterVec) CurryWith(labels Labels) (*CounterVec, error) {
	vec, err := v.curryWith(labels)
	if vec != nil {
		return &CounterVec{vec}, err
	}
	return nil, err
}

// MustCurryWith works as CurryWith but panics where CurryWith would have
// returned an error.
func (v *CounterVec) MustCurryWith(labels Labels) *CounterVec {
	vec, err := v.CurryWith(labels)
	if err != nil {
		panic(err)
	}
	return vec
}

// CounterFunc is a Counter whose value is determined at collect time by calling a
// provided function.
//
// To create CounterFunc instances, use NewCounterFunc.
type CounterFunc interface {
	Metric
	Collector
}

// NewCounterFunc creates a new CounterFunc based on the provided
// CounterOpts. The value reported is determined by calling the given function
// from within the Write method. Take into account that metric collection may
// happen concurrently. If that results in concurrent calls to Write, like in
// the case where a CounterFunc is directly registered with Prometheus, the
// provided function must be concurrency-safe. The function should also honor
// the contract for a Counter (values only go up, not down), but compliance will
// not be checked.
func NewCounterFunc(opts CounterOpts, function func() float64) CounterFunc {
	return newValueFunc(NewDesc(
		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
		opts.Help,
		nil,
		opts.ConstLabels,
	), CounterValue, function)
}
<|endoftext|>


<commit_before>// Copyright 2012 Gonéri Le Bouder. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "notmuch"
import "log"
import "encoding/json"
import "os"
import "io"
import "fmt"
import "regexp"
import "net/mail"
import "path"

type Filter struct {
	Field   string
	Pattern string
	Re      *regexp.Regexp
	Tags    string
}

type Result struct {
	MessageID string
	Tags      string
	Die       bool
        Filename  string
}

const NCPU = 4 // number of CPU cores

func getMaildirLoc() string {
	// honor NOTMUCH_CONFIG
	home := os.Getenv("NOTMUCH_CONFIG")
	if home == "" {
		home = os.Getenv("HOME")
	}

	return path.Join(home, "Maildir")
}

func RefreshFlags(nmdb *notmuch.Database) {

	query := nmdb.CreateQuery("tag:inbox and tag:delete")
	msgs := query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:archive")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and not tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.AddTag("archive")
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:bug")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.AddTag("inbox")
		}
	}

	query = nmdb.CreateQuery("tag:inbox and tag:killed")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.RemoveTag("inbox")
		}
	}

	nmdb.Close()
	fmt.Print("Ok\n")

}

func studyMsg(filter []Filter, filenameIn chan string, resultOut chan Result, quit chan bool) {
	for {
		filename := <-filenameIn

		if filename == "" {
			var result Result
			result.Die = true
			resultOut <- result

			return
		}
		// We can use Notmuch for this directly because Xappian will
		// fails as soon as we have 2 concurrent goroutine
		file, err := os.Open(filename) // For read access.
		if err != nil {
			log.Fatal(err)
		}
		var msg *mail.Message
		msg, err = mail.ReadMessage(file)

		if err != nil {
			log.Fatal(err)
		}

		var result Result
		result.MessageID = msg.Header.Get("Message-Id")
                if (result.MessageID == "") {
                    fmt.Printf("No message ID for %s\n", filename)
                    continue;
                }
		result.Filename = filename
		for _, f := range filter {
			if f.Re.MatchString(msg.Header.Get(f.Field)) {
				result.Tags += " "
				result.Tags += f.Tags
			}

		}
		file.Close()

		resultOut <- result
	}
}

func loadFilter() (filter []Filter) {

	file, err := os.Open(fmt.Sprintf("/%s/notmuch-filter.json", getMaildirLoc())) // For read access.
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	dec := json.NewDecoder(file)
	for {
		var f Filter
		if err := dec.Decode(&f); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		var err error = nil
		if f.Re, err = regexp.Compile(f.Pattern); err != nil {
			log.Printf("error: %v\n", err)
		}

		filter = append(filter, f)
	}

	return filter
}

func studyMsgs(resultOut chan Result, quit chan bool, filenames []string) {

	filter := loadFilter()

	filenameIn := make(chan string)
	for i := 0; i < NCPU+1; i++ {
		go studyMsg(filter, filenameIn, resultOut, quit)
	}
	for _, filename := range filenames {
		filenameIn <- filename
	}

	for i := 0; i < NCPU+1; i++ {
		filenameIn <- ""
	}

	quit <- true
}

func main() {
	var query *notmuch.Query
	var nmdb *notmuch.Database

	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		notmuch.DATABASE_MODE_READ_ONLY); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}

	quit := make(chan bool)
	resultOut := make(chan Result)

	query = nmdb.CreateQuery("tag:new")

	println(">", query.CountMessages(), "<")
	msgs := query.SearchMessages()

	var filenames []string
	if query.CountMessages() > 0 {
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()

			filenames = append(filenames, msg.GetFileName())
		}
	}

	go studyMsgs(resultOut, quit, filenames)

	//	var query *notmuch.Query
	var msgIDRegexp = regexp.MustCompile("^<(.*)>$")
	var tagRegexp = regexp.MustCompile("([\\+-])(\\S+)")

	// open the database
	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		1); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}
	defer nmdb.Close()

	var running int = NCPU + 1
	for {
		result := <-resultOut

		if result.Die {

			running--

			if running > 0 {
				continue
			} else {
                            break
			}
		}

		// Message-ID without the <>
                fmt.Printf("MessageID: %s\n", result.MessageID)
		reResult := msgIDRegexp.FindStringSubmatch(result.MessageID)
		if (reResult == nil) {
			fmt.Printf("Can't parse MessageID for mail %s\n", result.Filename)
			continue
		}

		msgID := reResult[1]
		filter := "id:"
		filter += msgID
		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		msg := msgs.Get()
		if msg == nil {
                    fmt.Printf("Can't find MessageID %s for mail %s\n", msgID, result.Filename)
                    continue
                }

		fmt.Printf("%s, tags: %s\n", msgID, result.Tags)
		msg.Freeze()
		for _, v := range tagRegexp.FindAllStringSubmatch(result.Tags, -1) {
			if v[1] == "+" {
				msg.AddTag(v[2])
			} else if v[1] == "-" {
				msg.RemoveTag(v[2])
			}
		}
		msg.Thaw()

	}
        RefreshFlags(nmdb)
        fmt.Printf("exit\n")
        os.Exit(0)

}
<commit_msg>use only one CPU<commit_after>// Copyright 2012 Gonéri Le Bouder. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "notmuch"
import "log"
import "encoding/json"
import "os"
import "io"
import "fmt"
import "regexp"
import "net/mail"
import "path"

type Filter struct {
	Field   string
	Pattern string
	Re      *regexp.Regexp
	Tags    string
}

type Result struct {
	MessageID string
	Tags      string
	Die       bool
        Filename  string
}

const NCPU = 1 // number of CPU cores

func getMaildirLoc() string {
	// honor NOTMUCH_CONFIG
	home := os.Getenv("NOTMUCH_CONFIG")
	if home == "" {
		home = os.Getenv("HOME")
	}

	return path.Join(home, "Maildir")
}

func RefreshFlags(nmdb *notmuch.Database) {

	query := nmdb.CreateQuery("tag:inbox and tag:delete")
	msgs := query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:archive")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and not tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.AddTag("archive")
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:bug")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.AddTag("inbox")
		}
	}

	query = nmdb.CreateQuery("tag:inbox and tag:killed")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.RemoveTag("inbox")
		}
	}

	nmdb.Close()
	fmt.Print("Ok\n")

}

func studyMsg(filter []Filter, filenameIn chan string, resultOut chan Result, quit chan bool) {
	for {
		filename := <-filenameIn

		if filename == "" {
			var result Result
			result.Die = true
			resultOut <- result

			return
		}
		// We can use Notmuch for this directly because Xappian will
		// fails as soon as we have 2 concurrent goroutine
		file, err := os.Open(filename) // For read access.
		if err != nil {
			log.Fatal(err)
		}
		var msg *mail.Message
		msg, err = mail.ReadMessage(file)

		if err != nil {
			log.Fatal(err)
		}

		var result Result
		result.MessageID = msg.Header.Get("Message-Id")
                if (result.MessageID == "") {
                    fmt.Printf("No message ID for %s\n", filename)
                    continue;
                }
		result.Filename = filename
		for _, f := range filter {
			if f.Re.MatchString(msg.Header.Get(f.Field)) {
				result.Tags += " "
				result.Tags += f.Tags
			}

		}
		file.Close()

		resultOut <- result
	}
}

func loadFilter() (filter []Filter) {

	file, err := os.Open(fmt.Sprintf("/%s/notmuch-filter.json", getMaildirLoc())) // For read access.
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	dec := json.NewDecoder(file)
	for {
		var f Filter
		if err := dec.Decode(&f); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		var err error = nil
		if f.Re, err = regexp.Compile(f.Pattern); err != nil {
			log.Printf("error: %v\n", err)
		}

		filter = append(filter, f)
	}

	return filter
}

func studyMsgs(resultOut chan Result, quit chan bool, filenames []string) {

	filter := loadFilter()

	filenameIn := make(chan string)
	for i := 0; i < NCPU+1; i++ {
		go studyMsg(filter, filenameIn, resultOut, quit)
	}
	for _, filename := range filenames {
		filenameIn <- filename
	}

	for i := 0; i < NCPU+1; i++ {
		filenameIn <- ""
	}

	quit <- true
}

func main() {
	var query *notmuch.Query
	var nmdb *notmuch.Database

	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		notmuch.DATABASE_MODE_READ_ONLY); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}

	quit := make(chan bool)
	resultOut := make(chan Result)

	query = nmdb.CreateQuery("tag:new")

	println(">", query.CountMessages(), "<")
	msgs := query.SearchMessages()

	var filenames []string
	if query.CountMessages() > 0 {
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()

			filenames = append(filenames, msg.GetFileName())
		}
	}

	go studyMsgs(resultOut, quit, filenames)

	//	var query *notmuch.Query
	var msgIDRegexp = regexp.MustCompile("^<(.*)>$")
	var tagRegexp = regexp.MustCompile("([\\+-])(\\S+)")

	// open the database
	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		1); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}
	defer nmdb.Close()

	var running int = NCPU + 1
	for {
		result := <-resultOut

		if result.Die {

			running--

			if running > 0 {
				continue
			} else {
                            break
			}
		}

		// Message-ID without the <>
                fmt.Printf("MessageID: %s\n", result.MessageID)
		reResult := msgIDRegexp.FindStringSubmatch(result.MessageID)
		if (reResult == nil) {
			fmt.Printf("Can't parse MessageID for mail %s\n", result.Filename)
			continue
		}

		msgID := reResult[1]
		filter := "id:"
		filter += msgID
		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		msg := msgs.Get()
		if msg == nil {
                    fmt.Printf("Can't find MessageID %s for mail %s\n", msgID, result.Filename)
                    continue
                }

		fmt.Printf("%s, tags: %s\n", msgID, result.Tags)
		msg.Freeze()
		for _, v := range tagRegexp.FindAllStringSubmatch(result.Tags, -1) {
			if v[1] == "+" {
				msg.AddTag(v[2])
			} else if v[1] == "-" {
				msg.RemoveTag(v[2])
			}
		}
		msg.Thaw()

	}
        RefreshFlags(nmdb)
        fmt.Printf("exit\n")
        os.Exit(0)

}
<|endoftext|>


<commit_before>package match

import (
	"fmt"

	"github.com/tisp-lang/tisp/src/lib/ast"
	"github.com/tisp-lang/tisp/src/lib/debug"
	"github.com/tisp-lang/tisp/src/lib/gensym"
	"github.com/tisp-lang/tisp/src/lib/scalar"
)

type desugarer struct {
	letBoundNames, lets []interface{}
}

func newDesugarer() *desugarer {
	return &desugarer{nil, nil}
}

func (d *desugarer) desugar(x interface{}) interface{} {
	switch x := x.(type) {
	case ast.App:
		return ast.NewApp(
			d.desugar(x.Function()),
			d.desugar(x.Arguments()).(ast.Arguments),
			x.DebugInfo())
	case ast.Arguments:
		ps := make([]ast.PositionalArgument, 0, len(x.Positionals()))

		for _, p := range x.Positionals() {
			ps = append(ps, d.desugar(p).(ast.PositionalArgument))
		}

		ks := make([]ast.KeywordArgument, 0, len(x.Keywords()))

		for _, k := range x.Keywords() {
			ks = append(ks, d.desugar(k).(ast.KeywordArgument))
		}

		dicts := make([]interface{}, 0, len(x.ExpandedDicts()))

		for _, dict := range x.ExpandedDicts() {
			dicts = append(dicts, d.desugar(dict))
		}

		return ast.NewArguments(ps, ks, dicts)
	case ast.KeywordArgument:
		return ast.NewKeywordArgument(x.Name(), d.desugar(x.Value()))
	case ast.LetFunction:
		ls := make([]interface{}, 0, len(x.Lets()))

		for _, l := range x.Lets() {
			l := d.desugar(l)
			ls = append(ls, append(d.takeLets(), l)...)
		}

		b := d.desugar(x.Body())
		ls = append(ls, d.takeLets()...)

		return ast.NewLetFunction(
			x.Name(),
			x.Signature(),
			ls,
			b,
			x.DebugInfo())
	case ast.LetVar:
		return ast.NewLetVar(x.Name(), d.desugar(x.Expr()))
	case ast.Match:
		cs := make([]ast.MatchCase, 0, len(x.Cases()))

		for _, c := range x.Cases() {
			cs = append(cs, renameBoundNamesInCase(ast.NewMatchCase(c.Pattern(), d.desugar(c.Value()))))
		}

		return d.resultApp(d.createMatchFunction(cs), d.desugar(x.Value()))
	case ast.Output:
		return ast.NewOutput(d.desugar(x.Expr()), x.Expanded())
	case ast.PositionalArgument:
		return ast.NewPositionalArgument(d.desugar(x.Value()), x.Expanded())
	default:
		return x
	}
}

func (d *desugarer) takeLets() []interface{} {
	ls := append(d.letBoundNames, d.lets...)
	d.letBoundNames = nil
	d.lets = nil
	return ls
}

func (d *desugarer) letVar(s string, v interface{}) string {
	d.lets = append(d.lets, ast.NewLetVar(s, v))
	return s
}

func (d *desugarer) bindName(s string, v interface{}) string {
	d.letBoundNames = append(d.letBoundNames, ast.NewLetVar(s, v))
	return s
}

// matchedApp applies a function to arguments and creates a matched value of
// match expression.
func (d *desugarer) matchedApp(f interface{}, args ...interface{}) string {
	return d.bindName(gensym.GenSym("match", "app"), app(f, args...))
}

// resultApp applies a function to arguments and creates a result value of match
// expression.
func (d *desugarer) resultApp(f interface{}, args ...interface{}) string {
	return d.letVar(gensym.GenSym("match", "app"), app(f, args...))
}

func (d *desugarer) createMatchFunction(cs []ast.MatchCase) interface{} {
	arg := gensym.GenSym("match", "argument")
	body := d.desugarCases(arg, cs, "$matchError")

	f := ast.NewLetFunction(
		gensym.GenSym("match", "function"),
		ast.NewSignature([]string{arg}, nil, "", nil, nil, ""),
		d.takeLets(),
		body,
		debug.NewGoInfo(0))

	d.lets = append(d.lets, f)

	return f.Name()
}

func (d *desugarer) desugarCases(v interface{}, cs []ast.MatchCase, dc interface{}) interface{} {
	css := groupCases(cs)

	if cs, ok := css[namePattern]; ok {
		c := cs[0]
		d.bindName(c.Pattern().(string), v)
		dc = c.Value()
	}

	ks := []ast.SwitchCase{}

	if cs, ok := css[listPattern]; ok {
		ks = append(ks, ast.NewSwitchCase("\"list\"", d.desugarListCases(v, cs, dc)))
	}

	if cs, ok := css[dictPattern]; ok {
		ks = append(ks, ast.NewSwitchCase("\"dict\"", d.desugarDictCases(v, cs, dc)))
	}

	if cs, ok := css[scalarPattern]; ok {
		dc = d.desugarScalarCases(v, cs, dc)
	}

	return newSwitch(d.resultApp("$typeOf", v), ks, dc)
}

func groupCases(cs []ast.MatchCase) map[patternType][]ast.MatchCase {
	css := map[patternType][]ast.MatchCase{}

	for i, c := range cs {
		t := getPatternType(c.Pattern())

		if t == namePattern && i < len(cs)-1 {
			panic("A wildcard pattern is found while some patterns are left")
		}

		css[t] = append(css[t], c)
	}

	return css
}

func getPatternType(p interface{}) patternType {
	switch x := p.(type) {
	case string:
		if scalar.Defined(x) {
			return scalarPattern
		}

		return namePattern
	case ast.App:
		switch x.Function().(string) {
		case "$list":
			return listPattern
		case "$dict":
			return dictPattern
		}
	}

	panic(fmt.Errorf("Invalid pattern: %#v", p))
}

func (d *desugarer) desugarListCases(list interface{}, cs []ast.MatchCase, dc interface{}) interface{} {
	type group struct {
		first interface{}
		cases []ast.MatchCase
	}

	gs := []group{}
	first := d.matchedApp("$first", list)
	rest := d.matchedApp("$rest", list)

	for i, c := range cs {
		ps := c.Pattern().(ast.App).Arguments().Positionals()

		if len(ps) == 0 {
			dc = d.resultApp("$if", app("$=", list, "$emptyList"), c.Value(), dc)
			continue
		}

		if ps[0].Expanded() {
			panic("Not implemented")
		}

		v := ps[0].Value()

		c = ast.NewMatchCase(
			ast.NewApp("$list", ast.NewArguments(ps[1:], nil, nil), debug.NewGoInfo(0)),
			c.Value())

		if getPatternType(v) == namePattern {
			d.bindName(v.(string), first)
			dc = d.desugarCases(
				rest,
				[]ast.MatchCase{c},
				d.desugarListCases(list, cs[i+1:], dc))
			break
		}

		groupExist := false

		for i, g := range gs {
			if equalPatterns(v, g.first) {
				groupExist = true
				gs[i].cases = append(gs[i].cases, c)
			}
		}

		if !groupExist {
			gs = append(gs, group{v, []ast.MatchCase{c}})
		}
	}

	ks := make([]ast.MatchCase, 0, len(gs))

	for _, g := range gs {
		ks = append(ks, ast.NewMatchCase(g.first, d.desugarCases(rest, g.cases, dc)))
	}

	return d.desugarCases(first, ks, dc)
}

func (d *desugarer) desugarDictCases(v interface{}, cs []ast.MatchCase, dc interface{}) interface{} {
	type group struct {
		key   interface{}
		cases []ast.MatchCase
	}

	gs := []group{}

	for _, c := range cs {
		ps := c.Pattern().(ast.App).Arguments().Positionals()

		if len(ps) == 0 {
			dc = d.resultApp("$if", app("$=", v, "$emptyDict"), c.Value(), dc)
			continue
		}

		if ps[0].Expanded() {
			panic("Not implemented")
		}

		g := group{ps[0].Value(), []ast.MatchCase{c}}

		if len(gs) == 0 {
			gs = append(gs, g)
		} else if last := gs[len(gs)-1]; equalPatterns(g.key, last.key) {
			last.cases = append(last.cases, c)
		} else {
			gs = append(gs, g)
		}
	}

	for i := len(gs) - 1; i >= 0; i-- {
		g := gs[i]
		dc = d.resultApp("$if",
			app("$include", v, g.key),
			d.desugarDictCasesOfSameKey(v, g.cases, dc),
			dc)
	}

	return dc
}

func (d *desugarer) desugarDictCasesOfSameKey(dict interface{}, cs []ast.MatchCase, dc interface{}) interface{} {
	type group struct {
		value interface{}
		cases []ast.MatchCase
	}

	key := cs[0].Pattern().(ast.App).Arguments().Positionals()[0].Value()
	value := d.matchedApp(dict, key)
	newDict := d.matchedApp("delete", dict, key)
	gs := []group{}

	for i, c := range cs {
		ps := c.Pattern().(ast.App).Arguments().Positionals()
		v := ps[1].Value()

		c = ast.NewMatchCase(
			ast.NewApp("$dict", ast.NewArguments(ps[2:], nil, nil), debug.NewGoInfo(0)),
			c.Value())

		if getPatternType(v) == namePattern {
			d.bindName(v.(string), value)

			if rest := cs[i+1:]; len(rest) != 0 {
				dc = d.desugarDictCasesOfSameKey(dict, rest, dc)
			}

			dc = d.desugarCases(newDict, []ast.MatchCase{c}, dc)

			break
		}

		groupExist := false

		for i, g := range gs {
			if equalPatterns(v, g.value) {
				groupExist = true
				gs[i].cases = append(gs[i].cases, c)
			}
		}

		if !groupExist {
			gs = append(gs, group{v, []ast.MatchCase{c}})
		}
	}

	cs = make([]ast.MatchCase, 0, len(gs))

	for _, g := range gs {
		cs = append(
			cs,
			ast.NewMatchCase(g.value, d.desugarCases(newDict, g.cases, dc)))
	}

	return d.desugarCases(value, cs, dc)
}

func (d *desugarer) desugarScalarCases(v interface{}, cs []ast.MatchCase, dc interface{}) interface{} {
	ks := []ast.SwitchCase{}

	for _, c := range cs {
		ks = append(ks, ast.NewSwitchCase(c.Pattern().(string), c.Value()))
	}

	return newSwitch(v, ks, dc)
}

func renameBoundNamesInCase(c ast.MatchCase) ast.MatchCase {
	p, ns := newPatternRenamer().rename(c.Pattern())
	return ast.NewMatchCase(p, newValueRenamer(ns).rename(c.Value()))
}

func equalPatterns(p, q interface{}) bool {
	switch x := p.(type) {
	case string:
		y, ok := q.(string)

		if !ok {
			return false
		}

		return x == y
	case ast.App:
		y, ok := q.(ast.App)

		if !ok ||
			x.Function().(string) != y.Function().(string) ||
			len(x.Arguments().Positionals()) != len(y.Arguments().Positionals()) {
			return false
		}

		for i := range x.Arguments().Positionals() {
			p := x.Arguments().Positionals()[i]
			q := y.Arguments().Positionals()[i]

			if p.Expanded() != q.Expanded() || !equalPatterns(p.Value(), q.Value()) {
				return false
			}
		}

		return true
	}

	panic(fmt.Errorf("Invalid pattern: %#v, %#v", p, q))
}
<commit_msg>Create intermediate variables of default cases<commit_after>package match

import (
	"fmt"

	"github.com/tisp-lang/tisp/src/lib/ast"
	"github.com/tisp-lang/tisp/src/lib/debug"
	"github.com/tisp-lang/tisp/src/lib/gensym"
	"github.com/tisp-lang/tisp/src/lib/scalar"
)

type desugarer struct {
	letBoundNames, lets []interface{}
}

func newDesugarer() *desugarer {
	return &desugarer{nil, nil}
}

func (d *desugarer) desugar(x interface{}) interface{} {
	switch x := x.(type) {
	case ast.App:
		return ast.NewApp(
			d.desugar(x.Function()),
			d.desugar(x.Arguments()).(ast.Arguments),
			x.DebugInfo())
	case ast.Arguments:
		ps := make([]ast.PositionalArgument, 0, len(x.Positionals()))

		for _, p := range x.Positionals() {
			ps = append(ps, d.desugar(p).(ast.PositionalArgument))
		}

		ks := make([]ast.KeywordArgument, 0, len(x.Keywords()))

		for _, k := range x.Keywords() {
			ks = append(ks, d.desugar(k).(ast.KeywordArgument))
		}

		dicts := make([]interface{}, 0, len(x.ExpandedDicts()))

		for _, dict := range x.ExpandedDicts() {
			dicts = append(dicts, d.desugar(dict))
		}

		return ast.NewArguments(ps, ks, dicts)
	case ast.KeywordArgument:
		return ast.NewKeywordArgument(x.Name(), d.desugar(x.Value()))
	case ast.LetFunction:
		ls := make([]interface{}, 0, len(x.Lets()))

		for _, l := range x.Lets() {
			l := d.desugar(l)
			ls = append(ls, append(d.takeLets(), l)...)
		}

		b := d.desugar(x.Body())
		ls = append(ls, d.takeLets()...)

		return ast.NewLetFunction(
			x.Name(),
			x.Signature(),
			ls,
			b,
			x.DebugInfo())
	case ast.LetVar:
		return ast.NewLetVar(x.Name(), d.desugar(x.Expr()))
	case ast.Match:
		cs := make([]ast.MatchCase, 0, len(x.Cases()))

		for _, c := range x.Cases() {
			cs = append(cs, renameBoundNamesInCase(ast.NewMatchCase(c.Pattern(), d.desugar(c.Value()))))
		}

		return d.resultApp(d.createMatchFunction(cs), d.desugar(x.Value()))
	case ast.Output:
		return ast.NewOutput(d.desugar(x.Expr()), x.Expanded())
	case ast.PositionalArgument:
		return ast.NewPositionalArgument(d.desugar(x.Value()), x.Expanded())
	default:
		return x
	}
}

func (d *desugarer) takeLets() []interface{} {
	ls := append(d.letBoundNames, d.lets...)
	d.letBoundNames = nil
	d.lets = nil
	return ls
}

func (d *desugarer) letTempVar(v interface{}) string {
	s := gensym.GenSym("match", "tmp")
	d.lets = append(d.lets, ast.NewLetVar(s, v))
	return s
}

func (d *desugarer) bindName(s string, v interface{}) string {
	d.letBoundNames = append(d.letBoundNames, ast.NewLetVar(s, v))
	return s
}

// matchedApp applies a function to arguments and creates a matched value of
// match expression.
func (d *desugarer) matchedApp(f interface{}, args ...interface{}) string {
	return d.bindName(gensym.GenSym("match", "app"), app(f, args...))
}

// resultApp applies a function to arguments and creates a result value of match
// expression.
func (d *desugarer) resultApp(f interface{}, args ...interface{}) string {
	return d.letTempVar(app(f, args...))
}

func (d *desugarer) createMatchFunction(cs []ast.MatchCase) interface{} {
	arg := gensym.GenSym("match", "argument")
	body := d.desugarCases(arg, cs, "$matchError")

	f := ast.NewLetFunction(
		gensym.GenSym("match", "function"),
		ast.NewSignature([]string{arg}, nil, "", nil, nil, ""),
		d.takeLets(),
		body,
		debug.NewGoInfo(0))

	d.lets = append(d.lets, f)

	return f.Name()
}

func (d *desugarer) desugarCases(v interface{}, cs []ast.MatchCase, dc interface{}) interface{} {
	css := groupCases(cs)

	if cs, ok := css[namePattern]; ok {
		c := cs[0]
		d.bindName(c.Pattern().(string), v)
		dc = c.Value()
	}

	ks := []ast.SwitchCase{}

	if cs, ok := css[listPattern]; ok {
		ks = append(ks, ast.NewSwitchCase("\"list\"", d.desugarListCases(v, cs, dc)))
	}

	if cs, ok := css[dictPattern]; ok {
		ks = append(ks, ast.NewSwitchCase("\"dict\"", d.desugarDictCases(v, cs, dc)))
	}

	if cs, ok := css[scalarPattern]; ok {
		dc = d.desugarScalarCases(v, cs, dc)
	}

	return newSwitch(d.resultApp("$typeOf", v), ks, dc)
}

func groupCases(cs []ast.MatchCase) map[patternType][]ast.MatchCase {
	css := map[patternType][]ast.MatchCase{}

	for i, c := range cs {
		t := getPatternType(c.Pattern())

		if t == namePattern && i < len(cs)-1 {
			panic("A wildcard pattern is found while some patterns are left")
		}

		css[t] = append(css[t], c)
	}

	return css
}

func getPatternType(p interface{}) patternType {
	switch x := p.(type) {
	case string:
		if scalar.Defined(x) {
			return scalarPattern
		}

		return namePattern
	case ast.App:
		switch x.Function().(string) {
		case "$list":
			return listPattern
		case "$dict":
			return dictPattern
		}
	}

	panic(fmt.Errorf("Invalid pattern: %#v", p))
}

func (d *desugarer) desugarListCases(list interface{}, cs []ast.MatchCase, dc interface{}) interface{} {
	type group struct {
		first interface{}
		cases []ast.MatchCase
	}

	gs := []group{}
	first := d.matchedApp("$first", list)
	rest := d.matchedApp("$rest", list)

	for i, c := range cs {
		ps := c.Pattern().(ast.App).Arguments().Positionals()

		if len(ps) == 0 {
			dc = d.resultApp("$if", app("$=", list, "$emptyList"), c.Value(), dc)
			continue
		}

		if ps[0].Expanded() {
			panic("Not implemented")
		}

		v := ps[0].Value()

		c = ast.NewMatchCase(
			ast.NewApp("$list", ast.NewArguments(ps[1:], nil, nil), debug.NewGoInfo(0)),
			c.Value())

		if getPatternType(v) == namePattern {
			d.bindName(v.(string), first)
			dc = d.desugarCases(
				rest,
				[]ast.MatchCase{c},
				d.desugarListCases(list, cs[i+1:], dc))
			break
		}

		groupExist := false

		for i, g := range gs {
			if equalPatterns(v, g.first) {
				groupExist = true
				gs[i].cases = append(gs[i].cases, c)
			}
		}

		if !groupExist {
			gs = append(gs, group{v, []ast.MatchCase{c}})
		}
	}

	ks := make([]ast.MatchCase, 0, len(gs))
	dc = d.letTempVar(dc)

	for _, g := range gs {
		ks = append(ks, ast.NewMatchCase(g.first, d.desugarCases(rest, g.cases, dc)))
	}

	return d.desugarCases(first, ks, dc)
}

func (d *desugarer) desugarDictCases(v interface{}, cs []ast.MatchCase, dc interface{}) interface{} {
	type group struct {
		key   interface{}
		cases []ast.MatchCase
	}

	gs := []group{}

	for _, c := range cs {
		ps := c.Pattern().(ast.App).Arguments().Positionals()

		if len(ps) == 0 {
			dc = d.resultApp("$if", app("$=", v, "$emptyDict"), c.Value(), dc)
			continue
		}

		if ps[0].Expanded() {
			panic("Not implemented")
		}

		g := group{ps[0].Value(), []ast.MatchCase{c}}

		if len(gs) == 0 {
			gs = append(gs, g)
		} else if last := gs[len(gs)-1]; equalPatterns(g.key, last.key) {
			last.cases = append(last.cases, c)
		} else {
			gs = append(gs, g)
		}
	}

	for i := len(gs) - 1; i >= 0; i-- {
		g := gs[i]
		dc = d.resultApp("$if",
			app("$include", v, g.key),
			d.desugarDictCasesOfSameKey(v, g.cases, dc),
			dc)
	}

	return dc
}

func (d *desugarer) desugarDictCasesOfSameKey(dict interface{}, cs []ast.MatchCase, dc interface{}) interface{} {
	type group struct {
		value interface{}
		cases []ast.MatchCase
	}

	key := cs[0].Pattern().(ast.App).Arguments().Positionals()[0].Value()
	value := d.matchedApp(dict, key)
	newDict := d.matchedApp("delete", dict, key)
	gs := []group{}

	for i, c := range cs {
		ps := c.Pattern().(ast.App).Arguments().Positionals()
		v := ps[1].Value()

		c = ast.NewMatchCase(
			ast.NewApp("$dict", ast.NewArguments(ps[2:], nil, nil), debug.NewGoInfo(0)),
			c.Value())

		if getPatternType(v) == namePattern {
			d.bindName(v.(string), value)

			if rest := cs[i+1:]; len(rest) != 0 {
				dc = d.desugarDictCasesOfSameKey(dict, rest, dc)
			}

			dc = d.desugarCases(newDict, []ast.MatchCase{c}, dc)

			break
		}

		groupExist := false

		for i, g := range gs {
			if equalPatterns(v, g.value) {
				groupExist = true
				gs[i].cases = append(gs[i].cases, c)
			}
		}

		if !groupExist {
			gs = append(gs, group{v, []ast.MatchCase{c}})
		}
	}

	cs = make([]ast.MatchCase, 0, len(gs))
	dc = d.letTempVar(dc)

	for _, g := range gs {
		cs = append(
			cs,
			ast.NewMatchCase(g.value, d.desugarCases(newDict, g.cases, dc)))
	}

	return d.desugarCases(value, cs, dc)
}

func (d *desugarer) desugarScalarCases(v interface{}, cs []ast.MatchCase, dc interface{}) interface{} {
	ks := []ast.SwitchCase{}

	for _, c := range cs {
		ks = append(ks, ast.NewSwitchCase(c.Pattern().(string), c.Value()))
	}

	return newSwitch(v, ks, dc)
}

func renameBoundNamesInCase(c ast.MatchCase) ast.MatchCase {
	p, ns := newPatternRenamer().rename(c.Pattern())
	return ast.NewMatchCase(p, newValueRenamer(ns).rename(c.Value()))
}

func equalPatterns(p, q interface{}) bool {
	switch x := p.(type) {
	case string:
		y, ok := q.(string)

		if !ok {
			return false
		}

		return x == y
	case ast.App:
		y, ok := q.(ast.App)

		if !ok ||
			x.Function().(string) != y.Function().(string) ||
			len(x.Arguments().Positionals()) != len(y.Arguments().Positionals()) {
			return false
		}

		for i := range x.Arguments().Positionals() {
			p := x.Arguments().Positionals()[i]
			q := y.Arguments().Positionals()[i]

			if p.Expanded() != q.Expanded() || !equalPatterns(p.Value(), q.Value()) {
				return false
			}
		}

		return true
	}

	panic(fmt.Errorf("Invalid pattern: %#v, %#v", p, q))
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package lsp

import (
	"bytes"
	"context"
	"fmt"

	"golang.org/x/tools/internal/jsonrpc2"
	"golang.org/x/tools/internal/lsp/protocol"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/span"
	errors "golang.org/x/xerrors"
)

// ModificationSource identifies the originating cause of a file modification.
type ModificationSource int

const (
	// FromDidOpen is a file modification caused by opening a file.
	FromDidOpen = ModificationSource(iota)
	// FromDidChange is a file modification caused by changing a file.
	FromDidChange
	// FromDidChangeWatchedFiles is a file modification caused by a change to a watched file.
	FromDidChangeWatchedFiles
	// FromDidSave is a file modification caused by a file save.
	FromDidSave
	// FromDidClose is a file modification caused by closing a file.
	FromDidClose
	FromRegenerateCgo
)

func (m ModificationSource) String() string {
	switch m {
	case FromDidOpen:
		return "opened files"
	case FromDidChange:
		return "changed files"
	case FromDidChangeWatchedFiles:
		return "files changed on disk"
	case FromDidSave:
		return "saved files"
	case FromRegenerateCgo:
		return "regenerate cgo"
	default:
		return "unknown file modification"
	}
}

func (s *Server) didOpen(ctx context.Context, params *protocol.DidOpenTextDocumentParams) error {
	uri := params.TextDocument.URI.SpanURI()
	if !uri.IsFile() {
		return nil
	}

	_, err := s.didModifyFiles(ctx, []source.FileModification{
		{
			URI:        uri,
			Action:     source.Open,
			Version:    params.TextDocument.Version,
			Text:       []byte(params.TextDocument.Text),
			LanguageID: params.TextDocument.LanguageID,
		},
	}, FromDidOpen)
	return err
}

func (s *Server) didChange(ctx context.Context, params *protocol.DidChangeTextDocumentParams) error {
	uri := params.TextDocument.URI.SpanURI()
	if !uri.IsFile() {
		return nil
	}

	text, err := s.changedText(ctx, uri, params.ContentChanges)
	if err != nil {
		return err
	}
	c := source.FileModification{
		URI:     uri,
		Action:  source.Change,
		Version: params.TextDocument.Version,
		Text:    text,
	}
	snapshots, err := s.didModifyFiles(ctx, []source.FileModification{c}, FromDidChange)
	if err != nil {
		return err
	}
	snapshot := snapshots[uri]
	if snapshot == nil {
		return errors.Errorf("no snapshot for %s", uri)
	}
	// Ideally, we should be able to specify that a generated file should be opened as read-only.
	// Tell the user that they should not be editing a generated file.
	if s.wasFirstChange(uri) && source.IsGenerated(ctx, snapshot, uri) {
		if err := s.client.ShowMessage(ctx, &protocol.ShowMessageParams{
			Message: fmt.Sprintf("Do not edit this file! %s is a generated file.", uri.Filename()),
			Type:    protocol.Warning,
		}); err != nil {
			return err
		}
	}
	return nil
}

func (s *Server) didChangeWatchedFiles(ctx context.Context, params *protocol.DidChangeWatchedFilesParams) error {
	var modifications []source.FileModification
	deletions := make(map[span.URI]struct{})
	for _, change := range params.Changes {
		uri := change.URI.SpanURI()
		if !uri.IsFile() {
			continue
		}
		action := changeTypeToFileAction(change.Type)
		modifications = append(modifications, source.FileModification{
			URI:    uri,
			Action: action,
			OnDisk: true,
		})
		// Keep track of deleted files so that we can clear their diagnostics.
		// A file might be re-created after deletion, so only mark files that
		// have truly been deleted.
		switch action {
		case source.Delete:
			deletions[uri] = struct{}{}
		case source.Close:
		default:
			delete(deletions, uri)
		}
	}
	snapshots, err := s.didModifyFiles(ctx, modifications, FromDidChangeWatchedFiles)
	if err != nil {
		return err
	}
	// Clear the diagnostics for any deleted files that are not open in the editor.
	for uri := range deletions {
		if snapshot := snapshots[uri]; snapshot == nil || snapshot.IsOpen(uri) {
			continue
		}
		if err := s.client.PublishDiagnostics(ctx, &protocol.PublishDiagnosticsParams{
			URI:         protocol.URIFromSpanURI(uri),
			Diagnostics: []protocol.Diagnostic{},
			Version:     0,
		}); err != nil {
			return err
		}
	}
	return nil
}

func (s *Server) didSave(ctx context.Context, params *protocol.DidSaveTextDocumentParams) error {
	uri := params.TextDocument.URI.SpanURI()
	if !uri.IsFile() {
		return nil
	}
	c := source.FileModification{
		URI:     uri,
		Action:  source.Save,
		Version: params.TextDocument.Version,
	}
	if params.Text != nil {
		c.Text = []byte(*params.Text)
	}
	_, err := s.didModifyFiles(ctx, []source.FileModification{c}, FromDidSave)
	return err
}

func (s *Server) didClose(ctx context.Context, params *protocol.DidCloseTextDocumentParams) error {
	uri := params.TextDocument.URI.SpanURI()
	if !uri.IsFile() {
		return nil
	}
	snapshots, err := s.didModifyFiles(ctx, []source.FileModification{
		{
			URI:     uri,
			Action:  source.Close,
			Version: -1,
			Text:    nil,
		},
	}, FromDidClose)
	if err != nil {
		return err
	}
	snapshot := snapshots[uri]
	if snapshot == nil {
		return errors.Errorf("no snapshot for %s", uri)
	}
	fh, err := snapshot.GetFile(uri)
	if err != nil {
		return err
	}
	// If a file has been closed and is not on disk, clear its diagnostics.
	if _, _, err := fh.Read(ctx); err != nil {
		return s.client.PublishDiagnostics(ctx, &protocol.PublishDiagnosticsParams{
			URI:         protocol.URIFromSpanURI(uri),
			Diagnostics: []protocol.Diagnostic{},
			Version:     0,
		})
	}
	return nil
}

func (s *Server) didModifyFiles(ctx context.Context, modifications []source.FileModification, cause ModificationSource) (map[span.URI]source.Snapshot, error) {
	snapshots, err := s.session.DidModifyFiles(ctx, modifications)
	if err != nil {
		return nil, err
	}
	snapshotByURI := make(map[span.URI]source.Snapshot)
	for _, c := range modifications {
		snapshotByURI[c.URI] = nil
	}
	// Avoid diagnosing the same snapshot twice.
	snapshotSet := make(map[source.Snapshot][]span.URI)
	for uri := range snapshotByURI {
		view, err := s.session.ViewOf(uri)
		if err != nil {
			return nil, err
		}
		var snapshot source.Snapshot
		for _, s := range snapshots {
			if s.View() == view {
				if snapshot != nil {
					return nil, errors.Errorf("duplicate snapshots for the same view")
				}
				snapshot = s
			}
		}
		// If the file isn't in any known views (for example, if it's in a dependency),
		// we may not have a snapshot to map it to. As a result, we won't try to
		// diagnose it. TODO(rstambler): Figure out how to handle this better.
		if snapshot == nil {
			continue
		}
		snapshotByURI[uri] = snapshot
		snapshotSet[snapshot] = append(snapshotSet[snapshot], uri)
	}
	for snapshot, uris := range snapshotSet {
		// If a modification comes in for the view's go.mod file and the view
		// was never properly initialized, or the view does not have
		// a go.mod file, try to recreate the associated view.
		if modfile, _ := snapshot.View().ModFiles(); modfile == "" {
			for _, uri := range uris {
				// Don't rebuild the view until the go.mod is on disk.
				if !snapshot.IsSaved(uri) {
					continue
				}
				fh, err := snapshot.GetFile(uri)
				if err != nil {
					return nil, err
				}
				switch fh.Identity().Kind {
				case source.Mod:
					newSnapshot, err := snapshot.View().Rebuild(ctx)
					if err != nil {
						return nil, err
					}
					// Update the snapshot to the rebuilt one.
					snapshot = newSnapshot
					snapshotByURI[uri] = newSnapshot
				}
			}
		}
		go func(snapshot source.Snapshot) {
			if s.session.Options().VerboseWorkDoneProgress {
				work := s.StartWork(ctx, DiagnosticWorkTitle(cause), "Calculating file diagnostics...", nil)
				defer work.End(ctx, "Done.")
			}
			s.diagnoseSnapshot(snapshot)
		}(snapshot)
	}
	return snapshotByURI, nil
}

// DiagnosticWorkTitle returns the title of the diagnostic work resulting from a
// file change originating from the given cause.
func DiagnosticWorkTitle(cause ModificationSource) string {
	return fmt.Sprintf("diagnosing %v", cause)
}

func (s *Server) wasFirstChange(uri span.URI) bool {
	if s.changedFiles == nil {
		s.changedFiles = make(map[span.URI]struct{})
	}
	_, ok := s.changedFiles[uri]
	return ok
}

func (s *Server) changedText(ctx context.Context, uri span.URI, changes []protocol.TextDocumentContentChangeEvent) ([]byte, error) {
	if len(changes) == 0 {
		return nil, fmt.Errorf("%w: no content changes provided", jsonrpc2.ErrInternal)
	}

	// Check if the client sent the full content of the file.
	// We accept a full content change even if the server expected incremental changes.
	if len(changes) == 1 && changes[0].Range == nil && changes[0].RangeLength == 0 {
		return []byte(changes[0].Text), nil
	}
	return s.applyIncrementalChanges(ctx, uri, changes)
}

func (s *Server) applyIncrementalChanges(ctx context.Context, uri span.URI, changes []protocol.TextDocumentContentChangeEvent) ([]byte, error) {
	content, _, err := s.session.GetFile(uri).Read(ctx)
	if err != nil {
		return nil, fmt.Errorf("%w: file not found (%v)", jsonrpc2.ErrInternal, err)
	}
	for _, change := range changes {
		// Make sure to update column mapper along with the content.
		converter := span.NewContentConverter(uri.Filename(), content)
		m := &protocol.ColumnMapper{
			URI:       uri,
			Converter: converter,
			Content:   content,
		}
		if change.Range == nil {
			return nil, fmt.Errorf("%w: unexpected nil range for change", jsonrpc2.ErrInternal)
		}
		spn, err := m.RangeSpan(*change.Range)
		if err != nil {
			return nil, err
		}
		if !spn.HasOffset() {
			return nil, fmt.Errorf("%w: invalid range for content change", jsonrpc2.ErrInternal)
		}
		start, end := spn.Start().Offset(), spn.End().Offset()
		if end < start {
			return nil, fmt.Errorf("%w: invalid range for content change", jsonrpc2.ErrInternal)
		}
		var buf bytes.Buffer
		buf.Write(content[:start])
		buf.WriteString(change.Text)
		buf.Write(content[end:])
		content = buf.Bytes()
	}
	return content, nil
}

func changeTypeToFileAction(ct protocol.FileChangeType) source.FileAction {
	switch ct {
	case protocol.Changed:
		return source.Change
	case protocol.Created:
		return source.Create
	case protocol.Deleted:
		return source.Delete
	}
	return source.UnknownFileAction
}
<commit_msg>internal/lsp: lift up workdone instrumentation to didModifyFiles<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package lsp

import (
	"bytes"
	"context"
	"fmt"
	"sync"

	"golang.org/x/tools/internal/jsonrpc2"
	"golang.org/x/tools/internal/lsp/protocol"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/span"
	errors "golang.org/x/xerrors"
)

// ModificationSource identifies the originating cause of a file modification.
type ModificationSource int

const (
	// FromDidOpen is a file modification caused by opening a file.
	FromDidOpen = ModificationSource(iota)
	// FromDidChange is a file modification caused by changing a file.
	FromDidChange
	// FromDidChangeWatchedFiles is a file modification caused by a change to a watched file.
	FromDidChangeWatchedFiles
	// FromDidSave is a file modification caused by a file save.
	FromDidSave
	// FromDidClose is a file modification caused by closing a file.
	FromDidClose
	FromRegenerateCgo
)

func (m ModificationSource) String() string {
	switch m {
	case FromDidOpen:
		return "opened files"
	case FromDidChange:
		return "changed files"
	case FromDidChangeWatchedFiles:
		return "files changed on disk"
	case FromDidSave:
		return "saved files"
	case FromRegenerateCgo:
		return "regenerate cgo"
	default:
		return "unknown file modification"
	}
}

func (s *Server) didOpen(ctx context.Context, params *protocol.DidOpenTextDocumentParams) error {
	uri := params.TextDocument.URI.SpanURI()
	if !uri.IsFile() {
		return nil
	}

	_, err := s.didModifyFiles(ctx, []source.FileModification{
		{
			URI:        uri,
			Action:     source.Open,
			Version:    params.TextDocument.Version,
			Text:       []byte(params.TextDocument.Text),
			LanguageID: params.TextDocument.LanguageID,
		},
	}, FromDidOpen)
	return err
}

func (s *Server) didChange(ctx context.Context, params *protocol.DidChangeTextDocumentParams) error {
	uri := params.TextDocument.URI.SpanURI()
	if !uri.IsFile() {
		return nil
	}

	text, err := s.changedText(ctx, uri, params.ContentChanges)
	if err != nil {
		return err
	}
	c := source.FileModification{
		URI:     uri,
		Action:  source.Change,
		Version: params.TextDocument.Version,
		Text:    text,
	}
	snapshots, err := s.didModifyFiles(ctx, []source.FileModification{c}, FromDidChange)
	if err != nil {
		return err
	}
	snapshot := snapshots[uri]
	if snapshot == nil {
		return errors.Errorf("no snapshot for %s", uri)
	}
	// Ideally, we should be able to specify that a generated file should be opened as read-only.
	// Tell the user that they should not be editing a generated file.
	if s.wasFirstChange(uri) && source.IsGenerated(ctx, snapshot, uri) {
		if err := s.client.ShowMessage(ctx, &protocol.ShowMessageParams{
			Message: fmt.Sprintf("Do not edit this file! %s is a generated file.", uri.Filename()),
			Type:    protocol.Warning,
		}); err != nil {
			return err
		}
	}
	return nil
}

func (s *Server) didChangeWatchedFiles(ctx context.Context, params *protocol.DidChangeWatchedFilesParams) error {
	var modifications []source.FileModification
	deletions := make(map[span.URI]struct{})
	for _, change := range params.Changes {
		uri := change.URI.SpanURI()
		if !uri.IsFile() {
			continue
		}
		action := changeTypeToFileAction(change.Type)
		modifications = append(modifications, source.FileModification{
			URI:    uri,
			Action: action,
			OnDisk: true,
		})
		// Keep track of deleted files so that we can clear their diagnostics.
		// A file might be re-created after deletion, so only mark files that
		// have truly been deleted.
		switch action {
		case source.Delete:
			deletions[uri] = struct{}{}
		case source.Close:
		default:
			delete(deletions, uri)
		}
	}
	snapshots, err := s.didModifyFiles(ctx, modifications, FromDidChangeWatchedFiles)
	if err != nil {
		return err
	}
	// Clear the diagnostics for any deleted files that are not open in the editor.
	for uri := range deletions {
		if snapshot := snapshots[uri]; snapshot == nil || snapshot.IsOpen(uri) {
			continue
		}
		if err := s.client.PublishDiagnostics(ctx, &protocol.PublishDiagnosticsParams{
			URI:         protocol.URIFromSpanURI(uri),
			Diagnostics: []protocol.Diagnostic{},
			Version:     0,
		}); err != nil {
			return err
		}
	}
	return nil
}

func (s *Server) didSave(ctx context.Context, params *protocol.DidSaveTextDocumentParams) error {
	uri := params.TextDocument.URI.SpanURI()
	if !uri.IsFile() {
		return nil
	}
	c := source.FileModification{
		URI:     uri,
		Action:  source.Save,
		Version: params.TextDocument.Version,
	}
	if params.Text != nil {
		c.Text = []byte(*params.Text)
	}
	_, err := s.didModifyFiles(ctx, []source.FileModification{c}, FromDidSave)
	return err
}

func (s *Server) didClose(ctx context.Context, params *protocol.DidCloseTextDocumentParams) error {
	uri := params.TextDocument.URI.SpanURI()
	if !uri.IsFile() {
		return nil
	}
	snapshots, err := s.didModifyFiles(ctx, []source.FileModification{
		{
			URI:     uri,
			Action:  source.Close,
			Version: -1,
			Text:    nil,
		},
	}, FromDidClose)
	if err != nil {
		return err
	}
	snapshot := snapshots[uri]
	if snapshot == nil {
		return errors.Errorf("no snapshot for %s", uri)
	}
	fh, err := snapshot.GetFile(uri)
	if err != nil {
		return err
	}
	// If a file has been closed and is not on disk, clear its diagnostics.
	if _, _, err := fh.Read(ctx); err != nil {
		return s.client.PublishDiagnostics(ctx, &protocol.PublishDiagnosticsParams{
			URI:         protocol.URIFromSpanURI(uri),
			Diagnostics: []protocol.Diagnostic{},
			Version:     0,
		})
	}
	return nil
}

func (s *Server) didModifyFiles(ctx context.Context, modifications []source.FileModification, cause ModificationSource) (map[span.URI]source.Snapshot, error) {
	// diagnosticWG tracks outstanding diagnostic work as a result of this file
	// modification.
	var diagnosticWG sync.WaitGroup
	if s.session.Options().VerboseWorkDoneProgress {
		work := s.StartWork(ctx, DiagnosticWorkTitle(cause), "Calculating file diagnostics...", nil)
		defer func() {
			go func() {
				diagnosticWG.Wait()
				work.End(ctx, "Done.")
			}()
		}()
	}
	snapshots, err := s.session.DidModifyFiles(ctx, modifications)
	if err != nil {
		return nil, err
	}
	snapshotByURI := make(map[span.URI]source.Snapshot)
	for _, c := range modifications {
		snapshotByURI[c.URI] = nil
	}
	// Avoid diagnosing the same snapshot twice.
	snapshotSet := make(map[source.Snapshot][]span.URI)
	for uri := range snapshotByURI {
		view, err := s.session.ViewOf(uri)
		if err != nil {
			return nil, err
		}
		var snapshot source.Snapshot
		for _, s := range snapshots {
			if s.View() == view {
				if snapshot != nil {
					return nil, errors.Errorf("duplicate snapshots for the same view")
				}
				snapshot = s
			}
		}
		// If the file isn't in any known views (for example, if it's in a dependency),
		// we may not have a snapshot to map it to. As a result, we won't try to
		// diagnose it. TODO(rstambler): Figure out how to handle this better.
		if snapshot == nil {
			continue
		}
		snapshotByURI[uri] = snapshot
		snapshotSet[snapshot] = append(snapshotSet[snapshot], uri)
	}
	for snapshot, uris := range snapshotSet {
		// If a modification comes in for the view's go.mod file and the view
		// was never properly initialized, or the view does not have
		// a go.mod file, try to recreate the associated view.
		if modfile, _ := snapshot.View().ModFiles(); modfile == "" {
			for _, uri := range uris {
				// Don't rebuild the view until the go.mod is on disk.
				if !snapshot.IsSaved(uri) {
					continue
				}
				fh, err := snapshot.GetFile(uri)
				if err != nil {
					return nil, err
				}
				switch fh.Identity().Kind {
				case source.Mod:
					newSnapshot, err := snapshot.View().Rebuild(ctx)
					if err != nil {
						return nil, err
					}
					// Update the snapshot to the rebuilt one.
					snapshot = newSnapshot
					snapshotByURI[uri] = newSnapshot
				}
			}
		}
		diagnosticWG.Add(1)
		go func(snapshot source.Snapshot) {
			defer diagnosticWG.Done()
			s.diagnoseSnapshot(snapshot)
		}(snapshot)
	}
	return snapshotByURI, nil
}

// DiagnosticWorkTitle returns the title of the diagnostic work resulting from a
// file change originating from the given cause.
func DiagnosticWorkTitle(cause ModificationSource) string {
	return fmt.Sprintf("diagnosing %v", cause)
}

func (s *Server) wasFirstChange(uri span.URI) bool {
	if s.changedFiles == nil {
		s.changedFiles = make(map[span.URI]struct{})
	}
	_, ok := s.changedFiles[uri]
	return ok
}

func (s *Server) changedText(ctx context.Context, uri span.URI, changes []protocol.TextDocumentContentChangeEvent) ([]byte, error) {
	if len(changes) == 0 {
		return nil, fmt.Errorf("%w: no content changes provided", jsonrpc2.ErrInternal)
	}

	// Check if the client sent the full content of the file.
	// We accept a full content change even if the server expected incremental changes.
	if len(changes) == 1 && changes[0].Range == nil && changes[0].RangeLength == 0 {
		return []byte(changes[0].Text), nil
	}
	return s.applyIncrementalChanges(ctx, uri, changes)
}

func (s *Server) applyIncrementalChanges(ctx context.Context, uri span.URI, changes []protocol.TextDocumentContentChangeEvent) ([]byte, error) {
	content, _, err := s.session.GetFile(uri).Read(ctx)
	if err != nil {
		return nil, fmt.Errorf("%w: file not found (%v)", jsonrpc2.ErrInternal, err)
	}
	for _, change := range changes {
		// Make sure to update column mapper along with the content.
		converter := span.NewContentConverter(uri.Filename(), content)
		m := &protocol.ColumnMapper{
			URI:       uri,
			Converter: converter,
			Content:   content,
		}
		if change.Range == nil {
			return nil, fmt.Errorf("%w: unexpected nil range for change", jsonrpc2.ErrInternal)
		}
		spn, err := m.RangeSpan(*change.Range)
		if err != nil {
			return nil, err
		}
		if !spn.HasOffset() {
			return nil, fmt.Errorf("%w: invalid range for content change", jsonrpc2.ErrInternal)
		}
		start, end := spn.Start().Offset(), spn.End().Offset()
		if end < start {
			return nil, fmt.Errorf("%w: invalid range for content change", jsonrpc2.ErrInternal)
		}
		var buf bytes.Buffer
		buf.Write(content[:start])
		buf.WriteString(change.Text)
		buf.Write(content[end:])
		content = buf.Bytes()
	}
	return content, nil
}

func changeTypeToFileAction(ct protocol.FileChangeType) source.FileAction {
	switch ct {
	case protocol.Changed:
		return source.Change
	case protocol.Created:
		return source.Create
	case protocol.Deleted:
		return source.Delete
	}
	return source.UnknownFileAction
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validation

import (
	"reflect"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	unversionedvalidation "k8s.io/kubernetes/pkg/api/unversioned/validation"
	apivalidation "k8s.io/kubernetes/pkg/api/validation"
	"k8s.io/kubernetes/pkg/apis/apps"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/util/validation/field"
)

// ValidateStatefulSetName can be used to check whether the given StatefulSet name is valid.
// Prefix indicates this name will be used as part of generation, in which case
// trailing dashes are allowed.
func ValidateStatefulSetName(name string, prefix bool) []string {
	// TODO: Validate that there's name for the suffix inserted by the pods.
	// Currently this is just "-index". In the future we may allow a user
	// specified list of suffixes and we need  to validate the longest one.
	return apivalidation.NameIsDNSSubdomain(name, prefix)
}

// Validates the given template and ensures that it is in accordance with the desired selector.
func ValidatePodTemplateSpecForStatefulSet(template *api.PodTemplateSpec, selector labels.Selector, fldPath *field.Path) field.ErrorList {
	allErrs := field.ErrorList{}
	if template == nil {
		allErrs = append(allErrs, field.Required(fldPath, ""))
	} else {
		if !selector.Empty() {
			// Verify that the StatefulSet selector matches the labels in template.
			labels := labels.Set(template.Labels)
			if !selector.Matches(labels) {
				allErrs = append(allErrs, field.Invalid(fldPath.Child("metadata", "labels"), template.Labels, "`selector` does not match template `labels`"))
			}
		}
		// TODO: Add validation for PodSpec, currently this will check volumes, which we know will
		// fail. We should really check that the union of the given volumes and volumeClaims match
		// volume mounts in the containers.
		// allErrs = append(allErrs, apivalidation.ValidatePodTemplateSpec(template, fldPath)...)
		allErrs = append(allErrs, unversionedvalidation.ValidateLabels(template.Labels, fldPath.Child("labels"))...)
		allErrs = append(allErrs, apivalidation.ValidateAnnotations(template.Annotations, fldPath.Child("annotations"))...)
		allErrs = append(allErrs, apivalidation.ValidatePodSpecificAnnotations(template.Annotations, &template.Spec, fldPath.Child("annotations"))...)
	}
	return allErrs
}

// ValidateStatefulSetSpec tests if required fields in the StatefulSet spec are set.
func ValidateStatefulSetSpec(spec *apps.StatefulSetSpec, fldPath *field.Path) field.ErrorList {
	allErrs := field.ErrorList{}

	allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(spec.Replicas), fldPath.Child("replicas"))...)
	if spec.Selector == nil {
		allErrs = append(allErrs, field.Required(fldPath.Child("selector"), ""))
	} else {
		allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(spec.Selector, fldPath.Child("selector"))...)
		if len(spec.Selector.MatchLabels)+len(spec.Selector.MatchExpressions) == 0 {
			allErrs = append(allErrs, field.Invalid(fldPath.Child("selector"), spec.Selector, "empty selector is not valid for statefulset."))
		}
	}

	selector, err := unversioned.LabelSelectorAsSelector(spec.Selector)
	if err != nil {
		allErrs = append(allErrs, field.Invalid(fldPath.Child("selector"), spec.Selector, ""))
	} else {
		allErrs = append(allErrs, ValidatePodTemplateSpecForStatefulSet(&spec.Template, selector, fldPath.Child("template"))...)
	}

	if spec.Template.Spec.RestartPolicy != api.RestartPolicyAlways {
		allErrs = append(allErrs, field.NotSupported(fldPath.Child("template", "spec", "restartPolicy"), spec.Template.Spec.RestartPolicy, []string{string(api.RestartPolicyAlways)}))
	}

	return allErrs
}

// ValidateStatefulSet validates a StatefulSet.
func ValidateStatefulSet(statefulSet *apps.StatefulSet) field.ErrorList {
	allErrs := apivalidation.ValidateObjectMeta(&statefulSet.ObjectMeta, true, ValidateStatefulSetName, field.NewPath("metadata"))
	allErrs = append(allErrs, ValidateStatefulSetSpec(&statefulSet.Spec, field.NewPath("spec"))...)
	return allErrs
}

// ValidateStatefulSetUpdate tests if required fields in the StatefulSet are set.
func ValidateStatefulSetUpdate(statefulSet, oldStatefulSet *apps.StatefulSet) field.ErrorList {
	allErrs := apivalidation.ValidateObjectMetaUpdate(&statefulSet.ObjectMeta, &oldStatefulSet.ObjectMeta, field.NewPath("metadata"))

	// TODO: For now we're taking the safe route and disallowing all updates to
	// spec except for Replicas, for scaling, and Template.Spec.containers.image
	// for rolling-update. Enable others on a case by case basis.
	restoreReplicas := statefulSet.Spec.Replicas
	statefulSet.Spec.Replicas = oldStatefulSet.Spec.Replicas

	restoreContainers := statefulSet.Spec.Template.Spec.Containers
	statefulSet.Spec.Template.Spec.Containers = oldStatefulSet.Spec.Template.Spec.Containers

	if !reflect.DeepEqual(statefulSet.Spec, oldStatefulSet.Spec) {
		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "updates to statefulset spec for fields other than 'replicas' are forbidden."))
	}
	statefulSet.Spec.Replicas = restoreReplicas
	statefulSet.Spec.Template.Spec.Containers = restoreContainers

	allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(statefulSet.Spec.Replicas), field.NewPath("spec", "replicas"))...)
	containerErrs, _ := apivalidation.ValidateContainerUpdates(statefulSet.Spec.Template.Spec.Containers, oldStatefulSet.Spec.Template.Spec.Containers, field.NewPath("spec").Child("template").Child("containers"))
	allErrs = append(allErrs, containerErrs...)
	return allErrs
}

// ValidateStatefulSetStatusUpdate tests if required fields in the StatefulSet are set.
func ValidateStatefulSetStatusUpdate(statefulSet, oldStatefulSet *apps.StatefulSet) field.ErrorList {
	allErrs := field.ErrorList{}
	allErrs = append(allErrs, apivalidation.ValidateObjectMetaUpdate(&statefulSet.ObjectMeta, &oldStatefulSet.ObjectMeta, field.NewPath("metadata"))...)
	// TODO: Validate status.
	return allErrs
}
<commit_msg>Update validation message.<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validation

import (
	"reflect"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	unversionedvalidation "k8s.io/kubernetes/pkg/api/unversioned/validation"
	apivalidation "k8s.io/kubernetes/pkg/api/validation"
	"k8s.io/kubernetes/pkg/apis/apps"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/util/validation/field"
)

// ValidateStatefulSetName can be used to check whether the given StatefulSet name is valid.
// Prefix indicates this name will be used as part of generation, in which case
// trailing dashes are allowed.
func ValidateStatefulSetName(name string, prefix bool) []string {
	// TODO: Validate that there's name for the suffix inserted by the pods.
	// Currently this is just "-index". In the future we may allow a user
	// specified list of suffixes and we need  to validate the longest one.
	return apivalidation.NameIsDNSSubdomain(name, prefix)
}

// Validates the given template and ensures that it is in accordance with the desired selector.
func ValidatePodTemplateSpecForStatefulSet(template *api.PodTemplateSpec, selector labels.Selector, fldPath *field.Path) field.ErrorList {
	allErrs := field.ErrorList{}
	if template == nil {
		allErrs = append(allErrs, field.Required(fldPath, ""))
	} else {
		if !selector.Empty() {
			// Verify that the StatefulSet selector matches the labels in template.
			labels := labels.Set(template.Labels)
			if !selector.Matches(labels) {
				allErrs = append(allErrs, field.Invalid(fldPath.Child("metadata", "labels"), template.Labels, "`selector` does not match template `labels`"))
			}
		}
		// TODO: Add validation for PodSpec, currently this will check volumes, which we know will
		// fail. We should really check that the union of the given volumes and volumeClaims match
		// volume mounts in the containers.
		// allErrs = append(allErrs, apivalidation.ValidatePodTemplateSpec(template, fldPath)...)
		allErrs = append(allErrs, unversionedvalidation.ValidateLabels(template.Labels, fldPath.Child("labels"))...)
		allErrs = append(allErrs, apivalidation.ValidateAnnotations(template.Annotations, fldPath.Child("annotations"))...)
		allErrs = append(allErrs, apivalidation.ValidatePodSpecificAnnotations(template.Annotations, &template.Spec, fldPath.Child("annotations"))...)
	}
	return allErrs
}

// ValidateStatefulSetSpec tests if required fields in the StatefulSet spec are set.
func ValidateStatefulSetSpec(spec *apps.StatefulSetSpec, fldPath *field.Path) field.ErrorList {
	allErrs := field.ErrorList{}

	allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(spec.Replicas), fldPath.Child("replicas"))...)
	if spec.Selector == nil {
		allErrs = append(allErrs, field.Required(fldPath.Child("selector"), ""))
	} else {
		allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(spec.Selector, fldPath.Child("selector"))...)
		if len(spec.Selector.MatchLabels)+len(spec.Selector.MatchExpressions) == 0 {
			allErrs = append(allErrs, field.Invalid(fldPath.Child("selector"), spec.Selector, "empty selector is not valid for statefulset."))
		}
	}

	selector, err := unversioned.LabelSelectorAsSelector(spec.Selector)
	if err != nil {
		allErrs = append(allErrs, field.Invalid(fldPath.Child("selector"), spec.Selector, ""))
	} else {
		allErrs = append(allErrs, ValidatePodTemplateSpecForStatefulSet(&spec.Template, selector, fldPath.Child("template"))...)
	}

	if spec.Template.Spec.RestartPolicy != api.RestartPolicyAlways {
		allErrs = append(allErrs, field.NotSupported(fldPath.Child("template", "spec", "restartPolicy"), spec.Template.Spec.RestartPolicy, []string{string(api.RestartPolicyAlways)}))
	}

	return allErrs
}

// ValidateStatefulSet validates a StatefulSet.
func ValidateStatefulSet(statefulSet *apps.StatefulSet) field.ErrorList {
	allErrs := apivalidation.ValidateObjectMeta(&statefulSet.ObjectMeta, true, ValidateStatefulSetName, field.NewPath("metadata"))
	allErrs = append(allErrs, ValidateStatefulSetSpec(&statefulSet.Spec, field.NewPath("spec"))...)
	return allErrs
}

// ValidateStatefulSetUpdate tests if required fields in the StatefulSet are set.
func ValidateStatefulSetUpdate(statefulSet, oldStatefulSet *apps.StatefulSet) field.ErrorList {
	allErrs := apivalidation.ValidateObjectMetaUpdate(&statefulSet.ObjectMeta, &oldStatefulSet.ObjectMeta, field.NewPath("metadata"))

	// TODO: For now we're taking the safe route and disallowing all updates to
	// spec except for Replicas, for scaling, and Template.Spec.containers.image
	// for rolling-update. Enable others on a case by case basis.
	restoreReplicas := statefulSet.Spec.Replicas
	statefulSet.Spec.Replicas = oldStatefulSet.Spec.Replicas

	restoreContainers := statefulSet.Spec.Template.Spec.Containers
	statefulSet.Spec.Template.Spec.Containers = oldStatefulSet.Spec.Template.Spec.Containers

	if !reflect.DeepEqual(statefulSet.Spec, oldStatefulSet.Spec) {
		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "updates to statefulset spec for fields other than 'replicas' and 'containers' is forbidden."))
	}
	statefulSet.Spec.Replicas = restoreReplicas
	statefulSet.Spec.Template.Spec.Containers = restoreContainers

	allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(statefulSet.Spec.Replicas), field.NewPath("spec", "replicas"))...)
	containerErrs, _ := apivalidation.ValidateContainerUpdates(statefulSet.Spec.Template.Spec.Containers, oldStatefulSet.Spec.Template.Spec.Containers, field.NewPath("spec").Child("template").Child("containers"))
	allErrs = append(allErrs, containerErrs...)
	return allErrs
}

// ValidateStatefulSetStatusUpdate tests if required fields in the StatefulSet are set.
func ValidateStatefulSetStatusUpdate(statefulSet, oldStatefulSet *apps.StatefulSet) field.ErrorList {
	allErrs := field.ErrorList{}
	allErrs = append(allErrs, apivalidation.ValidateObjectMetaUpdate(&statefulSet.ObjectMeta, &oldStatefulSet.ObjectMeta, field.NewPath("metadata"))...)
	// TODO: Validate status.
	return allErrs
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"time"
	"bufio"
	"strings"
	"strconv"
	"fmt"
	"net/http"
	_ "net/http/pprof"
)

func (self *AlmazServer) StartHttpface(bindAddress string) {
    log.Printf("Http interface available at %s", bindAddress)
    http.HandleFunc("/", self.http_main)
    http.HandleFunc("/list/all/", self.http_list_all)
    http.HandleFunc("/list/group/", self.http_list_group)
    http.HandleFunc("/almaz/list/all/", self.http_list_all)
    http.HandleFunc("/almaz/list/group/", self.http_list_group)
    http.ListenAndServe(bindAddress, nil)
}

func (self *AlmazServer) http_main(w http.ResponseWriter, r *http.Request) {
	self.RLock()
	defer self.RUnlock()
	fmt.Fprintf(w, "Metrics count: %d\n", self.storage.MetricCount())
}

func (self *AlmazServer) http_list_all(w http.ResponseWriter, r *http.Request) {
	self.RLock()
	defer self.RUnlock()

	periods := []int64{60, 15*60, 60*60, 4*60*60, 24*60*60}
	now := time.Now().Unix()

	for k := range self.storage.metrics {
		fmt.Fprintf(w, "%s", k)
		counts_per_period := self.storage.metrics[k].GetSumsPerPeriodUntilNow(periods, now)
		for _, el := range counts_per_period {
			fmt.Fprintf(w, "\t%f", el)
		}
		fmt.Fprintf(w, "\n")
	}
}

func (self *AlmazServer) http_list_group(w http.ResponseWriter, r *http.Request) {
	self.RLock()
	defer self.RUnlock()

	now := time.Now().Unix()

	defer r.Body.Close()
	scanner := bufio.NewScanner(r.Body)
	ok := scanner.Scan()
	if !ok {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "400 Bad Request\r\n")
		fmt.Fprintf(w, "Use POST method and specify period durations in seconds ")
		fmt.Fprintf(w, "on the first line of request data.\n")
		return
	}
	periods_str := strings.Split(scanner.Text(), " ")

	periods := make([]int64, len(periods_str))
	for i := range periods_str {
		period, err := strconv.ParseInt(periods_str[i], 10, 64)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "400 Bad Request\r\n")
			fmt.Fprintf(w, "%s", err)
			return
		}
		periods[i] = period
	}

	groups := make([]string, 0)
	for scanner.Scan() {
		groups = append(groups, scanner.Text())
	}

	var results [][]float64
	results = self.storage.SumByPeriodGroupingQuery(groups, periods, now, true)
	for k := range groups {
		fmt.Fprintf(w, "%s", groups[k])
		for _, el := range results[k] {
			fmt.Fprintf(w, "\t%f", el)
		}
		fmt.Fprintf(w, "\n")
	}
}
<commit_msg>separate http views for hard and interpolated counters<commit_after>package main

import (
	"log"
	"time"
	"bufio"
	"strings"
	"strconv"
	"fmt"
	"net/http"
	_ "net/http/pprof"
)

func (self *AlmazServer) StartHttpface(bindAddress string) {
    log.Printf("Http interface available at %s", bindAddress)
    http.HandleFunc("/", self.http_main)
    http.HandleFunc("/list/all/", self.http_list_all)
    http.HandleFunc("/list/all-interpolated/", self.http_list_all_smooth)
    http.HandleFunc("/list/group/", self.http_list_group)
    http.HandleFunc("/almaz/list/all/", self.http_list_all)
    http.HandleFunc("/almaz/list/all-interpolated/", self.http_list_all_smooth)
    http.HandleFunc("/almaz/list/group/", self.http_list_group)
    http.ListenAndServe(bindAddress, nil)
}

func (self *AlmazServer) http_main(w http.ResponseWriter, r *http.Request) {
	self.RLock()
	defer self.RUnlock()
	fmt.Fprintf(w, "Metrics count: %d\n", self.storage.MetricCount())
}

func (self *AlmazServer) http_list_all(w http.ResponseWriter, r *http.Request) {
	self.http_list_all_with_interpolation(w, r, false)
}

func (self *AlmazServer) http_list_all_smooth(w http.ResponseWriter, r *http.Request) {
	self.http_list_all_with_interpolation(w, r, true)
}

func (self *AlmazServer) http_list_all_with_interpolation(w http.ResponseWriter, r *http.Request, interpolate bool) {
	self.RLock()
	defer self.RUnlock()

	periods := []int64{60, 15*60, 60*60, 4*60*60, 24*60*60}
	now := time.Now().Unix()

	for k := range self.storage.metrics {
		fmt.Fprintf(w, "%s", k)
		counts_per_period := self.storage.metrics[k].GetSumsPerPeriodUntilNowWithInterpolation(periods, now, interpolate)
		for _, el := range counts_per_period {
			fmt.Fprintf(w, "\t%f", el)
		}
		fmt.Fprintf(w, "\n")
	}
}

func (self *AlmazServer) http_list_group(w http.ResponseWriter, r *http.Request) {
	self.RLock()
	defer self.RUnlock()

	now := time.Now().Unix()

	defer r.Body.Close()
	scanner := bufio.NewScanner(r.Body)
	ok := scanner.Scan()
	if !ok {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "400 Bad Request\r\n")
		fmt.Fprintf(w, "Use POST method and specify period durations in seconds ")
		fmt.Fprintf(w, "on the first line of request data.\n")
		return
	}
	periods_str := strings.Split(scanner.Text(), " ")

	periods := make([]int64, len(periods_str))
	for i := range periods_str {
		period, err := strconv.ParseInt(periods_str[i], 10, 64)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "400 Bad Request\r\n")
			fmt.Fprintf(w, "%s", err)
			return
		}
		periods[i] = period
	}

	groups := make([]string, 0)
	for scanner.Scan() {
		groups = append(groups, scanner.Text())
	}

	var results [][]float64
	results = self.storage.SumByPeriodGroupingQuery(groups, periods, now, true)
	for k := range groups {
		fmt.Fprintf(w, "%s", groups[k])
		for _, el := range results[k] {
			fmt.Fprintf(w, "\t%f", el)
		}
		fmt.Fprintf(w, "\n")
	}
}
<|endoftext|>


<commit_before>package connectors

import (
	"encoding/json"
	"fmt"
	"github.com/projectjane/jane/models"
	"github.com/projectjane/jane/parse"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
)

// ImageMe Struct representing the image me connector
type ImageMe struct {
}

//Listen Not implemented
func (x ImageMe) Listen(commandMsgs chan<- models.Message, connector models.Connector) {
	defer Recovery(connector)
	return
}

// Command Takes in animateme or imageme command
func (x ImageMe) Command(message models.Message, publishMsgs chan<- models.Message, connector models.Connector) {
	if match, tokens := parse.Match("image me*", message.In.Text); match {
		message.In.Tags = parse.TagAppend(message.In.Tags, connector.Tags)
		message.Out.Text = callImageMe(tokens["*"], connector.KeyValues["Key"], connector.Pass, false)
		publishMsgs <- message
	}
	if match, tokens := parse.Match("animate me*", message.In.Text); match {
		message.In.Tags = parse.TagAppend(message.In.Tags, connector.Tags)
		message.Out.Text = callImageMe(tokens["*"], connector.KeyValues["Key"], connector.Pass, true)
		publishMsgs <- message
	}
}

// Publish Not implemented
func (x ImageMe) Publish(publishMsgs <-chan models.Message, connector models.Connector) {
	return
}

// Help Returns help data
func (x ImageMe) Help(connector models.Connector) (help []string) {
	help = make([]string, 0)
	help = append(help, "image me <image keywords> - pulls back an image url")
	help = append(help, "animate me <image keywords> - pulls back an animated gif url")
	return help
}

type searchResult struct {
	Items []items `json:"items"`
}

type items struct {
	Link string `json:"link"`
}

var imageClient = &http.Client{}

var baseURL = "https://www.googleapis.com/customsearch/v1?key="
var errorMessage = "Error retrieving image"
var animated bool

func callImageMe(msg string, apiKey string, cx string, animated bool) string {
	start := rand.Intn(3)
	if start < 1 {
		start = 1
	}

	cx = "&cx=" + cx
	returnFields := fmt.Sprintf("&fields=items(link)&start=%v", start)
	query := "&q=" + url.QueryEscape(msg)
	fields := "&searchType=image"
	if animated {
		fields += "&fileType=gif&hq=animated&tbs=itp:animated"
	}

	url := baseURL + apiKey + cx + returnFields + query + fields

	resp, err := imageClient.Get(url)
	if err != nil {
		log.Print(err)
		return errorMessage
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return findDeprecatedImage(msg, animated)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Print(err)
		return errorMessage
	}

	var result searchResult
	err = json.Unmarshal(body, &result)
	if err != nil {
		log.Print(err)
		return errorMessage
	}

	if len(result.Items) > 0 {
		randomLink := result.Items[rand.Intn(len(result.Items))]
		return randomLink.Link
	}

	return findDeprecatedImage(msg, animated)
}

type deprecatedResult struct {
	ResponseData responseData `json:"responseData"`
}

type responseData struct {
	Results []result `json:"results"`
}

type result struct {
	URL string `json:"url"`
}

func findDeprecatedImage(query string, animated bool) string {
	baseURL := "https://ajax.googleapis.com/ajax/services/search/images?v=1.0&rsz=8"
	if animated {
		baseURL += "&as_filetype=gif"
	}

	baseURL += "&q="
	searchURL := baseURL + url.QueryEscape(query)

	if animated {
		searchURL += url.QueryEscape(" animated")
	}

	resp, err := imageClient.Get(searchURL)
	if err != nil {
		log.Print(err)
		return errorMessage
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Print(err)
		return errorMessage
	}

	var result deprecatedResult
	err = json.Unmarshal(body, &result)
	if err != nil {
		log.Print(err)
		return errorMessage
	}

	index := rand.Intn(len(result.ResponseData.Results))

	if len(result.ResponseData.Results) > 0 {
		return result.ResponseData.Results[index].URL
	}

	return "No results found"
}
<commit_msg>Rolling back a change to imageme<commit_after>package connectors

import (
	"encoding/json"
	"fmt"
	"github.com/projectjane/jane/models"
	"github.com/projectjane/jane/parse"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
)

// ImageMe Struct representing the image me connector
type ImageMe struct {
}

//Listen Not implemented
func (x ImageMe) Listen(commandMsgs chan<- models.Message, connector models.Connector) {
	defer Recovery(connector)
	return
}

// Command Takes in animateme or imageme command
func (x ImageMe) Command(message models.Message, publishMsgs chan<- models.Message, connector models.Connector) {
	if match, tokens := parse.Match("image me*", message.In.Text); match {
		message.In.Tags = parse.TagAppend(message.In.Tags, connector.Tags)
		message.Out.Text = callImageMe(tokens["*"], connector.Key, connector.Pass, false)
		publishMsgs <- message
	}
	if match, tokens := parse.Match("animate me*", message.In.Text); match {
		message.In.Tags = parse.TagAppend(message.In.Tags, connector.Tags)
		message.Out.Text = callImageMe(tokens["*"], connector.Key, connector.Pass, true)
		publishMsgs <- message
	}
}

// Publish Not implemented
func (x ImageMe) Publish(publishMsgs <-chan models.Message, connector models.Connector) {
	return
}

// Help Returns help data
func (x ImageMe) Help(connector models.Connector) (help []string) {
	help = make([]string, 0)
	help = append(help, "image me <image keywords> - pulls back an image url")
	help = append(help, "animate me <image keywords> - pulls back an animated gif url")
	return help
}

type searchResult struct {
	Items []items `json:"items"`
}

type items struct {
	Link string `json:"link"`
}

var imageClient = &http.Client{}

var baseURL = "https://www.googleapis.com/customsearch/v1?key="
var errorMessage = "Error retrieving image"
var animated bool

func callImageMe(msg string, apiKey string, cx string, animated bool) string {
	start := rand.Intn(3)
	if start < 1 {
		start = 1
	}

	cx = "&cx=" + cx
	returnFields := fmt.Sprintf("&fields=items(link)&start=%v", start)
	query := "&q=" + url.QueryEscape(msg)
	fields := "&searchType=image"
	if animated {
		fields += "&fileType=gif&hq=animated&tbs=itp:animated"
	}

	url := baseURL + apiKey + cx + returnFields + query + fields

	resp, err := imageClient.Get(url)
	if err != nil {
		log.Print(err)
		return errorMessage
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return findDeprecatedImage(msg, animated)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Print(err)
		return errorMessage
	}

	var result searchResult
	err = json.Unmarshal(body, &result)
	if err != nil {
		log.Print(err)
		return errorMessage
	}

	if len(result.Items) > 0 {
		randomLink := result.Items[rand.Intn(len(result.Items))]
		return randomLink.Link
	}

	return findDeprecatedImage(msg, animated)
}

type deprecatedResult struct {
	ResponseData responseData `json:"responseData"`
}

type responseData struct {
	Results []result `json:"results"`
}

type result struct {
	URL string `json:"url"`
}

func findDeprecatedImage(query string, animated bool) string {
	baseURL := "https://ajax.googleapis.com/ajax/services/search/images?v=1.0&rsz=8"
	if animated {
		baseURL += "&as_filetype=gif"
	}

	baseURL += "&q="
	searchURL := baseURL + url.QueryEscape(query)

	if animated {
		searchURL += url.QueryEscape(" animated")
	}

	resp, err := imageClient.Get(searchURL)
	if err != nil {
		log.Print(err)
		return errorMessage
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Print(err)
		return errorMessage
	}

	var result deprecatedResult
	err = json.Unmarshal(body, &result)
	if err != nil {
		log.Print(err)
		return errorMessage
	}

	index := rand.Intn(len(result.ResponseData.Results))

	if len(result.ResponseData.Results) > 0 {
		return result.ResponseData.Results[index].URL
	}

	return "No results found"
}
<|endoftext|>