<commit_before>package aggregator

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/percona/go-mysql/event"
	"github.com/percona/percona-toolkit/src/go/mongolib/fingerprinter"
	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
	mongostats "github.com/percona/percona-toolkit/src/go/mongolib/stats"
	pc "github.com/percona/pmm/proto/config"
	"github.com/percona/pmm/proto/qan"
	"github.com/percona/qan-agent/qan/analyzer/mongo/status"
	"github.com/percona/qan-agent/qan/analyzer/report"
)

const (
	DefaultInterval       = 60 // in seconds
	DefaultExampleQueries = true
	ReportChanBuffer      = 1000
)

// New returns configured *Aggregator
func New(timeStart time.Time, config pc.QAN) *Aggregator {
	// verify config
	if config.Interval == 0 {
		config.Interval = DefaultInterval
		*config.ExampleQueries = DefaultExampleQueries
	}

	aggregator := &Aggregator{
		config: config,
	}

	// create duration from interval
	aggregator.d = time.Duration(config.Interval) * time.Second

	// create mongolib stats
	fp := fingerprinter.NewFingerprinter(fingerprinter.DEFAULT_KEY_FILTERS)
	aggregator.mongostats = mongostats.New(fp)

	// create new interval
	aggregator.newInterval(timeStart)

	return aggregator
}

// Aggregator aggregates system.profile document
type Aggregator struct {
	// dependencies
	config pc.QAN

	// status
	status *status.Status
	stats  *stats

	// provides
	reportChan chan *qan.Report

	// interval
	timeStart  time.Time
	timeEnd    time.Time
	d          time.Duration
	t          *time.Timer
	mongostats *mongostats.Stats

	// state
	sync.RWMutex                 // Lock() to protect internal consistency of the service
	running      bool            // Is this service running?
	doneChan     chan struct{}   // close(doneChan) to notify goroutines that they should shutdown
	wg           *sync.WaitGroup // Wait() for goroutines to stop after being notified they should shutdown
}

// Add aggregates new system.profile document
func (self *Aggregator) Add(doc proto.SystemProfile) error {
	self.Lock()
	defer self.Unlock()
	if !self.running {
		return fmt.Errorf("aggregator is not running")
	}

	ts := doc.Ts.UTC()

	// skip old metrics
	if ts.Before(self.timeStart) {
		self.stats.DocsSkippedOld.Add(1)
		return nil
	}

	// if new doc is outside of interval then finish old interval and flush it
	if !ts.Before(self.timeEnd) {
		self.flush(ts)
	}

	// we had some activity so reset timer
	self.t.Reset(self.d)

	// add new doc to stats
	self.stats.DocsIn.Add(1)
	return self.mongostats.Add(doc)
}

func (self *Aggregator) Start() <-chan *qan.Report {
	self.Lock()
	defer self.Unlock()
	if self.running {
		return self.reportChan
	}

	// create new channels over which we will communicate to...
	// ... outside world by sending collected docs
	self.reportChan = make(chan *qan.Report, ReportChanBuffer)
	// ... inside goroutine to close it
	self.doneChan = make(chan struct{})

	// set status
	self.stats = &stats{}
	self.status = status.New(self.stats)

	// timeout after not receiving data for interval time
	self.t = time.NewTimer(self.d)

	// start a goroutine and Add() it to WaitGroup
	// so we could later Wait() for it to finish
	self.wg = &sync.WaitGroup{}
	self.wg.Add(1)
	go start(
		self.wg,
		self,
		self.doneChan,
		self.stats,
	)

	self.running = true
	return self.reportChan
}

func (self *Aggregator) Stop() {
	self.Lock()
	defer self.Unlock()
	if !self.running {
		return
	}
	self.running = false

	// notify goroutine to close
	close(self.doneChan)

	// wait for goroutines to exit
	self.wg.Wait()

	// close reportChan
	close(self.reportChan)
}

func (self *Aggregator) Status() map[string]string {
	self.RLock()
	defer self.RUnlock()
	if !self.running {
		return nil
	}

	return self.status.Map()
}

func start(
	wg *sync.WaitGroup,
	aggregator *Aggregator,
	doneChan <-chan struct{},
	stats *stats,
) {
	// signal WaitGroup when goroutine finished
	defer wg.Done()

	// update stats
	stats.IntervalStart.Set(aggregator.TimeStart().Format("2006-01-02 15:04:05"))
	stats.IntervalEnd.Set(aggregator.TimeEnd().Format("2006-01-02 15:04:05"))
	for {
		select {
		case <-aggregator.t.C:
			// When Tail()ing system.profile collection you don't know if sample
			// is last sample in the collection until you get sample with higher timestamp than interval.
			// For this, in cases where we generate only few test queries,
			// but still expect them to show after interval expires, we need to implement timeout.
			// This introduces another issue, that in case something goes wrong, and we get metrics for old interval too late, they will be skipped.
			// A proper solution would be to allow fixing old samples, but API and qan-agent doesn't allow this, yet.
			aggregator.Flush()
		case <-doneChan:
			// Check if we should shutdown.
			return
		}
	}
}

func (self *Aggregator) Flush() {
	self.Lock()
	defer self.Unlock()
	self.flush(time.Now())
}

func (self *Aggregator) flush(ts time.Time) {
	r := self.interval(ts)
	if r != nil {
		self.reportChan <- r
		self.stats.ReportsOut.Add(1)
	}
}

// interval sets interval if necessary and returns *qan.Report for old interval if not empty
func (self *Aggregator) interval(ts time.Time) *qan.Report {
	// create new interval
	defer self.newInterval(ts)

	// let's check if we have anything to send for current interval
	if len(self.mongostats.Queries()) == 0 {
		// if there are no queries then we don't create report #PMM-927
		return nil
	}

	// create result
	result := self.createResult()

	// translate result into report and return it
	return report.MakeReport(self.config, self.timeStart, self.timeEnd, nil, result)
}

// TimeStart returns start time for current interval
func (self *Aggregator) TimeStart() time.Time {
	return self.timeStart
}

// TimeEnd returns end time for current interval
func (self *Aggregator) TimeEnd() time.Time {
	return self.timeEnd
}

func (self *Aggregator) newInterval(ts time.Time) {
	// reset stats
	self.mongostats.Reset()

	// truncate to the duration e.g 12:15:35 with 1 minute duration it will be 12:15:00
	self.timeStart = ts.UTC().Truncate(self.d)
	// create ending time by adding interval
	self.timeEnd = self.timeStart.Add(self.d)
}

func (self *Aggregator) createResult() *report.Result {
	queries := self.mongostats.Queries()
	global := event.NewClass("", "", false)
	queryStats := queries.CalcQueriesStats(int64(self.config.Interval))
	classes := []*event.Class{}
	for _, queryInfo := range queryStats {
		class := event.NewClass(queryInfo.ID, queryInfo.Fingerprint, *self.config.ExampleQueries)
		if *self.config.ExampleQueries {
			db := ""
			s := strings.SplitN(queryInfo.Namespace, ".", 2)
			if len(s) == 2 {
				db = s[0]
			}

			class.Example = &event.Example{
				QueryTime: queryInfo.QueryTime.Total,
				Db:        db,
				Query:     queryInfo.Query,
			}
		}

		metrics := event.NewMetrics()

		metrics.TimeMetrics["Query_time"] = newEventTimeStatsInMilliseconds(queryInfo.QueryTime)

		// @todo we map below metrics to MySQL equivalents according to PMM-830
		metrics.NumberMetrics["Bytes_sent"] = newEventNumberStats(queryInfo.ResponseLength)
		metrics.NumberMetrics["Rows_sent"] = newEventNumberStats(queryInfo.Returned)
		metrics.NumberMetrics["Rows_examined"] = newEventNumberStats(queryInfo.Scanned)

		class.Metrics = metrics
		class.TotalQueries = uint(queryInfo.Count)
		class.UniqueQueries = 1
		classes = append(classes, class)

		// Add the class to the global metrics.
		global.AddClass(class)
	}

	return &report.Result{
		Global: global,
		Class:  classes,
	}

}

func newEventNumberStats(s mongostats.Statistics) *event.NumberStats {
	return &event.NumberStats{
		Sum: uint64(s.Total),
		Min: event.Uint64(uint64(s.Min)),
		Avg: event.Uint64(uint64(s.Avg)),
		Med: event.Uint64(uint64(s.Median)),
		P95: event.Uint64(uint64(s.Pct95)),
		Max: event.Uint64(uint64(s.Max)),
	}
}

func newEventTimeStatsInMilliseconds(s mongostats.Statistics) *event.TimeStats {
	return &event.TimeStats{
		Sum: s.Total / 1000,
		Min: event.Float64(s.Min / 1000),
		Avg: event.Float64(s.Avg / 1000),
		Med: event.Float64(s.Median / 1000),
		P95: event.Float64(s.Pct95 / 1000),
		Max: event.Float64(s.Max / 1000),
	}
}
<commit_msg>PMM-2214: Fix passing pointer bool.<commit_after>package aggregator

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/percona/go-mysql/event"
	"github.com/percona/percona-toolkit/src/go/mongolib/fingerprinter"
	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
	mongostats "github.com/percona/percona-toolkit/src/go/mongolib/stats"
	pc "github.com/percona/pmm/proto/config"
	"github.com/percona/pmm/proto/qan"
	"github.com/percona/qan-agent/qan/analyzer/mongo/status"
	"github.com/percona/qan-agent/qan/analyzer/report"
)

const (
	DefaultInterval       = 60 // in seconds
	DefaultExampleQueries = true
	ReportChanBuffer      = 1000
)

// New returns configured *Aggregator
func New(timeStart time.Time, config pc.QAN) *Aggregator {
	defaultExampleQueries := DefaultExampleQueries
	// verify config
	if config.Interval == 0 {
		config.Interval = DefaultInterval
		config.ExampleQueries = &defaultExampleQueries
	}

	aggregator := &Aggregator{
		config: config,
	}

	// create duration from interval
	aggregator.d = time.Duration(config.Interval) * time.Second

	// create mongolib stats
	fp := fingerprinter.NewFingerprinter(fingerprinter.DEFAULT_KEY_FILTERS)
	aggregator.mongostats = mongostats.New(fp)

	// create new interval
	aggregator.newInterval(timeStart)

	return aggregator
}

// Aggregator aggregates system.profile document
type Aggregator struct {
	// dependencies
	config pc.QAN

	// status
	status *status.Status
	stats  *stats

	// provides
	reportChan chan *qan.Report

	// interval
	timeStart  time.Time
	timeEnd    time.Time
	d          time.Duration
	t          *time.Timer
	mongostats *mongostats.Stats

	// state
	sync.RWMutex                 // Lock() to protect internal consistency of the service
	running      bool            // Is this service running?
	doneChan     chan struct{}   // close(doneChan) to notify goroutines that they should shutdown
	wg           *sync.WaitGroup // Wait() for goroutines to stop after being notified they should shutdown
}

// Add aggregates new system.profile document
func (self *Aggregator) Add(doc proto.SystemProfile) error {
	self.Lock()
	defer self.Unlock()
	if !self.running {
		return fmt.Errorf("aggregator is not running")
	}

	ts := doc.Ts.UTC()

	// skip old metrics
	if ts.Before(self.timeStart) {
		self.stats.DocsSkippedOld.Add(1)
		return nil
	}

	// if new doc is outside of interval then finish old interval and flush it
	if !ts.Before(self.timeEnd) {
		self.flush(ts)
	}

	// we had some activity so reset timer
	self.t.Reset(self.d)

	// add new doc to stats
	self.stats.DocsIn.Add(1)
	return self.mongostats.Add(doc)
}

func (self *Aggregator) Start() <-chan *qan.Report {
	self.Lock()
	defer self.Unlock()
	if self.running {
		return self.reportChan
	}

	// create new channels over which we will communicate to...
	// ... outside world by sending collected docs
	self.reportChan = make(chan *qan.Report, ReportChanBuffer)
	// ... inside goroutine to close it
	self.doneChan = make(chan struct{})

	// set status
	self.stats = &stats{}
	self.status = status.New(self.stats)

	// timeout after not receiving data for interval time
	self.t = time.NewTimer(self.d)

	// start a goroutine and Add() it to WaitGroup
	// so we could later Wait() for it to finish
	self.wg = &sync.WaitGroup{}
	self.wg.Add(1)
	go start(
		self.wg,
		self,
		self.doneChan,
		self.stats,
	)

	self.running = true
	return self.reportChan
}

func (self *Aggregator) Stop() {
	self.Lock()
	defer self.Unlock()
	if !self.running {
		return
	}
	self.running = false

	// notify goroutine to close
	close(self.doneChan)

	// wait for goroutines to exit
	self.wg.Wait()

	// close reportChan
	close(self.reportChan)
}

func (self *Aggregator) Status() map[string]string {
	self.RLock()
	defer self.RUnlock()
	if !self.running {
		return nil
	}

	return self.status.Map()
}

func start(
	wg *sync.WaitGroup,
	aggregator *Aggregator,
	doneChan <-chan struct{},
	stats *stats,
) {
	// signal WaitGroup when goroutine finished
	defer wg.Done()

	// update stats
	stats.IntervalStart.Set(aggregator.TimeStart().Format("2006-01-02 15:04:05"))
	stats.IntervalEnd.Set(aggregator.TimeEnd().Format("2006-01-02 15:04:05"))
	for {
		select {
		case <-aggregator.t.C:
			// When Tail()ing system.profile collection you don't know if sample
			// is last sample in the collection until you get sample with higher timestamp than interval.
			// For this, in cases where we generate only few test queries,
			// but still expect them to show after interval expires, we need to implement timeout.
			// This introduces another issue, that in case something goes wrong, and we get metrics for old interval too late, they will be skipped.
			// A proper solution would be to allow fixing old samples, but API and qan-agent doesn't allow this, yet.
			aggregator.Flush()
		case <-doneChan:
			// Check if we should shutdown.
			return
		}
	}
}

func (self *Aggregator) Flush() {
	self.Lock()
	defer self.Unlock()
	self.flush(time.Now())
}

func (self *Aggregator) flush(ts time.Time) {
	r := self.interval(ts)
	if r != nil {
		self.reportChan <- r
		self.stats.ReportsOut.Add(1)
	}
}

// interval sets interval if necessary and returns *qan.Report for old interval if not empty
func (self *Aggregator) interval(ts time.Time) *qan.Report {
	// create new interval
	defer self.newInterval(ts)

	// let's check if we have anything to send for current interval
	if len(self.mongostats.Queries()) == 0 {
		// if there are no queries then we don't create report #PMM-927
		return nil
	}

	// create result
	result := self.createResult()

	// translate result into report and return it
	return report.MakeReport(self.config, self.timeStart, self.timeEnd, nil, result)
}

// TimeStart returns start time for current interval
func (self *Aggregator) TimeStart() time.Time {
	return self.timeStart
}

// TimeEnd returns end time for current interval
func (self *Aggregator) TimeEnd() time.Time {
	return self.timeEnd
}

func (self *Aggregator) newInterval(ts time.Time) {
	// reset stats
	self.mongostats.Reset()

	// truncate to the duration e.g 12:15:35 with 1 minute duration it will be 12:15:00
	self.timeStart = ts.UTC().Truncate(self.d)
	// create ending time by adding interval
	self.timeEnd = self.timeStart.Add(self.d)
}

func (self *Aggregator) createResult() *report.Result {
	queries := self.mongostats.Queries()
	global := event.NewClass("", "", false)
	queryStats := queries.CalcQueriesStats(int64(self.config.Interval))
	classes := []*event.Class{}
	exampleQueries := boolValue(self.config.ExampleQueries)
	for _, queryInfo := range queryStats {
		class := event.NewClass(queryInfo.ID, queryInfo.Fingerprint, exampleQueries)
		if exampleQueries {
			db := ""
			s := strings.SplitN(queryInfo.Namespace, ".", 2)
			if len(s) == 2 {
				db = s[0]
			}

			class.Example = &event.Example{
				QueryTime: queryInfo.QueryTime.Total,
				Db:        db,
				Query:     queryInfo.Query,
			}
		}

		metrics := event.NewMetrics()

		metrics.TimeMetrics["Query_time"] = newEventTimeStatsInMilliseconds(queryInfo.QueryTime)

		// @todo we map below metrics to MySQL equivalents according to PMM-830
		metrics.NumberMetrics["Bytes_sent"] = newEventNumberStats(queryInfo.ResponseLength)
		metrics.NumberMetrics["Rows_sent"] = newEventNumberStats(queryInfo.Returned)
		metrics.NumberMetrics["Rows_examined"] = newEventNumberStats(queryInfo.Scanned)

		class.Metrics = metrics
		class.TotalQueries = uint(queryInfo.Count)
		class.UniqueQueries = 1
		classes = append(classes, class)

		// Add the class to the global metrics.
		global.AddClass(class)
	}

	return &report.Result{
		Global: global,
		Class:  classes,
	}

}

func newEventNumberStats(s mongostats.Statistics) *event.NumberStats {
	return &event.NumberStats{
		Sum: uint64(s.Total),
		Min: event.Uint64(uint64(s.Min)),
		Avg: event.Uint64(uint64(s.Avg)),
		Med: event.Uint64(uint64(s.Median)),
		P95: event.Uint64(uint64(s.Pct95)),
		Max: event.Uint64(uint64(s.Max)),
	}
}

func newEventTimeStatsInMilliseconds(s mongostats.Statistics) *event.TimeStats {
	return &event.TimeStats{
		Sum: s.Total / 1000,
		Min: event.Float64(s.Min / 1000),
		Avg: event.Float64(s.Avg / 1000),
		Med: event.Float64(s.Median / 1000),
		P95: event.Float64(s.Pct95 / 1000),
		Max: event.Float64(s.Max / 1000),
	}
}

// boolValue returns the value of the bool pointer passed in or
// false if the pointer is nil.
func boolValue(v *bool) bool {
	if v != nil {
		return *v
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package filter

import (
	"fmt"
	"net/url"
	"testing"
)

func TestFilterDuplicates(t *testing.T) {
	fmt.Println("Adding tasks...")
	src := make(chan *url.URL, 5)
	src <- &url.URL{Path: "/a"}
	src <- &url.URL{Path: "/b"}
	src <- &url.URL{Path: "/a"}
	src <- &url.URL{Path: "/c"}
	src <- &url.URL{Path: "/a"}
	dupes := 0
	dupefunc := func(i int) { dupes += i }
	filter := NewWorkFilter(&ScanSettings{}, dupefunc)
	fmt.Println("Starting filtering...")
	close(src)
	out := filter.Filter(src)
	for _, p := range []string{"/a", "/b", "/c"} {
		if u, ok := <-out; ok {
			if u.Path != p {
				t.Errorf("Expected %s, got %s.", p, u.Path)
			}
		} else {
			t.Error("Expected output, channel was closed.")
		}
	}
	if _, ok := <-out; ok {
		t.Error("Expected closed channel, got read.")
	}
	if dupes != 2 {
		t.Errorf("Expected 2 dupes, got %d", dupes)
	}
}
<commit_msg>Fix filter testing.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package filter

import (
	"fmt"
	"github.com/Matir/gobuster/settings"
	"net/url"
	"testing"
)

func TestFilterDuplicates(t *testing.T) {
	fmt.Println("Adding tasks...")
	src := make(chan *url.URL, 5)
	src <- &url.URL{Path: "/a"}
	src <- &url.URL{Path: "/b"}
	src <- &url.URL{Path: "/a"}
	src <- &url.URL{Path: "/c"}
	src <- &url.URL{Path: "/a"}
	dupes := 0
	dupefunc := func(i int) { dupes += i }
	filter := NewWorkFilter(&settings.ScanSettings{}, dupefunc)
	fmt.Println("Starting filtering...")
	close(src)
	out := filter.Filter(src)
	for _, p := range []string{"/a", "/b", "/c"} {
		if u, ok := <-out; ok {
			if u.Path != p {
				t.Errorf("Expected %s, got %s.", p, u.Path)
			}
		} else {
			t.Error("Expected output, channel was closed.")
		}
	}
	if _, ok := <-out; ok {
		t.Error("Expected closed channel, got read.")
	}
	if dupes != 2 {
		t.Errorf("Expected 2 dupes, got %d", dupes)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package agent

// Some comments on use of mocking framework in helpers_test.go.

import (
	"errors"
	"net"
	"strings"
	"testing"
)

// TestNewChains is checking that detectMissingChains correctly detects which Pani chains
// must be created for given NetIf.
func TestNewChains(t *testing.T) {
	agent := mockAgent()
	// when

	// detectMissingChains calls isChainExist which is reading FakeExecutor
	// isChainExist doesn't care for output but must receive not nil error
	// otherwise it would decide that chain exist already and skip
	E := &FakeExecutor{nil, errors.New("bla"), nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	newChains := fw.detectMissingChains()

	// expect
	if len(newChains) != 3 {
		t.Error("TestNewChains failed")
	}

	// TODO test case with some chains already exist requires support for
	// stack of output in FakeExecutor
}

// TestCreateChains is checking that CreateChains generates correct OS commands
// for iptables to create firewall chains.
func TestCreateChains(t *testing.T) {
	agent := mockAgent()
	// when

	// CreateChains doesn't care for output and we don't any errors
	// we only want to analize which command generated by the function
	E := &FakeExecutor{nil, nil, nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	_ = fw.CreateChains([]int{0, 1, 2})

	// expect
	expect := strings.Join([]string{"/sbin/iptables -N pani-T0S0-INPUT",
		"/sbin/iptables -N pani-T0S0-OUTPUT",
		"/sbin/iptables -N pani-T0S0-FORWARD"}, "\n")

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestCreateChains, expect\n%s, got\n%s", expect, *E.Commands)
	}
}

// TestDivertTraffic is checking that DivertTrafficToPaniIptablesChain generates correct commands for
// firewall to divert traffic into pani chains.
func TestDivertTraffic(t *testing.T) {
	agent := mockAgent()
	// when

	// we only care for recorded commands, no need for fake output or errors
	E := &FakeExecutor{nil, nil, nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	// 0 is a first standard chain - INPUT
	fw.DivertTrafficToPaniIptablesChain(0)

	// expect
	expect := "/sbin/iptables -A INPUT -i eth0 -j pani-T0S0-INPUT"

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestDivertTraffic, expect\n%s, got\n%s", expect, *E.Commands)
	}
}

// TestCreateRules is checking that CreateRules generates correct commands to create
// firewall rules.
func TestCreateRules(t *testing.T) {
	agent := mockAgent()
	// when

	// we only care for recorded commands, no need for fake output or errors
	E := &FakeExecutor{nil, nil, nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	// 0 is a first standard chain - INPUT
	fw.CreateRules(0)

	// expect
	expect := strings.Join([]string{"/sbin/iptables -A pani-T0S0-INPUT -d 172.17.0.1 -j ACCEPT",
		"/sbin/iptables -A pani-T0S0-INPUT -d 127.0.0.1/8 -j ACCEPT",
		"/sbin/iptables -A pani-T0S0-INPUT -p udp --sport 68 --dport 67 -d 255.255.255.255 -j ACCEPT",
		"/sbin/iptables -A pani-T0S0-INPUT -p tcp -m tcp --sport 22 -d 172.17.0.1 -j ACCEPT"},
		"\n")

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestCreateRules, expect\n%s, got\n%s", expect, *E.Commands)
	}
}

// TestCreateU32Rule is checking that CreateU32Rules generates correct commands to
// create firewall rules.
func TestCreateU32Rules(t *testing.T) {
	agent := mockAgent()
	// when

	// we only care for recorded commands, no need for fake output or errors
	E := &FakeExecutor{nil, nil, nil}

	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	// 0 is a first standard chain - INPUT
	fw.CreateU32Rules(0)

	// expect
	expect := strings.Join([]string{"/sbin/iptables -A pani-T0S0-INPUT -m u32 --u32 12&0xFF00FF00=0x7F000000 && 16&0xFF00FF00=0x7F000000 -j ACCEPT"}, "\n")

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestCreateU32Rules, expect\n%s, got\n%s", expect, *E.Commands)
	}
}
<commit_msg>Comments firewall_test.go<commit_after>// Copyright (c) 2015 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Tests for firewall.go
package agent

// Some comments on use of mocking framework in helpers_test.go.

import (
	"errors"
	"net"
	"strings"
	"testing"
)

// TestNewChains is checking that detectMissingChains correctly detects which Pani chains
// must be created for given NetIf.
func TestNewChains(t *testing.T) {
	agent := mockAgent()
	// when

	// detectMissingChains calls isChainExist which is reading FakeExecutor
	// isChainExist doesn't care for output but must receive not nil error
	// otherwise it would decide that chain exist already and skip
	E := &FakeExecutor{nil, errors.New("bla"), nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	newChains := fw.detectMissingChains()

	// expect
	if len(newChains) != 3 {
		t.Error("TestNewChains failed")
	}

	// TODO test case with some chains already exist requires support for
	// stack of output in FakeExecutor
}

// TestCreateChains is checking that CreateChains generates correct OS commands
// for iptables to create firewall chains.
func TestCreateChains(t *testing.T) {
	agent := mockAgent()
	// when

	// CreateChains doesn't care for output and we don't any errors
	// we only want to analize which command generated by the function
	E := &FakeExecutor{nil, nil, nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	_ = fw.CreateChains([]int{0, 1, 2})

	// expect
	expect := strings.Join([]string{"/sbin/iptables -N pani-T0S0-INPUT",
		"/sbin/iptables -N pani-T0S0-OUTPUT",
		"/sbin/iptables -N pani-T0S0-FORWARD"}, "\n")

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestCreateChains, expect\n%s, got\n%s", expect, *E.Commands)
	}
}

// TestDivertTraffic is checking that DivertTrafficToPaniIptablesChain generates correct commands for
// firewall to divert traffic into pani chains.
func TestDivertTraffic(t *testing.T) {
	agent := mockAgent()
	// when

	// we only care for recorded commands, no need for fake output or errors
	E := &FakeExecutor{nil, nil, nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	// 0 is a first standard chain - INPUT
	fw.DivertTrafficToPaniIptablesChain(0)

	// expect
	expect := "/sbin/iptables -A INPUT -i eth0 -j pani-T0S0-INPUT"

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestDivertTraffic, expect\n%s, got\n%s", expect, *E.Commands)
	}
}

// TestCreateRules is checking that CreateRules generates correct commands to create
// firewall rules.
func TestCreateRules(t *testing.T) {
	agent := mockAgent()
	// when

	// we only care for recorded commands, no need for fake output or errors
	E := &FakeExecutor{nil, nil, nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	// 0 is a first standard chain - INPUT
	fw.CreateRules(0)

	// expect
	expect := strings.Join([]string{"/sbin/iptables -A pani-T0S0-INPUT -d 172.17.0.1 -j ACCEPT",
		"/sbin/iptables -A pani-T0S0-INPUT -d 127.0.0.1/8 -j ACCEPT",
		"/sbin/iptables -A pani-T0S0-INPUT -p udp --sport 68 --dport 67 -d 255.255.255.255 -j ACCEPT",
		"/sbin/iptables -A pani-T0S0-INPUT -p tcp -m tcp --sport 22 -d 172.17.0.1 -j ACCEPT"},
		"\n")

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestCreateRules, expect\n%s, got\n%s", expect, *E.Commands)
	}
}

// TestCreateU32Rule is checking that CreateU32Rules generates correct commands to
// create firewall rules.
func TestCreateU32Rules(t *testing.T) {
	agent := mockAgent()
	// when

	// we only care for recorded commands, no need for fake output or errors
	E := &FakeExecutor{nil, nil, nil}

	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	// 0 is a first standard chain - INPUT
	fw.CreateU32Rules(0)

	// expect
	expect := strings.Join([]string{"/sbin/iptables -A pani-T0S0-INPUT -m u32 --u32 12&0xFF00FF00=0x7F000000 && 16&0xFF00FF00=0x7F000000 -j ACCEPT"}, "\n")

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestCreateU32Rules, expect\n%s, got\n%s", expect, *E.Commands)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package text offers functions to draw texts on an Ebiten's image.
//
// Note: This package is experimental and API might be changed.
package text

import (
	"image"
	"image/color"
	"math"

	"golang.org/x/image/font"
	"golang.org/x/image/math/fixed"

	"github.com/hajimehoshi/ebiten"
	"github.com/hajimehoshi/ebiten/internal/graphics" // TODO: Move NextPowerOf2Int to a new different package
	"github.com/hajimehoshi/ebiten/internal/sync"
)

var (
	monotonicClock int64
)

func now() int64 {
	monotonicClock++
	return monotonicClock
}

var (
	charBounds = map[char]fixed.Rectangle26_6{}
)

type char struct {
	face font.Face
	rune rune
}

func (c *char) bounds() fixed.Rectangle26_6 {
	if b, ok := charBounds[*c]; ok {
		return b
	}
	b, _, _ := c.face.GlyphBounds(c.rune)
	charBounds[*c] = b
	return b
}

func (c *char) size() fixed.Point26_6 {
	b := c.bounds()
	return b.Max.Sub(b.Min)
}

func (c *char) empty() bool {
	s := c.size()
	return s.X == 0 || s.Y == 0
}

func (c *char) atlasGroup() int {
	s := c.size()
	w, h := s.X.Ceil(), s.Y.Ceil()
	t := w
	if t < h {
		t = h
	}

	// Different images for small runes are inefficient.
	// Let's use a same texture atlas for typical character sizes.
	if t < 32 {
		return 32
	}
	return graphics.NextPowerOf2Int(t)
}

type glyph struct {
	char  char
	index int
	atime int64
}

func fixed26_6ToFloat64(x fixed.Int26_6) float64 {
	return float64(x) / (1 << 6)
}

func (g *glyph) draw(dst *ebiten.Image, x, y fixed.Int26_6, clr color.Color) {
	cr, cg, cb, ca := clr.RGBA()
	if ca == 0 {
		return
	}

	b := g.char.bounds()
	op := &ebiten.DrawImageOptions{}
	op.GeoM.Translate(fixed26_6ToFloat64(x), fixed26_6ToFloat64(y))
	op.GeoM.Translate(fixed26_6ToFloat64(b.Min.X), fixed26_6ToFloat64(b.Min.Y))

	rf := float64(cr) / float64(ca)
	gf := float64(cg) / float64(ca)
	bf := float64(cb) / float64(ca)
	af := float64(ca) / 0xffff
	op.ColorM.Scale(rf, gf, bf, af)

	a := atlases[g.char.atlasGroup()]
	sx, sy := a.at(g)
	r := image.Rect(sx, sy, sx+a.size, sy+a.size)
	op.SourceRect = &r

	dst.DrawImage(a.image, op)
}

var (
	atlases = map[int]*atlas{}
)

type atlas struct {
	// image is the back-end image to hold glyph cache.
	image *ebiten.Image

	// tmpImage is the temporary image as a renderer source for glyph.
	tmpImage *ebiten.Image

	// size is the size of one glyph in the cache.
	// This value is always power of 2.
	size int

	charToGlyph map[char]*glyph

	// glyphs is the set of glyph information.
	glyphs []*glyph

	// num is the number of glyphs the atlas holds.
	num int
}

func (a *atlas) at(glyph *glyph) (int, int) {
	if a.size != glyph.char.atlasGroup() {
		panic("not reached")
	}
	w, _ := a.image.Size()
	xnum := w / a.size
	x, y := glyph.index%xnum, glyph.index/xnum
	return x * a.size, y * a.size
}

func (a *atlas) append(glyph *glyph) {
	if a.num == len(a.glyphs) {
		idx := -1
		t := int64(math.MaxInt64)
		for i, g := range a.glyphs {
			if g.atime < t {
				t = g.atime
				idx = i
			}
		}
		if idx < 0 {
			panic("not reached")
		}
		oldest := a.glyphs[idx]
		delete(a.charToGlyph, oldest.char)

		glyph.index = idx
		a.glyphs[idx] = glyph
		a.charToGlyph[glyph.char] = glyph
		a.draw(glyph)
		return
	}
	idx := -1
	for i, g := range a.glyphs {
		if g == nil {
			idx = i
			break
		}
	}
	if idx < 0 {
		panic("not reached")
	}
	a.num++
	glyph.index = idx
	a.glyphs[idx] = glyph
	a.charToGlyph[glyph.char] = glyph
	a.draw(glyph)
}

func (a *atlas) draw(glyph *glyph) {
	if a.tmpImage == nil {
		a.tmpImage, _ = ebiten.NewImage(a.size, a.size, ebiten.FilterNearest)
	}

	dst := image.NewRGBA(image.Rect(0, 0, a.size, a.size))
	d := font.Drawer{
		Dst:  dst,
		Src:  image.White,
		Face: glyph.char.face,
	}
	b := glyph.char.bounds()
	d.Dot = fixed.Point26_6{-b.Min.X, -b.Min.Y}
	d.DrawString(string(glyph.char.rune))
	a.tmpImage.ReplacePixels(dst.Pix)

	op := &ebiten.DrawImageOptions{}
	x, y := a.at(glyph)
	op.GeoM.Translate(float64(x), float64(y))
	op.CompositeMode = ebiten.CompositeModeCopy
	a.image.DrawImage(a.tmpImage, op)

	a.tmpImage.Clear()
}

func getGlyphFromCache(face font.Face, r rune, now int64) *glyph {
	ch := char{face, r}
	a, ok := atlases[ch.atlasGroup()]
	if ok {
		g, ok := a.charToGlyph[ch]
		if ok {
			g.atime = now
			return g
		}
	}

	g := &glyph{
		char:  ch,
		atime: now,
	}
	if ch.empty() {
		return g
	}

	if !ok {
		// Don't use ebiten.MaxImageSize here.
		// It's because the back-end image pixels will be restored from GPU
		// whenever a new glyph is rendered on the image, and restoring cost is
		// expensive if the image is big.
		// The back-end image is updated a temporary image, and the temporary image is
		// always cleared after used. This means that there is no clue to restore
		// the back-end image without reading from GPU
		// (see the package 'restorable' implementation).
		//
		// TODO: How about making a new function for 'flagile' image?
		const size = 1024
		i, _ := ebiten.NewImage(size, size, ebiten.FilterNearest)
		a = &atlas{
			image:       i,
			size:        g.char.atlasGroup(),
			charToGlyph: map[char]*glyph{},
		}
		w, h := a.image.Size()
		xnum := w / a.size
		ynum := h / a.size
		a.glyphs = make([]*glyph, xnum*ynum)
		atlases[g.char.atlasGroup()] = a
	}

	a.append(g)
	return g
}

var textM sync.Mutex

// Draw draws a given text on a given destination image dst.
//
// face is the font for text rendering.
// (x, y) represents a 'dot' position. Be careful that this doesn't represent left-upper corner position.
// clr is the color for text rendering.
//
// Glyphs used for rendering are cached in least-recently-used way.
// It is OK to call this function with a same text and a same face at every frame.
//
// This function is concurrent-safe.
func Draw(dst *ebiten.Image, face font.Face, text string, x, y int, clr color.Color) {
	textM.Lock()

	n := now()
	fx := fixed.I(x)
	prevC := rune(-1)

	runes := []rune(text)
	for _, c := range runes {
		if prevC >= 0 {
			fx += face.Kern(prevC, c)
		}
		if g := getGlyphFromCache(face, c, n); g != nil {
			if !g.char.empty() {
				g.draw(dst, fx, fixed.I(y), clr)
			}
			a, _ := face.GlyphAdvance(c)
			fx += a
		}
		prevC = c
	}

	textM.Unlock()
}
<commit_msg>text: Refactoring: Remove some members from atlas<commit_after>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package text offers functions to draw texts on an Ebiten's image.
//
// Note: This package is experimental and API might be changed.
package text

import (
	"image"
	"image/color"
	"math"

	"golang.org/x/image/font"
	"golang.org/x/image/math/fixed"

	"github.com/hajimehoshi/ebiten"
	"github.com/hajimehoshi/ebiten/internal/graphics" // TODO: Move NextPowerOf2Int to a new different package
	"github.com/hajimehoshi/ebiten/internal/sync"
)

var (
	monotonicClock int64
)

func now() int64 {
	monotonicClock++
	return monotonicClock
}

var (
	charBounds = map[char]fixed.Rectangle26_6{}
)

type char struct {
	face font.Face
	rune rune
}

func (c *char) bounds() fixed.Rectangle26_6 {
	if b, ok := charBounds[*c]; ok {
		return b
	}
	b, _, _ := c.face.GlyphBounds(c.rune)
	charBounds[*c] = b
	return b
}

func (c *char) size() fixed.Point26_6 {
	b := c.bounds()
	return b.Max.Sub(b.Min)
}

func (c *char) empty() bool {
	s := c.size()
	return s.X == 0 || s.Y == 0
}

func (c *char) atlasGroup() int {
	s := c.size()
	w, h := s.X.Ceil(), s.Y.Ceil()
	t := w
	if t < h {
		t = h
	}

	// Different images for small runes are inefficient.
	// Let's use a same texture atlas for typical character sizes.
	if t < 32 {
		return 32
	}
	return graphics.NextPowerOf2Int(t)
}

type glyph struct {
	char  char
	index int
	atime int64
}

func fixed26_6ToFloat64(x fixed.Int26_6) float64 {
	return float64(x) / (1 << 6)
}

func (g *glyph) draw(dst *ebiten.Image, x, y fixed.Int26_6, clr color.Color) {
	cr, cg, cb, ca := clr.RGBA()
	if ca == 0 {
		return
	}

	b := g.char.bounds()
	op := &ebiten.DrawImageOptions{}
	op.GeoM.Translate(fixed26_6ToFloat64(x), fixed26_6ToFloat64(y))
	op.GeoM.Translate(fixed26_6ToFloat64(b.Min.X), fixed26_6ToFloat64(b.Min.Y))

	rf := float64(cr) / float64(ca)
	gf := float64(cg) / float64(ca)
	bf := float64(cb) / float64(ca)
	af := float64(ca) / 0xffff
	op.ColorM.Scale(rf, gf, bf, af)

	a := atlases[g.char.atlasGroup()]
	sx, sy := a.at(g)
	r := image.Rect(sx, sy, sx+a.size, sy+a.size)
	op.SourceRect = &r

	dst.DrawImage(a.image, op)
}

var (
	atlases = map[int]*atlas{}
)

type atlas struct {
	// image is the back-end image to hold glyph cache.
	image *ebiten.Image

	// tmpImage is the temporary image as a renderer source for glyph.
	tmpImage *ebiten.Image

	// size is the size of one glyph in the cache.
	// This value is always power of 2.
	size int

	charToGlyph map[char]*glyph
}

func (a *atlas) at(glyph *glyph) (int, int) {
	if a.size != glyph.char.atlasGroup() {
		panic("not reached")
	}
	w, _ := a.image.Size()
	xnum := w / a.size
	x, y := glyph.index%xnum, glyph.index/xnum
	return x * a.size, y * a.size
}

func (a *atlas) maxGlyphNum() int {
	w, h := a.image.Size()
	xnum := w / a.size
	ynum := h / a.size
	return xnum * ynum
}

func (a *atlas) append(g *glyph) {
	if len(a.charToGlyph) == a.maxGlyphNum() {
		var oldest *glyph
		t := int64(math.MaxInt64)
		for _, g := range a.charToGlyph {
			if g.atime < t {
				t = g.atime
				oldest = g
			}
		}
		if oldest == nil {
			panic("not reached")
		}
		idx := oldest.index
		delete(a.charToGlyph, oldest.char)

		g.index = idx
	} else {
		g.index = len(a.charToGlyph)
	}
	a.charToGlyph[g.char] = g
	a.draw(g)
}

func (a *atlas) draw(glyph *glyph) {
	if a.tmpImage == nil {
		a.tmpImage, _ = ebiten.NewImage(a.size, a.size, ebiten.FilterNearest)
	}

	dst := image.NewRGBA(image.Rect(0, 0, a.size, a.size))
	d := font.Drawer{
		Dst:  dst,
		Src:  image.White,
		Face: glyph.char.face,
	}
	b := glyph.char.bounds()
	d.Dot = fixed.Point26_6{-b.Min.X, -b.Min.Y}
	d.DrawString(string(glyph.char.rune))
	a.tmpImage.ReplacePixels(dst.Pix)

	op := &ebiten.DrawImageOptions{}
	x, y := a.at(glyph)
	op.GeoM.Translate(float64(x), float64(y))
	op.CompositeMode = ebiten.CompositeModeCopy
	a.image.DrawImage(a.tmpImage, op)

	a.tmpImage.Clear()
}

func getGlyphFromCache(face font.Face, r rune, now int64) *glyph {
	ch := char{face, r}
	a, ok := atlases[ch.atlasGroup()]
	if ok {
		g, ok := a.charToGlyph[ch]
		if ok {
			g.atime = now
			return g
		}
	}

	g := &glyph{
		char:  ch,
		atime: now,
	}
	if ch.empty() {
		return g
	}

	if !ok {
		// Don't use ebiten.MaxImageSize here.
		// It's because the back-end image pixels will be restored from GPU
		// whenever a new glyph is rendered on the image, and restoring cost is
		// expensive if the image is big.
		// The back-end image is updated a temporary image, and the temporary image is
		// always cleared after used. This means that there is no clue to restore
		// the back-end image without reading from GPU
		// (see the package 'restorable' implementation).
		//
		// TODO: How about making a new function for 'flagile' image?
		const size = 1024
		i, _ := ebiten.NewImage(size, size, ebiten.FilterNearest)
		a = &atlas{
			image:       i,
			size:        g.char.atlasGroup(),
			charToGlyph: map[char]*glyph{},
		}
		atlases[g.char.atlasGroup()] = a
	}

	a.append(g)
	return g
}

var textM sync.Mutex

// Draw draws a given text on a given destination image dst.
//
// face is the font for text rendering.
// (x, y) represents a 'dot' position. Be careful that this doesn't represent left-upper corner position.
// clr is the color for text rendering.
//
// Glyphs used for rendering are cached in least-recently-used way.
// It is OK to call this function with a same text and a same face at every frame.
//
// This function is concurrent-safe.
func Draw(dst *ebiten.Image, face font.Face, text string, x, y int, clr color.Color) {
	textM.Lock()

	n := now()
	fx := fixed.I(x)
	prevC := rune(-1)

	runes := []rune(text)
	for _, c := range runes {
		if prevC >= 0 {
			fx += face.Kern(prevC, c)
		}
		if g := getGlyphFromCache(face, c, n); g != nil {
			if !g.char.empty() {
				g.draw(dst, fx, fixed.I(y), clr)
			}
			a, _ := face.GlyphAdvance(c)
			fx += a
		}
		prevC = c
	}

	textM.Unlock()
}
<|endoftext|>


<commit_before>package textmagic

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"time"

	"github.com/MJKWoolnough/memio"
)

var apiURLPrefix = "https://www.textmagic.com/app/api?"

const (
	cmdAccount       = "account"
	cmdCheckNumber   = "check_number"
	cmdDeleteReply   = "delete_reply"
	cmdMessageStatus = "message_status"
	cmdReceive       = "receive"
	cmdSend          = "send"
)

type TextMagic struct {
	username, password string
}

func New(username, password string) TextMagic {
	return TextMagic{username, password}
}

func (t TextMagic) sendAPI(cmd string, params url.Values, data interface{}) error {
	params.Set("username", t.username)
	params.Set("password", t.password)
	params.Set("cmd", cmd)
	r, err := http.Get(apiURLPrefix + params.Encode())
	if err != nil {
		return RequestError{cmd, err}
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return StatusError{cmd, r.StatusCode}
	}
	jsonData := make([]byte, r.ContentLength) // avoid allocation using io.Pipe?
	var apiError APIError
	err = json.NewDecoder(io.TeeReader(r.Body, memio.Create(&jsonData))).Decode(&apiError)
	if err != nil {
		return JSONError{cmd, err}
	}
	if apiError.Code != 0 {
		apiError.Cmd = cmd
		return apiError
	}
	json.Unmarshal(jsonData, data)
	return nil
}

type balance struct {
	Balance float32 `json:"balance"`
}

func (t TextMagic) Account() (float32, error) {
	var b balance
	if err := t.sendAPI(cmdAccount, url.Values{}, &b); err != nil {
		return 0, err
	}
	return b.Balance, nil
}

type DeliveryNotificationCode string

func (d DeliveryNotificationCode) Status() string {
	switch d {
	case "q", "r", "a", "b", "s":
		return "intermediate"
	case "d", "f", "e", "j", "u":
		return "final"
	}
	return "unknown"
}

func (d DeliveryNotificationCode) String() string {
	switch d {
	case "q":
		return "The message is queued on the TextMagic server."
	case "r":
		return "The message has been sent to the mobile operator."
	case "a":
		return "The mobile operator has acknowledged the message."
	case "b":
		return "The mobile operator has queued the message."
	case "d":
		return "The message has been successfully delivered to the handset."
	case "f":
		return "An error occurred while delivering message."
	case "e":
		return "An error occurred while sending message."
	case "j":
		return "The mobile operator has rejected the message."
	case "s":
		return "This message is scheduled to be sent later."
	default:
		return "The status is unknown."

	}
}

type Status struct {
	Text      string                   `json:"text"`
	Status    DeliveryNotificationCode `json:"status"`
	Created   int64                    `json:"created_time"`
	Reply     string                   `json:"reply_number"`
	Cost      float32                  `json:"credits_cost"`
	Completed int64                    `json:"completed_time"`
}

func (t TextMagic) MessageStatus(ids []uint64) (map[uint64]Status, error) {
	statuses := make(map[uint64]Status)
	for _, tIds := range splitSlice(ids) {
		messageIds := joinUints(tIds)
		strStatuses := make(map[string]Status)
		err := t.sendAPI(cmdMessageStatus, url.Values{"ids": {messageIds}}, strStatuses)
		if err != nil {
			return statuses, err
		}
		for messageID, status := range strStatuses {
			if id, isNum := stou(messageID); isNum {
				statuses[id] = status
			}
		}
	}
	return statuses, nil
}

type Number struct {
	Price   float32 `json:"price"`
	Country string  `json:"country"`
}

func (t TextMagic) CheckNumber(numbers []uint64) (map[uint64]Number, error) {
	ns := make(map[string]Number)
	if err := t.sendAPI(cmdCheckNumber, url.Values{"phone": {joinUints(numbers)}}, ns); err != nil {
		return nil, err
	}
	toRet := make(map[uint64]Number)
	for number, data := range ns {
		if n, isNum := stou(number); isNum {
			toRet[n] = data
		}
	}
	return toRet, nil
}

type deleted struct {
	Deleted []uint64 `json:"deleted"`
}

func (t TextMagic) DeleteReply(ids []uint64) ([]uint64, error) {
	toRet := make([]uint64, 0, len(ids))
	for _, tIds := range splitSlice(ids) {
		var d deleted
		if err := t.sendAPI(cmdDeleteReply, url.Values{"deleted": {joinUints(tIds)}}, &d); err != nil {
			return toRet, err
		}
		toRet = append(toRet, d.Deleted...)
	}
	return toRet, nil
}

type Message struct {
	ID        uint64 `json:"message_id"`
	From      uint64 `json:"from"`
	Timestamp int64  `json:"timestamp"`
	Text      string `json:"text"`
}

type received struct {
	Messages []Message `json:"messages"`
	Unread   uint64    `json:"unread"`
}

func (t TextMagic) Receive(lastRetrieved uint64) (uint64, []Message, error) {
	var r received
	err := t.sendAPI(cmdReceive, url.Values{"last_retrieved_id": {utos(lastRetrieved)}}, &r)
	return r.Unread, r.Messages, err
}

type option func(u url.Values)

func From(from uint64) option {
	return func(u url.Values) {
		u.Set("from", utos(from))
	}
}

func MaxLength(length uint64) option {
	if length > 3 {
		length = 3
	}
	return func(u url.Values) {
		u.Set("max_length", utos(length))
	}
}

func CutExtra() option {
	return func(u url.Values) {
		u.Set("cut_extra", "1")
	}
}

func SendTime(t time.Time) option {
	return func(u url.Values) {
		u.Set("send_time", t.Format(time.RFC3339))
	}
}

type messageResponse struct {
	IDs   map[string]string `json:"message_id"`
	Text  string            `json:"sent_text"`
	Parts uint              `json:"parts_count"`
}

func (t TextMagic) Send(message string, to []uint64, options ...option) (map[string]uint64, string, uint, error) {
	var (
		params url.Values
		text   string
		parts  uint
		ids    = make(map[string]uint64)
	)
	// check message for unicode/invalid chars
	params.Set("text", message)
	for _, o := range options {
		o(params)
	}
	for _, numbers := range splitSlice(to) {
		params.Set("phone", joinUints(numbers))
		var m messageResponse
		if err := t.sendAPI(cmdSend, params, &m); err != nil {
			return ids, text, parts, err
		}
		if parts == 0 {
			parts = m.Parts
			text = m.Text
		}
		for id, number := range m.IDs {
			if n, isNum := stou(number); isNum {
				ids[id] = n
			}
		}
	}
	return ids, text, parts, nil
}

const joinSep = ','

func joinUints(u []uint64) string {
	toStr := make([]byte, 0, 10*len(u))
	var digits [21]byte
	for n, num := range u {
		if n > 0 {
			toStr = append(toStr, joinSep)
		}
		if num == 0 {
			toStr = append(toStr, '0')
			continue
		}
		pos := 21
		for ; num > 0; num /= 10 {
			pos--
			digits[pos] = '0' + byte(num%10)
		}
		toStr = append(toStr, digits[pos:]...)
	}
	return string(toStr)
}

const maxInSlice = 100

func splitSlice(slice []uint64) [][]uint64 {
	toRet := make([][]uint64, 0, len(slice)/maxInSlice+1)
	for len(slice) > 100 {
		toRet = append(toRet, slice[:100])
		slice = slice[100:]
	}
	if len(slice) > 0 {
		toRet = append(toRet, slice)
	}
	return toRet
}

func utos(num uint64) string {
	if num == 0 {
		return "0"
	}
	var digits [21]byte
	pos := 21
	for ; num > 0; num /= 10 {
		pos--
		digits[pos] = '0' + byte(num%10)
	}
	return string(digits[pos:])
}

func stou(str string) (uint64, bool) {
	var num uint64
	for _, c := range str {
		if c > '9' || c < '0' {
			return 0, false
		}
		num *= 10
		num += uint64(c - '0')
	}
	return num, true
}

// Errors

type APIError struct {
	Cmd     string
	Code    int    `json:"error_code"`
	Message string `json:"error_message"`
}

func (a APIError) Error() string {
	return "command " + a.Cmd + " returned the following API error: " + a.Message
}

type RequestError struct {
	Cmd string
	Err error
}

func (r RequestError) Error() string {
	return "command " + r.Cmd + " returned the following error while makeing the API call: " + r.Err.Error()
}

type StatusError struct {
	Cmd        string
	StatusCode int
}

func (s StatusError) Error() string {
	return "command " + s.Cmd + " returned a non-200 OK response: " + http.StatusText(s.StatusCode)
}

type JSONError struct {
	Cmd string
	Err error
}

func (j JSONError) Error() string {
	return "command " + j.Cmd + " returned malformed JSON: " + j.Err.Error()
}
<commit_msg>Replaced constants<commit_after>package textmagic

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"time"

	"github.com/MJKWoolnough/memio"
)

var apiURLPrefix = "https://www.textmagic.com/app/api?"

const (
	cmdAccount       = "account"
	cmdCheckNumber   = "check_number"
	cmdDeleteReply   = "delete_reply"
	cmdMessageStatus = "message_status"
	cmdReceive       = "receive"
	cmdSend          = "send"
)

type TextMagic struct {
	username, password string
}

func New(username, password string) TextMagic {
	return TextMagic{username, password}
}

func (t TextMagic) sendAPI(cmd string, params url.Values, data interface{}) error {
	params.Set("username", t.username)
	params.Set("password", t.password)
	params.Set("cmd", cmd)
	r, err := http.Get(apiURLPrefix + params.Encode())
	if err != nil {
		return RequestError{cmd, err}
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return StatusError{cmd, r.StatusCode}
	}
	jsonData := make([]byte, r.ContentLength) // avoid allocation using io.Pipe?
	var apiError APIError
	err = json.NewDecoder(io.TeeReader(r.Body, memio.Create(&jsonData))).Decode(&apiError)
	if err != nil {
		return JSONError{cmd, err}
	}
	if apiError.Code != 0 {
		apiError.Cmd = cmd
		return apiError
	}
	json.Unmarshal(jsonData, data)
	return nil
}

type balance struct {
	Balance float32 `json:"balance"`
}

func (t TextMagic) Account() (float32, error) {
	var b balance
	if err := t.sendAPI(cmdAccount, url.Values{}, &b); err != nil {
		return 0, err
	}
	return b.Balance, nil
}

type DeliveryNotificationCode string

func (d DeliveryNotificationCode) Status() string {
	switch d {
	case "q", "r", "a", "b", "s":
		return "intermediate"
	case "d", "f", "e", "j", "u":
		return "final"
	}
	return "unknown"
}

func (d DeliveryNotificationCode) String() string {
	switch d {
	case "q":
		return "The message is queued on the TextMagic server."
	case "r":
		return "The message has been sent to the mobile operator."
	case "a":
		return "The mobile operator has acknowledged the message."
	case "b":
		return "The mobile operator has queued the message."
	case "d":
		return "The message has been successfully delivered to the handset."
	case "f":
		return "An error occurred while delivering message."
	case "e":
		return "An error occurred while sending message."
	case "j":
		return "The mobile operator has rejected the message."
	case "s":
		return "This message is scheduled to be sent later."
	default:
		return "The status is unknown."

	}
}

type Status struct {
	Text      string                   `json:"text"`
	Status    DeliveryNotificationCode `json:"status"`
	Created   int64                    `json:"created_time"`
	Reply     string                   `json:"reply_number"`
	Cost      float32                  `json:"credits_cost"`
	Completed int64                    `json:"completed_time"`
}

func (t TextMagic) MessageStatus(ids []uint64) (map[uint64]Status, error) {
	statuses := make(map[uint64]Status)
	for _, tIds := range splitSlice(ids) {
		messageIds := joinUints(tIds)
		strStatuses := make(map[string]Status)
		err := t.sendAPI(cmdMessageStatus, url.Values{"ids": {messageIds}}, strStatuses)
		if err != nil {
			return statuses, err
		}
		for messageID, status := range strStatuses {
			if id, isNum := stou(messageID); isNum {
				statuses[id] = status
			}
		}
	}
	return statuses, nil
}

type Number struct {
	Price   float32 `json:"price"`
	Country string  `json:"country"`
}

func (t TextMagic) CheckNumber(numbers []uint64) (map[uint64]Number, error) {
	ns := make(map[string]Number)
	if err := t.sendAPI(cmdCheckNumber, url.Values{"phone": {joinUints(numbers)}}, ns); err != nil {
		return nil, err
	}
	toRet := make(map[uint64]Number)
	for number, data := range ns {
		if n, isNum := stou(number); isNum {
			toRet[n] = data
		}
	}
	return toRet, nil
}

type deleted struct {
	Deleted []uint64 `json:"deleted"`
}

func (t TextMagic) DeleteReply(ids []uint64) ([]uint64, error) {
	toRet := make([]uint64, 0, len(ids))
	for _, tIds := range splitSlice(ids) {
		var d deleted
		if err := t.sendAPI(cmdDeleteReply, url.Values{"deleted": {joinUints(tIds)}}, &d); err != nil {
			return toRet, err
		}
		toRet = append(toRet, d.Deleted...)
	}
	return toRet, nil
}

type Message struct {
	ID        uint64 `json:"message_id"`
	From      uint64 `json:"from"`
	Timestamp int64  `json:"timestamp"`
	Text      string `json:"text"`
}

type received struct {
	Messages []Message `json:"messages"`
	Unread   uint64    `json:"unread"`
}

func (t TextMagic) Receive(lastRetrieved uint64) (uint64, []Message, error) {
	var r received
	err := t.sendAPI(cmdReceive, url.Values{"last_retrieved_id": {utos(lastRetrieved)}}, &r)
	return r.Unread, r.Messages, err
}

type option func(u url.Values)

func From(from uint64) option {
	return func(u url.Values) {
		u.Set("from", utos(from))
	}
}

func MaxLength(length uint64) option {
	if length > 3 {
		length = 3
	}
	return func(u url.Values) {
		u.Set("max_length", utos(length))
	}
}

func CutExtra() option {
	return func(u url.Values) {
		u.Set("cut_extra", "1")
	}
}

func SendTime(t time.Time) option {
	return func(u url.Values) {
		u.Set("send_time", t.Format(time.RFC3339))
	}
}

type messageResponse struct {
	IDs   map[string]string `json:"message_id"`
	Text  string            `json:"sent_text"`
	Parts uint              `json:"parts_count"`
}

func (t TextMagic) Send(message string, to []uint64, options ...option) (map[string]uint64, string, uint, error) {
	var (
		params url.Values
		text   string
		parts  uint
		ids    = make(map[string]uint64)
	)
	// check message for unicode/invalid chars
	params.Set("text", message)
	for _, o := range options {
		o(params)
	}
	for _, numbers := range splitSlice(to) {
		params.Set("phone", joinUints(numbers))
		var m messageResponse
		if err := t.sendAPI(cmdSend, params, &m); err != nil {
			return ids, text, parts, err
		}
		if parts == 0 {
			parts = m.Parts
			text = m.Text
		}
		for id, number := range m.IDs {
			if n, isNum := stou(number); isNum {
				ids[id] = n
			}
		}
	}
	return ids, text, parts, nil
}

const joinSep = ','

func joinUints(u []uint64) string {
	toStr := make([]byte, 0, 10*len(u))
	var digits [21]byte
	for n, num := range u {
		if n > 0 {
			toStr = append(toStr, joinSep)
		}
		if num == 0 {
			toStr = append(toStr, '0')
			continue
		}
		pos := 21
		for ; num > 0; num /= 10 {
			pos--
			digits[pos] = '0' + byte(num%10)
		}
		toStr = append(toStr, digits[pos:]...)
	}
	return string(toStr)
}

const maxInSlice = 100

func splitSlice(slice []uint64) [][]uint64 {
	toRet := make([][]uint64, 0, len(slice)/maxInSlice+1)
	for len(slice) > maxInSlice {
		toRet = append(toRet, slice[:maxInSlice])
		slice = slice[maxInSlice:]
	}
	if len(slice) > 0 {
		toRet = append(toRet, slice)
	}
	return toRet
}

func utos(num uint64) string {
	if num == 0 {
		return "0"
	}
	var digits [21]byte
	pos := 21
	for ; num > 0; num /= 10 {
		pos--
		digits[pos] = '0' + byte(num%10)
	}
	return string(digits[pos:])
}

func stou(str string) (uint64, bool) {
	var num uint64
	for _, c := range str {
		if c > '9' || c < '0' {
			return 0, false
		}
		num *= 10
		num += uint64(c - '0')
	}
	return num, true
}

// Errors

type APIError struct {
	Cmd     string
	Code    int    `json:"error_code"`
	Message string `json:"error_message"`
}

func (a APIError) Error() string {
	return "command " + a.Cmd + " returned the following API error: " + a.Message
}

type RequestError struct {
	Cmd string
	Err error
}

func (r RequestError) Error() string {
	return "command " + r.Cmd + " returned the following error while makeing the API call: " + r.Err.Error()
}

type StatusError struct {
	Cmd        string
	StatusCode int
}

func (s StatusError) Error() string {
	return "command " + s.Cmd + " returned a non-200 OK response: " + http.StatusText(s.StatusCode)
}

type JSONError struct {
	Cmd string
	Err error
}

func (j JSONError) Error() string {
	return "command " + j.Cmd + " returned malformed JSON: " + j.Err.Error()
}
<|endoftext|>


<commit_before>package textmagic

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"strconv"

	"github.com/MJKWoolnough/memio"
)

var apiURLPrefix = "https://www.textmagic.com/app/api?"

const (
	cmdAccount       = "account"
	cmdCheckNumber   = "check_number"
	cmdDeleteReply   = "delete_reply"
	cmdMessageStatus = "message_status"
	cmdReceive       = "receive"
	cmdSend          = "send"
)

type TextMagic struct {
	username, password string
}

func New(username, password string) TextMagic {
	return TextMagic{username, password}
}

func (t TextMagic) sendAPI(cmd string, params url.Values, data interface{}) error {
	params.Add("username", t.username)
	params.Add("password", t.password)
	params.Add("cmd", cmd)
	r, err := http.Get(apiURLPrefix + params.Encode())
	if err != nil {
		return RequestError{cmd, err}
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return StatusError{cmd, r.StatusCode}
	}
	jsonData := make([]byte, r.ContentLength) // avoid allocation using io.Pipe?
	var apiError APIError
	err = json.NewDecoder(io.TeeReader(r.Body, memio.Create(&jsonData))).Decode(&apiError)
	if err != nil {
		return JSONError{cmd, err}
	}
	if apiError.Code != 0 {
		apiError.Cmd = cmd
		return apiError
	}
	json.Unmarshal(jsonData, data)
	return nil
}

type balance struct {
	Balance float32 `json:"balance"`
}

func (t TextMagic) Account() (float32, error) {
	var b balance
	if err := t.sendAPI(cmdAccount, url.Values{}, &b); err != nil {
		return 0, err
	}
	return b.Balance, nil
}

type Status struct {
	Text      string  `json:"text"`
	Status    string  `json:"status"`
	Created   int64   `json:"created_time"`
	Reply     string  `json:"reply_number"`
	Cost      float32 `json:"credits_cost"`
	Completed int64   `json:"completed_time"`
}

func (t TextMagic) MessageStatus(ids ...uint) (map[uint]Status, error) {
	statuses := make(map[uint]Status)
	for _, tIds := range splitSlice(ids) {
		messageIds := joinUints(tIds...)
		strStatuses := make(map[string]Status)
		err := t.sendAPI(cmdMessageStatus, url.Values{"ids": {messageIds}}, strStatuses)
		if err != nil {
			return statuses, err
		}
		for messageID, status := range strStatuses {
			id, err := strconv.Atoi(messageID)
			if err != nil {
				continue
			}
			statuses[uint(id)] = status
		}
	}
	return statuses, nil
}

type Number struct {
	Price   float32 `json:"price"`
	Country string  `json:"country"`
}

func (t TextMagic) CheckNumber(numbers ...uint) (map[uint]Number, error) {
	ns := make(map[string]Number)
	if err := t.sendAPI(cmdCheckNumber, url.Values{"phone": {joinUints(numbers...)}}, ns); err != nil {
		return nil, err
	}
	toRet := make(map[uint]Number)
	for n, data := range ns {
		number, err := strconv.Atoi(n)
		if err != nil {
			continue
		}
		toRet[uint(number)] = data
	}
	return toRet, nil
}

type deleted struct {
	Deleted []uint `json:"deleted"`
}

func (t TextMagic) DeleteReply(ids ...uint) ([]uint, error) {
	toRet := make([]uint, 0, len(ids))
	for _, tIds := range splitSlice(ids) {
		var d deleted
		if err := t.sendAPI(cmdDeleteReply, url.Values{"deleted": {joinUints(tIds...)}}, &d); err != nil {
			return toRet, err
		}
		toRet = append(toRet, d.Deleted...)
	}
	return toRet, nil
}

type Message struct {
	ID        uint   `json:"message_id"`
	From      uint   `json:"from"`
	Timestamp int64  `json:"timestamp"`
	Text      string `json:"text"`
}

type received struct {
	Messages []Message `json:"messages"`
	Unread   uint      `json:"unread"`
}

func (t TextMagic) Receive(lastRetrieved uint) (uint, []Message, error) {
	var r received
	err := t.sendAPI(cmdReceive, url.Values{"last_retrieved_id": {strconv.Itoa(int(lastRetrieved))}}, &r)
	return r.Unread, r.Messages, err
}

const joinSep = ','

func joinUints(u ...uint) string {
	toStr := make([]byte, 0, 10*len(u))
	var digits [21]byte
	for n, num := range u {
		if n > 0 {
			toStr = append(toStr, joinSep)
		}
		if num == 0 {
			toStr = append(toStr, '0')
			continue
		}
		pos := 21
		for ; num > 0; num /= 10 {
			pos--
			digits[pos] = '0' + byte(num%10)
		}
		toStr = append(toStr, digits[pos:]...)
	}
	return string(toStr)
}

const maxInSlice = 100

func splitSlice(slice []uint) [][]uint {
	toRet := make([][]uint, 0, len(slice)/maxInSlice+1)
	for len(slice) > 100 {
		toRet = append(toRet, slice[:100])
		slice = slice[100:]
	}
	if len(slice) > 0 {
		toRet = append(toRet, slice)
	}
	return toRet
}

// Errors

type APIError struct {
	Cmd     string
	Code    int    `json:"error_code"`
	Message string `json:"error_message"`
}

func (a APIError) Error() string {
	return "command " + a.Cmd + " returned the following API error: " + a.Message
}

type RequestError struct {
	Cmd string
	Err error
}

func (r RequestError) Error() string {
	return "command " + r.Cmd + " returned the following error while makeing the API call: " + r.Err.Error()
}

type StatusError struct {
	Cmd        string
	StatusCode int
}

func (s StatusError) Error() string {
	return "command " + s.Cmd + " returned a non-200 OK response: " + http.StatusText(s.StatusCode)
}

type JSONError struct {
	Cmd string
	Err error
}

func (j JSONError) Error() string {
	return "command " + j.Cmd + " returned malformed JSON: " + j.Err.Error()
}
<commit_msg>Added helper type DeliveryNotificationCode<commit_after>package textmagic

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"strconv"

	"github.com/MJKWoolnough/memio"
)

var apiURLPrefix = "https://www.textmagic.com/app/api?"

const (
	cmdAccount       = "account"
	cmdCheckNumber   = "check_number"
	cmdDeleteReply   = "delete_reply"
	cmdMessageStatus = "message_status"
	cmdReceive       = "receive"
	cmdSend          = "send"
)

type TextMagic struct {
	username, password string
}

func New(username, password string) TextMagic {
	return TextMagic{username, password}
}

func (t TextMagic) sendAPI(cmd string, params url.Values, data interface{}) error {
	params.Add("username", t.username)
	params.Add("password", t.password)
	params.Add("cmd", cmd)
	r, err := http.Get(apiURLPrefix + params.Encode())
	if err != nil {
		return RequestError{cmd, err}
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return StatusError{cmd, r.StatusCode}
	}
	jsonData := make([]byte, r.ContentLength) // avoid allocation using io.Pipe?
	var apiError APIError
	err = json.NewDecoder(io.TeeReader(r.Body, memio.Create(&jsonData))).Decode(&apiError)
	if err != nil {
		return JSONError{cmd, err}
	}
	if apiError.Code != 0 {
		apiError.Cmd = cmd
		return apiError
	}
	json.Unmarshal(jsonData, data)
	return nil
}

type balance struct {
	Balance float32 `json:"balance"`
}

func (t TextMagic) Account() (float32, error) {
	var b balance
	if err := t.sendAPI(cmdAccount, url.Values{}, &b); err != nil {
		return 0, err
	}
	return b.Balance, nil
}

type DeliveryNotificationCode string

func (d DeliveryNotificationCode) Status() string {
	switch d {
	case "q", "r", "a", "b", "s":
		return "intermediate"
	case "d", "f", "e", "j", "u":
		return "final"
	}
	return "unknown"
}

func (d DeliveryNotificationCode) String() string {
	switch d {
	case "q":
		return "The message is queued on the TextMagic server."
	case "r":
		return "The message has been sent to the mobile operator."
	case "a":
		return "The mobile operator has acknowledged the message."
	case "b":
		return "The mobile operator has queued the message."
	case "d":
		return "The message has been successfully delivered to the handset."
	case "f":
		return "An error occurred while delivering message."
	case "e":
		return "An error occurred while sending message."
	case "j":
		return "The mobile operator has rejected the message."
	case "s":
		return "This message is scheduled to be sent later."
	default:
		return "The status is unknown."

	}
}

type Status struct {
	Text      string                   `json:"text"`
	Status    DeliveryNotificationCode `json:"status"`
	Created   int64                    `json:"created_time"`
	Reply     string                   `json:"reply_number"`
	Cost      float32                  `json:"credits_cost"`
	Completed int64                    `json:"completed_time"`
}

func (t TextMagic) MessageStatus(ids ...uint) (map[uint]Status, error) {
	statuses := make(map[uint]Status)
	for _, tIds := range splitSlice(ids) {
		messageIds := joinUints(tIds...)
		strStatuses := make(map[string]Status)
		err := t.sendAPI(cmdMessageStatus, url.Values{"ids": {messageIds}}, strStatuses)
		if err != nil {
			return statuses, err
		}
		for messageID, status := range strStatuses {
			id, err := strconv.Atoi(messageID)
			if err != nil {
				continue
			}
			statuses[uint(id)] = status
		}
	}
	return statuses, nil
}

type Number struct {
	Price   float32 `json:"price"`
	Country string  `json:"country"`
}

func (t TextMagic) CheckNumber(numbers ...uint) (map[uint]Number, error) {
	ns := make(map[string]Number)
	if err := t.sendAPI(cmdCheckNumber, url.Values{"phone": {joinUints(numbers...)}}, ns); err != nil {
		return nil, err
	}
	toRet := make(map[uint]Number)
	for n, data := range ns {
		number, err := strconv.Atoi(n)
		if err != nil {
			continue
		}
		toRet[uint(number)] = data
	}
	return toRet, nil
}

type deleted struct {
	Deleted []uint `json:"deleted"`
}

func (t TextMagic) DeleteReply(ids ...uint) ([]uint, error) {
	toRet := make([]uint, 0, len(ids))
	for _, tIds := range splitSlice(ids) {
		var d deleted
		if err := t.sendAPI(cmdDeleteReply, url.Values{"deleted": {joinUints(tIds...)}}, &d); err != nil {
			return toRet, err
		}
		toRet = append(toRet, d.Deleted...)
	}
	return toRet, nil
}

type Message struct {
	ID        uint   `json:"message_id"`
	From      uint   `json:"from"`
	Timestamp int64  `json:"timestamp"`
	Text      string `json:"text"`
}

type received struct {
	Messages []Message `json:"messages"`
	Unread   uint      `json:"unread"`
}

func (t TextMagic) Receive(lastRetrieved uint) (uint, []Message, error) {
	var r received
	err := t.sendAPI(cmdReceive, url.Values{"last_retrieved_id": {strconv.Itoa(int(lastRetrieved))}}, &r)
	return r.Unread, r.Messages, err
}

const joinSep = ','

func joinUints(u ...uint) string {
	toStr := make([]byte, 0, 10*len(u))
	var digits [21]byte
	for n, num := range u {
		if n > 0 {
			toStr = append(toStr, joinSep)
		}
		if num == 0 {
			toStr = append(toStr, '0')
			continue
		}
		pos := 21
		for ; num > 0; num /= 10 {
			pos--
			digits[pos] = '0' + byte(num%10)
		}
		toStr = append(toStr, digits[pos:]...)
	}
	return string(toStr)
}

const maxInSlice = 100

func splitSlice(slice []uint) [][]uint {
	toRet := make([][]uint, 0, len(slice)/maxInSlice+1)
	for len(slice) > 100 {
		toRet = append(toRet, slice[:100])
		slice = slice[100:]
	}
	if len(slice) > 0 {
		toRet = append(toRet, slice)
	}
	return toRet
}

// Errors

type APIError struct {
	Cmd     string
	Code    int    `json:"error_code"`
	Message string `json:"error_message"`
}

func (a APIError) Error() string {
	return "command " + a.Cmd + " returned the following API error: " + a.Message
}

type RequestError struct {
	Cmd string
	Err error
}

func (r RequestError) Error() string {
	return "command " + r.Cmd + " returned the following error while makeing the API call: " + r.Err.Error()
}

type StatusError struct {
	Cmd        string
	StatusCode int
}

func (s StatusError) Error() string {
	return "command " + s.Cmd + " returned a non-200 OK response: " + http.StatusText(s.StatusCode)
}

type JSONError struct {
	Cmd string
	Err error
}

func (j JSONError) Error() string {
	return "command " + j.Cmd + " returned malformed JSON: " + j.Err.Error()
}
<|endoftext|>


<commit_before>package main

import (
	//"github.com/masterminds/cookoo/src/cookoo"
	"cookoo"
	//"fmt"
	"flag"
	"os"
	"path"
	"time"
)

func main() {
	homedir := os.ExpandEnv("${HOME}/.skunk")
	flag.StringVar(&homedir, "confd", homedir, "Set the directory with settings.json")
	flag.Parse()

	project := flag.Arg(0)
	projectdir := os.ExpandEnv("${PWD}/" + project)
	registry, router, cxt := cookoo.Cookoo()

	cxt.Add("homedir", homedir)
	cxt.Add("basedir", projectdir)
	cxt.Add("project", project)
	cxt.Add("now", time.Now())

	registry.
		Route("scaffold", "Scaffold a new app.").
		Does(LoadSettings, "settings").
		Using("file").WithDefault(path.Join(homedir, "settings.json")).From("cxt:SettingsFile").
		Does(MakeDirectories, "dirs").
		Using("basedir").From("cxt:basedir").
		Using("directories").From("cxt:directories").
		Does(RenderTemplates, "template").
		Using("tpldir").From("cxt:homedir").
		Using("basedir").From("cxt:basedir").
		Using("templates").From("cxt:templates").
		Route("help", "Print help").
		Does(Usage, "HelpText").
		Done()

	//router.HandleRequest("help", cxt, false)
	router.HandleRequest("scaffold", cxt, false)
}
<commit_msg>Added type support (stage 1).<commit_after>package main

import (
	//"github.com/masterminds/cookoo/src/cookoo"
	"cookoo"
	"fmt"
	"flag"
	"os"
	"path"
	"time"
	"strings"
)

func main() {
	homedir := os.ExpandEnv("${HOME}/.skunk")
	var sets templateSet
	flag.StringVar(&homedir, "confd", homedir, "Set the directory with settings.json")
	flag.Var(&sets, "type", "Project type (e.g. 'go', 'php'). Separate multiple values with ','")
	flag.Parse()

	project := flag.Arg(0)
	projectdir := os.ExpandEnv("${PWD}/" + project)
	registry, router, cxt := cookoo.Cookoo()

	cxt.Add("homedir", homedir)
	cxt.Add("basedir", projectdir)
	cxt.Add("project", project)
	cxt.Add("now", time.Now())

	registry.
		Route("scaffold", "Scaffold a new app.").
		Does(LoadSettings, "settings").
		Using("file").WithDefault(path.Join(homedir, "settings.json")).From("cxt:SettingsFile").
		Does(MakeDirectories, "dirs").
		Using("basedir").From("cxt:basedir").
		Using("directories").From("cxt:directories").
		Does(RenderTemplates, "template").
		Using("tpldir").From("cxt:homedir").
		Using("basedir").From("cxt:basedir").
		Using("templates").From("cxt:templates").
		Route("help", "Print help").
		Does(Usage, "HelpText").
		Done()

	//router.HandleRequest("help", cxt, false)
	router.HandleRequest("scaffold", cxt, false)
}

type templateSet []string

func (t *templateSet) Set(arg string) error {
	// Split the string
	/*for _, str := range strings.Split(value, ",") {
		// Clean up string
		// append to the templateSet
		*t = append(*t, str)
	}*/
	*t = append(*t, strings.Split(arg, ",")...)
	return nil
}

func (t *templateSet) String() string {
	//strings.Join(t, ",")
	return fmt.Sprint(*t)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"sort"
	"text/template"

	"appengine"
	"appengine/datastore"
	"appengine/delay"
	"appengine/mail"
	"appengine/urlfetch"
)

const (
	mailFrom   = "builder@golang.org" // use this for sending any mail
	failMailTo = "golang-dev@googlegroups.com"
	domain     = "build.golang.org"
	gobotBase  = "http://research.swtch.com/gobot_codereview"
)

// ignoreFailure is a set of builders that we don't email about because
// they are not yet production-ready.
var ignoreFailure = map[string]bool{
	"dragonfly-386":           true,
	"dragonfly-amd64":         true,
	"netbsd-arm-rpi":          true,
	"solaris-amd64-smartos":   true,
	"solaris-amd64-solaris11": true,
}

// notifyOnFailure checks whether the supplied Commit or the subsequent
// Commit (if present) breaks the build for this builder.
// If either of those commits break the build an email notification is sent
// from a delayed task. (We use a task because this way the mail won't be
// sent if the enclosing datastore transaction fails.)
//
// This must be run in a datastore transaction, and the provided *Commit must
// have been retrieved from the datastore within that transaction.
func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
	if ignoreFailure[builder] {
		return nil
	}

	// TODO(adg): implement notifications for packages
	if com.PackagePath != "" {
		return nil
	}

	p := &Package{Path: com.PackagePath}
	var broken *Commit
	cr := com.Result(builder, "")
	if cr == nil {
		return fmt.Errorf("no result for %s/%s", com.Hash, builder)
	}
	q := datastore.NewQuery("Commit").Ancestor(p.Key(c))
	if cr.OK {
		// This commit is OK. Notify if next Commit is broken.
		next := new(Commit)
		q = q.Filter("ParentHash=", com.Hash)
		if err := firstMatch(c, q, next); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// OK at tip, no notification necessary.
				return nil
			}
			return err
		}
		if nr := next.Result(builder, ""); nr != nil && !nr.OK {
			c.Debugf("commit ok: %#v\nresult: %#v", com, cr)
			c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr)
			broken = next
		}
	} else {
		// This commit is broken. Notify if the previous Commit is OK.
		prev := new(Commit)
		q = q.Filter("Hash=", com.ParentHash)
		if err := firstMatch(c, q, prev); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// No previous result, let the backfill of
				// this result trigger the notification.
				return nil
			}
			return err
		}
		if pr := prev.Result(builder, ""); pr != nil && pr.OK {
			c.Debugf("commit broken: %#v\nresult: %#v", com, cr)
			c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr)
			broken = com
		}
	}
	if broken == nil {
		return nil
	}
	r := broken.Result(builder, "")
	if r == nil {
		return fmt.Errorf("finding result for %q: %+v", builder, com)
	}
	return commonNotify(c, broken, builder, r.LogHash)
}

// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
	t := q.Limit(1).Run(c)
	_, err := t.Next(v)
	if err == datastore.Done {
		err = datastore.ErrNoSuchEntity
	}
	return err
}

var notifyLater = delay.Func("notify", notify)

// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder, logHash string) {
	if !updateCL(c, com, builder, logHash) {
		// Send a mail notification if the CL can't be found.
		sendFailMail(c, com, builder, logHash)
	}
}

// updateCL updates the CL for the given Commit with a failure message
// for the given builder.
func updateCL(c appengine.Context, com *Commit, builder, logHash string) bool {
	cl, err := lookupCL(c, com)
	if err != nil {
		c.Errorf("could not find CL for %v: %v", com.Hash, err)
		return false
	}
	url := fmt.Sprintf("%v?cl=%v&brokebuild=%v&log=%v", gobotBase, cl, builder, logHash)
	r, err := urlfetch.Client(c).Post(url, "text/plain", nil)
	if err != nil {
		c.Errorf("could not update CL %v: %v", cl, err)
		return false
	}
	r.Body.Close()
	if r.StatusCode != http.StatusOK {
		c.Errorf("could not update CL %v: %v", cl, r.Status)
		return false
	}
	return true
}

var clURL = regexp.MustCompile(`https://codereview.appspot.com/([0-9]+)`)

// lookupCL consults code.google.com for the full change description for the
// provided Commit, and returns the relevant CL number.
func lookupCL(c appengine.Context, com *Commit) (string, error) {
	url := "https://code.google.com/p/go/source/detail?r=" + com.Hash
	r, err := urlfetch.Client(c).Get(url)
	if err != nil {
		return "", err
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return "", fmt.Errorf("retrieving %v: %v", url, r.Status)
	}
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return "", err
	}
	m := clURL.FindAllSubmatch(b, -1)
	if m == nil {
		return "", errors.New("no CL URL found on changeset page")
	}
	// Return the last visible codereview URL on the page,
	// in case the change description refers to another CL.
	return string(m[len(m)-1][1]), nil
}

var sendFailMailTmpl = template.Must(template.New("notify.txt").
	Funcs(template.FuncMap(tmplFuncs)).
	ParseFiles("build/notify.txt"))

func init() {
	gob.Register(&Commit{}) // for delay
}

var (
	sendPerfMailLater = delay.Func("sendPerfMail", sendPerfMailFunc)
	sendPerfMailTmpl  = template.Must(
		template.New("perf_notify.txt").
			Funcs(template.FuncMap(tmplFuncs)).
			ParseFiles("build/perf_notify.txt"),
	)
)

func sendPerfFailMail(c appengine.Context, builder string, res *PerfResult) error {
	com := &Commit{Hash: res.CommitHash}
	if err := datastore.Get(c, com.Key(c), com); err != nil {
		return fmt.Errorf("getting commit %v: %v", com.Hash, err)
	}
	logHash := ""
	parsed := res.ParseData()
	for _, data := range parsed[builder] {
		if !data.OK {
			logHash = data.Artifacts["log"]
			break
		}
	}
	if logHash == "" {
		return fmt.Errorf("can not find failed result for commit %v on builder %v", com.Hash, builder)
	}
	return commonNotify(c, com, builder, logHash)
}

func commonNotify(c appengine.Context, com *Commit, builder, logHash string) error {
	if com.FailNotificationSent {
		return nil
	}
	c.Infof("%s is broken commit; notifying", com.Hash)
	notifyLater.Call(c, com, builder, logHash) // add task to queue
	com.FailNotificationSent = true
	_, err := datastore.Put(c, com.Key(c), com)
	return err
}

// sendFailMail sends a mail notification that the build failed on the
// provided commit and builder.
func sendFailMail(c appengine.Context, com *Commit, builder, logHash string) {
	// get Log
	k := datastore.NewKey(c, "Log", logHash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, k, l); err != nil {
		c.Errorf("finding Log record %v: %v", logHash, err)
		return
	}
	logText, err := l.Text()
	if err != nil {
		c.Errorf("unpacking Log record %v: %v", logHash, err)
		return
	}

	// prepare mail message
	var body bytes.Buffer
	err = sendFailMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "LogHash": logHash, "LogText": logText,
		"Hostname": domain,
	})
	if err != nil {
		c.Errorf("rendering mail template: %v", err)
		return
	}
	subject := fmt.Sprintf("%s broken by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}

type PerfChangeBenchmark struct {
	Name    string
	Metrics []*PerfChangeMetric
}

type PerfChangeMetric struct {
	Name  string
	Old   uint64
	New   uint64
	Delta float64
}

type PerfChangeBenchmarkSlice []*PerfChangeBenchmark

func (l PerfChangeBenchmarkSlice) Len() int      { return len(l) }
func (l PerfChangeBenchmarkSlice) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
func (l PerfChangeBenchmarkSlice) Less(i, j int) bool {
	b1, p1 := splitBench(l[i].Name)
	b2, p2 := splitBench(l[j].Name)
	if b1 != b2 {
		return b1 < b2
	}
	return p1 < p2
}

type PerfChangeMetricSlice []*PerfChangeMetric

func (l PerfChangeMetricSlice) Len() int           { return len(l) }
func (l PerfChangeMetricSlice) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
func (l PerfChangeMetricSlice) Less(i, j int) bool { return l[i].Name < l[j].Name }

func sendPerfMailFunc(c appengine.Context, com *Commit, prevCommitHash, builder string, changes []*PerfChange) {
	// Sort the changes into the right order.
	var benchmarks []*PerfChangeBenchmark
	for _, ch := range changes {
		// Find the benchmark.
		var b *PerfChangeBenchmark
		for _, b1 := range benchmarks {
			if b1.Name == ch.Bench {
				b = b1
				break
			}
		}
		if b == nil {
			b = &PerfChangeBenchmark{Name: ch.Bench}
			benchmarks = append(benchmarks, b)
		}
		b.Metrics = append(b.Metrics, &PerfChangeMetric{Name: ch.Metric, Old: ch.Old, New: ch.New, Delta: ch.Diff})
	}
	for _, b := range benchmarks {
		sort.Sort(PerfChangeMetricSlice(b.Metrics))
	}
	sort.Sort(PerfChangeBenchmarkSlice(benchmarks))

	url := fmt.Sprintf("http://%v/perfdetail?commit=%v&commit0=%v&kind=builder&builder=%v", domain, com.Hash, prevCommitHash, builder)

	// prepare mail message
	var body bytes.Buffer
	err := sendPerfMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "Hostname": domain, "Url": url, "Benchmarks": benchmarks,
	})
	if err != nil {
		c.Errorf("rendering perf mail template: %v", err)
		return
	}
	subject := fmt.Sprintf("Perf changes on %s by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}
<commit_msg>go.tools/dashboard/app: update CL threads with perf notifications<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"text/template"

	"appengine"
	"appengine/datastore"
	"appengine/delay"
	"appengine/mail"
	"appengine/urlfetch"
)

const (
	mailFrom   = "builder@golang.org" // use this for sending any mail
	failMailTo = "golang-dev@googlegroups.com"
	domain     = "build.golang.org"
	gobotBase  = "http://research.swtch.com/gobot_codereview"
)

// ignoreFailure is a set of builders that we don't email about because
// they are not yet production-ready.
var ignoreFailure = map[string]bool{
	"dragonfly-386":           true,
	"dragonfly-amd64":         true,
	"netbsd-arm-rpi":          true,
	"solaris-amd64-smartos":   true,
	"solaris-amd64-solaris11": true,
}

// notifyOnFailure checks whether the supplied Commit or the subsequent
// Commit (if present) breaks the build for this builder.
// If either of those commits break the build an email notification is sent
// from a delayed task. (We use a task because this way the mail won't be
// sent if the enclosing datastore transaction fails.)
//
// This must be run in a datastore transaction, and the provided *Commit must
// have been retrieved from the datastore within that transaction.
func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
	if ignoreFailure[builder] {
		return nil
	}

	// TODO(adg): implement notifications for packages
	if com.PackagePath != "" {
		return nil
	}

	p := &Package{Path: com.PackagePath}
	var broken *Commit
	cr := com.Result(builder, "")
	if cr == nil {
		return fmt.Errorf("no result for %s/%s", com.Hash, builder)
	}
	q := datastore.NewQuery("Commit").Ancestor(p.Key(c))
	if cr.OK {
		// This commit is OK. Notify if next Commit is broken.
		next := new(Commit)
		q = q.Filter("ParentHash=", com.Hash)
		if err := firstMatch(c, q, next); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// OK at tip, no notification necessary.
				return nil
			}
			return err
		}
		if nr := next.Result(builder, ""); nr != nil && !nr.OK {
			c.Debugf("commit ok: %#v\nresult: %#v", com, cr)
			c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr)
			broken = next
		}
	} else {
		// This commit is broken. Notify if the previous Commit is OK.
		prev := new(Commit)
		q = q.Filter("Hash=", com.ParentHash)
		if err := firstMatch(c, q, prev); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// No previous result, let the backfill of
				// this result trigger the notification.
				return nil
			}
			return err
		}
		if pr := prev.Result(builder, ""); pr != nil && pr.OK {
			c.Debugf("commit broken: %#v\nresult: %#v", com, cr)
			c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr)
			broken = com
		}
	}
	if broken == nil {
		return nil
	}
	r := broken.Result(builder, "")
	if r == nil {
		return fmt.Errorf("finding result for %q: %+v", builder, com)
	}
	return commonNotify(c, broken, builder, r.LogHash)
}

// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
	t := q.Limit(1).Run(c)
	_, err := t.Next(v)
	if err == datastore.Done {
		err = datastore.ErrNoSuchEntity
	}
	return err
}

var notifyLater = delay.Func("notify", notify)

// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder, logHash string) {
	v := url.Values{"brokebuild": {builder}, "log": {logHash}}
	if !updateCL(c, com, v) {
		// Send a mail notification if the CL can't be found.
		sendFailMail(c, com, builder, logHash)
	}
}

// updateCL tells gobot to update the CL for the given Commit with
// the provided query values.
func updateCL(c appengine.Context, com *Commit, v url.Values) bool {
	cl, err := lookupCL(c, com)
	if err != nil {
		c.Errorf("could not find CL for %v: %v", com.Hash, err)
		return false
	}
	u := fmt.Sprintf("%v?cl=%v&%s", gobotBase, cl, v.Encode())
	r, err := urlfetch.Client(c).Post(u, "text/plain", nil)
	if err != nil {
		c.Errorf("could not update CL %v: %v", cl, err)
		return false
	}
	r.Body.Close()
	if r.StatusCode != http.StatusOK {
		c.Errorf("could not update CL %v: %v", cl, r.Status)
		return false
	}
	return true
}

var clURL = regexp.MustCompile(`https://codereview.appspot.com/([0-9]+)`)

// lookupCL consults code.google.com for the full change description for the
// provided Commit, and returns the relevant CL number.
func lookupCL(c appengine.Context, com *Commit) (string, error) {
	url := "https://code.google.com/p/go/source/detail?r=" + com.Hash
	r, err := urlfetch.Client(c).Get(url)
	if err != nil {
		return "", err
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return "", fmt.Errorf("retrieving %v: %v", url, r.Status)
	}
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return "", err
	}
	m := clURL.FindAllSubmatch(b, -1)
	if m == nil {
		return "", errors.New("no CL URL found on changeset page")
	}
	// Return the last visible codereview URL on the page,
	// in case the change description refers to another CL.
	return string(m[len(m)-1][1]), nil
}

var sendFailMailTmpl = template.Must(template.New("notify.txt").
	Funcs(template.FuncMap(tmplFuncs)).
	ParseFiles("build/notify.txt"))

func init() {
	gob.Register(&Commit{}) // for delay
}

var (
	sendPerfMailLater = delay.Func("sendPerfMail", sendPerfMailFunc)
	sendPerfMailTmpl  = template.Must(
		template.New("perf_notify.txt").
			Funcs(template.FuncMap(tmplFuncs)).
			ParseFiles("build/perf_notify.txt"),
	)
)

func sendPerfFailMail(c appengine.Context, builder string, res *PerfResult) error {
	com := &Commit{Hash: res.CommitHash}
	logHash := ""
	parsed := res.ParseData()
	for _, data := range parsed[builder] {
		if !data.OK {
			logHash = data.Artifacts["log"]
			break
		}
	}
	if logHash == "" {
		return fmt.Errorf("can not find failed result for commit %v on builder %v", com.Hash, builder)
	}
	return commonNotify(c, com, builder, logHash)
}

func commonNotify(c appengine.Context, com *Commit, builder, logHash string) error {
	if com.FailNotificationSent {
		return nil
	}
	c.Infof("%s is broken commit; notifying", com.Hash)
	notifyLater.Call(c, com, builder, logHash) // add task to queue
	com.FailNotificationSent = true
	_, err := datastore.Put(c, com.Key(c), com)
	return err
}

// sendFailMail sends a mail notification that the build failed on the
// provided commit and builder.
func sendFailMail(c appengine.Context, com *Commit, builder, logHash string) {
	// get Log
	k := datastore.NewKey(c, "Log", logHash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, k, l); err != nil {
		c.Errorf("finding Log record %v: %v", logHash, err)
		return
	}
	logText, err := l.Text()
	if err != nil {
		c.Errorf("unpacking Log record %v: %v", logHash, err)
		return
	}

	// prepare mail message
	var body bytes.Buffer
	err = sendFailMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "LogHash": logHash, "LogText": logText,
		"Hostname": domain,
	})
	if err != nil {
		c.Errorf("rendering mail template: %v", err)
		return
	}
	subject := fmt.Sprintf("%s broken by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}

type PerfChangeBenchmark struct {
	Name    string
	Metrics []*PerfChangeMetric
}

type PerfChangeMetric struct {
	Name  string
	Old   uint64
	New   uint64
	Delta float64
}

type PerfChangeBenchmarkSlice []*PerfChangeBenchmark

func (l PerfChangeBenchmarkSlice) Len() int      { return len(l) }
func (l PerfChangeBenchmarkSlice) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
func (l PerfChangeBenchmarkSlice) Less(i, j int) bool {
	b1, p1 := splitBench(l[i].Name)
	b2, p2 := splitBench(l[j].Name)
	if b1 != b2 {
		return b1 < b2
	}
	return p1 < p2
}

type PerfChangeMetricSlice []*PerfChangeMetric

func (l PerfChangeMetricSlice) Len() int           { return len(l) }
func (l PerfChangeMetricSlice) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
func (l PerfChangeMetricSlice) Less(i, j int) bool { return l[i].Name < l[j].Name }

func sendPerfMailFunc(c appengine.Context, com *Commit, prevCommitHash, builder string, changes []*PerfChange) {
	// Sort the changes into the right order.
	var benchmarks []*PerfChangeBenchmark
	for _, ch := range changes {
		// Find the benchmark.
		var b *PerfChangeBenchmark
		for _, b1 := range benchmarks {
			if b1.Name == ch.Bench {
				b = b1
				break
			}
		}
		if b == nil {
			b = &PerfChangeBenchmark{Name: ch.Bench}
			benchmarks = append(benchmarks, b)
		}
		b.Metrics = append(b.Metrics, &PerfChangeMetric{Name: ch.Metric, Old: ch.Old, New: ch.New, Delta: ch.Diff})
	}
	for _, b := range benchmarks {
		sort.Sort(PerfChangeMetricSlice(b.Metrics))
	}
	sort.Sort(PerfChangeBenchmarkSlice(benchmarks))

	u := fmt.Sprintf("http://%v/perfdetail?commit=%v&commit0=%v&kind=builder&builder=%v", domain, com.Hash, prevCommitHash, builder)

	// prepare mail message
	var body bytes.Buffer
	err := sendPerfMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "Hostname": domain, "Url": u, "Benchmarks": benchmarks,
	})
	if err != nil {
		c.Errorf("rendering perf mail template: %v", err)
		return
	}

	// First, try to update the CL.
	v := url.Values{"textmsg": {body.String()}}
	if updateCL(c, com, v) {
		return
	}

	// Otherwise, send mail.
	subject := fmt.Sprintf("Perf changes on %s by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"time"

	"github.com/jasonlvhit/gocron"
)

func main() {
	scheduler := gocron.NewScheduler()
	scheduler.Every(1).Day().At("00:00").Do(runSchedulerTask, nil)
	<-scheduler.Start()
}

func runSchedulerTask() {
	archive, err := buildArchive(os.Getenv("TOGLACIER_PATH"))
	if err != nil {
		log.Fatal(err)
	}
	defer archive.Close()

	result, err := sendArchive(archive, os.Getenv("AWS_ACCOUNT_ID"), os.Getenv("AWS_VAULT_NAME"))
	if err != nil {
		log.Fatal(err)
	}

	auditFile, err := os.OpenFile(os.Getenv("TOGLACIER_AUDIT"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		log.Fatalf("error opening the audit file. details: %s", err)
	}
	defer auditFile.Close()

	audit := fmt.Sprintf("%s %s %s\n", result.time.Format(time.RFC3339), result.location, result.checksum)
	if _, err = auditFile.WriteString(audit); err != nil {
		log.Fatalf("error writing the audit file. details: %s", err)
	}
}
<commit_msg>Remove temporary tarball after using it<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"time"

	"github.com/jasonlvhit/gocron"
)

func main() {
	scheduler := gocron.NewScheduler()
	scheduler.Every(1).Day().At("00:00").Do(runSchedulerTask, nil)
	<-scheduler.Start()
}

func runSchedulerTask() {
	archive, err := buildArchive(os.Getenv("TOGLACIER_PATH"))
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		archive.Close()
		// remove the temporary tarball
		os.Remove(archive.Name())
	}()

	result, err := sendArchive(archive, os.Getenv("AWS_ACCOUNT_ID"), os.Getenv("AWS_VAULT_NAME"))
	if err != nil {
		log.Fatal(err)
	}

	auditFile, err := os.OpenFile(os.Getenv("TOGLACIER_AUDIT"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		log.Fatalf("error opening the audit file. details: %s", err)
	}
	defer auditFile.Close()

	audit := fmt.Sprintf("%s %s %s\n", result.time.Format(time.RFC3339), result.location, result.checksum)
	if _, err = auditFile.WriteString(audit); err != nil {
		log.Fatalf("error writing the audit file. details: %s", err)
	}
}
<|endoftext|>