<commit_before>package expr

import (
	"fmt"
	"math"

	"github.com/grafana/metrictank/api/models"
)

var results []models.Series

func equalOutput(exp, got []models.Series, expErr, gotErr error) error {
	if expErr == nil && gotErr != nil {
		return fmt.Errorf("err should be nil. got %q", gotErr)
	}
	if expErr != nil && gotErr == nil {
		return fmt.Errorf("err should be error %v. got %q", expErr, gotErr)
	}
	if len(got) != len(exp) {
		return fmt.Errorf("perSecond len output expected %d, got %d", len(exp), len(got))
	}
	for i := range got {
		if err := equalSeries(exp[i], got[i]); err != nil {
			return fmt.Errorf("series %d: %s", i, err)
		}
	}
	return nil
}

// cannot just use reflect.DeepEqual because NaN != NaN, whereas we want NaN == NaN
// https://github.com/golang/go/issues/12025
func equalSeries(exp, got models.Series) error {
	if got.Target != exp.Target {
		return fmt.Errorf("Target %q, got %q", exp.Target, got.Target)
	}
	if got.Interval != exp.Interval {
		return fmt.Errorf("Interval %d, got %d", exp.Interval, got.Interval)
	}
	if got.QueryPatt != exp.QueryPatt {
		return fmt.Errorf("QueryPatt %q, got %q", exp.QueryPatt, got.QueryPatt)
	}
	if got.QueryFrom != exp.QueryFrom {
		return fmt.Errorf("QueryFrom %d, got %d", exp.QueryFrom, got.QueryFrom)
	}
	if got.QueryTo != exp.QueryTo {
		return fmt.Errorf("QueryTo %d, got %d", exp.QueryTo, got.QueryTo)
	}
	if got.QueryCons != exp.QueryCons {
		return fmt.Errorf("QueryCons %v, got %v", exp.QueryCons, got.QueryCons)
	}
	if got.Consolidator != exp.Consolidator {
		return fmt.Errorf("Consolidator %v, got %v", exp.Consolidator, got.Consolidator)
	}
	if len(got.Datapoints) != len(exp.Datapoints) {
		return fmt.Errorf("output expected %d, got %d", len(exp.Datapoints), len(got.Datapoints))
	}
	for j, p := range got.Datapoints {
		bothNaN := math.IsNaN(p.Val) && math.IsNaN(exp.Datapoints[j].Val)
		if (bothNaN || p.Val == exp.Datapoints[j].Val) && p.Ts == exp.Datapoints[j].Ts {
			continue
		}
		return fmt.Errorf("point %d - expected %v got %v", j, exp.Datapoints[j], p)
	}
	return nil
}
<commit_msg>Add epsilon to double comparisons<commit_after>package expr

import (
	"fmt"
	"math"

	"github.com/grafana/metrictank/api/models"
)

var results []models.Series

func equalOutput(exp, got []models.Series, expErr, gotErr error) error {
	if expErr == nil && gotErr != nil {
		return fmt.Errorf("err should be nil. got %q", gotErr)
	}
	if expErr != nil && gotErr == nil {
		return fmt.Errorf("err should be error %v. got %q", expErr, gotErr)
	}
	if len(got) != len(exp) {
		return fmt.Errorf("perSecond len output expected %d, got %d", len(exp), len(got))
	}
	for i := range got {
		if err := equalSeries(exp[i], got[i]); err != nil {
			return fmt.Errorf("series %d: %s", i, err)
		}
	}
	return nil
}

// cannot just use reflect.DeepEqual because NaN != NaN, whereas we want NaN == NaN
// https://github.com/golang/go/issues/12025
func equalSeries(exp, got models.Series) error {
	if got.Target != exp.Target {
		return fmt.Errorf("Target %q, got %q", exp.Target, got.Target)
	}
	if got.Interval != exp.Interval {
		return fmt.Errorf("Interval %d, got %d", exp.Interval, got.Interval)
	}
	if got.QueryPatt != exp.QueryPatt {
		return fmt.Errorf("QueryPatt %q, got %q", exp.QueryPatt, got.QueryPatt)
	}
	if got.QueryFrom != exp.QueryFrom {
		return fmt.Errorf("QueryFrom %d, got %d", exp.QueryFrom, got.QueryFrom)
	}
	if got.QueryTo != exp.QueryTo {
		return fmt.Errorf("QueryTo %d, got %d", exp.QueryTo, got.QueryTo)
	}
	if got.QueryCons != exp.QueryCons {
		return fmt.Errorf("QueryCons %v, got %v", exp.QueryCons, got.QueryCons)
	}
	if got.Consolidator != exp.Consolidator {
		return fmt.Errorf("Consolidator %v, got %v", exp.Consolidator, got.Consolidator)
	}
	if len(got.Datapoints) != len(exp.Datapoints) {
		return fmt.Errorf("output expected %d, got %d", len(exp.Datapoints), len(got.Datapoints))
	}
	for j, p := range got.Datapoints {
		if (doubleFuzzyEqual(p.Val, exp.Datapoints[j].Val)) && p.Ts == exp.Datapoints[j].Ts {
			continue
		}
		return fmt.Errorf("point %d - expected %v got %v", j, exp.Datapoints[j], p)
	}
	return nil
}

func doubleFuzzyEqual(a, b float64) bool {
	if math.IsNaN(a) && math.IsNaN(b) {
		return true
	}
	var epsilon = 1e-10
	return a == b || math.Abs(a-b) < epsilon
}
<|endoftext|>


<commit_before>package main

import (
	"gitlab.com/flimzy/ale/errors"

	"foo/bar"
)

func main() {
	sErr := bar.SomeError{}
	// sErr := &bar.SomeError{}
	err := bar.Fail()

	if errors.As(err, &sErr) && sErr.Code() == 42 {
		panic("omg that's a big number")
	}
	panic("phew all is fine")
}
<commit_msg>fail: include the error message<commit_after>package main

import (
	"fmt"

	"gitlab.com/flimzy/ale/errors"

	"foo/bar"
)

func main() {
	sErr := bar.SomeError{}
	// sErr := &bar.SomeError{}
	err := bar.Fail()

	if errors.As(err, &sErr) && sErr.Code() == 42 {
		panic(fmt.Sprintf("omg that's a big number: %v", sErr))
	}
	panic("phew all is fine")
}
<|endoftext|>


<commit_before>package docker

type HostConfig struct {
	Binds   []string
	LxcConf map[string]string
}
<commit_msg>add PortBindings to HostConfig<commit_after>package docker

type Port string

type HostConfig struct {
	Binds        []string
	PortBindings map[Port][]PortBinding
	LxcConf      map[string]string
}

type PortBinding struct {
	HostIp   string
	HostPort string
}
<|endoftext|>


<commit_before>package main

const NGINX_BUILD_VERSION = "0.3.3"

// nginx
const (
	NGINX_VERSION             = "1.7.12"
	NGINX_DOWNLOAD_URL_PREFIX = "http://nginx.org/download"
)

// pcre
const (
	PCRE_VERSION             = "8.36"
	PCRE_DOWNLOAD_URL_PREFIX = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OPENSSL_VERSION             = "1.0.2a"
	OPENSSL_DOWNLOAD_URL_PREFIX = "http://www.openssl.org/source"
)

// zlib
const (
	ZLIB_VERSION             = "1.2.8"
	ZLIB_DOWNLOAD_URL_PREFIX = "http://zlib.net"
)

// openResty
const (
	OPENRESTY_VERSION             = "1.7.10.1"
	OPENRESTY_DOWNLOAD_URL_PREFIX = "http://openresty.org/download"
)

// tengine
const (
	TENGINE_VERSION             = "2.1.0"
	TENGINE_DOWNLOAD_URL_PREFIX = "http://tengine.taobao.org/download"
)

// component enumerations
const (
	COMPONENT_NGINX = iota
	COMPONENT_OPENRESTY
	COMPONENT_TENGINE
	COMPONENT_PCRE
	COMPONENT_OPENSSL
	COMPONENT_ZLIB
	COMPONENT_MAX
)
<commit_msg>bumped version to 1.8.0.<commit_after>package main

const NGINX_BUILD_VERSION = "0.3.3"

// nginx
const (
	NGINX_VERSION             = "1.8.0"
	NGINX_DOWNLOAD_URL_PREFIX = "http://nginx.org/download"
)

// pcre
const (
	PCRE_VERSION             = "8.36"
	PCRE_DOWNLOAD_URL_PREFIX = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OPENSSL_VERSION             = "1.0.2a"
	OPENSSL_DOWNLOAD_URL_PREFIX = "http://www.openssl.org/source"
)

// zlib
const (
	ZLIB_VERSION             = "1.2.8"
	ZLIB_DOWNLOAD_URL_PREFIX = "http://zlib.net"
)

// openResty
const (
	OPENRESTY_VERSION             = "1.7.10.1"
	OPENRESTY_DOWNLOAD_URL_PREFIX = "http://openresty.org/download"
)

// tengine
const (
	TENGINE_VERSION             = "2.1.0"
	TENGINE_DOWNLOAD_URL_PREFIX = "http://tengine.taobao.org/download"
)

// component enumerations
const (
	COMPONENT_NGINX = iota
	COMPONENT_OPENRESTY
	COMPONENT_TENGINE
	COMPONENT_PCRE
	COMPONENT_OPENSSL
	COMPONENT_ZLIB
	COMPONENT_MAX
)
<|endoftext|>


<commit_before>package controller

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/goadesign/goa"
	influxClient "github.com/influxdata/influxdb/client/v2"
	"github.com/pkg/errors"
	"gitlab.com/remp/remp/Beam/go/cmd/tracker/app"
	"gitlab.com/remp/remp/Beam/go/model"
)

// TrackController implements the track resource.
type TrackController struct {
	*goa.Controller
	EventProducer   sarama.AsyncProducer
	PropertyStorage model.PropertyStorage
}

// Event represents Influx event structure
type Event struct {
	Action   string                 `json:"action"`
	Category string                 `json:"category"`
	Fields   map[string]interface{} `json:"fields"`
	Value    float64                `json:"value"`
}

// NewTrackController creates a track controller.
func NewTrackController(service *goa.Service, ep sarama.AsyncProducer, ps model.PropertyStorage) *TrackController {
	return &TrackController{
		Controller:      service.NewController("TrackController"),
		EventProducer:   ep,
		PropertyStorage: ps,
	}
}

// Commerce runs the commerce action.
func (c *TrackController) Commerce(ctx *app.CommerceTrackContext) error {
	_, ok, err := c.PropertyStorage.Get(ctx.Payload.System.PropertyToken.String())
	if err != nil {
		return err
	}
	if !ok {
		return ctx.NotFound()
	}

	tags := map[string]string{
		"step": ctx.Payload.Step,
	}
	values := map[string]interface{}{}

	if ctx.Payload.Article != nil {
		at, av := articleValues(ctx.Payload.Article)
		for key, tag := range at {
			tags[key] = tag
		}
		for key, val := range av {
			values[key] = val
		}
	}

	switch ctx.Payload.Step {
	case "checkout":
		values["funnel_id"] = ctx.Payload.Checkout.FunnelID
	case "payment":
		if ctx.Payload.Payment.FunnelID != nil {
			values["funnel_id"] = *ctx.Payload.Payment.FunnelID
		}
		values["product_ids"] = strings.Join(ctx.Payload.Payment.ProductIds, ",")
		values["revenue"] = ctx.Payload.Payment.Revenue.Amount
		values["transaction_id"] = ctx.Payload.Payment.TransactionID
		tags["currency"] = ctx.Payload.Payment.Revenue.Currency
	case "purchase":
		if ctx.Payload.Purchase.FunnelID != nil {
			values["funnel_id"] = *ctx.Payload.Purchase.FunnelID
		}
		values["product_ids"] = strings.Join(ctx.Payload.Purchase.ProductIds, ",")
		values["revenue"] = ctx.Payload.Purchase.Revenue.Amount
		values["transaction_id"] = ctx.Payload.Purchase.TransactionID
		tags["currency"] = ctx.Payload.Purchase.Revenue.Currency
	case "refund":
		if ctx.Payload.Refund.FunnelID != nil {
			values["funnel_id"] = *ctx.Payload.Refund.FunnelID
		}
		values["product_ids"] = strings.Join(ctx.Payload.Refund.ProductIds, ",")
		values["revenue"] = ctx.Payload.Refund.Revenue.Amount
		values["transaction_id"] = ctx.Payload.Refund.TransactionID
		tags["currency"] = ctx.Payload.Refund.Revenue.Currency
	default:
		return fmt.Errorf("unhandled commerce step: %s", ctx.Payload.Step)
	}

	if err := c.pushInternal(ctx.Payload.System, ctx.Payload.User, model.TableCommerce, tags, values); err != nil {
		return err
	}

	topic := fmt.Sprintf("%s_%s", "commerce", ctx.Payload.Step)
	value, err := json.Marshal(ctx.Payload)
	if err != nil {
		return errors.Wrap(err, "unable to marshal payload for kafka")
	}
	c.pushPublic(topic, value)

	return ctx.Accepted()
}

// Event runs the event action.
func (c *TrackController) Event(ctx *app.EventTrackContext) error {
	_, ok, err := c.PropertyStorage.Get(ctx.Payload.System.PropertyToken.String())
	if err != nil {
		return err
	}
	if !ok {
		return ctx.NotFound()
	}

	tags := map[string]string{
		"category": ctx.Payload.Category,
		"action":   ctx.Payload.Action,
	}
	fields := map[string]interface{}{}
	if ctx.Payload.Value != nil {
		fields["value"] = *ctx.Payload.Value
	}
	for key, val := range ctx.Payload.Fields {
		fields[key] = val
	}
	if err := c.pushInternal(ctx.Payload.System, ctx.Payload.User, model.TableEvents, tags, fields); err != nil {
		return err
	}

	// push public

	topic := fmt.Sprintf("%s_%s", ctx.Payload.Category, ctx.Payload.Action)
	value, err := json.Marshal(ctx.Payload)
	if err != nil {
		return errors.Wrap(err, "unable to marshal payload for kafka")
	}
	c.pushPublic(topic, value)

	return ctx.Accepted()
}

// Pageview runs the pageview action.
func (c *TrackController) Pageview(ctx *app.PageviewTrackContext) error {
	_, ok, err := c.PropertyStorage.Get(ctx.Payload.System.PropertyToken.String())
	if err != nil {
		return err
	}
	if !ok {
		return ctx.NotFound()
	}

	tags := map[string]string{
		"category": model.CategoryPageview,
		"action":   model.ActionPageviewLoad,
	}
	values := map[string]interface{}{}

	if ctx.Payload.Article != nil {
		at, av := articleValues(ctx.Payload.Article)
		for key, tag := range at {
			tags[key] = tag
		}
		for key, val := range av {
			values[key] = val
		}
	}

	if err := c.pushInternal(ctx.Payload.System, ctx.Payload.User, model.TablePageviews, tags, values); err != nil {
		return err
	}
	return ctx.Accepted()
}

func articleValues(article *app.Article) (map[string]string, map[string]interface{}) {
	tags := map[string]string{
		"article_id": article.ID,
	}
	values := map[string]interface{}{}
	if article.AuthorID != nil {
		tags["author_id"] = *article.AuthorID
	}
	if article.Category != nil {
		tags["category"] = *article.Category
	}
	if article.Tags != nil {
		values["tags"] = strings.Join(article.Tags, ",")
	}
	return tags, values
}

// pushInternal pushes new event to the InfluxDB.
func (c *TrackController) pushInternal(system *app.System, user *app.User,
	name string, tags map[string]string, fields map[string]interface{}) error {
	fields["token"] = system.PropertyToken

	if user != nil {
		if user.IPAddress != nil {
			fields["ip"] = *user.IPAddress
		}
		if user.URL != nil {
			fields["url"] = *user.URL
		}
		if user.UserAgent != nil {
			fields["user_agent"] = *user.UserAgent
		}
		if user.ID != nil {
			tags["user_id"] = *user.ID
		}

		if user.Source != nil {
			if user.Source.Social != nil {
				tags["social"] = *user.Source.Social
			}
			if user.Source.UtmSource != nil {
				tags["utm_source"] = *user.Source.UtmSource
			}
			if user.Source.UtmMedium != nil {
				tags["utm_medium"] = *user.Source.UtmMedium
			}
			if user.Source.UtmCampaign != nil {
				tags["utm_campaign"] = *user.Source.UtmCampaign
			}
			if user.Source.UtmContent != nil {
				tags["utm_content"] = *user.Source.UtmContent
			}
		}
	}

	p, err := influxClient.NewPoint(name, tags, fields, system.Time)
	if err != nil {
		return err
	}
	c.EventProducer.Input() <- &sarama.ProducerMessage{
		Topic: "beam_events",
		Value: sarama.StringEncoder(p.String()),
	}
	return nil
}

func (c *TrackController) pushPublic(topic string, value []byte) {
	c.EventProducer.Input() <- &sarama.ProducerMessage{
		Topic: topic,
		Value: sarama.ByteEncoder(value),
	}
}
<commit_msg>Tracking info whether article is present or not.<commit_after>package controller

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/goadesign/goa"
	influxClient "github.com/influxdata/influxdb/client/v2"
	"github.com/pkg/errors"
	"gitlab.com/remp/remp/Beam/go/cmd/tracker/app"
	"gitlab.com/remp/remp/Beam/go/model"
)

// TrackController implements the track resource.
type TrackController struct {
	*goa.Controller
	EventProducer   sarama.AsyncProducer
	PropertyStorage model.PropertyStorage
}

// Event represents Influx event structure
type Event struct {
	Action   string                 `json:"action"`
	Category string                 `json:"category"`
	Fields   map[string]interface{} `json:"fields"`
	Value    float64                `json:"value"`
}

// NewTrackController creates a track controller.
func NewTrackController(service *goa.Service, ep sarama.AsyncProducer, ps model.PropertyStorage) *TrackController {
	return &TrackController{
		Controller:      service.NewController("TrackController"),
		EventProducer:   ep,
		PropertyStorage: ps,
	}
}

// Commerce runs the commerce action.
func (c *TrackController) Commerce(ctx *app.CommerceTrackContext) error {
	_, ok, err := c.PropertyStorage.Get(ctx.Payload.System.PropertyToken.String())
	if err != nil {
		return err
	}
	if !ok {
		return ctx.NotFound()
	}

	tags := map[string]string{
		"step": ctx.Payload.Step,
	}
	values := map[string]interface{}{}

	if ctx.Payload.Article != nil {
		at, av := articleValues(ctx.Payload.Article)
		for key, tag := range at {
			tags[key] = tag
		}
		for key, val := range av {
			values[key] = val
		}
	}

	switch ctx.Payload.Step {
	case "checkout":
		values["funnel_id"] = ctx.Payload.Checkout.FunnelID
	case "payment":
		if ctx.Payload.Payment.FunnelID != nil {
			values["funnel_id"] = *ctx.Payload.Payment.FunnelID
		}
		values["product_ids"] = strings.Join(ctx.Payload.Payment.ProductIds, ",")
		values["revenue"] = ctx.Payload.Payment.Revenue.Amount
		values["transaction_id"] = ctx.Payload.Payment.TransactionID
		tags["currency"] = ctx.Payload.Payment.Revenue.Currency
	case "purchase":
		if ctx.Payload.Purchase.FunnelID != nil {
			values["funnel_id"] = *ctx.Payload.Purchase.FunnelID
		}
		values["product_ids"] = strings.Join(ctx.Payload.Purchase.ProductIds, ",")
		values["revenue"] = ctx.Payload.Purchase.Revenue.Amount
		values["transaction_id"] = ctx.Payload.Purchase.TransactionID
		tags["currency"] = ctx.Payload.Purchase.Revenue.Currency
	case "refund":
		if ctx.Payload.Refund.FunnelID != nil {
			values["funnel_id"] = *ctx.Payload.Refund.FunnelID
		}
		values["product_ids"] = strings.Join(ctx.Payload.Refund.ProductIds, ",")
		values["revenue"] = ctx.Payload.Refund.Revenue.Amount
		values["transaction_id"] = ctx.Payload.Refund.TransactionID
		tags["currency"] = ctx.Payload.Refund.Revenue.Currency
	default:
		return fmt.Errorf("unhandled commerce step: %s", ctx.Payload.Step)
	}

	if err := c.pushInternal(ctx.Payload.System, ctx.Payload.User, model.TableCommerce, tags, values); err != nil {
		return err
	}

	topic := fmt.Sprintf("%s_%s", "commerce", ctx.Payload.Step)
	value, err := json.Marshal(ctx.Payload)
	if err != nil {
		return errors.Wrap(err, "unable to marshal payload for kafka")
	}
	c.pushPublic(topic, value)

	return ctx.Accepted()
}

// Event runs the event action.
func (c *TrackController) Event(ctx *app.EventTrackContext) error {
	_, ok, err := c.PropertyStorage.Get(ctx.Payload.System.PropertyToken.String())
	if err != nil {
		return err
	}
	if !ok {
		return ctx.NotFound()
	}

	tags := map[string]string{
		"category": ctx.Payload.Category,
		"action":   ctx.Payload.Action,
	}
	fields := map[string]interface{}{}
	if ctx.Payload.Value != nil {
		fields["value"] = *ctx.Payload.Value
	}
	for key, val := range ctx.Payload.Fields {
		fields[key] = val
	}
	if err := c.pushInternal(ctx.Payload.System, ctx.Payload.User, model.TableEvents, tags, fields); err != nil {
		return err
	}

	// push public

	topic := fmt.Sprintf("%s_%s", ctx.Payload.Category, ctx.Payload.Action)
	value, err := json.Marshal(ctx.Payload)
	if err != nil {
		return errors.Wrap(err, "unable to marshal payload for kafka")
	}
	c.pushPublic(topic, value)

	return ctx.Accepted()
}

// Pageview runs the pageview action.
func (c *TrackController) Pageview(ctx *app.PageviewTrackContext) error {
	_, ok, err := c.PropertyStorage.Get(ctx.Payload.System.PropertyToken.String())
	if err != nil {
		return err
	}
	if !ok {
		return ctx.NotFound()
	}

	tags := map[string]string{
		"category": model.CategoryPageview,
		"action":   model.ActionPageviewLoad,
	}
	values := map[string]interface{}{}

	if ctx.Payload.Article != nil {
		tags["_article"] = "1"
		at, av := articleValues(ctx.Payload.Article)
		for key, tag := range at {
			tags[key] = tag
		}
		for key, val := range av {
			values[key] = val
		}
	} else {
		tags["_article"] = "0"
	}

	if err := c.pushInternal(ctx.Payload.System, ctx.Payload.User, model.TablePageviews, tags, values); err != nil {
		return err
	}
	return ctx.Accepted()
}

func articleValues(article *app.Article) (map[string]string, map[string]interface{}) {
	tags := map[string]string{
		"article_id": article.ID,
	}
	values := map[string]interface{}{}
	if article.AuthorID != nil {
		tags["author_id"] = *article.AuthorID
	}
	if article.Category != nil {
		tags["category"] = *article.Category
	}
	if article.Tags != nil {
		values["tags"] = strings.Join(article.Tags, ",")
	}
	return tags, values
}

// pushInternal pushes new event to the InfluxDB.
func (c *TrackController) pushInternal(system *app.System, user *app.User,
	name string, tags map[string]string, fields map[string]interface{}) error {
	fields["token"] = system.PropertyToken

	if user != nil {
		if user.IPAddress != nil {
			fields["ip"] = *user.IPAddress
		}
		if user.URL != nil {
			fields["url"] = *user.URL
		}
		if user.UserAgent != nil {
			fields["user_agent"] = *user.UserAgent
		}
		if user.ID != nil {
			tags["user_id"] = *user.ID
		}

		if user.Source != nil {
			if user.Source.Social != nil {
				tags["social"] = *user.Source.Social
			}
			if user.Source.UtmSource != nil {
				tags["utm_source"] = *user.Source.UtmSource
			}
			if user.Source.UtmMedium != nil {
				tags["utm_medium"] = *user.Source.UtmMedium
			}
			if user.Source.UtmCampaign != nil {
				tags["utm_campaign"] = *user.Source.UtmCampaign
			}
			if user.Source.UtmContent != nil {
				tags["utm_content"] = *user.Source.UtmContent
			}
		}
	}

	p, err := influxClient.NewPoint(name, tags, fields, system.Time)
	if err != nil {
		return err
	}
	c.EventProducer.Input() <- &sarama.ProducerMessage{
		Topic: "beam_events",
		Value: sarama.StringEncoder(p.String()),
	}
	return nil
}

func (c *TrackController) pushPublic(topic string, value []byte) {
	c.EventProducer.Input() <- &sarama.ProducerMessage{
		Topic: topic,
		Value: sarama.ByteEncoder(value),
	}
}
<|endoftext|>


<commit_before>package god

import (
	"log"
	"os/exec"
	"syscall"
	"time"
)

var MIMIMUM_AGE = 2.0

type God struct {
	cmd      *exec.Cmd
	name     string
	args     []string
	started  time.Time
	stopping bool
	exited   bool
}

func NewGod(name string, args []string) *God {
	d := new(God)
	d.name = name
	d.args = args
	return d
}

func (d *God) Start() {
	cmd := exec.Command(d.name, d.args...)
	err := cmd.Start()
	if err != nil {
		panic(err)
	}
	d.cmd = cmd
	d.started = time.Now()
	d.exited = false
	d.Watch()
}

func (d *God) Watch() {
	if d.cmd == nil {
		panic("You must call Start first")
	}
	log.Printf("Waiting for command to finish...")
	err := d.cmd.Wait()
	if err == nil {
		log.Println("Terminate without error")
		d.exited = true
		return
	}

	if d.stopping {
		log.Printf("Stopping. Process %s exited with %v", d.name, err)
		d.exited = true
		return
	}

	d.exited = true
	log.Printf("Command finished with error: %v", err)
	if time.Now().Sub(d.started).Seconds() < MIMIMUM_AGE {
		log.Printf("Program '%s' restart too fast. No restart!", d.name)
		return
	}
	d.Restart()
}

func (d *God) Restart() {
	if d.cmd == nil {
		panic("You must call Start first")
	}
	log.Printf("Restart program %s", d.name)
	d.Stop()
	d.Start()
}

func (d *God) Stop() {
	if d.cmd == nil {
		panic("You must call Start first")
	}
	if d.Exited() {
		return
	}
	d.stopping = true
	d.cmd.Process.Signal(syscall.SIGTERM)
	d.waitExited()
	// if not exited with SIGTERM we force with SIGKILL
	if !d.Exited() {
		d.cmd.Process.Signal(syscall.SIGKILL)
		d.waitExited()
	}
	d.stopping = false
}

func (d *God) Exited() bool {
	return d.cmd.ProcessState != nil && d.exited
}

func (d *God) waitExited() {
	for i := 0; i < 400; i++ {
		if d.Exited() {
			return
		}
		time.Sleep(30 * time.Millisecond)
	}
}
<commit_msg>Remove redundant panic<commit_after>package god

import (
	"log"
	"os/exec"
	"syscall"
	"time"
)

var MIMIMUM_AGE = 2.0

type God struct {
	cmd      *exec.Cmd
	name     string
	args     []string
	started  time.Time
	stopping bool
	exited   bool
}

func NewGod(name string, args []string) *God {
	d := new(God)
	d.name = name
	d.args = args
	return d
}

func (d *God) Start() {
	cmd := exec.Command(d.name, d.args...)
	err := cmd.Start()
	if err != nil {
		panic(err)
	}
	d.cmd = cmd
	d.started = time.Now()
	d.exited = false
	d.Watch()
}

func (d *God) Watch() {
	if d.cmd == nil {
		panic("You must call Start first")
	}
	log.Printf("Waiting for command to finish...")
	err := d.cmd.Wait()
	if err == nil {
		log.Println("Terminate without error")
		d.exited = true
		return
	}

	if d.stopping {
		log.Printf("Stopping. Process %s exited with %v", d.name, err)
		d.exited = true
		return
	}

	d.exited = true
	log.Printf("Command finished with error: %v", err)
	if time.Now().Sub(d.started).Seconds() < MIMIMUM_AGE {
		log.Printf("Program '%s' restart too fast. No restart!", d.name)
		return
	}
	d.Restart()
}

func (d *God) Restart() {
	log.Printf("Restart program %s", d.name)
	d.Stop()
	d.Start()
}

func (d *God) Stop() {
	if d.cmd == nil {
		panic("You must call Start first")
	}
	if d.Exited() {
		return
	}
	d.stopping = true
	d.cmd.Process.Signal(syscall.SIGTERM)
	d.waitExited()
	// if not exited with SIGTERM we force with SIGKILL
	if !d.Exited() {
		d.cmd.Process.Signal(syscall.SIGKILL)
		d.waitExited()
	}
	d.stopping = false
}

func (d *God) Exited() bool {
	return d.cmd.ProcessState != nil && d.exited
}

func (d *God) waitExited() {
	for i := 0; i < 400; i++ {
		if d.Exited() {
			return
		}
		time.Sleep(30 * time.Millisecond)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce

import (
	"context"
	"fmt"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"strings"
	"time"

	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
	"k8s.io/autoscaler/cluster-autoscaler/utils/glogx"

	gce "google.golang.org/api/compute/v1"
	"k8s.io/klog"
)

const (
	defaultOperationWaitTimeout  = 20 * time.Second
	defaultOperationPollInterval = 100 * time.Millisecond

	// ErrorCodeQuotaExceeded is error code used in InstanceErrorInfo if quota exceeded error occurs.
	ErrorCodeQuotaExceeded = "QUOTA_EXCEEDED"

	// ErrorCodeResourcePoolExhausted is error code used in InstanceErrorInfo if requested resources
	// cannot be provisioned by cloud provider.
	ErrorCodeResourcePoolExhausted = "RESOURCE_POOL_EXHAUSTED"

	// ErrorCodeOther is error code used in InstanceErrorInfo if other error occurs.
	ErrorCodeOther = "OTHER"
)

// AutoscalingGceClient is used for communicating with GCE API.
type AutoscalingGceClient interface {
	// reading resources
	FetchMachineType(zone, machineType string) (*gce.MachineType, error)
	FetchMachineTypes(zone string) ([]*gce.MachineType, error)
	FetchAllMigs(zone string) ([]*gce.InstanceGroupManager, error)
	FetchMigTargetSize(GceRef) (int64, error)
	FetchMigBasename(GceRef) (string, error)
	FetchMigInstances(GceRef) ([]cloudprovider.Instance, error)
	FetchMigTemplate(GceRef) (*gce.InstanceTemplate, error)
	FetchMigsWithName(zone string, filter *regexp.Regexp) ([]string, error)
	FetchZones(region string) ([]string, error)
	FetchAvailableCpuPlatforms() (map[string][]string, error)

	// modifying resources
	ResizeMig(GceRef, int64) error
	DeleteInstances(migRef GceRef, instances []GceRef) error
}

type autoscalingGceClientV1 struct {
	gceService *gce.Service

	projectId string

	// These can be overridden, e.g. for testing.
	operationWaitTimeout  time.Duration
	operationPollInterval time.Duration
}

// NewAutoscalingGceClientV1 creates a new client for communicating with GCE v1 API.
func NewAutoscalingGceClientV1(client *http.Client, projectId string) (*autoscalingGceClientV1, error) {
	gceService, err := gce.New(client)
	if err != nil {
		return nil, err
	}

	return &autoscalingGceClientV1{
		projectId:             projectId,
		gceService:            gceService,
		operationWaitTimeout:  defaultOperationWaitTimeout,
		operationPollInterval: defaultOperationPollInterval,
	}, nil
}

// NewCustomAutoscalingGceClientV1 creates a new client using custom server url and timeouts
// for communicating with GCE v1 API.
func NewCustomAutoscalingGceClientV1(client *http.Client, projectId, serverUrl string,
	waitTimeout, pollInterval time.Duration) (*autoscalingGceClientV1, error) {
	gceService, err := gce.New(client)
	if err != nil {
		return nil, err
	}
	gceService.BasePath = serverUrl

	return &autoscalingGceClientV1{
		projectId:             projectId,
		gceService:            gceService,
		operationWaitTimeout:  waitTimeout,
		operationPollInterval: pollInterval,
	}, nil
}

func (client *autoscalingGceClientV1) FetchMachineType(zone, machineType string) (*gce.MachineType, error) {
	registerRequest("machine_types", "get")
	return client.gceService.MachineTypes.Get(client.projectId, zone, machineType).Do()
}

func (client *autoscalingGceClientV1) FetchMachineTypes(zone string) ([]*gce.MachineType, error) {
	registerRequest("machine_types", "list")
	machines, err := client.gceService.MachineTypes.List(client.projectId, zone).Do()
	if err != nil {
		return nil, err
	}
	return machines.Items, nil
}

func (client *autoscalingGceClientV1) FetchAllMigs(zone string) ([]*gce.InstanceGroupManager, error) {
	registerRequest("instance_group_managers", "list")

	var migs []*gce.InstanceGroupManager
	err := client.gceService.InstanceGroupManagers.List(client.projectId, zone).Pages(
		context.TODO(),
		func(page *gce.InstanceGroupManagerList) error {
			migs = append(migs, page.Items...)
			return nil
		})
	if err != nil {
		return nil, err
	}
	return migs, nil
}

func (client *autoscalingGceClientV1) FetchMigTargetSize(migRef GceRef) (int64, error) {
	registerRequest("instance_group_managers", "get")
	igm, err := client.gceService.InstanceGroupManagers.Get(migRef.Project, migRef.Zone, migRef.Name).Do()
	if err != nil {
		return 0, err
	}
	return igm.TargetSize, nil
}

func (client *autoscalingGceClientV1) FetchMigBasename(migRef GceRef) (string, error) {
	registerRequest("instance_group_managers", "get")
	igm, err := client.gceService.InstanceGroupManagers.Get(migRef.Project, migRef.Zone, migRef.Name).Do()
	if err != nil {
		return "", err
	}
	return igm.BaseInstanceName, nil
}

func (client *autoscalingGceClientV1) ResizeMig(migRef GceRef, size int64) error {
	registerRequest("instance_group_managers", "resize")
	op, err := client.gceService.InstanceGroupManagers.Resize(migRef.Project, migRef.Zone, migRef.Name, size).Do()
	if err != nil {
		return err
	}
	return client.waitForOp(op, migRef.Project, migRef.Zone)
}

func (client *autoscalingGceClientV1) waitForOp(operation *gce.Operation, project, zone string) error {
	for start := time.Now(); time.Since(start) < client.operationWaitTimeout; time.Sleep(client.operationPollInterval) {
		klog.V(4).Infof("Waiting for operation %s %s %s", project, zone, operation.Name)
		registerRequest("zone_operations", "get")
		if op, err := client.gceService.ZoneOperations.Get(project, zone, operation.Name).Do(); err == nil {
			klog.V(4).Infof("Operation %s %s %s status: %s", project, zone, operation.Name, op.Status)
			if op.Status == "DONE" {
				return nil
			}
		} else {
			klog.Warningf("Error while getting operation %s on %s: %v", operation.Name, operation.TargetLink, err)
		}
	}
	return fmt.Errorf("timeout while waiting for operation %s on %s to complete.", operation.Name, operation.TargetLink)
}

func (client *autoscalingGceClientV1) DeleteInstances(migRef GceRef, instances []GceRef) error {
	registerRequest("instance_group_managers", "delete_instances")
	req := gce.InstanceGroupManagersDeleteInstancesRequest{
		Instances: []string{},
	}
	for _, i := range instances {
		req.Instances = append(req.Instances, GenerateInstanceUrl(i))
	}
	op, err := client.gceService.InstanceGroupManagers.DeleteInstances(migRef.Project, migRef.Zone, migRef.Name, &req).Do()
	if err != nil {
		return err
	}
	return client.waitForOp(op, migRef.Project, migRef.Zone)
}

func (client *autoscalingGceClientV1) FetchMigInstances(migRef GceRef) ([]cloudprovider.Instance, error) {
	registerRequest("instance_group_managers", "list_managed_instances")
	gceInstances, err := client.gceService.InstanceGroupManagers.ListManagedInstances(migRef.Project, migRef.Zone, migRef.Name).Do()
	if err != nil {
		klog.V(4).Infof("Failed MIG info request for %s %s %s: %v", migRef.Project, migRef.Zone, migRef.Name, err)
		return nil, err
	}
	infos := []cloudprovider.Instance{}
	errorCodeCounts := make(map[string]int)
	errorLoggingQuota := glogx.NewLoggingQuota(100)
	for _, gceInstance := range gceInstances.ManagedInstances {
		ref, err := ParseInstanceUrlRef(gceInstance.Instance)
		if err != nil {
			return nil, err
		}

		instance := cloudprovider.Instance{
			Id:     ref.ToProviderId(),
			Status: &cloudprovider.InstanceStatus{},
		}

		switch gceInstance.CurrentAction {
		case "CREATING", "RECREATING", "CREATING_WITHOUT_RETRIES":
			instance.Status.State = cloudprovider.InstanceCreating
		case "ABANDONING", "DELETING":
			instance.Status.State = cloudprovider.InstanceDeleting
		default:
			instance.Status.State = cloudprovider.InstanceRunning
		}

		if instance.Status.State == cloudprovider.InstanceCreating {
			var errorInfo cloudprovider.InstanceErrorInfo
			errorMessages := []string{}
			errorFound := false
			lastAttemptErrors := getLastAttemptErrors(gceInstance)
			for _, instanceError := range lastAttemptErrors {
				errorCodeCounts[instanceError.Code]++
				if isResourcePoolExhaustedErrorCode(instanceError.Code) {
					errorInfo.ErrorClass = cloudprovider.OutOfResourcesErrorClass
					errorInfo.ErrorCode = ErrorCodeResourcePoolExhausted
				} else if isQuotaExceededErrorCoce(instanceError.Code) {
					errorInfo.ErrorClass = cloudprovider.OutOfResourcesErrorClass
					errorInfo.ErrorCode = ErrorCodeQuotaExceeded
				} else if isInstanceNotRunningYet(gceInstance) {
					if !errorFound {
						// do not override error code with OTHER
						errorInfo.ErrorClass = cloudprovider.OtherErrorClass
						errorInfo.ErrorCode = ErrorCodeOther
					}
				} else {
					// no error
					continue
				}
				errorFound = true
				if instanceError.Message != "" {
					errorMessages = append(errorMessages, instanceError.Message)
				}
			}
			errorInfo.ErrorMessage = strings.Join(errorMessages, "; ")
			if errorFound {
				instance.Status.ErrorInfo = &errorInfo
			}

			if len(lastAttemptErrors) > 0 {
				gceInstanceJSONBytes, err := gceInstance.MarshalJSON()
				var gceInstanceJSON string
				if err != nil {
					gceInstanceJSON = fmt.Sprintf("Got error from MarshalJSON; %v", err)
				} else {
					gceInstanceJSON = string(gceInstanceJSONBytes)
				}
				glogx.V(4).UpTo(errorLoggingQuota).Infof("Got GCE instance which is being created and has lastAttemptErrors; gceInstance=%v; errorInfo=%#v", gceInstanceJSON, errorInfo)
			}
		}
		infos = append(infos, instance)
	}
	glogx.V(4).Over(errorLoggingQuota).Infof("Got %v other GCE instances being created with lastAttemptErrors", -errorLoggingQuota.Left())
	if len(errorCodeCounts) > 0 {
		klog.V(4).Infof("Spotted following instance creation error codes: %#v", errorCodeCounts)
	}
	return infos, nil
}

func getLastAttemptErrors(instance *gce.ManagedInstance) []*gce.ManagedInstanceLastAttemptErrorsErrors {
	if instance.LastAttempt != nil && instance.LastAttempt.Errors != nil {
		return instance.LastAttempt.Errors.Errors
	}
	return nil
}

func isResourcePoolExhaustedErrorCode(errorCode string) bool {
	return errorCode == "RESOURCE_POOL_EXHAUSTED" || errorCode == "ZONE_RESOURCE_POOL_EXHAUSTED" || errorCode == "ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS"
}

func isQuotaExceededErrorCoce(errorCode string) bool {
	return strings.Contains(errorCode, "QUOTA")
}

func isInstanceNotRunningYet(gceInstance *gce.ManagedInstance) bool {
	return gceInstance.InstanceStatus == "" || gceInstance.InstanceStatus == "PROVISIONING" || gceInstance.InstanceStatus == "STAGING"
}

func (client *autoscalingGceClientV1) FetchZones(region string) ([]string, error) {
	registerRequest("regions", "get")
	r, err := client.gceService.Regions.Get(client.projectId, region).Do()
	if err != nil {
		return nil, fmt.Errorf("cannot get zones for GCE region %s: %v", region, err)
	}
	zones := make([]string, len(r.Zones))
	for i, link := range r.Zones {
		zones[i] = path.Base(link)
	}
	return zones, nil
}

func (client *autoscalingGceClientV1) FetchAvailableCpuPlatforms() (map[string][]string, error) {
	availableCpuPlatforms := make(map[string][]string)
	err := client.gceService.Zones.List(client.projectId).Pages(
		context.TODO(),
		func(zones *gce.ZoneList) error {
			for _, zone := range zones.Items {
				availableCpuPlatforms[zone.Name] = zone.AvailableCpuPlatforms
			}
			return nil
		})
	if err != nil {
		return nil, err
	}
	return availableCpuPlatforms, nil
}

func (client *autoscalingGceClientV1) FetchMigTemplate(migRef GceRef) (*gce.InstanceTemplate, error) {
	registerRequest("instance_group_managers", "get")
	igm, err := client.gceService.InstanceGroupManagers.Get(migRef.Project, migRef.Zone, migRef.Name).Do()
	if err != nil {
		return nil, err
	}
	templateUrl, err := url.Parse(igm.InstanceTemplate)
	if err != nil {
		return nil, err
	}
	_, templateName := path.Split(templateUrl.EscapedPath())
	registerRequest("instance_templates", "get")
	return client.gceService.InstanceTemplates.Get(migRef.Project, templateName).Do()
}

func (client *autoscalingGceClientV1) FetchMigsWithName(zone string, name *regexp.Regexp) ([]string, error) {
	filter := fmt.Sprintf("name eq %s", name)
	links := make([]string, 0)
	registerRequest("instance_groups", "list")
	req := client.gceService.InstanceGroups.List(client.projectId, zone).Filter(filter)
	if err := req.Pages(context.TODO(), func(page *gce.InstanceGroupList) error {
		for _, ig := range page.Items {
			links = append(links, ig.SelfLink)
			klog.V(3).Infof("found managed instance group %s matching regexp %s", ig.Name, name)
		}
		return nil
	}); err != nil {
		return nil, fmt.Errorf("cannot list managed instance groups: %v", err)
	}
	return links, nil
}
<commit_msg>Use pagination to list MachineTypes<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce

import (
	"context"
	"fmt"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"strings"
	"time"

	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
	"k8s.io/autoscaler/cluster-autoscaler/utils/glogx"

	gce "google.golang.org/api/compute/v1"
	"k8s.io/klog"
)

const (
	defaultOperationWaitTimeout  = 20 * time.Second
	defaultOperationPollInterval = 100 * time.Millisecond

	// ErrorCodeQuotaExceeded is error code used in InstanceErrorInfo if quota exceeded error occurs.
	ErrorCodeQuotaExceeded = "QUOTA_EXCEEDED"

	// ErrorCodeResourcePoolExhausted is error code used in InstanceErrorInfo if requested resources
	// cannot be provisioned by cloud provider.
	ErrorCodeResourcePoolExhausted = "RESOURCE_POOL_EXHAUSTED"

	// ErrorCodeOther is error code used in InstanceErrorInfo if other error occurs.
	ErrorCodeOther = "OTHER"
)

// AutoscalingGceClient is used for communicating with GCE API.
type AutoscalingGceClient interface {
	// reading resources
	FetchMachineType(zone, machineType string) (*gce.MachineType, error)
	FetchMachineTypes(zone string) ([]*gce.MachineType, error)
	FetchAllMigs(zone string) ([]*gce.InstanceGroupManager, error)
	FetchMigTargetSize(GceRef) (int64, error)
	FetchMigBasename(GceRef) (string, error)
	FetchMigInstances(GceRef) ([]cloudprovider.Instance, error)
	FetchMigTemplate(GceRef) (*gce.InstanceTemplate, error)
	FetchMigsWithName(zone string, filter *regexp.Regexp) ([]string, error)
	FetchZones(region string) ([]string, error)
	FetchAvailableCpuPlatforms() (map[string][]string, error)

	// modifying resources
	ResizeMig(GceRef, int64) error
	DeleteInstances(migRef GceRef, instances []GceRef) error
}

type autoscalingGceClientV1 struct {
	gceService *gce.Service

	projectId string

	// These can be overridden, e.g. for testing.
	operationWaitTimeout  time.Duration
	operationPollInterval time.Duration
}

// NewAutoscalingGceClientV1 creates a new client for communicating with GCE v1 API.
func NewAutoscalingGceClientV1(client *http.Client, projectId string) (*autoscalingGceClientV1, error) {
	gceService, err := gce.New(client)
	if err != nil {
		return nil, err
	}

	return &autoscalingGceClientV1{
		projectId:             projectId,
		gceService:            gceService,
		operationWaitTimeout:  defaultOperationWaitTimeout,
		operationPollInterval: defaultOperationPollInterval,
	}, nil
}

// NewCustomAutoscalingGceClientV1 creates a new client using custom server url and timeouts
// for communicating with GCE v1 API.
func NewCustomAutoscalingGceClientV1(client *http.Client, projectId, serverUrl string,
	waitTimeout, pollInterval time.Duration) (*autoscalingGceClientV1, error) {
	gceService, err := gce.New(client)
	if err != nil {
		return nil, err
	}
	gceService.BasePath = serverUrl

	return &autoscalingGceClientV1{
		projectId:             projectId,
		gceService:            gceService,
		operationWaitTimeout:  waitTimeout,
		operationPollInterval: pollInterval,
	}, nil
}

func (client *autoscalingGceClientV1) FetchMachineType(zone, machineType string) (*gce.MachineType, error) {
	registerRequest("machine_types", "get")
	return client.gceService.MachineTypes.Get(client.projectId, zone, machineType).Do()
}

func (client *autoscalingGceClientV1) FetchMachineTypes(zone string) ([]*gce.MachineType, error) {
	registerRequest("machine_types", "list")
	var machineTypes []*gce.MachineType
	err := client.gceService.MachineTypes.List(client.projectId, zone).Pages(
		context.TODO(),
		func(page *gce.MachineTypeList) error {
			machineTypes = append(machineTypes, page.Items...)
			return nil
		})
	if err != nil {
		return nil, err
	}
	return machineTypes, nil
}

func (client *autoscalingGceClientV1) FetchAllMigs(zone string) ([]*gce.InstanceGroupManager, error) {
	registerRequest("instance_group_managers", "list")
	var migs []*gce.InstanceGroupManager
	err := client.gceService.InstanceGroupManagers.List(client.projectId, zone).Pages(
		context.TODO(),
		func(page *gce.InstanceGroupManagerList) error {
			migs = append(migs, page.Items...)
			return nil
		})
	if err != nil {
		return nil, err
	}
	return migs, nil
}

func (client *autoscalingGceClientV1) FetchMigTargetSize(migRef GceRef) (int64, error) {
	registerRequest("instance_group_managers", "get")
	igm, err := client.gceService.InstanceGroupManagers.Get(migRef.Project, migRef.Zone, migRef.Name).Do()
	if err != nil {
		return 0, err
	}
	return igm.TargetSize, nil
}

func (client *autoscalingGceClientV1) FetchMigBasename(migRef GceRef) (string, error) {
	registerRequest("instance_group_managers", "get")
	igm, err := client.gceService.InstanceGroupManagers.Get(migRef.Project, migRef.Zone, migRef.Name).Do()
	if err != nil {
		return "", err
	}
	return igm.BaseInstanceName, nil
}

func (client *autoscalingGceClientV1) ResizeMig(migRef GceRef, size int64) error {
	registerRequest("instance_group_managers", "resize")
	op, err := client.gceService.InstanceGroupManagers.Resize(migRef.Project, migRef.Zone, migRef.Name, size).Do()
	if err != nil {
		return err
	}
	return client.waitForOp(op, migRef.Project, migRef.Zone)
}

func (client *autoscalingGceClientV1) waitForOp(operation *gce.Operation, project, zone string) error {
	for start := time.Now(); time.Since(start) < client.operationWaitTimeout; time.Sleep(client.operationPollInterval) {
		klog.V(4).Infof("Waiting for operation %s %s %s", project, zone, operation.Name)
		registerRequest("zone_operations", "get")
		if op, err := client.gceService.ZoneOperations.Get(project, zone, operation.Name).Do(); err == nil {
			klog.V(4).Infof("Operation %s %s %s status: %s", project, zone, operation.Name, op.Status)
			if op.Status == "DONE" {
				return nil
			}
		} else {
			klog.Warningf("Error while getting operation %s on %s: %v", operation.Name, operation.TargetLink, err)
		}
	}
	return fmt.Errorf("timeout while waiting for operation %s on %s to complete.", operation.Name, operation.TargetLink)
}

func (client *autoscalingGceClientV1) DeleteInstances(migRef GceRef, instances []GceRef) error {
	registerRequest("instance_group_managers", "delete_instances")
	req := gce.InstanceGroupManagersDeleteInstancesRequest{
		Instances: []string{},
	}
	for _, i := range instances {
		req.Instances = append(req.Instances, GenerateInstanceUrl(i))
	}
	op, err := client.gceService.InstanceGroupManagers.DeleteInstances(migRef.Project, migRef.Zone, migRef.Name, &req).Do()
	if err != nil {
		return err
	}
	return client.waitForOp(op, migRef.Project, migRef.Zone)
}

func (client *autoscalingGceClientV1) FetchMigInstances(migRef GceRef) ([]cloudprovider.Instance, error) {
	registerRequest("instance_group_managers", "list_managed_instances")
	gceInstances, err := client.gceService.InstanceGroupManagers.ListManagedInstances(migRef.Project, migRef.Zone, migRef.Name).Do()
	if err != nil {
		klog.V(4).Infof("Failed MIG info request for %s %s %s: %v", migRef.Project, migRef.Zone, migRef.Name, err)
		return nil, err
	}
	infos := []cloudprovider.Instance{}
	errorCodeCounts := make(map[string]int)
	errorLoggingQuota := glogx.NewLoggingQuota(100)
	for _, gceInstance := range gceInstances.ManagedInstances {
		ref, err := ParseInstanceUrlRef(gceInstance.Instance)
		if err != nil {
			return nil, err
		}

		instance := cloudprovider.Instance{
			Id:     ref.ToProviderId(),
			Status: &cloudprovider.InstanceStatus{},
		}

		switch gceInstance.CurrentAction {
		case "CREATING", "RECREATING", "CREATING_WITHOUT_RETRIES":
			instance.Status.State = cloudprovider.InstanceCreating
		case "ABANDONING", "DELETING":
			instance.Status.State = cloudprovider.InstanceDeleting
		default:
			instance.Status.State = cloudprovider.InstanceRunning
		}

		if instance.Status.State == cloudprovider.InstanceCreating {
			var errorInfo cloudprovider.InstanceErrorInfo
			errorMessages := []string{}
			errorFound := false
			lastAttemptErrors := getLastAttemptErrors(gceInstance)
			for _, instanceError := range lastAttemptErrors {
				errorCodeCounts[instanceError.Code]++
				if isResourcePoolExhaustedErrorCode(instanceError.Code) {
					errorInfo.ErrorClass = cloudprovider.OutOfResourcesErrorClass
					errorInfo.ErrorCode = ErrorCodeResourcePoolExhausted
				} else if isQuotaExceededErrorCoce(instanceError.Code) {
					errorInfo.ErrorClass = cloudprovider.OutOfResourcesErrorClass
					errorInfo.ErrorCode = ErrorCodeQuotaExceeded
				} else if isInstanceNotRunningYet(gceInstance) {
					if !errorFound {
						// do not override error code with OTHER
						errorInfo.ErrorClass = cloudprovider.OtherErrorClass
						errorInfo.ErrorCode = ErrorCodeOther
					}
				} else {
					// no error
					continue
				}
				errorFound = true
				if instanceError.Message != "" {
					errorMessages = append(errorMessages, instanceError.Message)
				}
			}
			errorInfo.ErrorMessage = strings.Join(errorMessages, "; ")
			if errorFound {
				instance.Status.ErrorInfo = &errorInfo
			}

			if len(lastAttemptErrors) > 0 {
				gceInstanceJSONBytes, err := gceInstance.MarshalJSON()
				var gceInstanceJSON string
				if err != nil {
					gceInstanceJSON = fmt.Sprintf("Got error from MarshalJSON; %v", err)
				} else {
					gceInstanceJSON = string(gceInstanceJSONBytes)
				}
				glogx.V(4).UpTo(errorLoggingQuota).Infof("Got GCE instance which is being created and has lastAttemptErrors; gceInstance=%v; errorInfo=%#v", gceInstanceJSON, errorInfo)
			}
		}
		infos = append(infos, instance)
	}
	glogx.V(4).Over(errorLoggingQuota).Infof("Got %v other GCE instances being created with lastAttemptErrors", -errorLoggingQuota.Left())
	if len(errorCodeCounts) > 0 {
		klog.V(4).Infof("Spotted following instance creation error codes: %#v", errorCodeCounts)
	}
	return infos, nil
}

func getLastAttemptErrors(instance *gce.ManagedInstance) []*gce.ManagedInstanceLastAttemptErrorsErrors {
	if instance.LastAttempt != nil && instance.LastAttempt.Errors != nil {
		return instance.LastAttempt.Errors.Errors
	}
	return nil
}

func isResourcePoolExhaustedErrorCode(errorCode string) bool {
	return errorCode == "RESOURCE_POOL_EXHAUSTED" || errorCode == "ZONE_RESOURCE_POOL_EXHAUSTED" || errorCode == "ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS"
}

func isQuotaExceededErrorCoce(errorCode string) bool {
	return strings.Contains(errorCode, "QUOTA")
}

func isInstanceNotRunningYet(gceInstance *gce.ManagedInstance) bool {
	return gceInstance.InstanceStatus == "" || gceInstance.InstanceStatus == "PROVISIONING" || gceInstance.InstanceStatus == "STAGING"
}

func (client *autoscalingGceClientV1) FetchZones(region string) ([]string, error) {
	registerRequest("regions", "get")
	r, err := client.gceService.Regions.Get(client.projectId, region).Do()
	if err != nil {
		return nil, fmt.Errorf("cannot get zones for GCE region %s: %v", region, err)
	}
	zones := make([]string, len(r.Zones))
	for i, link := range r.Zones {
		zones[i] = path.Base(link)
	}
	return zones, nil
}

func (client *autoscalingGceClientV1) FetchAvailableCpuPlatforms() (map[string][]string, error) {
	availableCpuPlatforms := make(map[string][]string)
	err := client.gceService.Zones.List(client.projectId).Pages(
		context.TODO(),
		func(zones *gce.ZoneList) error {
			for _, zone := range zones.Items {
				availableCpuPlatforms[zone.Name] = zone.AvailableCpuPlatforms
			}
			return nil
		})
	if err != nil {
		return nil, err
	}
	return availableCpuPlatforms, nil
}

func (client *autoscalingGceClientV1) FetchMigTemplate(migRef GceRef) (*gce.InstanceTemplate, error) {
	registerRequest("instance_group_managers", "get")
	igm, err := client.gceService.InstanceGroupManagers.Get(migRef.Project, migRef.Zone, migRef.Name).Do()
	if err != nil {
		return nil, err
	}
	templateUrl, err := url.Parse(igm.InstanceTemplate)
	if err != nil {
		return nil, err
	}
	_, templateName := path.Split(templateUrl.EscapedPath())
	registerRequest("instance_templates", "get")
	return client.gceService.InstanceTemplates.Get(migRef.Project, templateName).Do()
}

func (client *autoscalingGceClientV1) FetchMigsWithName(zone string, name *regexp.Regexp) ([]string, error) {
	filter := fmt.Sprintf("name eq %s", name)
	links := make([]string, 0)
	registerRequest("instance_groups", "list")
	req := client.gceService.InstanceGroups.List(client.projectId, zone).Filter(filter)
	if err := req.Pages(context.TODO(), func(page *gce.InstanceGroupList) error {
		for _, ig := range page.Items {
			links = append(links, ig.SelfLink)
			klog.V(3).Infof("found managed instance group %s matching regexp %s", ig.Name, name)
		}
		return nil
	}); err != nil {
		return nil, fmt.Errorf("cannot list managed instance groups: %v", err)
	}
	return links, nil
}
<|endoftext|>


<commit_before>package ar

import (
	"bufio"
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"syscall"
	"time"
)

const (
	magic     = "!<arch>\n"
	filemagic = "\x60\x0A"
)

type file struct {
	name [16]uint8 // Filename in  ASCII

}

type fileInfo struct {
	name  string
	mode  os.FileMode
	size  int64
	mtime time.Time
}

// IsDir returns always false for ar archive members, because we don't support directories.
func (f *fileInfo) IsDir() bool { return false }

func (f *fileInfo) ModTime() time.Time { return f.mtime }
func (f *fileInfo) Mode() os.FileMode  { return f.mode }
func (f *fileInfo) Name() string       { return f.name }
func (f *fileInfo) Size() int64        { return f.size }
func (f *fileInfo) Sys() interface{}   { return nil }

// Reader can read ar archives
type Reader struct {
	buffer  *bufio.Reader
	valid   bool
	err     error
	section io.LimitedReader
}

// Reset cancels all internal state/buffering and starts to read from in.
// Useful to avoid allocations, but otherwise has the same effect as r := NewReader(in)
func (r *Reader) Reset(in io.Reader) {
	r.buffer.Reset(in)
	r.valid = false
	r.err = nil
	r.section.R, r.section.N = nil, 0
}

// NewReader will start parsing a possible archive from r
func NewReader(r io.Reader) *Reader {
	reader := &Reader{}
	reader.buffer = bufio.NewReader(r)
	return reader
}

// sticks an error to the reader. From now on this error is returned
// for each following operation until Reset is called.
func (r *Reader) stick(err error) error {
	r.err = err
	return err
}

// Next will advance to the next available file in the archive and return it's meta data.
// After calling r.Next, you can use r.Read() to actually read the file contained.
func (r *Reader) Next() (os.FileInfo, error) {
	if r.err != nil {
		return nil, r.err
	}
	if !r.valid {
		err := checkMagic(r.buffer)
		if err != nil {
			return nil, r.stick(err)
		}

		r.valid = true
	}

	if r.section.R != nil {
		if r.section.N > 0 {
			_, err := io.Copy(ioutil.Discard, &r.section)
			return nil, r.stick(err)
		}
		// skip padding byte.
		if c, err := r.buffer.ReadByte(); err != nil {
			return nil, r.stick(err)
		} else if c != '\n' {
			// If it wasn't padding, put it back
			r.buffer.UnreadByte()
		}
		r.section.R, r.section.N = nil, 0
	}

	fi, err := readFileHeader(r.buffer)
	if err != nil {
		return nil, r.stick(err)
	}
	r.section.R, r.section.N = r.buffer, fi.Size()
	return fi, nil
}

func (r *Reader) Read(b []byte) (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	if r.section.R != nil {
		return r.section.Read(b)
	}

	return 0, os.ErrNotExist
}

// NotImplementedError will be returned for any features not implemented in this package.
// It means the archive may be valid, but it uses features detected and not (yet) supported by this archive
type NotImplementedError string

func (feature NotImplementedError) Error() string {
	return "feature not implemented: " + string(feature)
}

// CorruptArchiveError will be returned, if this archive cannot be parsed.
type CorruptArchiveError string

func (c CorruptArchiveError) Error() string {
	return "corrupt archive: " + string(c)
}

func parseFileMode(s string) (filemode os.FileMode, err error) {
	mode, err := strconv.ParseUint(s, 8, 32)
	if err != nil {
		return filemode, CorruptArchiveError(err.Error())
	}

	if os.FileMode(mode) != (os.FileMode(mode) & (os.ModePerm | syscall.S_IFMT)) {
		return filemode, CorruptArchiveError("invalid file mode")
	}

	switch mode & syscall.S_IFMT {
	case 0: // no file type sepcified, assume regular file
	case syscall.S_IFREG: // regular file, nothing to add
	default:
		return filemode, NotImplementedError("non-regular files")
	}

	return os.FileMode(mode) & os.ModePerm, nil
}

func readFileHeader(r io.Reader) (*fileInfo, error) {
	fh := make([]byte, 60)
	_, err := io.ReadFull(r, fh)
	if err != nil {
		return nil, err
	}

	if string(fh[58:58+2]) != filemagic {
		return nil, CorruptArchiveError("per file magic not found")
	}

	name := string(bytes.TrimSpace(fh[0:16]))
	secs, err := strconv.ParseInt(string(bytes.TrimSpace(fh[16:16+12])), 10, 64)
	if err != nil {
		return nil, CorruptArchiveError(err.Error())
	}

	filemode, err := parseFileMode(string(bytes.TrimSpace(fh[40 : 40+8])))
	if err != nil {
		return nil, err
	}

	filesize, err := strconv.ParseInt(string(bytes.TrimSpace(fh[48:48+10])), 10, 64)
	if err != nil {
		return nil, CorruptArchiveError(err.Error())
	}

	fi := &fileInfo{
		name:  name,
		mtime: time.Unix(secs, 0),
		mode:  filemode,
		size:  filesize,
	}

	return fi, nil
}

func checkMagic(r io.Reader) error {
	m := make([]byte, len(magic))
	_, err := io.ReadFull(r, m)
	if err != nil {
		return err
	}

	if string(m) != magic {
		return CorruptArchiveError("global archive header not found")
	}

	return nil
}
<commit_msg>Add package documentation<commit_after>// Package ar can read common ar archives. Those are often used in software development tools.
// Even *.deb files are actually a special case of the common ar archive.
// See http://en.wikipedia.org/wiki/Ar_(Unix) for more information on this file format.
package ar

import (
	"bufio"
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"syscall"
	"time"
)

const (
	magic     = "!<arch>\n"
	filemagic = "\x60\x0A"
)

type file struct {
	name [16]uint8 // Filename in  ASCII

}

type fileInfo struct {
	name  string
	mode  os.FileMode
	size  int64
	mtime time.Time
}

// IsDir returns always false for ar archive members, because we don't support directories.
func (f *fileInfo) IsDir() bool { return false }

func (f *fileInfo) ModTime() time.Time { return f.mtime }
func (f *fileInfo) Mode() os.FileMode  { return f.mode }
func (f *fileInfo) Name() string       { return f.name }
func (f *fileInfo) Size() int64        { return f.size }
func (f *fileInfo) Sys() interface{}   { return nil }

// Reader can read ar archives
type Reader struct {
	buffer  *bufio.Reader
	valid   bool
	err     error
	section io.LimitedReader
}

// Reset cancels all internal state/buffering and starts to read from in.
// Useful to avoid allocations, but otherwise has the same effect as r := NewReader(in)
func (r *Reader) Reset(in io.Reader) {
	r.buffer.Reset(in)
	r.valid = false
	r.err = nil
	r.section.R, r.section.N = nil, 0
}

// NewReader will start parsing a possible archive from r
func NewReader(r io.Reader) *Reader {
	reader := &Reader{}
	reader.buffer = bufio.NewReader(r)
	return reader
}

// sticks an error to the reader. From now on this error is returned
// for each following operation until Reset is called.
func (r *Reader) stick(err error) error {
	r.err = err
	return err
}

// Next will advance to the next available file in the archive and return it's meta data.
// After calling r.Next, you can use r.Read() to actually read the file contained.
func (r *Reader) Next() (os.FileInfo, error) {
	if r.err != nil {
		return nil, r.err
	}
	if !r.valid {
		err := checkMagic(r.buffer)
		if err != nil {
			return nil, r.stick(err)
		}

		r.valid = true
	}

	if r.section.R != nil {
		if r.section.N > 0 {
			_, err := io.Copy(ioutil.Discard, &r.section)
			return nil, r.stick(err)
		}
		// skip padding byte.
		if c, err := r.buffer.ReadByte(); err != nil {
			return nil, r.stick(err)
		} else if c != '\n' {
			// If it wasn't padding, put it back
			r.buffer.UnreadByte()
		}
		r.section.R, r.section.N = nil, 0
	}

	fi, err := readFileHeader(r.buffer)
	if err != nil {
		return nil, r.stick(err)
	}
	r.section.R, r.section.N = r.buffer, fi.Size()
	return fi, nil
}

func (r *Reader) Read(b []byte) (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	if r.section.R != nil {
		return r.section.Read(b)
	}

	return 0, os.ErrNotExist
}

// NotImplementedError will be returned for any features not implemented in this package.
// It means the archive may be valid, but it uses features detected and not (yet) supported by this archive
type NotImplementedError string

func (feature NotImplementedError) Error() string {
	return "feature not implemented: " + string(feature)
}

// CorruptArchiveError will be returned, if this archive cannot be parsed.
type CorruptArchiveError string

func (c CorruptArchiveError) Error() string {
	return "corrupt archive: " + string(c)
}

func parseFileMode(s string) (filemode os.FileMode, err error) {
	mode, err := strconv.ParseUint(s, 8, 32)
	if err != nil {
		return filemode, CorruptArchiveError(err.Error())
	}

	if os.FileMode(mode) != (os.FileMode(mode) & (os.ModePerm | syscall.S_IFMT)) {
		return filemode, CorruptArchiveError("invalid file mode")
	}

	switch mode & syscall.S_IFMT {
	case 0: // no file type sepcified, assume regular file
	case syscall.S_IFREG: // regular file, nothing to add
	default:
		return filemode, NotImplementedError("non-regular files")
	}

	return os.FileMode(mode) & os.ModePerm, nil
}

func readFileHeader(r io.Reader) (*fileInfo, error) {
	fh := make([]byte, 60)
	_, err := io.ReadFull(r, fh)
	if err != nil {
		return nil, err
	}

	if string(fh[58:58+2]) != filemagic {
		return nil, CorruptArchiveError("per file magic not found")
	}

	name := string(bytes.TrimSpace(fh[0:16]))
	secs, err := strconv.ParseInt(string(bytes.TrimSpace(fh[16:16+12])), 10, 64)
	if err != nil {
		return nil, CorruptArchiveError(err.Error())
	}

	filemode, err := parseFileMode(string(bytes.TrimSpace(fh[40 : 40+8])))
	if err != nil {
		return nil, err
	}

	filesize, err := strconv.ParseInt(string(bytes.TrimSpace(fh[48:48+10])), 10, 64)
	if err != nil {
		return nil, CorruptArchiveError(err.Error())
	}

	fi := &fileInfo{
		name:  name,
		mtime: time.Unix(secs, 0),
		mode:  filemode,
		size:  filesize,
	}

	return fi, nil
}

func checkMagic(r io.Reader) error {
	m := make([]byte, len(magic))
	_, err := io.ReadFull(r, m)
	if err != nil {
		return err
	}

	if string(m) != magic {
		return CorruptArchiveError("global archive header not found")
	}

	return nil
}
<|endoftext|>


<commit_before>package admin

import (
	"database/sql"
	"fmt"
	"github.com/oal/admin/fields"
	"reflect"
	"strings"
)

// queryModel is used in list view to display all rows.
func (a *Admin) queryModel(mdl *model, search, sortBy string, sortDesc bool, page int) ([][]interface{}, int, error) {
	page--

	// Ugly search. Will fix later.
	doSearch := false
	whereStr := ""
	var searchList []interface{}
	if len(search) > 0 {
		if len(mdl.searchableColumns) > 0 {
			searchCols := make([]string, len(mdl.searchableColumns))
			searchList = make([]interface{}, len(searchCols))
			for i, _ := range searchList {
				searchList[i] = search
			}
			for i, _ := range searchCols {
				searchCols[i] = fmt.Sprintf("%v.%v LIKE ?", mdl.tableName, mdl.searchableColumns[i])
			}
			whereStr = fmt.Sprintf("WHERE (%v)", strings.Join(searchCols, " OR "))
			doSearch = true
		}

	}

	cols := []string{}
	tables := []string{mdl.tableName}
	fkWhere := []string{}
	for _, field := range mdl.fields {
		if field.Attrs().List {
			colName := fmt.Sprintf("%v.%v", mdl.tableName, field.Attrs().ColumnName)
			if relField, ok := field.(fields.RelationalField); ok && len(relField.GetListColumn()) > 0 {
				relTable := relField.GetRelatedTable()
				fkColName := fmt.Sprintf("%v.%v", relTable, relField.GetListColumn())
				fkWhere = append(fkWhere, fmt.Sprintf("%v = %v.id", colName, relTable))
				colName = fkColName
				tables = append(tables, relTable)
			}
			cols = append(cols, colName)
		}
	}

	if len(fkWhere) > 0 {
		if len(whereStr) > 0 {
			whereStr += fmt.Sprintf(" AND (%v)", strings.Join(fkWhere, " AND "))
		} else {
			whereStr = "WHERE " + strings.Join(fkWhere, " AND ")
		}
	}

	sqlColumns := strings.Join(cols, ", ")
	sqlTables := strings.Join(tables, ", ")

	if len(sortBy) > 0 {
		sortCol := sortBy
		if a.NameTransform != nil {
			sortCol = a.NameTransform(sortBy)
		}

		direction := "ASC"
		if sortDesc {
			direction = "DESC"
		}

		sortBy = fmt.Sprintf(" ORDER BY %v.%v %v", mdl.tableName, sortCol, direction)
	}

	fromWhere := fmt.Sprintf("FROM %v %v", sqlTables, whereStr)
	rowQuery := fmt.Sprintf("SELECT %v %v%v LIMIT %v,%v", sqlColumns, fromWhere, sortBy, page*25, 25)
	countQuery := fmt.Sprintf("SELECT COUNT(*) %v", fromWhere)
	fmt.Println(rowQuery)

	var rows *sql.Rows
	var countRow *sql.Row
	var err error
	if doSearch {
		rows, err = a.db.Query(rowQuery, searchList...)
		countRow = a.db.QueryRow(countQuery, searchList...)
	} else {
		rows, err = a.db.Query(rowQuery)
		countRow = a.db.QueryRow(countQuery)
	}

	numRows := 0
	err = countRow.Scan(&numRows)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(numRows)

	if err != nil {
		return nil, numRows, err
	}

	numCols := len(cols)
	results := [][]interface{}{}

	for rows.Next() {
		result, err := scanRow(numCols, rows)
		if err != nil {
			return nil, numRows, err
		}
		results = append(results, result)
	}

	return results, numRows, nil
}

// querySingleModel is used in edit view.
func (a *Admin) querySingleModel(mdl *model, id int) (map[string]interface{}, error) {
	numCols := len(mdl.fieldNames)

	// Can't do * as column order in the DB might not match struct
	cols := make([]string, numCols)
	for i, fieldName := range mdl.fieldNames {
		if a.NameTransform != nil {
			fieldName = a.NameTransform(fieldName)
		}
		cols[i] = fieldName
	}

	q := fmt.Sprintf("SELECT %v FROM %v WHERE id = ?", strings.Join(cols, ", "), mdl.tableName)
	row := a.db.QueryRow(q, id)

	result, err := scanRow(numCols, row)
	if err != nil {
		return nil, err
	}

	resultMap := map[string]interface{}{}
	for i, val := range result {
		resultMap[mdl.fieldNames[i]] = val
	}

	return resultMap, nil
}

// MultiScanner is like the db.Scan interface, but scans to a slice.
type MultiScanner interface {
	Scan(src ...interface{}) error
}

// scanRow loads all data from a row into a string slice.
func scanRow(numCols int, scanner MultiScanner) ([]interface{}, error) {
	// We can only scan into pointers, so create result and destination slices
	result := make([]interface{}, numCols)
	dest := make([]interface{}, numCols)
	for i, _ := range result {
		dest[i] = &result[i]
	}

	err := scanner.Scan(dest...)
	if err != nil {
		return nil, err
	}

	// These are *interface{}, so get the interface{} and check if we can convert byte slice to string
	for i := 0; i < numCols; i++ {
		val := reflect.ValueOf(dest[i]).Elem().Interface()
		if str, ok := val.([]uint8); ok {
			result[i] = string(str)
		}
	}

	return result, nil
}
<commit_msg>More M2M progress.<commit_after>package admin

import (
	"database/sql"
	"fmt"
	"github.com/oal/admin/fields"
	"reflect"
	"strings"
)

// queryModel is used in list view to display all rows.
func (a *Admin) queryModel(mdl *model, search, sortBy string, sortDesc bool, page int) ([][]interface{}, int, error) {
	page--

	// Ugly search. Will fix later.
	doSearch := false
	whereStr := ""
	var searchList []interface{}
	if len(search) > 0 {
		if len(mdl.searchableColumns) > 0 {
			searchCols := make([]string, len(mdl.searchableColumns))
			searchList = make([]interface{}, len(searchCols))
			for i, _ := range searchList {
				searchList[i] = search
			}
			for i, _ := range searchCols {
				searchCols[i] = fmt.Sprintf("%v.%v LIKE ?", mdl.tableName, mdl.searchableColumns[i])
			}
			whereStr = fmt.Sprintf("WHERE (%v)", strings.Join(searchCols, " OR "))
			doSearch = true
		}

	}

	cols := []string{}
	tables := []string{mdl.tableName}
	fkWhere := []string{}
	for _, field := range mdl.fields {
		if field.Attrs().List {
			colName := fmt.Sprintf("%v.%v", mdl.tableName, field.Attrs().ColumnName)
			if relField, ok := field.(fields.RelationalField); ok && len(relField.GetListColumn()) > 0 {
				relTable := relField.GetRelatedTable()
				fkColName := fmt.Sprintf("%v.%v", relTable, relField.GetListColumn())
				fkWhere = append(fkWhere, fmt.Sprintf("%v = %v.id", colName, relTable))
				colName = fkColName
				tables = append(tables, relTable)
			}
			cols = append(cols, colName)
		}
	}

	if len(fkWhere) > 0 {
		if len(whereStr) > 0 {
			whereStr += fmt.Sprintf(" AND (%v)", strings.Join(fkWhere, " AND "))
		} else {
			whereStr = "WHERE " + strings.Join(fkWhere, " AND ")
		}
	}

	sqlColumns := strings.Join(cols, ", ")
	sqlTables := strings.Join(tables, ", ")

	if len(sortBy) > 0 {
		sortCol := sortBy
		if a.NameTransform != nil {
			sortCol = a.NameTransform(sortBy)
		}

		direction := "ASC"
		if sortDesc {
			direction = "DESC"
		}

		sortBy = fmt.Sprintf(" ORDER BY %v.%v %v", mdl.tableName, sortCol, direction)
	}

	fromWhere := fmt.Sprintf("FROM %v %v", sqlTables, whereStr)
	rowQuery := fmt.Sprintf("SELECT %v %v%v LIMIT %v,%v", sqlColumns, fromWhere, sortBy, page*25, 25)
	countQuery := fmt.Sprintf("SELECT COUNT(*) %v", fromWhere)
	fmt.Println(rowQuery)

	var rows *sql.Rows
	var countRow *sql.Row
	var err error
	if doSearch {
		rows, err = a.db.Query(rowQuery, searchList...)
		countRow = a.db.QueryRow(countQuery, searchList...)
	} else {
		rows, err = a.db.Query(rowQuery)
		countRow = a.db.QueryRow(countQuery)
	}

	numRows := 0
	err = countRow.Scan(&numRows)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(numRows)

	if err != nil {
		return nil, numRows, err
	}

	numCols := len(cols)
	results := [][]interface{}{}

	for rows.Next() {
		result, err := scanRow(numCols, rows)
		if err != nil {
			return nil, numRows, err
		}
		results = append(results, result)
	}

	return results, numRows, nil
}

// querySingleModel is used in edit view.
func (a *Admin) querySingleModel(mdl *model, id int) (map[string]interface{}, error) {
	cols := make([]string, 0, len(mdl.fieldNames))
	m2mFields := map[string]struct{}{}

	// Can't do * as column order in the DB might not match struct
	for _, fieldName := range mdl.fieldNames {
		// Add to m2mFields so we can load it later
		if _, ok := mdl.fieldByName(fieldName).(*fields.ManyToManyField); ok {
			m2mFields[fieldName] = struct{}{}
			continue
		}

		// Normal columns will be loaded directly in the main query
		if a.NameTransform != nil {
			fieldName = a.NameTransform(fieldName)
		}
		cols = append(cols, fieldName)
	}

	q := fmt.Sprintf("SELECT %v FROM %v WHERE id = ?", strings.Join(cols, ", "), mdl.tableName)
	row := a.db.QueryRow(q, id)

	result, err := scanRow(len(cols), row)
	if err != nil {
		return nil, err
	}

	// Loop over fields, and run separate query for M2Ms. Iterator index i only increases if there is value for column in main query.
	resultMap := map[string]interface{}{}
	i := 0
	for _, fieldName := range mdl.fieldNames {
		if _, ok := m2mFields[fieldName]; ok {
			// TODO: Separate query.
			resultMap[fieldName] = []int{1, 2, 3}
			continue
		}

		resultMap[fieldName] = result[i]
		i++
	}

	fmt.Println(resultMap)

	return resultMap, nil
}

// MultiScanner is like the db.Scan interface, but scans to a slice.
type MultiScanner interface {
	Scan(src ...interface{}) error
}

// scanRow loads all data from a row into a string slice.
func scanRow(numCols int, scanner MultiScanner) ([]interface{}, error) {
	// We can only scan into pointers, so create result and destination slices
	result := make([]interface{}, numCols)
	dest := make([]interface{}, numCols)
	for i, _ := range result {
		dest[i] = &result[i]
	}

	err := scanner.Scan(dest...)
	if err != nil {
		return nil, err
	}

	// These are *interface{}, so get the interface{} and check if we can convert byte slice to string
	for i := 0; i < numCols; i++ {
		val := reflect.ValueOf(dest[i]).Elem().Interface()
		if str, ok := val.([]uint8); ok {
			result[i] = string(str)
		}
	}

	return result, nil
}
<|endoftext|>


<commit_before>package sqldb

import (
	"bytes"
	"fmt"
	"io"
)

type DBConfig struct {
	Type     string
	Host     string
	Port     int
	DBName   string
	User     string
	Password string
	Options  map[string]string
}

func (d *DBConfig) JoinOptions(kvSep, optSep string) string {
	var buf bytes.Buffer
	for k, v := range d.Options {
		if buf.Len() > 0 {
			buf.WriteString(optSep)
		}
		buf.WriteString(k)
		buf.WriteString(kvSep)
		buf.WriteString(v)
	}
	return buf.String()
}

type NameMapper func(string) string

type DBDialect interface {
	Type(typ, precision, val string) (dbtyp, defaultVal string, err error)
	DSN(config DBConfig) string
}

func defaultVal(def, val string, quote bool) string {
	if val == "" {
		val = def
	}
	if quote {
		val = `'` + val + `'`
	}
	return val
}

type Postgres struct{}

func (Postgres) Type(typ, precision, val string) (dbtyp, defval string, err error) {
	switch typ {
	case "bool":
		return "BOOLEAN", defaultVal("false", val, false), nil
	case "int":
		return "INTEGER", defaultVal("0", val, false), nil
	case "int8":
		return "SMALLINT", defaultVal("0", val, false), nil
	case "int16":
		return "SMALLINT", defaultVal("0", val, false), nil
	case "int32":
		return "INTEGER", defaultVal("0", val, false), nil
	case "int64":
		return "BIGINT", defaultVal("0", val, false), nil
	case "uint":
		return "BIGINT", defaultVal("0", val, false), nil
	case "uint8":
		return "SMALLINT", defaultVal("0", val, false), nil
	case "uint16":
		return "INTEGER", defaultVal("0", val, false), nil
	case "uint32":
		return "BIGINT", defaultVal("0", val, false), nil
	case "uint64":
		return "BIGINT", defaultVal("0", val, false), nil
	case "float32", "float64":
		if precision != "" {
			typ = fmt.Sprintf("NUMERIC(%s)", precision)
		} else if typ == "float32" {
			typ = "REAL"
		} else {
			typ = "DOUBLE PRECISION"
		}
		return typ, defaultVal("0", val, false), nil
	case "string":
		if precision == "" {
			precision = "1024"
		}
		return fmt.Sprintf("VARCHAR(%s)", precision), defaultVal("", val, true), nil
	case "char":
		if precision == "" {
			precision = "256"
		}
		return fmt.Sprintf("CHAR(%s)", precision), defaultVal("", val, true), nil
	case "text":
		return "text", defaultVal("", val, true), nil
	default:
		return "", "", fmt.Errorf("postgres: unsupported type: %s", typ)
	}
}

func (Postgres) DSN(config DBConfig) string {
	if config.Host == "" {
		config.Host = "localhost"
	}
	if config.Port == 0 {
		config.Port = 5432
	}
	userPass := config.User
	if userPass != "" {
		if config.Password != "" {
			userPass += ":" + config.Password
		}
		userPass += "@"
	}
	return fmt.Sprintf(
		"postgres://%s%s:%d/%s?%s",
		userPass,
		config.Host,
		config.Port,
		config.DBName,
		config.JoinOptions("=", "&"),
	)
}

type Tx interface {
	Commit() error
	Rollback() error
}

func TxDone(tx Tx, err *error) error {
	var e error
	if err != nil && *err != nil {
		e = tx.Rollback()
	} else {
		e = tx.Commit()
	}
	return e
}

type TxCloser interface {
	Tx
	io.Closer
}

func TxDoneClose(tx TxCloser, err *error) error {
	e := TxDone(tx, err)
	tx.Close()
	return e
}
<commit_msg>add Open function<commit_after>package sqldb

import (
	"bytes"
	"database/sql"
	"fmt"
	"io"
	"time"
)

type DBConfig struct {
	Type        string
	Host        string
	Port        int
	DBName      string
	User        string
	Password    string
	MaxIdle     int
	MaxOpen     int
	MaxLifetime int
	Options     map[string]string
}

func (d *DBConfig) JoinOptions(kvSep, optSep string) string {
	var buf bytes.Buffer
	for k, v := range d.Options {
		if buf.Len() > 0 {
			buf.WriteString(optSep)
		}
		buf.WriteString(k)
		buf.WriteString(kvSep)
		buf.WriteString(v)
	}
	return buf.String()
}

type NameMapper func(string) string

type DBDialect interface {
	Type(typ, precision, val string) (dbtyp, defaultVal string, err error)
	DSN(config DBConfig) string
}

func Open(dialect DBDialect, config DBConfig) (*sql.DB, error) {
	db, err := sql.Open(config.Type, dialect.DSN(config))
	if err != nil {
		return nil, err
	}
	if config.MaxIdle > 0 {
		db.SetMaxIdleConns(config.MaxIdle)
	}
	if config.MaxOpen > 0 {
		db.SetMaxOpenConns(config.MaxOpen)
	}
	if config.MaxLifetime > 0 {
		db.SetConnMaxLifetime(time.Duration(config.MaxLifetime) * time.Second)
	}
	return db, nil
}

func defaultVal(def, val string, quote bool) string {
	if val == "" {
		val = def
	}
	if quote {
		val = `'` + val + `'`
	}
	return val
}

type Postgres struct{}

func (Postgres) Type(typ, precision, val string) (dbtyp, defval string, err error) {
	switch typ {
	case "bool":
		return "BOOLEAN", defaultVal("false", val, false), nil
	case "int":
		return "INTEGER", defaultVal("0", val, false), nil
	case "int8":
		return "SMALLINT", defaultVal("0", val, false), nil
	case "int16":
		return "SMALLINT", defaultVal("0", val, false), nil
	case "int32":
		return "INTEGER", defaultVal("0", val, false), nil
	case "int64":
		return "BIGINT", defaultVal("0", val, false), nil
	case "uint":
		return "BIGINT", defaultVal("0", val, false), nil
	case "uint8":
		return "SMALLINT", defaultVal("0", val, false), nil
	case "uint16":
		return "INTEGER", defaultVal("0", val, false), nil
	case "uint32":
		return "BIGINT", defaultVal("0", val, false), nil
	case "uint64":
		return "BIGINT", defaultVal("0", val, false), nil
	case "float32", "float64":
		if precision != "" {
			typ = fmt.Sprintf("NUMERIC(%s)", precision)
		} else if typ == "float32" {
			typ = "REAL"
		} else {
			typ = "DOUBLE PRECISION"
		}
		return typ, defaultVal("0", val, false), nil
	case "string":
		if precision == "" {
			precision = "1024"
		}
		return fmt.Sprintf("VARCHAR(%s)", precision), defaultVal("", val, true), nil
	case "char":
		if precision == "" {
			precision = "256"
		}
		return fmt.Sprintf("CHAR(%s)", precision), defaultVal("", val, true), nil
	case "text":
		return "text", defaultVal("", val, true), nil
	default:
		return "", "", fmt.Errorf("postgres: unsupported type: %s", typ)
	}
}

func (Postgres) DSN(config DBConfig) string {
	if config.Host == "" {
		config.Host = "localhost"
	}
	if config.Port == 0 {
		config.Port = 5432
	}
	userPass := config.User
	if userPass != "" {
		if config.Password != "" {
			userPass += ":" + config.Password
		}
		userPass += "@"
	}
	return fmt.Sprintf(
		"postgres://%s%s:%d/%s?%s",
		userPass,
		config.Host,
		config.Port,
		config.DBName,
		config.JoinOptions("=", "&"),
	)
}

type Tx interface {
	Commit() error
	Rollback() error
}

func TxDone(tx Tx, err *error) error {
	var e error
	if err != nil && *err != nil {
		e = tx.Rollback()
	} else {
		e = tx.Commit()
	}
	return e
}

type TxCloser interface {
	Tx
	io.Closer
}

func TxDoneClose(tx TxCloser, err *error) error {
	e := TxDone(tx, err)
	tx.Close()
	return e
}
<|endoftext|>


<commit_before>package appgo

import (
	"bytes"
	"encoding/base64"
	"encoding/binary"
	"encoding/json"
	"math"
	"strconv"
)

var (
	base64Encoding = base64.RawURLEncoding
	binaryEndian   = binary.BigEndian
)

type Id int64

func (id Id) String() string {
	return strconv.FormatInt(int64(id), 10)
}

func (id Id) MarshalJSON() ([]byte, error) {
	var buffer bytes.Buffer
	buffer.WriteByte('"')
	buffer.WriteString(id.String())
	buffer.WriteByte('"')
	return buffer.Bytes(), nil
}

func (id *Id) UnmarshalJSON(b []byte) error {
	var s string
	if err := json.Unmarshal(b, &s); err != nil {
		return err
	}
	if val, err := strconv.ParseInt(s, 10, 64); err != nil {
		return err
	} else {
		*id = Id(val)
	}
	return nil
}

func IdFromStr(str string) Id {
	i, _ := strconv.ParseInt(str, 10, 64)
	return Id(i)
}

func IdFromBase64(str string) Id {
	if len(str) == 0 {
		return 0
	} else if data, err := base64Encoding.DecodeString(str); err != nil {
		return 0
	} else {
		val := binaryEndian.Uint64(data)
		return Id(int64(val))
	}
}

func IdMax() Id {
	return Id(math.MaxInt64)
}

func (id Id) Base64() string {
	buf := make([]byte, 8)
	binaryEndian.PutUint64(buf, uint64(int64(id)))
	return base64Encoding.EncodeToString(buf)
}
<commit_msg>add UnmarshalText for gorilla/schema<commit_after>package appgo

import (
	"bytes"
	"encoding/base64"
	"encoding/binary"
	"encoding/json"
	"math"
	"strconv"
)

var (
	base64Encoding = base64.RawURLEncoding
	binaryEndian   = binary.BigEndian
)

type Id int64

func (id Id) String() string {
	return strconv.FormatInt(int64(id), 10)
}

func (id Id) MarshalJSON() ([]byte, error) {
	var buffer bytes.Buffer
	buffer.WriteByte('"')
	buffer.WriteString(id.String())
	buffer.WriteByte('"')
	return buffer.Bytes(), nil
}

func (id *Id) UnmarshalJSON(b []byte) error {
	var s string
	if err := json.Unmarshal(b, &s); err != nil {
		return err
	}
	if val, err := strconv.ParseInt(s, 10, 64); err != nil {
		return err
	} else {
		*id = Id(val)
	}
	return nil
}

// for github.com/gorilla/schema
func (id *Id) UnmarshalText(text []byte) (err error) {
	*id = IdFromStr(string(text))
	return nil
}

func IdFromStr(str string) Id {
	i, _ := strconv.ParseInt(str, 10, 64)
	return Id(i)
}

func IdFromBase64(str string) Id {
	if len(str) == 0 {
		return 0
	} else if data, err := base64Encoding.DecodeString(str); err != nil {
		return 0
	} else {
		val := binaryEndian.Uint64(data)
		return Id(int64(val))
	}
}

func IdMax() Id {
	return Id(math.MaxInt64)
}

func (id Id) Base64() string {
	buf := make([]byte, 8)
	binaryEndian.PutUint64(buf, uint64(int64(id)))
	return base64Encoding.EncodeToString(buf)
}
<|endoftext|>


<commit_before>package pcap

import (
	"encoding/binary"
	"fmt"
	"io"
	"time"
)

// FileHeader is the parsed header of a pcap file.
// http://wiki.wireshark.org/Development/LibpcapFileFormat
type FileHeader struct {
	MagicNumber  uint32
	VersionMajor uint16
	VersionMinor uint16
	TimeZone     int32
	SigFigs      uint32
	SnapLen      uint32

	// NOTE: 'Network' property has been changed to `linktype`
	// Please see pcap/pcap.h header file.
	//     Network      uint32
	LinkType int32
}

type PacketTime struct {
	Sec  int32
	Usec int32
}

// Packet is a single packet parsed from a pcap file.
type Packet struct {
	Time   time.Time // packet send/receive time
	Caplen uint32    // bytes stored in the file (caplen <= len)
	Len    uint32    // bytes sent/received
	Data   []byte    // packet data

	Type    int // protocol type, see LINKTYPE_*
	DestMac uint64
	SrcMac  uint64

	Headers []interface{} // decoded headers, in order
	Payload []byte        // remaining non-header bytes
}

// Reader parses pcap files.
type Reader struct {
	flip         bool
	buf          io.Reader
	err          error
	fourBytes    []byte
	twoBytes     []byte
	sixteenBytes []byte
	Header       FileHeader
}

// NewReader reads pcap data from an io.Reader.
func NewReader(reader io.Reader) (*Reader, error) {
	r := &Reader{
		buf:          reader,
		fourBytes:    make([]byte, 4),
		twoBytes:     make([]byte, 2),
		sixteenBytes: make([]byte, 16),
	}
	switch magic := r.readUint32(); magic {
	case 0xa1b2c3d4:
		r.flip = false
	case 0xd4c3b2a1:
		r.flip = true
	default:
		return nil, fmt.Errorf("pcap: bad magic number: %0x", magic)
	}
	r.Header = FileHeader{
		MagicNumber:  0xa1b2c3d4,
		VersionMajor: r.readUint16(),
		VersionMinor: r.readUint16(),
		TimeZone:     r.readInt32(),
		SigFigs:      r.readUint32(),
		SnapLen:      r.readUint32(),
		LinkType:      r.readUint32(),
	}
	return r, nil
}

// Next returns the next packet or nil if no more packets can be read.
func (r *Reader) Next() *Packet {
	d := r.sixteenBytes
	r.err = r.read(d)
	if r.err != nil {
		return nil
	}
	timeSec := asUint32(d[0:4], r.flip)
	timeUsec := asUint32(d[4:8], r.flip)
	capLen := asUint32(d[8:12], r.flip)
	origLen := asUint32(d[12:16], r.flip)

	data := make([]byte, capLen)
	if r.err = r.read(data); r.err != nil {
		return nil
	}
	return &Packet{
		Time: time.Unix(int64(timeSec), int64(timeUsec)),
		Caplen: capLen,
		Len:    origLen,
		Data:   data,
	}
}

func (r *Reader) read(data []byte) error {
	var err error
	n, err := r.buf.Read(data)
	for err == nil && n != len(data) {
		var chunk int
		chunk, err = r.buf.Read(data[n:])
		n += chunk
	}
	if len(data) == n {
		return nil
	}
	return err
}

func (r *Reader) readUint32() uint32 {
	data := r.fourBytes
	if r.err = r.read(data); r.err != nil {
		return 0
	}
	return asUint32(data, r.flip)
}

func (r *Reader) readInt32() int32 {
	data := r.fourBytes
	if r.err = r.read(data); r.err != nil {
		return 0
	}
	return int32(asUint32(data, r.flip))
}

func (r *Reader) readUint16() uint16 {
	data := r.twoBytes
	if r.err = r.read(data); r.err != nil {
		return 0
	}
	return asUint16(data, r.flip)
}

// Writer writes a pcap file.
type Writer struct {
	writer io.Writer
	buf    []byte
}

// NewWriter creates a Writer that stores output in an io.Writer.
// The FileHeader is written immediately.
func NewWriter(writer io.Writer, header *FileHeader) (*Writer, error) {
	w := &Writer{
		writer: writer,
		buf:    make([]byte, 24),
	}
	binary.LittleEndian.PutUint32(w.buf, header.MagicNumber)
	binary.LittleEndian.PutUint16(w.buf[4:], header.VersionMajor)
	binary.LittleEndian.PutUint16(w.buf[6:], header.VersionMinor)
	binary.LittleEndian.PutUint32(w.buf[8:], uint32(header.TimeZone))
	binary.LittleEndian.PutUint32(w.buf[12:], header.SigFigs)
	binary.LittleEndian.PutUint32(w.buf[16:], header.SnapLen)
	binary.LittleEndian.PutUint32(w.buf[20:], header.LinkType)
	if _, err := writer.Write(w.buf); err != nil {
		return nil, err
	}
	return w, nil
}

// Writer writes a packet to the underlying writer.
func (w *Writer) Write(pkt *Packet) error {
	binary.LittleEndian.PutUint32(w.buf, uint32(pkt.Time.Unix()))
	binary.LittleEndian.PutUint32(w.buf[4:], uint32(pkt.Time.Nanosecond()))
	binary.LittleEndian.PutUint32(w.buf[8:], pkt.Caplen)
	binary.LittleEndian.PutUint32(w.buf[12:], pkt.Len)
	if _, err := w.writer.Write(w.buf[:16]); err != nil {
		return err
	}
	_, err := w.writer.Write(pkt.Data)
	return err
}

func asUint32(data []byte, flip bool) uint32 {
	if flip {
		return binary.BigEndian.Uint32(data)
	}
	return binary.LittleEndian.Uint32(data)
}

func asUint16(data []byte, flip bool) uint16 {
	if flip {
		return binary.BigEndian.Uint16(data)
	}
	return binary.LittleEndian.Uint16(data)
}
<commit_msg>LinkType should be uint32<commit_after>package pcap

import (
	"encoding/binary"
	"fmt"
	"io"
	"time"
)

// FileHeader is the parsed header of a pcap file.
// http://wiki.wireshark.org/Development/LibpcapFileFormat
type FileHeader struct {
	MagicNumber  uint32
	VersionMajor uint16
	VersionMinor uint16
	TimeZone     int32
	SigFigs      uint32
	SnapLen      uint32

	// NOTE: 'Network' property has been changed to `linktype`
	// Please see pcap/pcap.h header file.
	//     Network      uint32
	LinkType     uint32
}

type PacketTime struct {
	Sec  int32
	Usec int32
}

// Packet is a single packet parsed from a pcap file.
type Packet struct {
	Time   time.Time // packet send/receive time
	Caplen uint32    // bytes stored in the file (caplen <= len)
	Len    uint32    // bytes sent/received
	Data   []byte    // packet data

	Type    int // protocol type, see LINKTYPE_*
	DestMac uint64
	SrcMac  uint64

	Headers []interface{} // decoded headers, in order
	Payload []byte        // remaining non-header bytes
}

// Reader parses pcap files.
type Reader struct {
	flip         bool
	buf          io.Reader
	err          error
	fourBytes    []byte
	twoBytes     []byte
	sixteenBytes []byte
	Header       FileHeader
}

// NewReader reads pcap data from an io.Reader.
func NewReader(reader io.Reader) (*Reader, error) {
	r := &Reader{
		buf:          reader,
		fourBytes:    make([]byte, 4),
		twoBytes:     make([]byte, 2),
		sixteenBytes: make([]byte, 16),
	}
	switch magic := r.readUint32(); magic {
	case 0xa1b2c3d4:
		r.flip = false
	case 0xd4c3b2a1:
		r.flip = true
	default:
		return nil, fmt.Errorf("pcap: bad magic number: %0x", magic)
	}
	r.Header = FileHeader{
		MagicNumber:  0xa1b2c3d4,
		VersionMajor: r.readUint16(),
		VersionMinor: r.readUint16(),
		TimeZone:     r.readInt32(),
		SigFigs:      r.readUint32(),
		SnapLen:      r.readUint32(),
		LinkType:      r.readUint32(),
	}
	return r, nil
}

// Next returns the next packet or nil if no more packets can be read.
func (r *Reader) Next() *Packet {
	d := r.sixteenBytes
	r.err = r.read(d)
	if r.err != nil {
		return nil
	}
	timeSec := asUint32(d[0:4], r.flip)
	timeUsec := asUint32(d[4:8], r.flip)
	capLen := asUint32(d[8:12], r.flip)
	origLen := asUint32(d[12:16], r.flip)

	data := make([]byte, capLen)
	if r.err = r.read(data); r.err != nil {
		return nil
	}
	return &Packet{
		Time: time.Unix(int64(timeSec), int64(timeUsec)),
		Caplen: capLen,
		Len:    origLen,
		Data:   data,
	}
}

func (r *Reader) read(data []byte) error {
	var err error
	n, err := r.buf.Read(data)
	for err == nil && n != len(data) {
		var chunk int
		chunk, err = r.buf.Read(data[n:])
		n += chunk
	}
	if len(data) == n {
		return nil
	}
	return err
}

func (r *Reader) readUint32() uint32 {
	data := r.fourBytes
	if r.err = r.read(data); r.err != nil {
		return 0
	}
	return asUint32(data, r.flip)
}

func (r *Reader) readInt32() int32 {
	data := r.fourBytes
	if r.err = r.read(data); r.err != nil {
		return 0
	}
	return int32(asUint32(data, r.flip))
}

func (r *Reader) readUint16() uint16 {
	data := r.twoBytes
	if r.err = r.read(data); r.err != nil {
		return 0
	}
	return asUint16(data, r.flip)
}

// Writer writes a pcap file.
type Writer struct {
	writer io.Writer
	buf    []byte
}

// NewWriter creates a Writer that stores output in an io.Writer.
// The FileHeader is written immediately.
func NewWriter(writer io.Writer, header *FileHeader) (*Writer, error) {
	w := &Writer{
		writer: writer,
		buf:    make([]byte, 24),
	}
	binary.LittleEndian.PutUint32(w.buf, header.MagicNumber)
	binary.LittleEndian.PutUint16(w.buf[4:], header.VersionMajor)
	binary.LittleEndian.PutUint16(w.buf[6:], header.VersionMinor)
	binary.LittleEndian.PutUint32(w.buf[8:], uint32(header.TimeZone))
	binary.LittleEndian.PutUint32(w.buf[12:], header.SigFigs)
	binary.LittleEndian.PutUint32(w.buf[16:], header.SnapLen)
	binary.LittleEndian.PutUint32(w.buf[20:], header.LinkType)
	if _, err := writer.Write(w.buf); err != nil {
		return nil, err
	}
	return w, nil
}

// Writer writes a packet to the underlying writer.
func (w *Writer) Write(pkt *Packet) error {
	binary.LittleEndian.PutUint32(w.buf, uint32(pkt.Time.Unix()))
	binary.LittleEndian.PutUint32(w.buf[4:], uint32(pkt.Time.Nanosecond()))
	binary.LittleEndian.PutUint32(w.buf[8:], pkt.Caplen)
	binary.LittleEndian.PutUint32(w.buf[12:], pkt.Len)
	if _, err := w.writer.Write(w.buf[:16]); err != nil {
		return err
	}
	_, err := w.writer.Write(pkt.Data)
	return err
}

func asUint32(data []byte, flip bool) uint32 {
	if flip {
		return binary.BigEndian.Uint32(data)
	}
	return binary.LittleEndian.Uint32(data)
}

func asUint16(data []byte, flip bool) uint16 {
	if flip {
		return binary.BigEndian.Uint16(data)
	}
	return binary.LittleEndian.Uint16(data)
}
<|endoftext|>


<commit_before>// +build !js

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
)

func Replay(file string) error {
	ui := &gameui{}
	g := &game{}
	ui.g = g
	g.ui = ui
	err := g.LoadReplay(file)
	if err != nil {
		return fmt.Errorf("loading replay: %v", err)
	}
	err = ui.Init()
	if err != nil {
		fmt.Fprintf(os.Stderr, "boohu: %v\n", err)
		os.Exit(1)
	}
	defer ui.Close()
	ui.DrawBufferInit()
	ui.Replay()
	ui.Close()
	return nil
}

func (g *game) DataDir() (string, error) {
	var xdg string
	if os.Getenv("GOOS") == "windows" {
		xdg = os.Getenv("LOCALAPPDATA")
	} else {
		xdg = os.Getenv("XDG_DATA_HOME")
	}
	if xdg == "" {
		xdg = filepath.Join(os.Getenv("HOME"), ".local", "share")
	}
	dataDir := filepath.Join(xdg, "boohu")
	_, err := os.Stat(dataDir)
	if err != nil {
		err = os.MkdirAll(dataDir, 0755)
		if err != nil {
			return "", fmt.Errorf("%v\n", err)
		}
	}
	return dataDir, nil
}

func (g *game) Save() error {
	dataDir, err := g.DataDir()
	if err != nil {
		g.Print(err.Error())
		return err
	}
	saveFile := filepath.Join(dataDir, "save")
	data, err := g.GameSave()
	if err != nil {
		g.Print(err.Error())
		return err
	}
	err = ioutil.WriteFile(saveFile, data, 0644)
	if err != nil {
		g.Print(err.Error())
		return err
	}
	return nil
}

func (g *game) RemoveSaveFile() error {
	return g.RemoveDataFile("save")
}

func (g *game) Load() (bool, error) {
	dataDir, err := g.DataDir()
	if err != nil {
		return false, err
	}
	saveFile := filepath.Join(dataDir, "save")
	_, err = os.Stat(saveFile)
	if err != nil {
		// no save file, new game
		return false, err
	}
	data, err := ioutil.ReadFile(saveFile)
	if err != nil {
		return true, err
	}
	lg, err := g.DecodeGameSave(data)
	if err != nil {
		return true, err
	}
	if lg.Version != Version {
		return true, fmt.Errorf("saved game for previous version %s.", lg.Version)
	}
	*g = *lg
	return true, nil
}

func (g *game) SaveConfig() error {
	dataDir, err := g.DataDir()
	if err != nil {
		g.Print(err.Error())
		return err
	}
	saveFile := filepath.Join(dataDir, "config.gob")
	data, err := gameConfig.ConfigSave()
	if err != nil {
		g.Print(err.Error())
		return err
	}
	err = ioutil.WriteFile(saveFile, data, 0644)
	if err != nil {
		g.Print(err.Error())
		return err
	}
	return nil
}

func (g *game) LoadConfig() (bool, error) {
	dataDir, err := g.DataDir()
	if err != nil {
		return false, err
	}
	saveFile := filepath.Join(dataDir, "config.gob")
	_, err = os.Stat(saveFile)
	if err != nil {
		// no save file, new game
		return false, err
	}
	data, err := ioutil.ReadFile(saveFile)
	if err != nil {
		return true, err
	}
	c, err := g.DecodeConfigSave(data)
	if err != nil {
		return true, err
	}
	gameConfig = *c
	ApplyConfig()
	return true, nil
}

func (g *game) RemoveDataFile(file string) error {
	dataDir, err := g.DataDir()
	if err != nil {
		return err
	}
	dataFile := filepath.Join(dataDir, file)
	_, err = os.Stat(dataFile)
	if err == nil {
		err := os.Remove(dataFile)
		if err != nil {
			return err
		}
	}
	return nil
}

func (g *game) SaveReplay() error {
	dataDir, err := g.DataDir()
	if err != nil {
		g.Print(err.Error())
		return err
	}
	saveFile := filepath.Join(dataDir, "replay")
	data, err := g.EncodeDrawLog()
	if err != nil {
		g.Print(err.Error())
		return err
	}
	err = ioutil.WriteFile(saveFile, data, 0644)
	if err != nil {
		g.Print(err.Error())
		return err
	}
	return nil
}

func (g *game) LoadReplay(file string) error {
	dataDir, err := g.DataDir()
	if err != nil {
		return err
	}
	replayFile := filepath.Join(dataDir, "replay")
	if file != "_" {
		replayFile = file
	}
	_, err = os.Stat(replayFile)
	if err != nil {
		// no save file, new game
		return err
	}
	data, err := ioutil.ReadFile(replayFile)
	if err != nil {
		return err
	}
	dl, err := g.DecodeDrawLog(data)
	if err != nil {
		return err
	}
	g.DrawLog = dl
	return nil
}

func (g *game) WriteDump() error {
	dataDir, err := g.DataDir()
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(filepath.Join(dataDir, "dump"), []byte(g.Dump()), 0644)
	if err != nil {
		return fmt.Errorf("writing game statistics: %v", err)
	}
	err = g.SaveReplay()
	if err != nil {
		return fmt.Errorf("writing replay: %v", err)
	}
	return nil
}
<commit_msg>remove redundant ui.Close()<commit_after>// +build !js

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
)

func Replay(file string) error {
	ui := &gameui{}
	g := &game{}
	ui.g = g
	g.ui = ui
	err := g.LoadReplay(file)
	if err != nil {
		return fmt.Errorf("loading replay: %v", err)
	}
	err = ui.Init()
	if err != nil {
		fmt.Fprintf(os.Stderr, "boohu: %v\n", err)
		os.Exit(1)
	}
	defer ui.Close()
	ui.DrawBufferInit()
	ui.Replay()
	return nil
}

func (g *game) DataDir() (string, error) {
	var xdg string
	if os.Getenv("GOOS") == "windows" {
		xdg = os.Getenv("LOCALAPPDATA")
	} else {
		xdg = os.Getenv("XDG_DATA_HOME")
	}
	if xdg == "" {
		xdg = filepath.Join(os.Getenv("HOME"), ".local", "share")
	}
	dataDir := filepath.Join(xdg, "boohu")
	_, err := os.Stat(dataDir)
	if err != nil {
		err = os.MkdirAll(dataDir, 0755)
		if err != nil {
			return "", fmt.Errorf("%v\n", err)
		}
	}
	return dataDir, nil
}

func (g *game) Save() error {
	dataDir, err := g.DataDir()
	if err != nil {
		g.Print(err.Error())
		return err
	}
	saveFile := filepath.Join(dataDir, "save")
	data, err := g.GameSave()
	if err != nil {
		g.Print(err.Error())
		return err
	}
	err = ioutil.WriteFile(saveFile, data, 0644)
	if err != nil {
		g.Print(err.Error())
		return err
	}
	return nil
}

func (g *game) RemoveSaveFile() error {
	return g.RemoveDataFile("save")
}

func (g *game) Load() (bool, error) {
	dataDir, err := g.DataDir()
	if err != nil {
		return false, err
	}
	saveFile := filepath.Join(dataDir, "save")
	_, err = os.Stat(saveFile)
	if err != nil {
		// no save file, new game
		return false, err
	}
	data, err := ioutil.ReadFile(saveFile)
	if err != nil {
		return true, err
	}
	lg, err := g.DecodeGameSave(data)
	if err != nil {
		return true, err
	}
	if lg.Version != Version {
		return true, fmt.Errorf("saved game for previous version %s.", lg.Version)
	}
	*g = *lg
	return true, nil
}

func (g *game) SaveConfig() error {
	dataDir, err := g.DataDir()
	if err != nil {
		g.Print(err.Error())
		return err
	}
	saveFile := filepath.Join(dataDir, "config.gob")
	data, err := gameConfig.ConfigSave()
	if err != nil {
		g.Print(err.Error())
		return err
	}
	err = ioutil.WriteFile(saveFile, data, 0644)
	if err != nil {
		g.Print(err.Error())
		return err
	}
	return nil
}

func (g *game) LoadConfig() (bool, error) {
	dataDir, err := g.DataDir()
	if err != nil {
		return false, err
	}
	saveFile := filepath.Join(dataDir, "config.gob")
	_, err = os.Stat(saveFile)
	if err != nil {
		// no save file, new game
		return false, err
	}
	data, err := ioutil.ReadFile(saveFile)
	if err != nil {
		return true, err
	}
	c, err := g.DecodeConfigSave(data)
	if err != nil {
		return true, err
	}
	gameConfig = *c
	ApplyConfig()
	return true, nil
}

func (g *game) RemoveDataFile(file string) error {
	dataDir, err := g.DataDir()
	if err != nil {
		return err
	}
	dataFile := filepath.Join(dataDir, file)
	_, err = os.Stat(dataFile)
	if err == nil {
		err := os.Remove(dataFile)
		if err != nil {
			return err
		}
	}
	return nil
}

func (g *game) SaveReplay() error {
	dataDir, err := g.DataDir()
	if err != nil {
		g.Print(err.Error())
		return err
	}
	saveFile := filepath.Join(dataDir, "replay")
	data, err := g.EncodeDrawLog()
	if err != nil {
		g.Print(err.Error())
		return err
	}
	err = ioutil.WriteFile(saveFile, data, 0644)
	if err != nil {
		g.Print(err.Error())
		return err
	}
	return nil
}

func (g *game) LoadReplay(file string) error {
	dataDir, err := g.DataDir()
	if err != nil {
		return err
	}
	replayFile := filepath.Join(dataDir, "replay")
	if file != "_" {
		replayFile = file
	}
	_, err = os.Stat(replayFile)
	if err != nil {
		// no save file, new game
		return err
	}
	data, err := ioutil.ReadFile(replayFile)
	if err != nil {
		return err
	}
	dl, err := g.DecodeDrawLog(data)
	if err != nil {
		return err
	}
	g.DrawLog = dl
	return nil
}

func (g *game) WriteDump() error {
	dataDir, err := g.DataDir()
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(filepath.Join(dataDir, "dump"), []byte(g.Dump()), 0644)
	if err != nil {
		return fmt.Errorf("writing game statistics: %v", err)
	}
	err = g.SaveReplay()
	if err != nil {
		return fmt.Errorf("writing replay: %v", err)
	}
	return nil
}
<|endoftext|>