<commit_before>package opentsdb

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"

	"github.com/StackExchange/slog"
)

var l = log.New(os.Stdout, "", log.LstdFlags)

type ResponseSet []*Response

type Point float64

type Response struct {
	Metric        string           `json:"metric"`
	Tags          TagSet           `json:"tags"`
	AggregateTags []string         `json:"aggregateTags"`
	DPS           map[string]Point `json:"dps"`
}

type DataPoint struct {
	Metric    string      `json:"metric"`
	Timestamp int64       `json:"timestamp"`
	Value     interface{} `json:"value"`
	Tags      TagSet      `json:"tags"`
}

func (d *DataPoint) Telnet() string {
	m := ""
	d.clean()
	for k, v := range d.Tags {
		m += fmt.Sprintf(" %s=%s", k, v)
	}
	return fmt.Sprintf("put %s %d %v%s\n", d.Metric, d.Timestamp, d.Value, m)
}

func (m MultiDataPoint) Json() ([]byte, error) {
	var md MultiDataPoint
	for _, d := range m {
		err := d.clean()
		if err != nil {
			slog.Infoln(err, "Removing Datapoint", d)
			continue
		}
		md = append(md, d)
	}
	return json.Marshal(md)
}

type MultiDataPoint []*DataPoint

type TagSet map[string]string

func (t TagSet) Equal(o TagSet) bool {
	if len(t) != len(o) {
		return false
	}
	for k, v := range t {
		if ov, ok := o[k]; !ok || ov != v {
			return false
		}
	}
	return true
}

// Subset returns true if all k=v pairs in o are in t.
func (t TagSet) Subset(o TagSet) bool {
	for k, v := range o {
		if tv, ok := t[k]; !ok || tv != v {
			return false
		}
	}
	return true
}

// String converts t to an OpenTSDB-style {a=b,c=b} string, alphabetized by key.
func (t TagSet) String() string {
	var keys []string
	for k := range t {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	b := bytes.NewBufferString("{")
	for i, k := range keys {
		if i > 0 {
			fmt.Fprint(b, ",")
		}
		fmt.Fprintf(b, "%s=%s", k, t[k])
	}
	fmt.Fprint(b, "}")
	return b.String()
}

func (d *DataPoint) clean() error {
	err := d.Tags.clean()
	if err != nil {
		return err
	}
	om := d.Metric
	d.Metric, err = Clean(d.Metric)
	if err != nil {
		return fmt.Errorf("%s. Orginal: [%s] Cleaned: [%s]", err.Error(), om, d.Metric)
	}
	if sv, ok := d.Value.(string); ok {
		if i, err := strconv.ParseInt(sv, 10, 64); err == nil {
			d.Value = i
		} else if f, err := strconv.ParseFloat(sv, 64); err == nil {
			d.Value = f
		} else {
			return fmt.Errorf("Unparseable number %v", sv)
		}
	}
	return nil
}

func (t TagSet) clean() error {
	for k, v := range t {
		kc, err := Clean(k)
		if err != nil {
			return fmt.Errorf("%s. Orginal: [%s] Cleaned: [%s]", err.Error(), k, kc)
		}
		vc, err := Clean(v)
		if err != nil {
			return fmt.Errorf("%s. Orginal: [%s] Cleaned: [%s]", err.Error(), v, vc)
		}
		delete(t, k)
		t[kc] = vc
	}
	return nil
}

// Clean removes characters from s that are invalid for OpenTSDB metric and tag
// values.
// See: http://opentsdb.net/docs/build/html/user_guide/writing.html#metrics-and-tags
func Clean(s string) (string, error) {
	var c string
	if len(s) == 0 {
		// This one is perhaps better checked earlier in the pipeline, but since
		// it makes sense to check that the resulting cleaned tag is not Zero length here I'm including it
		// It also might be the case that this just shouldn't be happening and this is yet another side
		// effect of WMI turning to Garbage....
		return s, errors.New("Metric/Tagk/Tagv Cleaning Passed a Zero Length String")
	}
	for len(s) > 0 {
		r, size := utf8.DecodeRuneInString(s)
		if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '-' || r == '_' || r == '.' || r == '/' {
			c += string(r)
		}
		s = s[size:]
	}

	if len(c) == 0 {
		return c, errors.New("Cleaning Metric/Tagk/Tagv resulted in a Zero Length String")
	}
	return c, nil
}

type Request struct {
	Start             interface{} `json:"start"`
	End               interface{} `json:"end,omitempty"`
	Queries           []*Query    `json:"queries"`
	NoAnnotations     bool        `json:"noAnnotations,omitempty"`
	GlobalAnnotations bool        `json:"globalAnnotations,omitempty"`
	MsResolution      bool        `json:"msResolution,omitempty"`
	ShowTSUIDs        bool        `json:"showTSUIDs,omitempty"`
}

type Query struct {
	Aggregator  string `json:"aggregator"`
	Metric      string `json:"metric"`
	Rate        bool   `json:"rate,omitempty"`
	RateOptions `json:"rateOptions,omitempty"`
	Downsample  string `json:"downsample,omitempty"`
	Tags        TagSet `json:"tags,omitempty"`
}

type RateOptions struct {
	Counter    bool `json:"counter,omitempty"`
	CounterMax int  `json:"counterMax,omitempty"`
	ResetValue int  `json:"resetValue,omitempty"`
}

// ParsesRequest parses OpenTSDB requests of the form: start=1h-ago&m=avg:cpu.
func ParseRequest(req string) (*Request, error) {
	v, err := url.ParseQuery(req)
	if err != nil {
		return nil, err
	}
	r := Request{}
	if s := v.Get("start"); s == "" {
		return nil, fmt.Errorf("tsdb: missing start: %s", req)
	} else {
		r.Start = s
	}
	for _, m := range v["m"] {
		q, err := ParseQuery(m)
		if err != nil {
			return nil, err
		}
		r.Queries = append(r.Queries, q)
	}
	if len(r.Queries) == 0 {
		return nil, fmt.Errorf("tsdb: missing m: %s", req)
	}
	return &r, nil
}

var qRE = regexp.MustCompile(`^(\w+):(?:(\w+-\w+):)?(?:(rate):)?([\w./]+)(?:\{([\w./,=*-|]+)\})?$`)

// ParseQuery parses OpenTSDB queries of the form: avg:rate:cpu{k=v}.
func ParseQuery(query string) (*Query, error) {
	q := Query{}
	m := qRE.FindStringSubmatch(query)
	if m == nil {
		return nil, fmt.Errorf("tsdb: bad query format: %s", query)
	}
	q.Aggregator = m[1]
	q.Downsample = m[2]
	q.Rate = m[3] == "rate"
	q.Metric = m[4]
	if m[5] != "" {
		tags, err := ParseTags(m[5])
		if err != nil {
			return nil, err
		}
		q.Tags = tags
	}
	return &q, nil
}

// ParseTags parses OpenTSDB tagk=tagv pairs of the form: k=v,m=o.
func ParseTags(t string) (TagSet, error) {
	ts := make(TagSet)
	for _, v := range strings.Split(t, ",") {
		sp := strings.SplitN(v, "=", 2)
		if len(sp) != 2 {
			return nil, fmt.Errorf("tsdb: bad tag: %s", v)
		}
		ts[sp[0]] = sp[1]
	}
	return ts, nil
}

func (q Query) String() string {
	s := q.Aggregator + ":"
	if q.Downsample != "" {
		s += q.Downsample + ":"
	}
	if q.Rate {
		s += "rate:"
	}
	s += q.Metric
	if len(q.Tags) > 0 {
		s += "{"
		first := true
		for k, v := range q.Tags {
			if first {
				first = false
			} else {
				s += ","
			}
			s += k + "=" + v
		}
		s += "}"
	}
	return s
}

func (r Request) String() string {
	v := make(url.Values)
	for _, q := range r.Queries {
		v.Add("m", q.String())
	}
	v.Add("start", fmt.Sprint(r.Start))
	if e := fmt.Sprint(r.End); r.End != nil && e != "" {
		v.Add("end", e)
	}
	return v.Encode()
}

type RequestError struct {
	Err struct {
		Code    int    `json:"code"`
		Message string `json:"message"`
		Details string `json:"details"`
	} `json:"error"`
}

func (r *RequestError) Error() string {
	return fmt.Sprintf("tsdb: %s", r.Err.Message)
}

// Query performs a v2 OpenTSDB request to the given host. host should be of the
// form hostname:port. Can return a RequestError.
func (r Request) Query(host string) (ResponseSet, error) {
	u := url.URL{
		Scheme: "http",
		Host:   host,
		Path:   "/api/query",
	}
	b, err := json.Marshal(&r)
	if err != nil {
		return nil, err
	}
	resp, err := http.Post(u.String(), "application/json", bytes.NewReader(b))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		b, err = ioutil.ReadAll(resp.Body)
		fmt.Println(string(b))
		e := RequestError{}
		if err := json.Unmarshal(b, &e); err == nil {
			return nil, &e
		}
		return nil, fmt.Errorf("tsdb: %s", b)
	}
	b, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var tr ResponseSet
	if err := json.Unmarshal(b, &tr); err != nil {
		return nil, err
	}
	return tr, nil
}

type Context interface {
	Query(Request) (ResponseSet, error)
}

type Host string

func (h Host) Query(r Request) (ResponseSet, error) {
	return r.Query(string(h))
}

type Cache struct {
	host  string
	cache map[string]*cacheResult
}

type cacheResult struct {
	ResponseSet
	Err error
}

func NewCache(host string) *Cache {
	return &Cache{
		host:  host,
		cache: make(map[string]*cacheResult),
	}
}

func (c *Cache) Query(r Request) (ResponseSet, error) {
	b, err := json.Marshal(&r)
	if err != nil {
		return nil, err
	}
	s := string(b)
	if v, ok := c.cache[s]; ok {
		log.Println("CACHE HIT", s)
		return v.ResponseSet, v.Err
	}
	log.Println("CACHE MISS", s)
	rs, e := r.Query(c.host)
	c.cache[s] = &cacheResult{rs, e}
	return rs, e
}
<commit_msg>cmd/scollector: Remove debugs<commit_after>package opentsdb

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"

	"github.com/StackExchange/slog"
)

var l = log.New(os.Stdout, "", log.LstdFlags)

type ResponseSet []*Response

type Point float64

type Response struct {
	Metric        string           `json:"metric"`
	Tags          TagSet           `json:"tags"`
	AggregateTags []string         `json:"aggregateTags"`
	DPS           map[string]Point `json:"dps"`
}

type DataPoint struct {
	Metric    string      `json:"metric"`
	Timestamp int64       `json:"timestamp"`
	Value     interface{} `json:"value"`
	Tags      TagSet      `json:"tags"`
}

func (d *DataPoint) Telnet() string {
	m := ""
	d.clean()
	for k, v := range d.Tags {
		m += fmt.Sprintf(" %s=%s", k, v)
	}
	return fmt.Sprintf("put %s %d %v%s\n", d.Metric, d.Timestamp, d.Value, m)
}

func (m MultiDataPoint) Json() ([]byte, error) {
	var md MultiDataPoint
	for _, d := range m {
		err := d.clean()
		if err != nil {
			slog.Infoln(err, "Removing Datapoint", d)
			continue
		}
		md = append(md, d)
	}
	return json.Marshal(md)
}

type MultiDataPoint []*DataPoint

type TagSet map[string]string

func (t TagSet) Equal(o TagSet) bool {
	if len(t) != len(o) {
		return false
	}
	for k, v := range t {
		if ov, ok := o[k]; !ok || ov != v {
			return false
		}
	}
	return true
}

// Subset returns true if all k=v pairs in o are in t.
func (t TagSet) Subset(o TagSet) bool {
	for k, v := range o {
		if tv, ok := t[k]; !ok || tv != v {
			return false
		}
	}
	return true
}

// String converts t to an OpenTSDB-style {a=b,c=b} string, alphabetized by key.
func (t TagSet) String() string {
	var keys []string
	for k := range t {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	b := bytes.NewBufferString("{")
	for i, k := range keys {
		if i > 0 {
			fmt.Fprint(b, ",")
		}
		fmt.Fprintf(b, "%s=%s", k, t[k])
	}
	fmt.Fprint(b, "}")
	return b.String()
}

func (d *DataPoint) clean() error {
	err := d.Tags.clean()
	if err != nil {
		return err
	}
	om := d.Metric
	d.Metric, err = Clean(d.Metric)
	if err != nil {
		return fmt.Errorf("%s. Orginal: [%s] Cleaned: [%s]", err.Error(), om, d.Metric)
	}
	if sv, ok := d.Value.(string); ok {
		if i, err := strconv.ParseInt(sv, 10, 64); err == nil {
			d.Value = i
		} else if f, err := strconv.ParseFloat(sv, 64); err == nil {
			d.Value = f
		} else {
			return fmt.Errorf("Unparseable number %v", sv)
		}
	}
	return nil
}

func (t TagSet) clean() error {
	for k, v := range t {
		kc, err := Clean(k)
		if err != nil {
			return fmt.Errorf("%s. Orginal: [%s] Cleaned: [%s]", err.Error(), k, kc)
		}
		vc, err := Clean(v)
		if err != nil {
			return fmt.Errorf("%s. Orginal: [%s] Cleaned: [%s]", err.Error(), v, vc)
		}
		delete(t, k)
		t[kc] = vc
	}
	return nil
}

// Clean removes characters from s that are invalid for OpenTSDB metric and tag
// values.
// See: http://opentsdb.net/docs/build/html/user_guide/writing.html#metrics-and-tags
func Clean(s string) (string, error) {
	var c string
	if len(s) == 0 {
		// This one is perhaps better checked earlier in the pipeline, but since
		// it makes sense to check that the resulting cleaned tag is not Zero length here I'm including it
		// It also might be the case that this just shouldn't be happening and this is yet another side
		// effect of WMI turning to Garbage....
		return s, errors.New("Metric/Tagk/Tagv Cleaning Passed a Zero Length String")
	}
	for len(s) > 0 {
		r, size := utf8.DecodeRuneInString(s)
		if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '-' || r == '_' || r == '.' || r == '/' {
			c += string(r)
		}
		s = s[size:]
	}

	if len(c) == 0 {
		return c, errors.New("Cleaning Metric/Tagk/Tagv resulted in a Zero Length String")
	}
	return c, nil
}

type Request struct {
	Start             interface{} `json:"start"`
	End               interface{} `json:"end,omitempty"`
	Queries           []*Query    `json:"queries"`
	NoAnnotations     bool        `json:"noAnnotations,omitempty"`
	GlobalAnnotations bool        `json:"globalAnnotations,omitempty"`
	MsResolution      bool        `json:"msResolution,omitempty"`
	ShowTSUIDs        bool        `json:"showTSUIDs,omitempty"`
}

type Query struct {
	Aggregator  string `json:"aggregator"`
	Metric      string `json:"metric"`
	Rate        bool   `json:"rate,omitempty"`
	RateOptions `json:"rateOptions,omitempty"`
	Downsample  string `json:"downsample,omitempty"`
	Tags        TagSet `json:"tags,omitempty"`
}

type RateOptions struct {
	Counter    bool `json:"counter,omitempty"`
	CounterMax int  `json:"counterMax,omitempty"`
	ResetValue int  `json:"resetValue,omitempty"`
}

// ParsesRequest parses OpenTSDB requests of the form: start=1h-ago&m=avg:cpu.
func ParseRequest(req string) (*Request, error) {
	v, err := url.ParseQuery(req)
	if err != nil {
		return nil, err
	}
	r := Request{}
	if s := v.Get("start"); s == "" {
		return nil, fmt.Errorf("tsdb: missing start: %s", req)
	} else {
		r.Start = s
	}
	for _, m := range v["m"] {
		q, err := ParseQuery(m)
		if err != nil {
			return nil, err
		}
		r.Queries = append(r.Queries, q)
	}
	if len(r.Queries) == 0 {
		return nil, fmt.Errorf("tsdb: missing m: %s", req)
	}
	return &r, nil
}

var qRE = regexp.MustCompile(`^(\w+):(?:(\w+-\w+):)?(?:(rate):)?([\w./]+)(?:\{([\w./,=*-|]+)\})?$`)

// ParseQuery parses OpenTSDB queries of the form: avg:rate:cpu{k=v}.
func ParseQuery(query string) (*Query, error) {
	q := Query{}
	m := qRE.FindStringSubmatch(query)
	if m == nil {
		return nil, fmt.Errorf("tsdb: bad query format: %s", query)
	}
	q.Aggregator = m[1]
	q.Downsample = m[2]
	q.Rate = m[3] == "rate"
	q.Metric = m[4]
	if m[5] != "" {
		tags, err := ParseTags(m[5])
		if err != nil {
			return nil, err
		}
		q.Tags = tags
	}
	return &q, nil
}

// ParseTags parses OpenTSDB tagk=tagv pairs of the form: k=v,m=o.
func ParseTags(t string) (TagSet, error) {
	ts := make(TagSet)
	for _, v := range strings.Split(t, ",") {
		sp := strings.SplitN(v, "=", 2)
		if len(sp) != 2 {
			return nil, fmt.Errorf("tsdb: bad tag: %s", v)
		}
		ts[sp[0]] = sp[1]
	}
	return ts, nil
}

func (q Query) String() string {
	s := q.Aggregator + ":"
	if q.Downsample != "" {
		s += q.Downsample + ":"
	}
	if q.Rate {
		s += "rate:"
	}
	s += q.Metric
	if len(q.Tags) > 0 {
		s += "{"
		first := true
		for k, v := range q.Tags {
			if first {
				first = false
			} else {
				s += ","
			}
			s += k + "=" + v
		}
		s += "}"
	}
	return s
}

func (r Request) String() string {
	v := make(url.Values)
	for _, q := range r.Queries {
		v.Add("m", q.String())
	}
	v.Add("start", fmt.Sprint(r.Start))
	if e := fmt.Sprint(r.End); r.End != nil && e != "" {
		v.Add("end", e)
	}
	return v.Encode()
}

type RequestError struct {
	Err struct {
		Code    int    `json:"code"`
		Message string `json:"message"`
		Details string `json:"details"`
	} `json:"error"`
}

func (r *RequestError) Error() string {
	return fmt.Sprintf("tsdb: %s", r.Err.Message)
}

// Query performs a v2 OpenTSDB request to the given host. host should be of the
// form hostname:port. Can return a RequestError.
func (r Request) Query(host string) (ResponseSet, error) {
	u := url.URL{
		Scheme: "http",
		Host:   host,
		Path:   "/api/query",
	}
	b, err := json.Marshal(&r)
	if err != nil {
		return nil, err
	}
	resp, err := http.Post(u.String(), "application/json", bytes.NewReader(b))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		b, err = ioutil.ReadAll(resp.Body)
		fmt.Println(string(b))
		e := RequestError{}
		if err := json.Unmarshal(b, &e); err == nil {
			return nil, &e
		}
		return nil, fmt.Errorf("tsdb: %s", b)
	}
	b, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var tr ResponseSet
	if err := json.Unmarshal(b, &tr); err != nil {
		return nil, err
	}
	return tr, nil
}

type Context interface {
	Query(Request) (ResponseSet, error)
}

type Host string

func (h Host) Query(r Request) (ResponseSet, error) {
	return r.Query(string(h))
}

type Cache struct {
	host  string
	cache map[string]*cacheResult
}

type cacheResult struct {
	ResponseSet
	Err error
}

func NewCache(host string) *Cache {
	return &Cache{
		host:  host,
		cache: make(map[string]*cacheResult),
	}
}

func (c *Cache) Query(r Request) (ResponseSet, error) {
	b, err := json.Marshal(&r)
	if err != nil {
		return nil, err
	}
	s := string(b)
	if v, ok := c.cache[s]; ok {
		return v.ResponseSet, v.Err
	}
	rs, e := r.Query(c.host)
	c.cache[s] = &cacheResult{rs, e}
	return rs, e
}
<|endoftext|>


<commit_before>package shell

import (
	"context"
	"flag"
	"io"

	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
)

func init() {
	Commands = append(Commands, &commandVacuum{})
}

type commandVacuum struct {
}

func (c *commandVacuum) Name() string {
	return "volume.vacuum"
}

func (c *commandVacuum) Help() string {
	return `compact volumes if deleted entries are more than the limit

	volume.vacuum [-garbageThreshold=0.3]

`
}

func (c *commandVacuum) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	volumeVacuumCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	garbageThreshold := volumeVacuumCommand.Float64("garbageThreshold", 0.3, "vacuum when garbage is more than this limit")
	if err = volumeVacuumCommand.Parse(args); err != nil {
		return nil
	}

	if err = commandEnv.confirmIsLocked(args); err != nil {
		return
	}

	err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
		_, err = client.VacuumVolume(context.Background(), &master_pb.VacuumVolumeRequest{
			GarbageThreshold: float32(*garbageThreshold),
		})
		return err
	})
	if err != nil {
		return
	}

	return nil
}
<commit_msg>fix a return bug in func (c *commandVacuum) Do<commit_after>package shell

import (
	"context"
	"flag"
	"io"

	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
)

func init() {
	Commands = append(Commands, &commandVacuum{})
}

type commandVacuum struct {
}

func (c *commandVacuum) Name() string {
	return "volume.vacuum"
}

func (c *commandVacuum) Help() string {
	return `compact volumes if deleted entries are more than the limit

	volume.vacuum [-garbageThreshold=0.3]

`
}

func (c *commandVacuum) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	volumeVacuumCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	garbageThreshold := volumeVacuumCommand.Float64("garbageThreshold", 0.3, "vacuum when garbage is more than this limit")
	if err = volumeVacuumCommand.Parse(args); err != nil {
		return
	}

	if err = commandEnv.confirmIsLocked(args); err != nil {
		return
	}

	err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
		_, err = client.VacuumVolume(context.Background(), &master_pb.VacuumVolumeRequest{
			GarbageThreshold: float32(*garbageThreshold),
		})
		return err
	})
	if err != nil {
		return
	}

	return nil
}
<|endoftext|>


<commit_before>package mysql

import (
	"database/sql"
	"fmt"

	"github.com/elwinar/rambler/driver"
	_ "github.com/go-sql-driver/mysql" // Where are working with the go-sql-driver/mysql driver for database/sql.
)

func init() {
	driver.Register("mysql", Driver{})
}

// Driver is the type that initialize new connections.
type Driver struct{}

func (d Driver) New(dsn, schema, table string) (driver.Conn, error) {
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		return nil, err
	}

	return Conn{
		db:     db,
		schema: schema,
		table:  table,
	}, nil
}

// Conn holds a connection to a MySQL database schema.
type Conn struct {
	db     *sql.DB
	schema string
	table  string
}

// HasTable check if the schema has the migration table needed for Rambler to operate on it.
func (c Conn) HasTable() (bool, error) {
	var table string
	err := c.db.QueryRow(`select table_name from information_schema.tables where table_schema = ? and table_name = ?`, c.schema, c.table).Scan(&table)
	if err != nil && err != sql.ErrNoRows {
		return false, err
	}

	if err == sql.ErrNoRows {
		return false, nil
	}

	return true, nil
}

// CreateTable create the migration table using a MySQL-compatible syntax.
func (c Conn) CreateTable() error {
	_, err := c.db.Exec(fmt.Sprintf(`CREATE TABLE %s ( migration VARCHAR(255) NOT NULL ) DEFAULT CHARSET=utf8`, c.table))
	return err
}

// GetApplied returns the list of already applied migrations.
func (c Conn) GetApplied() ([]string, error) {
	rows, err := c.db.Query(fmt.Sprintf(`SELECT migration FROM %s ORDER BY migration ASC`, c.table))
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var migrations []string
	for rows.Next() {
		var migration string
		err := rows.Scan(&migration)
		if err != nil {
			return nil, err
		}

		migrations = append(migrations, migration)
	}

	return migrations, nil
}

// AddApplied record that a migration was applied.
func (c Conn) AddApplied(migration string) error {
	_, err := c.db.Exec(fmt.Sprintf(`INSERT INTO %s (migration) VALUES (?)`, c.table), migration)
	return err
}

// RemoveApplied record that a migration was reversed.
func (c Conn) RemoveApplied(migration string) error {
	_, err := c.db.Exec(fmt.Sprintf(`DELETE FROM %s WHERE migration = ?`, c.table), migration)
	return err
}

// Execute run a statement on the schema.
func (c Conn) Execute(statement string) error {
	_, err := c.db.Exec(statement)
	return err
}
<commit_msg>The column migration in the migrations table is now the primary key. This is necessary to avoid problems with Percona Xtra DB clusters<commit_after>package mysql

import (
	"database/sql"
	"fmt"

	"github.com/elwinar/rambler/driver"
	_ "github.com/go-sql-driver/mysql" // Where are working with the go-sql-driver/mysql driver for database/sql.
)

func init() {
	driver.Register("mysql", Driver{})
}

// Driver is the type that initialize new connections.
type Driver struct{}

func (d Driver) New(dsn, schema, table string) (driver.Conn, error) {
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		return nil, err
	}

	return Conn{
		db:     db,
		schema: schema,
		table:  table,
	}, nil
}

// Conn holds a connection to a MySQL database schema.
type Conn struct {
	db     *sql.DB
	schema string
	table  string
}

// HasTable check if the schema has the migration table needed for Rambler to operate on it.
func (c Conn) HasTable() (bool, error) {
	var table string
	err := c.db.QueryRow(`select table_name from information_schema.tables where table_schema = ? and table_name = ?`, c.schema, c.table).Scan(&table)
	if err != nil && err != sql.ErrNoRows {
		return false, err
	}

	if err == sql.ErrNoRows {
		return false, nil
	}

	return true, nil
}

// CreateTable create the migration table using a MySQL-compatible syntax.
func (c Conn) CreateTable() error {
	_, err := c.db.Exec(fmt.Sprintf(`CREATE TABLE %s ( migration VARCHAR(255) NOT NULL, PRIMARY KEY(migration) ) DEFAULT CHARSET=utf8`, c.table))
	return err
}

// GetApplied returns the list of already applied migrations.
func (c Conn) GetApplied() ([]string, error) {
	rows, err := c.db.Query(fmt.Sprintf(`SELECT migration FROM %s ORDER BY migration ASC`, c.table))
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var migrations []string
	for rows.Next() {
		var migration string
		err := rows.Scan(&migration)
		if err != nil {
			return nil, err
		}

		migrations = append(migrations, migration)
	}

	return migrations, nil
}

// AddApplied record that a migration was applied.
func (c Conn) AddApplied(migration string) error {
	_, err := c.db.Exec(fmt.Sprintf(`INSERT INTO %s (migration) VALUES (?)`, c.table), migration)
	return err
}

// RemoveApplied record that a migration was reversed.
func (c Conn) RemoveApplied(migration string) error {
	_, err := c.db.Exec(fmt.Sprintf(`DELETE FROM %s WHERE migration = ?`, c.table), migration)
	return err
}

// Execute run a statement on the schema.
func (c Conn) Execute(statement string) error {
	_, err := c.db.Exec(statement)
	return err
}
<|endoftext|>


<commit_before>package vcfgo

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/brentp/irelate/interfaces"
)

// Variant holds the information about a single site. It is analagous to a row in a VCF file.
type Variant struct {
	Chromosome string
	Pos        uint64
	Id_        string
	Reference  string
	Alternate  []string
	Quality    float32
	Filter     string
	Info_      interfaces.Info
	Format     []string
	Samples    []*SampleGenotype
	// if lazy parsing, then just save the sample strings here.
	sampleString string
	Header       *Header
	LineNumber   int64
}

func (v *Variant) Info() interfaces.Info {
	return v.Info_
}

func (v *Variant) Id() string {
	return v.Id_
}

func (v *Variant) Ref() string {
	return v.Reference
}

func (v *Variant) Alt() []string {
	return v.Alternate
}

// Chrom returns the chromosome name.
func (v *Variant) Chrom() string {
	return v.Chromosome
}

// Start returns the 0-based start
func (v *Variant) Start() uint32 {
	return uint32(v.Pos - 1)
}

// CIPos reports the Left and Right end of an SV using the CIPOS tag. It is in
// bed format so the end is +1'ed. E.g. If there is not CIPOS, the return value
// is v.Start(), v.Start() + 1
func (v *Variant) CIPos() (uint32, uint32, bool) {
	if v.Header == nil {
		v.Header = NewHeader()
	}
	if _, ok := v.Header.Infos["CIPOS"]; !ok {
		v.Header.Infos["CIPOS"] = &Info{Id: "CIPOS", Number: "2", Description: "CIPOS", Type: "Integer"}
	}
	s := v.Start()
	ipair, err := v.Info().Get("CIPOS")
	if ipair == nil && err != nil {
		return s, s + 1, false
	}
	pair, ok := ipair.([]int)
	if !ok {
		return s, s + 1, false
	}
	left := pair[0]
	right := pair[1]
	return uint32(int(s) + left), uint32(int(s) + right + 1), true
}

// CIEnd reports the Left and Right end of an SV using the CIEND tag. It is in
// bed format so the end is +1'ed. E.g. If there is no CIEND, the return value
// is v.End() - 1, v.End()
func (v *Variant) CIEnd() (uint32, uint32, bool) {
	if v.Header == nil {
		v.Header = NewHeader()
	}
	if _, ok := v.Header.Infos["CIEND"]; !ok {
		v.Header.Lock()
		v.Header.Infos["CIEND"] = &Info{Id: "CIEND", Number: "2", Description: "CIEND", Type: "Integer"}
		v.Header.Unlock()
	}
	e := v.End()
	ipair, err := v.Info().Get("CIEND")
	if ipair == nil && err != nil {
		return e - 1, e, false
	}
	pair, ok := ipair.([]int)
	if !ok {
		return e - 1, e, false
	}
	left := pair[0]
	right := pair[1]
	return uint32(int(e)+left) - 1, uint32(int(e) + right), true
}

// End returns the 0-based start + the length of the reference allele.
func (v *Variant) End() uint32 {
	/*
		if len(v.Alt()[0]) == 0 {
			log.Println(v.Chrom(), v.Start()+1, v.Ref(), v.Alt())
		}*/
	a := v.Alt()[0]
	if len(a) == 0 || a[0] != '<' {
		return uint32(v.Pos-1) + uint32(len(v.Ref()))
	}
	if strings.HasPrefix(v.Alt()[0], "<DEL") || strings.HasPrefix(v.Alt()[0], "<DUP") || strings.HasPrefix(v.Alt()[0], "<INV") || strings.HasPrefix(v.Alt()[0], "<CN") {
		if svlen, err := v.Info().Get("SVLEN"); err == nil || strings.Contains(err.Error(), "not found in header") {
			var slen int
			err = nil
			switch svlen.(type) {
			case int:
				slen = svlen.(int)
			case string:
				var e error
				if svlen.(string) == "" {
					return uint32(v.Pos)
				}
				slen, e = strconv.Atoi(svlen.(string))
				if e != nil {
					log.Fatalf("bad value for svlen: %s\n", svlen)
				}
			case []interface{}:
				slen = svlen.([]interface{})[0].(int)
			case interface{}:
				slen = svlen.(int)
			default:
				log.Fatalf("non int type for SVLEN")
			}
			if slen < 0 {
				slen = -slen
			}
			return uint32(int(v.Pos) + slen)

		} else if end, err := v.Info().Get("END"); err == nil {
			return uint32(end.(int))
		}
		log.Printf("no svlen for variant %s:%d\n%s\nUsing %d", v.Chromosome, v.Pos, v, v.Pos+1)
	}
	// <INS and BND's get handled by this.
	return uint32(v.Pos-1) + uint32(len(v.Ref()))
}

func fmtFloat32(v float32) string {
	var val string
	if v > 0.02 || v < -0.02 {
		val = fmt.Sprintf("%.4f", v)
	} else {
		val = fmt.Sprintf("%.5g", v)
	}
	val = strings.TrimRight(strings.TrimRight(val, "0"), ".")
	if val == "" {
		val = "0"
	}
	return val
}

func fmtFloat64(v float64) string {
	var val string
	if v > 0.02 || v < -0.02 {
		val = fmt.Sprintf("%.4f", v)
	} else {
		val = fmt.Sprintf("%.5g", v)
	}
	val = strings.TrimRight(strings.TrimRight(val, "0"), ".")
	if val == "" {
		val = "0"
	}
	return val
}

// SampleGenotype holds the information about a sample. Several fields are pre-parsed, but
// all fields are kept in Fields as well.
type SampleGenotype struct {
	Phased bool
	GT     []int
	DP     int
	GL     []float64
	GQ     int
	MQ     int
	Fields map[string]string
}

// RefDepths returns the depths of the alternates for this sample
func (s *SampleGenotype) RefDepth() (int, error) {
	if ad, ok := s.Fields["AD"]; ok {
		idx := strings.Index(ad, ",")
		return strconv.Atoi(ad[:idx])
	}
	if ro, ok := s.Fields["RO"]; ok {
		return strconv.Atoi(ro)
	}
	return 0, errors.New("only freebayes and gatk depths supported at this time")
}

// AltDepths returns the depths of the alternates for this sample
func (s *SampleGenotype) AltDepths() ([]int, error) {
	var svals []string
	if ad, ok := s.Fields["AD"]; ok {
		idx := strings.Index(ad, ",")
		svals = strings.Split(ad[idx+1:], ",")
	} else if ro, ok := s.Fields["AO"]; ok {
		svals = strings.Split(ro, ",")
	} else {
		return []int{}, errors.New("only freebayes and GATK supported for ref/alt depths")
	}
	vals := make([]int, len(svals))
	for i := range svals {
		v, err := strconv.Atoi(svals[i])
		if err != nil {
			return []int{}, err
		}
		vals[i] = v
	}
	return vals, nil
}

// String returns the string representation of the sample field.
func (sg *SampleGenotype) String(fields []string) string {
	if len(fields) == 0 {
		return "."
	}

	s := make([]string, len(fields))
	for i, f := range fields {
		s[i] = sg.Fields[f]
	}
	return strings.Join(s, ":")
}

// NewSampleGenotype allocates the internals and returns a *SampleGenotype
func NewSampleGenotype() *SampleGenotype {
	s := &SampleGenotype{}
	s.GT = make([]int, 0, 2)
	s.GL = make([]float64, 0, 3)
	s.Fields = make(map[string]string)
	return s
}

// String gives a string representation of a variant
func (v *Variant) String() string {
	var qual string
	if v.Quality == MISSING_VAL {
		qual = "."
	} else {
		qual = fmt.Sprintf("%.1f", v.Quality)
	}
	s := fmt.Sprintf("%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s", v.Chromosome, v.Pos, v.Id_, v.Ref(), strings.Join(v.Alt(), ","), qual, v.Filter, v.Info())
	if len(v.Samples) > 0 {
		samps := make([]string, len(v.Samples))
		for i, s := range v.Samples {
			samps[i] = s.String(v.Format)
		}
		s += fmt.Sprintf("\t%s\t%s", strings.Join(v.Format, ":"), strings.Join(samps, "\t"))
	} else if v.sampleString != "" {
		s += fmt.Sprintf("\t%s\t%s", strings.Join(v.Format, ":"), v.sampleString)
	}
	return s
}

// GetGenotypeField uses the information from the header to parse the correct time from a genotype field.
// It returns an interface that can be asserted to the expected type.
func (v *Variant) GetGenotypeField(g *SampleGenotype, field string, missing interface{}) (interface{}, error) {
	if g == nil {
		return missing, fmt.Errorf("GetGenotypeField: empty genotype when requesting %s", field)
	}
	h := v.Header
	format, ok := h.SampleFormats[field]
	if !ok {
		return nil, fmt.Errorf("GetGenotypeField: field not found in formats: %s", field)
	}
	value, ok := g.Fields[field]
	if !ok {
		return nil, fmt.Errorf("GetGenotypeField: field not found in genotypes: %s", field)
	}
	switch format.Type {
	case "Integer":
		var mv int
		var ok bool
		if mv, ok = missing.(int); !ok {
			return nil, fmt.Errorf("GetGenotypeField: bad non-int missing value: %v", missing)
		}
		return handleNumberType(format.Number, value, len(v.Alt()), len(g.GT), true, mv)

	case "Float":
		var mv float32
		var ok bool
		if mv, ok = missing.(float32); !ok {
			return nil, fmt.Errorf("GetGenotypeField: bad non-float missing value: %v", missing)
		}
		return handleNumberType(format.Number, value, len(v.Alt()), len(g.GT), false, mv)

	case "String", "Character", "Unknown":
		return value, nil

	case "Flag":
		return field, nil

	}

	return nil, fmt.Errorf("unknown format: %s", format.Type)
}

func handleNumberType(number string, value string, nAlts int, nGTs int, isInt bool, mv interface{}) (interface{}, error) {
	if number == "1" || !strings.Contains(value, ",") || number == "." || number == "" {
		if isInt {
			if value == "" || value == "." {
				return (mv).(int), nil
			}
			return strconv.Atoi(value)
		}
		if value == "" || value == "." {
			return (mv).(float32), nil
		}
		return strconv.ParseFloat(value, 32)
	}
	if count, err := strconv.Atoi(number); err == nil || number == "G" || number == "A" || number == "R" {
		if err != nil {
			switch number {
			case "G":
				count = nGTs * (nGTs + 1) / 2
			case "A":
				count = nAlts
			case "R":
				count = nAlts + 1
			}
			err = nil
		}
		var ret interface{}
		split := strings.Split(value, ",")
		if isInt {
			ret = make([]int, len(split), len(split))
		} else {
			ret = make([]float32, len(split), len(split))
		}

		var countErr error

		// caller can ignore error if they want, we still fill what we can.
		if len(split) != count {
			countErr = fmt.Errorf("number of fields (%d) does not match expected (%d) in '%s'", len(split), count, value)
		}
		for i, s := range split {
			if isInt {
				ri, err := strconv.Atoi(s) //, 10, 32)
				if err != nil {
					// if it's an error, we allow empty
					if s == "" || s == "." {
						ret.([]int)[i] = mv.(int)
						err = nil
					} else {
						return nil, fmt.Errorf("non integer type: %s", s)
					}
				} else {
					ret.([]int)[i] = int(ri)
				}
			} else {
				rf, err := strconv.ParseFloat(s, 32)
				if err != nil {
					// if it's an error, we allow empty
					if s == "" || s == "." {
						ret.([]float32)[i] = mv.(float32)
						err = nil
					} else {
						return nil, fmt.Errorf("non float type: %s", s)
					}
				} else {
					ret.([]float32)[i] = float32(rf)
				}
			}
		}
		return ret, countErr
	} else if number == "." || number == "" {
		return value, nil
	} else {
		return nil, fmt.Errorf("unknown number field: %s", number)
	}
}
<commit_msg>fix race condition with CIPos/CIEnd<commit_after>package vcfgo

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/brentp/irelate/interfaces"
)

// Variant holds the information about a single site. It is analagous to a row in a VCF file.
type Variant struct {
	Chromosome string
	Pos        uint64
	Id_        string
	Reference  string
	Alternate  []string
	Quality    float32
	Filter     string
	Info_      interfaces.Info
	Format     []string
	Samples    []*SampleGenotype
	// if lazy parsing, then just save the sample strings here.
	sampleString string
	Header       *Header
	LineNumber   int64
}

func (v *Variant) Info() interfaces.Info {
	return v.Info_
}

func (v *Variant) Id() string {
	return v.Id_
}

func (v *Variant) Ref() string {
	return v.Reference
}

func (v *Variant) Alt() []string {
	return v.Alternate
}

// Chrom returns the chromosome name.
func (v *Variant) Chrom() string {
	return v.Chromosome
}

// Start returns the 0-based start
func (v *Variant) Start() uint32 {
	return uint32(v.Pos - 1)
}

// CIPos reports the Left and Right end of an SV using the CIPOS tag. It is in
// bed format so the end is +1'ed. E.g. If there is not CIPOS, the return value
// is v.Start(), v.Start() + 1
func (v *Variant) CIPos() (uint32, uint32, bool) {
	if v.Header == nil {
		v.Header = NewHeader()
	}
	s := v.Start()
	v.Header.RLock()
	if _, ok := v.Header.Infos["CIPOS"]; !ok {
		v.Header.RUnlock()
		v.Header.Lock()
		v.Header.Infos["CIPOS"] = &Info{Id: "CIPOS", Number: "2", Description: "CIPOS", Type: "Integer"}
		v.Header.Unlock()
		v.Header.RLock()
	}
	ipair, err := v.Info().Get("CIPOS")
	v.Header.RUnlock()
	if ipair == nil && err != nil {
		return s, s + 1, false
	}
	pair, ok := ipair.([]int)
	if !ok {
		return s, s + 1, false
	}
	left := pair[0]
	right := pair[1]
	return uint32(int(s) + left), uint32(int(s) + right + 1), true
}

// CIEnd reports the Left and Right end of an SV using the CIEND tag. It is in
// bed format so the end is +1'ed. E.g. If there is no CIEND, the return value
// is v.End() - 1, v.End()
func (v *Variant) CIEnd() (uint32, uint32, bool) {
	if v.Header == nil {
		v.Header = NewHeader()
	}
	e := v.End()
	v.Header.RLock()
	if _, ok := v.Header.Infos["CIEND"]; !ok {
		v.Header.RUnlock()
		v.Header.Lock()
		v.Header.Infos["CIEND"] = &Info{Id: "CIEND", Number: "2", Description: "CIEND", Type: "Integer"}
		v.Header.Unlock()
		v.Header.RLock()
	}
	ipair, err := v.Info().Get("CIEND")
	v.Header.RUnlock()
	if ipair == nil && err != nil {
		return e - 1, e, false
	}
	pair, ok := ipair.([]int)
	if !ok {
		return e - 1, e, false
	}
	left := pair[0]
	right := pair[1]
	return uint32(int(e)+left) - 1, uint32(int(e) + right), true
}

// End returns the 0-based start + the length of the reference allele.
func (v *Variant) End() uint32 {
	/*
		if len(v.Alt()[0]) == 0 {
			log.Println(v.Chrom(), v.Start()+1, v.Ref(), v.Alt())
		}*/
	a := v.Alt()[0]
	if len(a) == 0 || a[0] != '<' {
		return uint32(v.Pos-1) + uint32(len(v.Ref()))
	}
	if strings.HasPrefix(v.Alt()[0], "<DEL") || strings.HasPrefix(v.Alt()[0], "<DUP") || strings.HasPrefix(v.Alt()[0], "<INV") || strings.HasPrefix(v.Alt()[0], "<CN") {
		if svlen, err := v.Info().Get("SVLEN"); err == nil || strings.Contains(err.Error(), "not found in header") {
			var slen int
			err = nil
			switch svlen.(type) {
			case int:
				slen = svlen.(int)
			case string:
				var e error
				if svlen.(string) == "" {
					return uint32(v.Pos)
				}
				slen, e = strconv.Atoi(svlen.(string))
				if e != nil {
					log.Fatalf("bad value for svlen: %s\n", svlen)
				}
			case []interface{}:
				slen = svlen.([]interface{})[0].(int)
			case interface{}:
				slen = svlen.(int)
			default:
				log.Fatalf("non int type for SVLEN")
			}
			if slen < 0 {
				slen = -slen
			}
			return uint32(int(v.Pos) + slen)

		} else if end, err := v.Info().Get("END"); err == nil {
			return uint32(end.(int))
		}
		log.Printf("no svlen for variant %s:%d\n%s\nUsing %d", v.Chromosome, v.Pos, v, v.Pos+1)
	}
	// <INS and BND's get handled by this.
	return uint32(v.Pos-1) + uint32(len(v.Ref()))
}

func fmtFloat32(v float32) string {
	var val string
	if v > 0.02 || v < -0.02 {
		val = fmt.Sprintf("%.4f", v)
	} else {
		val = fmt.Sprintf("%.5g", v)
	}
	val = strings.TrimRight(strings.TrimRight(val, "0"), ".")
	if val == "" {
		val = "0"
	}
	return val
}

func fmtFloat64(v float64) string {
	var val string
	if v > 0.02 || v < -0.02 {
		val = fmt.Sprintf("%.4f", v)
	} else {
		val = fmt.Sprintf("%.5g", v)
	}
	val = strings.TrimRight(strings.TrimRight(val, "0"), ".")
	if val == "" {
		val = "0"
	}
	return val
}

// SampleGenotype holds the information about a sample. Several fields are pre-parsed, but
// all fields are kept in Fields as well.
type SampleGenotype struct {
	Phased bool
	GT     []int
	DP     int
	GL     []float64
	GQ     int
	MQ     int
	Fields map[string]string
}

// RefDepths returns the depths of the alternates for this sample
func (s *SampleGenotype) RefDepth() (int, error) {
	if ad, ok := s.Fields["AD"]; ok {
		idx := strings.Index(ad, ",")
		return strconv.Atoi(ad[:idx])
	}
	if ro, ok := s.Fields["RO"]; ok {
		return strconv.Atoi(ro)
	}
	return 0, errors.New("only freebayes and gatk depths supported at this time")
}

// AltDepths returns the depths of the alternates for this sample
func (s *SampleGenotype) AltDepths() ([]int, error) {
	var svals []string
	if ad, ok := s.Fields["AD"]; ok {
		idx := strings.Index(ad, ",")
		svals = strings.Split(ad[idx+1:], ",")
	} else if ro, ok := s.Fields["AO"]; ok {
		svals = strings.Split(ro, ",")
	} else {
		return []int{}, errors.New("only freebayes and GATK supported for ref/alt depths")
	}
	vals := make([]int, len(svals))
	for i := range svals {
		v, err := strconv.Atoi(svals[i])
		if err != nil {
			return []int{}, err
		}
		vals[i] = v
	}
	return vals, nil
}

// String returns the string representation of the sample field.
func (sg *SampleGenotype) String(fields []string) string {
	if len(fields) == 0 {
		return "."
	}

	s := make([]string, len(fields))
	for i, f := range fields {
		s[i] = sg.Fields[f]
	}
	return strings.Join(s, ":")
}

// NewSampleGenotype allocates the internals and returns a *SampleGenotype
func NewSampleGenotype() *SampleGenotype {
	s := &SampleGenotype{}
	s.GT = make([]int, 0, 2)
	s.GL = make([]float64, 0, 3)
	s.Fields = make(map[string]string)
	return s
}

// String gives a string representation of a variant
func (v *Variant) String() string {
	var qual string
	if v.Quality == MISSING_VAL {
		qual = "."
	} else {
		qual = fmt.Sprintf("%.1f", v.Quality)
	}
	s := fmt.Sprintf("%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s", v.Chromosome, v.Pos, v.Id_, v.Ref(), strings.Join(v.Alt(), ","), qual, v.Filter, v.Info())
	if len(v.Samples) > 0 {
		samps := make([]string, len(v.Samples))
		for i, s := range v.Samples {
			samps[i] = s.String(v.Format)
		}
		s += fmt.Sprintf("\t%s\t%s", strings.Join(v.Format, ":"), strings.Join(samps, "\t"))
	} else if v.sampleString != "" {
		s += fmt.Sprintf("\t%s\t%s", strings.Join(v.Format, ":"), v.sampleString)
	}
	return s
}

// GetGenotypeField uses the information from the header to parse the correct time from a genotype field.
// It returns an interface that can be asserted to the expected type.
func (v *Variant) GetGenotypeField(g *SampleGenotype, field string, missing interface{}) (interface{}, error) {
	if g == nil {
		return missing, fmt.Errorf("GetGenotypeField: empty genotype when requesting %s", field)
	}
	h := v.Header
	format, ok := h.SampleFormats[field]
	if !ok {
		return nil, fmt.Errorf("GetGenotypeField: field not found in formats: %s", field)
	}
	value, ok := g.Fields[field]
	if !ok {
		return nil, fmt.Errorf("GetGenotypeField: field not found in genotypes: %s", field)
	}
	switch format.Type {
	case "Integer":
		var mv int
		var ok bool
		if mv, ok = missing.(int); !ok {
			return nil, fmt.Errorf("GetGenotypeField: bad non-int missing value: %v", missing)
		}
		return handleNumberType(format.Number, value, len(v.Alt()), len(g.GT), true, mv)

	case "Float":
		var mv float32
		var ok bool
		if mv, ok = missing.(float32); !ok {
			return nil, fmt.Errorf("GetGenotypeField: bad non-float missing value: %v", missing)
		}
		return handleNumberType(format.Number, value, len(v.Alt()), len(g.GT), false, mv)

	case "String", "Character", "Unknown":
		return value, nil

	case "Flag":
		return field, nil

	}

	return nil, fmt.Errorf("unknown format: %s", format.Type)
}

func handleNumberType(number string, value string, nAlts int, nGTs int, isInt bool, mv interface{}) (interface{}, error) {
	if number == "1" || !strings.Contains(value, ",") || number == "." || number == "" {
		if isInt {
			if value == "" || value == "." {
				return (mv).(int), nil
			}
			return strconv.Atoi(value)
		}
		if value == "" || value == "." {
			return (mv).(float32), nil
		}
		return strconv.ParseFloat(value, 32)
	}
	if count, err := strconv.Atoi(number); err == nil || number == "G" || number == "A" || number == "R" {
		if err != nil {
			switch number {
			case "G":
				count = nGTs * (nGTs + 1) / 2
			case "A":
				count = nAlts
			case "R":
				count = nAlts + 1
			}
			err = nil
		}
		var ret interface{}
		split := strings.Split(value, ",")
		if isInt {
			ret = make([]int, len(split), len(split))
		} else {
			ret = make([]float32, len(split), len(split))
		}

		var countErr error

		// caller can ignore error if they want, we still fill what we can.
		if len(split) != count {
			countErr = fmt.Errorf("number of fields (%d) does not match expected (%d) in '%s'", len(split), count, value)
		}
		for i, s := range split {
			if isInt {
				ri, err := strconv.Atoi(s) //, 10, 32)
				if err != nil {
					// if it's an error, we allow empty
					if s == "" || s == "." {
						ret.([]int)[i] = mv.(int)
						err = nil
					} else {
						return nil, fmt.Errorf("non integer type: %s", s)
					}
				} else {
					ret.([]int)[i] = int(ri)
				}
			} else {
				rf, err := strconv.ParseFloat(s, 32)
				if err != nil {
					// if it's an error, we allow empty
					if s == "" || s == "." {
						ret.([]float32)[i] = mv.(float32)
						err = nil
					} else {
						return nil, fmt.Errorf("non float type: %s", s)
					}
				} else {
					ret.([]float32)[i] = float32(rf)
				}
			}
		}
		return ret, countErr
	} else if number == "." || number == "" {
		return value, nil
	} else {
		return nil, fmt.Errorf("unknown number field: %s", number)
	}
}
<|endoftext|>


<commit_before>// vcfanno is a command-line application and an api for annotating intervals (bed or vcf).
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	//_ "net/http/pprof"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/brentp/irelate"
	"github.com/brentp/irelate/interfaces"
	"github.com/brentp/irelate/parsers"
	. "github.com/brentp/vcfanno/api"
	. "github.com/brentp/vcfanno/shared"
	"github.com/brentp/vcfgo"
	"github.com/brentp/xopen"
)

var VERSION = "0.2.1-dev"

func envGet(name string, vdefault int) int {
	sval := os.Getenv(name)
	var err error
	if sval != "" {
		vdefault, err = strconv.Atoi(sval)
		if err != nil {
			log.Printf("couldn't parse %s using %d\n", name, vdefault)
		} else {
			log.Printf("using %s of %d\n", name, vdefault)
		}
	}
	return vdefault
}

func init() {
	log.SetFlags(log.Lshortfile)
}

func main() {
	fmt.Fprintf(os.Stderr, `
=============================================
vcfanno version %s [built with %s]

see: https://github.com/brentp/vcfanno
=============================================
`, VERSION, runtime.Version())

	ends := flag.Bool("ends", false, "annotate the start and end as well as the interval itself.")
	notstrict := flag.Bool("permissive-overlap", false, "annotate with an overlapping variant even it doesn't"+
		" share the same ref and alt alleles. Default is to require exact match between variants.")
	lua := flag.String("lua", "", "optional path to a file containing custom javascript functions to be used as ops")
	base := flag.String("base-path", "", "optional base-path to prepend to annotation files in the config")
	procs := flag.Int("p", 2, "number of processes to use.")
	flag.Parse()
	inFiles := flag.Args()
	if len(inFiles) != 2 {
		fmt.Printf(`Usage:
%s config.toml input.vcf > annotated.vcf

`, os.Args[0])
		flag.PrintDefaults()
		return
	}
	queryFile := inFiles[1]
	if !(xopen.Exists(queryFile) || queryFile == "") {
		fmt.Fprintf(os.Stderr, "\nERROR: can't find query file: %s\n", queryFile)
		os.Exit(2)
	}
	runtime.GOMAXPROCS(*procs)

	var config Config
	if _, err := toml.DecodeFile(inFiles[0], &config); err != nil {
		if strings.Contains(err.Error(), "Expected value but found") {
			fmt.Fprintln(os.Stderr, "\nNOTE: you must quote values in the conf file, e.g. fields=['AC', 'AN'] instead of fields=[AC, AN]")
		}
		panic(err)
	}
	config.Base = *base
	for _, a := range config.Annotation {
		err := CheckAnno(&a)
		if err != nil {
			log.Fatal("CheckAnno err:", err)
		}
		for _, op := range a.Ops {
			if len(op) > 4 && op[:4] == "lua:" && *lua == "" {
				log.Fatal("ERROR: requested lua op without specifying -lua flag")
			}
		}
	}
	for i := range config.PostAnnotation {
		r := config.PostAnnotation[i]
		err := CheckPostAnno(&r)
		if err != nil {
			log.Fatal(fmt.Sprintf("error in postannotaion section %s err: %s", r.Name, err))
		}
		if len(r.Op) > 4 && r.Op[:4] == "lua:" && *lua == "" {
			log.Fatal("ERROR: requested lua op without specifying -lua flag")
		}
	}

	sources, e := config.Sources()
	if e != nil {
		log.Fatal(e)
	}

	log.Printf("found %d sources from %d files\n", len(sources), len(config.Annotation))
	/*
		go func() {
			log.Println(http.ListenAndServe("localhost:6060", nil))
		}()
	*/

	luaString := ReadLua(*lua)
	strict := !*notstrict
	var a = NewAnnotator(sources, luaString, *ends, strict, config.PostAnnotation)

	var out io.Writer = os.Stdout
	defer os.Stdout.Close()

	var err error
	qrdr, err := xopen.Ropen(queryFile)
	if err != nil {
		log.Fatal(fmt.Errorf("error opening query file %s: %s", queryFile, err))
	}
	qstream, query, err := parsers.VCFIterator(qrdr)
	if err != nil {
		log.Fatal(fmt.Errorf("error parsing VCF query file %s: %s", queryFile, err))
	}

	queryables, err := a.Setup(query)
	if err != nil {
		log.Fatal(err)
	}
	aends := INTERVAL
	if *ends {
		aends = BOTH
	}

	lastMsg := struct {
		sync.RWMutex
		s [10]string
		i int
	}{}

	fn := func(v interfaces.Relatable) {
		e := a.AnnotateEnds(v, aends)
		if e != nil {
			lastMsg.RLock()
			em := e.Error()
			found := false
			for i := len(lastMsg.s) - 1; i >= 0; i-- {
				if em == lastMsg.s[i] {
					found = true
					break
				}
			}
			if !found {
				log.Println(e, ">> this error/warning may occur many times. reporting once here...")
				lastMsg.RUnlock()
				lastMsg.Lock()
				lastMsg.s[lastMsg.i] = em
				if lastMsg.i == len(lastMsg.s)-1 {
					lastMsg.i = -1
				}
				lastMsg.i++

				lastMsg.Unlock()
			} else {
				lastMsg.RUnlock()
			}
		}
	}

	maxGap := envGet("IRELATE_MAX_GAP", 20000)
	maxChunk := envGet("IRELATE_MAX_CHUNK", 8000)

	// make a new writer from the string header.
	out, err = vcfgo.NewWriter(out, query.Header)

	stream := irelate.PIRelate(maxChunk, maxGap, qstream, *ends, fn, queryables...)

	if err != nil {
		log.Fatal(err)
	}

	start := time.Now()
	n := 0

	/*
		if os.Getenv("IRELATE_PROFILE") == "TRUE" {
			log.Println("profiling to: irelate.pprof")
			f, err := os.Create("irelate.pprof")
			if err != nil {
				panic(err)
			}
			pprof.StartCPUProfile(f)
			defer pprof.StopCPUProfile()
		}
	*/

	for interval := range stream {
		//log.Printf("%v\n", interval)
		fmt.Fprintln(out, interval)
		n++
	}
	printTime(start, n)
}

func printTime(start time.Time, n int) {
	dur := time.Since(start)
	duri, duru := dur.Seconds(), "second"
	log.Printf("annotated %d variants in %.2f %ss (%.1f / %s)", n, duri, duru, float64(n)/duri, duru)
}
<commit_msg>make new version<commit_after>// vcfanno is a command-line application and an api for annotating intervals (bed or vcf).
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	//_ "net/http/pprof"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/brentp/irelate"
	"github.com/brentp/irelate/interfaces"
	"github.com/brentp/irelate/parsers"
	. "github.com/brentp/vcfanno/api"
	. "github.com/brentp/vcfanno/shared"
	"github.com/brentp/vcfgo"
	"github.com/brentp/xopen"
)

var VERSION = "0.2.1"

func envGet(name string, vdefault int) int {
	sval := os.Getenv(name)
	var err error
	if sval != "" {
		vdefault, err = strconv.Atoi(sval)
		if err != nil {
			log.Printf("couldn't parse %s using %d\n", name, vdefault)
		} else {
			log.Printf("using %s of %d\n", name, vdefault)
		}
	}
	return vdefault
}

func init() {
	log.SetFlags(log.Lshortfile)
}

func main() {
	fmt.Fprintf(os.Stderr, `
=============================================
vcfanno version %s [built with %s]

see: https://github.com/brentp/vcfanno
=============================================
`, VERSION, runtime.Version())

	ends := flag.Bool("ends", false, "annotate the start and end as well as the interval itself.")
	notstrict := flag.Bool("permissive-overlap", false, "annotate with an overlapping variant even it doesn't"+
		" share the same ref and alt alleles. Default is to require exact match between variants.")
	lua := flag.String("lua", "", "optional path to a file containing custom javascript functions to be used as ops")
	base := flag.String("base-path", "", "optional base-path to prepend to annotation files in the config")
	procs := flag.Int("p", 2, "number of processes to use.")
	flag.Parse()
	inFiles := flag.Args()
	if len(inFiles) != 2 {
		fmt.Printf(`Usage:
%s config.toml input.vcf > annotated.vcf

`, os.Args[0])
		flag.PrintDefaults()
		return
	}
	queryFile := inFiles[1]
	if !(xopen.Exists(queryFile) || queryFile == "") {
		fmt.Fprintf(os.Stderr, "\nERROR: can't find query file: %s\n", queryFile)
		os.Exit(2)
	}
	runtime.GOMAXPROCS(*procs)

	var config Config
	if _, err := toml.DecodeFile(inFiles[0], &config); err != nil {
		if strings.Contains(err.Error(), "Expected value but found") {
			fmt.Fprintln(os.Stderr, "\nNOTE: you must quote values in the conf file, e.g. fields=['AC', 'AN'] instead of fields=[AC, AN]")
		}
		panic(err)
	}
	config.Base = *base
	for _, a := range config.Annotation {
		err := CheckAnno(&a)
		if err != nil {
			log.Fatal("CheckAnno err:", err)
		}
		for _, op := range a.Ops {
			if len(op) > 4 && op[:4] == "lua:" && *lua == "" {
				log.Fatal("ERROR: requested lua op without specifying -lua flag")
			}
		}
	}
	for i := range config.PostAnnotation {
		r := config.PostAnnotation[i]
		err := CheckPostAnno(&r)
		if err != nil {
			log.Fatal(fmt.Sprintf("error in postannotaion section %s err: %s", r.Name, err))
		}
		if len(r.Op) > 4 && r.Op[:4] == "lua:" && *lua == "" {
			log.Fatal("ERROR: requested lua op without specifying -lua flag")
		}
	}

	sources, e := config.Sources()
	if e != nil {
		log.Fatal(e)
	}

	log.Printf("found %d sources from %d files\n", len(sources), len(config.Annotation))
	/*
		go func() {
			log.Println(http.ListenAndServe("localhost:6060", nil))
		}()
	*/

	luaString := ReadLua(*lua)
	strict := !*notstrict
	var a = NewAnnotator(sources, luaString, *ends, strict, config.PostAnnotation)

	var out io.Writer = os.Stdout
	defer os.Stdout.Close()

	var err error
	qrdr, err := xopen.Ropen(queryFile)
	if err != nil {
		log.Fatal(fmt.Errorf("error opening query file %s: %s", queryFile, err))
	}
	qstream, query, err := parsers.VCFIterator(qrdr)
	if err != nil {
		log.Fatal(fmt.Errorf("error parsing VCF query file %s: %s", queryFile, err))
	}

	queryables, err := a.Setup(query)
	if err != nil {
		log.Fatal(err)
	}
	aends := INTERVAL
	if *ends {
		aends = BOTH
	}

	lastMsg := struct {
		sync.RWMutex
		s [10]string
		i int
	}{}

	fn := func(v interfaces.Relatable) {
		e := a.AnnotateEnds(v, aends)
		if e != nil {
			lastMsg.RLock()
			em := e.Error()
			found := false
			for i := len(lastMsg.s) - 1; i >= 0; i-- {
				if em == lastMsg.s[i] {
					found = true
					break
				}
			}
			if !found {
				log.Println(e, ">> this error/warning may occur many times. reporting once here...")
				lastMsg.RUnlock()
				lastMsg.Lock()
				lastMsg.s[lastMsg.i] = em
				if lastMsg.i == len(lastMsg.s)-1 {
					lastMsg.i = -1
				}
				lastMsg.i++

				lastMsg.Unlock()
			} else {
				lastMsg.RUnlock()
			}
		}
	}

	maxGap := envGet("IRELATE_MAX_GAP", 20000)
	maxChunk := envGet("IRELATE_MAX_CHUNK", 8000)

	// make a new writer from the string header.
	out, err = vcfgo.NewWriter(out, query.Header)

	stream := irelate.PIRelate(maxChunk, maxGap, qstream, *ends, fn, queryables...)

	if err != nil {
		log.Fatal(err)
	}

	start := time.Now()
	n := 0

	/*
		if os.Getenv("IRELATE_PROFILE") == "TRUE" {
			log.Println("profiling to: irelate.pprof")
			f, err := os.Create("irelate.pprof")
			if err != nil {
				panic(err)
			}
			pprof.StartCPUProfile(f)
			defer pprof.StopCPUProfile()
		}
	*/

	for interval := range stream {
		//log.Printf("%v\n", interval)
		fmt.Fprintln(out, interval)
		n++
	}
	printTime(start, n)
}

func printTime(start time.Time, n int) {
	dur := time.Since(start)
	duri, duru := dur.Seconds(), "second"
	log.Printf("annotated %d variants in %.2f %ss (%.1f / %s)", n, duri, duru, float64(n)/duri, duru)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/fatih/color"
)

type SemanticVersion struct {
	major int
	minor int
	patch int
}

func (v *SemanticVersion) compare(other *SemanticVersion) int {
	if v.major != other.major {
		return compare(v.major, other.major)
	}
	if v.minor != other.minor {
		return compare(v.minor, other.minor)
	}
	if v.patch != other.patch {
		return compare(v.patch, other.patch)
	}

	return 0
}

func compare(lhs int, rhs int) int {
	if lhs == rhs {
		return 0
	} else if lhs < rhs {
		return -1
	}

	return 1
}

func printInfo(str string) {
	color.Yellow(str)
}

func printYay(str string) {
	color.Green(str)
	fmt.Println("")
}

func printError(err error) {
	message := fmt.Sprintf("Error: %s", err.Error())
	color.Red(message)
}

func main() {
	programCall := os.Args
	if len(programCall) < 2 {
		log.Fatal(errors.New("Usage: ./release <new-version>"))
	}

	args := programCall[1:]
	newVersion := args[0]
	printInfo("Fetching current version string...")
	currentVersion, fetchCurrentErr := fetchCurrentVersionString()
	if fetchCurrentErr != nil {
		printError(fetchCurrentErr)
		return
	}
	foundCurrentVersion := fmt.Sprintf("Found current version: %s", currentVersion)
	printYay(foundCurrentVersion)

	printInfo("Verifying new version is greater than current version...")
	if isNewVersionErr := verifyNewerThanCurrentVersion(newVersion, currentVersion); isNewVersionErr != nil {
		printError(isNewVersionErr)
		return
	}
	confirmIsNewVersionMessage := fmt.Sprintf("Confirmed: %s > %s", newVersion, currentVersion)
	printYay(confirmIsNewVersionMessage)

	bumpingVersionMessage := fmt.Sprintf("Bumping Fleet version to %s", newVersion)
	printInfo(bumpingVersionMessage)
	if bumpVersionErr := bumpVersionTo(newVersion, currentVersion); bumpVersionErr != nil {
		printError(bumpVersionErr)
		return
	}
	didBumpVersionMessage := fmt.Sprintf("Finished setting Fleet version to %s", newVersion)
	printYay(didBumpVersionMessage)

	printInfo("Committing new release...")
	if commitErr := commitRelease(newVersion); commitErr != nil {
		printError(commitErr)
		return
	}
	printYay("Committed")

	printInfo("Pushing new release to Github...")
	if pushErr := pushRelease(); pushErr != nil {
		printError(pushErr)
		return
	}
	printYay("Pushed")

	printInfo("Tagging new release and pushing tags...")
	if tagErr := tagRelease(); tagErr != nil {
		printError(tagErr)
		return
	}
	tagMessage := fmt.Sprintf("Created and pushed tag %s", newVersion)
	printYay(tagMessage)

	printInfo("Updating Cocoapod...")
	if podPushErr := updateCocoapod(); podPushErr != nil {
		printError(podPushErr)
		return
	}
	updatedPodMessage := fmt.Sprintf("Cocoapod updated to version %s", newVersion)
	printYay(updatedPodMessage)

	successMessage := fmt.Sprintf("Finished successfully updating Fleet to version %s", newVersion)
	printYay(successMessage)
}

func verifyNewerThanCurrentVersion(newVersionString string, currentVersionString string) error {
	newVersion, err := convertToSemanticVersion(newVersionString)
	if err != nil {
		return err
	}

	currentVersion, err := convertToSemanticVersion(currentVersionString)
	if err != nil {
		return err
	}

	compareResult := newVersion.compare(currentVersion)
	if compareResult != 1 {
		message := fmt.Sprintf("'%s' is not a greater semantic version than '%v.%v.%v'.", newVersionString, currentVersion.major, currentVersion.minor, currentVersion.patch)
		return errors.New(message)
	}

	return nil
}

func fetchCurrentVersionString() (string, error) {
	versionToolOutput, err := exec.Command("agvtool", "vers").Output()
	if err != nil {
		return "", err
	}

	reader := bytes.NewBuffer(versionToolOutput)
	bufReader := bufio.NewReader(reader)

	_, firstLineErr := bufReader.ReadString('\n')
	if firstLineErr != nil {
		return "", firstLineErr
	}

	lineWithVersion, versionLineErr := bufReader.ReadString('\n')
	if versionLineErr != nil {
		return "", versionLineErr
	}

	strippedVersion := strings.Trim(lineWithVersion, " \n\t\r")
	return strippedVersion, nil
}

func convertToSemanticVersion(str string) (*SemanticVersion, error) {
	if strings.ContainsAny(str, " \n\t\r") {
		message := fmt.Sprintf("'%s' not a semantic verison: Semantic versions cannot contain whitespace.", str)
		return nil, errors.New(message)
	}

	components := strings.Split(str, ".")
	if len(components) != 3 {
		message := fmt.Sprintf("'%s' not a semantic verison: Semantic versions have 3 dot-separated components.", str)
		return nil, errors.New(message)
	}

	major, conversionErr := strconv.Atoi(components[0])
	if conversionErr != nil {
		message := fmt.Sprintf("'%s' not a semantic verison: Each dot-separated component must be an integer.", str)
		return nil, errors.New(message)
	}

	minor, conversionErr := strconv.Atoi(components[1])
	if conversionErr != nil {
		message := fmt.Sprintf("'%s' not a semantic verison: Each dot-separated component must be an integer.", str)
		return nil, errors.New(message)
	}

	patch, conversionErr := strconv.Atoi(components[2])
	if conversionErr != nil {
		message := fmt.Sprintf("'%s' not a semantic verison: Each dot-separated component must be an integer.", str)
		return nil, errors.New(message)
	}

	return &SemanticVersion{
		major: major,
		minor: minor,
		patch: patch,
	}, nil
}

func bumpVersionTo(newVersion string, currentVersion string) error {
	agvToolErr := agvBumpVersion(newVersion)
	if agvToolErr != nil {
		return agvToolErr
	}

	infoPlistErr := bumpInfoPlistVersion(newVersion)
	if infoPlistErr != nil {
		return infoPlistErr
	}

	podspecErr := bumpPodspec(newVersion, currentVersion)
	if podspecErr != nil {
		return podspecErr
	}

	return nil
}

func agvBumpVersion(newVersion string) error {
	_, err := exec.Command("agvtool", "new-version", newVersion).Output()
	return err
}

func bumpInfoPlistVersion(newVersion string) error {
	_, err := exec.Command("plutil", "-replace", "CFBundleShortVersionString", "-string", newVersion, "Fleet/Info.plist").Output()
	return err
}

func bumpPodspec(newVersion string, currentVersion string) error {
	fileBytes, readErr := ioutil.ReadFile("Fleet.podspec")
	if readErr != nil {
		return readErr
	}

	fileString := string(fileBytes)
	newFileString := strings.Replace(fileString, currentVersion, newVersion, -1)
	writeErr := ioutil.WriteFile("Fleet.podspec", []byte(newFileString), 0644)
	if writeErr != nil {
		return writeErr
	}

	return nil
}

func commitRelease(newVersion string) error {
	if addProjectErr := addToCommit("Fleet.xcodeproj/"); addProjectErr != nil {
		return addProjectErr
	}
	if addInfoPlistErr := addToCommit("Fleet/Info.plist"); addInfoPlistErr != nil {
		return addInfoPlistErr
	}
	if addPodspecErr := addToCommit("Fleet.podspec"); addPodspecErr != nil {
		return addPodspecErr
	}

	releaseCommitMessage := fmt.Sprintf("Bump version to %s", newVersion)
	_, commitErr := exec.Command("git", "commit", "-m", releaseCommitMessage).Output()
	return commitErr
}

func addToCommit(fileName string) error {
	_, err := exec.Command("git", "add", fileName).Output()
	return err
}

func pushRelease() error {
	_, err := exec.Command("git", "push", "origin", "head").Output()
	return err
}

func tagRelease(newVersion string) error {
	_, tagErr := exec.Command("git", "tag", newVersion).Output()
	if tagErr != nil {
		return tagErr
	}

	_, pushErr := exec.Command("git", "push", "--tags").Output()
	return pushErr
}

func updateCocoapod() error {
	_, err := exec.Command("pod", "trunk", "push", "Fleet.podspec").Output()
	return err
}
<commit_msg>Fix missing argument in part of release script<commit_after>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/fatih/color"
)

type SemanticVersion struct {
	major int
	minor int
	patch int
}

func (v *SemanticVersion) compare(other *SemanticVersion) int {
	if v.major != other.major {
		return compare(v.major, other.major)
	}
	if v.minor != other.minor {
		return compare(v.minor, other.minor)
	}
	if v.patch != other.patch {
		return compare(v.patch, other.patch)
	}

	return 0
}

func compare(lhs int, rhs int) int {
	if lhs == rhs {
		return 0
	} else if lhs < rhs {
		return -1
	}

	return 1
}

func printInfo(str string) {
	color.Yellow(str)
}

func printYay(str string) {
	color.Green(str)
	fmt.Println("")
}

func printError(err error) {
	message := fmt.Sprintf("Error: %s", err.Error())
	color.Red(message)
}

func main() {
	programCall := os.Args
	if len(programCall) < 2 {
		log.Fatal(errors.New("Usage: ./release <new-version>"))
	}

	args := programCall[1:]
	newVersion := args[0]
	printInfo("Fetching current version string...")
	currentVersion, fetchCurrentErr := fetchCurrentVersionString()
	if fetchCurrentErr != nil {
		printError(fetchCurrentErr)
		return
	}
	foundCurrentVersion := fmt.Sprintf("Found current version: %s", currentVersion)
	printYay(foundCurrentVersion)

	printInfo("Verifying new version is greater than current version...")
	if isNewVersionErr := verifyNewerThanCurrentVersion(newVersion, currentVersion); isNewVersionErr != nil {
		printError(isNewVersionErr)
		return
	}
	confirmIsNewVersionMessage := fmt.Sprintf("Confirmed: %s > %s", newVersion, currentVersion)
	printYay(confirmIsNewVersionMessage)

	bumpingVersionMessage := fmt.Sprintf("Bumping Fleet version to %s", newVersion)
	printInfo(bumpingVersionMessage)
	if bumpVersionErr := bumpVersionTo(newVersion, currentVersion); bumpVersionErr != nil {
		printError(bumpVersionErr)
		return
	}
	didBumpVersionMessage := fmt.Sprintf("Finished setting Fleet version to %s", newVersion)
	printYay(didBumpVersionMessage)

	printInfo("Committing new release...")
	if commitErr := commitRelease(newVersion); commitErr != nil {
		printError(commitErr)
		return
	}
	printYay("Committed")

	printInfo("Pushing new release to Github...")
	if pushErr := pushRelease(); pushErr != nil {
		printError(pushErr)
		return
	}
	printYay("Pushed")

	printInfo("Tagging new release and pushing tags...")
	if tagErr := tagRelease(newVersion); tagErr != nil {
		printError(tagErr)
		return
	}
	tagMessage := fmt.Sprintf("Created and pushed tag %s", newVersion)
	printYay(tagMessage)

	printInfo("Updating Cocoapod...")
	if podPushErr := updateCocoapod(); podPushErr != nil {
		printError(podPushErr)
		return
	}
	updatedPodMessage := fmt.Sprintf("Cocoapod updated to version %s", newVersion)
	printYay(updatedPodMessage)

	successMessage := fmt.Sprintf("Finished successfully updating Fleet to version %s", newVersion)
	printYay(successMessage)
}

func verifyNewerThanCurrentVersion(newVersionString string, currentVersionString string) error {
	newVersion, err := convertToSemanticVersion(newVersionString)
	if err != nil {
		return err
	}

	currentVersion, err := convertToSemanticVersion(currentVersionString)
	if err != nil {
		return err
	}

	compareResult := newVersion.compare(currentVersion)
	if compareResult != 1 {
		message := fmt.Sprintf("'%s' is not a greater semantic version than '%v.%v.%v'.", newVersionString, currentVersion.major, currentVersion.minor, currentVersion.patch)
		return errors.New(message)
	}

	return nil
}

func fetchCurrentVersionString() (string, error) {
	versionToolOutput, err := exec.Command("agvtool", "vers").Output()
	if err != nil {
		return "", err
	}

	reader := bytes.NewBuffer(versionToolOutput)
	bufReader := bufio.NewReader(reader)

	_, firstLineErr := bufReader.ReadString('\n')
	if firstLineErr != nil {
		return "", firstLineErr
	}

	lineWithVersion, versionLineErr := bufReader.ReadString('\n')
	if versionLineErr != nil {
		return "", versionLineErr
	}

	strippedVersion := strings.Trim(lineWithVersion, " \n\t\r")
	return strippedVersion, nil
}

func convertToSemanticVersion(str string) (*SemanticVersion, error) {
	if strings.ContainsAny(str, " \n\t\r") {
		message := fmt.Sprintf("'%s' not a semantic verison: Semantic versions cannot contain whitespace.", str)
		return nil, errors.New(message)
	}

	components := strings.Split(str, ".")
	if len(components) != 3 {
		message := fmt.Sprintf("'%s' not a semantic verison: Semantic versions have 3 dot-separated components.", str)
		return nil, errors.New(message)
	}

	major, conversionErr := strconv.Atoi(components[0])
	if conversionErr != nil {
		message := fmt.Sprintf("'%s' not a semantic verison: Each dot-separated component must be an integer.", str)
		return nil, errors.New(message)
	}

	minor, conversionErr := strconv.Atoi(components[1])
	if conversionErr != nil {
		message := fmt.Sprintf("'%s' not a semantic verison: Each dot-separated component must be an integer.", str)
		return nil, errors.New(message)
	}

	patch, conversionErr := strconv.Atoi(components[2])
	if conversionErr != nil {
		message := fmt.Sprintf("'%s' not a semantic verison: Each dot-separated component must be an integer.", str)
		return nil, errors.New(message)
	}

	return &SemanticVersion{
		major: major,
		minor: minor,
		patch: patch,
	}, nil
}

func bumpVersionTo(newVersion string, currentVersion string) error {
	agvToolErr := agvBumpVersion(newVersion)
	if agvToolErr != nil {
		return agvToolErr
	}

	infoPlistErr := bumpInfoPlistVersion(newVersion)
	if infoPlistErr != nil {
		return infoPlistErr
	}

	podspecErr := bumpPodspec(newVersion, currentVersion)
	if podspecErr != nil {
		return podspecErr
	}

	return nil
}

func agvBumpVersion(newVersion string) error {
	_, err := exec.Command("agvtool", "new-version", newVersion).Output()
	return err
}

func bumpInfoPlistVersion(newVersion string) error {
	_, err := exec.Command("plutil", "-replace", "CFBundleShortVersionString", "-string", newVersion, "Fleet/Info.plist").Output()
	return err
}

func bumpPodspec(newVersion string, currentVersion string) error {
	fileBytes, readErr := ioutil.ReadFile("Fleet.podspec")
	if readErr != nil {
		return readErr
	}

	fileString := string(fileBytes)
	newFileString := strings.Replace(fileString, currentVersion, newVersion, -1)
	writeErr := ioutil.WriteFile("Fleet.podspec", []byte(newFileString), 0644)
	if writeErr != nil {
		return writeErr
	}

	return nil
}

func commitRelease(newVersion string) error {
	if addProjectErr := addToCommit("Fleet.xcodeproj/"); addProjectErr != nil {
		return addProjectErr
	}
	if addInfoPlistErr := addToCommit("Fleet/Info.plist"); addInfoPlistErr != nil {
		return addInfoPlistErr
	}
	if addPodspecErr := addToCommit("Fleet.podspec"); addPodspecErr != nil {
		return addPodspecErr
	}

	releaseCommitMessage := fmt.Sprintf("Bump version to %s", newVersion)
	_, commitErr := exec.Command("git", "commit", "-m", releaseCommitMessage).Output()
	return commitErr
}

func addToCommit(fileName string) error {
	_, err := exec.Command("git", "add", fileName).Output()
	return err
}

func pushRelease() error {
	_, err := exec.Command("git", "push", "origin", "head").Output()
	return err
}

func tagRelease(newVersion string) error {
	_, tagErr := exec.Command("git", "tag", newVersion).Output()
	if tagErr != nil {
		return tagErr
	}

	_, pushErr := exec.Command("git", "push", "--tags").Output()
	return pushErr
}

func updateCocoapod() error {
	_, err := exec.Command("pod", "trunk", "push", "Fleet.podspec").Output()
	return err
}
<|endoftext|>


<commit_before>package main

// CreatedDate is the date we're built. This would be nice to generate
// dynamically, but I don't want to complicate the build.
const CreatedDate = "2019-07-06"

// Version is our version.
const Version = "catbox-1.12.0"
<commit_msg>v1.13.0<commit_after>package main

// CreatedDate is the date we're built. This would be nice to generate
// dynamically, but I don't want to complicate the build.
const CreatedDate = "2019-07-08"

// Version is our version.
const Version = "catbox-1.13.0"
<|endoftext|>


<commit_before>package main

const VERSION = "0.1.0"
<commit_msg>Bump up 0.1.1<commit_after>package main

const VERSION = "0.1.1"
<|endoftext|>


<commit_before>package main
var VERSION string = "v0.4.0.11"
<commit_msg>Forced again<commit_after>package main
var VERSION string = "v0.4.0.12"
<|endoftext|>


<commit_before>package containerd

const Version = "0.0.1"
<commit_msg>Bump to alpha v0.0.2<commit_after>package containerd

const Version = "0.0.2"
<|endoftext|>


<commit_before>package mssql

import (
	"strconv"
	"strings"

	// DRIVER: mssql
	"github.com/denisenkom/go-mssqldb"

	"github.com/knq/usql/drivers"
)

func init() {
	drivers.Register("mssql", drivers.Driver{
		ReqPP: true,
		V: func(db drivers.DB) (string, error) {
			var ver, level, edition string
			err := db.QueryRow(
				`SELECT SERVERPROPERTY('productversion'), SERVERPROPERTY ('productlevel'), SERVERPROPERTY ('edition')`,
			).Scan(&ver, &level, &edition)
			if err != nil {
				return "", err
			}
			return "Microsoft SQL Server " + ver + ", " + level + ", " + edition, nil
		},
		ChPw: func(db drivers.DB, user, new, old string) error {
			_, err := db.Exec(`alter login ` + user + ` with password = '` + new + `' old_password = '` + old + `'`)
			return err
		},
		E: func(err error) (string, string) {
			if e, ok := err.(mssql.Error); ok {
				return strconv.Itoa(int(e.Number)), e.Message
			}

			msg := err.Error()
			if i := strings.LastIndex(msg, "mssql:"); i != -1 {
				msg = msg[i:]
			}

			return "", msg
		},
		PwErr: func(err error) bool {
			return strings.Contains(err.Error(), "Login failed for")
		},
	})
}
<commit_msg>Enabling multi-line comments for mssql driver<commit_after>package mssql

import (
	"strconv"
	"strings"

	// DRIVER: mssql
	"github.com/denisenkom/go-mssqldb"

	"github.com/knq/usql/drivers"
)

func init() {
	drivers.Register("mssql", drivers.Driver{
		AMC:   true,
		ReqPP: true,
		V: func(db drivers.DB) (string, error) {
			var ver, level, edition string
			err := db.QueryRow(
				`SELECT SERVERPROPERTY('productversion'), SERVERPROPERTY ('productlevel'), SERVERPROPERTY ('edition')`,
			).Scan(&ver, &level, &edition)
			if err != nil {
				return "", err
			}
			return "Microsoft SQL Server " + ver + ", " + level + ", " + edition, nil
		},
		ChPw: func(db drivers.DB, user, new, old string) error {
			_, err := db.Exec(`alter login ` + user + ` with password = '` + new + `' old_password = '` + old + `'`)
			return err
		},
		E: func(err error) (string, string) {
			if e, ok := err.(mssql.Error); ok {
				return strconv.Itoa(int(e.Number)), e.Message
			}

			msg := err.Error()
			if i := strings.LastIndex(msg, "mssql:"); i != -1 {
				msg = msg[i:]
			}

			return "", msg
		},
		PwErr: func(err error) bool {
			return strings.Contains(err.Error(), "Login failed for")
		},
	})
}
<|endoftext|>


<commit_before>package superast

import (
	"go/ast"
	"go/token"
	"log"
	"strconv"
)

var allowedImports = map[string]struct{}{
	"fmt": struct{}{},
}

type AST struct {
	curID      int
	RootBlock  *block
	nodeStack  []ast.Node
	stmtsStack []*[]stmt
	fset       *token.FileSet
	position   token.Position
}

func NewAST(fset *token.FileSet) *AST {
	a := &AST{
		fset: fset,
	}
	a.RootBlock = &block{
		id:    a.newID(),
		Stmts: make([]stmt, 0),
	}
	a.pushStmts(&a.RootBlock.Stmts)
	return a
}

func (a *AST) newID() id {
	i := a.curID
	a.curID++
	return id{ID: i}
}

func (a *AST) pos() pos {
	return pos{Line: a.position.Line, Col: a.position.Column}
}

func (a *AST) pushNode(node ast.Node) {
	a.nodeStack = append(a.nodeStack, node)
}

func (a *AST) curNode() ast.Node {
	if len(a.nodeStack) == 0 {
		return nil
	}
	return a.nodeStack[len(a.nodeStack)-1]
}

func (a *AST) popNode() {
	if len(a.nodeStack) == 0 {
		return
	}
	a.nodeStack = a.nodeStack[:len(a.nodeStack)-1]
}

func (a *AST) pushStmts(stmts *[]stmt) {
	a.stmtsStack = append(a.stmtsStack, stmts)
}

func (a *AST) addStmt(s stmt) {
	if len(a.stmtsStack) == 0 {
		return
	}
	curStmts := a.stmtsStack[len(a.stmtsStack)-1]
	*curStmts = append(*curStmts, s)
}

func (a *AST) popStmts() {
	if len(a.stmtsStack) == 0 {
		return
	}
	a.stmtsStack = a.stmtsStack[:len(a.stmtsStack)-1]
}

func strUnquote(s string) string {
	u, err := strconv.Unquote(s)
	if err != nil {
		return s
	}
	return u
}

func exprString(x ast.Expr) string {
	switch t := x.(type) {
	case *ast.Ident:
		return t.Name
	case *ast.BasicLit:
		return t.Value
	case *ast.SelectorExpr:
		return exprString(t.X) + "." + t.Sel.Name
	case *ast.StarExpr:
		return exprString(t.X)
	}
	return ""
}

func exprType(x ast.Expr) *dataType {
	s := exprString(x)
	if s != "" {
		return &dataType{
			Name: s,
		}
	}
	switch t := x.(type) {
	case *ast.ArrayType:
		return &dataType{
			Name:    "vector",
			SubType: exprType(t.Elt),
		}
	}
	return nil
}

type namedType struct {
	vName string
	dType *dataType
}

func flattenNames(baseType ast.Expr, names []*ast.Ident) []namedType {
	var types []namedType
	t := exprType(baseType)
	if len(names) == 0 {
		types = append(types, namedType{
			vName: "",
			dType: t,
		})
	}
	for _, n := range names {
		types = append(types, namedType{
			vName: n.Name,
			dType: t,
		})
	}
	return types
}

func flattenFieldList(fieldList *ast.FieldList) []namedType {
	if fieldList == nil {
		return nil
	}
	var types []namedType
	for _, f := range fieldList.List {
		for _, t := range flattenNames(f.Type, f.Names) {
			types = append(types, t)
		}
	}
	return types
}

var basicLitName = map[token.Token]string{
	token.INT:    "int",
	token.FLOAT:  "double",
	token.CHAR:   "char",
	token.STRING: "string",
}

var zeroValues = map[string]value{
	"int":    0,
	"double": 0.0,
	"char":   `'\0'`,
	"string": "",
}

func (a *AST) assignIdToDataType(dType *dataType) *dataType {
	if dType == nil {
		return nil
	}
	dTypeCopy := *dType
	dTypeCopy.id = a.newID()
	if dTypeCopy.SubType != nil {
		dTypeCopy.SubType = a.assignIdToDataType(dTypeCopy.SubType)
	}
	return &dTypeCopy
}

func (a *AST) Visit(node ast.Node) ast.Visitor {
	parentNode := a.curNode()
	if node == nil {
		switch parentNode.(type) {
		case *ast.CallExpr:
			a.popStmts()
		case *ast.FuncDecl:
			a.popStmts()
		}
		a.popNode()
		return nil
	}
	a.position = a.fset.Position(node.Pos())
	switch x := node.(type) {
	case *ast.File:
		imports := x.Imports
		for _, imp := range imports {
			path := strUnquote(imp.Path.Value)
			if _, e := allowedImports[path]; !e {
				log.Fatalf(`Import path not allowed: "%s"`, path)
			}
		}
	case *ast.TypeSpec:
		n := ""
		if x.Name != nil {
			n = exprString(x.Name)
		}
		switch t := x.Type.(type) {
		case *ast.StructType:
			decl := &structDecl{
				id:   a.newID(),
				pos:  a.pos(),
				Type: "struct-declaration",
				Name: n,
			}
			for _, f := range flattenFieldList(t.Fields) {
				attr := varDecl{
					id:       a.newID(),
					pos:      a.pos(),
					Type:     "variable-declaration",
					Name:     f.vName,
					DataType: a.assignIdToDataType(f.dType),
				}
				decl.Attrs = append(decl.Attrs, attr)
			}
			a.addStmt(decl)
		}
	case *ast.Ident:
		switch parentNode.(type) {
		case *ast.CallExpr:
		default:
			return nil
		}
		id := &identifier{
			id:    a.newID(),
			pos:   a.pos(),
			Type:  "identifier",
			Value: x.Name,
		}
		a.addStmt(id)
	case *ast.BasicLit:
		lit := &identifier{
			id:    a.newID(),
			pos:   a.pos(),
			Type:  "string",
			Value: strUnquote(x.Value),
		}
		a.addStmt(lit)
	case *ast.CallExpr:
		name := exprString(x.Fun)
		call := &funcCall{
			id:   a.newID(),
			pos:  a.pos(),
			Type: "function-call",
			Name: name,
		}
		a.addStmt(call)
		a.pushStmts(&call.Args)
	case *ast.FuncDecl:
		name := x.Name.Name
		var retType *dataType
		results := flattenFieldList(x.Type.Results)
		switch len(results) {
		case 1:
			retType = results[0].dType
		}
		fn := &funcDecl{
			id:      a.newID(),
			pos:     a.pos(),
			Type:    "function-declaration",
			Name:    name,
			RetType: a.assignIdToDataType(retType),
			Block: &block{
				id:    a.newID(),
				Stmts: make([]stmt, 0),
			},
		}
		for _, f := range flattenFieldList(x.Type.Params) {
			param := varDecl{
				id:       a.newID(),
				pos:      a.pos(),
				Type:     "variable-declaration",
				Name:     f.vName,
				DataType: a.assignIdToDataType(f.dType),
			}
			fn.Params = append(fn.Params, param)
		}
		a.addStmt(fn)
		a.pushStmts(&fn.Block.Stmts)
	case *ast.DeclStmt:
		gd, _ := x.Decl.(*ast.GenDecl)
		for _, spec := range gd.Specs {
			s, e := spec.(*ast.ValueSpec)
			if !e {
				continue
			}
			for i, t := range flattenNames(s.Type, s.Names) {
				vType := t.dType.Name
				v, _ := zeroValues[vType]
				if s.Values != nil {
					v = exprString(s.Values[i])
				}
				decl := &varDecl{
					id:       a.newID(),
					pos:      a.pos(),
					Type:     "variable-declaration",
					Name:     t.vName,
					DataType: a.assignIdToDataType(t.dType),
					Init: &identifier{
						id:    a.newID(),
						pos:   a.pos(),
						Type:  vType,
						Value: v,
					},
				}
				a.addStmt(decl)
			}
		}
	case *ast.AssignStmt:
		for i, expr := range x.Lhs {
			n := exprString(expr)
			var v value
			var t string
			switch r := x.Rhs[i].(type) {
			case *ast.BasicLit:
				t, _ = basicLitName[r.Kind]
				v = strUnquote(r.Value)
			case *ast.CompositeLit:
				t = exprString(r.Type)
			default:
			}
			asg := &varDecl{
				id:   a.newID(),
				pos:  a.pos(),
				Type: "variable-declaration",
				Name: n,
				DataType: &dataType{
					id:   a.newID(),
					Name: t,
				},
				Init: &identifier{
					id:    a.newID(),
					pos:   a.pos(),
					Type:  t,
					Value: v,
				},
			}
			a.addStmt(asg)
		}
		return nil
	case *ast.BlockStmt:
	case *ast.ExprStmt:
	case *ast.GenDecl:
	default:
		return nil
	}
	a.pushNode(node)
	return a
}
<commit_msg>Remove import whitelisting<commit_after>package superast

import (
	"go/ast"
	"go/token"
	"log"
	"strconv"
)

type AST struct {
	curID      int
	RootBlock  *block
	nodeStack  []ast.Node
	stmtsStack []*[]stmt
	fset       *token.FileSet
	position   token.Position
}

func NewAST(fset *token.FileSet) *AST {
	a := &AST{
		fset: fset,
	}
	a.RootBlock = &block{
		id:    a.newID(),
		Stmts: make([]stmt, 0),
	}
	a.pushStmts(&a.RootBlock.Stmts)
	return a
}

func (a *AST) newID() id {
	i := a.curID
	a.curID++
	return id{ID: i}
}

func (a *AST) pos() pos {
	return pos{Line: a.position.Line, Col: a.position.Column}
}

func (a *AST) pushNode(node ast.Node) {
	a.nodeStack = append(a.nodeStack, node)
}

func (a *AST) curNode() ast.Node {
	if len(a.nodeStack) == 0 {
		return nil
	}
	return a.nodeStack[len(a.nodeStack)-1]
}

func (a *AST) popNode() {
	if len(a.nodeStack) == 0 {
		return
	}
	a.nodeStack = a.nodeStack[:len(a.nodeStack)-1]
}

func (a *AST) pushStmts(stmts *[]stmt) {
	a.stmtsStack = append(a.stmtsStack, stmts)
}

func (a *AST) addStmt(s stmt) {
	if len(a.stmtsStack) == 0 {
		return
	}
	curStmts := a.stmtsStack[len(a.stmtsStack)-1]
	*curStmts = append(*curStmts, s)
}

func (a *AST) popStmts() {
	if len(a.stmtsStack) == 0 {
		return
	}
	a.stmtsStack = a.stmtsStack[:len(a.stmtsStack)-1]
}

func strUnquote(s string) string {
	u, err := strconv.Unquote(s)
	if err != nil {
		return s
	}
	return u
}

func exprString(x ast.Expr) string {
	switch t := x.(type) {
	case *ast.Ident:
		return t.Name
	case *ast.BasicLit:
		return t.Value
	case *ast.SelectorExpr:
		return exprString(t.X) + "." + t.Sel.Name
	case *ast.StarExpr:
		return exprString(t.X)
	}
	return ""
}

func exprType(x ast.Expr) *dataType {
	s := exprString(x)
	if s != "" {
		return &dataType{
			Name: s,
		}
	}
	switch t := x.(type) {
	case *ast.ArrayType:
		return &dataType{
			Name:    "vector",
			SubType: exprType(t.Elt),
		}
	}
	return nil
}

type namedType struct {
	vName string
	dType *dataType
}

func flattenNames(baseType ast.Expr, names []*ast.Ident) []namedType {
	var types []namedType
	t := exprType(baseType)
	if len(names) == 0 {
		types = append(types, namedType{
			vName: "",
			dType: t,
		})
	}
	for _, n := range names {
		types = append(types, namedType{
			vName: n.Name,
			dType: t,
		})
	}
	return types
}

func flattenFieldList(fieldList *ast.FieldList) []namedType {
	if fieldList == nil {
		return nil
	}
	var types []namedType
	for _, f := range fieldList.List {
		for _, t := range flattenNames(f.Type, f.Names) {
			types = append(types, t)
		}
	}
	return types
}

var basicLitName = map[token.Token]string{
	token.INT:    "int",
	token.FLOAT:  "double",
	token.CHAR:   "char",
	token.STRING: "string",
}

var zeroValues = map[string]value{
	"int":    0,
	"double": 0.0,
	"char":   `'\0'`,
	"string": "",
}

func (a *AST) assignIdToDataType(dType *dataType) *dataType {
	if dType == nil {
		return nil
	}
	dTypeCopy := *dType
	dTypeCopy.id = a.newID()
	if dTypeCopy.SubType != nil {
		dTypeCopy.SubType = a.assignIdToDataType(dTypeCopy.SubType)
	}
	return &dTypeCopy
}

func (a *AST) Visit(node ast.Node) ast.Visitor {
	parentNode := a.curNode()
	if node == nil {
		switch parentNode.(type) {
		case *ast.CallExpr:
			a.popStmts()
		case *ast.FuncDecl:
			a.popStmts()
		}
		a.popNode()
		return nil
	}
	a.position = a.fset.Position(node.Pos())
	switch x := node.(type) {
	case *ast.TypeSpec:
		n := ""
		if x.Name != nil {
			n = exprString(x.Name)
		}
		switch t := x.Type.(type) {
		case *ast.StructType:
			decl := &structDecl{
				id:   a.newID(),
				pos:  a.pos(),
				Type: "struct-declaration",
				Name: n,
			}
			for _, f := range flattenFieldList(t.Fields) {
				attr := varDecl{
					id:       a.newID(),
					pos:      a.pos(),
					Type:     "variable-declaration",
					Name:     f.vName,
					DataType: a.assignIdToDataType(f.dType),
				}
				decl.Attrs = append(decl.Attrs, attr)
			}
			a.addStmt(decl)
		}
	case *ast.Ident:
		switch parentNode.(type) {
		case *ast.CallExpr:
		default:
			return nil
		}
		id := &identifier{
			id:    a.newID(),
			pos:   a.pos(),
			Type:  "identifier",
			Value: x.Name,
		}
		a.addStmt(id)
	case *ast.BasicLit:
		lit := &identifier{
			id:    a.newID(),
			pos:   a.pos(),
			Type:  "string",
			Value: strUnquote(x.Value),
		}
		a.addStmt(lit)
	case *ast.CallExpr:
		name := exprString(x.Fun)
		call := &funcCall{
			id:   a.newID(),
			pos:  a.pos(),
			Type: "function-call",
			Name: name,
		}
		a.addStmt(call)
		a.pushStmts(&call.Args)
	case *ast.FuncDecl:
		name := x.Name.Name
		var retType *dataType
		results := flattenFieldList(x.Type.Results)
		switch len(results) {
		case 1:
			retType = results[0].dType
		}
		fn := &funcDecl{
			id:      a.newID(),
			pos:     a.pos(),
			Type:    "function-declaration",
			Name:    name,
			RetType: a.assignIdToDataType(retType),
			Block: &block{
				id:    a.newID(),
				Stmts: make([]stmt, 0),
			},
		}
		for _, f := range flattenFieldList(x.Type.Params) {
			param := varDecl{
				id:       a.newID(),
				pos:      a.pos(),
				Type:     "variable-declaration",
				Name:     f.vName,
				DataType: a.assignIdToDataType(f.dType),
			}
			fn.Params = append(fn.Params, param)
		}
		a.addStmt(fn)
		a.pushStmts(&fn.Block.Stmts)
	case *ast.DeclStmt:
		gd, _ := x.Decl.(*ast.GenDecl)
		for _, spec := range gd.Specs {
			s, e := spec.(*ast.ValueSpec)
			if !e {
				continue
			}
			for i, t := range flattenNames(s.Type, s.Names) {
				vType := t.dType.Name
				v, _ := zeroValues[vType]
				if s.Values != nil {
					v = exprString(s.Values[i])
				}
				decl := &varDecl{
					id:       a.newID(),
					pos:      a.pos(),
					Type:     "variable-declaration",
					Name:     t.vName,
					DataType: a.assignIdToDataType(t.dType),
					Init: &identifier{
						id:    a.newID(),
						pos:   a.pos(),
						Type:  vType,
						Value: v,
					},
				}
				a.addStmt(decl)
			}
		}
	case *ast.AssignStmt:
		for i, expr := range x.Lhs {
			n := exprString(expr)
			var v value
			var t string
			switch r := x.Rhs[i].(type) {
			case *ast.BasicLit:
				t, _ = basicLitName[r.Kind]
				v = strUnquote(r.Value)
			case *ast.CompositeLit:
				t = exprString(r.Type)
			default:
			}
			asg := &varDecl{
				id:   a.newID(),
				pos:  a.pos(),
				Type: "variable-declaration",
				Name: n,
				DataType: &dataType{
					id:   a.newID(),
					Name: t,
				},
				Init: &identifier{
					id:    a.newID(),
					pos:   a.pos(),
					Type:  t,
					Value: v,
				},
			}
			a.addStmt(asg)
		}
		return nil
	case *ast.File:
	case *ast.BlockStmt:
	case *ast.ExprStmt:
	case *ast.GenDecl:
	default:
		return nil
	}
	a.pushNode(node)
	return a
}
<|endoftext|>


<commit_before>// Package bitmap provides a []bool/bitmap implementation with standardized
// iteration. Bitmaps are the equivalent of []bool, with improved compression
// for runs of similar values, and faster operations on ranges and the like.
package bitmap

import (
	"math"

	"github.com/RoaringBitmap/roaring"

	"github.com/anacrolix/missinggo/iter"
)

const MaxInt = -1

type BitIndex = int

type Interface interface {
	Len() int
}

// Bitmaps store the existence of values in [0,math.MaxUint32] more
// efficiently than []bool. The empty value starts with no bits set.
type Bitmap struct {
	RB *roaring.Bitmap
}

var ToEnd int = -1

// The number of set bits in the bitmap. Also known as cardinality.
func (me *Bitmap) Len() int {
	if me.RB == nil {
		return 0
	}
	return int(me.RB.GetCardinality())
}

func (me Bitmap) ToSortedSlice() (ret []int) {
	if me.RB == nil {
		return
	}
	for _, ui32 := range me.RB.ToArray() {
		ret = append(ret, int(int32(ui32)))
	}
	return
}

func (me *Bitmap) lazyRB() *roaring.Bitmap {
	if me.RB == nil {
		me.RB = roaring.NewBitmap()
	}
	return me.RB
}

func (me Bitmap) Iter(cb iter.Callback) {
	me.IterTyped(func(i int) bool {
		return cb(i)
	})
}

// Returns true if all values were traversed without early termination.
func (me Bitmap) IterTyped(f func(int) bool) bool {
	if me.RB == nil {
		return true
	}
	it := me.RB.Iterator()
	for it.HasNext() {
		if !f(int(it.Next())) {
			return false
		}
	}
	return true
}

func checkInt(i BitIndex) {
	if i < math.MinInt32 || i > math.MaxInt32 {
		panic("out of bounds")
	}
}

func (me *Bitmap) Add(is ...BitIndex) {
	rb := me.lazyRB()
	for _, i := range is {
		checkInt(i)
		rb.AddInt(i)
	}
}

func (me *Bitmap) AddRange(begin, end BitIndex) {
	if begin >= end {
		return
	}
	me.lazyRB().AddRange(uint64(begin), uint64(end))
}

func (me *Bitmap) Remove(i BitIndex) bool {
	if me.RB == nil {
		return false
	}
	return me.RB.CheckedRemove(uint32(i))
}

func (me *Bitmap) Union(other Bitmap) {
	me.lazyRB().Or(other.lazyRB())
}

func (me *Bitmap) Contains(i int) bool {
	if me.RB == nil {
		return false
	}
	return me.RB.Contains(uint32(i))
}

func (me *Bitmap) Sub(other Bitmap) {
	if other.RB == nil {
		return
	}
	if me.RB == nil {
		return
	}
	me.RB.AndNot(other.RB)
}

func (me *Bitmap) Clear() {
	if me.RB == nil {
		return
	}
	me.RB.Clear()
}

func (me Bitmap) Copy() (ret Bitmap) {
	ret = me
	if ret.RB != nil {
		ret.RB = ret.RB.Clone()
	}
	return
}

func (me *Bitmap) FlipRange(begin, end BitIndex) {
	me.lazyRB().FlipInt(begin, end)
}

func (me *Bitmap) Get(bit BitIndex) bool {
	return me.RB != nil && me.RB.ContainsInt(bit)
}

func (me *Bitmap) Set(bit BitIndex, value bool) {
	if value {
		me.lazyRB().AddInt(bit)
	} else {
		if me.RB != nil {
			me.RB.Remove(uint32(bit))
		}
	}
}

func (me *Bitmap) RemoveRange(begin, end BitIndex) *Bitmap {
	if me.RB == nil {
		return me
	}
	rangeEnd := uint64(end)
	if end == ToEnd {
		rangeEnd = 0x100000000
	}
	me.RB.RemoveRange(uint64(begin), rangeEnd)
	return me
}

func (me Bitmap) IsEmpty() bool {
	return me.RB == nil || me.RB.IsEmpty()
}
<commit_msg>bitmap.Bitmap.{Contains,Get}: Take value receiver<commit_after>// Package bitmap provides a []bool/bitmap implementation with standardized
// iteration. Bitmaps are the equivalent of []bool, with improved compression
// for runs of similar values, and faster operations on ranges and the like.
package bitmap

import (
	"math"

	"github.com/RoaringBitmap/roaring"

	"github.com/anacrolix/missinggo/iter"
)

const MaxInt = -1

type BitIndex = int

type Interface interface {
	Len() int
}

// Bitmaps store the existence of values in [0,math.MaxUint32] more
// efficiently than []bool. The empty value starts with no bits set.
type Bitmap struct {
	RB *roaring.Bitmap
}

var ToEnd int = -1

// The number of set bits in the bitmap. Also known as cardinality.
func (me *Bitmap) Len() int {
	if me.RB == nil {
		return 0
	}
	return int(me.RB.GetCardinality())
}

func (me Bitmap) ToSortedSlice() (ret []int) {
	if me.RB == nil {
		return
	}
	for _, ui32 := range me.RB.ToArray() {
		ret = append(ret, int(int32(ui32)))
	}
	return
}

func (me *Bitmap) lazyRB() *roaring.Bitmap {
	if me.RB == nil {
		me.RB = roaring.NewBitmap()
	}
	return me.RB
}

func (me Bitmap) Iter(cb iter.Callback) {
	me.IterTyped(func(i int) bool {
		return cb(i)
	})
}

// Returns true if all values were traversed without early termination.
func (me Bitmap) IterTyped(f func(int) bool) bool {
	if me.RB == nil {
		return true
	}
	it := me.RB.Iterator()
	for it.HasNext() {
		if !f(int(it.Next())) {
			return false
		}
	}
	return true
}

func checkInt(i BitIndex) {
	if i < math.MinInt32 || i > math.MaxInt32 {
		panic("out of bounds")
	}
}

func (me *Bitmap) Add(is ...BitIndex) {
	rb := me.lazyRB()
	for _, i := range is {
		checkInt(i)
		rb.AddInt(i)
	}
}

func (me *Bitmap) AddRange(begin, end BitIndex) {
	if begin >= end {
		return
	}
	me.lazyRB().AddRange(uint64(begin), uint64(end))
}

func (me *Bitmap) Remove(i BitIndex) bool {
	if me.RB == nil {
		return false
	}
	return me.RB.CheckedRemove(uint32(i))
}

func (me *Bitmap) Union(other Bitmap) {
	me.lazyRB().Or(other.lazyRB())
}

func (me Bitmap) Contains(i int) bool {
	if me.RB == nil {
		return false
	}
	return me.RB.Contains(uint32(i))
}

func (me *Bitmap) Sub(other Bitmap) {
	if other.RB == nil {
		return
	}
	if me.RB == nil {
		return
	}
	me.RB.AndNot(other.RB)
}

func (me *Bitmap) Clear() {
	if me.RB == nil {
		return
	}
	me.RB.Clear()
}

func (me Bitmap) Copy() (ret Bitmap) {
	ret = me
	if ret.RB != nil {
		ret.RB = ret.RB.Clone()
	}
	return
}

func (me *Bitmap) FlipRange(begin, end BitIndex) {
	me.lazyRB().FlipInt(begin, end)
}

func (me Bitmap) Get(bit BitIndex) bool {
	return me.RB != nil && me.RB.ContainsInt(bit)
}

func (me *Bitmap) Set(bit BitIndex, value bool) {
	if value {
		me.lazyRB().AddInt(bit)
	} else {
		if me.RB != nil {
			me.RB.Remove(uint32(bit))
		}
	}
}

func (me *Bitmap) RemoveRange(begin, end BitIndex) *Bitmap {
	if me.RB == nil {
		return me
	}
	rangeEnd := uint64(end)
	if end == ToEnd {
		rangeEnd = 0x100000000
	}
	me.RB.RemoveRange(uint64(begin), rangeEnd)
	return me
}

func (me Bitmap) IsEmpty() bool {
	return me.RB == nil || me.RB.IsEmpty()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/context"

	"github.com/currantlabs/ble"
	"github.com/currantlabs/ble/examples/lib/dev"
)

var (
	device = flag.String("device", "default", "implementation of ble")
	du     = flag.Duration("du", 15*time.Second, "scanning duration")
	dup    = flag.Bool("dup", true, "allow duplicate reported")
)

func main() {
	flag.Parse()

	d, err := dev.NewDevice(*device)
	if err != nil {
		log.Fatalf("can't new device : %s", err)
	}
	ble.SetDefaultDevice(d)

	// Scan for specified durantion, or until interrupted by user.
	fmt.Printf("Scanning for %s...\n", *du)
	ctx := ble.WithSigHandler(context.WithTimeout(context.Background(), *du))
	chkErr(ble.Scan(ctx, *dup, advHandler, nil))
}

func advHandler(a ble.Advertisement) {
	if a.Connectable() {
		fmt.Printf("[%s] C %3d:", a.Address(), a.RSSI())
	} else {
		fmt.Printf("[%s] N %3d:", a.Address(), a.RSSI())
	}
	comma := ""
	if len(a.LocalName()) > 0 {
		fmt.Printf(" Name: %s", a.LocalName())
		comma = ","
	}
	if len(a.Services()) > 0 {
		fmt.Printf("%s Svcs: %v", comma, a.Services())
		comma = ","
	}
	if len(a.ManufacturerData()) > 0 {
		fmt.Printf("%s MD: %X", comma, a.ManufacturerData())
	}
	fmt.Printf("\n")
}

func chkErr(err error) {
	switch errors.Cause(err) {
	case nil:
	case context.DeadlineExceeded:
		fmt.Printf("done\n")
	case context.Canceled:
		fmt.Printf("canceled\n")
	default:
		log.Fatalf(err.Error())
	}
}
<commit_msg>* adding advFilter<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/context"

	"github.com/currantlabs/ble"
	"github.com/currantlabs/ble/examples/lib/dev"
)

var (
	device = flag.String("device", "default", "implementation of ble")
	du     = flag.Duration("du", 15*time.Second, "scanning duration")
)

type beacon struct {
	Address string
	RSSI    int
}

func main() {
	flag.Parse()

	d, err := dev.NewDevice(*device)
	if err != nil {
		log.Fatalf("can't new device : %s", err)
	}
	ble.SetDefaultDevice(d)

	beacons := make(map[string]*beacon)
	advFilter := func(a ble.Advertisement) bool {
		b, exists := beacons[a.Address().String()]
		if !exists {
			b := &beacon{Address: a.Address().String(), RSSI: ((a.RSSI() / 5) * 5)}
			beacons[a.Address().String()] = b
			return true
		}
		if b.RSSI != ((a.RSSI() / 5) * 5) {
			b.RSSI = ((a.RSSI() / 5) * 5)
			return true
		}
		return false
	}

	// Scan for specified durantion, or until interrupted by user.
	fmt.Printf("Scanning for %s...\n", *du)
	ctx := ble.WithSigHandler(context.WithTimeout(context.Background(), *du))
	chkErr(ble.Scan(ctx, true, advHandler, advFilter))
}

func advHandler(a ble.Advertisement) {
	if a.Connectable() {
		fmt.Printf("[%s] C %3d:", a.Address(), a.RSSI())
	} else {
		fmt.Printf("[%s] N %3d:", a.Address(), a.RSSI())
	}
	comma := ""
	if len(a.LocalName()) > 0 {
		fmt.Printf(" Name: %s", a.LocalName())
		comma = ","
	}
	if len(a.Services()) > 0 {
		fmt.Printf("%s Svcs: %v", comma, a.Services())
		comma = ","
	}
	if len(a.ManufacturerData()) > 0 {
		fmt.Printf("%s MD: %X", comma, a.ManufacturerData())
	}
	fmt.Printf("\n")
}

func chkErr(err error) {
	switch errors.Cause(err) {
	case nil:
	case context.DeadlineExceeded:
		fmt.Printf("done\n")
	case context.Canceled:
		fmt.Printf("canceled\n")
	default:
		log.Fatalf(err.Error())
	}
}
<|endoftext|>