<commit_before>package orm

import (
	"fmt"
	"gnd.la/log"
	"gnd.la/orm/codec"
	"gnd.la/orm/driver"
	"gnd.la/orm/query"
	"gnd.la/types"
	"gnd.la/util"
	"reflect"
	"regexp"
	"sort"
	"strings"
	"time"
)

type nameRegistry map[string]*model
type typeRegistry map[reflect.Type]*model

var (
	// these keep track of the registered models,
	// using the driver tags as the key.
	_nameRegistry = map[string]nameRegistry{}
	_typeRegistry = map[string]typeRegistry{}
	timeType      = reflect.TypeOf(time.Time{})
	referencesRe  = regexp.MustCompile("([\\w\\.]+)(\\((\\w+)\\))?")
)

// Register registers a struct for future usage with the ORMs with
// the same driver. If you're using ORM instances with different drivers
// (e.g. postgres and mongodb)  you must register each object type with each
// driver (by creating an ORM of each type, calling Register() and then
// CommitTables()). The first returned value is a Table object, which must be
// using when querying the ORM in cases when an object is not provided
// (like e.g. Count()). If you want to use the same type in multiple
// tables, you must register it for every table and then use the Table
// object returned to specify on which table you want to operate. If
// no table is specified, the first registered table will be used.
func (o *Orm) Register(t interface{}, opt *Options) (*Table, error) {
	s, err := types.NewStruct(t, o.dtags())
	if err != nil {
		switch err {
		case types.ErrNoStruct:
			return nil, fmt.Errorf("only structs can be registered as models (tried to register %T)", t)
		case types.ErrNoFields:
			return nil, fmt.Errorf("type %T has no fields", t)
		}
		return nil, err
	}
	var table string
	if opt != nil {
		table = opt.Table
	}
	if table == "" {
		table = defaultTableName(s.Type)
	}
	if _nameRegistry[o.tags] == nil {
		_nameRegistry[o.tags] = nameRegistry{}
		_typeRegistry[o.tags] = typeRegistry{}
	}
	if _, ok := _nameRegistry[o.tags][table]; ok {
		return nil, fmt.Errorf("duplicate ORM table name %q", table)
	}
	fields, references, err := o.fields(table, s)
	if err != nil {
		return nil, err
	}
	var name string
	if opt != nil && opt.Name != "" {
		name = opt.Name
	} else {
		name = typeName(s.Type)
	}
	if opt != nil {
		if len(opt.PrimaryKey) > 0 {
			if fields.PrimaryKey >= 0 {
				return nil, fmt.Errorf("duplicate primary key in model %q. tags define %q as PK, options define %v",
					name, fields.QNames[fields.PrimaryKey], opt.PrimaryKey)
			}
			fields.CompositePrimaryKey = make([]int, len(opt.PrimaryKey))
			for ii, v := range opt.PrimaryKey {
				pos, ok := fields.QNameMap[v]
				if !ok {
					return nil, fmt.Errorf("can't map qualified name %q on model %q when creating composite key", v, name)
				}
				fields.CompositePrimaryKey[ii] = pos
			}
		}
	}
	model := &model{
		fields:     fields,
		name:       name,
		shortName:  s.Type.Name(),
		references: references,
		options:    opt,
		table:      table,
		tags:       o.tags,
	}
	_nameRegistry[o.tags][table] = model
	// The first registered table is the default for the type
	typ := model.Type()
	if _, ok := _typeRegistry[o.tags][typ]; !ok || opt != nil && opt.Default {
		_typeRegistry[o.tags][typ] = model
	}
	log.Debugf("Registered model %v (%q) with tags %q", typ, name, o.tags)
	return &Table{model: &joinModel{model: model}}, nil
}

// MustRegister works like Register, but panics if there's an
// error.
func (o *Orm) MustRegister(t interface{}, opt *Options) *Table {
	tbl, err := o.Register(t, opt)
	if err != nil {
		panic(err)
	}
	return tbl
}

// Initialize resolves model references and creates tables and
// indexes required by the registered models. You MUST call it
// AFTER all the models have been registered and BEFORE starting
// to use the ORM from several goroutines (for performance reasons,
// the access to some shared resources from several ORM instances
// is not thread safe).
func (o *Orm) Initialize() error {
	nr := _nameRegistry[o.tags]
	// Resolve references
	names := make(map[string]*model)
	for _, v := range nr {
		names[v.name] = v
	}
	for _, v := range nr {
		if c := len(v.references); c > 0 {
			v.fields.References = make(map[string]*driver.Reference, c)
			for k, r := range v.references {
				referenced := names[r.model]
				if referenced == nil {
					if !strings.Contains(r.model, ".") {
						referenced = names[v.Type().PkgPath()+"."+r.model]
					}
					if referenced == nil {
						return fmt.Errorf("can't find referenced model %q from model %q", r.model, v.name)
					}
				}
				if r.field == "" {
					// Map to PK
					if pk := referenced.fields.PrimaryKey; pk >= 0 {
						r.field = referenced.fields.QNames[pk]
					} else {
						return fmt.Errorf("referenced model %q does not have a non-composite primary key. Please, specify a field", r.model)
					}
				}
				_, ft, err := v.fields.Map(k)
				if err != nil {
					return err
				}
				_, fkt, err := referenced.fields.Map(r.field)
				if err != nil {
					return err
				}
				if ft != fkt {
					return fmt.Errorf("type mismatch: referenced field %q in model %q is of type %s, field %q in model %q is of type %s",
						r.field, referenced.name, fkt, k, v.name, ft)
				}
				v.fields.References[k] = &driver.Reference{
					Model: referenced,
					Field: r.field,
				}
				if v.modelReferences == nil {
					v.modelReferences = make(map[*model][]*join)
					v.modelReferences[referenced] = append(v.modelReferences[referenced], &join{
						model: &joinModel{model: referenced},
						q:     Eq(v.fullName(k), query.F(referenced.fullName(r.field))),
					})
				}
				if referenced.modelReferences == nil {
					referenced.modelReferences = make(map[*model][]*join)
					referenced.modelReferences[v] = append(referenced.modelReferences[v], &join{
						model: &joinModel{model: v},
						q:     Eq(referenced.fullName(r.field), query.F(v.fullName(k))),
					})
				}
			}
		}
	}
	models := make([]driver.Model, 0, len(nr))
	for _, v := range nr {
		models = append(models, v)
	}
	// Sort models to the ones with FKs are created after
	// the models they reference
	sort.Sort(sortModels(models))
	return o.driver.MakeTables(models)
}

// MustInitialize works like Initialize, but panics if
// there's an error.
func (o *Orm) MustInitialize() {
	if err := o.Initialize(); err != nil {
		panic(err)
	}
}

func (o *Orm) fields(table string, s *types.Struct) (*driver.Fields, map[string]*reference, error) {
	methods, err := driver.MakeMethods(s.Type)
	if err != nil {
		return nil, nil, err
	}
	fields := &driver.Fields{
		Struct:     s,
		PrimaryKey: -1,
		Methods:    methods,
	}
	var references map[string]*reference
	for ii, v := range s.QNames {
		// XXX: Check if this quoting is enough
		fields.QuotedNames = append(fields.QuotedNames, fmt.Sprintf("\"%s\".\"%s\"", table, s.MNames[ii]))
		t := s.Types[ii]
		k := t.Kind()
		ftag := s.Tags[ii]
		// Check encoded types
		if c := codec.FromTag(ftag); c != nil {
			if err := c.Try(t, o.dtags()); err != nil {
				return nil, nil, fmt.Errorf("can't encode field %q as %s: %s", v, c.Name(), err)
			}
		} else if ftag.CodecName() != "" {
			return nil, nil, fmt.Errorf("can't find ORM codec %q. Perhaps you missed an import?", ftag.CodecName())
		} else {
			switch k {
			case reflect.Array, reflect.Chan, reflect.Func, reflect.Interface, reflect.Map:
				return nil, nil, fmt.Errorf("field %q in struct %v has invalid type %v", v, t, k)
			}
		}
		fields.OmitEmpty = append(fields.OmitEmpty, ftag.Has("omitempty") || (ftag.Has("auto_increment") && !ftag.Has("notomitempty")))
		// Struct has flattened types, but we need to original type
		// to determine if it should be nullempty by default
		field := s.Type.FieldByIndex(s.Indexes[ii])
		fields.NullEmpty = append(fields.NullEmpty, ftag.Has("nullempty") || (defaultsToNullEmpty(field.Type, ftag) && !ftag.Has("notnullempty")))
		if ftag.Has("primary_key") {
			if fields.PrimaryKey >= 0 {
				return nil, nil, fmt.Errorf("duplicate primary_key in struct %v (%s and %s)", s.Type, s.QNames[fields.PrimaryKey], v)
			}
			fields.PrimaryKey = ii
			if ftag.Has("auto_increment") && (k == reflect.Int || k == reflect.Int8 || k == reflect.Int16 || k == reflect.Int32 || k == reflect.Int64) {
				fields.IntegerAutoincrementPk = true
			}
		}
		if ref := ftag.Value("references"); ref != "" {
			m := referencesRe.FindStringSubmatch(ref)
			if len(m) != 4 {
				return nil, nil, fmt.Errorf("field %q has invalid references %q. Must be in the form references=Model or references=Model(Field)", v, ref)
			}
			if references == nil {
				references = make(map[string]*reference)
			}
			references[v] = &reference{model: m[1], field: m[3]}
		}
	}
	return fields, references, nil
}

func (o *Orm) dtags() []string {
	return append(o.driver.Tags(), "orm")
}

// returns wheter the kind defaults to nullempty option
func defaultsToNullEmpty(typ reflect.Type, t *types.Tag) bool {
	if t.Has("references") || t.Has("codec") {
		return true
	}
	switch typ.Kind() {
	case reflect.Slice, reflect.Ptr, reflect.Interface, reflect.String:
		return true
	case reflect.Struct:
		return typ == timeType
	}
	return false
}

// Returns the default table name for a type
func defaultTableName(typ reflect.Type) string {
	n := typ.Name()
	if p := typ.PkgPath(); p != "main" {
		n = strings.Replace(p, "/", "_", -1) + n
	}
	return util.CamelCaseToLower(n, "_")
}

func typeName(typ reflect.Type) string {
	if p := typ.PkgPath(); p != "main" {
		return p + "." + typ.Name()
	}
	return typ.Name()
}
<commit_msg>Correctly add all the model references<commit_after>package orm

import (
	"fmt"
	"gnd.la/log"
	"gnd.la/orm/codec"
	"gnd.la/orm/driver"
	"gnd.la/orm/query"
	"gnd.la/types"
	"gnd.la/util"
	"reflect"
	"regexp"
	"sort"
	"strings"
	"time"
)

type nameRegistry map[string]*model
type typeRegistry map[reflect.Type]*model

var (
	// these keep track of the registered models,
	// using the driver tags as the key.
	_nameRegistry = map[string]nameRegistry{}
	_typeRegistry = map[string]typeRegistry{}
	timeType      = reflect.TypeOf(time.Time{})
	referencesRe  = regexp.MustCompile("([\\w\\.]+)(\\((\\w+)\\))?")
)

// Register registers a struct for future usage with the ORMs with
// the same driver. If you're using ORM instances with different drivers
// (e.g. postgres and mongodb)  you must register each object type with each
// driver (by creating an ORM of each type, calling Register() and then
// CommitTables()). The first returned value is a Table object, which must be
// using when querying the ORM in cases when an object is not provided
// (like e.g. Count()). If you want to use the same type in multiple
// tables, you must register it for every table and then use the Table
// object returned to specify on which table you want to operate. If
// no table is specified, the first registered table will be used.
func (o *Orm) Register(t interface{}, opt *Options) (*Table, error) {
	s, err := types.NewStruct(t, o.dtags())
	if err != nil {
		switch err {
		case types.ErrNoStruct:
			return nil, fmt.Errorf("only structs can be registered as models (tried to register %T)", t)
		case types.ErrNoFields:
			return nil, fmt.Errorf("type %T has no fields", t)
		}
		return nil, err
	}
	var table string
	if opt != nil {
		table = opt.Table
	}
	if table == "" {
		table = defaultTableName(s.Type)
	}
	if _nameRegistry[o.tags] == nil {
		_nameRegistry[o.tags] = nameRegistry{}
		_typeRegistry[o.tags] = typeRegistry{}
	}
	if _, ok := _nameRegistry[o.tags][table]; ok {
		return nil, fmt.Errorf("duplicate ORM table name %q", table)
	}
	fields, references, err := o.fields(table, s)
	if err != nil {
		return nil, err
	}
	var name string
	if opt != nil && opt.Name != "" {
		name = opt.Name
	} else {
		name = typeName(s.Type)
	}
	if opt != nil {
		if len(opt.PrimaryKey) > 0 {
			if fields.PrimaryKey >= 0 {
				return nil, fmt.Errorf("duplicate primary key in model %q. tags define %q as PK, options define %v",
					name, fields.QNames[fields.PrimaryKey], opt.PrimaryKey)
			}
			fields.CompositePrimaryKey = make([]int, len(opt.PrimaryKey))
			for ii, v := range opt.PrimaryKey {
				pos, ok := fields.QNameMap[v]
				if !ok {
					return nil, fmt.Errorf("can't map qualified name %q on model %q when creating composite key", v, name)
				}
				fields.CompositePrimaryKey[ii] = pos
			}
		}
	}
	model := &model{
		fields:     fields,
		name:       name,
		shortName:  s.Type.Name(),
		references: references,
		options:    opt,
		table:      table,
		tags:       o.tags,
	}
	_nameRegistry[o.tags][table] = model
	// The first registered table is the default for the type
	typ := model.Type()
	if _, ok := _typeRegistry[o.tags][typ]; !ok || opt != nil && opt.Default {
		_typeRegistry[o.tags][typ] = model
	}
	log.Debugf("Registered model %v (%q) with tags %q", typ, name, o.tags)
	return &Table{model: &joinModel{model: model}}, nil
}

// MustRegister works like Register, but panics if there's an
// error.
func (o *Orm) MustRegister(t interface{}, opt *Options) *Table {
	tbl, err := o.Register(t, opt)
	if err != nil {
		panic(err)
	}
	return tbl
}

// Initialize resolves model references and creates tables and
// indexes required by the registered models. You MUST call it
// AFTER all the models have been registered and BEFORE starting
// to use the ORM from several goroutines (for performance reasons,
// the access to some shared resources from several ORM instances
// is not thread safe).
func (o *Orm) Initialize() error {
	nr := _nameRegistry[o.tags]
	// Resolve references
	names := make(map[string]*model)
	for _, v := range nr {
		names[v.name] = v
	}
	for _, v := range nr {
		if c := len(v.references); c > 0 {
			v.fields.References = make(map[string]*driver.Reference, c)
			for k, r := range v.references {
				referenced := names[r.model]
				if referenced == nil {
					if !strings.Contains(r.model, ".") {
						referenced = names[v.Type().PkgPath()+"."+r.model]
					}
					if referenced == nil {
						return fmt.Errorf("can't find referenced model %q from model %q", r.model, v.name)
					}
				}
				if r.field == "" {
					// Map to PK
					if pk := referenced.fields.PrimaryKey; pk >= 0 {
						r.field = referenced.fields.QNames[pk]
					} else {
						return fmt.Errorf("referenced model %q does not have a non-composite primary key. Please, specify a field", r.model)
					}
				}
				_, ft, err := v.fields.Map(k)
				if err != nil {
					return err
				}
				_, fkt, err := referenced.fields.Map(r.field)
				if err != nil {
					return err
				}
				if ft != fkt {
					return fmt.Errorf("type mismatch: referenced field %q in model %q is of type %s, field %q in model %q is of type %s",
						r.field, referenced.name, fkt, k, v.name, ft)
				}
				v.fields.References[k] = &driver.Reference{
					Model: referenced,
					Field: r.field,
				}
				if v.modelReferences == nil {
					v.modelReferences = make(map[*model][]*join)
				}
				v.modelReferences[referenced] = append(v.modelReferences[referenced], &join{
					model: &joinModel{model: referenced},
					q:     Eq(v.fullName(k), query.F(referenced.fullName(r.field))),
				})
				if referenced.modelReferences == nil {
					referenced.modelReferences = make(map[*model][]*join)
				}
				referenced.modelReferences[v] = append(referenced.modelReferences[v], &join{
					model: &joinModel{model: v},
					q:     Eq(referenced.fullName(r.field), query.F(v.fullName(k))),
				})
			}
		}
	}
	models := make([]driver.Model, 0, len(nr))
	for _, v := range nr {
		models = append(models, v)
	}
	// Sort models to the ones with FKs are created after
	// the models they reference
	sort.Sort(sortModels(models))
	return o.driver.MakeTables(models)
}

// MustInitialize works like Initialize, but panics if
// there's an error.
func (o *Orm) MustInitialize() {
	if err := o.Initialize(); err != nil {
		panic(err)
	}
}

func (o *Orm) fields(table string, s *types.Struct) (*driver.Fields, map[string]*reference, error) {
	methods, err := driver.MakeMethods(s.Type)
	if err != nil {
		return nil, nil, err
	}
	fields := &driver.Fields{
		Struct:     s,
		PrimaryKey: -1,
		Methods:    methods,
	}
	var references map[string]*reference
	for ii, v := range s.QNames {
		// XXX: Check if this quoting is enough
		fields.QuotedNames = append(fields.QuotedNames, fmt.Sprintf("\"%s\".\"%s\"", table, s.MNames[ii]))
		t := s.Types[ii]
		k := t.Kind()
		ftag := s.Tags[ii]
		// Check encoded types
		if c := codec.FromTag(ftag); c != nil {
			if err := c.Try(t, o.dtags()); err != nil {
				return nil, nil, fmt.Errorf("can't encode field %q as %s: %s", v, c.Name(), err)
			}
		} else if ftag.CodecName() != "" {
			return nil, nil, fmt.Errorf("can't find ORM codec %q. Perhaps you missed an import?", ftag.CodecName())
		} else {
			switch k {
			case reflect.Array, reflect.Chan, reflect.Func, reflect.Interface, reflect.Map:
				return nil, nil, fmt.Errorf("field %q in struct %v has invalid type %v", v, t, k)
			}
		}
		fields.OmitEmpty = append(fields.OmitEmpty, ftag.Has("omitempty") || (ftag.Has("auto_increment") && !ftag.Has("notomitempty")))
		// Struct has flattened types, but we need to original type
		// to determine if it should be nullempty by default
		field := s.Type.FieldByIndex(s.Indexes[ii])
		fields.NullEmpty = append(fields.NullEmpty, ftag.Has("nullempty") || (defaultsToNullEmpty(field.Type, ftag) && !ftag.Has("notnullempty")))
		if ftag.Has("primary_key") {
			if fields.PrimaryKey >= 0 {
				return nil, nil, fmt.Errorf("duplicate primary_key in struct %v (%s and %s)", s.Type, s.QNames[fields.PrimaryKey], v)
			}
			fields.PrimaryKey = ii
			if ftag.Has("auto_increment") && (k == reflect.Int || k == reflect.Int8 || k == reflect.Int16 || k == reflect.Int32 || k == reflect.Int64) {
				fields.IntegerAutoincrementPk = true
			}
		}
		if ref := ftag.Value("references"); ref != "" {
			m := referencesRe.FindStringSubmatch(ref)
			if len(m) != 4 {
				return nil, nil, fmt.Errorf("field %q has invalid references %q. Must be in the form references=Model or references=Model(Field)", v, ref)
			}
			if references == nil {
				references = make(map[string]*reference)
			}
			references[v] = &reference{model: m[1], field: m[3]}
		}
	}
	return fields, references, nil
}

func (o *Orm) dtags() []string {
	return append(o.driver.Tags(), "orm")
}

// returns wheter the kind defaults to nullempty option
func defaultsToNullEmpty(typ reflect.Type, t *types.Tag) bool {
	if t.Has("references") || t.Has("codec") {
		return true
	}
	switch typ.Kind() {
	case reflect.Slice, reflect.Ptr, reflect.Interface, reflect.String:
		return true
	case reflect.Struct:
		return typ == timeType
	}
	return false
}

// Returns the default table name for a type
func defaultTableName(typ reflect.Type) string {
	n := typ.Name()
	if p := typ.PkgPath(); p != "main" {
		n = strings.Replace(p, "/", "_", -1) + n
	}
	return util.CamelCaseToLower(n, "_")
}

func typeName(typ reflect.Type) string {
	if p := typ.PkgPath(); p != "main" {
		return p + "." + typ.Name()
	}
	return typ.Name()
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Ludovic Fauvet
// Licensed under the MIT license

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"sort"
	"strconv"
	"strings"
)

var (
	TemplatesNotFound = errors.New("Please set a valid path to the templates directory.")
)

type PageRenderer interface {
	Write(ctx *Context, page *MirrorlistPage) (int, error)
	Type() string
}

type JsonRenderer struct{}

func (w *JsonRenderer) Type() string {
	return "JSON"
}

func (w *JsonRenderer) Write(ctx *Context, page *MirrorlistPage) (statusCode int, err error) {
	var output []byte
	if ctx.IsPretty() {
		output, err = json.MarshalIndent(page, "", "    ")
	} else {
		output, err = json.Marshal(page)
	}
	if err != nil {
		return http.StatusInternalServerError, err
	}
	ctx.ResponseWriter().Header().Set("Content-Type", "application/json; charset=utf-8")
	ctx.ResponseWriter().Header().Set("Content-Length", strconv.Itoa(len(output)))
	ctx.ResponseWriter().Write(output)
	return http.StatusOK, nil
}

type RedirectRenderer struct{}

func (w *RedirectRenderer) Type() string {
	return "REDIRECT"
}

func (w *RedirectRenderer) Write(ctx *Context, page *MirrorlistPage) (statusCode int, err error) {
	if len(page.MirrorList) > 0 {
		ctx.ResponseWriter().Header().Set("Content-Type", "text/html; charset=utf-8")

		for i, m := range page.MirrorList[1:] {
			ctx.ResponseWriter().Header().Add("Link", fmt.Sprintf("<%s>; rel=duplicate; pri=%d; geo=%s", m.HttpURL+page.FileInfo.Path[1:], i+1, strings.ToLower(m.CountryFields[0])))
		}
		http.Redirect(ctx.ResponseWriter(), ctx.Request(), page.MirrorList[0].HttpURL+page.FileInfo.Path[1:], http.StatusFound)
		return http.StatusFound, nil
	}
	http.NotFound(ctx.ResponseWriter(), ctx.Request())
	return http.StatusNotFound, nil
}

type MirrorListRenderer struct{}

func (w *MirrorListRenderer) Type() string {
	return "MIRRORLIST"
}

func (w *MirrorListRenderer) Write(ctx *Context, page *MirrorlistPage) (statusCode int, err error) {
	if ctx.Templates().mirrorlist == nil {
		return http.StatusInternalServerError, TemplatesNotFound
	}
	sort.Sort(ByExcludeReason{page.ExcludedList})
	page.MapURL = getMirrorMapUrl(page.MirrorList, page.ClientInfo)
	ctx.ResponseWriter().Header().Set("Content-Type", "text/html; charset=utf-8")
	err = ctx.Templates().mirrorlist.ExecuteTemplate(ctx.ResponseWriter(), "base", page)
	if err != nil {
		return http.StatusInternalServerError, err
	}
	return http.StatusOK, err
}
<commit_msg>pagerenderer: render the mirrorlist page into a buffer before writing it to the socket<commit_after>// Copyright (c) 2014 Ludovic Fauvet
// Licensed under the MIT license

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"sort"
	"strconv"
	"strings"
)

var (
	TemplatesNotFound = errors.New("Please set a valid path to the templates directory.")
)

type PageRenderer interface {
	Write(ctx *Context, page *MirrorlistPage) (int, error)
	Type() string
}

type JsonRenderer struct{}

func (w *JsonRenderer) Type() string {
	return "JSON"
}

func (w *JsonRenderer) Write(ctx *Context, page *MirrorlistPage) (statusCode int, err error) {
	var output []byte
	if ctx.IsPretty() {
		output, err = json.MarshalIndent(page, "", "    ")
	} else {
		output, err = json.Marshal(page)
	}
	if err != nil {
		return http.StatusInternalServerError, err
	}
	ctx.ResponseWriter().Header().Set("Content-Type", "application/json; charset=utf-8")
	ctx.ResponseWriter().Header().Set("Content-Length", strconv.Itoa(len(output)))
	ctx.ResponseWriter().Write(output)
	return http.StatusOK, nil
}

type RedirectRenderer struct{}

func (w *RedirectRenderer) Type() string {
	return "REDIRECT"
}

func (w *RedirectRenderer) Write(ctx *Context, page *MirrorlistPage) (statusCode int, err error) {
	if len(page.MirrorList) > 0 {
		ctx.ResponseWriter().Header().Set("Content-Type", "text/html; charset=utf-8")

		for i, m := range page.MirrorList[1:] {
			ctx.ResponseWriter().Header().Add("Link", fmt.Sprintf("<%s>; rel=duplicate; pri=%d; geo=%s", m.HttpURL+page.FileInfo.Path[1:], i+1, strings.ToLower(m.CountryFields[0])))
		}
		http.Redirect(ctx.ResponseWriter(), ctx.Request(), page.MirrorList[0].HttpURL+page.FileInfo.Path[1:], http.StatusFound)
		return http.StatusFound, nil
	}
	http.NotFound(ctx.ResponseWriter(), ctx.Request())
	return http.StatusNotFound, nil
}

type MirrorListRenderer struct{}

func (w *MirrorListRenderer) Type() string {
	return "MIRRORLIST"
}

func (w *MirrorListRenderer) Write(ctx *Context, page *MirrorlistPage) (statusCode int, err error) {
	if ctx.Templates().mirrorlist == nil {
		return http.StatusInternalServerError, TemplatesNotFound
	}
	sort.Sort(ByExcludeReason{page.ExcludedList})
	var buf bytes.Buffer
	page.MapURL = getMirrorMapUrl(page.MirrorList, page.ClientInfo)
	ctx.ResponseWriter().Header().Set("Content-Type", "text/html; charset=utf-8")
	err = ctx.Templates().mirrorlist.ExecuteTemplate(&buf, "base", page)
	if err != nil {
		return http.StatusInternalServerError, err
	}
	buf.WriteTo(ctx.ResponseWriter())
	return http.StatusOK, nil
}
<|endoftext|>


<commit_before>// Package passwordhash implements safe password hashing and comparison.
//
// Passwords are derived using PBKDF2-HMAC-SHA256 function with 5000 iterations
// (by default), 32-byte salt and 64-byte output.
package passwordhash

import (
	"crypto/rand"
	"crypto/sha256"
	"crypto/subtle"
	"fmt"
	"github.com/dchest/pbkdf2"
)

// PasswordHash represents storage for password hash and salt.
type PasswordHash struct {
	Iter int
	Salt []byte
	Hash []byte
}

const (
	// Default number of iterations for PBKDF2
	DefaultIterations = 5000
	// Default salt length
	SaltLen = 32
)

// getSalt returns a new random salt.
func getSalt() []byte {
	salt := make([]byte, SaltLen)
	if _, err := rand.Reader.Read(salt); err != nil {
		panic("can't read from random source: " + err.String())
	}
	return salt
}

// New returns a new password hash derived from the provided password,
// a random salt, and the default number of iterations.
func New(password string) *PasswordHash {
	return NewSaltIter(password, getSalt(), DefaultIterations)
}

// NewIter returns a new password hash derived from the provided password,
// the number of iterations, and a random salt.
func NewIter(password string, iter int) *PasswordHash {
	return NewSaltIter(password, getSalt(), iter)
}

// NewSaltIter creates a new password hash from the provided password, salt,
// and the number of iterations.
func NewSaltIter(password string, salt []byte, iter int) *PasswordHash {
	return &PasswordHash{iter, salt,
		pbkdf2.WithHMAC(sha256.New, []byte(password), salt, iter, 64)}
}

// EqualToPassword returns true if the password hash was derived from the provided password.
// This function uses constant time comparison.
func (ph *PasswordHash) EqualToPassword(password string) bool {
	provided := NewSaltIter(password, ph.Salt, ph.Iter)
	return subtle.ConstantTimeCompare(ph.Hash, provided.Hash) == 1
}

// String returns a string representation of the password hash.
func (ph *PasswordHash) String() string {
	return fmt.Sprintf("&PasswordHash{Iterations: %d, Salt: %x, Hash: %x}",
		ph.Iter, ph.Salt, ph.Hash)
}
<commit_msg>Update String() method with the new Iter name.<commit_after>// Package passwordhash implements safe password hashing and comparison.
//
// Passwords are derived using PBKDF2-HMAC-SHA256 function with 5000 iterations
// (by default), 32-byte salt and 64-byte output.
package passwordhash

import (
	"crypto/rand"
	"crypto/sha256"
	"crypto/subtle"
	"fmt"
	"github.com/dchest/pbkdf2"
)

// PasswordHash represents storage for password hash and salt.
type PasswordHash struct {
	Iter int
	Salt []byte
	Hash []byte
}

const (
	// Default number of iterations for PBKDF2
	DefaultIterations = 5000
	// Default salt length
	SaltLen = 32
)

// getSalt returns a new random salt.
func getSalt() []byte {
	salt := make([]byte, SaltLen)
	if _, err := rand.Reader.Read(salt); err != nil {
		panic("can't read from random source: " + err.String())
	}
	return salt
}

// New returns a new password hash derived from the provided password,
// a random salt, and the default number of iterations.
func New(password string) *PasswordHash {
	return NewSaltIter(password, getSalt(), DefaultIterations)
}

// NewIter returns a new password hash derived from the provided password,
// the number of iterations, and a random salt.
func NewIter(password string, iter int) *PasswordHash {
	return NewSaltIter(password, getSalt(), iter)
}

// NewSaltIter creates a new password hash from the provided password, salt,
// and the number of iterations.
func NewSaltIter(password string, salt []byte, iter int) *PasswordHash {
	return &PasswordHash{iter, salt,
		pbkdf2.WithHMAC(sha256.New, []byte(password), salt, iter, 64)}
}

// EqualToPassword returns true if the password hash was derived from the provided password.
// This function uses constant time comparison.
func (ph *PasswordHash) EqualToPassword(password string) bool {
	provided := NewSaltIter(password, ph.Salt, ph.Iter)
	return subtle.ConstantTimeCompare(ph.Hash, provided.Hash) == 1
}

// String returns a string representation of the password hash.
func (ph *PasswordHash) String() string {
	return fmt.Sprintf("&PasswordHash{Iter: %d, Salt: %x, Hash: %x}",
		ph.Iter, ph.Salt, ph.Hash)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"errors"
	"fmt"
	"io"
	"net/textproto"
	"strings"

	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/armor"
)

// PGPEncryptor implements PGP encryption; use PGPTransformer.NewEncryptor
// to get a properly configured instance.
type PGPEncryptor struct {
	pgpBuffer   *bytes.Buffer
	outBuffer   *bytes.Buffer
	plainBuffer *bytes.Buffer
	pgpWriter   io.WriteCloser
	asciiWriter io.WriteCloser
	keepHeaders []string
	headers     textproto.MIMEHeader
}

// NewPGPEncryptor returns a new PGPEncryptor instance, prepared for encrypting one single
// mail with the given parameters.
func NewPGPEncryptor(signingKey, encryptionKey *openpgp.Entity, keepHeaders []string) (*PGPEncryptor, error) {
	if encryptionKey == nil {
		return nil, errors.New("missing encryption key")
	}
	e := &PGPEncryptor{}
	e.keepHeaders = keepHeaders
	e.pgpBuffer = &bytes.Buffer{}
	e.plainBuffer = &bytes.Buffer{}
	var err error
	e.asciiWriter, err = armor.Encode(e.pgpBuffer, "PGP MESSAGE", nil)
	if err != nil {
		return nil, err
	}
	e.pgpWriter, err = openpgp.Encrypt(e.asciiWriter,
		[]*openpgp.Entity{encryptionKey}, signingKey,
		&openpgp.FileHints{IsBinary: true}, nil)
	if err != nil {
		return nil, err
	}
	return e, nil
}

// Write passes the given data to the underlying PGP encryptor.
func (e *PGPEncryptor) Write(data []byte) (int, error) {
	_, err := e.plainBuffer.Write(data)
	if err != nil {
		return 0, err
	}
	return e.pgpWriter.Write(data)
}

// GetBytes returns the encrypted message as a byte array.
func (e *PGPEncryptor) GetBytes() ([]byte, error) {
	err := e.finalizePGP()
	if err != nil {
		return nil, err
	}
	err = e.finalizeMIME()
	if err != nil {
		return nil, err
	}
	return e.outBuffer.Bytes(), nil
}

// finalizePGP ends the PGP encryption process and ascii-encoding process.
func (e *PGPEncryptor) finalizePGP() error {
	err := e.pgpWriter.Close()
	if err != nil {
		return err
	}
	return e.asciiWriter.Close()
}

// finalizeMIME finally encodes the PGP ascii-armored data in a MIME message.
func (e *PGPEncryptor) finalizeMIME() error {
	e.outBuffer = &bytes.Buffer{}
	err := e.writePlainHeaders()
	if err != nil {
		return err
	}
	return e.writeMIMEStructure()
}

// writePlainHeaders generates Message-Id and copies all the plain headers which are
// configured to be copied up from the original message to the output buffer.
func (e *PGPEncryptor) writePlainHeaders() error {
	plainReader := bufio.NewReader(e.plainBuffer)
	mimeReader := textproto.NewReader(plainReader)
	var err error
	e.headers, err = mimeReader.ReadMIMEHeader()
	if err != nil {
		return err
	}
	ctype := e.headers.Get("Content-Type")
	if strings.HasPrefix(ctype, "multipart/encrypted") {
		return errors.New("already encrypted")
	}
	e.writeMessageID()
	e.writeKeptHeaders()
	e.writeLemoncryptHeader()
	return nil
}

// writeMessageID outputs the current message's adapted message id.
func (e *PGPEncryptor) writeMessageID() {
	msgid := e.headers.Get("Message-Id")
	if msgid != "" {
		if len(msgid) > 1 && msgid[0] == '<' {
			msgid = msgid[0:1] + msgIDPrefix + msgid[1:]
		} else {
			msgid += msgIDPrefix + msgid[1:]
		}
	}
	e.outBuffer.WriteString("Message-Id: " + msgid + "\n")
}

func (e *PGPEncryptor) writeLemoncryptHeader() {
	e.outBuffer.WriteString(CustomHeader + ": v0.1\n")
}

// writeKeptHeaders outputs the current message's copied plaintext headers.
func (e *PGPEncryptor) writeKeptHeaders() {
	for _, key := range e.keepHeaders {
		val := e.headers.Get(key)
		if val == "" {
			// don't attempt to copy empty headers
			continue
		}
		//FIXME proper line wrapping; not obvious how go does it
		e.outBuffer.WriteString(key + ": " + val + "\n")
	}
}

// writeMIMEStructure writes the basic MIME structure and the encrypted content
// to the output buffer.
func (e *PGPEncryptor) writeMIMEStructure() error {
	boundary, err := generateBoundary()
	if err != nil {
		return err
	}
	e.outBuffer.WriteString(
		"MIME-Version: 1.0\n" +
			"Content-Type: multipart/encrypted;\n" +
			" protocol=\"application/pgp-encrypted\";\n" +
			" boundary=\"" + boundary + "\"\n\n" +
			"OpenPGP/MIME\n" +
			"--" + boundary + "\n" +
			"Content-Type: application/pgp-encrypted\n\n" +
			"Version: 1\n\n" +
			"--" + boundary + "\n" +
			"Content-Type: application/octet-stream; name=\"encrypted.asc\"\n" +
			"Content-Disposition: inline; filename=\"encrypted.asc\"\n\n" +
			string(e.pgpBuffer.Bytes()) + "\n--" + boundary + "--")
	return nil
}

// generateBoundary creates a random boundary string suitable for
// MIME part separation.
func generateBoundary() (string, error) {
	tmp := make([]byte, 30)
	_, err := io.ReadFull(rand.Reader, tmp)
	if err != nil {
		return "", err
	}
	boundary := fmt.Sprintf("%x", tmp)
	return boundary, nil
}
<commit_msg>pgp: use SHA256/AES256 by default<commit_after>package main

import (
	"bufio"
	"bytes"
	"crypto"
	"crypto/rand"
	"errors"
	"fmt"
	"io"
	"net/textproto"
	"strings"

	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/armor"
	"golang.org/x/crypto/openpgp/packet"
)

// PGPEncryptor implements PGP encryption; use PGPTransformer.NewEncryptor
// to get a properly configured instance.
type PGPEncryptor struct {
	pgpBuffer   *bytes.Buffer
	outBuffer   *bytes.Buffer
	plainBuffer *bytes.Buffer
	pgpWriter   io.WriteCloser
	asciiWriter io.WriteCloser
	keepHeaders []string
	headers     textproto.MIMEHeader
}

// NewPGPEncryptor returns a new PGPEncryptor instance, prepared for encrypting one single
// mail with the given parameters.
func NewPGPEncryptor(signingKey, encryptionKey *openpgp.Entity, keepHeaders []string) (*PGPEncryptor, error) {
	if encryptionKey == nil {
		return nil, errors.New("missing encryption key")
	}
	e := &PGPEncryptor{}
	e.keepHeaders = keepHeaders
	e.pgpBuffer = &bytes.Buffer{}
	e.plainBuffer = &bytes.Buffer{}
	var err error
	e.asciiWriter, err = armor.Encode(e.pgpBuffer, "PGP MESSAGE", nil)
	if err != nil {
		return nil, err
	}
	cfg := &packet.Config{
		DefaultCipher: packet.CipherAES256,
		DefaultHash:   crypto.SHA256,
	}
	e.pgpWriter, err = openpgp.Encrypt(e.asciiWriter,
		[]*openpgp.Entity{encryptionKey}, signingKey,
		&openpgp.FileHints{IsBinary: true}, cfg)
	if err != nil {
		return nil, err
	}
	return e, nil
}

// Write passes the given data to the underlying PGP encryptor.
func (e *PGPEncryptor) Write(data []byte) (int, error) {
	_, err := e.plainBuffer.Write(data)
	if err != nil {
		return 0, err
	}
	return e.pgpWriter.Write(data)
}

// GetBytes returns the encrypted message as a byte array.
func (e *PGPEncryptor) GetBytes() ([]byte, error) {
	err := e.finalizePGP()
	if err != nil {
		return nil, err
	}
	err = e.finalizeMIME()
	if err != nil {
		return nil, err
	}
	return e.outBuffer.Bytes(), nil
}

// finalizePGP ends the PGP encryption process and ascii-encoding process.
func (e *PGPEncryptor) finalizePGP() error {
	err := e.pgpWriter.Close()
	if err != nil {
		return err
	}
	return e.asciiWriter.Close()
}

// finalizeMIME finally encodes the PGP ascii-armored data in a MIME message.
func (e *PGPEncryptor) finalizeMIME() error {
	e.outBuffer = &bytes.Buffer{}
	err := e.writePlainHeaders()
	if err != nil {
		return err
	}
	return e.writeMIMEStructure()
}

// writePlainHeaders generates Message-Id and copies all the plain headers which are
// configured to be copied up from the original message to the output buffer.
func (e *PGPEncryptor) writePlainHeaders() error {
	plainReader := bufio.NewReader(e.plainBuffer)
	mimeReader := textproto.NewReader(plainReader)
	var err error
	e.headers, err = mimeReader.ReadMIMEHeader()
	if err != nil {
		return err
	}
	ctype := e.headers.Get("Content-Type")
	if strings.HasPrefix(ctype, "multipart/encrypted") {
		return errors.New("already encrypted")
	}
	e.writeMessageID()
	e.writeKeptHeaders()
	e.writeLemoncryptHeader()
	return nil
}

// writeMessageID outputs the current message's adapted message id.
func (e *PGPEncryptor) writeMessageID() {
	msgid := e.headers.Get("Message-Id")
	if msgid != "" {
		if len(msgid) > 1 && msgid[0] == '<' {
			msgid = msgid[0:1] + msgIDPrefix + msgid[1:]
		} else {
			msgid += msgIDPrefix + msgid[1:]
		}
	}
	e.outBuffer.WriteString("Message-Id: " + msgid + "\n")
}

func (e *PGPEncryptor) writeLemoncryptHeader() {
	e.outBuffer.WriteString(CustomHeader + ": v0.1\n")
}

// writeKeptHeaders outputs the current message's copied plaintext headers.
func (e *PGPEncryptor) writeKeptHeaders() {
	for _, key := range e.keepHeaders {
		val := e.headers.Get(key)
		if val == "" {
			// don't attempt to copy empty headers
			continue
		}
		//FIXME proper line wrapping; not obvious how go does it
		e.outBuffer.WriteString(key + ": " + val + "\n")
	}
}

// writeMIMEStructure writes the basic MIME structure and the encrypted content
// to the output buffer.
func (e *PGPEncryptor) writeMIMEStructure() error {
	boundary, err := generateBoundary()
	if err != nil {
		return err
	}
	e.outBuffer.WriteString(
		"MIME-Version: 1.0\n" +
			"Content-Type: multipart/encrypted;\n" +
			" protocol=\"application/pgp-encrypted\";\n" +
			" boundary=\"" + boundary + "\"\n\n" +
			"OpenPGP/MIME\n" +
			"--" + boundary + "\n" +
			"Content-Type: application/pgp-encrypted\n\n" +
			"Version: 1\n\n" +
			"--" + boundary + "\n" +
			"Content-Type: application/octet-stream; name=\"encrypted.asc\"\n" +
			"Content-Disposition: inline; filename=\"encrypted.asc\"\n\n" +
			string(e.pgpBuffer.Bytes()) + "\n--" + boundary + "--")
	return nil
}

// generateBoundary creates a random boundary string suitable for
// MIME part separation.
func generateBoundary() (string, error) {
	tmp := make([]byte, 30)
	_, err := io.ReadFull(rand.Reader, tmp)
	if err != nil {
		return "", err
	}
	boundary := fmt.Sprintf("%x", tmp)
	return boundary, nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"strings"
)

// AcceptTypes is a map of string to struct.
type AcceptTypes map[string]struct{}

// GetAcceptedTestTypes validates the AcceptedTestTypes of a VerifyCodeRequest
// and upconverts the list into a set (map) of the effective accepted test types.
// If an invalid test type is in the API request, an error is returned.
func (v *VerifyCodeRequest) GetAcceptedTestTypes() (AcceptTypes, error) {
	accepted := AcceptTypes{}
	if len(v.AcceptTestTypes) == 0 {
		accepted.AddAcceptTypes(TestTypeConfirmed, TestTypeLikely, TestTypeNegative)
		return accepted, nil
	}

	for _, testType := range v.AcceptTestTypes {
		testType = strings.ToLower(testType)
		switch testType {
		case TestTypeConfirmed:
			accepted.AddAcceptTypes(TestTypeConfirmed)
		case TestTypeLikely:
			accepted.AddAcceptTypes(TestTypeConfirmed, TestTypeLikely)
		case TestTypeNegative:
			accepted.AddAcceptTypes(TestTypeConfirmed, TestTypeLikely, TestTypeNegative)
		case TestTypeUserReport:
			accepted.AddAcceptTypes(TestTypeConfirmed, TestTypeUserReport)
		default:
			return nil, fmt.Errorf("invalid accepted test type: %v", testType)
		}
	}

	return accepted, nil
}

// AddAcceptTypes takes a list of acceptable types and converts it to a map.
func (a AcceptTypes) AddAcceptTypes(types ...string) {
	for _, t := range types {
		a[t] = struct{}{}
	}
}
<commit_msg>add user report to default accept (#2043)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"strings"
)

// AcceptTypes is a map of string to struct.
type AcceptTypes map[string]struct{}

// GetAcceptedTestTypes validates the AcceptedTestTypes of a VerifyCodeRequest
// and upconverts the list into a set (map) of the effective accepted test types.
// If an invalid test type is in the API request, an error is returned.
func (v *VerifyCodeRequest) GetAcceptedTestTypes() (AcceptTypes, error) {
	accepted := AcceptTypes{}
	if len(v.AcceptTestTypes) == 0 {
		accepted.AddAcceptTypes(TestTypeConfirmed, TestTypeLikely, TestTypeNegative, TestTypeUserReport)
		return accepted, nil
	}

	for _, testType := range v.AcceptTestTypes {
		testType = strings.ToLower(testType)
		switch testType {
		case TestTypeConfirmed:
			accepted.AddAcceptTypes(TestTypeConfirmed)
		case TestTypeLikely:
			accepted.AddAcceptTypes(TestTypeConfirmed, TestTypeLikely)
		case TestTypeNegative:
			accepted.AddAcceptTypes(TestTypeConfirmed, TestTypeLikely, TestTypeNegative)
		case TestTypeUserReport:
			accepted.AddAcceptTypes(TestTypeConfirmed, TestTypeUserReport)
		default:
			return nil, fmt.Errorf("invalid accepted test type: %v", testType)
		}
	}

	return accepted, nil
}

// AddAcceptTypes takes a list of acceptable types and converts it to a map.
func (a AcceptTypes) AddAcceptTypes(types ...string) {
	for _, t := range types {
		a[t] = struct{}{}
	}
}
<|endoftext|>


<commit_before>package cfg

import (
	"flag"
	"os"

	"github.com/cortexproject/cortex/pkg/util/flagext"
	"github.com/pkg/errors"
)

// Defaults registers flags to the command line using dst as the
// flagext.Registerer
func Defaults() Source {
	return dDefaults(flag.CommandLine)
}

// dDefaults registers flags to the flagSet using dst as the flagext.Registerer
func dDefaults(fs *flag.FlagSet) Source {
	return func(dst interface{}) error {
		r, ok := dst.(flagext.Registerer)
		if !ok {
			return errors.New("dst does not satisfy flagext.Registerer")
		}

		// already sets the defaults on r
		r.RegisterFlags(fs)
		return nil
	}
}

// Flags parses the flag from the command line, setting only user-supplied
// values on the flagext.Registerer passed to Defaults()
func Flags() Source {
	return dFlags(flag.CommandLine, os.Args[1:])
}

// dFlags parses the flagset, applying all values set on the slice
func dFlags(fs *flag.FlagSet, args []string) Source {
	return func(dst interface{}) error {
		// parse the final flagset
		return fs.Parse(args)
	}
}
<commit_msg>feat(cli): order flags by categories (#1168)<commit_after>package cfg

import (
	"flag"
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/cortexproject/cortex/pkg/util/flagext"
	"github.com/pkg/errors"
)

// Defaults registers flags to the command line using dst as the
// flagext.Registerer
func Defaults() Source {
	return dDefaults(flag.CommandLine)
}

// dDefaults registers flags to the flagSet using dst as the flagext.Registerer
func dDefaults(fs *flag.FlagSet) Source {
	return func(dst interface{}) error {
		r, ok := dst.(flagext.Registerer)
		if !ok {
			return errors.New("dst does not satisfy flagext.Registerer")
		}

		// already sets the defaults on r
		r.RegisterFlags(fs)
		return nil
	}
}

// Flags parses the flag from the command line, setting only user-supplied
// values on the flagext.Registerer passed to Defaults()
func Flags() Source {
	flag.Usage = categorizedUsage(flag.CommandLine)
	return dFlags(flag.CommandLine, os.Args[1:])
}

// dFlags parses the flagset, applying all values set on the slice
func dFlags(fs *flag.FlagSet, args []string) Source {
	return func(dst interface{}) error {
		// parse the final flagset
		return fs.Parse(args)
	}
}

func categorizedUsage(fs *flag.FlagSet) func() {
	categories := make(map[string][]string)
	return func() {
		if fs.Name() == "" {
			fmt.Fprintf(fs.Output(), "Usage:\n")
		} else {
			fmt.Fprintf(fs.Output(), "Usage of %s:\n", fs.Name())
		}

		fs.VisitAll(func(f *flag.Flag) {
			id := ""
			if strings.Contains(f.Name, ".") {
				id = strings.Split(f.Name, ".")[0]
			}

			kind, usage := flag.UnquoteUsage(f)
			if kind != "" {
				kind = " " + kind
			}
			def := f.DefValue
			if def != "" {
				def = fmt.Sprintf(" (default %s)", def)
			}
			categories[id] = append(categories[id], fmt.Sprintf("   -%s%s:\n      %s%s", f.Name, kind, usage, def))
		})

		for name, flags := range categories {
			if len(flags) == 1 {
				categories[""] = append(categories[""], flags[0])
				delete(categories, name)
			}
		}

		for name := range categories {
			sort.Strings(categories[name])
		}

		for _, u := range categories[""] {
			fmt.Fprintln(fs.Output(), u)
		}
		fmt.Fprintln(fs.Output())

		keys := make([]string, 0, len(categories))
		for k := range categories {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, name := range keys {
			if name == "" {
				continue
			}
			fmt.Fprintf(fs.Output(), " %s:\n", strings.Title(name))
			for _, u := range categories[name] {
				fmt.Fprintln(fs.Output(), u)
			}
			fmt.Fprintln(fs.Output())
		}
	}
}
<|endoftext|>


<commit_before>package geo

import "testing"

func TestPolygonCompare(t *testing.T) {
	// Same
	for _, testcase := range []struct {
		P1 Polygon
		P2 *Polygon
	}{
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
			P2: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
		},
	} {

		if same := testcase.P1.Compare(testcase.P2); !same {
			t.Fatalf("expected %s and %s to be the same", testcase.P1.String(), testcase.P2.String())
		}
	}
	// Different
	for _, testcase := range []struct {
		P1 Polygon
		P2 Geometry
	}{
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
			P2: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
				},
			},
		},
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
				},
			},
			P2: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
				},
			},
		},
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
			P2: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.4, 7.3},
				},
			},
		},
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
			P2: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.5},
				},
			},
		},
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
			P2: &Line{
				{1.2, 3.4},
				{5.6, 7.8},
				{1.4, 9.3},
				{-1.7, 7.5},
			},
		},
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
			P2: &Point{1.2, 3.4},
		},
	} {

		if same := testcase.P1.Compare(testcase.P2); same {
			t.Fatalf("expected %s to not equal %s", testcase.P1.String(), testcase.P2.String())
		}
	}
}

func TestPolygonMarshal(t *testing.T) {
	// Pass
	marshalTestcases{
		{
			Input: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
				},
			},
			Expected: `{"type":"Polygon","coordinates":[[[1.2,3.4],[5.6,7.8]]]}`,
		},
		{
			Input: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{2, 6},
				},
				{
					{0, 0},
					{0, 1},
					{1, 0},
				},
			},
			Expected: `{"type":"Polygon","coordinates":[[[1.2,3.4],[5.6,7.8],[2,6]],[[0,0],[0,1],[1,0]]]}`,
		},
	}.pass(t)
}

func TestPolygonScan(t *testing.T) {
	// Pass
	for _, testcase := range []struct {
		WKT      string
		Expected Polygon
	}{
		{
			WKT: "POLYGON((1.2 3.4, 5.6 7.8, 6.2 1.5, 1.2 3.4))",
			Expected: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{6.2, 1.5},
					{1.2, 3.4},
				},
			},
		},
	} {
		p := &Polygon{}
		if err := p.Scan(testcase.WKT); err != nil {
			t.Fatal(err)
		}
		for i, coord := range testcase.Expected {
			if expected, got := coord[0], (*p)[i][0]; expected != got {
				t.Fatalf("expected %f, got %f", expected, got)
			}
			if expected, got := coord[1], (*p)[i][1]; expected != got {
				t.Fatalf("expected %f, got %f", expected, got)
			}
		}
	}
	// Fail
	for _, testcase := range []interface{}{
		"POLYGON((1.2, 3.4, 5.6, 7.8))",
		[]byte("POLYGON((1.2, 3.4, 5.6, 7.8))"),
		7,
		"POLYGON(1.2 3.4 5.6 7.8)",
		"POLYGON((1.2 3.4 5.6 7.8)}",
		"PIKACHU",
	} {
		p := &Polygon{}
		if err := p.Scan(testcase); err == nil {
			t.Fatalf("expected err, got nil for %s", testcase.(string))
		}
	}
}

func TestPolygonValue(t *testing.T) {
	var (
		p = Polygon{
			{
				{1.2, 3.4},
				{5.6, 7.8},
				{8.7, 6.5},
				{4.3, 2.1},
			},
		}
		expected = `POLYGON((1.2 3.4, 5.6 7.8, 8.7 6.5, 4.3 2.1))`
	)
	value, err := p.Value()
	if err != nil {
		t.Fatal(err)
	}
	got, ok := value.(string)
	if !ok {
		t.Fatalf("expected string, got %T", value)
	}
	if expected != got {
		t.Fatalf("expected %s, got %s", expected, got)
	}
}

func TestPolygonEmpty(t *testing.T) {
	var (
		p        = Polygon{}
		expected = "POLYGON EMPTY"
	)
	value, err := p.Value()
	if err != nil {
		t.Fatal(err)
	}
	got, ok := value.(string)
	if !ok {
		t.Fatalf("expected string, got %T", value)
	}
	if expected != got {
		t.Fatalf("expected %s, got %s", expected, got)
	}
}

func TestPolygonContains(t *testing.T) {
	for _, testcase := range []struct {
		Poly    Polygon
		Inside  []Point
		Outside []Point
	}{
		// Square
		{
			Poly: Polygon{
				{
					{0, 0},
					{2, 0},
					{2, 2},
					{0, 2},
					{0, 0},
				},
			},
			Inside: []Point{
				{1, 1},
			},
			Outside: []Point{
				{4, 1},
			},
		},
		// Hexagon
		{
			Poly: Polygon{
				{
					{0, 1},
					{1, 2},
					{2, 1},
					{2, 0},
					{1, -1},
					{0, 0},
					{0, 1},
				},
			},
			Inside: []Point{
				{1, 0},
			},
		},
		// A tilted quadrilateral
		{
			Poly: Polygon{
				{
					{-1, 10},
					{10, 1},
					{1, -10},
					{-10, -1},
					{-1, 10},
				},
			},
			Inside: []Point{
				{2, 2},
				{2, -2},
			},
		},
	} {
		if testcase.Inside != nil {
			for _, point := range testcase.Inside {
				if !testcase.Poly.Contains(point) {
					t.Fatalf("Expected polygon %v to contain point %v", testcase.Poly, point)
				}
			}
		}
		if testcase.Outside != nil {
			for _, point := range testcase.Outside {
				if testcase.Poly.Contains(point) {
					t.Fatalf("Expected polygon %v to not contain point %v", testcase.Poly, point)
				}
			}
		}
	}
}

func TestPolygonString(t *testing.T) {
	for _, c := range []struct {
		Input    Polygon
		Expected string
	}{
		{
			Input: Polygon{
				{
					{-4, -4},
					{-4, 4},
					{4, 4},
					{4, -4},
				},
				{
					{0, 0},
					{1, 0},
					{0, 1},
				},
			},
			Expected: `POLYGON((-4 -4, -4 4, 4 4, 4 -4),(0 0, 1 0, 0 1))`,
		},
	} {
		if expected, got := c.Expected, c.Input.String(); expected != got {
			t.Fatalf("expected %s, got %s", expected, got)
		}
	}
}
<commit_msg>add polygon scan test<commit_after>package geo

import "testing"

func TestPolygonCompare(t *testing.T) {
	// Same
	for _, testcase := range []struct {
		P1 Polygon
		P2 *Polygon
	}{
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
			P2: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
		},
	} {

		if same := testcase.P1.Compare(testcase.P2); !same {
			t.Fatalf("expected %s and %s to be the same", testcase.P1.String(), testcase.P2.String())
		}
	}
	// Different
	for _, testcase := range []struct {
		P1 Polygon
		P2 Geometry
	}{
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
			P2: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
				},
			},
		},
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
				},
			},
			P2: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
				},
			},
		},
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
			P2: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.4, 7.3},
				},
			},
		},
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
			P2: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.5},
				},
			},
		},
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
			P2: &Line{
				{1.2, 3.4},
				{5.6, 7.8},
				{1.4, 9.3},
				{-1.7, 7.5},
			},
		},
		{
			P1: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{1.4, 9.3},
					{-1.7, 7.3},
				},
			},
			P2: &Point{1.2, 3.4},
		},
	} {

		if same := testcase.P1.Compare(testcase.P2); same {
			t.Fatalf("expected %s to not equal %s", testcase.P1.String(), testcase.P2.String())
		}
	}
}

func TestPolygonMarshal(t *testing.T) {
	// Pass
	marshalTestcases{
		{
			Input: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
				},
			},
			Expected: `{"type":"Polygon","coordinates":[[[1.2,3.4],[5.6,7.8]]]}`,
		},
		{
			Input: &Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{2, 6},
				},
				{
					{0, 0},
					{0, 1},
					{1, 0},
				},
			},
			Expected: `{"type":"Polygon","coordinates":[[[1.2,3.4],[5.6,7.8],[2,6]],[[0,0],[0,1],[1,0]]]}`,
		},
	}.pass(t)
}

func TestPolygonScan(t *testing.T) {
	// Pass
	for _, testcase := range []struct {
		WKT      string
		Expected Polygon
	}{
		{
			WKT: "POLYGON((1.2 3.4, 5.6 7.8, 6.2 1.5, 1.2 3.4))",
			Expected: Polygon{
				{
					{1.2, 3.4},
					{5.6, 7.8},
					{6.2, 1.5},
					{1.2, 3.4},
				},
			},
		},
		{
			WKT: `POLYGON((-113.14448537305 33.4192544895836,-113.140408415347 33.4192634445438,-113.140419144183 33.4209917345629,-113.144506830722 33.4210096441239,-113.14448537305 33.4192544895836))`,
			Expected: Polygon{
				{
					{-113.14448537305, 33.4192544895836},
					{-113.140408415347, 33.4192634445438},
					{-113.140419144183, 33.4209917345629},
					{-113.144506830722, 33.4210096441239},
					{-113.14448537305, 33.4192544895836},
				},
			},
		},
	} {
		p := &Polygon{}
		if err := p.Scan(testcase.WKT); err != nil {
			t.Fatal(err)
		}
		for i, coord := range testcase.Expected {
			if expected, got := coord[0], (*p)[i][0]; expected != got {
				t.Fatalf("expected %f, got %f", expected, got)
			}
			if expected, got := coord[1], (*p)[i][1]; expected != got {
				t.Fatalf("expected %f, got %f", expected, got)
			}
		}
	}
	// Fail
	for _, testcase := range []interface{}{
		"POLYGON((1.2, 3.4, 5.6, 7.8))",
		[]byte("POLYGON((1.2, 3.4, 5.6, 7.8))"),
		7,
		"POLYGON(1.2 3.4 5.6 7.8)",
		"POLYGON((1.2 3.4 5.6 7.8)}",
		"PIKACHU",
	} {
		p := &Polygon{}
		if err := p.Scan(testcase); err == nil {
			t.Fatalf("expected err, got nil for %s", testcase.(string))
		}
	}
}

func TestPolygonValue(t *testing.T) {
	var (
		p = Polygon{
			{
				{1.2, 3.4},
				{5.6, 7.8},
				{8.7, 6.5},
				{4.3, 2.1},
			},
		}
		expected = `POLYGON((1.2 3.4, 5.6 7.8, 8.7 6.5, 4.3 2.1))`
	)
	value, err := p.Value()
	if err != nil {
		t.Fatal(err)
	}
	got, ok := value.(string)
	if !ok {
		t.Fatalf("expected string, got %T", value)
	}
	if expected != got {
		t.Fatalf("expected %s, got %s", expected, got)
	}
}

func TestPolygonEmpty(t *testing.T) {
	var (
		p        = Polygon{}
		expected = "POLYGON EMPTY"
	)
	value, err := p.Value()
	if err != nil {
		t.Fatal(err)
	}
	got, ok := value.(string)
	if !ok {
		t.Fatalf("expected string, got %T", value)
	}
	if expected != got {
		t.Fatalf("expected %s, got %s", expected, got)
	}
}

func TestPolygonContains(t *testing.T) {
	for _, testcase := range []struct {
		Poly    Polygon
		Inside  []Point
		Outside []Point
	}{
		// Square
		{
			Poly: Polygon{
				{
					{0, 0},
					{2, 0},
					{2, 2},
					{0, 2},
					{0, 0},
				},
			},
			Inside: []Point{
				{1, 1},
			},
			Outside: []Point{
				{4, 1},
			},
		},
		// Hexagon
		{
			Poly: Polygon{
				{
					{0, 1},
					{1, 2},
					{2, 1},
					{2, 0},
					{1, -1},
					{0, 0},
					{0, 1},
				},
			},
			Inside: []Point{
				{1, 0},
			},
		},
		// A tilted quadrilateral
		{
			Poly: Polygon{
				{
					{-1, 10},
					{10, 1},
					{1, -10},
					{-10, -1},
					{-1, 10},
				},
			},
			Inside: []Point{
				{2, 2},
				{2, -2},
			},
		},
	} {
		if testcase.Inside != nil {
			for _, point := range testcase.Inside {
				if !testcase.Poly.Contains(point) {
					t.Fatalf("Expected polygon %v to contain point %v", testcase.Poly, point)
				}
			}
		}
		if testcase.Outside != nil {
			for _, point := range testcase.Outside {
				if testcase.Poly.Contains(point) {
					t.Fatalf("Expected polygon %v to not contain point %v", testcase.Poly, point)
				}
			}
		}
	}
}

func TestPolygonString(t *testing.T) {
	for _, c := range []struct {
		Input    Polygon
		Expected string
	}{
		{
			Input: Polygon{
				{
					{-4, -4},
					{-4, 4},
					{4, 4},
					{4, -4},
				},
				{
					{0, 0},
					{1, 0},
					{0, 1},
				},
			},
			Expected: `POLYGON((-4 -4, -4 4, 4 4, 4 -4),(0 0, 1 0, 0 1))`,
		},
	} {
		if expected, got := c.Expected, c.Input.String(); expected != got {
			t.Fatalf("expected %s, got %s", expected, got)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Russell Haering.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package queue

import (
	"sync"

	"tux21b.org/v1/gocql"
)

type BatchResult struct {
	idx int64
	err error
}

type QueueItemBatchRequest struct {
	items      []QueueItemValue
	resultChan chan BatchResult
}

type queueWriter struct {
	id              string
	nextIndex       int64
	pendingRequests chan *QueueItemBatchRequest
}

func newQueueWriter(mgr *QueueManager, id string) (*queueWriter, error) {
	var lastIndex int64
	err := mgr.db.Query(`SELECT item_id FROM queue_items WHERE queue_id = ? ORDER BY item_id DESC LIMIT 1`, id).Scan(&lastIndex)
	// Scan returns an ErrNotFound if the writer didn't previously exist. If
	// that happens we default lastIndex to -1 so that nextIndex will be 0. If
	// any other error occurs, return it.
	if err == gocql.ErrNotFound {
		lastIndex = -1
	} else if err != nil {
		return nil, err
	}

	writer := &queueWriter{
		id:              id,
		nextIndex:       lastIndex + int64(1),
		pendingRequests: make(chan *QueueItemBatchRequest),
	}

	go writer.process(mgr.db, mgr.done)
	return writer, nil
}

func (writer *queueWriter) publish(items []QueueItemValue) (idx int64, err error) {
	request := QueueItemBatchRequest{
		items:      items,
		resultChan: make(chan BatchResult),
	}

	defer func() {
		if r := recover(); r != nil {
			err = ErrManagerShutdown
		}
	}()

	writer.pendingRequests <- &request

	result := <-request.resultChan
	return result.idx, result.err
}

func (writer *queueWriter) process(db *gocql.Session, done *sync.WaitGroup) {
	done.Add(1)

	for {
		request := <-writer.pendingRequests

		if request == nil {
			break
		}

		// We got a request - drain any additional requests that are pending, then
		// flush them all to the database
		requests := []*QueueItemBatchRequest{request}
		requests = append(requests, writer.drainPending()...)
		writer.writeBatch(db, requests)
	}
	done.Done()
}

func (writer *queueWriter) shutdown() {
	close(writer.pendingRequests)
}

func (writer *queueWriter) drainPending() []*QueueItemBatchRequest {
	requests := []*QueueItemBatchRequest{}

	for {
		select {
		case request := <-writer.pendingRequests:
			if request == nil {
				// This will happen (_not_ the default case) if the channel is closed
				return requests
			}
			requests = append(requests, request)

		default:
			return requests
		}
	}
}

func (writer *queueWriter) writeBatch(db *gocql.Session, requests []*QueueItemBatchRequest) {
	dbBatch := gocql.NewBatch(gocql.UnloggedBatch)
	i := int64(0)

	for _, request := range requests {
		for _, item := range request.items {
			itemID := writer.nextIndex + i
			dbBatch.Query(`INSERT INTO queue_items (queue_id, item_id, item_value) VALUES (?, ?, ?)`, writer.id, itemID, item)
			i++
		}
	}

	err := db.ExecuteBatch(dbBatch)
	writer.respond(requests, err)
}

func (writer *queueWriter) respond(requests []*QueueItemBatchRequest, err error) {
	i := int64(0)

	for _, request := range requests {
		result := BatchResult{err: err}

		if err == nil {
			result.idx = writer.nextIndex + i
		}

		request.resultChan <- result
		i += int64(len(request.items))
	}

	writer.nextIndex += i
}
<commit_msg>make all queue writer types private<commit_after>// Copyright 2013 Russell Haering.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package queue

import (
	"sync"

	"tux21b.org/v1/gocql"
)

type batchResult struct {
	idx int64
	err error
}

type batchRequest struct {
	items      []QueueItemValue
	resultChan chan batchResult
}

type queueWriter struct {
	id              string
	nextIndex       int64
	pendingRequests chan *batchRequest
}

func newQueueWriter(mgr *QueueManager, id string) (*queueWriter, error) {
	var lastIndex int64
	err := mgr.db.Query(`SELECT item_id FROM queue_items WHERE queue_id = ? ORDER BY item_id DESC LIMIT 1`, id).Scan(&lastIndex)
	// Scan returns an ErrNotFound if the writer didn't previously exist. If
	// that happens we default lastIndex to -1 so that nextIndex will be 0. If
	// any other error occurs, return it.
	if err == gocql.ErrNotFound {
		lastIndex = -1
	} else if err != nil {
		return nil, err
	}

	writer := &queueWriter{
		id:              id,
		nextIndex:       lastIndex + int64(1),
		pendingRequests: make(chan *batchRequest),
	}

	go writer.process(mgr.db, mgr.done)
	return writer, nil
}

func (writer *queueWriter) publish(items []QueueItemValue) (idx int64, err error) {
	request := batchRequest{
		items:      items,
		resultChan: make(chan batchResult),
	}

	defer func() {
		if r := recover(); r != nil {
			err = ErrManagerShutdown
		}
	}()

	writer.pendingRequests <- &request

	result := <-request.resultChan
	return result.idx, result.err
}

func (writer *queueWriter) process(db *gocql.Session, done *sync.WaitGroup) {
	done.Add(1)

	for {
		request := <-writer.pendingRequests

		if request == nil {
			break
		}

		// We got a request - drain any additional requests that are pending, then
		// flush them all to the database
		requests := []*batchRequest{request}
		requests = append(requests, writer.drainPending()...)
		writer.writeBatch(db, requests)
	}
	done.Done()
}

func (writer *queueWriter) shutdown() {
	close(writer.pendingRequests)
}

func (writer *queueWriter) drainPending() []*batchRequest {
	requests := []*batchRequest{}

	for {
		select {
		case request := <-writer.pendingRequests:
			if request == nil {
				// This will happen (_not_ the default case) if the channel is closed
				return requests
			}
			requests = append(requests, request)

		default:
			return requests
		}
	}
}

func (writer *queueWriter) writeBatch(db *gocql.Session, requests []*batchRequest) {
	dbBatch := gocql.NewBatch(gocql.UnloggedBatch)
	i := int64(0)

	for _, request := range requests {
		for _, item := range request.items {
			itemID := writer.nextIndex + i
			dbBatch.Query(`INSERT INTO queue_items (queue_id, item_id, item_value) VALUES (?, ?, ?)`, writer.id, itemID, item)
			i++
		}
	}

	err := db.ExecuteBatch(dbBatch)
	writer.respond(requests, err)
}

func (writer *queueWriter) respond(requests []*batchRequest, err error) {
	i := int64(0)

	for _, request := range requests {
		result := batchResult{err: err}

		if err == nil {
			result.idx = writer.nextIndex + i
		}

		request.resultChan <- result
		i += int64(len(request.items))
	}

	writer.nextIndex += i
}
<|endoftext|>


<commit_before>package quickfix

import "honnef.co/go/tools/analysis/lint"

var Docs = lint.Markdownify(map[string]*lint.Documentation{
	"QF1000": {
		Title: "Use byte-specific indexing function",
		Since: "Unreleased",
	},
	"QF1001": {
		Title: "Apply De Morgan's law",
		Since: "Unreleased",
	},
	"QF1002": {
		Title: "Convert untagged switch to tagged switch",
		Text: `An untagged switch that compares a single variable against a series of values can be replaced with a tagged switch.

Before:

	switch {
	case x == 1 || x == 2, x == 3:
		...
	case x == 4:
		...
	default:
		...
	}

After:

	switch x {
	case 1, 2, 3:
		...
	case 4:
		...
	default:
		...
	}
`,
		Since: "Unreleased",
	},
	"QF1003": {
		Title: "Convert if/else-if chain to tagged switch",
		Text: ` A series of if/else-if checks comparing the same variable against values can be replaced with a tagged switch.

Before:

	if x == 1 || x == 2 {
		...
	} else if x == 3 {
		...
	} else {
		...
	}

After:

	switch x {
	case 1, 2:
		...
	case 3:
		...
	default:
		...
	}`,
		Since: "Unreleased",
	},
	"QF1004": {
		Since: "Unreleased",
	},
	"QF1005": {
		Title: "Expand call to `math.Pow`",
		Text: `Some uses of \'math.Pow\' can be simplified to basic multiplication.

Before:

	math.Pow(x, 2)

After:

	x * x`,
		Since: "Unreleased",
	},

	"QF1006": {
		Title: "Lift if+break into loop condition",
		Text: `Before:

	for {
		if done {
			break
		}
		...
	}

After:

	for !done {
		...
	}`,
		Since: "Unreleased",
	},

	"QF1007": {
		Title: "Merge conditional assignment into variable declaration",
		Text: `Before:

	x := false
	if someCondition {
		x = true
	}

After:

	x := someCondition
`,
		Since: "Unreleased",
	},

	"QF1008": {
		Title: "Omit embedded fields from selector expression",
		Since: "Unreleased",
	},
})
<commit_msg>quickfix: remove superfluous space in documentation<commit_after>package quickfix

import "honnef.co/go/tools/analysis/lint"

var Docs = lint.Markdownify(map[string]*lint.Documentation{
	"QF1000": {
		Title: "Use byte-specific indexing function",
		Since: "Unreleased",
	},
	"QF1001": {
		Title: "Apply De Morgan's law",
		Since: "Unreleased",
	},
	"QF1002": {
		Title: "Convert untagged switch to tagged switch",
		Text: `An untagged switch that compares a single variable against a series of values can be replaced with a tagged switch.

Before:

	switch {
	case x == 1 || x == 2, x == 3:
		...
	case x == 4:
		...
	default:
		...
	}

After:

	switch x {
	case 1, 2, 3:
		...
	case 4:
		...
	default:
		...
	}
`,
		Since: "Unreleased",
	},
	"QF1003": {
		Title: "Convert if/else-if chain to tagged switch",
		Text: `A series of if/else-if checks comparing the same variable against values can be replaced with a tagged switch.

Before:

	if x == 1 || x == 2 {
		...
	} else if x == 3 {
		...
	} else {
		...
	}

After:

	switch x {
	case 1, 2:
		...
	case 3:
		...
	default:
		...
	}`,
		Since: "Unreleased",
	},
	"QF1004": {
		Since: "Unreleased",
	},
	"QF1005": {
		Title: "Expand call to `math.Pow`",
		Text: `Some uses of \'math.Pow\' can be simplified to basic multiplication.

Before:

	math.Pow(x, 2)

After:

	x * x`,
		Since: "Unreleased",
	},

	"QF1006": {
		Title: "Lift if+break into loop condition",
		Text: `Before:

	for {
		if done {
			break
		}
		...
	}

After:

	for !done {
		...
	}`,
		Since: "Unreleased",
	},

	"QF1007": {
		Title: "Merge conditional assignment into variable declaration",
		Text: `Before:

	x := false
	if someCondition {
		x = true
	}

After:

	x := someCondition
`,
		Since: "Unreleased",
	},

	"QF1008": {
		Title: "Omit embedded fields from selector expression",
		Since: "Unreleased",
	},
})
<|endoftext|>


<commit_before>// 12 december 2015

package ui

import (
	"unsafe"
)

// #include "ui.h"
import "C"

// RadioButtons is a Control that represents a set of checkable
// buttons from which exactly one may be chosen by the user.
type RadioButtons struct {
	c	*C.uiControl
	r	*C.uiRadioButtons
}

// NewRadioButtons creates a new RadioButtons.
func NewRadioButtons() *RadioButtons {
	r := new(RadioButtons)

	r.r = C.uiNewRadioButtons()
	r.c = (*C.uiControl)(unsafe.Pointer(r.r))

	return r
}

// Destroy destroys the RadioButtons.
func (r *RadioButtons) Destroy() {
	C.uiControlDestroy(r.c)
}

// LibuiControl returns the libui uiControl pointer that backs
// the Window. This is only used by package ui itself and should
// not be called by programs.
func (r *RadioButtons) LibuiControl() uintptr {
	return uintptr(unsafe.Pointer(r.c))
}

// Handle returns the OS-level handle associated with this RadioButtons.
// On Windows this is an HWND of a libui-internal class; its
// child windows are instances of the standard Windows API
// BUTTON class (as provided by Common Controls version 6).
// On GTK+ this is a pointer to a GtkBox containing GtkRadioButtons.
// On OS X this is a pointer to a NSView with each radio button as a NSButton subview.
func (r *RadioButtons) Handle() uintptr {
	return uintptr(C.uiControlHandle(r.c))
}

// Show shows the RadioButtons.
func (r *RadioButtons) Show() {
	C.uiControlShow(r.c)
}

// Hide hides the RadioButtons.
func (r *RadioButtons) Hide() {
	C.uiControlHide(r.c)
}

// Enable enables the RadioButtons.
func (r *RadioButtons) Enable() {
	C.uiControlEnable(r.c)
}

// Disable disables the RadioButtons.
func (r *RadioButtons) Disable() {
	C.uiControlDisable(r.c)
}

// Append adds the named button to the end of the RadioButtons.
// If this button is the first button, it is automatically selected.
func (r *RadioButtons) Append(text string) {
	ctext := C.CString(text)
	C.uiRadioButtonsAppend(r.r, ctext)
	freestr(ctext)
}
<commit_msg>radio button selection support<commit_after>// 12 december 2015

package ui

import "unsafe"

// #include "ui.h"
// extern void doRadioButtonsOnSelected(uiRadioButtons *, void *);
// static inline void realuiRadioButtonsOnSelected(uiRadioButtons *r)
// {
// 	uiRadioButtonsOnSelected(r, doRadioButtonsOnSelected, NULL);
// }
import "C"

// no need to lock this; only the GUI thread can access it
var radioButtons = make(map[*C.uiRadioButtons]*RadioButtons)

// RadioButtons is a Control that represents a set of checkable
// buttons from which exactly one may be chosen by the user.
type RadioButtons struct {
	c *C.uiControl
	r *C.uiRadioButtons

	onSelected func(*RadioButtons)
}

// NewRadioButtons creates a new RadioButtons.
func NewRadioButtons() *RadioButtons {
	r := new(RadioButtons)

	r.r = C.uiNewRadioButtons()
	r.c = (*C.uiControl)(unsafe.Pointer(r.r))

	C.realuiRadioButtonsOnSelected(r.r)
	radioButtons[r.r] = r

	return r
}

// Destroy destroys the RadioButtons.
func (r *RadioButtons) Destroy() {
	C.uiControlDestroy(r.c)
}

// LibuiControl returns the libui uiControl pointer that backs
// the Window. This is only used by package ui itself and should
// not be called by programs.
func (r *RadioButtons) LibuiControl() uintptr {
	return uintptr(unsafe.Pointer(r.c))
}

// Handle returns the OS-level handle associated with this RadioButtons.
// On Windows this is an HWND of a libui-internal class; its
// child windows are instances of the standard Windows API
// BUTTON class (as provided by Common Controls version 6).
// On GTK+ this is a pointer to a GtkBox containing GtkRadioButtons.
// On OS X this is a pointer to a NSView with each radio button as a NSButton subview.
func (r *RadioButtons) Handle() uintptr {
	return uintptr(C.uiControlHandle(r.c))
}

// Show shows the RadioButtons.
func (r *RadioButtons) Show() {
	C.uiControlShow(r.c)
}

// Hide hides the RadioButtons.
func (r *RadioButtons) Hide() {
	C.uiControlHide(r.c)
}

// Enable enables the RadioButtons.
func (r *RadioButtons) Enable() {
	C.uiControlEnable(r.c)
}

// Disable disables the RadioButtons.
func (r *RadioButtons) Disable() {
	C.uiControlDisable(r.c)
}

// Append adds the named button to the end of the RadioButtons.
// If this button is the first button, it is automatically selected.
func (r *RadioButtons) Append(text string) {
	ctext := C.CString(text)
	C.uiRadioButtonsAppend(r.r, ctext)
	freestr(ctext)
}

// Selected returns the index of the currently selected item in the
// RadioButtons.
func (r *RadioButtons) Selected() int {
	return int(C.uiRadioButtonsSelected(r.r))
}

// SetSelected sets the currently select item in the RadioButtons
// to index.
func (r *RadioButtons) SetSelected(index int) {
	C.uiRadioButtonsSetSelected(r.r, C.int(index))
}

// OnSelected registers f to be run when the user selects an item in
// the RadioButtons. Only one function can be registered at a time.
func (r *RadioButtons) OnSelected(f func(*RadioButtons)) {
	r.onSelected = f
}

//export doRadioButtonsOnSelected
func doRadioButtonsOnSelected(rr *C.uiRadioButtons, data unsafe.Pointer) {
	r := radioButtons[rr]
	if r.onSelected != nil {
		r.onSelected(r)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package raft

import (
	"errors"

	pb "go.etcd.io/etcd/raft/v3/raftpb"
	"go.etcd.io/etcd/raft/v3/tracker"
)

// ErrStepLocalMsg is returned when try to step a local raft message
var ErrStepLocalMsg = errors.New("raft: cannot step raft local message")

// ErrStepPeerNotFound is returned when try to step a response message
// but there is no peer found in raft.prs for that node.
var ErrStepPeerNotFound = errors.New("raft: cannot step as peer not found")

// RawNode is a thread-unsafe Node.
// The methods of this struct correspond to the methods of Node and are described
// more fully there.
type RawNode struct {
	raft       *raft
	prevSoftSt *SoftState
	prevHardSt pb.HardState
}

// NewRawNode instantiates a RawNode from the given configuration.
//
// See Bootstrap() for bootstrapping an initial state; this replaces the former
// 'peers' argument to this method (with identical behavior). However, It is
// recommended that instead of calling Bootstrap, applications bootstrap their
// state manually by setting up a Storage that has a first index > 1 and which
// stores the desired ConfState as its InitialState.
func NewRawNode(config *Config) (*RawNode, error) {
	r := newRaft(config)
	rn := &RawNode{
		raft: r,
	}
	rn.prevSoftSt = r.softState()
	rn.prevHardSt = r.hardState()
	return rn, nil
}

// Tick advances the internal logical clock by a single tick.
func (rn *RawNode) Tick() {
	rn.raft.tick()
}

// TickQuiesced advances the internal logical clock by a single tick without
// performing any other state machine processing. It allows the caller to avoid
// periodic heartbeats and elections when all of the peers in a Raft group are
// known to be at the same state. Expected usage is to periodically invoke Tick
// or TickQuiesced depending on whether the group is "active" or "quiesced".
//
// WARNING: Be very careful about using this method as it subverts the Raft
// state machine. You should probably be using Tick instead.
func (rn *RawNode) TickQuiesced() {
	rn.raft.electionElapsed++
}

// Campaign causes this RawNode to transition to candidate state.
func (rn *RawNode) Campaign() error {
	return rn.raft.Step(pb.Message{
		Type: pb.MsgHup,
	})
}

// Propose proposes data be appended to the raft log.
func (rn *RawNode) Propose(data []byte) error {
	return rn.raft.Step(pb.Message{
		Type: pb.MsgProp,
		From: rn.raft.id,
		Entries: []pb.Entry{
			{Data: data},
		}})
}

// ProposeConfChange proposes a config change. See (Node).ProposeConfChange for
// details.
func (rn *RawNode) ProposeConfChange(cc pb.ConfChangeI) error {
	m, err := confChangeToMsg(cc)
	if err != nil {
		return err
	}
	return rn.raft.Step(m)
}

// ApplyConfChange applies a config change to the local node. The app must call
// this when it applies a configuration change, except when it decides to reject
// the configuration change, in which case no call must take place.
func (rn *RawNode) ApplyConfChange(cc pb.ConfChangeI) *pb.ConfState {
	cs := rn.raft.applyConfChange(cc.AsV2())
	return &cs
}

// Step advances the state machine using the given message.
func (rn *RawNode) Step(m pb.Message) error {
	// ignore unexpected local messages receiving over network
	if IsLocalMsg(m.Type) {
		return ErrStepLocalMsg
	}
	if pr := rn.raft.prs.Progress[m.From]; pr != nil || !IsResponseMsg(m.Type) {
		return rn.raft.Step(m)
	}
	return ErrStepPeerNotFound
}

// Ready returns the outstanding work that the application needs to handle. This
// includes appending and applying entries or a snapshot, updating the HardState,
// and sending messages. The returned Ready() *must* be handled and subsequently
// passed back via Advance().
func (rn *RawNode) Ready() Ready {
	rd := rn.readyWithoutAccept()
	rn.acceptReady(rd)
	return rd
}

// readyWithoutAccept returns a Ready. This is a read-only operation, i.e. there
// is no obligation that the Ready must be handled.
func (rn *RawNode) readyWithoutAccept() Ready {
	return newReady(rn.raft, rn.prevSoftSt, rn.prevHardSt)
}

// acceptReady is called when the consumer of the RawNode has decided to go
// ahead and handle a Ready. Nothing must alter the state of the RawNode between
// this call and the prior call to Ready().
func (rn *RawNode) acceptReady(rd Ready) {
	if rd.SoftState != nil {
		rn.prevSoftSt = rd.SoftState
	}
	if len(rd.ReadStates) != 0 {
		rn.raft.readStates = nil
	}
	rn.raft.msgs = nil
}

// HasReady called when RawNode user need to check if any Ready pending.
// Checking logic in this method should be consistent with Ready.containsUpdates().
func (rn *RawNode) HasReady() bool {
	r := rn.raft
	if !r.softState().equal(rn.prevSoftSt) {
		return true
	}
	if hardSt := r.hardState(); !IsEmptyHardState(hardSt) && !isHardStateEqual(hardSt, rn.prevHardSt) {
		return true
	}
	if r.raftLog.hasPendingSnapshot() {
		return true
	}
	if len(r.msgs) > 0 || len(r.raftLog.unstableEntries()) > 0 || r.raftLog.hasNextEnts() {
		return true
	}
	if len(r.readStates) != 0 {
		return true
	}
	return false
}

// Advance notifies the RawNode that the application has applied and saved progress in the
// last Ready results.
func (rn *RawNode) Advance(rd Ready) {
	if !IsEmptyHardState(rd.HardState) {
		rn.prevHardSt = rd.HardState
	}
	rn.raft.advance(rd)
}

// Status returns the current status of the given group. This allocates, see
// BasicStatus and WithProgress for allocation-friendlier choices.
func (rn *RawNode) Status() Status {
	status := getStatus(rn.raft)
	return status
}

// BasicStatus returns a BasicStatus. Notably this does not contain the
// Progress map; see WithProgress for an allocation-free way to inspect it.
func (rn *RawNode) BasicStatus() BasicStatus {
	return getBasicStatus(rn.raft)
}

// ProgressType indicates the type of replica a Progress corresponds to.
type ProgressType byte

const (
	// ProgressTypePeer accompanies a Progress for a regular peer replica.
	ProgressTypePeer ProgressType = iota
	// ProgressTypeLearner accompanies a Progress for a learner replica.
	ProgressTypeLearner
)

// WithProgress is a helper to introspect the Progress for this node and its
// peers.
func (rn *RawNode) WithProgress(visitor func(id uint64, typ ProgressType, pr tracker.Progress)) {
	rn.raft.prs.Visit(func(id uint64, pr *tracker.Progress) {
		typ := ProgressTypePeer
		if pr.IsLearner {
			typ = ProgressTypeLearner
		}
		p := *pr
		p.Inflights = nil
		visitor(id, typ, p)
	})
}

// ReportUnreachable reports the given node is not reachable for the last send.
func (rn *RawNode) ReportUnreachable(id uint64) {
	_ = rn.raft.Step(pb.Message{Type: pb.MsgUnreachable, From: id})
}

// ReportSnapshot reports the status of the sent snapshot.
func (rn *RawNode) ReportSnapshot(id uint64, status SnapshotStatus) {
	rej := status == SnapshotFailure

	_ = rn.raft.Step(pb.Message{Type: pb.MsgSnapStatus, From: id, Reject: rej})
}

// TransferLeader tries to transfer leadership to the given transferee.
func (rn *RawNode) TransferLeader(transferee uint64) {
	_ = rn.raft.Step(pb.Message{Type: pb.MsgTransferLeader, From: transferee})
}

// ReadIndex requests a read state. The read state will be set in ready.
// Read State has a read index. Once the application advances further than the read
// index, any linearizable read requests issued before the read request can be
// processed safely. The read state will have the same rctx attached.
func (rn *RawNode) ReadIndex(rctx []byte) {
	_ = rn.raft.Step(pb.Message{Type: pb.MsgReadIndex, Entries: []pb.Entry{{Data: rctx}}})
}
<commit_msg>raft: update prevHardSt on Ready accept, not advance<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package raft

import (
	"errors"

	pb "go.etcd.io/etcd/raft/v3/raftpb"
	"go.etcd.io/etcd/raft/v3/tracker"
)

// ErrStepLocalMsg is returned when try to step a local raft message
var ErrStepLocalMsg = errors.New("raft: cannot step raft local message")

// ErrStepPeerNotFound is returned when try to step a response message
// but there is no peer found in raft.prs for that node.
var ErrStepPeerNotFound = errors.New("raft: cannot step as peer not found")

// RawNode is a thread-unsafe Node.
// The methods of this struct correspond to the methods of Node and are described
// more fully there.
type RawNode struct {
	raft       *raft
	prevSoftSt *SoftState
	prevHardSt pb.HardState
}

// NewRawNode instantiates a RawNode from the given configuration.
//
// See Bootstrap() for bootstrapping an initial state; this replaces the former
// 'peers' argument to this method (with identical behavior). However, It is
// recommended that instead of calling Bootstrap, applications bootstrap their
// state manually by setting up a Storage that has a first index > 1 and which
// stores the desired ConfState as its InitialState.
func NewRawNode(config *Config) (*RawNode, error) {
	r := newRaft(config)
	rn := &RawNode{
		raft: r,
	}
	rn.prevSoftSt = r.softState()
	rn.prevHardSt = r.hardState()
	return rn, nil
}

// Tick advances the internal logical clock by a single tick.
func (rn *RawNode) Tick() {
	rn.raft.tick()
}

// TickQuiesced advances the internal logical clock by a single tick without
// performing any other state machine processing. It allows the caller to avoid
// periodic heartbeats and elections when all of the peers in a Raft group are
// known to be at the same state. Expected usage is to periodically invoke Tick
// or TickQuiesced depending on whether the group is "active" or "quiesced".
//
// WARNING: Be very careful about using this method as it subverts the Raft
// state machine. You should probably be using Tick instead.
func (rn *RawNode) TickQuiesced() {
	rn.raft.electionElapsed++
}

// Campaign causes this RawNode to transition to candidate state.
func (rn *RawNode) Campaign() error {
	return rn.raft.Step(pb.Message{
		Type: pb.MsgHup,
	})
}

// Propose proposes data be appended to the raft log.
func (rn *RawNode) Propose(data []byte) error {
	return rn.raft.Step(pb.Message{
		Type: pb.MsgProp,
		From: rn.raft.id,
		Entries: []pb.Entry{
			{Data: data},
		}})
}

// ProposeConfChange proposes a config change. See (Node).ProposeConfChange for
// details.
func (rn *RawNode) ProposeConfChange(cc pb.ConfChangeI) error {
	m, err := confChangeToMsg(cc)
	if err != nil {
		return err
	}
	return rn.raft.Step(m)
}

// ApplyConfChange applies a config change to the local node. The app must call
// this when it applies a configuration change, except when it decides to reject
// the configuration change, in which case no call must take place.
func (rn *RawNode) ApplyConfChange(cc pb.ConfChangeI) *pb.ConfState {
	cs := rn.raft.applyConfChange(cc.AsV2())
	return &cs
}

// Step advances the state machine using the given message.
func (rn *RawNode) Step(m pb.Message) error {
	// ignore unexpected local messages receiving over network
	if IsLocalMsg(m.Type) {
		return ErrStepLocalMsg
	}
	if pr := rn.raft.prs.Progress[m.From]; pr != nil || !IsResponseMsg(m.Type) {
		return rn.raft.Step(m)
	}
	return ErrStepPeerNotFound
}

// Ready returns the outstanding work that the application needs to handle. This
// includes appending and applying entries or a snapshot, updating the HardState,
// and sending messages. The returned Ready() *must* be handled and subsequently
// passed back via Advance().
func (rn *RawNode) Ready() Ready {
	rd := rn.readyWithoutAccept()
	rn.acceptReady(rd)
	return rd
}

// readyWithoutAccept returns a Ready. This is a read-only operation, i.e. there
// is no obligation that the Ready must be handled.
func (rn *RawNode) readyWithoutAccept() Ready {
	return newReady(rn.raft, rn.prevSoftSt, rn.prevHardSt)
}

// acceptReady is called when the consumer of the RawNode has decided to go
// ahead and handle a Ready. Nothing must alter the state of the RawNode between
// this call and the prior call to Ready().
func (rn *RawNode) acceptReady(rd Ready) {
	if rd.SoftState != nil {
		rn.prevSoftSt = rd.SoftState
	}
	if !IsEmptyHardState(rd.HardState) {
		rn.prevHardSt = rd.HardState
	}
	if len(rd.ReadStates) != 0 {
		rn.raft.readStates = nil
	}
	rn.raft.msgs = nil
}

// HasReady called when RawNode user need to check if any Ready pending.
// Checking logic in this method should be consistent with Ready.containsUpdates().
func (rn *RawNode) HasReady() bool {
	r := rn.raft
	if !r.softState().equal(rn.prevSoftSt) {
		return true
	}
	if hardSt := r.hardState(); !IsEmptyHardState(hardSt) && !isHardStateEqual(hardSt, rn.prevHardSt) {
		return true
	}
	if r.raftLog.hasPendingSnapshot() {
		return true
	}
	if len(r.msgs) > 0 || len(r.raftLog.unstableEntries()) > 0 || r.raftLog.hasNextEnts() {
		return true
	}
	if len(r.readStates) != 0 {
		return true
	}
	return false
}

// Advance notifies the RawNode that the application has applied and saved progress in the
// last Ready results.
func (rn *RawNode) Advance(rd Ready) {
	rn.raft.advance(rd)
}

// Status returns the current status of the given group. This allocates, see
// BasicStatus and WithProgress for allocation-friendlier choices.
func (rn *RawNode) Status() Status {
	status := getStatus(rn.raft)
	return status
}

// BasicStatus returns a BasicStatus. Notably this does not contain the
// Progress map; see WithProgress for an allocation-free way to inspect it.
func (rn *RawNode) BasicStatus() BasicStatus {
	return getBasicStatus(rn.raft)
}

// ProgressType indicates the type of replica a Progress corresponds to.
type ProgressType byte

const (
	// ProgressTypePeer accompanies a Progress for a regular peer replica.
	ProgressTypePeer ProgressType = iota
	// ProgressTypeLearner accompanies a Progress for a learner replica.
	ProgressTypeLearner
)

// WithProgress is a helper to introspect the Progress for this node and its
// peers.
func (rn *RawNode) WithProgress(visitor func(id uint64, typ ProgressType, pr tracker.Progress)) {
	rn.raft.prs.Visit(func(id uint64, pr *tracker.Progress) {
		typ := ProgressTypePeer
		if pr.IsLearner {
			typ = ProgressTypeLearner
		}
		p := *pr
		p.Inflights = nil
		visitor(id, typ, p)
	})
}

// ReportUnreachable reports the given node is not reachable for the last send.
func (rn *RawNode) ReportUnreachable(id uint64) {
	_ = rn.raft.Step(pb.Message{Type: pb.MsgUnreachable, From: id})
}

// ReportSnapshot reports the status of the sent snapshot.
func (rn *RawNode) ReportSnapshot(id uint64, status SnapshotStatus) {
	rej := status == SnapshotFailure

	_ = rn.raft.Step(pb.Message{Type: pb.MsgSnapStatus, From: id, Reject: rej})
}

// TransferLeader tries to transfer leadership to the given transferee.
func (rn *RawNode) TransferLeader(transferee uint64) {
	_ = rn.raft.Step(pb.Message{Type: pb.MsgTransferLeader, From: transferee})
}

// ReadIndex requests a read state. The read state will be set in ready.
// Read State has a read index. Once the application advances further than the read
// index, any linearizable read requests issued before the read request can be
// processed safely. The read state will have the same rctx attached.
func (rn *RawNode) ReadIndex(rctx []byte) {
	_ = rn.raft.Step(pb.Message{Type: pb.MsgReadIndex, Entries: []pb.Entry{{Data: rctx}}})
}
<|endoftext|>


<commit_before>package receiver

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"strings"
	"sync/atomic"
	"time"

	"github.com/lomik/go-carbon/points"

	"github.com/Sirupsen/logrus"
)

// UDP receive metrics from UDP socket
type UDP struct {
	out                chan *points.Points
	exit               chan bool
	graphPrefix        string
	metricsReceived    uint32
	incompleteReceived uint32
	errors             uint32
	conn               *net.UDPConn
}

// NewUDP create new instance of UDP
func NewUDP(out chan *points.Points) *UDP {
	return &UDP{
		out:  out,
		exit: make(chan bool),
	}
}

type incompleteRecord struct {
	added time.Time
	data  []byte
}

// incompleteStorage store incomplete lines
type incompleteStorage struct {
	Records map[string]*incompleteRecord
	Expires time.Duration
	MaxSize int
}

func newIncompleteStorage() *incompleteStorage {
	return &incompleteStorage{
		Records: make(map[string]*incompleteRecord, 0),
		Expires: 5 * time.Second,
		MaxSize: 10000,
	}
}

func (storage *incompleteStorage) store(addr string, data []byte) {
	storage.Records[addr] = &incompleteRecord{
		added: time.Now(),
		data:  data,
	}
	storage.checkAndClear()
}

func (storage *incompleteStorage) pop(addr string) []byte {
	if record, ok := storage.Records[addr]; ok {
		delete(storage.Records, addr)
		if record.added.Before(time.Now().Add(-storage.Expires)) {
			return nil
		}
		return record.data
	}
	return nil
}

func (storage *incompleteStorage) purge() {
	die := time.Now().Add(-storage.Expires)
	for key, record := range storage.Records {
		if record.added.Before(die) {
			delete(storage.Records, key)
		}
	}
}

func (storage *incompleteStorage) checkAndClear() {
	if len(storage.Records) > storage.MaxSize {
		storage.purge()
	}
}

// Addr returns binded socket address. For bind port 0 in tests
func (rcv *UDP) Addr() net.Addr {
	if rcv.conn == nil {
		return nil
	}
	return rcv.conn.LocalAddr()
}

// SetGraphPrefix for internal cache metrics
func (rcv *UDP) SetGraphPrefix(prefix string) {
	rcv.graphPrefix = prefix
}

// Stat sends internal statistics to cache
func (rcv *UDP) Stat(metric string, value float64) {
	rcv.out <- points.OnePoint(
		fmt.Sprintf("%s%s", rcv.graphPrefix, metric),
		value,
		time.Now().Unix(),
	)
}

// Listen bind port. Receive messages and send to out channel
func (rcv *UDP) Listen(addr *net.UDPAddr) error {
	var err error
	rcv.conn, err = net.ListenUDP("udp", addr)
	if err != nil {
		return err
	}

	go func() {
		ticker := time.NewTicker(time.Minute)
		defer ticker.Stop()

		for {
			select {
			case <-ticker.C:
				cnt := atomic.LoadUint32(&rcv.metricsReceived)
				atomic.AddUint32(&rcv.metricsReceived, -cnt)
				rcv.Stat("udp.metricsReceived", float64(cnt))

				cnt = atomic.LoadUint32(&rcv.incompleteReceived)
				atomic.AddUint32(&rcv.incompleteReceived, -cnt)
				rcv.Stat("udp.incompleteReceived", float64(cnt))

				cnt = atomic.LoadUint32(&rcv.errors)
				atomic.AddUint32(&rcv.errors, -cnt)
				rcv.Stat("udp.errors", float64(cnt))
			case <-rcv.exit:
				rcv.conn.Close()
				return
			}
		}
	}()

	go func() {
		defer rcv.conn.Close()

		var buf [2048]byte

		var data *bytes.Buffer

		lines := newIncompleteStorage()

		for {
			rlen, peer, err := rcv.conn.ReadFromUDP(buf[:])
			if err != nil {
				if strings.Contains(err.Error(), "use of closed network connection") {
					break
				}
				atomic.AddUint32(&rcv.errors, 1)
				logrus.Error(err)
				continue
			}

			prev := lines.pop(peer.String())

			if prev != nil {
				data = bytes.NewBuffer(prev)
				data.Write(buf[:rlen])
			} else {
				data = bytes.NewBuffer(buf[:rlen])
			}

			for {
				line, err := data.ReadBytes('\n')

				if err != nil {
					if err == io.EOF {
						if len(line) > 0 {
							lines.store(peer.String(), line)
							atomic.AddUint32(&rcv.incompleteReceived, 1)
						}
					} else {
						atomic.AddUint32(&rcv.errors, 1)
						logrus.Error(err)
					}
					break
				}
				if len(line) > 0 { // skip empty lines
					if msg, err := points.ParseText(string(line)); err != nil {
						atomic.AddUint32(&rcv.errors, 1)
						logrus.Info(err)
					} else {
						atomic.AddUint32(&rcv.metricsReceived, 1)
						rcv.out <- msg
					}
				}
			}
		}

	}()

	return nil
}

// Stop all listeners
func (rcv *UDP) Stop() {
	close(rcv.exit)
}
<commit_msg>Log udp checkpoint<commit_after>package receiver

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"strings"
	"sync/atomic"
	"time"

	"github.com/lomik/go-carbon/points"

	"github.com/Sirupsen/logrus"
)

// UDP receive metrics from UDP socket
type UDP struct {
	out                chan *points.Points
	exit               chan bool
	graphPrefix        string
	metricsReceived    uint32
	incompleteReceived uint32
	errors             uint32
	conn               *net.UDPConn
}

// NewUDP create new instance of UDP
func NewUDP(out chan *points.Points) *UDP {
	return &UDP{
		out:  out,
		exit: make(chan bool),
	}
}

type incompleteRecord struct {
	added time.Time
	data  []byte
}

// incompleteStorage store incomplete lines
type incompleteStorage struct {
	Records map[string]*incompleteRecord
	Expires time.Duration
	MaxSize int
}

func newIncompleteStorage() *incompleteStorage {
	return &incompleteStorage{
		Records: make(map[string]*incompleteRecord, 0),
		Expires: 5 * time.Second,
		MaxSize: 10000,
	}
}

func (storage *incompleteStorage) store(addr string, data []byte) {
	storage.Records[addr] = &incompleteRecord{
		added: time.Now(),
		data:  data,
	}
	storage.checkAndClear()
}

func (storage *incompleteStorage) pop(addr string) []byte {
	if record, ok := storage.Records[addr]; ok {
		delete(storage.Records, addr)
		if record.added.Before(time.Now().Add(-storage.Expires)) {
			return nil
		}
		return record.data
	}
	return nil
}

func (storage *incompleteStorage) purge() {
	die := time.Now().Add(-storage.Expires)
	for key, record := range storage.Records {
		if record.added.Before(die) {
			delete(storage.Records, key)
		}
	}
}

func (storage *incompleteStorage) checkAndClear() {
	if len(storage.Records) > storage.MaxSize {
		storage.purge()
	}
}

// Addr returns binded socket address. For bind port 0 in tests
func (rcv *UDP) Addr() net.Addr {
	if rcv.conn == nil {
		return nil
	}
	return rcv.conn.LocalAddr()
}

// SetGraphPrefix for internal cache metrics
func (rcv *UDP) SetGraphPrefix(prefix string) {
	rcv.graphPrefix = prefix
}

// Stat sends internal statistics to cache
func (rcv *UDP) Stat(metric string, value float64) {
	rcv.out <- points.OnePoint(
		fmt.Sprintf("%s%s", rcv.graphPrefix, metric),
		value,
		time.Now().Unix(),
	)
}

// Listen bind port. Receive messages and send to out channel
func (rcv *UDP) Listen(addr *net.UDPAddr) error {
	var err error
	rcv.conn, err = net.ListenUDP("udp", addr)
	if err != nil {
		return err
	}

	go func() {
		ticker := time.NewTicker(time.Minute)
		defer ticker.Stop()

		for {
			select {
			case <-ticker.C:
				metricsReceived := atomic.LoadUint32(&rcv.metricsReceived)
				atomic.AddUint32(&rcv.metricsReceived, -metricsReceived)
				rcv.Stat("udp.metricsReceived", float64(metricsReceived))

				incompleteReceived := atomic.LoadUint32(&rcv.incompleteReceived)
				atomic.AddUint32(&rcv.incompleteReceived, -incompleteReceived)
				rcv.Stat("udp.incompleteReceived", float64(incompleteReceived))

				errors := atomic.LoadUint32(&rcv.errors)
				atomic.AddUint32(&rcv.errors, -errors)
				rcv.Stat("udp.errors", float64(errors))

				logrus.WithFields(logrus.Fields{
					"metricsReceived":    metricsReceived,
					"incompleteReceived": incompleteReceived,
					"errors":             errors,
				}).Info("[udp] doCheckpoint()")

			case <-rcv.exit:
				rcv.conn.Close()
				return
			}
		}
	}()

	go func() {
		defer rcv.conn.Close()

		var buf [2048]byte

		var data *bytes.Buffer

		lines := newIncompleteStorage()

		for {
			rlen, peer, err := rcv.conn.ReadFromUDP(buf[:])
			if err != nil {
				if strings.Contains(err.Error(), "use of closed network connection") {
					break
				}
				atomic.AddUint32(&rcv.errors, 1)
				logrus.Error(err)
				continue
			}

			prev := lines.pop(peer.String())

			if prev != nil {
				data = bytes.NewBuffer(prev)
				data.Write(buf[:rlen])
			} else {
				data = bytes.NewBuffer(buf[:rlen])
			}

			for {
				line, err := data.ReadBytes('\n')

				if err != nil {
					if err == io.EOF {
						if len(line) > 0 {
							lines.store(peer.String(), line)
							atomic.AddUint32(&rcv.incompleteReceived, 1)
						}
					} else {
						atomic.AddUint32(&rcv.errors, 1)
						logrus.Error(err)
					}
					break
				}
				if len(line) > 0 { // skip empty lines
					if msg, err := points.ParseText(string(line)); err != nil {
						atomic.AddUint32(&rcv.errors, 1)
						logrus.Info(err)
					} else {
						atomic.AddUint32(&rcv.metricsReceived, 1)
						rcv.out <- msg
					}
				}
			}
		}

	}()

	return nil
}

// Stop all listeners
func (rcv *UDP) Stop() {
	close(rcv.exit)
}
<|endoftext|>