<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"os"
	"text/tabwriter"
	"text/template"
	"time"

	"upspin.io/access"
	"upspin.io/pack"
	"upspin.io/path"
	"upspin.io/upspin"
)

func (s *State) info(args ...string) {
	const help = `
Info prints to standard output a thorough description of all the
information about named paths, including information provided by
ls but also storage references, sizes, and other metadata.

If the path names an Access or Group file, it is also checked for
validity. If it is a link, the command attempts to access the target
of the link.
`
	fs := flag.NewFlagSet("info", flag.ExitOnError)
	s.parseFlags(fs, args, help, "info path...")

	if fs.NArg() == 0 {
		fs.Usage()
	}
	for _, name := range s.globAllUpspin(fs.Args()) {
		// We don't want to follow links, so don't use Client.
		entry, err := s.DirServer().Lookup(name)
		if err != nil && err != upspin.ErrFollowLink {
			s.exit(err)
		}
		s.printInfo(entry)
		switch {
		case access.IsAccessFile(name):
			s.checkAccessFile(name)
		case access.IsGroupFile(name):
			s.checkGroupFile(name)
		}
	}
}

// infoDirEntry wraps a DirEntry to allow new methods for easy formatting.
// It also has fields that hold relevant information as we acquire it.
type infoDirEntry struct {
	*upspin.DirEntry
	state *State
	// The following fields are computed as we run.
	access    *access.Access
	lastUsers string
}

func (d *infoDirEntry) TimeString() string {
	return d.Time.Go().In(time.Local).Format("Mon Jan 2 15:04:05 MST 2006")
}

func (d *infoDirEntry) AttrString() string {
	return attrFormat(d.Attr)
}

func (d *infoDirEntry) Rights() []access.Right {
	return []access.Right{access.Read, access.Write, access.List, access.Create, access.Delete}
}

func (d *infoDirEntry) Readers() string {
	d.state.sharer.addAccess(d.DirEntry)
	d.lastUsers = "<nobody>"
	if d.IsDir() {
		return "is a directory"
	}
	_, users, _, err := d.state.sharer.readers(d.DirEntry)
	if err != nil {
		return err.Error()
	}
	d.lastUsers = users
	return users
}

func (d *infoDirEntry) Sequence() int64 {
	return upspin.SeqVersion(d.DirEntry.Sequence)
}

func (d *infoDirEntry) Hashes() string {
	h := ""
	if d.IsDir() || d.Packing != upspin.EEPack {
		return h
	}
	packer := pack.Lookup(d.Packing)
	hashes, err := packer.ReaderHashes(d.Packdata)
	if err != nil {
		return h
	}
	for _, r := range hashes {
		if h == "" {
			h += " "
		}
		h += fmt.Sprintf("%x...", r[:4])
	}
	return h
}

func (d *infoDirEntry) Users(right access.Right) string {
	users := userListToString(d.state.usersWithAccess(d.state.client, d.access, right))
	if users == d.lastUsers {
		return "(same)"
	}
	d.lastUsers = users
	return users
}

func (d *infoDirEntry) WhichAccess() string {
	var acc *access.Access
	accEntry, err := d.state.whichAccessFollowLinks(d.Name)
	if err != nil {
		return err.Error()
	}
	accFile := "owner only"
	if accEntry == nil {
		// No access file applies.
		acc, err = access.New(d.Name)
		if err != nil {
			// Can't happen, since the name must be valid.
			d.state.exitf("%q: %s", d.Name, err)
		}
	} else {
		accFile = string(accEntry.Name)
		data, err := read(d.state.client, accEntry.Name)
		if err != nil {
			fmt.Fprintf(os.Stderr, "cannot open access file %q: %s\n", accFile, err)
		}
		acc, err = access.Parse(accEntry.Name, data)
		if err != nil {
			fmt.Fprintf(os.Stderr, "cannot parse access file %q: %s\n", accFile, err)
		}
	}
	d.access = acc
	return accFile
}

// printInfo prints, in human-readable form, most of the information about
// the entry, including the users that have permission to access it.
// TODO: Present this more neatly.
// TODO: Present group information.
func (s *State) printInfo(entry *upspin.DirEntry) {
	infoDir := &infoDirEntry{
		state:    s,
		DirEntry: entry,
	}
	writer := tabwriter.NewWriter(os.Stdout, 4, 4, 1, ' ', 0)
	err := infoTmpl.Execute(writer, infoDir)
	if err != nil {
		s.exitf("executing info template: %v", err)
	}
	if writer.Flush() != nil {
		s.exitf("flushing template output: %v", err)
	}
	if !entry.IsLink() {
		return
	}
	// Check and print information about the link target.
	target, err := s.client.Lookup(entry.Link, true)
	if err != nil {
		// Print the whole error indented, starting on the next line. This helps it stand out.
		s.exitf("Error: link %s has invalid target %s:\n\t%v", entry.Name, entry.Link, err)
	}
	fmt.Printf("Target of link %s:\n", entry.Name)
	s.printInfo(target)
}

func attrFormat(attr upspin.Attribute) string {
	tail := ""
	if attr | upspin.AttrIncomplete {
		tail = " (incomplete)"
		attr ^= upspin.AttrIncomplete
	}
	switch {
	case upspin.AttrNone:
		return "none (plain file)"
	case upspin.AttrDirectory:
		return "directory"
	case upspin.AttrLink:
		return "link"
	}
	return fmt.Sprintf("attribute(%#x)", attr)
}

var infoTmpl = template.Must(template.New("info").Parse(infoText))

const infoText = `{{.Name}}
	packing:	{{.Packing}}
	size:	{{.Size}}
	time:	{{.TimeString}}
	writer:	{{.Writer}}
	attributes:	{{.AttrString}}
	sequence:	{{.Sequence}}
	access file:	{{.WhichAccess}}
	key holders: 	{{.Readers}}
	key hashes:     {{.Hashes}}
	{{range $right := .Rights -}}
	can {{$right}}:	{{$.Users $right}}
	{{end -}}
	Block#	Offset	Size	Location
	{{range $index, $block := .Blocks -}}
	{{$index}}	{{.Offset}}	{{.Size}}	{{.Location}}
	{{end}}`

// checkGroupFile diagnoses likely problems with the contents and rights
// of the Group file.
// TODO: We could check that packing is Plain but that should never be a problem.
func (s *State) checkGroupFile(name upspin.PathName) {
	parsed, err := path.Parse(name)
	if err != nil {
		s.exit(err) // Should never happen.
	}
	groupSeen := make(map[upspin.PathName]bool)
	userSeen := make(map[upspin.UserName]bool)
	s.doCheckGroupFile(parsed, groupSeen, userSeen)
}

// doCheckGroupFile is the inner, recursive implementation of checkGroupFile.
func (s *State) doCheckGroupFile(parsed path.Parsed, groupSeen map[upspin.PathName]bool, userSeen map[upspin.UserName]bool) {
	group := parsed.Path()
	if groupSeen[group] {
		return
	}
	groupSeen[group] = true
	data, err := s.client.Get(group)
	if err != nil {
		s.exitf("cannot read Group file: %v", err)
	}

	// Get the Access file, if any, that applies.
	// TODO: We've already got it in earlier code, so could save it.
	whichAccess, err := s.DirServer().WhichAccess(group)
	if err != nil {
		s.exitf("unexpected error finding Access file for Group file %s: %v", group, err)
	}
	var accessFile *access.Access
	if whichAccess == nil {
		accessFile, err = access.New(group)
		if err != nil {
			s.exitf("cannot create default Access file: %v", err)
		}
	} else {
		data, err := s.client.Get(whichAccess.Name)
		if err != nil {
			s.exitf("cannot get Access file: %v", err)
		}
		accessFile, err = access.Parse(whichAccess.Name, data)
		if err != nil {
			s.exitf("cannot parse Access file: %v", err)
		}
	}

	// Each member should be either a plain user or a group and be able to access the Group file.
	members, err := access.ParseGroup(parsed, data)
	if err != nil {
		s.exitf("error parsing Group file %s: %v", group, err)
	}
	for _, member := range members {
		if member.IsRoot() {
			// Normal user.
			user := member.User()
			if !s.userExists(user, userSeen) {
				s.failf("user %s in Group file %s not found in key server", user, group)
				continue
			}
			// Member must be able to read the Group file.
			canRead, err := accessFile.Can(user, access.Read, group, s.client.Get)
			if err != nil {
				s.exitf("error checking permissions in Group file %s for user %s: %v", group, user, err)
				continue
			}
			if !canRead {
				s.failf("user %s is missing read access for group %s", user, group)
			}
			continue
		}
		if !access.IsGroupFile(member.Path()) {
			s.failf("do not understand member %s of Group file %s", member, parsed) // Should never happen.
			continue
		}
		// Member is a group. Recur using Group file.
		s.doCheckGroupFile(member, groupSeen, userSeen)
	}
}

func (s *State) checkAccessFile(name upspin.PathName) {
	data, err := s.client.Get(name)
	if err != nil {
		s.exitf("cannot get Access file: %v", err)
	}
	accessFile, err := access.Parse(name, data)
	if err != nil {
		s.exitf("cannot parse Access file: %v", err)
	}
	users := accessFile.List(access.AnyRight)

	groupSeen := make(map[upspin.PathName]bool)
	userSeen := make(map[upspin.UserName]bool)
	for _, user := range users {
		if user.IsRoot() {
			// Normal user.
			if !s.userExists(user.User(), userSeen) {
				s.failf("user %s in Access file %s not found in key server", user.User(), name)
			}
			continue
		}
		// Member is a group.
		s.doCheckGroupFile(user, groupSeen, userSeen)
	}
}

func (s *State) userExists(user upspin.UserName, userSeen map[upspin.UserName]bool) bool {
	if userSeen[user] || user == access.AllUsers { // all@upspin.io is baked in.
		return true // Previous answer will do.
	}
	userSeen[user] = true
	_, err := s.KeyServer().Lookup(user)
	return err == nil
}
<commit_msg>cmd/upspin: fix build<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"os"
	"text/tabwriter"
	"text/template"
	"time"

	"upspin.io/access"
	"upspin.io/pack"
	"upspin.io/path"
	"upspin.io/upspin"
)

func (s *State) info(args ...string) {
	const help = `
Info prints to standard output a thorough description of all the
information about named paths, including information provided by
ls but also storage references, sizes, and other metadata.

If the path names an Access or Group file, it is also checked for
validity. If it is a link, the command attempts to access the target
of the link.
`
	fs := flag.NewFlagSet("info", flag.ExitOnError)
	s.parseFlags(fs, args, help, "info path...")

	if fs.NArg() == 0 {
		fs.Usage()
	}
	for _, name := range s.globAllUpspin(fs.Args()) {
		// We don't want to follow links, so don't use Client.
		entry, err := s.DirServer().Lookup(name)
		if err != nil && err != upspin.ErrFollowLink {
			s.exit(err)
		}
		s.printInfo(entry)
		switch {
		case access.IsAccessFile(name):
			s.checkAccessFile(name)
		case access.IsGroupFile(name):
			s.checkGroupFile(name)
		}
	}
}

// infoDirEntry wraps a DirEntry to allow new methods for easy formatting.
// It also has fields that hold relevant information as we acquire it.
type infoDirEntry struct {
	*upspin.DirEntry
	state *State
	// The following fields are computed as we run.
	access    *access.Access
	lastUsers string
}

func (d *infoDirEntry) TimeString() string {
	return d.Time.Go().In(time.Local).Format("Mon Jan 2 15:04:05 MST 2006")
}

func (d *infoDirEntry) AttrString() string {
	return attrFormat(d.Attr)
}

func (d *infoDirEntry) Rights() []access.Right {
	return []access.Right{access.Read, access.Write, access.List, access.Create, access.Delete}
}

func (d *infoDirEntry) Readers() string {
	d.state.sharer.addAccess(d.DirEntry)
	d.lastUsers = "<nobody>"
	if d.IsDir() {
		return "is a directory"
	}
	_, users, _, err := d.state.sharer.readers(d.DirEntry)
	if err != nil {
		return err.Error()
	}
	d.lastUsers = users
	return users
}

func (d *infoDirEntry) Sequence() int64 {
	return upspin.SeqVersion(d.DirEntry.Sequence)
}

func (d *infoDirEntry) Hashes() string {
	h := ""
	if d.IsDir() || d.Packing != upspin.EEPack {
		return h
	}
	packer := pack.Lookup(d.Packing)
	hashes, err := packer.ReaderHashes(d.Packdata)
	if err != nil {
		return h
	}
	for _, r := range hashes {
		if h == "" {
			h += " "
		}
		h += fmt.Sprintf("%x...", r[:4])
	}
	return h
}

func (d *infoDirEntry) Users(right access.Right) string {
	users := userListToString(d.state.usersWithAccess(d.state.client, d.access, right))
	if users == d.lastUsers {
		return "(same)"
	}
	d.lastUsers = users
	return users
}

func (d *infoDirEntry) WhichAccess() string {
	var acc *access.Access
	accEntry, err := d.state.whichAccessFollowLinks(d.Name)
	if err != nil {
		return err.Error()
	}
	accFile := "owner only"
	if accEntry == nil {
		// No access file applies.
		acc, err = access.New(d.Name)
		if err != nil {
			// Can't happen, since the name must be valid.
			d.state.exitf("%q: %s", d.Name, err)
		}
	} else {
		accFile = string(accEntry.Name)
		data, err := read(d.state.client, accEntry.Name)
		if err != nil {
			fmt.Fprintf(os.Stderr, "cannot open access file %q: %s\n", accFile, err)
		}
		acc, err = access.Parse(accEntry.Name, data)
		if err != nil {
			fmt.Fprintf(os.Stderr, "cannot parse access file %q: %s\n", accFile, err)
		}
	}
	d.access = acc
	return accFile
}

// printInfo prints, in human-readable form, most of the information about
// the entry, including the users that have permission to access it.
// TODO: Present this more neatly.
// TODO: Present group information.
func (s *State) printInfo(entry *upspin.DirEntry) {
	infoDir := &infoDirEntry{
		state:    s,
		DirEntry: entry,
	}
	writer := tabwriter.NewWriter(os.Stdout, 4, 4, 1, ' ', 0)
	err := infoTmpl.Execute(writer, infoDir)
	if err != nil {
		s.exitf("executing info template: %v", err)
	}
	if writer.Flush() != nil {
		s.exitf("flushing template output: %v", err)
	}
	if !entry.IsLink() {
		return
	}
	// Check and print information about the link target.
	target, err := s.client.Lookup(entry.Link, true)
	if err != nil {
		// Print the whole error indented, starting on the next line. This helps it stand out.
		s.exitf("Error: link %s has invalid target %s:\n\t%v", entry.Name, entry.Link, err)
	}
	fmt.Printf("Target of link %s:\n", entry.Name)
	s.printInfo(target)
}

func attrFormat(attr upspin.Attribute) string {
	a := attr
	tail := ""
	if a|upspin.AttrIncomplete > 0 {
		tail = " (incomplete)"
		a ^= upspin.AttrIncomplete
	}
	switch a {
	case upspin.AttrNone:
		return "none (plain file)" + tail
	case upspin.AttrDirectory:
		return "directory" + tail
	case upspin.AttrLink:
		return "link" + tail
	}
	return fmt.Sprintf("attribute(%#x)", attr)
}

var infoTmpl = template.Must(template.New("info").Parse(infoText))

const infoText = `{{.Name}}
	packing:	{{.Packing}}
	size:	{{.Size}}
	time:	{{.TimeString}}
	writer:	{{.Writer}}
	attributes:	{{.AttrString}}
	sequence:	{{.Sequence}}
	access file:	{{.WhichAccess}}
	key holders: 	{{.Readers}}
	key hashes:     {{.Hashes}}
	{{range $right := .Rights -}}
	can {{$right}}:	{{$.Users $right}}
	{{end -}}
	Block#	Offset	Size	Location
	{{range $index, $block := .Blocks -}}
	{{$index}}	{{.Offset}}	{{.Size}}	{{.Location}}
	{{end}}`

// checkGroupFile diagnoses likely problems with the contents and rights
// of the Group file.
// TODO: We could check that packing is Plain but that should never be a problem.
func (s *State) checkGroupFile(name upspin.PathName) {
	parsed, err := path.Parse(name)
	if err != nil {
		s.exit(err) // Should never happen.
	}
	groupSeen := make(map[upspin.PathName]bool)
	userSeen := make(map[upspin.UserName]bool)
	s.doCheckGroupFile(parsed, groupSeen, userSeen)
}

// doCheckGroupFile is the inner, recursive implementation of checkGroupFile.
func (s *State) doCheckGroupFile(parsed path.Parsed, groupSeen map[upspin.PathName]bool, userSeen map[upspin.UserName]bool) {
	group := parsed.Path()
	if groupSeen[group] {
		return
	}
	groupSeen[group] = true
	data, err := s.client.Get(group)
	if err != nil {
		s.exitf("cannot read Group file: %v", err)
	}

	// Get the Access file, if any, that applies.
	// TODO: We've already got it in earlier code, so could save it.
	whichAccess, err := s.DirServer().WhichAccess(group)
	if err != nil {
		s.exitf("unexpected error finding Access file for Group file %s: %v", group, err)
	}
	var accessFile *access.Access
	if whichAccess == nil {
		accessFile, err = access.New(group)
		if err != nil {
			s.exitf("cannot create default Access file: %v", err)
		}
	} else {
		data, err := s.client.Get(whichAccess.Name)
		if err != nil {
			s.exitf("cannot get Access file: %v", err)
		}
		accessFile, err = access.Parse(whichAccess.Name, data)
		if err != nil {
			s.exitf("cannot parse Access file: %v", err)
		}
	}

	// Each member should be either a plain user or a group and be able to access the Group file.
	members, err := access.ParseGroup(parsed, data)
	if err != nil {
		s.exitf("error parsing Group file %s: %v", group, err)
	}
	for _, member := range members {
		if member.IsRoot() {
			// Normal user.
			user := member.User()
			if !s.userExists(user, userSeen) {
				s.failf("user %s in Group file %s not found in key server", user, group)
				continue
			}
			// Member must be able to read the Group file.
			canRead, err := accessFile.Can(user, access.Read, group, s.client.Get)
			if err != nil {
				s.exitf("error checking permissions in Group file %s for user %s: %v", group, user, err)
				continue
			}
			if !canRead {
				s.failf("user %s is missing read access for group %s", user, group)
			}
			continue
		}
		if !access.IsGroupFile(member.Path()) {
			s.failf("do not understand member %s of Group file %s", member, parsed) // Should never happen.
			continue
		}
		// Member is a group. Recur using Group file.
		s.doCheckGroupFile(member, groupSeen, userSeen)
	}
}

func (s *State) checkAccessFile(name upspin.PathName) {
	data, err := s.client.Get(name)
	if err != nil {
		s.exitf("cannot get Access file: %v", err)
	}
	accessFile, err := access.Parse(name, data)
	if err != nil {
		s.exitf("cannot parse Access file: %v", err)
	}
	users := accessFile.List(access.AnyRight)

	groupSeen := make(map[upspin.PathName]bool)
	userSeen := make(map[upspin.UserName]bool)
	for _, user := range users {
		if user.IsRoot() {
			// Normal user.
			if !s.userExists(user.User(), userSeen) {
				s.failf("user %s in Access file %s not found in key server", user.User(), name)
			}
			continue
		}
		// Member is a group.
		s.doCheckGroupFile(user, groupSeen, userSeen)
	}
}

func (s *State) userExists(user upspin.UserName, userSeen map[upspin.UserName]bool) bool {
	if userSeen[user] || user == access.AllUsers { // all@upspin.io is baked in.
		return true // Previous answer will do.
	}
	userSeen[user] = true
	_, err := s.KeyServer().Lookup(user)
	return err == nil
}
<|endoftext|>


<commit_before>package dsunit

import (
	"github.com/viant/dsc"
	"github.com/viant/toolbox"
)

//DatasetTransformer represents a dataset transformer.
type DatasetTransformer struct{}

func getDataset(datasets map[string]*Dataset, table string, registry dsc.TableDescriptorRegistry, tables *[]string) *Dataset {
	if result, found := datasets[table]; found {
		return result
	}
	descriptor := registry.Get(table)
	result := &Dataset{
		TableDescriptor: descriptor,
		Rows:            make([]*Row, 0),
	}
	datasets[table] = result
	(*tables) = append((*tables), table)
	return result
}

func (dt *DatasetTransformer) mapRowToDataset(row *Row, mapping *DatasetMapping) map[string]interface{} {
	var values = make(map[string]interface{})
	for _, column := range mapping.Columns {
		fromColumn := column.FromColumn
		if fromColumn == "" {
			fromColumn = column.Name
		}
		if rowValue, found := row.Values[fromColumn]; found {
			values[column.Name] = rowValue
		} else {
			if column.DefaultValue != "" {
				values[column.Name] = column.DefaultValue
			} else if column.Required {
				return nil
			}

		}
	}
	return values
}

func (dt *DatasetTransformer) isRowDuplicated(dataset *Dataset, mapping *DatasetMapping, row *Row, tablesPk map[string]bool) bool {
	if !dataset.Autoincrement {
		pkValue := "__ " + mapping.Table + ":"
		for _, pkColumn := range dataset.PkColumns {
			if value, ok := row.Values[pkColumn]; ok {
				pkValue += toolbox.AsString(value) + ":"
			}
		}
		if _, found := tablesPk[pkValue]; found {
			return true
		}
		tablesPk[pkValue] = true
	}
	return false
}

func (dt *DatasetTransformer) mapRowToDatasets(row *Row, mapping *DatasetMapping, datasets map[string]*Dataset, registry dsc.TableDescriptorRegistry, tables *[]string, tablesPk map[string]bool) {
	values := dt.mapRowToDataset(row, mapping)
	if values != nil {
		dataset := getDataset(datasets, mapping.Table, registry, tables)
		mappedRow := &Row{Source: row.Source, Values: values}
		if dt.isRowDuplicated(dataset, mapping, mappedRow, tablesPk) {
			return
		}
		dataset.Rows = append(dataset.Rows, mappedRow)
	}
	if mapping.Associations != nil {
		for _, association := range mapping.Associations {
			dt.mapRowToDatasets(row, association, datasets, registry, tables, tablesPk)
		}
	}
}

//Transform routes source dataset data into mapping dataset, it uses source dataset and table descriptor registry to build resulting datasets.
func (dt *DatasetTransformer) Transform(datastore string, sourceDataset *Dataset, mapping *DatasetMapping, registry dsc.TableDescriptorRegistry) *Datasets {
	var datasets = make(map[string]*Dataset)
	var tables = make([]string, 0)
	var tablesPk = make(map[string]bool)
	for _, row := range sourceDataset.Rows {
		dt.mapRowToDatasets(row, mapping, datasets, registry, &tables, tablesPk)
	}
	result := &Datasets{
		Datastore: datastore,
		Datasets:  make([]*Dataset, 0),
	}

	for _, table := range tables {
		value := datasets[table]
		if value == nil || (*value).Rows == nil || len((*value).Rows) == 0 {
			continue
		}
		result.Datasets = append(result.Datasets, value)
	}
	return result
}

//NewDatasetTransformer returns a new NewDatasetTransformer
func NewDatasetTransformer() *DatasetTransformer {
	return &DatasetTransformer{}
}

type datasetTransformerRegistry struct {
	registry map[string]*DatasetMapping
}

//Register register dataset mapping with name
func (r *datasetTransformerRegistry) register(name string, mapping *DatasetMapping) {
	r.registry[name] = mapping
}

//Get returns dataset mapping for passed in name.
func (r *datasetTransformerRegistry) get(name string) *DatasetMapping {
	if result, ok := r.registry[name]; ok {
		return result
	}
	return nil
}

//Has returns true if data mapping has been registered.
func (r *datasetTransformerRegistry) has(name string) bool {
	if _, ok := r.registry[name]; ok {
		return true
	}
	return false
}

//Names returns names of registered dataset mapping.
func (r *datasetTransformerRegistry) names() []string {
	return toolbox.MapKeysToStringSlice(r.registry)
}

//NewDatasetTransformerRegistry returns new NewDatasetTransformerRegistry
func newDatasetTransformerRegistry() *datasetTransformerRegistry {
	return &datasetTransformerRegistry{
		registry: make(map[string]*DatasetMapping),
	}
}
<commit_msg>patched dataset generation<commit_after>package dsunit

import (
	"github.com/viant/dsc"
	"github.com/viant/toolbox"
)

//DatasetTransformer represents a dataset transformer.
type DatasetTransformer struct{}

func getDataset(datasets map[string]*Dataset, table string, registry dsc.TableDescriptorRegistry, tables *[]string) *Dataset {
	if result, found := datasets[table]; found {
		return result
	}
	descriptor := registry.Get(table)
	result := &Dataset{
		TableDescriptor: descriptor,
		Rows:            make([]*Row, 0),
	}
	datasets[table] = result
	(*tables) = append((*tables), table)
	return result
}

func (dt *DatasetTransformer) mapRowToDataset(row *Row, mapping *DatasetMapping) map[string]interface{} {
	var values = make(map[string]interface{})
	for _, column := range mapping.Columns {
		fromColumn := column.FromColumn
		if fromColumn == "" {
			fromColumn = column.Name
		}
		if rowValue, found := row.Values[fromColumn]; found {
			values[column.Name] = rowValue
		} else {
			if column.DefaultValue != "" {
				values[column.Name] = column.DefaultValue
			} else if column.Required {
				return nil
			}

		}
	}
	return values
}

func (dt *DatasetTransformer) isRowDuplicated(dataset *Dataset, mapping *DatasetMapping, row *Row, tablesPk map[string]bool) bool {
	if !dataset.Autoincrement {
		var required = ""
		for _, column := range mapping.Columns {
			if column.Required {
				required += column.Name
			}
		}
		pkValue := "__ " + mapping.Table + required + ":"
		var hasPk = false;
		for _, pkColumn := range dataset.PkColumns {
			if value, ok := row.Values[pkColumn]; ok {
				pkValue += toolbox.AsString(value) + ":"
				hasPk = true
			}
		}
		if ! hasPk {
			return false
		}

		if _, found := tablesPk[pkValue]; found {
			return true
		}
		tablesPk[pkValue] = true
	}
	return false
}

func (dt *DatasetTransformer) mapRowToDatasets(row *Row, mapping *DatasetMapping, datasets map[string]*Dataset, registry dsc.TableDescriptorRegistry, tables *[]string, tablesPk map[string]bool) {
	values := dt.mapRowToDataset(row, mapping)
	if values != nil {
		dataset := getDataset(datasets, mapping.Table, registry, tables)
		mappedRow := &Row{Source: row.Source, Values: values}

		if dt.isRowDuplicated(dataset, mapping, mappedRow, tablesPk) {
			return
		}
		dataset.Rows = append(dataset.Rows, mappedRow)
	}
	if len(mapping.Associations) > 0 {
		for _, association := range mapping.Associations {
			dt.mapRowToDatasets(row, association, datasets, registry, tables, tablesPk)
		}
	}
}

//Transform routes source dataset data into mapping dataset, it uses source dataset and table descriptor registry to build resulting datasets.
func (dt *DatasetTransformer) Transform(datastore string, sourceDataset *Dataset, mapping *DatasetMapping, registry dsc.TableDescriptorRegistry) *Datasets {
	var datasets = make(map[string]*Dataset)
	var tables = make([]string, 0)
	var tablesPk = make(map[string]bool)

	for _, row := range sourceDataset.Rows {
		dt.mapRowToDatasets(row, mapping, datasets, registry, &tables, tablesPk)
	}
	result := &Datasets{
		Datastore: datastore,
		Datasets:  make([]*Dataset, 0),
	}

	for _, table := range tables {
		value := datasets[table]
		if value == nil || (*value).Rows == nil || len((*value).Rows) == 0 {
			continue
		}
		result.Datasets = append(result.Datasets, value)
	}
	return result
}

//NewDatasetTransformer returns a new NewDatasetTransformer
func NewDatasetTransformer() *DatasetTransformer {
	return &DatasetTransformer{}
}

type datasetTransformerRegistry struct {
	registry map[string]*DatasetMapping
}

//Register register dataset mapping with name
func (r *datasetTransformerRegistry) register(name string, mapping *DatasetMapping) {
	r.registry[name] = mapping
}

//Get returns dataset mapping for passed in name.
func (r *datasetTransformerRegistry) get(name string) *DatasetMapping {
	if result, ok := r.registry[name]; ok {
		return result
	}
	return nil
}

//Has returns true if data mapping has been registered.
func (r *datasetTransformerRegistry) has(name string) bool {
	if _, ok := r.registry[name]; ok {
		return true
	}
	return false
}

//Names returns names of registered dataset mapping.
func (r *datasetTransformerRegistry) names() []string {
	return toolbox.MapKeysToStringSlice(r.registry)
}

//NewDatasetTransformerRegistry returns new NewDatasetTransformerRegistry
func newDatasetTransformerRegistry() *datasetTransformerRegistry {
	return &datasetTransformerRegistry{
		registry: make(map[string]*DatasetMapping),
	}
}
<|endoftext|>


<commit_before>package spnego

import (
	"context"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"

	"github.com/jcmturner/gofork/encoding/asn1"
	"gopkg.in/jcmturner/gokrb5.v7/asn1tools"
	"gopkg.in/jcmturner/gokrb5.v7/client"
	"gopkg.in/jcmturner/gokrb5.v7/credentials"
	"gopkg.in/jcmturner/gokrb5.v7/gssapi"
	"gopkg.in/jcmturner/gokrb5.v7/iana/chksumtype"
	"gopkg.in/jcmturner/gokrb5.v7/iana/msgtype"
	"gopkg.in/jcmturner/gokrb5.v7/krberror"
	"gopkg.in/jcmturner/gokrb5.v7/messages"
	"gopkg.in/jcmturner/gokrb5.v7/service"
	"gopkg.in/jcmturner/gokrb5.v7/types"
)

// GSSAPI KRB5 MechToken IDs.
const (
	TOK_ID_KRB_AP_REQ = "0100"
	TOK_ID_KRB_AP_REP = "0200"
	TOK_ID_KRB_ERROR  = "0300"
)

// KRB5Token context token implementation for GSSAPI.
type KRB5Token struct {
	OID      asn1.ObjectIdentifier
	tokID    []byte
	APReq    messages.APReq
	APRep    messages.APRep
	KRBError messages.KRBError
	settings *service.Settings
	context  context.Context
}

// Marshal a KRB5Token into a slice of bytes.
func (m *KRB5Token) Marshal() ([]byte, error) {
	// Create the header
	b, _ := asn1.Marshal(m.OID)
	b = append(b, m.tokID...)
	var tb []byte
	var err error
	switch hex.EncodeToString(m.tokID) {
	case TOK_ID_KRB_AP_REQ:
		tb, err = m.APReq.Marshal()
		if err != nil {
			return []byte{}, fmt.Errorf("error marshalling AP_REQ for MechToken: %v", err)
		}
	case TOK_ID_KRB_AP_REP:
		return []byte{}, errors.New("marshal of AP_REP GSSAPI MechToken not supported by gokrb5")
	case TOK_ID_KRB_ERROR:
		return []byte{}, errors.New("marshal of KRB_ERROR GSSAPI MechToken not supported by gokrb5")
	}
	if err != nil {
		return []byte{}, fmt.Errorf("error mashalling kerberos message within mech token: %v", err)
	}
	b = append(b, tb...)
	return asn1tools.AddASNAppTag(b, 0), nil
}

// Unmarshal a KRB5Token.
func (m *KRB5Token) Unmarshal(b []byte) error {
	var oid asn1.ObjectIdentifier
	r, err := asn1.UnmarshalWithParams(b, &oid, fmt.Sprintf("application,explicit,tag:%v", 0))
	if err != nil {
		return fmt.Errorf("error unmarshalling KRB5Token OID: %v", err)
	}
	m.OID = oid
	m.tokID = r[0:2]
	switch hex.EncodeToString(m.tokID) {
	case TOK_ID_KRB_AP_REQ:
		var a messages.APReq
		err = a.Unmarshal(r[2:])
		if err != nil {
			return fmt.Errorf("error unmarshalling KRB5Token AP_REQ: %v", err)
		}
		m.APReq = a
	case TOK_ID_KRB_AP_REP:
		var a messages.APRep
		err = a.Unmarshal(r[2:])
		if err != nil {
			return fmt.Errorf("error unmarshalling KRB5Token AP_REP: %v", err)
		}
		m.APRep = a
	case TOK_ID_KRB_ERROR:
		var a messages.KRBError
		err = a.Unmarshal(r[2:])
		if err != nil {
			return fmt.Errorf("error unmarshalling KRB5Token KRBError: %v", err)
		}
		m.KRBError = a
	}
	return nil
}

// Verify a KRB5Token.
func (m *KRB5Token) Verify() (bool, gssapi.Status) {
	switch hex.EncodeToString(m.tokID) {
	case TOK_ID_KRB_AP_REQ:
		ok, creds, err := service.VerifyAPREQ(m.APReq, m.settings)
		if err != nil {
			return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: err.Error()}
		}
		if !ok {
			return false, gssapi.Status{Code: gssapi.StatusDefectiveCredential, Message: "KRB5_AP_REQ token not valid"}
		}
		m.context = context.Background()
		m.context = context.WithValue(m.context, CTXKeyCredentials, creds)
		m.context = context.WithValue(m.context, CTXKeyAuthenticated, ok)
		return true, gssapi.Status{Code: gssapi.StatusComplete}
	case TOK_ID_KRB_AP_REP:
		// Client side
		// TODO how to verify the AP_REP - not yet implemented
		return false, gssapi.Status{Code: gssapi.StatusFailure, Message: "verifying an AP_REP is not currently supported by gokrb5"}
	case TOK_ID_KRB_ERROR:
		if m.KRBError.MsgType != msgtype.KRB_ERROR {
			return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: "KRB5_Error token not valid"}
		}
		return true, gssapi.Status{Code: gssapi.StatusUnavailable}
	}
	return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: "unknown TOK_ID in KRB5 token"}
}

// IsAPReq tests if the MechToken contains an AP_REQ.
func (m *KRB5Token) IsAPReq() bool {
	if hex.EncodeToString(m.tokID) == TOK_ID_KRB_AP_REQ {
		return true
	}
	return false
}

// IsAPRep tests if the MechToken contains an AP_REP.
func (m *KRB5Token) IsAPRep() bool {
	if hex.EncodeToString(m.tokID) == TOK_ID_KRB_AP_REP {
		return true
	}
	return false
}

// IsKRBError tests if the MechToken contains an KRB_ERROR.
func (m *KRB5Token) IsKRBError() bool {
	if hex.EncodeToString(m.tokID) == TOK_ID_KRB_ERROR {
		return true
	}
	return false
}

// Context returns the KRB5 token's context which will contain any verify user identity information.
func (m *KRB5Token) Context() context.Context {
	return m.context
}

// NewKRB5TokenAPREQ creates a new KRB5 token with AP_REQ
func NewKRB5TokenAPREQ(cl *client.Client, tkt messages.Ticket, sessionKey types.EncryptionKey, GSSAPIFlags []int, APOptions []int) (KRB5Token, error) {
	// TODO consider providing the SPN rather than the specific tkt and key and get these from the krb client.
	var m KRB5Token
	m.OID = gssapi.OID(gssapi.OIDKRB5)
	tb, _ := hex.DecodeString(TOK_ID_KRB_AP_REQ)
	m.tokID = tb

	auth, err := krb5TokenAuthenticator(cl.Credentials, GSSAPIFlags)
	if err != nil {
		return m, err
	}
	APReq, err := messages.NewAPReq(
		tkt,
		sessionKey,
		auth,
	)
	if err != nil {
		return m, err
	}
	for _, o := range APOptions {
		types.SetFlag(&APReq.APOptions, o)
	}
	m.APReq = APReq
	return m, nil
}

// krb5TokenAuthenticator creates a new kerberos authenticator for kerberos MechToken
func krb5TokenAuthenticator(creds *credentials.Credentials, flags []int) (types.Authenticator, error) {
	//RFC 4121 Section 4.1.1
	auth, err := types.NewAuthenticator(creds.Domain(), creds.CName())
	if err != nil {
		return auth, krberror.Errorf(err, krberror.KRBMsgError, "error generating new authenticator")
	}
	auth.Cksum = types.Checksum{
		CksumType: chksumtype.GSSAPI,
		Checksum:  newAuthenticatorChksum(flags),
	}
	return auth, nil
}

// Create new authenticator checksum for kerberos MechToken
func newAuthenticatorChksum(flags []int) []byte {
	a := make([]byte, 24)
	binary.LittleEndian.PutUint32(a[:4], 16)
	for _, i := range flags {
		if i == gssapi.ContextFlagDeleg {
			x := make([]byte, 28-len(a))
			a = append(a, x...)
		}
		f := binary.LittleEndian.Uint32(a[20:24])
		f |= uint32(i)
		binary.LittleEndian.PutUint32(a[20:24], f)
	}
	return a
}

/*
The authenticator checksum field SHALL have the following format:

Octet        Name      Description
-----------------------------------------------------------------
0..3         Lgth    Number of octets in Bnd field;  Represented
			in little-endian order;  Currently contains
			hex value 10 00 00 00 (16).
4..19        Bnd     Channel binding information, as described in
			section 4.1.1.2.
20..23       Flags   Four-octet context-establishment flags in
			little-endian order as described in section
			4.1.1.1.
24..25       DlgOpt  The delegation option identifier (=1) in
			little-endian order [optional].  This field
			and the next two fields are present if and
			only if GSS_C_DELEG_FLAG is set as described
			in section 4.1.1.1.
26..27       Dlgth   The length of the Deleg field in little-endian order [optional].
28..(n-1)    Deleg   A KRB_CRED message (n = Dlgth + 28) [optional].
n..last      Exts    Extensions [optional].
*/
<commit_msg>length check when parsing krb5tokens (#341)<commit_after>package spnego

import (
	"context"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"

	"github.com/jcmturner/gofork/encoding/asn1"
	"gopkg.in/jcmturner/gokrb5.v7/asn1tools"
	"gopkg.in/jcmturner/gokrb5.v7/client"
	"gopkg.in/jcmturner/gokrb5.v7/credentials"
	"gopkg.in/jcmturner/gokrb5.v7/gssapi"
	"gopkg.in/jcmturner/gokrb5.v7/iana/chksumtype"
	"gopkg.in/jcmturner/gokrb5.v7/iana/msgtype"
	"gopkg.in/jcmturner/gokrb5.v7/krberror"
	"gopkg.in/jcmturner/gokrb5.v7/messages"
	"gopkg.in/jcmturner/gokrb5.v7/service"
	"gopkg.in/jcmturner/gokrb5.v7/types"
)

// GSSAPI KRB5 MechToken IDs.
const (
	TOK_ID_KRB_AP_REQ = "0100"
	TOK_ID_KRB_AP_REP = "0200"
	TOK_ID_KRB_ERROR  = "0300"
)

// KRB5Token context token implementation for GSSAPI.
type KRB5Token struct {
	OID      asn1.ObjectIdentifier
	tokID    []byte
	APReq    messages.APReq
	APRep    messages.APRep
	KRBError messages.KRBError
	settings *service.Settings
	context  context.Context
}

// Marshal a KRB5Token into a slice of bytes.
func (m *KRB5Token) Marshal() ([]byte, error) {
	// Create the header
	b, _ := asn1.Marshal(m.OID)
	b = append(b, m.tokID...)
	var tb []byte
	var err error
	switch hex.EncodeToString(m.tokID) {
	case TOK_ID_KRB_AP_REQ:
		tb, err = m.APReq.Marshal()
		if err != nil {
			return []byte{}, fmt.Errorf("error marshalling AP_REQ for MechToken: %v", err)
		}
	case TOK_ID_KRB_AP_REP:
		return []byte{}, errors.New("marshal of AP_REP GSSAPI MechToken not supported by gokrb5")
	case TOK_ID_KRB_ERROR:
		return []byte{}, errors.New("marshal of KRB_ERROR GSSAPI MechToken not supported by gokrb5")
	}
	if err != nil {
		return []byte{}, fmt.Errorf("error mashalling kerberos message within mech token: %v", err)
	}
	b = append(b, tb...)
	return asn1tools.AddASNAppTag(b, 0), nil
}

// Unmarshal a KRB5Token.
func (m *KRB5Token) Unmarshal(b []byte) error {
	var oid asn1.ObjectIdentifier
	r, err := asn1.UnmarshalWithParams(b, &oid, fmt.Sprintf("application,explicit,tag:%v", 0))
	if err != nil {
		return fmt.Errorf("error unmarshalling KRB5Token OID: %v", err)
	}
	m.OID = oid
	if len(r) < 2 {
		return fmt.Errorf("krb5token too short")
	}
	m.tokID = r[0:2]
	switch hex.EncodeToString(m.tokID) {
	case TOK_ID_KRB_AP_REQ:
		var a messages.APReq
		err = a.Unmarshal(r[2:])
		if err != nil {
			return fmt.Errorf("error unmarshalling KRB5Token AP_REQ: %v", err)
		}
		m.APReq = a
	case TOK_ID_KRB_AP_REP:
		var a messages.APRep
		err = a.Unmarshal(r[2:])
		if err != nil {
			return fmt.Errorf("error unmarshalling KRB5Token AP_REP: %v", err)
		}
		m.APRep = a
	case TOK_ID_KRB_ERROR:
		var a messages.KRBError
		err = a.Unmarshal(r[2:])
		if err != nil {
			return fmt.Errorf("error unmarshalling KRB5Token KRBError: %v", err)
		}
		m.KRBError = a
	}
	return nil
}

// Verify a KRB5Token.
func (m *KRB5Token) Verify() (bool, gssapi.Status) {
	switch hex.EncodeToString(m.tokID) {
	case TOK_ID_KRB_AP_REQ:
		ok, creds, err := service.VerifyAPREQ(m.APReq, m.settings)
		if err != nil {
			return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: err.Error()}
		}
		if !ok {
			return false, gssapi.Status{Code: gssapi.StatusDefectiveCredential, Message: "KRB5_AP_REQ token not valid"}
		}
		m.context = context.Background()
		m.context = context.WithValue(m.context, CTXKeyCredentials, creds)
		m.context = context.WithValue(m.context, CTXKeyAuthenticated, ok)
		return true, gssapi.Status{Code: gssapi.StatusComplete}
	case TOK_ID_KRB_AP_REP:
		// Client side
		// TODO how to verify the AP_REP - not yet implemented
		return false, gssapi.Status{Code: gssapi.StatusFailure, Message: "verifying an AP_REP is not currently supported by gokrb5"}
	case TOK_ID_KRB_ERROR:
		if m.KRBError.MsgType != msgtype.KRB_ERROR {
			return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: "KRB5_Error token not valid"}
		}
		return true, gssapi.Status{Code: gssapi.StatusUnavailable}
	}
	return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: "unknown TOK_ID in KRB5 token"}
}

// IsAPReq tests if the MechToken contains an AP_REQ.
func (m *KRB5Token) IsAPReq() bool {
	if hex.EncodeToString(m.tokID) == TOK_ID_KRB_AP_REQ {
		return true
	}
	return false
}

// IsAPRep tests if the MechToken contains an AP_REP.
func (m *KRB5Token) IsAPRep() bool {
	if hex.EncodeToString(m.tokID) == TOK_ID_KRB_AP_REP {
		return true
	}
	return false
}

// IsKRBError tests if the MechToken contains an KRB_ERROR.
func (m *KRB5Token) IsKRBError() bool {
	if hex.EncodeToString(m.tokID) == TOK_ID_KRB_ERROR {
		return true
	}
	return false
}

// Context returns the KRB5 token's context which will contain any verify user identity information.
func (m *KRB5Token) Context() context.Context {
	return m.context
}

// NewKRB5TokenAPREQ creates a new KRB5 token with AP_REQ
func NewKRB5TokenAPREQ(cl *client.Client, tkt messages.Ticket, sessionKey types.EncryptionKey, GSSAPIFlags []int, APOptions []int) (KRB5Token, error) {
	// TODO consider providing the SPN rather than the specific tkt and key and get these from the krb client.
	var m KRB5Token
	m.OID = gssapi.OID(gssapi.OIDKRB5)
	tb, _ := hex.DecodeString(TOK_ID_KRB_AP_REQ)
	m.tokID = tb

	auth, err := krb5TokenAuthenticator(cl.Credentials, GSSAPIFlags)
	if err != nil {
		return m, err
	}
	APReq, err := messages.NewAPReq(
		tkt,
		sessionKey,
		auth,
	)
	if err != nil {
		return m, err
	}
	for _, o := range APOptions {
		types.SetFlag(&APReq.APOptions, o)
	}
	m.APReq = APReq
	return m, nil
}

// krb5TokenAuthenticator creates a new kerberos authenticator for kerberos MechToken
func krb5TokenAuthenticator(creds *credentials.Credentials, flags []int) (types.Authenticator, error) {
	//RFC 4121 Section 4.1.1
	auth, err := types.NewAuthenticator(creds.Domain(), creds.CName())
	if err != nil {
		return auth, krberror.Errorf(err, krberror.KRBMsgError, "error generating new authenticator")
	}
	auth.Cksum = types.Checksum{
		CksumType: chksumtype.GSSAPI,
		Checksum:  newAuthenticatorChksum(flags),
	}
	return auth, nil
}

// Create new authenticator checksum for kerberos MechToken
func newAuthenticatorChksum(flags []int) []byte {
	a := make([]byte, 24)
	binary.LittleEndian.PutUint32(a[:4], 16)
	for _, i := range flags {
		if i == gssapi.ContextFlagDeleg {
			x := make([]byte, 28-len(a))
			a = append(a, x...)
		}
		f := binary.LittleEndian.Uint32(a[20:24])
		f |= uint32(i)
		binary.LittleEndian.PutUint32(a[20:24], f)
	}
	return a
}

/*
The authenticator checksum field SHALL have the following format:

Octet        Name      Description
-----------------------------------------------------------------
0..3         Lgth    Number of octets in Bnd field;  Represented
			in little-endian order;  Currently contains
			hex value 10 00 00 00 (16).
4..19        Bnd     Channel binding information, as described in
			section 4.1.1.2.
20..23       Flags   Four-octet context-establishment flags in
			little-endian order as described in section
			4.1.1.1.
24..25       DlgOpt  The delegation option identifier (=1) in
			little-endian order [optional].  This field
			and the next two fields are present if and
			only if GSS_C_DELEG_FLAG is set as described
			in section 4.1.1.1.
26..27       Dlgth   The length of the Deleg field in little-endian order [optional].
28..(n-1)    Deleg   A KRB_CRED message (n = Dlgth + 28) [optional].
n..last      Exts    Extensions [optional].
*/
<|endoftext|>


<commit_before>package turnpike

import (
	"fmt"
	"sync"
	"time"

	logrus "github.com/sirupsen/logrus"
	"github.com/streamrail/concurrent-map"
)

const (
	defaultAuthTimeout = 2 * time.Minute
)

// A Realm is a WAMP routing and administrative domain.
//
// Clients that have connected to a WAMP router are joined to a realm and all
// message delivery is handled by the realm.
type Realm struct {
	_                string
	URI              URI
	Broker           Broker
	Dealer           Dealer
	Authorizer       Authorizer
	Interceptor      Interceptor
	CRAuthenticators map[string]CRAuthenticator
	Authenticators   map[string]Authenticator
	// DefaultAuth      func(details map[string]interface{}) (map[string]interface{}, error)
	AuthTimeout time.Duration
	clients     cmap.ConcurrentMap
	localClient *localClient

	lock sync.RWMutex
}

type localClient struct {
	*Client
	sync.Mutex
}

func (r *Realm) getPeer(details map[string]interface{}) (Peer, error) {
	peerA, peerB := localPipe()
	sess := &Session{Peer: peerA, Id: NewID(), Details: details, kill: make(chan URI, 1)}
	if details == nil {
		details = make(map[string]interface{})
	}
	go func() {
		r.handleSession(sess)
		sess.Close()
	}()
	log.WithField("session_id", sess.Id).Info("established internal session")
	return peerB, nil
}

// Close disconnects all clients after sending a goodbye message
func (r Realm) Close() {
	iter := r.clients.Iter()
	for client := range iter {
		sess, isSession := client.Val.(*Session)
		if !isSession {
			continue
		}
		sess.kill <- ErrSystemShutdown
	}

	close(r.acts)
}

func (r *Realm) init() {
	r.lock.Lock()
	defer r.lock.Unlock()

	r.clients = cmap.New()

	if r.localClient == nil {
		p, _ := r.getPeer(nil)
		client := NewClient(p)
		r.localClient = new(localClient)
		r.localClient.Client = client
	}

	if r.Broker == nil {
		r.Broker = NewDefaultBroker()
	}
	if r.Dealer == nil {
		r.Dealer = NewDefaultDealer()
	}
	if r.Authorizer == nil {
		r.Authorizer = NewDefaultAuthorizer()
	}
	if r.Interceptor == nil {
		r.Interceptor = NewDefaultInterceptor()
	}
	if r.AuthTimeout == 0 {
		r.AuthTimeout = defaultAuthTimeout
	}
	go r.localClient.Receive()
	go r.run()
}

func (r *Realm) run() {
	for {
		if act, ok := <-r.acts; ok {
			act()
		} else {
			return
		}
	}
}

func (l *localClient) onJoin(details map[string]interface{}) {
	l.Lock()
	defer l.Unlock()
	l.Publish("wamp.session.on_join", []interface{}{details}, nil)
}

func (l *localClient) onLeave(session ID) {
	l.Lock()
	defer l.Unlock()
	l.Publish("wamp.session.on_leave", []interface{}{session}, nil)
}

func (r *Realm) doOne(c <-chan Message, sess *Session) bool {
	r.lock.RLock()
	defer r.lock.RUnlock()

	var msg Message
	var open bool
	select {
	case msg, open = <-c:
		if !open {
			log.WithField("session_id", sess.Id).Error("lost session")
			return false
		}
	case reason := <-sess.kill:
		logErr(sess.Send(&Goodbye{Reason: reason, Details: make(map[string]interface{})}))
		log.Printf("kill session %s: %v", sess, reason)
		// TODO: wait for client Goodbye?
		return false
	}

redactedMsg := redactMessage(msg)

log.WithFields(logrus.Fields{
	"session_id":   sess.Id,
	"message_type": msg.MessageType().String(),
	"message":      redactedMsg,
}).Info("new message")

	if isAuthz, err := r.Authorizer.Authorize(sess, msg); !isAuthz {
		errMsg := &Error{Type: msg.MessageType()}
		if err != nil {
			errMsg.Error = ErrAuthorizationFailed
			log.WithFields(logrus.Fields{
				"session_id":   sess.Id,
				"message_type": msg.MessageType().String(),
				"message":      redactedMsg,
				"error": err,
			}).Info("authorization failed")
		} else {
			errMsg.Error = ErrNotAuthorized
			log.WithFields(logrus.Fields{
				"session_id":   sess.Id,
				"message_type": msg.MessageType().String(),
				"message":      redactedMsg,
				"error": err,
			}).Info("UNAUTHORIZED")
		}
		logErr(sess.Send(errMsg))
		return true
	}

	r.Interceptor.Intercept(sess, &msg)

	switch msg := msg.(type) {
	case *Goodbye:
		logErr(sess.Send(&Goodbye{Reason: ErrGoodbyeAndOut, Details: make(map[string]interface{})}))
		log.WithFields(logrus.Fields{
			"session_id": sess.Id,
			"reason": msg.Reason,
			}).Warning("leaving")
		return false

	// Broker messages
	case *Publish:
		r.Broker.Publish(sess, msg)
	case *Subscribe:
		r.Broker.Subscribe(sess, msg)
	case *Unsubscribe:
		r.Broker.Unsubscribe(sess, msg)

	// Dealer messages
	case *Register:
		r.Dealer.Register(sess, msg)
	case *Unregister:
		r.Dealer.Unregister(sess, msg)
	case *Call:
		r.Dealer.Call(sess, msg)
	case *Yield:
		r.Dealer.Yield(sess, msg)

	// Error messages
	case *Error:
		if msg.Type == INVOCATION {
			// the only type of ERROR message the router should receive
			r.Dealer.Error(sess, msg)
		} else {
			log.Infof("invalid ERROR message received: %v", msg)
		}

	default:
		log.Infof("Unhandled message:", msg.MessageType())
	}
	return true
}

func redactMessage(msg Message) Message {
	switch msg := msg.(type) {
	case *Call:
		var redacted Call
		redacted.Request = msg.Request
		redacted.Arguments = append(redacted.Arguments, msg.Arguments...)
		redacted.ArgumentsKw = make(map[string]interface{})
		for k, v := range msg.ArgumentsKw {
			if k == "token" {
				v = "redacted"
			}
			redacted.ArgumentsKw[k] = v
		}
		redacted.Options = make(map[string]interface{})
		for k, v := range msg.Options {
			redacted.Options[k] = v
		}
		return &redacted
}

func (r *Realm) handleSession(sess *Session) {
	r.lock.RLock()
	r.clients.Set(string(sess.Id), sess)
	r.localClient.onJoin(sess.Details)
	r.lock.RUnlock()

	defer func() {
		r.lock.RLock()
		defer r.lock.RUnlock()

		r.clients.Remove(string(sess.Id))
		r.Broker.RemoveSession(sess)
		r.Dealer.RemoveSession(sess)
		r.localClient.onLeave(sess.Id)
	}()
	c := sess.Receive()
	// TODO: what happens if the realm is closed?

	for r.doOne(c, sess) {
	}
	return msg
}

func (r *Realm) handleAuth(client Peer, details map[string]interface{}) (*Welcome, error) {
	msg, err := r.authenticate(details)
	if err != nil {
		return nil, err
	}
	// we should never get anything besides WELCOME and CHALLENGE
	if msg.MessageType() == WELCOME {
		return msg.(*Welcome), nil
	}
	// Challenge response
	challenge := msg.(*Challenge)
	if err := client.Send(challenge); err != nil {
		return nil, err
	}

	log.WithFields(logrus.Fields{
		"message_type": msg.MessageType().String(),
		"message":      msg,
	}).Info("challenge")

	msg, err = GetMessageTimeout(client, r.AuthTimeout)
	if err != nil {
		return nil, err
	}

	if authenticate, ok := msg.(*Authenticate); !ok {
		return nil, fmt.Errorf("unexpected %s message received", msg.MessageType())
	} else {
		redacted := map[string]interface{}{
			"Extra":     authenticate.Extra,
			"Signature": "redacted",
		}
		log.WithFields(logrus.Fields{
			"message_type": msg.MessageType().String(),
			"message":      redacted,
		}).Info("authenticate")

		return r.checkResponse(challenge, authenticate)
	}
}

// Authenticate either authenticates a client or returns a challenge message if
// challenge/response authentication is to be used.
func (r Realm) authenticate(details map[string]interface{}) (Message, error) {
	if len(r.Authenticators) == 0 && len(r.CRAuthenticators) == 0 {
		return &Welcome{}, nil
	}
	// TODO: this might not always be a []interface{}. Using the JSON unmarshaller it will be,
	// but we may have serializations that preserve more of the original type.
	// For now, the tests just explicitly send a []interface{}
	_authmethods, ok := details["authmethods"].([]interface{})
	if !ok {
		return nil, fmt.Errorf("No authentication supplied")
	}
	authmethods := []string{}
	for _, method := range _authmethods {
		if m, ok := method.(string); ok {
			authmethods = append(authmethods, m)
		} else {
			log.Errorf("invalid authmethod value: %v", method)
		}
	}
	for _, method := range authmethods {
		if auth, ok := r.CRAuthenticators[method]; ok {
			if challenge, err := auth.Challenge(details); err != nil {
				return nil, err
			} else {
				return &Challenge{AuthMethod: method, Extra: challenge}, nil
			}
		}
		if auth, ok := r.Authenticators[method]; ok {
			if authDetails, err := auth.Authenticate(details); err != nil {
				return nil, err
			} else {
				return &Welcome{Details: addAuthMethod(authDetails, method)}, nil
			}
		}
	}
	// TODO: check default auth (special '*' auth?)
	return nil, fmt.Errorf("could not authenticate with any method")
}

// checkResponse determines whether the response to the challenge is sufficient to gain access to the Realm.
func (r Realm) checkResponse(chal *Challenge, auth *Authenticate) (*Welcome, error) {
	authenticator, ok := r.CRAuthenticators[chal.AuthMethod]
	if !ok {
		return nil, fmt.Errorf("authentication method has been removed")
	}
	if details, err := authenticator.Authenticate(chal.Extra, auth.Signature); err != nil {
		return nil, err
	} else {
		return &Welcome{Details: addAuthMethod(details, chal.AuthMethod)}, nil
	}
}

func addAuthMethod(details map[string]interface{}, method string) map[string]interface{} {
	if details == nil {
		details = make(map[string]interface{})
	}
	details["authmethod"] = method
	return details
}

// r := Realm{
// 	Authenticators: map[string]gowamp.Authenticator{
// 		"wampcra": gowamp.NewCRAAuthenticatorFactoryFactory(mySecret),
// 		"ticket": gowamp.NewTicketAuthenticator(myTicket),
// 		"asdfasdf": myAsdfAuthenticator,
// 	},
// 	BasicAuthenticators: map[string]turnpike.BasicAuthenticator{
// 		"anonymous": nil,
// 	},
// }
<commit_msg>clean up realm.go<commit_after>package turnpike

import (
	"fmt"
	"sync"
	"time"

	logrus "github.com/sirupsen/logrus"
	"github.com/streamrail/concurrent-map"
)

const (
	defaultAuthTimeout = 2 * time.Minute
)

// A Realm is a WAMP routing and administrative domain.
//
// Clients that have connected to a WAMP router are joined to a realm and all
// message delivery is handled by the realm.
type Realm struct {
	_                string
	URI              URI
	Broker           Broker
	Dealer           Dealer
	Authorizer       Authorizer
	Interceptor      Interceptor
	CRAuthenticators map[string]CRAuthenticator
	Authenticators   map[string]Authenticator
	// DefaultAuth      func(details map[string]interface{}) (map[string]interface{}, error)
	AuthTimeout time.Duration
	clients     cmap.ConcurrentMap
	localClient *localClient

	lock sync.RWMutex
}

type localClient struct {
	*Client
	sync.Mutex
}

func (r *Realm) getPeer(details map[string]interface{}) (Peer, error) {
	peerA, peerB := localPipe()
	sess := &Session{Peer: peerA, Id: NewID(), Details: details, kill: make(chan URI, 1)}
	if details == nil {
		details = make(map[string]interface{})
	}
	go func() {
		r.handleSession(sess)
		sess.Close()
	}()
	log.WithField("session_id", sess.Id).Info("established internal session")
	return peerB, nil
}

// Close disconnects all clients after sending a goodbye message
func (r Realm) Close() {
	iter := r.clients.Iter()
	for client := range iter {
		sess, isSession := client.Val.(*Session)
		if !isSession {
			continue
		}
		sess.kill <- ErrSystemShutdown
	}

	close(r.acts)
}

func (r *Realm) init() {
	r.lock.Lock()
	defer r.lock.Unlock()

	r.clients = cmap.New()

	if r.localClient == nil {
		p, _ := r.getPeer(nil)
		client := NewClient(p)
		r.localClient = new(localClient)
		r.localClient.Client = client
	}

	if r.Broker == nil {
		r.Broker = NewDefaultBroker()
	}
	if r.Dealer == nil {
		r.Dealer = NewDefaultDealer()
	}
	if r.Authorizer == nil {
		r.Authorizer = NewDefaultAuthorizer()
	}
	if r.Interceptor == nil {
		r.Interceptor = NewDefaultInterceptor()
	}
	if r.AuthTimeout == 0 {
		r.AuthTimeout = defaultAuthTimeout
	}
	go r.localClient.Receive()
	go r.run()
}

func (r *Realm) run() {
	for {
		if act, ok := <-r.acts; ok {
			act()
		} else {
			return
		}
	}
}

func (l *localClient) onJoin(details map[string]interface{}) {
	l.Lock()
	defer l.Unlock()
	l.Publish("wamp.session.on_join", []interface{}{details}, nil)
}

func (l *localClient) onLeave(session ID) {
	l.Lock()
	defer l.Unlock()
	l.Publish("wamp.session.on_leave", []interface{}{session}, nil)
}

func (r *Realm) doOne(c <-chan Message, sess *Session) bool {
	r.lock.RLock()
	defer r.lock.RUnlock()

	var msg Message
	var open bool
	select {
	case msg, open = <-c:
		if !open {
			log.WithField("session_id", sess.Id).Error("lost session")
			return false
		}
	case reason := <-sess.kill:
		logErr(sess.Send(&Goodbye{Reason: reason, Details: make(map[string]interface{})}))
		log.Printf("kill session %s: %v", sess, reason)
		// TODO: wait for client Goodbye?
		return false
	}

	redactedMsg := redactMessage(msg)

	log.WithFields(logrus.Fields{
		"session_id":   sess.Id,
		"message_type": msg.MessageType().String(),
		"message":      redactedMsg,
	}).Info("new message")

	if isAuthz, err := r.Authorizer.Authorize(sess, msg); !isAuthz {
		errMsg := &Error{Type: msg.MessageType()}
		if err != nil {
			errMsg.Error = ErrAuthorizationFailed
			log.WithFields(logrus.Fields{
				"session_id":   sess.Id,
				"message_type": msg.MessageType().String(),
				"message":      redactedMsg,
				"error":        err,
			}).Info("authorization failed")
		} else {
			errMsg.Error = ErrNotAuthorized
			log.WithFields(logrus.Fields{
				"session_id":   sess.Id,
				"message_type": msg.MessageType().String(),
				"message":      redactedMsg,
				"error":        err,
			}).Info("UNAUTHORIZED")
		}
		logErr(sess.Send(errMsg))
		return true
	}

	r.Interceptor.Intercept(sess, &msg)

	switch msg := msg.(type) {
	case *Goodbye:
		logErr(sess.Send(&Goodbye{Reason: ErrGoodbyeAndOut, Details: make(map[string]interface{})}))
		log.WithFields(logrus.Fields{
			"session_id": sess.Id,
			"reason":     msg.Reason,
		}).Warning("leaving")
		return false

	// Broker messages
	case *Publish:
		r.Broker.Publish(sess, msg)
	case *Subscribe:
		r.Broker.Subscribe(sess, msg)
	case *Unsubscribe:
		r.Broker.Unsubscribe(sess, msg)

	// Dealer messages
	case *Register:
		r.Dealer.Register(sess, msg)
	case *Unregister:
		r.Dealer.Unregister(sess, msg)
	case *Call:
		r.Dealer.Call(sess, msg)
	case *Yield:
		r.Dealer.Yield(sess, msg)

	// Error messages
	case *Error:
		if msg.Type == INVOCATION {
			// the only type of ERROR message the router should receive
			r.Dealer.Error(sess, msg)
		} else {
			log.Infof("invalid ERROR message received: %v", msg)
		}

	default:
		log.Infof("Unhandled message:", msg.MessageType())
	}
	return true
}

func redactMessage(msg Message) Message {
	switch msg := msg.(type) {
	case *Call:
		var redacted Call
		redacted.Request = msg.Request
		redacted.Arguments = append(redacted.Arguments, msg.Arguments...)
		redacted.ArgumentsKw = make(map[string]interface{})
		for k, v := range msg.ArgumentsKw {
			if k == "token" {
				v = "redacted"
			}
			redacted.ArgumentsKw[k] = v
		}
		redacted.Options = make(map[string]interface{})
		for k, v := range msg.Options {
			redacted.Options[k] = v
		}
		return &redacted
	}
	return msg
}

func (r *Realm) handleSession(sess *Session) {
	r.lock.RLock()
	r.clients.Set(string(sess.Id), sess)
	r.localClient.onJoin(sess.Details)
	r.lock.RUnlock()

	defer func() {
		r.lock.RLock()
		defer r.lock.RUnlock()

		r.clients.Remove(string(sess.Id))
		r.Broker.RemoveSession(sess)
		r.Dealer.RemoveSession(sess)
		r.localClient.onLeave(sess.Id)
	}()
	c := sess.Receive()
	// TODO: what happens if the realm is closed?

	for r.doOne(c, sess) {
	}
	return msg
}

func (r *Realm) handleAuth(client Peer, details map[string]interface{}) (*Welcome, error) {
	msg, err := r.authenticate(details)
	if err != nil {
		return nil, err
	}
	// we should never get anything besides WELCOME and CHALLENGE
	if msg.MessageType() == WELCOME {
		return msg.(*Welcome), nil
	}
	// Challenge response
	challenge := msg.(*Challenge)
	if err := client.Send(challenge); err != nil {
		return nil, err
	}

	log.WithFields(logrus.Fields{
		"message_type": msg.MessageType().String(),
		"message":      msg,
	}).Info("challenge")

	msg, err = GetMessageTimeout(client, r.AuthTimeout)
	if err != nil {
		return nil, err
	}

	if authenticate, ok := msg.(*Authenticate); !ok {
		return nil, fmt.Errorf("unexpected %s message received", msg.MessageType())
	} else {
		redacted := map[string]interface{}{
			"Extra":     authenticate.Extra,
			"Signature": "redacted",
		}
		log.WithFields(logrus.Fields{
			"message_type": msg.MessageType().String(),
			"message":      redacted,
		}).Info("authenticate")

		return r.checkResponse(challenge, authenticate)
	}
}

// Authenticate either authenticates a client or returns a challenge message if
// challenge/response authentication is to be used.
func (r Realm) authenticate(details map[string]interface{}) (Message, error) {
	if len(r.Authenticators) == 0 && len(r.CRAuthenticators) == 0 {
		return &Welcome{}, nil
	}
	// TODO: this might not always be a []interface{}. Using the JSON unmarshaller it will be,
	// but we may have serializations that preserve more of the original type.
	// For now, the tests just explicitly send a []interface{}
	_authmethods, ok := details["authmethods"].([]interface{})
	if !ok {
		return nil, fmt.Errorf("No authentication supplied")
	}
	authmethods := []string{}
	for _, method := range _authmethods {
		if m, ok := method.(string); ok {
			authmethods = append(authmethods, m)
		} else {
			log.Errorf("invalid authmethod value: %v", method)
		}
	}
	for _, method := range authmethods {
		if auth, ok := r.CRAuthenticators[method]; ok {
			if challenge, err := auth.Challenge(details); err != nil {
				return nil, err
			} else {
				return &Challenge{AuthMethod: method, Extra: challenge}, nil
			}
		}
		if auth, ok := r.Authenticators[method]; ok {
			if authDetails, err := auth.Authenticate(details); err != nil {
				return nil, err
			} else {
				return &Welcome{Details: addAuthMethod(authDetails, method)}, nil
			}
		}
	}
	// TODO: check default auth (special '*' auth?)
	return nil, fmt.Errorf("could not authenticate with any method")
}

// checkResponse determines whether the response to the challenge is sufficient to gain access to the Realm.
func (r Realm) checkResponse(chal *Challenge, auth *Authenticate) (*Welcome, error) {
	authenticator, ok := r.CRAuthenticators[chal.AuthMethod]
	if !ok {
		return nil, fmt.Errorf("authentication method has been removed")
	}
	if details, err := authenticator.Authenticate(chal.Extra, auth.Signature); err != nil {
		return nil, err
	} else {
		return &Welcome{Details: addAuthMethod(details, chal.AuthMethod)}, nil
	}
}

func addAuthMethod(details map[string]interface{}, method string) map[string]interface{} {
	if details == nil {
		details = make(map[string]interface{})
	}
	details["authmethod"] = method
	return details
}

// r := Realm{
// 	Authenticators: map[string]gowamp.Authenticator{
// 		"wampcra": gowamp.NewCRAAuthenticatorFactoryFactory(mySecret),
// 		"ticket": gowamp.NewTicketAuthenticator(myTicket),
// 		"asdfasdf": myAsdfAuthenticator,
// 	},
// 	BasicAuthenticators: map[string]turnpike.BasicAuthenticator{
// 		"anonymous": nil,
// 	},
// }
<|endoftext|>


<commit_before>package db_manager

import (
	"fmt"
	"github.com/garyburd/redigo/redis"
	"log"
	"strings"
	"warcluster/entities"
)

// SetEntity takes an Entity (struct used as template for all data containers to ease the managing of the DB)
// and generates an unique key in order to add the record to the DB.
func SetEntity(entity entities.Entity) bool {
	key, prepared_entity, err := entity.Serialize()
	if err != nil {
		return false
	}

	mutex.Lock()
	send_err := connection.Send("SET", key, prepared_entity)
	mutex.Unlock()
	if send_err != nil {
		log.Print(send_err)
	}

	mutex.Lock()
	flush_err := connection.Flush()
	mutex.Unlock()
	if flush_err != nil {
		log.Print(flush_err)
	}
	return true
}

// GetEntity is used to pull information from the DB in order to be used by the server.
// GetEntity operates as read only function and does not modify the data in the DB.
func GetEntity(key string) (entities.Entity, error) {
	mutex.Lock()
	result, err := redis.Bytes(connection.Do("GET", key))
	mutex.Unlock()
	if err != nil {
		return nil, err
	}
	return entities.Construct(key, result), nil
}

// GetList is a special function needed to parse a list of keys stored in the DB for quick acsess.
// For instance provide a userna in order to acsess the list of planets owned by this player.
// The list will be iterated upon in order to call GetEntity for every key.
func GetList(group_type string, username string) []entities.Entity {
	var entity_list []entities.Entity
	var coord string

	mutex.Lock()
	result, err := redis.String(connection.Do("GET", fmt.Sprintf("%v.%v", group_type, username)))
	mutex.Unlock()
	if err != nil {
		return nil
	}

	for _, coord = range strings.Split(result, ",") {
		key := fmt.Sprintf("%v.%v", group_type[:len(group_type)-1], coord)
		entity, _ := GetEntity(key)
		entity_list = append(entity_list, entity)
	}
	return entity_list
}

// GetEntities operates as GetEntity but instead of an unique key it takes a patern in order to return
// a lyst of Entitys that reflect the entered patern.
func GetEntities(pattern string) []entities.Entity {
	mutex.Lock()
	result, err := redis.Values(connection.Do("KEYS", pattern))
	mutex.Unlock()
	if err != nil {
		return nil
	}

	results := fmt.Sprintf("%s", result)
	var entity_list []entities.Entity
	for _, key := range strings.Split(results[1:len(results)-1], " ") {
		if entity, err := GetEntity(key); err == nil {
			entity_list = append(entity_list, entity)
		}
	}
	return entity_list
}

// I think DeleteEntity speaks for itself but still. This function is used to remove entrys from the DB.
func DeleteEntity(key string) error {
	mutex.Lock()
	_, err := redis.Bytes(connection.Do("DEL", key))
	mutex.Unlock()
	return err
}
<commit_msg>db_manager now uses the native json marshaling<commit_after>package db_manager

import (
	"encoding/json"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"log"
	"strings"
	"warcluster/entities"
)

// SetEntity takes an Entity (struct used as template for all data containers to ease the managing of the DB)
// and generates an unique key in order to add the record to the DB.
func SetEntity(entity entities.Entity) bool {
	key := entity.GetKey()
	prepared_entity, err := json.Marshal(entity)
	if err != nil {
		return false
	}

	mutex.Lock()
	send_err := connection.Send("SET", key, prepared_entity)
	mutex.Unlock()
	if send_err != nil {
		log.Print(send_err)
	}

	mutex.Lock()
	flush_err := connection.Flush()
	mutex.Unlock()
	if flush_err != nil {
		log.Print(flush_err)
	}
	return true
}

// GetEntity is used to pull information from the DB in order to be used by the server.
// GetEntity operates as read only function and does not modify the data in the DB.
func GetEntity(key string) (entities.Entity, error) {
	mutex.Lock()
	result, err := redis.Bytes(connection.Do("GET", key))
	mutex.Unlock()
	if err != nil {
		return nil, err
	}
	return entities.Construct(key, result), nil
}

// GetList is a special function needed to parse a list of keys stored in the DB for quick acsess.
// For instance provide a userna in order to acsess the list of planets owned by this player.
// The list will be iterated upon in order to call GetEntity for every key.
func GetList(group_type string, username string) []entities.Entity {
	var entity_list []entities.Entity
	var coord string

	mutex.Lock()
	result, err := redis.String(connection.Do("GET", fmt.Sprintf("%v.%v", group_type, username)))
	mutex.Unlock()
	if err != nil {
		return nil
	}

	for _, coord = range strings.Split(result, ",") {
		key := fmt.Sprintf("%v.%v", group_type[:len(group_type)-1], coord)
		entity, _ := GetEntity(key)
		entity_list = append(entity_list, entity)
	}
	return entity_list
}

// GetEntities operates as GetEntity but instead of an unique key it takes a patern in order to return
// a lyst of Entitys that reflect the entered patern.
func GetEntities(pattern string) []entities.Entity {
	mutex.Lock()
	result, err := redis.Values(connection.Do("KEYS", pattern))
	mutex.Unlock()
	if err != nil {
		return nil
	}

	results := fmt.Sprintf("%s", result)
	var entity_list []entities.Entity
	for _, key := range strings.Split(results[1:len(results)-1], " ") {
		if entity, err := GetEntity(key); err == nil {
			entity_list = append(entity_list, entity)
		}
	}
	return entity_list
}

// I think DeleteEntity speaks for itself but still. This function is used to remove entrys from the DB.
func DeleteEntity(key string) error {
	mutex.Lock()
	_, err := redis.Bytes(connection.Do("DEL", key))
	mutex.Unlock()
	return err
}
<|endoftext|>


<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.

package apd

import "math/big"

// Round sets d to rounded x. The result is stored in d and returned. If
// d has zero Precision, no modification of x is done. If d has no Rounding
// specified, RoundHalfUp is used.
func (c *Context) Round(d, x *Decimal) (Condition, error) {
	return c.round(d, x).GoError(c.Traps)
}

func (c *Context) round(d, x *Decimal) Condition {
	if c.Precision == 0 {
		d.Set(x)
		return d.setExponent(c, int64(d.Exponent))
	}
	rounder := c.Rounding
	if rounder == nil {
		rounder = roundHalfUp
	}
	res := rounder(c, d, x)
	res |= d.setExponent(c, int64(d.Exponent))
	return res
}

// Rounder sets d to rounded x.
type Rounder func(c *Context, d, x *Decimal) Condition

var (
	// RoundDown rounds toward 0; truncate.
	RoundDown Rounder = roundDown
	// RoundHalfUp rounds up if the digits are >= 0.5.
	RoundHalfUp Rounder = roundHalfUp
	// RoundHalfEven rounds up if the digits are > 0.5. If the digits are equal
	// to 0.5, it rounds up if the previous digit is odd, always producing an
	// even digit.
	RoundHalfEven Rounder = roundHalfEven
	// RoundCeiling towards +Inf: rounds up if digits are > 0 and the number
	// is positive.
	RoundCeiling Rounder = roundCeiling
	// RoundFloor towards -Inf: rounds up if digits are > 0 and the number
	// is negative.
	RoundFloor Rounder = roundFloor
	// RoundHalfDown rounds up if the digits are > 0.5.
	RoundHalfDown Rounder = roundHalfDown
	// RoundUp rounds up if the digits > 0.
	RoundUp Rounder = roundUp
)

func roundDown(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		return false
	})
}

func roundHalfUp(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		m.Abs(m)
		m.Mul(m, bigTwo)
		return m.Cmp(e) >= 0
	})
}

func roundHalfEven(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		m.Abs(m)
		m.Mul(m, bigTwo)
		if c := m.Cmp(e); c > 0 {
			return true
		} else if c == 0 && y.Bit(0) == 1 {
			return true
		}
		return false
	})
}

func roundHalfDown(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		m.Abs(m)
		m.Mul(m, bigTwo)
		return m.Cmp(e) > 0
	})
}

func roundUp(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		return m.Sign() != 0
	})
}

func roundFloor(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		return m.Sign() != 0 && y.Sign() < 0
	})
}

func roundCeiling(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		return m.Sign() != 0 && y.Sign() >= 0
	})
}

func roundFunc(c *Context, d, x *Decimal, f func(m, y, e *big.Int) bool) Condition {
	d.Set(x)
	nd := x.NumDigits()
	var res Condition
	if diff := nd - int64(c.Precision); diff > 0 {
		tmp := new(Decimal).Set(x)
		res |= Rounded
		y := big.NewInt(diff)
		e := new(big.Int).Exp(bigTen, y, nil)
		m := new(big.Int)
		y.QuoRem(&d.Coeff, e, m)
		if f(m, y, e) {
			roundAddOne(y, &diff)
		}
		d.Coeff.Set(y)
		res |= d.setExponent(c, int64(d.Exponent), diff)
		if cmp, _ := d.Cmp(tmp); cmp != 0 {
			res |= Inexact
		}
	}
	return res
}

// roundAddOne adds 1 to abs(b).
func roundAddOne(b *big.Int, diff *int64) {
	nd := NumDigits(b)
	if b.Sign() >= 0 {
		b.Add(b, bigOne)
	} else {
		b.Sub(b, bigOne)
	}
	nd2 := NumDigits(b)
	if nd2 > nd {
		b.Quo(b, bigTen)
		*diff++
	}
}
<commit_msg>More clear description of RoundUp<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.

package apd

import "math/big"

// Round sets d to rounded x. The result is stored in d and returned. If
// d has zero Precision, no modification of x is done. If d has no Rounding
// specified, RoundHalfUp is used.
func (c *Context) Round(d, x *Decimal) (Condition, error) {
	return c.round(d, x).GoError(c.Traps)
}

func (c *Context) round(d, x *Decimal) Condition {
	if c.Precision == 0 {
		d.Set(x)
		return d.setExponent(c, int64(d.Exponent))
	}
	rounder := c.Rounding
	if rounder == nil {
		rounder = roundHalfUp
	}
	res := rounder(c, d, x)
	res |= d.setExponent(c, int64(d.Exponent))
	return res
}

// Rounder sets d to rounded x.
type Rounder func(c *Context, d, x *Decimal) Condition

var (
	// RoundDown rounds toward 0; truncate.
	RoundDown Rounder = roundDown
	// RoundHalfUp rounds up if the digits are >= 0.5.
	RoundHalfUp Rounder = roundHalfUp
	// RoundHalfEven rounds up if the digits are > 0.5. If the digits are equal
	// to 0.5, it rounds up if the previous digit is odd, always producing an
	// even digit.
	RoundHalfEven Rounder = roundHalfEven
	// RoundCeiling towards +Inf: rounds up if digits are > 0 and the number
	// is positive.
	RoundCeiling Rounder = roundCeiling
	// RoundFloor towards -Inf: rounds up if digits are > 0 and the number
	// is negative.
	RoundFloor Rounder = roundFloor
	// RoundHalfDown rounds up if the digits are > 0.5.
	RoundHalfDown Rounder = roundHalfDown
	// RoundUp rounds away from 0.
	RoundUp Rounder = roundUp
)

func roundDown(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		return false
	})
}

func roundHalfUp(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		m.Abs(m)
		m.Mul(m, bigTwo)
		return m.Cmp(e) >= 0
	})
}

func roundHalfEven(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		m.Abs(m)
		m.Mul(m, bigTwo)
		if c := m.Cmp(e); c > 0 {
			return true
		} else if c == 0 && y.Bit(0) == 1 {
			return true
		}
		return false
	})
}

func roundHalfDown(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		m.Abs(m)
		m.Mul(m, bigTwo)
		return m.Cmp(e) > 0
	})
}

func roundUp(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		return m.Sign() != 0
	})
}

func roundFloor(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		return m.Sign() != 0 && y.Sign() < 0
	})
}

func roundCeiling(c *Context, d, x *Decimal) Condition {
	return roundFunc(c, d, x, func(m, y, e *big.Int) bool {
		return m.Sign() != 0 && y.Sign() >= 0
	})
}

func roundFunc(c *Context, d, x *Decimal, f func(m, y, e *big.Int) bool) Condition {
	d.Set(x)
	nd := x.NumDigits()
	var res Condition
	if diff := nd - int64(c.Precision); diff > 0 {
		tmp := new(Decimal).Set(x)
		res |= Rounded
		y := big.NewInt(diff)
		e := new(big.Int).Exp(bigTen, y, nil)
		m := new(big.Int)
		y.QuoRem(&d.Coeff, e, m)
		if f(m, y, e) {
			roundAddOne(y, &diff)
		}
		d.Coeff.Set(y)
		res |= d.setExponent(c, int64(d.Exponent), diff)
		if cmp, _ := d.Cmp(tmp); cmp != 0 {
			res |= Inexact
		}
	}
	return res
}

// roundAddOne adds 1 to abs(b).
func roundAddOne(b *big.Int, diff *int64) {
	nd := NumDigits(b)
	if b.Sign() >= 0 {
		b.Add(b, bigOne)
	} else {
		b.Sub(b, bigOne)
	}
	nd2 := NumDigits(b)
	if nd2 > nd {
		b.Quo(b, bigTen)
		*diff++
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Cutehacks AS. All rights reserved.
// License can be found in the LICENSE file.

package vcs

import (
	"fmt"
	"strings"
	"net/http"
	"encoding/json"
	"path/filepath"
	"io"
	"os"
	"os/exec"
	"compress/gzip"
	"archive/tar"
	"qpm.io/common"
	msg "qpm.io/common/messages"
	"bufio"
)

const (
	GitHubURL     = "https://api.github.com/repos"
	Tarball       = "tarball"
	TarSuffix     = ".tar.gz"
)

func LastCommitSHA1() (string, error) {
	// TODO: refactor this to an interface for all VCSs
	out, err := exec.Command("git","rev-parse", "HEAD").Output()
	return strings.TrimSpace(string(out)), err
}

func LastCommitAuthorName() (string, error) {
	// TODO: refactor this to an interface for all VCSs
	args := []string{"log", "-1", "--format=%an"}
	out, err := exec.Command("git", args...).Output()
	return strings.TrimSpace(string(out)), err
}

func LastCommitEmail() (string, error) {
	// TODO: refactor this to an interface for all VCSs
	args := []string{"log", "-1", "--format=%ae"}
	out, err := exec.Command("git", args...).Output()
	return strings.TrimSpace(string(out)), err
}

func RepositorySubURL() (string, error) {
	// TODO: refactor this to an interface for all VCSs and hosts
	out, err := exec.Command("git", "config", "remote.origin.url").Output()
	if err != nil {
		return "", err
	}

	// assumes github
	str := strings.TrimSpace(string(out))
	if strings.HasPrefix(str, "git@github.com:") {
		str = strings.TrimPrefix(str, "git@github.com:")
		str = strings.TrimSuffix(str, ".git")
	} else if strings.HasPrefix(str, "https://github.com/") {
		str = strings.TrimPrefix(str, "https://github.com/")
	} else {
		return "", fmt.Errorf("This does not seem to be a GitHub repository.")
	}
	return str, nil
}

func RepositoryFileList() ([]string, error) {
	// TODO: refactor this to an interface for all VCSs

	var paths []string
	out, err := exec.Command("git","ls-files").Output()
	if err != nil {
		return paths, err
	}

	// TODO: this may not work on Windows - we need to test this
	output := string(out)
	paths = strings.Split(strings.Trim(output, "\n"), "\n")

	return paths, nil
}

func Tag(name string) error {
	_, err := exec.Command("git", "tag", name).Output()
	if err != nil {
		return err
	}
	return nil
}

// Validate that the given commit has been published to the origin repository.
func ValidateCommit(commit string) error {

	// First run 'git ls-remote' to get the HEADs of all published remotes
	lsRemote := exec.Command("git", "ls-remote")
	stdout, err := lsRemote.StdoutPipe()
	if err != nil {
		return err
	}

	if err := lsRemote.Start(); err != nil {
		return err
	}

	remotes := make(map[string]string)
	scanner := bufio.NewScanner(stdout)
	scanner.Split(bufio.ScanWords)
	for scanner.Scan() {
		sha1 := scanner.Text()
		if !scanner.Scan() {
			break
		}
		remote := scanner.Text()
		remotes[remote] = sha1
	}

	if err := lsRemote.Wait(); err != nil {
		return err
	}

	// Check if commit is an ancestor of a published remote
	for _, s := range remotes {
		err = exec.Command("git" ,"merge-base", "--is-ancestor", commit, s).Run()
		if err == nil {
			return nil
		}
	}

	if err != nil {
		err = fmt.Errorf("Commit %s has not been pushed yet.", commit[:8])
	}

	return err
}

func Install(dependency *msg.Dependency, destination string) (*common.PackageWrapper, error) {
	fileName, err := download(dependency, destination)
	if err != nil {
		return nil, err
	}

	pkg, err := extract(fileName, destination)
	if err != nil {
		return nil, err
	}

	return pkg, os.Remove(fileName)
}

func download(dependency *msg.Dependency, destination string) (fileName string, err error) {

	url := GitHubURL + "/" + dependency.Repository.Url + "/" + Tarball + "/" + dependency.Version.Revision
	tokens := strings.Split(url, "/")
	fileName = destination + string(filepath.Separator) + tokens[len(tokens)-2] + TarSuffix // FIXME: we assume it's a tarball

	var output *os.File
	output, err = os.Create(fileName)
	if err != nil {
		// TODO: check file existence first with os.IsExist(err)
		return "", err
	}
	defer output.Close()

	var response *http.Response
	response, err = http.Get(url)
	if err != nil {
		return "", err
	}
	defer response.Body.Close()

	if response.StatusCode >= 400 {
		errResp := make(map[string]string)
		dec := json.NewDecoder(response.Body)
		if err = dec.Decode(&errResp); err != nil {
			return "", err
		}
		errMsg, ok := errResp["message"]
		if !ok {
			errMsg = response.Status
		}
		err = fmt.Errorf("Error fetching %s: %s", url, errMsg)
		return "", err
	}

	//proxy := &ProgressProxyReader{ Reader: response.Body, length: response.ContentLength }
	//var written int64
	_, err = io.Copy(output, response.Body)
	if err != nil {
		return "", err
	}

	return fileName, nil
}

func extract(fileName string, destination string) (*common.PackageWrapper, error) {

	file, err := os.Open(fileName)

	if err != nil {
		return nil, err
	}

	defer file.Close()

	var fileReader io.ReadCloser = file

	// add a filter to handle gzipped file
	if strings.HasSuffix(fileName, ".gz") {
		if fileReader, err = gzip.NewReader(file); err != nil {
			return nil, err
		}
		defer fileReader.Close()
	}

	tarBallReader := tar.NewReader(fileReader)
	var topDir string

	for {
		header, err := tarBallReader.Next()
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, err
		}

		filename := destination + string(filepath.Separator) + header.Name

		switch header.Typeflag {
		case tar.TypeDir:
			tokens := strings.Split(header.Name, string(filepath.Separator))
			topDir = tokens[0]
			err = os.MkdirAll(filename, os.FileMode(header.Mode)) // or use 0755
			if err != nil {
				return nil, err
			}

		case tar.TypeReg:
			writer, err := os.Create(filename)
			if err != nil {
				return nil, err
			}
			io.Copy(writer, tarBallReader)
			err = os.Chmod(filename, os.FileMode(header.Mode))
			if err != nil {
				return nil, err
			}
			writer.Close()

		case tar.TypeXGlobalHeader:
		// Ignore this

		default:
		//i.Info("Unable to extract type : %c in file %s\n", header.Typeflag, filename)
		}
	}

	if topDir != "" {

		src := filepath.Join(destination, topDir)

		pkg, err := common.LoadPackage(src)
		if err != nil {
			return pkg, err
		}

		path := filepath.Join(destination, pkg.QrcPrefix())

		if err := os.MkdirAll(path, 0755); err != nil {
			return pkg, err
		}

		os.RemoveAll(path)

		if err = os.Rename(src, path); err != nil {
			return pkg, err
		}

		return pkg, err
	}

	return nil, nil
}<commit_msg>Don't fail if we are not able to get the remote origin url from git.<commit_after>// Copyright 2015 Cutehacks AS. All rights reserved.
// License can be found in the LICENSE file.

package vcs

import (
	"fmt"
	"strings"
	"net/http"
	"encoding/json"
	"path/filepath"
	"io"
	"os"
	"os/exec"
	"compress/gzip"
	"archive/tar"
	"qpm.io/common"
	msg "qpm.io/common/messages"
	"bufio"
)

const (
	GitHubURL     = "https://api.github.com/repos"
	Tarball       = "tarball"
	TarSuffix     = ".tar.gz"
)

func LastCommitSHA1() (string, error) {
	// TODO: refactor this to an interface for all VCSs
	out, err := exec.Command("git","rev-parse", "HEAD").Output()
	return strings.TrimSpace(string(out)), err
}

func LastCommitAuthorName() (string, error) {
	// TODO: refactor this to an interface for all VCSs
	args := []string{"log", "-1", "--format=%an"}
	out, err := exec.Command("git", args...).Output()
	return strings.TrimSpace(string(out)), err
}

func LastCommitEmail() (string, error) {
	// TODO: refactor this to an interface for all VCSs
	args := []string{"log", "-1", "--format=%ae"}
	out, err := exec.Command("git", args...).Output()
	return strings.TrimSpace(string(out)), err
}

func RepositorySubURL() (string, error) {
	// TODO: refactor this to an interface for all VCSs and hosts
	out, err := exec.Command("git", "config", "remote.origin.url").Output()
	if err != nil {
		return "", fmt.Errorf("We could not get the repository remote origin URL.")
	}

	// assumes github
	str := strings.TrimSpace(string(out))
	if strings.HasPrefix(str, "git@github.com:") {
		str = strings.TrimPrefix(str, "git@github.com:")
		str = strings.TrimSuffix(str, ".git")
	} else if strings.HasPrefix(str, "https://github.com/") {
		str = strings.TrimPrefix(str, "https://github.com/")
	} else {
		return "", fmt.Errorf("This does not seem to be a GitHub repository.")
	}
	return str, nil
}

func RepositoryFileList() ([]string, error) {
	// TODO: refactor this to an interface for all VCSs

	var paths []string
	out, err := exec.Command("git","ls-files").Output()
	if err != nil {
		return paths, err
	}

	// TODO: this may not work on Windows - we need to test this
	output := string(out)
	paths = strings.Split(strings.Trim(output, "\n"), "\n")

	return paths, nil
}

func Tag(name string) error {
	_, err := exec.Command("git", "tag", name).Output()
	if err != nil {
		return err
	}
	return nil
}

// Validate that the given commit has been published to the origin repository.
func ValidateCommit(commit string) error {

	// First run 'git ls-remote' to get the HEADs of all published remotes
	lsRemote := exec.Command("git", "ls-remote")
	stdout, err := lsRemote.StdoutPipe()
	if err != nil {
		return err
	}

	if err := lsRemote.Start(); err != nil {
		return err
	}

	remotes := make(map[string]string)
	scanner := bufio.NewScanner(stdout)
	scanner.Split(bufio.ScanWords)
	for scanner.Scan() {
		sha1 := scanner.Text()
		if !scanner.Scan() {
			break
		}
		remote := scanner.Text()
		remotes[remote] = sha1
	}

	if err := lsRemote.Wait(); err != nil {
		return err
	}

	// Check if commit is an ancestor of a published remote
	for _, s := range remotes {
		err = exec.Command("git" ,"merge-base", "--is-ancestor", commit, s).Run()
		if err == nil {
			return nil
		}
	}

	if err != nil {
		err = fmt.Errorf("Commit %s has not been pushed yet.", commit[:8])
	}

	return err
}

func Install(dependency *msg.Dependency, destination string) (*common.PackageWrapper, error) {
	fileName, err := download(dependency, destination)
	if err != nil {
		return nil, err
	}

	pkg, err := extract(fileName, destination)
	if err != nil {
		return nil, err
	}

	return pkg, os.Remove(fileName)
}

func download(dependency *msg.Dependency, destination string) (fileName string, err error) {

	url := GitHubURL + "/" + dependency.Repository.Url + "/" + Tarball + "/" + dependency.Version.Revision
	tokens := strings.Split(url, "/")
	fileName = destination + string(filepath.Separator) + tokens[len(tokens)-2] + TarSuffix // FIXME: we assume it's a tarball

	var output *os.File
	output, err = os.Create(fileName)
	if err != nil {
		// TODO: check file existence first with os.IsExist(err)
		return "", err
	}
	defer output.Close()

	var response *http.Response
	response, err = http.Get(url)
	if err != nil {
		return "", err
	}
	defer response.Body.Close()

	if response.StatusCode >= 400 {
		errResp := make(map[string]string)
		dec := json.NewDecoder(response.Body)
		if err = dec.Decode(&errResp); err != nil {
			return "", err
		}
		errMsg, ok := errResp["message"]
		if !ok {
			errMsg = response.Status
		}
		err = fmt.Errorf("Error fetching %s: %s", url, errMsg)
		return "", err
	}

	//proxy := &ProgressProxyReader{ Reader: response.Body, length: response.ContentLength }
	//var written int64
	_, err = io.Copy(output, response.Body)
	if err != nil {
		return "", err
	}

	return fileName, nil
}

func extract(fileName string, destination string) (*common.PackageWrapper, error) {

	file, err := os.Open(fileName)

	if err != nil {
		return nil, err
	}

	defer file.Close()

	var fileReader io.ReadCloser = file

	// add a filter to handle gzipped file
	if strings.HasSuffix(fileName, ".gz") {
		if fileReader, err = gzip.NewReader(file); err != nil {
			return nil, err
		}
		defer fileReader.Close()
	}

	tarBallReader := tar.NewReader(fileReader)
	var topDir string

	for {
		header, err := tarBallReader.Next()
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, err
		}

		filename := destination + string(filepath.Separator) + header.Name

		switch header.Typeflag {
		case tar.TypeDir:
			tokens := strings.Split(header.Name, string(filepath.Separator))
			topDir = tokens[0]
			err = os.MkdirAll(filename, os.FileMode(header.Mode)) // or use 0755
			if err != nil {
				return nil, err
			}

		case tar.TypeReg:
			writer, err := os.Create(filename)
			if err != nil {
				return nil, err
			}
			io.Copy(writer, tarBallReader)
			err = os.Chmod(filename, os.FileMode(header.Mode))
			if err != nil {
				return nil, err
			}
			writer.Close()

		case tar.TypeXGlobalHeader:
		// Ignore this

		default:
		//i.Info("Unable to extract type : %c in file %s\n", header.Typeflag, filename)
		}
	}

	if topDir != "" {

		src := filepath.Join(destination, topDir)

		pkg, err := common.LoadPackage(src)
		if err != nil {
			return pkg, err
		}

		path := filepath.Join(destination, pkg.QrcPrefix())

		if err := os.MkdirAll(path, 0755); err != nil {
			return pkg, err
		}

		os.RemoveAll(path)

		if err = os.Rename(src, path); err != nil {
			return pkg, err
		}

		return pkg, err
	}

	return nil, nil
}<|endoftext|>


<commit_before>package workq

import (
	"fmt"
	"sj/translator"
)

type Item struct {
	Sender     string
	Message    string
	Translated string
	Src        string
	Dest       string
	Done       chan bool
}

func (i *Item) Translate() {
	var err error

	if i.Src == i.Dest {
		i.Translated = i.Message
	} else {
		i.Translated, err = translator.Translate(i.Src, i.Dest, i.Message)
		if err != nil {
			i.Translated = fmt.Sprintf("(untranslated) %s", i.Message)
		}
	}

	i.Done <- true
	close(i.Done)
}
<commit_msg>Fix the package path for translator in workq/item.go<commit_after>package workq

import (
	"fmt"
	"github.com/Vladimiroff/mazungumzo/translator"
)

type Item struct {
	Sender     string
	Message    string
	Translated string
	Src        string
	Dest       string
	Done       chan bool
}

func (i *Item) Translate() {
	var err error

	if i.Src == i.Dest {
		i.Translated = i.Message
	} else {
		i.Translated, err = translator.Translate(i.Src, i.Dest, i.Message)
		if err != nil {
			i.Translated = fmt.Sprintf("(untranslated) %s", i.Message)
		}
	}

	i.Done <- true
	close(i.Done)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ratelimit

import (
	"time"

	"github.com/jacobsa/gcloud/syncutil"

	"golang.org/x/net/context"
)

// A simple interface for limiting the rate of some event. Unlike TokenBucket,
// does not allow the user control over what time means.
//
// Safe for concurrent access.
type Throttle interface {
	// Return the maximum number of tokens that can be requested in a call to
	// Wait.
	Capacity() (c uint64)

	// Acquire the given number of tokens from the underlying token bucket, then
	// sleep until when it says to wake, returning early with false if the
	// context is cancelled before then.
	//
	// REQUIRES: tokens <= capacity
	Wait(ctx context.Context, tokens uint64) (ok bool)
}

// Create a throttle that uses time.Now to judge the time given to the
// underlying token bucket.
//
// Be aware of the monotonicity issues. In particular:
//
//  *  If the system clock jumps into the future, the throttle will let through
//     a burst of traffic.
//
//  *  If the system clock jumps into the past, it will halt all traffic for
//     a potentially very long amount of time.
//
func NewThrottle(
	rateHz float64,
	capacity uint64) (t Throttle) {
	typed := &throttle{
		startTime: time.Now(),
		bucket:    NewTokenBucket(rateHz, capacity),
	}

	typed.mu = syncutil.NewInvariantMutex(typed.checkInvariants)

	t = typed
	return
}

type throttle struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	startTime time.Time

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// INVARIANT: bucket.CheckInvariants()
	//
	// GUARDED_BY(mu)
	bucket TokenBucket
}

// LOCKS_REQUIRED(t.mu)
func (t *throttle) checkInvariants() {
	// INVARIANT: bucket.CheckInvariants()
	t.bucket.CheckInvariants()
}

// LOCKS_EXCLUDED(t.mu)
func (t *throttle) Capacity() (c uint64) {
	t.mu.Lock()
	c = t.bucket.Capacity()
	t.mu.Unlock()

	return
}

// LOCKS_EXCLUDED(t.mu)
func (t *throttle) Wait(
	ctx context.Context,
	tokens uint64) (ok bool) {
	now := MonotonicTime(time.Now().Sub(t.startTime))

	t.mu.Lock()
	sleepUntil := t.bucket.Remove(now, tokens)
	t.mu.Unlock()

	select {
	case <-ctx.Done():
		return

	case <-time.After(time.Duration(sleepUntil - now)):
		ok = true
		return
	}
}
<commit_msg>Changed the interface.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ratelimit

import (
	"time"

	"github.com/jacobsa/gcloud/syncutil"

	"golang.org/x/net/context"
)

// A simple interface for limiting the rate of some event. Unlike TokenBucket,
// does not allow the user control over what time means.
//
// Safe for concurrent access.
type Throttle interface {
	// Return the maximum number of tokens that can be requested in a call to
	// Wait.
	Capacity() (c uint64)

	// Acquire the given number of tokens from the underlying token bucket, then
	// sleep until when it says to wake. If the context is cancelled before then,
	// return early with an error.
	//
	// REQUIRES: tokens <= capacity
	Wait(ctx context.Context, tokens uint64) (err error)
}

// Create a throttle that uses time.Now to judge the time given to the
// underlying token bucket.
//
// Be aware of the monotonicity issues. In particular:
//
//  *  If the system clock jumps into the future, the throttle will let through
//     a burst of traffic.
//
//  *  If the system clock jumps into the past, it will halt all traffic for
//     a potentially very long amount of time.
//
func NewThrottle(
	rateHz float64,
	capacity uint64) (t Throttle) {
	typed := &throttle{
		startTime: time.Now(),
		bucket:    NewTokenBucket(rateHz, capacity),
	}

	typed.mu = syncutil.NewInvariantMutex(typed.checkInvariants)

	t = typed
	return
}

type throttle struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	startTime time.Time

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// INVARIANT: bucket.CheckInvariants()
	//
	// GUARDED_BY(mu)
	bucket TokenBucket
}

// LOCKS_REQUIRED(t.mu)
func (t *throttle) checkInvariants() {
	// INVARIANT: bucket.CheckInvariants()
	t.bucket.CheckInvariants()
}

// LOCKS_EXCLUDED(t.mu)
func (t *throttle) Capacity() (c uint64) {
	t.mu.Lock()
	c = t.bucket.Capacity()
	t.mu.Unlock()

	return
}

// LOCKS_EXCLUDED(t.mu)
func (t *throttle) Wait(
	ctx context.Context,
	tokens uint64) (ok bool) {
	now := MonotonicTime(time.Now().Sub(t.startTime))

	t.mu.Lock()
	sleepUntil := t.bucket.Remove(now, tokens)
	t.mu.Unlock()

	select {
	case <-ctx.Done():
		return

	case <-time.After(time.Duration(sleepUntil - now)):
		ok = true
		return
	}
}
<|endoftext|>