<commit_before>package koding

import (
	"bytes"
	"errors"
	"sort"
	"strconv"
	"strings"
	"time"

	"koding/kites/kloud/kloud"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/protocol"
	"koding/kites/kloud/provider/amazon"

	"github.com/dgrijalva/jwt-go"
	kiteprotocol "github.com/koding/kite/protocol"
	"github.com/mitchellh/goamz/ec2"
	"github.com/nu7hatch/gouuid"
)

var (
	DefaultCustomAMITag = "koding-stable" // Only use AMI's that have this tag
	DefaultInstanceType = "t2.micro"
)

const (
	DefaultApachePort = 80
	DefaultKitePort   = 3000
)

func (p *Provider) Build(m *protocol.Machine) (resArt *protocol.Artifact, err error) {
	a, err := p.NewClient(m)
	if err != nil {
		return nil, err
	}

	return p.build(a, m, &pushValues{Start: 10, Finish: 90})
}

func (p *Provider) build(a *amazon.AmazonClient, m *protocol.Machine, v *pushValues) (resArt *protocol.Artifact, err error) {
	// returns the normalized step according to the initial start and finish
	// values. i.e for a start,finish pair of (10,90) percentages of
	// 0,15,20,50,80,100 will be according to the function: 10,18,26,50,74,90
	normalize := func(percentage int) int {
		base := v.Finish - v.Start
		step := float64(base) * (float64(percentage) / 100)
		normalized := float64(v.Start) + step
		return int(normalized)
	}

	// Check for total amachine allowance
	checker, err := p.PlanChecker(m)
	if err != nil {
		return nil, err
	}

	p.Log.Info("[%s] checking machine limit for user '%s'", m.Id, m.Username)
	if err := checker.Total(); err != nil {
		return nil, err
	}

	p.Log.Info("[%s] checking alwaysOn limit for user '%s'", m.Id, m.Username)
	if err := checker.AlwaysOn(); err != nil {
		return nil, err
	}

	instanceName := m.Builder["instanceName"].(string)

	a.Push("Initializing data", normalize(10), machinestate.Building)

	infoLog := p.GetCustomLogger(m.Id, "info")
	errLog := p.GetCustomLogger(m.Id, "error")

	a.InfoLog = infoLog

	// this can happen when an Info method is called on a terminated instance.
	// This updates the DB records with the name that EC2 gives us, which is a
	// "terminated-instance"
	if instanceName == "terminated-instance" {
		instanceName = "user-" + m.Username + "-" + strconv.FormatInt(time.Now().UTC().UnixNano(), 10)
		infoLog("Instance name is an artifact (terminated), changing to %s", instanceName)
	}

	a.Push("Checking network requirements", normalize(20), machinestate.Building)

	// get all subnets belonging to Kloud
	kloudKeyName := "Kloud"
	infoLog("Searching for subnets with tag-key %s", kloudKeyName)
	subnets, err := a.SubnetsWithTag(kloudKeyName)
	if err != nil {
		errLog("Searching subnet err: %v", err)
		return nil, errors.New("searching network configuration failed")
	}

	// sort and get the lowest
	infoLog("Searching a subnet with most IPs amongst '%d' subnets", len(subnets))
	subnet := subnetWithMostIPs(subnets)

	infoLog("Using subnet id %s, which has %d available IPs", subnet.SubnetId, subnet.AvailableIpAddressCount)
	a.Builder.SubnetId = subnet.SubnetId

	infoLog("Checking if security group for VPC id %s exists.", subnet.VpcId)
	group, err := a.SecurityGroupFromVPC(subnet.VpcId, kloudKeyName)
	if err != nil {
		errLog("Checking security group err: %v", err)
		return nil, errors.New("checking security requirements failed")
	}

	// add now our security group
	a.Builder.SecurityGroupId = group.Id

	// Use koding plans instead of those later
	a.Builder.InstanceType = DefaultInstanceType

	infoLog("Check if user is allowed to create instance type %s", a.Builder.InstanceType)
	// check if the user is egligible to create a vm with this size
	if err := checker.AllowedInstances(instances[a.Builder.InstanceType]); err != nil {
		return nil, err
	}

	a.Push("Checking base build image", normalize(30), machinestate.Building)

	infoLog("Checking if AMI with tag '%s' exists", DefaultCustomAMITag)
	image, err := a.ImageByTag(DefaultCustomAMITag)
	if err != nil {
		errLog("Checking ami tag failed err: %v", err)
		return nil, errors.New("checking base image failed")
	}

	// Use this ami id, which is going to be a stable one
	a.Builder.SourceAmi = image.Id

	storageSize := 3 // default AMI 3GB size
	if a.Builder.StorageSize != 0 && a.Builder.StorageSize > 3 {
		storageSize = a.Builder.StorageSize
	}

	infoLog("Check if user is allowed to create machine with '%dGB' storage", storageSize)
	// check if the user is egligible to create a vm with this size
	if err := checker.Storage(storageSize); err != nil {
		return nil, err
	}

	// Increase storage if it's passed to us, otherwise the default 3GB is
	// created already with the default AMI
	if a.Builder.StorageSize != 0 {
		for _, device := range image.BlockDevices {
			a.Builder.BlockDeviceMapping = &ec2.BlockDeviceMapping{
				DeviceName:          device.DeviceName,
				VirtualName:         device.VirtualName,
				SnapshotId:          device.SnapshotId,
				VolumeType:          device.VolumeType,
				VolumeSize:          int64(a.Builder.StorageSize),
				DeleteOnTermination: true,
				Encrypted:           false,
			}

			break
		}
	}

	kiteId, err := uuid.NewV4()
	if err != nil {
		panic(err)
	}

	kiteKey, err := p.createKey(m.Username, kiteId.String())
	if err != nil {
		return nil, err
	}

	latestKlientPath, err := p.Bucket.LatestDeb()
	if err != nil {
		errLog("Checking klient S3 path failed: %v", err)
		return nil, errors.New("machine initialization requirements failed [1]")
	}

	latestKlientUrl := p.Bucket.URL(latestKlientPath)

	// Use cloud-init for initial configuration of the VM
	cloudInitConfig := &CloudInitConfig{
		Username:        m.Username,
		UserDomain:      m.Domain.Name,
		Hostname:        m.Username, // no typo here. hostname = username
		KiteKey:         kiteKey,
		LatestKlientURL: latestKlientUrl,
		ApachePort:      DefaultApachePort,
		KitePort:        DefaultKitePort,
		Test:            p.Test,
	}

	// check if the user has some keys
	if keyData, ok := m.Builder["user_ssh_keys"]; ok {
		if keys, ok := keyData.([]string); ok && len(keys) > 0 {
			cloudInitConfig.UserSSHKeys = keys
		}
	}

	cloudInitConfig.setupMigrateScript()

	var userdata bytes.Buffer
	err = cloudInitTemplate.Funcs(funcMap).Execute(&userdata, *cloudInitConfig)
	if err != nil {
		errLog("Template execution failed: %v", err)
		return nil, errors.New("machine initialization requirements failed [2]")
	}

	// user data is now ready!
	a.Builder.UserData = userdata.Bytes()

	// add our Koding keypair
	a.Builder.KeyPair = p.KeyName

	// build now our instance!!
	buildArtifact, err := a.Build(instanceName, normalize(45), normalize(60))
	if err != nil {
		return nil, err
	}
	buildArtifact.MachineId = m.Id

	// cleanup build if something goes wrong here
	defer func() {
		if err != nil {
			p.Log.Warning("[%s] Cleaning up instance. Terminating instance: %s",
				m.Id, buildArtifact.InstanceId)

			if _, err := a.Client.TerminateInstances([]string{buildArtifact.InstanceId}); err != nil {
				p.Log.Warning("[%s] Cleaning up instance failed: %v", m.Id, err)
			}
		}
	}()

	a.Push("Updating/Creating domain", normalize(70), machinestate.Building)
	if err := p.UpdateDomain(buildArtifact.IpAddress, m.Domain.Name, m.Username); err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			p.Log.Warning("[%s] Cleaning up domain record. Deleting domain record: %s",
				m.Id, m.Domain.Name)
			if err := p.DNS.DeleteDomain(m.Domain.Name, buildArtifact.IpAddress); err != nil {
				p.Log.Warning("[%s] Cleaning up domain failed: %v", m.Id, err)
			}

		}
	}()

	tags := []ec2.Tag{
		{Key: "Name", Value: buildArtifact.InstanceName},
		{Key: "koding-user", Value: m.Username},
		{Key: "koding-env", Value: p.Kite.Config.Environment},
		{Key: "koding-machineId", Value: m.Id},
		{Key: "koding-domain", Value: m.Domain.Name},
	}

	infoLog("Adding user tags %v", tags)
	if err := a.AddTags(buildArtifact.InstanceId, tags); err != nil {
		errLog("Adding tags failed: %v", err)
		return nil, errors.New("machine initialization requirements failed [3]")
	}

	buildArtifact.DomainName = m.Domain.Name

	query := kiteprotocol.Kite{ID: kiteId.String()}
	buildArtifact.KiteQuery = query.String()

	a.Push("Checking connectivity", normalize(75), machinestate.Building)
	infoLog("Connecting to remote Klient instance")
	if p.IsKlientReady(query.String()) {
		p.Log.Info("[%s] klient is ready.", m.Id)
	} else {
		p.Log.Warning("[%s] klient is not ready. I couldn't connect to it.", m.Id)
	}

	return buildArtifact, nil
}

// CreateKey signs a new key and returns the token back
func (p *Provider) createKey(username, kiteId string) (string, error) {
	if username == "" {
		return "", kloud.NewError(kloud.ErrSignUsernameEmpty)
	}

	if p.KontrolURL == "" {
		return "", kloud.NewError(kloud.ErrSignKontrolURLEmpty)
	}

	if p.KontrolPrivateKey == "" {
		return "", kloud.NewError(kloud.ErrSignPrivateKeyEmpty)
	}

	if p.KontrolPublicKey == "" {
		return "", kloud.NewError(kloud.ErrSignPublicKeyEmpty)
	}

	token := jwt.New(jwt.GetSigningMethod("RS256"))

	token.Claims = map[string]interface{}{
		"iss":        "koding",                              // Issuer, should be the same username as kontrol
		"sub":        username,                              // Subject
		"iat":        time.Now().UTC().Unix(),               // Issued At
		"jti":        kiteId,                                // JWT ID
		"kontrolURL": p.KontrolURL,                          // Kontrol URL
		"kontrolKey": strings.TrimSpace(p.KontrolPublicKey), // Public key of kontrol
	}

	return token.SignedString([]byte(p.KontrolPrivateKey))
}

type ByMostIP []ec2.Subnet

func (a ByMostIP) Len() int      { return len(a) }
func (a ByMostIP) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByMostIP) Less(i, j int) bool {
	return a[i].AvailableIpAddressCount > a[j].AvailableIpAddressCount
}

func subnetWithMostIPs(subnets []ec2.Subnet) ec2.Subnet {
	sort.Sort(ByMostIP(subnets))
	return subnets[0]
}
<commit_msg>kloud: Validate the final result of cloudinit file.<commit_after>package koding

import (
	"bytes"
	"errors"
	"sort"
	"strconv"
	"strings"
	"time"

	"koding/kites/kloud/kloud"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/protocol"
	"koding/kites/kloud/provider/amazon"

	"github.com/dgrijalva/jwt-go"
	kiteprotocol "github.com/koding/kite/protocol"
	"github.com/mitchellh/goamz/ec2"
	"github.com/nu7hatch/gouuid"
	"gopkg.in/yaml.v2"
)

var (
	DefaultCustomAMITag = "koding-stable" // Only use AMI's that have this tag
	DefaultInstanceType = "t2.micro"
)

const (
	DefaultApachePort = 80
	DefaultKitePort   = 3000
)

func (p *Provider) Build(m *protocol.Machine) (resArt *protocol.Artifact, err error) {
	a, err := p.NewClient(m)
	if err != nil {
		return nil, err
	}

	return p.build(a, m, &pushValues{Start: 10, Finish: 90})
}

func (p *Provider) build(a *amazon.AmazonClient, m *protocol.Machine, v *pushValues) (resArt *protocol.Artifact, err error) {
	// returns the normalized step according to the initial start and finish
	// values. i.e for a start,finish pair of (10,90) percentages of
	// 0,15,20,50,80,100 will be according to the function: 10,18,26,50,74,90
	normalize := func(percentage int) int {
		base := v.Finish - v.Start
		step := float64(base) * (float64(percentage) / 100)
		normalized := float64(v.Start) + step
		return int(normalized)
	}

	// Check for total amachine allowance
	checker, err := p.PlanChecker(m)
	if err != nil {
		return nil, err
	}

	p.Log.Info("[%s] checking machine limit for user '%s'", m.Id, m.Username)
	if err := checker.Total(); err != nil {
		return nil, err
	}

	p.Log.Info("[%s] checking alwaysOn limit for user '%s'", m.Id, m.Username)
	if err := checker.AlwaysOn(); err != nil {
		return nil, err
	}

	instanceName := m.Builder["instanceName"].(string)

	a.Push("Initializing data", normalize(10), machinestate.Building)

	infoLog := p.GetCustomLogger(m.Id, "info")
	errLog := p.GetCustomLogger(m.Id, "error")

	a.InfoLog = infoLog

	// this can happen when an Info method is called on a terminated instance.
	// This updates the DB records with the name that EC2 gives us, which is a
	// "terminated-instance"
	if instanceName == "terminated-instance" {
		instanceName = "user-" + m.Username + "-" + strconv.FormatInt(time.Now().UTC().UnixNano(), 10)
		infoLog("Instance name is an artifact (terminated), changing to %s", instanceName)
	}

	a.Push("Checking network requirements", normalize(20), machinestate.Building)

	// get all subnets belonging to Kloud
	kloudKeyName := "Kloud"
	infoLog("Searching for subnets with tag-key %s", kloudKeyName)
	subnets, err := a.SubnetsWithTag(kloudKeyName)
	if err != nil {
		errLog("Searching subnet err: %v", err)
		return nil, errors.New("searching network configuration failed")
	}

	// sort and get the lowest
	infoLog("Searching a subnet with most IPs amongst '%d' subnets", len(subnets))
	subnet := subnetWithMostIPs(subnets)

	infoLog("Using subnet id %s, which has %d available IPs", subnet.SubnetId, subnet.AvailableIpAddressCount)
	a.Builder.SubnetId = subnet.SubnetId

	infoLog("Checking if security group for VPC id %s exists.", subnet.VpcId)
	group, err := a.SecurityGroupFromVPC(subnet.VpcId, kloudKeyName)
	if err != nil {
		errLog("Checking security group err: %v", err)
		return nil, errors.New("checking security requirements failed")
	}

	// add now our security group
	a.Builder.SecurityGroupId = group.Id

	// Use koding plans instead of those later
	a.Builder.InstanceType = DefaultInstanceType

	infoLog("Check if user is allowed to create instance type %s", a.Builder.InstanceType)
	// check if the user is egligible to create a vm with this size
	if err := checker.AllowedInstances(instances[a.Builder.InstanceType]); err != nil {
		return nil, err
	}

	a.Push("Checking base build image", normalize(30), machinestate.Building)

	infoLog("Checking if AMI with tag '%s' exists", DefaultCustomAMITag)
	image, err := a.ImageByTag(DefaultCustomAMITag)
	if err != nil {
		errLog("Checking ami tag failed err: %v", err)
		return nil, errors.New("checking base image failed")
	}

	// Use this ami id, which is going to be a stable one
	a.Builder.SourceAmi = image.Id

	storageSize := 3 // default AMI 3GB size
	if a.Builder.StorageSize != 0 && a.Builder.StorageSize > 3 {
		storageSize = a.Builder.StorageSize
	}

	infoLog("Check if user is allowed to create machine with '%dGB' storage", storageSize)
	// check if the user is egligible to create a vm with this size
	if err := checker.Storage(storageSize); err != nil {
		return nil, err
	}

	// Increase storage if it's passed to us, otherwise the default 3GB is
	// created already with the default AMI
	if a.Builder.StorageSize != 0 {
		for _, device := range image.BlockDevices {
			a.Builder.BlockDeviceMapping = &ec2.BlockDeviceMapping{
				DeviceName:          device.DeviceName,
				VirtualName:         device.VirtualName,
				SnapshotId:          device.SnapshotId,
				VolumeType:          device.VolumeType,
				VolumeSize:          int64(a.Builder.StorageSize),
				DeleteOnTermination: true,
				Encrypted:           false,
			}

			break
		}
	}

	kiteId, err := uuid.NewV4()
	if err != nil {
		panic(err)
	}

	kiteKey, err := p.createKey(m.Username, kiteId.String())
	if err != nil {
		return nil, err
	}

	latestKlientPath, err := p.Bucket.LatestDeb()
	if err != nil {
		errLog("Checking klient S3 path failed: %v", err)
		return nil, errors.New("machine initialization requirements failed [1]")
	}

	latestKlientUrl := p.Bucket.URL(latestKlientPath)

	// Use cloud-init for initial configuration of the VM
	cloudInitConfig := &CloudInitConfig{
		Username:        m.Username,
		UserDomain:      m.Domain.Name,
		Hostname:        m.Username, // no typo here. hostname = username
		KiteKey:         kiteKey,
		LatestKlientURL: latestKlientUrl,
		ApachePort:      DefaultApachePort,
		KitePort:        DefaultKitePort,
		Test:            p.Test,
	}

	// check if the user has some keys
	if keyData, ok := m.Builder["user_ssh_keys"]; ok {
		if keys, ok := keyData.([]string); ok && len(keys) > 0 {
			cloudInitConfig.UserSSHKeys = keys
		}
	}

	cloudInitConfig.setupMigrateScript()

	var userdata bytes.Buffer
	err = cloudInitTemplate.Funcs(funcMap).Execute(&userdata, *cloudInitConfig)
	if err != nil {
		errLog("Template execution failed: %v", err)
		return nil, errors.New("machine initialization requirements failed [2]")
	}

	// validate the userdata first before sending
	if err = yaml.Unmarshal(userdata.Bytes(), struct{}{}); err != nil {
		return nil, errors.New("Cloudinit template is not a valid YAML file.")
	}

	// user data is now ready!
	a.Builder.UserData = userdata.Bytes()

	// add our Koding keypair
	a.Builder.KeyPair = p.KeyName

	// build now our instance!!
	buildArtifact, err := a.Build(instanceName, normalize(45), normalize(60))
	if err != nil {
		return nil, err
	}
	buildArtifact.MachineId = m.Id

	// cleanup build if something goes wrong here
	defer func() {
		if err != nil {
			p.Log.Warning("[%s] Cleaning up instance. Terminating instance: %s",
				m.Id, buildArtifact.InstanceId)

			if _, err := a.Client.TerminateInstances([]string{buildArtifact.InstanceId}); err != nil {
				p.Log.Warning("[%s] Cleaning up instance failed: %v", m.Id, err)
			}
		}
	}()

	a.Push("Updating/Creating domain", normalize(70), machinestate.Building)
	if err := p.UpdateDomain(buildArtifact.IpAddress, m.Domain.Name, m.Username); err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			p.Log.Warning("[%s] Cleaning up domain record. Deleting domain record: %s",
				m.Id, m.Domain.Name)
			if err := p.DNS.DeleteDomain(m.Domain.Name, buildArtifact.IpAddress); err != nil {
				p.Log.Warning("[%s] Cleaning up domain failed: %v", m.Id, err)
			}

		}
	}()

	tags := []ec2.Tag{
		{Key: "Name", Value: buildArtifact.InstanceName},
		{Key: "koding-user", Value: m.Username},
		{Key: "koding-env", Value: p.Kite.Config.Environment},
		{Key: "koding-machineId", Value: m.Id},
		{Key: "koding-domain", Value: m.Domain.Name},
	}

	infoLog("Adding user tags %v", tags)
	if err := a.AddTags(buildArtifact.InstanceId, tags); err != nil {
		errLog("Adding tags failed: %v", err)
		return nil, errors.New("machine initialization requirements failed [3]")
	}

	buildArtifact.DomainName = m.Domain.Name

	query := kiteprotocol.Kite{ID: kiteId.String()}
	buildArtifact.KiteQuery = query.String()

	a.Push("Checking connectivity", normalize(75), machinestate.Building)
	infoLog("Connecting to remote Klient instance")
	if p.IsKlientReady(query.String()) {
		p.Log.Info("[%s] klient is ready.", m.Id)
	} else {
		p.Log.Warning("[%s] klient is not ready. I couldn't connect to it.", m.Id)
	}

	return buildArtifact, nil
}

// CreateKey signs a new key and returns the token back
func (p *Provider) createKey(username, kiteId string) (string, error) {
	if username == "" {
		return "", kloud.NewError(kloud.ErrSignUsernameEmpty)
	}

	if p.KontrolURL == "" {
		return "", kloud.NewError(kloud.ErrSignKontrolURLEmpty)
	}

	if p.KontrolPrivateKey == "" {
		return "", kloud.NewError(kloud.ErrSignPrivateKeyEmpty)
	}

	if p.KontrolPublicKey == "" {
		return "", kloud.NewError(kloud.ErrSignPublicKeyEmpty)
	}

	token := jwt.New(jwt.GetSigningMethod("RS256"))

	token.Claims = map[string]interface{}{
		"iss":        "koding",                              // Issuer, should be the same username as kontrol
		"sub":        username,                              // Subject
		"iat":        time.Now().UTC().Unix(),               // Issued At
		"jti":        kiteId,                                // JWT ID
		"kontrolURL": p.KontrolURL,                          // Kontrol URL
		"kontrolKey": strings.TrimSpace(p.KontrolPublicKey), // Public key of kontrol
	}

	return token.SignedString([]byte(p.KontrolPrivateKey))
}

type ByMostIP []ec2.Subnet

func (a ByMostIP) Len() int      { return len(a) }
func (a ByMostIP) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByMostIP) Less(i, j int) bool {
	return a[i].AvailableIpAddressCount > a[j].AvailableIpAddressCount
}

func subnetWithMostIPs(subnets []ec2.Subnet) ec2.Subnet {
	sort.Sort(ByMostIP(subnets))
	return subnets[0]
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/boltdb/bolt"
	"log"
	"time"
)

type mig2prevPlayer struct {
	Person         *Person `json:"person"`
	Color          string  `json:"color"`
	PreferredColor string  `json:"preferred_color"`
	Shots          int     `json:"shots"`
	Sweeps         int     `json:"sweeps"`
	Kills          int     `json:"kills"`
	Self           int     `json:"self"`
	Explosions     int     `json:"explosions"`
	Matches        int     `json:"matches"`
	TotalScore     int     `json:"score"`
	Match          *Match  `json:"-"`
}

type mig2prevMatch struct {
	Players    []mig2prevPlayer `json:"players"`
	Judges     []Judge          `json:"judges"`
	Kind       string           `json:"kind"`
	Index      int              `json:"index"`
	Length     int              `json:"length"`
	Pause      time.Duration    `json:"pause"`
	Scheduled  time.Time        `json:"scheduled"`
	Started    time.Time        `json:"started"`
	Ended      time.Time        `json:"ended"`
	Tournament *Tournament      `json:"-"`
	ScoreOrder []int            `json:"score_order"`
	Commits    []Round          `json:"commits"`
}

type mig2prevTournament struct {
	Name      string           `json:"name"`
	ID        string           `json:"id"`
	Players   []mig2prevPlayer `json:"players"`
	Winners   []mig2prevPlayer `json:"winners"`
	Runnerups []string         `json:"runnerups"`
	Judges    []Judge          `json:"judges"`
	Tryouts   []*mig2prevMatch `json:"tryouts"`
	Semis     []*mig2prevMatch `json:"semis"`
	Final     *mig2prevMatch   `json:"final"`
	Current   CurrentMatch     `json:"current"`
	Opened    time.Time        `json:"opened"`
	Scheduled time.Time        `json:"scheduled"`
	Started   time.Time        `json:"started"`
	Ended     time.Time        `json:"ended"`
}

type mig2curPerson struct {
	ID              string   `json:"id"`
	Name            string   `json:"name"`
	Email           string   `json:"email"`
	Nick            string   `json:"nick"`
	ColorPreference []string `json:"color_preference"`
	FacebookID      string   `json:"facebook_id"`
	FacebookToken   string   `json:"facebook_token"`
	AvatarURL       string   `json:"avatar_url"`
	Userlevel       int      `json:"userlevel"`
}

type mig2curTournament struct {
	Name      string           `json:"name"`
	ID        string           `json:"id"`
	Players   []mig2prevPlayer `json:"players"`
	Winners   []mig2prevPlayer `json:"winners"`
	Runnerups []*mig2curPerson `json:"runnerups"`
	Judges    []Judge          `json:"judges"`
	Tryouts   []*mig2prevMatch `json:"tryouts"`
	Semis     []*mig2prevMatch `json:"semis"`
	Final     *mig2prevMatch   `json:"final"`
	Current   CurrentMatch     `json:"current"`
	Opened    time.Time        `json:"opened"`
	Scheduled time.Time        `json:"scheduled"`
	Started   time.Time        `json:"started"`
	Ended     time.Time        `json:"ended"`
}

// MigrateTournamentRunnerupStringPerson changes Tournament.Runnerups to be a
// list of Person{} objects rather than a list of string names.
func MigrateTournamentRunnerupStringPerson(db *bolt.DB) error {
	return db.Update(func(tx *bolt.Tx) error {
		// Load the tournaments
		var ts []*mig2prevTournament
		err := db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket(TournamentKey)
			err := b.ForEach(func(k []byte, v []byte) error {
				t := &mig2prevTournament{}
				err := json.Unmarshal(v, t)
				if err != nil {
					log.Print(err)
					return err
				}
				if err != nil {
					return err
				}

				ts = append(ts, t)
				return nil
			})
			return err
		})

		if err != nil {
			log.Fatal(err)
		}

		var out []*mig2curTournament
		for _, pt := range ts {
			t := mig2curTournament{
				Name:      pt.Name,
				ID:        pt.ID,
				Players:   pt.Players,
				Winners:   pt.Winners,
				Judges:    pt.Judges,
				Tryouts:   pt.Tryouts,
				Semis:     pt.Semis,
				Final:     pt.Final,
				Current:   pt.Current,
				Opened:    pt.Opened,
				Scheduled: pt.Scheduled,
				Started:   pt.Started,
				Ended:     pt.Ended,
			}

			load := func(id string) *mig2curPerson {
				tx, err = db.Begin(false)
				if err != nil {
					log.Fatal(err)
					return nil
				}
				defer tx.Rollback()

				b := tx.Bucket(PeopleKey)
				bs := b.Get([]byte(id))
				p := &mig2curPerson{}
				_ = json.Unmarshal(bs, p)
				return p
			}

			// Convert runnerups from strings to objects...
			for _, n := range pt.Runnerups {
				// ...by using the Person object on the players.
				for _, pl := range pt.Players {
					if pl.Person.Nick == n {
						p := load(pl.Person.ID)
						t.Runnerups = append(t.Runnerups, p)
					}
				}
			}

			out = append(out, &t)
		}

		// Loop the tournament results and save them into the db
		for _, t := range out {
			j, err := json.Marshal(t)
			if err != nil {
				log.Fatal(err)
			}

			err = tx.Bucket(TournamentKey).Put([]byte(t.ID), j)
			if err != nil {
				log.Fatal(err)
			}
		}

		return setVersion(tx, 3)
	})
}
<commit_msg>Fix accidental shadowing error<commit_after>package main

import (
	"encoding/json"
	"github.com/boltdb/bolt"
	"log"
	"time"
)

type mig2prevPlayer struct {
	Person         *Person `json:"person"`
	Color          string  `json:"color"`
	PreferredColor string  `json:"preferred_color"`
	Shots          int     `json:"shots"`
	Sweeps         int     `json:"sweeps"`
	Kills          int     `json:"kills"`
	Self           int     `json:"self"`
	Explosions     int     `json:"explosions"`
	Matches        int     `json:"matches"`
	TotalScore     int     `json:"score"`
	Match          *Match  `json:"-"`
}

type mig2prevMatch struct {
	Players    []mig2prevPlayer `json:"players"`
	Judges     []Judge          `json:"judges"`
	Kind       string           `json:"kind"`
	Index      int              `json:"index"`
	Length     int              `json:"length"`
	Pause      time.Duration    `json:"pause"`
	Scheduled  time.Time        `json:"scheduled"`
	Started    time.Time        `json:"started"`
	Ended      time.Time        `json:"ended"`
	Tournament *Tournament      `json:"-"`
	ScoreOrder []int            `json:"score_order"`
	Commits    []Round          `json:"commits"`
}

type mig2prevTournament struct {
	Name      string           `json:"name"`
	ID        string           `json:"id"`
	Players   []mig2prevPlayer `json:"players"`
	Winners   []mig2prevPlayer `json:"winners"`
	Runnerups []string         `json:"runnerups"`
	Judges    []Judge          `json:"judges"`
	Tryouts   []*mig2prevMatch `json:"tryouts"`
	Semis     []*mig2prevMatch `json:"semis"`
	Final     *mig2prevMatch   `json:"final"`
	Current   CurrentMatch     `json:"current"`
	Opened    time.Time        `json:"opened"`
	Scheduled time.Time        `json:"scheduled"`
	Started   time.Time        `json:"started"`
	Ended     time.Time        `json:"ended"`
}

type mig2curPerson struct {
	ID              string   `json:"id"`
	Name            string   `json:"name"`
	Email           string   `json:"email"`
	Nick            string   `json:"nick"`
	ColorPreference []string `json:"color_preference"`
	FacebookID      string   `json:"facebook_id"`
	FacebookToken   string   `json:"facebook_token"`
	AvatarURL       string   `json:"avatar_url"`
	Userlevel       int      `json:"userlevel"`
}

type mig2curTournament struct {
	Name      string           `json:"name"`
	ID        string           `json:"id"`
	Players   []mig2prevPlayer `json:"players"`
	Winners   []mig2prevPlayer `json:"winners"`
	Runnerups []*mig2curPerson `json:"runnerups"`
	Judges    []Judge          `json:"judges"`
	Tryouts   []*mig2prevMatch `json:"tryouts"`
	Semis     []*mig2prevMatch `json:"semis"`
	Final     *mig2prevMatch   `json:"final"`
	Current   CurrentMatch     `json:"current"`
	Opened    time.Time        `json:"opened"`
	Scheduled time.Time        `json:"scheduled"`
	Started   time.Time        `json:"started"`
	Ended     time.Time        `json:"ended"`
}

// MigrateTournamentRunnerupStringPerson changes Tournament.Runnerups to be a
// list of Person{} objects rather than a list of string names.
func MigrateTournamentRunnerupStringPerson(db *bolt.DB) error {
	return db.Update(func(tx *bolt.Tx) error {
		// Load the tournaments
		var ts []*mig2prevTournament
		err := db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket(TournamentKey)
			err := b.ForEach(func(k []byte, v []byte) error {
				t := &mig2prevTournament{}
				err := json.Unmarshal(v, t)
				if err != nil {
					log.Print(err)
					return err
				}
				if err != nil {
					return err
				}

				ts = append(ts, t)
				return nil
			})
			return err
		})

		if err != nil {
			log.Fatal(err)
		}

		var out []*mig2curTournament
		for _, pt := range ts {
			t := mig2curTournament{
				Name:      pt.Name,
				ID:        pt.ID,
				Players:   pt.Players,
				Winners:   pt.Winners,
				Judges:    pt.Judges,
				Tryouts:   pt.Tryouts,
				Semis:     pt.Semis,
				Final:     pt.Final,
				Current:   pt.Current,
				Opened:    pt.Opened,
				Scheduled: pt.Scheduled,
				Started:   pt.Started,
				Ended:     pt.Ended,
			}

			load := func(id string) *mig2curPerson {
				// Prefixing tx and err with l because otherwise heavy shadowing will
				// break angrily.
				ltx, lerr := db.Begin(false)
				if lerr != nil {
					log.Fatal(lerr)
					return nil
				}
				defer ltx.Rollback()

				b := ltx.Bucket(PeopleKey)
				bs := b.Get([]byte(id))
				p := &mig2curPerson{}
				_ = json.Unmarshal(bs, p)
				return p
			}

			// Convert runnerups from strings to objects...
			for _, n := range pt.Runnerups {
				// ...by using the Person object on the players.
				for _, pl := range pt.Players {
					if pl.Person.Nick == n {
						p := load(pl.Person.ID)
						t.Runnerups = append(t.Runnerups, p)
					}
				}
			}

			out = append(out, &t)
		}

		// Loop the tournament results and save them into the db
		for _, t := range out {
			j, err := json.Marshal(t)
			if err != nil {
				log.Fatal(err)
			}

			err = tx.Bucket(TournamentKey).Put([]byte(t.ID), j)
			if err != nil {
				log.Fatal(err)
			}
		}

		return setVersion(tx, 3)
	})
}
<|endoftext|>


<commit_before>package log

import (
	"context"
	"strings"
	"context"

	"v2ray.com/core/common/serial"
)

type logKey int

const (
	accessMessageKey logKey = iota
)

type AccessStatus string

const (
	AccessAccepted = AccessStatus("accepted")
	AccessRejected = AccessStatus("rejected")
)

type AccessMessage struct {
	From   interface{}
	To     interface{}
	Status AccessStatus
	Reason interface{}
	Email  string
	Detour string
}

func (m *AccessMessage) String() string {
	builder := strings.Builder{}
	builder.WriteString(serial.ToString(m.From))
	builder.WriteByte(' ')
	builder.WriteString(string(m.Status))
	builder.WriteByte(' ')
	builder.WriteString(serial.ToString(m.To))
	builder.WriteByte(' ')
	if len(m.Detour) > 0 {
		builder.WriteByte('[')
		builder.WriteString(m.Detour)
		builder.WriteString("] ")
	}
	builder.WriteString(serial.ToString(m.Reason))

	if len(m.Email) > 0 {
		builder.WriteString("email:")
		builder.WriteString(m.Email)
		builder.WriteByte(' ')
	}
	return builder.String()
}

func ContextWithAccessMessage(ctx context.Context, accessMessage *AccessMessage) context.Context {
	return context.WithValue(ctx, accessMessageKey, accessMessage)
}

func AccessMessageFromContext(ctx context.Context) *AccessMessage {
	if accessMessage, ok := ctx.Value(accessMessageKey).(*AccessMessage); ok {
		return accessMessage
	}
	return nil
}
<commit_msg>Fix merge error<commit_after>package log

import (
	"context"
	"strings"

	"v2ray.com/core/common/serial"
)

type logKey int

const (
	accessMessageKey logKey = iota
)

type AccessStatus string

const (
	AccessAccepted = AccessStatus("accepted")
	AccessRejected = AccessStatus("rejected")
)

type AccessMessage struct {
	From   interface{}
	To     interface{}
	Status AccessStatus
	Reason interface{}
	Email  string
	Detour string
}

func (m *AccessMessage) String() string {
	builder := strings.Builder{}
	builder.WriteString(serial.ToString(m.From))
	builder.WriteByte(' ')
	builder.WriteString(string(m.Status))
	builder.WriteByte(' ')
	builder.WriteString(serial.ToString(m.To))
	builder.WriteByte(' ')
	if len(m.Detour) > 0 {
		builder.WriteByte('[')
		builder.WriteString(m.Detour)
		builder.WriteString("] ")
	}
	builder.WriteString(serial.ToString(m.Reason))

	if len(m.Email) > 0 {
		builder.WriteString("email:")
		builder.WriteString(m.Email)
		builder.WriteByte(' ')
	}
	return builder.String()
}

func ContextWithAccessMessage(ctx context.Context, accessMessage *AccessMessage) context.Context {
	return context.WithValue(ctx, accessMessageKey, accessMessage)
}

func AccessMessageFromContext(ctx context.Context) *AccessMessage {
	if accessMessage, ok := ctx.Value(accessMessageKey).(*AccessMessage); ok {
		return accessMessage
	}
	return nil
}
<|endoftext|>


<commit_before>package httpd

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strings"
)

type Store interface {
	// Get returns the value for the given key.
	Get(key string) (string, error)

	// Set sets the value for the given key, via distributed consensus.
	Set(key, value string) error

	// Delete removes the give key, via distributed consensus.
	Delete(key string) error

	// Join joins the node, reachable at addr, to the cluster.
	Join(addr string) error
}

type Service struct {
	addr string
	ln   net.Listener

	store Store
}

func New(addr string, store Store) *Service {
	return &Service{
		addr:  addr,
		store: store,
	}
}

func (s *Service) Start() error {
	server := http.Server{
		Handler: s,
	}

	ln, err := net.Listen("tcp", s.addr)
	if err != nil {
		return err
	}
	s.ln = ln

	http.Handle("/", s)

	go func() {
		err := server.Serve(s.ln)
		if err != nil {
			log.Fatalf("HTTP serve: %s", err)
		}
	}()

	return nil
}

func (s *Service) Close() {
	s.ln.Close()
	return
}

func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if strings.HasPrefix(r.URL.Path, "/key") {
		s.handleKeyRequest(w, r)
	} else if r.URL.Path == "/join" {
		s.handleJoin(w, r)
	} else {
		w.WriteHeader(http.StatusNotFound)
	}
}

func (s *Service) handleJoin(w http.ResponseWriter, r *http.Request) {
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	m := map[string]string{}
	if err := json.Unmarshal(b, &m); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if len(m) != 1 {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	remoteAddr, ok := m["addr"]
	if !ok {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if err := s.store.Join(remoteAddr); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
}

func (s *Service) handleKeyRequest(w http.ResponseWriter, r *http.Request) {
	getKey := func() string {
		parts := strings.Split(r.URL.Path, "/")
		if len(parts) != 3 {
			return ""
		}
		return parts[2]
	}

	switch r.Method {
	case "GET":
		k := getKey()
		if k == "" {
			w.WriteHeader(http.StatusBadRequest)
		}
		v, err := s.store.Get(k)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		b, err := json.Marshal(map[string]string{k: v})
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		io.WriteString(w, string(b))

	case "POST":
		// Read the value from the POST body.
		b, err := ioutil.ReadAll(r.Body)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		m := map[string]string{}
		if err := json.Unmarshal(b, &m); err != nil {
			w.WriteHeader(http.StatusBadRequest)
		}
		for k, v := range m {
			if err := s.store.Set(k, v); err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
		}

	case "DELETE":
		k := getKey()
		if k == "" {
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		if err := s.store.Delete(k); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		s.store.Delete(k)

	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
	}
	return
}

func (s *Service) Addr() net.Addr {
	return s.ln.Addr()
}
<commit_msg>Fix typo in comment<commit_after>package httpd

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strings"
)

type Store interface {
	// Get returns the value for the given key.
	Get(key string) (string, error)

	// Set sets the value for the given key, via distributed consensus.
	Set(key, value string) error

	// Delete removes the given key, via distributed consensus.
	Delete(key string) error

	// Join joins the node, reachable at addr, to the cluster.
	Join(addr string) error
}

type Service struct {
	addr string
	ln   net.Listener

	store Store
}

func New(addr string, store Store) *Service {
	return &Service{
		addr:  addr,
		store: store,
	}
}

func (s *Service) Start() error {
	server := http.Server{
		Handler: s,
	}

	ln, err := net.Listen("tcp", s.addr)
	if err != nil {
		return err
	}
	s.ln = ln

	http.Handle("/", s)

	go func() {
		err := server.Serve(s.ln)
		if err != nil {
			log.Fatalf("HTTP serve: %s", err)
		}
	}()

	return nil
}

func (s *Service) Close() {
	s.ln.Close()
	return
}

func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if strings.HasPrefix(r.URL.Path, "/key") {
		s.handleKeyRequest(w, r)
	} else if r.URL.Path == "/join" {
		s.handleJoin(w, r)
	} else {
		w.WriteHeader(http.StatusNotFound)
	}
}

func (s *Service) handleJoin(w http.ResponseWriter, r *http.Request) {
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	m := map[string]string{}
	if err := json.Unmarshal(b, &m); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if len(m) != 1 {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	remoteAddr, ok := m["addr"]
	if !ok {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if err := s.store.Join(remoteAddr); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
}

func (s *Service) handleKeyRequest(w http.ResponseWriter, r *http.Request) {
	getKey := func() string {
		parts := strings.Split(r.URL.Path, "/")
		if len(parts) != 3 {
			return ""
		}
		return parts[2]
	}

	switch r.Method {
	case "GET":
		k := getKey()
		if k == "" {
			w.WriteHeader(http.StatusBadRequest)
		}
		v, err := s.store.Get(k)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		b, err := json.Marshal(map[string]string{k: v})
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		io.WriteString(w, string(b))

	case "POST":
		// Read the value from the POST body.
		b, err := ioutil.ReadAll(r.Body)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		m := map[string]string{}
		if err := json.Unmarshal(b, &m); err != nil {
			w.WriteHeader(http.StatusBadRequest)
		}
		for k, v := range m {
			if err := s.store.Set(k, v); err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
		}

	case "DELETE":
		k := getKey()
		if k == "" {
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		if err := s.store.Delete(k); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		s.store.Delete(k)

	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
	}
	return
}

func (s *Service) Addr() net.Addr {
	return s.ln.Addr()
}
<|endoftext|>


<commit_before>package cache

import (
	"bytes"
	"encoding/binary"
	"net"
	"reflect"
	"testing"
	"time"

	"github.com/lomik/go-carbon/points"
	"github.com/stretchr/testify/assert"
)

const sampleCacheQuery = "\x00\x00\x00Y\x80\x02}q\x01(U\x06metricq\x02U,carbon.agents.carbon_agent_server.cache.sizeq\x03U\x04typeq\x04U\x0bcache-queryq\x05u."
const sampleCacheQuery2 = "\x00\x00\x00Y\x80\x02}q\x01(U\x04typeq\x04U\x0bcache-queryq\x05U\x06metricq\x02U,carbon.agents.carbon_agent_server.param.sizeq\x03u."
const sampleCacheQuery3 = "\x00\x00\x00R\x80\x02}(U\x06metricX,\x00\x00\x00carbon.agents.carbon_agent_server.param.sizeU\x04typeU\x0bcache-queryu." // unicode metric

func TestCarbonlink(t *testing.T) {
	assert := assert.New(t)

	cache := New()
	cache.SetOutputChanSize(0)
	cache.Start()

	msg1 := points.OnePoint(
		"carbon.agents.carbon_agent_server.cache.size",
		42.17,
		1422797285,
	)

	msg2 := points.OnePoint(
		"carbon.agents.carbon_agent_server.param.size",
		-42.14,
		1422797267,
	)

	msg3 := points.OnePoint(
		"carbon.agents.carbon_agent_server.param.size",
		15,
		1422795966,
	)

	cache.In() <- msg1
	cache.In() <- msg2
	cache.In() <- msg3

	defer cache.Stop()

	time.Sleep(50 * time.Millisecond)

	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	assert.NoError(err)

	carbonlink := NewCarbonlinkListener(cache.Query())
	defer carbonlink.Stop()

	assert.NoError(carbonlink.Listen(addr))

	NewClient := func() (conn net.Conn, cleanup func()) {
		conn, err := net.Dial("tcp", carbonlink.Addr().String())
		assert.NoError(err)

		conn.SetDeadline(time.Now().Add(time.Second))
		return conn, func() { conn.Close() }
	}

	var replyLength int32
	var data []byte

	/* MESSAGE 1 */
	conn, cleanup := NewClient()
	_, err = conn.Write([]byte(sampleCacheQuery))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	// {'datapoints': [(1422797285, 42.17)]}
	assert.Equal([]byte("\x80\x02}U\ndatapoints]J\xe5)\xceTG@E\x15\xc2\x8f\\(\xf6\x86as."), data)
	cleanup()

	/* MESSAGE 2 */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte(sampleCacheQuery2))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	// {'datapoints': [(1422797267, -42.14), (1422795966, 15.0)]}
	assert.Equal("\x80\x02}U\ndatapoints](J\xd3)\xceTG\xc0E\x11\xeb\x85\x1e\xb8R\x86J\xbe$\xceTG@.\x00\x00\x00\x00\x00\x00\x86es.",
		string(data))
	cleanup()

	/* MESSAGE 2.5 - unicode */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte(sampleCacheQuery3))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	// {'datapoints': [(1422797267, -42.14), (1422795966, 15.0)]}
	assert.Equal("\x80\x02}U\ndatapoints](J\xd3)\xceTG\xc0E\x11\xeb\x85\x1e\xb8R\x86J\xbe$\xceTG@.\x00\x00\x00\x00\x00\x00\x86es.",
		string(data))
	cleanup()

	/* MESSAGE 3 */
	/* Remove carbon.agents.carbon_agent_server.param.size from cache and request again */
	conn, cleanup = NewClient()
	for {
		c := <-cache.Out()
		cache.Confirm() <- c
		if c.Metric == "carbon.agents.carbon_agent_server.param.size" {
			break
		}
	}

	_, err = conn.Write([]byte(sampleCacheQuery2))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	assert.Equal("\x80\x02}U\ndatapoints]s.", string(data))
	cleanup()

	/* WRONG MESSAGE TEST */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte("\x00\x00\x00\x05aaaaa"))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)

	assert.Error(err)
	cleanup()

	/* Invalid querytype */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte("\x00\x00\x00(\x80\x02}q\x00(U\x06metricq\x01U\x03barq\x02U\x04typeq\x03U\x03fooq\x04u."))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)

	assert.NoError(err)
	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	assert.Equal("\x80\x02}q\x00U\x05errorq\x01U\x1aInvalid request type \"foo\"q\x02s.", string(data))
	cleanup()
}

func TestCarbonlinkErrors(t *testing.T) {
	assert := assert.New(t)

	cache := New()
	cache.SetOutputChanSize(0)
	cache.Start()

	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	assert.NoError(err)

	carbonlink := NewCarbonlinkListener(cache.Query())
	listenerTimeout := 10 * time.Millisecond
	carbonlink.SetReadTimeout(listenerTimeout)
	defer carbonlink.Stop()

	err = carbonlink.Listen(addr)
	if err != nil {
		t.Errorf("Received %q when attempting to start CarbonlinkListener", err)
	}

	table := []*struct {
		waitAfterWrite time.Duration
		msg            []byte
	}{
		{ // connect and disconnect
			time.Millisecond * 0,
			[]byte{},
		},
		{ // connect, send msg length and disconnect
			time.Millisecond * 0,
			[]byte(sampleCacheQuery2[:4]),
		},
		{ // connect and wait timeout
			listenerTimeout * 2,
			[]byte{},
		},
		{ // connect, send msg length and wait timeout
			listenerTimeout * 2,
			[]byte(sampleCacheQuery2[:4]),
		},
		{ // send broken pickle
			listenerTimeout * 2,
			[]byte(sampleCacheQuery2[:len(sampleCacheQuery2)-1] + "a"),
		},
	}

	for _, test := range table {
		remoteaddr, _ := net.ResolveTCPAddr("tcp", carbonlink.Addr().String())
		conn, err := net.DialTCP("tcp", addr, remoteaddr)
		assert.NoError(err)

		if len(test.msg) > 0 {
			_, err = conn.Write(test.msg)
			if err != nil {
				t.Errorf("Received %q when attempting to write to CarbonlinkListener", err)
			}
		}
		time.Sleep(test.waitAfterWrite)
		if test.waitAfterWrite > listenerTimeout {
			conn.SetWriteBuffer(0)
			n, err := conn.Write([]byte("x"))
			if err == nil || n > 0 {
				t.Errorf("CarbonlinkListener did not close connection as expected")
			}
		}
		conn.Close()

	}
}

func TestReadCarbonlinkRequest(t *testing.T) {

	tests := []struct {
		name    string
		data    []byte
		want    []byte
		wantErr bool
	}{
		{name: "Empty reader",
			data:    []byte{},
			wantErr: true},
		// Length is expected to be 32 bits
		{name: "Short Length",
			data:    []byte("\x00"),
			wantErr: true},
		{name: "Length, but no body",
			data:    []byte("\x00\x00\x00\x01"),
			wantErr: true},
		{name: "Valid Length and body",
			data: []byte("\x00\x00\x00\x01x"),
			want: []byte("x"),
		},
		{name: "Length zero, no body",
			data: []byte("\x00\x00\x00\x00"),
			want: []byte(""),
		},
	}
	for _, tt := range tests {
		got, err := ReadCarbonlinkRequest(bytes.NewReader(tt.data))
		if (err != nil) != tt.wantErr {
			t.Errorf("%q. ReadCarbonlinkRequest() error = %v, wantErr %v", tt.name, err, tt.wantErr)
			continue
		}
		if !reflect.DeepEqual(got, tt.want) {
			t.Errorf("%q. ReadCarbonlinkRequest() = %v, want %v", tt.name, got, tt.want)
		}
	}
}

func TestNewCarbonlinkRequest(t *testing.T) {
	want := &CarbonlinkRequest{}
	got := NewCarbonlinkRequest()
	if !reflect.DeepEqual(got, want) {
		t.Errorf("NewCarbonlinkRequest() = %v, want %v", got, want)
	}
}

func TestParseCarbonlinkRequest(t *testing.T) {
	tests := []struct {
		name    string
		data    []byte
		want    *CarbonlinkRequest
		wantErr bool
	}{
		{name: "Good query",
			data: []byte(sampleCacheQuery)[4:],
			want: &CarbonlinkRequest{
				Type:   "cache-query",
				Metric: "carbon.agents.carbon_agent_server.cache.size"},
		},
		{name: "Good query2",
			data: []byte(sampleCacheQuery2)[4:],
			want: &CarbonlinkRequest{
				Type:   "cache-query",
				Metric: "carbon.agents.carbon_agent_server.param.size"},
		},
		{name: "Invalid query type",
			data: []byte("\x80\x02}q\x00(U\x06metricq\x01U\x03barq\x02U\x04typeq\x03U\x03fooq\x04u."),
			want: &CarbonlinkRequest{
				Type:   "foo",
				Metric: "bar",
			},
		},
		{name: "Garbage",
			data:    []byte("garbage"),
			wantErr: true,
		},
	}
	for _, tt := range tests {
		got, err := ParseCarbonlinkRequest(tt.data)
		if (err != nil) != tt.wantErr {
			t.Errorf("%q. ParseCarbonlinkRequest() error = %v, wantErr %v", tt.name, err, tt.wantErr)
			continue
		}
		if !reflect.DeepEqual(got, tt.want) {
			t.Errorf("%q. ParseCarbonlinkRequest() = %v, want %v", tt.name, got, tt.want)
		}
	}
}

func BenchmarkCarbonLinkPickleParse(b *testing.B) {
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			ParseCarbonlinkRequest([]byte(sampleCacheQuery))
		}
	})
}

func BenchmarkCarbonLinkPackReply(b *testing.B) {
	p := points.OnePoint("carbon.agents.carbon_agent_server.param.size", 15, 1422795966).Add(15, 9000000).Add(16, 9000000)

	q := Query{CacheData: p}

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			packReply(&q)
		}
	})
}
<commit_msg>Resolves #87<commit_after>package cache

import (
	"bytes"
	"encoding/binary"
	"net"
	"reflect"
	"testing"
	"time"

	"github.com/lomik/go-carbon/points"
	"github.com/stretchr/testify/assert"
)

const sampleCacheQuery = "\x00\x00\x00Y\x80\x02}q\x01(U\x06metricq\x02U,carbon.agents.carbon_agent_server.cache.sizeq\x03U\x04typeq\x04U\x0bcache-queryq\x05u."
const sampleCacheQuery2 = "\x00\x00\x00Y\x80\x02}q\x01(U\x04typeq\x04U\x0bcache-queryq\x05U\x06metricq\x02U,carbon.agents.carbon_agent_server.param.sizeq\x03u."
const sampleCacheQuery3 = "\x00\x00\x00R\x80\x02}(U\x06metricX,\x00\x00\x00carbon.agents.carbon_agent_server.param.sizeU\x04typeU\x0bcache-queryu." // unicode metric

func TestCarbonlink(t *testing.T) {
	assert := assert.New(t)

	cache := New()
	cache.SetOutputChanSize(0)
	cache.Start()

	msg1 := points.OnePoint(
		"carbon.agents.carbon_agent_server.cache.size",
		42.17,
		1422797285,
	)

	msg2 := points.OnePoint(
		"carbon.agents.carbon_agent_server.param.size",
		-42.14,
		1422797267,
	)

	msg3 := points.OnePoint(
		"carbon.agents.carbon_agent_server.param.size",
		15,
		1422795966,
	)

	cache.In() <- msg1
	cache.In() <- msg2
	cache.In() <- msg3

	defer cache.Stop()
	for {
		if len(cache.In()) == 0 {
			break
		}
	}

	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	assert.NoError(err)

	carbonlink := NewCarbonlinkListener(cache.Query())
	defer carbonlink.Stop()

	assert.NoError(carbonlink.Listen(addr))

	NewClient := func() (conn net.Conn, cleanup func()) {
		conn, err := net.Dial("tcp", carbonlink.Addr().String())
		assert.NoError(err)

		conn.SetDeadline(time.Now().Add(time.Second))
		return conn, func() { conn.Close() }
	}

	var replyLength int32
	var data []byte

	/* MESSAGE 1 */
	conn, cleanup := NewClient()
	_, err = conn.Write([]byte(sampleCacheQuery))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	// {'datapoints': [(1422797285, 42.17)]}
	assert.Equal([]byte("\x80\x02}U\ndatapoints]J\xe5)\xceTG@E\x15\xc2\x8f\\(\xf6\x86as."), data)
	cleanup()

	/* MESSAGE 2 */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte(sampleCacheQuery2))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	// {'datapoints': [(1422797267, -42.14), (1422795966, 15.0)]}
	assert.Equal("\x80\x02}U\ndatapoints](J\xd3)\xceTG\xc0E\x11\xeb\x85\x1e\xb8R\x86J\xbe$\xceTG@.\x00\x00\x00\x00\x00\x00\x86es.",
		string(data))
	cleanup()

	/* MESSAGE 2.5 - unicode */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte(sampleCacheQuery3))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	// {'datapoints': [(1422797267, -42.14), (1422795966, 15.0)]}
	assert.Equal("\x80\x02}U\ndatapoints](J\xd3)\xceTG\xc0E\x11\xeb\x85\x1e\xb8R\x86J\xbe$\xceTG@.\x00\x00\x00\x00\x00\x00\x86es.",
		string(data))
	cleanup()

	/* MESSAGE 3 */
	/* Remove carbon.agents.carbon_agent_server.param.size from cache and request again */
	conn, cleanup = NewClient()
	for {
		c := <-cache.Out()
		cache.Confirm() <- c
		if c.Metric == "carbon.agents.carbon_agent_server.param.size" {
			break
		}
	}

	_, err = conn.Write([]byte(sampleCacheQuery2))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	assert.Equal("\x80\x02}U\ndatapoints]s.", string(data))
	cleanup()

	/* WRONG MESSAGE TEST */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte("\x00\x00\x00\x05aaaaa"))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)

	assert.Error(err)
	cleanup()

	/* Invalid querytype */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte("\x00\x00\x00(\x80\x02}q\x00(U\x06metricq\x01U\x03barq\x02U\x04typeq\x03U\x03fooq\x04u."))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)

	assert.NoError(err)
	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	assert.Equal("\x80\x02}q\x00U\x05errorq\x01U\x1aInvalid request type \"foo\"q\x02s.", string(data))
	cleanup()
}

func TestCarbonlinkErrors(t *testing.T) {
	assert := assert.New(t)

	cache := New()
	cache.SetOutputChanSize(0)
	cache.Start()

	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	assert.NoError(err)

	carbonlink := NewCarbonlinkListener(cache.Query())
	listenerTimeout := 10 * time.Millisecond
	carbonlink.SetReadTimeout(listenerTimeout)
	defer carbonlink.Stop()

	err = carbonlink.Listen(addr)
	if err != nil {
		t.Errorf("Received %q when attempting to start CarbonlinkListener", err)
	}

	table := []*struct {
		waitAfterWrite time.Duration
		msg            []byte
	}{
		{ // connect and disconnect
			time.Millisecond * 0,
			[]byte{},
		},
		{ // connect, send msg length and disconnect
			time.Millisecond * 0,
			[]byte(sampleCacheQuery2[:4]),
		},
		{ // connect and wait timeout
			listenerTimeout * 2,
			[]byte{},
		},
		{ // connect, send msg length and wait timeout
			listenerTimeout * 2,
			[]byte(sampleCacheQuery2[:4]),
		},
		{ // send broken pickle
			listenerTimeout * 2,
			[]byte(sampleCacheQuery2[:len(sampleCacheQuery2)-1] + "a"),
		},
	}

	for _, test := range table {
		remoteaddr, _ := net.ResolveTCPAddr("tcp", carbonlink.Addr().String())
		conn, err := net.DialTCP("tcp", addr, remoteaddr)
		assert.NoError(err)

		if len(test.msg) > 0 {
			_, err = conn.Write(test.msg)
			if err != nil {
				t.Errorf("Received %q when attempting to write to CarbonlinkListener", err)
			}
		}
		time.Sleep(test.waitAfterWrite)
		if test.waitAfterWrite > listenerTimeout {
			conn.SetWriteBuffer(0)
			n, err := conn.Write([]byte("x"))
			if err == nil || n > 0 {
				t.Errorf("CarbonlinkListener did not close connection as expected")
			}
		}
		conn.Close()

	}
}

func TestReadCarbonlinkRequest(t *testing.T) {

	tests := []struct {
		name    string
		data    []byte
		want    []byte
		wantErr bool
	}{
		{name: "Empty reader",
			data:    []byte{},
			wantErr: true},
		// Length is expected to be 32 bits
		{name: "Short Length",
			data:    []byte("\x00"),
			wantErr: true},
		{name: "Length, but no body",
			data:    []byte("\x00\x00\x00\x01"),
			wantErr: true},
		{name: "Valid Length and body",
			data: []byte("\x00\x00\x00\x01x"),
			want: []byte("x"),
		},
		{name: "Length zero, no body",
			data: []byte("\x00\x00\x00\x00"),
			want: []byte(""),
		},
	}
	for _, tt := range tests {
		got, err := ReadCarbonlinkRequest(bytes.NewReader(tt.data))
		if (err != nil) != tt.wantErr {
			t.Errorf("%q. ReadCarbonlinkRequest() error = %v, wantErr %v", tt.name, err, tt.wantErr)
			continue
		}
		if !reflect.DeepEqual(got, tt.want) {
			t.Errorf("%q. ReadCarbonlinkRequest() = %v, want %v", tt.name, got, tt.want)
		}
	}
}

func TestNewCarbonlinkRequest(t *testing.T) {
	want := &CarbonlinkRequest{}
	got := NewCarbonlinkRequest()
	if !reflect.DeepEqual(got, want) {
		t.Errorf("NewCarbonlinkRequest() = %v, want %v", got, want)
	}
}

func TestParseCarbonlinkRequest(t *testing.T) {
	tests := []struct {
		name    string
		data    []byte
		want    *CarbonlinkRequest
		wantErr bool
	}{
		{name: "Good query",
			data: []byte(sampleCacheQuery)[4:],
			want: &CarbonlinkRequest{
				Type:   "cache-query",
				Metric: "carbon.agents.carbon_agent_server.cache.size"},
		},
		{name: "Good query2",
			data: []byte(sampleCacheQuery2)[4:],
			want: &CarbonlinkRequest{
				Type:   "cache-query",
				Metric: "carbon.agents.carbon_agent_server.param.size"},
		},
		{name: "Invalid query type",
			data: []byte("\x80\x02}q\x00(U\x06metricq\x01U\x03barq\x02U\x04typeq\x03U\x03fooq\x04u."),
			want: &CarbonlinkRequest{
				Type:   "foo",
				Metric: "bar",
			},
		},
		{name: "Garbage",
			data:    []byte("garbage"),
			wantErr: true,
		},
	}
	for _, tt := range tests {
		got, err := ParseCarbonlinkRequest(tt.data)
		if (err != nil) != tt.wantErr {
			t.Errorf("%q. ParseCarbonlinkRequest() error = %v, wantErr %v", tt.name, err, tt.wantErr)
			continue
		}
		if !reflect.DeepEqual(got, tt.want) {
			t.Errorf("%q. ParseCarbonlinkRequest() = %v, want %v", tt.name, got, tt.want)
		}
	}
}

func BenchmarkCarbonLinkPickleParse(b *testing.B) {
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			ParseCarbonlinkRequest([]byte(sampleCacheQuery))
		}
	})
}

func BenchmarkCarbonLinkPackReply(b *testing.B) {
	p := points.OnePoint("carbon.agents.carbon_agent_server.param.size", 15, 1422795966).Add(15, 9000000).Add(16, 9000000)

	q := Query{CacheData: p}

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			packReply(&q)
		}
	})
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Compilebench benchmarks the speed of the Go compiler.
//
// Usage:
//
//	compilebench [options]
//
// It times the compilation of various packages and prints results in
// the format used by package testing (and expected by rsc.io/benchstat).
//
// The options are:
//
//	-alloc
//		Report allocations.
//
//	-compile exe
//		Use exe as the path to the cmd/compile binary.
//
//	-compileflags 'list'
//		Pass the space-separated list of flags to the compilation.
//
//	-count n
//		Run each benchmark n times (default 1).
//
//	-cpuprofile file
//		Write a CPU profile of the compiler to file.
//
//	-memprofile file
//		Write a memory profile of the compiler to file.
//
//	-memprofilerate rate
//		Set runtime.MemProfileRate during compilation.
//
//	-run regexp
//		Only run benchmarks with names matching regexp.
//
// Although -cpuprofile and -memprofile are intended to write a
// combined profile for all the executed benchmarks to file,
// today they write only the profile for the last benchmark executed.
//
// The default memory profiling rate is one profile sample per 512 kB
// allocated (see ``go doc runtime.MemProfileRate'').
// Lowering the rate (for example, -memprofilerate 64000) produces
// a more fine-grained and therefore accurate profile, but it also incurs
// execution cost. For benchmark comparisons, never use timings
// obtained with a low -memprofilerate option.
//
// Example
//
// Assuming the base version of the compiler has been saved with
// ``toolstash save,'' this sequence compares the old and new compiler:
//
//	compilebench -count 10 -compile $(toolstash -n compile) >old.txt
//	compilebench -count 10 >new.txt
//	benchstat old.txt new.txt
//
package main

import (
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var (
	goroot   = runtime.GOROOT()
	compiler string
	runRE    *regexp.Regexp
	is6g     bool
)

var (
	flagAlloc          = flag.Bool("alloc", false, "report allocations")
	flagCompiler       = flag.String("compile", "", "use `exe` as the cmd/compile binary")
	flagCompilerFlags  = flag.String("compileflags", "", "additional `flags` to pass to compile")
	flagRun            = flag.String("run", "", "run benchmarks matching `regexp`")
	flagCount          = flag.Int("count", 1, "run benchmarks `n` times")
	flagCpuprofile     = flag.String("cpuprofile", "", "write CPU profile to `file`")
	flagMemprofile     = flag.String("memprofile", "", "write memory profile to `file`")
	flagMemprofilerate = flag.Int64("memprofilerate", -1, "set memory profile `rate`")
	flagShort          = flag.Bool("short", false, "skip long-running benchmarks")
)

var tests = []struct {
	name string
	dir  string
	long bool
}{
	{"BenchmarkTemplate", "html/template", false},
	{"BenchmarkUnicode", "unicode", false},
	{"BenchmarkGoTypes", "go/types", false},
	{"BenchmarkCompiler", "cmd/compile/internal/gc", false},
	{"BenchmarkMakeBash", "", true},
	{"BenchmarkHelloSize", "", false},
	{"BenchmarkCmdGoSize", "", true},
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: compilebench [options]\n")
	fmt.Fprintf(os.Stderr, "options:\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("compilebench: ")
	flag.Usage = usage
	flag.Parse()
	if flag.NArg() != 0 {
		usage()
	}

	compiler = *flagCompiler
	if compiler == "" {
		out, err := exec.Command("go", "tool", "-n", "compile").CombinedOutput()
		if err != nil {
			out, err = exec.Command("go", "tool", "-n", "6g").CombinedOutput()
			is6g = true
			if err != nil {
				out, err = exec.Command("go", "tool", "-n", "compile").CombinedOutput()
				log.Fatalf("go tool -n compiler: %v\n%s", err, out)
			}
		}
		compiler = strings.TrimSpace(string(out))
	}

	if *flagRun != "" {
		r, err := regexp.Compile(*flagRun)
		if err != nil {
			log.Fatalf("invalid -run argument: %v", err)
		}
		runRE = r
	}

	for i := 0; i < *flagCount; i++ {
		for _, tt := range tests {
			if tt.long && *flagShort {
				continue
			}
			if runRE == nil || runRE.MatchString(tt.name) {
				runBuild(tt.name, tt.dir)
			}
		}
	}
}

func runCmd(name string, cmd *exec.Cmd) {
	start := time.Now()
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("%v: %v\n%s", name, err, out)
		return
	}
	fmt.Printf("%s 1 %d ns/op\n", name, time.Since(start).Nanoseconds())
}

func runMakeBash() {
	cmd := exec.Command("./make.bash")
	cmd.Dir = filepath.Join(runtime.GOROOT(), "src")
	runCmd("BenchmarkMakeBash", cmd)
}

func runCmdGoSize() {
	runSize("BenchmarkCmdGoSize", filepath.Join(runtime.GOROOT(), "bin/go"))
}

func runHelloSize() {
	cmd := exec.Command("go", "build", "-o", "_hello_", filepath.Join(runtime.GOROOT(), "test/helloworld.go"))
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		log.Print(err)
		return
	}
	defer os.Remove("_hello_")
	runSize("BenchmarkHelloSize", "_hello_")
}

func runSize(name, file string) {
	info, err := os.Stat(file)
	if err != nil {
		log.Print(err)
		return
	}
	out, err := exec.Command("size", file).CombinedOutput()
	if err != nil {
		log.Printf("size: %v\n%s", err, out)
		return
	}
	lines := strings.Split(string(out), "\n")
	if len(lines) < 2 {
		log.Printf("not enough output from size: %s", out)
		return
	}
	f := strings.Fields(lines[1])
	if strings.HasPrefix(lines[0], "__TEXT") && len(f) >= 2 { // OS X
		fmt.Printf("%s 1 %s text-bytes %s data-bytes %v exe-bytes\n", name, f[0], f[1], info.Size())
	} else if strings.Contains(lines[0], "bss") && len(f) >= 3 {
		fmt.Printf("%s 1 %s text-bytes %s data-bytes %s bss-bytes %v exe-bytes\n", name, f[0], f[1], f[2], info.Size())
	}
}

func runBuild(name, dir string) {
	switch name {
	case "BenchmarkMakeBash":
		runMakeBash()
		return
	case "BenchmarkCmdGoSize":
		runCmdGoSize()
		return
	case "BenchmarkHelloSize":
		runHelloSize()
		return
	}

	pkg, err := build.Import(dir, ".", 0)
	if err != nil {
		log.Print(err)
		return
	}
	args := []string{"-o", "_compilebench_.o"}
	if is6g {
		*flagMemprofilerate = -1
		*flagAlloc = false
		*flagCpuprofile = ""
		*flagMemprofile = ""
	}
	if *flagMemprofilerate >= 0 {
		args = append(args, "-memprofilerate", fmt.Sprint(*flagMemprofilerate))
	}
	args = append(args, strings.Fields(*flagCompilerFlags)...)
	if *flagAlloc || *flagCpuprofile != "" || *flagMemprofile != "" {
		if *flagAlloc || *flagMemprofile != "" {
			args = append(args, "-memprofile", "_compilebench_.memprof")
		}
		if *flagCpuprofile != "" {
			args = append(args, "-cpuprofile", "_compilebench_.cpuprof")
		}
	}
	args = append(args, pkg.GoFiles...)
	cmd := exec.Command(compiler, args...)
	cmd.Dir = pkg.Dir
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	start := time.Now()
	err = cmd.Run()
	if err != nil {
		log.Printf("%v: %v", name, err)
		return
	}
	end := time.Now()

	var allocs, bytes int64
	if *flagAlloc || *flagMemprofile != "" {
		out, err := ioutil.ReadFile(pkg.Dir + "/_compilebench_.memprof")
		if err != nil {
			log.Print("cannot find memory profile after compilation")
		}
		for _, line := range strings.Split(string(out), "\n") {
			f := strings.Fields(line)
			if len(f) < 4 || f[0] != "#" || f[2] != "=" {
				continue
			}
			val, err := strconv.ParseInt(f[3], 0, 64)
			if err != nil {
				continue
			}
			switch f[1] {
			case "TotalAlloc":
				bytes = val
			case "Mallocs":
				allocs = val
			}
		}

		if *flagMemprofile != "" {
			if err := ioutil.WriteFile(*flagMemprofile, out, 0666); err != nil {
				log.Print(err)
			}
		}
		os.Remove(pkg.Dir + "/_compilebench_.memprof")
	}

	if *flagCpuprofile != "" {
		out, err := ioutil.ReadFile(pkg.Dir + "/_compilebench_.cpuprof")
		if err != nil {
			log.Print(err)
		}
		if err := ioutil.WriteFile(*flagCpuprofile, out, 0666); err != nil {
			log.Print(err)
		}
		os.Remove(pkg.Dir + "/_compilebench_.cpuprof")
	}

	wallns := end.Sub(start).Nanoseconds()
	userns := cmd.ProcessState.UserTime().Nanoseconds()

	if *flagAlloc {
		fmt.Printf("%s 1 %d ns/op %d user-ns/op %d B/op %d allocs/op\n", name, wallns, userns, bytes, allocs)
	} else {
		fmt.Printf("%s 1 %d ns/op %d user-ns/op\n", name, wallns, userns)
	}

	os.Remove(pkg.Dir + "/_compilebench_.o")
}
<commit_msg>cmd/compilebench: update list of packages<commit_after>// Copyright 2015 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Compilebench benchmarks the speed of the Go compiler.
//
// Usage:
//
//	compilebench [options]
//
// It times the compilation of various packages and prints results in
// the format used by package testing (and expected by rsc.io/benchstat).
//
// The options are:
//
//	-alloc
//		Report allocations.
//
//	-compile exe
//		Use exe as the path to the cmd/compile binary.
//
//	-compileflags 'list'
//		Pass the space-separated list of flags to the compilation.
//
//	-count n
//		Run each benchmark n times (default 1).
//
//	-cpuprofile file
//		Write a CPU profile of the compiler to file.
//
//	-memprofile file
//		Write a memory profile of the compiler to file.
//
//	-memprofilerate rate
//		Set runtime.MemProfileRate during compilation.
//
//	-run regexp
//		Only run benchmarks with names matching regexp.
//
// Although -cpuprofile and -memprofile are intended to write a
// combined profile for all the executed benchmarks to file,
// today they write only the profile for the last benchmark executed.
//
// The default memory profiling rate is one profile sample per 512 kB
// allocated (see ``go doc runtime.MemProfileRate'').
// Lowering the rate (for example, -memprofilerate 64000) produces
// a more fine-grained and therefore accurate profile, but it also incurs
// execution cost. For benchmark comparisons, never use timings
// obtained with a low -memprofilerate option.
//
// Example
//
// Assuming the base version of the compiler has been saved with
// ``toolstash save,'' this sequence compares the old and new compiler:
//
//	compilebench -count 10 -compile $(toolstash -n compile) >old.txt
//	compilebench -count 10 >new.txt
//	benchstat old.txt new.txt
//
package main

import (
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var (
	goroot   = runtime.GOROOT()
	compiler string
	runRE    *regexp.Regexp
	is6g     bool
)

var (
	flagAlloc          = flag.Bool("alloc", false, "report allocations")
	flagCompiler       = flag.String("compile", "", "use `exe` as the cmd/compile binary")
	flagCompilerFlags  = flag.String("compileflags", "", "additional `flags` to pass to compile")
	flagRun            = flag.String("run", "", "run benchmarks matching `regexp`")
	flagCount          = flag.Int("count", 1, "run benchmarks `n` times")
	flagCpuprofile     = flag.String("cpuprofile", "", "write CPU profile to `file`")
	flagMemprofile     = flag.String("memprofile", "", "write memory profile to `file`")
	flagMemprofilerate = flag.Int64("memprofilerate", -1, "set memory profile `rate`")
	flagShort          = flag.Bool("short", false, "skip long-running benchmarks")
)

var tests = []struct {
	name string
	dir  string
	long bool
}{
	{"BenchmarkTemplate", "html/template", false},
	{"BenchmarkUnicode", "unicode", false},
	{"BenchmarkGoTypes", "go/types", false},
	{"BenchmarkCompiler", "cmd/compile/internal/gc", false},
	{"BenchmarkSSA", "cmd/compile/internal/ssa", false},
	{"BenchmarkFlate", "compress/flate", false},
	{"BenchmarkGoParser", "go/parser", false},
	{"BenchmarkReflect", "reflect", false},
	{"BenchmarkTar", "archive/tar", false},
	{"BenchmarkXML", "encoding/xml", false},
	{"BenchmarkMakeBash", "", true},
	{"BenchmarkHelloSize", "", false},
	{"BenchmarkCmdGoSize", "", true},
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: compilebench [options]\n")
	fmt.Fprintf(os.Stderr, "options:\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("compilebench: ")
	flag.Usage = usage
	flag.Parse()
	if flag.NArg() != 0 {
		usage()
	}

	compiler = *flagCompiler
	if compiler == "" {
		out, err := exec.Command("go", "tool", "-n", "compile").CombinedOutput()
		if err != nil {
			out, err = exec.Command("go", "tool", "-n", "6g").CombinedOutput()
			is6g = true
			if err != nil {
				out, err = exec.Command("go", "tool", "-n", "compile").CombinedOutput()
				log.Fatalf("go tool -n compiler: %v\n%s", err, out)
			}
		}
		compiler = strings.TrimSpace(string(out))
	}

	if *flagRun != "" {
		r, err := regexp.Compile(*flagRun)
		if err != nil {
			log.Fatalf("invalid -run argument: %v", err)
		}
		runRE = r
	}

	for i := 0; i < *flagCount; i++ {
		for _, tt := range tests {
			if tt.long && *flagShort {
				continue
			}
			if runRE == nil || runRE.MatchString(tt.name) {
				runBuild(tt.name, tt.dir)
			}
		}
	}
}

func runCmd(name string, cmd *exec.Cmd) {
	start := time.Now()
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("%v: %v\n%s", name, err, out)
		return
	}
	fmt.Printf("%s 1 %d ns/op\n", name, time.Since(start).Nanoseconds())
}

func runMakeBash() {
	cmd := exec.Command("./make.bash")
	cmd.Dir = filepath.Join(runtime.GOROOT(), "src")
	runCmd("BenchmarkMakeBash", cmd)
}

func runCmdGoSize() {
	runSize("BenchmarkCmdGoSize", filepath.Join(runtime.GOROOT(), "bin/go"))
}

func runHelloSize() {
	cmd := exec.Command("go", "build", "-o", "_hello_", filepath.Join(runtime.GOROOT(), "test/helloworld.go"))
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		log.Print(err)
		return
	}
	defer os.Remove("_hello_")
	runSize("BenchmarkHelloSize", "_hello_")
}

func runSize(name, file string) {
	info, err := os.Stat(file)
	if err != nil {
		log.Print(err)
		return
	}
	out, err := exec.Command("size", file).CombinedOutput()
	if err != nil {
		log.Printf("size: %v\n%s", err, out)
		return
	}
	lines := strings.Split(string(out), "\n")
	if len(lines) < 2 {
		log.Printf("not enough output from size: %s", out)
		return
	}
	f := strings.Fields(lines[1])
	if strings.HasPrefix(lines[0], "__TEXT") && len(f) >= 2 { // OS X
		fmt.Printf("%s 1 %s text-bytes %s data-bytes %v exe-bytes\n", name, f[0], f[1], info.Size())
	} else if strings.Contains(lines[0], "bss") && len(f) >= 3 {
		fmt.Printf("%s 1 %s text-bytes %s data-bytes %s bss-bytes %v exe-bytes\n", name, f[0], f[1], f[2], info.Size())
	}
}

func runBuild(name, dir string) {
	switch name {
	case "BenchmarkMakeBash":
		runMakeBash()
		return
	case "BenchmarkCmdGoSize":
		runCmdGoSize()
		return
	case "BenchmarkHelloSize":
		runHelloSize()
		return
	}

	pkg, err := build.Import(dir, ".", 0)
	if err != nil {
		log.Print(err)
		return
	}
	args := []string{"-o", "_compilebench_.o"}
	if is6g {
		*flagMemprofilerate = -1
		*flagAlloc = false
		*flagCpuprofile = ""
		*flagMemprofile = ""
	}
	if *flagMemprofilerate >= 0 {
		args = append(args, "-memprofilerate", fmt.Sprint(*flagMemprofilerate))
	}
	args = append(args, strings.Fields(*flagCompilerFlags)...)
	if *flagAlloc || *flagCpuprofile != "" || *flagMemprofile != "" {
		if *flagAlloc || *flagMemprofile != "" {
			args = append(args, "-memprofile", "_compilebench_.memprof")
		}
		if *flagCpuprofile != "" {
			args = append(args, "-cpuprofile", "_compilebench_.cpuprof")
		}
	}
	args = append(args, pkg.GoFiles...)
	cmd := exec.Command(compiler, args...)
	cmd.Dir = pkg.Dir
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	start := time.Now()
	err = cmd.Run()
	if err != nil {
		log.Printf("%v: %v", name, err)
		return
	}
	end := time.Now()

	var allocs, bytes int64
	if *flagAlloc || *flagMemprofile != "" {
		out, err := ioutil.ReadFile(pkg.Dir + "/_compilebench_.memprof")
		if err != nil {
			log.Print("cannot find memory profile after compilation")
		}
		for _, line := range strings.Split(string(out), "\n") {
			f := strings.Fields(line)
			if len(f) < 4 || f[0] != "#" || f[2] != "=" {
				continue
			}
			val, err := strconv.ParseInt(f[3], 0, 64)
			if err != nil {
				continue
			}
			switch f[1] {
			case "TotalAlloc":
				bytes = val
			case "Mallocs":
				allocs = val
			}
		}

		if *flagMemprofile != "" {
			if err := ioutil.WriteFile(*flagMemprofile, out, 0666); err != nil {
				log.Print(err)
			}
		}
		os.Remove(pkg.Dir + "/_compilebench_.memprof")
	}

	if *flagCpuprofile != "" {
		out, err := ioutil.ReadFile(pkg.Dir + "/_compilebench_.cpuprof")
		if err != nil {
			log.Print(err)
		}
		if err := ioutil.WriteFile(*flagCpuprofile, out, 0666); err != nil {
			log.Print(err)
		}
		os.Remove(pkg.Dir + "/_compilebench_.cpuprof")
	}

	wallns := end.Sub(start).Nanoseconds()
	userns := cmd.ProcessState.UserTime().Nanoseconds()

	if *flagAlloc {
		fmt.Printf("%s 1 %d ns/op %d user-ns/op %d B/op %d allocs/op\n", name, wallns, userns, bytes, allocs)
	} else {
		fmt.Printf("%s 1 %d ns/op %d user-ns/op\n", name, wallns, userns)
	}

	os.Remove(pkg.Dir + "/_compilebench_.o")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"github.com/realglobe-Inc/edo/util"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"math/big"
	"math/rand"
	"sync/atomic"
	"time"
)

type idGenerator struct {
	// 文字数の下界。
	minIdLen int
	// インスタンス内での ID 被りを防ぐための通し番号。
	// 別インスタンスでは保証できない。
	ser int64
}

func newIdGenerator(minIdLen int) idGenerator {
	return idGenerator{
		minIdLen: minIdLen,
		ser:      rand.New(rand.NewSource(time.Now().UnixNano())).Int63(),
	}
}

func (this *idGenerator) newId() (id string, err error) {
	id, err = util.SecureRandomString(this.minIdLen)
	if err != nil {
		return "", erro.Wrap(err)
	}
	buff := big.NewInt(atomic.AddInt64(&this.ser, 1)).Bytes()
	id += base64.URLEncoding.EncodeToString(buff)[:(len(buff)*8+5)/6]
	return id, nil
}
<commit_msg>ID 生成の通し番号部分で使い切れない桁が出ないようにした<commit_after>package main

import (
	"encoding/base64"
	"github.com/realglobe-Inc/edo/util"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"math/big"
	"math/rand"
	"sync/atomic"
	"time"
)

type idGenerator struct {
	// 乱数文字数の長さ。
	randLen int
	// インスタンス内での ID 被りを防ぐための通し番号。
	ser int64
}

func newIdGenerator(randLen int) idGenerator {
	return idGenerator{
		randLen: randLen,
		ser:     rand.New(rand.NewSource(time.Now().UnixNano())).Int63(),
	}
}

func (this *idGenerator) newId() (id string, err error) {
	id, err = util.SecureRandomString(this.randLen)
	if err != nil {
		return "", erro.Wrap(err)
	}

	const bLen = 64 / 8       // int64
	const sLen = bLen * 8 / 6 // BASE64 にして文字を使い切れない上位ビットは捨てる。

	v := big.NewInt(atomic.AddInt64(&this.ser, 1))
	v = v.Lsh(v, bLen*8-sLen*6) // BASE64 の 6 ビット区切りと最下位ビットの位置を揃える。
	buff := v.Bytes()
	if len(buff) < bLen {
		// 上位を 0 詰め。
		buff = append(make([]byte, bLen-len(buff)), buff...)
	} else if len(buff) > bLen {
		// 上位を捨てる。
		buff = buff[len(buff)-bLen:]
	}
	id += base64.URLEncoding.EncodeToString(buff)[:sLen]

	return id, nil
}
<|endoftext|>


<commit_before>package thor

type Query []*Node

func (q Query) KeyRange(key string, start, end int) Query {
	match := Query{}
	for _, node := range q {
		submatch := Query{}
		for _, kid := range node.Kids {
			if kid.Key == key {
				submatch = append(submatch, kid)
			}
		}
		s, e := calcRange(start, end, len(submatch))
		match = append(match, submatch[s:e]...)
	}
	return match
}

func (q Query) Key(key string) Query {
	return q.KeyRange(key, 0, -1)
}

func (q Query) KeyN(key string, n int) Query {
	return q.KeyRange(key, n, n)
}

func (q Query) FirstKey(key string) Query {
	return q.KeyN(key, 0)
}

func (q Query) LastKey(key string) Query {
	return q.KeyN(key, -1)
}

func (q Query) Range(start, end int) Query {
	match := Query{}
	for _, node := range q {
		s, e := calcRange(start, end, len(node.Kids))
		match = append(match, node.Kids[s:e]...)
	}
	return match
}

func (q Query) All() Query {
	return q.Range(0, -1)
}

func (q Query) At(n int) Query {
	return q.Range(n, n)
}

func (q Query) First() Query {
	return q.At(0)
}

func (q Query) Last() Query {
	return q.At(-1)
}

func calcRange(start, end, max int) (int, int) {
	if start < 0 {
		start += max
	}
	if end < 0 {
		end += max
	}
	end++
	if end <= start || start < 0 || end < 0 {
		return 0, 0
	}
	if start > max {
		start = max
	}
	if end > max {
		end = max
	}
	return start, end
}

<commit_msg>Added first/last n query types.  Renamed key first/last queries for consistency.<commit_after>package thor

type Query []*Node

func (q Query) KeyRange(key string, start, end int) Query {
	match := Query{}
	for _, node := range q {
		submatch := Query{}
		for _, kid := range node.Kids {
			if kid.Key == key {
				submatch = append(submatch, kid)
			}
		}
		s, e := calcRange(start, end, len(submatch))
		match = append(match, submatch[s:e]...)
	}
	return match
}

func (q Query) Key(key string) Query {
	return q.KeyRange(key, 0, -1)
}

func (q Query) KeyN(key string, n int) Query {
	return q.KeyRange(key, n, n)
}

func (q Query) KeyFirst(key string) Query {
	return q.KeyN(key, 0)
}

func (q Query) KeyLast(key string) Query {
	return q.KeyN(key, -1)
}

func (q Query) KeyFirstN(key string, n int) Query {
	return q.KeyRange(key, 0, n-1)
}

func (q Query) KeyLastN(key string, n int) Query {
	return q.KeyRange(key, -n, -1)
}

func (q Query) Range(start, end int) Query {
	match := Query{}
	for _, node := range q {
		s, e := calcRange(start, end, len(node.Kids))
		match = append(match, node.Kids[s:e]...)
	}
	return match
}

func (q Query) All() Query {
	return q.Range(0, -1)
}

func (q Query) At(n int) Query {
	return q.Range(n, n)
}

func (q Query) First() Query {
	return q.At(0)
}

func (q Query) Last() Query {
	return q.At(-1)
}

func (q Query) FirstN(n int) Query {
	return q.Range(0, n-1)
}

func (q Query) LastN(n int) Query {
	return q.Range(-n, -1)
}

func calcRange(start, end, max int) (int, int) {
	if start < 0 {
		start += max
	}
	if end < 0 {
		end += max
	}
	end++
	if end <= start || start < 0 || end < 0 {
		return 0, 0
	}
	if start > max {
		start = max
	}
	if end > max {
		end = max
	}
	return start, end
}
<|endoftext|>


<commit_before>package xql

import (
    "errors"
    //log "github.com/Sirupsen/logrus"
    "reflect"
    "database/sql"
)

func (qc QueryColumn) String(as...bool) string {
    s := ""
    if qc.Function != "" {
        s = qc.Function+"("+qc.FieldName+")"
    }else{
        s = qc.FieldName
    }
    if qc.Alias != "" && len(as)>0 && as[0] {
        s = s + " AS " + qc.Alias
    }
    return s
}

type QuerySet struct {
    session *Session
    table   *Table
    queries []QueryColumn
    filters []QueryFilter
    orders  []QueryOrder
    offset  int64
    limit   int64
}

type XRow struct {
    row *sql.Row
    qs *QuerySet
}

func (self *XRow) Scan(dest ...interface{}) error {
    if nil == self.row {
        return errors.New("Nil row.")
    }
    if len(dest) < 1 {
        panic("Empty output!")
    }
    if len(dest) == 1 {
        d := dest[0]
        if reflect.TypeOf(d) == reflect.TypeOf(self.qs.table.Entity) {
            var outputs []interface{}
            r := reflect.ValueOf(d)
            for _, qc := range self.qs.queries {
                c, _ := self.qs.table.MappedColumns[qc.FieldName]
                vp := r.Elem().FieldByName(c.PropertyName).Addr().Interface()
                outputs = append(outputs, vp)
            }
            return self.row.Scan(outputs...)
        }
    }
    return self.row.Scan(dest...)
}

type XRows struct {
    rows *sql.Rows
    qs *QuerySet
}

func (self *XRows) Scan(dest ...interface{}) error {
    if nil == self.rows {
        return errors.New("No rows.")
    }
    if len(dest) < 1 {
        panic("Empty output!")
    }
    if len(dest) == 1 {
        d := dest[0]
        if reflect.TypeOf(d) == reflect.TypeOf(self.qs.table.Entity) {
            var outputs []interface{}
            r := reflect.ValueOf(d)
            for _, qc := range self.qs.queries {
                c, _ := self.qs.table.MappedColumns[qc.FieldName]
                vp := r.Elem().FieldByName(c.PropertyName).Addr().Interface()
                outputs = append(outputs, vp)
            }
            return self.rows.Scan(outputs...)
        }
    }
    return self.rows.Scan(dest...)
}

func (self *XRows) Next() bool {
    if nil == self.rows {
        return false
    }
    return self.rows.Next()
}

func (self *XRows) Close() {
    self.rows.Close()
    self.rows = nil
}

func makeQueryOrder(table *Table, s string) QueryOrder {
    qo := QueryOrder{}
    if s[:1] == "-" {
        qo.Type = ORDER_DESC
        qo.Field = s[1:]
    } else {
        qo.Type = ORDER_ASC
        qo.Field = s
    }
    return qo
}

func (self *QuerySet) Filter(cons ...interface{}) *QuerySet {
    for _, con := range cons {
        if vs, ok := con.(string); ok {
            self.filters = append(self.filters, QueryFilter{
                Field: vs,
            })
        }else if vm, ok := con.(map[string]interface{}); ok {
            for k, v := range vm {
                self.filters = append(self.filters, QueryFilter{
                    Field:k,
                    Value:v,
                    Operator: "=",
                })
            }
        }else if vf, ok := con.(*QueryFilter); ok {
            self.filters = append(self.filters, *vf)
        }else if vf, ok := con.(QueryFilter); ok {
            self.filters = append(self.filters, vf)
        }else{
            panic("Unknow Filter!")
        }
    }
    return self
}

func (self *QuerySet) OrderBy(orders ...interface{}) *QuerySet {
    for _, x := range orders {
        switch x.(type) {
        case string:
            qo := makeQueryOrder(self.table, x.(string))
            self.orders = append(self.orders, qo)
        case *QueryOrder:
            self.orders = append(self.orders, *x.(*QueryOrder))
        case QueryOrder:
            self.orders = append(self.orders, x.(QueryOrder))
        default:
            panic("Not supported parameter type.")
        }
    }
    return self
}

func (self *QuerySet) Offset(offset int64) *QuerySet {
    self.offset = offset
    return self
}

func (self *QuerySet) Limit(limit int64) *QuerySet {
    self.limit = limit
    return self
}

func (self *QuerySet) Count(cols...string) (int64,error) {
    s, args, err := self.session.getDialect().Select(self.table,
        []QueryColumn{{Function:"COUNT", FieldName:"*"}},
        self.filters, nil, -1, -1)
    if nil != err {
        return 0, err
    }
    row := self.session.doQueryRow(s, args...)
    var n int64
    if e := row.Scan(&n); nil != e {
        return 0, e
    }
    return n, nil
}

func (self *QuerySet) All() (*XRows, error) {
    if len(self.queries) < 1 {
        for _, col := range self.table.MappedColumns {
            self.queries = append(self.queries, QueryColumn{FieldName:col.FieldName, Alias:col.FieldName})
        }
    }
    s, args, err := self.session.getDialect().Select(self.table, self.queries,
        self.filters, self.orders, self.offset, self.limit)
    if nil != err {
        return nil, err
    }
    rows, err := self.session.doQuery(s, args...)
    if nil != err {
        return nil, err
    }
    xrows := &XRows{rows:rows, qs:self}
    return xrows, nil
}

func (self *QuerySet) One() *XRow {
    if len(self.queries) < 1 {
        for _, col := range self.table.MappedColumns {
            self.queries = append(self.queries, QueryColumn{FieldName:col.FieldName, Alias:col.FieldName})
        }
    }
    s, args, err := self.session.getDialect().Select(self.table, self.queries,
        self.filters, self.orders, self.offset, 1)
    if nil != err {
        return nil
    }
    row := self.session.doQueryRow(s, args...)
    xrow := &XRow{row:row, qs:self}
    return xrow
}

func (self *QuerySet) Update(vals interface{}) (int64, error) {
    cols := []UpdateColumn{}
    if cm, ok := vals.(map[string]interface{}); ok {
        for k, v := range cm {
            var fk string
            if c, ok := self.table.MappedColumns[k]; ok {
                fk = c.FieldName
            }else if c, ok := self.table.JTaggedColumns[k]; ok {
                fk = c.FieldName
            }else{
                return 0, errors.New("Invalid column:"+k)
            }
            uc := UpdateColumn{Field:fk, Value:v, Operator:"="}
            cols = append(cols, uc)
        }
    }else if cx, ok := vals.([]UpdateColumn); ok {
        cols = cx
    }
    s, args, err := self.session.getDialect().Update(self.table, self.filters, cols...)
    if nil != err {
        return 0, err
    }
    var ret sql.Result
    ret, err = self.session.doExec(s, args...)
    if nil != err {
        return 0, err
    }else{
        rows, e := ret.RowsAffected()
        return rows, e
    }
    return 0, nil
}

func (self *QuerySet) Delete() (int64, error) {
    s, args, err := self.session.getDialect().Delete(self.table, self.filters)
    if nil != err {
        return 0, err
    }
    var ret sql.Result
    ret, err = self.session.doExec(s, args...)
    if nil != err {
        return 0, err
    }else{
        rows, e := ret.RowsAffected()
        return rows, e
    }
    return 0, nil
}

func (self *QuerySet) Insert(objs ...interface{}) (int64, error) {
    //log.Debugln("Insert:> ", objs)
    //log.Debugln("Table:> ", self.table)
    //var ret sql.Result
    var rows int64 = 0
    var cols []string
    if len(self.queries) > 0 {
        for _,x := range self.queries {
            cols = append(cols, x.FieldName)
        }
    }
    for _, obj := range objs {
        //log.Debugln("reflect.TypeOf(obj)>", reflect.TypeOf(obj))
        //log.Debugln("reflect.TypeOf(self.table.Entity)>", reflect.TypeOf(self.table.Entity))
        if reflect.TypeOf(obj) != reflect.TypeOf(self.table.Entity) {
           return 0, errors.New("Invalid data type.")
        }
        s, args, err := self.session.getDialect().Insert(self.table, obj, cols...)
        if nil != err {
            return 0, err
        }
        _, err = self.session.doExec(s, args...)
        if nil != err {
            return 0, err
        }else{
            //if nil != ret {
            //    n, e := ret.LastInsertId()
            //    if nil == e {
            //        rows += n
            //    }else{
            //        rows += 1
            //    }
            //    //log.Debugln("Insert:> ", n)
            //}else{
            //    rows += 1
            //}
            rows += 1
        }
    }
    return rows, nil
}
<commit_msg>Updated: fixed issues for special field names.<commit_after>package xql

import (
    "errors"
    //log "github.com/Sirupsen/logrus"
    "reflect"
    "database/sql"
    "fmt"
)

func (qc QueryColumn) String(as...bool) string {
    s := ""
    if qc.Function != "" {
        s = fmt.Sprintf(`%s("%s")`, qc.Function, qc.FieldName) //qc.Function+"("+qc.FieldName+")"
    }else{
        s = fmt.Sprintf(`"%s"`, qc.FieldName)
    }
    if qc.Alias != "" && len(as)>0 && as[0] {
        s = s + " AS " + qc.Alias
    }
    return s
}

type QuerySet struct {
    session *Session
    table   *Table
    queries []QueryColumn
    filters []QueryFilter
    orders  []QueryOrder
    offset  int64
    limit   int64
}

type XRow struct {
    row *sql.Row
    qs *QuerySet
}

func (self *XRow) Scan(dest ...interface{}) error {
    if nil == self.row {
        return errors.New("Nil row.")
    }
    if len(dest) < 1 {
        panic("Empty output!")
    }
    if len(dest) == 1 {
        d := dest[0]
        if reflect.TypeOf(d) == reflect.TypeOf(self.qs.table.Entity) {
            var outputs []interface{}
            r := reflect.ValueOf(d)
            for _, qc := range self.qs.queries {
                c, _ := self.qs.table.MappedColumns[qc.FieldName]
                vp := r.Elem().FieldByName(c.PropertyName).Addr().Interface()
                outputs = append(outputs, vp)
            }
            return self.row.Scan(outputs...)
        }
    }
    return self.row.Scan(dest...)
}

type XRows struct {
    rows *sql.Rows
    qs *QuerySet
}

func (self *XRows) Scan(dest ...interface{}) error {
    if nil == self.rows {
        return errors.New("No rows.")
    }
    if len(dest) < 1 {
        panic("Empty output!")
    }
    if len(dest) == 1 {
        d := dest[0]
        if reflect.TypeOf(d) == reflect.TypeOf(self.qs.table.Entity) {
            var outputs []interface{}
            r := reflect.ValueOf(d)
            for _, qc := range self.qs.queries {
                c, _ := self.qs.table.MappedColumns[qc.FieldName]
                vp := r.Elem().FieldByName(c.PropertyName).Addr().Interface()
                outputs = append(outputs, vp)
            }
            return self.rows.Scan(outputs...)
        }
    }
    return self.rows.Scan(dest...)
}

func (self *XRows) Next() bool {
    if nil == self.rows {
        return false
    }
    return self.rows.Next()
}

func (self *XRows) Close() {
    self.rows.Close()
    self.rows = nil
}

func makeQueryOrder(table *Table, s string) QueryOrder {
    qo := QueryOrder{}
    if s[:1] == "-" {
        qo.Type = ORDER_DESC
        qo.Field = s[1:]
    } else {
        qo.Type = ORDER_ASC
        qo.Field = s
    }
    return qo
}

func (self *QuerySet) Filter(cons ...interface{}) *QuerySet {
    for _, con := range cons {
        if vs, ok := con.(string); ok {
            self.filters = append(self.filters, QueryFilter{
                Field: vs,
            })
        }else if vm, ok := con.(map[string]interface{}); ok {
            for k, v := range vm {
                self.filters = append(self.filters, QueryFilter{
                    Field:k,
                    Value:v,
                    Operator: "=",
                })
            }
        }else if vf, ok := con.(*QueryFilter); ok {
            self.filters = append(self.filters, *vf)
        }else if vf, ok := con.(QueryFilter); ok {
            self.filters = append(self.filters, vf)
        }else{
            panic("Unknow Filter!")
        }
    }
    return self
}

func (self *QuerySet) OrderBy(orders ...interface{}) *QuerySet {
    for _, x := range orders {
        switch x.(type) {
        case string:
            qo := makeQueryOrder(self.table, x.(string))
            self.orders = append(self.orders, qo)
        case *QueryOrder:
            self.orders = append(self.orders, *x.(*QueryOrder))
        case QueryOrder:
            self.orders = append(self.orders, x.(QueryOrder))
        default:
            panic("Not supported parameter type.")
        }
    }
    return self
}

func (self *QuerySet) Offset(offset int64) *QuerySet {
    self.offset = offset
    return self
}

func (self *QuerySet) Limit(limit int64) *QuerySet {
    self.limit = limit
    return self
}

func (self *QuerySet) Count(cols...string) (int64,error) {
    var fieldname string
    if len(cols) > 0 {
        fieldname = cols[0]
    }else if len(self.table.PrimaryKey) > 0 {
        fieldname = self.table.PrimaryKey[0]
    }else{
        fieldname = self.table.ListedColumns[0].FieldName
    }
    s, args, err := self.session.getDialect().Select(self.table,
        []QueryColumn{{Function:"COUNT", FieldName:fieldname}},
        self.filters, nil, -1, -1)
    if nil != err {
        return 0, err
    }
    row := self.session.doQueryRow(s, args...)
    var n int64
    if e := row.Scan(&n); nil != e {
        return 0, e
    }
    return n, nil
}

func (self *QuerySet) All() (*XRows, error) {
    if len(self.queries) < 1 {
        for _, col := range self.table.MappedColumns {
            self.queries = append(self.queries, QueryColumn{FieldName:col.FieldName, Alias:col.FieldName})
        }
    }
    s, args, err := self.session.getDialect().Select(self.table, self.queries,
        self.filters, self.orders, self.offset, self.limit)
    if nil != err {
        return nil, err
    }
    rows, err := self.session.doQuery(s, args...)
    if nil != err {
        return nil, err
    }
    xrows := &XRows{rows:rows, qs:self}
    return xrows, nil
}

func (self *QuerySet) One() *XRow {
    if len(self.queries) < 1 {
        for _, col := range self.table.MappedColumns {
            self.queries = append(self.queries, QueryColumn{FieldName:col.FieldName, Alias:col.FieldName})
        }
    }
    s, args, err := self.session.getDialect().Select(self.table, self.queries,
        self.filters, self.orders, self.offset, 1)
    if nil != err {
        return nil
    }
    row := self.session.doQueryRow(s, args...)
    xrow := &XRow{row:row, qs:self}
    return xrow
}

func (self *QuerySet) Update(vals interface{}) (int64, error) {
    cols := []UpdateColumn{}
    if cm, ok := vals.(map[string]interface{}); ok {
        for k, v := range cm {
            var fk string
            if c, ok := self.table.MappedColumns[k]; ok {
                fk = c.FieldName
            }else if c, ok := self.table.JTaggedColumns[k]; ok {
                fk = c.FieldName
            }else{
                return 0, errors.New("Invalid column:"+k)
            }
            uc := UpdateColumn{Field:fk, Value:v, Operator:"="}
            cols = append(cols, uc)
        }
    }else if cx, ok := vals.([]UpdateColumn); ok {
        cols = cx
    }
    s, args, err := self.session.getDialect().Update(self.table, self.filters, cols...)
    if nil != err {
        return 0, err
    }
    var ret sql.Result
    ret, err = self.session.doExec(s, args...)
    if nil != err {
        return 0, err
    }else{
        rows, e := ret.RowsAffected()
        return rows, e
    }
    return 0, nil
}

func (self *QuerySet) Delete() (int64, error) {
    s, args, err := self.session.getDialect().Delete(self.table, self.filters)
    if nil != err {
        return 0, err
    }
    var ret sql.Result
    ret, err = self.session.doExec(s, args...)
    if nil != err {
        return 0, err
    }else{
        rows, e := ret.RowsAffected()
        return rows, e
    }
    return 0, nil
}

func (self *QuerySet) Insert(objs ...interface{}) (int64, error) {
    //log.Debugln("Insert:> ", objs)
    //log.Debugln("Table:> ", self.table)
    //var ret sql.Result
    var rows int64 = 0
    var cols []string
    if len(self.queries) > 0 {
        for _,x := range self.queries {
            cols = append(cols, x.FieldName)
        }
    }
    for _, obj := range objs {
        //log.Debugln("reflect.TypeOf(obj)>", reflect.TypeOf(obj))
        //log.Debugln("reflect.TypeOf(self.table.Entity)>", reflect.TypeOf(self.table.Entity))
        if reflect.TypeOf(obj) != reflect.TypeOf(self.table.Entity) {
           return 0, errors.New("Invalid data type.")
        }
        s, args, err := self.session.getDialect().Insert(self.table, obj, cols...)
        if nil != err {
            return 0, err
        }
        _, err = self.session.doExec(s, args...)
        if nil != err {
            return 0, err
        }else{
            //if nil != ret {
            //    n, e := ret.LastInsertId()
            //    if nil == e {
            //        rows += n
            //    }else{
            //        rows += 1
            //    }
            //    //log.Debugln("Insert:> ", n)
            //}else{
            //    rows += 1
            //}
            rows += 1
        }
    }
    return rows, nil
}
<|endoftext|>