<commit_before>// Copyright (c) 2017 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package clientv3

import (
	"context"

	"fmt"

	apiv3 "github.com/projectcalico/libcalico-go/lib/apis/v3"
	"github.com/projectcalico/libcalico-go/lib/errors"
	"github.com/projectcalico/libcalico-go/lib/names"
	"github.com/projectcalico/libcalico-go/lib/net"
	cnet "github.com/projectcalico/libcalico-go/lib/net"
	"github.com/projectcalico/libcalico-go/lib/options"
	validator "github.com/projectcalico/libcalico-go/lib/validator/v3"
	"github.com/projectcalico/libcalico-go/lib/watch"
	log "github.com/sirupsen/logrus"
)

// NodeInterface has methods to work with Node resources.
type NodeInterface interface {
	Create(ctx context.Context, res *apiv3.Node, opts options.SetOptions) (*apiv3.Node, error)
	Update(ctx context.Context, res *apiv3.Node, opts options.SetOptions) (*apiv3.Node, error)
	Delete(ctx context.Context, name string, opts options.DeleteOptions) (*apiv3.Node, error)
	Get(ctx context.Context, name string, opts options.GetOptions) (*apiv3.Node, error)
	List(ctx context.Context, opts options.ListOptions) (*apiv3.NodeList, error)
	Watch(ctx context.Context, opts options.ListOptions) (watch.Interface, error)
}

// nodes implements NodeInterface
type nodes struct {
	client client
}

// Create takes the representation of a Node and creates it.  Returns the stored
// representation of the Node, and an error, if there is any.
func (r nodes) Create(ctx context.Context, res *apiv3.Node, opts options.SetOptions) (*apiv3.Node, error) {
	if err := validator.Validate(res); err != nil {
		return nil, err
	}

	// For host-protection only clusters, we instruct the user to create a Node as the first
	// operation.  Piggy-back the datastore initialisation on that to ensure the Ready flag gets
	// set.  Since we're likely being called from calicoctl, we don't know the Calico version.
	err := r.client.EnsureInitialized(ctx, "", "")
	if err != nil {
		return nil, err
	}
	out, err := r.client.resources.Create(ctx, opts, apiv3.KindNode, res)
	if out != nil {
		return out.(*apiv3.Node), err
	}
	return nil, err
}

// Update takes the representation of a Node and updates it. Returns the stored
// representation of the Node, and an error, if there is any.
func (r nodes) Update(ctx context.Context, res *apiv3.Node, opts options.SetOptions) (*apiv3.Node, error) {
	if err := validator.Validate(res); err != nil {
		return nil, err
	}

	out, err := r.client.resources.Update(ctx, opts, apiv3.KindNode, res)
	if out != nil {
		return out.(*apiv3.Node), err
	}
	return nil, err
}

// Delete takes name of the Node and deletes it. Returns an error if one occurs.
func (r nodes) Delete(ctx context.Context, name string, opts options.DeleteOptions) (*apiv3.Node, error) {
	pname, err := names.WorkloadEndpointIdentifiers{Node: name}.CalculateWorkloadEndpointName(true)
	if err != nil {
		return nil, err
	}

	// Get all weps belonging to the node
	weps, err := r.client.WorkloadEndpoints().List(ctx, options.ListOptions{
		Prefix: true,
		Name:   pname,
	})
	if err != nil {
		return nil, err
	}

	// Collate all IPs across all endpoints, and then release those IPs.
	ips := []net.IP{}
	for _, wep := range weps.Items {
		// The prefix match is unfortunately not a perfect match on the Node (since it is theoretically possible for
		// another node to match the prefix (e.g. a node name of the format <thisnode>-foobar would also match a prefix
		// search of the node <thisnode>). Therefore, we will also need to check that the Spec.Node field matches the Node.
		if wep.Spec.Node != name {
			continue
		}
		for _, ip := range wep.Spec.IPNetworks {
			ipAddr, _, err := cnet.ParseCIDROrIP(ip)
			if err == nil {
				ips = append(ips, *ipAddr)
			} else {
				// Validation for wep insists upon CIDR, so we should always succeed
				log.WithError(err).Warnf("Failed to parse CIDR: %s", ip)
			}
		}
	}
	_, err = r.client.IPAM().ReleaseIPs(context.Background(), ips)
	switch err.(type) {
	case nil, errors.ErrorResourceDoesNotExist, errors.ErrorOperationNotSupported:
	default:
		return nil, err
	}

	// Delete the weps.
	for _, wep := range weps.Items {
		if wep.Spec.Node != name {
			continue
		}

		_, err = r.client.WorkloadEndpoints().Delete(ctx, wep.Namespace, wep.Name, options.DeleteOptions{})
		switch err.(type) {
		case nil, errors.ErrorResourceDoesNotExist, errors.ErrorOperationNotSupported:
		default:
			return nil, err
		}
	}

	// Remove the node from the IPAM data if it exists.
	err = r.client.IPAM().RemoveIPAMHost(ctx, name)
	switch err.(type) {
	case nil, errors.ErrorResourceDoesNotExist, errors.ErrorOperationNotSupported:
	default:
		return nil, err
	}

	// Remove BGPPeers.
	bgpPeers, err := r.client.BGPPeers().List(ctx, options.ListOptions{})
	if err != nil {
		return nil, err
	}
	for _, peer := range bgpPeers.Items {
		if peer.Spec.Node != name {
			continue
		}
		_, err = r.client.BGPPeers().Delete(ctx, peer.Name, options.DeleteOptions{})
		switch err.(type) {
		case nil, errors.ErrorResourceDoesNotExist, errors.ErrorOperationNotSupported:
		default:
			return nil, err
		}
	}

	// Delete felix configuration
	nodeConfName := fmt.Sprintf("node.%s", name)
	_, err = r.client.FelixConfigurations().Delete(ctx, nodeConfName, options.DeleteOptions{})
	switch err.(type) {
	case nil, errors.ErrorResourceDoesNotExist, errors.ErrorOperationNotSupported:
	default:
		return nil, err
	}

	// Delete bgp configuration
	_, err = r.client.BGPConfigurations().Delete(ctx, nodeConfName, options.DeleteOptions{})
	switch err.(type) {
	case nil, errors.ErrorResourceDoesNotExist, errors.ErrorOperationNotSupported:
	default:
		return nil, err
	}

	// Delete the node.
	out, err := r.client.resources.Delete(ctx, opts, apiv3.KindNode, noNamespace, name)
	if out != nil {
		return out.(*apiv3.Node), err
	}
	return nil, err
}

// Get takes name of the Node, and returns the corresponding Node object,
// and an error if there is any.
func (r nodes) Get(ctx context.Context, name string, opts options.GetOptions) (*apiv3.Node, error) {
	out, err := r.client.resources.Get(ctx, opts, apiv3.KindNode, noNamespace, name)
	if out != nil {
		return out.(*apiv3.Node), err
	}
	return nil, err
}

// List returns the list of Node objects that match the supplied options.
func (r nodes) List(ctx context.Context, opts options.ListOptions) (*apiv3.NodeList, error) {
	res := &apiv3.NodeList{}
	if err := r.client.resources.List(ctx, opts, apiv3.KindNode, apiv3.KindNodeList, res); err != nil {
		return nil, err
	}
	return res, nil
}

// Watch returns a watch.Interface that watches the Nodes that match the
// supplied options.
func (r nodes) Watch(ctx context.Context, opts options.ListOptions) (watch.Interface, error) {
	return r.client.resources.Watch(ctx, opts, apiv3.KindNode, nil)
}
<commit_msg>Release IPIP and VXLAN addresses on node deletion<commit_after>// Copyright (c) 2017 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package clientv3

import (
	"context"

	"fmt"

	apiv3 "github.com/projectcalico/libcalico-go/lib/apis/v3"
	"github.com/projectcalico/libcalico-go/lib/errors"
	"github.com/projectcalico/libcalico-go/lib/names"
	"github.com/projectcalico/libcalico-go/lib/net"
	cnet "github.com/projectcalico/libcalico-go/lib/net"
	"github.com/projectcalico/libcalico-go/lib/options"
	validator "github.com/projectcalico/libcalico-go/lib/validator/v3"
	"github.com/projectcalico/libcalico-go/lib/watch"
	log "github.com/sirupsen/logrus"
)

// NodeInterface has methods to work with Node resources.
type NodeInterface interface {
	Create(ctx context.Context, res *apiv3.Node, opts options.SetOptions) (*apiv3.Node, error)
	Update(ctx context.Context, res *apiv3.Node, opts options.SetOptions) (*apiv3.Node, error)
	Delete(ctx context.Context, name string, opts options.DeleteOptions) (*apiv3.Node, error)
	Get(ctx context.Context, name string, opts options.GetOptions) (*apiv3.Node, error)
	List(ctx context.Context, opts options.ListOptions) (*apiv3.NodeList, error)
	Watch(ctx context.Context, opts options.ListOptions) (watch.Interface, error)
}

// nodes implements NodeInterface
type nodes struct {
	client client
}

// Create takes the representation of a Node and creates it.  Returns the stored
// representation of the Node, and an error, if there is any.
func (r nodes) Create(ctx context.Context, res *apiv3.Node, opts options.SetOptions) (*apiv3.Node, error) {
	if err := validator.Validate(res); err != nil {
		return nil, err
	}

	// For host-protection only clusters, we instruct the user to create a Node as the first
	// operation.  Piggy-back the datastore initialisation on that to ensure the Ready flag gets
	// set.  Since we're likely being called from calicoctl, we don't know the Calico version.
	err := r.client.EnsureInitialized(ctx, "", "")
	if err != nil {
		return nil, err
	}
	out, err := r.client.resources.Create(ctx, opts, apiv3.KindNode, res)
	if out != nil {
		return out.(*apiv3.Node), err
	}
	return nil, err
}

// Update takes the representation of a Node and updates it. Returns the stored
// representation of the Node, and an error, if there is any.
func (r nodes) Update(ctx context.Context, res *apiv3.Node, opts options.SetOptions) (*apiv3.Node, error) {
	if err := validator.Validate(res); err != nil {
		return nil, err
	}

	out, err := r.client.resources.Update(ctx, opts, apiv3.KindNode, res)
	if out != nil {
		return out.(*apiv3.Node), err
	}
	return nil, err
}

// Delete takes name of the Node and deletes it. Returns an error if one occurs.
func (r nodes) Delete(ctx context.Context, name string, opts options.DeleteOptions) (*apiv3.Node, error) {
	pname, err := names.WorkloadEndpointIdentifiers{Node: name}.CalculateWorkloadEndpointName(true)
	if err != nil {
		return nil, err
	}

	// Get all weps belonging to the node
	weps, err := r.client.WorkloadEndpoints().List(ctx, options.ListOptions{
		Prefix: true,
		Name:   pname,
	})
	if err != nil {
		return nil, err
	}

	// Collate all IPs across all endpoints, and then release those IPs.
	ips := []net.IP{}
	for _, wep := range weps.Items {
		// The prefix match is unfortunately not a perfect match on the Node (since it is theoretically possible for
		// another node to match the prefix (e.g. a node name of the format <thisnode>-foobar would also match a prefix
		// search of the node <thisnode>). Therefore, we will also need to check that the Spec.Node field matches the Node.
		if wep.Spec.Node != name {
			continue
		}
		for _, ip := range wep.Spec.IPNetworks {
			ipAddr, _, err := cnet.ParseCIDROrIP(ip)
			if err == nil {
				ips = append(ips, *ipAddr)
			} else {
				// Validation for wep insists upon CIDR, so we should always succeed
				log.WithError(err).Warnf("Failed to parse CIDR: %s", ip)
			}
		}
	}

	// Add in tunnel addresses if they exist for the node.
	if n, err := r.client.Nodes().Get(ctx, name, options.GetOptions{}); err != nil {
		if _, ok := err.(errors.ErrorResourceDoesNotExist); !ok {
			return nil, err
		}
		// Resource does not exist, carry on and clean up as much as we can.
	} else {
		if n.Spec.BGP != nil && n.Spec.BGP.IPv4IPIPTunnelAddr != "" {
			ipAddr, _, err := cnet.ParseCIDROrIP(n.Spec.BGP.IPv4IPIPTunnelAddr)
			if err == nil {
				ips = append(ips, *ipAddr)
			} else {
				log.WithError(err).Warnf("Failed to parse IPIP tunnel address CIDR: %s", n.Spec.BGP.IPv4IPIPTunnelAddr)
			}
		}
		if n.Spec.IPv4VXLANTunnelAddr != "" {
			ipAddr, _, err := cnet.ParseCIDROrIP(n.Spec.IPv4VXLANTunnelAddr)
			if err == nil {
				ips = append(ips, *ipAddr)
			} else {
				log.WithError(err).Warnf("Failed to parse VXLAN tunnel address CIDR: %s", n.Spec.IPv4VXLANTunnelAddr)
			}
		}
	}

	_, err = r.client.IPAM().ReleaseIPs(context.Background(), ips)
	switch err.(type) {
	case nil, errors.ErrorResourceDoesNotExist, errors.ErrorOperationNotSupported:
	default:
		return nil, err
	}

	// Delete the weps.
	for _, wep := range weps.Items {
		if wep.Spec.Node != name {
			continue
		}

		_, err = r.client.WorkloadEndpoints().Delete(ctx, wep.Namespace, wep.Name, options.DeleteOptions{})
		switch err.(type) {
		case nil, errors.ErrorResourceDoesNotExist, errors.ErrorOperationNotSupported:
		default:
			return nil, err
		}
	}

	// Remove the node from the IPAM data if it exists.
	err = r.client.IPAM().RemoveIPAMHost(ctx, name)
	switch err.(type) {
	case nil, errors.ErrorResourceDoesNotExist, errors.ErrorOperationNotSupported:
	default:
		return nil, err
	}

	// Remove BGPPeers.
	bgpPeers, err := r.client.BGPPeers().List(ctx, options.ListOptions{})
	if err != nil {
		return nil, err
	}
	for _, peer := range bgpPeers.Items {
		if peer.Spec.Node != name {
			continue
		}
		_, err = r.client.BGPPeers().Delete(ctx, peer.Name, options.DeleteOptions{})
		switch err.(type) {
		case nil, errors.ErrorResourceDoesNotExist, errors.ErrorOperationNotSupported:
		default:
			return nil, err
		}
	}

	// Delete felix configuration
	nodeConfName := fmt.Sprintf("node.%s", name)
	_, err = r.client.FelixConfigurations().Delete(ctx, nodeConfName, options.DeleteOptions{})
	switch err.(type) {
	case nil, errors.ErrorResourceDoesNotExist, errors.ErrorOperationNotSupported:
	default:
		return nil, err
	}

	// Delete bgp configuration
	_, err = r.client.BGPConfigurations().Delete(ctx, nodeConfName, options.DeleteOptions{})
	switch err.(type) {
	case nil, errors.ErrorResourceDoesNotExist, errors.ErrorOperationNotSupported:
	default:
		return nil, err
	}

	// Delete the node.
	out, err := r.client.resources.Delete(ctx, opts, apiv3.KindNode, noNamespace, name)
	if out != nil {
		return out.(*apiv3.Node), err
	}
	return nil, err
}

// Get takes name of the Node, and returns the corresponding Node object,
// and an error if there is any.
func (r nodes) Get(ctx context.Context, name string, opts options.GetOptions) (*apiv3.Node, error) {
	out, err := r.client.resources.Get(ctx, opts, apiv3.KindNode, noNamespace, name)
	if out != nil {
		return out.(*apiv3.Node), err
	}
	return nil, err
}

// List returns the list of Node objects that match the supplied options.
func (r nodes) List(ctx context.Context, opts options.ListOptions) (*apiv3.NodeList, error) {
	res := &apiv3.NodeList{}
	if err := r.client.resources.List(ctx, opts, apiv3.KindNode, apiv3.KindNodeList, res); err != nil {
		return nil, err
	}
	return res, nil
}

// Watch returns a watch.Interface that watches the Nodes that match the
// supplied options.
func (r nodes) Watch(ctx context.Context, opts options.ListOptions) (watch.Interface, error) {
	return r.client.resources.Watch(ctx, opts, apiv3.KindNode, nil)
}
<|endoftext|>


<commit_before>package uatparse

// AIRMET = AIRMET/SIGMET/ (TFR?)

const (
	UATMSG_TEXT   = 1
	UATMSG_NEXRAD = 2
	UATMSG_AIRMET = 3 // AIRMET. Decoded.

	// How the coordinates should be used in a graphical AIRMET.
	AIRMET_POLYGON = 1
	AIRMET_ELLIPSE = 2
	AIRMET_PRISM   = 3
	AIRMET_3D      = 4
)

// Points can be in 3D - take care that altitude is used correctly.
type GeoPoints struct {
	Lat float64
	Lon float64
	Alt int32
}

type UATAirmet struct {
	Points []GeoPoints // Points
}

type UATMsgDecoded struct {
	Type int
}
<commit_msg>Revert "Beginning."<commit_after><|endoftext|>


<commit_before>package goose

import (
	"database/sql"
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"text/template"
	"time"

	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
	_ "github.com/ziutek/mymysql/godrv"
)

var (
	ErrTableDoesNotExist = errors.New("table does not exist")
	ErrNoPreviousVersion = errors.New("no previous version found")
)

type MigrationRecord struct {
	VersionId int64
	TStamp    time.Time
	IsApplied bool // was this a result of up() or down()
}

type Migration struct {
	Version  int64
	Next     int64  // next version, or -1 if none
	Previous int64  // previous version, -1 if none
	Source   string // path to .go or .sql script
}

type migrationSorter []*Migration

// helpers so we can use pkg sort
func (ms migrationSorter) Len() int           { return len(ms) }
func (ms migrationSorter) Swap(i, j int)      { ms[i], ms[j] = ms[j], ms[i] }
func (ms migrationSorter) Less(i, j int) bool { return ms[i].Version < ms[j].Version }

func newMigration(v int64, src string) *Migration {
	return &Migration{v, -1, -1, src}
}

func RunMigrations(conf *DBConf, migrationsDir string, target int64) (err error) {

	db, err := OpenDBFromDBConf(conf)
	if err != nil {
		return err
	}
	defer db.Close()

	return RunMigrationsOnDb(conf, migrationsDir, target, db)
}

// Runs migration on a specific database instance.
func RunMigrationsOnDb(conf *DBConf, migrationsDir string, target int64, db *sql.DB) (err error) {
	current, err := EnsureDBVersion(conf, db)
	if err != nil {
		return err
	}

	migrations, err := CollectMigrations(migrationsDir, current, target)
	if err != nil {
		return err
	}

	if len(migrations) == 0 {
		fmt.Printf("goose: no migrations to run. current version: %d\n", current)
		return nil
	}

	ms := migrationSorter(migrations)
	direction := current < target
	ms.Sort(direction)

	fmt.Printf("goose: migrating db environment '%v', current version: %d, target: %d\n",
		conf.Env, current, target)

	for _, m := range ms {

		switch filepath.Ext(m.Source) {
		case ".sql":
			err = runSQLMigration(conf, db, m.Source, m.Version, direction)
		}

		if err != nil {
			return errors.New(fmt.Sprintf("FAIL %v, quitting migration", err))
		}

		fmt.Println("OK   ", filepath.Base(m.Source))
	}

	return nil
}

// collect all the valid looking migration scripts in the
// migrations folder, and key them by version
func CollectMigrations(dirpath string, current, target int64) (m []*Migration, err error) {

	// extract the numeric component of each migration,
	// filter out any uninteresting files,
	// and ensure we only have one file per migration version.
	filepath.Walk(dirpath, func(name string, info os.FileInfo, err error) error {

		if v, e := NumericComponent(name); e == nil {

			for _, g := range m {
				if v == g.Version {
					log.Fatalf("more than one file specifies the migration for version %d (%s and %s)",
						v, g.Source, filepath.Join(dirpath, name))
				}
			}

			if versionFilter(v, current, target) {
				m = append(m, newMigration(v, name))
			}
		}

		return nil
	})

	return m, nil
}

func versionFilter(v, current, target int64) bool {

	if target > current {
		return v > current && v <= target
	}

	if target < current {
		return v <= current && v > target
	}

	return false
}

func (ms migrationSorter) Sort(direction bool) {

	// sort ascending or descending by version
	if direction {
		sort.Sort(ms)
	} else {
		sort.Sort(sort.Reverse(ms))
	}

	// now that we're sorted in the appropriate direction,
	// populate next and previous for each migration
	for i, m := range ms {
		prev := int64(-1)
		if i > 0 {
			prev = ms[i-1].Version
			ms[i-1].Next = m.Version
		}
		ms[i].Previous = prev
	}
}

// look for migration scripts with names in the form:
//  XXX_descriptivename.ext
// where XXX specifies the version number
// and ext specifies the type of migration
func NumericComponent(name string) (int64, error) {

	base := filepath.Base(name)

	if ext := filepath.Ext(base); ext != ".sql" {
		return 0, errors.New("not a recognized migration file type")
	}

	idx := strings.Index(base, "_")
	if idx < 0 {
		return 0, errors.New("no separator found")
	}

	n, e := strconv.ParseInt(base[:idx], 10, 64)
	if e == nil && n <= 0 {
		return 0, errors.New("migration IDs must be greater than zero")
	}

	return n, e
}

// retrieve the current version for this DB.
// Create and initialize the DB version table if it doesn't exist.
func EnsureDBVersion(conf *DBConf, db *sql.DB) (int64, error) {

	rows, err := conf.Driver.Dialect.dbVersionQuery(db)
	if err != nil {
		if err == ErrTableDoesNotExist {
			return 0, createVersionTable(conf, db)
		}
		return 0, err
	}
	defer rows.Close()

	// The most recent record for each migration specifies
	// whether it has been applied or rolled back.
	// The first version we find that has been applied is the current version.

	toSkip := make([]int64, 0)

	for rows.Next() {
		var row MigrationRecord
		if err = rows.Scan(&row.VersionId, &row.IsApplied); err != nil {
			log.Fatal("error scanning rows:", err)
		}

		// have we already marked this version to be skipped?
		skip := false
		for _, v := range toSkip {
			if v == row.VersionId {
				skip = true
				break
			}
		}

		if skip {
			continue
		}

		// if version has been applied we're done
		if row.IsApplied {
			return row.VersionId, nil
		}

		// latest version of migration has not been applied.
		toSkip = append(toSkip, row.VersionId)
	}

	panic("failure in EnsureDBVersion()")
}

// Create the goose_db_version table
// and insert the initial 0 value into it
func createVersionTable(conf *DBConf, db *sql.DB) error {
	txn, err := db.Begin()
	if err != nil {
		return err
	}

	d := conf.Driver.Dialect

	if _, err := txn.Exec(d.createVersionTableSql()); err != nil {
		txn.Rollback()
		return err
	}

	version := 0
	applied := true
	if _, err := txn.Exec(d.insertVersionSql(), version, applied); err != nil {
		txn.Rollback()
		return err
	}

	return txn.Commit()
}

// wrapper for EnsureDBVersion for callers that don't already have
// their own DB instance
func GetDBVersion(conf *DBConf) (version int64, err error) {

	db, err := OpenDBFromDBConf(conf)
	if err != nil {
		return -1, err
	}
	defer db.Close()

	version, err = EnsureDBVersion(conf, db)
	if err != nil {
		return -1, err
	}

	return version, nil
}

func GetPreviousDBVersion(dirpath string, version int64) (previous int64, err error) {

	previous = -1
	sawGivenVersion := false

	filepath.Walk(dirpath, func(name string, info os.FileInfo, walkerr error) error {

		if !info.IsDir() {
			if v, e := NumericComponent(name); e == nil {
				if v > previous && v < version {
					previous = v
				}
				if v == version {
					sawGivenVersion = true
				}
			}
		}

		return nil
	})

	if previous == -1 {
		if sawGivenVersion {
			// the given version is (likely) valid but we didn't find
			// anything before it.
			// 'previous' must reflect that no migrations have been applied.
			previous = 0
		} else {
			err = ErrNoPreviousVersion
		}
	}

	return
}

// helper to identify the most recent possible version
// within a folder of migration scripts
func GetMostRecentDBVersion(dirpath string) (version int64, err error) {

	version = -1

	filepath.Walk(dirpath, func(name string, info os.FileInfo, walkerr error) error {
		if walkerr != nil {
			return walkerr
		}

		if !info.IsDir() {
			if v, e := NumericComponent(name); e == nil {
				if v > version {
					version = v
				}
			}
		}

		return nil
	})

	if version == -1 {
		err = errors.New("no valid version found")
	}

	return
}

func CreateMigration(name, migrationType, dir string, t time.Time) (path string, err error) {

	if migrationType != "sql" {
		return "", errors.New("migration type must be 'sql'")
	}

	timestamp := t.Format("20060102150405")
	filename := fmt.Sprintf("%v_%v.%v", timestamp, name, migrationType)

	fpath := filepath.Join(dir, filename)

	var tmpl *template.Template
	if migrationType == "sql" {
		tmpl = sqlMigrationTemplate
	}

	path, err = writeTemplateToFile(fpath, tmpl, timestamp)

	return
}

var sqlMigrationTemplate = template.Must(template.New("goose.sql-migration").Parse(`
-- +goose Up
-- SQL in section 'Up' is executed when this migration is applied

-- +goose Down
-- SQL section 'Down' is executed when this migration is rolled back
`))
<commit_msg>Remove whitespace at top of goose create template<commit_after>package goose

import (
	"database/sql"
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"text/template"
	"time"

	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
	_ "github.com/ziutek/mymysql/godrv"
)

var (
	ErrTableDoesNotExist = errors.New("table does not exist")
	ErrNoPreviousVersion = errors.New("no previous version found")
)

type MigrationRecord struct {
	VersionId int64
	TStamp    time.Time
	IsApplied bool // was this a result of up() or down()
}

type Migration struct {
	Version  int64
	Next     int64  // next version, or -1 if none
	Previous int64  // previous version, -1 if none
	Source   string // path to .go or .sql script
}

type migrationSorter []*Migration

// helpers so we can use pkg sort
func (ms migrationSorter) Len() int           { return len(ms) }
func (ms migrationSorter) Swap(i, j int)      { ms[i], ms[j] = ms[j], ms[i] }
func (ms migrationSorter) Less(i, j int) bool { return ms[i].Version < ms[j].Version }

func newMigration(v int64, src string) *Migration {
	return &Migration{v, -1, -1, src}
}

func RunMigrations(conf *DBConf, migrationsDir string, target int64) (err error) {

	db, err := OpenDBFromDBConf(conf)
	if err != nil {
		return err
	}
	defer db.Close()

	return RunMigrationsOnDb(conf, migrationsDir, target, db)
}

// Runs migration on a specific database instance.
func RunMigrationsOnDb(conf *DBConf, migrationsDir string, target int64, db *sql.DB) (err error) {
	current, err := EnsureDBVersion(conf, db)
	if err != nil {
		return err
	}

	migrations, err := CollectMigrations(migrationsDir, current, target)
	if err != nil {
		return err
	}

	if len(migrations) == 0 {
		fmt.Printf("goose: no migrations to run. current version: %d\n", current)
		return nil
	}

	ms := migrationSorter(migrations)
	direction := current < target
	ms.Sort(direction)

	fmt.Printf("goose: migrating db environment '%v', current version: %d, target: %d\n",
		conf.Env, current, target)

	for _, m := range ms {

		switch filepath.Ext(m.Source) {
		case ".sql":
			err = runSQLMigration(conf, db, m.Source, m.Version, direction)
		}

		if err != nil {
			return errors.New(fmt.Sprintf("FAIL %v, quitting migration", err))
		}

		fmt.Println("OK   ", filepath.Base(m.Source))
	}

	return nil
}

// collect all the valid looking migration scripts in the
// migrations folder, and key them by version
func CollectMigrations(dirpath string, current, target int64) (m []*Migration, err error) {

	// extract the numeric component of each migration,
	// filter out any uninteresting files,
	// and ensure we only have one file per migration version.
	filepath.Walk(dirpath, func(name string, info os.FileInfo, err error) error {

		if v, e := NumericComponent(name); e == nil {

			for _, g := range m {
				if v == g.Version {
					log.Fatalf("more than one file specifies the migration for version %d (%s and %s)",
						v, g.Source, filepath.Join(dirpath, name))
				}
			}

			if versionFilter(v, current, target) {
				m = append(m, newMigration(v, name))
			}
		}

		return nil
	})

	return m, nil
}

func versionFilter(v, current, target int64) bool {

	if target > current {
		return v > current && v <= target
	}

	if target < current {
		return v <= current && v > target
	}

	return false
}

func (ms migrationSorter) Sort(direction bool) {

	// sort ascending or descending by version
	if direction {
		sort.Sort(ms)
	} else {
		sort.Sort(sort.Reverse(ms))
	}

	// now that we're sorted in the appropriate direction,
	// populate next and previous for each migration
	for i, m := range ms {
		prev := int64(-1)
		if i > 0 {
			prev = ms[i-1].Version
			ms[i-1].Next = m.Version
		}
		ms[i].Previous = prev
	}
}

// look for migration scripts with names in the form:
//  XXX_descriptivename.ext
// where XXX specifies the version number
// and ext specifies the type of migration
func NumericComponent(name string) (int64, error) {

	base := filepath.Base(name)

	if ext := filepath.Ext(base); ext != ".sql" {
		return 0, errors.New("not a recognized migration file type")
	}

	idx := strings.Index(base, "_")
	if idx < 0 {
		return 0, errors.New("no separator found")
	}

	n, e := strconv.ParseInt(base[:idx], 10, 64)
	if e == nil && n <= 0 {
		return 0, errors.New("migration IDs must be greater than zero")
	}

	return n, e
}

// retrieve the current version for this DB.
// Create and initialize the DB version table if it doesn't exist.
func EnsureDBVersion(conf *DBConf, db *sql.DB) (int64, error) {

	rows, err := conf.Driver.Dialect.dbVersionQuery(db)
	if err != nil {
		if err == ErrTableDoesNotExist {
			return 0, createVersionTable(conf, db)
		}
		return 0, err
	}
	defer rows.Close()

	// The most recent record for each migration specifies
	// whether it has been applied or rolled back.
	// The first version we find that has been applied is the current version.

	toSkip := make([]int64, 0)

	for rows.Next() {
		var row MigrationRecord
		if err = rows.Scan(&row.VersionId, &row.IsApplied); err != nil {
			log.Fatal("error scanning rows:", err)
		}

		// have we already marked this version to be skipped?
		skip := false
		for _, v := range toSkip {
			if v == row.VersionId {
				skip = true
				break
			}
		}

		if skip {
			continue
		}

		// if version has been applied we're done
		if row.IsApplied {
			return row.VersionId, nil
		}

		// latest version of migration has not been applied.
		toSkip = append(toSkip, row.VersionId)
	}

	panic("failure in EnsureDBVersion()")
}

// Create the goose_db_version table
// and insert the initial 0 value into it
func createVersionTable(conf *DBConf, db *sql.DB) error {
	txn, err := db.Begin()
	if err != nil {
		return err
	}

	d := conf.Driver.Dialect

	if _, err := txn.Exec(d.createVersionTableSql()); err != nil {
		txn.Rollback()
		return err
	}

	version := 0
	applied := true
	if _, err := txn.Exec(d.insertVersionSql(), version, applied); err != nil {
		txn.Rollback()
		return err
	}

	return txn.Commit()
}

// wrapper for EnsureDBVersion for callers that don't already have
// their own DB instance
func GetDBVersion(conf *DBConf) (version int64, err error) {

	db, err := OpenDBFromDBConf(conf)
	if err != nil {
		return -1, err
	}
	defer db.Close()

	version, err = EnsureDBVersion(conf, db)
	if err != nil {
		return -1, err
	}

	return version, nil
}

func GetPreviousDBVersion(dirpath string, version int64) (previous int64, err error) {

	previous = -1
	sawGivenVersion := false

	filepath.Walk(dirpath, func(name string, info os.FileInfo, walkerr error) error {

		if !info.IsDir() {
			if v, e := NumericComponent(name); e == nil {
				if v > previous && v < version {
					previous = v
				}
				if v == version {
					sawGivenVersion = true
				}
			}
		}

		return nil
	})

	if previous == -1 {
		if sawGivenVersion {
			// the given version is (likely) valid but we didn't find
			// anything before it.
			// 'previous' must reflect that no migrations have been applied.
			previous = 0
		} else {
			err = ErrNoPreviousVersion
		}
	}

	return
}

// helper to identify the most recent possible version
// within a folder of migration scripts
func GetMostRecentDBVersion(dirpath string) (version int64, err error) {

	version = -1

	filepath.Walk(dirpath, func(name string, info os.FileInfo, walkerr error) error {
		if walkerr != nil {
			return walkerr
		}

		if !info.IsDir() {
			if v, e := NumericComponent(name); e == nil {
				if v > version {
					version = v
				}
			}
		}

		return nil
	})

	if version == -1 {
		err = errors.New("no valid version found")
	}

	return
}

func CreateMigration(name, migrationType, dir string, t time.Time) (path string, err error) {

	if migrationType != "sql" {
		return "", errors.New("migration type must be 'sql'")
	}

	timestamp := t.Format("20060102150405")
	filename := fmt.Sprintf("%v_%v.%v", timestamp, name, migrationType)

	fpath := filepath.Join(dir, filename)

	var tmpl *template.Template
	if migrationType == "sql" {
		tmpl = sqlMigrationTemplate
	}

	path, err = writeTemplateToFile(fpath, tmpl, timestamp)

	return
}

var sqlMigrationTemplate = template.Must(template.New("goose.sql-migration").Parse(
	`-- +goose Up
-- SQL in section 'Up' is executed when this migration is applied

-- +goose Down
-- SQL section 'Down' is executed when this migration is rolled back
`))
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"json"
	"math"
	"./nbt"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
)

var (
	out        *bufio.Writer
	yMin       int
	blockFaces bool
	hideBottom bool
	noColor    bool

	faceCount int
	faceLimit int

	chunkCount int
	chunkLimit int

	chunkMask ChunkMask
)

func main() {
	var bx, bz float64
	var cx, cz int
	var square int
	var rectx, rectz int
	var maxProcs = runtime.GOMAXPROCS(0)
	var prt bool
	var solidSides bool
	var mtlNumber bool

	var defaultObjOutFilename = "a.obj"
	var defaultPrtOutFilename = "a.prt"

	var outFilename string
	flag.IntVar(&maxProcs, "cpu", maxProcs, "Number of cores to use")
	flag.StringVar(&outFilename, "o", defaultObjOutFilename, "Name for output file")
	flag.IntVar(&yMin, "y", 0, "Omit all blocks below this height. 63 is sea level")
	flag.BoolVar(&solidSides, "sides", false, "Solid sides, rather than showing underground")
	flag.BoolVar(&blockFaces, "bf", false, "Don't combine adjacent faces of the same block within a column")
	flag.BoolVar(&hideBottom, "hb", false, "Hide bottom of world")
	flag.BoolVar(&noColor, "g", false, "Omit materials")
	flag.Float64Var(&bx, "x", 0, "Center x coordinate in blocks")
	flag.Float64Var(&bz, "z", 0, "Center z coordinate in blocks")
	flag.IntVar(&cx, "cx", 0, "Center x coordinate in chunks")
	flag.IntVar(&cz, "cz", 0, "Center z coordinate in chunks")
	flag.IntVar(&square, "s", math.MaxInt32, "Chunk square size")
	flag.IntVar(&rectx, "rx", math.MaxInt32, "Width(x) of rectangle size")
	flag.IntVar(&rectz, "rz", math.MaxInt32, "Height(z) of rectangle size")
	flag.IntVar(&faceLimit, "fk", math.MaxInt32, "Face limit (thousands of faces)")
	flag.BoolVar(&prt, "prt", false, "Write out PRT file instead of Obj file")
	flag.BoolVar(&mtlNumber, "mtlnum", false, "Number materials instead of using names")
	var showHelp = flag.Bool("h", false, "Show Help")
	flag.Parse()

	runtime.GOMAXPROCS(maxProcs)
	fmt.Printf("mcobj %v (cpu: %d) Copyright (c) 2011 Jonathan Wright\n", version, runtime.GOMAXPROCS(0))

	if *showHelp || flag.NArg() == 0 {
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Usage: mcobj -cpu 4 -s 20 -o world1.obj", exampleWorldPath)
		fmt.Fprintln(os.Stderr)
		flag.PrintDefaults()
		return
	}

	if faceLimit != math.MaxInt32 {
		faceLimit *= 1000
	}

	if mtlNumber {
		MaterialNamer = new(NumberBlockIdNamer)
	} else {
		MaterialNamer = new(NameBlockIdNamer)
	}

	switch {
	case bx == 0 && cx == 0:
		cx = 0
	case cx == 0:
		cx = int(bx / 16)
	}

	switch {
	case bz == 0 && cz == 0:
		cz = 0
	case cz == 0:
		cz = int(bz / 16)
	}

	if square != math.MaxInt32 {
		chunkLimit = square * square
		var h = square / 2
		chunkMask = &RectangeChunkMask{cx - h, cz - h, cx - h + square, cz - h + square}
	} else if rectx != math.MaxInt32 || rectz != math.MaxInt32 {
		switch {
		case rectx != math.MaxInt32 && rectz != math.MaxInt32:
			chunkLimit = rectx * rectz
			var (
				hx = rectx / 2
				hz = rectz / 2
			)
			chunkMask = &RectangeChunkMask{cx - hx, cz - hz, cx - hx + rectx, cz - hz + rectz}
		case rectx != math.MaxInt32:
			chunkLimit = math.MaxInt32
			var hx = rectx / 2
			chunkMask = &RectangeChunkMask{cx - hx, math.MinInt32, cx - hx + rectx, math.MaxInt32}
		case rectz != math.MaxInt32:
			chunkLimit = math.MaxInt32
			var hz = rectz / 2
			chunkMask = &RectangeChunkMask{math.MinInt32, cz - hz, math.MaxInt32, cz - hz + rectz}
		}
	} else {
		chunkLimit = math.MaxInt32
		chunkMask = &AllChunksMask{}
	}

	if prt && outFilename == defaultObjOutFilename {
		outFilename = defaultPrtOutFilename
	}

	if solidSides {
		defaultSide = &emptySide
	}

	{
		var dir, _ = filepath.Split(strings.Replace(os.Args[0], "\\", "/", -1))
		var jsonError = loadBlockTypesJson(filepath.Join(dir, "blocks.json"))
		if jsonError != nil {
			fmt.Fprintln(os.Stderr, jsonError)
			return
		}
	}

	for i := 0; i < flag.NArg(); i++ {
		var dirpath = flag.Arg(i)
		var fi, err = os.Stat(dirpath)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		} else if !fi.IsDirectory() {
			fmt.Fprintln(os.Stderr, dirpath, "is not a directory")
		}

		var world = OpenWorld(dirpath, chunkMask)
		var pool, poolErr = world.ChunkPool()
		if poolErr != nil {
			fmt.Println(poolErr)
			continue
		}

		var generator OutputGenerator
		if prt {
			generator = new(PrtGenerator)
		} else {
			generator = new(ObjGenerator)
		}
		var boundary = new(BoundaryLocator)
		boundary.Init()
		generator.Start(outFilename, pool.Remaining(), maxProcs, boundary)

		if walkEnclosedChunks(pool, world, cx, cz, generator.GetEnclosedJobsChan()) {
			<-generator.GetCompleteChan()
		}

		generator.Close()
	}
}

type OutputGenerator interface {
	Start(outFilename string, total int, maxProcs int, boundary *BoundaryLocator)
	GetEnclosedJobsChan() chan *EnclosedChunkJob
	GetCompleteChan() chan bool
	Close()
}

type EnclosedChunkJob struct {
	last     bool
	enclosed *EnclosedChunk
}

func walkEnclosedChunks(pool ChunkPool, opener ChunkOpener, cx, cz int, enclosedsChan chan *EnclosedChunkJob) bool {
	var (
		sideCache = new(SideCache)
		started   = false
	)

	for i := 0; moreChunks(pool.Remaining()); i++ {
		for x := 0; x < i && moreChunks(pool.Remaining()); x++ {
			for z := 0; z < i && moreChunks(pool.Remaining()); z++ {
				var (
					ax = cx + unzigzag(x)
					az = cz + unzigzag(z)
				)

				if pool.Pop(ax, az) {
					loadSide(sideCache, opener, ax-1, az)
					loadSide(sideCache, opener, ax+1, az)
					loadSide(sideCache, opener, ax, az-1)
					loadSide(sideCache, opener, ax, az+1)

					var chunk, loadErr = loadChunk2(opener, ax, az)
					if loadErr != nil {
						fmt.Println(loadErr)
					} else {
						var enclosed = sideCache.EncloseChunk(chunk)
						sideCache.AddChunk(chunk)
						chunkCount++
						enclosedsChan <- &EnclosedChunkJob{!moreChunks(pool.Remaining()), enclosed}
						started = true
					}
				}
			}
		}
	}

	return started
}

type Blocks []uint16

type BlockColumn []uint16

func (b *Blocks) Get(x, y, z int) uint16 {
	return (*b)[y+(z*128+(x*128*16))]
}

func (b *Blocks) Column(x, z int) BlockColumn {
	var i = 128 * (z + x*16)
	return BlockColumn((*b)[i : i+128])
}

func zigzag(n int) int {
	return (n << 1) ^ (n >> 31)
}

func unzigzag(n int) int {
	return (n >> 1) ^ (-(n & 1))
}

func moreChunks(unprocessedCount int) bool {
	return unprocessedCount > 0 && faceCount < faceLimit && chunkCount < chunkLimit
}

func loadChunk(filename string) (*nbt.Chunk, os.Error) {
	var file, fileErr = os.Open(filename)
	defer file.Close()
	if fileErr != nil {
		return nil, fileErr
	}
	var chunk, err = nbt.ReadDat(file)
	if err == os.EOF {
		err = nil
	}
	return chunk, err
}

func loadChunk2(opener ChunkOpener, x, z int) (*nbt.Chunk, os.Error) {
	var r, openErr = opener.OpenChunk(x, z)
	if openErr != nil {
		return nil, openErr
	}
	defer r.Close()

	var chunk, nbtErr = nbt.ReadNbt(r)
	if nbtErr != nil {
		return nil, nbtErr
	}
	return chunk, nil
}

func loadSide(sideCache *SideCache, opener ChunkOpener, x, z int) {
	if !sideCache.HasSide(x, z) && !chunkMask.IsMasked(x, z) {
		var chunk, loadErr = loadChunk2(opener, x, z)
		if loadErr != nil {
			fmt.Println(loadErr)
		} else {
			sideCache.AddChunk(chunk)
		}
	}
}

func loadBlockTypesJson(filename string) os.Error {
	var jsonBytes, jsonIoError = ioutil.ReadFile(filename)

	if jsonIoError != nil {
		return jsonIoError
	}

	var f interface{}
	var unmarshalError = json.Unmarshal(jsonBytes, &f)
	if unmarshalError != nil {
		return unmarshalError
	}

	var lines, linesOk = f.([]interface{})
	if linesOk {
		for _, line := range lines {
			var fields, fieldsOk = line.(map[string]interface{})
			if fieldsOk {
				var (
					blockId      byte
					data         byte = 255
					dataArray    []byte
					name         string
					mass         SingularOrAggregate = Mass
					transparency Transparency        = Opaque
					empty        bool                = false
					color        uint32
				)
				for k, v := range fields {
					switch k {
					case "name":
						name = v.(string)
					case "color":
						switch len(v.(string)) {
						case 7:
							var n, numErr = strconv.Btoui64(v.(string)[1:], 16)
							if numErr == nil {
								color = uint32(n*0x100 + 0xff)
							}
						case 9:
							var n, numErr = strconv.Btoui64(v.(string)[1:], 16)
							if numErr == nil {
								color = uint32(n)
							}
						}
					case "blockId":
						blockId = byte(v.(float64))
					case "data":
						switch d := v.(type) {
						case float64:
							data = byte(d)
						case []interface{}:
							dataArray = make([]byte, len(d))
							for i, value := range d {
								dataArray[i] = byte(value.(float64))
							}
						}
					case "item":
						if v.(bool) {
							mass = Item
							transparency = Transparent
						} else {
							mass = Mass
							transparency = Opaque
						}
					case "transparent":
						if v.(bool) {
							transparency = Transparent
						} else {
							transparency = Opaque
						}
					case "empty":
						if v.(bool) {
							empty = true
							transparency = Transparent
							mass = Mass
						} else {
							empty = false
						}
					}
				}

				blockTypeMap[blockId] = &BlockType{blockId, mass, transparency, empty}
				if dataArray == nil {
					if data != 255 {
						extraData[blockId] = true
						colors = append(colors, MTL{blockId, data, color, name})
					} else {
						colors[blockId] = MTL{blockId, data, color, name}
					}
				} else {
					extraData[blockId] = true
					for _, data = range dataArray {
						colors = append(colors, MTL{blockId, data, color, fmt.Sprintf("%s_%d", name, data)})
					}
				}
			}
		}
	}

	return nil
}
<commit_msg>Fix off-by-one bug when -x or -z are negative<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"json"
	"math"
	"./nbt"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
)

var (
	out        *bufio.Writer
	yMin       int
	blockFaces bool
	hideBottom bool
	noColor    bool

	faceCount int
	faceLimit int

	chunkCount int
	chunkLimit int

	chunkMask ChunkMask
)

func main() {
	var bx, bz float64
	var cx, cz int
	var square int
	var rectx, rectz int
	var maxProcs = runtime.GOMAXPROCS(0)
	var prt bool
	var solidSides bool
	var mtlNumber bool

	var defaultObjOutFilename = "a.obj"
	var defaultPrtOutFilename = "a.prt"

	var outFilename string
	flag.IntVar(&maxProcs, "cpu", maxProcs, "Number of cores to use")
	flag.StringVar(&outFilename, "o", defaultObjOutFilename, "Name for output file")
	flag.IntVar(&yMin, "y", 0, "Omit all blocks below this height. 63 is sea level")
	flag.BoolVar(&solidSides, "sides", false, "Solid sides, rather than showing underground")
	flag.BoolVar(&blockFaces, "bf", false, "Don't combine adjacent faces of the same block within a column")
	flag.BoolVar(&hideBottom, "hb", false, "Hide bottom of world")
	flag.BoolVar(&noColor, "g", false, "Omit materials")
	flag.Float64Var(&bx, "x", 0, "Center x coordinate in blocks")
	flag.Float64Var(&bz, "z", 0, "Center z coordinate in blocks")
	flag.IntVar(&cx, "cx", 0, "Center x coordinate in chunks")
	flag.IntVar(&cz, "cz", 0, "Center z coordinate in chunks")
	flag.IntVar(&square, "s", math.MaxInt32, "Chunk square size")
	flag.IntVar(&rectx, "rx", math.MaxInt32, "Width(x) of rectangle size")
	flag.IntVar(&rectz, "rz", math.MaxInt32, "Height(z) of rectangle size")
	flag.IntVar(&faceLimit, "fk", math.MaxInt32, "Face limit (thousands of faces)")
	flag.BoolVar(&prt, "prt", false, "Write out PRT file instead of Obj file")
	flag.BoolVar(&mtlNumber, "mtlnum", false, "Number materials instead of using names")
	var showHelp = flag.Bool("h", false, "Show Help")
	flag.Parse()

	runtime.GOMAXPROCS(maxProcs)
	fmt.Printf("mcobj %v (cpu: %d) Copyright (c) 2011 Jonathan Wright\n", version, runtime.GOMAXPROCS(0))

	if *showHelp || flag.NArg() == 0 {
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Usage: mcobj -cpu 4 -s 20 -o world1.obj", exampleWorldPath)
		fmt.Fprintln(os.Stderr)
		flag.PrintDefaults()
		return
	}

	if faceLimit != math.MaxInt32 {
		faceLimit *= 1000
	}

	if mtlNumber {
		MaterialNamer = new(NumberBlockIdNamer)
	} else {
		MaterialNamer = new(NameBlockIdNamer)
	}

	switch {
	case bx == 0 && cx == 0:
		cx = 0
	case cx == 0:
		cx = int(math.Floor(bx / 16))
	}

	switch {
	case bz == 0 && cz == 0:
		cz = 0
	case cz == 0:
		cz = int(math.Floor(bz / 16))
	}

	if square != math.MaxInt32 {
		chunkLimit = square * square
		var h = square / 2
		chunkMask = &RectangeChunkMask{cx - h, cz - h, cx - h + square, cz - h + square}
	} else if rectx != math.MaxInt32 || rectz != math.MaxInt32 {
		switch {
		case rectx != math.MaxInt32 && rectz != math.MaxInt32:
			chunkLimit = rectx * rectz
			var (
				hx = rectx / 2
				hz = rectz / 2
			)
			chunkMask = &RectangeChunkMask{cx - hx, cz - hz, cx - hx + rectx, cz - hz + rectz}
		case rectx != math.MaxInt32:
			chunkLimit = math.MaxInt32
			var hx = rectx / 2
			chunkMask = &RectangeChunkMask{cx - hx, math.MinInt32, cx - hx + rectx, math.MaxInt32}
		case rectz != math.MaxInt32:
			chunkLimit = math.MaxInt32
			var hz = rectz / 2
			chunkMask = &RectangeChunkMask{math.MinInt32, cz - hz, math.MaxInt32, cz - hz + rectz}
		}
	} else {
		chunkLimit = math.MaxInt32
		chunkMask = &AllChunksMask{}
	}

	if prt && outFilename == defaultObjOutFilename {
		outFilename = defaultPrtOutFilename
	}

	if solidSides {
		defaultSide = &emptySide
	}

	{
		var dir, _ = filepath.Split(strings.Replace(os.Args[0], "\\", "/", -1))
		var jsonError = loadBlockTypesJson(filepath.Join(dir, "blocks.json"))
		if jsonError != nil {
			fmt.Fprintln(os.Stderr, jsonError)
			return
		}
	}

	for i := 0; i < flag.NArg(); i++ {
		var dirpath = flag.Arg(i)
		var fi, err = os.Stat(dirpath)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		} else if !fi.IsDirectory() {
			fmt.Fprintln(os.Stderr, dirpath, "is not a directory")
		}

		var world = OpenWorld(dirpath, chunkMask)
		var pool, poolErr = world.ChunkPool()
		if poolErr != nil {
			fmt.Println(poolErr)
			continue
		}

		var generator OutputGenerator
		if prt {
			generator = new(PrtGenerator)
		} else {
			generator = new(ObjGenerator)
		}
		var boundary = new(BoundaryLocator)
		boundary.Init()
		generator.Start(outFilename, pool.Remaining(), maxProcs, boundary)

		if walkEnclosedChunks(pool, world, cx, cz, generator.GetEnclosedJobsChan()) {
			<-generator.GetCompleteChan()
		}

		generator.Close()
	}
}

type OutputGenerator interface {
	Start(outFilename string, total int, maxProcs int, boundary *BoundaryLocator)
	GetEnclosedJobsChan() chan *EnclosedChunkJob
	GetCompleteChan() chan bool
	Close()
}

type EnclosedChunkJob struct {
	last     bool
	enclosed *EnclosedChunk
}

func walkEnclosedChunks(pool ChunkPool, opener ChunkOpener, cx, cz int, enclosedsChan chan *EnclosedChunkJob) bool {
	var (
		sideCache = new(SideCache)
		started   = false
	)

	for i := 0; moreChunks(pool.Remaining()); i++ {
		for x := 0; x < i && moreChunks(pool.Remaining()); x++ {
			for z := 0; z < i && moreChunks(pool.Remaining()); z++ {
				var (
					ax = cx + unzigzag(x)
					az = cz + unzigzag(z)
				)

				if pool.Pop(ax, az) {
					loadSide(sideCache, opener, ax-1, az)
					loadSide(sideCache, opener, ax+1, az)
					loadSide(sideCache, opener, ax, az-1)
					loadSide(sideCache, opener, ax, az+1)

					var chunk, loadErr = loadChunk2(opener, ax, az)
					if loadErr != nil {
						fmt.Println(loadErr)
					} else {
						var enclosed = sideCache.EncloseChunk(chunk)
						sideCache.AddChunk(chunk)
						chunkCount++
						enclosedsChan <- &EnclosedChunkJob{!moreChunks(pool.Remaining()), enclosed}
						started = true
					}
				}
			}
		}
	}

	return started
}

type Blocks []uint16

type BlockColumn []uint16

func (b *Blocks) Get(x, y, z int) uint16 {
	return (*b)[y+(z*128+(x*128*16))]
}

func (b *Blocks) Column(x, z int) BlockColumn {
	var i = 128 * (z + x*16)
	return BlockColumn((*b)[i : i+128])
}

func zigzag(n int) int {
	return (n << 1) ^ (n >> 31)
}

func unzigzag(n int) int {
	return (n >> 1) ^ (-(n & 1))
}

func moreChunks(unprocessedCount int) bool {
	return unprocessedCount > 0 && faceCount < faceLimit && chunkCount < chunkLimit
}

func loadChunk(filename string) (*nbt.Chunk, os.Error) {
	var file, fileErr = os.Open(filename)
	defer file.Close()
	if fileErr != nil {
		return nil, fileErr
	}
	var chunk, err = nbt.ReadDat(file)
	if err == os.EOF {
		err = nil
	}
	return chunk, err
}

func loadChunk2(opener ChunkOpener, x, z int) (*nbt.Chunk, os.Error) {
	var r, openErr = opener.OpenChunk(x, z)
	if openErr != nil {
		return nil, openErr
	}
	defer r.Close()

	var chunk, nbtErr = nbt.ReadNbt(r)
	if nbtErr != nil {
		return nil, nbtErr
	}
	return chunk, nil
}

func loadSide(sideCache *SideCache, opener ChunkOpener, x, z int) {
	if !sideCache.HasSide(x, z) && !chunkMask.IsMasked(x, z) {
		var chunk, loadErr = loadChunk2(opener, x, z)
		if loadErr != nil {
			fmt.Println(loadErr)
		} else {
			sideCache.AddChunk(chunk)
		}
	}
}

func loadBlockTypesJson(filename string) os.Error {
	var jsonBytes, jsonIoError = ioutil.ReadFile(filename)

	if jsonIoError != nil {
		return jsonIoError
	}

	var f interface{}
	var unmarshalError = json.Unmarshal(jsonBytes, &f)
	if unmarshalError != nil {
		return unmarshalError
	}

	var lines, linesOk = f.([]interface{})
	if linesOk {
		for _, line := range lines {
			var fields, fieldsOk = line.(map[string]interface{})
			if fieldsOk {
				var (
					blockId      byte
					data         byte = 255
					dataArray    []byte
					name         string
					mass         SingularOrAggregate = Mass
					transparency Transparency        = Opaque
					empty        bool                = false
					color        uint32
				)
				for k, v := range fields {
					switch k {
					case "name":
						name = v.(string)
					case "color":
						switch len(v.(string)) {
						case 7:
							var n, numErr = strconv.Btoui64(v.(string)[1:], 16)
							if numErr == nil {
								color = uint32(n*0x100 + 0xff)
							}
						case 9:
							var n, numErr = strconv.Btoui64(v.(string)[1:], 16)
							if numErr == nil {
								color = uint32(n)
							}
						}
					case "blockId":
						blockId = byte(v.(float64))
					case "data":
						switch d := v.(type) {
						case float64:
							data = byte(d)
						case []interface{}:
							dataArray = make([]byte, len(d))
							for i, value := range d {
								dataArray[i] = byte(value.(float64))
							}
						}
					case "item":
						if v.(bool) {
							mass = Item
							transparency = Transparent
						} else {
							mass = Mass
							transparency = Opaque
						}
					case "transparent":
						if v.(bool) {
							transparency = Transparent
						} else {
							transparency = Opaque
						}
					case "empty":
						if v.(bool) {
							empty = true
							transparency = Transparent
							mass = Mass
						} else {
							empty = false
						}
					}
				}

				blockTypeMap[blockId] = &BlockType{blockId, mass, transparency, empty}
				if dataArray == nil {
					if data != 255 {
						extraData[blockId] = true
						colors = append(colors, MTL{blockId, data, color, name})
					} else {
						colors[blockId] = MTL{blockId, data, color, name}
					}
				} else {
					extraData[blockId] = true
					for _, data = range dataArray {
						colors = append(colors, MTL{blockId, data, color, fmt.Sprintf("%s_%d", name, data)})
					}
				}
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package fscache

import (
	"bytes"
	"errors"
	"io"
	"os"
	"sync"
	"time"

	"gopkg.in/djherbis/stream.v1"
)

type memFS struct {
	mu    sync.RWMutex
	files map[string]*memFile
}

// NewMemFs creates an in-memory FileSystem.
// It does not support persistence (Reload is a nop).
func NewMemFs() FileSystem {
	return &memFS{
		files: make(map[string]*memFile),
	}
}

func (fs *memFS) Stat(name string) (FileInfo, error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()
	f, ok := fs.files[name]
	if !ok {
		return FileInfo{}, errors.New("file has not been read")
	}

	size, err := fs.size(name)
	if err != nil {
		return FileInfo{}, err
	}

	return FileInfo{
		name:     name,
		size:     size,
		fileMode: os.ModeIrregular,
		isDir:    false,
		sys:      nil,
		rt:       f.rt,
		wt:       f.wt,
	}, nil
}

func (fs *memFS) size(name string) (int64, error) {
	f, ok := fs.files[name]
	if ok {
		return int64(len(f.Bytes())), nil
	}
	return 0, errors.New("file has not been read")
}

func (fs *memFS) Reload(add func(key, name string)) error {
	return nil
}

func (fs *memFS) Create(key string) (stream.File, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if _, ok := fs.files[key]; ok {
		return nil, errors.New("file exists")
	}
	file := &memFile{
		name: key,
		r:    bytes.NewBuffer(nil),
		wt:   time.Now(),
	}
	file.memReader.memFile = file
	fs.files[key] = file
	return file, nil
}

func (fs *memFS) Open(name string) (stream.File, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if f, ok := fs.files[name]; ok {
		f.rt = time.Now()
		return &memReader{memFile: f}, nil
	}
	return nil, errors.New("file does not exist")
}

func (fs *memFS) Remove(key string) error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	delete(fs.files, key)
	return nil
}

func (fs *memFS) RemoveAll() error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	fs.files = make(map[string]*memFile)
	return nil
}

type memFile struct {
	mu   sync.RWMutex
	name string
	r    *bytes.Buffer
	memReader
	rt, wt time.Time
}

func (f *memFile) Name() string {
	return f.name
}

func (f *memFile) Write(p []byte) (int, error) {
	if len(p) > 0 {
		f.mu.Lock()
		defer f.mu.Unlock()
		return f.r.Write(p)
	}
	return len(p), nil
}

func (f *memFile) Bytes() []byte {
	f.mu.RLock()
	defer f.mu.RUnlock()
	return f.r.Bytes()
}

func (f *memFile) Close() error {
	return nil
}

type memReader struct {
	*memFile
	n int
}

func (r *memReader) ReadAt(p []byte, off int64) (n int, err error) {
	data := r.Bytes()
	if int64(len(data)) < off {
		return 0, io.EOF
	}
	n, err = bytes.NewReader(data[off:]).ReadAt(p, 0)
	return n, err
}

func (r *memReader) Read(p []byte) (n int, err error) {
	n, err = bytes.NewReader(r.Bytes()[r.n:]).Read(p)
	r.n += n
	return n, err
}

func (r *memReader) Close() error {
	return nil
}
<commit_msg>Remove redudant `memFs.size` method<commit_after>package fscache

import (
	"bytes"
	"errors"
	"io"
	"os"
	"sync"
	"time"

	"gopkg.in/djherbis/stream.v1"
)

type memFS struct {
	mu    sync.RWMutex
	files map[string]*memFile
}

// NewMemFs creates an in-memory FileSystem.
// It does not support persistence (Reload is a nop).
func NewMemFs() FileSystem {
	return &memFS{
		files: make(map[string]*memFile),
	}
}

func (fs *memFS) Stat(name string) (FileInfo, error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()
	f, ok := fs.files[name]
	if !ok {
		return FileInfo{}, errors.New("file has not been read")
	}

	size := int64(len(f.Bytes()))

	return FileInfo{
		name:     name,
		size:     size,
		fileMode: os.ModeIrregular,
		isDir:    false,
		sys:      nil,
		rt:       f.rt,
		wt:       f.wt,
	}, nil
}

func (fs *memFS) Reload(add func(key, name string)) error {
	return nil
}

func (fs *memFS) Create(key string) (stream.File, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if _, ok := fs.files[key]; ok {
		return nil, errors.New("file exists")
	}
	file := &memFile{
		name: key,
		r:    bytes.NewBuffer(nil),
		wt:   time.Now(),
	}
	file.memReader.memFile = file
	fs.files[key] = file
	return file, nil
}

func (fs *memFS) Open(name string) (stream.File, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if f, ok := fs.files[name]; ok {
		f.rt = time.Now()
		return &memReader{memFile: f}, nil
	}
	return nil, errors.New("file does not exist")
}

func (fs *memFS) Remove(key string) error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	delete(fs.files, key)
	return nil
}

func (fs *memFS) RemoveAll() error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	fs.files = make(map[string]*memFile)
	return nil
}

type memFile struct {
	mu   sync.RWMutex
	name string
	r    *bytes.Buffer
	memReader
	rt, wt time.Time
}

func (f *memFile) Name() string {
	return f.name
}

func (f *memFile) Write(p []byte) (int, error) {
	if len(p) > 0 {
		f.mu.Lock()
		defer f.mu.Unlock()
		return f.r.Write(p)
	}
	return len(p), nil
}

func (f *memFile) Bytes() []byte {
	f.mu.RLock()
	defer f.mu.RUnlock()
	return f.r.Bytes()
}

func (f *memFile) Close() error {
	return nil
}

type memReader struct {
	*memFile
	n int
}

func (r *memReader) ReadAt(p []byte, off int64) (n int, err error) {
	data := r.Bytes()
	if int64(len(data)) < off {
		return 0, io.EOF
	}
	n, err = bytes.NewReader(data[off:]).ReadAt(p, 0)
	return n, err
}

func (r *memReader) Read(p []byte) (n int, err error) {
	n, err = bytes.NewReader(r.Bytes()[r.n:]).Read(p)
	r.n += n
	return n, err
}

func (r *memReader) Close() error {
	return nil
}
<|endoftext|>


<commit_before>package grpsink

import (
	"context"
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/sirupsen/logrus"
	"github.com/stripe/veneur/protocol"
	"github.com/stripe/veneur/sinks"
	"github.com/stripe/veneur/ssf"
	"github.com/stripe/veneur/trace"
	"github.com/stripe/veneur/trace/metrics"
	"google.golang.org/grpc"
	"google.golang.org/grpc/connectivity"
	"google.golang.org/grpc/status"
)

// GRPCStreamingSpanSink is a sink that streams spans to a configurable target
// service over gRPC. The sink is only tied to the grpc_sink.proto definition of
// a SpanSink service, and thus is generic with respect to the specific server
// it is connecting to.
type GRPCStreamingSpanSink struct {
	name                 string
	target               string
	grpcConn             *grpc.ClientConn
	sinkClient           SpanSinkClient
	stream               SpanSink_SendSpansClient
	streamMut            sync.Mutex
	stats                *statsd.Client
	commonTags           map[string]string
	traceClient          *trace.Client
	log                  *logrus.Logger
	sentCount, dropCount uint32 // Total counts of sent and errors, respectively
	bad                  uint32 // If 1, indicates that the current stream is dead (has seen an EOF)
}

var _ sinks.SpanSink = &GRPCStreamingSpanSink{}

// NewGRPCStreamingSpanSink creates a sinks.SpanSink that can write to any
// compliant gRPC server.
//
// The target parameter should be of the "host:port"; the name parameter is
// prepended with "grpc-", and is used when reporting logs in order to permit
// differentiation between various services.
//
// Any grpc.CallOpts that are provided will be used while first establishing the
// connection to the target server (in grpc.DialContext()).
func NewGRPCStreamingSpanSink(ctx context.Context, target, name string, commonTags map[string]string, log *logrus.Logger, opts ...grpc.DialOption) (*GRPCStreamingSpanSink, error) {
	name = "grpc-" + name
	// We want the stream in fail-fast mode. This is the default, but it's
	// important to the design, so it's worth being explicit: if the streams
	// weren't in fail-fast mode, then Ingest() calls will block while connections
	// get re-established in the background, resulting in undesirable backpressure.
	// For a use case like this sink, unequivocally preferred to fail fast instead,
	// resulting in dropped spans.
	dco := grpc.WithDefaultCallOptions(grpc.FailFast(true))
	conn, err := grpc.DialContext(ctx, target, append(opts, dco)...)
	if err != nil {
		log.WithError(err).WithFields(logrus.Fields{
			"name":   name,
			"target": target,
		}).Error("Error establishing connection to gRPC server")
		return nil, err
	}

	return &GRPCStreamingSpanSink{
		grpcConn:   conn,
		name:       name,
		target:     target,
		sinkClient: NewSpanSinkClient(conn),
		commonTags: commonTags,
		log:        log,
	}, nil
}

// Start performs final preparations on the sink before it is
// ready to begin ingesting spans.
func (gs *GRPCStreamingSpanSink) Start(cl *trace.Client) error {
	gs.traceClient = cl

	var err error
	gs.stream, err = gs.sinkClient.SendSpans(context.TODO())
	if err != nil {
		gs.log.WithFields(logrus.Fields{
			"name":          gs.name,
			"target":        gs.target,
			"chanstate":     gs.grpcConn.GetState().String(),
			logrus.ErrorKey: err,
		}).Error("Failed to set up a stream over gRPC channel to sink target")
		return err
	}

	go gs.maintainStream(context.TODO())
	return nil
}

// maintainStream is intended to be run in a background goroutine. It is kicked
// off by Start(), and is only kept as a standalone method for testing purposes.
//
// This method runs an endless loop that reacts to state changes in the
// underlying channel by automatically attempting to re-establish the stream
// connection when it moves back into the 'READY' state. See
// https://github.com/grpc/grpc/blob/master/doc/connectivity-semantics-and-api.mdA
// for details about gRPC's connectivity state machine.
func (gs *GRPCStreamingSpanSink) maintainStream(ctx context.Context) {
	for {
		// This call will block on a channel receive until the gRPC connection
		// becomes unhealthy. Then, spring into action!
		state := gs.grpcConn.GetState()
		switch state {
		case connectivity.Idle:
			// gRPC considers an open stream to be an active RPC, and channels
			// normally only go idle if there are no active RPC within the
			// channel's timeout window. It should be nigh-impossible for that
			// to happen here, so we record entering the Idle state as an error.
			gs.log.WithFields(logrus.Fields{
				"name":          gs.name,
				"target":        gs.target,
				"chanstate":     gs.grpcConn.GetState().String(),
				logrus.ErrorKey: fmt.Errorf("gRPC sink became idle; should be nearly impossible"),
			}).Error("gRPC channel transitioned to idle")

			// With the error sent, now fall through to recreating the stream
			// as if the channel were ready, as stream creation should force
			// the channel back out of idle. This isn't ideal - it'll end up
			// recreating the stream again after re-reaching READY - but it's
			// not a huge problem, as this case is essentially unreachable.
			fallthrough
		case connectivity.Ready:
			gs.streamMut.Lock()
			stream, err := gs.sinkClient.SendSpans(ctx)
			if err != nil {
				// This is a weird case and probably shouldn't be reachable, but
				// if stream setup fails after recovery, then just wait for a
				// second and try again.
				gs.log.WithFields(logrus.Fields{
					"name":          gs.name,
					"target":        gs.target,
					"chanstate":     gs.grpcConn.GetState().String(),
					logrus.ErrorKey: err,
				}).Error("Failed to set up a new stream after gRPC channel recovered")
				gs.streamMut.Unlock()
				// Wait for 1s before re-attempting to set up the stream.
				time.Sleep(1 * time.Second)
				continue
			}
			// CAS inside the mutex so that we can't ever get a double-send of the
			// "first time channel degradation" log error.
			atomic.CompareAndSwapUint32(&gs.bad, 1, 0)
			gs.stream = stream
			gs.streamMut.Unlock()
			gs.log.WithFields(logrus.Fields{
				"name":      gs.name,
				"target":    gs.target,
				"chanstate": gs.grpcConn.GetState().String(),
			}).Info("gRPC channel and stream re-established")
		case connectivity.Connecting:
			gs.log.WithFields(logrus.Fields{
				"name":      gs.name,
				"target":    gs.target,
				"chanstate": state,
			}).Info("Attempting to re-establish gRPC channel connection")
			// Nothing to do here except wait.
		case connectivity.TransientFailure:
			gs.log.WithFields(logrus.Fields{
				"name":      gs.name,
				"target":    gs.target,
				"chanstate": state,
			}).Warn("gRPC channel now in transient failure")
		case connectivity.Shutdown:
			// The current design has no path to actually shutting down the
			// connection, so this should be unreachable.
			return
		}
		gs.grpcConn.WaitForStateChange(context.Background(), state)
	}
}

// Name returns this sink's name. As the gRPC sink is generic, it's expected
// that this is set via configuration and injected.
func (gs *GRPCStreamingSpanSink) Name() string {
	return gs.name
}

// Ingest takes in a span and streams it over gRPC to the connected server.
func (gs *GRPCStreamingSpanSink) Ingest(ssfSpan *ssf.SSFSpan) error {
	if err := protocol.ValidateTrace(ssfSpan); err != nil {
		return err
	}
	// TODO(sdboyer) validation of span, e.g. time bounds like in datadog sink?

	// Apply any common tags, superseding defaults passed in at sink creation
	// in the event of overlap.
	for k, v := range gs.commonTags {
		ssfSpan.Tags[k] = v
	}

	// gRPC indicates that it is safe to send and receive on a stream at the
	// same time, but it is not safe to simultaneously send. Guard against that
	// with a mutex.
	gs.streamMut.Lock()
	err := gs.stream.Send(ssfSpan)
	gs.streamMut.Unlock()

	if err != nil {
		atomic.AddUint32(&gs.dropCount, 1)

		// gRPC guarantees that an error returned from an RPC call will be of
		// type status.Status. In the unexpected event that they're not, this
		// call creates a dummy type, so there's no risk of panic.
		serr := status.Convert(err)
		state := gs.grpcConn.GetState()

		if err == io.EOF {
			// EOF should be a terminal state for the stream; we only need to
			// log that we've seen it once.
			if atomic.CompareAndSwapUint32(&gs.bad, 0, 1) {
				gs.log.WithFields(logrus.Fields{
					logrus.ErrorKey: err,
					"target":        gs.target,
					"name":          gs.name,
					"chanstate":     state.String(),
				}).Warn("Target server closed the span stream")
			}
		} else {
			// Errors that are not io.EOF are more interesting and, presumably,
			// ephemeral. It's worth logging each one of them.
			gs.log.WithFields(logrus.Fields{
				logrus.ErrorKey: err,
				"target":        gs.target,
				"name":          gs.name,
				"chanstate":     state.String(),
				"code":          serr.Code(),
				"details":       serr.Details(),
				"message":       serr.Message(),
			}).Warn("Error while streaming trace to gRPC target server")
		}
	} else {
		atomic.AddUint32(&gs.sentCount, 1)
	}

	return err
}

// Flush reports metrics about the number of failed and successful
//
// No data is sent to the target sink on Flush(); this sink operates
// entirely over a stream on Ingest().
func (gs *GRPCStreamingSpanSink) Flush() {
	samples := &ssf.Samples{}
	samples.Add(
		ssf.Count(
			sinks.MetricKeyTotalSpansFlushed,
			float32(atomic.LoadUint32(&gs.sentCount)),
			map[string]string{"sink": gs.Name()}),
		ssf.Count(
			sinks.MetricKeyTotalSpansDropped,
			float32(atomic.LoadUint32(&gs.dropCount)),
			map[string]string{"sink": gs.Name()},
		),
	)

	metrics.Report(gs.traceClient, samples)
	return
}
<commit_msg>Comment improvements<commit_after>package grpsink

import (
	"context"
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/sirupsen/logrus"
	"github.com/stripe/veneur/protocol"
	"github.com/stripe/veneur/sinks"
	"github.com/stripe/veneur/ssf"
	"github.com/stripe/veneur/trace"
	"github.com/stripe/veneur/trace/metrics"
	"google.golang.org/grpc"
	"google.golang.org/grpc/connectivity"
	"google.golang.org/grpc/status"
)

// GRPCStreamingSpanSink is a sink that streams spans to a configurable target
// service over gRPC. The sink is only tied to the grpc_sink.proto definition of
// a SpanSink service, and thus is generic with respect to the specific server
// it is connecting to.
type GRPCStreamingSpanSink struct {
	name   string
	target string
	// The underlying gRPC connection ("channel") to the target server.
	grpcConn *grpc.ClientConn
	// The gRPC client that can generate stream connections.
	sinkClient SpanSinkClient
	stream     SpanSink_SendSpansClient
	// streamMut guards two classes of interaction with the stream: actually
	// sending over the stream (gRPC requires that only one goroutine send on
	// a stream at a time), and automatically recreating the stream after a
	// connection drops.
	streamMut sync.Mutex
	// Total counts of sent and dropped spans, respectively
	sentCount, dropCount uint32
	// If 1, indicates that the current stream is dead (has seen an EOF)
	bad         uint32
	stats       *statsd.Client
	commonTags  map[string]string
	traceClient *trace.Client
	log         *logrus.Logger
}

var _ sinks.SpanSink = &GRPCStreamingSpanSink{}

// NewGRPCStreamingSpanSink creates a sinks.SpanSink that can write to any
// compliant gRPC server.
//
// The target parameter should be of the "host:port"; the name parameter is
// prepended with "grpc-", and is used when reporting logs in order to permit
// differentiation between various services.
//
// Any grpc.CallOpts that are provided will be used while first establishing the
// connection to the target server (in grpc.DialContext()).
func NewGRPCStreamingSpanSink(ctx context.Context, target, name string, commonTags map[string]string, log *logrus.Logger, opts ...grpc.DialOption) (*GRPCStreamingSpanSink, error) {
	name = "grpc-" + name
	// We want the stream in fail-fast mode. This is the default, but it's
	// important to the design, so it's worth being explicit: if the streams
	// weren't in fail-fast mode, then Ingest() calls will block while connections
	// get re-established in the background, resulting in undesirable backpressure.
	// For a use case like this sink, unequivocally preferred to fail fast instead,
	// resulting in dropped spans.
	dco := grpc.WithDefaultCallOptions(grpc.FailFast(true))
	conn, err := grpc.DialContext(ctx, target, append(opts, dco)...)
	if err != nil {
		log.WithError(err).WithFields(logrus.Fields{
			"name":   name,
			"target": target,
		}).Error("Error establishing connection to gRPC server")
		return nil, err
	}

	return &GRPCStreamingSpanSink{
		grpcConn:   conn,
		name:       name,
		target:     target,
		sinkClient: NewSpanSinkClient(conn),
		commonTags: commonTags,
		log:        log,
	}, nil
}

// Start performs final preparations on the sink before it is
// ready to begin ingesting spans.
func (gs *GRPCStreamingSpanSink) Start(cl *trace.Client) error {
	gs.traceClient = cl

	var err error
	gs.stream, err = gs.sinkClient.SendSpans(context.TODO())
	if err != nil {
		gs.log.WithFields(logrus.Fields{
			"name":          gs.name,
			"target":        gs.target,
			"chanstate":     gs.grpcConn.GetState().String(),
			logrus.ErrorKey: err,
		}).Error("Failed to set up a stream over gRPC channel to sink target")
		return err
	}

	go gs.maintainStream(context.TODO())
	return nil
}

// maintainStream is intended to be run in a background goroutine. It is kicked
// off by Start(), and is only kept as a standalone method for testing purposes.
//
// This method runs an endless loop that reacts to state changes in the
// underlying channel by automatically attempting to re-establish the stream
// connection when it moves back into the 'READY' state. See
// https://github.com/grpc/grpc/blob/master/doc/connectivity-semantics-and-api.mdA
// for details about gRPC's connectivity state machine.
func (gs *GRPCStreamingSpanSink) maintainStream(ctx context.Context) {
	for {
		// This call will block on a channel receive until the gRPC connection
		// becomes unhealthy. Then, spring into action!
		state := gs.grpcConn.GetState()
		switch state {
		case connectivity.Idle:
			// gRPC considers an open stream to be an active RPC, and channels
			// normally only go idle if there are no active RPC within the
			// channel's timeout window. It should be nigh-impossible for that
			// to happen here, so we record entering the Idle state as an error.
			gs.log.WithFields(logrus.Fields{
				"name":          gs.name,
				"target":        gs.target,
				"chanstate":     gs.grpcConn.GetState().String(),
				logrus.ErrorKey: fmt.Errorf("gRPC sink became idle; should be nearly impossible"),
			}).Error("gRPC channel transitioned to idle")

			// With the error sent, now fall through to recreating the stream
			// as if the channel were ready, as stream creation should force
			// the channel back out of idle. This isn't ideal - it'll end up
			// recreating the stream again after re-reaching READY - but it's
			// not a huge problem, as this case is essentially unreachable.
			fallthrough
		case connectivity.Ready:
			gs.streamMut.Lock()
			stream, err := gs.sinkClient.SendSpans(ctx)
			if err != nil {
				// This is a weird case and probably shouldn't be reachable, but
				// if stream setup fails after recovery, then just wait and retry.
				gs.log.WithFields(logrus.Fields{
					"name":          gs.name,
					"target":        gs.target,
					"chanstate":     gs.grpcConn.GetState().String(),
					logrus.ErrorKey: err,
				}).Error("Failed to set up a new stream after gRPC channel recovered")
				gs.streamMut.Unlock()
				// Wait for 1s before re-attempting to set up the stream.
				time.Sleep(1 * time.Second)
				continue
			}
			// CAS inside the mutex means that it's impossible for for the same
			// stream instance to log an io.EOF twice.
			atomic.CompareAndSwapUint32(&gs.bad, 1, 0)
			gs.stream = stream
			gs.streamMut.Unlock()
			gs.log.WithFields(logrus.Fields{
				"name":      gs.name,
				"target":    gs.target,
				"chanstate": gs.grpcConn.GetState().String(),
			}).Info("gRPC channel and stream re-established")
		case connectivity.Connecting:
			gs.log.WithFields(logrus.Fields{
				"name":      gs.name,
				"target":    gs.target,
				"chanstate": state,
			}).Info("Attempting to re-establish gRPC channel connection")
			// Nothing to do here except wait.
		case connectivity.TransientFailure:
			gs.log.WithFields(logrus.Fields{
				"name":      gs.name,
				"target":    gs.target,
				"chanstate": state,
			}).Warn("gRPC channel now in transient failure")
		case connectivity.Shutdown:
			// The current design has no path to actually shutting down the
			// connection, so this should be unreachable.
			return
		}
		gs.grpcConn.WaitForStateChange(context.Background(), state)
	}
}

// Name returns this sink's name. As the gRPC sink is generic, it's expected
// that this is set via configuration and injected.
func (gs *GRPCStreamingSpanSink) Name() string {
	return gs.name
}

// Ingest takes in a span and streams it over gRPC to the connected server.
func (gs *GRPCStreamingSpanSink) Ingest(ssfSpan *ssf.SSFSpan) error {
	if err := protocol.ValidateTrace(ssfSpan); err != nil {
		return err
	}
	// TODO(sdboyer) validation of span, e.g. time bounds like in datadog sink?

	// Apply any common tags, superseding defaults passed in at sink creation
	// in the event of overlap.
	for k, v := range gs.commonTags {
		ssfSpan.Tags[k] = v
	}

	// gRPC indicates that it is safe to send and receive on a stream at the
	// same time, but it is not safe to simultaneously send. Guard against that
	// with a mutex.
	gs.streamMut.Lock()
	err := gs.stream.Send(ssfSpan)
	gs.streamMut.Unlock()

	if err != nil {
		atomic.AddUint32(&gs.dropCount, 1)

		// gRPC guarantees that an error returned from an RPC call will be of
		// type status.Status. In the unexpected event that they're not, this
		// call creates a dummy type, so there's no risk of panic.
		serr := status.Convert(err)
		state := gs.grpcConn.GetState()

		if err == io.EOF {
			// EOF should be a terminal state for the stream; we only need to
			// log that we've seen it once.
			if atomic.CompareAndSwapUint32(&gs.bad, 0, 1) {
				gs.log.WithFields(logrus.Fields{
					logrus.ErrorKey: err,
					"target":        gs.target,
					"name":          gs.name,
					"chanstate":     state.String(),
				}).Warn("Target server closed the span stream")
			}
		} else {
			// Errors that are not io.EOF are more interesting and, presumably,
			// ephemeral. It's worth logging each one of them.
			gs.log.WithFields(logrus.Fields{
				logrus.ErrorKey: err,
				"target":        gs.target,
				"name":          gs.name,
				"chanstate":     state.String(),
				"code":          serr.Code(),
				"details":       serr.Details(),
				"message":       serr.Message(),
			}).Warn("Error while streaming trace to gRPC target server")
		}
	} else {
		atomic.AddUint32(&gs.sentCount, 1)
	}

	return err
}

// Flush reports total counts of the number of sent and dropped spans over its
// lifetime.
//
// No data is sent to the target sink on Flush(), as this sink operates entirely
// over a stream on Ingest().
func (gs *GRPCStreamingSpanSink) Flush() {
	samples := &ssf.Samples{}
	samples.Add(
		ssf.Count(
			sinks.MetricKeyTotalSpansFlushed,
			float32(atomic.LoadUint32(&gs.sentCount)),
			map[string]string{"sink": gs.Name()}),
		ssf.Count(
			sinks.MetricKeyTotalSpansDropped,
			float32(atomic.LoadUint32(&gs.dropCount)),
			map[string]string{"sink": gs.Name()},
		),
	)

	metrics.Report(gs.traceClient, samples)
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"fmt"
	"time"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/pkg/kubelet/events"
	runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
	"k8s.io/kubernetes/test/e2e/framework"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	imageutils "k8s.io/kubernetes/test/utils/image"
	utilpointer "k8s.io/utils/pointer"

	"github.com/onsi/ginkgo"
)

const (
	// PreconfiguredRuntimeHandler is the name of the runtime handler that is expected to be
	// preconfigured in the test environment.
	PreconfiguredRuntimeHandler = "test-handler"
	// DockerRuntimeHandler is a hardcoded runtime handler that is accepted by dockershim, and
	// treated equivalently to a nil runtime handler.
	DockerRuntimeHandler = "docker"
)

var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() {
	f := framework.NewDefaultFramework("runtimeclass")

	ginkgo.It("should reject a Pod requesting a non-existent RuntimeClass", func() {
		rcName := f.Namespace.Name + "-nonexistent"
		pod := createRuntimeClassPod(f, rcName)
		expectSandboxFailureEvent(f, pod, fmt.Sprintf("\"%s\" not found", rcName))
	})

	ginkgo.It("should reject a Pod requesting a RuntimeClass with an unconfigured handler", func() {
		handler := f.Namespace.Name + "-handler"
		rcName := createRuntimeClass(f, "unconfigured-handler", handler)
		pod := createRuntimeClassPod(f, rcName)
		expectSandboxFailureEvent(f, pod, handler)
	})

	// This test requires that the PreconfiguredRuntimeHandler has already been set up on nodes.
	ginkgo.It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func() {
		// The built-in docker runtime does not support configuring runtime handlers.
		handler := PreconfiguredRuntimeHandler
		if framework.TestContext.ContainerRuntime == "docker" {
			handler = DockerRuntimeHandler
		}

		rcName := createRuntimeClass(f, "preconfigured-handler", handler)
		pod := createRuntimeClassPod(f, rcName)
		expectPodSuccess(f, pod)
	})

	ginkgo.It("should reject a Pod requesting a deleted RuntimeClass", func() {
		rcName := createRuntimeClass(f, "delete-me", "runc")
		rcClient := f.ClientSet.NodeV1beta1().RuntimeClasses()

		ginkgo.By("Deleting RuntimeClass "+rcName, func() {
			err := rcClient.Delete(rcName, nil)
			framework.ExpectNoError(err, "failed to delete RuntimeClass %s", rcName)

			ginkgo.By("Waiting for the RuntimeClass to disappear")
			framework.ExpectNoError(wait.PollImmediate(framework.Poll, time.Minute, func() (bool, error) {
				_, err := rcClient.Get(rcName, metav1.GetOptions{})
				if errors.IsNotFound(err) {
					return true, nil // done
				}
				if err != nil {
					return true, err // stop wait with error
				}
				return false, nil
			}))
		})

		pod := createRuntimeClassPod(f, rcName)
		expectSandboxFailureEvent(f, pod, fmt.Sprintf("\"%s\" not found", rcName))
	})
})

// createRuntimeClass generates a RuntimeClass with the desired handler and a "namespaced" name,
// synchronously creates it, and returns the generated name.
func createRuntimeClass(f *framework.Framework, name, handler string) string {
	uniqueName := fmt.Sprintf("%s-%s", f.Namespace.Name, name)
	rc := runtimeclasstest.NewRuntimeClass(uniqueName, handler)
	rc, err := f.ClientSet.NodeV1beta1().RuntimeClasses().Create(rc)
	framework.ExpectNoError(err, "failed to create RuntimeClass resource")
	return rc.GetName()
}

// createRuntimeClass creates a test pod with the given runtimeClassName.
func createRuntimeClassPod(f *framework.Framework, runtimeClassName string) *v1.Pod {
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
		},
		Spec: v1.PodSpec{
			RuntimeClassName: &runtimeClassName,
			Containers: []v1.Container{{
				Name:    "test",
				Image:   imageutils.GetE2EImage(imageutils.BusyBox),
				Command: []string{"true"},
			}},
			RestartPolicy:                v1.RestartPolicyNever,
			AutomountServiceAccountToken: utilpointer.BoolPtr(false),
		},
	}
	return f.PodClient().Create(pod)
}

// expectPodSuccess waits for the given pod to terminate successfully.
func expectPodSuccess(f *framework.Framework, pod *v1.Pod) {
	framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespace(
		f.ClientSet, pod.Name, f.Namespace.Name))
}

// expectSandboxFailureEvent polls for an event with reason "FailedCreatePodSandBox" containing the
// expected message string.
func expectSandboxFailureEvent(f *framework.Framework, pod *v1.Pod, msg string) {
	eventSelector := fields.Set{
		"involvedObject.kind":      "Pod",
		"involvedObject.name":      pod.Name,
		"involvedObject.namespace": f.Namespace.Name,
		"reason":                   events.FailedCreatePodSandBox,
	}.AsSelector().String()
	framework.ExpectNoError(WaitTimeoutForEvent(
		f.ClientSet, f.Namespace.Name, eventSelector, msg, framework.PodEventTimeout))
}
<commit_msg>Update RuntimeClass E2E expectations<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"fmt"
	"time"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/pkg/kubelet/events"
	runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
	"k8s.io/kubernetes/test/e2e/framework"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	imageutils "k8s.io/kubernetes/test/utils/image"
	utilpointer "k8s.io/utils/pointer"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
)

const (
	// PreconfiguredRuntimeHandler is the name of the runtime handler that is expected to be
	// preconfigured in the test environment.
	PreconfiguredRuntimeHandler = "test-handler"
	// DockerRuntimeHandler is a hardcoded runtime handler that is accepted by dockershim, and
	// treated equivalently to a nil runtime handler.
	DockerRuntimeHandler = "docker"
)

var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() {
	f := framework.NewDefaultFramework("runtimeclass")

	ginkgo.It("should reject a Pod requesting a non-existent RuntimeClass", func() {
		rcName := f.Namespace.Name + "-nonexistent"
		expectPodRejection(f, newRuntimeClassPod(rcName))
	})

	ginkgo.It("should reject a Pod requesting a RuntimeClass with an unconfigured handler", func() {
		handler := f.Namespace.Name + "-handler"
		rcName := createRuntimeClass(f, "unconfigured-handler", handler)
		pod := f.PodClient().Create(newRuntimeClassPod(rcName))
		expectSandboxFailureEvent(f, pod, handler)
	})

	// This test requires that the PreconfiguredRuntimeHandler has already been set up on nodes.
	ginkgo.It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func() {
		// The built-in docker runtime does not support configuring runtime handlers.
		handler := PreconfiguredRuntimeHandler
		if framework.TestContext.ContainerRuntime == "docker" {
			handler = DockerRuntimeHandler
		}

		rcName := createRuntimeClass(f, "preconfigured-handler", handler)
		pod := f.PodClient().Create(newRuntimeClassPod(rcName))
		expectPodSuccess(f, pod)
	})

	ginkgo.It("should reject a Pod requesting a deleted RuntimeClass", func() {
		rcName := createRuntimeClass(f, "delete-me", "runc")
		rcClient := f.ClientSet.NodeV1beta1().RuntimeClasses()

		ginkgo.By("Deleting RuntimeClass "+rcName, func() {
			err := rcClient.Delete(rcName, nil)
			framework.ExpectNoError(err, "failed to delete RuntimeClass %s", rcName)

			ginkgo.By("Waiting for the RuntimeClass to disappear")
			framework.ExpectNoError(wait.PollImmediate(framework.Poll, time.Minute, func() (bool, error) {
				_, err := rcClient.Get(rcName, metav1.GetOptions{})
				if errors.IsNotFound(err) {
					return true, nil // done
				}
				if err != nil {
					return true, err // stop wait with error
				}
				return false, nil
			}))
		})

		expectPodRejection(f, newRuntimeClassPod(rcName))
	})
})

// createRuntimeClass generates a RuntimeClass with the desired handler and a "namespaced" name,
// synchronously creates it, and returns the generated name.
func createRuntimeClass(f *framework.Framework, name, handler string) string {
	uniqueName := fmt.Sprintf("%s-%s", f.Namespace.Name, name)
	rc := runtimeclasstest.NewRuntimeClass(uniqueName, handler)
	rc, err := f.ClientSet.NodeV1beta1().RuntimeClasses().Create(rc)
	framework.ExpectNoError(err, "failed to create RuntimeClass resource")
	return rc.GetName()
}

// newRuntimeClassPod generates a test pod with the given runtimeClassName.
func newRuntimeClassPod(runtimeClassName string) *v1.Pod {
	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
		},
		Spec: v1.PodSpec{
			RuntimeClassName: &runtimeClassName,
			Containers: []v1.Container{{
				Name:    "test",
				Image:   imageutils.GetE2EImage(imageutils.BusyBox),
				Command: []string{"true"},
			}},
			RestartPolicy:                v1.RestartPolicyNever,
			AutomountServiceAccountToken: utilpointer.BoolPtr(false),
		},
	}
}

func expectPodRejection(f *framework.Framework, pod *v1.Pod) {
	// The Node E2E doesn't run the RuntimeClass admission controller, so we expect the rejection to
	// happen by the Kubelet.
	if framework.TestContext.NodeE2E {
		pod = f.PodClient().Create(pod)
		expectSandboxFailureEvent(f, pod, fmt.Sprintf("\"%s\" not found", *pod.Spec.RuntimeClassName))
	} else {
		_, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
		framework.ExpectError(err, "should be forbidden")
		gomega.Expect(apierrs.IsForbidden(err)).To(gomega.BeTrue(), "should be forbidden error")
	}
}

// expectPodSuccess waits for the given pod to terminate successfully.
func expectPodSuccess(f *framework.Framework, pod *v1.Pod) {
	framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespace(
		f.ClientSet, pod.Name, f.Namespace.Name))
}

// expectSandboxFailureEvent polls for an event with reason "FailedCreatePodSandBox" containing the
// expected message string.
func expectSandboxFailureEvent(f *framework.Framework, pod *v1.Pod, msg string) {
	eventSelector := fields.Set{
		"involvedObject.kind":      "Pod",
		"involvedObject.name":      pod.Name,
		"involvedObject.namespace": f.Namespace.Name,
		"reason":                   events.FailedCreatePodSandBox,
	}.AsSelector().String()
	framework.ExpectNoError(WaitTimeoutForEvent(
		f.ClientSet, f.Namespace.Name, eventSelector, msg, framework.PodEventTimeout))
}
<|endoftext|>


<commit_before>package main //for test

import (
	"encoding/hex"
	"fmt"
)

type Header struct {
	Name, Value string
}

func ParseIntRepresentation(buf []byte, N byte) (I int, cursor byte) {
	I = int(buf[0] & ((1 << N) - 1)) // byte could be used as byte
	cursor = 1
	if I < ((1 << N) - 1) {
		return I, cursor
	} else {
		var M byte = 0
		for (buf[cursor] & 0x80) > 0 {
			I += int(buf[cursor]&0x7f) * (1 << M)
			M += 7
			cursor += 1
		}
		I += int(buf[cursor]&0x7f) * (1 << M)
		return I, cursor + 1
	}
}

func ExtractContent(buf []byte, length int, isHuffman bool) (content string) {
	if isHuffman {
		return content
	} else {
		content = string(buf[:length])
		return content
	}
}

func ParseFromByte(buf []byte) (content string, cursor byte) {
	isHuffman := false
	if buf[0]&0x80 > 0 {
		isHuffman = true
	}
	length, cursor := ParseIntRepresentation(buf, 7)
	content = ExtractContent(buf[cursor:], length, isHuffman)
	cursor += byte(length)
	return content, cursor
}

func ParseHeader(index int, table int, buf []byte, isIndexed bool) (name, value string, cursor byte) {
	if c := byte(0); !isIndexed {
		if index == 0 {
			name, c = ParseFromByte(buf[cursor:])
			cursor += c
		}
		value, c = ParseFromByte(buf[cursor:])
		cursor += c
	}

	if index > 0 {
		//get header from table
	}

	return name, value, cursor
}

func decode(wire string) []Header {
	var Headers []Header
	var buf *[]byte
	nums, err := hex.DecodeString(string(wire))
	if err != nil {
		panic(err)
	}
	buf = &nums

	var cursor byte = 0
	for cursor < byte(len(nums)) {
		isIndexed := false
		isIncremental := false
		var index int
		var c byte
		if (*buf)[cursor]&0xe0 == 0x20 {
			// 7.3 Header Table Size Update
			cursor = 1
		} else if ((*buf)[cursor] & 0x80) > 0 {
			// 7.1 Indexed Header Field
			if ((*buf)[cursor] & 0x7f) == 0 {
				panic('a')
			}
			index, c = ParseIntRepresentation((*buf)[cursor:], 7)
			isIndexed = true
		} else {
			if (*buf)[cursor]&0xc0 == 0x40 {
				// 7.2.1 Literal Header Field with Incremental Indexing
				index, c = ParseIntRepresentation((*buf)[cursor:], 6)
				isIncremental = true
			} else if (*buf)[cursor]&0xf0 == 0xf0 {
				// 7.2.3 Literal Header Field never Indexed
				index, c = ParseIntRepresentation((*buf)[cursor:], 4)
			} else {
				// 7.2.2 Literal Header Field without Indexing
				index, c = ParseIntRepresentation((*buf)[cursor:], 4)
			}
		}
		cursor += c

		table := 1 //for test
		name, value, c := ParseHeader(index, table, (*buf)[cursor:], isIndexed)
		cursor += c

		if isIncremental {
			//add to table
		}
		Headers = append(Headers, []Header{{name, value}}...)
	}

	//d := hex.EncodeToString(nums)
	//fmt.Println(d)
	return Headers
}

func main() {
	//nums, _ := hex.DecodeString(string("1FA18DB701"))
	//fmt.Println(nums)
	//fmt.Println(ParseIntRepresentation(nums, 5))
	//decode("ff80000111")
	fmt.Println(decode("00073a6d6574686f640347455400073a736368656d650468747470000a3a617574686f726974790f7777772e7961686f6f2e636f2e6a7000053a70617468012f"))
	fmt.Println()
}
<commit_msg>change return value []Header to []map[string]string<commit_after>//package main //for test
package hpack

import (
	"encoding/hex"
	"fmt"
	"huffman"
)

type Header struct {
	Name, Value string
}

func ParseIntRepresentation(buf []byte, N byte) (I int, cursor byte) {
	I = int(buf[0] & ((1 << N) - 1)) // byte could be used as byte
	cursor = 1
	if I < ((1 << N) - 1) {
		return I, cursor
	} else {
		var M byte = 0
		for (buf[cursor] & 0x80) > 0 {
			I += int(buf[cursor]&0x7f) * (1 << M)
			M += 7
			cursor += 1
		}
		I += int(buf[cursor]&0x7f) * (1 << M)
		return I, cursor + 1
	}
}

func ExtractContent(buf []byte, length int, isHuffman bool) (content string) {
	if isHuffman {
		return content
	} else {
		content = string(buf[:length])
		return content
	}
}

func ParseFromByte(buf []byte) (content string, cursor byte) {
	isHuffman := false
	if buf[0]&0x80 > 0 {
		isHuffman = true
	}
	length, cursor := ParseIntRepresentation(buf, 7)
	content = ExtractContent(buf[cursor:], length, isHuffman)
	cursor += byte(length)
	return content, cursor
}

func ParseHeader(index int, table int, buf []byte, isIndexed bool) (name, value string, cursor byte) {
	if c := byte(0); !isIndexed {
		if index == 0 {
			name, c = ParseFromByte(buf[cursor:])
			cursor += c
		}
		value, c = ParseFromByte(buf[cursor:])
		cursor += c
	}

	if index > 0 {
		//get header from table
	}

	return name, value, cursor
}

func Decode(wire string) (Headers []map[string]string) {
	var buf *[]byte
	nums, err := hex.DecodeString(string(wire))
	if err != nil {
		panic(err)
	}
	buf = &nums

	var cursor byte = 0
	for cursor < byte(len(nums)) {
		isIndexed := false
		isIncremental := false
		var index int
		var c byte
		if (*buf)[cursor]&0xe0 == 0x20 {
			// 7.3 Header Table Size Update
			cursor = 1
		} else if ((*buf)[cursor] & 0x80) > 0 {
			// 7.1 Indexed Header Field
			if ((*buf)[cursor] & 0x7f) == 0 {
				panic('a')
			}
			index, c = ParseIntRepresentation((*buf)[cursor:], 7)
			isIndexed = true
		} else {
			if (*buf)[cursor]&0xc0 == 0x40 {
				// 7.2.1 Literal Header Field with Incremental Indexing
				index, c = ParseIntRepresentation((*buf)[cursor:], 6)
				isIncremental = true
			} else if (*buf)[cursor]&0xf0 == 0xf0 {
				// 7.2.3 Literal Header Field never Indexed
				index, c = ParseIntRepresentation((*buf)[cursor:], 4)
			} else {
				// 7.2.2 Literal Header Field without Indexing
				index, c = ParseIntRepresentation((*buf)[cursor:], 4)
			}
		}
		cursor += c

		table := 1 //for test
		name, value, c := ParseHeader(index, table, (*buf)[cursor:], isIndexed)
		cursor += c

		if isIncremental {
			//add to table
		}
		header := map[string]string{name: value}
		Headers = append(Headers, header)
	}

	//d := hex.EncodeToString(nums)
	//fmt.Println(d)
	return Headers
}

func main() {
	//nums, _ := hex.DecodeString(string("1FA18DB701"))
	//fmt.Println(nums)
	//fmt.Println(ParseIntRepresentation(nums, 5))
	//decode("ff80000111")
	fmt.Println(Decode("00073a6d6574686f640347455400073a736368656d650468747470000a3a617574686f726974790f7777772e7961686f6f2e636f2e6a7000053a70617468012f"))
	fmt.Println(huffman.HUFFMAN_TABLE)
}
<|endoftext|>