<commit_before>/*
 *
 * Copyright 2021 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package priority implements the priority balancer.
//
// This balancer will be kept in internal until we use it in the xds balancers,
// and are confident its functionalities are stable. It will then be exported
// for more users.
package priority

import (
	"fmt"
	"sync"
	"time"

	"google.golang.org/grpc/balancer"
	"google.golang.org/grpc/internal/buffer"
	"google.golang.org/grpc/internal/grpclog"
	"google.golang.org/grpc/internal/grpcsync"
	"google.golang.org/grpc/internal/hierarchy"
	"google.golang.org/grpc/resolver"
	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
)

const priorityBalancerName = "priority_experimental"

func init() {
	balancer.Register(priorityBB{})
}

type priorityBB struct{}

func (priorityBB) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
	b := &priorityBalancer{
		cc:                       cc,
		done:                     grpcsync.NewEvent(),
		childToPriority:          make(map[string]int),
		children:                 make(map[string]*childBalancer),
		childBalancerStateUpdate: buffer.NewUnbounded(),
	}

	b.logger = prefixLogger(b)
	b.bg = balancergroup.New(cc, b, nil, b.logger)
	b.bg.Start()
	go b.run()
	b.logger.Infof("Created")
	return b

}

func (priorityBB) Name() string {
	return priorityBalancerName
}

// timerWrapper wraps a timer with a boolean. So that when a race happens
// between AfterFunc and Stop, the func is guaranteed to not execute.
type timerWrapper struct {
	stopped bool
	timer   *time.Timer
}

type priorityBalancer struct {
	logger                   *grpclog.PrefixLogger
	cc                       balancer.ClientConn
	bg                       *balancergroup.BalancerGroup
	done                     *grpcsync.Event
	childBalancerStateUpdate *buffer.Unbounded

	mu         sync.Mutex
	childInUse string
	// priority of the child that's current in use. Int starting from 0, and 0
	// is the higher priority.
	priorityInUse int
	// priorities is a list of child names from higher to lower priority.
	priorities []string
	// childToPriority is a map from the child name to it's priority. Priority
	// is an int start from 0, and 0 is the higher priority.
	childToPriority map[string]int
	// children is a map from child name to sub-balancers.
	children map[string]*childBalancer
	// The timer to give a priority some time to connect. And if the priority
	// doesn't go into Ready/Failure, the next priority will be started.
	//
	// One timer is enough because there can be at most one priority in init
	// state.
	priorityInitTimer *timerWrapper
}

func (b *priorityBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
	newConfig, ok := s.BalancerConfig.(*lbConfig)
	if !ok {
		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
	}
	addressesSplit := hierarchy.Group(s.ResolverState.Addresses)

	b.mu.Lock()
	defer b.mu.Unlock()
	// Create and remove children, since we know all children from the config
	// are used by some priority.
	for name, newSubConfig := range newConfig.Children {
		bb := balancer.Get(newSubConfig.Config.Name)
		if bb == nil {
			b.logger.Errorf("balancer name %v from config is not registered", newSubConfig.Config.Name)
			continue
		}

		currentChild, ok := b.children[name]
		if !ok {
			// This is a new child, add it to the children list. But note that
			// the balancer isn't built, because this child can be a low
			// priority. If necessary, it will be built when syncing priorities.
			cb := newChildBalancer(name, b, bb)
			cb.updateConfig(newSubConfig.Config.Config, resolver.State{
				Addresses:     addressesSplit[name],
				ServiceConfig: s.ResolverState.ServiceConfig,
				Attributes:    s.ResolverState.Attributes,
			})
			b.children[name] = cb
			continue
		}

		// This is not a new child. But the config/addresses could change.

		// The balancing policy name is changed, close the old child. But don't
		// rebuild, rebuild will happen when syncing priorities.
		if currentChild.bb.Name() != bb.Name() {
			currentChild.stop()
			currentChild.bb = bb
		}

		// Update config and address, but note that this doesn't send the
		// updates to child balancer (the child balancer might not be built, if
		// it's a low priority).
		currentChild.updateConfig(newSubConfig.Config.Config, resolver.State{
			Addresses:     addressesSplit[name],
			ServiceConfig: s.ResolverState.ServiceConfig,
			Attributes:    s.ResolverState.Attributes,
		})
	}

	// Remove child from children if it's not in new config.
	for name, oldChild := range b.children {
		if _, ok := newConfig.Children[name]; !ok {
			oldChild.stop()
		}
	}

	// Update priorities and handle priority changes.
	b.priorities = newConfig.Priorities
	b.childToPriority = make(map[string]int, len(newConfig.Priorities))
	for pi, pName := range newConfig.Priorities {
		b.childToPriority[pName] = pi
	}
	// Sync the states of all children to the new updated priorities. This
	// include starting/stopping child balancers when necessary.
	b.syncPriority()

	return nil
}

func (b *priorityBalancer) ResolverError(err error) {
	b.bg.ResolverError(err)
}

func (b *priorityBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
	b.bg.UpdateSubConnState(sc, state)
}

func (b *priorityBalancer) Close() {
	b.bg.Close()

	b.mu.Lock()
	defer b.mu.Unlock()
	b.done.Fire()
	// Clear states of the current child in use, so if there's a race in picker
	// update, it will be dropped.
	b.childInUse = ""
	b.stopPriorityInitTimer()
}

// stopPriorityInitTimer stops the priorityInitTimer if it's not nil, and set it
// to nil.
//
// Caller must hold b.mu.
func (b *priorityBalancer) stopPriorityInitTimer() {
	timerW := b.priorityInitTimer
	if timerW == nil {
		return
	}
	b.priorityInitTimer = nil
	timerW.stopped = true
	timerW.timer.Stop()
}

// UpdateState implements balancergroup.BalancerStateAggregator interface. The
// balancer group sends new connectivity state and picker here.
func (b *priorityBalancer) UpdateState(childName string, state balancer.State) {
	b.childBalancerStateUpdate.Put(&childBalancerState{
		name: childName,
		s:    state,
	})
}

type childBalancerState struct {
	name string
	s    balancer.State
}

// run handles child update in a separate goroutine, so if the child sends
// updates inline (when called by parent), it won't cause deadlocks (by trying
// to hold the same mutex).
func (b *priorityBalancer) run() {
	for {
		select {
		case u := <-b.childBalancerStateUpdate.Get():
			b.childBalancerStateUpdate.Load()
			s := u.(*childBalancerState)
			// Needs to handle state update in a goroutine, because each state
			// update needs to start/close child policy, could result in
			// deadlock.
			b.handleChildStateUpdate(s.name, s.s)
		case <-b.done.Done():
			return
		}
	}
}
<commit_msg>priority: pass build options to balancergroup (#4202)<commit_after>/*
 *
 * Copyright 2021 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package priority implements the priority balancer.
//
// This balancer will be kept in internal until we use it in the xds balancers,
// and are confident its functionalities are stable. It will then be exported
// for more users.
package priority

import (
	"fmt"
	"sync"
	"time"

	"google.golang.org/grpc/balancer"
	"google.golang.org/grpc/internal/buffer"
	"google.golang.org/grpc/internal/grpclog"
	"google.golang.org/grpc/internal/grpcsync"
	"google.golang.org/grpc/internal/hierarchy"
	"google.golang.org/grpc/resolver"
	"google.golang.org/grpc/xds/internal/balancer/balancergroup"
)

const priorityBalancerName = "priority_experimental"

func init() {
	balancer.Register(priorityBB{})
}

type priorityBB struct{}

func (priorityBB) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
	b := &priorityBalancer{
		cc:                       cc,
		done:                     grpcsync.NewEvent(),
		childToPriority:          make(map[string]int),
		children:                 make(map[string]*childBalancer),
		childBalancerStateUpdate: buffer.NewUnbounded(),
	}

	b.logger = prefixLogger(b)
	b.bg = balancergroup.New(cc, bOpts, b, nil, b.logger)
	b.bg.Start()
	go b.run()
	b.logger.Infof("Created")
	return b

}

func (priorityBB) Name() string {
	return priorityBalancerName
}

// timerWrapper wraps a timer with a boolean. So that when a race happens
// between AfterFunc and Stop, the func is guaranteed to not execute.
type timerWrapper struct {
	stopped bool
	timer   *time.Timer
}

type priorityBalancer struct {
	logger                   *grpclog.PrefixLogger
	cc                       balancer.ClientConn
	bg                       *balancergroup.BalancerGroup
	done                     *grpcsync.Event
	childBalancerStateUpdate *buffer.Unbounded

	mu         sync.Mutex
	childInUse string
	// priority of the child that's current in use. Int starting from 0, and 0
	// is the higher priority.
	priorityInUse int
	// priorities is a list of child names from higher to lower priority.
	priorities []string
	// childToPriority is a map from the child name to it's priority. Priority
	// is an int start from 0, and 0 is the higher priority.
	childToPriority map[string]int
	// children is a map from child name to sub-balancers.
	children map[string]*childBalancer
	// The timer to give a priority some time to connect. And if the priority
	// doesn't go into Ready/Failure, the next priority will be started.
	//
	// One timer is enough because there can be at most one priority in init
	// state.
	priorityInitTimer *timerWrapper
}

func (b *priorityBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
	newConfig, ok := s.BalancerConfig.(*lbConfig)
	if !ok {
		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
	}
	addressesSplit := hierarchy.Group(s.ResolverState.Addresses)

	b.mu.Lock()
	defer b.mu.Unlock()
	// Create and remove children, since we know all children from the config
	// are used by some priority.
	for name, newSubConfig := range newConfig.Children {
		bb := balancer.Get(newSubConfig.Config.Name)
		if bb == nil {
			b.logger.Errorf("balancer name %v from config is not registered", newSubConfig.Config.Name)
			continue
		}

		currentChild, ok := b.children[name]
		if !ok {
			// This is a new child, add it to the children list. But note that
			// the balancer isn't built, because this child can be a low
			// priority. If necessary, it will be built when syncing priorities.
			cb := newChildBalancer(name, b, bb)
			cb.updateConfig(newSubConfig.Config.Config, resolver.State{
				Addresses:     addressesSplit[name],
				ServiceConfig: s.ResolverState.ServiceConfig,
				Attributes:    s.ResolverState.Attributes,
			})
			b.children[name] = cb
			continue
		}

		// This is not a new child. But the config/addresses could change.

		// The balancing policy name is changed, close the old child. But don't
		// rebuild, rebuild will happen when syncing priorities.
		if currentChild.bb.Name() != bb.Name() {
			currentChild.stop()
			currentChild.bb = bb
		}

		// Update config and address, but note that this doesn't send the
		// updates to child balancer (the child balancer might not be built, if
		// it's a low priority).
		currentChild.updateConfig(newSubConfig.Config.Config, resolver.State{
			Addresses:     addressesSplit[name],
			ServiceConfig: s.ResolverState.ServiceConfig,
			Attributes:    s.ResolverState.Attributes,
		})
	}

	// Remove child from children if it's not in new config.
	for name, oldChild := range b.children {
		if _, ok := newConfig.Children[name]; !ok {
			oldChild.stop()
		}
	}

	// Update priorities and handle priority changes.
	b.priorities = newConfig.Priorities
	b.childToPriority = make(map[string]int, len(newConfig.Priorities))
	for pi, pName := range newConfig.Priorities {
		b.childToPriority[pName] = pi
	}
	// Sync the states of all children to the new updated priorities. This
	// include starting/stopping child balancers when necessary.
	b.syncPriority()

	return nil
}

func (b *priorityBalancer) ResolverError(err error) {
	b.bg.ResolverError(err)
}

func (b *priorityBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
	b.bg.UpdateSubConnState(sc, state)
}

func (b *priorityBalancer) Close() {
	b.bg.Close()

	b.mu.Lock()
	defer b.mu.Unlock()
	b.done.Fire()
	// Clear states of the current child in use, so if there's a race in picker
	// update, it will be dropped.
	b.childInUse = ""
	b.stopPriorityInitTimer()
}

// stopPriorityInitTimer stops the priorityInitTimer if it's not nil, and set it
// to nil.
//
// Caller must hold b.mu.
func (b *priorityBalancer) stopPriorityInitTimer() {
	timerW := b.priorityInitTimer
	if timerW == nil {
		return
	}
	b.priorityInitTimer = nil
	timerW.stopped = true
	timerW.timer.Stop()
}

// UpdateState implements balancergroup.BalancerStateAggregator interface. The
// balancer group sends new connectivity state and picker here.
func (b *priorityBalancer) UpdateState(childName string, state balancer.State) {
	b.childBalancerStateUpdate.Put(&childBalancerState{
		name: childName,
		s:    state,
	})
}

type childBalancerState struct {
	name string
	s    balancer.State
}

// run handles child update in a separate goroutine, so if the child sends
// updates inline (when called by parent), it won't cause deadlocks (by trying
// to hold the same mutex).
func (b *priorityBalancer) run() {
	for {
		select {
		case u := <-b.childBalancerStateUpdate.Get():
			b.childBalancerStateUpdate.Load()
			s := u.(*childBalancerState)
			// Needs to handle state update in a goroutine, because each state
			// update needs to start/close child policy, could result in
			// deadlock.
			b.handleChildStateUpdate(s.name, s.s)
		case <-b.done.Done():
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"compress/gzip"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/appc/spec/aci"
	"github.com/appc/spec/pkg/tarheader"
	"github.com/appc/spec/schema"
)

var (
	buildNocompress bool
	buildOverwrite  bool
	cmdBuild        = &Command{
		Name: "build",
		Description: `Build an ACI from a given directory. The directory should
contain an Image Layout. The Image Layout will be validated
before the ACI is created. The produced ACI will be
gzip-compressed by default.`,
		Summary: "Build an ACI from an Image Layout",
		Usage:   `[--overwrite] [--no-compression] DIRECTORY OUTPUT_FILE`,
		Run:     runBuild,
	}
)

func init() {
	cmdBuild.Flags.BoolVar(&buildOverwrite, "overwrite", false, "Overwrite target file if it already exists")
	cmdBuild.Flags.BoolVar(&buildNocompress, "no-compression", false, "Do not gzip-compress the produced ACI")
}

func buildWalker(root string, aw aci.ArchiveWriter) filepath.WalkFunc {
	// cache of inode -> filepath, used to leverage hard links in the archive
	inos := map[uint64]string{}
	return func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		relpath, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		if relpath == "." {
			return nil
		}
		if relpath == aci.ManifestFile {
			// ignore; this will be written by the archive writer
			// TODO(jonboulle): does this make sense? maybe just remove from archivewriter?
			return nil
		}

		link := ""
		var r io.Reader
		switch info.Mode() & os.ModeType {
		case os.ModeDir:
		case os.ModeSymlink:
			target, err := os.Readlink(path)
			if err != nil {
				return err
			}
			link = target
		default:
			file, err := os.Open(path)
			if err != nil {
				return err
			}
			defer file.Close()
			r = file
		}

		hdr, err := tar.FileInfoHeader(info, link)
		if err != nil {
			panic(err)
		}
		// Because os.FileInfo's Name method returns only the base
		// name of the file it describes, it may be necessary to
		// modify the Name field of the returned header to provide the
		// full path name of the file.
		hdr.Name = relpath
		tarheader.Populate(hdr, info, inos)
		// If the file is a hard link to a file we've already seen, we
		// don't need the contents
		if hdr.Typeflag == tar.TypeLink {
			hdr.Size = 0
			r = nil
		}
		if err := aw.AddFile(relpath, hdr, r); err != nil {
			return err
		}

		return nil
	}
}

func runBuild(args []string) (exit int) {
	if len(args) != 2 {
		stderr("build: Must provide directory and output file")
		return 1
	}

	root := args[0]
	tgt := args[1]
	ext := filepath.Ext(tgt)
	if ext != schema.ACIExtension {
		stderr("build: Extension must be %s (given %s)", schema.ACIExtension, ext)
		return 1
	}

	mode := os.O_CREATE | os.O_WRONLY
	if !buildOverwrite {
		mode |= os.O_EXCL
	}
	fh, err := os.OpenFile(tgt, mode, 0644)
	if err != nil {
		if os.IsExist(err) {
			stderr("build: Target file exists (try --overwrite)")
		} else {
			stderr("build: Unable to open target %s: %v", tgt, err)
		}
		return 1
	}

	var gw *gzip.Writer
	var r io.WriteCloser = fh
	if !buildNocompress {
		gw = gzip.NewWriter(fh)
		r = gw
	}
	tr := tar.NewWriter(r)

	defer func() {
		tr.Close()
		if !buildNocompress {
			gw.Close()
		}
		fh.Close()
		if exit != 0 && !buildOverwrite {
			os.Remove(tgt)
		}
	}()

	// TODO(jonboulle): stream the validation so we don't have to walk the rootfs twice
	if err := aci.ValidateLayout(root); err != nil {
		stderr("build: Layout failed validation: %v", err)
		return 1
	}
	mpath := filepath.Join(root, aci.ManifestFile)
	b, err := ioutil.ReadFile(mpath)
	if err != nil {
		stderr("build: Unable to read Image Manifest: %v", err)
		return 1
	}
	var im schema.ImageManifest
	if err := im.UnmarshalJSON(b); err != nil {
		stderr("build: Unable to load Image Manifest: %v", err)
		return 1
	}
	iw := aci.NewImageWriter(im, tr)

	err = filepath.Walk(root, buildWalker(root, iw))
	if err != nil {
		stderr("build: Error walking rootfs: %v", err)
		return 1
	}

	err = iw.Close()
	if err != nil {
		stderr("build: Unable to close image %s: %v", tgt, err)
		return 1
	}

	return
}
<commit_msg>build: support device files, add experimental note<commit_after>package main

import (
	"archive/tar"
	"compress/gzip"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/appc/spec/aci"
	"github.com/appc/spec/pkg/tarheader"
	"github.com/appc/spec/schema"
)

var (
	buildNocompress bool
	buildOverwrite  bool
	cmdBuild        = &Command{
		Name: "build",
		Description: `Build an ACI from a given directory. The directory should
contain an Image Layout. The Image Layout will be validated
before the ACI is created. The produced ACI will be
gzip-compressed by default.`,
		Summary: "Build an ACI from an Image Layout (experimental)",
		Usage:   `[--overwrite] [--no-compression] DIRECTORY OUTPUT_FILE`,
		Run:     runBuild,
	}
)

func init() {
	cmdBuild.Flags.BoolVar(&buildOverwrite, "overwrite", false, "Overwrite target file if it already exists")
	cmdBuild.Flags.BoolVar(&buildNocompress, "no-compression", false, "Do not gzip-compress the produced ACI")
}

func buildWalker(root string, aw aci.ArchiveWriter) filepath.WalkFunc {
	// cache of inode -> filepath, used to leverage hard links in the archive
	inos := map[uint64]string{}
	return func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		relpath, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		if relpath == "." {
			return nil
		}
		if relpath == aci.ManifestFile {
			// ignore; this will be written by the archive writer
			// TODO(jonboulle): does this make sense? maybe just remove from archivewriter?
			return nil
		}

		link := ""
		var r io.Reader
		switch info.Mode() & os.ModeType {
		case os.ModeCharDevice:
		case os.ModeDevice:
		case os.ModeDir:
		case os.ModeSymlink:
			target, err := os.Readlink(path)
			if err != nil {
				return err
			}
			link = target
		default:
			file, err := os.Open(path)
			if err != nil {
				return err
			}
			defer file.Close()
			r = file
		}

		hdr, err := tar.FileInfoHeader(info, link)
		if err != nil {
			panic(err)
		}
		// Because os.FileInfo's Name method returns only the base
		// name of the file it describes, it may be necessary to
		// modify the Name field of the returned header to provide the
		// full path name of the file.
		hdr.Name = relpath
		tarheader.Populate(hdr, info, inos)
		// If the file is a hard link to a file we've already seen, we
		// don't need the contents
		if hdr.Typeflag == tar.TypeLink {
			hdr.Size = 0
			r = nil
		}
		if err := aw.AddFile(relpath, hdr, r); err != nil {
			return err
		}

		return nil
	}
}

func runBuild(args []string) (exit int) {
	if len(args) != 2 {
		stderr("build: Must provide directory and output file")
		return 1
	}

	root := args[0]
	tgt := args[1]
	ext := filepath.Ext(tgt)
	if ext != schema.ACIExtension {
		stderr("build: Extension must be %s (given %s)", schema.ACIExtension, ext)
		return 1
	}

	mode := os.O_CREATE | os.O_WRONLY
	if !buildOverwrite {
		mode |= os.O_EXCL
	}
	fh, err := os.OpenFile(tgt, mode, 0644)
	if err != nil {
		if os.IsExist(err) {
			stderr("build: Target file exists (try --overwrite)")
		} else {
			stderr("build: Unable to open target %s: %v", tgt, err)
		}
		return 1
	}

	var gw *gzip.Writer
	var r io.WriteCloser = fh
	if !buildNocompress {
		gw = gzip.NewWriter(fh)
		r = gw
	}
	tr := tar.NewWriter(r)

	defer func() {
		tr.Close()
		if !buildNocompress {
			gw.Close()
		}
		fh.Close()
		if exit != 0 && !buildOverwrite {
			os.Remove(tgt)
		}
	}()

	// TODO(jonboulle): stream the validation so we don't have to walk the rootfs twice
	if err := aci.ValidateLayout(root); err != nil {
		stderr("build: Layout failed validation: %v", err)
		return 1
	}
	mpath := filepath.Join(root, aci.ManifestFile)
	b, err := ioutil.ReadFile(mpath)
	if err != nil {
		stderr("build: Unable to read Image Manifest: %v", err)
		return 1
	}
	var im schema.ImageManifest
	if err := im.UnmarshalJSON(b); err != nil {
		stderr("build: Unable to load Image Manifest: %v", err)
		return 1
	}
	iw := aci.NewImageWriter(im, tr)

	err = filepath.Walk(root, buildWalker(root, iw))
	if err != nil {
		stderr("build: Error walking rootfs: %v", err)
		return 1
	}

	err = iw.Close()
	if err != nil {
		stderr("build: Unable to close image %s: %v", tgt, err)
		return 1
	}

	return
}
<|endoftext|>


<commit_before>package copystructure

import (
	"reflect"
	"testing"
	"time"
)

func TestCopy_complex(t *testing.T) {
	v := map[string]interface{}{
		"foo": []string{"a", "b"},
		"bar": "baz",
	}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_primitive(t *testing.T) {
	cases := []interface{}{
		42,
		"foo",
		1.2,
	}

	for _, tc := range cases {
		result, err := Copy(tc)
		if err != nil {
			t.Fatalf("err: %s", err)
		}
		if result != tc {
			t.Fatalf("bad: %#v", result)
		}
	}
}

func TestCopy_primitivePtr(t *testing.T) {
	cases := []interface{}{
		42,
		"foo",
		1.2,
	}

	for _, tc := range cases {
		result, err := Copy(&tc)
		if err != nil {
			t.Fatalf("err: %s", err)
		}

		if !reflect.DeepEqual(result, &tc) {
			t.Fatalf("bad: %#v", result)
		}
	}
}

func TestCopy_map(t *testing.T) {
	v := map[string]interface{}{
		"bar": "baz",
	}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_slice(t *testing.T) {
	v := []string{"bar", "baz"}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_struct(t *testing.T) {
	type test struct {
		Value string
	}

	v := test{Value: "foo"}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_structPtr(t *testing.T) {
	type test struct {
		Value string
	}

	v := &test{Value: "foo"}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_structNil(t *testing.T) {
	type test struct {
		Value string
	}

	var v *test
	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if v, ok := result.(*test); !ok {
		t.Fatalf("bad: %#v", result)
	} else if v != nil {
		t.Fatalf("bad: %#v", v)
	}
}

func TestCopy_structNested(t *testing.T) {
	type TestInner struct{}

	type Test struct {
		Test *TestInner
	}

	v := Test{}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_structUnexported(t *testing.T) {
	type test struct {
		Value string

		private string
	}

	v := test{Value: "foo"}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_nestedStructUnexported(t *testing.T) {
	type subTest struct {
		mine string
	}

	type test struct {
		Value   string
		private subTest
	}

	v := test{Value: "foo"}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_time(t *testing.T) {
	type test struct {
		Value time.Time
	}

	v := test{Value: time.Now().UTC()}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_aliased(t *testing.T) {
	type (
		Int   int
		Str   string
		Map   map[Int]interface{}
		Slice []Str
	)

	v := Map{
		1: Map{10: 20},
		2: Map(nil),
		3: Slice{"a", "b"},
	}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}
<commit_msg>Add failing test for slice with nil pointer<commit_after>package copystructure

import (
	"reflect"
	"testing"
	"time"
)

func TestCopy_complex(t *testing.T) {
	v := map[string]interface{}{
		"foo": []string{"a", "b"},
		"bar": "baz",
	}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_primitive(t *testing.T) {
	cases := []interface{}{
		42,
		"foo",
		1.2,
	}

	for _, tc := range cases {
		result, err := Copy(tc)
		if err != nil {
			t.Fatalf("err: %s", err)
		}
		if result != tc {
			t.Fatalf("bad: %#v", result)
		}
	}
}

func TestCopy_primitivePtr(t *testing.T) {
	cases := []interface{}{
		42,
		"foo",
		1.2,
	}

	for _, tc := range cases {
		result, err := Copy(&tc)
		if err != nil {
			t.Fatalf("err: %s", err)
		}

		if !reflect.DeepEqual(result, &tc) {
			t.Fatalf("bad: %#v", result)
		}
	}
}

func TestCopy_map(t *testing.T) {
	v := map[string]interface{}{
		"bar": "baz",
	}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_slice(t *testing.T) {
	v := []string{"bar", "baz"}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_struct(t *testing.T) {
	type test struct {
		Value string
	}

	v := test{Value: "foo"}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_structPtr(t *testing.T) {
	type test struct {
		Value string
	}

	v := &test{Value: "foo"}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_structNil(t *testing.T) {
	type test struct {
		Value string
	}

	var v *test
	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if v, ok := result.(*test); !ok {
		t.Fatalf("bad: %#v", result)
	} else if v != nil {
		t.Fatalf("bad: %#v", v)
	}
}

func TestCopy_structNested(t *testing.T) {
	type TestInner struct{}

	type Test struct {
		Test *TestInner
	}

	v := Test{}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_structUnexported(t *testing.T) {
	type test struct {
		Value string

		private string
	}

	v := test{Value: "foo"}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_nestedStructUnexported(t *testing.T) {
	type subTest struct {
		mine string
	}

	type test struct {
		Value   string
		private subTest
	}

	v := test{Value: "foo"}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_time(t *testing.T) {
	type test struct {
		Value time.Time
	}

	v := test{Value: time.Now().UTC()}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_aliased(t *testing.T) {
	type (
		Int   int
		Str   string
		Map   map[Int]interface{}
		Slice []Str
	)

	v := Map{
		1: Map{10: 20},
		2: Map(nil),
		3: Slice{"a", "b"},
	}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}

func TestCopy_panicSliceWithNil(t *testing.T) {
	v := [](*int){nil}

	result, err := Copy(v)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !reflect.DeepEqual(result, v) {
		t.Fatalf("bad: %#v", result)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"regexp"
	"strings"
	"testing"
)

func testcmd(cmd string) (string, error) {
	fmt.Println("(T) " + cmd)
	switch {
	case cmd == "sudo ls -a1F /mnt/sda1/var/lib/docker/vfs/dir":
		return currenttest.vs.ls(), nil
	case cmd == "docker ps -aq --no-trunc":
		return "", nil
	case strings.HasPrefix(cmd, "docker inspect -f '{{ .Name }},{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' "):
		return "", nil
	case strings.HasPrefix(cmd, "sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):
		deleted := cmd[len("sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):]
		deletions = append(deletions, deleted)
		return "", nil
	case strings.HasPrefix(cmd, "sudo readlink /mnt/sda1/var/lib/docker/vfs/dir/"):
		if strings.Contains(cmd, ",nonexistent") {
			return "", errors.New("non-existent linked folder")
		}
		r := regexp.MustCompile(`.*\$([^#]+)###.*`)
		ss := r.FindStringSubmatch(cmd)
		if len(ss) == 2 {
			return ss[1], nil
		}
		return "", nil
	case strings.HasPrefix(cmd, "sudo ls /mnt/sda1/var/lib/docker/vfs/dir/"):
		if cmd == "sudo ls /mnt/sda1/var/lib/docker/vfs/dir/" {
			return "", errors.New("non-ls linked folder")
		}
		return "", nil
	default:
		currentT.Fatalf("test '%s': unknown command!\n", cmd)
		return fmt.Sprintf("test '%s'", cmd), errors.New("unknown command")
	}
}

type volspecs []string
type Test struct {
	title string
	vs    volspecs
	res   []int
}

func (vs volspecs) ls() string {
	if len(vs) == 0 {
		return ""
	}
	res := ""
	for i, spec := range vs {
		if strings.HasSuffix(spec, "/") {
			spec = spec[:len(spec)-1]
			res = res + spec + strings.Repeat(fmt.Sprintf("%d", i), 64-len(spec)) + "/\n"
		}
		if strings.HasSuffix(spec, "@") {
			mp := "." + strings.Replace(spec, ";", "###", -1)
			mp = strings.Replace(mp, "/", ",#,", -1)
			res = res + mp + "\n"
		}
	}
	return res
}

var deletions = []string{}
var tests = []Test{
	Test{"empty vfs", []string{}, []int{0, 0, 0, 0, 0}},
	Test{"two volumes", []string{"fa/", "fb/"}, []int{0, 0, 2, 2, 0}},
	Test{"Invalid (ill-formed) markers must be deleted", []string{"cainv/path/a@"}, []int{0, 0, 0, 0, -1}},
	Test{"Invalid (no readlink) markers must be deleted", []string{"ca;/path/nonexistenta@", "cb;/path/nonexistentb@"}, []int{0, 0, 0, 0, -2}},
	Test{"Invalid (no ls) markers must be deleted", []string{"ca;/path/nolsa@", "cb;/path/nolsb@"}, []int{0, 0, 0, 0, -2}},
}
var currenttest Test
var currentT *testing.T

// TestContainers test different vfs scenarios
func TestContainers(t *testing.T) {
	cmd = testcmd
	currentT = t
	for i, test := range tests {
		currenttest = test
		deletions = []string{}
		main()
		tc := Containers()
		toc := OrphanedContainers()
		tv := Volumes()
		tov := OrphanedVolumes()
		tm := Markers()
		if len(tc) != test.res[0] {
			t.Errorf("Test %d: '%s' expected '%d' containers, got '%d'", i+1, test.title, test.res[0], len(tc))
		}
		if len(toc) != test.res[1] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned containers, got '%d'", i+1, test.title, test.res[1], len(toc))
		}
		if len(tv) != test.res[2] {
			t.Errorf("Test %d: '%s' expected '%d' volumes, got '%d'", i+1, test.title, test.res[2], len(tv))
		}
		if len(tov) != test.res[3] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned volumes, got '%d'", i+1, test.title, test.res[3], len(tov))
		}
		if nbmarkers(tm) != test.res[4] {
			t.Errorf("Test %d: '%s' expected '%d' markers, got '%d'", i+1, test.title, test.res[4], nbmarkers(tm))
		}
		fmt.Println("----------------")
	}
}

func nbmarkers(tm markers) int {
	res := len(tm)
	for _, d := range deletions {
		if strings.HasPrefix(d, ".") {
			res = res - 1
		}
	}
	return res
}
<commit_msg>gcl_tst.go: test invalid marker (no vdir)<commit_after>package main

import (
	"errors"
	"fmt"
	"regexp"
	"strings"
	"testing"
)

func testcmd(cmd string) (string, error) {
	fmt.Println("(T) " + cmd)
	switch {
	case cmd == "sudo ls -a1F /mnt/sda1/var/lib/docker/vfs/dir":
		return currenttest.vs.ls(), nil
	case cmd == "docker ps -aq --no-trunc":
		return "", nil
	case strings.HasPrefix(cmd, "docker inspect -f '{{ .Name }},{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' "):
		return "", nil
	case strings.HasPrefix(cmd, "sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):
		deleted := cmd[len("sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):]
		deletions = append(deletions, deleted)
		return "", nil
	case strings.HasPrefix(cmd, "sudo readlink /mnt/sda1/var/lib/docker/vfs/dir/"):
		if strings.Contains(cmd, ",nonexistent") {
			return "", errors.New("non-existent linked folder")
		}
		r := regexp.MustCompile(`.*\$([^#]+)###.*`)
		ss := r.FindStringSubmatch(cmd)
		if len(ss) == 2 {
			return ss[1], nil
		}
		return "", nil
	case strings.HasPrefix(cmd, "sudo ls /mnt/sda1/var/lib/docker/vfs/dir/"):
		if cmd == "sudo ls /mnt/sda1/var/lib/docker/vfs/dir/" {
			return "", errors.New("non-ls linked folder")
		}
		return "", nil
	default:
		currentT.Fatalf("test '%s': unknown command!\n", cmd)
		return fmt.Sprintf("test '%s'", cmd), errors.New("unknown command")
	}
}

type volspecs []string
type Test struct {
	title string
	vs    volspecs
	res   []int
}

func (vs volspecs) ls() string {
	if len(vs) == 0 {
		return ""
	}
	res := ""
	for i, spec := range vs {
		if strings.HasSuffix(spec, "/") {
			spec = spec[:len(spec)-1]
			res = res + spec + strings.Repeat(fmt.Sprintf("%d", i), 64-len(spec)) + "/\n"
		}
		if strings.HasSuffix(spec, "@") {
			mp := "." + strings.Replace(spec, ";", "###", -1)
			mp = strings.Replace(mp, "/", ",#,", -1)
			res = res + mp + "\n"
		}
	}
	return res
}

var deletions = []string{}
var tests = []Test{
	Test{"empty vfs", []string{}, []int{0, 0, 0, 0, 0}},
	Test{"two volumes", []string{"fa/", "fb/"}, []int{0, 0, 2, 2, 0}},
	Test{"Invalid (ill-formed) markers must be deleted", []string{"cainv/path/a@"}, []int{0, 0, 0, 0, -1}},
	Test{"Invalid (no readlink) markers must be deleted", []string{"ca;/path/nonexistenta@", "cb;/path/nonexistentb@"}, []int{0, 0, 0, 0, -2}},
	Test{"Invalid (no ls) markers must be deleted", []string{"ca;/path/nolsa@", "cb;/path/nolsb@"}, []int{0, 0, 0, 0, -2}},
	Test{"Invalid (no vdir) markers must be deleted", []string{"ca$novdira;/path/nolsa@", "cb$novdirb;/path/nolsb@"}, []int{0, 0, 0, 0, -2}},
}
var currenttest Test
var currentT *testing.T

// TestContainers test different vfs scenarios
func TestContainers(t *testing.T) {
	cmd = testcmd
	currentT = t
	for i, test := range tests {
		currenttest = test
		deletions = []string{}
		main()
		tc := Containers()
		toc := OrphanedContainers()
		tv := Volumes()
		tov := OrphanedVolumes()
		tm := Markers()
		if len(tc) != test.res[0] {
			t.Errorf("Test %d: '%s' expected '%d' containers, got '%d'", i+1, test.title, test.res[0], len(tc))
		}
		if len(toc) != test.res[1] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned containers, got '%d'", i+1, test.title, test.res[1], len(toc))
		}
		if len(tv) != test.res[2] {
			t.Errorf("Test %d: '%s' expected '%d' volumes, got '%d'", i+1, test.title, test.res[2], len(tv))
		}
		if len(tov) != test.res[3] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned volumes, got '%d'", i+1, test.title, test.res[3], len(tov))
		}
		if nbmarkers(tm) != test.res[4] {
			t.Errorf("Test %d: '%s' expected '%d' markers, got '%d'", i+1, test.title, test.res[4], nbmarkers(tm))
		}
		fmt.Println("----------------")
	}
}

func nbmarkers(tm markers) int {
	res := len(tm)
	for _, d := range deletions {
		if strings.HasPrefix(d, ".") {
			res = res - 1
		}
	}
	return res
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Axel Smeets
package main

// Get an available key:
//  client -- GET /get --> server
//  client <-- key -- server (plain text)
//
// Register a key:
//  client -- POST /new {key:k} --> server (json)
//
// Tell server that key is fucked:
// client -- PUT /set {key:k,time:t} --> server (json)

import (
    "net/http"
    "container/list"
    "encoding/JSON"
    "errors"
    "fmt"
    "flag"
    "time"
)

var (
    Keys = list.New()
    currentKey *list.Element

    TimeLayout = "2006/01/02 15:04:05 0000"
)

var (
    ErrNoKeys = errors.New("Found no registered keys")
    ErrNoAvailable = errors.New("All keys are rate limited")
    ErrKeyNotFound = errors.New("Key wasn't found in pool")
    ErrKeyAlreadyExists = errors.New("Key has already been registered")
)

type KeyEntry struct {
    // The client_id of an app
    Key string

    // Some time in the past: available
    // Some time in the future: unavailable
    ResetDate time.Time
}
func (e *KeyEntry) Available() bool {
    return time.Now().After(e.ResetDate)
}

type apiError struct {
     Message string
     Code int
     Error error
 }

type handler func (http.ResponseWriter, *http.Request) *apiError

func (fn handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
    if err := fn(w, r); err != nil {
        fmt.Printf("%s %s %d (%s)\n", r.Method, r.URL, err.Code, err.Error.Error())
        http.Error(w, err.Message, err.Code)
    } else {
        fmt.Printf("%s %s %d\n", r.Method, r.URL, 200)
    }
}

// Try to register a key.
func registerKey(w http.ResponseWriter, r *http.Request) *apiError {
    decoder := json.NewDecoder(r.Body)
    defer r.Body.Close()

    var msg struct {
        Key string
    }

    if err := decoder.Decode(&msg); err != nil {
        return &apiError{
            Message: "You fucked now, son",
            Code: 400,
            Error: err,
        }
    }

    for e := Keys.Front(); e != nil; e = e.Next() {
        ke := e.Value.(*KeyEntry)
        if ke.Key == msg.Key {
            return &apiError{
                Message: "Key already exists",
                Code: 400,
                Error: ErrKeyAlreadyExists,
            }
        }
    }

    Keys.PushFront(&KeyEntry{
        Key: msg.Key,
        ResetDate: time.Now(),
    })

    if currentKey == nil {
        currentKey = Keys.Front()
    }

    w.Write([]byte(msg.Key))
    return nil
}

// Try to set the reset_time of a key.
func updateKey(w http.ResponseWriter, r *http.Request) *apiError {
    // parse body
    decoder := json.NewDecoder(r.Body)
    defer r.Body.Close()

    var msg struct {
        Key string
        Time string
    }

    if err := decoder.Decode(&msg); err != nil {
        return &apiError{
            Message: "You fucked now, son",
            Code: 400,
            Error: err,
        }
    }

    fmt.Printf("%v\n", msg)

    t, e := time.Parse(TimeLayout, msg.Time)
    if e != nil {
        return &apiError{
            Message: "Couldn't parse provided time!",
            Code: 400,
            Error: e,
        }
    }

    el := Keys.Front()
    if el == nil {
        return &apiError{
            Message: ErrNoKeys.Error(),
            Code: 500,
            Error: ErrNoKeys,
        }
    }

    ke := el.Value.(*KeyEntry)
    for ke.Key != msg.Key {
        el = el.Next()
        if el == nil {
            return &apiError{
                Message: ErrKeyNotFound.Error(),
                Code: 400,
                Error: ErrKeyNotFound,
            }
        }
        ke = el.Value.(*KeyEntry)
    }
    ke.ResetDate = t

    w.Write([]byte("Ok"))
    return nil
}

// Try to get a key.
func acquireKey(w http.ResponseWriter, r *http.Request) *apiError {
    if currentKey == nil {
        return &apiError{
            Message: ErrNoKeys.Error(),
            Code: 500,
            Error: ErrNoKeys,
        }
    }

    // start on currentKey.Next
    el := currentKey.Next()
    if el == nil {
        el = Keys.Front()
    }
    ke := el.Value.(*KeyEntry)

    // stop when we've gone full circle or have found a valid key
    // we only care about getting a functional key (not rate limited),
    // the round-robin is just for simplistic balancing
    for !ke.Available() && el != currentKey {
        el = el.Next()
        if el == nil {
            el = Keys.Front()
        }
        ke = el.Value.(*KeyEntry)
    }

    // so here we don't check if el == currentKey, because we don't care...
    currentKey = el
    if !ke.Available() {
        return &apiError{
            Message: ErrNoAvailable.Error(),
            Code: 500,
            Error: ErrNoAvailable,
        }
    }

    w.Write([]byte(ke.Key))
    return nil
}

func b2str(b bool) string {
    if b {
        return "available"
    }
    return "unavailable"
}

// Dump all keys and some info: KEY STATUS RESET_DATE
func dumpKeys(w http.ResponseWriter, r *http.Request) *apiError {
    for e := Keys.Front(); e != nil; e = e.Next() {
        ke := e.Value.(*KeyEntry)
        dur := ke.ResetDate.Sub(time.Now())
        fmt.Fprintf(w, "%s %s %s\n",
            ke.Key,
            b2str(dur.Seconds() <= 0),
            ke.ResetDate.Format(TimeLayout))
    }
    return nil
}

func main() {
    serverAddr := flag.String("addr", ":8000", "server address")
    flag.Parse()

    http.Handle("/new", handler(registerKey))
    http.Handle("/get", handler(acquireKey))
    http.Handle("/set", handler(updateKey))
    http.Handle("/keys", handler(dumpKeys))

    fmt.Printf("Starting scpool server at %s\n", *serverAddr)
    err := http.ListenAndServe(*serverAddr, nil)
    if err != nil {
        panic("ListenAndServe: " + err.Error())
    }
}<commit_msg>Update encoding/JSON --> encoding/json: maybe fix gobuilder<commit_after>// Copyright (c) 2015 Axel Smeets
package main

// Get an available key:
//  client -- GET /get --> server
//  client <-- key -- server (plain text)
//
// Register a key:
//  client -- POST /new {key:k} --> server (json)
//
// Tell server that key is fucked:
// client -- PUT /set {key:k,time:t} --> server (json)

import (
    "net/http"
    "container/list"
    "encoding/json"
    "errors"
    "fmt"
    "flag"
    "time"
)

var (
    Keys = list.New()
    currentKey *list.Element

    TimeLayout = "2006/01/02 15:04:05 0000"
)

var (
    ErrNoKeys = errors.New("Found no registered keys")
    ErrNoAvailable = errors.New("All keys are rate limited")
    ErrKeyNotFound = errors.New("Key wasn't found in pool")
    ErrKeyAlreadyExists = errors.New("Key has already been registered")
)

type KeyEntry struct {
    // The client_id of an app
    Key string

    // Some time in the past: available
    // Some time in the future: unavailable
    ResetDate time.Time
}
func (e *KeyEntry) Available() bool {
    return time.Now().After(e.ResetDate)
}

type apiError struct {
     Message string
     Code int
     Error error
 }

type handler func (http.ResponseWriter, *http.Request) *apiError

func (fn handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
    if err := fn(w, r); err != nil {
        fmt.Printf("%s %s %d (%s)\n", r.Method, r.URL, err.Code, err.Error.Error())
        http.Error(w, err.Message, err.Code)
    } else {
        fmt.Printf("%s %s %d\n", r.Method, r.URL, 200)
    }
}

// Try to register a key.
func registerKey(w http.ResponseWriter, r *http.Request) *apiError {
    decoder := json.NewDecoder(r.Body)
    defer r.Body.Close()

    var msg struct {
        Key string
    }

    if err := decoder.Decode(&msg); err != nil {
        return &apiError{
            Message: "You fucked now, son",
            Code: 400,
            Error: err,
        }
    }

    for e := Keys.Front(); e != nil; e = e.Next() {
        ke := e.Value.(*KeyEntry)
        if ke.Key == msg.Key {
            return &apiError{
                Message: "Key already exists",
                Code: 400,
                Error: ErrKeyAlreadyExists,
            }
        }
    }

    Keys.PushFront(&KeyEntry{
        Key: msg.Key,
        ResetDate: time.Now(),
    })

    if currentKey == nil {
        currentKey = Keys.Front()
    }

    w.Write([]byte(msg.Key))
    return nil
}

// Try to set the reset_time of a key.
func updateKey(w http.ResponseWriter, r *http.Request) *apiError {
    // parse body
    decoder := json.NewDecoder(r.Body)
    defer r.Body.Close()

    var msg struct {
        Key string
        Time string
    }

    if err := decoder.Decode(&msg); err != nil {
        return &apiError{
            Message: "You fucked now, son",
            Code: 400,
            Error: err,
        }
    }

    fmt.Printf("%v\n", msg)

    t, e := time.Parse(TimeLayout, msg.Time)
    if e != nil {
        return &apiError{
            Message: "Couldn't parse provided time!",
            Code: 400,
            Error: e,
        }
    }

    el := Keys.Front()
    if el == nil {
        return &apiError{
            Message: ErrNoKeys.Error(),
            Code: 500,
            Error: ErrNoKeys,
        }
    }

    ke := el.Value.(*KeyEntry)
    for ke.Key != msg.Key {
        el = el.Next()
        if el == nil {
            return &apiError{
                Message: ErrKeyNotFound.Error(),
                Code: 400,
                Error: ErrKeyNotFound,
            }
        }
        ke = el.Value.(*KeyEntry)
    }
    ke.ResetDate = t

    w.Write([]byte("Ok"))
    return nil
}

// Try to get a key.
func acquireKey(w http.ResponseWriter, r *http.Request) *apiError {
    if currentKey == nil {
        return &apiError{
            Message: ErrNoKeys.Error(),
            Code: 500,
            Error: ErrNoKeys,
        }
    }

    // start on currentKey.Next
    el := currentKey.Next()
    if el == nil {
        el = Keys.Front()
    }
    ke := el.Value.(*KeyEntry)

    // stop when we've gone full circle or have found a valid key
    // we only care about getting a functional key (not rate limited),
    // the round-robin is just for simplistic balancing
    for !ke.Available() && el != currentKey {
        el = el.Next()
        if el == nil {
            el = Keys.Front()
        }
        ke = el.Value.(*KeyEntry)
    }

    // so here we don't check if el == currentKey, because we don't care...
    currentKey = el
    if !ke.Available() {
        return &apiError{
            Message: ErrNoAvailable.Error(),
            Code: 500,
            Error: ErrNoAvailable,
        }
    }

    w.Write([]byte(ke.Key))
    return nil
}

func b2str(b bool) string {
    if b {
        return "available"
    }
    return "unavailable"
}

// Dump all keys and some info: KEY STATUS RESET_DATE
func dumpKeys(w http.ResponseWriter, r *http.Request) *apiError {
    for e := Keys.Front(); e != nil; e = e.Next() {
        ke := e.Value.(*KeyEntry)
        dur := ke.ResetDate.Sub(time.Now())
        fmt.Fprintf(w, "%s %s %s\n",
            ke.Key,
            b2str(dur.Seconds() <= 0),
            ke.ResetDate.Format(TimeLayout))
    }
    return nil
}

func main() {
    serverAddr := flag.String("addr", ":8000", "server address")
    flag.Parse()

    http.Handle("/new", handler(registerKey))
    http.Handle("/get", handler(acquireKey))
    http.Handle("/set", handler(updateKey))
    http.Handle("/keys", handler(dumpKeys))

    fmt.Printf("Starting scpool server at %s\n", *serverAddr)
    err := http.ListenAndServe(*serverAddr, nil)
    if err != nil {
        panic("ListenAndServe: " + err.Error())
    }
}<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"

	"github.com/prometheus/common/log"
	"github.com/prometheus/prometheus/config"

	"gopkg.in/yaml.v2"
)

type Page struct {
	JobExporterMap      map[string]string
	ErrorMessage        string
	PrometheusStatusUrl string
}

var (
	configFile      = flag.String("config.file", "prometheus.yml", "prometheus configuration file name.")
	exporterListDir = flag.String("exporter.list.dir", "/tmp", "file_sd_configs names file dir")
	listenAddress   = flag.String("listen-address", ":12345", "The address to listen on for HTTP requests.")
	prometheusUrl   = flag.String("prometheus.url", "http://localhost:9090", "prometheus")
)

func rewrite() error {
	conf, err := config.LoadFile(*configFile)
	if err != nil {
		return errors.Wrapf(err, "Couldn't load configuration (-config.file=%s): %v", *configFile)
	}
	yamlData, yamlErr := yaml.Marshal(&conf)
	if yamlErr != nil {
		return errors.Wrapf(yamlErr, "yaml marshal error: %v", yamlErr)
	}
	writeErr := ioutil.WriteFile(*configFile, yamlData, 0644)
	if writeErr != nil {
		return errors.Wrapf(writeErr, "yaml write error: %v", writeErr)
	}
	return nil
}

func create(job_name string, exporter_list string) error {
	newcfg := &config.ScrapeConfig{}
	newcfg.JobName = job_name
	fsdc := &config.FileSDConfig{}
	exporterListFile := filepath.Join(*exporterListDir, job_name+".yml")
	strs := "'" + strings.Replace(exporter_list, "\r\n", "','", -1) + "'"
	exporterListFileWriteErr := ioutil.WriteFile(exporterListFile, []byte(fmt.Sprintf("[{\"targets\":[%s]}]", strs)), 0644)
	if exporterListFileWriteErr != nil {
		return errors.Wrapf(exporterListFileWriteErr, "error: %v", exporterListFileWriteErr)
	}
	fsdc.Names = []string{exporterListFile}
	newcfg.FileSDConfigs = append(newcfg.FileSDConfigs, fsdc)
	rc := &config.RelabelConfig{}
	rc.Regex = config.MustNewRegexp("(.*):\\d+")
	newcfg.RelabelConfigs = append(newcfg.RelabelConfigs, rc)

	var newScrapeConfigs []*config.ScrapeConfig
	newScrapeConfigs = append(newScrapeConfigs, newcfg)
	conf, confErr := config.LoadFile(*configFile)
	if confErr != nil {
		return errors.Wrapf(confErr, "Couldn't load configuration (-config.file=%s): %v", *configFile, confErr)
	}
	for _, scfg := range conf.ScrapeConfigs {
		newScrapeConfigs = append(newScrapeConfigs, scfg)
	}
	conf.ScrapeConfigs = newScrapeConfigs
	newYamlData, err := yaml.Marshal(&conf)
	if err != nil {
		return errors.Wrapf(err, "yaml marshal error: %v", err)
	}
	writeErr := ioutil.WriteFile(*configFile, newYamlData, 0644)
	if writeErr != nil {
		return errors.Wrapf(writeErr, "write error: %v", writeErr)
	}
	rewriteErr := rewrite()
	if rewriteErr != nil {
		return errors.Wrapf(rewriteErr, "rewrite error: %v", rewriteErr)
	}
	_, postErr := http.PostForm(*prometheusUrl+"/-/reload", url.Values{})
	if postErr != nil {
		return errors.Wrapf(postErr, "post error: %v", postErr)
	}
	return nil
}

func main() {
	flag.Parse()

	log.Infof("Loading configuration file %s", *configFile)
	log.Infof("Starting Server: %s", *listenAddress)

	t := template.Must(template.ParseFiles("templates/toya.html"))
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		errorMessage := ""
		if r.Method == "POST" {
			if r.FormValue("job_name") == "" {
				errorMessage += "Job Name is required\n"
			}
			conf, err := config.LoadFile(*configFile)
			if err != nil {
				log.Errorf("Couldn't load configuration (-config.file=%s): %v", *configFile)
				return
			}
			for _, scfg := range conf.ScrapeConfigs {
				if r.FormValue("job_name") == scfg.JobName {
					errorMessage += "duplicate Job Name\n"
					break
				}
			}
			if r.FormValue("exporter_list") == "" {
				errorMessage += "Exporter lis is required\n"
			}
			for _, exporter := range strings.Split(r.FormValue("exporter_list"), "\r\n") {
				if strings.Contains(exporter, "/") {
					errorMessage += exporter + " is not a valid hostname\n"
				}
			}
			if errorMessage == "" {
				err := create(r.FormValue("job_name"), r.FormValue("exporter_list"))
				if err != nil {
					log.Errorf("create error: %v", err)
					return
				}
			}
		}
		conf, err := config.LoadFile(*configFile)
		if err != nil {
			log.Errorf("Couldn't load configuration (-config.file=%s): %v", *configFile, err)
			return
		}
		jobExporterMap := make(map[string]string)
		for _, scfg := range conf.ScrapeConfigs {
			for _, fsdc := range scfg.FileSDConfigs {
				content, e := ioutil.ReadFile(fsdc.Names[0])
				if e != nil {
					log.Fatalf("error: %v", e)
				}
				jobExporterMap[scfg.JobName] = string(content)
			}
		}
		page := Page{
			JobExporterMap:      jobExporterMap,
			ErrorMessage:        errorMessage,
			PrometheusStatusUrl: *prometheusUrl + "/status",
		}
		t.Execute(w, page)
	})
	err := http.ListenAndServe(*listenAddress, nil)
	if err != nil {
		log.Fatalf("error: %v", err)
	}

}
<commit_msg>Use Errorf, not Fatalf<commit_after>package main

import (
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"

	"github.com/prometheus/common/log"
	"github.com/prometheus/prometheus/config"

	"gopkg.in/yaml.v2"
)

type Page struct {
	JobExporterMap      map[string]string
	ErrorMessage        string
	PrometheusStatusUrl string
}

var (
	configFile      = flag.String("config.file", "prometheus.yml", "prometheus configuration file name.")
	exporterListDir = flag.String("exporter.list.dir", "/tmp", "file_sd_configs names file dir")
	listenAddress   = flag.String("listen-address", ":12345", "The address to listen on for HTTP requests.")
	prometheusUrl   = flag.String("prometheus.url", "http://localhost:9090", "prometheus")
)

func rewrite() error {
	conf, err := config.LoadFile(*configFile)
	if err != nil {
		return errors.Wrapf(err, "Couldn't load configuration (-config.file=%s): %v", *configFile)
	}
	yamlData, yamlErr := yaml.Marshal(&conf)
	if yamlErr != nil {
		return errors.Wrapf(yamlErr, "yaml marshal error: %v", yamlErr)
	}
	writeErr := ioutil.WriteFile(*configFile, yamlData, 0644)
	if writeErr != nil {
		return errors.Wrapf(writeErr, "yaml write error: %v", writeErr)
	}
	return nil
}

func create(job_name string, exporter_list string) error {
	newcfg := &config.ScrapeConfig{}
	newcfg.JobName = job_name
	fsdc := &config.FileSDConfig{}
	exporterListFile := filepath.Join(*exporterListDir, job_name+".yml")
	strs := "'" + strings.Replace(exporter_list, "\r\n", "','", -1) + "'"
	exporterListFileWriteErr := ioutil.WriteFile(exporterListFile, []byte(fmt.Sprintf("[{\"targets\":[%s]}]", strs)), 0644)
	if exporterListFileWriteErr != nil {
		return errors.Wrapf(exporterListFileWriteErr, "error: %v", exporterListFileWriteErr)
	}
	fsdc.Names = []string{exporterListFile}
	newcfg.FileSDConfigs = append(newcfg.FileSDConfigs, fsdc)
	rc := &config.RelabelConfig{}
	rc.Regex = config.MustNewRegexp("(.*):\\d+")
	newcfg.RelabelConfigs = append(newcfg.RelabelConfigs, rc)

	var newScrapeConfigs []*config.ScrapeConfig
	newScrapeConfigs = append(newScrapeConfigs, newcfg)
	conf, confErr := config.LoadFile(*configFile)
	if confErr != nil {
		return errors.Wrapf(confErr, "Couldn't load configuration (-config.file=%s): %v", *configFile, confErr)
	}
	for _, scfg := range conf.ScrapeConfigs {
		newScrapeConfigs = append(newScrapeConfigs, scfg)
	}
	conf.ScrapeConfigs = newScrapeConfigs
	newYamlData, err := yaml.Marshal(&conf)
	if err != nil {
		return errors.Wrapf(err, "yaml marshal error: %v", err)
	}
	writeErr := ioutil.WriteFile(*configFile, newYamlData, 0644)
	if writeErr != nil {
		return errors.Wrapf(writeErr, "write error: %v", writeErr)
	}
	rewriteErr := rewrite()
	if rewriteErr != nil {
		return errors.Wrapf(rewriteErr, "rewrite error: %v", rewriteErr)
	}
	_, postErr := http.PostForm(*prometheusUrl+"/-/reload", url.Values{})
	if postErr != nil {
		return errors.Wrapf(postErr, "post error: %v", postErr)
	}
	return nil
}

func main() {
	flag.Parse()

	log.Infof("Loading configuration file %s", *configFile)
	log.Infof("Starting Server: %s", *listenAddress)

	t := template.Must(template.ParseFiles("templates/toya.html"))
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		errorMessage := ""
		if r.Method == "POST" {
			if r.FormValue("job_name") == "" {
				errorMessage += "Job Name is required\n"
			}
			conf, err := config.LoadFile(*configFile)
			if err != nil {
				log.Errorf("Couldn't load configuration (-config.file=%s): %v", *configFile)
				return
			}
			for _, scfg := range conf.ScrapeConfigs {
				if r.FormValue("job_name") == scfg.JobName {
					errorMessage += "duplicate Job Name\n"
					break
				}
			}
			if r.FormValue("exporter_list") == "" {
				errorMessage += "Exporter lis is required\n"
			}
			for _, exporter := range strings.Split(r.FormValue("exporter_list"), "\r\n") {
				if strings.Contains(exporter, "/") {
					errorMessage += exporter + " is not a valid hostname\n"
				}
			}
			if errorMessage == "" {
				err := create(r.FormValue("job_name"), r.FormValue("exporter_list"))
				if err != nil {
					log.Errorf("create error: %v", err)
					return
				}
			}
		}
		conf, err := config.LoadFile(*configFile)
		if err != nil {
			log.Errorf("Couldn't load configuration (-config.file=%s): %v", *configFile, err)
			return
		}
		jobExporterMap := make(map[string]string)
		for _, scfg := range conf.ScrapeConfigs {
			for _, fsdc := range scfg.FileSDConfigs {
				content, e := ioutil.ReadFile(fsdc.Names[0])
				if e != nil {
					log.Errorf("error: %v", e)
					return
				}
				jobExporterMap[scfg.JobName] = string(content)
			}
		}
		page := Page{
			JobExporterMap:      jobExporterMap,
			ErrorMessage:        errorMessage,
			PrometheusStatusUrl: *prometheusUrl + "/status",
		}
		t.Execute(w, page)
	})
	err := http.ListenAndServe(*listenAddress, nil)
	if err != nil {
		log.Errorf("error: %v", err)
		return
	}

}
<|endoftext|>


<commit_before>package tree

import (
	"sort"
	"strings"
)

type node struct {
	nodes map[string]*node
}

func (n *node) isLeaf() bool {
	return nil == n.nodes || len(n.nodes) == 0
}

func (n *node) eat(segments []string) {
	if len(segments) == 0 {
		return
	}

	firstSegment := segments[0]
	var subNode *node
	if nil == n.nodes {
		subNode = &node{}
		n.nodes = make(map[string]*node)
		n.nodes[firstSegment] = subNode
	} else {
		if _, ok := n.nodes[firstSegment]; !ok {
			subNode = &node{}
			n.nodes[firstSegment] = subNode
		} else {
			subNode = n.nodes[firstSegment]
		}
	}

	if len(segments) > 1 {
		subNode.eat(segments[1:])
	}
}

func (n *node) Format(indent string) string {
	var lines []string
	if n.isLeaf() || len(n.nodes) == 0 {
		panic("")
	} else {
		keys := make([]string, len(n.nodes))
		i := 0
		for key := range n.nodes {
			keys[i] = key
			i++
		}

		sort.Strings(keys)

		i = len(n.nodes) - 1
		for _, key := range keys {
			c := n.nodes[key]
			newLine := indent[:len(indent)-8]
			if i == 0 {
				newLine += "└── "
				indent = indent[:len(indent)-8] + "    "
			} else if len(indent) == 8 && i == (len(n.nodes)-1) {
				newLine += "."
			} else {
				newLine += "├── "
			}
			newLine += key
			lines = append(
				lines,
				newLine,
			)

			if !c.isLeaf() {
				lines = append(
					lines,
					c.Format(indent+"│   "),
				)
			}
			i -= 1
		}
		return strings.Join(lines, "\n")
	}
}

type tree struct {
	separator string
	root      *node
}

func (g *tree) Format() string {
	return g.root.Format("│   ")
}

func New(separator string) *tree {
	return &tree{separator, &node{}}
}

func (g *tree) Eat(line string) {
	g.root.eat(strings.Split(line, g.separator))
}

func (g *tree) EatLines(lines []string) {
	for _, line := range lines {
		g.Eat(line)
	}
}
<commit_msg>Return, return<commit_after>package tree

import (
	"sort"
	"strings"
)

type node struct {
	nodes map[string]*node
}

func (n *node) isLeaf() bool {
	return nil == n.nodes || len(n.nodes) == 0
}

func (n *node) eat(segments []string) {
	if len(segments) == 0 {
		return
	}

	firstSegment := segments[0]
	var subNode *node
	if nil == n.nodes {
		subNode = &node{}
		n.nodes = make(map[string]*node)
		n.nodes[firstSegment] = subNode
	} else {
		if _, ok := n.nodes[firstSegment]; !ok {
			subNode = &node{}
			n.nodes[firstSegment] = subNode
		} else {
			subNode = n.nodes[firstSegment]
		}
	}

	if len(segments) > 1 {
		subNode.eat(segments[1:])
	}
}

func (n *node) Format(indent string) string {
	var lines []string
	if n.isLeaf() || len(n.nodes) == 0 {
		return ""
	} else {
		keys := make([]string, len(n.nodes))
		i := 0
		for key := range n.nodes {
			keys[i] = key
			i++
		}

		sort.Strings(keys)

		i = len(n.nodes) - 1
		for _, key := range keys {
			c := n.nodes[key]
			newLine := indent[:len(indent)-8]
			if i == 0 {
				newLine += "└── "
				indent = indent[:len(indent)-8] + "    "
			} else if len(indent) == 8 && i == (len(n.nodes)-1) {
				newLine += "."
			} else {
				newLine += "├── "
			}
			newLine += key
			lines = append(
				lines,
				newLine,
			)

			if !c.isLeaf() {
				lines = append(
					lines,
					c.Format(indent+"│   "),
				)
			}
			i -= 1
		}
		return strings.Join(lines, "\n")
	}
	return ""
}

type tree struct {
	separator string
	root      *node
}

func (g *tree) Format() string {
	return g.root.Format("│   ")
}

func New(separator string) *tree {
	return &tree{separator, &node{}}
}

func (g *tree) Eat(line string) {
	g.root.eat(strings.Split(line, g.separator))
}

func (g *tree) EatLines(lines []string) {
	for _, line := range lines {
		g.Eat(line)
	}
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"code.google.com/p/go.net/idna"
)

var (
	v, quick    bool
	concurrency int
	zones       []string
	prefixes    []string
)

func init() {
	flag.BoolVar(&v, "v", false, "verbose output (to stderr)")
	flag.BoolVar(&quick, "quick", false, "Only work on a subset of zones")
	flag.IntVar(&concurrency, "concurrency", 8, "Set maximum number of concurrent requests")
}

func main() {
	flag.Parse()

	if err := main1(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func main1() error {
	var err error
	zones, err = readLines("zones.txt")
	if err != nil {
		return err
	}
	prefixes, err = readLines("prefixes.txt")
	if err != nil {
		return err
	}

	// Quick for debugging?
	if quick {
		zones = zones[0:50]
	}

	// fmt.Println(strings.Join(zones, "\n"))
	// fmt.Println("\n\n-------------------\n\n")
	// fmt.Println(strings.Join(prefixes, "\n"))

	limiter := make(chan struct{}, concurrency) // semaphore to limit concurrency

	fmt.Fprintf(os.Stderr, "Querying whois for %d prefixes and %d zones\n", len(prefixes), len(zones))

	// Create 1 goroutine for each zone
	for _, zone := range zones {
		for _, prefix := range prefixes {
			go func(prefix, zone string) {
				limiter <- struct{}{} // acquire semaphore
				defer func() {        // release semaphore
					<-limiter
				}()

				fmt.Printf("%s.%s\n", prefix, zone)
			}(prefix, zone)
		}
	}

	return nil
}

var re = regexp.MustCompile("\\s+|#.+$")

func readLines(fn string) (out []string, err error) {
	fmt.Fprintf(os.Stderr, "Reading %s\n", fn)
	_, file, _, _ := runtime.Caller(0)
	buf, err := ioutil.ReadFile(filepath.Join(filepath.Dir(file), "data", fn))
	if err != nil {
		return
	}
	s := strings.Trim(string(buf), "\n")
	lines := strings.Split(s, "\n")
	for _, line := range lines {
		line = re.ReplaceAllLiteralString(line, "")
		if line != "" {
			line, _ = idna.ToASCII(line)
			out = append(out, line)
		}
	}
	return
}
<commit_msg>Generate directories for each whois server to store fixture data. FIXME: this breaks when run concurrently<commit_after>// +build ignore

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"

	"code.google.com/p/go.net/idna"
	whois "github.com/domainr/go-whois"
)

var (
	v, quick      bool
	concurrency   int
	zones         []string
	prefixes      []string
	_, FILE, _, _ = runtime.Caller(0)
	DIR           = filepath.Dir(FILE)
)

func init() {
	flag.BoolVar(&v, "v", false, "verbose output (to stderr)")
	flag.BoolVar(&quick, "quick", false, "Only work on a subset of zones")
	flag.IntVar(&concurrency, "concurrency", 8, "Set maximum number of concurrent requests")
}

func main() {
	flag.Parse()

	if err := main1(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func main1() error {
	var err error
	zones, err = readLines("zones.txt")
	if err != nil {
		return err
	}
	prefixes, err = readLines("prefixes.txt")
	if err != nil {
		return err
	}

	// Quick for debugging?
	if quick {
		fmt.Fprintf(os.Stderr, "Quick mode enabled\n")
		zones = zones[0:50]
	}

	limiter := make(chan struct{}, concurrency) // semaphore to limit concurrency
	var mutex sync.Mutex

	fmt.Fprintf(os.Stderr, "Querying whois for %d prefixes and %d zones\n", len(prefixes), len(zones))

	for _, zone := range zones {
		for _, prefix := range prefixes {
			domain := fmt.Sprintf("%s.%s", prefix, zone)
			fmt.Printf("%s\n", domain)
			// FIXME: this SHOULD run under a goroutine, but it currently behaves incorrectly
			// go func(domain string) {
			func(domain string) {
				limiter <- struct{}{} // acquire semaphore
				defer func() {        // release semaphore
					<-limiter
				}()

				req, err := whois.Resolve(domain)
				if err != nil {
					return
				}

				mutex.Lock()
				err = os.MkdirAll(filepath.Join(DIR, "data", req.Host), os.ModePerm)
				mutex.Unlock()
				if err != nil {
					return
				}
			}(domain)
		}
	}

	return nil
}

var re = regexp.MustCompile("\\s+|#.+$")

func readLines(fn string) (out []string, err error) {
	fmt.Fprintf(os.Stderr, "Reading %s\n", fn)
	buf, err := ioutil.ReadFile(filepath.Join(DIR, "data", fn))
	if err != nil {
		return
	}
	s := strings.Trim(string(buf), "\n")
	lines := strings.Split(s, "\n")
	for _, line := range lines {
		line = re.ReplaceAllLiteralString(line, "")
		if line != "" {
			line, _ = idna.ToASCII(line)
			out = append(out, line)
		}
	}
	return
}
<|endoftext|>


<commit_before>/**
 * Copyright 2015 Comcast Cable Communications Management, LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package jtl

import (
	"bufio"
	"encoding/json"
	"fmt"
	"net/http"
	"os"

	. "github.com/Comcast/eel/eel/util"
)

type TraceLogger struct {
	File         *os.File
	Writer       *bufio.Writer
	Settings     *EelTraceLogParams
	EventChannel chan *JDoc
}

func NewTraceLogger(ctx Context, config *EelSettings) *TraceLogger {
	tl := new(TraceLogger)
	if config.TraceLogParams != nil && config.TraceLogParams.FileName != "" {
		tl.Settings = config.TraceLogParams
		tl.EventChannel = make(chan *JDoc)
		ctx.Log().Info("SETTINGS", tl.Settings)
		var err error
		tl.File, err = os.Create(config.TraceLogParams.FileName)
		if err != nil {
			ctx.Log().Error("error_type", "io_error", "op", "trace_logger", "cause", "unable_to_create_file", "error", err.Error())
		} else {
			tl.Writer = bufio.NewWriter(tl.File)
		}
	}
	tl.processTraceLogLoop(ctx)
	return tl
}

func TraceLogConfigHandler(w http.ResponseWriter, r *http.Request) {
	ctx := Gctx.SubContext()
	w.Header().Set("Content-Type", "application/json")
	state := make(map[string]interface{}, 0)
	state["Version"] = GetConfig(ctx).Version
	if ctx.ConfigValue(EelTraceLogger) != nil {
		tl := ctx.ConfigValue(EelTraceLogger).(*TraceLogger)
		tl.Settings.Active = !tl.Settings.Active
		GetConfig(ctx).TraceLogParams.Active = !GetConfig(ctx).TraceLogParams.Active
		state["TraceLogConfigs"] = tl.Settings
	}
	buf, err := json.MarshalIndent(state, "", "\t")
	if err != nil {
		fmt.Fprintf(w, `{"error":"%s"}`, err.Error())
	} else {
		fmt.Fprintf(w, string(buf))
	}
}

func (t *TraceLogger) processTraceLogLoop(ctx Context) {
	go func() {
		for {
			event := <-t.EventChannel
			if t.Writer != nil && t.Settings.LogParams != nil {
				entry := make(map[string]interface{}, 0)
				for k, v := range t.Settings.LogParams {
					ev := event.ParseExpression(ctx, v)
					entry[k] = ev
				}
				buf, err := json.Marshal(entry)
				if err != nil {
					ctx.Log().Error("error_type", "json_error", "op", "trace_logger", "cause", "unable_to_marshal_entry", "error", err.Error())
				} else {
					t.Writer.WriteString(string(buf) + "\n")
				}
			}
		}
	}()
}

func (t *TraceLogger) TraceLog(ctx Context, event *JDoc, incoming bool) {
	if !t.Settings.Active {
		return
	}
	if incoming && !t.Settings.LogIncoming {
		return
	}
	if !incoming && !t.Settings.LogOutgoing {
		return
	}
	t.EventChannel <- event
}

func (t *TraceLogger) CloseTraceLog(ctx Context) {
	if t.File != nil {
		err := t.File.Close()
		if err != nil {
			ctx.Log().Error("error_type", "io_error", "op", "trace_logger", "cause", "unable_to_close_file", "error", err.Error())
		}
	}
}
<commit_msg>cleanup logging<commit_after>/**
 * Copyright 2015 Comcast Cable Communications Management, LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package jtl

import (
	"bufio"
	"encoding/json"
	"fmt"
	"net/http"
	"os"

	. "github.com/Comcast/eel/eel/util"
)

type TraceLogger struct {
	File         *os.File
	Writer       *bufio.Writer
	Settings     *EelTraceLogParams
	EventChannel chan *JDoc
}

func NewTraceLogger(ctx Context, config *EelSettings) *TraceLogger {
	tl := new(TraceLogger)
	if config.TraceLogParams != nil && config.TraceLogParams.FileName != "" {
		tl.Settings = config.TraceLogParams
		tl.EventChannel = make(chan *JDoc)
		var err error
		tl.File, err = os.Create(config.TraceLogParams.FileName)
		if err != nil {
			ctx.Log().Error("error_type", "io_error", "op", "trace_logger", "cause", "unable_to_create_file", "error", err.Error())
		} else {
			tl.Writer = bufio.NewWriter(tl.File)
		}
	}
	tl.processTraceLogLoop(ctx)
	return tl
}

func TraceLogConfigHandler(w http.ResponseWriter, r *http.Request) {
	ctx := Gctx.SubContext()
	w.Header().Set("Content-Type", "application/json")
	state := make(map[string]interface{}, 0)
	state["Version"] = GetConfig(ctx).Version
	if ctx.ConfigValue(EelTraceLogger) != nil {
		tl := ctx.ConfigValue(EelTraceLogger).(*TraceLogger)
		tl.Settings.Active = !tl.Settings.Active
		GetConfig(ctx).TraceLogParams.Active = !GetConfig(ctx).TraceLogParams.Active
		state["TraceLogConfigs"] = tl.Settings
	}
	buf, err := json.MarshalIndent(state, "", "\t")
	if err != nil {
		fmt.Fprintf(w, `{"error":"%s"}`, err.Error())
	} else {
		fmt.Fprintf(w, string(buf))
	}
}

func (t *TraceLogger) processTraceLogLoop(ctx Context) {
	go func() {
		for {
			event := <-t.EventChannel
			if t.Writer != nil && t.Settings.LogParams != nil {
				entry := make(map[string]interface{}, 0)
				for k, v := range t.Settings.LogParams {
					ev := event.ParseExpression(ctx, v)
					entry[k] = ev
				}
				buf, err := json.Marshal(entry)
				if err != nil {
					ctx.Log().Error("error_type", "json_error", "op", "trace_logger", "cause", "unable_to_marshal_entry", "error", err.Error())
				} else {
					t.Writer.WriteString(string(buf) + "\n")
				}
			}
		}
	}()
}

func (t *TraceLogger) TraceLog(ctx Context, event *JDoc, incoming bool) {
	if !t.Settings.Active {
		return
	}
	if incoming && !t.Settings.LogIncoming {
		return
	}
	if !incoming && !t.Settings.LogOutgoing {
		return
	}
	t.EventChannel <- event
}

func (t *TraceLogger) CloseTraceLog(ctx Context) {
	if t.File != nil {
		err := t.File.Close()
		if err != nil {
			ctx.Log().Error("error_type", "io_error", "op", "trace_logger", "cause", "unable_to_close_file", "error", err.Error())
		}
	}
}
<|endoftext|>


<commit_before>package orient

import (
	"log"
	"reflect"
	"time"
	"unsafe"
)

// OType is an enum for the various data types supported by OrientDB.
type OType byte

// in alignment with: http://orientdb.com/docs/last/Types.html
const (
	BOOLEAN      OType = 0
	INTEGER      OType = 1
	SHORT        OType = 2
	LONG         OType = 3
	FLOAT        OType = 4
	DOUBLE       OType = 5
	DATETIME     OType = 6
	STRING       OType = 7
	BINARY       OType = 8 // means []byte
	EMBEDDED     OType = 9
	EMBEDDEDLIST OType = 10
	EMBEDDEDSET  OType = 11
	EMBEDDEDMAP  OType = 12
	LINK         OType = 13
	LINKLIST     OType = 14
	LINKSET      OType = 15
	LINKMAP      OType = 16
	BYTE         OType = 17
	TRANSIENT    OType = 18
	DATE         OType = 19
	CUSTOM       OType = 20
	DECIMAL      OType = 21
	LINKBAG      OType = 22
	ANY          OType = 23
	UNKNOWN      OType = 255 // driver addition
)

// detect the int size (4 or 8 bytes)
const intSize = unsafe.Sizeof(int(0))

func (t OType) String() string { // do not change - it may be used as field type for SQL queries
	switch t {
	case BOOLEAN:
		return "BOOLEAN"
	case INTEGER:
		return "INTEGER"
	case LONG:
		return "LONG"
	case FLOAT:
		return "FLOAT"
	case DOUBLE:
		return "DOUBLE"
	case DATETIME:
		return "DATETIME"
	case STRING:
		return "STRING"
	case BINARY:
		return "BINARY"
	case EMBEDDED:
		return "EMBEDDED"
	case EMBEDDEDLIST:
		return "EMBEDDEDLIST"
	case EMBEDDEDSET:
		return "EMBEDDEDSET"
	case EMBEDDEDMAP:
		return "EMBEDDEDMAP"
	case LINK:
		return "LINK"
	case LINKLIST:
		return "LINKLIST"
	case LINKSET:
		return "LINKSET"
	case LINKMAP:
		return "LINKMAP"
	case BYTE:
		return "BYTE"
	case TRANSIENT:
		return "TRANSIENT"
	case DATE:
		return "DATE"
	case CUSTOM:
		return "CUSTOM"
	case DECIMAL:
		return "DECIMAL"
	case LINKBAG:
		return "LINKBAG"
	case ANY:
		return "ANY"
	default:
		return "UNKNOWN"
	}
}

func (t OType) ReflectKind() reflect.Kind {
	switch t {
	case BOOLEAN:
		return reflect.Bool
	case BYTE:
		return reflect.Uint8
	case SHORT:
		return reflect.Int16
	case INTEGER:
		return reflect.Int32
	case LONG:
		return reflect.Int64
	case FLOAT:
		return reflect.Float32
	case DOUBLE:
		return reflect.Float64
	case STRING:
		return reflect.String
	case EMBEDDEDLIST, EMBEDDEDSET:
		return reflect.Slice
	case EMBEDDEDMAP:
		return reflect.Map
	case LINKLIST, LINKSET:
		return reflect.Slice
	case LINKMAP:
		return reflect.Map
	default:
		return reflect.Invalid
	}
}

func (t OType) ReflectType() reflect.Type {
	switch t {
	case BOOLEAN:
		return reflect.TypeOf(bool(false))
	case INTEGER:
		return reflect.TypeOf(int32(0))
	case LONG:
		return reflect.TypeOf(int64(0))
	case FLOAT:
		return reflect.TypeOf(float32(0))
	case DOUBLE:
		return reflect.TypeOf(float64(0))
	case DATETIME, DATE:
		return reflect.TypeOf(time.Time{})
	case STRING:
		return reflect.TypeOf(string(""))
	case BINARY:
		return reflect.TypeOf([]byte{})
	case BYTE:
		return reflect.TypeOf(byte(0))
		//	case EMBEDDED:
		//		return "EMBEDDED"
		//	case EMBEDDEDLIST:
		//		return "EMBEDDEDLIST"
		//	case EMBEDDEDSET:
		//		return "EMBEDDEDSET"
		//	case EMBEDDEDMAP:
		//		return "EMBEDDEDMAP"
		//	case LINK:
		//		return "LINK"
		//	case LINKLIST:
		//		return "LINKLIST"
		//	case LINKSET:
		//		return "LINKSET"
		//	case LINKMAP:
		//		return "LINKMAP"
		//	case CUSTOM:
		//		return "CUSTOM"
		//	case DECIMAL:
		//		return "DECIMAL"
		//	case LINKBAG:
		//		return "LINKBAG"
	default: // and ANY, TRANSIENT
		return reflect.TypeOf((*interface{})(nil)).Elem()
	}
}

func OTypeForValue(val interface{}) (ftype OType) {
	ftype = UNKNOWN
	// TODO: need to add more types: LINKSET, LINKLIST, etc. ...
	switch val.(type) {
	case string:
		ftype = STRING
	case bool:
		ftype = BOOLEAN
	case int32:
		ftype = INTEGER
	case int64:
		ftype = LONG
	case int16:
		ftype = SHORT
	case int:
		if intSize == 4 {
			ftype = INTEGER
		} else {
			ftype = LONG
		}
	case byte, int8:
		ftype = BYTE
	case *Document, DocumentSerializable:
		ftype = EMBEDDED
	case float32:
		ftype = FLOAT
	case float64:
		ftype = DOUBLE
	case []byte:
		ftype = BINARY
	case OIdentifiable:
		ftype = LINK
	case []OIdentifiable, []RID:
		ftype = LINKLIST
	case *RidBag:
		ftype = LINKBAG
	// TODO: more types need to be added
	default:
		if isDecimal(val) {
			ftype = DECIMAL
			return
		}
		rt := reflect.TypeOf(val)
		if rt.Kind() == reflect.Ptr {
			rt = rt.Elem()
		}
		switch rt.Kind() {
		case reflect.Map:
			ftype = EMBEDDEDMAP
		case reflect.Slice, reflect.Array:
			if reflect.TypeOf(val).Elem() == reflect.TypeOf(byte(0)) {
				ftype = BINARY
			} else {
				ftype = EMBEDDEDLIST
			}
		case reflect.Bool:
			ftype = BOOLEAN
		case reflect.Uint8:
			ftype = BYTE
		case reflect.Int16:
			ftype = SHORT
		case reflect.Int32:
			ftype = INTEGER
		case reflect.Int64:
			ftype = LONG
		case reflect.Int:
			if intSize == 4 {
				ftype = INTEGER
			} else {
				ftype = LONG
			}
		case reflect.String:
			ftype = STRING
		case reflect.Struct:
			ftype = EMBEDDED
		default:
			log.Printf("unknown type in serialization: %T, kind: %v", val, reflect.TypeOf(val).Kind())
		}
	}
	return
}

func OTypeFromString(typ string) OType {
	switch typ {
	case "BOOLEAN":
		return BOOLEAN
	case "INTEGER":
		return INTEGER
	case "SHORT":
		return SHORT
	case "LONG":
		return LONG
	case "FLOAT":
		return FLOAT
	case "DOUBLE":
		return DOUBLE
	case "DATETIME":
		return DATETIME
	case "STRING":
		return STRING
	case "BINARY":
		return BINARY
	case "EMBEDDED":
		return EMBEDDED
	case "EMBEDDEDLIST":
		return EMBEDDEDLIST
	case "EMBEDDEDSET":
		return EMBEDDEDSET
	case "EMBEDDEDMAP":
		return EMBEDDEDMAP
	case "LINK":
		return LINK
	case "LINKLIST":
		return LINKLIST
	case "LINKSET":
		return LINKSET
	case "LINKMAP":
		return LINKMAP
	case "BYTE":
		return BYTE
	case "TRANSIENT":
		return TRANSIENT
	case "DATE":
		return DATE
	case "CUSTOM":
		return CUSTOM
	case "DECIMAL":
		return DECIMAL
	case "LINKBAG":
		return LINKBAG
	case "ANY":
		return ANY
	default:
		panic("Unkwown type: " + typ)
	}
}
<commit_msg>support Time, Uint and Uint64 added<commit_after>package orient

import (
	"log"
	"reflect"
	"time"
	"unsafe"
)

// OType is an enum for the various data types supported by OrientDB.
type OType byte

// in alignment with: http://orientdb.com/docs/last/Types.html
const (
	BOOLEAN      OType = 0
	INTEGER      OType = 1
	SHORT        OType = 2
	LONG         OType = 3
	FLOAT        OType = 4
	DOUBLE       OType = 5
	DATETIME     OType = 6
	STRING       OType = 7
	BINARY       OType = 8 // means []byte
	EMBEDDED     OType = 9
	EMBEDDEDLIST OType = 10
	EMBEDDEDSET  OType = 11
	EMBEDDEDMAP  OType = 12
	LINK         OType = 13
	LINKLIST     OType = 14
	LINKSET      OType = 15
	LINKMAP      OType = 16
	BYTE         OType = 17
	TRANSIENT    OType = 18
	DATE         OType = 19
	CUSTOM       OType = 20
	DECIMAL      OType = 21
	LINKBAG      OType = 22
	ANY          OType = 23
	UNKNOWN      OType = 255 // driver addition
)

// detect the int size (4 or 8 bytes)
const intSize = unsafe.Sizeof(int(0))

func (t OType) String() string { // do not change - it may be used as field type for SQL queries
	switch t {
	case BOOLEAN:
		return "BOOLEAN"
	case INTEGER:
		return "INTEGER"
	case LONG:
		return "LONG"
	case FLOAT:
		return "FLOAT"
	case DOUBLE:
		return "DOUBLE"
	case DATETIME:
		return "DATETIME"
	case STRING:
		return "STRING"
	case BINARY:
		return "BINARY"
	case EMBEDDED:
		return "EMBEDDED"
	case EMBEDDEDLIST:
		return "EMBEDDEDLIST"
	case EMBEDDEDSET:
		return "EMBEDDEDSET"
	case EMBEDDEDMAP:
		return "EMBEDDEDMAP"
	case LINK:
		return "LINK"
	case LINKLIST:
		return "LINKLIST"
	case LINKSET:
		return "LINKSET"
	case LINKMAP:
		return "LINKMAP"
	case BYTE:
		return "BYTE"
	case TRANSIENT:
		return "TRANSIENT"
	case DATE:
		return "DATE"
	case CUSTOM:
		return "CUSTOM"
	case DECIMAL:
		return "DECIMAL"
	case LINKBAG:
		return "LINKBAG"
	case ANY:
		return "ANY"
	default:
		return "UNKNOWN"
	}
}

func (t OType) ReflectKind() reflect.Kind {
	switch t {
	case BOOLEAN:
		return reflect.Bool
	case BYTE:
		return reflect.Uint8
	case SHORT:
		return reflect.Int16
	case INTEGER:
		return reflect.Int32
	case LONG:
		return reflect.Int64
	case FLOAT:
		return reflect.Float32
	case DOUBLE:
		return reflect.Float64
	case STRING:
		return reflect.String
	case EMBEDDEDLIST, EMBEDDEDSET:
		return reflect.Slice
	case EMBEDDEDMAP:
		return reflect.Map
	case LINKLIST, LINKSET:
		return reflect.Slice
	case LINKMAP:
		return reflect.Map
	default:
		return reflect.Invalid
	}
}

func (t OType) ReflectType() reflect.Type {
	switch t {
	case BOOLEAN:
		return reflect.TypeOf(bool(false))
	case INTEGER:
		return reflect.TypeOf(int32(0))
	case LONG:
		return reflect.TypeOf(int64(0))
	case FLOAT:
		return reflect.TypeOf(float32(0))
	case DOUBLE:
		return reflect.TypeOf(float64(0))
	case DATETIME, DATE:
		return reflect.TypeOf(time.Time{})
	case STRING:
		return reflect.TypeOf(string(""))
	case BINARY:
		return reflect.TypeOf([]byte{})
	case BYTE:
		return reflect.TypeOf(byte(0))
		//	case EMBEDDED:
		//		return "EMBEDDED"
		//	case EMBEDDEDLIST:
		//		return "EMBEDDEDLIST"
		//	case EMBEDDEDSET:
		//		return "EMBEDDEDSET"
		//	case EMBEDDEDMAP:
		//		return "EMBEDDEDMAP"
		//	case LINK:
		//		return "LINK"
		//	case LINKLIST:
		//		return "LINKLIST"
		//	case LINKSET:
		//		return "LINKSET"
		//	case LINKMAP:
		//		return "LINKMAP"
		//	case CUSTOM:
		//		return "CUSTOM"
		//	case DECIMAL:
		//		return "DECIMAL"
		//	case LINKBAG:
		//		return "LINKBAG"
	default: // and ANY, TRANSIENT
		return reflect.TypeOf((*interface{})(nil)).Elem()
	}
}

func OTypeForValue(val interface{}) (ftype OType) {
	ftype = UNKNOWN
	// TODO: need to add more types: LINKSET, LINKLIST, etc. ...
	switch val.(type) {
	case string:
		ftype = STRING
	case bool:
		ftype = BOOLEAN
	case int32:
		ftype = INTEGER
	case int64:
		ftype = LONG
	case int16:
		ftype = SHORT
	case int:
		if intSize == 4 {
			ftype = INTEGER
		} else {
			ftype = LONG
		}
	case byte, int8:
		ftype = BYTE
	case *Document, DocumentSerializable:
		ftype = EMBEDDED
	case float32:
		ftype = FLOAT
	case float64:
		ftype = DOUBLE
	case []byte:
		ftype = BINARY
	case OIdentifiable:
		ftype = LINK
	case []OIdentifiable, []RID:
		ftype = LINKLIST
	case *RidBag:
		ftype = LINKBAG
	case time.Time:
		ftype = DATETIME
	// TODO: more types need to be added
	default:
		if isDecimal(val) {
			ftype = DECIMAL
			return
		}
		rt := reflect.TypeOf(val)
		if rt.Kind() == reflect.Ptr {
			rt = rt.Elem()
		}
		switch rt.Kind() {
		case reflect.Map:
			ftype = EMBEDDEDMAP
		case reflect.Slice, reflect.Array:
			if reflect.TypeOf(val).Elem() == reflect.TypeOf(byte(0)) {
				ftype = BINARY
			} else {
				ftype = EMBEDDEDLIST
			}
		case reflect.Bool:
			ftype = BOOLEAN
		case reflect.Uint8:
			ftype = BYTE
		case reflect.Int16:
			ftype = SHORT
		case reflect.Int32:
			ftype = INTEGER
		case reflect.Int64:
			ftype = LONG
		case reflect.Int:
			if intSize == 4 {
				ftype = INTEGER
			} else {
				ftype = LONG
			}
		case reflect.Uint:
			if intSize == 4 {
				ftype = INTEGER
			} else {
				ftype = LONG
			}
		case reflect.Uint64:
			ftype = LONG
		case reflect.String:
			ftype = STRING
		case reflect.Struct:
			ftype = EMBEDDED
		default:
			log.Printf("unknown type in serialization: %T, kind: %v", val, reflect.TypeOf(val).Kind())
		}
	}
	return
}

func OTypeFromString(typ string) OType {
	switch typ {
	case "BOOLEAN":
		return BOOLEAN
	case "INTEGER":
		return INTEGER
	case "SHORT":
		return SHORT
	case "LONG":
		return LONG
	case "FLOAT":
		return FLOAT
	case "DOUBLE":
		return DOUBLE
	case "DATETIME":
		return DATETIME
	case "STRING":
		return STRING
	case "BINARY":
		return BINARY
	case "EMBEDDED":
		return EMBEDDED
	case "EMBEDDEDLIST":
		return EMBEDDEDLIST
	case "EMBEDDEDSET":
		return EMBEDDEDSET
	case "EMBEDDEDMAP":
		return EMBEDDEDMAP
	case "LINK":
		return LINK
	case "LINKLIST":
		return LINKLIST
	case "LINKSET":
		return LINKSET
	case "LINKMAP":
		return LINKMAP
	case "BYTE":
		return BYTE
	case "TRANSIENT":
		return TRANSIENT
	case "DATE":
		return DATE
	case "CUSTOM":
		return CUSTOM
	case "DECIMAL":
		return DECIMAL
	case "LINKBAG":
		return LINKBAG
	case "ANY":
		return ANY
	default:
		panic("Unkwown type: " + typ)
	}
}
<|endoftext|>