<commit_before>package cobe

import (
	"fmt"
	"log"
	"math/rand"
	"strings"
	"time"
)

type Brain struct {
	graph  *graph
	tok    tokenizer
	scorer scorer
}

const spaceTokenID tokenID = -1

func OpenBrain(path string) (*Brain, error) {
	graph, err := openGraph(path)
	if err != nil {
		return nil, err
	}

	version, err := graph.GetInfoString("version")
	if err != nil {
		return nil, err
	}

	if version != "2" {
		return nil, fmt.Errorf("cannot read version %s brain", version)
	}

	tokenizer, err := graph.GetInfoString("tokenizer")
	if err != nil {
		return nil, err
	}

	return &Brain{graph, getTokenizer(tokenizer), &cobeScorer{}}, nil
}

func (b *Brain) Close() {
	if b.graph != nil {
		b.graph.Close()
		b.graph = nil
	}
}

func getTokenizer(name string) tokenizer {
	switch strings.ToLower(name) {
	case "cobe":
		return newCobeTokenizer()
	case "megahal":
		return newMegaHALTokenizer()
	}

	return nil
}

func (b *Brain) Learn(text string) {
	tokens := b.tok.Split(text)

	// skip learning if too few tokens (but don't count spaces)
	if countGoodTokens(tokens) <= b.graph.getOrder() {
		return
	}

	var tokenIds []tokenID
	for _, text := range tokens {
		var tokenID tokenID
		if text == " " {
			tokenID = spaceTokenID
		} else {
			tokenID = b.graph.GetOrCreateToken(text)
		}

		tokenIds = append(tokenIds, tokenID)
	}

	var prevNode nodeID
	b.forEdges(tokenIds, func(prev, next []tokenID, hasSpace bool) {
		if prevNode == 0 {
			prevNode = b.graph.GetOrCreateNode(prev)
		}
		nextNode := b.graph.GetOrCreateNode(next)

		b.graph.addEdge(prevNode, nextNode, hasSpace)
		prevNode = nextNode
	})
}

func countGoodTokens(tokens []string) int {
	var count int
	for _, token := range tokens {
		if token != " " {
			count++
		}
	}

	return count
}

func (b *Brain) forEdges(tokenIds []tokenID, f func([]tokenID, []tokenID, bool)) {
	// Call f() on every N-gram (N = brain order) in tokenIds.
	order := b.graph.getOrder()

	chain := b.toChain(order, tokenIds)
	edges := toEdges(order, chain)

	for _, e := range edges {
		f(e.prev, e.next, e.hasSpace)
	}
}

func (b *Brain) toChain(order int, tokenIds []tokenID) []tokenID {
	var chain []tokenID
	for i := 0; i < order; i++ {
		chain = append(chain, b.graph.endTokenID)
	}

	chain = append(chain, tokenIds...)

	for i := 0; i < order; i++ {
		chain = append(chain, b.graph.endTokenID)
	}

	return chain
}

type edge struct {
	prev     []tokenID
	next     []tokenID
	hasSpace bool
}

func toEdges(order int, tokenIds []tokenID) []edge {
	var tokens []tokenID
	var spaces []int

	// Turn tokenIds (containing some SPACE_TOKEN_ID) into a list
	// of tokens and a list of positions in the tokens slice after
	// which spaces were found.

	for i := 0; i < len(tokenIds); i++ {
		tokens = append(tokens, tokenIds[i])

		if i < len(tokenIds)-1 && tokenIds[i+1] == spaceTokenID {
			spaces = append(spaces, len(tokens))
			i++
		}
	}

	var ret []edge

	prev := tokens[0:order]
	for i := 1; i < len(tokens)-order+1; i++ {
		next := tokens[i : i+order]

		var hasSpace bool
		if len(spaces) > 0 && spaces[0] == i+order-1 {
			hasSpace = true
			spaces = spaces[1:]
		}

		ret = append(ret, edge{prev, next, hasSpace})
		prev = next
	}

	return ret
}

func (b *Brain) Reply(text string) string {
	tokens := b.tok.Split(text)
	tokenIds := b.graph.filterPivots(unique(tokens))

	stemTokenIds := b.conflateStems(tokens)
	tokenIds = uniqueIds(append(tokenIds, stemTokenIds...))

	if len(tokenIds) == 0 {
		tokenIds = b.babble()
	}

	var count int

	var bestReply *reply
	var bestScore float64 = -1

	next := b.replySearch(tokenIds)

	endTime := time.Now().Add(500 * time.Millisecond)
	for time.Now().Before(endTime) {
		edges, valid := next()
		if !valid {
			// run another search
			next = b.replySearch(tokenIds)
		} else {
			reply := newReply(b.graph, edges)
			score := b.scorer.Score(reply)

			if score > bestScore {
				bestReply = reply
				bestScore = score
			}

			count++
		}
	}

	fmt.Printf("Got %d total replies\n", count)
	if bestReply == nil {
		return "no replies :  ("
	}

	return bestReply.ToString()
}

func (b *Brain) conflateStems(tokens []string) []tokenID {
	var ret []tokenID

	for _, token := range tokens {
		tokenIds := b.graph.getTokensByStem(token)
		ret = append(ret, tokenIds...)
	}

	return ret
}

func (b *Brain) babble() []tokenID {
	var tokenIds []tokenID

	for i := 0; i < 5; i++ {
		t := b.graph.getRandomToken()
		if t > 0 {
			tokenIds = append(tokenIds, tokenID(t))
		}
	}

	return tokenIds
}

// replySearch combines a forward and a reverse search over the graph
// into a series of replies.
func (b *Brain) replySearch(tokenIds []tokenID) func() ([]edgeID, bool) {
	pivotID := b.pickPivot(tokenIds)
	pivotNode := b.graph.getRandomNodeWithToken(pivotID)

	endNode := b.graph.endContextID

	revIter := b.graph.search(pivotNode, endNode, Reverse)
	fwdIter := b.graph.search(pivotNode, endNode, Forward)

	return func() ([]edgeID, bool) {
		if !revIter.Next() {
			return nil, false
		}

		if !fwdIter.Next() {
			return nil, false
		}

		return join(revIter.Result(), fwdIter.Result()), true
	}
}

func join(rev []edgeID, fwd []edgeID) []edgeID {
	edges := make([]edgeID, 0, len(rev)+len(fwd))

	// rev is a path from the pivot node to the beginning of a
	// reply: join its edges in reverse order.
	for i := len(rev) - 1; i >= 0; i-- {
		edges = append(edges, rev[i])
	}

	return append(edges, fwd...)
}

func (b *Brain) pickPivot(tokenIds []tokenID) tokenID {
	return tokenIds[rand.Intn(len(tokenIds))]
}

func unique(tokens []string) []string {
	// Reduce tokens to a unique set by sending them through a map.
	m := make(map[string]int)
	for _, token := range tokens {
		m[token]++
	}

	ret := make([]string, 0, len(m))
	for token := range m {
		ret = append(ret, token)
	}

	return ret
}

func uniqueIds(ids []tokenID) []tokenID {
	// Reduce token ids to a unique set by sending them through a map.
	m := make(map[tokenID]int)
	for _, id := range ids {
		m[id]++
	}

	ret := make([]tokenID, 0, len(m))
	for id := range m {
		ret = append(ret, id)
	}

	return ret
}

type reply struct {
	graph   *graph
	edges   []edgeID
	hasText bool
	text    string
}

func newReply(graph *graph, edges []edgeID) *reply {
	return &reply{graph, edges, false, ""}
}

func (r *reply) ToString() string {
	if !r.hasText {
		var parts []string

		for _, edge := range r.edges {
			word, hasSpace, err := r.graph.getTextByEdge(edge)
			if err != nil {
				log.Printf("ERROR: %s\n", err)
			}

			parts = append(parts, word)
			if hasSpace {
				parts = append(parts, " ")
			}
		}

		r.hasText = true
		r.text = strings.Join(parts, "")
	}

	return r.text
}
<commit_msg>Fix a nil dereference when replying on an empty brain<commit_after>package cobe

import (
	"fmt"
	"log"
	"math/rand"
	"strings"
	"time"
)

type Brain struct {
	graph  *graph
	tok    tokenizer
	scorer scorer
}

const spaceTokenID tokenID = -1

func OpenBrain(path string) (*Brain, error) {
	graph, err := openGraph(path)
	if err != nil {
		return nil, err
	}

	version, err := graph.GetInfoString("version")
	if err != nil {
		return nil, err
	}

	if version != "2" {
		return nil, fmt.Errorf("cannot read version %s brain", version)
	}

	tokenizer, err := graph.GetInfoString("tokenizer")
	if err != nil {
		return nil, err
	}

	return &Brain{graph, getTokenizer(tokenizer), &cobeScorer{}}, nil
}

func (b *Brain) Close() {
	if b.graph != nil {
		b.graph.Close()
		b.graph = nil
	}
}

func getTokenizer(name string) tokenizer {
	switch strings.ToLower(name) {
	case "cobe":
		return newCobeTokenizer()
	case "megahal":
		return newMegaHALTokenizer()
	}

	return nil
}

func (b *Brain) Learn(text string) {
	tokens := b.tok.Split(text)

	// skip learning if too few tokens (but don't count spaces)
	if countGoodTokens(tokens) <= b.graph.getOrder() {
		return
	}

	var tokenIds []tokenID
	for _, text := range tokens {
		var tokenID tokenID
		if text == " " {
			tokenID = spaceTokenID
		} else {
			tokenID = b.graph.GetOrCreateToken(text)
		}

		tokenIds = append(tokenIds, tokenID)
	}

	var prevNode nodeID
	b.forEdges(tokenIds, func(prev, next []tokenID, hasSpace bool) {
		if prevNode == 0 {
			prevNode = b.graph.GetOrCreateNode(prev)
		}
		nextNode := b.graph.GetOrCreateNode(next)

		b.graph.addEdge(prevNode, nextNode, hasSpace)
		prevNode = nextNode
	})
}

func countGoodTokens(tokens []string) int {
	var count int
	for _, token := range tokens {
		if token != " " {
			count++
		}
	}

	return count
}

func (b *Brain) forEdges(tokenIds []tokenID, f func([]tokenID, []tokenID, bool)) {
	// Call f() on every N-gram (N = brain order) in tokenIds.
	order := b.graph.getOrder()

	chain := b.toChain(order, tokenIds)
	edges := toEdges(order, chain)

	for _, e := range edges {
		f(e.prev, e.next, e.hasSpace)
	}
}

func (b *Brain) toChain(order int, tokenIds []tokenID) []tokenID {
	var chain []tokenID
	for i := 0; i < order; i++ {
		chain = append(chain, b.graph.endTokenID)
	}

	chain = append(chain, tokenIds...)

	for i := 0; i < order; i++ {
		chain = append(chain, b.graph.endTokenID)
	}

	return chain
}

type edge struct {
	prev     []tokenID
	next     []tokenID
	hasSpace bool
}

func toEdges(order int, tokenIds []tokenID) []edge {
	var tokens []tokenID
	var spaces []int

	// Turn tokenIds (containing some SPACE_TOKEN_ID) into a list
	// of tokens and a list of positions in the tokens slice after
	// which spaces were found.

	for i := 0; i < len(tokenIds); i++ {
		tokens = append(tokens, tokenIds[i])

		if i < len(tokenIds)-1 && tokenIds[i+1] == spaceTokenID {
			spaces = append(spaces, len(tokens))
			i++
		}
	}

	var ret []edge

	prev := tokens[0:order]
	for i := 1; i < len(tokens)-order+1; i++ {
		next := tokens[i : i+order]

		var hasSpace bool
		if len(spaces) > 0 && spaces[0] == i+order-1 {
			hasSpace = true
			spaces = spaces[1:]
		}

		ret = append(ret, edge{prev, next, hasSpace})
		prev = next
	}

	return ret
}

func (b *Brain) Reply(text string) string {
	tokens := b.tok.Split(text)
	tokenIds := b.graph.filterPivots(unique(tokens))

	stemTokenIds := b.conflateStems(tokens)
	tokenIds = uniqueIds(append(tokenIds, stemTokenIds...))

	if len(tokenIds) == 0 {
		tokenIds = b.babble()
	}

	if len(tokenIds) == 0 {
		return "I don't know enough to answer you yet!"
	}

	var count int

	var bestReply *reply
	var bestScore float64 = -1

	next := b.replySearch(tokenIds)

	endTime := time.Now().Add(500 * time.Millisecond)
	for time.Now().Before(endTime) {
		edges, valid := next()
		if !valid {
			// run another search
			next = b.replySearch(tokenIds)
		} else {
			reply := newReply(b.graph, edges)
			score := b.scorer.Score(reply)

			if score > bestScore {
				bestReply = reply
				bestScore = score
			}

			count++
		}
	}

	fmt.Printf("Got %d total replies\n", count)
	if bestReply == nil {
		return "I don't know enough to answer you yet!"
	}

	return bestReply.ToString()
}

func (b *Brain) conflateStems(tokens []string) []tokenID {
	var ret []tokenID

	for _, token := range tokens {
		tokenIds := b.graph.getTokensByStem(token)
		ret = append(ret, tokenIds...)
	}

	return ret
}

func (b *Brain) babble() []tokenID {
	var tokenIds []tokenID

	for i := 0; i < 5; i++ {
		t := b.graph.getRandomToken()
		if t > 0 {
			tokenIds = append(tokenIds, tokenID(t))
		}
	}

	return tokenIds
}

// replySearch combines a forward and a reverse search over the graph
// into a series of replies.
func (b *Brain) replySearch(tokenIds []tokenID) func() ([]edgeID, bool) {
	pivotID := b.pickPivot(tokenIds)
	pivotNode := b.graph.getRandomNodeWithToken(pivotID)

	endNode := b.graph.endContextID

	revIter := b.graph.search(pivotNode, endNode, Reverse)
	fwdIter := b.graph.search(pivotNode, endNode, Forward)

	return func() ([]edgeID, bool) {
		if !revIter.Next() {
			return nil, false
		}

		if !fwdIter.Next() {
			return nil, false
		}

		return join(revIter.Result(), fwdIter.Result()), true
	}
}

func join(rev []edgeID, fwd []edgeID) []edgeID {
	edges := make([]edgeID, 0, len(rev)+len(fwd))

	// rev is a path from the pivot node to the beginning of a
	// reply: join its edges in reverse order.
	for i := len(rev) - 1; i >= 0; i-- {
		edges = append(edges, rev[i])
	}

	return append(edges, fwd...)
}

func (b *Brain) pickPivot(tokenIds []tokenID) tokenID {
	return tokenIds[rand.Intn(len(tokenIds))]
}

func unique(tokens []string) []string {
	// Reduce tokens to a unique set by sending them through a map.
	m := make(map[string]int)
	for _, token := range tokens {
		m[token]++
	}

	ret := make([]string, 0, len(m))
	for token := range m {
		ret = append(ret, token)
	}

	return ret
}

func uniqueIds(ids []tokenID) []tokenID {
	// Reduce token ids to a unique set by sending them through a map.
	m := make(map[tokenID]int)
	for _, id := range ids {
		m[id]++
	}

	ret := make([]tokenID, 0, len(m))
	for id := range m {
		ret = append(ret, id)
	}

	return ret
}

type reply struct {
	graph   *graph
	edges   []edgeID
	hasText bool
	text    string
}

func newReply(graph *graph, edges []edgeID) *reply {
	return &reply{graph, edges, false, ""}
}

func (r *reply) ToString() string {
	if !r.hasText {
		var parts []string

		for _, edge := range r.edges {
			word, hasSpace, err := r.graph.getTextByEdge(edge)
			if err != nil {
				log.Printf("ERROR: %s\n", err)
			}

			parts = append(parts, word)
			if hasSpace {
				parts = append(parts, " ")
			}
		}

		r.hasText = true
		r.text = strings.Join(parts, "")
	}

	return r.text
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/giantswarm/conair/btrfs"
	"github.com/giantswarm/conair/layer"
	"github.com/giantswarm/conair/nspawn"
	"github.com/giantswarm/conair/parser"
)

var cmdBuild = &Command{
	Name:        "build",
	Description: "Build an image",
	Summary:     "Build an image",
	Run:         runBuild,
}

func readFile(filename string) (*parser.Conairfile, error) {
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		return nil, err
	}
	return parser.Parse(filename)
}

func runBuild(args []string) (exit int) {
	if len(args) < 1 {
		fmt.Fprintln(os.Stderr, "Image name missing.")
		return 1
	}

	newImage := args[0]
	newImagePath := fmt.Sprintf("images/%s", newImage)

	fs, _ := btrfs.Init(home)

	// remove existing layer
	if fs.Exists(newImagePath) {
		if err := fs.Remove(newImagePath); err != nil {
			fmt.Printf("%v", err)
			fmt.Fprintln(os.Stderr, fmt.Sprintf("Couldn't remove existing image. %v", err))
			return 1
		}
	}

	// read build file
	f, err := readFile("./Conairfile")
	if err != nil {
		f, err = readFile("./Dockerfile")
		if err != nil {
			fmt.Fprintln(os.Stderr, "Couldn't read Conairfile or Dockerfile.", err)
			return 1
		}
	}

	image := f.From
	parentPath := fmt.Sprintf("images/%s", image)

	for i, snap := range f.Snapshots {
		paths := strings.Split(snap, ":")
		if len(paths) < 2 {
			fmt.Fprintln(os.Stderr, "SNAPSHOT definition is unreadable. Please use <snapshot name>:<container path> notation.")
			return 1
		}

		// check if snapshot exists - otherwise create a new subvolume
		snapshotPath := fmt.Sprintf("snapshots/%s", paths[0])
		if !fs.Exists(snapshotPath) {
			if err := fs.Subvolume(snapshotPath); err != nil {
				fmt.Fprintln(os.Stderr, fmt.Sprintf("Couldn't create snapshot '%s'.", snapshotPath))
				return 1
			}
		}

		f.Snapshots[i] = fmt.Sprintf("%s/snapshots/%s", home, snap)
	}

	for _, cmd := range f.Commands {
		l, err := layer.Create(fs, cmd.Verb, cmd.Payload, parentPath)
		if err != nil {
			fmt.Fprintln(os.Stderr, fmt.Sprintf("Couldn't create layer: %v.", err))
			return 1
		}
		if l.Exists == true {
			parentPath = l.Path
			continue
		}

		fmt.Println(l.Hash, cmd.Verb, cmd.Payload)
		c := nspawn.Init(l.Hash, fmt.Sprintf("%s/%s", home, l.Path))
		c.SetBinds(append(f.Binds, f.Snapshots...))

		if err := c.Build(cmd.Verb, cmd.Payload); err != nil {
			fmt.Fprintln(os.Stderr, fmt.Sprintf("Buildstep failed: %v.", err))
			if err = l.Remove(); err != nil {
				fmt.Fprintln(os.Stderr, "Couldn't remove temporary build container.", err)
			}
			return 1
		}

		parentPath = l.Path
	}
	if err = fs.Snapshot(parentPath, newImagePath, true); err != nil {
		fmt.Fprintln(os.Stderr, "Couldn't create filesystem for new image.", err)
		return 1
	}

	return 0
}
<commit_msg>moved buildstep output before the layer check<commit_after>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/giantswarm/conair/btrfs"
	"github.com/giantswarm/conair/layer"
	"github.com/giantswarm/conair/nspawn"
	"github.com/giantswarm/conair/parser"
)

var cmdBuild = &Command{
	Name:        "build",
	Description: "Build an image",
	Summary:     "Build an image",
	Run:         runBuild,
}

func readFile(filename string) (*parser.Conairfile, error) {
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		return nil, err
	}
	return parser.Parse(filename)
}

func runBuild(args []string) (exit int) {
	if len(args) < 1 {
		fmt.Fprintln(os.Stderr, "Image name missing.")
		return 1
	}

	newImage := args[0]
	newImagePath := fmt.Sprintf("images/%s", newImage)

	fs, _ := btrfs.Init(home)

	// remove existing layer
	if fs.Exists(newImagePath) {
		if err := fs.Remove(newImagePath); err != nil {
			fmt.Printf("%v", err)
			fmt.Fprintln(os.Stderr, fmt.Sprintf("Couldn't remove existing image. %v", err))
			return 1
		}
	}

	// read build file
	f, err := readFile("./Conairfile")
	if err != nil {
		f, err = readFile("./Dockerfile")
		if err != nil {
			fmt.Fprintln(os.Stderr, "Couldn't read Conairfile or Dockerfile.", err)
			return 1
		}
	}

	image := f.From
	parentPath := fmt.Sprintf("images/%s", image)

	for i, snap := range f.Snapshots {
		paths := strings.Split(snap, ":")
		if len(paths) < 2 {
			fmt.Fprintln(os.Stderr, "SNAPSHOT definition is unreadable. Please use <snapshot name>:<container path> notation.")
			return 1
		}

		// check if snapshot exists - otherwise create a new subvolume
		snapshotPath := fmt.Sprintf("snapshots/%s", paths[0])
		if !fs.Exists(snapshotPath) {
			if err := fs.Subvolume(snapshotPath); err != nil {
				fmt.Fprintln(os.Stderr, fmt.Sprintf("Couldn't create snapshot '%s'.", snapshotPath))
				return 1
			}
		}

		f.Snapshots[i] = fmt.Sprintf("%s/snapshots/%s", home, snap)
	}

	for _, cmd := range f.Commands {
		l, err := layer.Create(fs, cmd.Verb, cmd.Payload, parentPath)
		if err != nil {
			fmt.Fprintln(os.Stderr, fmt.Sprintf("Couldn't create layer: %v.", err))
			return 1
		}
		fmt.Println(l.Hash, cmd.Verb, cmd.Payload)

		if l.Exists == true {
			parentPath = l.Path
			continue
		}

		c := nspawn.Init(l.Hash, fmt.Sprintf("%s/%s", home, l.Path))
		c.SetBinds(append(f.Binds, f.Snapshots...))

		if err := c.Build(cmd.Verb, cmd.Payload); err != nil {
			fmt.Fprintln(os.Stderr, fmt.Sprintf("Buildstep failed: %v.", err))
			if err = l.Remove(); err != nil {
				fmt.Fprintln(os.Stderr, "Couldn't remove temporary build container.", err)
			}
			return 1
		}

		parentPath = l.Path
	}
	if err = fs.Snapshot(parentPath, newImagePath, true); err != nil {
		fmt.Fprintln(os.Stderr, "Couldn't create filesystem for new image.", err)
		return 1
	}

	return 0
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"os/exec"
	"time"
)

// RunBuild builds the server (either for deployment or for local use)
func RunBuild(args []string) {
	// Remove fragmenta build from args list
	args = args[2:]

	if len(args) > 0 {
		buildDeployServer()
	} else {
		buildLocalServer()
	}

}

// buildServer removes the old binary and rebuilds the server
func buildServer(server string, env []string) error {

	// Remove old binary
	_, err := os.Stat(server)
	if err == nil {
		err = os.Remove(server)
		if err != nil {
			log.Printf("Error removing server %s", err)
		}
	}

	// If we have a goimports, run that, if not run go fmt
	_, err = os.Stat(os.ExpandEnv("$GOPATH/bin/goimports"))

	if err == nil {
		// Go imports behaviour differs from go fmt
		srcPath := "./"
		log.Printf("Running goimports at %s", srcPath)
		result, err := runCommand("goimports", "-w", srcPath)
		if err != nil {
			log.Printf("Error running goimports %s", err)
			return err
		}
		if len(result) > 0 {
			log.Printf(string(result))
		}

	} else {
		srcPath := "./..."
		// Run go fmt on any packages with src
		log.Printf("Running go fmt at %s", srcPath)
		result, err := runCommand("go", "fmt", srcPath)
		if err != nil {
			log.Printf("Error running fmt %s", err)
			return err
		}
		if len(result) > 0 {
			log.Printf(string(result))
		}
	}

	// Build new binary
	log.Printf("Building server at %s", server)
	started := time.Now()

	args := []string{"build", "-o", server, serverCompilePath(".")}

	// If a local build with no environment settings, use go build -i
	if len(env) == 0 {
		args = []string{"build", "-i", "-o", server, serverCompilePath(".")}
	}

	// Call go build with -i to install artefacts for local builds, and -o to output to ./bin
	// log.Printf("  %s", args)
	cmd := exec.Command("go", args...)
	cmd.Stderr = os.Stdout

	if env != nil {
		cmd.Env = env
	}

	output, err := cmd.Output()
	if err != nil {
		log.Printf("Error running build %s\n%s", err, string(output))
		return err
	}

	// Record the output of our build (success or failure)
	if len(output) == 0 {
		log.Printf("Build completed successfully in %s", time.Since(started).String())
	} else {
		log.Printf(string(output))
	}

	return nil

}

func buildLocalServer() {
	buildServer(localServerPath("."), nil)
}

func buildDeployServer() {
	env := append(os.Environ(), "GOOS=linux")
	env = append(env, "GOARCH=amd64")

	// When compiling with cgo, we get this error:
	// ./bin/server: error while loading shared libraries: /usr/lib/libSystem.B.dylib: cannot open shared object file: No such file or directory
	env = append(env, "CGO_ENABLED=0")
	buildServer(serverPath("."), env)
}
<commit_msg>Removed use of goimports, use go fmt only on build<commit_after>package main

import (
	"log"
	"os"
	"os/exec"
	"time"
)

// RunBuild builds the server (either for deployment or for local use)
func RunBuild(args []string) {
	// Remove fragmenta build from args list
	args = args[2:]

	if len(args) > 0 {
		buildDeployServer()
	} else {
		buildLocalServer()
	}

}

// buildServer removes the old binary and rebuilds the server
func buildServer(server string, env []string) error {

	// Remove old binary for server
	_, err := os.Stat(server)
	if err == nil {
		err = os.Remove(server)
		if err != nil {
			log.Printf("Error removing server %s", err)
		}
	}

	// Run go fmt on any packages below root
	srcPath := "./..."
	log.Printf("Running go fmt at %s", srcPath)
	result, err := runCommand("go", "fmt", srcPath)
	if err != nil {
		log.Printf("Error running fmt %s", err)
		return err
	}
	if len(result) > 0 {
		log.Printf(string(result))
	}

	// Build new binary for server
	log.Printf("Building server at %s", server)
	started := time.Now()

	args := []string{"build", "-o", server, serverCompilePath(".")}

	// If a local build with no environment settings, use go build -i
	if len(env) == 0 {
		args = []string{"build", "-i", "-o", server, serverCompilePath(".")}
	}

	// Call go build with -i to install artefacts for local builds, and -o to output to ./bin
	// log.Printf("  %s", args)
	cmd := exec.Command("go", args...)
	cmd.Stderr = os.Stdout

	if env != nil {
		cmd.Env = env
	}

	output, err := cmd.Output()
	if err != nil {
		log.Printf("Error running build %s\n%s", err, string(output))
		return err
	}

	// Record the output of our build (success or failure)
	if len(output) == 0 {
		log.Printf("Build completed successfully in %s", time.Since(started).String())
	} else {
		log.Printf(string(output))
	}

	return nil

}

func buildLocalServer() {
	buildServer(localServerPath("."), nil)
}

func buildDeployServer() {
	env := append(os.Environ(), "GOOS=linux")
	env = append(env, "GOARCH=amd64")

	// When compiling with cgo, we get this error:
	// ./bin/server: error while loading shared libraries: /usr/lib/libSystem.B.dylib: cannot open shared object file: No such file or directory
	env = append(env, "CGO_ENABLED=0")
	buildServer(serverPath("."), env)
}
<|endoftext|>


<commit_before>// Package token contains CLI client token related helpers
package token

import (
	"fmt"
	"strconv"
	"time"

	"github.com/micro/go-micro/v2/auth"
	"github.com/micro/micro/v2/internal/config"
)

// Get tries a best effort read of auth token from user config.
// Might have missing `RefreshToken` or `Expiry` fields in case of
// incomplete or corrupted user config.
func Get(envName string) (*auth.Token, error) {
	path := []string{"micro", "auth", envName}
	accessToken, _ := config.Get(append(path, "token")...)

	refreshToken, err := config.Get(append(path, "refresh-token")...)
	if err != nil {
		// Gracefully degrading here in case the user only has a temporary access token at hand.
		// The call will fail on the receiving end.
		return &auth.Token{
			AccessToken: accessToken,
		}, nil
	}

	// See if the access token has expired
	expiry, _ := config.Get(append(path, "expiry")...)
	if len(expiry) == 0 {
		return &auth.Token{
			AccessToken:  accessToken,
			RefreshToken: refreshToken,
		}, nil
	}
	expiryInt, err := strconv.ParseInt(expiry, 10, 64)
	if err != nil {
		return nil, err
	}
	return &auth.Token{
		AccessToken:  accessToken,
		RefreshToken: refreshToken,
		Expiry:       time.Unix(expiryInt, 0),
	}, nil
}

// Save saves the auth token to the user's local config file
func Save(envName string, token *auth.Token) error {
	if err := config.Set(token.AccessToken, "micro", "auth", envName, "token"); err != nil {
		return err
	}
	// Store the refresh token in micro config
	if err := config.Set(token.RefreshToken, "micro", "auth", envName, "refresh-token"); err != nil {
		return err
	}
	// Store the refresh token in micro config
	return config.Set(fmt.Sprintf("%v", token.Expiry.Unix()), "micro", "auth", envName, "expiry")
}
<commit_msg>client/cli/token: add remove func<commit_after>// Package token contains CLI client token related helpers
package token

import (
	"fmt"
	"strconv"
	"time"

	"github.com/micro/go-micro/v2/auth"
	"github.com/micro/micro/v2/internal/config"
)

// Get tries a best effort read of auth token from user config.
// Might have missing `RefreshToken` or `Expiry` fields in case of
// incomplete or corrupted user config.
func Get(envName string) (*auth.Token, error) {
	path := []string{"micro", "auth", envName}
	accessToken, _ := config.Get(append(path, "token")...)

	refreshToken, err := config.Get(append(path, "refresh-token")...)
	if err != nil {
		// Gracefully degrading here in case the user only has a temporary access token at hand.
		// The call will fail on the receiving end.
		return &auth.Token{
			AccessToken: accessToken,
		}, nil
	}

	// See if the access token has expired
	expiry, _ := config.Get(append(path, "expiry")...)
	if len(expiry) == 0 {
		return &auth.Token{
			AccessToken:  accessToken,
			RefreshToken: refreshToken,
		}, nil
	}
	expiryInt, err := strconv.ParseInt(expiry, 10, 64)
	if err != nil {
		return nil, err
	}
	return &auth.Token{
		AccessToken:  accessToken,
		RefreshToken: refreshToken,
		Expiry:       time.Unix(expiryInt, 0),
	}, nil
}

// Save saves the auth token to the user's local config file
func Save(envName string, token *auth.Token) error {
	if err := config.Set(token.AccessToken, "micro", "auth", envName, "token"); err != nil {
		return err
	}
	// Store the refresh token in micro config
	if err := config.Set(token.RefreshToken, "micro", "auth", envName, "refresh-token"); err != nil {
		return err
	}
	// Store the refresh token in micro config
	return config.Set(fmt.Sprintf("%v", token.Expiry.Unix()), "micro", "auth", envName, "expiry")
}

// Remove deletes a token. Useful when trying to reset test
// for example at testing: not having a token is a different state
// than having an invalid token.
func Remove(envName string) error {
	if err := config.Set("", "micro", "auth", envName, "token"); err != nil {
		return err
	}
	// Store the refresh token in micro config
	if err := config.Set("", "micro", "auth", envName, "refresh-token"); err != nil {
		return err
	}
	// Store the refresh token in micro config
	return config.Set("", "micro", "auth", envName, "expiry")
}
<|endoftext|>


<commit_before>// errchk $G $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

type I1 interface { I2 }	// ERROR "interface"
type I2 int

type I3 interface { int }	// ERROR "interface"
<commit_msg>more interface embedding tests 6g's error for I4 is not quite accurate (it calls I4 a non-interface) but also not a crash or infinite loop.<commit_after>// errchk $G $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

type I1 interface { I2 }	// ERROR "interface"
type I2 int

type I3 interface { int }	// ERROR "interface"

type S struct {
	x interface{ S }	// ERROR "interface"
}
type I4 interface {
	I4	// ERROR "interface"
}

type I5 interface {
	I6
}

type I6 interface {
	I5	// ERROR "interface"
}
<|endoftext|>


<commit_before>/*
** Copyright [2013-2016] [Megam Systems]
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
** http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
 */
package bills

import (
	"encoding/json"
	"fmt"
	"github.com/megamsys/libgo/api"
	"gopkg.in/yaml.v2"
	"strconv"
	"time"
)

const (
	UPDATE        = "/update"
	BALANCE       = "/balances/"
	EVENTBALANCEJSON = "Megam::Balances"
)

type BalanceOpts struct {
	Id       string
	Consumed string
}

type ApiBalances struct {
	JsonClaz string     `json:"json_claz" cql:"json_claz"`
	Results  []Balances `json:"results" json:"results"`
}
type Balances struct {
	Id        string    `json:"id" cql:"id"`
	AccountId string    `json:"account_id" cql:"account_id"`
	Credit    string    `json:"credit" cql:"credit"`
	JsonClaz  string    `json:"json_claz" cql:"json_claz"`
}

type updateBalances struct {
	Id        string    `json:"id" cql:"id"`
	AccountId string    `json:"account_id" cql:"account_id"`
	Credit    string    `json:"credit" cql:"credit"`
	JsonClaz  string    `json:"json_claz" cql:"json_claz"`
	CreatedAt time.Time `json:"created_at" cql:"created_at"`
	UpdatedAt time.Time `json:"updated_at" cql:"updated_at"`
}

func (b *Balances) String() string {
	if d, err := yaml.Marshal(b); err != nil {
		return err.Error()
	} else {
		return string(d)
	}
}

//Temporary hack to create an assembly from its id.
//This is used by SetStatus.
//We need add a Notifier interface duck typed by Box and Carton ?
func NewBalances(email string, m map[string]string) (*Balances, error) {
	// Here skips balances fetching for the VMs which is launched on opennebula,
	// that does not have records on vertice database
	if email == "" {
		return nil, fmt.Errorf("account_id should not be empty")
	}

	args := api.NewArgs(m)
	cl := api.NewClient(args, BALANCE + email)
	response, err := cl.Get()
	if err != nil {
		return nil, err
	}

	ac := &ApiBalances{}
	err = json.Unmarshal(response, ac)
	if err != nil {
		return nil, err
	}
	b := &ac.Results[0]

	return b, nil
}

func (b *Balances) Deduct(bopts *BalanceOpts, m map[string]string) error {

	avail, err := strconv.ParseFloat(b.Credit, 64)
	if err != nil {
		return err
	}

	consume, cerr := strconv.ParseFloat(bopts.Consumed, 64)
	if cerr != nil {
		return cerr
	}

	b.Credit = strconv.FormatFloat(avail-consume, 'f', 6, 64)

	args := api.NewArgs(m)
	cl := api.NewClient(args,BALANCE + "bill")
	_, err = cl.Post(b.toUpdate())
	if err != nil {
		return err
	}
	return nil
}

func (b *Balances) toUpdate() *updateBalances {
 return &updateBalances{
	 Id:        b.Id,
	 AccountId: b.AccountId,
	 Credit:    b.Credit,
	 JsonClaz:  b.JsonClaz,
	 CreatedAt: time.Now(),
	 UpdatedAt: time.Now(),
 }
}
<commit_msg>vm billing change<commit_after>/*
** Copyright [2013-2016] [Megam Systems]
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
** http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
 */
package bills

import (
	"encoding/json"
	"fmt"
	"github.com/megamsys/libgo/api"
	"gopkg.in/yaml.v2"
	"strconv"
	"time"
)

const (
	UPDATE        = "/update"
	BALANCE       = "/balances/"
	EVENTBALANCEJSON = "Megam::Balances"
)

type BalanceOpts struct {
	Id       string
	Consumed string
}

type ApiBalances struct {
	JsonClaz string     `json:"json_claz" cql:"json_claz"`
	Results  []Balances `json:"results" json:"results"`
}
type Balances struct {
	Id        string    `json:"id" cql:"id"`
	AccountId string    `json:"account_id" cql:"account_id"`
	Credit    string    `json:"credit" cql:"credit"`
	JsonClaz  string    `json:"json_claz" cql:"json_claz"`
}

type updateBalances struct {
	Id        string    `json:"id" cql:"id"`
	AccountId string    `json:"account_id" cql:"account_id"`
	Credit    string    `json:"credit" cql:"credit"`
	JsonClaz  string    `json:"json_claz" cql:"json_claz"`
	CreatedAt time.Time `json:"created_at" cql:"created_at"`
	UpdatedAt time.Time `json:"updated_at" cql:"updated_at"`
}

func (b *Balances) String() string {
	if d, err := yaml.Marshal(b); err != nil {
		return err.Error()
	} else {
		return string(d)
	}
}

//Temporary hack to create an assembly from its id.
//This is used by SetStatus.
//We need add a Notifier interface duck typed by Box and Carton ?
func NewBalances(email string, m map[string]string) (*Balances, error) {
	// Here skips balances fetching for the VMs which is launched on opennebula,
	// that does not have records on vertice database
	if email == "" {
		return nil, fmt.Errorf("account_id should not be empty")
	}

	args := api.NewArgs(m)
	cl := api.NewClient(args, BALANCE + email)
	response, err := cl.Get()
	if err != nil {
		return nil, err
	}

	ac := &ApiBalances{}
	err = json.Unmarshal(response, ac)
	if err != nil {
		return nil, err
	}
	b := &ac.Results[0]

	return b, nil
}

func (b *Balances) Deduct(bopts *BalanceOpts, m map[string]string) error {

	avail, err := strconv.ParseFloat(b.Credit, 64)
	if err != nil {
		return err
	}

	consume, cerr := strconv.ParseFloat(bopts.Consumed, 64)
	if cerr != nil {
		return cerr
	}

	b.Credit = strconv.FormatFloat(avail-consume, 'f', 6, 64)

	args := api.NewArgs(m)
        args.Email = bopts.Id
	cl := api.NewClient(args,BALANCE + "bill")
	_, err = cl.Post(b.toUpdate())
	if err != nil {
		return err
	}
	return nil
}

func (b *Balances) toUpdate() *updateBalances {
 return &updateBalances{
	 Id:        b.Id,
	 AccountId: b.AccountId,
	 Credit:    b.Credit,
	 JsonClaz:  b.JsonClaz,
	 CreatedAt: time.Now(),
	 UpdatedAt: time.Now(),
 }
}
<|endoftext|>


<commit_before>package network

import (
	"encoding/json"
	"fmt"
	"net/http"

	"golang.org/x/net/context"

	"github.com/docker/docker/api/server/httputils"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/filters"
	"github.com/docker/engine-api/types/network"
	"github.com/docker/libnetwork"
)

func (n *networkRouter) getNetworksList(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	filter := r.Form.Get("filters")
	netFilters, err := filters.FromParam(filter)
	if err != nil {
		return err
	}

	list := []types.NetworkResource{}

	if nr, err := n.clusterProvider.GetNetworks(); err == nil {
		for _, nw := range nr {
			list = append(list, nw)
		}
	}

	// Combine the network list returned by Docker daemon if it is not already
	// returned by the cluster manager
SKIP:
	for _, nw := range n.backend.GetNetworks() {
		for _, nl := range list {
			if nl.ID == nw.ID() {
				continue SKIP
			}
		}
		list = append(list, *n.buildNetworkResource(nw))
	}

	list, err = filterNetworks(list, netFilters)
	if err != nil {
		return err
	}
	return httputils.WriteJSON(w, http.StatusOK, list)
}

func (n *networkRouter) getNetwork(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	nw, err := n.backend.FindNetwork(vars["id"])
	if err != nil {
		if nr, err := n.clusterProvider.GetNetwork(vars["id"]); err == nil {
			return httputils.WriteJSON(w, http.StatusOK, nr)
		}
		return err
	}
	return httputils.WriteJSON(w, http.StatusOK, n.buildNetworkResource(nw))
}

func (n *networkRouter) postNetworkCreate(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	var create types.NetworkCreateRequest

	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	if err := json.NewDecoder(r.Body).Decode(&create); err != nil {
		return err
	}

	if _, err := n.clusterProvider.GetNetwork(create.Name); err == nil {
		return libnetwork.NetworkNameError(create.Name)
	}

	nw, err := n.backend.CreateNetwork(create)
	if err != nil {
		if _, ok := err.(libnetwork.ManagerRedirectError); !ok {
			return err
		}
		id, err := n.clusterProvider.CreateNetwork(create)
		if err != nil {
			return err
		}
		nw = &types.NetworkCreateResponse{ID: id}
	}

	return httputils.WriteJSON(w, http.StatusCreated, nw)
}

func (n *networkRouter) postNetworkConnect(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	var connect types.NetworkConnect
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	if err := json.NewDecoder(r.Body).Decode(&connect); err != nil {
		return err
	}

	nw, err := n.backend.FindNetwork(vars["id"])
	if err != nil {
		return err
	}

	if nw.Info().Dynamic() {
		return newNetworkForbiddenError("operation not supported for swarm scoped networks")
	}

	return n.backend.ConnectContainerToNetwork(connect.Container, nw.Name(), connect.EndpointConfig)
}

func (n *networkRouter) postNetworkDisconnect(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	var disconnect types.NetworkDisconnect
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	if err := json.NewDecoder(r.Body).Decode(&disconnect); err != nil {
		return err
	}

	nw, err := n.backend.FindNetwork(vars["id"])
	if err != nil {
		return err
	}

	if nw.Info().Dynamic() {
		return newNetworkForbiddenError("operation not supported for swarm scoped networks")
	}

	return n.backend.DisconnectContainerFromNetwork(disconnect.Container, nw, disconnect.Force)
}

func (n *networkRouter) deleteNetwork(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}
	if _, err := n.clusterProvider.GetNetwork(vars["id"]); err == nil {
		return n.clusterProvider.RemoveNetwork(vars["id"])
	}
	if err := n.backend.DeleteNetwork(vars["id"]); err != nil {
		return err
	}
	w.WriteHeader(http.StatusNoContent)
	return nil
}

func (n *networkRouter) buildNetworkResource(nw libnetwork.Network) *types.NetworkResource {
	r := &types.NetworkResource{}
	if nw == nil {
		return r
	}

	info := nw.Info()
	r.Name = nw.Name()
	r.ID = nw.ID()
	r.Scope = info.Scope()
	if n.clusterProvider.IsManager() {
		if _, err := n.clusterProvider.GetNetwork(nw.Name()); err == nil {
			r.Scope = "swarm"
		}
	} else if info.Dynamic() {
		r.Scope = "swarm"
	}
	r.Driver = nw.Type()
	r.EnableIPv6 = info.IPv6Enabled()
	r.Internal = info.Internal()
	r.Options = info.DriverOptions()
	r.Containers = make(map[string]types.EndpointResource)
	buildIpamResources(r, info)
	r.Internal = info.Internal()
	r.Labels = info.Labels()

	epl := nw.Endpoints()
	for _, e := range epl {
		ei := e.Info()
		if ei == nil {
			continue
		}
		sb := ei.Sandbox()
		key := "ep-" + e.ID()
		if sb != nil {
			key = sb.ContainerID()
		}

		r.Containers[key] = buildEndpointResource(e)
	}
	return r
}

func buildIpamResources(r *types.NetworkResource, nwInfo libnetwork.NetworkInfo) {
	id, opts, ipv4conf, ipv6conf := nwInfo.IpamConfig()

	ipv4Info, ipv6Info := nwInfo.IpamInfo()

	r.IPAM.Driver = id

	r.IPAM.Options = opts

	r.IPAM.Config = []network.IPAMConfig{}
	for _, ip4 := range ipv4conf {
		if ip4.PreferredPool == "" {
			continue
		}
		iData := network.IPAMConfig{}
		iData.Subnet = ip4.PreferredPool
		iData.IPRange = ip4.SubPool
		iData.Gateway = ip4.Gateway
		iData.AuxAddress = ip4.AuxAddresses
		r.IPAM.Config = append(r.IPAM.Config, iData)
	}

	if len(r.IPAM.Config) == 0 {
		for _, ip4Info := range ipv4Info {
			iData := network.IPAMConfig{}
			iData.Subnet = ip4Info.IPAMData.Pool.String()
			iData.Gateway = ip4Info.IPAMData.Gateway.String()
			r.IPAM.Config = append(r.IPAM.Config, iData)
		}
	}

	hasIpv6Conf := false
	for _, ip6 := range ipv6conf {
		if ip6.PreferredPool == "" {
			continue
		}
		hasIpv6Conf = true
		iData := network.IPAMConfig{}
		iData.Subnet = ip6.PreferredPool
		iData.IPRange = ip6.SubPool
		iData.Gateway = ip6.Gateway
		iData.AuxAddress = ip6.AuxAddresses
		r.IPAM.Config = append(r.IPAM.Config, iData)
	}

	if !hasIpv6Conf {
		for _, ip6Info := range ipv6Info {
			iData := network.IPAMConfig{}
			iData.Subnet = ip6Info.IPAMData.Pool.String()
			iData.Gateway = ip6Info.IPAMData.Gateway.String()
			r.IPAM.Config = append(r.IPAM.Config, iData)
		}
	}
}

func buildEndpointResource(e libnetwork.Endpoint) types.EndpointResource {
	er := types.EndpointResource{}
	if e == nil {
		return er
	}

	er.EndpointID = e.ID()
	er.Name = e.Name()
	ei := e.Info()
	if ei == nil {
		return er
	}

	if iface := ei.Iface(); iface != nil {
		if mac := iface.MacAddress(); mac != nil {
			er.MacAddress = mac.String()
		}
		if ip := iface.Address(); ip != nil && len(ip.IP) > 0 {
			er.IPv4Address = ip.String()
		}

		if ipv6 := iface.AddressIPv6(); ipv6 != nil && len(ipv6.IP) > 0 {
			er.IPv6Address = ipv6.String()
		}
	}
	return er
}

// networkForbiddenError represents an authorization deny error
type networkForbiddenError struct {
	error
}

// HTTPErrorStatusCode returns the authorization error status code (forbidden)
func (e networkForbiddenError) HTTPErrorStatusCode() int {
	return http.StatusForbidden
}

func newNetworkForbiddenError(msg string) networkForbiddenError {
	return networkForbiddenError{error: fmt.Errorf("%s", msg)}
}
<commit_msg>make network errors less DRY<commit_after>package network

import (
	"encoding/json"
	"fmt"
	"net/http"

	"golang.org/x/net/context"

	"github.com/docker/docker/api/server/httputils"
	"github.com/docker/docker/errors"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/filters"
	"github.com/docker/engine-api/types/network"
	"github.com/docker/libnetwork"
)

func (n *networkRouter) getNetworksList(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	filter := r.Form.Get("filters")
	netFilters, err := filters.FromParam(filter)
	if err != nil {
		return err
	}

	list := []types.NetworkResource{}

	if nr, err := n.clusterProvider.GetNetworks(); err == nil {
		for _, nw := range nr {
			list = append(list, nw)
		}
	}

	// Combine the network list returned by Docker daemon if it is not already
	// returned by the cluster manager
SKIP:
	for _, nw := range n.backend.GetNetworks() {
		for _, nl := range list {
			if nl.ID == nw.ID() {
				continue SKIP
			}
		}
		list = append(list, *n.buildNetworkResource(nw))
	}

	list, err = filterNetworks(list, netFilters)
	if err != nil {
		return err
	}
	return httputils.WriteJSON(w, http.StatusOK, list)
}

func (n *networkRouter) getNetwork(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	nw, err := n.backend.FindNetwork(vars["id"])
	if err != nil {
		if nr, err := n.clusterProvider.GetNetwork(vars["id"]); err == nil {
			return httputils.WriteJSON(w, http.StatusOK, nr)
		}
		return err
	}
	return httputils.WriteJSON(w, http.StatusOK, n.buildNetworkResource(nw))
}

func (n *networkRouter) postNetworkCreate(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	var create types.NetworkCreateRequest

	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	if err := json.NewDecoder(r.Body).Decode(&create); err != nil {
		return err
	}

	if _, err := n.clusterProvider.GetNetwork(create.Name); err == nil {
		return libnetwork.NetworkNameError(create.Name)
	}

	nw, err := n.backend.CreateNetwork(create)
	if err != nil {
		if _, ok := err.(libnetwork.ManagerRedirectError); !ok {
			return err
		}
		id, err := n.clusterProvider.CreateNetwork(create)
		if err != nil {
			return err
		}
		nw = &types.NetworkCreateResponse{ID: id}
	}

	return httputils.WriteJSON(w, http.StatusCreated, nw)
}

func (n *networkRouter) postNetworkConnect(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	var connect types.NetworkConnect
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	if err := json.NewDecoder(r.Body).Decode(&connect); err != nil {
		return err
	}

	nw, err := n.backend.FindNetwork(vars["id"])
	if err != nil {
		return err
	}

	if nw.Info().Dynamic() {
		err := fmt.Errorf("operation not supported for swarm scoped networks")
		return errors.NewRequestForbiddenError(err)
	}

	return n.backend.ConnectContainerToNetwork(connect.Container, nw.Name(), connect.EndpointConfig)
}

func (n *networkRouter) postNetworkDisconnect(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	var disconnect types.NetworkDisconnect
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	if err := json.NewDecoder(r.Body).Decode(&disconnect); err != nil {
		return err
	}

	nw, err := n.backend.FindNetwork(vars["id"])
	if err != nil {
		return err
	}

	if nw.Info().Dynamic() {
		err := fmt.Errorf("operation not supported for swarm scoped networks")
		return errors.NewRequestForbiddenError(err)
	}

	return n.backend.DisconnectContainerFromNetwork(disconnect.Container, nw, disconnect.Force)
}

func (n *networkRouter) deleteNetwork(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}
	if _, err := n.clusterProvider.GetNetwork(vars["id"]); err == nil {
		return n.clusterProvider.RemoveNetwork(vars["id"])
	}
	if err := n.backend.DeleteNetwork(vars["id"]); err != nil {
		return err
	}
	w.WriteHeader(http.StatusNoContent)
	return nil
}

func (n *networkRouter) buildNetworkResource(nw libnetwork.Network) *types.NetworkResource {
	r := &types.NetworkResource{}
	if nw == nil {
		return r
	}

	info := nw.Info()
	r.Name = nw.Name()
	r.ID = nw.ID()
	r.Scope = info.Scope()
	if n.clusterProvider.IsManager() {
		if _, err := n.clusterProvider.GetNetwork(nw.Name()); err == nil {
			r.Scope = "swarm"
		}
	} else if info.Dynamic() {
		r.Scope = "swarm"
	}
	r.Driver = nw.Type()
	r.EnableIPv6 = info.IPv6Enabled()
	r.Internal = info.Internal()
	r.Options = info.DriverOptions()
	r.Containers = make(map[string]types.EndpointResource)
	buildIpamResources(r, info)
	r.Internal = info.Internal()
	r.Labels = info.Labels()

	epl := nw.Endpoints()
	for _, e := range epl {
		ei := e.Info()
		if ei == nil {
			continue
		}
		sb := ei.Sandbox()
		key := "ep-" + e.ID()
		if sb != nil {
			key = sb.ContainerID()
		}

		r.Containers[key] = buildEndpointResource(e)
	}
	return r
}

func buildIpamResources(r *types.NetworkResource, nwInfo libnetwork.NetworkInfo) {
	id, opts, ipv4conf, ipv6conf := nwInfo.IpamConfig()

	ipv4Info, ipv6Info := nwInfo.IpamInfo()

	r.IPAM.Driver = id

	r.IPAM.Options = opts

	r.IPAM.Config = []network.IPAMConfig{}
	for _, ip4 := range ipv4conf {
		if ip4.PreferredPool == "" {
			continue
		}
		iData := network.IPAMConfig{}
		iData.Subnet = ip4.PreferredPool
		iData.IPRange = ip4.SubPool
		iData.Gateway = ip4.Gateway
		iData.AuxAddress = ip4.AuxAddresses
		r.IPAM.Config = append(r.IPAM.Config, iData)
	}

	if len(r.IPAM.Config) == 0 {
		for _, ip4Info := range ipv4Info {
			iData := network.IPAMConfig{}
			iData.Subnet = ip4Info.IPAMData.Pool.String()
			iData.Gateway = ip4Info.IPAMData.Gateway.String()
			r.IPAM.Config = append(r.IPAM.Config, iData)
		}
	}

	hasIpv6Conf := false
	for _, ip6 := range ipv6conf {
		if ip6.PreferredPool == "" {
			continue
		}
		hasIpv6Conf = true
		iData := network.IPAMConfig{}
		iData.Subnet = ip6.PreferredPool
		iData.IPRange = ip6.SubPool
		iData.Gateway = ip6.Gateway
		iData.AuxAddress = ip6.AuxAddresses
		r.IPAM.Config = append(r.IPAM.Config, iData)
	}

	if !hasIpv6Conf {
		for _, ip6Info := range ipv6Info {
			iData := network.IPAMConfig{}
			iData.Subnet = ip6Info.IPAMData.Pool.String()
			iData.Gateway = ip6Info.IPAMData.Gateway.String()
			r.IPAM.Config = append(r.IPAM.Config, iData)
		}
	}
}

func buildEndpointResource(e libnetwork.Endpoint) types.EndpointResource {
	er := types.EndpointResource{}
	if e == nil {
		return er
	}

	er.EndpointID = e.ID()
	er.Name = e.Name()
	ei := e.Info()
	if ei == nil {
		return er
	}

	if iface := ei.Iface(); iface != nil {
		if mac := iface.MacAddress(); mac != nil {
			er.MacAddress = mac.String()
		}
		if ip := iface.Address(); ip != nil && len(ip.IP) > 0 {
			er.IPv4Address = ip.String()
		}

		if ipv6 := iface.AddressIPv6(); ipv6 != nil && len(ipv6.IP) > 0 {
			er.IPv6Address = ipv6.String()
		}
	}
	return er
}
<|endoftext|>


<commit_before>package util

import (
	"container/heap"
)

// キャッシュ。

type Cache interface {
	// 入れる。
	Put(key, val, prio interface{})
	// 取り出す。
	Get(key interface{}) (val, prio interface{})
	// 優先度を変えつつ取り出す。
	Update(key, prio interface{}) (val interface{})
	// 基準以下を削除。
	CleanLesser(prioThres interface{})
}

func NewCache(less func(interface{}, interface{}) bool) Cache {
	ca := &cache{less, []*cacheElement{}, map[interface{}]int{}}
	heap.Init(ca)
	return ca
}

type cache struct {
	// Cache と heap.Interface を実装する。
	// Cache からは参照だけ直接行い、操作は heap.Interface 側でやる。

	less      func(interface{}, interface{}) bool
	prioQueue []*cacheElement
	keyToIdx  map[interface{}]int
}

func (ca *cache) Put(key, val, prio interface{}) {
	ca.Push(&cacheElement{key, val, prio})
}

func (ca *cache) Get(key interface{}) (val, prio interface{}) {
	idx, ok := ca.keyToIdx[key]
	if !ok {
		return nil, nil
	}
	elem := ca.prioQueue[idx]
	return elem.val, elem.prio
}

func (ca *cache) Update(key, prio interface{}) (val interface{}) {
	idx, ok := ca.keyToIdx[key]
	if !ok {
		return nil
	}
	elem := ca.prioQueue[idx]
	elem.prio = prio
	heap.Fix(ca, idx)
	return elem.val
}

func (ca *cache) CleanLesser(prioThres interface{}) {
	for ca.Len() > 0 {
		if ca.less(ca.prioQueue[0].prio, prioThres) {
			heap.Pop(ca)
		} else {
			break
		}
	}
}

type cacheElement struct {
	key  interface{}
	val  interface{}
	prio interface{}
}

func (ca *cache) Len() int {
	return len(ca.prioQueue)
}

func (ca *cache) Less(i, j int) bool {
	return ca.less(ca.prioQueue[i].prio, ca.prioQueue[j].prio)
}
func (ca *cache) Swap(i, j int) {
	ca.prioQueue[i], ca.prioQueue[j] = ca.prioQueue[j], ca.prioQueue[i]
	ca.keyToIdx[ca.prioQueue[i].key], ca.keyToIdx[ca.prioQueue[j].key] = i, j
}

func (ca *cache) Push(x interface{}) {
	elem := x.(*cacheElement)
	ca.prioQueue = append(ca.prioQueue, elem)
	ca.keyToIdx[elem.key] = len(ca.prioQueue) - 1
}

func (ca *cache) Pop() interface{} {
	n := len(ca.prioQueue)
	elem := ca.prioQueue[n-1]
	ca.prioQueue = ca.prioQueue[:n-1]
	delete(ca.keyToIdx, elem.key)
	return elem
}
<commit_msg>IdP 用のドライバを追加<commit_after>package util

import (
	"container/heap"
)

// キャッシュ。
// スレッドセーフではない。

type Cache interface {
	// 入れる。
	Put(key, val, prio interface{})
	// 取り出す。
	Get(key interface{}) (val, prio interface{})
	// 優先度を変えつつ取り出す。LRU のとき使う。
	Update(key, prio interface{}) (val interface{})
	// 基準以下を削除。
	CleanLesser(prioThres interface{})
}

func NewCache(less func(interface{}, interface{}) bool) Cache {
	ca := &cache{less, []*cacheElement{}, map[interface{}]int{}}
	heap.Init(ca)
	return ca
}

type cache struct {
	// Cache と heap.Interface を実装する。
	// Cache からは参照だけ直接行い、操作は heap.Interface 側でやる。

	less      func(interface{}, interface{}) bool
	prioQueue []*cacheElement
	keyToIdx  map[interface{}]int
}

func (ca *cache) Put(key, val, prio interface{}) {
	ca.Push(&cacheElement{key, val, prio})
}

func (ca *cache) Get(key interface{}) (val, prio interface{}) {
	idx, ok := ca.keyToIdx[key]
	if !ok {
		return nil, nil
	}
	elem := ca.prioQueue[idx]
	return elem.val, elem.prio
}

func (ca *cache) Update(key, prio interface{}) (val interface{}) {
	idx, ok := ca.keyToIdx[key]
	if !ok {
		return nil
	}
	elem := ca.prioQueue[idx]
	elem.prio = prio
	heap.Fix(ca, idx)
	return elem.val
}

func (ca *cache) CleanLesser(prioThres interface{}) {
	for ca.Len() > 0 {
		if ca.less(ca.prioQueue[0].prio, prioThres) {
			heap.Pop(ca)
		} else {
			break
		}
	}
}

type cacheElement struct {
	key  interface{}
	val  interface{}
	prio interface{}
}

func (ca *cache) Len() int {
	return len(ca.prioQueue)
}

func (ca *cache) Less(i, j int) bool {
	return ca.less(ca.prioQueue[i].prio, ca.prioQueue[j].prio)
}
func (ca *cache) Swap(i, j int) {
	ca.prioQueue[i], ca.prioQueue[j] = ca.prioQueue[j], ca.prioQueue[i]
	ca.keyToIdx[ca.prioQueue[i].key], ca.keyToIdx[ca.prioQueue[j].key] = i, j
}

func (ca *cache) Push(x interface{}) {
	elem := x.(*cacheElement)
	ca.prioQueue = append(ca.prioQueue, elem)
	ca.keyToIdx[elem.key] = len(ca.prioQueue) - 1
}

func (ca *cache) Pop() interface{} {
	n := len(ca.prioQueue)
	elem := ca.prioQueue[n-1]
	ca.prioQueue = ca.prioQueue[:n-1]
	delete(ca.keyToIdx, elem.key)
	return elem
}
<|endoftext|>


<commit_before>// redis cache backend implementation for persistent storage

package main

import (
	"bytes"
	"fmt"
	"os"
	"strconv"

	log "github.com/Sirupsen/logrus"
	"github.com/boltdb/bolt"
	"github.com/garyburd/redigo/redis"
)

const prefix = "genproxy:"
const requestsBucketName = "rqbucket"

type Cache struct {
	pool   *redis.Pool
	prefix string

	db             *bolt.DB
	requestsBucket []byte
}

// set records a key in cache (redis)
func (c *Cache) set(key string, value interface{}) error {

	if c.prefix == "" {
		c.prefix = prefix
	}

	client := c.pool.Get()
	defer client.Close()

	_, err := client.Do("SET", fmt.Sprintf(c.prefix+key), value)

	if err != nil {
		// getting payload for debugging
		var payload string
		switch v := value.(type) {
		case bytes.Buffer: // return as is
			payload = v.String() // Here v is of type bytes.Buffer
		}
		log.WithFields(log.Fields{
			"error":   err.Error(),
			"key":     fmt.Sprintf(c.prefix + key),
			"payload": payload,
		}).Error("Failed to SET key...")
	} else {
		log.WithFields(log.Fields{
			"key": fmt.Sprintf(c.prefix + key),
		}).Info("Key/value SET successfuly!")
	}

	return err
}

// getAllKeys returns all keys for specified (or default) prefix
func (c *Cache) getAllKeys() ([]string, error) {

	client := c.pool.Get()
	defer client.Close()

	values, err := redis.Strings(client.Do("KEYS", fmt.Sprintf(c.prefix+"*")))

	return values, err
}

// getAllValues returns values for specified keys
func (c *Cache) getAllValues(keys []string) ([]string, error) {

	log.WithFields(log.Fields{
		"keys": keys,
	}).Debug("Getting all supplied values")

	client := c.pool.Get()
	defer client.Close()

	// preparing keys
	var args []interface{}
	for _, k := range keys {
		args = append(args, k)
	}

	jsonStr, err := redis.Strings(client.Do("MGET", args...))

	log.WithFields(log.Fields{
		"keys": keys,
	}).Debug("Returning supplied values")

	return jsonStr, err

}

// get returns key from cache
func (c *Cache) get(key string) ([]byte, error) {

	client := c.pool.Get()
	defer client.Close()

	value, err := redis.Bytes(client.Do("GET", fmt.Sprintf(c.prefix+key)))

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
			"key":   fmt.Sprintf(c.prefix + key),
		}).Error("Failed to GET key...")
	} else {
		log.WithFields(log.Fields{
			"key": fmt.Sprintf(c.prefix + key),
		}).Debug("Key found!")
	}

	return value, err
}

// delete removes specified entry from cache
func (c *Cache) delete(key string) error {
	if c.prefix == "" {
		c.prefix = prefix
	}

	client := c.pool.Get()
	defer client.Close()

	_, err := client.Do("DEL", key)

	return err

}

// getRedisPool returns thread safe Redis connection pool
func getRedisPool() *redis.Pool {

	// getting redis connection
	maxConnections := 10
	mc := os.Getenv("MaxConnections")
	if mc != "" {
		maxCons, err := strconv.Atoi(mc)
		if err != nil {
			maxConnections = 10
		} else {
			maxConnections = maxCons
		}
	}

	// getting redis client for state storing
	redisPool := redis.NewPool(func() (redis.Conn, error) {
		c, err := redis.Dial("tcp", AppConfig.redisAddress)

		if err != nil {
			log.WithFields(log.Fields{
				"Error": err.Error(),
			}).Warn("Failed to create Redis connection pool!")
			return nil, err
		}
		if AppConfig.redisPassword != "" {
			if _, err := c.Do("AUTH", AppConfig.redisPassword); err != nil {
				log.WithFields(log.Fields{
					"Error":        err.Error(),
					"PasswordUsed": AppConfig.redisPassword,
				}).Warn("Failed to authenticate to Redis!")
				c.Close()
				return nil, err
			} else {
				log.Debug("Authenticated to Redis successfully! ")
			}
		}

		return c, err
	}, maxConnections)

	return redisPool
}
<commit_msg>bolt set method<commit_after>// redis cache backend implementation for persistent storage

package main

import (
	"bytes"
	"fmt"
	"os"
	"strconv"

	log "github.com/Sirupsen/logrus"
	"github.com/boltdb/bolt"
	"github.com/garyburd/redigo/redis"
)

const prefix = "genproxy:"
const requestsBucketName = "rqbucket"

type Cache struct {
	pool   *redis.Pool
	prefix string

	db             *bolt.DB
	requestsBucket []byte
}

// set records a key in cache (redis)
func (c *Cache) set(key string, value interface{}) error {

	if c.prefix == "" {
		c.prefix = prefix
	}

	client := c.pool.Get()
	defer client.Close()

	_, err := client.Do("SET", fmt.Sprintf(c.prefix+key), value)

	if err != nil {
		// getting payload for debugging
		var payload string
		switch v := value.(type) {
		case bytes.Buffer: // return as is
			payload = v.String() // Here v is of type bytes.Buffer
		}
		log.WithFields(log.Fields{
			"error":   err.Error(),
			"key":     fmt.Sprintf(c.prefix + key),
			"payload": payload,
		}).Error("Failed to SET key...")
	} else {
		log.WithFields(log.Fields{
			"key": fmt.Sprintf(c.prefix + key),
		}).Info("Key/value SET successfuly!")
	}

	return err
}

func (c *Cache) Set(key, value []byte) error {
	err := c.db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists(c.requestsBucket)
		if err != nil {
			return err
		}
		err = bucket.Put(key, value)
		if err != nil {
			return err
		}
		return nil
	})

	return err
}
// getAllKeys returns all keys for specified (or default) prefix
func (c *Cache) getAllKeys() ([]string, error) {

	client := c.pool.Get()
	defer client.Close()

	values, err := redis.Strings(client.Do("KEYS", fmt.Sprintf(c.prefix+"*")))

	return values, err
}

// getAllValues returns values for specified keys
func (c *Cache) getAllValues(keys []string) ([]string, error) {

	log.WithFields(log.Fields{
		"keys": keys,
	}).Debug("Getting all supplied values")

	client := c.pool.Get()
	defer client.Close()

	// preparing keys
	var args []interface{}
	for _, k := range keys {
		args = append(args, k)
	}

	jsonStr, err := redis.Strings(client.Do("MGET", args...))

	log.WithFields(log.Fields{
		"keys": keys,
	}).Debug("Returning supplied values")

	return jsonStr, err

}

// get returns key from cache
func (c *Cache) get(key string) ([]byte, error) {

	client := c.pool.Get()
	defer client.Close()

	value, err := redis.Bytes(client.Do("GET", fmt.Sprintf(c.prefix+key)))

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
			"key":   fmt.Sprintf(c.prefix + key),
		}).Error("Failed to GET key...")
	} else {
		log.WithFields(log.Fields{
			"key": fmt.Sprintf(c.prefix + key),
		}).Debug("Key found!")
	}

	return value, err
}

// delete removes specified entry from cache
func (c *Cache) delete(key string) error {
	if c.prefix == "" {
		c.prefix = prefix
	}

	client := c.pool.Get()
	defer client.Close()

	_, err := client.Do("DEL", key)

	return err

}

// getRedisPool returns thread safe Redis connection pool
func getRedisPool() *redis.Pool {

	// getting redis connection
	maxConnections := 10
	mc := os.Getenv("MaxConnections")
	if mc != "" {
		maxCons, err := strconv.Atoi(mc)
		if err != nil {
			maxConnections = 10
		} else {
			maxConnections = maxCons
		}
	}

	// getting redis client for state storing
	redisPool := redis.NewPool(func() (redis.Conn, error) {
		c, err := redis.Dial("tcp", AppConfig.redisAddress)

		if err != nil {
			log.WithFields(log.Fields{
				"Error": err.Error(),
			}).Warn("Failed to create Redis connection pool!")
			return nil, err
		}
		if AppConfig.redisPassword != "" {
			if _, err := c.Do("AUTH", AppConfig.redisPassword); err != nil {
				log.WithFields(log.Fields{
					"Error":        err.Error(),
					"PasswordUsed": AppConfig.redisPassword,
				}).Warn("Failed to authenticate to Redis!")
				c.Close()
				return nil, err
			} else {
				log.Debug("Authenticated to Redis successfully! ")
			}
		}

		return c, err
	}, maxConnections)

	return redisPool
}
<|endoftext|>


<commit_before>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package schema

import (
	"errors"
	"reflect"
	"strconv"
	"strings"
	"sync"
)

var invalidPath = errors.New("schema: invalid path")

// newCache returns a new cache.
func newCache() *cache {
	c := cache{
		m:       make(map[reflect.Type]*structInfo),
		conv:    make(map[reflect.Kind]Converter),
		regconv: make(map[reflect.Type]Converter),
		tag:     "schema",
	}
	for k, v := range converters {
		c.conv[k] = v
	}
	return &c
}

// cache caches meta-data about a struct.
type cache struct {
	l       sync.RWMutex
	m       map[reflect.Type]*structInfo
	conv    map[reflect.Kind]Converter
	regconv map[reflect.Type]Converter
	tag     string
}

// parsePath parses a path in dotted notation verifying that it is a valid
// path to a struct field.
//
// It returns "path parts" which contain indices to fields to be used by
// reflect.Value.FieldByString(). Multiple parts are required for slices of
// structs.
func (c *cache) parsePath(p string, t reflect.Type) ([]pathPart, error) {
	var struc *structInfo
	var field *fieldInfo
	var index64 int64
	var err error
	parts := make([]pathPart, 0)
	path := make([]string, 0)
	keys := strings.Split(p, ".")
	for i := 0; i < len(keys); i++ {
		if t.Kind() != reflect.Struct {
			return nil, invalidPath
		}
		if struc = c.get(t); struc == nil {
			return nil, invalidPath
		}
		if field = struc.get(keys[i]); field == nil {
			return nil, invalidPath
		}
		// Valid field. Append index.
		path = append(path, field.name)
		if field.ss {
			// Parse a special case: slices of structs.
			// i+1 must be the slice index.
			//
			// Now that struct can implements TextUnmarshaler interface,
			// we don't need to force the struct's fields to appear in the path.
			// So checking i+2 is not necessary anymore.
			i++
			if i+1 > len(keys) {
				return nil, invalidPath
			}
			if index64, err = strconv.ParseInt(keys[i], 10, 0); err != nil {
				return nil, invalidPath
			}
			parts = append(parts, pathPart{
				path:  path,
				field: field,
				index: int(index64),
			})
			path = make([]string, 0)

			// Get the next struct type, dropping ptrs.
			if field.typ.Kind() == reflect.Ptr {
				t = field.typ.Elem()
			} else {
				t = field.typ
			}
			if t.Kind() == reflect.Slice {
				t = t.Elem()
				if t.Kind() == reflect.Ptr {
					t = t.Elem()
				}
			}
		} else if field.typ.Kind() == reflect.Ptr {
			t = field.typ.Elem()
		} else {
			t = field.typ
		}
	}
	// Add the remaining.
	parts = append(parts, pathPart{
		path:  path,
		field: field,
		index: -1,
	})
	return parts, nil
}

// get returns a cached structInfo, creating it if necessary.
func (c *cache) get(t reflect.Type) *structInfo {
	c.l.RLock()
	info := c.m[t]
	c.l.RUnlock()
	if info == nil {
		info = c.create(t, nil)
		c.l.Lock()
		c.m[t] = info
		c.l.Unlock()
	}
	return info
}

// create creates a structInfo with meta-data about a struct.
func (c *cache) create(t reflect.Type, info *structInfo) *structInfo {
	if info == nil {
		info = &structInfo{fields: []*fieldInfo{}}
	}
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		if field.Anonymous {
			ft := field.Type
			if ft.Kind() == reflect.Ptr {
				ft = ft.Elem()
			}
			if ft.Kind() == reflect.Struct {
				bef := len(info.fields)
				c.create(ft, info)
				for _, fi := range info.fields[bef:len(info.fields)] {
					// exclude required check because duplicated to embedded field
					fi.required = false
				}
			}
		}
		c.createField(field, info)
	}
	return info
}

// createField creates a fieldInfo for the given field.
func (c *cache) createField(field reflect.StructField, info *structInfo) {
	alias, options := fieldAlias(field, c.tag)
	if alias == "-" {
		// Ignore this field.
		return
	}
	// Check if the type is supported and don't cache it if not.
	// First let's get the basic type.
	isSlice, isStruct := false, false
	ft := field.Type
	if ft.Kind() == reflect.Ptr {
		ft = ft.Elem()
	}
	if isSlice = ft.Kind() == reflect.Slice; isSlice {
		ft = ft.Elem()
		if ft.Kind() == reflect.Ptr {
			ft = ft.Elem()
		}
	}
	if ft.Kind() == reflect.Array {
		ft = ft.Elem()
		if ft.Kind() == reflect.Ptr {
			ft = ft.Elem()
		}
	}
	if isStruct = ft.Kind() == reflect.Struct; !isStruct {
		if conv := c.conv[ft.Kind()]; conv == nil {
			// Type is not supported.
			return
		}
	}

	info.fields = append(info.fields, &fieldInfo{
		typ:      field.Type,
		name:     field.Name,
		ss:       isSlice && isStruct,
		alias:    alias,
		anon:     field.Anonymous,
		required: options.Contains("required"),
	})
}

// converter returns the converter for a type.
func (c *cache) converter(t reflect.Type) Converter {
	conv := c.regconv[t]
	if conv == nil {
		conv = c.conv[t.Kind()]
	}
	return conv
}

// ----------------------------------------------------------------------------

type structInfo struct {
	fields []*fieldInfo
}

func (i *structInfo) get(alias string) *fieldInfo {
	for _, field := range i.fields {
		if strings.EqualFold(field.alias, alias) {
			return field
		}
	}
	return nil
}

type fieldInfo struct {
	typ      reflect.Type
	name     string // field name in the struct.
	ss       bool   // true if this is a slice of structs.
	alias    string
	anon     bool // is an embedded field
	required bool // tag option
}

type pathPart struct {
	field *fieldInfo
	path  []string // path to the field: walks structs using field names.
	index int      // struct index in slices of structs.
}

// ----------------------------------------------------------------------------

// fieldAlias parses a field tag to get a field alias.
func fieldAlias(field reflect.StructField, tagName string) (alias string, options tagOptions) {
	if tag := field.Tag.Get(tagName); tag != "" {
		alias, options = parseTag(tag)
	}
	if alias == "" {
		alias = field.Name
	}
	return alias, options
}

// tagOptions is the string following a comma in a struct field's tag, or
// the empty string. It does not include the leading comma.
type tagOptions []string

// parseTag splits a struct field's url tag into its name and comma-separated
// options.
func parseTag(tag string) (string, tagOptions) {
	s := strings.Split(tag, ",")
	return s[0], s[1:]
}

// Contains checks whether the tagOptions contains the specified option.
func (o tagOptions) Contains(option string) bool {
	for _, s := range o {
		if s == option {
			return true
		}
	}
	return false
}
<commit_msg>Checking for registered converters when creating structInfo.<commit_after>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package schema

import (
	"errors"
	"reflect"
	"strconv"
	"strings"
	"sync"
)

var invalidPath = errors.New("schema: invalid path")

// newCache returns a new cache.
func newCache() *cache {
	c := cache{
		m:       make(map[reflect.Type]*structInfo),
		conv:    make(map[reflect.Kind]Converter),
		regconv: make(map[reflect.Type]Converter),
		tag:     "schema",
	}
	for k, v := range converters {
		c.conv[k] = v
	}
	return &c
}

// cache caches meta-data about a struct.
type cache struct {
	l       sync.RWMutex
	m       map[reflect.Type]*structInfo
	conv    map[reflect.Kind]Converter
	regconv map[reflect.Type]Converter
	tag     string
}

// parsePath parses a path in dotted notation verifying that it is a valid
// path to a struct field.
//
// It returns "path parts" which contain indices to fields to be used by
// reflect.Value.FieldByString(). Multiple parts are required for slices of
// structs.
func (c *cache) parsePath(p string, t reflect.Type) ([]pathPart, error) {
	var struc *structInfo
	var field *fieldInfo
	var index64 int64
	var err error
	parts := make([]pathPart, 0)
	path := make([]string, 0)
	keys := strings.Split(p, ".")
	for i := 0; i < len(keys); i++ {
		if t.Kind() != reflect.Struct {
			return nil, invalidPath
		}
		if struc = c.get(t); struc == nil {
			return nil, invalidPath
		}
		if field = struc.get(keys[i]); field == nil {
			return nil, invalidPath
		}
		// Valid field. Append index.
		path = append(path, field.name)
		if field.ss {
			// Parse a special case: slices of structs.
			// i+1 must be the slice index.
			//
			// Now that struct can implements TextUnmarshaler interface,
			// we don't need to force the struct's fields to appear in the path.
			// So checking i+2 is not necessary anymore.
			i++
			if i+1 > len(keys) {
				return nil, invalidPath
			}
			if index64, err = strconv.ParseInt(keys[i], 10, 0); err != nil {
				return nil, invalidPath
			}
			parts = append(parts, pathPart{
				path:  path,
				field: field,
				index: int(index64),
			})
			path = make([]string, 0)

			// Get the next struct type, dropping ptrs.
			if field.typ.Kind() == reflect.Ptr {
				t = field.typ.Elem()
			} else {
				t = field.typ
			}
			if t.Kind() == reflect.Slice {
				t = t.Elem()
				if t.Kind() == reflect.Ptr {
					t = t.Elem()
				}
			}
		} else if field.typ.Kind() == reflect.Ptr {
			t = field.typ.Elem()
		} else {
			t = field.typ
		}
	}
	// Add the remaining.
	parts = append(parts, pathPart{
		path:  path,
		field: field,
		index: -1,
	})
	return parts, nil
}

// get returns a cached structInfo, creating it if necessary.
func (c *cache) get(t reflect.Type) *structInfo {
	c.l.RLock()
	info := c.m[t]
	c.l.RUnlock()
	if info == nil {
		info = c.create(t, nil)
		c.l.Lock()
		c.m[t] = info
		c.l.Unlock()
	}
	return info
}

// create creates a structInfo with meta-data about a struct.
func (c *cache) create(t reflect.Type, info *structInfo) *structInfo {
	if info == nil {
		info = &structInfo{fields: []*fieldInfo{}}
	}
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		if field.Anonymous {
			ft := field.Type
			if ft.Kind() == reflect.Ptr {
				ft = ft.Elem()
			}
			if ft.Kind() == reflect.Struct {
				bef := len(info.fields)
				c.create(ft, info)
				for _, fi := range info.fields[bef:len(info.fields)] {
					// exclude required check because duplicated to embedded field
					fi.required = false
				}
			}
		}
		c.createField(field, info)
	}
	return info
}

// createField creates a fieldInfo for the given field.
func (c *cache) createField(field reflect.StructField, info *structInfo) {
	alias, options := fieldAlias(field, c.tag)
	if alias == "-" {
		// Ignore this field.
		return
	}
	// Check if the type is supported and don't cache it if not.
	// First let's get the basic type.
	isSlice, isStruct := false, false
	ft := field.Type
	if ft.Kind() == reflect.Ptr {
		ft = ft.Elem()
	}
	if isSlice = ft.Kind() == reflect.Slice; isSlice {
		ft = ft.Elem()
		if ft.Kind() == reflect.Ptr {
			ft = ft.Elem()
		}
	}
	if ft.Kind() == reflect.Array {
		ft = ft.Elem()
		if ft.Kind() == reflect.Ptr {
			ft = ft.Elem()
		}
	}
	if isStruct = ft.Kind() == reflect.Struct; !isStruct {
		if conv := c.converter(ft); conv == nil {
			// Type is not supported.
			return
		}
	}

	info.fields = append(info.fields, &fieldInfo{
		typ:      field.Type,
		name:     field.Name,
		ss:       isSlice && isStruct,
		alias:    alias,
		anon:     field.Anonymous,
		required: options.Contains("required"),
	})
}

// converter returns the converter for a type.
func (c *cache) converter(t reflect.Type) Converter {
	conv := c.regconv[t]
	if conv == nil {
		conv = c.conv[t.Kind()]
	}
	return conv
}

// ----------------------------------------------------------------------------

type structInfo struct {
	fields []*fieldInfo
}

func (i *structInfo) get(alias string) *fieldInfo {
	for _, field := range i.fields {
		if strings.EqualFold(field.alias, alias) {
			return field
		}
	}
	return nil
}

type fieldInfo struct {
	typ      reflect.Type
	name     string // field name in the struct.
	ss       bool   // true if this is a slice of structs.
	alias    string
	anon     bool // is an embedded field
	required bool // tag option
}

type pathPart struct {
	field *fieldInfo
	path  []string // path to the field: walks structs using field names.
	index int      // struct index in slices of structs.
}

// ----------------------------------------------------------------------------

// fieldAlias parses a field tag to get a field alias.
func fieldAlias(field reflect.StructField, tagName string) (alias string, options tagOptions) {
	if tag := field.Tag.Get(tagName); tag != "" {
		alias, options = parseTag(tag)
	}
	if alias == "" {
		alias = field.Name
	}
	return alias, options
}

// tagOptions is the string following a comma in a struct field's tag, or
// the empty string. It does not include the leading comma.
type tagOptions []string

// parseTag splits a struct field's url tag into its name and comma-separated
// options.
func parseTag(tag string) (string, tagOptions) {
	s := strings.Split(tag, ",")
	return s[0], s[1:]
}

// Contains checks whether the tagOptions contains the specified option.
func (o tagOptions) Contains(option string) bool {
	for _, s := range o {
		if s == option {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 turbobytes
//
// This file is part of geoipdb, a library of GeoIP related helper functions
// for TurboBytes stack.
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package geoipdb

import (
	"time"
	"sync"
)

// cacheTTL is the expiration time of a cache entry.
const cacheTTL = time.Hour * 24

// cacheEntry is the data we want to keep cached.
type cacheEntry struct {
	// ASN number
	asn string
	// ASN description
	descr string
	// Due date of this entry
	due time.Time
}

// cache allows manipulating cached data.
type cache struct {
	// Concurrent access control to maps
	sync.RWMutex
	// IP to ASN data
	ip map[string]cacheEntry
	// ASN to IP list
	asn map[string]map[string]interface{}
}

// newCache returns an empty initialized cache.
func newCache() cache {
	return cache{
		ip:  make(map[string]cacheEntry),
		asn: make(map[string]map[string]interface{}),
	}
}

// store updates the cache.
func (c cache) store(ip string, asn string, descr string) {
	if ip == "" {
		return
	}
	c.Lock()
	defer c.Unlock()
	// Purge ASN map of given ip
	for _, ips := range c.asn {
		delete(ips, ip)
	}
	// Purge ASN map of empty entries
	for asn, ips := range c.asn {
		if len(ips) < 1 {
			delete(c.asn, asn)
		}
	}
	// Update IP map
	c.ip[ip] = cacheEntry{
		asn:   asn,
		descr: descr,
		due:   time.Now().Add(cacheTTL),
	}
	// Update ASN map
	if c.asn[asn] == nil {
		c.asn[asn] = make(map[string]interface{})
	}
	c.asn[asn][ip] = nil
}

// lookupByIP retrieves cached data by IP address.
//
// Returns
// the ASN identification and description,
// if cached data is expired,
// and if ip was found in cache.
func (c cache) lookupByIP(ip string) (asn string, descr string, expired bool, found bool) {
	c.RLock()
	defer c.RUnlock()
	entry, ok := c.ip[ip]
	if !ok {
		return "", "", false, false
	}
	return entry.asn, entry.descr, time.Now().After(entry.due), true
}

// lookupByASN retrieves the list of cached IPs associated with a given ASN.
//
// Returns a non nil list of IP addresses.
func (c cache) lookupByASN(asn string) map[string]interface{} {
	c.RLock()
	defer c.RUnlock()
	answer, ok := c.asn[asn]
	if !ok || answer == nil {
		return make(map[string]interface{})
	}
	return answer
}

// purgeASN removes from the cache all information related to a given ASN.
func (c cache) purgeASN(asn string) {
	c.Lock()
	defer c.Unlock()
	// Purge ip map of given asn
	for ip, entry := range c.ip {
		if entry.asn == asn {
			delete(c.ip, ip)
		}
	}
	// Purge asn map of given asn
	delete(c.asn, asn)
}

// purgeAll removes all entries from the cache
func (c cache) purgeAll() {
	c.Lock()
	defer c.Unlock()
	for ip, _ := range c.ip {
		delete(c.ip, ip)
	}
	for asn, _ := range c.asn {
		delete(c.asn, asn)
	}
}

// asnList retrieves all ASNs known to the cache.
//
// Returns a non nil list of ASNs.
func (c cache) asnList() []string {
	c.RLock()
	defer c.RUnlock()
	answer := make([]string, len(c.asn))
	var i int
	for asn := range c.asn {
		answer[i] = asn
		i++
	}
	return answer
}
<commit_msg>make cache concurrency safe fixes #24<commit_after>// Copyright (c) 2016 turbobytes
//
// This file is part of geoipdb, a library of GeoIP related helper functions
// for TurboBytes stack.
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package geoipdb

import (
	"sync"
	"time"
)

// cacheTTL is the expiration time of a cache entry.
const cacheTTL = time.Hour * 24

// cacheEntry is the data we want to keep cached.
type cacheEntry struct {
	// ASN number
	asn string
	// ASN description
	descr string
	// Due date of this entry
	due time.Time
}

// cache allows manipulating cached data.
type cache struct {
	// Concurrent access control to maps
	*sync.RWMutex
	// IP to ASN data
	ip map[string]cacheEntry
	// ASN to IP list
	asn map[string]map[string]interface{}
}

// newCache returns an empty initialized cache.
func newCache() cache {
	return cache{
		&sync.RWMutex{},
		make(map[string]cacheEntry),
		make(map[string]map[string]interface{}),
	}
}

// store updates the cache.
func (c cache) store(ip string, asn string, descr string) {
	if ip == "" {
		return
	}
	c.Lock()
	defer c.Unlock()
	// Purge ASN map of given ip
	for _, ips := range c.asn {
		delete(ips, ip)
	}
	// Purge ASN map of empty entries
	for asn, ips := range c.asn {
		if len(ips) < 1 {
			delete(c.asn, asn)
		}
	}
	// Update IP map
	c.ip[ip] = cacheEntry{
		asn:   asn,
		descr: descr,
		due:   time.Now().Add(cacheTTL),
	}
	// Update ASN map
	if c.asn[asn] == nil {
		c.asn[asn] = make(map[string]interface{})
	}
	c.asn[asn][ip] = nil
}

// lookupByIP retrieves cached data by IP address.
//
// Returns
// the ASN identification and description,
// if cached data is expired,
// and if ip was found in cache.
func (c cache) lookupByIP(ip string) (asn string, descr string, expired bool, found bool) {
	c.RLock()
	defer c.RUnlock()
	entry, ok := c.ip[ip]
	if !ok {
		return "", "", false, false
	}
	return entry.asn, entry.descr, time.Now().After(entry.due), true
}

// lookupByASN retrieves the list of cached IPs associated with a given ASN.
//
// Returns a non nil list of IP addresses.
func (c cache) lookupByASN(asn string) map[string]interface{} {
	c.RLock()
	defer c.RUnlock()
	answer, ok := c.asn[asn]
	if !ok || answer == nil {
		return make(map[string]interface{})
	}
	return answer
}

// purgeASN removes from the cache all information related to a given ASN.
func (c cache) purgeASN(asn string) {
	c.Lock()
	defer c.Unlock()
	// Purge ip map of given asn
	for ip, entry := range c.ip {
		if entry.asn == asn {
			delete(c.ip, ip)
		}
	}
	// Purge asn map of given asn
	delete(c.asn, asn)
}

// purgeAll removes all entries from the cache
func (c cache) purgeAll() {
	c.Lock()
	defer c.Unlock()
	for ip, _ := range c.ip {
		delete(c.ip, ip)
	}
	for asn, _ := range c.asn {
		delete(c.asn, asn)
	}
}

// asnList retrieves all ASNs known to the cache.
//
// Returns a non nil list of ASNs.
func (c cache) asnList() []string {
	c.RLock()
	defer c.RUnlock()
	answer := make([]string, len(c.asn))
	var i int
	for asn := range c.asn {
		answer[i] = asn
		i++
	}
	return answer
}
<|endoftext|>