<commit_before>package main

import (
	"archive/tar"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"time"

	"github.com/codegangsta/cli"
	"github.com/kr/pty"
	"github.com/mattn/go-shellwords"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/pkg/term"

	"github.com/cloudfoundry-incubator/garden"
	gclient "github.com/cloudfoundry-incubator/garden/client"
	gconn "github.com/cloudfoundry-incubator/garden/client/connection"
)

func handleComplete(c *cli.Context) {
	// This will complete if no args are passed
	if c.Args().Present() {
		return
	}

	containers, err := client(c).Containers(nil)
	failIf(err)

	for _, container := range containers {
		fmt.Println(container.Handle())
	}
}

func fail(err error) {
	fmt.Fprintln(os.Stderr, "failed:", err)
	os.Exit(1)
}

func failIf(err error) {
	if err != nil {
		fail(err)
	}
}

func client(c *cli.Context) garden.Client {
	target := c.GlobalString("target")
	return gclient.New(gconn.New("tcp", target))
}

func handle(c *cli.Context) string {
	if len(c.Args()) == 0 {
		fail(errors.New("must provide container handle"))
	}
	return c.Args().First()
}

func main() {
	app := cli.NewApp()
	app.Name = "gaol"
	app.Usage = "a cli for garden"
	app.Version = "0.0.1"
	app.Author = "Chris Brown"
	app.Email = "cbrown@pivotal.io"
	app.EnableBashCompletion = true

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "target, t",
			Value:  "localhost:7777",
			Usage:  "server to which commands are sent",
			EnvVar: "GAOL_TARGET",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "ping",
			Usage: "check if the server is running",
			Action: func(c *cli.Context) {
				err := client(c).Ping()
				failIf(err)
			},
		},
		{
			Name:  "create",
			Usage: "create a container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "handle, n",
					Usage: "name to give container",
				},
				cli.StringFlag{
					Name:  "rootfs, r",
					Usage: "rootfs image with which to create the container",
				},
				cli.StringSliceFlag{
					Name:  "env, e",
					Usage: "set environment variables",
					Value: &cli.StringSlice{},
				},
				cli.DurationFlag{
					Name:  "grace, g",
					Usage: "grace time (resetting ttl) of container",
					Value: 5 * time.Minute,
				},
				cli.BoolFlag{
					Name:  "privileged, p",
					Usage: "privileged user in container is privileged in host",
				},
			},
			Action: func(c *cli.Context) {
				handle := c.String("handle")
				grace := c.Duration("grace")
				rootfs := c.String("rootfs")
				env := c.StringSlice("env")
				privileged := c.Bool("privileged")

				container, err := client(c).Create(garden.ContainerSpec{
					Handle:     handle,
					GraceTime:  grace,
					RootFSPath: rootfs,
					Privileged: privileged,
					Env:        env,
				})
				failIf(err)

				fmt.Println(container.Handle())
			},
		},
		{
			Name:         "destroy",
			Usage:        "destroy a container",
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				client := client(c)
				handles := c.Args()

				for _, handle := range handles {
					err := client.Destroy(handle)
					failIf(err)
				}
			},
		},
		{
			Name:  "list",
			Usage: "get a list of running containers",
			Action: func(c *cli.Context) {
				containers, err := client(c).Containers(nil)
				failIf(err)

				for _, container := range containers {
					fmt.Println(container.Handle())
				}
			},
		},
		{
			Name:  "run",
			Usage: "run a command in a container",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "attach, a",
					Usage: "attach to the process after it is started",
				},
				cli.StringFlag{
					Name:  "dir, d",
					Usage: "current working directory of process",
				},
				cli.StringFlag{
					Name:  "user, u",
					Usage: "user to run the process as",
				},
				cli.StringFlag{
					Name:  "command, c",
					Usage: "the command to run",
				},
				cli.StringSliceFlag{
					Name:  "env, e",
					Usage: "set environment variables",
					Value: &cli.StringSlice{},
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				attach := c.Bool("attach")
				dir := c.String("dir")
				user := c.String("user")
				command := c.String("command")
				env := c.StringSlice("env")

				handle := handle(c)
				container, err := client(c).Lookup(handle)
				failIf(err)

				var processIo garden.ProcessIO
				if attach {
					processIo = garden.ProcessIO{
						Stdin:  os.Stdin,
						Stdout: os.Stdout,
						Stderr: os.Stderr,
					}
				} else {
					processIo = garden.ProcessIO{}
				}

				args, err := shellwords.Parse(command)
				failIf(err)

				process, err := container.Run(garden.ProcessSpec{
					Path: args[0],
					Args: args[1:],
					Dir:  dir,
					User: user,
					Env:  env,
				}, processIo)
				failIf(err)

				if attach {
					status, err := process.Wait()
					failIf(err)
					os.Exit(status)
				} else {
					fmt.Println(process.ID())
				}
			},
		},
		{
			Name:  "attach",
			Usage: "attach to command running in the container",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "pid, p",
					Usage: "process id to connect to",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				pid := uint32(c.Int("pid"))
				if pid == 0 {
					err := errors.New("must specify pid to attach to")
					failIf(err)
				}

				handle := handle(c)
				container, err := client(c).Lookup(handle)
				failIf(err)

				process, err := container.Attach(pid, garden.ProcessIO{
					Stdin:  os.Stdin,
					Stdout: os.Stdout,
					Stderr: os.Stderr,
				})
				failIf(err)

				_, err = process.Wait()
				failIf(err)
			},
		},
		{
			Name:  "shell",
			Usage: "open a shell inside the running container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "user, u",
					Usage: "user to open shell as",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				container, err := client(c).Lookup(handle(c))
				failIf(err)

				term, err := term.Open(os.Stdin.Name())
				failIf(err)

				err = term.SetRaw()
				failIf(err)

				rows, cols, err := pty.Getsize(os.Stdin)
				failIf(err)

				process, err := container.Run(garden.ProcessSpec{
					User: c.String("user"),
					Path: "/bin/sh",
					Args: []string{"-l"},
					Env:  []string{"TERM=" + os.Getenv("TERM")},
					TTY: &garden.TTYSpec{
						WindowSize: &garden.WindowSize{
							Rows:    rows,
							Columns: cols,
						},
					},
				}, garden.ProcessIO{
					Stdin:  term,
					Stdout: term,
					Stderr: term,
				})
				if err != nil {
					term.Restore()
					failIf(err)
				}

				resized := make(chan os.Signal, 10)
				signal.Notify(resized, syscall.SIGWINCH)

				go func() {
					for {
						<-resized

						rows, cols, err := pty.Getsize(os.Stdin)
						if err == nil {
							process.SetTTY(garden.TTYSpec{
								WindowSize: &garden.WindowSize{
									Rows:    rows,
									Columns: cols,
								},
							})
						}
					}
				}()

				process.Wait()
				term.Restore()
			},
		},
		{
			Name:  "stream-in",
			Usage: "stream data into the container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "to-file, t",
					Usage: "destination path in the container",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				handle := handle(c)

				dst := c.String("to-file")
				if dst == "" {
					fail(errors.New("missing --to-file argument"))
				}

				container, err := client(c).Lookup(handle)
				failIf(err)

				// perform dance to get correct file names
				tmpDir, err := ioutil.TempDir("", "gaol")
				failIf(err)
				defer os.RemoveAll(tmpDir)

				tmp, err := os.Create(filepath.Join(tmpDir, filepath.Base(dst)))
				failIf(err)

				_, err = io.Copy(tmp, os.Stdin)
				failIf(err)

				err = tmp.Close()
				failIf(err)

				reader, writer := io.Pipe()
				go func(w io.WriteCloser) {
					err := compressor.WriteTar(tmp.Name(), w)
					failIf(err)
					w.Close()
				}(writer)

				streamInSpec := garden.StreamInSpec{
					Path:      filepath.Dir(dst),
					TarStream: reader,
				}
				err = container.StreamIn(streamInSpec)
				failIf(err)
			},
		},
		{
			Name:  "stream-out",
			Usage: "stream data out of the container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "from-file, f",
					Usage: "source path in the container",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				handle := handle(c)

				src := c.String("from-file")
				if src == "" {
					fail(errors.New("missing --from-file argument"))
				}

				container, err := client(c).Lookup(handle)
				failIf(err)

				streamOutSpec := garden.StreamOutSpec{Path: src}
				output, err := container.StreamOut(streamOutSpec)
				failIf(err)

				tr := tar.NewReader(output)
				_, err = tr.Next()
				failIf(err)

				_, err = io.Copy(os.Stdout, tr)
				failIf(err)
			},
		},
		{
			Name:  "net-in",
			Usage: "map a port on the host to a port in the container",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "port, p",
					Usage: "container port",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				target := c.GlobalString("target")
				requestedContainerPort := uint32(c.Int("port"))

				if target == "" {
					fail(errors.New("target must be set"))
				}

				handle := handle(c)
				container, err := client(c).Lookup(handle)
				failIf(err)

				hostPort, _, err := container.NetIn(0, requestedContainerPort)
				failIf(err)

				host, _, err := net.SplitHostPort(target)
				failIf(err)

				fmt.Println(net.JoinHostPort(host, fmt.Sprintf("%d", hostPort)))
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>filtering containers by/printing out properties<commit_after>package main

import (
	"archive/tar"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/codegangsta/cli"
	"github.com/kr/pty"
	"github.com/mattn/go-shellwords"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/pkg/term"

	"github.com/cloudfoundry-incubator/garden"
	gclient "github.com/cloudfoundry-incubator/garden/client"
	gconn "github.com/cloudfoundry-incubator/garden/client/connection"
)

func handleComplete(c *cli.Context) {
	// This will complete if no args are passed
	if c.Args().Present() {
		return
	}

	containers, err := client(c).Containers(nil)
	failIf(err)

	for _, container := range containers {
		fmt.Println(container.Handle())
	}
}

func fail(err error) {
	fmt.Fprintln(os.Stderr, "failed:", err)
	os.Exit(1)
}

func failIf(err error) {
	if err != nil {
		fail(err)
	}
}

func client(c *cli.Context) garden.Client {
	target := c.GlobalString("target")
	return gclient.New(gconn.New("tcp", target))
}

func handle(c *cli.Context) string {
	if len(c.Args()) == 0 {
		fail(errors.New("must provide container handle"))
	}
	return c.Args().First()
}

func main() {
	app := cli.NewApp()
	app.Name = "gaol"
	app.Usage = "a cli for garden"
	app.Version = "0.0.1"
	app.Author = "Chris Brown"
	app.Email = "cbrown@pivotal.io"
	app.EnableBashCompletion = true

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "target, t",
			Value:  "localhost:7777",
			Usage:  "server to which commands are sent",
			EnvVar: "GAOL_TARGET",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "ping",
			Usage: "check if the server is running",
			Action: func(c *cli.Context) {
				err := client(c).Ping()
				failIf(err)
			},
		},
		{
			Name:  "create",
			Usage: "create a container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "handle, n",
					Usage: "name to give container",
				},
				cli.StringFlag{
					Name:  "rootfs, r",
					Usage: "rootfs image with which to create the container",
				},
				cli.StringSliceFlag{
					Name:  "env, e",
					Usage: "set environment variables",
					Value: &cli.StringSlice{},
				},
				cli.DurationFlag{
					Name:  "grace, g",
					Usage: "grace time (resetting ttl) of container",
					Value: 5 * time.Minute,
				},
				cli.BoolFlag{
					Name:  "privileged, p",
					Usage: "privileged user in container is privileged in host",
				},
			},
			Action: func(c *cli.Context) {
				handle := c.String("handle")
				grace := c.Duration("grace")
				rootfs := c.String("rootfs")
				env := c.StringSlice("env")
				privileged := c.Bool("privileged")

				container, err := client(c).Create(garden.ContainerSpec{
					Handle:     handle,
					GraceTime:  grace,
					RootFSPath: rootfs,
					Privileged: privileged,
					Env:        env,
				})
				failIf(err)

				fmt.Println(container.Handle())
			},
		},
		{
			Name:         "destroy",
			Usage:        "destroy a container",
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				client := client(c)
				handles := c.Args()

				for _, handle := range handles {
					err := client.Destroy(handle)
					failIf(err)
				}
			},
		},
		{
			Name:  "list",
			Usage: "get a list of running containers",
			Flags: []cli.Flag{
				cli.StringSliceFlag{
					Name:  "properties, p",
					Usage: "filter by properties (name=val)",
					Value: &cli.StringSlice{},
				},
			},
			Action: func(c *cli.Context) {
				properties := garden.Properties{}
				for _, prop := range c.StringSlice("properties") {
					segs := strings.SplitN(prop, "=", 2)
					if len(segs) < 2 {
						fail(errors.New("malformed property pair (must be name=value)"))
					}

					properties[segs[0]] = segs[1]
				}

				containers, err := client(c).Containers(nil)
				failIf(err)

				for _, container := range containers {
					fmt.Println(container.Handle())

					props, _ := container.Properties()
					for k, v := range props {
						fmt.Printf("  %s=%s\n", k, v)
					}
				}
			},
		},
		{
			Name:  "run",
			Usage: "run a command in a container",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "attach, a",
					Usage: "attach to the process after it is started",
				},
				cli.StringFlag{
					Name:  "dir, d",
					Usage: "current working directory of process",
				},
				cli.StringFlag{
					Name:  "user, u",
					Usage: "user to run the process as",
				},
				cli.StringFlag{
					Name:  "command, c",
					Usage: "the command to run",
				},
				cli.StringSliceFlag{
					Name:  "env, e",
					Usage: "set environment variables",
					Value: &cli.StringSlice{},
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				attach := c.Bool("attach")
				dir := c.String("dir")
				user := c.String("user")
				command := c.String("command")
				env := c.StringSlice("env")

				handle := handle(c)
				container, err := client(c).Lookup(handle)
				failIf(err)

				var processIo garden.ProcessIO
				if attach {
					processIo = garden.ProcessIO{
						Stdin:  os.Stdin,
						Stdout: os.Stdout,
						Stderr: os.Stderr,
					}
				} else {
					processIo = garden.ProcessIO{}
				}

				args, err := shellwords.Parse(command)
				failIf(err)

				process, err := container.Run(garden.ProcessSpec{
					Path: args[0],
					Args: args[1:],
					Dir:  dir,
					User: user,
					Env:  env,
				}, processIo)
				failIf(err)

				if attach {
					status, err := process.Wait()
					failIf(err)
					os.Exit(status)
				} else {
					fmt.Println(process.ID())
				}
			},
		},
		{
			Name:  "attach",
			Usage: "attach to command running in the container",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "pid, p",
					Usage: "process id to connect to",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				pid := uint32(c.Int("pid"))
				if pid == 0 {
					err := errors.New("must specify pid to attach to")
					failIf(err)
				}

				handle := handle(c)
				container, err := client(c).Lookup(handle)
				failIf(err)

				process, err := container.Attach(pid, garden.ProcessIO{
					Stdin:  os.Stdin,
					Stdout: os.Stdout,
					Stderr: os.Stderr,
				})
				failIf(err)

				_, err = process.Wait()
				failIf(err)
			},
		},
		{
			Name:  "shell",
			Usage: "open a shell inside the running container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "user, u",
					Usage: "user to open shell as",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				container, err := client(c).Lookup(handle(c))
				failIf(err)

				term, err := term.Open(os.Stdin.Name())
				failIf(err)

				err = term.SetRaw()
				failIf(err)

				rows, cols, err := pty.Getsize(os.Stdin)
				failIf(err)

				process, err := container.Run(garden.ProcessSpec{
					User: c.String("user"),
					Path: "/bin/sh",
					Args: []string{"-l"},
					Env:  []string{"TERM=" + os.Getenv("TERM")},
					TTY: &garden.TTYSpec{
						WindowSize: &garden.WindowSize{
							Rows:    rows,
							Columns: cols,
						},
					},
				}, garden.ProcessIO{
					Stdin:  term,
					Stdout: term,
					Stderr: term,
				})
				if err != nil {
					term.Restore()
					failIf(err)
				}

				resized := make(chan os.Signal, 10)
				signal.Notify(resized, syscall.SIGWINCH)

				go func() {
					for {
						<-resized

						rows, cols, err := pty.Getsize(os.Stdin)
						if err == nil {
							process.SetTTY(garden.TTYSpec{
								WindowSize: &garden.WindowSize{
									Rows:    rows,
									Columns: cols,
								},
							})
						}
					}
				}()

				process.Wait()
				term.Restore()
			},
		},
		{
			Name:  "stream-in",
			Usage: "stream data into the container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "to-file, t",
					Usage: "destination path in the container",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				handle := handle(c)

				dst := c.String("to-file")
				if dst == "" {
					fail(errors.New("missing --to-file argument"))
				}

				container, err := client(c).Lookup(handle)
				failIf(err)

				// perform dance to get correct file names
				tmpDir, err := ioutil.TempDir("", "gaol")
				failIf(err)
				defer os.RemoveAll(tmpDir)

				tmp, err := os.Create(filepath.Join(tmpDir, filepath.Base(dst)))
				failIf(err)

				_, err = io.Copy(tmp, os.Stdin)
				failIf(err)

				err = tmp.Close()
				failIf(err)

				reader, writer := io.Pipe()
				go func(w io.WriteCloser) {
					err := compressor.WriteTar(tmp.Name(), w)
					failIf(err)
					w.Close()
				}(writer)

				streamInSpec := garden.StreamInSpec{
					Path:      filepath.Dir(dst),
					TarStream: reader,
				}
				err = container.StreamIn(streamInSpec)
				failIf(err)
			},
		},
		{
			Name:  "stream-out",
			Usage: "stream data out of the container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "from-file, f",
					Usage: "source path in the container",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				handle := handle(c)

				src := c.String("from-file")
				if src == "" {
					fail(errors.New("missing --from-file argument"))
				}

				container, err := client(c).Lookup(handle)
				failIf(err)

				streamOutSpec := garden.StreamOutSpec{Path: src}
				output, err := container.StreamOut(streamOutSpec)
				failIf(err)

				tr := tar.NewReader(output)
				_, err = tr.Next()
				failIf(err)

				_, err = io.Copy(os.Stdout, tr)
				failIf(err)
			},
		},
		{
			Name:  "net-in",
			Usage: "map a port on the host to a port in the container",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "port, p",
					Usage: "container port",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				target := c.GlobalString("target")
				requestedContainerPort := uint32(c.Int("port"))

				if target == "" {
					fail(errors.New("target must be set"))
				}

				handle := handle(c)
				container, err := client(c).Lookup(handle)
				failIf(err)

				hostPort, _, err := container.NetIn(0, requestedContainerPort)
				failIf(err)

				host, _, err := net.SplitHostPort(target)
				failIf(err)

				fmt.Println(net.JoinHostPort(host, fmt.Sprintf("%d", hostPort)))
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>/*
 * Copyright 2016 Dgraph Labs, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 		http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package posting

import (
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"io/ioutil"
	"time"

	"golang.org/x/net/trace"

	"github.com/dgraph-io/dgraph/posting/types"
	"github.com/dgraph-io/dgraph/store"
	"github.com/dgraph-io/dgraph/x"
)

const (
	// Posting list keys are prefixed with this rune if it is a mutation meant for
	// the index.
	indexRune = ':'
)

type indexConfigs struct {
	Cfg []*indexConfig `json:"config"`
}

type indexConfig struct {
	Attr string `json:"attribute"`
	// TODO(jchiu): Add other tokenizer here in future.
}

var (
	indexLog        trace.EventLog
	indexStore      *store.Store
	indexCfgs       indexConfigs
	indexConfigFile = flag.String("indexconfig", "index.json", "File containing index config. If empty, we create an empty config.")
	indexedAttr     = make(map[string]bool)
)

func init() {
	indexLog = trace.NewEventLog("index", "Logger")
	x.AddInit(func() {
		x.Assert(indexConfigFile != nil && len(*indexConfigFile) > 0)
		f, err := ioutil.ReadFile(*indexConfigFile)
		x.Check(err)
		indexLog.Printf("Reading index configs from [%s]", *indexConfigFile)
		ReadIndexConfigs(f)
	})
}

func ReadIndexConfigs(f []byte) {
	x.Check(json.Unmarshal(f, &indexCfgs))
	for _, c := range indexCfgs.Cfg {
		indexedAttr[c.Attr] = true
	}
	if len(indexedAttr) == 0 {
		indexLog.Printf("No indexed attributes!")
	} else {
		for k := range indexedAttr {
			indexLog.Printf("Indexed attribute [%s]", k)
		}
	}
}

// InitIndex initializes the index with the given data store.
func InitIndex(ds *store.Store) {
	if ds == nil {
		return
	}
	indexStore = ds
}

func IndexKey(attr string, term []byte) []byte {
	buf := bytes.NewBuffer(make([]byte, 0, len(attr)+len(term)+2))
	_, err := buf.WriteRune(indexRune)
	x.Check(err)
	_, err = buf.WriteString(attr)
	x.Check(err)
	_, err = buf.WriteRune('|')
	x.Check(err)
	_, err = buf.Write(term)
	x.Check(err)
	return buf.Bytes()
}

// processIndexTerm adds mutation(s) for a single term, to maintain index.
func processIndexTerm(attr string, uid uint64, term []byte, del bool) {
	edge := x.DirectedEdge{
		Timestamp: time.Now(),
		ValueId:   uid,
		Attribute: attr,
	}
	key := IndexKey(edge.Attribute, term)
	plist := GetOrCreate(key, indexStore)
	x.Assertf(plist != nil, "plist is nil [%s] %d %s", key, edge.ValueId, edge.Attribute)

	ctx := context.Background()
	if del {
		plist.AddMutation(ctx, edge, Del)
		indexLog.Printf("DEL [%s] [%d] OldTerm [%s]", edge.Attribute, edge.Entity, string(term))
	} else {
		plist.AddMutation(ctx, edge, Set)
		indexLog.Printf("SET [%s] [%d] NewTerm [%s]", edge.Attribute, edge.Entity, string(term))
	}
}

// AddMutationWithIndex is AddMutation with support for indexing.
func (l *List) AddMutationWithIndex(ctx context.Context, t x.DirectedEdge, op byte) error {
	x.Assertf(len(t.Attribute) > 0 && t.Attribute[0] != indexRune,
		"[%s] [%d] [%s] %d %d\n", t.Attribute, t.Entity, string(t.Value), t.ValueId, op)

	var lastPost types.Posting
	var hasLastPost bool
	doUpdateIndex := indexStore != nil && (t.Value != nil) && indexedAttr[t.Attribute]
	if doUpdateIndex {
		// Check last posting for original value BEFORE any mutation actually happens.
		if l.Length() >= 1 {
			x.Assert(l.Get(&lastPost, l.Length()-1))
			hasLastPost = true
		}
	}
	hasMutated, err := l.AddMutation(ctx, t, op)
	if err != nil {
		return err
	}
	if !hasMutated || !doUpdateIndex {
		return nil
	}
	if hasLastPost && lastPost.ValueBytes() != nil {
		processIndexTerm(t.Attribute, t.Entity, lastPost.ValueBytes(), true)
	}
	if op == Set {
		processIndexTerm(t.Attribute, t.Entity, t.Value, false)
	}
	return nil
}
<commit_msg>Allow index to be undefined and disabled<commit_after>/*
 * Copyright 2016 Dgraph Labs, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 		http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package posting

import (
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"io/ioutil"
	"time"

	"golang.org/x/net/trace"

	"github.com/dgraph-io/dgraph/posting/types"
	"github.com/dgraph-io/dgraph/store"
	"github.com/dgraph-io/dgraph/x"
)

const (
	// Posting list keys are prefixed with this rune if it is a mutation meant for
	// the index.
	indexRune = ':'
)

type indexConfigs struct {
	Cfg []*indexConfig `json:"config"`
}

type indexConfig struct {
	Attr string `json:"attribute"`
	// TODO(jchiu): Add other tokenizer here in future.
}

var (
	indexLog        trace.EventLog
	indexStore      *store.Store
	indexCfgs       indexConfigs
	indexConfigFile = flag.String("indexconfig", "",
		"File containing index config. If empty, we assume no index.")
	indexedAttr = make(map[string]bool)
)

func init() {
	indexLog = trace.NewEventLog("index", "Logger")
	x.AddInit(func() {
		if indexConfigFile == nil || len(*indexConfigFile) == 0 {
			indexLog.Printf("No valid config file", *indexConfigFile)
			return
		}
		f, err := ioutil.ReadFile(*indexConfigFile)
		x.Check(err)
		indexLog.Printf("Reading index configs from [%s]", *indexConfigFile)
		ReadIndexConfigs(f)
	})
}

func ReadIndexConfigs(f []byte) {
	x.Check(json.Unmarshal(f, &indexCfgs))
	for _, c := range indexCfgs.Cfg {
		indexedAttr[c.Attr] = true
	}
	if len(indexedAttr) == 0 {
		indexLog.Printf("No indexed attributes!")
	} else {
		for k := range indexedAttr {
			indexLog.Printf("Indexed attribute [%s]", k)
		}
	}
}

// InitIndex initializes the index with the given data store.
func InitIndex(ds *store.Store) {
	if ds == nil {
		return
	}
	indexStore = ds
}

func IndexKey(attr string, term []byte) []byte {
	buf := bytes.NewBuffer(make([]byte, 0, len(attr)+len(term)+2))
	_, err := buf.WriteRune(indexRune)
	x.Check(err)
	_, err = buf.WriteString(attr)
	x.Check(err)
	_, err = buf.WriteRune('|')
	x.Check(err)
	_, err = buf.Write(term)
	x.Check(err)
	return buf.Bytes()
}

// processIndexTerm adds mutation(s) for a single term, to maintain index.
func processIndexTerm(attr string, uid uint64, term []byte, del bool) {
	edge := x.DirectedEdge{
		Timestamp: time.Now(),
		ValueId:   uid,
		Attribute: attr,
	}
	key := IndexKey(edge.Attribute, term)
	plist := GetOrCreate(key, indexStore)
	x.Assertf(plist != nil, "plist is nil [%s] %d %s", key, edge.ValueId, edge.Attribute)

	ctx := context.Background()
	if del {
		plist.AddMutation(ctx, edge, Del)
		indexLog.Printf("DEL [%s] [%d] OldTerm [%s]", edge.Attribute, edge.Entity, string(term))
	} else {
		plist.AddMutation(ctx, edge, Set)
		indexLog.Printf("SET [%s] [%d] NewTerm [%s]", edge.Attribute, edge.Entity, string(term))
	}
}

// AddMutationWithIndex is AddMutation with support for indexing.
func (l *List) AddMutationWithIndex(ctx context.Context, t x.DirectedEdge, op byte) error {
	x.Assertf(len(t.Attribute) > 0 && t.Attribute[0] != indexRune,
		"[%s] [%d] [%s] %d %d\n", t.Attribute, t.Entity, string(t.Value), t.ValueId, op)

	var lastPost types.Posting
	var hasLastPost bool
	doUpdateIndex := indexStore != nil && (t.Value != nil) && indexedAttr[t.Attribute]
	if doUpdateIndex {
		// Check last posting for original value BEFORE any mutation actually happens.
		if l.Length() >= 1 {
			x.Assert(l.Get(&lastPost, l.Length()-1))
			hasLastPost = true
		}
	}
	hasMutated, err := l.AddMutation(ctx, t, op)
	if err != nil {
		return err
	}
	if !hasMutated || !doUpdateIndex {
		return nil
	}
	if hasLastPost && lastPost.ValueBytes() != nil {
		processIndexTerm(t.Attribute, t.Entity, lastPost.ValueBytes(), true)
	}
	if op == Set {
		processIndexTerm(t.Attribute, t.Entity, t.Value, false)
	}
	return nil
}
<|endoftext|>


<commit_before>package awstasks

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/golang/glog"
	"k8s.io/kube-deploy/upup/pkg/fi"
	"k8s.io/kube-deploy/upup/pkg/fi/cloudup/awsup"
	"k8s.io/kube-deploy/upup/pkg/fi/cloudup/terraform"
	"reflect"
	"strings"
)

//go:generate fitask -type=AutoscalingGroup
type AutoscalingGroup struct {
	Name *string

	MinSize *int64
	MaxSize *int64
	Subnets []*Subnet
	Tags    map[string]string

	LaunchConfiguration *LaunchConfiguration
}

var _ fi.CompareWithID = &AutoscalingGroup{}

func (e *AutoscalingGroup) CompareWithID() *string {
	return e.Name
}

func findAutoscalingGroup(cloud *awsup.AWSCloud, name string) (*autoscaling.Group, error) {
	request := &autoscaling.DescribeAutoScalingGroupsInput{
		AutoScalingGroupNames: []*string{&name},
	}

	var found []*autoscaling.Group
	err := cloud.Autoscaling.DescribeAutoScalingGroupsPages(request, func(p *autoscaling.DescribeAutoScalingGroupsOutput, lastPage bool) (shouldContinue bool) {
		for _, g := range p.AutoScalingGroups {
			if aws.StringValue(g.AutoScalingGroupName) == name {
				found = append(found, g)
			} else {
				glog.Warningf("Got ASG with unexpected name")
			}
		}

		return true
	})

	if err != nil {
		return nil, fmt.Errorf("error listing AutoscalingGroups: %v", err)
	}

	if len(found) == 0 {
		return nil, nil
	}

	if len(found) != 1 {
		return nil, fmt.Errorf("Found multiple AutoscalingGroups with name %q", name)
	}

	return found[0], nil
}

func (e *AutoscalingGroup) Find(c *fi.Context) (*AutoscalingGroup, error) {
	cloud := c.Cloud.(*awsup.AWSCloud)

	g, err := findAutoscalingGroup(cloud, *e.Name)
	if err != nil {
		return nil, err
	}
	if g == nil {
		return nil, nil
	}

	actual := &AutoscalingGroup{}
	actual.Name = g.AutoScalingGroupName
	actual.MinSize = g.MinSize
	actual.MaxSize = g.MaxSize

	if g.VPCZoneIdentifier != nil {
		subnets := strings.Split(*g.VPCZoneIdentifier, ",")
		for _, subnet := range subnets {
			actual.Subnets = append(actual.Subnets, &Subnet{ID: aws.String(subnet)})
		}
	}

	if len(g.Tags) != 0 {
		actual.Tags = make(map[string]string)
		for _, tag := range g.Tags {
			actual.Tags[*tag.Key] = *tag.Value
		}
	}

	if g.LaunchConfigurationName == nil {
		return nil, fmt.Errorf("autoscaling Group %q had no LaunchConfiguration", *actual.Name)
	}
	actual.LaunchConfiguration = &LaunchConfiguration{ID: g.LaunchConfigurationName}

	if subnetSlicesEqualIgnoreOrder(actual.Subnets, e.Subnets) {
		actual.Subnets = e.Subnets
	}

	return actual, nil
}

func (e *AutoscalingGroup) Run(c *fi.Context) error {
	c.Cloud.(*awsup.AWSCloud).AddTags(e.Name, e.Tags)
	return fi.DefaultDeltaRunMethod(e, c)
}

func (s *AutoscalingGroup) CheckChanges(a, e, changes *AutoscalingGroup) error {
	if a != nil {
		if e.Name == nil {
			return fi.RequiredField("Name")
		}
	}
	return nil
}

func (e *AutoscalingGroup) buildTags(cloud fi.Cloud) map[string]string {
	tags := make(map[string]string)
	for k, v := range e.Tags {
		tags[k] = v
	}
	return tags
}

func (_ *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *AutoscalingGroup) error {
	if a == nil {
		glog.V(2).Infof("Creating autoscaling Group with Name:%q", *e.Name)

		request := &autoscaling.CreateAutoScalingGroupInput{}
		request.AutoScalingGroupName = e.Name
		request.LaunchConfigurationName = e.LaunchConfiguration.ID
		request.MinSize = e.MinSize
		request.MaxSize = e.MaxSize

		var subnetIDs []string
		for _, s := range e.Subnets {
			subnetIDs = append(subnetIDs, *s.ID)
		}
		request.VPCZoneIdentifier = aws.String(strings.Join(subnetIDs, ","))

		tags := []*autoscaling.Tag{}
		for k, v := range e.buildTags(t.Cloud) {
			tags = append(tags, &autoscaling.Tag{
				Key:          aws.String(k),
				Value:        aws.String(v),
				ResourceId:   e.Name,
				ResourceType: aws.String("auto-scaling-group"),
			})
		}
		request.Tags = tags

		_, err := t.Cloud.Autoscaling.CreateAutoScalingGroup(request)
		if err != nil {
			return fmt.Errorf("error creating AutoscalingGroup: %v", err)
		}
	} else {
		request := &autoscaling.UpdateAutoScalingGroupInput{
			AutoScalingGroupName: e.Name,
		}

		if changes.LaunchConfiguration != nil {
			request.LaunchConfigurationName = e.LaunchConfiguration.ID
			changes.LaunchConfiguration = nil
		}
		if changes.MinSize != nil {
			request.MinSize = e.MinSize
			changes.MinSize = nil
		}
		if changes.MaxSize != nil {
			request.MaxSize = e.MaxSize
			changes.MaxSize = nil
		}

		empty := &AutoscalingGroup{}
		if !reflect.DeepEqual(empty, changes) {
			glog.Warningf("cannot apply changes to AutoScalingGroup: %v", changes)
		}

		glog.V(2).Infof("Updating autoscaling group %s", *e.Name)

		_, err := t.Cloud.Autoscaling.UpdateAutoScalingGroup(request)
		if err != nil {
			return fmt.Errorf("error updating AutoscalingGroup: %v", err)
		}
	}

	// TODO: Use PropagateAtLaunch = false for tagging?

	return nil // We have
}

type terraformASGTag struct {
	Key               *string `json:"key"`
	Value             *string `json:"value"`
	PropagateAtLaunch *bool   `json:"propagate_at_launch"`
}
type terraformAutoscalingGroup struct {
	Name                    *string              `json:"name,omitempty"`
	LaunchConfigurationName *terraform.Literal   `json:"launch_configuration,omitempty"`
	MaxSize                 *int64               `json:"max_size,omitempty"`
	MinSize                 *int64               `json:"min_size,omitempty"`
	VPCZoneIdentifier       []*terraform.Literal `json:"vpc_zone_identifier,omitempty"`
	Tags                    []*terraformASGTag   `json:"tag,omitempty"`
}

func (_ *AutoscalingGroup) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *AutoscalingGroup) error {
	tf := &terraformAutoscalingGroup{
		Name:                    e.Name,
		MinSize:                 e.MinSize,
		MaxSize:                 e.MaxSize,
		LaunchConfigurationName: e.LaunchConfiguration.TerraformLink(),
	}

	for _, s := range e.Subnets {
		tf.VPCZoneIdentifier = append(tf.VPCZoneIdentifier, s.TerraformLink())
	}

	for k, v := range e.buildTags(t.Cloud) {
		tf.Tags = append(tf.Tags, &terraformASGTag{
			Key:               fi.String(k),
			Value:             fi.String(v),
			PropagateAtLaunch: fi.Bool(true),
		})
	}

	return t.RenderResource("aws_autoscaling_group", *e.Name, tf)
}

func (e *AutoscalingGroup) TerraformLink() *terraform.Literal {
	return terraform.LiteralProperty("aws_autoscaling_group", *e.Name, "id")
}
<commit_msg>upup: enable subnet changes on ASG<commit_after>package awstasks

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/golang/glog"
	"k8s.io/kube-deploy/upup/pkg/fi"
	"k8s.io/kube-deploy/upup/pkg/fi/cloudup/awsup"
	"k8s.io/kube-deploy/upup/pkg/fi/cloudup/terraform"
	"reflect"
	"strings"
)

//go:generate fitask -type=AutoscalingGroup
type AutoscalingGroup struct {
	Name *string

	MinSize *int64
	MaxSize *int64
	Subnets []*Subnet
	Tags    map[string]string

	LaunchConfiguration *LaunchConfiguration
}

var _ fi.CompareWithID = &AutoscalingGroup{}

func (e *AutoscalingGroup) CompareWithID() *string {
	return e.Name
}

func findAutoscalingGroup(cloud *awsup.AWSCloud, name string) (*autoscaling.Group, error) {
	request := &autoscaling.DescribeAutoScalingGroupsInput{
		AutoScalingGroupNames: []*string{&name},
	}

	var found []*autoscaling.Group
	err := cloud.Autoscaling.DescribeAutoScalingGroupsPages(request, func(p *autoscaling.DescribeAutoScalingGroupsOutput, lastPage bool) (shouldContinue bool) {
		for _, g := range p.AutoScalingGroups {
			if aws.StringValue(g.AutoScalingGroupName) == name {
				found = append(found, g)
			} else {
				glog.Warningf("Got ASG with unexpected name")
			}
		}

		return true
	})

	if err != nil {
		return nil, fmt.Errorf("error listing AutoscalingGroups: %v", err)
	}

	if len(found) == 0 {
		return nil, nil
	}

	if len(found) != 1 {
		return nil, fmt.Errorf("Found multiple AutoscalingGroups with name %q", name)
	}

	return found[0], nil
}

func (e *AutoscalingGroup) Find(c *fi.Context) (*AutoscalingGroup, error) {
	cloud := c.Cloud.(*awsup.AWSCloud)

	g, err := findAutoscalingGroup(cloud, *e.Name)
	if err != nil {
		return nil, err
	}
	if g == nil {
		return nil, nil
	}

	actual := &AutoscalingGroup{}
	actual.Name = g.AutoScalingGroupName
	actual.MinSize = g.MinSize
	actual.MaxSize = g.MaxSize

	if g.VPCZoneIdentifier != nil {
		subnets := strings.Split(*g.VPCZoneIdentifier, ",")
		for _, subnet := range subnets {
			actual.Subnets = append(actual.Subnets, &Subnet{ID: aws.String(subnet)})
		}
	}

	if len(g.Tags) != 0 {
		actual.Tags = make(map[string]string)
		for _, tag := range g.Tags {
			actual.Tags[*tag.Key] = *tag.Value
		}
	}

	if g.LaunchConfigurationName == nil {
		return nil, fmt.Errorf("autoscaling Group %q had no LaunchConfiguration", *actual.Name)
	}
	actual.LaunchConfiguration = &LaunchConfiguration{ID: g.LaunchConfigurationName}

	if subnetSlicesEqualIgnoreOrder(actual.Subnets, e.Subnets) {
		actual.Subnets = e.Subnets
	}

	return actual, nil
}

func (e *AutoscalingGroup) Run(c *fi.Context) error {
	c.Cloud.(*awsup.AWSCloud).AddTags(e.Name, e.Tags)
	return fi.DefaultDeltaRunMethod(e, c)
}

func (s *AutoscalingGroup) CheckChanges(a, e, changes *AutoscalingGroup) error {
	if a != nil {
		if e.Name == nil {
			return fi.RequiredField("Name")
		}
	}
	return nil
}

func (e *AutoscalingGroup) buildTags(cloud fi.Cloud) map[string]string {
	tags := make(map[string]string)
	for k, v := range e.Tags {
		tags[k] = v
	}
	return tags
}

func (_ *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *AutoscalingGroup) error {
	if a == nil {
		glog.V(2).Infof("Creating autoscaling Group with Name:%q", *e.Name)

		request := &autoscaling.CreateAutoScalingGroupInput{}
		request.AutoScalingGroupName = e.Name
		request.LaunchConfigurationName = e.LaunchConfiguration.ID
		request.MinSize = e.MinSize
		request.MaxSize = e.MaxSize

		var subnetIDs []string
		for _, s := range e.Subnets {
			subnetIDs = append(subnetIDs, *s.ID)
		}
		request.VPCZoneIdentifier = aws.String(strings.Join(subnetIDs, ","))

		tags := []*autoscaling.Tag{}
		for k, v := range e.buildTags(t.Cloud) {
			tags = append(tags, &autoscaling.Tag{
				Key:          aws.String(k),
				Value:        aws.String(v),
				ResourceId:   e.Name,
				ResourceType: aws.String("auto-scaling-group"),
			})
		}
		request.Tags = tags

		_, err := t.Cloud.Autoscaling.CreateAutoScalingGroup(request)
		if err != nil {
			return fmt.Errorf("error creating AutoscalingGroup: %v", err)
		}
	} else {
		request := &autoscaling.UpdateAutoScalingGroupInput{
			AutoScalingGroupName: e.Name,
		}

		if changes.LaunchConfiguration != nil {
			request.LaunchConfigurationName = e.LaunchConfiguration.ID
			changes.LaunchConfiguration = nil
		}
		if changes.MinSize != nil {
			request.MinSize = e.MinSize
			changes.MinSize = nil
		}
		if changes.MaxSize != nil {
			request.MaxSize = e.MaxSize
			changes.MaxSize = nil
		}
		if changes.Subnets != nil {
			var subnetIDs []string
			for _, s := range e.Subnets {
				subnetIDs = append(subnetIDs, *s.ID)
			}
			request.VPCZoneIdentifier = aws.String(strings.Join(subnetIDs, ","))
			changes.Subnets = nil
		}

		empty := &AutoscalingGroup{}
		if !reflect.DeepEqual(empty, changes) {
			glog.Warningf("cannot apply changes to AutoScalingGroup: %v", changes)
		}

		glog.V(2).Infof("Updating autoscaling group %s", *e.Name)

		_, err := t.Cloud.Autoscaling.UpdateAutoScalingGroup(request)
		if err != nil {
			return fmt.Errorf("error updating AutoscalingGroup: %v", err)
		}
	}

	// TODO: Use PropagateAtLaunch = false for tagging?

	return nil // We have
}

type terraformASGTag struct {
	Key               *string `json:"key"`
	Value             *string `json:"value"`
	PropagateAtLaunch *bool   `json:"propagate_at_launch"`
}
type terraformAutoscalingGroup struct {
	Name                    *string              `json:"name,omitempty"`
	LaunchConfigurationName *terraform.Literal   `json:"launch_configuration,omitempty"`
	MaxSize                 *int64               `json:"max_size,omitempty"`
	MinSize                 *int64               `json:"min_size,omitempty"`
	VPCZoneIdentifier       []*terraform.Literal `json:"vpc_zone_identifier,omitempty"`
	Tags                    []*terraformASGTag   `json:"tag,omitempty"`
}

func (_ *AutoscalingGroup) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *AutoscalingGroup) error {
	tf := &terraformAutoscalingGroup{
		Name:                    e.Name,
		MinSize:                 e.MinSize,
		MaxSize:                 e.MaxSize,
		LaunchConfigurationName: e.LaunchConfiguration.TerraformLink(),
	}

	for _, s := range e.Subnets {
		tf.VPCZoneIdentifier = append(tf.VPCZoneIdentifier, s.TerraformLink())
	}

	for k, v := range e.buildTags(t.Cloud) {
		tf.Tags = append(tf.Tags, &terraformASGTag{
			Key:               fi.String(k),
			Value:             fi.String(v),
			PropagateAtLaunch: fi.Bool(true),
		})
	}

	return t.RenderResource("aws_autoscaling_group", *e.Name, tf)
}

func (e *AutoscalingGroup) TerraformLink() *terraform.Literal {
	return terraform.LiteralProperty("aws_autoscaling_group", *e.Name, "id")
}
<|endoftext|>


<commit_before>package state

import (
	"math"
	"sort"

	"github.com/kzahedi/goent/continuous"

	pb "gopkg.in/cheggaaa/pb.v1"
)

// KraskovStoegbauerGrassberger1 is an implementation of the first
// algorithm presented in
// A. Kraskov, H. Stoegbauer, and P. Grassberger.
// Estimating mutual information. Phys. Rev. E, 69:066138, Jun 2004.
func KraskovStoegbauerGrassberger1(xy [][]float64, xIndices, yIndices []int, k int, eta bool) []float64 {

	N := float64(len(xy))
	r := make([]float64, len(xy), len(xy))

	hk := continuous.Harmonic(k) / N            // h(k)
	hN := continuous.Harmonic(int(len(xy))) / N // h(N)

	var bar *pb.ProgressBar

	if eta == true {
		bar = pb.StartNew(len(xy))
	}
	for t := 0; t < len(xy); t++ {
		epsilon := ksgGetEpsilon(k, xy[t], xy, xIndices, yIndices)

		cNx := ksgCount(epsilon, xy[t], xy, xIndices) // N_x
		hNx := continuous.Harmonic(cNx + 1)           // h(N_x)

		cNy := ksgCount(epsilon, xy[t], xy, yIndices) // N_y
		hNy := continuous.Harmonic(cNy + 1)           // h(N_y)

		r[t] = -hNx - hNy + hk + hN

		if eta == true {
			bar.Increment()
		}
	}

	if eta == true {
		bar.FinishPrint("Finished")
	}

	return r
}

// KraskovStoegbauerGrassberger2 is an implementation of the second
// algorithm presented in
// A. Kraskov, H. Stoegbauer, and P. Grassberger.
// Estimating mutual information. Phys. Rev. E, 69:066138, Jun 2004.
func KraskovStoegbauerGrassberger2(xy [][]float64, xIndices, yIndices []int, k int, eta bool) []float64 {

	N := float64(len(xy))
	r := make([]float64, len(xy), len(xy))

	hk := continuous.Harmonic(k) / N
	hN := continuous.Harmonic(int(len(xy))) / N
	kfactor := 1.0 / (float64(k) * N)

	var bar *pb.ProgressBar

	if eta == true {
		bar = pb.StartNew(len(xy))
	}
	for t := 0; t < len(xy); t++ {
		epsilon := ksgGetEpsilon(k, xy[t], xy, xIndices, yIndices)

		cNx := ksgCount(epsilon, xy[t], xy, xIndices)
		hNx := continuous.Harmonic(cNx)

		cNy := ksgCount(epsilon, xy[t], xy, yIndices)
		hNy := continuous.Harmonic(cNy)

		r[t] = -hNx - hNy + hk + hN - kfactor
		if eta == true {
			bar.Increment()
		}
	}

	if eta == true {
		bar.FinishPrint("Finished")
	}

	return r
}

// ksgGetEpsilon calculate epsilon_k(t) as defined by Frenzel & Pompe, 2007
// epsilon_k(t) is the Distance of the k-th nearest neighbour. The function
// takes k, the point from which the Distance is calculated (xyz), and the
// data from which the k-th nearest neighbour should be determined
func ksgGetEpsilon(k int, xy []float64, data [][]float64, xIndices, yIndices []int) float64 {
	distances := make([]float64, len(data), len(data))

	for t := 0; t < len(data); t++ {
		distances[t] = ksgMaxNorm2(xy, data[t], xIndices, yIndices)
	}

	sort.Float64s(distances)

	return distances[k-1] // we start to ksgCount at zero
}

func ksgMaxNorm2(a, b []float64, xIndices, yIndices []int) float64 {
	xDistance := continuous.Distance(a, b, xIndices)
	yDistance := continuous.Distance(a, b, yIndices)
	return math.Max(xDistance, yDistance)
}

// ksgCount count the number of points for which the x or y coordinate is
// closer than epsilon, where the ksgDistance is measured by the max-norm
func ksgCount(epsilon float64, xy []float64, data [][]float64, indices []int) (c int) {

	for t := 0; t < len(data); t++ {
		if continuous.Distance(xy, data[t], indices) < epsilon {
			c++
		}
	}

	return
}
<commit_msg>Bug fixes in continuous.state<commit_after>package state

import (
	"math"
	"sort"

	"github.com/kzahedi/goent/continuous"

	pb "gopkg.in/cheggaaa/pb.v1"
)

// KraskovStoegbauerGrassberger1 is an implementation of the first
// algorithm presented in
// A. Kraskov, H. Stoegbauer, and P. Grassberger.
// Estimating mutual information. Phys. Rev. E, 69:066138, Jun 2004.
func KraskovStoegbauerGrassberger1(xy [][]float64, xIndices, yIndices []int, k int, eta bool) []float64 {

	N := float64(len(xy))
	r := make([]float64, len(xy), len(xy))

	hk := continuous.Harmonic(k)            // h(k)
	hN := continuous.Harmonic(int(len(xy))) // h(N)

	var bar *pb.ProgressBar

	if eta == true {
		bar = pb.StartNew(len(xy))
	}
	for t := 0; t < len(xy); t++ {
		epsilon := ksgGetEpsilon(k, xy[t], xy, xIndices, yIndices)

		cNx := ksgCount(epsilon, xy[t], xy, xIndices) // N_x
		hNx := continuous.Harmonic(cNx + 1)           // h(N_x)

		cNy := ksgCount(epsilon, xy[t], xy, yIndices) // N_y
		hNy := continuous.Harmonic(cNy + 1)           // h(N_y)

		r[t] = (-hNx - hNy + hk + hN) / N

		if eta == true {
			bar.Increment()
		}
	}

	if eta == true {
		bar.FinishPrint("Finished")
	}

	return r
}

// KraskovStoegbauerGrassberger2 is an implementation of the second
// algorithm presented in
// A. Kraskov, H. Stoegbauer, and P. Grassberger.
// Estimating mutual information. Phys. Rev. E, 69:066138, Jun 2004.
func KraskovStoegbauerGrassberger2(xy [][]float64, xIndices, yIndices []int, k int, eta bool) []float64 {

	n := len(xy)
	r := make([]float64, n, n)

	N := float64(len(xy))

	hk := continuous.Harmonic(k)
	hN := continuous.Harmonic(n)
	k1 := 1.0 / float64(k)

	var bar *pb.ProgressBar

	if eta == true {
		bar = pb.StartNew(len(xy))
	}
	for t := 0; t < len(xy); t++ {
		epsilon := ksgGetEpsilon(k, xy[t], xy, xIndices, yIndices)

		cNx := ksgCount(epsilon, xy[t], xy, xIndices)
		hNx := continuous.Harmonic(cNx)

		cNy := ksgCount(epsilon, xy[t], xy, yIndices)
		hNy := continuous.Harmonic(cNy)

		r[t] = (-hNx - hNy + hk + hN - k1) / N

		if eta == true {
			bar.Increment()
		}
	}

	if eta == true {
		bar.FinishPrint("Finished")
	}

	return r
}

// ksgGetEpsilon calculate epsilon_k(t) as defined by Frenzel & Pompe, 2007
// epsilon_k(t) is the Distance of the k-th nearest neighbour. The function
// takes k, the point from which the Distance is calculated (xyz), and the
// data from which the k-th nearest neighbour should be determined
func ksgGetEpsilon(k int, xy []float64, data [][]float64, xIndices, yIndices []int) float64 {
	distances := make([]float64, len(data), len(data))

	for t := 0; t < len(data); t++ {
		distances[t] = ksgMaxNorm2(xy, data[t], xIndices, yIndices)
	}

	sort.Float64s(distances)

	return distances[k-1] // we start to ksgCount at zero
}

func ksgMaxNorm2(a, b []float64, xIndices, yIndices []int) float64 {
	xDistance := continuous.Distance(a, b, xIndices)
	yDistance := continuous.Distance(a, b, yIndices)
	return math.Max(xDistance, yDistance)
}

// ksgCount count the number of points for which the x or y coordinate is
// closer than epsilon, where the ksgDistance is measured by the max-norm
func ksgCount(epsilon float64, xy []float64, data [][]float64, indices []int) (c int) {

	for t := 0; t < len(data); t++ {
		if continuous.Distance(xy, data[t], indices) < epsilon {
			c++
		}
	}

	return
}
<|endoftext|>


<commit_before>// Copyright (c) 2020-2022 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

/*
Package schnorr provides custom Schnorr signing and verification via secp256k1.

This package provides data structures and functions necessary to produce and
verify deterministic canonical Schnorr signatures using a custom scheme named
EC-Schnorr-DCRv0 that is described herein.  The signatures and implementation
are optimized specifically for the secp256k1 curve.  See
https://www.secg.org/sec2-v2.pdf for details on the secp256k1 standard.

It also provides functions to parse and serialize the Schnorr signatures
according to the specification described herein.

A comprehensive suite of tests is provided to ensure proper functionality.

# Overview

A Schnorr signature is a digital signature scheme that is known for its
simplicity, provable security and efficient generation of short signatures.

It provides many advantages over ECDSA signatures that make them ideal for use
with the only real downside being that they are not well standardized at the
time of this writing.

Some of the advantages over ECDSA include:

 * They are linear which makes them easier to aggregate and use in protocols that
   build on them such as multi-party signatures, threshold signatures, adaptor
   signatures, and blind signatures
 * They are provably secure with weaker assumptions than the best known security
   proofs for ECDSA
   *  Specifically Schnorr signatures are provably secure under SUF-CMA (Strong
      Existential Unforgeability under Chosen Message Attack) in the ROM (Random
      Oracle Model) which guarantees that as long as the hash function behaves
      ideally, the only way to break Schnorr signatures is by solving the ECDLP
      (Elliptic Curve Discrete Logarithm Problem).
 * Their relatively straightforward and efficient aggregation properties make
   them excellent for scalability and allow them to provide some nice privacy
   characteristics
 * They support faster batch verification unlike the standardized version of
   ECDSA signatures

Custom Schnorr-based Signature Scheme

As mentioned in the overview, the primary downside of Schnorr signatures for
elliptic curves is that they are not standardized as well as ECDSA signatures
which means there are a number of variations that are not compatible with each
other.

In addition, many of the standardization attempts have various disadvantages
that make them unsuitable for use in Decred.  Some of these details and some
insight into the design decisions made are discussed further in the README.md
file.

Consequently, this package implements a custom Schnorr-based signature scheme
named EC-Schnorr-DCRv0 suitable for use in Decred.

The following provides a high-level overview of the key design features of the
scheme:

  - Uses signatures of the form (R, s)
  - Produces 64-byte signatures by only encoding the x coordinate of R
  - Enforces even y coordinates for R to support efficient verification by
    disambiguating the two possible y coordinates
  - Canonically encodes by both components of the signature with 32-bytes each
  - Uses BLAKE-256 with 14 rounds for the hash function to calculate challenge e
  - Uses RFC6979 to obviate the need for an entropy source at signing time
  - Produces deterministic signatures for a given message and private key pair

# EC-Schnorr-DCRv0 Specification

See the README.md file for the specific details of the signing and verification
algorithm as well as the signature serialization format.

# Future Design Considerations

It is worth noting that there are some additional optimizations and
modifications that have been identified since the introduction of
EC-Schnorr-DCRv0 that can be made to further harden security for multi-party and
threshold signature use cases as well provide the opportunity for faster
signature verification with a sufficiently optimized implementation.

However, the v0 scheme is used in the existing consensus rules and any changes
to the signature scheme would invalidate existing uses.  Therefore changes in
this regard will need to come in the form of a v1 signature scheme and be
accompanied by the necessary consensus updates.

# Schnorr use in Decred

At the time of this writing, Schnorr signatures are not yet in widespread use on
the Decred network, largely due to the current lack of support in wallets and
infrastructure for secure multi-party and threshold signatures.

However, the consensus rules and scripting engine supports the necessary
primitives and given many of the beneficial properties of Schnorr signatures, a
good goal is to work towards providing the additional infrastructure to increase
their usage.
*/
package schnorr
<commit_msg>schnorr: Go 1.19 doc comment formatting.<commit_after>// Copyright (c) 2020-2022 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

/*
Package schnorr provides custom Schnorr signing and verification via secp256k1.

This package provides data structures and functions necessary to produce and
verify deterministic canonical Schnorr signatures using a custom scheme named
EC-Schnorr-DCRv0 that is described herein.  The signatures and implementation
are optimized specifically for the secp256k1 curve.  See
https://www.secg.org/sec2-v2.pdf for details on the secp256k1 standard.

It also provides functions to parse and serialize the Schnorr signatures
according to the specification described herein.

A comprehensive suite of tests is provided to ensure proper functionality.

# Overview

A Schnorr signature is a digital signature scheme that is known for its
simplicity, provable security and efficient generation of short signatures.

It provides many advantages over ECDSA signatures that make them ideal for use
with the only real downside being that they are not well standardized at the
time of this writing.

Some of the advantages over ECDSA include:

  - They are linear which makes them easier to aggregate and use in protocols that
    build on them such as multi-party signatures, threshold signatures, adaptor
    signatures, and blind signatures
  - They are provably secure with weaker assumptions than the best known security
    proofs for ECDSA
  - Specifically Schnorr signatures are provably secure under SUF-CMA (Strong
    Existential Unforgeability under Chosen Message Attack) in the ROM (Random
    Oracle Model) which guarantees that as long as the hash function behaves
    ideally, the only way to break Schnorr signatures is by solving the ECDLP
    (Elliptic Curve Discrete Logarithm Problem).
  - Their relatively straightforward and efficient aggregation properties make
    them excellent for scalability and allow them to provide some nice privacy
    characteristics
  - They support faster batch verification unlike the standardized version of
    ECDSA signatures

# Custom Schnorr-based Signature Scheme

As mentioned in the overview, the primary downside of Schnorr signatures for
elliptic curves is that they are not standardized as well as ECDSA signatures
which means there are a number of variations that are not compatible with each
other.

In addition, many of the standardization attempts have various disadvantages
that make them unsuitable for use in Decred.  Some of these details and some
insight into the design decisions made are discussed further in the README.md
file.

Consequently, this package implements a custom Schnorr-based signature scheme
named EC-Schnorr-DCRv0 suitable for use in Decred.

The following provides a high-level overview of the key design features of the
scheme:

  - Uses signatures of the form (R, s)
  - Produces 64-byte signatures by only encoding the x coordinate of R
  - Enforces even y coordinates for R to support efficient verification by
    disambiguating the two possible y coordinates
  - Canonically encodes by both components of the signature with 32-bytes each
  - Uses BLAKE-256 with 14 rounds for the hash function to calculate challenge e
  - Uses RFC6979 to obviate the need for an entropy source at signing time
  - Produces deterministic signatures for a given message and private key pair

# EC-Schnorr-DCRv0 Specification

See the README.md file for the specific details of the signing and verification
algorithm as well as the signature serialization format.

# Future Design Considerations

It is worth noting that there are some additional optimizations and
modifications that have been identified since the introduction of
EC-Schnorr-DCRv0 that can be made to further harden security for multi-party and
threshold signature use cases as well provide the opportunity for faster
signature verification with a sufficiently optimized implementation.

However, the v0 scheme is used in the existing consensus rules and any changes
to the signature scheme would invalidate existing uses.  Therefore changes in
this regard will need to come in the form of a v1 signature scheme and be
accompanied by the necessary consensus updates.

# Schnorr use in Decred

At the time of this writing, Schnorr signatures are not yet in widespread use on
the Decred network, largely due to the current lack of support in wallets and
infrastructure for secure multi-party and threshold signatures.

However, the consensus rules and scripting engine supports the necessary
primitives and given many of the beneficial properties of Schnorr signatures, a
good goal is to work towards providing the additional infrastructure to increase
their usage.
*/
package schnorr
<|endoftext|>


<commit_before>package firego

import "encoding/json"

// Push creates a reference to an auto-generated child location
func (fb *Firebase) Push(v interface{}) (*Firebase, error) {
	bytes, err := json.Marshal(v)
	if err != nil {
		return nil, err
	}
	bytes, err = fb.doRequest("POST", bytes)
	var m map[string]string
	if err := json.Unmarshal(bytes, &m); err != nil {
		return nil, err
	}
	return &Firebase{
		url:    fb.url + "/" + m["name"],
		client: fb.client,
	}, err
}
<commit_msg>Fixes a missing error check in firebase.Push()<commit_after>package firego

import "encoding/json"

// Push creates a reference to an auto-generated child location
func (fb *Firebase) Push(v interface{}) (*Firebase, error) {
	bytes, err := json.Marshal(v)
	if err != nil {
		return nil, err
	}
	bytes, err = fb.doRequest("POST", bytes)
	if err != nil {
		return nil, err
	}
	var m map[string]string
	if err := json.Unmarshal(bytes, &m); err != nil {
		return nil, err
	}
	return &Firebase{
		url:    fb.url + "/" + m["name"],
		client: fb.client,
	}, err
}
<|endoftext|>


<commit_before>/*
 * ############################################################
 * #                       DEAPEXER                           #
 * ############################################################
 * # Written by Xan Manning, PyratLabs 2016. http://pyrat.io/ #
 * ############################################################
 * #                                                          #
 * # Released under the MIT License. Please see LICENSE       #
 * # -------------------------------------------------------- #
 * #                                                          #
 * # This application has a very specific use case for        #
 * # dealing with redirecting the apex record of a domain to  #
 * # a subdomain. The main reason for this is that a domain   #
 * # apex record must not be a CNAME, it must be an A record. #
 * #                                                          #
 * # A good example of when to use this is when working with  #
 * # AWS Elastic Load Balancers when you want to avoid using  #
 * # Route 53. This has been something that I have had to     #
 * # work around with customers. Often the solution is to 301 #
 * # redirect in Apache on one of the load balanced EC2       #
 * # instances. This causes disproportionate load on EC2s.    #
 * #                                                          #
 * # What DeApexer allows you to do is host one shared        #
 * # service that generates these redirects for any domain    #
 * # with a low cost to disk I/O. Configuration is stored in  #
 * # memory so this is fairly light on resources.             #
 * #                                                          #
 * ############################################################
 */

package main

/*
 * Import our required Go modules
 */
import (
    "encoding/json"
    "flag"
    "io/ioutil"
    "log"
    "log/syslog"
    "net/http"
    "regexp"
    "strconv"
)

/*
 * Define our Configuration and UrlOverride data struct
 *
 * These will be populated from JSON data so we need to define in the struct
 * the key value relationship.
 */
type Configuration struct {
    ListenPort          int             `json:"listen_port"`
    DefaultSubdomain    string          `json:"default_subdomain"`
    Debug               bool            `json:"debug"`
    LogRedirect         bool            `json:"log_redirects"`
    Overrides           []UrlOverride   `json:"url_overrides"`
}

type UrlOverride struct {
    Hostname            string          `json:"hostname"`
    Subdomain           string          `json:"subdomain"`
    DefaultPath         string          `json:"default_path"`
    PathOverrides       []PathOverride  `json:"path_overrides"`
}

type PathOverride struct {
    Source              string          `json:"path_source"`
    Destination         string          `json:"path_destination"`
}

/*
 * Define our global configuration variable.
 */
var config Configuration

/*
 * Our init() function, used to build the configuration and set up syslog.
 */
func init() {
    // Do we have a specific location for our configuration file?
    configFile := flag.String("c",
        "/etc/deapexer/config.json",
        "Configuration File")

    // Are we wanting to avoid using Syslog?
    noLogs := flag.Bool("n", false, "Don't Log to Syslog")

    // Parse flag input
    flag.Parse()

    // Set up our syslog, throw error if there is a problem.
    if *noLogs == false {
        logfile, syslog_err := syslog.New(syslog.LOG_NOTICE, "deapexer")
        check(syslog_err)

        // Make log function write to log file.
        log.SetOutput(logfile)
    }

    // Read config data file, throw error if there is a problem.
    config_data, err := ioutil.ReadFile(*configFile)
    check(err)

    // Unmarshal the configuration file.
    unmarshal_err := json.Unmarshal(config_data, &config)
    check(unmarshal_err)

    // Debugging:
    //      - config file name
    //      - listening port
    //      - default subdomain.
    if config.Debug == true {
        log.Print("configFile: ", *configFile)
        log.Print("Loaded Configuration")
        log.Print("Listening Port: ", config.ListenPort)
        log.Print("Default Subdomain: ", config.DefaultSubdomain)
    }
}

/*
 * Define our function for checking errors and throwing panic.
 */
func check(e error) {
    if e != nil {
        panic(e)
    }
}

/*
 * Our deapex() function, generates the redirect
 */
func deapex(w http.ResponseWriter, r *http.Request) {
    // Are we logging redirects?
    if config.Debug == true || config.LogRedirect == true {
        log.Print("Request Host: ", r.Host)
        log.Print("Request URI: ", r.RequestURI)
    }

    // Obtain our host and URI from request data.
    requestHost := r.Host
    requestURI := r.RequestURI

    // Create the regexp to match the port number
    rpPort := regexp.MustCompile(":[0-9]+")

    // Strip the port number out of request host
    hostName := rpPort.ReplaceAllString(requestHost, "")

    // Variable to contain our subdomain redirect.
    var redirectSubdomain string

    // Subdomain Redirect Key
    var oKey int

    // Variable to contain our path redirect.
    var redirectPath string

    // Iterate through all the overrides we are storing in memory.
    for i := 0 ; i < len(config.Overrides) ; i++ {
        // Does the hostname value match override hostname?
        if config.Overrides[i].Hostname == hostName {
            // Set our redirect subdomain to this override.
            redirectSubdomain = config.Overrides[i].Subdomain
            oKey = i
        }
    }

    // If there is no redirect subdomain in overrides, use the default.
    if redirectSubdomain == "" {
        redirectSubdomain = config.DefaultSubdomain + "." + hostName
    } else {
        // Iterate through the path overrides for this hostname.
        for i := 0 ; i < len(config.Overrides[oKey].PathOverrides) ; i++ {
            // Does the request URI value match override path?
            if config.Overrides[oKey].PathOverrides[i].Source == requestURI {
                // Set our URI to this path.
                redirectPath = config.Overrides[oKey].PathOverrides[i].
                    Destination
            }
        }
    }

    // How are we going to redirect the path?
    if redirectPath == "" {
        if config.Overrides[oKey].DefaultPath != "" {
            requestURI = config.Overrides[oKey].DefaultPath
        }
    } else {
        requestURI = redirectPath
    }

    // Debug, where are we going?
    if config.Debug == true || config.LogRedirect == true {
        log.Print("New Host: ", redirectSubdomain)
        log.Print("New Request URI: ", requestURI)
    }

    // Build our forward address.
    forwardAddr := "http://" + redirectSubdomain + requestURI

    // Serve our 301 redirect.
    http.Redirect(w, r, forwardAddr, 301)
}

/*
 * Our main() function, sets up the http listener to handle deapex() function.
 */
func main() {
    // Define the function deapex() as our handler for.
    http.HandleFunc("/", deapex)

    // Listen and serve HTTP on our defined port.
    initErr := http.ListenAndServe(":" + strconv.Itoa(config.ListenPort), nil)

    // If there was an error setting up HTTP listener then log and throw panic.
    if initErr != nil {
        if config.Debug == true {
            log.Fatal("ListenAndServe: ", initErr)
        }
        panic(initErr)
    }
}
<commit_msg>Added a break<commit_after>/*
 * ############################################################
 * #                       DEAPEXER                           #
 * ############################################################
 * # Written by Xan Manning, PyratLabs 2016. http://pyrat.io/ #
 * ############################################################
 * #                                                          #
 * # Released under the MIT License. Please see LICENSE       #
 * # -------------------------------------------------------- #
 * #                                                          #
 * # This application has a very specific use case for        #
 * # dealing with redirecting the apex record of a domain to  #
 * # a subdomain. The main reason for this is that a domain   #
 * # apex record must not be a CNAME, it must be an A record. #
 * #                                                          #
 * # A good example of when to use this is when working with  #
 * # AWS Elastic Load Balancers when you want to avoid using  #
 * # Route 53. This has been something that I have had to     #
 * # work around with customers. Often the solution is to 301 #
 * # redirect in Apache on one of the load balanced EC2       #
 * # instances. This causes disproportionate load on EC2s.    #
 * #                                                          #
 * # What DeApexer allows you to do is host one shared        #
 * # service that generates these redirects for any domain    #
 * # with a low cost to disk I/O. Configuration is stored in  #
 * # memory so this is fairly light on resources.             #
 * #                                                          #
 * ############################################################
 */

package main

/*
 * Import our required Go modules
 */
import (
    "encoding/json"
    "flag"
    "io/ioutil"
    "log"
    "log/syslog"
    "net/http"
    "regexp"
    "strconv"
)

/*
 * Define our Configuration and UrlOverride data struct
 *
 * These will be populated from JSON data so we need to define in the struct
 * the key value relationship.
 */
type Configuration struct {
    ListenPort          int             `json:"listen_port"`
    DefaultSubdomain    string          `json:"default_subdomain"`
    Debug               bool            `json:"debug"`
    LogRedirect         bool            `json:"log_redirects"`
    Overrides           []UrlOverride   `json:"url_overrides"`
}

type UrlOverride struct {
    Hostname            string          `json:"hostname"`
    Subdomain           string          `json:"subdomain"`
    DefaultPath         string          `json:"default_path"`
    PathOverrides       []PathOverride  `json:"path_overrides"`
}

type PathOverride struct {
    Source              string          `json:"path_source"`
    Destination         string          `json:"path_destination"`
}

/*
 * Define our global configuration variable.
 */
var config Configuration

/*
 * Our init() function, used to build the configuration and set up syslog.
 */
func init() {
    // Do we have a specific location for our configuration file?
    configFile := flag.String("c",
        "/etc/deapexer/config.json",
        "Configuration File")

    // Are we wanting to avoid using Syslog?
    noLogs := flag.Bool("n", false, "Don't Log to Syslog")

    // Parse flag input
    flag.Parse()

    // Set up our syslog, throw error if there is a problem.
    if *noLogs == false {
        logfile, syslog_err := syslog.New(syslog.LOG_NOTICE, "deapexer")
        check(syslog_err)

        // Make log function write to log file.
        log.SetOutput(logfile)
    }

    // Read config data file, throw error if there is a problem.
    config_data, err := ioutil.ReadFile(*configFile)
    check(err)

    // Unmarshal the configuration file.
    unmarshal_err := json.Unmarshal(config_data, &config)
    check(unmarshal_err)

    // Debugging:
    //      - config file name
    //      - listening port
    //      - default subdomain.
    if config.Debug == true {
        log.Print("configFile: ", *configFile)
        log.Print("Loaded Configuration")
        log.Print("Listening Port: ", config.ListenPort)
        log.Print("Default Subdomain: ", config.DefaultSubdomain)
    }
}

/*
 * Define our function for checking errors and throwing panic.
 */
func check(e error) {
    if e != nil {
        panic(e)
    }
}

/*
 * Our deapex() function, generates the redirect
 */
func deapex(w http.ResponseWriter, r *http.Request) {
    // Are we logging redirects?
    if config.Debug == true || config.LogRedirect == true {
        log.Print("Request Host: ", r.Host)
        log.Print("Request URI: ", r.RequestURI)
    }

    // Obtain our host and URI from request data.
    requestHost := r.Host
    requestURI := r.RequestURI

    // Create the regexp to match the port number
    rpPort := regexp.MustCompile(":[0-9]+")

    // Strip the port number out of request host
    hostName := rpPort.ReplaceAllString(requestHost, "")

    // Variable to contain our subdomain redirect.
    var redirectSubdomain string

    // Subdomain Redirect Key
    var oKey int

    // Variable to contain our path redirect.
    var redirectPath string

    // Iterate through all the overrides we are storing in memory.
    for i := 0 ; i < len(config.Overrides) ; i++ {
        // Does the hostname value match override hostname?
        if config.Overrides[i].Hostname == hostName {
            // Set our redirect subdomain to this override.
            redirectSubdomain = config.Overrides[i].Subdomain
            oKey = i
            break
        }
    }

    // If there is no redirect subdomain in overrides, use the default.
    if redirectSubdomain == "" {
        redirectSubdomain = config.DefaultSubdomain + "." + hostName
    } else {
        // Iterate through the path overrides for this hostname.
        for i := 0 ; i < len(config.Overrides[oKey].PathOverrides) ; i++ {
            // Does the request URI value match override path?
            if config.Overrides[oKey].PathOverrides[i].Source == requestURI {
                // Set our URI to this path.
                redirectPath = config.Overrides[oKey].PathOverrides[i].
                    Destination
                break
            }
        }
    }

    // How are we going to redirect the path?
    if redirectPath == "" {
        if config.Overrides[oKey].DefaultPath != "" {
            requestURI = config.Overrides[oKey].DefaultPath
        }
    } else {
        requestURI = redirectPath
    }

    // Debug, where are we going?
    if config.Debug == true || config.LogRedirect == true {
        log.Print("New Host: ", redirectSubdomain)
        log.Print("New Request URI: ", requestURI)
    }

    // Build our forward address.
    forwardAddr := "http://" + redirectSubdomain + requestURI

    // Serve our 301 redirect.
    http.Redirect(w, r, forwardAddr, 301)
}

/*
 * Our main() function, sets up the http listener to handle deapex() function.
 */
func main() {
    // Define the function deapex() as our handler for.
    http.HandleFunc("/", deapex)

    // Listen and serve HTTP on our defined port.
    initErr := http.ListenAndServe(":" + strconv.Itoa(config.ListenPort), nil)

    // If there was an error setting up HTTP listener then log and throw panic.
    if initErr != nil {
        if config.Debug == true {
            log.Fatal("ListenAndServe: ", initErr)
        }
        panic(initErr)
    }
}
<|endoftext|>


<commit_before>package dexcom

import (
	"log"
	"time"
)

func (cgm *Cgm) ReadHistory(pageType PageType, since time.Time) []Record {
	first, last := cgm.ReadPageRange(pageType)
	if cgm.Error() != nil {
		return nil
	}
	results := []Record{}
	proc := func(r Record) (bool, error) {
		t := r.Timestamp.DisplayTime
		if t.Before(since) {
			log.Printf("stopping at timestamp %s", t.Format(time.RFC3339))
			return true, nil
		}
		results = append(results, r)
		return false, nil
	}
	cgm.IterRecords(pageType, first, last, proc)
	return results
}

func (cgm *Cgm) ReadCount(pageType PageType, count int) []Record {
	first, last := cgm.ReadPageRange(pageType)
	if cgm.Error() != nil {
		return nil
	}
	results := []Record{}
	proc := func(r Record) (bool, error) {
		results = append(results, r)
		return len(results) == count, nil
	}
	cgm.IterRecords(pageType, first, last, proc)
	return results
}

const (
	// Time window within which EGV and sensor readings will be merged.
	glucoseReadingWindow = 2 * time.Second
)

func (cgm *Cgm) GlucoseReadings(since time.Time) []Record {
	sensor := cgm.ReadHistory(SENSOR_DATA, since)
	if cgm.Error() != nil {
		return nil
	}
	numSensor := len(sensor)
	egv := cgm.ReadHistory(EGV_DATA, since)
	if cgm.Error() != nil {
		return nil
	}
	numEgv := len(egv)
	readings := []Record{}
	i, j := 0, 0
	for {
		r := Record{}
		if i < numSensor && j < numEgv {
			sensorTime := sensor[i].Timestamp.DisplayTime
			egvTime := egv[j].Timestamp.DisplayTime
			delta := egvTime.Sub(sensorTime)
			if 0 <= delta && delta < glucoseReadingWindow {
				// Merge using sensor[i]'s slightly earlier time.
				r = sensor[i]
				r.Egv = egv[j].Egv
				i++
				j++
			} else if 0 <= -delta && -delta < glucoseReadingWindow {
				// Merge using egv[j]'s slightly earlier time.
				r = egv[j]
				r.Sensor = sensor[i].Sensor
				i++
				j++
			} else if sensorTime.After(egvTime) {
				r = sensor[i]
				i++
			} else {
				r = egv[j]
				j++
			}
		} else if i < numSensor {
			r = sensor[i]
			i++
		} else if j < numEgv {
			r = egv[j]
			j++
		} else {
			break
		}
		readings = append(readings, r)
	}
	return readings
}
<commit_msg>Add MergeHistory function<commit_after>package dexcom

import (
	"log"
	"time"
)

func (cgm *Cgm) ReadHistory(pageType PageType, since time.Time) []Record {
	first, last := cgm.ReadPageRange(pageType)
	if cgm.Error() != nil {
		return nil
	}
	results := []Record{}
	proc := func(r Record) (bool, error) {
		t := r.Timestamp.DisplayTime
		if t.Before(since) {
			log.Printf("stopping at timestamp %s", t.Format(time.RFC3339))
			return true, nil
		}
		results = append(results, r)
		return false, nil
	}
	cgm.IterRecords(pageType, first, last, proc)
	return results
}

func (cgm *Cgm) ReadCount(pageType PageType, count int) []Record {
	first, last := cgm.ReadPageRange(pageType)
	if cgm.Error() != nil {
		return nil
	}
	results := []Record{}
	proc := func(r Record) (bool, error) {
		results = append(results, r)
		return len(results) == count, nil
	}
	cgm.IterRecords(pageType, first, last, proc)
	return results
}

// Merge slices of records that are already in reverse chronological order
// into a single ordered slice.
func MergeHistory(slices ...[]Record) []Record {
	n := len(slices)
	if n == 0 {
		return nil
	}
	length := make([]int, n)
	total := 0
	for i, v := range slices {
		length[i] = len(v)
		total += len(v)
	}
	results := make([]Record, total)
	index := make([]int, n)
	for next, _ := range results {
		// Find slice with latest current value.
		which := -1
		max := time.Time{}
		for i, v := range slices {
			if index[i] < len(v) {
				t := v[index[i]].Timestamp.DisplayTime
				if t.After(max) {
					which = i
					max = t
				}
			}
		}
		results[next] = slices[which][index[which]]
		index[which]++
	}
	return results
}

const (
	// Time window within which EGV and sensor readings will be merged.
	glucoseReadingWindow = 2 * time.Second
)

func (cgm *Cgm) GlucoseReadings(since time.Time) []Record {
	sensor := cgm.ReadHistory(SENSOR_DATA, since)
	if cgm.Error() != nil {
		return nil
	}
	numSensor := len(sensor)
	egv := cgm.ReadHistory(EGV_DATA, since)
	if cgm.Error() != nil {
		return nil
	}
	numEgv := len(egv)
	readings := []Record{}
	i, j := 0, 0
	for {
		r := Record{}
		if i < numSensor && j < numEgv {
			sensorTime := sensor[i].Timestamp.DisplayTime
			egvTime := egv[j].Timestamp.DisplayTime
			delta := egvTime.Sub(sensorTime)
			if 0 <= delta && delta < glucoseReadingWindow {
				// Merge using sensor[i]'s slightly earlier time.
				r = sensor[i]
				r.Egv = egv[j].Egv
				i++
				j++
			} else if 0 <= -delta && -delta < glucoseReadingWindow {
				// Merge using egv[j]'s slightly earlier time.
				r = egv[j]
				r.Sensor = sensor[i].Sensor
				i++
				j++
			} else if sensorTime.After(egvTime) {
				r = sensor[i]
				i++
			} else {
				r = egv[j]
				j++
			}
		} else if i < numSensor {
			r = sensor[i]
			i++
		} else if j < numEgv {
			r = egv[j]
			j++
		} else {
			break
		}
		readings = append(readings, r)
	}
	return readings
}
<|endoftext|>


<commit_before>package goPy

import (
	"encoding/json"
	"fmt"
	"net"
)

//TCProute is the first struct in this program
type TCProute struct {
	Host string
	Port string
}

type data struct {
	Method string        `json:"method"`
	Args   []interface{} `json:"args"`
	Time   string        `json:"time"`
}

type dataResponse struct {
	Response string `json:"response"`
}

type connection struct {
	conn net.Conn
}

func (obj *TCProute) connect() (*connection, error) {
	addr := obj.Host + ":" + obj.Port
	fmt.Println("host and port to connect", addr)
	client, err := net.Dial("tcp", addr)
	ret := connection{client}
	if err != nil {
		return nil, err
	}
	return &ret, nil
}

func (obj *connection) sendData(dataIN *data) (bool, error) {
	jsonData, err := json.Marshal(*dataIN)
	if err != nil {
		return false, err
	}
	obj.conn.Write(jsonData)
	return true, nil
}

func (obj *connection) recvData() (dataResponse, error) {
	r := make([]byte, 4096)
	n, _ := obj.conn.Read(r)
	response := r[:n]
	var v dataResponse
	err := json.Unmarshal(response, &v)
	if err != nil {
		return v, err
	}
	return v, nil
}

// func main() {
// 	cli := &TCProute{"localhost", "9000"}
// 	conn, _ := cli.connect()
// 	t0 := time.Now()
// 	tIni := (strconv.Itoa(t0.Hour())) + ":" + (strconv.Itoa(t0.Minute())) + ":" + strconv.Itoa(t0.Second()) + "." + strconv.Itoa(t0.Nanosecond())[:6]
// 	d := &data{"show", []interface{}{10, 20.2, 30, 40, "i am nikhil"}, tIni}
// 	conn.sendData(d)
// 	recv, _ := conn.recvData()
// 	fmt.Println(recv)
//
// }
<commit_msg>Added comments<commit_after>package goPy

import (
	"encoding/json"
	"fmt"
	"net"
)

//TCProute is the first struct in this program
type TCProute struct {
	Host string
	Port string
}

// the data structure to be used to make RPC call
type data struct {
	Method string        `json:"method"`
	Args   []interface{} `json:"args"`
	Time   string        `json:"time"`
}

// The Response of data from RPC CALL
type dataResponse struct {
	Response string `json:"response"`
}

// Using the simple connection interface from net package
type connection struct {
	conn net.Conn
}

func (obj *TCProute) connect() (*connection, error) {
	addr := obj.Host + ":" + obj.Port
	fmt.Println("host and port to connect", addr)
	client, err := net.Dial("tcp", addr)
	ret := connection{client}
	if err != nil {
		return nil, err
	}
	return &ret, nil
}

func (obj *connection) sendData(dataIN *data) (bool, error) {
	jsonData, err := json.Marshal(*dataIN)
	if err != nil {
		return false, err
	}
	obj.conn.Write(jsonData)
	return true, nil
}

func (obj *connection) recvData() (dataResponse, error) {
	r := make([]byte, 4096)
	n, _ := obj.conn.Read(r)
	response := r[:n]
	var v dataResponse
	err := json.Unmarshal(response, &v)
	if err != nil {
		return v, err
	}
	return v, nil
}

// func main() {
// 	cli := &TCProute{"localhost", "9000"}
// 	conn, _ := cli.connect()
// 	t0 := time.Now()
// 	tIni := (strconv.Itoa(t0.Hour())) + ":" + (strconv.Itoa(t0.Minute())) + ":" + strconv.Itoa(t0.Second()) + "." + strconv.Itoa(t0.Nanosecond())[:6]
// 	d := &data{"show", []interface{}{10, 20.2, 30, 40, "i am nikhil"}, tIni}
// 	conn.sendData(d)
// 	recv, _ := conn.recvData()
// 	fmt.Println(recv)
//
// }
<|endoftext|>


<commit_before>package goku

import (
	"encoding/json"
	"errors"
	"log"
	"net"
	"reflect"
	"runtime"
	"time"

	"github.com/garyburd/redigo/redis"
)

var (
	ErrPointer = errors.New("method receiver was a pointer when it shouldn't be")
)

var (
	registry map[string]Job
	rc       redis.Conn
)

type Config struct {
	Hostport string
	Timeout  time.Duration
}

func Configure(cfg Config) error {
	registry = make(map[string]Job)
	conn, err := net.Dial("tcp", cfg.Hostport)
	if err != nil {
		return err
	}
	rc = redis.NewConn(conn, cfg.Timeout, cfg.Timeout)
	return nil
}

type Job interface {
	Execute() error
}

func Register(jobs ...Job) {
	for _, j := range jobs {
		registry[getFunctionName(j.Execute)] = j
	}
}

type marshalledJob struct {
	N string
	A map[string]interface{}
}

func Run(j Job, queue string) error {
	args := make(map[string]interface{})

	rv := reflect.ValueOf(j)
	rt := reflect.TypeOf(j)

	for rv.Kind() == reflect.Ptr {
		return ErrPointer
	}

	for i := 0; i < rv.NumField(); i++ {
		field := rt.Field(i)
		value := rv.Field(i)
		args[field.Name] = value.Interface()
	}

	jsn, err := json.Marshal(marshalledJob{
		N: getFunctionName(j.Execute),
		A: args,
	})

	if err != nil {
		return err
	}

	if _, err := rc.Do("RPUSH", queue, jsn); err != nil {
		return err
	}
	return nil
}

func getFunctionName(i interface{}) string {
	return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
}

type WorkerConfig struct {
	NumWorkers   int
	PollInterval time.Duration
	Failure      func(worker int, jobName string, r interface{})
	Queue        string
}

type FailureFunc func(worker int, jobName string, r interface{})

func Work(config WorkerConfig, jobs []Job) error {
	ch := make(chan marshalledJob)

	for i := 0; i < config.NumWorkers; i++ {
		go worker(i, ch, config.Failure)
	}

	for ; ; time.Sleep(time.Second) {
		jsn, err := redis.Bytes(rc.Do("LPOP", config.Queue))
		if err != nil {
			continue
		}

		var j marshalledJob
		if err := json.Unmarshal(jsn, &j); err != nil {
			log.Fatal(err)
		}
		ch <- j
	}
}

func worker(n int, ch chan marshalledJob, failure FailureFunc) {
	var jobName string

	if failure != nil {
		defer func() {
			if r := recover(); r != nil {
				failure(n, jobName, r)
			}
		}()
	}

	for j := range ch {
		jobName = j.N
		job, ok := registry[j.N]
		if !ok {
			log.Fatal("Invalid job")
		}

		nj := reflect.New(reflect.TypeOf(job)).Elem()
		for k, v := range j.A {
			field := nj.FieldByName(k)
			if field.CanSet() {
				field.Set(reflect.ValueOf(v))
			}
		}
		nj.Interface().(Job).Execute()
	}
}
<commit_msg>Requeue jobs when the worker can't process it<commit_after>package goku

import (
	"encoding/json"
	"errors"
	"log"
	"net"
	"reflect"
	"runtime"
	"time"

	"github.com/garyburd/redigo/redis"
)

// goku errors
var (
	ErrPointer = errors.New("method receiver was a pointer when it shouldn't be")
)

var (
	registry map[string]Job
	rc       redis.Conn
)

type Config struct {
	Hostport string
	Timeout  time.Duration
}

func Configure(cfg Config) error {
	registry = make(map[string]Job)
	conn, err := net.Dial("tcp", cfg.Hostport)
	if err != nil {
		return err
	}
	rc = redis.NewConn(conn, cfg.Timeout, cfg.Timeout)
	return nil
}

type Job interface {
	Execute() error
}

func Register(jobs ...Job) {
	for _, j := range jobs {
		registry[getFunctionName(j.Execute)] = j
	}
}

type marshalledJob struct {
	N string
	A map[string]interface{}
}

func Run(j Job, queue string) error {
	args := make(map[string]interface{})

	rv := reflect.ValueOf(j)
	rt := reflect.TypeOf(j)

	for rv.Kind() == reflect.Ptr {
		return ErrPointer
	}

	for i := 0; i < rv.NumField(); i++ {
		field := rt.Field(i)
		value := rv.Field(i)
		args[field.Name] = value.Interface()
	}

	jsn, err := json.Marshal(marshalledJob{
		N: getFunctionName(j.Execute),
		A: args,
	})

	if err != nil {
		return err
	}

	if _, err := rc.Do("RPUSH", queue, jsn); err != nil {
		return err
	}
	return nil
}

func getFunctionName(i interface{}) string {
	return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
}

type WorkerConfig struct {
	NumWorkers   int
	PollInterval time.Duration
	Failure      func(worker int, jobName string, r interface{})
	Queue        string
}

type FailureFunc func(worker int, jobName string, r interface{})

func Work(config WorkerConfig, jobs []Job) error {
	ch := make(chan []byte)
	requeuerCh := make(chan []byte)

	for i := 0; i < config.NumWorkers; i++ {
		go worker(i, ch, requeuerCh, config.Failure)
	}

	for ; ; time.Sleep(config.PollInterval) {
		jsn, err := redis.Bytes(rc.Do("LPOP", config.Queue))
		if err != nil {
			continue
		}
		ch <- jsn
	}
}

func reqeuer(qn string, ch chan []byte) {
	for jsn := range ch {
		if _, err := rc.Do("RPUSH", qn, jsn); err != nil {
			// TODO (retry)
		}
	}
}

func worker(n int, ch chan []byte, requeuerCh chan []byte, failure FailureFunc) {
	var jobName string

	if failure != nil {
		defer func() {
			if r := recover(); r != nil {
				failure(n, jobName, r)
			}
		}()
	}

	for jsn := range ch {
		var j marshalledJob
		if err := json.Unmarshal(jsn, &j); err != nil {
			log.Fatal(err)
		}

		jobName = j.N
		job, ok := registry[jobName]
		if !ok {
			requeuerCh <- jsn
			continue
		}

		nj := reflect.New(reflect.TypeOf(job)).Elem()
		for k, v := range j.A {
			field := nj.FieldByName(k)
			if field.CanSet() {
				field.Set(reflect.ValueOf(v))
			}
		}
		nj.Interface().(Job).Execute()
	}
}
<|endoftext|>


<commit_before>/**
 * Copyright 2015 @ at3.net.
 * name : gord.go
 * author : jarryliu
 * date : -- :
 * description :
 * history :
 */
package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

func main() {
	var port int
	var dir string
	flag.IntVar(&port, "port", 8302, "port")
	flag.StringVar(&dir, "dir", "./", "config file")
	baseDir := dir

	app := http.Server{
		Addr:    fmt.Sprintf(":%d", port),
		Handler: getHandler(baseDir),
	}

	log.Printf("[ Service] - running on port %s\n", app.Addr)
	if err := app.ListenAndServe(); err != nil {
		log.Fatalf(" Server aborted!reason:%s\n", err.Error())
	}
}

func getHandler(baseDir string) http.Handler {
	re := &redirectHandler{
		itemManager: &ItemManager{
			basePath: baseDir,
		},
	}
	re.itemManager.Load()
	return re
}

type Item struct {
	//主机头，*表示通配
	Host string `json:"host"`
	//全局请求跳转路径,{path}表示完整的路径；
	//{#序号}表示路径片段的序号
	To string `json:"to"`
	//如果未设定全局请求跳转路径，那么将启用路径字典
	//如果{"a/b/c":"http://abc.com"}，访问/a/b/c将跳转
	//到"http://abc.com"
	Location map[string]string `json:"location"`
}

type ItemManager struct {
	items    map[string]*Item
	basePath string
}

// 检查目录，并初始化
func (i *ItemManager) checkDir(path string) {
	_, err := os.Stat(path)
	//创建目录
	if os.IsNotExist(err) {
		os.MkdirAll(path, os.ModePerm)
		i.initExample()
	} else {
		//是否存在.conf文件,不存在，则初始化
		fi, _ := os.Open(path)
		exits := false
		list, _ := fi.Readdirnames(-1)
		for _, v := range list {
			if strings.HasSuffix(v, ".conf") {
				exits = true
			}
		}
		if !exits {
			i.initExample()
		}
	}
}

func (i *ItemManager) initExample() {
	var defaultItems []*Item = []*Item{
		{
			Host: "*.at3.net",
			To:   "http://www.at3.net/{path}{query}",
			Location: map[string]string{
				"/a":      "http://a.com",
				"/b/*":    "http://b.com/{*}",
				"/a/b":    "http://a.com/{path}{query}",
				"/1/2/3/": "http://a.com/{#0}-{#1}-{#2}",
			},
		},
	}

	//创建文件
	bytes, _ := json.MarshalIndent(defaultItems, " ", " ")
	f, err := os.Create("example.conf")
	if err == nil {
		wr := bufio.NewWriter(f)
		wr.Write(bytes)
		err = wr.Flush()
		f.Close()
	}
	if err != nil {
		log.Println(" init example config fail ! error :", err.Error())
	}
}

// 加载配置
func (i *ItemManager) Load() {
	i.checkDir(i.basePath)
	filepath.Walk(i.basePath, func(path string, info os.FileInfo, err error) error {
		if err == nil && !info.IsDir() && strings.HasSuffix(path, ".conf") {
			items := i.GetItemsFromFile(path)
			i.Append(items)
		}
		return nil
	})
}

func (i *ItemManager) checkItem(item *Item) error {
	if len(item.Host) < 2 {
		return errors.New("主机头长度不正确")
	}
	return nil
}

// 从文件中加载配置项目
func (i *ItemManager) GetItemsFromFile(path string) []*Item {
	bytes, err := ioutil.ReadFile(path)
	if err == nil {
		//从文件中反序列化
		items := make([]*Item, 0)
		err = json.Unmarshal(bytes, &items)
		//检查配置是否正确
		if err == nil {
			for _, v := range items {
				if err = i.checkItem(v); err != nil {
					log.Println(fmt.Sprintf("配置有误! 文件：%s; host: %s; error:%s",
						path, v.Host, err.Error()))
					os.Exit(1)
				}
			}
			return items
		}
	}
	log.Println("加载配置:" + path + "出错：" + err.Error())
	os.Exit(1)
	return nil
}

// 增加配置项
func (i *ItemManager) Append(items []*Item) {
	if i.items == nil {
		i.items = make(map[string]*Item, 0)
	}
	if items != nil {
		for _, v := range items {
			if _, ok := i.items[v.Host]; ok {
				panic("has exists host " + v.Host)
			}
			i.items[v.Host] = v
		}
	}
}

// 根据主机名获取相应的配置
func (i *ItemManager) GetItemByHost(host string) *Item {
	for k, v := range i.items {
		if i.matchHost(k, host) {
			return v
		}
	}
	return nil
}

// 匹配主机
func (i *ItemManager) matchHost(hostKey, host string) bool {
	if host == hostKey {
		return true
	}
	// 判断是否泛解析
	if strings.HasPrefix(hostKey, "*.") {
		return strings.HasSuffix(host, hostKey[2:])
	}
	return false
}

var _ http.Handler = new(redirectHandler)

type redirectHandler struct {
	itemManager *ItemManager
}

func (r *redirectHandler) ServeHTTP(rsp http.ResponseWriter, req *http.Request) {
	host := req.Host
	host = "www.at3.net" // "z3q.net" use for test
	var item *Item = r.itemManager.GetItemByHost(host)
	if item != nil {
		if location, b := r.getLocation(rsp, req, item); b {
			rsp.Header().Add("Location", location)
			rsp.WriteHeader(302)
			return
		}
	}
	rsp.Write([]byte("Not match any host"))
}

func (r *redirectHandler) getLocation(rsp http.ResponseWriter,
	req *http.Request, item *Item) (string, bool) {
	path := req.URL.Path
	query := req.URL.RawQuery
	concat := ""
	if len(query) != 0 {
		concat = "?"
	}
	//查找匹配
	to := item.To
	for k, v := range item.Location {
		//判断路径是否相同
		if path == k {
			to = v
			break
		}
		//匹配如：/d/* 含通配符的路径
		if strings.HasSuffix(k, "*") {
			if strings.HasPrefix(path, k[:len(k)-1]) {
				to = v
				break
			}
		}
	}
	//无匹配
	to = strings.TrimSpace(to)
	if to == "" {
		return "", false
	}

	//全局请求跳转路径,{path}表示完整的路径；
	if strings.Contains(to, "{path}") {
		to = strings.Replace(to, "{path}", path[1:], -1)
	}
	if strings.Contains(to, "{query}") {
		to = strings.Replace(to, "{query}", concat+query, -1)
	}
	//匹配含有路径片段的URL,{#序号}表示指定的路径片段
	if strings.Contains(to, "{#") {
		segments := strings.Split(path[1:], "/")
		for i, l := 0, len(segments); i < l; i++ {
			to = strings.Replace(to, "{#"+strconv.Itoa(i)+"}",
				segments[i], -1)
		}
	}
	//log.Println("--- origin:", path, "; target:", to)
	return to, true
}
<commit_msg>gord.go<commit_after>/**
 * Copyright 2015 @ at3.net.
 * name : gord.go
 * author : jarryliu
 * date : -- :
 * description :
 * history :
 */
package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

func main() {
	var port int
	var dir string
	flag.IntVar(&port, "port", 8302, "port")
	flag.StringVar(&dir, "dir", "./", "config file")
	baseDir := dir

	app := http.Server{
		Addr:    fmt.Sprintf(":%d", port),
		Handler: getHandler(baseDir),
	}

	log.Printf("[ Service] - running on port %s\n", app.Addr)
	if err := app.ListenAndServe(); err != nil {
		log.Fatalf(" Server aborted!reason:%s\n", err.Error())
	}
}

func getHandler(baseDir string) http.Handler {
	re := &redirectHandler{
		itemManager: &ItemManager{
			basePath: baseDir,
		},
	}
	re.itemManager.Load()
	return re
}

type Item struct {
	//主机头，*表示通配
	Host string `json:"host"`
	//全局请求跳转路径,{path}表示完整的路径；
	//{#序号}表示路径片段的序号
	To string `json:"to"`
	//如果未设定全局请求跳转路径，那么将启用路径字典
	//如果{"a/b/c":"http://abc.com"}，访问/a/b/c将跳转
	//到"http://abc.com"
	Location map[string]string `json:"location"`
}

type ItemManager struct {
	items    map[string]*Item
	basePath string
}

// 检查目录，并初始化
func (i *ItemManager) checkDir(path string) {
	_, err := os.Stat(path)
	//创建目录
	if os.IsNotExist(err) {
		os.MkdirAll(path, os.ModePerm)
		i.initExample()
	} else {
		//是否存在.conf文件,不存在，则初始化
		fi, _ := os.Open(path)
		exits := false
		list, _ := fi.Readdirnames(-1)
		for _, v := range list {
			if strings.HasSuffix(v, ".conf") {
				exits = true
			}
		}
		if !exits {
			i.initExample()
		}
	}
}

func (i *ItemManager) initExample() {
	var defaultItems []*Item = []*Item{
		{
			Host: "*.at3.net",
			To:   "http://www.at3.net/{path}{query}",
			Location: map[string]string{
				"/a":      "http://a.com",
				"/b/*":    "http://b.com/{*}",
				"/a/b":    "http://a.com/{path}{query}",
				"/1/2/3/": "http://a.com/{#0}-{#1}-{#2}",
			},
		},
	}

	//创建文件
	bytes, _ := json.MarshalIndent(defaultItems, " ", " ")
	f, err := os.Create("example.conf")
	if err == nil {
		wr := bufio.NewWriter(f)
		wr.Write(bytes)
		err = wr.Flush()
		f.Close()
	}
	if err != nil {
		log.Println(" init example config fail ! error :", err.Error())
	}
}

// 加载配置
func (i *ItemManager) Load() {
	i.checkDir(i.basePath)
	filepath.Walk(i.basePath, func(path string, info os.FileInfo, err error) error {
		if err == nil && !info.IsDir() && strings.HasSuffix(path, ".conf") {
			items := i.GetItemsFromFile(path)
			i.Append(items)
		}
		return nil
	})
}

func (i *ItemManager) checkItem(item *Item) error {
	if len(item.Host) < 2 {
		return errors.New("主机头长度不正确")
	}
	return nil
}

// 从文件中加载配置项目
func (i *ItemManager) GetItemsFromFile(path string) []*Item {
	bytes, err := ioutil.ReadFile(path)
	if err == nil {
		//从文件中反序列化
		items := make([]*Item, 0)
		err = json.Unmarshal(bytes, &items)
		//检查配置是否正确
		if err == nil {
			for _, v := range items {
				if err = i.checkItem(v); err != nil {
					log.Println(fmt.Sprintf("配置有误! 文件：%s; host: %s; error:%s",
						path, v.Host, err.Error()))
					os.Exit(1)
				}
			}
			return items
		}
	}
	log.Println("加载配置:" + path + "出错：" + err.Error())
	os.Exit(1)
	return nil
}

// 增加配置项
func (i *ItemManager) Append(items []*Item) {
	if i.items == nil {
		i.items = make(map[string]*Item, 0)
	}
	if items != nil {
		for _, v := range items {
			if _, ok := i.items[v.Host]; ok {
				panic("has exists host " + v.Host)
			}
			i.items[v.Host] = v
		}
	}
}

// 根据主机名获取相应的配置
func (i *ItemManager) GetItemByHost(host string) *Item {
	for k, v := range i.items {
		if i.matchHost(k, host) {
			return v
		}
	}
	return nil
}

// 匹配主机
func (i *ItemManager) matchHost(hostKey, host string) bool {
	if host == hostKey {
		return true
	}
	// 判断是否泛解析
	if strings.HasPrefix(hostKey, "*.") {
		return strings.HasSuffix(host, hostKey[2:])
	}
	return false
}

var _ http.Handler = new(redirectHandler)

type redirectHandler struct {
	itemManager *ItemManager
}

func (r *redirectHandler) ServeHTTP(rsp http.ResponseWriter, req *http.Request) {
	host := req.Host
	//host = "www.at3.net" // "z3q.net" use for test
	var item *Item = r.itemManager.GetItemByHost(host)
	if item != nil {
		if location, b := r.getLocation(rsp, req, item); b {
			rsp.Header().Add("Location", location)
			rsp.WriteHeader(302)
			return
		}
	}
	rsp.Write([]byte("Not match any host"))
}

func (r *redirectHandler) getLocation(rsp http.ResponseWriter,
	req *http.Request, item *Item) (string, bool) {
	path := req.URL.Path
	query := req.URL.RawQuery
	concat := ""
	if len(query) != 0 {
		concat = "?"
	}
	//查找匹配
	to := item.To
	for k, v := range item.Location {
		//判断路径是否相同
		if path == k {
			to = v
			break
		}
		//匹配如：/d/* 含通配符的路径
		if strings.HasSuffix(k, "*") {
			if strings.HasPrefix(path, k[:len(k)-1]) {
				to = v
				break
			}
		}
	}
	//无匹配
	to = strings.TrimSpace(to)
	if to == "" {
		return "", false
	}

	//全局请求跳转路径,{path}表示完整的路径；
	if strings.Contains(to, "{path}") {
		to = strings.Replace(to, "{path}", path[1:], -1)
	}
	if strings.Contains(to, "{query}") {
		to = strings.Replace(to, "{query}", concat+query, -1)
	}
	//匹配含有路径片段的URL,{#序号}表示指定的路径片段
	if strings.Contains(to, "{#") {
		segments := strings.Split(path[1:], "/")
		for i, l := 0, len(segments); i < l; i++ {
			to = strings.Replace(to, "{#"+strconv.Itoa(i)+"}",
				segments[i], -1)
		}
	}
	//log.Println("--- origin:", path, "; target:", to)
	return to, true
}
<|endoftext|>