<commit_before>/*
 Copyright 2017 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package templates

const Pocketmine = `{
  "pufferd": {
    "type": "pocketmine",
    "display": "PocketMine-MP",
    "install": {
      "commands": [
        {
          "type": "download",
          "files": "https://raw.githubusercontent.com/PocketMine/php-build-scripts/master/installer.sh"
        },
        {
          "commands": [
            "chmod +x ./installer.sh",
            "./installer.sh"
          ],
          "type": "command"
        },
        {
          "type": "writefile",
          "text": "server-ip=${ip}\nserver-port=${port}\n",
          "target": "server.properties"
        }
      ]
    },
    "run": {
      "stop": "stop",
      "pre": [],
      "post": [],
      "arguments": [
      	"--no-wizard"
      ],
      "program": "./start.sh"
    },
    "environment": {
      "type": "standard"
    },
    "data": {
      "ip": {
        "value": "0.0.0.0",
        "required": true,
        "desc": "What IP to bind the server to",
        "display": "IP",
        "internal": false
      },
      "port": {
        "value": "19132",
        "required": true,
        "desc": "What port to bind the server to",
        "display": "Port",
        "internal": false
      }
    }
  }
}`
<commit_msg>Update PocketMine template<commit_after>/*
 Copyright 2017 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package templates

const Pocketmine = `{
  "pufferd": {
    "type": "pocketmine",
    "display": "PocketMine-MP",
    "install": {
      "commands": [
        {
          "type": "download",
          "files": "https://raw.githubusercontent.com/pmmp/php-build-scripts/master/installer.sh"
        },
        {
          "commands": [
            "chmod +x ./installer.sh",
            "./installer.sh"
          ],
          "type": "command"
        },
        {
          "type": "writefile",
          "text": "server-ip=${ip}\nserver-port=${port}\n",
          "target": "server.properties"
        }
      ]
    },
    "run": {
      "stop": "stop",
      "pre": [],
      "post": [],
      "arguments": [
      	"--no-wizard"
      ],
      "program": "./start.sh"
    },
    "environment": {
      "type": "standard"
    },
    "data": {
      "ip": {
        "value": "0.0.0.0",
        "required": true,
        "desc": "What IP to bind the server to",
        "display": "IP",
        "internal": false
      },
      "port": {
        "value": "19132",
        "required": true,
        "desc": "What port to bind the server to",
        "display": "Port",
        "internal": false
      }
    }
  }
}`
<|endoftext|>


<commit_before>/*package shellfish contains code for computing the splashback shells of
halos in N-body simulations.*/
package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"

	"github.com/phil-mansfield/shellfish/cmd"
	"github.com/phil-mansfield/shellfish/cmd/env"
	"github.com/phil-mansfield/shellfish/version"
)

var helpStrings = map[string]string{
	"setup": `The setup mode isn't implemented yet.`,
	"id":    `Mode specifcations will be documented in version 0.3.0.`,
	"tree":  `Mode specifcations will be documented in version 0.3.0.`,
	"coord": `Mode specifcations will be documented in version 0.3.0.`,
	"shell": `Mode specifcations will be documented in version 0.3.0.`,
	"stats": `Mode specifcations will be documented in version 0.3.0.`,

	"config":       new(cmd.GlobalConfig).ExampleConfig(),
	"setup.config": `The setup mode does not have a non-global config file.`,
	"id.config":    cmd.ModeNames["id"].ExampleConfig(),
	"tree.config":  `The tree mode does not have a non-global config file.`,
	"coord.config": `The coord mode does not have a non-global config file.`,
	"shell.config": cmd.ModeNames["shell"].ExampleConfig(),
	"stats.config": cmd.ModeNames["stats"].ExampleConfig(),
}

var modeDescriptions = `My help modes are:
shellfish help
shellfish help [ setup | id | tree | coord | shell | stats ]
shellfish help [ config | id.config | shell.config | stats.config ]

My setup mode is:
shellfish setup ____.config

My analysis modes are:
shellfish id     [flags] ____.config [____.id.config]
shellfish tree ____.config
shellfish coord ____.config
shellfish shell  [flags] ____.config [____.shell.config]
shellfish stats  [flags] ____.config [____.stats.config]`

func main() {
	args := os.Args
	if len(args) <= 1 {
		fmt.Fprintf(
			os.Stderr, "I was not supplied with a mode.\nFor help, type "+
				"'./shellfish help'.\n",
		)
		os.Exit(1)
	}

	if args[1] == "setup" {
		// TODO: Implement the setup command.
		panic("NYI")
	} else if args[1] == "help" {
		switch len(args) - 2 {
		case 0:
			fmt.Println(modeDescriptions)
		case 1:
			text, ok := helpStrings[args[2]]
			if !ok {
				fmt.Printf("I don't recognize the help target '%s'\n", args[2])
			} else {
				fmt.Println(text)
			}
		case 2:
			fmt.Println("The help mode can only take a single argument.")
		}
		os.Exit(0)
		// TODO: Implement the help command.
	} else if args[1] == "version" {
		fmt.Printf("Shellfish version %s\n", version.SourceVersion)
		os.Exit(0)
	}

	mode, ok := cmd.ModeNames[args[1]]
	if !ok {
		fmt.Fprintf(
			os.Stderr, "You passed me the mode '%s', which I don't "+
				"recognize.\nFor help, type './shellfish help'\n", args[1],
		)
		os.Exit(1)
	}

	var lines []string
	switch args[1] {
	case "tree", "coord", "shell", "stats":
		var err error
		lines, err = stdinLines()
		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(1)
		}
	}

	flags := getFlags(args)
	config, ok := getConfig(args)
	gConfigName, gConfig, err := getGlobalConfig(args)
	if err != nil {
		log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
	}

	if ok {
		if err = mode.ReadConfig(config); err != nil {
			log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
		}
	} else {
		if err = mode.ReadConfig(""); err != nil {
			log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
		}
	}

	if err = checkMemoDir(gConfig.MemoDir, gConfigName); err != nil {
		log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
	}

	e := &env.Environment{MemoDir: gConfig.MemoDir}
	initCatalogs(gConfig, e)
	initHalos(args[1], gConfig, e)

	if err != nil {
		log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
	}

	if err != nil {
		log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
	}

	out, err := mode.Run(flags, gConfig, e, lines)
	if err != nil {
		log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
	}

	for i := range out {
		fmt.Println(out[i])
	}
}

// stdinLines reads stdin and splits it into lines.
func stdinLines() ([]string, error) {
	bs, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, fmt.Errorf(
			"Error reading stdin: %s.", err.Error(),
		)
	}
	text := string(bs)
	lines := strings.Split(text, "\n")
	if lines[len(lines)-1] == "" {
		lines = lines[:len(lines)-1]
	}
	return lines, nil
}

// getFlags reutrns the flag tokens from the command line arguments.
func getFlags(args []string) []string {
	return args[1 : len(args)-1-configNum(args)]
}

// getGlobalConfig returns the name of the base config file from the command
// line arguments.
func getGlobalConfig(args []string) (string, *cmd.GlobalConfig, error) {
	name := os.Getenv("SHELLFISH_GLOBAL_CONFIG")
	if name != "" {
		if configNum(args) > 1 {
			return "", nil, fmt.Errorf("$SHELLFISH_GLOBAL_CONFIG has been " +
				"set, so you may only pass a single config file as a " +
				"parameter.")
		}

		config := &cmd.GlobalConfig{}
		err := config.ReadConfig(name)
		if err != nil {
			return "", nil, err
		}
		return name, config, nil
	}

	switch configNum(args) {
	case 0:
		return "", nil, fmt.Errorf("No config files provided in command " +
			"line arguments.")
	case 1:
		name = args[len(args)-1]
	case 2:
		name = args[len(args)-2]
	default:
		return "", nil, fmt.Errorf("Passed too many config files as arguments.")
	}

	config := &cmd.GlobalConfig{}
	err := config.ReadConfig(name)
	if err != nil {
		return "", nil, err
	}
	return name, config, nil
}

// getConfig return the name of the mode-specific config file from the command
// line arguments.
func getConfig(args []string) (string, bool) {
	if os.Getenv("SHELLFISH_GLOBAL_CONFIG") != "" && configNum(args) == 1 {
		return args[len(args)-1], true
	} else if os.Getenv("SHELLFISH_GLOBAL_CONFIG") == "" &&
		configNum(args) == 2 {

		return args[len(args)-1], true
	}
	return "", false
}

// configNum returns the number of configuration files at the end of the
// argument list (up to 2).
func configNum(args []string) int {
	num := 0
	for i := len(args) - 1; i >= 0; i-- {
		if isConfig(args[i]) {
			num++
		} else {
			break
		}
	}
	return num
}

// isConfig returns true if the fiven string is a config file name.
func isConfig(s string) bool {
	return len(s) >= 7 && s[len(s)-7:] == ".config"
}

// cehckMemoDir checks whether the given MemoDir corresponds to a GlobalConfig
// file with the exact same variables. If not, a non-nil error is returned.
// If the MemoDir does not have an associated GlobalConfig file, the current
// one will be copied in.
func checkMemoDir(memoDir, configFile string) error {
	memoConfigFile := path.Join(memoDir, "memo.config")

	if _, err := os.Stat(memoConfigFile); err != nil {
		// File doesn't exist, directory is clean.
		err = copyFile(memoConfigFile, configFile)
		return err
	}

	config, memoConfig := &cmd.GlobalConfig{}, &cmd.GlobalConfig{}
	if err := config.ReadConfig(configFile); err != nil {
		return err
	}
	if err := memoConfig.ReadConfig(memoConfigFile); err != nil {
		return err
	}

	if !configEqual(config, memoConfig) {
		return fmt.Errorf("The variables in the config file '%s' do not "+
			"match the varables used when creating the MemoDir, '%s.' These "+
			"variables can be compared by inspecting '%s' and '%s'",
			configFile, memoDir, configFile, memoConfigFile,
		)
	}
	return nil
}

// copyFile copies a file from src to dst.
func copyFile(dst, src string) error {
	srcFile, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	dstFile, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer dstFile.Close()

	if _, err = io.Copy(dstFile, srcFile); err != nil {
		return err
	}
	return dstFile.Sync()
}

func configEqual(m, c *cmd.GlobalConfig) bool {
	// Well, equal up to the variables that actually matter.
	// (i.e. changing something like Threads shouldn't flush the memoization
	// buffer. Otherwise, I'd just use reflection.)
	return c.Version == m.Version &&
		c.SnapshotFormat == m.SnapshotFormat &&
		c.SnapshotType == m.SnapshotType &&
		c.HaloDir == m.HaloDir &&
		c.HaloType == m.HaloType &&
		c.TreeDir == m.TreeDir &&
		c.MemoDir == m.MemoDir && // (this is impossible)
		int64sEqual(c.BlockMins, m.BlockMins) &&
		int64sEqual(c.BlockMaxes, m.BlockMaxes) &&
		c.SnapMin == m.SnapMin &&
		c.SnapMax == m.SnapMax &&
		stringsEqual(c.SnapshotFormatMeanings, m.SnapshotFormatMeanings) &&
		c.HaloPositionUnits == m.HaloPositionUnits &&
		c.HaloMassUnits == m.HaloMassUnits &&
		c.HaloIDColumn == m.HaloIDColumn &&
		c.HaloM200mColumn == m.HaloM200mColumn &&
		int64sEqual(c.HaloPositionColumns, m.HaloPositionColumns) &&
		c.Endianness == m.Endianness
}

func int64sEqual(xs, ys []int64) bool {
	if len(xs) != len(ys) {
		return false
	}
	for i := range xs {
		if xs[i] != ys[i] {
			return false
		}
	}
	return true
}

func stringsEqual(xs, ys []string) bool {
	if len(xs) != len(ys) {
		return false
	}
	for i := range xs {
		if xs[i] != ys[i] {
			return false
		}
	}
	return true
}

func initHalos(
	mode string, gConfig *cmd.GlobalConfig, e *env.Environment,
) error {
	switch mode {
	case "shell", "stats":
		return nil
	}

	switch gConfig.HaloType {
	case "nil":
		return fmt.Errorf("You may not use nil as a HaloType for the "+
			"mode '%s.'\n", mode)
	case "Text":
		return e.InitTextHalo(&gConfig.HaloInfo)
		if gConfig.TreeType != "consistent-trees" {
			return fmt.Errorf("You're trying to use the '%s' TreeType with " +
				"the 'Text' HaloType.")
		}
	}
	if gConfig.TreeType == "nil" {
		return fmt.Errorf("You may not use nil as a TreeType for the "+
			"mode '%s.'\n", mode)
	}

	panic("Impossible")
}

func initCatalogs(gConfig *cmd.GlobalConfig, e *env.Environment) error {
	switch gConfig.SnapshotType {
	case "gotetra":
		return e.InitGotetra(&gConfig.ParticleInfo, gConfig.ValidateFormats)
	case "LGadget-2":
		return e.InitLGadget2(&gConfig.ParticleInfo, gConfig.ValidateFormats)
	case "ARTIO":
		return e.InitARTIO(&gConfig.ParticleInfo, gConfig.ValidateFormats)
	}
	panic("Impossible.")
}
<commit_msg>Added error checking to catalog initializers.<commit_after>/*package shellfish contains code for computing the splashback shells of
halos in N-body simulations.*/
package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"

	"github.com/phil-mansfield/shellfish/cmd"
	"github.com/phil-mansfield/shellfish/cmd/env"
	"github.com/phil-mansfield/shellfish/version"
)

var helpStrings = map[string]string{
	"setup": `The setup mode isn't implemented yet.`,
	"id":    `Mode specifcations will be documented in version 0.3.0.`,
	"tree":  `Mode specifcations will be documented in version 0.3.0.`,
	"coord": `Mode specifcations will be documented in version 0.3.0.`,
	"shell": `Mode specifcations will be documented in version 0.3.0.`,
	"stats": `Mode specifcations will be documented in version 0.3.0.`,

	"config":       new(cmd.GlobalConfig).ExampleConfig(),
	"setup.config": `The setup mode does not have a non-global config file.`,
	"id.config":    cmd.ModeNames["id"].ExampleConfig(),
	"tree.config":  `The tree mode does not have a non-global config file.`,
	"coord.config": `The coord mode does not have a non-global config file.`,
	"shell.config": cmd.ModeNames["shell"].ExampleConfig(),
	"stats.config": cmd.ModeNames["stats"].ExampleConfig(),
}

var modeDescriptions = `My help modes are:
shellfish help
shellfish help [ setup | id | tree | coord | shell | stats ]
shellfish help [ config | id.config | shell.config | stats.config ]

My setup mode is:
shellfish setup ____.config

My analysis modes are:
shellfish id     [flags] ____.config [____.id.config]
shellfish tree ____.config
shellfish coord ____.config
shellfish shell  [flags] ____.config [____.shell.config]
shellfish stats  [flags] ____.config [____.stats.config]`

func main() {
	args := os.Args
	if len(args) <= 1 {
		fmt.Fprintf(
			os.Stderr, "I was not supplied with a mode.\nFor help, type "+
				"'./shellfish help'.\n",
		)
		os.Exit(1)
	}

	if args[1] == "setup" {
		// TODO: Implement the setup command.
		panic("NYI")
	} else if args[1] == "help" {
		switch len(args) - 2 {
		case 0:
			fmt.Println(modeDescriptions)
		case 1:
			text, ok := helpStrings[args[2]]
			if !ok {
				fmt.Printf("I don't recognize the help target '%s'\n", args[2])
			} else {
				fmt.Println(text)
			}
		case 2:
			fmt.Println("The help mode can only take a single argument.")
		}
		os.Exit(0)
		// TODO: Implement the help command.
	} else if args[1] == "version" {
		fmt.Printf("Shellfish version %s\n", version.SourceVersion)
		os.Exit(0)
	}

	mode, ok := cmd.ModeNames[args[1]]
	if !ok {
		fmt.Fprintf(
			os.Stderr, "You passed me the mode '%s', which I don't "+
				"recognize.\nFor help, type './shellfish help'\n", args[1],
		)
		os.Exit(1)
	}

	var lines []string
	switch args[1] {
	case "tree", "coord", "shell", "stats":
		var err error
		lines, err = stdinLines()
		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(1)
		}
	}

	flags := getFlags(args)
	config, ok := getConfig(args)
	gConfigName, gConfig, err := getGlobalConfig(args)
	if err != nil {
		log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
	}

	if ok {
		if err = mode.ReadConfig(config); err != nil {
			log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
		}
	} else {
		if err = mode.ReadConfig(""); err != nil {
			log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
		}
	}

	if err = checkMemoDir(gConfig.MemoDir, gConfigName); err != nil {
		log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
	}

	e := &env.Environment{MemoDir: gConfig.MemoDir}
	err = initCatalogs(gConfig, e)
	if err != nil {
		log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
	}
	err = initHalos(args[1], gConfig, e)
	if err != nil {
		log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
	}

	if err != nil {
		log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
	}

	if err != nil {
		log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
	}

	out, err := mode.Run(flags, gConfig, e, lines)
	if err != nil {
		log.Fatalf("Error running mode %s:\n%s\n", args[1], err.Error())
	}

	for i := range out {
		fmt.Println(out[i])
	}
}

// stdinLines reads stdin and splits it into lines.
func stdinLines() ([]string, error) {
	bs, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, fmt.Errorf(
			"Error reading stdin: %s.", err.Error(),
		)
	}
	text := string(bs)
	lines := strings.Split(text, "\n")
	if lines[len(lines)-1] == "" {
		lines = lines[:len(lines)-1]
	}
	return lines, nil
}

// getFlags reutrns the flag tokens from the command line arguments.
func getFlags(args []string) []string {
	return args[1 : len(args)-1-configNum(args)]
}

// getGlobalConfig returns the name of the base config file from the command
// line arguments.
func getGlobalConfig(args []string) (string, *cmd.GlobalConfig, error) {
	name := os.Getenv("SHELLFISH_GLOBAL_CONFIG")
	if name != "" {
		if configNum(args) > 1 {
			return "", nil, fmt.Errorf("$SHELLFISH_GLOBAL_CONFIG has been " +
				"set, so you may only pass a single config file as a " +
				"parameter.")
		}

		config := &cmd.GlobalConfig{}
		err := config.ReadConfig(name)
		if err != nil {
			return "", nil, err
		}
		return name, config, nil
	}

	switch configNum(args) {
	case 0:
		return "", nil, fmt.Errorf("No config files provided in command " +
			"line arguments.")
	case 1:
		name = args[len(args)-1]
	case 2:
		name = args[len(args)-2]
	default:
		return "", nil, fmt.Errorf("Passed too many config files as arguments.")
	}

	config := &cmd.GlobalConfig{}
	err := config.ReadConfig(name)
	if err != nil {
		return "", nil, err
	}
	return name, config, nil
}

// getConfig return the name of the mode-specific config file from the command
// line arguments.
func getConfig(args []string) (string, bool) {
	if os.Getenv("SHELLFISH_GLOBAL_CONFIG") != "" && configNum(args) == 1 {
		return args[len(args)-1], true
	} else if os.Getenv("SHELLFISH_GLOBAL_CONFIG") == "" &&
		configNum(args) == 2 {

		return args[len(args)-1], true
	}
	return "", false
}

// configNum returns the number of configuration files at the end of the
// argument list (up to 2).
func configNum(args []string) int {
	num := 0
	for i := len(args) - 1; i >= 0; i-- {
		if isConfig(args[i]) {
			num++
		} else {
			break
		}
	}
	return num
}

// isConfig returns true if the fiven string is a config file name.
func isConfig(s string) bool {
	return len(s) >= 7 && s[len(s)-7:] == ".config"
}

// cehckMemoDir checks whether the given MemoDir corresponds to a GlobalConfig
// file with the exact same variables. If not, a non-nil error is returned.
// If the MemoDir does not have an associated GlobalConfig file, the current
// one will be copied in.
func checkMemoDir(memoDir, configFile string) error {
	memoConfigFile := path.Join(memoDir, "memo.config")

	if _, err := os.Stat(memoConfigFile); err != nil {
		// File doesn't exist, directory is clean.
		err = copyFile(memoConfigFile, configFile)
		return err
	}

	config, memoConfig := &cmd.GlobalConfig{}, &cmd.GlobalConfig{}
	if err := config.ReadConfig(configFile); err != nil {
		return err
	}
	if err := memoConfig.ReadConfig(memoConfigFile); err != nil {
		return err
	}

	if !configEqual(config, memoConfig) {
		return fmt.Errorf("The variables in the config file '%s' do not "+
			"match the varables used when creating the MemoDir, '%s.' These "+
			"variables can be compared by inspecting '%s' and '%s'",
			configFile, memoDir, configFile, memoConfigFile,
		)
	}
	return nil
}

// copyFile copies a file from src to dst.
func copyFile(dst, src string) error {
	srcFile, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	dstFile, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer dstFile.Close()

	if _, err = io.Copy(dstFile, srcFile); err != nil {
		return err
	}
	return dstFile.Sync()
}

func configEqual(m, c *cmd.GlobalConfig) bool {
	// Well, equal up to the variables that actually matter.
	// (i.e. changing something like Threads shouldn't flush the memoization
	// buffer. Otherwise, I'd just use reflection.)
	return c.Version == m.Version &&
		c.SnapshotFormat == m.SnapshotFormat &&
		c.SnapshotType == m.SnapshotType &&
		c.HaloDir == m.HaloDir &&
		c.HaloType == m.HaloType &&
		c.TreeDir == m.TreeDir &&
		c.MemoDir == m.MemoDir && // (this is impossible)
		int64sEqual(c.BlockMins, m.BlockMins) &&
		int64sEqual(c.BlockMaxes, m.BlockMaxes) &&
		c.SnapMin == m.SnapMin &&
		c.SnapMax == m.SnapMax &&
		stringsEqual(c.SnapshotFormatMeanings, m.SnapshotFormatMeanings) &&
		c.HaloPositionUnits == m.HaloPositionUnits &&
		c.HaloMassUnits == m.HaloMassUnits &&
		c.HaloIDColumn == m.HaloIDColumn &&
		c.HaloM200mColumn == m.HaloM200mColumn &&
		int64sEqual(c.HaloPositionColumns, m.HaloPositionColumns) &&
		c.Endianness == m.Endianness
}

func int64sEqual(xs, ys []int64) bool {
	if len(xs) != len(ys) {
		return false
	}
	for i := range xs {
		if xs[i] != ys[i] {
			return false
		}
	}
	return true
}

func stringsEqual(xs, ys []string) bool {
	if len(xs) != len(ys) {
		return false
	}
	for i := range xs {
		if xs[i] != ys[i] {
			return false
		}
	}
	return true
}

func initHalos(
	mode string, gConfig *cmd.GlobalConfig, e *env.Environment,
) error {
	switch mode {
	case "shell", "stats":
		return nil
	}

	switch gConfig.HaloType {
	case "nil":
		return fmt.Errorf("You may not use nil as a HaloType for the "+
			"mode '%s.'\n", mode)
	case "Text":
		return e.InitTextHalo(&gConfig.HaloInfo)
		if gConfig.TreeType != "consistent-trees" {
			return fmt.Errorf("You're trying to use the '%s' TreeType with " +
				"the 'Text' HaloType.")
		}
	}
	if gConfig.TreeType == "nil" {
		return fmt.Errorf("You may not use nil as a TreeType for the "+
			"mode '%s.'\n", mode)
	}

	panic("Impossible")
}

func initCatalogs(gConfig *cmd.GlobalConfig, e *env.Environment) error {
	switch gConfig.SnapshotType {
	case "gotetra":
		return e.InitGotetra(&gConfig.ParticleInfo, gConfig.ValidateFormats)
	case "LGadget-2":
		return e.InitLGadget2(&gConfig.ParticleInfo, gConfig.ValidateFormats)
	case "ARTIO":
		return e.InitARTIO(&gConfig.ParticleInfo, gConfig.ValidateFormats)
	}
	panic("Impossible.")
}
<|endoftext|>


<commit_before>// Test APP
package main

import (
	"github.com/dim13/gold"
	"log"
	"net/http"
	"sort"
	"strconv"
	"text/template"
)

const listen = ":8000"

var (
	conf *gold.Config
	data *gold.Data
	tmpl *template.Template
)

type Page struct {
	Config   *gold.Config
	Title    string
	Articles gold.Articles
	Article  *gold.Article
	Error    error
	PrevPage int
	NextPage int
	Expand   bool
	TagCloud gold.TagCloud
}

func adminList(w http.ResponseWriter, r *http.Request, s []string) {
	p := Page{
		Config:   conf,
		Title:    "Admin interface",
		Articles: data.Articles,
	}
	err := tmpl.ExecuteTemplate(w, "admin.tmpl", p)
	if err != nil {
		log.Fatal(err)
	}
}

func adminSlug(w http.ResponseWriter, r *http.Request, s []string) {
	var p Page

	a, err := data.Articles.Find(s[0])
	if err != nil {
		p = Page{
			Config: conf,
			Error:  err,
		}
	} else {
		p = Page{
			Config:  conf,
			Title:   a.Title,
			Article: a,
		}
	}

	err = tmpl.ExecuteTemplate(w, "admin.tmpl", p)
	if err != nil {
		log.Fatal(err)
	}
}

func index(w http.ResponseWriter, r *http.Request, s []string) {
	var p Page

	a, err := data.Articles.Find(s[0])
	if err == nil {
		p = Page{
			Title:    a.Title,
			Articles: gold.Articles{a},
			Expand:   true,
		}
		genpage(w, p)
	} else {
		page(w, r, []string{"1"})
	}
}

func page(w http.ResponseWriter, r *http.Request, s []string) {
	pg, err := strconv.Atoi(s[0])
	if err != nil {
		log.Fatal(err)
	}
	app := conf.Blog.ArticlesPerPage

	a, next, prev := data.Articles.Enabled().Page(pg, app)

	p := Page{
		Title:    conf.Blog.Title,
		Articles: a,
		NextPage: next,
		PrevPage: prev,
	}

	genpage(w, p)
}

func tags(w http.ResponseWriter, r *http.Request, s []string) {
	var a gold.Articles
	for _, v := range data.Articles {
		if v.Tags.Has(s[0]) {
			a = append(a, v)
		}
	}
	p := Page{
		Title:    conf.Blog.Title + " - " + s[0],
		Articles: a,
	}
	genpage(w, p)
}

func assets(w http.ResponseWriter, r *http.Request, s []string) {
	http.ServeFile(w, r, r.URL.Path[1:])
}

func rss(w http.ResponseWriter, r *http.Request, s []string) {
	a := data.Articles.Enabled()
	app := conf.Blog.ArticlesPerPage

	p := Page{
		Config:   conf,
		Articles: a[:app],
	}

	err := tmpl.ExecuteTemplate(w, "rss.tmpl", p)
	if err != nil {
		log.Fatal(err)
	}
}

func year(w http.ResponseWriter, r *http.Request, s []string) {
	y, err := strconv.Atoi(s[0])
	if err != nil {
		log.Fatal(err)
	}
	a := data.Articles.Year(y)
	p := Page{
		Title:    conf.Blog.Title + " - " + s[0],
		Articles: a,
	}
	genpage(w, p)
}

func genpage(w http.ResponseWriter, p Page) {
	p.TagCloud = data.Articles.TagCloud(conf.Blog.TagsInCloud)
	p.Config = conf

	err := tmpl.ExecuteTemplate(w, "index.tmpl", p)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	var err error

	conf, err = gold.ReadConf("config/config.ini")
	if err != nil {
		log.Fatal(err)
	}

	data = gold.Open(conf.Blog.DataBase)
	if err := data.Read(); err != nil {
		log.Println(err)
	}
	sort.Sort(sort.Reverse(data.Articles))

	tmpl = template.Must(template.ParseGlob("templates/*.tmpl"))

	re := new(gold.ReHandler)
	re.AddRoute("^/assets/", assets)
	re.AddRoute("^/images/", assets)
	re.AddRoute("^/admin/(.+)$", adminSlug)
	re.AddRoute("^/admin/?$", adminList)
	re.AddRoute("^/tags?/(.+)$", tags)
	re.AddRoute("^/page/(\\d+)$", page)
	re.AddRoute("^/rss\\.xml$", rss)
	re.AddRoute("^/(\\d+)/$", year)
	re.AddRoute("^/(\\d+)/(\\d+)/(.*)$", index)
	re.AddRoute("^/(.*)$", index)

	if err := http.ListenAndServe(listen, re); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>sitemap stub<commit_after>// Test APP
package main

import (
	"github.com/dim13/gold"
	"log"
	"net/http"
	"sort"
	"strconv"
	"text/template"
)

const listen = ":8000"

var (
	conf *gold.Config
	data *gold.Data
	tmpl *template.Template
)

type Page struct {
	Config   *gold.Config
	Title    string
	Articles gold.Articles
	Article  *gold.Article
	Error    error
	PrevPage int
	NextPage int
	Expand   bool
	TagCloud gold.TagCloud
}

func adminList(w http.ResponseWriter, r *http.Request, s []string) {
	p := Page{
		Config:   conf,
		Title:    "Admin interface",
		Articles: data.Articles,
	}
	err := tmpl.ExecuteTemplate(w, "admin.tmpl", p)
	if err != nil {
		log.Fatal(err)
	}
}

func adminSlug(w http.ResponseWriter, r *http.Request, s []string) {
	var p Page

	a, err := data.Articles.Find(s[0])
	if err != nil {
		p = Page{
			Config: conf,
			Error:  err,
		}
	} else {
		p = Page{
			Config:  conf,
			Title:   a.Title,
			Article: a,
		}
	}

	err = tmpl.ExecuteTemplate(w, "admin.tmpl", p)
	if err != nil {
		log.Fatal(err)
	}
}

func index(w http.ResponseWriter, r *http.Request, s []string) {
	var p Page

	a, err := data.Articles.Find(s[0])
	if err == nil {
		p = Page{
			Title:    a.Title,
			Articles: gold.Articles{a},
			Expand:   true,
		}
		genpage(w, p)
	} else {
		page(w, r, []string{"1"})
	}
}

func page(w http.ResponseWriter, r *http.Request, s []string) {
	pg, err := strconv.Atoi(s[0])
	if err != nil {
		log.Fatal(err)
	}
	app := conf.Blog.ArticlesPerPage

	a, next, prev := data.Articles.Enabled().Page(pg, app)

	p := Page{
		Title:    conf.Blog.Title,
		Articles: a,
		NextPage: next,
		PrevPage: prev,
	}

	genpage(w, p)
}

func tags(w http.ResponseWriter, r *http.Request, s []string) {
	var a gold.Articles
	for _, v := range data.Articles {
		if v.Tags.Has(s[0]) {
			a = append(a, v)
		}
	}
	p := Page{
		Title:    conf.Blog.Title + " - " + s[0],
		Articles: a,
	}
	genpage(w, p)
}

func assets(w http.ResponseWriter, r *http.Request, s []string) {
	http.ServeFile(w, r, r.URL.Path[1:])
}

func rss(w http.ResponseWriter, r *http.Request, s []string) {
	a := data.Articles.Enabled()
	app := conf.Blog.ArticlesPerPage

	p := Page{
		Config:   conf,
		Articles: a[:app],
	}

	err := tmpl.ExecuteTemplate(w, "rss.tmpl", p)
	if err != nil {
		log.Fatal(err)
	}
}

func sitemap(w http.ResponseWriter, r *http.Request, s []string) {
}

func year(w http.ResponseWriter, r *http.Request, s []string) {
	y, err := strconv.Atoi(s[0])
	if err != nil {
		log.Fatal(err)
	}
	a := data.Articles.Year(y)
	p := Page{
		Title:    conf.Blog.Title + " - " + s[0],
		Articles: a,
	}
	genpage(w, p)
}

func genpage(w http.ResponseWriter, p Page) {
	p.TagCloud = data.Articles.TagCloud(conf.Blog.TagsInCloud)
	p.Config = conf

	err := tmpl.ExecuteTemplate(w, "index.tmpl", p)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	var err error

	conf, err = gold.ReadConf("config/config.ini")
	if err != nil {
		log.Fatal(err)
	}

	data = gold.Open(conf.Blog.DataBase)
	if err := data.Read(); err != nil {
		log.Println(err)
	}
	sort.Sort(sort.Reverse(data.Articles))

	tmpl = template.Must(template.ParseGlob("templates/*.tmpl"))

	re := new(gold.ReHandler)
	re.AddRoute("^/assets/", assets)
	re.AddRoute("^/images/", assets)
	re.AddRoute("^/admin/(.+)$", adminSlug)
	re.AddRoute("^/admin/?$", adminList)
	re.AddRoute("^/tags?/(.+)$", tags)
	re.AddRoute("^/page/(\\d+)$", page)
	re.AddRoute("^/rss\\.xml$", rss)
	re.AddRoute("^/sitemap\\.xml$", sitemap)
	re.AddRoute("^/(\\d+)/$", year)
	re.AddRoute("^/(\\d+)/(\\d+)/(.*)$", index)
	re.AddRoute("^/(.*)$", index)

	if err := http.ListenAndServe(listen, re); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package statsd

// In package metrics so we can stub tick.

import (
	"bytes"
	"fmt"
	"runtime"
	"testing"
	"time"
)

func TestCounter(t *testing.T) {
	ch := make(chan time.Time)
	tick = func(time.Duration) <-chan time.Time { return ch }
	defer func() { tick = time.Tick }()

	buf := &bytes.Buffer{}
	c := NewCounter(buf, "test_statsd_counter", time.Second)

	c.Add(1)
	c.Add(2)
	ch <- time.Now()

	for i := 0; i < 10 && buf.Len() == 0; i++ {
		time.Sleep(time.Millisecond)
	}

	if want, have := "test_statsd_counter:1|c\ntest_statsd_counter:2|c\n", buf.String(); want != have {
		t.Errorf("want %q, have %q", want, have)
	}
}

func TestGauge(t *testing.T) {
	ch := make(chan time.Time)
	tick = func(time.Duration) <-chan time.Time { return ch }
	defer func() { tick = time.Tick }()

	buf := &bytes.Buffer{}
	g := NewGauge(buf, "test_statsd_gauge", time.Second)

	delta := 1.0
	g.Add(delta)      // send command
	runtime.Gosched() // yield to buffer write
	ch <- time.Now()  // signal flush
	runtime.Gosched() // yield to flush
	if want, have := fmt.Sprintf("test_statsd_gauge:+%f|g\n", delta), buf.String(); want != have {
		t.Errorf("want %q, have %q", want, have)
	}

	buf.Reset()

	delta = -2.0
	g.Add(delta)
	runtime.Gosched()
	ch <- time.Now()
	runtime.Gosched()
	if want, have := fmt.Sprintf("test_statsd_gauge:%f|g\n", delta), buf.String(); want != have {
		t.Errorf("want %q, have %q", want, have)
	}

	buf.Reset()

	value := 3.0
	g.Set(value)
	runtime.Gosched()
	ch <- time.Now()
	runtime.Gosched()
	if want, have := fmt.Sprintf("test_statsd_gauge:%f|g\n", value), buf.String(); want != have {
		t.Errorf("want %q, have %q", want, have)
	}
}

func TestCallbackGauge(t *testing.T) {
	ch := make(chan time.Time)
	tick = func(time.Duration) <-chan time.Time { return ch }
	defer func() { tick = time.Tick }()

	buf := &bytes.Buffer{}
	value := 55.55
	cb := func() float64 { return value }
	NewCallbackGauge(buf, "test_statsd_callback_gauge", time.Second, time.Nanosecond, cb)

	ch <- time.Now()  // signal emitter
	runtime.Gosched() // yield to emitter
	ch <- time.Now()  // signal flush
	runtime.Gosched() // yield to flush

	// Travis is annoying
	by(t, time.Second,
		func() bool { return buf.String() != "" },
		func() { runtime.Gosched(); time.Sleep(time.Millisecond) },
		"buffer never got write+flush",
	)

	if want, have := fmt.Sprintf("test_statsd_callback_gauge:%f|g\n", value), buf.String(); want != have {
		t.Errorf("want %q, have %q", want, have)
	}
}

func TestHistogram(t *testing.T) {
	ch := make(chan time.Time)
	tick = func(time.Duration) <-chan time.Time { return ch }
	defer func() { tick = time.Tick }()

	buf := &bytes.Buffer{}
	h := NewHistogram(buf, "test_statsd_histogram", time.Second)

	h.Observe(123)

	runtime.Gosched()
	ch <- time.Now()
	runtime.Gosched()
	if want, have := "test_statsd_histogram:123|ms\n", buf.String(); want != have {
		t.Errorf("want %q, have %q", want, have)
	}
}

func by(t *testing.T, d time.Duration, b func() bool, c func(), msg string) {
	deadline := time.Now().Add(d)
	for !b() {
		if time.Now().After(deadline) {
			t.Fatal(msg)
		}
		c()
	}
}
<commit_msg>Fix flaky Travis test?<commit_after>package statsd

// In package metrics so we can stub tick.

import (
	"bytes"
	"fmt"
	"runtime"
	"testing"
	"time"
)

func TestCounter(t *testing.T) {
	ch := make(chan time.Time)
	tick = func(time.Duration) <-chan time.Time { return ch }
	defer func() { tick = time.Tick }()

	buf := &bytes.Buffer{}
	c := NewCounter(buf, "test_statsd_counter", time.Second)

	c.Add(1)
	c.Add(2)
	ch <- time.Now()

	for i := 0; i < 10 && buf.Len() == 0; i++ {
		time.Sleep(time.Millisecond)
	}

	if want, have := "test_statsd_counter:1|c\ntest_statsd_counter:2|c\n", buf.String(); want != have {
		t.Errorf("want %q, have %q", want, have)
	}
}

func TestGauge(t *testing.T) {
	ch := make(chan time.Time)
	tick = func(time.Duration) <-chan time.Time { return ch }
	defer func() { tick = time.Tick }()

	buf := &bytes.Buffer{}
	g := NewGauge(buf, "test_statsd_gauge", time.Second)

	delta := 1.0
	g.Add(delta)      // send command
	runtime.Gosched() // yield to buffer write
	ch <- time.Now()  // signal flush
	runtime.Gosched() // yield to flush
	if want, have := fmt.Sprintf("test_statsd_gauge:+%f|g\n", delta), buf.String(); want != have {
		t.Errorf("want %q, have %q", want, have)
	}

	buf.Reset()

	delta = -2.0
	g.Add(delta)
	runtime.Gosched()
	ch <- time.Now()
	runtime.Gosched()
	if want, have := fmt.Sprintf("test_statsd_gauge:%f|g\n", delta), buf.String(); want != have {
		t.Errorf("want %q, have %q", want, have)
	}

	buf.Reset()

	value := 3.0
	g.Set(value)
	runtime.Gosched()
	ch <- time.Now()
	runtime.Gosched()
	if want, have := fmt.Sprintf("test_statsd_gauge:%f|g\n", value), buf.String(); want != have {
		t.Errorf("want %q, have %q", want, have)
	}
}

func TestCallbackGauge(t *testing.T) {
	ch := make(chan time.Time)
	tick = func(time.Duration) <-chan time.Time { return ch }
	defer func() { tick = time.Tick }()

	buf := &bytes.Buffer{}
	value := 55.55
	cb := func() float64 { return value }
	NewCallbackGauge(buf, "test_statsd_callback_gauge", time.Second, time.Nanosecond, cb)

	ch <- time.Now()  // signal emitter
	runtime.Gosched() // yield to emitter
	ch <- time.Now()  // signal flush
	runtime.Gosched() // yield to flush

	// Travis is annoying
	by(t, time.Second,
		func() bool { return buf.String() != "" },
		func() { ch <- time.Now(); runtime.Gosched(); time.Sleep(5 * time.Millisecond) },
		"buffer never got write+flush",
	)

	if want, have := fmt.Sprintf("test_statsd_callback_gauge:%f|g\n", value), buf.String(); want != have {
		t.Errorf("want %q, have %q", want, have)
	}
}

func TestHistogram(t *testing.T) {
	ch := make(chan time.Time)
	tick = func(time.Duration) <-chan time.Time { return ch }
	defer func() { tick = time.Tick }()

	buf := &bytes.Buffer{}
	h := NewHistogram(buf, "test_statsd_histogram", time.Second)

	h.Observe(123)

	runtime.Gosched()
	ch <- time.Now()
	runtime.Gosched()
	if want, have := "test_statsd_histogram:123|ms\n", buf.String(); want != have {
		t.Errorf("want %q, have %q", want, have)
	}
}

func by(t *testing.T, d time.Duration, b func() bool, c func(), msg string) {
	deadline := time.Now().Add(d)
	for !b() {
		if time.Now().After(deadline) {
			t.Fatal(msg)
		}
		c()
	}
}
<|endoftext|>


<commit_before>package scheduler

import (
	"fmt"
	"time"

	"github.com/NetSys/di/db"
	"github.com/NetSys/di/join"
	"github.com/NetSys/di/minion/docker"

	log "github.com/Sirupsen/logrus"
)

type scheduler interface {
	list() ([]docker.Container, error)

	boot(toBoot []db.Container)

	terminate(ids []string)
}

func Run(conn db.Conn) {
	var sched scheduler
	for range conn.TriggerTick(30, db.MinionTable, db.EtcdTable, db.ContainerTable).C {
		minions := conn.SelectFromMinion(nil)
		etcdRows := conn.SelectFromEtcd(nil)
		if len(minions) != 1 || len(etcdRows) != 1 || minions[0].Role != db.Master ||
			minions[0].PrivateIP == "" || !etcdRows[0].Leader {
			sched = nil
			continue
		}

		if sched == nil {
			ip := minions[0].PrivateIP
			sched = newSwarm(docker.New(fmt.Sprintf("tcp://%s:2377", ip)))
			time.Sleep(60 * time.Second)
		}

		// Each time we run through this loop, we may boot or terminate
		// containers.  These modification should, in turn, be reflected in the
		// database themselves.  For this reason, we attempt to sync until no
		// database modifications happen (up to an arbitrary limit of three
		// tries).
		for i := 0; i < 3; i++ {
			dkc, err := sched.list()
			if err != nil {
				log.WithError(err).Warning("Failed to get containers.")
				break
			}

			var boot []db.Container
			var term []string
			conn.Transact(func(view db.Database) error {
				term, boot = syncDB(view, dkc)
				return nil
			})

			if len(term) == 0 && len(boot) == 0 {
				break
			}
			sched.terminate(term)
			sched.boot(boot)
		}
	}
}

func syncDB(view db.Database, dkcs_ []docker.Container) ([]string, []db.Container) {
	score := func(left, right interface{}) int {
		dbc := left.(db.Container)
		dkc := right.(docker.Container)

		// Depending on the container, the command in the database could be
		// either The command plus it's arguments, or just it's arguments.  To
		// handle that case, we check both.
		cmd1 := dkc.Args
		cmd2 := append([]string{dkc.Path}, dkc.Args...)

		switch {
		case dkc.Image != dbc.Image:
			return -1
		case !strEq(dbc.Command, cmd1) && !strEq(dbc.Command, cmd2):
			return -1
		case dkc.ID == dbc.SchedID:
			return 0
		default:
			return 1
		}
	}
	pairs, dbcs, dkcs := join.Join(view.SelectFromContainer(nil), dkcs_, score)

	for _, pair := range pairs {
		dbc := pair.L.(db.Container)
		dbc.SchedID = pair.R.(docker.Container).ID
		view.Commit(dbc)
	}

	var term []string
	for _, dkc := range dkcs {
		term = append(term, dkc.(docker.Container).ID)
	}

	var boot []db.Container
	for _, dbc := range dbcs {
		boot = append(boot, dbc.(db.Container))
	}

	return term, boot
}

func strEq(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}

	for i := range a {
		if a[i] != b[i] {
			return false
		}
	}

	return true
}
<commit_msg>scheduler: Match empty command with all containers<commit_after>package scheduler

import (
	"fmt"
	"time"

	"github.com/NetSys/di/db"
	"github.com/NetSys/di/join"
	"github.com/NetSys/di/minion/docker"

	log "github.com/Sirupsen/logrus"
)

type scheduler interface {
	list() ([]docker.Container, error)

	boot(toBoot []db.Container)

	terminate(ids []string)
}

func Run(conn db.Conn) {
	var sched scheduler
	for range conn.TriggerTick(30, db.MinionTable, db.EtcdTable, db.ContainerTable).C {
		minions := conn.SelectFromMinion(nil)
		etcdRows := conn.SelectFromEtcd(nil)
		if len(minions) != 1 || len(etcdRows) != 1 || minions[0].Role != db.Master ||
			minions[0].PrivateIP == "" || !etcdRows[0].Leader {
			sched = nil
			continue
		}

		if sched == nil {
			ip := minions[0].PrivateIP
			sched = newSwarm(docker.New(fmt.Sprintf("tcp://%s:2377", ip)))
			time.Sleep(60 * time.Second)
		}

		// Each time we run through this loop, we may boot or terminate
		// containers.  These modification should, in turn, be reflected in the
		// database themselves.  For this reason, we attempt to sync until no
		// database modifications happen (up to an arbitrary limit of three
		// tries).
		for i := 0; i < 3; i++ {
			dkc, err := sched.list()
			if err != nil {
				log.WithError(err).Warning("Failed to get containers.")
				break
			}

			var boot []db.Container
			var term []string
			conn.Transact(func(view db.Database) error {
				term, boot = syncDB(view, dkc)
				return nil
			})

			if len(term) == 0 && len(boot) == 0 {
				break
			}
			sched.terminate(term)
			sched.boot(boot)
		}
	}
}

func syncDB(view db.Database, dkcs_ []docker.Container) ([]string, []db.Container) {
	score := func(left, right interface{}) int {
		dbc := left.(db.Container)
		dkc := right.(docker.Container)

		// Depending on the container, the command in the database could be
		// either The command plus it's arguments, or just it's arguments.  To
		// handle that case, we check both.
		cmd1 := dkc.Args
		cmd2 := append([]string{dkc.Path}, dkc.Args...)
		dbcCmd := dbc.Command

		switch {
		case dkc.Image != dbc.Image:
			return -1
		case len(dbcCmd) != 0 && !strEq(dbcCmd, cmd1) && !strEq(dbcCmd, cmd2):
			return -1
		case dkc.ID == dbc.SchedID:
			return 0
		default:
			return 1
		}
	}
	pairs, dbcs, dkcs := join.Join(view.SelectFromContainer(nil), dkcs_, score)

	for _, pair := range pairs {
		dbc := pair.L.(db.Container)
		dbc.SchedID = pair.R.(docker.Container).ID
		view.Commit(dbc)
	}

	var term []string
	for _, dkc := range dkcs {
		term = append(term, dkc.(docker.Container).ID)
	}

	var boot []db.Container
	for _, dbc := range dbcs {
		boot = append(boot, dbc.(db.Container))
	}

	return term, boot
}

func strEq(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}

	for i := range a {
		if a[i] != b[i] {
			return false
		}
	}

	return true
}
<|endoftext|>


<commit_before>package version

import (
	"fmt"
	"os"

	"github.com/containerd/containerd/cmd/ctr/commands"
	"github.com/containerd/containerd/version"
	"github.com/urfave/cli"
)

// Command is a cli ommand to output the client and containerd server version
var Command = cli.Command{
	Name:  "version",
	Usage: "print the client and server versions",
	Action: func(context *cli.Context) error {
		fmt.Println("Client:")
		fmt.Printf("  Version: %s\n", version.Version)
		fmt.Printf("  Revision: %s\n", version.Revision)
		fmt.Println("")
		client, ctx, cancel, err := commands.NewClient(context)
		if err != nil {
			return err
		}
		defer cancel()
		v, err := client.Version(ctx)
		if err != nil {
			return err
		}
		fmt.Println("Server:")
		fmt.Printf("  Version: %s\n", v.Version)
		fmt.Printf("  Revision: %s\n", v.Revision)
		if v.Version != version.Version {
			fmt.Fprintf(os.Stderr, "WARNING: version mismatch\n")
		}
		if v.Revision != version.Revision {
			fmt.Fprintf(os.Stderr, "WARNING: revision mismatch\n")
		}
		return nil
	},
}
<commit_msg>Align version output and minor code cleanup<commit_after>package version

import (
	"fmt"
	"os"

	"github.com/containerd/containerd/cmd/ctr/commands"
	"github.com/containerd/containerd/version"
	"github.com/urfave/cli"
)

// Command is a cli command to output the client and containerd server version
var Command = cli.Command{
	Name:  "version",
	Usage: "print the client and server versions",
	Action: func(context *cli.Context) error {
		fmt.Println("Client:")
		fmt.Println("  Version: ", version.Version)
		fmt.Println("  Revision:", version.Revision)
		fmt.Println("")
		client, ctx, cancel, err := commands.NewClient(context)
		if err != nil {
			return err
		}
		defer cancel()
		v, err := client.Version(ctx)
		if err != nil {
			return err
		}
		fmt.Println("Server:")
		fmt.Println("  Version: ", v.Version)
		fmt.Println("  Revision:", v.Revision)
		if v.Version != version.Version {
			fmt.Fprintln(os.Stderr, "WARNING: version mismatch")
		}
		if v.Revision != version.Revision {
			fmt.Fprintln(os.Stderr, "WARNING: revision mismatch")
		}
		return nil
	},
}
<|endoftext|>


<commit_before>// Copyright 2014 Brighcove Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package rq provides a simple queue abstraction that is backed by Redis.
package rq

import "github.com/garyburd/redigo/redis"

type Queue struct {
	pooledConnection *redis.Pool
	key              string
}

// Connect to the Redis server at the specified address and create a queue
// corresponding to the given key
func QueueConnect(pooledConnection *redis.Pool, key string) *Queue {
	return &Queue{pooledConnection: pooledConnection, key: key}
}

// Push will perform a right-push onto a Redis list/queue with the supplied
// key and value.  An error will be returned if the operation failed.
func (queue *Queue) Push(value string) error {
	c := queue.pooledConnection.Get()
	defer c.Close()

	err := c.Send("RPUSH", queue.key, value)
	if err == nil {
		return c.Flush()
	} else {
		return err
	}
}

// Pop will perform a blocking left-pop from a Redis list/queue with the supplied
// key.  An error will be returned if the operation failed.
func (queue *Queue) Pop(timeout int) (string, error) {
	c := queue.pooledConnection.Get()
	defer c.Close()

	rep, err := redis.Strings(c.Do("BLPOP", queue.key, timeout))
	if err == nil {
		return rep[1], nil
	} else {
		return "", err
	}
}

// Length will return the number of items in the specified list/queue
func (queue *Queue) Length() (int, error) {
	c := queue.pooledConnection.Get()
	defer c.Close()

	rep, err := redis.Int(c.Do("LLEN", queue.key))
	if err == nil {
		return rep, nil
	} else {
		return 0, err
	}
}
<commit_msg>Switch to BRPOP for pop, and LPUSH for push<commit_after>// Copyright 2014 Brighcove Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package rq provides a simple queue abstraction that is backed by Redis.
package rq

import "github.com/garyburd/redigo/redis"

type Queue struct {
	pooledConnection *redis.Pool
	key              string
}

// Connect to the Redis server at the specified address and create a queue
// corresponding to the given key
func QueueConnect(pooledConnection *redis.Pool, key string) *Queue {
	return &Queue{pooledConnection: pooledConnection, key: key}
}

// Push will perform a right-push onto a Redis list/queue with the supplied
// key and value.  An error will be returned if the operation failed.
func (queue *Queue) Push(value string) error {
	c := queue.pooledConnection.Get()
	defer c.Close()

	err := c.Send("LPUSH", queue.key, value)
	if err == nil {
		return c.Flush()
	} else {
		return err
	}
}

// Pop will perform a blocking left-pop from a Redis list/queue with the supplied
// key.  An error will be returned if the operation failed.
func (queue *Queue) Pop(timeout int) (string, error) {
	c := queue.pooledConnection.Get()
	defer c.Close()

	rep, err := redis.Strings(c.Do("BRPOP", queue.key, timeout))
	if err == nil {
		return rep[1], nil
	} else {
		return "", err
	}
}

// Length will return the number of items in the specified list/queue
func (queue *Queue) Length() (int, error) {
	c := queue.pooledConnection.Get()
	defer c.Close()

	rep, err := redis.Int(c.Do("LLEN", queue.key))
	if err == nil {
		return rep, nil
	} else {
		return 0, err
	}
}
<|endoftext|>


<commit_before>package dht

import (
	"sync"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	inet "github.com/jbenet/go-ipfs/net"
	peer "github.com/jbenet/go-ipfs/peer"
	"github.com/jbenet/go-ipfs/routing"
	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
	kb "github.com/jbenet/go-ipfs/routing/kbucket"
	u "github.com/jbenet/go-ipfs/util"
)

// This file implements the Routing interface for the IpfsDHT struct.

// Basic Put/Get

// PutValue adds value corresponding to given Key.
// This is the top level "Store" operation of the DHT
func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error {
	log.Debugf("PutValue %s", key)
	err := dht.putLocal(key, value)
	if err != nil {
		return err
	}

	rec, err := dht.makePutRecord(key, value)
	if err != nil {
		log.Error("Creation of record failed!")
		return err
	}

	var peers []peer.Peer
	for _, route := range dht.routingTables {
		npeers := route.NearestPeers(kb.ConvertKey(key), KValue)
		peers = append(peers, npeers...)
	}

	query := newQuery(key, dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {
		log.Debugf("%s PutValue qry part %v", dht.self, p)
		err := dht.putValueToNetwork(ctx, p, string(key), rec)
		if err != nil {
			return nil, err
		}
		return &dhtQueryResult{success: true}, nil
	})

	_, err = query.Run(ctx, peers)
	return err
}

// GetValue searches for the value corresponding to given Key.
// If the search does not succeed, a multiaddr string of a closer peer is
// returned along with util.ErrSearchIncomplete
func (dht *IpfsDHT) GetValue(ctx context.Context, key u.Key) ([]byte, error) {
	log.Debugf("Get Value [%s]", key)

	// If we have it local, dont bother doing an RPC!
	// NOTE: this might not be what we want to do...
	val, err := dht.getLocal(key)
	if err == nil {
		log.Debug("Got value locally!")
		return val, nil
	}

	// get closest peers in the routing tables
	routeLevel := 0
	closest := dht.routingTables[routeLevel].NearestPeers(kb.ConvertKey(key), PoolSize)
	if closest == nil || len(closest) == 0 {
		log.Warning("Got no peers back from routing table!")
		return nil, kb.ErrLookupFailure
	}

	// setup the Query
	query := newQuery(key, dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {

		val, peers, err := dht.getValueOrPeers(ctx, p, key, routeLevel)
		if err != nil {
			return nil, err
		}

		res := &dhtQueryResult{value: val, closerPeers: peers}
		if val != nil {
			res.success = true
		}

		return res, nil
	})

	// run it!
	result, err := query.Run(ctx, closest)
	if err != nil {
		return nil, err
	}

	log.Debugf("GetValue %v %v", key, result.value)
	if result.value == nil {
		return nil, routing.ErrNotFound
	}

	return result.value, nil
}

// Value provider layer of indirection.
// This is what DSHTs (Coral and MainlineDHT) do to store large values in a DHT.

// Provide makes this node announce that it can provide a value for the given key
func (dht *IpfsDHT) Provide(ctx context.Context, key u.Key) error {

	dht.providers.AddProvider(key, dht.self)
	peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), PoolSize)
	if len(peers) == 0 {
		return nil
	}

	//TODO FIX: this doesn't work! it needs to be sent to the actual nearest peers.
	// `peers` are the closest peers we have, not the ones that should get the value.
	for _, p := range peers {
		err := dht.putProvider(ctx, p, string(key))
		if err != nil {
			return err
		}
	}
	return nil
}

// FindProvidersAsync is the same thing as FindProviders, but returns a channel.
// Peers will be returned on the channel as soon as they are found, even before
// the search query completes.
func (dht *IpfsDHT) FindProvidersAsync(ctx context.Context, key u.Key, count int) <-chan peer.Peer {
	log.Event(ctx, "findProviders", &key)
	peerOut := make(chan peer.Peer, count)
	go func() {
		defer close(peerOut)

		ps := newPeerSet()
		// TODO may want to make this function async to hide latency
		provs := dht.providers.GetProviders(ctx, key)
		for _, p := range provs {
			count--
			// NOTE: assuming that this list of peers is unique
			ps.Add(p)
			select {
			case peerOut <- p:
			case <-ctx.Done():
				return
			}
			if count <= 0 {
				return
			}
		}

		var wg sync.WaitGroup
		peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), AlphaValue)
		for _, pp := range peers {
			wg.Add(1)
			go func(p peer.Peer) {
				defer wg.Done()
				pmes, err := dht.findProvidersSingle(ctx, p, key, 0)
				if err != nil {
					log.Error(err)
					return
				}
				dht.addPeerListAsync(ctx, key, pmes.GetProviderPeers(), ps, count, peerOut)
			}(pp)
		}
		wg.Wait()
	}()
	return peerOut
}

func (dht *IpfsDHT) addPeerListAsync(ctx context.Context, k u.Key, peers []*pb.Message_Peer, ps *peerSet, count int, out chan peer.Peer) {
	var wg sync.WaitGroup
	for _, pbp := range peers {
		wg.Add(1)
		go func(mp *pb.Message_Peer) {
			defer wg.Done()
			// construct new peer
			p, err := dht.ensureConnectedToPeer(ctx, mp)
			if err != nil {
				log.Errorf("%s", err)
				return
			}
			if p == nil {
				log.Error("Got nil peer from ensureConnectedToPeer")
				return
			}

			dht.providers.AddProvider(k, p)
			if ps.AddIfSmallerThan(p, count) {
				select {
				case out <- p:
				case <-ctx.Done():
					return
				}
			} else if ps.Size() >= count {
				return
			}
		}(pbp)
	}
	wg.Wait()
}

// FindPeer searches for a peer with given ID.
func (dht *IpfsDHT) FindPeer(ctx context.Context, id peer.ID) (peer.Peer, error) {

	// Check if were already connected to them
	p, _ := dht.FindLocal(id)
	if p != nil {
		return p, nil
	}

	routeLevel := 0
	closest := dht.routingTables[routeLevel].NearestPeers(kb.ConvertPeerID(id), AlphaValue)
	if closest == nil || len(closest) == 0 {
		return nil, kb.ErrLookupFailure
	}

	// Sanity...
	for _, p := range closest {
		if p.ID().Equal(id) {
			log.Error("Found target peer in list of closest peers...")
			return p, nil
		}
	}

	// setup the Query
	query := newQuery(u.Key(id), dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {

		pmes, err := dht.findPeerSingle(ctx, p, id, routeLevel)
		if err != nil {
			return nil, err
		}

		closer := pmes.GetCloserPeers()
		clpeers, errs := pb.PBPeersToPeers(dht.peerstore, closer)
		for _, err := range errs {
			if err != nil {
				log.Warning(err)
			}
		}

		// see it we got the peer here
		for _, np := range clpeers {
			if string(np.ID()) == string(id) {
				return &dhtQueryResult{
					peer:    np,
					success: true,
				}, nil
			}
		}

		return &dhtQueryResult{closerPeers: clpeers}, nil
	})

	// run it!
	result, err := query.Run(ctx, closest)
	if err != nil {
		return nil, err
	}

	log.Debugf("FindPeer %v %v", id, result.success)
	if result.peer == nil {
		return nil, routing.ErrNotFound
	}

	return result.peer, nil
}

// FindPeersConnectedToPeer searches for peers directly connected to a given peer.
func (dht *IpfsDHT) FindPeersConnectedToPeer(ctx context.Context, id peer.ID) (<-chan peer.Peer, error) {

	peerchan := make(chan peer.Peer, 10)
	peersSeen := map[string]peer.Peer{}

	routeLevel := 0
	closest := dht.routingTables[routeLevel].NearestPeers(kb.ConvertPeerID(id), AlphaValue)
	if closest == nil || len(closest) == 0 {
		return nil, kb.ErrLookupFailure
	}

	// setup the Query
	query := newQuery(u.Key(id), dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {

		pmes, err := dht.findPeerSingle(ctx, p, id, routeLevel)
		if err != nil {
			return nil, err
		}

		var clpeers []peer.Peer
		closer := pmes.GetCloserPeers()
		for _, pbp := range closer {
			// skip peers already seen
			if _, found := peersSeen[string(pbp.GetId())]; found {
				continue
			}

			// skip peers that fail to unmarshal
			p, err := pb.PBPeerToPeer(dht.peerstore, pbp)
			if err != nil {
				log.Warning(err)
				continue
			}

			// if peer is connected, send it to our client.
			if pb.Connectedness(*pbp.Connection) == inet.Connected {
				select {
				case <-ctx.Done():
					return nil, ctx.Err()
				case peerchan <- p:
				}
			}

			peersSeen[string(p.ID())] = p

			// if peer is the peer we're looking for, don't bother querying it.
			if pb.Connectedness(*pbp.Connection) != inet.Connected {
				clpeers = append(clpeers, p)
			}
		}

		return &dhtQueryResult{closerPeers: clpeers}, nil
	})

	// run it! run it asynchronously to gen peers as results are found.
	// this does no error checking
	go func() {
		if _, err := query.Run(ctx, closest); err != nil {
			log.Error(err)
		}

		// close the peerchan channel when done.
		close(peerchan)
	}()

	return peerchan, nil
}

// Ping a peer, log the time it took
func (dht *IpfsDHT) Ping(ctx context.Context, p peer.Peer) error {
	// Thoughts: maybe this should accept an ID and do a peer lookup?
	log.Debugf("ping %s start", p)

	pmes := pb.NewMessage(pb.Message_PING, "", 0)
	_, err := dht.sendRequest(ctx, p, pmes)
	log.Debugf("ping %s end (err = %s)", p, err)
	return err
}
<commit_msg>dht: comment for asyncQueryBuffer<commit_after>package dht

import (
	"sync"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	inet "github.com/jbenet/go-ipfs/net"
	peer "github.com/jbenet/go-ipfs/peer"
	"github.com/jbenet/go-ipfs/routing"
	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
	kb "github.com/jbenet/go-ipfs/routing/kbucket"
	u "github.com/jbenet/go-ipfs/util"
)

// asyncQueryBuffer is the size of buffered channels in async queries. This
// buffer allows multiple queries to execute simultaneously, return their
// results and continue querying closer peers. Note that different query
// results will wait for the channel to drain.
var asyncQueryBuffer = 10

// This file implements the Routing interface for the IpfsDHT struct.

// Basic Put/Get

// PutValue adds value corresponding to given Key.
// This is the top level "Store" operation of the DHT
func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error {
	log.Debugf("PutValue %s", key)
	err := dht.putLocal(key, value)
	if err != nil {
		return err
	}

	rec, err := dht.makePutRecord(key, value)
	if err != nil {
		log.Error("Creation of record failed!")
		return err
	}

	var peers []peer.Peer
	for _, route := range dht.routingTables {
		npeers := route.NearestPeers(kb.ConvertKey(key), KValue)
		peers = append(peers, npeers...)
	}

	query := newQuery(key, dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {
		log.Debugf("%s PutValue qry part %v", dht.self, p)
		err := dht.putValueToNetwork(ctx, p, string(key), rec)
		if err != nil {
			return nil, err
		}
		return &dhtQueryResult{success: true}, nil
	})

	_, err = query.Run(ctx, peers)
	return err
}

// GetValue searches for the value corresponding to given Key.
// If the search does not succeed, a multiaddr string of a closer peer is
// returned along with util.ErrSearchIncomplete
func (dht *IpfsDHT) GetValue(ctx context.Context, key u.Key) ([]byte, error) {
	log.Debugf("Get Value [%s]", key)

	// If we have it local, dont bother doing an RPC!
	// NOTE: this might not be what we want to do...
	val, err := dht.getLocal(key)
	if err == nil {
		log.Debug("Got value locally!")
		return val, nil
	}

	// get closest peers in the routing tables
	routeLevel := 0
	closest := dht.routingTables[routeLevel].NearestPeers(kb.ConvertKey(key), PoolSize)
	if closest == nil || len(closest) == 0 {
		log.Warning("Got no peers back from routing table!")
		return nil, kb.ErrLookupFailure
	}

	// setup the Query
	query := newQuery(key, dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {

		val, peers, err := dht.getValueOrPeers(ctx, p, key, routeLevel)
		if err != nil {
			return nil, err
		}

		res := &dhtQueryResult{value: val, closerPeers: peers}
		if val != nil {
			res.success = true
		}

		return res, nil
	})

	// run it!
	result, err := query.Run(ctx, closest)
	if err != nil {
		return nil, err
	}

	log.Debugf("GetValue %v %v", key, result.value)
	if result.value == nil {
		return nil, routing.ErrNotFound
	}

	return result.value, nil
}

// Value provider layer of indirection.
// This is what DSHTs (Coral and MainlineDHT) do to store large values in a DHT.

// Provide makes this node announce that it can provide a value for the given key
func (dht *IpfsDHT) Provide(ctx context.Context, key u.Key) error {

	dht.providers.AddProvider(key, dht.self)
	peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), PoolSize)
	if len(peers) == 0 {
		return nil
	}

	//TODO FIX: this doesn't work! it needs to be sent to the actual nearest peers.
	// `peers` are the closest peers we have, not the ones that should get the value.
	for _, p := range peers {
		err := dht.putProvider(ctx, p, string(key))
		if err != nil {
			return err
		}
	}
	return nil
}

// FindProvidersAsync is the same thing as FindProviders, but returns a channel.
// Peers will be returned on the channel as soon as they are found, even before
// the search query completes.
func (dht *IpfsDHT) FindProvidersAsync(ctx context.Context, key u.Key, count int) <-chan peer.Peer {
	log.Event(ctx, "findProviders", &key)
	peerOut := make(chan peer.Peer, count)
	go func() {
		defer close(peerOut)

		ps := newPeerSet()
		// TODO may want to make this function async to hide latency
		provs := dht.providers.GetProviders(ctx, key)
		for _, p := range provs {
			count--
			// NOTE: assuming that this list of peers is unique
			ps.Add(p)
			select {
			case peerOut <- p:
			case <-ctx.Done():
				return
			}
			if count <= 0 {
				return
			}
		}

		var wg sync.WaitGroup
		peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), AlphaValue)
		for _, pp := range peers {
			wg.Add(1)
			go func(p peer.Peer) {
				defer wg.Done()
				pmes, err := dht.findProvidersSingle(ctx, p, key, 0)
				if err != nil {
					log.Error(err)
					return
				}
				dht.addPeerListAsync(ctx, key, pmes.GetProviderPeers(), ps, count, peerOut)
			}(pp)
		}
		wg.Wait()
	}()
	return peerOut
}

func (dht *IpfsDHT) addPeerListAsync(ctx context.Context, k u.Key, peers []*pb.Message_Peer, ps *peerSet, count int, out chan peer.Peer) {
	var wg sync.WaitGroup
	for _, pbp := range peers {
		wg.Add(1)
		go func(mp *pb.Message_Peer) {
			defer wg.Done()
			// construct new peer
			p, err := dht.ensureConnectedToPeer(ctx, mp)
			if err != nil {
				log.Errorf("%s", err)
				return
			}
			if p == nil {
				log.Error("Got nil peer from ensureConnectedToPeer")
				return
			}

			dht.providers.AddProvider(k, p)
			if ps.AddIfSmallerThan(p, count) {
				select {
				case out <- p:
				case <-ctx.Done():
					return
				}
			} else if ps.Size() >= count {
				return
			}
		}(pbp)
	}
	wg.Wait()
}

// FindPeer searches for a peer with given ID.
func (dht *IpfsDHT) FindPeer(ctx context.Context, id peer.ID) (peer.Peer, error) {

	// Check if were already connected to them
	p, _ := dht.FindLocal(id)
	if p != nil {
		return p, nil
	}

	routeLevel := 0
	closest := dht.routingTables[routeLevel].NearestPeers(kb.ConvertPeerID(id), AlphaValue)
	if closest == nil || len(closest) == 0 {
		return nil, kb.ErrLookupFailure
	}

	// Sanity...
	for _, p := range closest {
		if p.ID().Equal(id) {
			log.Error("Found target peer in list of closest peers...")
			return p, nil
		}
	}

	// setup the Query
	query := newQuery(u.Key(id), dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {

		pmes, err := dht.findPeerSingle(ctx, p, id, routeLevel)
		if err != nil {
			return nil, err
		}

		closer := pmes.GetCloserPeers()
		clpeers, errs := pb.PBPeersToPeers(dht.peerstore, closer)
		for _, err := range errs {
			if err != nil {
				log.Warning(err)
			}
		}

		// see it we got the peer here
		for _, np := range clpeers {
			if string(np.ID()) == string(id) {
				return &dhtQueryResult{
					peer:    np,
					success: true,
				}, nil
			}
		}

		return &dhtQueryResult{closerPeers: clpeers}, nil
	})

	// run it!
	result, err := query.Run(ctx, closest)
	if err != nil {
		return nil, err
	}

	log.Debugf("FindPeer %v %v", id, result.success)
	if result.peer == nil {
		return nil, routing.ErrNotFound
	}

	return result.peer, nil
}

// FindPeersConnectedToPeer searches for peers directly connected to a given peer.
func (dht *IpfsDHT) FindPeersConnectedToPeer(ctx context.Context, id peer.ID) (<-chan peer.Peer, error) {

	peerchan := make(chan peer.Peer, asyncQueryBuffer)
	peersSeen := map[string]peer.Peer{}

	routeLevel := 0
	closest := dht.routingTables[routeLevel].NearestPeers(kb.ConvertPeerID(id), AlphaValue)
	if closest == nil || len(closest) == 0 {
		return nil, kb.ErrLookupFailure
	}

	// setup the Query
	query := newQuery(u.Key(id), dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {

		pmes, err := dht.findPeerSingle(ctx, p, id, routeLevel)
		if err != nil {
			return nil, err
		}

		var clpeers []peer.Peer
		closer := pmes.GetCloserPeers()
		for _, pbp := range closer {
			// skip peers already seen
			if _, found := peersSeen[string(pbp.GetId())]; found {
				continue
			}

			// skip peers that fail to unmarshal
			p, err := pb.PBPeerToPeer(dht.peerstore, pbp)
			if err != nil {
				log.Warning(err)
				continue
			}

			// if peer is connected, send it to our client.
			if pb.Connectedness(*pbp.Connection) == inet.Connected {
				select {
				case <-ctx.Done():
					return nil, ctx.Err()
				case peerchan <- p:
				}
			}

			peersSeen[string(p.ID())] = p

			// if peer is the peer we're looking for, don't bother querying it.
			if pb.Connectedness(*pbp.Connection) != inet.Connected {
				clpeers = append(clpeers, p)
			}
		}

		return &dhtQueryResult{closerPeers: clpeers}, nil
	})

	// run it! run it asynchronously to gen peers as results are found.
	// this does no error checking
	go func() {
		if _, err := query.Run(ctx, closest); err != nil {
			log.Error(err)
		}

		// close the peerchan channel when done.
		close(peerchan)
	}()

	return peerchan, nil
}

// Ping a peer, log the time it took
func (dht *IpfsDHT) Ping(ctx context.Context, p peer.Peer) error {
	// Thoughts: maybe this should accept an ID and do a peer lookup?
	log.Debugf("ping %s start", p)

	pmes := pb.NewMessage(pb.Message_PING, "", 0)
	_, err := dht.sendRequest(ctx, p, pmes)
	log.Debugf("ping %s end (err = %s)", p, err)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"../lib/libocit"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
)

type ServerConfig struct {
	TSurl string
	CPurl string
	Debug bool
}

//public variable
var pub_conf ServerConfig
var pub_casename string
var pub_debug bool
var pub_casedir string

func ts_validation(ts_demo libocit.TestCase) (validate bool, err_string string) {
	if len(ts_demo.Name) > 0 {
	} else {
		err_string = "Cannot find the name"
		return false, err_string
	}

	if len(ts_demo.Requires) > 0 {
	} else {
		err_string = "Cannot find the libocit.Required resource"
		return false, err_string
	}
	return true, "OK"
}

func get_url(req libocit.Require, path string) (apiurl string) {
	var apiuri string
	data := url.Values{}
	if req.Type == "os" {
		apiuri = pub_conf.TSurl
	} else {
		apiuri = pub_conf.CPurl
	}
	if len(req.Distribution) > 1 {
		data.Add("Distribution", req.Distribution)
	}
	data.Add("Version", strconv.Itoa(req.Version))

	u, _ := url.ParseRequestURI(apiuri)
	u.Path = path
	u.RawQuery = data.Encode()
	apiurl = fmt.Sprintf("%v", u)

	return apiurl
}

func apply_os(req libocit.Require) (resource libocit.Resource) {
	var apiurl string

	apiurl = get_url(req, "/os")
	if pub_debug {
		fmt.Println("get url: ", apiurl)
	}
	resp, err := http.Get(apiurl)
	defer resp.Body.Close()
	if err != nil {
		// handle error
		fmt.Println("err in get")
		resource.ID = ""
		resource.Msg = "err in get os"
		resource.Status = false
	} else {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Println("err in read os")
			resource.ID = ""
			resource.Msg = "err in read os"
			resource.Status = false
		} else {
			if pub_debug {
				fmt.Println("Get OS reply ", string(body))
			}
			json.Unmarshal([]byte(body), &resource)
			resource.Req = req
			fmt.Println(resource)
		}
	}

	return resource
}

func apply_container(req libocit.Require) (resource libocit.Resource) {
	tar_url := libocit.TarFilelist(req.Files, pub_casedir, req.Class)
	post_url := pub_conf.CPurl + "/upload"
	libocit.SendFile(post_url, tar_url, tar_url)

	apiurl := pub_conf.CPurl + "/build"
	b, jerr := json.Marshal(req)
	if jerr != nil {
		fmt.Println("Failed to marshal json:", jerr)
		return
	}
	libocit.SendCommand(apiurl, []byte(b))
	return resource
}

func setContainerClass(deploys []libocit.Deploy, req libocit.Require) {
	for index := 0; index < len(deploys); index++ {
		deploy := deploys[index]
		for c_index := 0; c_index < len(deploy.Containers); c_index++ {
			if deploy.Containers[c_index].Class == req.Class {
				deploy.Containers[c_index].Distribution = req.Distribution
				deploy.Containers[c_index].Version = req.Version
			}
		}
	}
}

func apply_resources(ts_demo libocit.TestCase) (resources []libocit.Resource) {
	for index := 0; index < len(ts_demo.Requires); index++ {
		var resource libocit.Resource
		req := ts_demo.Requires[index]
		if req.Type == "os" {
			resource = apply_os(req)
		} else if req.Type == "container" {
			//FIXME: change the democase
			resource = apply_container(req)
			setContainerClass(ts_demo.Deploys, req)
		} else {
			fmt.Println("What is the new type? How can it pass the validation test")
		}
		resource.Used = false
		if len(resource.ID) > 1 {
			resources = append(resources, resource)
		}
	}
	return resources
}

func ar_validation(ar_demo []libocit.Resource) (validate bool, err_string string) {
	return true, "OK"
}

func main() {
	var ts_demo libocit.TestCase
	var validate bool
	var msg string
	var case_file string

	config_content := libocit.ReadFile("./scheduler.conf")
	json.Unmarshal([]byte(config_content), &pub_conf)

	pub_debug = pub_conf.Debug
	arg_num := len(os.Args)
	if arg_num < 2 {
		case_file = "./case01/Network-iperf.json"
	} else {
		case_file = os.Args[1]
	}
	pub_casedir = path.Dir(case_file)
	fmt.Println(case_file)
	test_json_str := libocit.ReadFile(case_file)
	json.Unmarshal([]byte(test_json_str), &ts_demo)
	if pub_debug {
		fmt.Println(ts_demo)
	}
	validate, msg = ts_validation(ts_demo)
	if !validate {
		fmt.Println(msg)
		return
	}
	if pub_debug {
		fmt.Println(ts_demo)
	}

	//libocit.Require Session
	var resources []libocit.Resource
	//TODO: async in the future
	resources = apply_resources(ts_demo)
	validate, msg = ar_validation(resources)
	if !validate {
		fmt.Println(msg)
		return
	}

	//Deploy Session

	// Prepare deploys
	for index := 0; index < len(ts_demo.Deploys); index++ {
		var deploy libocit.Deploy
		deploy = ts_demo.Deploys[index]
		for r_index := 0; r_index < len(resources); r_index++ {
			var resource libocit.Resource
			resource = resources[r_index]
			if resource.Used {
				continue
			}
			if resource.Req.Class == deploy.Class {
				ts_demo.Deploys[index].ResourceID = resource.ID
				resources[r_index].Used = true
				continue
			}
			// TODO should do it after apply resource
			fmt.Println("Cannot get here, failed to get enough resource")
		}
	}
	if pub_debug {
		fmt.Println(ts_demo.Deploys)
	}

	// Send deploys
	for index := 0; index < len(ts_demo.Deploys); index++ {
		if len(ts_demo.Deploys[index].ResourceID) > 0 {
			filelist := GetDeployFiles(ts_demo.Deploys[index])
			//FIXME: change the democase
			tar_url := libocit.TarFilelist(filelist, pub_casedir, ts_demo.Deploys[index].Object)
			post_url := pub_conf.TSurl + "/casefile/" + ts_demo.Deploys[index].ResourceID
			fmt.Println("Send file  -- ", post_url, tar_url)
			libocit.SendFile(post_url, tar_url, tar_url)

			apiurl := pub_conf.TSurl + "/deploy"
			b, jerr := json.Marshal(ts_demo.Deploys[index])
			if jerr != nil {
				fmt.Println("Failed to marshal json:", jerr)
				return
			}
			fmt.Println("Send command ", apiurl)
			libocit.SendCommand(apiurl, []byte(b))
		}
	}

	// Send 'Run' -- do we really need this?

	// Prepare collect
	for index := 0; index < len(ts_demo.Collects); index++ {
		for r_index := 0; r_index < len(ts_demo.Deploys); r_index++ {
			if ts_demo.Collects[index].Object == ts_demo.Deploys[r_index].Object {
				ts_demo.Collects[index].ResourceID = ts_demo.Deploys[r_index].ResourceID
			}
		}
	}

	// Send collects
	for index := 0; index < len(ts_demo.Collects); index++ {
		if len(ts_demo.Collects[index].ResourceID) > 0 {
			collect := ts_demo.Collects[index]
			for f_index := 0; f_index < len(collect.Files); f_index++ {
				file := collect.Files[f_index]
				apiurl := pub_conf.TSurl + "/report/" + ts_demo.Collects[index].ResourceID + "?file=" + file
				fmt.Println("Send collect cmd ", apiurl)
				resp, err := http.Get(apiurl)
				if err != nil {
					fmt.Println("Error ", err)
					continue
				}
				defer resp.Body.Close()
				resp_body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					return
				}
				fmt.Println(resp.Status)
				fmt.Println(string(resp_body))
				cache_dir := "/tmp/test_scheduler_result"
				real_url := libocit.PreparePath(cache_dir, file)
				f, err := os.Create(real_url)
				defer f.Close()
				f.Write(resp_body)
				f.Sync()

			}
		}
	}

}

func GetDeployFiles(deploy libocit.Deploy) (filelist []string) {
	for index := 0; index < len(deploy.Files); index++ {
		filelist = append(filelist, deploy.Files[index])
	}
	for index := 0; index < len(deploy.Containers); index++ {
		container := deploy.Containers[index]
		for c_index := 0; c_index < len(container.Files); c_index++ {
			filelist = append(filelist, container.Files[c_index])
		}
	}
	return filelist
}
<commit_msg>Update scheduler.go<commit_after>package main

import (
	"../lib/libocit"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
)

type ServerConfig struct {
	TSurl string
	CPurl string
	Debug bool
}

//public variable
var pub_conf ServerConfig
var pub_casename string
var pub_debug bool
var pub_casedir string

func ts_validation(ts_demo libocit.TestCase) (validate bool, err_string string) {
	if len(ts_demo.Name) > 0 {
	} else {
		err_string = "Cannot find the name"
		return false, err_string
	}

	if len(ts_demo.Requires) > 0 {
	} else {
		err_string = "Cannot find the libocit.Requires resource"
		return false, err_string
	}
	return true, "OK"
}

func get_url(req libocit.Require, path string) (apiurl string) {
	var apiuri string
	data := url.Values{}
	if req.Type == "os" {
		apiuri = pub_conf.TSurl
	} else {
		apiuri = pub_conf.CPurl
	}
	if len(req.Distribution) > 1 {
		data.Add("Distribution", req.Distribution)
	}
	data.Add("Version", strconv.Itoa(req.Version))

	u, _ := url.ParseRequestURI(apiuri)
	u.Path = path
	u.RawQuery = data.Encode()
	apiurl = fmt.Sprintf("%v", u)

	return apiurl
}

func apply_os(req libocit.Require) (resource libocit.Resource) {
	var apiurl string

	apiurl = get_url(req, "/os")
	if pub_debug {
		fmt.Println("get url: ", apiurl)
	}
	resp, err := http.Get(apiurl)
	defer resp.Body.Close()
	if err != nil {
		// handle error
		fmt.Println("err in get")
		resource.ID = ""
		resource.Msg = "err in get os"
		resource.Status = false
	} else {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Println("err in read os")
			resource.ID = ""
			resource.Msg = "err in read os"
			resource.Status = false
		} else {
			if pub_debug {
				fmt.Println("Get OS reply ", string(body))
			}
			json.Unmarshal([]byte(body), &resource)
			resource.Req = req
			fmt.Println(resource)
		}
	}

	return resource
}

func apply_container(req libocit.Require) (resource libocit.Resource) {
	tar_url := libocit.TarFilelist(req.Files, pub_casedir, req.Class)
	post_url := pub_conf.CPurl + "/upload"
	libocit.SendFile(post_url, tar_url, tar_url)

	apiurl := pub_conf.CPurl + "/build"
	b, jerr := json.Marshal(req)
	if jerr != nil {
		fmt.Println("Failed to marshal json:", jerr)
		return
	}
	libocit.SendCommand(apiurl, []byte(b))
	return resource
}

func setContainerClass(deploys []libocit.Deploy, req libocit.Require) {
	for index := 0; index < len(deploys); index++ {
		deploy := deploys[index]
		for c_index := 0; c_index < len(deploy.Containers); c_index++ {
			if deploy.Containers[c_index].Class == req.Class {
				deploy.Containers[c_index].Distribution = req.Distribution
				deploy.Containers[c_index].Version = req.Version
			}
		}
	}
}

func apply_resources(ts_demo libocit.TestCase) (resources []libocit.Resource) {
	for index := 0; index < len(ts_demo.Requires); index++ {
		var resource libocit.Resource
		req := ts_demo.Requires[index]
		if req.Type == "os" {
			resource = apply_os(req)
		} else if req.Type == "container" {
			//FIXME: change the democase
			resource = apply_container(req)
			setContainerClass(ts_demo.Deploys, req)
		} else {
			fmt.Println("What is the new type? How can it pass the validation test")
		}
		resource.Used = false
		if len(resource.ID) > 1 {
			resources = append(resources, resource)
		}
	}
	return resources
}

func ar_validation(ar_demo []libocit.Resource) (validate bool, err_string string) {
	return true, "OK"
}

func main() {
	var ts_demo libocit.TestCase
	var validate bool
	var msg string
	var case_file string

	config_content := libocit.ReadFile("./scheduler.conf")
	json.Unmarshal([]byte(config_content), &pub_conf)

	pub_debug = pub_conf.Debug
	arg_num := len(os.Args)
	if arg_num < 2 {
		case_file = "./case01/Network-iperf.json"
	} else {
		case_file = os.Args[1]
	}
	pub_casedir = path.Dir(case_file)
	fmt.Println(case_file)
	test_json_str := libocit.ReadFile(case_file)
	json.Unmarshal([]byte(test_json_str), &ts_demo)
	if pub_debug {
		fmt.Println(ts_demo)
	}
	validate, msg = ts_validation(ts_demo)
	if !validate {
		fmt.Println(msg)
		return
	}
	if pub_debug {
		fmt.Println(ts_demo)
	}

	//libocit.Require Session
	var resources []libocit.Resource
	//TODO: async in the future
	resources = apply_resources(ts_demo)
	validate, msg = ar_validation(resources)
	if !validate {
		fmt.Println(msg)
		return
	}

	//Deploy Session

	// Prepare deploys
	for index := 0; index < len(ts_demo.Deploys); index++ {
		var deploy libocit.Deploy
		deploy = ts_demo.Deploys[index]
		for r_index := 0; r_index < len(resources); r_index++ {
			var resource libocit.Resource
			resource = resources[r_index]
			if resource.Used {
				continue
			}
			if resource.Req.Class == deploy.Class {
				ts_demo.Deploys[index].ResourceID = resource.ID
				resources[r_index].Used = true
				continue
			}
			// TODO should do it after apply resource
			fmt.Println("Cannot get here, failed to get enough resource")
		}
	}
	if pub_debug {
		fmt.Println(ts_demo.Deploys)
	}

	// Send deploys
	for index := 0; index < len(ts_demo.Deploys); index++ {
		if len(ts_demo.Deploys[index].ResourceID) > 0 {
			filelist := GetDeployFiles(ts_demo.Deploys[index])
			//FIXME: change the democase
			tar_url := libocit.TarFilelist(filelist, pub_casedir, ts_demo.Deploys[index].Object)
			post_url := pub_conf.TSurl + "/casefile/" + ts_demo.Deploys[index].ResourceID
			fmt.Println("Send file  -- ", post_url, tar_url)
			libocit.SendFile(post_url, tar_url, tar_url)

			apiurl := pub_conf.TSurl + "/deploy"
			b, jerr := json.Marshal(ts_demo.Deploys[index])
			if jerr != nil {
				fmt.Println("Failed to marshal json:", jerr)
				return
			}
			fmt.Println("Send command ", apiurl)
			libocit.SendCommand(apiurl, []byte(b))
		}
	}

	// Send 'Run' -- do we really need this?

	// Prepare collect
	for index := 0; index < len(ts_demo.Collects); index++ {
		for r_index := 0; r_index < len(ts_demo.Deploys); r_index++ {
			if ts_demo.Collects[index].Object == ts_demo.Deploys[r_index].Object {
				ts_demo.Collects[index].ResourceID = ts_demo.Deploys[r_index].ResourceID
			}
		}
	}

	// Send collects
	for index := 0; index < len(ts_demo.Collects); index++ {
		if len(ts_demo.Collects[index].ResourceID) > 0 {
			collect := ts_demo.Collects[index]
			for f_index := 0; f_index < len(collect.Files); f_index++ {
				file := collect.Files[f_index]
				apiurl := pub_conf.TSurl + "/report/" + ts_demo.Collects[index].ResourceID + "?file=" + file
				fmt.Println("Send collect cmd ", apiurl)
				resp, err := http.Get(apiurl)
				if err != nil {
					fmt.Println("Error ", err)
					continue
				}
				defer resp.Body.Close()
				resp_body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					return
				}
				fmt.Println(resp.Status)
				fmt.Println(string(resp_body))
				cache_dir := "/tmp/test_scheduler_result"
				real_url := libocit.PreparePath(cache_dir, file)
				f, err := os.Create(real_url)
				defer f.Close()
				f.Write(resp_body)
				f.Sync()

			}
		}
	}

}

func GetDeployFiles(deploy libocit.Deploy) (filelist []string) {
	for index := 0; index < len(deploy.Files); index++ {
		filelist = append(filelist, deploy.Files[index])
	}
	for index := 0; index < len(deploy.Containers); index++ {
		container := deploy.Containers[index]
		for c_index := 0; c_index < len(container.Files); c_index++ {
			filelist = append(filelist, container.Files[c_index])
		}
	}
	return filelist
}
<|endoftext|>


<commit_before>// Package site provides site annotations.
package site

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"time"

	"github.com/m-lab/go/content"
	"github.com/m-lab/go/flagx"
	"github.com/m-lab/go/memoryless"
	"github.com/m-lab/go/rtx"
	uuid "github.com/m-lab/uuid-annotator/annotator"
)

var (
	// For example of how siteinfo is loaded on production servers, see
	// https://github.com/m-lab/k8s-support/blob/ff5b53faef7828d11d45c2a4f27d53077ddd080c/k8s/daemonsets/templates.jsonnet#L350
	siteinfo        = flagx.URL{}
	siteinfoRetired = flagx.URL{}
	globalAnnotator *annotator
)

func init() {
	flag.Var(&siteinfo, "siteinfo.url", "The URL for the Siteinfo JSON file containing server location and ASN metadata. gs:// and file:// schemes accepted.")
	flag.Var(&siteinfoRetired, "siteinfo.retired-url", "The URL for the Siteinfo retired JSON file. gs:// and file:// schemes accepted.")
	globalAnnotator = nil
}

// Annotate adds site annotation for a site/machine
func Annotate(ip string, server *uuid.ServerAnnotations) {
	if globalAnnotator != nil {
		globalAnnotator.Annotate(ip, server)
	}
}

// LoadFrom loads the site annotation source from the provider.
func LoadFrom(ctx context.Context, js content.Provider, retiredJS content.Provider) error {
	globalAnnotator = &annotator{
		siteinfoSource:        js,
		siteinfoRetiredSource: retiredJS,
		networks:              make(map[string]uuid.ServerAnnotations, 400),
	}
	err := globalAnnotator.load(ctx)
	log.Println(len(globalAnnotator.networks), "sites loaded")
	return err
}

// MustLoad loads the site annotations source and will call log.Fatal if the
// loading fails.
func MustLoad(timeout time.Duration) {
	err := Load(timeout)
	rtx.Must(err, "Could not load annotation db")
}

// MustReload runs a memoryless timer that guarantees reload at least once a
// day. The first load must succeed; subsequent loads may fail without exiting.
func MustReload(ctx context.Context) {
	MustLoad(time.Minute)
	c := memoryless.Config{
		Expected: 12 * time.Hour,
		Min:      time.Hour,
		Max:      24 * time.Hour,
	}
	rtx.Must(memoryless.Run(ctx, func() {
		log.Println(Load(time.Minute))
	}, c), "failed to run site annotation reloader")
}

// Load loads the site annotations source. Will try at least once, retry up to
// timeout and return an error if unsuccessful.
func Load(timeout time.Duration) error {
	js, err := content.FromURL(context.Background(), siteinfo.URL)
	rtx.Must(err, "Invalid server annotations URL", siteinfo.URL.String())

	retiredJS, err := content.FromURL(context.Background(), siteinfoRetired.URL)
	rtx.Must(err, "Invalid retired server annotations URL", siteinfoRetired.URL.String())

	// When annotations are read via HTTP, which is the default, a timeout of
	// 1 minute is used for the GET request.
	// The timeout specified here must be > 1 * time.Minute for the retry loop
	// to make sense.
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	for ; ctx.Err() == nil; time.Sleep(time.Second) {
		err = LoadFrom(context.Background(), js, retiredJS)
		if err == nil {
			break
		}
	}
	return err
}

// annotator stores the annotations, and provides Annotate method.
type annotator struct {
	siteinfoSource        content.Provider
	siteinfoRetiredSource content.Provider
	// Each site has a single ServerAnnotations struct, which
	// is later customized for each machine.
	networks map[string]uuid.ServerAnnotations
}

// missing is used if annotation is requested for a non-existant server.
var missing = uuid.ServerAnnotations{
	Geo: &uuid.Geolocation{
		Missing: true,
	},
	Network: &uuid.Network{
		Missing: true,
	},
}

// Annotate annotates the server with the appropriate annotations.
func (sa *annotator) Annotate(ip string, server *uuid.ServerAnnotations) {
	if server == nil {
		return
	}

	parsedIP := net.ParseIP(ip)
	if parsedIP == nil {
		return
	}

	// Find CIDR corresponding to the provided ip.
	// All of our subnets are /26 if IPv4, /64 if IPv6.
	var cidr string
	if parsedIP.To4() == nil {
		mask := net.CIDRMask(64, 128)
		cidr = fmt.Sprintf("%s/64", parsedIP.Mask(mask))
	} else {
		mask := net.CIDRMask(26, 32)
		cidr = fmt.Sprintf("%s/26", parsedIP.Mask(mask))
	}

	if ann, ok := sa.networks[cidr]; ok {
		ann.Network.CIDR = cidr
		*server = ann
	} else {
		*server = missing
	}
}

// load loads siteinfo dataset and returns them.
func (sa *annotator) load(ctx context.Context) error {
	// siteinfoAnnotation struct is used for parsing the json annotation source.
	type siteinfoAnnotation struct {
		Site    string
		Network struct {
			IPv4 string
			IPv6 string
		}
		Annotation uuid.ServerAnnotations
	}

	js, err := sa.siteinfoSource.Get(ctx)
	if err != nil {
		return err
	}
	var s []siteinfoAnnotation
	err = json.Unmarshal(js, &s)
	if err != nil {
		return err
	}
	// Read the retired sites JSON file, and merge it with the current sites.
	retiredJS, err := sa.siteinfoRetiredSource.Get(ctx)
	if err != nil {
		return err
	}
	var retired []siteinfoAnnotation
	err = json.Unmarshal(retiredJS, &retired)
	if err != nil {
		return err
	}
	s = append(s, retired...)
	for _, ann := range s {
		// Machine should always be empty, filled in later.
		ann.Annotation.Machine = ""

		// Make a map of CIDR -> Annotation.
		// Verify that the CIDRs are valid by trying to parse them.
		// If either the IPv4 or IPv6 CIDRs are wrong, the entry is
		// discarded. The IPv6 CIDR can be empty in some cases.
		if ann.Network.IPv4 == "" {
			continue
		}
		_, _, err := net.ParseCIDR(ann.Network.IPv4)
		if err != nil {
			log.Printf("Found incorrect IPv4 in siteinfo: %s\n",
				ann.Network.IPv4)
			continue
		}

		// Check the IPv6 CIDR only if not empty.
		if ann.Network.IPv6 != "" {
			_, _, err = net.ParseCIDR(ann.Network.IPv6)
			if err != nil {
				log.Printf("Found incorrect IPv6 in siteinfo: %s\n",
					ann.Network.IPv6)
				continue
			}
			sa.networks[ann.Network.IPv6] = ann.Annotation
		}

		sa.networks[ann.Network.IPv4] = ann.Annotation
	}

	return nil
}
<commit_msg>Set default siteinfo URLs and update loaded log message (#288)<commit_after>// Package site provides site annotations.
package site

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"time"

	"github.com/m-lab/go/content"
	"github.com/m-lab/go/flagx"
	"github.com/m-lab/go/memoryless"
	"github.com/m-lab/go/rtx"
	uuid "github.com/m-lab/uuid-annotator/annotator"
)

var (
	// For example of how siteinfo is loaded on production servers, see
	// https://github.com/m-lab/k8s-support/blob/ff5b53faef7828d11d45c2a4f27d53077ddd080c/k8s/daemonsets/templates.jsonnet#L350
	siteinfo        = flagx.MustNewURL("https://siteinfo.mlab-oti.measurementlab.net/v1/sites/annotations.json")
	siteinfoRetired = flagx.MustNewURL("https://siteinfo.mlab-oti.measurementlab.net/v1/retired/annotations.json")
	globalAnnotator *annotator
)

func init() {
	flag.Var(&siteinfo, "siteinfo.url", "The URL for the Siteinfo JSON file containing server location and ASN metadata. gs:// and file:// schemes accepted.")
	flag.Var(&siteinfoRetired, "siteinfo.retired-url", "The URL for the Siteinfo retired JSON file. gs:// and file:// schemes accepted.")
	globalAnnotator = nil
}

// Annotate adds site annotation for a site/machine
func Annotate(ip string, server *uuid.ServerAnnotations) {
	if globalAnnotator != nil {
		globalAnnotator.Annotate(ip, server)
	}
}

// LoadFrom loads the site annotation source from the provider.
func LoadFrom(ctx context.Context, js content.Provider, retiredJS content.Provider) error {
	globalAnnotator = &annotator{
		siteinfoSource:        js,
		siteinfoRetiredSource: retiredJS,
		networks:              make(map[string]uuid.ServerAnnotations, 400),
	}
	err := globalAnnotator.load(ctx)
	log.Println(globalAnnotator.sites, "sites loaded with", len(globalAnnotator.networks), "networks")
	return err
}

// MustLoad loads the site annotations source and will call log.Fatal if the
// loading fails.
func MustLoad(timeout time.Duration) {
	err := Load(timeout)
	rtx.Must(err, "Could not load annotation db")
}

// MustReload runs a memoryless timer that guarantees reload at least once a
// day. The first load must succeed; subsequent loads may fail without exiting.
func MustReload(ctx context.Context) {
	MustLoad(time.Minute)
	c := memoryless.Config{
		Expected: 12 * time.Hour,
		Min:      time.Hour,
		Max:      24 * time.Hour,
	}
	rtx.Must(memoryless.Run(ctx, func() {
		log.Println(Load(time.Minute))
	}, c), "failed to run site annotation reloader")
}

// Load loads the site annotations source. Will try at least once, retry up to
// timeout and return an error if unsuccessful.
func Load(timeout time.Duration) error {
	js, err := content.FromURL(context.Background(), siteinfo.URL)
	rtx.Must(err, "Invalid server annotations URL", siteinfo.URL.String())

	retiredJS, err := content.FromURL(context.Background(), siteinfoRetired.URL)
	rtx.Must(err, "Invalid retired server annotations URL", siteinfoRetired.URL.String())

	// When annotations are read via HTTP, which is the default, a timeout of
	// 1 minute is used for the GET request.
	// The timeout specified here must be > 1 * time.Minute for the retry loop
	// to make sense.
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	for ; ctx.Err() == nil; time.Sleep(time.Second) {
		err = LoadFrom(context.Background(), js, retiredJS)
		if err == nil {
			break
		}
	}
	return err
}

// annotator stores the annotations, and provides Annotate method.
type annotator struct {
	siteinfoSource        content.Provider
	siteinfoRetiredSource content.Provider
	// Each site network (v4 or v6) has a single ServerAnnotations struct,
	// which is later customized for each machine.
	networks map[string]uuid.ServerAnnotations
	sites    int
}

// missing is used if annotation is requested for a non-existant server.
var missing = uuid.ServerAnnotations{
	Geo: &uuid.Geolocation{
		Missing: true,
	},
	Network: &uuid.Network{
		Missing: true,
	},
}

// Annotate annotates the server with the appropriate annotations.
func (sa *annotator) Annotate(ip string, server *uuid.ServerAnnotations) {
	if server == nil {
		return
	}

	parsedIP := net.ParseIP(ip)
	if parsedIP == nil {
		return
	}

	// Find CIDR corresponding to the provided ip.
	// All of our subnets are /26 if IPv4, /64 if IPv6.
	var cidr string
	if parsedIP.To4() == nil {
		mask := net.CIDRMask(64, 128)
		cidr = fmt.Sprintf("%s/64", parsedIP.Mask(mask))
	} else {
		mask := net.CIDRMask(26, 32)
		cidr = fmt.Sprintf("%s/26", parsedIP.Mask(mask))
	}

	if ann, ok := sa.networks[cidr]; ok {
		ann.Network.CIDR = cidr
		*server = ann
	} else {
		*server = missing
	}
}

// load loads siteinfo dataset and returns them.
func (sa *annotator) load(ctx context.Context) error {
	// siteinfoAnnotation struct is used for parsing the json annotation source.
	type siteinfoAnnotation struct {
		Site    string
		Network struct {
			IPv4 string
			IPv6 string
		}
		Annotation uuid.ServerAnnotations
	}

	js, err := sa.siteinfoSource.Get(ctx)
	if err != nil {
		return err
	}
	var s []siteinfoAnnotation
	err = json.Unmarshal(js, &s)
	if err != nil {
		return err
	}
	// Read the retired sites JSON file, and merge it with the current sites.
	retiredJS, err := sa.siteinfoRetiredSource.Get(ctx)
	if err != nil {
		return err
	}
	var retired []siteinfoAnnotation
	err = json.Unmarshal(retiredJS, &retired)
	if err != nil {
		return err
	}
	s = append(s, retired...)
	for _, ann := range s {
		// Machine should always be empty, filled in later.
		ann.Annotation.Machine = ""

		// Make a map of CIDR -> Annotation.
		// Verify that the CIDRs are valid by trying to parse them.
		// If either the IPv4 or IPv6 CIDRs are wrong, the entry is
		// discarded. The IPv6 CIDR can be empty in some cases.
		if ann.Network.IPv4 == "" {
			continue
		}
		_, _, err := net.ParseCIDR(ann.Network.IPv4)
		if err != nil {
			log.Printf("Found incorrect IPv4 in siteinfo: %s\n",
				ann.Network.IPv4)
			continue
		}

		// Check the IPv6 CIDR only if not empty.
		if ann.Network.IPv6 != "" {
			_, _, err = net.ParseCIDR(ann.Network.IPv6)
			if err != nil {
				log.Printf("Found incorrect IPv6 in siteinfo: %s\n",
					ann.Network.IPv6)
				continue
			}
			sa.networks[ann.Network.IPv6] = ann.Annotation
		}

		sa.networks[ann.Network.IPv4] = ann.Annotation
		sa.sites++
	}

	return nil
}
<|endoftext|>


<commit_before>package api

import "fmt"
import "errors"
import "io/ioutil"
import "net/url"
import "net/http"
import "encoding/json"

type Client interface {
	GetAuthToken(string, string) (string, error)
}

type HTTPClient struct {
	BaseURL string
}

type AuthResponse struct {
	Token string
}

func (api HTTPClient) GetAuthToken(username string, password string) (string, error) {
	resp, err := http.PostForm(api.BaseURL+"/signin",
		url.Values{"username": {username}, "password": {password}})

	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	if resp.StatusCode != 200 {
		return "", errors.New(fmt.Sprintf("erroneous API response: %s", body))
	}

	var authResponse AuthResponse
	if err := json.Unmarshal(body, &authResponse); err != nil {
		return "", err
	}

	return authResponse.Token, nil
}
<commit_msg>Extract JSON decoding<commit_after>package api

import "fmt"
import "errors"
import "io/ioutil"
import "net/url"
import "net/http"
import "encoding/json"

type Client interface {
	GetAuthToken(string, string) (string, error)
}

type HTTPClient struct {
	BaseURL string
}

type AuthResponse struct {
	Token string
}

func (api HTTPClient) GetAuthToken(username string, password string) (string, error) {
	resp, err := http.PostForm(api.BaseURL+"/signin",
		url.Values{"username": {username}, "password": {password}})

	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	var authResponse AuthResponse
	if err := DecodeResponse(resp, &authResponse); err != nil {
		return "", err
	}

	return authResponse.Token, nil
}

func DecodeResponse(resp *http.Response, v interface{}) error {
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	if resp.StatusCode != 200 {
		return errors.New(fmt.Sprintf("erroneous API response: %s", body))
	}

	if err := json.Unmarshal(body, &v); err != nil {
		return err
	}

	return nil
}
<|endoftext|>