<commit_before>package main

import (
	"flag"
	"log"
	"net/url"
	"os"
)

import (
	"github.com/qiniu/api/resumable/io"
)

func main() {
	token := flag.String("t", "", "upload token")
	file := flag.String("f", "", "file path")
	key := flag.String("k", "", "file key")
	custom := flag.String("x", "", "custom args foo=1&bar=2")
	flag.Parse()
	if token == nil || file == nil || key == nil {
		log.Fatalln("invalid args")
		return
	}

	f, err := os.Open(*file)
	if err != nil {
		log.Fatalln("file not exist")
		return
	}
	stat, err := f.Stat()
	if err != nil || stat.IsDir() {
		log.Fatalln("invalid file")
		return
	}

	blockNotify := func(blkIdx int, blkSize int, ret *io.BlkputRet) {
		log.Println("size", stat.Size(), "block id", blkIdx, "offset", ret.Offset)
	}

	params := map[string]string{}
	extra := &io.PutExtra{
		ChunkSize: 8192,
		Notify:    blockNotify,
		Params:    params,
	}
	if custom != nil && *custom != "" {
		values, err := url.ParseQuery(*custom)
		if err != nil {
			log.Fatalln(err.Error())
			return
		}
		for k, v := range values {
			params["x:"+k] = v[0]
		}
		extra.Params = params
	}
	var ret io.PutRet
	err = io.PutFile(nil, &ret, *token, *key, *file, extra)
	if err != nil {
		log.Fatalln(err)
	}
}
<commit_msg>args usage<commit_after>package main

import (
	"flag"
	"log"
	"net/url"
	"os"
)

import (
	"github.com/qiniu/api/resumable/io"
)

func main() {
	token := flag.String("t", "", "upload token")
	file := flag.String("f", "", "file path")
	key := flag.String("k", "", "file key")
	custom := flag.String("x", "", "custom args foo=1&bar=2")
	flag.Parse()
	if *token == "" || *file == "" || *key == "" {
		flag.PrintDefaults()
		log.Fatalln("invalid args")
		return
	}

	f, err := os.Open(*file)
	if err != nil {
		log.Fatalln("file not exist")
		return
	}
	stat, err := f.Stat()
	if err != nil || stat.IsDir() {
		log.Fatalln("invalid file")
		return
	}

	blockNotify := func(blkIdx int, blkSize int, ret *io.BlkputRet) {
		log.Println("size", stat.Size(), "block id", blkIdx, "offset", ret.Offset)
	}

	params := map[string]string{}
	extra := &io.PutExtra{
		ChunkSize: 8192,
		Notify:    blockNotify,
		Params:    params,
	}
	if custom != nil && *custom != "" {
		values, err := url.ParseQuery(*custom)
		if err != nil {
			log.Fatalln(err.Error())
			return
		}
		for k, v := range values {
			params["x:"+k] = v[0]
		}
		extra.Params = params
	}
	var ret io.PutRet
	err = io.PutFile(nil, &ret, *token, *key, *file, extra)
	if err != nil {
		log.Fatalln(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate ./COMPILE-PROTOS.sh

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/golang/protobuf/proto"
	"github.com/googleapis/gnostic/OpenAPIv2"
	"github.com/googleapis/gnostic/OpenAPIv3"
	"github.com/googleapis/gnostic/compiler"
	plugins "github.com/googleapis/gnostic/plugins"
)

const ( // OpenAPI Version
	OpenAPIvUnknown = 0
	OpenAPIv2       = 2
	OpenAPIv3       = 3
)

func openapi_version(info interface{}) int {
	m, ok := compiler.UnpackMap(info)
	if !ok {
		return OpenAPIvUnknown
	}
	swagger, ok := compiler.MapValueForKey(m, "swagger").(string)
	if ok && swagger == "2.0" {
		return OpenAPIv2
	}
	openapi, ok := compiler.MapValueForKey(m, "openapi").(string)
	if ok && openapi == "3.0" {
		return OpenAPIv3
	}
	return OpenAPIvUnknown
}

type PluginCall struct {
	Name       string
	Invocation string
}

func (pluginCall *PluginCall) perform(document proto.Message, openapi_version int, sourceName string) error {
	if pluginCall.Name != "" {
		request := &plugins.Request{}

		// Infer the name of the executable by adding the prefix.
		executableName := "gnostic_" + pluginCall.Name

		// validate invocation string with regular expression
		invocation := pluginCall.Invocation

		//
		// Plugin invocations must consist of
		// zero or more comma-separated key=value pairs followed by a path.
		// If pairs are present, a colon separates them from the path.
		// Keys and values must be alphanumeric strings and may contain
		// dashes, underscores, periods, or forward slashes.
		// A path can contain any characters other than the separators ',', ':', and '='.
		//
		invocation_regex := regexp.MustCompile(`^([\w-_\/\.]+=[\w-_\/\.]+(,[\w-_\/\.]+=[\w-_\/\.]+)*:)?[^,:=]+$`)
		if !invocation_regex.Match([]byte(pluginCall.Invocation)) {
			return errors.New(fmt.Sprintf("Invalid invocation of %s: %s", executableName, invocation))
		}

		invocationParts := strings.Split(pluginCall.Invocation, ":")
		var outputLocation string
		switch len(invocationParts) {
		case 1:
			outputLocation = invocationParts[0]
		case 2:
			parameters := strings.Split(invocationParts[0], ",")
			for _, keyvalue := range parameters {
				pair := strings.Split(keyvalue, "=")
				if len(pair) == 2 {
					request.Parameters = append(request.Parameters, &plugins.Parameter{Name: pair[0], Value: pair[1]})
				}
			}
			outputLocation = invocationParts[1]
		default:
			// badly-formed request
			outputLocation = invocationParts[len(invocationParts)-1]
		}

		version := &plugins.Version{}
		version.Major = 0
		version.Minor = 1
		version.Patch = 0
		request.CompilerVersion = version

		request.OutputPath = outputLocation

		wrapper := &plugins.Wrapper{}
		wrapper.Name = sourceName
		switch openapi_version {
		case OpenAPIv2:
			wrapper.Version = "v2"
		case OpenAPIv3:
			wrapper.Version = "v3"
		default:
			wrapper.Version = "unknown"
		}
		protoBytes, _ := proto.Marshal(document)
		wrapper.Value = protoBytes
		request.Wrapper = wrapper
		requestBytes, _ := proto.Marshal(request)

		cmd := exec.Command(executableName)
		cmd.Stdin = bytes.NewReader(requestBytes)
		cmd.Stderr = os.Stderr
		output, err := cmd.Output()
		if err != nil {
			return err
		}
		response := &plugins.Response{}
		err = proto.Unmarshal(output, response)
		if err != nil {
			return err
		}

		// write files to the specified directory
		var writer io.Writer
		if outputLocation == "!" {
			// write nothing
		} else if outputLocation == "-" {
			writer = os.Stdout
			for _, file := range response.Files {
				writer.Write([]byte("\n\n" + file.Name + " -------------------- \n"))
				writer.Write(file.Data)
			}
		} else if isFile(outputLocation) {
			return errors.New(fmt.Sprintf("Error, unable to overwrite %s\n", outputLocation))
		} else {
			if !isDirectory(outputLocation) {
				os.Mkdir(outputLocation, 0755)
			}
			for _, file := range response.Files {
				path := outputLocation + "/" + file.Name
				f, _ := os.Create(path)
				defer f.Close()
				f.Write(file.Data)
			}
		}
	}
	return nil
}

func isFile(path string) bool {
	fileInfo, err := os.Stat(path)
	if err != nil {
		return false
	}
	return !fileInfo.IsDir()
}

func isDirectory(path string) bool {
	fileInfo, err := os.Stat(path)
	if err != nil {
		return false
	}
	return fileInfo.IsDir()
}

func writeFile(name string, bytes []byte, source string, extension string) {
	var writer io.Writer
	if name == "!" {
		return
	} else if name == "-" {
		writer = os.Stdout
	} else if name == "=" {
		writer = os.Stderr
	} else if isDirectory(name) {
		base := filepath.Base(source)
		// remove the original source extension
		base = base[0 : len(base)-len(filepath.Ext(base))]
		// build the path that puts the result in the passed-in directory
		filename := name + "/" + base + "." + extension
		file, _ := os.Create(filename)
		defer file.Close()
		writer = file
	} else {
		file, _ := os.Create(name)
		defer file.Close()
		writer = file
	}
	writer.Write(bytes)
	if name == "-" || name == "=" {
		writer.Write([]byte("\n"))
	}
}

func main() {
	usage := `
Usage: gnostic OPENAPI_SOURCE [OPTIONS]
  OPENAPI_SOURCE is the filename or URL of an OpenAPI description to read.
Options:
  --pb_out=PATH       Write a binary proto to the specified location.
  --json_out=PATH     Write a json proto to the specified location.
  --text_out=PATH     Write a text proto to the specified location.
  --errors_out=PATH   Write compilation errors to the specified location.
  --PLUGIN_out=PATH   Run the plugin named gnostic_PLUGIN and write results to the specified location.
  --resolve_refs      Explicitly resolve $ref references (this could have problems with recursive definitions).
  --extension=NAME    Name of the installed extension handler that will accept vendor extensions values and construct a strongly typed google.protobuf.Any proto.
`
	// default values for all options
	sourceName := ""
	binaryProtoPath := ""
	jsonProtoPath := ""
	textProtoPath := ""
	errorPath := ""
	pluginCalls := make([]*PluginCall, 0)
	resolveReferences := false
	extensionHandlers := make([]compiler.ExtensionHandler, 0)

	// arg processing matches patterns of the form "--PLUGIN_out=PATH"
	plugin_regex := regexp.MustCompile("--(.+)_out=(.+)")

	// arg processing matches patterns of the form "--extension=GENERATOR_NAME"
	extensionHandler_regex, err := regexp.Compile("--extension=(.+)")
	defaultPrefixForExtensions := "openapi_extensions_"
	for i, arg := range os.Args {
		if i == 0 {
			continue // skip the tool name
		}
		var m [][]byte
		if m = plugin_regex.FindSubmatch([]byte(arg)); m != nil {
			pluginName := string(m[1])
			invocation := string(m[2])
			switch pluginName {
			case "pb":
				binaryProtoPath = invocation
			case "json":
				jsonProtoPath = invocation
			case "text":
				textProtoPath = invocation
			case "errors":
				errorPath = invocation
			default:
				pluginCall := &PluginCall{Name: pluginName, Invocation: invocation}
				pluginCalls = append(pluginCalls, pluginCall)
			}
		} else if m = extensionHandler_regex.FindSubmatch([]byte(arg)); m != nil {
			extensionHandlers = append(extensionHandlers, compiler.ExtensionHandler{Name: defaultPrefixForExtensions + string(m[1])})
		} else if arg == "--resolve_refs" {
			resolveReferences = true
		} else if arg[0] == '-' {
			fmt.Fprintf(os.Stderr, "Unknown option: %s.\n%s\n", arg, usage)
			os.Exit(-1)
		} else {
			sourceName = arg
		}
	}

	if binaryProtoPath == "" &&
		jsonProtoPath == "" &&
		textProtoPath == "" &&
		errorPath == "" &&
		len(pluginCalls) == 0 {
		fmt.Fprintf(os.Stderr, "Missing output directives.\n%s\n", usage)
		os.Exit(-1)
	}

	if sourceName == "" {
		fmt.Fprintf(os.Stderr, "No input specified.\n%s\n", usage)
		os.Exit(-1)
	}

	// If we get here and the error output is unspecified, write errors to stderr.
	if errorPath == "" {
		errorPath = "="
	}

	// Read the OpenAPI source.
	info, err := compiler.ReadInfoForFile(sourceName)
	if err != nil {
		writeFile(errorPath, []byte(err.Error()), sourceName, "errors")
		os.Exit(-1)
	}

	// Determine the OpenAPI version.
	openapi_version := openapi_version(info)
	if openapi_version == OpenAPIvUnknown {
		fmt.Fprintf(os.Stderr, "Unknown OpenAPI Version\n")
		os.Exit(-1)
	}

	var message proto.Message
	if openapi_version == OpenAPIv2 {
		document, err := openapi_v2.NewDocument(info, compiler.NewContextWithExtensions("$root", nil, &extensionHandlers))
		if err != nil {
			writeFile(errorPath, []byte(err.Error()), sourceName, "errors")
			os.Exit(-1)
		}
		// optionally resolve internal references
		if resolveReferences {
			_, err = document.ResolveReferences(sourceName)
			if err != nil {
				writeFile(errorPath, []byte(err.Error()), sourceName, "errors")
				os.Exit(-1)
			}
		}
		message = document
	} else if openapi_version == OpenAPIv3 {
		document, err := openapi_v3.NewDocument(info, compiler.NewContextWithExtensions("$root", nil, &extensionHandlers))
		if err != nil {
			writeFile(errorPath, []byte(err.Error()), sourceName, "errors")
			os.Exit(-1)
		}
		// optionally resolve internal references
		if resolveReferences {
			_, err = document.ResolveReferences(sourceName)
			if err != nil {
				writeFile(errorPath, []byte(err.Error()), sourceName, "errors")
				os.Exit(-1)
			}
		}
		message = document
	}

	// perform all specified actions
	if binaryProtoPath != "" {
		// write proto in binary format
		protoBytes, _ := proto.Marshal(message)
		writeFile(binaryProtoPath, protoBytes, sourceName, "pb")
	}
	if jsonProtoPath != "" {
		// write proto in json format
		jsonBytes, _ := json.Marshal(message)
		writeFile(jsonProtoPath, jsonBytes, sourceName, "json")
	}
	if textProtoPath != "" {
		// write proto in text format
		bytes := []byte(proto.MarshalTextString(message))
		writeFile(textProtoPath, bytes, sourceName, "text")
	}
	for _, pluginCall := range pluginCalls {
		err = pluginCall.perform(message, openapi_version, sourceName)
		if err != nil {
			writeFile(errorPath, []byte(err.Error()), sourceName, "errors")
			defer os.Exit(-1)
		}
	}
}
<commit_msg>Basic error reporting from plugins.<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate ./COMPILE-PROTOS.sh

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/golang/protobuf/proto"
	"github.com/googleapis/gnostic/OpenAPIv2"
	"github.com/googleapis/gnostic/OpenAPIv3"
	"github.com/googleapis/gnostic/compiler"
	plugins "github.com/googleapis/gnostic/plugins"
)

const ( // OpenAPI Version
	OpenAPIvUnknown = 0
	OpenAPIv2       = 2
	OpenAPIv3       = 3
)

func openapi_version(info interface{}) int {
	m, ok := compiler.UnpackMap(info)
	if !ok {
		return OpenAPIvUnknown
	}
	swagger, ok := compiler.MapValueForKey(m, "swagger").(string)
	if ok && swagger == "2.0" {
		return OpenAPIv2
	}
	openapi, ok := compiler.MapValueForKey(m, "openapi").(string)
	if ok && openapi == "3.0" {
		return OpenAPIv3
	}
	return OpenAPIvUnknown
}

type PluginCall struct {
	Name       string
	Invocation string
}

func (pluginCall *PluginCall) perform(document proto.Message, openapi_version int, sourceName string) error {
	if pluginCall.Name != "" {
		request := &plugins.Request{}

		// Infer the name of the executable by adding the prefix.
		executableName := "gnostic_" + pluginCall.Name

		// validate invocation string with regular expression
		invocation := pluginCall.Invocation

		//
		// Plugin invocations must consist of
		// zero or more comma-separated key=value pairs followed by a path.
		// If pairs are present, a colon separates them from the path.
		// Keys and values must be alphanumeric strings and may contain
		// dashes, underscores, periods, or forward slashes.
		// A path can contain any characters other than the separators ',', ':', and '='.
		//
		invocation_regex := regexp.MustCompile(`^([\w-_\/\.]+=[\w-_\/\.]+(,[\w-_\/\.]+=[\w-_\/\.]+)*:)?[^,:=]+$`)
		if !invocation_regex.Match([]byte(pluginCall.Invocation)) {
			return errors.New(fmt.Sprintf("Invalid invocation of %s: %s", executableName, invocation))
		}

		invocationParts := strings.Split(pluginCall.Invocation, ":")
		var outputLocation string
		switch len(invocationParts) {
		case 1:
			outputLocation = invocationParts[0]
		case 2:
			parameters := strings.Split(invocationParts[0], ",")
			for _, keyvalue := range parameters {
				pair := strings.Split(keyvalue, "=")
				if len(pair) == 2 {
					request.Parameters = append(request.Parameters, &plugins.Parameter{Name: pair[0], Value: pair[1]})
				}
			}
			outputLocation = invocationParts[1]
		default:
			// badly-formed request
			outputLocation = invocationParts[len(invocationParts)-1]
		}

		version := &plugins.Version{}
		version.Major = 0
		version.Minor = 1
		version.Patch = 0
		request.CompilerVersion = version

		request.OutputPath = outputLocation

		wrapper := &plugins.Wrapper{}
		wrapper.Name = sourceName
		switch openapi_version {
		case OpenAPIv2:
			wrapper.Version = "v2"
		case OpenAPIv3:
			wrapper.Version = "v3"
		default:
			wrapper.Version = "unknown"
		}
		protoBytes, _ := proto.Marshal(document)
		wrapper.Value = protoBytes
		request.Wrapper = wrapper
		requestBytes, _ := proto.Marshal(request)

		cmd := exec.Command(executableName)
		cmd.Stdin = bytes.NewReader(requestBytes)
		cmd.Stderr = os.Stderr
		output, err := cmd.Output()
		if err != nil {
			return err
		}
		response := &plugins.Response{}
		err = proto.Unmarshal(output, response)
		if err != nil {
			return err
		}

		if response.Errors != nil {
			return errors.New(fmt.Sprintf("Plugin error: %+v", response.Errors))
		}

		// write files to the specified directory
		var writer io.Writer
		if outputLocation == "!" {
			// write nothing
		} else if outputLocation == "-" {
			writer = os.Stdout
			for _, file := range response.Files {
				writer.Write([]byte("\n\n" + file.Name + " -------------------- \n"))
				writer.Write(file.Data)
			}
		} else if isFile(outputLocation) {
			return errors.New(fmt.Sprintf("Error, unable to overwrite %s\n", outputLocation))
		} else {
			if !isDirectory(outputLocation) {
				os.Mkdir(outputLocation, 0755)
			}
			for _, file := range response.Files {
				path := outputLocation + "/" + file.Name
				f, _ := os.Create(path)
				defer f.Close()
				f.Write(file.Data)
			}
		}
	}
	return nil
}

func isFile(path string) bool {
	fileInfo, err := os.Stat(path)
	if err != nil {
		return false
	}
	return !fileInfo.IsDir()
}

func isDirectory(path string) bool {
	fileInfo, err := os.Stat(path)
	if err != nil {
		return false
	}
	return fileInfo.IsDir()
}

func writeFile(name string, bytes []byte, source string, extension string) {
	var writer io.Writer
	if name == "!" {
		return
	} else if name == "-" {
		writer = os.Stdout
	} else if name == "=" {
		writer = os.Stderr
	} else if isDirectory(name) {
		base := filepath.Base(source)
		// remove the original source extension
		base = base[0 : len(base)-len(filepath.Ext(base))]
		// build the path that puts the result in the passed-in directory
		filename := name + "/" + base + "." + extension
		file, _ := os.Create(filename)
		defer file.Close()
		writer = file
	} else {
		file, _ := os.Create(name)
		defer file.Close()
		writer = file
	}
	writer.Write(bytes)
	if name == "-" || name == "=" {
		writer.Write([]byte("\n"))
	}
}

func main() {
	usage := `
Usage: gnostic OPENAPI_SOURCE [OPTIONS]
  OPENAPI_SOURCE is the filename or URL of an OpenAPI description to read.
Options:
  --pb_out=PATH       Write a binary proto to the specified location.
  --json_out=PATH     Write a json proto to the specified location.
  --text_out=PATH     Write a text proto to the specified location.
  --errors_out=PATH   Write compilation errors to the specified location.
  --PLUGIN_out=PATH   Run the plugin named gnostic_PLUGIN and write results to the specified location.
  --resolve_refs      Explicitly resolve $ref references (this could have problems with recursive definitions).
  --extension=NAME    Name of the installed extension handler that will accept vendor extensions values and construct a strongly typed google.protobuf.Any proto.
`
	// default values for all options
	sourceName := ""
	binaryProtoPath := ""
	jsonProtoPath := ""
	textProtoPath := ""
	errorPath := ""
	pluginCalls := make([]*PluginCall, 0)
	resolveReferences := false
	extensionHandlers := make([]compiler.ExtensionHandler, 0)

	// arg processing matches patterns of the form "--PLUGIN_out=PATH"
	plugin_regex := regexp.MustCompile("--(.+)_out=(.+)")

	// arg processing matches patterns of the form "--extension=GENERATOR_NAME"
	extensionHandler_regex, err := regexp.Compile("--extension=(.+)")
	defaultPrefixForExtensions := "openapi_extensions_"
	for i, arg := range os.Args {
		if i == 0 {
			continue // skip the tool name
		}
		var m [][]byte
		if m = plugin_regex.FindSubmatch([]byte(arg)); m != nil {
			pluginName := string(m[1])
			invocation := string(m[2])
			switch pluginName {
			case "pb":
				binaryProtoPath = invocation
			case "json":
				jsonProtoPath = invocation
			case "text":
				textProtoPath = invocation
			case "errors":
				errorPath = invocation
			default:
				pluginCall := &PluginCall{Name: pluginName, Invocation: invocation}
				pluginCalls = append(pluginCalls, pluginCall)
			}
		} else if m = extensionHandler_regex.FindSubmatch([]byte(arg)); m != nil {
			extensionHandlers = append(extensionHandlers, compiler.ExtensionHandler{Name: defaultPrefixForExtensions + string(m[1])})
		} else if arg == "--resolve_refs" {
			resolveReferences = true
		} else if arg[0] == '-' {
			fmt.Fprintf(os.Stderr, "Unknown option: %s.\n%s\n", arg, usage)
			os.Exit(-1)
		} else {
			sourceName = arg
		}
	}

	if binaryProtoPath == "" &&
		jsonProtoPath == "" &&
		textProtoPath == "" &&
		errorPath == "" &&
		len(pluginCalls) == 0 {
		fmt.Fprintf(os.Stderr, "Missing output directives.\n%s\n", usage)
		os.Exit(-1)
	}

	if sourceName == "" {
		fmt.Fprintf(os.Stderr, "No input specified.\n%s\n", usage)
		os.Exit(-1)
	}

	// If we get here and the error output is unspecified, write errors to stderr.
	if errorPath == "" {
		errorPath = "="
	}

	// Read the OpenAPI source.
	info, err := compiler.ReadInfoForFile(sourceName)
	if err != nil {
		writeFile(errorPath, []byte(err.Error()), sourceName, "errors")
		os.Exit(-1)
	}

	// Determine the OpenAPI version.
	openapi_version := openapi_version(info)
	if openapi_version == OpenAPIvUnknown {
		fmt.Fprintf(os.Stderr, "Unknown OpenAPI Version\n")
		os.Exit(-1)
	}

	var message proto.Message
	if openapi_version == OpenAPIv2 {
		document, err := openapi_v2.NewDocument(info, compiler.NewContextWithExtensions("$root", nil, &extensionHandlers))
		if err != nil {
			writeFile(errorPath, []byte(err.Error()), sourceName, "errors")
			os.Exit(-1)
		}
		// optionally resolve internal references
		if resolveReferences {
			_, err = document.ResolveReferences(sourceName)
			if err != nil {
				writeFile(errorPath, []byte(err.Error()), sourceName, "errors")
				os.Exit(-1)
			}
		}
		message = document
	} else if openapi_version == OpenAPIv3 {
		document, err := openapi_v3.NewDocument(info, compiler.NewContextWithExtensions("$root", nil, &extensionHandlers))
		if err != nil {
			writeFile(errorPath, []byte(err.Error()), sourceName, "errors")
			os.Exit(-1)
		}
		// optionally resolve internal references
		if resolveReferences {
			_, err = document.ResolveReferences(sourceName)
			if err != nil {
				writeFile(errorPath, []byte(err.Error()), sourceName, "errors")
				os.Exit(-1)
			}
		}
		message = document
	}

	// perform all specified actions
	if binaryProtoPath != "" {
		// write proto in binary format
		protoBytes, _ := proto.Marshal(message)
		writeFile(binaryProtoPath, protoBytes, sourceName, "pb")
	}
	if jsonProtoPath != "" {
		// write proto in json format
		jsonBytes, _ := json.Marshal(message)
		writeFile(jsonProtoPath, jsonBytes, sourceName, "json")
	}
	if textProtoPath != "" {
		// write proto in text format
		bytes := []byte(proto.MarshalTextString(message))
		writeFile(textProtoPath, bytes, sourceName, "text")
	}
	for _, pluginCall := range pluginCalls {
		err = pluginCall.perform(message, openapi_version, sourceName)
		if err != nil {
			writeFile(errorPath, []byte(err.Error()), sourceName, "errors")
			defer os.Exit(-1)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	path "path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/howeyc/fsnotify"
	"github.com/mitchellh/go-ps"
	"github.com/silenceper/log"
)

var (
	cmd          *exec.Cmd
	state        sync.Mutex
	eventTime    = make(map[string]int64)
	scheduleTime time.Time
)

// NewWatcher new watcher
func NewWatcher(paths []string, files []string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Errorf(" Fail to create new Watcher[ %s ]\n", err)
		os.Exit(2)
	}

	go func() {
		for {
			select {
			case e := <-watcher.Event:
				isbuild := true

				// Skip ignored files
				if shouldIgnoreFile(e.Name) {
					continue
				}
				if !checkIfWatchExt(e.Name) {
					continue
				}

				mt := getFileModTime(e.Name)
				if t := eventTime[e.Name]; mt == t {
					// log.Infof("[SKIP] # %s #\n", e.String())
					isbuild = false
				}

				eventTime[e.Name] = mt

				if isbuild {
					go func() {
						// Wait 1s before autobuild util there is no file change.
						scheduleTime = time.Now().Add(1 * time.Second)
						for {
							time.Sleep(time.Until(scheduleTime))
							if time.Now().After(scheduleTime) {
								break
							}
						}

						Autobuild(files)
					}()
				}
			case err := <-watcher.Error:
				log.Errorf("%v", err)
				log.Warnf(" %s\n", err.Error()) // No need to exit here
			}
		}
	}()

	log.Infof("Initializing watcher...\n")
	for _, path := range paths {
		log.Infof("Directory( %s )\n", path)
		err = watcher.Watch(path)
		if err != nil {
			log.Errorf("Fail to watch directory[ %s ]\n", err)
			os.Exit(2)
		}
	}
}

// getFileModTime retuens unix timestamp of `os.File.ModTime` by given path.
func getFileModTime(path string) int64 {
	path = strings.ReplaceAll(path, "\\", "/")
	f, err := os.Open(path)
	if err != nil {
		log.Errorf("Fail to open file[ %s ]\n", err)
		return time.Now().Unix()
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		log.Errorf("Fail to get file information[ %s ]\n", err)
		return time.Now().Unix()
	}

	return fi.ModTime().Unix()
}

// Autobuild auto build
func Autobuild(files []string) {
	state.Lock()
	defer state.Unlock()

	log.Infof("Start building...\n")

	if err := os.Chdir(currpath); err != nil {
		log.Errorf("Chdir Error: %+v\n", err)
		return
	}

	for _, prevCmd := range cfg.PrevBuildCmds {
		log.Infof("Run external cmd '%s'", prevCmd)
		cmdArr := strings.Split(prevCmd, " ")
		prevCmdExec := exec.Command(cmdArr[0])
		prevCmdExec.Env = append(os.Environ(), cfg.Envs...)
		prevCmdExec.Args = cmdArr
		prevCmdExec.Stdout = os.Stdout
		prevCmdExec.Stderr = os.Stderr
		err := prevCmdExec.Run()
		if err != nil {
			panic(err)
		}
	}

	cmdName := "go"

	var err error

	args := []string{"build"}
	args = append(args, "-o", cfg.Output)
	args = append(args, cfg.BuildArgs...)
	if cfg.BuildTags != "" {
		args = append(args, "-tags", cfg.BuildTags)
	}
	args = append(args, files...)

	bcmd := exec.Command(cmdName, args...)
	bcmd.Env = append(os.Environ(), "GOGC=off")
	bcmd.Stdout = os.Stdout
	bcmd.Stderr = os.Stderr
	log.Infof("Build Args: %s %s", cmdName, strings.Join(args, " "))
	err = bcmd.Run()

	if err != nil {
		log.Errorf("============== Build failed ===================\n")
		log.Errorf("%+v\n", err)
		return
	}
	log.Infof("Build was successful\n")
	if !cfg.DisableRun {
		if len(cfg.RunCmd) != 0 {
			Restart(cfg.RunCmd, true)
		} else {
			Restart(cfg.Output, false)
		}
	}
}

// Kill kill main process and all its children
func Kill() {
	defer func() {
		if e := recover(); e != nil {
			fmt.Println("Kill.recover -> ", e)
		}
	}()
	if cmd != nil && cmd.Process != nil {
		// err := cmd.Process.Kill()
		err := killAllProcesses(cmd.Process.Pid)
		if err != nil {
			fmt.Println("Kill -> ", err)
		}
	}
}

// kill main process and all its children
func killAllProcesses(pid int) (err error) {
	hasAllKilled := make(chan bool)
	go func() {
		pids, err := psTree(pid)
		if err != nil {
			log.Fatalf("getting all sub processes error: %v\n", err)
			return
		}
		log.Debugf("main pid: %d", pid)
		log.Debugf("pids: %+v", pids)

		for _, subPid := range pids {
			_ = killProcess(subPid)
		}

		waitForProcess(pid, hasAllKilled)
	}()

	// finally kill the main process
	<-hasAllKilled
	log.Debugf("killing MAIN process pid: %d", pid)
	err = cmd.Process.Kill()
	if err != nil {
		return
	}
	log.Debugf("kill MAIN process succeed")

	return
}

func killProcess(pid int) (err error) {
	log.Debugf("killing process pid: %d", pid)
	ps, err := os.FindProcess(pid)
	if err != nil {
		log.Errorf("find process %d error: %v\n", pid, err)
		return
	}
	err = ps.Kill()
	if err != nil {
		log.Errorf("killing process %d error: %v\n", pid, err)
		// retry
		time.AfterFunc(2*time.Second, func() {
			log.Debugf("retry killing process pid: %d", pid)
			_ = killProcess(pid)
		})
		return
	}
	return
}

// implement pstree based on the cross-platform ps utility in go, go-ps
func psTree(rootPid int) (res []int, err error) {
	pidOfInterest := map[int]struct{}{rootPid: {}}
	pss, err := ps.Processes()
	if err != nil {
		fmt.Println("ERROR: ", err)
		return
	}

	// we must sort the ps by ppid && pid first, otherwise we probably will miss some sub-processes
	// of the root process during for-range searching
	sort.Slice(pss, func(i, j int) bool {
		ppidLess := pss[i].PPid() < pss[j].PPid()
		pidLess := pss[i].PPid() == pss[j].PPid() && pss[i].Pid() < pss[j].Pid()

		return ppidLess || pidLess
	})

	for _, ps := range pss {
		ppid := ps.PPid()
		if _, exists := pidOfInterest[ppid]; exists {
			pidOfInterest[ps.Pid()] = struct{}{}
		}
	}

	for pid := range pidOfInterest {
		if pid != rootPid {
			res = append(res, pid)
		}
	}

	return
}

func waitForProcess(pid int, hasAllKilled chan bool) {
	pids, _ := psTree(pid)
	if len(pids) == 0 {
		hasAllKilled <- true
		return
	}

	log.Infof("still waiting for %d processes %+v to exit", len(pids), pids)
	time.AfterFunc(time.Second, func() {
		waitForProcess(pid, hasAllKilled)
	})
}

// Restart restart app
func Restart(appname string, isCmd bool) {
	// log.Debugf("kill running process")
	Kill()
	go Start(appname, isCmd)
}

// Start start app
func Start(appname string, isCmd bool) {
	log.Infof("Restarting %s ...\n", appname)
	if !strings.HasPrefix(appname, "./") && !isCmd {
		appname = "./" + appname
	}

	cmd = exec.Command(appname)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Args = append([]string{appname}, cfg.CmdArgs...)
	cmd.Env = append(os.Environ(), cfg.Envs...)
	log.Infof("Run %s", strings.Join(cmd.Args, " "))
	go func() {
		_ = cmd.Run()
	}()

	log.Infof("%s is running...\n", appname)
	started <- true
}

// Should ignore filenames generated by
// Emacs, Vim or SublimeText
func shouldIgnoreFile(filename string) bool {
	for _, regex := range ignoredFilesRegExps {
		r, err := regexp.Compile(regex)
		if err != nil {
			panic("Could not compile the regex: " + regex)
		}
		if r.MatchString(filename) {
			return true
		}
		continue
	}
	return false
}

// checkIfWatchExt returns true if the name HasSuffix <watch_ext>.
func checkIfWatchExt(name string) bool {
	for _, s := range cfg.WatchExts {
		if strings.HasSuffix(name, s) {
			return true
		}
	}
	return false
}

func readAppDirectories(directory string, paths *[]string) {
	fileInfos, err := ioutil.ReadDir(directory)
	if err != nil {
		return
	}

	useDirectory := false
	for _, fileInfo := range fileInfos {
		if strings.HasSuffix(fileInfo.Name(), "docs") {
			continue
		}
		if strings.HasSuffix(fileInfo.Name(), "swagger") {
			continue
		}

		if !cfg.VendorWatch && strings.HasSuffix(fileInfo.Name(), "vendor") {
			continue
		}

		if isExcluded(path.Join(directory, fileInfo.Name())) {
			continue
		}

		if fileInfo.IsDir() && fileInfo.Name()[0] != '.' {
			readAppDirectories(directory+"/"+fileInfo.Name(), paths)
			continue
		}
		if useDirectory {
			continue
		}
		*paths = append(*paths, directory)
		useDirectory = true
	}
}

// If a file is excluded
func isExcluded(filePath string) bool {
	for _, p := range cfg.ExcludedPaths {
		absP, err := path.Abs(p)
		if err != nil {
			log.Errorf("err =%v", err)
			log.Errorf("Can not get absolute path of [ %s ]\n", p)
			continue
		}
		absFilePath, err := path.Abs(filePath)
		if err != nil {
			log.Errorf("Can not get absolute path of [ %s ]\n", filePath)
			break
		}
		if strings.HasPrefix(absFilePath, absP) {
			log.Infof("Excluding from watching [ %s ]\n", filePath)
			return true
		}
	}
	return false
}
<commit_msg>fix #26<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	path "path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/howeyc/fsnotify"
	"github.com/mitchellh/go-ps"
	"github.com/silenceper/log"
)

var (
	cmd          *exec.Cmd
	state        sync.Mutex
	eventTime    = make(map[string]int64)
	scheduleTime time.Time
)

// NewWatcher new watcher
func NewWatcher(paths []string, files []string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Errorf(" Fail to create new Watcher[ %s ]\n", err)
		os.Exit(2)
	}

	go func() {
		for {
			select {
			case e := <-watcher.Event:
				isbuild := true

				// Skip ignored files
				if shouldIgnoreFile(e.Name) {
					continue
				}
				if !checkIfWatchExt(e.Name) {
					continue
				}

				mt := getFileModTime(e.Name)
				if t := eventTime[e.Name]; mt == t {
					// log.Infof("[SKIP] # %s #\n", e.String())
					isbuild = false
				}

				eventTime[e.Name] = mt

				if isbuild {
					go func() {
						// Wait 1s before autobuild util there is no file change.
						scheduleTime = time.Now().Add(1 * time.Second)
						for {
							time.Sleep(time.Until(scheduleTime))
							if time.Now().After(scheduleTime) {
								break
							}
						}

						Autobuild(files)
					}()
				}
			case err := <-watcher.Error:
				log.Errorf("%v", err)
				log.Warnf(" %s\n", err.Error()) // No need to exit here
			}
		}
	}()

	log.Infof("Initializing watcher...\n")
	for _, path := range paths {
		log.Infof("Directory( %s )\n", path)
		err = watcher.Watch(path)
		if err != nil {
			log.Errorf("Fail to watch directory[ %s ]\n", err)
			os.Exit(2)
		}
	}
}

// getFileModTime retuens unix timestamp of `os.File.ModTime` by given path.
func getFileModTime(path string) int64 {
	path = strings.ReplaceAll(path, "\\", "/")
	f, err := os.Open(path)
	if err != nil {
		log.Errorf("Fail to open file[ %s ]\n", err)
		return time.Now().Unix()
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		log.Errorf("Fail to get file information[ %s ]\n", err)
		return time.Now().Unix()
	}

	return fi.ModTime().Unix()
}

var building bool

// Autobuild auto build
func Autobuild(files []string) {
	if building {
		log.Infof("still in building...\n")
		return
	}
	building = true
	defer func() {
		building = false
	}()
	state.Lock()
	defer state.Unlock()

	log.Infof("Start building...\n")

	if err := os.Chdir(currpath); err != nil {
		log.Errorf("Chdir Error: %+v\n", err)
		return
	}

	for _, prevCmd := range cfg.PrevBuildCmds {
		log.Infof("Run external cmd '%s'", prevCmd)
		cmdArr := strings.Split(prevCmd, " ")
		prevCmdExec := exec.Command(cmdArr[0])
		prevCmdExec.Env = append(os.Environ(), cfg.Envs...)
		prevCmdExec.Args = cmdArr
		prevCmdExec.Stdout = os.Stdout
		prevCmdExec.Stderr = os.Stderr
		err := prevCmdExec.Run()
		if err != nil {
			panic(err)
		}
	}

	cmdName := "go"

	var err error

	args := []string{"build"}
	args = append(args, "-o", cfg.Output)
	args = append(args, cfg.BuildArgs...)
	if cfg.BuildTags != "" {
		args = append(args, "-tags", cfg.BuildTags)
	}
	args = append(args, files...)

	bcmd := exec.Command(cmdName, args...)
	bcmd.Env = append(os.Environ(), "GOGC=off")
	bcmd.Stdout = os.Stdout
	bcmd.Stderr = os.Stderr
	log.Infof("Build Args: %s %s", cmdName, strings.Join(args, " "))
	err = bcmd.Run()

	if err != nil {
		log.Errorf("============== Build failed ===================\n")
		log.Errorf("%+v\n", err)
		return
	}
	log.Infof("Build was successful\n")
	if !cfg.DisableRun {
		if len(cfg.RunCmd) != 0 {
			Restart(cfg.RunCmd, true)
		} else {
			Restart(cfg.Output, false)
		}
	}
}

// Kill kill main process and all its children
func Kill() {
	defer func() {
		if e := recover(); e != nil {
			fmt.Println("Kill.recover -> ", e)
		}
	}()
	if cmd != nil && cmd.Process != nil {
		// err := cmd.Process.Kill()
		err := killAllProcesses(cmd.Process.Pid)
		if err != nil {
			fmt.Println("Kill -> ", err)
		}
	}
}

// kill main process and all its children
func killAllProcesses(pid int) (err error) {
	hasAllKilled := make(chan bool)
	go func() {
		pids, err := psTree(pid)
		if err != nil {
			log.Fatalf("getting all sub processes error: %v\n", err)
			return
		}
		log.Debugf("main pid: %d", pid)
		log.Debugf("pids: %+v", pids)

		for _, subPid := range pids {
			_ = killProcess(subPid)
		}

		waitForProcess(pid, hasAllKilled)
	}()

	// finally kill the main process
	<-hasAllKilled
	log.Debugf("killing MAIN process pid: %d", pid)
	err = cmd.Process.Kill()
	if err != nil {
		return
	}
	log.Debugf("kill MAIN process succeed")

	return
}

func killProcess(pid int) (err error) {
	log.Debugf("killing process pid: %d", pid)
	ps, err := os.FindProcess(pid)
	if err != nil {
		log.Errorf("find process %d error: %v\n", pid, err)
		return
	}
	err = ps.Kill()
	if err != nil {
		log.Errorf("killing process %d error: %v\n", pid, err)
		// retry
		time.AfterFunc(2*time.Second, func() {
			log.Debugf("retry killing process pid: %d", pid)
			_ = killProcess(pid)
		})
		return
	}
	return
}

// implement pstree based on the cross-platform ps utility in go, go-ps
func psTree(rootPid int) (res []int, err error) {
	pidOfInterest := map[int]struct{}{rootPid: {}}
	pss, err := ps.Processes()
	if err != nil {
		fmt.Println("ERROR: ", err)
		return
	}

	// we must sort the ps by ppid && pid first, otherwise we probably will miss some sub-processes
	// of the root process during for-range searching
	sort.Slice(pss, func(i, j int) bool {
		ppidLess := pss[i].PPid() < pss[j].PPid()
		pidLess := pss[i].PPid() == pss[j].PPid() && pss[i].Pid() < pss[j].Pid()

		return ppidLess || pidLess
	})

	for _, ps := range pss {
		ppid := ps.PPid()
		if _, exists := pidOfInterest[ppid]; exists {
			pidOfInterest[ps.Pid()] = struct{}{}
		}
	}

	for pid := range pidOfInterest {
		if pid != rootPid {
			res = append(res, pid)
		}
	}

	return
}

func waitForProcess(pid int, hasAllKilled chan bool) {
	pids, _ := psTree(pid)
	if len(pids) == 0 {
		hasAllKilled <- true
		return
	}

	log.Infof("still waiting for %d processes %+v to exit", len(pids), pids)
	time.AfterFunc(time.Second, func() {
		waitForProcess(pid, hasAllKilled)
	})
}

// Restart restart app
func Restart(appname string, isCmd bool) {
	// log.Debugf("kill running process")
	Kill()
	go Start(appname, isCmd)
}

// Start start app
func Start(appname string, isCmd bool) {
	log.Infof("Restarting %s ...\n", appname)
	if !strings.HasPrefix(appname, "./") && !isCmd {
		appname = "./" + appname
	}

	cmd = exec.Command(appname)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Args = append([]string{appname}, cfg.CmdArgs...)
	cmd.Env = append(os.Environ(), cfg.Envs...)
	log.Infof("Run %s", strings.Join(cmd.Args, " "))
	go func() {
		_ = cmd.Run()
	}()

	log.Infof("%s is running...\n", appname)
	started <- true
}

// Should ignore filenames generated by
// Emacs, Vim or SublimeText
func shouldIgnoreFile(filename string) bool {
	for _, regex := range ignoredFilesRegExps {
		r, err := regexp.Compile(regex)
		if err != nil {
			panic("Could not compile the regex: " + regex)
		}
		if r.MatchString(filename) {
			return true
		}
		continue
	}
	return false
}

// checkIfWatchExt returns true if the name HasSuffix <watch_ext>.
func checkIfWatchExt(name string) bool {
	for _, s := range cfg.WatchExts {
		if strings.HasSuffix(name, s) {
			return true
		}
	}
	return false
}

func readAppDirectories(directory string, paths *[]string) {
	fileInfos, err := ioutil.ReadDir(directory)
	if err != nil {
		return
	}

	useDirectory := false
	for _, fileInfo := range fileInfos {
		if strings.HasSuffix(fileInfo.Name(), "docs") {
			continue
		}
		if strings.HasSuffix(fileInfo.Name(), "swagger") {
			continue
		}

		if !cfg.VendorWatch && strings.HasSuffix(fileInfo.Name(), "vendor") {
			continue
		}

		if isExcluded(path.Join(directory, fileInfo.Name())) {
			continue
		}

		if fileInfo.IsDir() && fileInfo.Name()[0] != '.' {
			readAppDirectories(directory+"/"+fileInfo.Name(), paths)
			continue
		}
		if useDirectory {
			continue
		}
		*paths = append(*paths, directory)
		useDirectory = true
	}
}

// If a file is excluded
func isExcluded(filePath string) bool {
	for _, p := range cfg.ExcludedPaths {
		absP, err := path.Abs(p)
		if err != nil {
			log.Errorf("err =%v", err)
			log.Errorf("Can not get absolute path of [ %s ]\n", p)
			continue
		}
		absFilePath, err := path.Abs(filePath)
		if err != nil {
			log.Errorf("Can not get absolute path of [ %s ]\n", filePath)
			break
		}
		if strings.HasPrefix(absFilePath, absP) {
			log.Infof("Excluding from watching [ %s ]\n", filePath)
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"compress/gzip"
	"crypto/md5"
	"errors"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	ini "gopkg.in/ini.v1"

	"github.com/asaskevich/govalidator"
	"github.com/goware/cors"
	"github.com/murdinc/terminal"
	"github.com/pressly/chi"
	"github.com/pressly/chi/middleware"
)

// Global
var menu *Menu

type Menu struct {
	URL      string
	Host     string
	Port     string
	Root     string
	Profiles []Profile
}

type Profile struct {
	Name        string   `ini:"-"` // actually the section name
	Source      string   `ini:"source"`
	Destination string   `ini:"destination"`
	Github      string   `ini:"github"`
	Universal   bool     `ini:"universal"`
	Extra       []string `ini:"extra"`
	URL         string   `ini:"-"`
}

func main() {
	terminal.Information("loading config...")
	var err error
	menu, err = loadMenu()
	if err != nil {
		terminal.ErrorLine(err.Error())
		return
	}
	terminal.Delta("config loaded.")

	r := chi.NewRouter()

	cors := cors.New(cors.Options{
		AllowedOrigins:   []string{"*"},
		AllowedMethods:   []string{"GET"},
		AllowCredentials: true,
	})

	r.Use(cors.Handler)
	r.Use(middleware.StripSlashes)
	r.Use(middleware.Recoverer)

	r.Route("/get", func(r chi.Router) {
		r.Route("/:name", func(r chi.Router) {
			r.Get("/", getScript)
		})
	})

	r.Route("/hash", func(r chi.Router) {
		r.Route("/:name", func(r chi.Router) {
			r.Get("/", getHash)
			r.Route("/:os/:arch", func(r chi.Router) {
				r.Get("/", getHash)
			})
		})
	})

	r.Route("/download", func(r chi.Router) {
		r.Route("/:name", func(r chi.Router) {
			r.Get("/", getDownload)
			r.Route("/:os/:arch", func(r chi.Router) {
				r.Get("/", getDownload)
			})
		})
	})

	// Default port
	if menu.Port == "" {
		menu.Port = "2468"
	}

	// Default host
	if menu.Host == "" {
		menu.Host = "localhost"
	}

	// Default URL
	if menu.URL == "" {
		menu.URL = menu.Host + ":" + menu.Port
	}

	terminal.Delta("server started: " + menu.Host + ":" + menu.Port + ", with URL: " + menu.URL)
	http.ListenAndServe(menu.Host+":"+menu.Port, r)

}

func getScript(w http.ResponseWriter, r *http.Request) {
	profileName := chi.URLParam(r, "name")

Loop:
	for _, profile := range menu.Profiles {
		if profile.Name == profileName {
			profile.URL = menu.URL

			_, err := os.Stat(menu.Root + profile.Source)
			if err != nil {
				terminal.ErrorLine(err.Error() + " : " + profile.Source)
				break Loop
			}

			// If universal, pass them the universal script
			if profile.Universal {
				t, _ := template.New("script").Parse(universalScriptTemplate)
				t.Execute(w, profile)
				return
			}

			t, _ := template.New("script").Parse(scriptTemplate)
			t.Execute(w, profile)
			return
		}
	}

	t, _ := template.New("script").Parse(errorScriptTemplate)
	t.Execute(w, profileName)
}

func getDownload(w http.ResponseWriter, r *http.Request) {
	profileName := chi.URLParam(r, "name")
	osName := chi.URLParam(r, "os")
	archName := chi.URLParam(r, "arch")

	switch archName {
	case "x86_64":
		archName = "amd64"
	}

	terminal.Information(fmt.Sprintf("Download request for: %s, OS: %s, Arch: %s\n", profileName, osName, archName))

	for _, profile := range menu.Profiles {
		if profile.Name == profileName {
			source := menu.Root + profile.Source + "/"

			if !profile.Universal {
				if osName != "" && archName != "" {
					source = source + osName + "/" + archName + "/"
				} else {
					terminal.ErrorLine("Invalid request.")
					http.Error(w, "Invalid request.", 500)
					return
				}
			}

			terminal.Information(fmt.Sprintf("Download request file source: %s\n", source))

			err := zipIt(profile.Name, source, w)
			if err != nil {
				terminal.ErrorLine(err.Error())
			}

			return
		}
	}
}

func getHash(w http.ResponseWriter, r *http.Request) {
	profileName := chi.URLParam(r, "name")
	osName := chi.URLParam(r, "os")
	archName := chi.URLParam(r, "arch")

	switch archName {
	case "x86_64":
		archName = "amd64"
	}

	terminal.Information(fmt.Sprintf("Download request for: %s, OS: %s, Arch: %s\n", profileName, osName, archName))

	for _, profile := range menu.Profiles {
		if profile.Name == profileName {

			source := menu.Root + profile.Source + "/"

			if !profile.Universal {
				if osName != "" && archName != "" {
					source = source + osName + "/" + archName + "/"
				} else {
					terminal.ErrorLine("Invalid request.")
					http.Error(w, "Invalid request.", 500)
					return
				}
			}

			terminal.Information(fmt.Sprintf("Download request file source: %s\n", source))

			md5Hash := md5.New()

			err := zipIt(profile.Name, source, md5Hash)
			if err != nil {
				terminal.ErrorLine(err.Error())
				http.Error(w, err.Error(), 500)
				return
			}

			hashString := fmt.Sprintf("%x", md5Hash.Sum(nil))

			io.WriteString(w, hashString)

			terminal.Information("Returned Hash: " + hashString + " for file source: " + source + ".\n")
			return
		}
	}
}

func zipIt(name, source string, w io.Writer) (err error) {
	// gzip writer
	gz := gzip.NewWriter(w)
	defer gz.Close()
	gz.Name = name

	// tarball
	tarball := tar.NewWriter(gz)
	defer tarball.Close()

	info, err := os.Stat(source)
	if err != nil {
		return
	}

	var baseDir string
	if info.IsDir() {
		baseDir = filepath.Base(name)
	}

	return filepath.Walk(source,
		func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			header, err := tar.FileInfoHeader(info, info.Name())
			if err != nil {
				return err
			}

			if baseDir != "" {
				header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
			}

			if err := tarball.WriteHeader(header); err != nil {
				return err
			}

			if info.IsDir() {
				return nil
			}

			file, err := os.Open(path)
			if err != nil {
				return err
			}
			defer file.Close()
			_, err = io.Copy(tarball, file)
			return err
		})
}

func loadMenu() (*Menu, error) {
	menu := new(Menu)

	cfg, err := ini.Load("/etc/drivethru/drivethru.conf")
	if err != nil {
		return menu, err
	}

	apps := cfg.Sections()

	for _, app := range apps {

		if app.Name() == "DEFAULT" {
			menu.URL = app.Key("url").String()
			menu.Host = app.Key("host").String()
			menu.Port = app.Key("port").String()
			menu.Root = app.Key("root").String()
			continue
		}

		profile := new(Profile)

		err := app.MapTo(profile)
		if err != nil {
			return menu, err
		}

		profile.Name = app.Name()
		menu.Profiles = append(menu.Profiles, *profile)

		// Add path separators if needed

		// Source
		if profile.Source[0] != os.PathSeparator {
			profile.Source = fmt.Sprintf("%s%s", string(os.PathSeparator), profile.Source)
		}
		if profile.Source[len(profile.Source)-1] != os.PathSeparator {
			profile.Source = fmt.Sprintf("%s%s", profile.Source, string(os.PathSeparator))
		}

		// Destination
		if profile.Destination[0] != os.PathSeparator {
			profile.Destination = fmt.Sprintf("%s%s", string(os.PathSeparator), profile.Destination)
		}
		if profile.Destination[len(profile.Destination)-1] != os.PathSeparator {
			profile.Destination = fmt.Sprintf("%s%s", profile.Destination, string(os.PathSeparator))
		}

		terminal.Information(fmt.Sprintf("-	found profile named %s.", profile.Name))
	}

	if len(menu.Host) > 0 && !govalidator.IsHost(menu.Host) {
		return menu, errors.New("host specified in conf is invalid: " + menu.Host)
	}

	if len(menu.Port) > 0 && !govalidator.IsPort(menu.Port) {
		return menu, errors.New("port specified in conf is invalid: " + menu.Host)
	}

	// Add path separator if needed
	if menu.Root[len(menu.Root)-1] != os.PathSeparator {
		menu.Root = fmt.Sprintf("%s%s", menu.Root, string(os.PathSeparator))
	}

	terminal.Delta(fmt.Sprintf("loaded %d profiles.", len(menu.Profiles)))

	return menu, err
}

// TEMPLATES
////////////////

var scriptTemplate = `#!/bin/sh

FORMAT="tar.gz"
TEMPFOLDER="/tmp/drivethru-{{ .Name }}-$$"
TARBALL="$TEMPFOLDER/tar/{{ .Name }}.$FORMAT"
OS=$(uname)
ARCH=$(uname -m)
URL="http://{{ .URL }}/download/{{ .Name }}/$OS/$ARCH/"
DEST={{ .Destination }}

sudo mkdir -p /tmp/$$/ && sudo chmod 777 /tmp/$$/

sudo mkdir -p "$TEMPFOLDER/tar"
sudo mkdir -p "$TEMPFOLDER/expanded"
sudo chmod -R 777 "$TEMPFOLDER"

echo "Downloading $URL"

curl -o $TARBALL -L -f $URL
if [ $? -eq 0 ]
then
    echo "\nCopying {{ .Name }} into $DEST\n"
    sudo mkdir -p $DEST/
    tar -xzf $TARBALL -C $TEMPFOLDER/expanded && sudo cp -av $TEMPFOLDER/expanded/{{ .Name }}/* $DEST/ && rm -rf {{ .Name }}
    if [ $? -eq 0 ]
    then
        sudo rm -rf "$TEMPFOLDER"
        echo "\n{{ .Name }} has been installed into $DEST\n"
        {{ if .Extra }}{{ $url := .URL }}
        {{ range .Extra }}curl -s http://{{ $url }}/get/{{ . }} | sh
        {{end}}{{ else }}echo "Done!"{{end}}
        exit 0
    fi
else
    echo "Failed to install {{ .Name }}.\nPlease try downloading from {{ .Github }} instead."
    sudo rm -rf "$TEMPFOLDER"
fi

exit 1
`

var universalScriptTemplate = `#!/bin/sh

FORMAT="tar.gz"
TEMPFOLDER="/tmp/drivethru-{{ .Name }}-$$"
TARBALL="$TEMPFOLDER/tar/{{ .Name }}.$FORMAT"
URL="http://{{ .URL }}/download/{{ .Name }}/"
DEST="{{ .Destination }}"

sudo mkdir -p "$TEMPFOLDER/tar"
sudo mkdir -p "$TEMPFOLDER/expanded"
sudo chmod -R 777 "$TEMPFOLDER"

echo "Downloading $URL"

curl -o $TARBALL -L -f $URL
if [ $? -eq 0 ]
then
    echo "\nCopying {{ .Name }} into $DEST\n"
    sudo mkdir -p $DEST/
    tar -xzf $TARBALL -C $TEMPFOLDER/expanded && sudo cp -av $TEMPFOLDER/expanded/{{ .Name }}/* $DEST/ && rm -rf {{ .Name }}
    if [ $? -eq 0 ]
    then
        sudo rm -rf "$TEMPFOLDER"
        echo "\n{{ .Name }} has been installed into $DEST\n"
        {{ if .Extra }}{{range .Extra}}http://{{ .URL }}/get/{{ . }}/ | sh
        {{end}}{{end}}
        echo "Done!"
        exit 0
    fi
else
    echo "Failed to install {{ .Name }}.\nPlease try downloading from {{ .Github }} instead."
    sudo rm -rf "$TEMPFOLDER"
fi

exit 1
`

var errorScriptTemplate = `#!/bin/sh

echo "There was an error building your script for the download of {{ . }}, please contact the developer."
exit 1
`
<commit_msg>update deps<commit_after>package main

import (
	"archive/tar"
	"compress/gzip"
	"crypto/md5"
	"errors"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	ini "gopkg.in/ini.v1"

	"github.com/asaskevich/govalidator"
	"github.com/go-chi/chi"
	"github.com/go-chi/chi/middleware"
	"github.com/goware/cors"
	"github.com/murdinc/terminal"
)

// Global
var menu *Menu

type Menu struct {
	URL      string
	Host     string
	Port     string
	Root     string
	Profiles []Profile
}

type Profile struct {
	Name        string   `ini:"-"` // actually the section name
	Source      string   `ini:"source"`
	Destination string   `ini:"destination"`
	Github      string   `ini:"github"`
	Universal   bool     `ini:"universal"`
	Extra       []string `ini:"extra"`
	URL         string   `ini:"-"`
}

func main() {
	terminal.Information("loading config...")
	var err error
	menu, err = loadMenu()
	if err != nil {
		terminal.ErrorLine(err.Error())
		return
	}
	terminal.Delta("config loaded.")

	r := chi.NewRouter()

	cors := cors.New(cors.Options{
		AllowedOrigins:   []string{"*"},
		AllowedMethods:   []string{"GET"},
		AllowCredentials: true,
	})

	r.Use(cors.Handler)
	r.Use(middleware.StripSlashes)
	r.Use(middleware.Recoverer)

	r.Route("/get", func(r chi.Router) {
		r.Route("/:name", func(r chi.Router) {
			r.Get("/", getScript)
		})
	})

	r.Route("/hash", func(r chi.Router) {
		r.Route("/:name", func(r chi.Router) {
			r.Get("/", getHash)
			r.Route("/:os/:arch", func(r chi.Router) {
				r.Get("/", getHash)
			})
		})
	})

	r.Route("/download", func(r chi.Router) {
		r.Route("/:name", func(r chi.Router) {
			r.Get("/", getDownload)
			r.Route("/:os/:arch", func(r chi.Router) {
				r.Get("/", getDownload)
			})
		})
	})

	// Default port
	if menu.Port == "" {
		menu.Port = "2468"
	}

	// Default host
	if menu.Host == "" {
		menu.Host = "localhost"
	}

	// Default URL
	if menu.URL == "" {
		menu.URL = menu.Host + ":" + menu.Port
	}

	terminal.Delta("server started: " + menu.Host + ":" + menu.Port + ", with URL: " + menu.URL)
	http.ListenAndServe(menu.Host+":"+menu.Port, r)

}

func getScript(w http.ResponseWriter, r *http.Request) {
	profileName := chi.URLParam(r, "name")

Loop:
	for _, profile := range menu.Profiles {
		if profile.Name == profileName {
			profile.URL = menu.URL

			_, err := os.Stat(menu.Root + profile.Source)
			if err != nil {
				terminal.ErrorLine(err.Error() + " : " + profile.Source)
				break Loop
			}

			// If universal, pass them the universal script
			if profile.Universal {
				t, _ := template.New("script").Parse(universalScriptTemplate)
				t.Execute(w, profile)
				return
			}

			t, _ := template.New("script").Parse(scriptTemplate)
			t.Execute(w, profile)
			return
		}
	}

	t, _ := template.New("script").Parse(errorScriptTemplate)
	t.Execute(w, profileName)
}

func getDownload(w http.ResponseWriter, r *http.Request) {
	profileName := chi.URLParam(r, "name")
	osName := chi.URLParam(r, "os")
	archName := chi.URLParam(r, "arch")

	switch archName {
	case "x86_64":
		archName = "amd64"
	}

	terminal.Information(fmt.Sprintf("Download request for: %s, OS: %s, Arch: %s\n", profileName, osName, archName))

	for _, profile := range menu.Profiles {
		if profile.Name == profileName {
			source := menu.Root + profile.Source + "/"

			if !profile.Universal {
				if osName != "" && archName != "" {
					source = source + osName + "/" + archName + "/"
				} else {
					terminal.ErrorLine("Invalid request.")
					http.Error(w, "Invalid request.", 500)
					return
				}
			}

			terminal.Information(fmt.Sprintf("Download request file source: %s\n", source))

			err := zipIt(profile.Name, source, w)
			if err != nil {
				terminal.ErrorLine(err.Error())
			}

			return
		}
	}
}

func getHash(w http.ResponseWriter, r *http.Request) {
	profileName := chi.URLParam(r, "name")
	osName := chi.URLParam(r, "os")
	archName := chi.URLParam(r, "arch")

	switch archName {
	case "x86_64":
		archName = "amd64"
	}

	terminal.Information(fmt.Sprintf("Download request for: %s, OS: %s, Arch: %s\n", profileName, osName, archName))

	for _, profile := range menu.Profiles {
		if profile.Name == profileName {

			source := menu.Root + profile.Source + "/"

			if !profile.Universal {
				if osName != "" && archName != "" {
					source = source + osName + "/" + archName + "/"
				} else {
					terminal.ErrorLine("Invalid request.")
					http.Error(w, "Invalid request.", 500)
					return
				}
			}

			terminal.Information(fmt.Sprintf("Download request file source: %s\n", source))

			md5Hash := md5.New()

			err := zipIt(profile.Name, source, md5Hash)
			if err != nil {
				terminal.ErrorLine(err.Error())
				http.Error(w, err.Error(), 500)
				return
			}

			hashString := fmt.Sprintf("%x", md5Hash.Sum(nil))

			io.WriteString(w, hashString)

			terminal.Information("Returned Hash: " + hashString + " for file source: " + source + ".\n")
			return
		}
	}
}

func zipIt(name, source string, w io.Writer) (err error) {
	// gzip writer
	gz := gzip.NewWriter(w)
	defer gz.Close()
	gz.Name = name

	// tarball
	tarball := tar.NewWriter(gz)
	defer tarball.Close()

	info, err := os.Stat(source)
	if err != nil {
		return
	}

	var baseDir string
	if info.IsDir() {
		baseDir = filepath.Base(name)
	}

	return filepath.Walk(source,
		func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			header, err := tar.FileInfoHeader(info, info.Name())
			if err != nil {
				return err
			}

			if baseDir != "" {
				header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
			}

			if err := tarball.WriteHeader(header); err != nil {
				return err
			}

			if info.IsDir() {
				return nil
			}

			file, err := os.Open(path)
			if err != nil {
				return err
			}
			defer file.Close()
			_, err = io.Copy(tarball, file)
			return err
		})
}

func loadMenu() (*Menu, error) {
	menu := new(Menu)

	cfg, err := ini.Load("/etc/drivethru/drivethru.conf")
	if err != nil {
		return menu, err
	}

	apps := cfg.Sections()

	for _, app := range apps {

		if app.Name() == "DEFAULT" {
			menu.URL = app.Key("url").String()
			menu.Host = app.Key("host").String()
			menu.Port = app.Key("port").String()
			menu.Root = app.Key("root").String()
			continue
		}

		profile := new(Profile)

		err := app.MapTo(profile)
		if err != nil {
			return menu, err
		}

		profile.Name = app.Name()
		menu.Profiles = append(menu.Profiles, *profile)

		// Add path separators if needed

		// Source
		if profile.Source[0] != os.PathSeparator {
			profile.Source = fmt.Sprintf("%s%s", string(os.PathSeparator), profile.Source)
		}
		if profile.Source[len(profile.Source)-1] != os.PathSeparator {
			profile.Source = fmt.Sprintf("%s%s", profile.Source, string(os.PathSeparator))
		}

		// Destination
		if profile.Destination[0] != os.PathSeparator {
			profile.Destination = fmt.Sprintf("%s%s", string(os.PathSeparator), profile.Destination)
		}
		if profile.Destination[len(profile.Destination)-1] != os.PathSeparator {
			profile.Destination = fmt.Sprintf("%s%s", profile.Destination, string(os.PathSeparator))
		}

		terminal.Information(fmt.Sprintf("-	found profile named %s.", profile.Name))
	}

	if len(menu.Host) > 0 && !govalidator.IsHost(menu.Host) {
		return menu, errors.New("host specified in conf is invalid: " + menu.Host)
	}

	if len(menu.Port) > 0 && !govalidator.IsPort(menu.Port) {
		return menu, errors.New("port specified in conf is invalid: " + menu.Host)
	}

	// Add path separator if needed
	if menu.Root[len(menu.Root)-1] != os.PathSeparator {
		menu.Root = fmt.Sprintf("%s%s", menu.Root, string(os.PathSeparator))
	}

	terminal.Delta(fmt.Sprintf("loaded %d profiles.", len(menu.Profiles)))

	return menu, err
}

// TEMPLATES
////////////////

var scriptTemplate = `#!/bin/sh

FORMAT="tar.gz"
TEMPFOLDER="/tmp/drivethru-{{ .Name }}-$$"
TARBALL="$TEMPFOLDER/tar/{{ .Name }}.$FORMAT"
OS=$(uname)
ARCH=$(uname -m)
URL="http://{{ .URL }}/download/{{ .Name }}/$OS/$ARCH/"
DEST={{ .Destination }}

sudo mkdir -p /tmp/$$/ && sudo chmod 777 /tmp/$$/

sudo mkdir -p "$TEMPFOLDER/tar"
sudo mkdir -p "$TEMPFOLDER/expanded"
sudo chmod -R 777 "$TEMPFOLDER"

echo "Downloading $URL"

curl -o $TARBALL -L -f $URL
if [ $? -eq 0 ]
then
    echo "\nCopying {{ .Name }} into $DEST\n"
    sudo mkdir -p $DEST/
    tar -xzf $TARBALL -C $TEMPFOLDER/expanded && sudo cp -av $TEMPFOLDER/expanded/{{ .Name }}/* $DEST/ && rm -rf {{ .Name }}
    if [ $? -eq 0 ]
    then
        sudo rm -rf "$TEMPFOLDER"
        echo "\n{{ .Name }} has been installed into $DEST\n"
        {{ if .Extra }}{{ $url := .URL }}
        {{ range .Extra }}curl -s http://{{ $url }}/get/{{ . }} | sh
        {{end}}{{ else }}echo "Done!"{{end}}
        exit 0
    fi
else
    echo "Failed to install {{ .Name }}.\nPlease try downloading from {{ .Github }} instead."
    sudo rm -rf "$TEMPFOLDER"
fi

exit 1
`

var universalScriptTemplate = `#!/bin/sh

FORMAT="tar.gz"
TEMPFOLDER="/tmp/drivethru-{{ .Name }}-$$"
TARBALL="$TEMPFOLDER/tar/{{ .Name }}.$FORMAT"
URL="http://{{ .URL }}/download/{{ .Name }}/"
DEST="{{ .Destination }}"

sudo mkdir -p "$TEMPFOLDER/tar"
sudo mkdir -p "$TEMPFOLDER/expanded"
sudo chmod -R 777 "$TEMPFOLDER"

echo "Downloading $URL"

curl -o $TARBALL -L -f $URL
if [ $? -eq 0 ]
then
    echo "\nCopying {{ .Name }} into $DEST\n"
    sudo mkdir -p $DEST/
    tar -xzf $TARBALL -C $TEMPFOLDER/expanded && sudo cp -av $TEMPFOLDER/expanded/{{ .Name }}/* $DEST/ && rm -rf {{ .Name }}
    if [ $? -eq 0 ]
    then
        sudo rm -rf "$TEMPFOLDER"
        echo "\n{{ .Name }} has been installed into $DEST\n"
        {{ if .Extra }}{{range .Extra}}http://{{ .URL }}/get/{{ . }}/ | sh
        {{end}}{{end}}
        echo "Done!"
        exit 0
    fi
else
    echo "Failed to install {{ .Name }}.\nPlease try downloading from {{ .Github }} instead."
    sudo rm -rf "$TEMPFOLDER"
fi

exit 1
`

var errorScriptTemplate = `#!/bin/sh

echo "There was an error building your script for the download of {{ . }}, please contact the developer."
exit 1
`
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd openbsd

package syscall_test

import (
	"syscall"
	"testing"
)

const MNT_WAIT = 1

func TestGetfsstat(t *testing.T) {
	n, err := syscall.Getfsstat(nil, MNT_WAIT)
	if err != nil {
		t.Fatal(err)
	}

	data := make([]syscall.Statfs_t, n)
	n, err = syscall.Getfsstat(data, MNT_WAIT)
	if err != nil {
		t.Fatal(err)
	}

	empty := syscall.Statfs_t{}
	for _, stat := range data {
		if stat == empty {
			t.Fatal("an empty Statfs_t struct was returned")
		}
	}
}
<commit_msg>[release-branch.go1.7] syscall: use MNT_NOWAIT in TestGetfsstat<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd openbsd

package syscall_test

import (
	"os/exec"
	"syscall"
	"testing"
)

const MNT_WAIT = 1
const MNT_NOWAIT = 2

func TestGetfsstat(t *testing.T) {
	const flags = MNT_NOWAIT // see Issue 16937
	n, err := syscall.Getfsstat(nil, flags)
	t.Logf("Getfsstat(nil, %d) = (%v, %v)", flags, n, err)
	if err != nil {
		t.Fatal(err)
	}

	data := make([]syscall.Statfs_t, n)
	n2, err := syscall.Getfsstat(data, flags)
	t.Logf("Getfsstat([]syscall.Statfs_t, %d) = (%v, %v)", flags, n2, err)
	if err != nil {
		t.Fatal(err)
	}
	if n != n2 {
		t.Errorf("Getfsstat(nil) = %d, but subsequent Getfsstat(slice) = %d", n, n2)
	}
	for i, stat := range data {
		if stat == (syscall.Statfs_t{}) {
			t.Errorf("index %v is an empty Statfs_t struct", i)
		}
	}
	if t.Failed() {
		for i, stat := range data[:n2] {
			t.Logf("data[%v] = %+v", i, stat)
		}
		mount, err := exec.Command("mount").CombinedOutput()
		if err != nil {
			t.Logf("mount: %v\n%s", err, mount)
		} else {
			t.Logf("mount: %s", mount)
		}
	}
}
<|endoftext|>


<commit_before>package templates

var DefaultTemplate *Template = &Template{
	AWSTemplateFormatVersion: "2010-09-09",
	Description:              "BOSH 101 Classroom CloudFormation Template",
	Parameters: map[string]Parameter{
		"InstanceType": Parameter{
			Description:           "EC2 Instance Type for Classroom VMs",
			Type:                  "String",
			Default:               "m3.medium",
			ConstraintDescription: "must be a valid EC2 instance type.",
		},

		"AMI": Parameter{
			Description: "AMI to boot",
			Type:        "String",
			ConstraintDescription: "should be an AMI from the latest cloudfoundry/bosh-lite Vagrant box",
		},

		"KeyName": Parameter{
			Description: "The EC2 Key Pair to allow SSH access to the instances",
			Type:        "AWS::EC2::KeyPair::KeyName",
			ConstraintDescription: "must be the name of an existing EC2 KeyPair",
		},

		"InstanceCount": Parameter{
			Description: "Number of EC2 instances to boot for the classroom",
			Type:        "Number",
			Default:     "1",
		},

		"SSHLocation": Parameter{
			Description:           "The IP address range that can be used to SSH to the EC2 instances",
			Type:                  "String",
			Default:               "0.0.0.0/0",
			ConstraintDescription: "must be a valid IP CIDR range of the form x.x.x.x/x.",
		},
	},
	Resources: map[string]Resource{
		"AutoScalingGroup": Resource{
			Type: "AWS::AutoScaling::AutoScalingGroup",
			Properties: map[string]interface{}{
				"AvailabilityZones":       Fn("GetAZs", ""),
				"LaunchConfigurationName": Ref{"LaunchConfig"},
				"MinSize":                 Ref{"InstanceCount"},
				"MaxSize":                 Ref{"InstanceCount"},
			},
		},

		"LaunchConfig": Resource{
			Type: "AWS::AutoScaling::LaunchConfiguration",
			Properties: map[string]interface{}{
				"KeyName":        Ref{"KeyName"},
				"ImageId":        Ref{"AMI"},
				"SecurityGroups": []Ref{{"InstanceSecurityGroup"}},
				"InstanceType":   Ref{"InstanceType"},
				"UserData": Fn("Base64", FnJoin("",
					"#!/bin/bash -xe\n",
					"apt-get update -y && apt-get install -y unzip\n",
					"cd /home/ubuntu\n",
					"sudo -u ubuntu mkdir -p /home/ubuntu/workspace\n",
					"sudo -u ubuntu curl -L -o /home/ubuntu/workspace/stemcell.tgz https://s3.amazonaws.com/bosh-warden-stemcells/bosh-stemcell-2776-warden-boshlite-ubuntu-trusty-go_agent.tgz\n",
					"sudo -u ubuntu curl -L -o /home/ubuntu/workspace/dummy.zip https://github.com/pivotal-cf-experimental/dummy-boshrelease/archive/master.zip\n",
					"sudo -u ubuntu unzip /home/ubuntu/workspace/dummy.zip -d /home/ubuntu/workspace \n",
					"sudo -u ubuntu rm /home/ubuntu/workspace/dummy.zip \n",
					"sudo -u ubuntu mv /home/ubuntu/workspace/dummy-boshrelease-master /home/ubuntu/workspace/dummy \n",
					"\n")),
			},
		},

		"InstanceSecurityGroup": Resource{
			Type: "AWS::EC2::SecurityGroup",
			Properties: map[string]interface{}{
				"GroupDescription": FnJoin("", "SSH Access for Classroom ", Ref{"AWS::StackName"}),
				"SecurityGroupIngress": []interface{}{
					map[string]interface{}{
						"IpProtocol": "tcp",
						"FromPort":   "22",
						"ToPort":     "22",
						"CidrIp":     Ref{"SSHLocation"},
					},
				},
			},
		},
	},
}
<commit_msg>generate deployment manifests<commit_after>package templates

var DefaultTemplate *Template = &Template{
	AWSTemplateFormatVersion: "2010-09-09",
	Description:              "BOSH 101 Classroom CloudFormation Template",
	Parameters: map[string]Parameter{
		"InstanceType": Parameter{
			Description:           "EC2 Instance Type for Classroom VMs",
			Type:                  "String",
			Default:               "m3.xlarge",
			ConstraintDescription: "must be a valid EC2 instance type.",
		},

		"AMI": Parameter{
			Description: "AMI to boot",
			Type:        "String",
			ConstraintDescription: "should be an AMI from the latest cloudfoundry/bosh-lite Vagrant box",
		},

		"KeyName": Parameter{
			Description: "The EC2 Key Pair to allow SSH access to the instances",
			Type:        "AWS::EC2::KeyPair::KeyName",
			ConstraintDescription: "must be the name of an existing EC2 KeyPair",
		},

		"InstanceCount": Parameter{
			Description: "Number of EC2 instances to boot for the classroom",
			Type:        "Number",
			Default:     "1",
		},

		"SSHLocation": Parameter{
			Description:           "The IP address range that can be used to SSH to the EC2 instances",
			Type:                  "String",
			Default:               "0.0.0.0/0",
			ConstraintDescription: "must be a valid IP CIDR range of the form x.x.x.x/x.",
		},
	},
	Resources: map[string]Resource{
		"AutoScalingGroup": Resource{
			Type: "AWS::AutoScaling::AutoScalingGroup",
			Properties: map[string]interface{}{
				"AvailabilityZones":       Fn("GetAZs", ""),
				"LaunchConfigurationName": Ref{"LaunchConfig"},
				"MinSize":                 Ref{"InstanceCount"},
				"MaxSize":                 Ref{"InstanceCount"},
			},
		},

		"LaunchConfig": Resource{
			Type: "AWS::AutoScaling::LaunchConfiguration",
			Properties: map[string]interface{}{
				"KeyName":        Ref{"KeyName"},
				"ImageId":        Ref{"AMI"},
				"SecurityGroups": []Ref{{"InstanceSecurityGroup"}},
				"InstanceType":   Ref{"InstanceType"},
				"UserData": Fn("Base64", FnJoin("",
					"#!/bin/bash -xue\n",
					"apt-get update -y && apt-get install -y unzip git\n",
					"cd /home/ubuntu\n",
					"sudo -u ubuntu mkdir -p workspace\n",
					"sudo -u ubuntu curl -L -o workspace/stemcell.tgz https://s3.amazonaws.com/bosh-warden-stemcells/bosh-stemcell-2776-warden-boshlite-ubuntu-trusty-go_agent.tgz\n",
					"sudo -u ubuntu git clone git://github.com/pivotal-cf-experimental/dummy-boshrelease workspace/dummy \n",
					"sudo -u ubuntu curl -L -o tmp/spiff.zip https://github.com/cloudfoundry-incubator/spiff/releases/download/v1.0.6/spiff_linux_amd64.zip \n",
					"sudo -u ubuntu unzip tmp/spiff.zip -d tmp/ \n",
					"mv tmp/spiff /usr/local/bin/spiff \n",
					"export  UUID=$(bosh status --uuid) \n",
					"echo -e \"---\nname: first\ndirector_uuid:  <%= \\`bosh status --uuid\\` %> \" >  tmp/stub-first.yml \n",
					"echo -e \"---\nname: second\ndirector_uuid: <%= \\`bosh status --uuid\\` %> \" >  tmp/stub-second.yml \n",
					"pushd workspace/dummy \n",
					"    spiff merge templates/dummy-deployment.yml ../../tmp/stub-first.yml >  ../first-deployment-manifest.yml \n",
					"    spiff merge templates/all-jobs-dummy-deployment.yml ../../tmp/stub-second.yml >  ../second-deployment-manifest.yml \n",
					"popd \n",
					"chown -R ubuntu workspace \n",
					"\n")),
			},
		},

		"InstanceSecurityGroup": Resource{
			Type: "AWS::EC2::SecurityGroup",
			Properties: map[string]interface{}{
				"GroupDescription": FnJoin("", "SSH Access for Classroom ", Ref{"AWS::StackName"}),
				"SecurityGroupIngress": []interface{}{
					map[string]interface{}{
						"IpProtocol": "tcp",
						"FromPort":   "22",
						"ToPort":     "22",
						"CidrIp":     Ref{"SSHLocation"},
					},
				},
			},
		},
	},
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"sort"
	"sync"
	"time"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/version"
	"github.com/pkg/errors"
)

func containersGet(d *Daemon, r *http.Request) Response {
	for i := 0; i < 100; i++ {
		result, err := doContainersGet(d, r)
		if err == nil {
			return SyncResponse(true, result)
		}
		if !db.IsRetriableError(err) {
			logger.Debugf("DBERR: containersGet: error %q", err)
			return SmartError(err)
		}
		// 1 s may seem drastic, but we really don't want to thrash
		// perhaps we should use a random amount
		time.Sleep(100 * time.Millisecond)
	}

	logger.Debugf("DBERR: containersGet, db is locked")
	logger.Debugf(logger.GetStack())
	return InternalError(fmt.Errorf("DB is locked"))
}

func doContainersGet(d *Daemon, r *http.Request) (interface{}, error) {
	var result map[string][]string
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		result, err = tx.ContainersListByNodeAddress()
		return err
	})
	if err != nil {
		return []string{}, err
	}

	recursion := util.IsRecursionRequest(r)
	resultString := []string{}
	resultList := []*api.Container{}
	resultMu := sync.Mutex{}

	resultAppend := func(name string, c *api.Container, err error) {
		if err != nil {
			c = &api.Container{
				Name:       name,
				Status:     api.Error.String(),
				StatusCode: api.Error}
		}
		resultMu.Lock()
		resultList = append(resultList, c)
		resultMu.Unlock()
	}

	wg := sync.WaitGroup{}
	for address, containers := range result {
		// Mark containers on unavailable nodes as down
		if recursion && address == "0.0.0.0" {
			for _, container := range containers {
				resultAppend(container, nil, fmt.Errorf("unavailable"))
			}
		}

		// For recursion requests we need to fetch the state of remote
		// containers from their respective nodes.
		if recursion && address != "" && !isClusterNotification(r) {
			wg.Add(1)
			go func(address string) {
				cert := d.endpoints.NetworkCert()
				cs, err := doContainersGetFromNode(address, cert)
				for _, c := range cs {
					resultAppend(c.Name, &c, err)
				}
				wg.Done()
			}(address)
			continue
		}

		for _, container := range containers {
			if !recursion {
				url := fmt.Sprintf("/%s/containers/%s", version.APIVersion, container)
				resultString = append(resultString, url)
				continue
			}

			c, err := doContainerGet(d.State(), container)
			resultAppend(container, c, err)
		}
	}
	wg.Wait()

	if !recursion {
		return resultString, nil
	}

	// Sort the result list by name.
	sort.Slice(resultList, func(i, j int) bool {
		return resultList[i].Name < resultList[j].Name
	})

	return resultList, nil
}

func doContainerGet(s *state.State, cname string) (*api.Container, error) {
	c, err := containerLoadByName(s, cname)
	if err != nil {
		return nil, err
	}

	cts, _, err := c.Render()
	if err != nil {
		return nil, err
	}

	return cts.(*api.Container), nil
}

// Fetch information about the containers on the given remote node, using the
// rest API and with a timeout of 30 seconds.
func doContainersGetFromNode(node string, cert *shared.CertInfo) ([]api.Container, error) {
	f := func() ([]api.Container, error) {
		client, err := cluster.Connect(node, cert, true)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to connect to node %s", node)
		}
		containers, err := client.GetContainers()
		if err != nil {
			return nil, errors.Wrapf(err, "failed to get containers from node %s", node)
		}
		return containers, nil
	}

	timeout := time.After(30 * time.Second)
	done := make(chan struct{})

	var containers []api.Container
	var err error

	go func() {
		containers, err = f()
		done <- struct{}{}
	}()

	select {
	case <-timeout:
		err = fmt.Errorf("timeout getting containers from node %s", node)
	case <-done:
	}

	return containers, err
}
<commit_msg>Fix duplicates in containers listing when fetching remote nodes<commit_after>package main

import (
	"fmt"
	"net/http"
	"sort"
	"sync"
	"time"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/version"
	"github.com/pkg/errors"
)

func containersGet(d *Daemon, r *http.Request) Response {
	for i := 0; i < 100; i++ {
		result, err := doContainersGet(d, r)
		if err == nil {
			return SyncResponse(true, result)
		}
		if !db.IsRetriableError(err) {
			logger.Debugf("DBERR: containersGet: error %q", err)
			return SmartError(err)
		}
		// 1 s may seem drastic, but we really don't want to thrash
		// perhaps we should use a random amount
		time.Sleep(100 * time.Millisecond)
	}

	logger.Debugf("DBERR: containersGet, db is locked")
	logger.Debugf(logger.GetStack())
	return InternalError(fmt.Errorf("DB is locked"))
}

func doContainersGet(d *Daemon, r *http.Request) (interface{}, error) {
	var result map[string][]string
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		result, err = tx.ContainersListByNodeAddress()
		return err
	})
	if err != nil {
		return []string{}, err
	}

	recursion := util.IsRecursionRequest(r)
	resultString := []string{}
	resultList := []*api.Container{}
	resultMu := sync.Mutex{}

	resultAppend := func(name string, c api.Container, err error) {
		if err != nil {
			c = api.Container{
				Name:       name,
				Status:     api.Error.String(),
				StatusCode: api.Error}
		}
		resultMu.Lock()
		resultList = append(resultList, &c)
		resultMu.Unlock()
	}

	wg := sync.WaitGroup{}
	for address, containers := range result {
		// Mark containers on unavailable nodes as down
		if recursion && address == "0.0.0.0" {
			for _, container := range containers {
				resultAppend(container, api.Container{}, fmt.Errorf("unavailable"))
			}
		}

		// If this is an internal request from another cluster node,
		// ignore containers from other nodes, and return only the ones
		// on this node
		if isClusterNotification(r) && address != "" {
			continue
		}

		// For recursion requests we need to fetch the state of remote
		// containers from their respective nodes.
		if recursion && address != "" && !isClusterNotification(r) {
			wg.Add(1)
			go func(address string) {
				cert := d.endpoints.NetworkCert()
				cs, err := doContainersGetFromNode(address, cert)
				for _, c := range cs {
					resultAppend(c.Name, c, err)
				}
				wg.Done()
			}(address)
			continue
		}

		for _, container := range containers {
			if !recursion {
				url := fmt.Sprintf("/%s/containers/%s", version.APIVersion, container)
				resultString = append(resultString, url)
				continue
			}

			c, err := doContainerGet(d.State(), container)
			resultAppend(container, *c, err)
		}
	}
	wg.Wait()

	if !recursion {
		return resultString, nil
	}

	// Sort the result list by name.
	sort.Slice(resultList, func(i, j int) bool {
		return resultList[i].Name < resultList[j].Name
	})

	return resultList, nil
}

func doContainerGet(s *state.State, cname string) (*api.Container, error) {
	c, err := containerLoadByName(s, cname)
	if err != nil {
		return nil, err
	}

	cts, _, err := c.Render()
	if err != nil {
		return nil, err
	}

	return cts.(*api.Container), nil
}

// Fetch information about the containers on the given remote node, using the
// rest API and with a timeout of 30 seconds.
func doContainersGetFromNode(node string, cert *shared.CertInfo) ([]api.Container, error) {
	f := func() ([]api.Container, error) {
		client, err := cluster.Connect(node, cert, true)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to connect to node %s", node)
		}
		containers, err := client.GetContainers()
		if err != nil {
			return nil, errors.Wrapf(err, "failed to get containers from node %s", node)
		}
		return containers, nil
	}

	timeout := time.After(30 * time.Second)
	done := make(chan struct{})

	var containers []api.Container
	var err error

	go func() {
		containers, err = f()
		done <- struct{}{}
	}()

	select {
	case <-timeout:
		err = fmt.Errorf("timeout getting containers from node %s", node)
	case <-done:
	}

	return containers, err
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"math"
	"math/big"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

func networkGetIP(subnet *net.IPNet, host int64) net.IP {
	// Convert IP to a big int
	bigIP := big.NewInt(0)
	bigIP.SetBytes(subnet.IP.To16())

	// Deal with negative offsets
	bigHost := big.NewInt(host)
	bigCount := big.NewInt(host)
	if host < 0 {
		mask, size := subnet.Mask.Size()

		bigHosts := big.NewFloat(0)
		bigHosts.SetFloat64((math.Pow(2, float64(size-mask))))
		bigHostsInt, _ := bigHosts.Int(nil)

		bigCount.Set(bigHostsInt)
		bigCount.Add(bigCount, bigHost)
	}

	// Get the new IP int
	bigIP.Add(bigIP, bigCount)

	// Generate an IPv6
	if subnet.IP.To4() == nil {
		newIp := make(net.IP, 16)
		newIp = bigIP.Bytes()
		return newIp
	}

	// Generate an IPv4
	newIp := make(net.IP, 4)
	binary.BigEndian.PutUint32(newIp, uint32(bigIP.Int64()))
	return newIp
}

func networkPingSubnet(subnet *net.IPNet) bool {
	var fail bool
	var failLock sync.Mutex
	var wgChecks sync.WaitGroup

	ping := func(ip net.IP) {
		defer wgChecks.Done()

		cmd := "ping"
		if ip.To4() == nil {
			cmd = "ping6"
		}

		_, err := exec.Command(cmd, "-n", "-q", ip.String(), "-c", "1", "-W", "1").CombinedOutput()
		if err != nil {
			// Remote didn't answer
			return
		}

		// Remote answered
		failLock.Lock()
		fail = true
		failLock.Unlock()
	}

	poke := func(ip net.IP) {
		defer wgChecks.Done()

		addr := fmt.Sprintf("%s:22", ip.String())
		if ip.To4() == nil {
			addr = fmt.Sprintf("[%s]:22", ip.String())
		}

		_, err := net.DialTimeout("tcp", addr, time.Second)
		if err == nil {
			// Remote answered
			failLock.Lock()
			fail = true
			failLock.Unlock()
			return
		}
	}

	// Ping first IP
	wgChecks.Add(1)
	go ping(networkGetIP(subnet, 1))

	// Poke port on first IP
	wgChecks.Add(1)
	go poke(networkGetIP(subnet, 1))

	// Ping check
	if subnet.IP.To4() != nil {
		// Ping last IP
		wgChecks.Add(1)
		go ping(networkGetIP(subnet, -2))

		// Poke port on last IP
		wgChecks.Add(1)
		go poke(networkGetIP(subnet, -2))
	}

	wgChecks.Wait()

	return fail
}

func networkInRoutingTable(subnet *net.IPNet) bool {
	filename := "route"
	if subnet.IP.To4() == nil {
		filename = "ipv6_route"
	}

	file, err := os.Open(fmt.Sprintf("/proc/net/%s", filename))
	if err != nil {
		return false
	}
	defer file.Close()

	scanner := bufio.NewReader(file)
	for {
		line, _, err := scanner.ReadLine()
		if err != nil {
			break
		}

		fields := strings.Fields(string(line))

		// Get the IP
		ip := net.IP{}
		if filename == "ipv6_route" {
			ip, err = hex.DecodeString(fields[0])
			if err != nil {
				continue
			}
		} else {
			bytes, err := hex.DecodeString(fields[1])
			if err != nil {
				continue
			}

			ip = net.IPv4(bytes[3], bytes[2], bytes[1], bytes[0])
		}

		// Get the mask
		mask := net.IPMask{}
		if filename == "ipv6_route" {
			size, err := strconv.ParseInt(fmt.Sprintf("0x%s", fields[1]), 0, 64)
			if err != nil {
				continue
			}

			mask = net.CIDRMask(int(size), 128)
		} else {
			bytes, err := hex.DecodeString(fields[7])
			if err != nil {
				continue
			}

			mask = net.IPv4Mask(bytes[3], bytes[2], bytes[1], bytes[0])
		}

		// Generate a new network
		lineNet := net.IPNet{IP: ip, Mask: mask}

		// Ignore default gateway
		if lineNet.IP.Equal(net.ParseIP("::")) {
			continue
		}

		if lineNet.IP.Equal(net.ParseIP("0.0.0.0")) {
			continue
		}

		// Check if we have a route to our new subnet
		if lineNet.Contains(subnet.IP) {
			return true
		}
	}

	return false
}

func networkRandomSubnetV4() string {
	var cidr string

	for {
		cidr = fmt.Sprintf("10.%d.%d.1/24", rand.Intn(255), rand.Intn(255))
		_, subnet, err := net.ParseCIDR(cidr)
		if err != nil {
			continue
		}

		if networkInRoutingTable(subnet) {
			continue
		}

		if networkPingSubnet(subnet) {
			continue
		}

		break
	}

	return cidr
}

func networkRandomSubnetV6() string {
	var cidr string

	for {
		cidr = fmt.Sprintf("fd00:%x:%x:%x::1/64", rand.Intn(65535), rand.Intn(65535), rand.Intn(65535))
		_, subnet, err := net.ParseCIDR(cidr)
		if err != nil {
			continue
		}

		if networkInRoutingTable(subnet) {
			continue
		}

		if networkPingSubnet(subnet) {
			continue
		}

		break
	}

	return cidr
}

func networkValidName(value string) error {
	// Validate the length
	if len(value) < 2 {
		return fmt.Errorf("Interface name is too short (minimum 2 characters)")
	}

	if len(value) > 15 {
		return fmt.Errorf("Interface name is too long (maximum 15 characters)")
	}

	// Validate the character set
	match, _ := regexp.MatchString("^[-a-zA-Z0-9]*$", value)
	if !match {
		return fmt.Errorf("Interface name contains invalid characters")
	}

	return nil
}

func networkValidPort(value string) error {
	if value == "" {
		return nil
	}

	valueInt, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	if valueInt < 1 || valueInt > 65536 {
		return fmt.Errorf("Invalid port number: %s", value)
	}

	return nil
}

func networkValidAddress(value string) error {
	err := networkValidAddressV4(value)
	if err == nil {
		return nil
	}

	err = networkValidAddressV6(value)
	if err == nil {
		return nil
	}

	return fmt.Errorf("Not a valid address: %s", value)
}

func networkValidAddressV6(value string) error {
	if value == "" {
		return nil
	}

	ip, net, err := net.ParseCIDR(value)
	if err != nil {
		return err
	}

	if ip.To4() != nil {
		return fmt.Errorf("Not an IPv6 address: %s", value)
	}

	if ip.String() == net.IP.String() {
		return fmt.Errorf("Not a usable IPv6 address: %s", value)
	}

	return nil
}

func networkValidNetworkV6(value string) error {
	if value == "" {
		return nil
	}

	ip, net, err := net.ParseCIDR(value)
	if err != nil {
		return err
	}

	if ip.To4() != nil {
		return fmt.Errorf("Not an IPv6 network: %s", value)
	}

	if ip.String() != net.IP.String() {
		return fmt.Errorf("Not an IPv6 network address: %s", value)
	}

	return nil
}

func networkValidAddressV4(value string) error {
	if value == "" {
		return nil
	}

	ip, net, err := net.ParseCIDR(value)
	if err != nil {
		return err
	}

	if ip.To4() == nil {
		return fmt.Errorf("Not an IPv4 address: %s", value)
	}

	if ip.String() == net.IP.String() {
		return fmt.Errorf("Not a usable IPv4 address: %s", value)
	}

	return nil
}

func networkValidNetworkV4(value string) error {
	if value == "" {
		return nil
	}

	ip, net, err := net.ParseCIDR(value)
	if err != nil {
		return err
	}

	if ip.To4() == nil {
		return fmt.Errorf("Not an IPv4 network: %s", value)
	}

	if ip.String() != net.IP.String() {
		return fmt.Errorf("Not an IPv4 network address: %s", value)
	}

	return nil
}
<commit_msg>network: Add auto fan underlay selection<commit_after>package main

import (
	"bufio"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"math"
	"math/big"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

func networkGetIP(subnet *net.IPNet, host int64) net.IP {
	// Convert IP to a big int
	bigIP := big.NewInt(0)
	bigIP.SetBytes(subnet.IP.To16())

	// Deal with negative offsets
	bigHost := big.NewInt(host)
	bigCount := big.NewInt(host)
	if host < 0 {
		mask, size := subnet.Mask.Size()

		bigHosts := big.NewFloat(0)
		bigHosts.SetFloat64((math.Pow(2, float64(size-mask))))
		bigHostsInt, _ := bigHosts.Int(nil)

		bigCount.Set(bigHostsInt)
		bigCount.Add(bigCount, bigHost)
	}

	// Get the new IP int
	bigIP.Add(bigIP, bigCount)

	// Generate an IPv6
	if subnet.IP.To4() == nil {
		newIp := make(net.IP, 16)
		newIp = bigIP.Bytes()
		return newIp
	}

	// Generate an IPv4
	newIp := make(net.IP, 4)
	binary.BigEndian.PutUint32(newIp, uint32(bigIP.Int64()))
	return newIp
}

func networkPingSubnet(subnet *net.IPNet) bool {
	var fail bool
	var failLock sync.Mutex
	var wgChecks sync.WaitGroup

	ping := func(ip net.IP) {
		defer wgChecks.Done()

		cmd := "ping"
		if ip.To4() == nil {
			cmd = "ping6"
		}

		_, err := exec.Command(cmd, "-n", "-q", ip.String(), "-c", "1", "-W", "1").CombinedOutput()
		if err != nil {
			// Remote didn't answer
			return
		}

		// Remote answered
		failLock.Lock()
		fail = true
		failLock.Unlock()
	}

	poke := func(ip net.IP) {
		defer wgChecks.Done()

		addr := fmt.Sprintf("%s:22", ip.String())
		if ip.To4() == nil {
			addr = fmt.Sprintf("[%s]:22", ip.String())
		}

		_, err := net.DialTimeout("tcp", addr, time.Second)
		if err == nil {
			// Remote answered
			failLock.Lock()
			fail = true
			failLock.Unlock()
			return
		}
	}

	// Ping first IP
	wgChecks.Add(1)
	go ping(networkGetIP(subnet, 1))

	// Poke port on first IP
	wgChecks.Add(1)
	go poke(networkGetIP(subnet, 1))

	// Ping check
	if subnet.IP.To4() != nil {
		// Ping last IP
		wgChecks.Add(1)
		go ping(networkGetIP(subnet, -2))

		// Poke port on last IP
		wgChecks.Add(1)
		go poke(networkGetIP(subnet, -2))
	}

	wgChecks.Wait()

	return fail
}

func networkInRoutingTable(subnet *net.IPNet) bool {
	filename := "route"
	if subnet.IP.To4() == nil {
		filename = "ipv6_route"
	}

	file, err := os.Open(fmt.Sprintf("/proc/net/%s", filename))
	if err != nil {
		return false
	}
	defer file.Close()

	scanner := bufio.NewReader(file)
	for {
		line, _, err := scanner.ReadLine()
		if err != nil {
			break
		}

		fields := strings.Fields(string(line))

		// Get the IP
		ip := net.IP{}
		if filename == "ipv6_route" {
			ip, err = hex.DecodeString(fields[0])
			if err != nil {
				continue
			}
		} else {
			bytes, err := hex.DecodeString(fields[1])
			if err != nil {
				continue
			}

			ip = net.IPv4(bytes[3], bytes[2], bytes[1], bytes[0])
		}

		// Get the mask
		mask := net.IPMask{}
		if filename == "ipv6_route" {
			size, err := strconv.ParseInt(fmt.Sprintf("0x%s", fields[1]), 0, 64)
			if err != nil {
				continue
			}

			mask = net.CIDRMask(int(size), 128)
		} else {
			bytes, err := hex.DecodeString(fields[7])
			if err != nil {
				continue
			}

			mask = net.IPv4Mask(bytes[3], bytes[2], bytes[1], bytes[0])
		}

		// Generate a new network
		lineNet := net.IPNet{IP: ip, Mask: mask}

		// Ignore default gateway
		if lineNet.IP.Equal(net.ParseIP("::")) {
			continue
		}

		if lineNet.IP.Equal(net.ParseIP("0.0.0.0")) {
			continue
		}

		// Check if we have a route to our new subnet
		if lineNet.Contains(subnet.IP) {
			return true
		}
	}

	return false
}

func networkRandomSubnetV4() string {
	var cidr string

	for {
		cidr = fmt.Sprintf("10.%d.%d.1/24", rand.Intn(255), rand.Intn(255))
		_, subnet, err := net.ParseCIDR(cidr)
		if err != nil {
			continue
		}

		if networkInRoutingTable(subnet) {
			continue
		}

		if networkPingSubnet(subnet) {
			continue
		}

		break
	}

	return cidr
}

func networkRandomSubnetV6() string {
	var cidr string

	for {
		cidr = fmt.Sprintf("fd00:%x:%x:%x::1/64", rand.Intn(65535), rand.Intn(65535), rand.Intn(65535))
		_, subnet, err := net.ParseCIDR(cidr)
		if err != nil {
			continue
		}

		if networkInRoutingTable(subnet) {
			continue
		}

		if networkPingSubnet(subnet) {
			continue
		}

		break
	}

	return cidr
}

func networkDefaultGatewaySubnetV4() (*net.IPNet, error) {
	file, err := os.Open("/proc/net/route")
	if err != nil {
		return nil, err
	}
	defer file.Close()

	ifaceName := ""

	scanner := bufio.NewReader(file)
	for {
		line, _, err := scanner.ReadLine()
		if err != nil {
			break
		}

		fields := strings.Fields(string(line))

		if fields[1] == "00000000" && fields[7] == "00000000" {
			ifaceName = fields[0]
			break
		}
	}

	if ifaceName == "" {
		return nil, fmt.Errorf("No default gateway for IPv4")
	}

	iface, err := net.InterfaceByName(ifaceName)
	if err != nil {
		return nil, err
	}

	addrs, err := iface.Addrs()
	if err != nil {
		return nil, err
	}

	var subnet *net.IPNet

	for _, addr := range addrs {
		addrIP, addrNet, err := net.ParseCIDR(addr.String())
		if err != nil {
			return nil, err
		}

		if addrIP.To4() == nil {
			continue
		}

		if subnet != nil {
			return nil, fmt.Errorf("More than one IPv4 subnet on default interface")
		}

		subnet = addrNet
	}

	if subnet == nil {
		return nil, fmt.Errorf("No IPv4 subnet on default interface")
	}

	return subnet, nil
}

func networkValidName(value string) error {
	// Validate the length
	if len(value) < 2 {
		return fmt.Errorf("Interface name is too short (minimum 2 characters)")
	}

	if len(value) > 15 {
		return fmt.Errorf("Interface name is too long (maximum 15 characters)")
	}

	// Validate the character set
	match, _ := regexp.MatchString("^[-a-zA-Z0-9]*$", value)
	if !match {
		return fmt.Errorf("Interface name contains invalid characters")
	}

	return nil
}

func networkValidPort(value string) error {
	if value == "" {
		return nil
	}

	valueInt, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	if valueInt < 1 || valueInt > 65536 {
		return fmt.Errorf("Invalid port number: %s", value)
	}

	return nil
}

func networkValidAddress(value string) error {
	err := networkValidAddressV4(value)
	if err == nil {
		return nil
	}

	err = networkValidAddressV6(value)
	if err == nil {
		return nil
	}

	return fmt.Errorf("Not a valid address: %s", value)
}

func networkValidAddressV6(value string) error {
	if value == "" {
		return nil
	}

	ip, net, err := net.ParseCIDR(value)
	if err != nil {
		return err
	}

	if ip.To4() != nil {
		return fmt.Errorf("Not an IPv6 address: %s", value)
	}

	if ip.String() == net.IP.String() {
		return fmt.Errorf("Not a usable IPv6 address: %s", value)
	}

	return nil
}

func networkValidNetworkV6(value string) error {
	if value == "" {
		return nil
	}

	ip, net, err := net.ParseCIDR(value)
	if err != nil {
		return err
	}

	if ip.To4() != nil {
		return fmt.Errorf("Not an IPv6 network: %s", value)
	}

	if ip.String() != net.IP.String() {
		return fmt.Errorf("Not an IPv6 network address: %s", value)
	}

	return nil
}

func networkValidAddressV4(value string) error {
	if value == "" {
		return nil
	}

	ip, net, err := net.ParseCIDR(value)
	if err != nil {
		return err
	}

	if ip.To4() == nil {
		return fmt.Errorf("Not an IPv4 address: %s", value)
	}

	if ip.String() == net.IP.String() {
		return fmt.Errorf("Not a usable IPv4 address: %s", value)
	}

	return nil
}

func networkValidNetworkV4(value string) error {
	if value == "" {
		return nil
	}

	ip, net, err := net.ParseCIDR(value)
	if err != nil {
		return err
	}

	if ip.To4() == nil {
		return fmt.Errorf("Not an IPv4 network: %s", value)
	}

	if ip.String() != net.IP.String() {
		return fmt.Errorf("Not an IPv4 network address: %s", value)
	}

	return nil
}
<|endoftext|>