<commit_before>package git

// This file just contains the Git-specific portions of sshd.

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"text/template"

	"github.com/Masterminds/cookoo"
	"github.com/Masterminds/cookoo/log"
	"golang.org/x/crypto/ssh"
)

// PrereceiveHookTmpl is a pre-receive hook.
//
// This is overridable. The following template variables are passed into it:
//
// 	.GitHome: the path to Git's home directory.
var PrereceiveHookTpl = `#!/bin/bash
strip_remote_prefix() {
    stdbuf -i0 -o0 -e0 sed "s/^/"$'\e[1G'"/"
}

while read oldrev newrev refname
do
  LOCKFILE="/tmp/$RECEIVE_REPO.lock"
  if ( set -o noclobber; echo "$$" > "$LOCKFILE" ) 2> /dev/null; then
	trap 'rm -f "$LOCKFILE"; exit 1' INT TERM EXIT

	# check for authorization on this repo
	{{.GitHome}}/receiver "$RECEIVE_REPO" "$newrev" "$RECEIVE_USER" "$RECEIVE_FINGERPRINT"
	rc=$?
	if [[ $rc != 0 ]] ; then
	  echo "      ERROR: failed on rev $newrev - push denied"
	  exit $rc
	fi
	# builder assumes that we are running this script from $GITHOME
	cd {{.GitHome}}
	# if we're processing a receive-pack on an existing repo, run a build
	if [[ $SSH_ORIGINAL_COMMAND == git-receive-pack* ]]; then
		{{.GitHome}}/builder "$RECEIVE_USER" "$RECEIVE_REPO" "$newrev" 2>&1 | strip_remote_prefix
	fi

	rm -f "$LOCKFILE"
	trap - INT TERM EXIT
  else
	echo "Another git push is ongoing. Aborting..."
	exit 1
  fi
done
`

// Receive receives a Git repo.
// This will only work for git-receive-pack.
//
// Params:
// 	- operation (string): e.g. git-receive-pack
// 	- repoName (string): The repository name, in the form '/REPO.git'.
// 	- channel (ssh.Channel): The channel.
// 	- request (*ssh.Request): The channel.
// 	- gitHome (string): Defaults to /home/git.
// 	- fingerprint (string): The fingerprint of the user's SSH key.
// 	- user (string): The name of the Deis user.
//
// Returns:
// 	- nothing
func Receive(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
	if ok, z := p.Requires("channel", "request", "fingerprint", "permissions"); !ok {
		return nil, fmt.Errorf("Missing requirements %q", z)
	}
	repoName := p.Get("repoName", "").(string)
	operation := p.Get("operation", "").(string)
	channel := p.Get("channel", nil).(ssh.Channel)
	gitHome := p.Get("gitHome", "/home/git").(string)
	fingerprint := p.Get("fingerprint", nil).(string)
	user := p.Get("user", "").(string)

	repo, err := cleanRepoName(repoName)
	if err != nil {
		log.Warnf(c, "Illegal repo name: %s.", err)
		channel.Stderr().Write([]byte("No repo given"))
		return nil, err
	}
	repo += ".git"

	if _, err := createRepo(c, filepath.Join(gitHome, repo), gitHome); err != nil {
		log.Infof(c, "Did not create new repo: %s", err)
	}
	cmd := exec.Command("git-shell", "-c", fmt.Sprintf("%s '%s'", operation, repo))
	log.Infof(c, strings.Join(cmd.Args, " "))

	var errbuff bytes.Buffer

	cmd.Dir = gitHome
	cmd.Env = []string{
		fmt.Sprintf("RECEIVE_USER=%s", user),
		fmt.Sprintf("RECEIVE_REPO=%s", repo),
		fmt.Sprintf("RECEIVE_FINGERPRINT=%s", fingerprint),
		fmt.Sprintf("SSH_ORIGINAL_COMMAND=%s '%s'", operation, repo),
		fmt.Sprintf("SSH_CONNECTION=%s", c.Get("SSH_CONNECTION", "0 0 0 0").(string)),
	}
	cmd.Env = append(cmd.Env, os.Environ()...)

	done := plumbCommand(cmd, channel, &errbuff)

	if err := cmd.Start(); err != nil {
		log.Warnf(c, "Failed git receive immediately: %s %s", err, errbuff.Bytes())
		return nil, err
	}
	fmt.Printf("Waiting for git-receive to run.\n")
	done.Wait()
	fmt.Printf("Waiting for deploy.\n")
	if err := cmd.Wait(); err != nil {
		log.Errf(c, "Error on command: %s %s", err, errbuff.Bytes())
		return nil, err
	}
	if errbuff.Len() > 0 {
		log.Warnf(c, "Unreported error: %s", errbuff.Bytes())
	}
	log.Infof(c, "Deploy complete.\n")

	return nil, nil
}

func execAs(user, cmd string, args ...string) *exec.Cmd {
	fullCmd := cmd + " " + strings.Join(args, " ")
	return exec.Command("su", user, "-c", fullCmd)
}

// cleanRepoName cleans a repository name for a git-sh operation.
func cleanRepoName(name string) (string, error) {
	if len(name) == 0 {
		return name, errors.New("Empty repo name.")
	}
	if strings.Contains(name, "..") {
		return "", errors.New("Cannot change directory in file name.")
	}
	name = strings.Replace(name, "'", "", -1)
	return strings.TrimPrefix(strings.TrimSuffix(name, ".git"), "/"), nil
}

// plumbCommand connects the exec in/output and the channel in/output.
//
// The sidechannel is for sending errors to logs.
func plumbCommand(cmd *exec.Cmd, channel ssh.Channel, sidechannel io.Writer) *sync.WaitGroup {
	var wg sync.WaitGroup
	inpipe, _ := cmd.StdinPipe()
	go func() {
		io.Copy(inpipe, channel)
		inpipe.Close()
	}()

	cmd.Stdout = channel
	cmd.Stderr = channel.Stderr()

	return &wg
}

var createLock sync.Mutex

// createRepo creates a new Git repo if it is not present already.
//
// Largely inspired by gitreceived from Flynn.
//
// Returns a bool indicating whether a project was created (true) or already
// existed (false).
func createRepo(c cookoo.Context, repoPath, gitHome string) (bool, error) {
	createLock.Lock()
	defer createLock.Unlock()

	if fi, err := os.Stat(repoPath); err == nil && fi.IsDir() {
		// Nothing to do.
		log.Infof(c, "Directory %s already exists.", repoPath)
		return false, nil
	} else if os.IsNotExist(err) {

		log.Infof(c, "Creating new directory at %s", repoPath)
		// Create directory
		if err := os.MkdirAll(repoPath, 0755); err != nil {
			log.Warnf(c, "Failed to create repository: %s", err)
			return false, err
		}
		cmd := exec.Command("git", "init", "--bare")
		cmd.Dir = repoPath
		if out, err := cmd.CombinedOutput(); err != nil {
			log.Warnf(c, "git init output: %s", out)
			return false, err
		}

		hook, err := prereceiveHook(map[string]string{"GitHome": gitHome})
		if err != nil {
			return true, err
		}
		ioutil.WriteFile(filepath.Join(repoPath, "hooks", "pre-receive"), hook, 0755)

		return true, nil
	} else if err == nil {
		return false, errors.New("Expected directory, found file.")
	} else {
		return false, err
	}
}

//prereceiveHook templates a pre-receive hook for Git.
func prereceiveHook(vars map[string]string) ([]byte, error) {
	var out bytes.Buffer
	// We parse the template anew each receive in case it has changed.
	t, err := template.New("hooks").Parse(PrereceiveHookTpl)
	if err != nil {
		return []byte{}, err
	}

	err = t.Execute(&out, vars)
	return out.Bytes(), err
}
<commit_msg>fix(builder): Fix problem with missed git repos after buider restart<commit_after>package git

// This file just contains the Git-specific portions of sshd.

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"text/template"

	"github.com/Masterminds/cookoo"
	"github.com/Masterminds/cookoo/log"
	"golang.org/x/crypto/ssh"
)

// PrereceiveHookTmpl is a pre-receive hook.
//
// This is overridable. The following template variables are passed into it:
//
// 	.GitHome: the path to Git's home directory.
var PrereceiveHookTpl = `#!/bin/bash
strip_remote_prefix() {
    stdbuf -i0 -o0 -e0 sed "s/^/"$'\e[1G'"/"
}

while read oldrev newrev refname
do
  LOCKFILE="/tmp/$RECEIVE_REPO.lock"
  if ( set -o noclobber; echo "$$" > "$LOCKFILE" ) 2> /dev/null; then
	trap 'rm -f "$LOCKFILE"; exit 1' INT TERM EXIT

	# check for authorization on this repo
	{{.GitHome}}/receiver "$RECEIVE_REPO" "$newrev" "$RECEIVE_USER" "$RECEIVE_FINGERPRINT"
	rc=$?
	if [[ $rc != 0 ]] ; then
	  echo "      ERROR: failed on rev $newrev - push denied"
	  exit $rc
	fi
	# builder assumes that we are running this script from $GITHOME
	cd {{.GitHome}}
	# if we're processing a receive-pack on an existing repo, run a build
	if [[ $SSH_ORIGINAL_COMMAND == git-receive-pack* ]]; then
		{{.GitHome}}/builder "$RECEIVE_USER" "$RECEIVE_REPO" "$newrev" 2>&1 | strip_remote_prefix
	fi

	rm -f "$LOCKFILE"
	trap - INT TERM EXIT
  else
	echo "Another git push is ongoing. Aborting..."
	exit 1
  fi
done
`

// Receive receives a Git repo.
// This will only work for git-receive-pack.
//
// Params:
// 	- operation (string): e.g. git-receive-pack
// 	- repoName (string): The repository name, in the form '/REPO.git'.
// 	- channel (ssh.Channel): The channel.
// 	- request (*ssh.Request): The channel.
// 	- gitHome (string): Defaults to /home/git.
// 	- fingerprint (string): The fingerprint of the user's SSH key.
// 	- user (string): The name of the Deis user.
//
// Returns:
// 	- nothing
func Receive(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
	if ok, z := p.Requires("channel", "request", "fingerprint", "permissions"); !ok {
		return nil, fmt.Errorf("Missing requirements %q", z)
	}
	repoName := p.Get("repoName", "").(string)
	operation := p.Get("operation", "").(string)
	channel := p.Get("channel", nil).(ssh.Channel)
	gitHome := p.Get("gitHome", "/home/git").(string)
	fingerprint := p.Get("fingerprint", nil).(string)
	user := p.Get("user", "").(string)

	repo, err := cleanRepoName(repoName)
	if err != nil {
		log.Warnf(c, "Illegal repo name: %s.", err)
		channel.Stderr().Write([]byte("No repo given"))
		return nil, err
	}
	repo += ".git"

	if _, err := createRepo(c, filepath.Join(gitHome, repo), gitHome); err != nil {
		log.Infof(c, "Did not create new repo: %s", err)
	}
	cmd := exec.Command("git-shell", "-c", fmt.Sprintf("%s '%s'", operation, repo))
	log.Infof(c, strings.Join(cmd.Args, " "))

	var errbuff bytes.Buffer

	cmd.Dir = gitHome
	cmd.Env = []string{
		fmt.Sprintf("RECEIVE_USER=%s", user),
		fmt.Sprintf("RECEIVE_REPO=%s", repo),
		fmt.Sprintf("RECEIVE_FINGERPRINT=%s", fingerprint),
		fmt.Sprintf("SSH_ORIGINAL_COMMAND=%s '%s'", operation, repo),
		fmt.Sprintf("SSH_CONNECTION=%s", c.Get("SSH_CONNECTION", "0 0 0 0").(string)),
	}
	cmd.Env = append(cmd.Env, os.Environ()...)

	done := plumbCommand(cmd, channel, &errbuff)

	if err := cmd.Start(); err != nil {
		log.Warnf(c, "Failed git receive immediately: %s %s", err, errbuff.Bytes())
		return nil, err
	}
	fmt.Printf("Waiting for git-receive to run.\n")
	done.Wait()
	fmt.Printf("Waiting for deploy.\n")
	if err := cmd.Wait(); err != nil {
		log.Errf(c, "Error on command: %s %s", err, errbuff.Bytes())
		return nil, err
	}
	if errbuff.Len() > 0 {
		log.Warnf(c, "Unreported error: %s", errbuff.Bytes())
	}
	log.Infof(c, "Deploy complete.\n")

	return nil, nil
}

func execAs(user, cmd string, args ...string) *exec.Cmd {
	fullCmd := cmd + " " + strings.Join(args, " ")
	return exec.Command("su", user, "-c", fullCmd)
}

// cleanRepoName cleans a repository name for a git-sh operation.
func cleanRepoName(name string) (string, error) {
	if len(name) == 0 {
		return name, errors.New("Empty repo name.")
	}
	if strings.Contains(name, "..") {
		return "", errors.New("Cannot change directory in file name.")
	}
	name = strings.Replace(name, "'", "", -1)
	return strings.TrimPrefix(strings.TrimSuffix(name, ".git"), "/"), nil
}

// plumbCommand connects the exec in/output and the channel in/output.
//
// The sidechannel is for sending errors to logs.
func plumbCommand(cmd *exec.Cmd, channel ssh.Channel, sidechannel io.Writer) *sync.WaitGroup {
	var wg sync.WaitGroup
	inpipe, _ := cmd.StdinPipe()
	go func() {
		io.Copy(inpipe, channel)
		inpipe.Close()
	}()

	cmd.Stdout = channel
	cmd.Stderr = channel.Stderr()

	return &wg
}

var createLock sync.Mutex

// initRepo create a directory and init a new Git repo
func initRepo(repoPath, gitHome string, c cookoo.Context) (bool, error) {
	log.Infof(c, "Creating new directory at %s", repoPath)
	// Create directory
	if err := os.MkdirAll(repoPath, 0755); err != nil {
		log.Warnf(c, "Failed to create repository: %s", err)
		return false, err
	}
	cmd := exec.Command("git", "init", "--bare")
	cmd.Dir = repoPath
	if out, err := cmd.CombinedOutput(); err != nil {
		log.Warnf(c, "git init output: %s", out)
		return false, err
	}

	hook, err := prereceiveHook(map[string]string{"GitHome": gitHome})
	if err != nil {
		return true, err
	}
	ioutil.WriteFile(filepath.Join(repoPath, "hooks", "pre-receive"), hook, 0755)

	return true, nil
}

// createRepo creates a new Git repo if it is not present already.
//
// Largely inspired by gitreceived from Flynn.
//
// Returns a bool indicating whether a project was created (true) or already
// existed (false).
func createRepo(c cookoo.Context, repoPath, gitHome string) (bool, error) {
	createLock.Lock()
	defer createLock.Unlock()

	if fi, err := os.Stat(repoPath); err == nil {
		if fi.IsDir() {
			configPath := filepath.Join(repoPath, "config")
			if _, cerr := os.Stat(configPath); cerr == nil {
				log.Infof(c, "Directory '%s' already exists.", repoPath)
				return true, nil
			} else {
				log.Warnf(c, "No config file found at path `%s`; removing it and recreating.", repoPath)
				if err := os.RemoveAll(repoPath); err != nil {
					return false, fmt.Errorf("Unable to remove path '%s': %s", repoPath, err)
				}
			}
		} else {
			log.Warnf(c, "Path '%s' is not a directory; removing it and recreating.", repoPath)
			if err := os.RemoveAll(repoPath); err != nil {
				return false, fmt.Errorf("Unable to remove path '%s': %s", repoPath, err)
			}
		}
	} else if os.IsNotExist(err) {
		log.Infof(c, "Unable to get stat for path '%s': %s .", repoPath, err)
	} else {
		return false, err
	}
	return initRepo(repoPath, gitHome, c)
}

//prereceiveHook templates a pre-receive hook for Git.
func prereceiveHook(vars map[string]string) ([]byte, error) {
	var out bytes.Buffer
	// We parse the template anew each receive in case it has changed.
	t, err := template.New("hooks").Parse(PrereceiveHookTpl)
	if err != nil {
		return []byte{}, err
	}

	err = t.Execute(&out, vars)
	return out.Bytes(), err
}
<|endoftext|>


<commit_before>// nflag is an open source alternative to Golang's flag package.
// nflag allows OS-specific flag styling, type option simplification, and allowance of empty values.

package nflag

import (
	"errors"
	"fmt"
	"os"
	"reflect"
	"runtime"
	"strconv"
	"strings"
)

var Config ConfigOptions
var Flags map[string]Flag

// Package Init
func init() {
	Flags = make(map[string]Flag)      // Make the Flags map
	Config.OSSpecificFlags = true      // Default to being true
	Config.OSSpecificFlagString = "--" // Default to using a double dash for declaring flags

	if runtime.GOOS == "windows" { // If we are on Windows
		Config.OSSpecificFlagString = "/" // Use a slash, since Windows likes to be "special"
	}
}

// Configure
// This function is for configuration of nflag prior to usage.
func Configure(providedConfig ConfigOptions) {
	if providedConfig.OSSpecificFlags == false { // If we are overriding OSSpecificFlags
		Config.OSSpecificFlags = false
		Config.OSSpecificFlagString = "--" // Enforce --
	}

	if providedConfig.OSSpecificFlagString != "" { // If we are overriding OSSpecificFlagString
		Config.OSSpecificFlagString = providedConfig.OSSpecificFlagString // Set OSSpecificFlagString to whatever is provided on config
	}

	if providedConfig.ShowHelpIfNoArgs { // If ShowHelpIfNoArgs was set to true
		Config.ShowHelpIfNoArgs = true // Change to true
	}
}

// Get
// This function will get the flag value and returns it, or an error if the flag does not exist.
func Get(flagName string) (interface{}, error) {
	flag, exists := Flags[flagName] // Get the trimmedFlag and the exists bool of this flagName in Flags
	var val interface{}
	var err error

	if exists { // If the flag exists
		val = flag.Value
	} else {
		err = errors.New("Flag does not exist.")
	}

	return val, err
}

// #region GetAs functions

// GetAsBool
// This function will get the flag value and convert it to bool, or an error if the flag does not exist.
func GetAsBool(flagName string) (bool, error) {
	var flagValue bool
	flagValueInterface, flagValueError := Get(flagName)

	if flagValueError == nil { // If this flag does exist
		flagValue = flagValueInterface.(bool) // Type inference to bool
	}

	return flagValue, flagValueError
}

// GetAsInt
// This function will get the flag value and convert it to int, or an error if the flag does not exist.
func GetAsInt(flagName string) (int, error) {
	var flagValue int
	flagValueInterface, flagValueError := Get(flagName)

	if flagValueError == nil { // If this flag does exist
		flagValue = flagValueInterface.(int) // Type inference to int
	}

	return flagValue, flagValueError
}

// GetAsFloat64
// This function will get the flag value and convert it to float64, or an error if the flag does not exist.
func GetAsFloat64(flagName string) (float64, error) {
	var flagValue float64
	flagValueInterface, flagValueError := Get(flagName)

	if flagValueError == nil { // If this flag does exist
		flagValue = flagValueInterface.(float64) // Type inference to float64
	}

	return flagValue, flagValueError
}

// GetAsString
// This function will get the flag value and convert it to string, or an error if the flag does not exist.
func GetAsString(flagName string) (string, error) {
	var flagValue string
	flagValueInterface, flagValueError := Get(flagName)

	if flagValueError == nil { // If this flag does exist
		flagValue = flagValueInterface.(string) // Type inference to string
	}

	return flagValue, flagValueError
}

// IsDefaultValue
// This function will return a boolean as to whether or not the value of the flag is the same as DefaultValue
func IsDefaultValue(flagName string) (bool, error) {
	var isDefaultValue bool
	flagValue, flagValueError := Get(flagName)

	if flagValueError == nil { // If the flag exists
		if Flags[flagName].DefaultValue == flagValue { // If the DefaultValue is the same as the flagValue
			isDefaultValue = true // Change isDefaultValue to true
		}
	}

	return isDefaultValue, flagValueError
}

// Set
// This function is for setting a flag.
func Set(flagName string, providedFlag Flag) error {
	var errorResponse error

	if providedFlag.Type == "" { // If no Type was provided
		providedFlag.Type = "bool" // Default to bool
	}

	allowedTypes := []string{"bool", "float64", "int", "string"} // Define allowed types
	isAllowedType := false                                       // Default to providedFlag.Type not being the type

	for _, val := range allowedTypes {
		if providedFlag.Type == val { // If the Type is the same as what is allowed (this specific value)
			isAllowedType = true
		}
	}

	if isAllowedType { // If this is an allowed type
		if providedFlag.DefaultValue == nil { // If no default value was provided
			if providedFlag.Type == "bool" { // If the type is bool
				providedFlag.DefaultValue = false
			} else if providedFlag.Type == "float64" { // If the type is float64
				providedFlag.DefaultValue = 1.1
			} else if providedFlag.Type == "int" { // If the type is int
				providedFlag.DefaultValue = 1
			} else if providedFlag.Type == "string" {
				providedFlag.DefaultValue = ""
			}
		} else if (providedFlag.DefaultValue != nil) && (!providedFlag.AllowNothing) { // If a Default value was provided
			providedFlag.AllowNothing = true                                             // Enforce AllowNothing since a default value is provided
			if providedFlag.Type != reflect.TypeOf(providedFlag.DefaultValue).String() { // If the Type and DefaultValue aren't the same type
				errorResponse = errors.New("Mismatch Flag and DefaultValue types.")
			}
		}

		if errorResponse == nil { // If there was no error
			Flags[flagName] = providedFlag // Set the flag name in Flags to the providedFlag struct
		}
	} else { // If this is a non-allowed type
		errorResponse = errors.New("Type is not allowed. Please use bool, float64, int, or string.")
	}

	return errorResponse
}

// Parse
// This function will parse input for flags
func Parse() {
	providedFlags := os.Args[1:] // ProvidedFlags start at index 1 (after binary name)

	if len(providedFlags) != 0 { // If flags were provided
		if providedFlags[0] != (Config.OSSpecificFlagString + "help") { // If the first providedFlag is not help
			for _, flag := range providedFlags { // For each flag provided
				flagNameValueSplit := strings.Split(flag, "=")                                          // Split the flag name and the value
				flagName := strings.Replace(flagNameValueSplit[0], Config.OSSpecificFlagString, "", -1) // Get the flagName and remove the OSSpecificFlagString
				flagValue := ""                                                                         // Default to nothing

				if len(flagNameValueSplit) == 2 { // If a value was provided
					flagValue = flagNameValueSplit[1] // Set to the value in flagNameValueSplit
				} else { // If the flag was passed but no value
					trimmedFlag, exists := Flags[flagName] // Get the trimmedFlag and the exists bool of this flagName in Flags

					if exists && (trimmedFlag.Type == "bool") { // If the flag exists and its type is bool
						flagValue = "true" // By passing just --flag, it implies --flag is true
					}
				}

				ParseVal(flagName, flagValue) // Parse the value
			}
		} else { // If we should output the help message
			PrintFlags() // Print the flags
			os.Exit(1)
		}
	} else if Config.ShowHelpIfNoArgs { // If no arguments are passed and ShowHelpIfNoArgs is true
		PrintFlags()
		os.Exit(1)
	}

	// Executed in the event some or no flags are passed and we haven't exited
	for flagName, _ := range Flags { // For each flagName in Flags
		if Flags[flagName].Value == nil { // If no value is set for this Flag (which happens if it wasn't parsed via
			ParseVal(flagName, "flag-not-provided")
		}
	}
}

// ParseVal
// This function will check Flags for the provided flag and parse the provided value
func ParseVal(flagName string, flagValue string) {
	trimmedFlag, exists := Flags[flagName] // Get the trimmedFlag and the exists bool of this flagName in Flags

	if exists { // If the flag exists
		if (flagValue != "") && (flagValue != "flag-not-provided") { // If a value was defined
			var conversionError error // Define conversionError as any error from not correctly converting the value to the type

			if trimmedFlag.Type == "bool" { // If the type is bool
				trimmedFlag.Value, conversionError = strconv.ParseBool(flagValue) // Convert to bool
			} else if trimmedFlag.Type == "float64" { // If the type is float64
				trimmedFlag.Value, conversionError = strconv.ParseFloat(flagValue, 64) // Convert to float64
			} else if trimmedFlag.Type == "int" { // If the type is int
				trimmedFlag.Value, conversionError = strconv.Atoi(flagValue) // Convert to int
			} else { // Remaining type, being int
				trimmedFlag.Value = flagValue
			}

			if conversionError != nil { // If there was a conversionError
				fmt.Println("An incorrect value was provided when using " + Config.OSSpecificFlagString + flagName + ". Exiting.")
				os.Exit(1)
			}
		} else { // If no value was provided by the user
			returnRequiredValueMessage := false

			if flagValue == "" { // If the flag was provided by the user but no content
				if trimmedFlag.AllowNothing { // If providing no content is permitted
					if trimmedFlag.Type == "bool" { // If the flag is bool
						trimmedFlag.Value = true // Force true
					} else { // If the flag is not bool
						trimmedFlag.Value = trimmedFlag.DefaultValue // Set to defaultvalue
					}
				} else { // If providing no content is NOT permitted
					returnRequiredValueMessage = true
				}
			} else { // If this flag was not provided
				if !trimmedFlag.Required { // If the flag is NOT required
					trimmedFlag.Value = trimmedFlag.DefaultValue // Set the value to the default
				} else { // If the flag IS required
					returnRequiredValueMessage = true
				}
			}

			if returnRequiredValueMessage {
				fmt.Println("A required value for " + Config.OSSpecificFlagString + flagName + " was not provided. Exiting.")
				os.Exit(1)
			}
		}

		Flags[flagName] = trimmedFlag // Update the map
	} else { // If the flag does not exist.
		fmt.Println(Config.OSSpecificFlagString + flagName + " does not exist. Exiting.")
		os.Exit(1)
	}
}

// PrintFlags
// This function will print all the flags that are set and their defaults
func PrintFlags() {
	fmt.Println("Usage: " + Config.OSSpecificFlagString + "example=value")
	fmt.Println("The following options are available:")

	for flagName, flag := range Flags { // For each flagName and trimmedFlag struct in Flags
		fmt.Println(Config.OSSpecificFlagString + flagName + " " + flag.Type)
		fmt.Println("\t" + flag.Descriptor)

		if flag.DefaultValue != nil { // If DefaultValue is not nil
			fmt.Println("\t" + "Default Value: " + reflect.ValueOf(flag.DefaultValue).String())
		}

		fmt.Println("\t" + "Allows Providing Only Flag: " + strconv.FormatBool(flag.AllowNothing))
		fmt.Println("")
	}
}
<commit_msg>Prettified output of flags drastically. No longer is unnecessarily verbose, flag descriptions will now align, and flags are organized in alphabetical order.<commit_after>// nflag is an open source alternative to Golang's flag package.
// nflag allows OS-specific flag styling, type option simplification, and allowance of empty values.

package nflag

import (
	"errors"
	"fmt"
	"os"
	"reflect"
	"runtime"
	"sort"
	"strconv"
	"strings"
)

var Config ConfigOptions
var Flags map[string]Flag
var LongestFlagLength int

// Package Init
func init() {
	Flags = make(map[string]Flag)      // Make the Flags map
	Config.OSSpecificFlags = true      // Default to being true
	Config.OSSpecificFlagString = "--" // Default to using a double dash for declaring flags

	if runtime.GOOS == "windows" { // If we are on Windows
		Config.OSSpecificFlagString = "/" // Use a slash, since Windows likes to be "special"
	}
}

// Configure
// This function is for configuration of nflag prior to usage.
func Configure(providedConfig ConfigOptions) {
	if providedConfig.OSSpecificFlags == false { // If we are overriding OSSpecificFlags
		Config.OSSpecificFlags = false
		Config.OSSpecificFlagString = "--" // Enforce --
	}

	if providedConfig.OSSpecificFlagString != "" { // If we are overriding OSSpecificFlagString
		Config.OSSpecificFlagString = providedConfig.OSSpecificFlagString // Set OSSpecificFlagString to whatever is provided on config
	}

	if providedConfig.ShowHelpIfNoArgs { // If ShowHelpIfNoArgs was set to true
		Config.ShowHelpIfNoArgs = true // Change to true
	}
}

// Get
// This function will get the flag value and returns it, or an error if the flag does not exist.
func Get(flagName string) (interface{}, error) {
	flag, exists := Flags[flagName] // Get the trimmedFlag and the exists bool of this flagName in Flags
	var val interface{}
	var err error

	if exists { // If the flag exists
		val = flag.Value
	} else {
		err = errors.New("Flag does not exist.")
	}

	return val, err
}

// #region GetAs functions

// GetAsBool
// This function will get the flag value and convert it to bool, or an error if the flag does not exist.
func GetAsBool(flagName string) (bool, error) {
	var flagValue bool
	flagValueInterface, flagValueError := Get(flagName)

	if flagValueError == nil { // If this flag does exist
		flagValue = flagValueInterface.(bool) // Type inference to bool
	}

	return flagValue, flagValueError
}

// GetAsInt
// This function will get the flag value and convert it to int, or an error if the flag does not exist.
func GetAsInt(flagName string) (int, error) {
	var flagValue int
	flagValueInterface, flagValueError := Get(flagName)

	if flagValueError == nil { // If this flag does exist
		flagValue = flagValueInterface.(int) // Type inference to int
	}

	return flagValue, flagValueError
}

// GetAsFloat64
// This function will get the flag value and convert it to float64, or an error if the flag does not exist.
func GetAsFloat64(flagName string) (float64, error) {
	var flagValue float64
	flagValueInterface, flagValueError := Get(flagName)

	if flagValueError == nil { // If this flag does exist
		flagValue = flagValueInterface.(float64) // Type inference to float64
	}

	return flagValue, flagValueError
}

// GetAsString
// This function will get the flag value and convert it to string, or an error if the flag does not exist.
func GetAsString(flagName string) (string, error) {
	var flagValue string
	flagValueInterface, flagValueError := Get(flagName)

	if flagValueError == nil { // If this flag does exist
		flagValue = flagValueInterface.(string) // Type inference to string
	}

	return flagValue, flagValueError
}

// IsDefaultValue
// This function will return a boolean as to whether or not the value of the flag is the same as DefaultValue
func IsDefaultValue(flagName string) (bool, error) {
	var isDefaultValue bool
	flagValue, flagValueError := Get(flagName)

	if flagValueError == nil { // If the flag exists
		if Flags[flagName].DefaultValue == flagValue { // If the DefaultValue is the same as the flagValue
			isDefaultValue = true // Change isDefaultValue to true
		}
	}

	return isDefaultValue, flagValueError
}

// Set
// This function is for setting a flag.
func Set(flagName string, providedFlag Flag) error {
	var errorResponse error

	if providedFlag.Type == "" { // If no Type was provided
		providedFlag.Type = "bool" // Default to bool
	}

	allowedTypes := []string{"bool", "float64", "int", "string"} // Define allowed types
	isAllowedType := false                                       // Default to providedFlag.Type not being the type

	for _, val := range allowedTypes {
		if providedFlag.Type == val { // If the Type is the same as what is allowed (this specific value)
			isAllowedType = true
		}
	}

	if isAllowedType { // If this is an allowed type
		if providedFlag.DefaultValue == nil { // If no default value was provided
			if providedFlag.Type == "bool" { // If the type is bool
				providedFlag.DefaultValue = false
			} else if providedFlag.Type == "float64" { // If the type is float64
				providedFlag.DefaultValue = 1.1
			} else if providedFlag.Type == "int" { // If the type is int
				providedFlag.DefaultValue = 1
			} else if providedFlag.Type == "string" {
				providedFlag.DefaultValue = ""
			}
		} else if (providedFlag.DefaultValue != nil) && (!providedFlag.AllowNothing) { // If a Default value was provided
			providedFlag.AllowNothing = true                                             // Enforce AllowNothing since a default value is provided
			if providedFlag.Type != reflect.TypeOf(providedFlag.DefaultValue).String() { // If the Type and DefaultValue aren't the same type
				errorResponse = errors.New("Mismatch Flag and DefaultValue types.")
			}
		}

		if errorResponse == nil { // If there was no error
			Flags[flagName] = providedFlag // Set the flag name in Flags to the providedFlag struct

			flagNameLength := len(flagName) // Get the length of the flagName

			if LongestFlagLength < flagNameLength { // If the currently stored LongestFlagLength is less than the current flagNameLength
				LongestFlagLength = (flagNameLength + 4)// Change to this flagNameLength, adding 4 for spacing for longest flag
			}
		}
	} else { // If this is a non-allowed type
		errorResponse = errors.New("Type is not allowed. Please use bool, float64, int, or string.")
	}

	return errorResponse
}

// Parse
// This function will parse input for flags
func Parse() {
	providedFlags := os.Args[1:] // ProvidedFlags start at index 1 (after binary name)

	if len(providedFlags) != 0 { // If flags were provided
		if providedFlags[0] != (Config.OSSpecificFlagString + "help") { // If the first providedFlag is not help
			for _, flag := range providedFlags { // For each flag provided
				flagNameValueSplit := strings.Split(flag, "=")                                          // Split the flag name and the value
				flagName := strings.Replace(flagNameValueSplit[0], Config.OSSpecificFlagString, "", -1) // Get the flagName and remove the OSSpecificFlagString
				flagValue := ""                                                                         // Default to nothing

				if len(flagNameValueSplit) == 2 { // If a value was provided
					flagValue = flagNameValueSplit[1] // Set to the value in flagNameValueSplit
				} else { // If the flag was passed but no value
					trimmedFlag, exists := Flags[flagName] // Get the trimmedFlag and the exists bool of this flagName in Flags

					if exists && (trimmedFlag.Type == "bool") { // If the flag exists and its type is bool
						flagValue = "true" // By passing just --flag, it implies --flag is true
					}
				}

				ParseVal(flagName, flagValue) // Parse the value
			}
		} else { // If we should output the help message
			PrintFlags() // Print the flags
			os.Exit(1)
		}
	} else if Config.ShowHelpIfNoArgs { // If no arguments are passed and ShowHelpIfNoArgs is true
		PrintFlags()
		os.Exit(1)
	}

	// Executed in the event some or no flags are passed and we haven't exited
	for flagName, _ := range Flags { // For each flagName in Flags
		if Flags[flagName].Value == nil { // If no value is set for this Flag (which happens if it wasn't parsed via
			ParseVal(flagName, "flag-not-provided")
		}
	}
}

// ParseVal
// This function will check Flags for the provided flag and parse the provided value
func ParseVal(flagName string, flagValue string) {
	trimmedFlag, exists := Flags[flagName] // Get the trimmedFlag and the exists bool of this flagName in Flags

	if exists { // If the flag exists
		if (flagValue != "") && (flagValue != "flag-not-provided") { // If a value was defined
			var conversionError error // Define conversionError as any error from not correctly converting the value to the type

			if trimmedFlag.Type == "bool" { // If the type is bool
				trimmedFlag.Value, conversionError = strconv.ParseBool(flagValue) // Convert to bool
			} else if trimmedFlag.Type == "float64" { // If the type is float64
				trimmedFlag.Value, conversionError = strconv.ParseFloat(flagValue, 64) // Convert to float64
			} else if trimmedFlag.Type == "int" { // If the type is int
				trimmedFlag.Value, conversionError = strconv.Atoi(flagValue) // Convert to int
			} else { // Remaining type, being int
				trimmedFlag.Value = flagValue
			}

			if conversionError != nil { // If there was a conversionError
				fmt.Println("An incorrect value was provided when using " + Config.OSSpecificFlagString + flagName + ". Exiting.")
				os.Exit(1)
			}
		} else { // If no value was provided by the user
			returnRequiredValueMessage := false

			if flagValue == "" { // If the flag was provided by the user but no content
				if trimmedFlag.AllowNothing { // If providing no content is permitted
					if trimmedFlag.Type == "bool" { // If the flag is bool
						trimmedFlag.Value = true // Force true
					} else { // If the flag is not bool
						trimmedFlag.Value = trimmedFlag.DefaultValue // Set to defaultvalue
					}
				} else { // If providing no content is NOT permitted
					returnRequiredValueMessage = true
				}
			} else { // If this flag was not provided
				if !trimmedFlag.Required { // If the flag is NOT required
					trimmedFlag.Value = trimmedFlag.DefaultValue // Set the value to the default
				} else { // If the flag IS required
					returnRequiredValueMessage = true
				}
			}

			if returnRequiredValueMessage {
				fmt.Println("A required value for " + Config.OSSpecificFlagString + flagName + " was not provided. Exiting.")
				os.Exit(1)
			}
		}

		Flags[flagName] = trimmedFlag // Update the map
	} else { // If the flag does not exist.
		fmt.Println(Config.OSSpecificFlagString + flagName + " does not exist. Exiting.")
		os.Exit(1)
	}
}

// PrintFlags
// This function will print all the flags that are set and their defaults
func PrintFlags() {
	fmt.Println("Usage: " + Config.OSSpecificFlagString + "example=value")
	fmt.Println("The following options are available:")

	// #region Sort Flags

	var flagNames []string

	for flagName, _ := range Flags { // For each flagName in Flags
		flagNames = append(flagNames, flagName) // Append flagName
	}

	sort.Strings(flagNames) // Sort the flagNames

	// #endregion

	for _, flagName := range flagNames { // For each flagName and trimmedFlag struct in Flags
		flag := Flags[flagName] // Get the flag

		thisFlagNameLength := len(flagName) // Get the length of this flagName
		flagLengthDiff := (LongestFlagLength - thisFlagNameLength) // Get the difference in flag name length compared to the longest one

		fmt.Println(Config.OSSpecificFlagString + flagName + strings.Repeat(" ", flagLengthDiff) + flag.Descriptor) // Output the flag, creating enough spacing to along descriptor
		if flag.DefaultValue != nil { // If DefaultValue is not nil

			if (flag.DefaultValue != "") && (flag.DefaultValue != false) { // If the default value is not an empty string and not false
				fmt.Println("Default Value: ", flag.DefaultValue)
			}
		}

		if flag.AllowNothing { // If we are allowing to pass only the flag
			fmt.Println("Allows Providing Only Flag")
		}

		fmt.Println("")
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Serge Gebhardt. All rights reserved.
//
// Use of this source code is governed by the ISC
// license that can be found in the LICENSE file.

package acd

import (
	"fmt"
	"net/http"
	"net/url"
	"reflect"

	"errors"
	"github.com/google/go-querystring/query"
)

// NodesService provides access to the nodes in the Amazon Cloud Drive API.
//
// See: https://developer.amazon.com/public/apis/experience/cloud-drive/content/nodes
type NodesService struct {
	client *Client
}

// Gets the root folder of the Amazon Cloud Drive.
func (s *NodesService) GetRoot() (*Folder, *http.Response, error) {
	opts := &NodeListOptions{Filters: "kind:FOLDER AND isRoot:true"}

	roots, resp, err := s.GetNodes(opts)
	if err != nil {
		return nil, resp, err
	}

	if len(roots) < 1 {
		return nil, resp, errors.New("No root found")
	}

	return &Folder{roots[0]}, resp, nil
}

// Gets the list of all nodes.
func (s *NodesService) GetAllNodes(opts *NodeListOptions) ([]*Node, *http.Response, error) {
	return s.listAllNodes("nodes", opts)
}

// Gets a list of nodes, up until the limit (either default or the one set in opts).
func (s *NodesService) GetNodes(opts *NodeListOptions) ([]*Node, *http.Response, error) {
	return s.listNodes("nodes", opts)
}

func (s *NodesService) listAllNodes(url string, opts *NodeListOptions) ([]*Node, *http.Response, error) {
	// Need opts to maintain state (NodeListOptions.reachedEnd)
	if opts == nil {
		opts = &NodeListOptions{}
	}

	result := make([]*Node, 0, 200)

	for {
		nodes, resp, err := s.listNodes(url, opts)
		if err != nil {
			return result, resp, err
		}
		if nodes == nil {
			break
		}

		result = append(result, nodes...)
	}

	return result, nil, nil
}

func (s *NodesService) listNodes(url string, opts *NodeListOptions) ([]*Node, *http.Response, error) {
	if opts.reachedEnd {
		return nil, nil, nil
	}

	url, err := addOptions(url, opts)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url, nil)
	if err != nil {
		return nil, nil, err
	}

	nodeList := &nodeListInternal{}
	resp, err := s.client.Do(req, nodeList)
	if err != nil {
		return nil, resp, err
	}

	if nodeList.NextToken != nil {
		opts.StartToken = *nodeList.NextToken
	} else {
		opts.reachedEnd = true
	}

	nodes := nodeList.Data
	// iterate over index since iterating over value would create a copy
	for i := range nodes {
		nodes[i].service = s
	}

	return nodes, resp, nil
}

type nodeListInternal struct {
	Count     *uint64 `json:"count"`
	NextToken *string `json:"nextToken"`
	Data      []*Node `json:"data"`
}

// Node represents a digital asset on the Amazon Cloud Drive, including files
// and folders, in a parent-child relationship. A node contains only metadata
// (e.g. folder) or it contains metadata and content (e.g. file).
type Node struct {
	Id                *string `json:"id"`
	Name              *string `json:"name"`
	Kind              *string `json:"kind"`
	ContentProperties *struct {
		Size *uint64 `json:"size"`
	} `json:"contentProperties"`
	service *NodesService
}

func (n *Node) IsFile() bool {
	return n.Kind != nil && *n.Kind == "FILE"
}

func (n *Node) IsFolder() bool {
	return n.Kind != nil && *n.Kind == "FOLDER"
}

func (n *Node) Typed() interface{} {
	if n.IsFile() {
		return &File{n}
	}

	if n.IsFolder() {
		return &Folder{n}
	}

	return n
}

// Represents a file and contains only metadata.
type File struct {
	*Node
}

// Represents a folder and contains only metadata.
type Folder struct {
	*Node
}

// Gets the list of all children.
func (f *Folder) GetAllChildren(opts *NodeListOptions) ([]*Node, *http.Response, error) {
	url := fmt.Sprintf("nodes/%s/children", *f.Id)
	return f.service.listAllNodes(url, opts)
}

// Gets a list of children, up until the limit (either default or the one set in opts).
func (f *Folder) GetChildren(opts *NodeListOptions) ([]*Node, *http.Response, error) {
	url := fmt.Sprintf("nodes/%s/children", *f.Id)
	return f.service.listNodes(url, opts)
}

// Gets the subfolder by name. It is an error if not exactly one subfolder is found.
func (f *Folder) GetFolder(name string) (*Folder, *http.Response, error) {
	n, resp, err := f.GetNode(name)
	if err != nil {
		return nil, resp, err
	}

	res, ok := n.Typed().(*Folder)
	if !ok {
		err := errors.New(fmt.Sprintf("Node '%s' is not a folder", name))
		return nil, resp, err
	}

	return res, resp, nil
}

// Gets the file by name. It is an error if not exactly one file is found.
func (f *Folder) GetFile(name string) (*File, *http.Response, error) {
	n, resp, err := f.GetNode(name)
	if err != nil {
		return nil, resp, err
	}

	res, ok := n.Typed().(*File)
	if !ok {
		err := errors.New(fmt.Sprintf("Node '%s' is not a file", name))
		return nil, resp, err
	}

	return res, resp, nil
}

// Gets the node by name. It is an error if not exactly one node is found.
func (f *Folder) GetNode(name string) (*Node, *http.Response, error) {
	filter := "parents:" + *f.Id + " AND name:" + name
	opts := &NodeListOptions{Filters: filter}

	nodes, resp, err := f.service.GetNodes(opts)
	if err != nil {
		return nil, resp, err
	}

	if len(nodes) < 1 {
		err := errors.New(fmt.Sprintf("No node '%s' found", name))
		return nil, resp, err
	}
	if len(nodes) > 1 {
		err := errors.New(fmt.Sprintf("Too many nodes '%s' found (%v)", name, len(nodes)))
		return nil, resp, err
	}

	return nodes[0], resp, nil
}

// WalkNodes walks the given node hierarchy, getting each node along the way, and returns
// the deepest node. If an error occurs, returns the furthest successful node and the list
// of HTTP responses.
func (f *Folder) WalkNodes(names ...string) (*Node, []*http.Response, error) {
	resps := make([]*http.Response, 0, len(names))

	if len(names) == 0 {
		return f.Node, resps, nil
	}

	// process each node except the last one
	fp := f
	for _, name := range names[:len(names)-1] {
		fn, resp, err := fp.GetFolder(name)
		resps = append(resps, resp)
		if err != nil {
			return fp.Node, resps, err
		}

		fp = fn
	}

	// process the last node
	nl, resp, err := fp.GetNode(names[len(names)-1])
	resps = append(resps, resp)
	if err != nil {
		return fp.Node, resps, err
	}

	return nl, resps, nil
}

// NodeListOptions holds the options when getting a list of nodes, such as the filter,
// sorting and pagination.
type NodeListOptions struct {
	Limit   uint   `url:"limit,omitempty"`
	Filters string `url:"filters,omitempty"`
	Sort    string `url:"sort,omitempty"`

	// Token where to start for next page (internal)
	StartToken string `url:"startToken,omitempty"`
	reachedEnd bool
}

// addOptions adds the parameters in opts as URL query parameters to s.  opts
// must be a struct whose fields may contain "url" tags.
func addOptions(s string, opts interface{}) (string, error) {
	v := reflect.ValueOf(opts)
	if v.Kind() == reflect.Ptr && v.IsNil() {
		return s, nil
	}

	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	qs, err := query.Values(opts)
	if err != nil {
		return s, err
	}

	u.RawQuery = qs.Encode()
	return u.String(), nil
}
<commit_msg>Iterate over value now that nodes are pointers<commit_after>// Copyright (c) 2015 Serge Gebhardt. All rights reserved.
//
// Use of this source code is governed by the ISC
// license that can be found in the LICENSE file.

package acd

import (
	"fmt"
	"net/http"
	"net/url"
	"reflect"

	"errors"
	"github.com/google/go-querystring/query"
)

// NodesService provides access to the nodes in the Amazon Cloud Drive API.
//
// See: https://developer.amazon.com/public/apis/experience/cloud-drive/content/nodes
type NodesService struct {
	client *Client
}

// Gets the root folder of the Amazon Cloud Drive.
func (s *NodesService) GetRoot() (*Folder, *http.Response, error) {
	opts := &NodeListOptions{Filters: "kind:FOLDER AND isRoot:true"}

	roots, resp, err := s.GetNodes(opts)
	if err != nil {
		return nil, resp, err
	}

	if len(roots) < 1 {
		return nil, resp, errors.New("No root found")
	}

	return &Folder{roots[0]}, resp, nil
}

// Gets the list of all nodes.
func (s *NodesService) GetAllNodes(opts *NodeListOptions) ([]*Node, *http.Response, error) {
	return s.listAllNodes("nodes", opts)
}

// Gets a list of nodes, up until the limit (either default or the one set in opts).
func (s *NodesService) GetNodes(opts *NodeListOptions) ([]*Node, *http.Response, error) {
	return s.listNodes("nodes", opts)
}

func (s *NodesService) listAllNodes(url string, opts *NodeListOptions) ([]*Node, *http.Response, error) {
	// Need opts to maintain state (NodeListOptions.reachedEnd)
	if opts == nil {
		opts = &NodeListOptions{}
	}

	result := make([]*Node, 0, 200)

	for {
		nodes, resp, err := s.listNodes(url, opts)
		if err != nil {
			return result, resp, err
		}
		if nodes == nil {
			break
		}

		result = append(result, nodes...)
	}

	return result, nil, nil
}

func (s *NodesService) listNodes(url string, opts *NodeListOptions) ([]*Node, *http.Response, error) {
	if opts.reachedEnd {
		return nil, nil, nil
	}

	url, err := addOptions(url, opts)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url, nil)
	if err != nil {
		return nil, nil, err
	}

	nodeList := &nodeListInternal{}
	resp, err := s.client.Do(req, nodeList)
	if err != nil {
		return nil, resp, err
	}

	if nodeList.NextToken != nil {
		opts.StartToken = *nodeList.NextToken
	} else {
		opts.reachedEnd = true
	}

	nodes := nodeList.Data
	for _, node := range nodes {
		node.service = s
	}

	return nodes, resp, nil
}

type nodeListInternal struct {
	Count     *uint64 `json:"count"`
	NextToken *string `json:"nextToken"`
	Data      []*Node `json:"data"`
}

// Node represents a digital asset on the Amazon Cloud Drive, including files
// and folders, in a parent-child relationship. A node contains only metadata
// (e.g. folder) or it contains metadata and content (e.g. file).
type Node struct {
	Id                *string `json:"id"`
	Name              *string `json:"name"`
	Kind              *string `json:"kind"`
	ContentProperties *struct {
		Size *uint64 `json:"size"`
	} `json:"contentProperties"`
	service *NodesService
}

func (n *Node) IsFile() bool {
	return n.Kind != nil && *n.Kind == "FILE"
}

func (n *Node) IsFolder() bool {
	return n.Kind != nil && *n.Kind == "FOLDER"
}

func (n *Node) Typed() interface{} {
	if n.IsFile() {
		return &File{n}
	}

	if n.IsFolder() {
		return &Folder{n}
	}

	return n
}

// Represents a file and contains only metadata.
type File struct {
	*Node
}

// Represents a folder and contains only metadata.
type Folder struct {
	*Node
}

// Gets the list of all children.
func (f *Folder) GetAllChildren(opts *NodeListOptions) ([]*Node, *http.Response, error) {
	url := fmt.Sprintf("nodes/%s/children", *f.Id)
	return f.service.listAllNodes(url, opts)
}

// Gets a list of children, up until the limit (either default or the one set in opts).
func (f *Folder) GetChildren(opts *NodeListOptions) ([]*Node, *http.Response, error) {
	url := fmt.Sprintf("nodes/%s/children", *f.Id)
	return f.service.listNodes(url, opts)
}

// Gets the subfolder by name. It is an error if not exactly one subfolder is found.
func (f *Folder) GetFolder(name string) (*Folder, *http.Response, error) {
	n, resp, err := f.GetNode(name)
	if err != nil {
		return nil, resp, err
	}

	res, ok := n.Typed().(*Folder)
	if !ok {
		err := errors.New(fmt.Sprintf("Node '%s' is not a folder", name))
		return nil, resp, err
	}

	return res, resp, nil
}

// Gets the file by name. It is an error if not exactly one file is found.
func (f *Folder) GetFile(name string) (*File, *http.Response, error) {
	n, resp, err := f.GetNode(name)
	if err != nil {
		return nil, resp, err
	}

	res, ok := n.Typed().(*File)
	if !ok {
		err := errors.New(fmt.Sprintf("Node '%s' is not a file", name))
		return nil, resp, err
	}

	return res, resp, nil
}

// Gets the node by name. It is an error if not exactly one node is found.
func (f *Folder) GetNode(name string) (*Node, *http.Response, error) {
	filter := "parents:" + *f.Id + " AND name:" + name
	opts := &NodeListOptions{Filters: filter}

	nodes, resp, err := f.service.GetNodes(opts)
	if err != nil {
		return nil, resp, err
	}

	if len(nodes) < 1 {
		err := errors.New(fmt.Sprintf("No node '%s' found", name))
		return nil, resp, err
	}
	if len(nodes) > 1 {
		err := errors.New(fmt.Sprintf("Too many nodes '%s' found (%v)", name, len(nodes)))
		return nil, resp, err
	}

	return nodes[0], resp, nil
}

// WalkNodes walks the given node hierarchy, getting each node along the way, and returns
// the deepest node. If an error occurs, returns the furthest successful node and the list
// of HTTP responses.
func (f *Folder) WalkNodes(names ...string) (*Node, []*http.Response, error) {
	resps := make([]*http.Response, 0, len(names))

	if len(names) == 0 {
		return f.Node, resps, nil
	}

	// process each node except the last one
	fp := f
	for _, name := range names[:len(names)-1] {
		fn, resp, err := fp.GetFolder(name)
		resps = append(resps, resp)
		if err != nil {
			return fp.Node, resps, err
		}

		fp = fn
	}

	// process the last node
	nl, resp, err := fp.GetNode(names[len(names)-1])
	resps = append(resps, resp)
	if err != nil {
		return fp.Node, resps, err
	}

	return nl, resps, nil
}

// NodeListOptions holds the options when getting a list of nodes, such as the filter,
// sorting and pagination.
type NodeListOptions struct {
	Limit   uint   `url:"limit,omitempty"`
	Filters string `url:"filters,omitempty"`
	Sort    string `url:"sort,omitempty"`

	// Token where to start for next page (internal)
	StartToken string `url:"startToken,omitempty"`
	reachedEnd bool
}

// addOptions adds the parameters in opts as URL query parameters to s.  opts
// must be a struct whose fields may contain "url" tags.
func addOptions(s string, opts interface{}) (string, error) {
	v := reflect.ValueOf(opts)
	if v.Kind() == reflect.Ptr && v.IsNil() {
		return s, nil
	}

	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	qs, err := query.Values(opts)
	if err != nil {
		return s, err
	}

	u.RawQuery = qs.Encode()
	return u.String(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"syscall"

	"github.com/golang/glog"
	"github.com/hyperhq/runv/agent"
	"github.com/hyperhq/runv/agent/proxy"
	"github.com/kardianos/osext"
	"github.com/urfave/cli"
	"google.golang.org/grpc"
)

var proxyCommand = cli.Command{
	Name:     "proxy",
	Usage:    "[internal command] proxy hyperstart API into vm",
	HideHelp: true,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "vmid",
			Usage: "the vm name",
		},
		cli.StringFlag{
			Name:  "hyperstart-ctl-sock",
			Usage: "the vm's ctl sock address to be connected",
		},
		cli.StringFlag{
			Name:  "hyperstart-stream-sock",
			Usage: "the vm's stream sock address to be connected",
		},
		cli.StringFlag{
			Name:  "proxy-hyperstart",
			Usage: "gprc sock address to be created for proxying hyperstart API",
		},
	},
	Before: func(context *cli.Context) error {
		return cmdPrepare(context, false, false)
	},
	Action: func(context *cli.Context) (err error) {
		if context.String("vmid") == "" || context.String("hyperstart-ctl-sock") == "" ||
			context.String("hyperstart-stream-sock") == "" || context.String("proxy-hyperstart") == "" {
			return err
		}
		glog.Infof("agent.NewJsonBasedHyperstart")
		h, _ := agent.NewJsonBasedHyperstart(context.String("vmid"), context.String("hyperstart-ctl-sock"), context.String("hyperstart-stream-sock"), 1, false, false)

		var s *grpc.Server
		grpcSock := context.String("proxy-hyperstart")
		glog.Infof("proxy.NewServer")
		s, err = proxy.NewServer(grpcSock, h)
		if err != nil {
			glog.Errorf("proxy.NewServer() failed with err: %#v", err)
			return err
		}
		if _, err := os.Stat(grpcSock); !os.IsNotExist(err) {
			return fmt.Errorf("%s existed, someone may be in service", grpcSock)
		}
		glog.Infof("net.Listen() to grpcsock: %s", grpcSock)
		l, err := net.Listen("unix", grpcSock)
		if err != nil {
			glog.Errorf("net.Listen() failed with err: %#v", err)
			return err
		}

		glog.Infof("proxy: grpc api on %s", grpcSock)
		if err = s.Serve(l); err != nil {
			glog.Errorf("proxy serve grpc with error: %v", err)
		}

		return err
	},
}

func createProxy(context *cli.Context, VMID, ctlSock, streamSock, grpcSock string) error {
	path, err := osext.Executable()
	if err != nil {
		return fmt.Errorf("cannot find self executable path for %s: %v", os.Args[0], err)
	}

	var cmd *exec.Cmd
	args := []string{
		"runv", "--root", context.GlobalString("root"),
	}
	if context.GlobalBool("debug") {
		args = append(args, "--debug")
	}
	if context.GlobalString("log_dir") != "" {
		args = append(args, "--log_dir", context.GlobalString("log_dir"))
	}
	args = append(args, "proxy", "--vmid", VMID, "--hyperstart-ctl-sock", ctlSock,
		"--hyperstart-stream-sock", streamSock, "--proxy-hyperstart", grpcSock)
	cmd = &exec.Cmd{
		Path: path,
		Args: args,
		Dir:  "/",
		SysProcAttr: &syscall.SysProcAttr{
			Setsid: true,
		},
	}

	glog.V(2).Infof("start proxy with argument: %v", args)
	err = cmd.Start()
	if err != nil {
		glog.Errorf("createProxy failed with err %#v", err)
		return err
	}
	glog.V(2).Infof("createProxy succeeded with proxy pid: %d", cmd.Process.Pid)

	return nil
}
<commit_msg>proxy: Wait READY message for before send messages to hyperstart<commit_after>package main

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"syscall"

	"github.com/golang/glog"
	"github.com/hyperhq/runv/agent"
	"github.com/hyperhq/runv/agent/proxy"
	"github.com/kardianos/osext"
	"github.com/urfave/cli"
	"google.golang.org/grpc"
)

var proxyCommand = cli.Command{
	Name:     "proxy",
	Usage:    "[internal command] proxy hyperstart API into vm",
	HideHelp: true,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "vmid",
			Usage: "the vm name",
		},
		cli.StringFlag{
			Name:  "hyperstart-ctl-sock",
			Usage: "the vm's ctl sock address to be connected",
		},
		cli.StringFlag{
			Name:  "hyperstart-stream-sock",
			Usage: "the vm's stream sock address to be connected",
		},
		cli.StringFlag{
			Name:  "proxy-hyperstart",
			Usage: "gprc sock address to be created for proxying hyperstart API",
		},
	},
	Before: func(context *cli.Context) error {
		return cmdPrepare(context, false, false)
	},
	Action: func(context *cli.Context) (err error) {
		if context.String("vmid") == "" || context.String("hyperstart-ctl-sock") == "" ||
			context.String("hyperstart-stream-sock") == "" || context.String("proxy-hyperstart") == "" {
			return err
		}

		glog.Infof("agent.NewJsonBasedHyperstart")
		h, _ := agent.NewJsonBasedHyperstart(context.String("vmid"), context.String("hyperstart-ctl-sock"), context.String("hyperstart-stream-sock"), 1, true, false)

		var s *grpc.Server
		grpcSock := context.String("proxy-hyperstart")
		glog.Infof("proxy.NewServer")
		s, err = proxy.NewServer(grpcSock, h)
		if err != nil {
			glog.Errorf("proxy.NewServer() failed with err: %#v", err)
			return err
		}
		if _, err := os.Stat(grpcSock); !os.IsNotExist(err) {
			return fmt.Errorf("%s existed, someone may be in service", grpcSock)
		}
		glog.Infof("net.Listen() to grpcsock: %s", grpcSock)
		l, err := net.Listen("unix", grpcSock)
		if err != nil {
			glog.Errorf("net.Listen() failed with err: %#v", err)
			return err
		}

		glog.Infof("proxy: grpc api on %s", grpcSock)
		if err = s.Serve(l); err != nil {
			glog.Errorf("proxy serve grpc with error: %v", err)
		}

		return err
	},
}

func createProxy(context *cli.Context, VMID, ctlSock, streamSock, grpcSock string) error {
	path, err := osext.Executable()
	if err != nil {
		return fmt.Errorf("cannot find self executable path for %s: %v", os.Args[0], err)
	}

	var cmd *exec.Cmd
	args := []string{
		"runv", "--root", context.GlobalString("root"),
	}
	if context.GlobalBool("debug") {
		args = append(args, "--debug")
	}
	if context.GlobalString("log_dir") != "" {
		args = append(args, "--log_dir", context.GlobalString("log_dir"))
	}
	args = append(args, "proxy", "--vmid", VMID, "--hyperstart-ctl-sock", ctlSock,
		"--hyperstart-stream-sock", streamSock, "--proxy-hyperstart", grpcSock)
	cmd = &exec.Cmd{
		Path: path,
		Args: args,
		Dir:  "/",
		SysProcAttr: &syscall.SysProcAttr{
			Setsid: true,
		},
	}

	glog.V(2).Infof("start proxy with argument: %v", args)
	err = cmd.Start()
	if err != nil {
		glog.Errorf("createProxy failed with err %#v", err)
		return err
	}
	glog.V(2).Infof("createProxy succeeded with proxy pid: %d", cmd.Process.Pid)

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package client

import (
	"bytes"
	"camli/blobref"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"http"
	"io"
	"json"
	"log"
	"os"
	"strings"
)

type UploadHandle struct {
	BlobRef  *blobref.BlobRef
	Size     int64
	Contents io.Reader
}

type PutResult struct {
	BlobRef  *blobref.BlobRef
	Size     int64
	Skipped  bool    // already present on blobserver
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() os.Error { return nil }

// Note: must not touch data after calling this.
func NewUploadHandleFromString(data string) *UploadHandle {
	s1 := sha1.New()
	s1.Write([]byte(data))
	bref := blobref.FromHash("sha1", s1)
	buf := bytes.NewBufferString(data)
	return &UploadHandle{BlobRef: bref, Size: int64(len(data)), Contents: buf}
}

func encodeBase64(s string) string {
	buf := make([]byte, base64.StdEncoding.EncodedLen(len(s)))
	base64.StdEncoding.Encode(buf, []byte(s))
	return string(buf)
}

func (c *Client) jsonFromResponse(resp *http.Response) (map[string]interface{}, os.Error) {
	if resp.StatusCode != 200 {
		log.Printf("Failed to JSON from response; status code is %d", resp.StatusCode)
		io.Copy(os.Stderr, resp.Body)
		return nil, os.NewError(fmt.Sprintf("HTTP response code is %d; no JSON to parse.", resp.StatusCode))
	}
	// TODO: LimitReader here for paranoia
	buf := new(bytes.Buffer)
	io.Copy(buf, resp.Body)
	resp.Body.Close()
	jmap := make(map[string]interface{})
	if jerr := json.Unmarshal(buf.Bytes(), &jmap); jerr != nil {
		return nil, jerr
	}
	return jmap, nil
}

func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) {
	error := func(msg string, e os.Error) (*PutResult, os.Error) {
		err := os.NewError(fmt.Sprintf("Error uploading blob %s: %s; err=%s",
			h.BlobRef, msg, e))
		c.log.Print(err.String())
		return nil, err
	}

	c.statsMutex.Lock()
	c.stats.UploadRequests.Blobs++
	c.stats.UploadRequests.Bytes += h.Size
	c.statsMutex.Unlock()

	blobRefString := h.BlobRef.String()

	// Pre-upload.  Check whether the blob already exists on the
	// server and if not, the URL to upload it to.
	url := fmt.Sprintf("%s/camli/stat", c.server)
	requestBody := "camliversion=1&blob1="+blobRefString
	req := c.newRequest("POST", url)
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Body = &nopCloser{strings.NewReader(requestBody)}
	req.ContentLength = int64(len(requestBody))
	req.TransferEncoding = nil

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return error("stat http error", err)
	}

	pur, err := c.jsonFromResponse(resp)
	if err != nil {
		return error("stat json parse error", err)
	}

	uploadUrl, ok := pur["uploadUrl"].(string)
	if uploadUrl == "" {
		return error("stat json validity error: no 'uploadUrl'", nil)
	}
	log.Printf("Got upload url: %q", uploadUrl)

	alreadyHave, ok := pur["stat"].([]interface{})
	if !ok {
		return error("stat json validity error: no 'stat'", nil)
	}

	pr := &PutResult{BlobRef: h.BlobRef, Size: h.Size}

	for _, haveObj := range alreadyHave {
		haveObj := haveObj.(map[string]interface{})
		if haveObj["blobRef"].(string) == h.BlobRef.String() {
			pr.Skipped = true
			return pr, nil
		}
	}

	// TODO: use a proper random boundary
	boundary := "sdf8sd8f7s9df9s7df9sd7sdf9s879vs7d8v7sd8v7sd8v"

	multiPartHeader := fmt.Sprintf(
		                "--%s\r\nContent-Type: application/octet-stream\r\n" +
		                "Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"\r\n\r\n",
				boundary,
				h.BlobRef, h.BlobRef)
	multiPartFooter := "\r\n--"+boundary+"--\r\n"

	c.log.Printf("Uploading to URL: %s", uploadUrl)
	req = c.newRequest("POST", uploadUrl)
	req.Header.Set("Content-Type", "multipart/form-data; boundary="+boundary)
	req.Body = &nopCloser{io.MultiReader(
		strings.NewReader(multiPartHeader),
		h.Contents,
			strings.NewReader(multiPartFooter))}

	req.ContentLength = int64(len(multiPartHeader)) + h.Size + int64(len(multiPartFooter))
	req.TransferEncoding = nil
	resp, err = http.DefaultClient.Do(req)
	if err != nil {
		return error("upload http error", err)
	}

	// The only valid HTTP responses are 200 and 303.
	if resp.StatusCode != 200 && resp.StatusCode != 303 {
		return error(fmt.Sprintf("invalid http response %d in upload response", resp.StatusCode), nil)
	}

	if resp.StatusCode == 303 {
		otherLocation := resp.Header.Get("Location")
		if otherLocation == "" {
			return error("303 without a Location", nil)
		}
		baseUrl, _ := http.ParseURL(uploadUrl)
		absUrl, err := baseUrl.ParseURL(otherLocation)
		if err != nil {
			return error("303 Location URL relative resolve error", err)
		}
		otherLocation = absUrl.String()
		resp, _, err = http.Get(otherLocation)
		if err != nil {
			return error("error following 303 redirect after upload", err)
		}
	}

	ures, err := c.jsonFromResponse(resp)
	if err != nil {
		return error("json parse from upload error", err)
	}

	errorText, ok := ures["errorText"].(string)
	if ok {
		c.log.Printf("Blob server reports error: %s", errorText)
	}

	received, ok := ures["received"].([]interface{})
	if !ok {
		return error("upload json validity error: no 'received'", nil)
	}

	for _, rit := range received {
		it, ok := rit.(map[string]interface{})
		if !ok {
			return error("upload json validity error: 'received' is malformed", nil)
		}
		if it["blobRef"] == blobRefString {
			switch size := it["size"].(type) {
			case nil:
				return error("upload json validity error: 'received' is missing 'size'", nil)
			case float64:
				if int64(size) == h.Size {
					// Success!
					c.statsMutex.Lock()
					c.stats.Uploads.Blobs++
					c.stats.Uploads.Bytes += h.Size
					c.statsMutex.Unlock()
					return pr, nil
				} else {
					return error(fmt.Sprintf("Server got blob, but reports wrong length (%v; expected %d)",
						size, h.Size), nil)
				}
			default:
				return error("unsupported type of 'size' in received response", nil)
			}
		}
	}

	return nil, os.NewError("Server didn't receive blob.")
}
<commit_msg>use fmt.Errorf instead of os.NewError(fmt.Sprintf(..))<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package client

import (
	"bytes"
	"camli/blobref"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"http"
	"io"
	"json"
	"log"
	"os"
	"strings"
)

type UploadHandle struct {
	BlobRef  *blobref.BlobRef
	Size     int64
	Contents io.Reader
}

type PutResult struct {
	BlobRef  *blobref.BlobRef
	Size     int64
	Skipped  bool    // already present on blobserver
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() os.Error { return nil }

// Note: must not touch data after calling this.
func NewUploadHandleFromString(data string) *UploadHandle {
	s1 := sha1.New()
	s1.Write([]byte(data))
	bref := blobref.FromHash("sha1", s1)
	buf := bytes.NewBufferString(data)
	return &UploadHandle{BlobRef: bref, Size: int64(len(data)), Contents: buf}
}

func encodeBase64(s string) string {
	buf := make([]byte, base64.StdEncoding.EncodedLen(len(s)))
	base64.StdEncoding.Encode(buf, []byte(s))
	return string(buf)
}

func (c *Client) jsonFromResponse(resp *http.Response) (map[string]interface{}, os.Error) {
	if resp.StatusCode != 200 {
		log.Printf("Failed to JSON from response; status code is %d", resp.StatusCode)
		io.Copy(os.Stderr, resp.Body)
		return nil, os.NewError(fmt.Sprintf("HTTP response code is %d; no JSON to parse.", resp.StatusCode))
	}
	// TODO: LimitReader here for paranoia
	buf := new(bytes.Buffer)
	io.Copy(buf, resp.Body)
	resp.Body.Close()
	jmap := make(map[string]interface{})
	if jerr := json.Unmarshal(buf.Bytes(), &jmap); jerr != nil {
		return nil, jerr
	}
	return jmap, nil
}

func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) {
	error := func(msg string, e os.Error) (*PutResult, os.Error) {
		err := fmt.Errorf("Error uploading blob %s: %s; err=%v", h.BlobRef, msg, e)
		c.log.Print(err.String())
		return nil, err
	}

	c.statsMutex.Lock()
	c.stats.UploadRequests.Blobs++
	c.stats.UploadRequests.Bytes += h.Size
	c.statsMutex.Unlock()

	blobRefString := h.BlobRef.String()

	// Pre-upload.  Check whether the blob already exists on the
	// server and if not, the URL to upload it to.
	url := fmt.Sprintf("%s/camli/stat", c.server)
	requestBody := "camliversion=1&blob1="+blobRefString
	req := c.newRequest("POST", url)
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Body = &nopCloser{strings.NewReader(requestBody)}
	req.ContentLength = int64(len(requestBody))
	req.TransferEncoding = nil

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return error("stat http error", err)
	}

	pur, err := c.jsonFromResponse(resp)
	if err != nil {
		return error("stat json parse error", err)
	}

	uploadUrl, ok := pur["uploadUrl"].(string)
	if uploadUrl == "" {
		return error("stat json validity error: no 'uploadUrl'", nil)
	}
	log.Printf("Got upload url: %q", uploadUrl)

	alreadyHave, ok := pur["stat"].([]interface{})
	if !ok {
		return error("stat json validity error: no 'stat'", nil)
	}

	pr := &PutResult{BlobRef: h.BlobRef, Size: h.Size}

	for _, haveObj := range alreadyHave {
		haveObj := haveObj.(map[string]interface{})
		if haveObj["blobRef"].(string) == h.BlobRef.String() {
			pr.Skipped = true
			return pr, nil
		}
	}

	// TODO: use a proper random boundary
	boundary := "sdf8sd8f7s9df9s7df9sd7sdf9s879vs7d8v7sd8v7sd8v"

	multiPartHeader := fmt.Sprintf(
		                "--%s\r\nContent-Type: application/octet-stream\r\n" +
		                "Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"\r\n\r\n",
				boundary,
				h.BlobRef, h.BlobRef)
	multiPartFooter := "\r\n--"+boundary+"--\r\n"

	c.log.Printf("Uploading to URL: %s", uploadUrl)
	req = c.newRequest("POST", uploadUrl)
	req.Header.Set("Content-Type", "multipart/form-data; boundary="+boundary)
	req.Body = &nopCloser{io.MultiReader(
		strings.NewReader(multiPartHeader),
		h.Contents,
			strings.NewReader(multiPartFooter))}

	req.ContentLength = int64(len(multiPartHeader)) + h.Size + int64(len(multiPartFooter))
	req.TransferEncoding = nil
	resp, err = http.DefaultClient.Do(req)
	if err != nil {
		return error("upload http error", err)
	}

	// The only valid HTTP responses are 200 and 303.
	if resp.StatusCode != 200 && resp.StatusCode != 303 {
		return error(fmt.Sprintf("invalid http response %d in upload response", resp.StatusCode), nil)
	}

	if resp.StatusCode == 303 {
		otherLocation := resp.Header.Get("Location")
		if otherLocation == "" {
			return error("303 without a Location", nil)
		}
		baseUrl, _ := http.ParseURL(uploadUrl)
		absUrl, err := baseUrl.ParseURL(otherLocation)
		if err != nil {
			return error("303 Location URL relative resolve error", err)
		}
		otherLocation = absUrl.String()
		resp, _, err = http.Get(otherLocation)
		if err != nil {
			return error("error following 303 redirect after upload", err)
		}
	}

	ures, err := c.jsonFromResponse(resp)
	if err != nil {
		return error("json parse from upload error", err)
	}

	errorText, ok := ures["errorText"].(string)
	if ok {
		c.log.Printf("Blob server reports error: %s", errorText)
	}

	received, ok := ures["received"].([]interface{})
	if !ok {
		return error("upload json validity error: no 'received'", nil)
	}

	for _, rit := range received {
		it, ok := rit.(map[string]interface{})
		if !ok {
			return error("upload json validity error: 'received' is malformed", nil)
		}
		if it["blobRef"] == blobRefString {
			switch size := it["size"].(type) {
			case nil:
				return error("upload json validity error: 'received' is missing 'size'", nil)
			case float64:
				if int64(size) == h.Size {
					// Success!
					c.statsMutex.Lock()
					c.stats.Uploads.Blobs++
					c.stats.Uploads.Bytes += h.Size
					c.statsMutex.Unlock()
					return pr, nil
				} else {
					return error(fmt.Sprintf("Server got blob, but reports wrong length (%v; expected %d)",
						size, h.Size), nil)
				}
			default:
				return error("unsupported type of 'size' in received response", nil)
			}
		}
	}

	return nil, os.NewError("Server didn't receive blob.")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"net"
	"time"
	//    "syscall"
	//"golang.org/x/net/ipv4"
)

type IP_Reader struct {
	incomingPackets chan []byte
	nr              *Network_Reader
	protocol        uint8
	ip              string
	fragBuf         map[string](chan []byte)
}

func (nr *Network_Reader) NewIP_Reader(ip string, protocol uint8) (*IP_Reader, error) {
	c, err := nr.bind(ip, protocol)
	if err != nil {
		return nil, err
	}

	return &IP_Reader{
		incomingPackets: c,
		nr:              nr,
		protocol:        protocol,
		ip:              ip,
		fragBuf:         make(map[string](chan []byte)),
	}, nil
}

func slicePacket(b []byte) (hrd, payload []byte) {
	hdrLen := int(b[0]&0x0f) * 4
	//fmt.Println("HdrLen: ", hdrLen)
	return b[:hdrLen], b[hdrLen:]
}

const FRAGMENT_TIMEOUT = 15

func (ipr *IP_Reader) ReadFrom() (ip string, b, payload []byte, e error) {
	fmt.Println("STARTING READ")
	b = <-ipr.incomingPackets
	fmt.Println("RAW READ COMPLETED")
	//fmt.Println("Read Length: ", len(b))
	//fmt.Println("Full Read Data: ", b)

	ip = net.IPv4(b[12], b[13], b[14], b[15]).String()
	hdr, p := slicePacket(b)

	packetOffset := uint16(hdr[6]&0x1f)<<8 + uint16(hdr[7])
	fmt.Println("PACK OFF", packetOffset, "HEADER FLAGS", (hdr[6] >> 5))
	if ((hdr[6]>>5)&0x01 == 0) && (packetOffset == 0) { // if not fragment
		// verify checksum
		if calcChecksum(hdr, false) != 0 {
			//fmt.Println("Header checksum verification failed. Packet dropped.")
			//fmt.Println("Wrong header: ", hdr)
			//fmt.Println("Payload (dropped): ", p)
			return "", nil, nil, errors.New("Header checksum incorrect, packet dropped")
		}

		//fmt.Println("Payload Length: ", len(p))
		//fmt.Println("Full payload: ", p)
		fmt.Println("PACKET COMPLETELY READ")
		return ip, b, p, nil
	} else {
		bufID := string([]byte{hdr[12], hdr[13], hdr[14], hdr[15], // the source IP
			hdr[16], hdr[17], hdr[18], hdr[19], // the destination IP
			hdr[9],         // protocol
			hdr[4], hdr[5], // identification
		})

		if c, ok := ipr.fragBuf[bufID]; ok {
			// the fragment has already started
			go func() { c <- b }()
		} else {
			// create the fragment buffer
			ipr.fragBuf[bufID] = make(chan []byte)

			// create the packet assembler in a goroutine to allow the program to continue
			go func(in <-chan []byte, finished chan<- []byte) {
				payload := <-in
				extraFrags := make(map[uint64]([]byte))
				//goalLen := int64(-1)
				t := time.Now()
				for time.Since(t).Seconds() <= FRAGMENT_TIMEOUT {
					select {
					case frag := <-in:
						hdr, p := slicePacket(frag)
						offset := 8 * (uint64(hdr[6]&0x1f)<<8 + uint64(hdr[7]))
						if (hdr[6]>>5)&0x01 == 0 {
							//totalLen := uint64(hdr[2]<<8 + hdr[3])
							//goalLen = int64(offset + totalLen)
						}
						fmt.Println("RECEIVED FRAG")
						fmt.Println("Offset:", offset)
						fmt.Println(len(payload))
						if offset == uint64(len(payload)) {
							payload = append(payload, p...)
							for storedFrag, found := extraFrags[uint64(len(payload))]; found; {
								delete(extraFrags, uint64(len(payload)))
								payload = append(payload, storedFrag...)
							}
							if (hdr[6]>>5)&0x01 == 0 {
								fullPacketHdr := hdr
								totalLen := uint16(fullPacketHdr[0]&0x0F)*4 + uint16(len(payload))
								fullPacketHdr[2] = byte(totalLen >> 8)
								fullPacketHdr[3] = byte(totalLen)
								fullPacketHdr[6] = 0
								fullPacketHdr[7] = 0

								// send the packet back into processing
								go func() { finished <- append(fullPacketHdr, payload...); fmt.Println("FINISHED") }()
								fmt.Println("Just wrote back in")
								return
							}
						} else {
							extraFrags[8*(uint64(p[6])<<3>>11+uint64(p[7]))] = p
						}
					default:
						// make the timeout actually have a chance of being hit
					}
				}

				// drop the packet upon timeout
				fmt.Println(errors.New("Fragments took too long, packet dropped"))
				return
			}(ipr.fragBuf[bufID], ipr.incomingPackets)

			// send in the first fragment
			ipr.fragBuf[bufID] <- p

			// TODO: Remove the fragment buffer after some time
		}

		// after dealing with the fragment, try reading again
		return ipr.ReadFrom()
	}
}

func (ipr *IP_Reader) Close() error {
	return ipr.nr.unbind(ipr.ip, ipr.protocol)
}

/* h := &ipv4.Header{
	Version:  ipv4.Version,      // protocol version
	Len:      20,                // header length
	TOS:      0,                 // type-of-service (0 is everything normal)
	TotalLen: len(x) + 20,       // packet total length (octets)
	ID:       0,                 // identification
	Flags:    ipv4.DontFragment, // flags
	FragOff:  0,                 // fragment offset
	TTL:      8,                 // time-to-live (maximum lifespan in seconds)
	Protocol: 17,                // next protocol (17 is UDP)
	Checksum: 0,                 // checksum (apparently autocomputed)
	//Src:    net.IPv4(127, 0, 0, 1), // source address, apparently done automatically
	Dst: net.ParseIP(c.manager.ipAddress), // destination address
	//Options                         // options, extension headers
}
*/
<commit_msg>Also made it print out the packet... It definitely is receiving.<commit_after>package main

import (
	"errors"
	"fmt"
	"net"
	"time"
	//    "syscall"
	//"golang.org/x/net/ipv4"
)

type IP_Reader struct {
	incomingPackets chan []byte
	nr              *Network_Reader
	protocol        uint8
	ip              string
	fragBuf         map[string](chan []byte)
}

func (nr *Network_Reader) NewIP_Reader(ip string, protocol uint8) (*IP_Reader, error) {
	c, err := nr.bind(ip, protocol)
	if err != nil {
		return nil, err
	}

	return &IP_Reader{
		incomingPackets: c,
		nr:              nr,
		protocol:        protocol,
		ip:              ip,
		fragBuf:         make(map[string](chan []byte)),
	}, nil
}

func slicePacket(b []byte) (hrd, payload []byte) {
	hdrLen := int(b[0]&0x0f) * 4
	//fmt.Println("HdrLen: ", hdrLen)
	return b[:hdrLen], b[hdrLen:]
}

const FRAGMENT_TIMEOUT = 15

func (ipr *IP_Reader) ReadFrom() (ip string, b, payload []byte, e error) {
	fmt.Println("STARTING READ")
	b = <-ipr.incomingPackets
	fmt.Println("RAW READ COMPLETED")
	//fmt.Println("Read Length: ", len(b))
	//fmt.Println("Full Read Data: ", b)

	ip = net.IPv4(b[12], b[13], b[14], b[15]).String()
	hdr, p := slicePacket(b)

	packetOffset := uint16(hdr[6]&0x1f)<<8 + uint16(hdr[7])
	fmt.Println("PACK OFF", packetOffset, "HEADER FLAGS", (hdr[6] >> 5))
	if ((hdr[6]>>5)&0x01 == 0) && (packetOffset == 0) { // if not fragment
		// verify checksum
		if calcChecksum(hdr, false) != 0 {
			//fmt.Println("Header checksum verification failed. Packet dropped.")
			//fmt.Println("Wrong header: ", hdr)
			//fmt.Println("Payload (dropped): ", p)
			return "", nil, nil, errors.New("Header checksum incorrect, packet dropped")
		}

		//fmt.Println("Payload Length: ", len(p))
		//fmt.Println("Full payload: ", p)
		fmt.Println("PACKET COMPLETELY READ")
		return ip, b, p, nil
	} else {
		bufID := string([]byte{hdr[12], hdr[13], hdr[14], hdr[15], // the source IP
			hdr[16], hdr[17], hdr[18], hdr[19], // the destination IP
			hdr[9],         // protocol
			hdr[4], hdr[5], // identification
		})

		if c, ok := ipr.fragBuf[bufID]; ok {
			// the fragment has already started
			go func() { c <- b }()
		} else {
			// create the fragment buffer
			ipr.fragBuf[bufID] = make(chan []byte)

			// create the packet assembler in a goroutine to allow the program to continue
			go func(in <-chan []byte, finished chan<- []byte) {
				payload := <-in
				extraFrags := make(map[uint64]([]byte))
				//goalLen := int64(-1)
				t := time.Now()
				for time.Since(t).Seconds() <= FRAGMENT_TIMEOUT {
					select {
					case frag := <-in:
						hdr, p := slicePacket(frag)
						offset := 8 * (uint64(hdr[6]&0x1f)<<8 + uint64(hdr[7]))
						if (hdr[6]>>5)&0x01 == 0 {
							//totalLen := uint64(hdr[2]<<8 + hdr[3])
							//goalLen = int64(offset + totalLen)
						}
						fmt.Println("RECEIVED FRAG")
						fmt.Println("Offset:", offset)
						fmt.Println(len(payload))
						if offset == uint64(len(payload)) {
							payload = append(payload, p...)
							for storedFrag, found := extraFrags[uint64(len(payload))]; found; {
								delete(extraFrags, uint64(len(payload)))
								payload = append(payload, storedFrag...)
							}
							if (hdr[6]>>5)&0x01 == 0 {
								fullPacketHdr := hdr
								totalLen := uint16(fullPacketHdr[0]&0x0F)*4 + uint16(len(payload))
								fullPacketHdr[2] = byte(totalLen >> 8)
								fullPacketHdr[3] = byte(totalLen)
								fullPacketHdr[6] = 0
								fullPacketHdr[7] = 0

								// send the packet back into processing
								go func() {
									finished <- append(fullPacketHdr, payload...)
									fmt.Println("FINISHED")
									fmt.Println(append(fullPacketHdr, payload...))
								}()
								fmt.Println("Just wrote back in")
								return
							}
						} else {
							extraFrags[8*(uint64(p[6])<<3>>11+uint64(p[7]))] = p
						}
					default:
						// make the timeout actually have a chance of being hit
					}
				}

				// drop the packet upon timeout
				fmt.Println(errors.New("Fragments took too long, packet dropped"))
				return
			}(ipr.fragBuf[bufID], ipr.incomingPackets)

			// send in the first fragment
			ipr.fragBuf[bufID] <- p

			// TODO: Remove the fragment buffer after some time
		}

		// after dealing with the fragment, try reading again
		return ipr.ReadFrom()
	}
}

func (ipr *IP_Reader) Close() error {
	return ipr.nr.unbind(ipr.ip, ipr.protocol)
}

/* h := &ipv4.Header{
	Version:  ipv4.Version,      // protocol version
	Len:      20,                // header length
	TOS:      0,                 // type-of-service (0 is everything normal)
	TotalLen: len(x) + 20,       // packet total length (octets)
	ID:       0,                 // identification
	Flags:    ipv4.DontFragment, // flags
	FragOff:  0,                 // fragment offset
	TTL:      8,                 // time-to-live (maximum lifespan in seconds)
	Protocol: 17,                // next protocol (17 is UDP)
	Checksum: 0,                 // checksum (apparently autocomputed)
	//Src:    net.IPv4(127, 0, 0, 1), // source address, apparently done automatically
	Dst: net.ParseIP(c.manager.ipAddress), // destination address
	//Options                         // options, extension headers
}
*/
<|endoftext|>