<commit_before>/*
    MP3Cat is a simple command line utility for concatenating MP3 files
    without re-encoding. It supports both constant bit rate (CBR) and
    variable bit rate (VBR) files.
*/
package main

import (
    "fmt"
    "io"
    "os"
    "path/filepath"
    "github.com/dmulholland/mp3lib"
    "github.com/dmulholland/clio/go/clio"
)

// Application version number.
const version = "2.2.0"

// Command line help text.
var helptext = fmt.Sprintf(`
Usage: %s [FLAGS] [OPTIONS] ARGUMENTS

  Concatenates MP3 files without re-encoding. Supports both constant bit rate
  (CBR) and variable bit rate (VBR) files. Strips ID3 tags and garbage data
  from the output.

Arguments:
  <files>           List of input files to merge.

Options:
  -o, --out <file>  Output filename. Defaults to 'output.mp3'.

Flags:
  -f, --force       Overwrite an existing output file.
      --help        Display this help text and exit.
  -v, --verbose     Report progress.
      --version     Display the application's version number and exit.
`, filepath.Base(os.Args[0]))

// Application entry point.
func main() {

    // Initialize an argument parser.
    parser := clio.NewParser(helptext, version)

    // Register flags.
    parser.AddFlag("force", 'f')
    parser.AddFlag("verbose", 'v')
    parser.AddFlag("debug")

    // Register options.
    parser.AddStrOpt("out", "output.mp3", 'o')

    // Parse the command line arguments.
    parser.Parse()

    // Make sure we have a list of input files.
    if !parser.HasArgs() {
        fmt.Fprintln(os.Stderr, "Error: you must supply a list of files to merge.")
        os.Exit(1)
    }

    // Set debug mode if the user supplied a --debug flag.
    if parser.GetFlag("debug") {
        mp3lib.DebugMode = true
    }

    // Merge the input files.
    mergeFiles(
        parser.GetStrOpt("out"),
        parser.GetArgs(),
        parser.GetFlag("force"),
        parser.GetFlag("verbose"))
}

// Create a new file at the specified output path containing the merged
// contents of the list of input files.
func mergeFiles(outputPath string, inputPaths []string, overwrite bool, verbose bool) {

    var totalFrames uint32
    var totalBytes uint32
    var totalFiles int
    var firstBitRate int
    var isVBR bool

    // Only overwrite an existing file if the --force flag has been used.
    if _, err := os.Stat(outputPath); err == nil {
        if !overwrite {
            fmt.Fprintf(os.Stderr, "Error: the file '%v' already exists. ", outputPath)
            fmt.Fprintf(os.Stderr, "Use --force to overwrite it.\n")
            os.Exit(1)
        }
    }

    // If the list of input files includes the output file we'll end up in an
    // infinite loop.
    for _, filepath := range inputPaths {
        if filepath == outputPath {
            fmt.Fprintln(os.Stderr, "Error: the list of input files includes the output file.")
            os.Exit(1)
        }
    }

    // Create the output file.
    outputFile, err := os.Create(outputPath)
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }

    // Loop over the input files and append their MP3 frames to the output file.
    for _, filepath := range inputPaths {

        if verbose {
            fmt.Println("Merging:", filepath)
        }

        inputFile, err := os.Open(filepath)
        if err != nil {
            fmt.Fprintln(os.Stderr, err)
            os.Exit(1)
        }

        isFirstFrame := true

        for {

            // Read the next frame from the input file.
            frame := mp3lib.NextFrame(inputFile)
            if frame == nil {
                break
            }

            // Skip the first frame if it's a VBR header.
            if isFirstFrame {
                isFirstFrame = false
                if mp3lib.IsXingHeader(frame) || mp3lib.IsVbriHeader(frame) {
                    continue
                }
            }

            // If we detect more than one bitrate we'll need to add a VBR
            // header to the output file.
            if firstBitRate == 0 {
                firstBitRate = frame.BitRate
            } else if frame.BitRate != firstBitRate {
                isVBR = true
            }

            // Write the frame to the output file.
            _, err := outputFile.Write(frame.RawBytes)
            if err != nil {
                fmt.Fprintln(os.Stderr, err)
                os.Exit(1)
            }

            totalFrames += 1
            totalBytes += uint32(len(frame.RawBytes))
        }

        inputFile.Close()
        totalFiles += 1
    }

    outputFile.Close()

    // If we detected multiple bitrates, prepend a VBR header to the file.
    if isVBR {
        if verbose {
            fmt.Println("VBR data detected. Adding Xing header.")
        }
        addXingHeader(outputPath, totalFrames, totalBytes)
    }

    if verbose {
        fmt.Printf("%v files merged.\n", totalFiles)
    }
}

// Prepend an Xing VBR header to the specified MP3 file.
func addXingHeader(filepath string, totalFrames, totalBytes uint32) {

    outputFile, err := os.Create(filepath + ".mp3cat.tmp")
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }

    inputFile, err := os.Open(filepath)
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }

    templateFrame := mp3lib.NextFrame(inputFile)
    inputFile.Seek(0, 0)

    xingHeader := mp3lib.NewXingHeader(templateFrame, totalFrames, totalBytes)

    _, err = outputFile.Write(xingHeader.RawBytes)
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }

    _, err = io.Copy(outputFile, inputFile)
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }

    outputFile.Close()
    inputFile.Close()

    err = os.Remove(filepath)
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }

    err = os.Rename(filepath + ".mp3cat.tmp", filepath)
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }
}
<commit_msg>Refactor arg parsing code for Clio 2.0<commit_after>/*
    MP3Cat is a fast command line utility for concatenating MP3 files
    without re-encoding. It supports both constant bit rate (CBR) and
    variable bit rate (VBR) files.
*/
package main

import (
    "fmt"
    "io"
    "os"
    "path/filepath"
    "github.com/dmulholland/mp3lib"
    "github.com/dmulholland/clio/go/clio"
)

// Application version number.
const version = "2.2.0"

// Command line help text.
var helptext = fmt.Sprintf(`
Usage: %s [FLAGS] [OPTIONS] ARGUMENTS

  Concatenates MP3 files without re-encoding. Supports both constant bit rate
  (CBR) and variable bit rate (VBR) files. Strips ID3 tags and garbage data
  from the output.

Arguments:
  <files>           List of input files to merge.

Options:
  -o, --out <file>  Output filename. Defaults to 'output.mp3'.

Flags:
  -f, --force       Overwrite an existing output file.
      --help        Display this help text and exit.
  -v, --verbose     Report progress.
      --version     Display the application's version number and exit.
`, filepath.Base(os.Args[0]))

// Application entry point.
func main() {

    // Initialize an argument parser.
    parser := clio.NewParser(helptext, version)

    // Register flags.
    parser.AddFlag("force f")
    parser.AddFlag("verbose v")
    parser.AddFlag("debug d")

    // Register options.
    parser.AddStr("out o", "output.mp3")

    // Parse the command line arguments.
    parser.Parse()

    // Make sure we have a list of input files.
    if !parser.HasArgs() {
        fmt.Fprintln(os.Stderr, "Error: you must supply a list of files to merge.")
        os.Exit(1)
    }

    // Set debug mode if the user supplied a --debug flag.
    if parser.GetFlag("debug") {
        mp3lib.DebugMode = true
    }

    // Merge the input files.
    mergeFiles(
        parser.GetStr("out"),
        parser.GetArgs(),
        parser.GetFlag("force"),
        parser.GetFlag("verbose"))
}

// Create a new file at the specified output path containing the merged
// contents of the list of input files.
func mergeFiles(outputPath string, inputPaths []string, force, verbose bool) {

    var totalFrames uint32
    var totalBytes uint32
    var totalFiles int
    var firstBitRate int
    var isVBR bool

    // Only overwrite an existing file if the --force flag has been used.
    if _, err := os.Stat(outputPath); err == nil {
        if !force {
            fmt.Fprintf(os.Stderr, "Error: the file '%v' already exists. ", outputPath)
            fmt.Fprintf(os.Stderr, "Use --force to overwrite it.\n")
            os.Exit(1)
        }
    }

    // If the list of input files includes the output file we'll end up in an
    // infinite loop.
    for _, filepath := range inputPaths {
        if filepath == outputPath {
            fmt.Fprintln(os.Stderr, "Error: the list of input files includes the output file.")
            os.Exit(1)
        }
    }

    // Create the output file.
    outputFile, err := os.Create(outputPath)
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }

    // Loop over the input files and append their MP3 frames to the output file.
    for _, filepath := range inputPaths {

        if verbose {
            fmt.Println("Merging:", filepath)
        }

        inputFile, err := os.Open(filepath)
        if err != nil {
            fmt.Fprintln(os.Stderr, err)
            os.Exit(1)
        }

        isFirstFrame := true

        for {

            // Read the next frame from the input file.
            frame := mp3lib.NextFrame(inputFile)
            if frame == nil {
                break
            }

            // Skip the first frame if it's a VBR header.
            if isFirstFrame {
                isFirstFrame = false
                if mp3lib.IsXingHeader(frame) || mp3lib.IsVbriHeader(frame) {
                    continue
                }
            }

            // If we detect more than one bitrate we'll need to add a VBR
            // header to the output file.
            if firstBitRate == 0 {
                firstBitRate = frame.BitRate
            } else if frame.BitRate != firstBitRate {
                isVBR = true
            }

            // Write the frame to the output file.
            _, err := outputFile.Write(frame.RawBytes)
            if err != nil {
                fmt.Fprintln(os.Stderr, err)
                os.Exit(1)
            }

            totalFrames += 1
            totalBytes += uint32(len(frame.RawBytes))
        }

        inputFile.Close()
        totalFiles += 1
    }

    outputFile.Close()

    // If we detected multiple bitrates, prepend a VBR header to the file.
    if isVBR {
        if verbose {
            fmt.Println("VBR data detected. Adding Xing header.")
        }
        addXingHeader(outputPath, totalFrames, totalBytes)
    }

    if verbose {
        fmt.Printf("%v files merged.\n", totalFiles)
    }
}

// Prepend an Xing VBR header to the specified MP3 file.
func addXingHeader(filepath string, totalFrames, totalBytes uint32) {

    outputFile, err := os.Create(filepath + ".mp3cat.tmp")
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }

    inputFile, err := os.Open(filepath)
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }

    templateFrame := mp3lib.NextFrame(inputFile)
    inputFile.Seek(0, 0)

    xingHeader := mp3lib.NewXingHeader(templateFrame, totalFrames, totalBytes)

    _, err = outputFile.Write(xingHeader.RawBytes)
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }

    _, err = io.Copy(outputFile, inputFile)
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }

    outputFile.Close()
    inputFile.Close()

    err = os.Remove(filepath)
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }

    err = os.Rename(filepath + ".mp3cat.tmp", filepath)
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package upgrades

import (
	api "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kubernetes/test/e2e/common"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gstruct"
)

// AppArmorUpgradeTest tests that AppArmor profiles are enforced & usable across upgrades.
type AppArmorUpgradeTest struct {
	pod *api.Pod
}

func (AppArmorUpgradeTest) Name() string { return "apparmor-upgrade" }

func (AppArmorUpgradeTest) Skip(upgCtx UpgradeContext) bool {
	supportedImages := make(map[string]bool)
	for _, d := range common.AppArmorDistros {
		supportedImages[d] = true
	}

	for _, vCtx := range upgCtx.Versions {
		if !supportedImages[vCtx.NodeImage] {
			return true
		}
	}
	return false
}

// Setup creates a secret and then verifies that a pod can consume it.
func (t *AppArmorUpgradeTest) Setup(f *framework.Framework) {
	By("Loading AppArmor profiles to nodes")
	common.LoadAppArmorProfiles(f)

	// Create the initial test pod.
	By("Creating a long-running AppArmor enabled pod.")
	t.pod = common.CreateAppArmorTestPod(f, false, false)

	// Verify initial state.
	t.verifyNodesAppArmorEnabled(f)
	t.verifyNewPodSucceeds(f)
}

// Test waits for the upgrade to complete, and then verifies that a
// pod can still consume the secret.
func (t *AppArmorUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade UpgradeType) {
	<-done
	if upgrade == MasterUpgrade {
		t.verifyPodStillUp(f)
	}
	t.verifyNodesAppArmorEnabled(f)
	t.verifyNewPodSucceeds(f)
}

// Teardown cleans up any remaining resources.
func (t *AppArmorUpgradeTest) Teardown(f *framework.Framework) {
	// rely on the namespace deletion to clean up everything
	By("Logging container failures")
	framework.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
}

func (t *AppArmorUpgradeTest) verifyPodStillUp(f *framework.Framework) {
	By("Verifying an AppArmor profile is continuously enforced for a pod")
	pod, err := f.PodClient().Get(t.pod.Name, metav1.GetOptions{})
	framework.ExpectNoError(err, "Should be able to get pod")
	Expect(pod.Status.Phase).To(Equal(api.PodRunning), "Pod should stay running")
	Expect(pod.Status.ContainerStatuses[0].State.Running).NotTo(BeNil(), "Container should be running")
	Expect(pod.Status.ContainerStatuses[0].RestartCount).To(BeZero(), "Container should not need to be restarted")
}

func (t *AppArmorUpgradeTest) verifyNewPodSucceeds(f *framework.Framework) {
	By("Verifying an AppArmor profile is enforced for a new pod")
	common.CreateAppArmorTestPod(f, false, true)

	By("Verifying an unconfined AppArmor profile is enforced for a new pod")
	common.CreateAppArmorTestPod(f, true, true)
}

func (t *AppArmorUpgradeTest) verifyNodesAppArmorEnabled(f *framework.Framework) {
	By("Verifying nodes are AppArmor enabled")
	nodes, err := f.ClientSet.CoreV1().Nodes().List(metav1.ListOptions{})
	framework.ExpectNoError(err, "Failed to list nodes")
	for _, node := range nodes.Items {
		Expect(node.Status.Conditions).To(gstruct.MatchElements(conditionType, gstruct.IgnoreExtras, gstruct.Elements{
			"Ready": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
				"Message": ContainSubstring("AppArmor enabled"),
			}),
		}))
	}
}

func conditionType(condition interface{}) string {
	return string(condition.(api.NodeCondition).Type)
}
<commit_msg>Fix AppArmor upgrade test<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package upgrades

import (
	api "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kubernetes/test/e2e/common"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gstruct"
)

// AppArmorUpgradeTest tests that AppArmor profiles are enforced & usable across upgrades.
type AppArmorUpgradeTest struct {
	pod *api.Pod
}

func (AppArmorUpgradeTest) Name() string { return "apparmor-upgrade" }

func (AppArmorUpgradeTest) Skip(upgCtx UpgradeContext) bool {
	supportedImages := make(map[string]bool)
	for _, d := range common.AppArmorDistros {
		supportedImages[d] = true
	}

	for _, vCtx := range upgCtx.Versions {
		if !supportedImages[vCtx.NodeImage] {
			return true
		}
	}
	return false
}

// Setup creates a secret and then verifies that a pod can consume it.
func (t *AppArmorUpgradeTest) Setup(f *framework.Framework) {
	By("Loading AppArmor profiles to nodes")
	common.LoadAppArmorProfiles(f)

	// Create the initial test pod.
	By("Creating a long-running AppArmor enabled pod.")
	t.pod = common.CreateAppArmorTestPod(f, false, false)

	// Verify initial state.
	t.verifyNodesAppArmorEnabled(f)
	t.verifyNewPodSucceeds(f)
}

// Test waits for the upgrade to complete, and then verifies that a
// pod can still consume the secret.
func (t *AppArmorUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade UpgradeType) {
	<-done
	if upgrade == MasterUpgrade {
		t.verifyPodStillUp(f)
	}
	t.verifyNodesAppArmorEnabled(f)
	t.verifyNewPodSucceeds(f)
}

// Teardown cleans up any remaining resources.
func (t *AppArmorUpgradeTest) Teardown(f *framework.Framework) {
	// rely on the namespace deletion to clean up everything
	By("Logging container failures")
	framework.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
}

func (t *AppArmorUpgradeTest) verifyPodStillUp(f *framework.Framework) {
	By("Verifying an AppArmor profile is continuously enforced for a pod")
	pod, err := f.PodClient().Get(t.pod.Name, metav1.GetOptions{})
	framework.ExpectNoError(err, "Should be able to get pod")
	Expect(pod.Status.Phase).To(Equal(api.PodRunning), "Pod should stay running")
	Expect(pod.Status.ContainerStatuses[0].State.Running).NotTo(BeNil(), "Container should be running")
	Expect(pod.Status.ContainerStatuses[0].RestartCount).To(BeZero(), "Container should not need to be restarted")
}

func (t *AppArmorUpgradeTest) verifyNewPodSucceeds(f *framework.Framework) {
	By("Verifying an AppArmor profile is enforced for a new pod")
	common.CreateAppArmorTestPod(f, false, true)
}

func (t *AppArmorUpgradeTest) verifyNodesAppArmorEnabled(f *framework.Framework) {
	By("Verifying nodes are AppArmor enabled")
	nodes, err := f.ClientSet.CoreV1().Nodes().List(metav1.ListOptions{})
	framework.ExpectNoError(err, "Failed to list nodes")
	for _, node := range nodes.Items {
		Expect(node.Status.Conditions).To(gstruct.MatchElements(conditionType, gstruct.IgnoreExtras, gstruct.Elements{
			"Ready": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
				"Message": ContainSubstring("AppArmor enabled"),
			}),
		}))
	}
}

func conditionType(condition interface{}) string {
	return string(condition.(api.NodeCondition).Type)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"os/exec"
	"strings"

	log "github.com/Sirupsen/logrus"
)

// To provide input to the pipeline, assign an io.Reader to the first's Stdin.
func Pipeline(cmds ...*exec.Cmd) (pipeLineOutput, collectedStandardError []byte, pipeLineError error) {
	// Require at least one command
	if len(cmds) < 1 {
		return nil, nil, nil
	}

	// Collect the output from the command(s)
	var output bytes.Buffer
	var stderr bytes.Buffer

	last := len(cmds) - 1
	for i, cmd := range cmds[:last] {
		// Connect each command's stdin to the previous command's stdout
		var err error
		if cmds[i+1].Stdin, err = cmd.StdoutPipe(); err != nil {
			return nil, nil, err
		}
		// Connect each command's stderr to a buffer
		cmd.Stderr = &stderr
	}

	// Connect the output and error for the last command
	cmds[last].Stdout, cmds[last].Stderr = &output, &stderr

	// Start each command
	for _, cmd := range cmds {
		if err := cmd.Start(); err != nil {
			return output.Bytes(), stderr.Bytes(), err
		}
	}

	// Wait for each command to complete
	for _, cmd := range cmds {
		if err := cmd.Wait(); err != nil {
			return output.Bytes(), stderr.Bytes(), err
		}
	}

	// Return the pipeline output and the collected standard error
	return output.Bytes(), stderr.Bytes(), nil
}

func ExecuteMiddleware(command string, payload Payload) (Payload, error) {
	commands := strings.Split(command, " ")

	log.WithFields(log.Fields{
		"commands": commands,
		"no":       len(commands),
	}).Info("Found commands")

	cmds := exec.Command(commands[0], commands[1:]...)

	// getting payload
	bts, err := json.Marshal(payload)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to marshal json")
	}
	cmds.Stdin = bytes.NewReader(bts)

	// Run the pipeline
	mwOutput, stderr, err := Pipeline(cmds)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to process pipeline")
	}

	// log stderr
	if len(stderr) > 0 {

		log.WithFields(log.Fields{
			"sdtderr": string(stderr),
		}).Warn("errors from middleware")

	} else if len(mwOutput) > 0 {
		var newPayload Payload

		err = json.Unmarshal(mwOutput, &newPayload)

		if err != nil {
			log.WithFields(log.Fields{
				"mwOutput": string(mwOutput),
			}).Error("Failed to unmarshal JSON from middleware")
		} else {
			// payload unmarshalled into Payload struct, returning it
			return newPayload, nil
		}
	} else {

		log.WithFields(log.Fields{
			"mwOutput": string(mwOutput),
		}).Warn("No response from middleware.")
	}

	return payload, nil

}
<commit_msg>treating stderr from middleware as logs rather than errors, easier to aggregate information<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"os/exec"
	"strings"

	log "github.com/Sirupsen/logrus"
)

// To provide input to the pipeline, assign an io.Reader to the first's Stdin.
func Pipeline(cmds ...*exec.Cmd) (pipeLineOutput, collectedStandardError []byte, pipeLineError error) {
	// Require at least one command
	if len(cmds) < 1 {
		return nil, nil, nil
	}

	// Collect the output from the command(s)
	var output bytes.Buffer
	var stderr bytes.Buffer

	last := len(cmds) - 1
	for i, cmd := range cmds[:last] {
		// Connect each command's stdin to the previous command's stdout
		var err error
		if cmds[i+1].Stdin, err = cmd.StdoutPipe(); err != nil {
			return nil, nil, err
		}
		// Connect each command's stderr to a buffer
		cmd.Stderr = &stderr
	}

	// Connect the output and error for the last command
	cmds[last].Stdout, cmds[last].Stderr = &output, &stderr

	// Start each command
	for _, cmd := range cmds {
		if err := cmd.Start(); err != nil {
			return output.Bytes(), stderr.Bytes(), err
		}
	}

	// Wait for each command to complete
	for _, cmd := range cmds {
		if err := cmd.Wait(); err != nil {
			return output.Bytes(), stderr.Bytes(), err
		}
	}

	// Return the pipeline output and the collected standard error
	return output.Bytes(), stderr.Bytes(), nil
}

func ExecuteMiddleware(command string, payload Payload) (Payload, error) {
	commands := strings.Split(command, " ")

	log.WithFields(log.Fields{
		"commands": commands,
		"no":       len(commands),
	}).Info("Found commands")

	cmds := exec.Command(commands[0], commands[1:]...)

	// getting payload
	bts, err := json.Marshal(payload)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to marshal json")
	}
	cmds.Stdin = bytes.NewReader(bts)

	// Run the pipeline
	mwOutput, stderr, err := Pipeline(cmds)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to process pipeline")
	}

	// log stderr
	if len(stderr) > 0 {

		log.WithFields(log.Fields{
			"sdtderr": string(stderr),
		}).Info("Information from middleware")

	}

	if len(mwOutput) > 0 {
		var newPayload Payload

		err = json.Unmarshal(mwOutput, &newPayload)

		if err != nil {
			log.WithFields(log.Fields{
				"mwOutput": string(mwOutput),
			}).Error("Failed to unmarshal JSON from middleware")
		} else {
			// payload unmarshalled into Payload struct, returning it
			return newPayload, nil
		}
	} else {

		log.WithFields(log.Fields{
			"mwOutput": string(mwOutput),
		}).Warn("No response from middleware.")
	}

	return payload, nil

}
<|endoftext|>


<commit_before>package lang

import (
	. "jvmgo/any"
	"jvmgo/jvm/rtda"
	rtc "jvmgo/jvm/rtda/class"
	"jvmgo/util"
	"strings"
)

func init() {
	_class(desiredAssertionStatus0, "desiredAssertionStatus0", "(Ljava/lang/Class;)Z")
	_class(forName0, "forName0", "(Ljava/lang/String;ZLjava/lang/ClassLoader;)Ljava/lang/Class;")
	_class(getClassLoader0, "getClassLoader0", "()Ljava/lang/ClassLoader;")
	_class(getComponentType, "getComponentType", "()Ljava/lang/Class;")
	_class(getConstantPool, "getConstantPool", "()Lsun/reflect/ConstantPool;")
	_class(getDeclaringClass, "getDeclaringClass", "()Ljava/lang/Class;")
	_class(getEnclosingMethod0, "getEnclosingMethod0", "()[Ljava/lang/Object;")
	_class(getInterfaces, "getInterfaces", "()[Ljava/lang/Class;")
	_class(getModifiers, "getModifiers", "()I")
	_class(getName0, "getName0", "()Ljava/lang/String;")
	_class(getPrimitiveClass, "getPrimitiveClass", "(Ljava/lang/String;)Ljava/lang/Class;")
	_class(getSuperclass, "getSuperclass", "()Ljava/lang/Class;")
	_class(isAssignableFrom, "isAssignableFrom", "(Ljava/lang/Class;)Z")
	_class(isArray, "isArray", "()Z")
	_class(isInterface, "isInterface", "()Z")
	_class(isPrimitive, "isPrimitive", "()Z")
}

func _class(method Any, name, desc string) {
	rtc.RegisterNativeMethod("java/lang/Class", name, desc, method)
}

// private static native boolean desiredAssertionStatus0(Class<?> clazz);
// (Ljava/lang/Class;)Z
func desiredAssertionStatus0(frame *rtda.Frame) {
	// todo
	stack := frame.OperandStack()
	//stack.PopRef() // this
	stack.PushBoolean(false)
}

// private static native Class<?> forName0(String name, boolean initialize, ClassLoader loader) throws ClassNotFoundException;
// (Ljava/lang/String;ZLjava/lang/ClassLoader;)Ljava/lang/Class;
func forName0(frame *rtda.Frame) {
	vars := frame.LocalVars()
	jName := vars.GetRef(0)
	initialize := vars.GetBoolean(1)
	//jLoader := vars.GetRef(2)

	goName := rtda.GoString(jName)
	goName = util.ReplaceAll(goName, ".", "/")
	goClass := frame.ClassLoader().LoadClass(goName)
	jClass := goClass.JClass()

	if initialize && goClass.InitializationNotStarted() {
		// undo forName0
		thread := frame.Thread()
		frame.SetNextPC(thread.PC())
		// init class
		thread.InitClass(goClass)
	} else {
		stack := frame.OperandStack()
		stack.PushRef(jClass)
	}
}

// native ClassLoader getClassLoader0();
// ()Ljava/lang/ClassLoader;
func getClassLoader0(frame *rtda.Frame) {
	// todo
	// _ = stack.PopRef() // this
	stack := frame.OperandStack()
	stack.PushRef(nil)
}

// public native Class<?> getComponentType();
// ()Ljava/lang/Class;
func getComponentType(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	componentClass := class.ComponentClass()
	componentClassObj := componentClass.JClass()

	stack := frame.OperandStack()
	stack.PushRef(componentClassObj)
}

// native ConstantPool getConstantPool();
// ()Lsun/reflect/ConstantPool;
func getConstantPool(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	cpClass := class.ClassLoader().LoadClass("sun/reflect/ConstantPool")
	if cpClass.InitializationNotStarted() {
		frame.RevertNextPC()
		frame.Thread().InitClass(cpClass)
		return
	}

	cp := class.ConstantPool()
	cpObj := cpClass.NewObjWithExtra(cp) // todo init cpObj
	frame.OperandStack().PushRef(cpObj)
}

// private native Class<?> getDeclaringClass();
// ()Ljava/lang/Class;
func getDeclaringClass(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	if class.IsArray() || class.IsPrimitive() {
		frame.OperandStack().PushRef(nil)
		return
	}

	lastDollerIndex := strings.LastIndex(class.Name(), "$")
	if lastDollerIndex < 0 {
		frame.OperandStack().PushRef(nil)
		return
	}

	// todo
	declaringClassName := class.Name()[:lastDollerIndex]
	declaringClass := frame.ClassLoader().LoadClass(declaringClassName)
	frame.OperandStack().PushRef(declaringClass.JClass())
}

// private native Object[] getEnclosingMethod0();
// ()[Ljava/lang/Object;
func getEnclosingMethod0(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	emInfo := class.Attributes().EnclosingMethod()
	emInfoObj := _createEnclosintMethodInfo(frame.ClassLoader(), emInfo)
	frame.OperandStack().PushRef(emInfoObj)
}

func _createEnclosintMethodInfo(classLoader *rtc.ClassLoader, emInfo *rtc.EnclosingMethod) *rtc.Obj {
	if emInfo == nil {
		return nil
	}

	enclosingClass := classLoader.LoadClass(emInfo.ClassName())
	enclosingClassObj := enclosingClass.JClass()
	var methodNameObj, methodDescriptorObj *rtc.Obj
	if emInfo.MethodName() != "" {
		methodNameObj = rtda.NewJString(emInfo.MethodName(), classLoader)
		methodDescriptorObj = rtda.NewJString(emInfo.MethodDescriptor(), classLoader)
	} else {
		methodNameObj, methodDescriptorObj = nil, nil
	}

	objs := []*rtc.Obj{enclosingClassObj, methodNameObj, methodDescriptorObj}
	return rtc.NewRefArray2(classLoader.JLObjectClass(), objs) // Object[]
}

// private native Class<?>[] getInterfaces();
// ()[Ljava/lang/Class;
func getInterfaces(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	interfaces := class.Interfaces()
	interfaceObjs := make([]*rtc.Obj, len(interfaces))
	for i, iface := range interfaces {
		interfaceObjs[i] = iface.JClass()
	}

	jlClassClass := class.ClassLoader().JLClassClass()
	interfaceArr := rtc.NewRefArray2(jlClassClass, interfaceObjs)

	stack := frame.OperandStack()
	stack.PushRef(interfaceArr)
}

// private native String getName0();
// ()Ljava/lang/String;
func getName0(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	name := class.JlsName()
	nameObj := rtda.NewJString(name, frame)

	stack := frame.OperandStack()
	stack.PushRef(nameObj)
}

// public native int getModifiers();
// ()I
func getModifiers(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	modifiers := class.GetAccessFlags()

	stack := frame.OperandStack()
	stack.PushInt(int32(modifiers))
}

// static native Class<?> getPrimitiveClass(String name);
// (Ljava/lang/String;)Ljava/lang/Class;
func getPrimitiveClass(frame *rtda.Frame) {
	vars := frame.LocalVars()
	nameObj := vars.GetRef(0)

	name := rtda.GoString(nameObj)
	classLoader := frame.ClassLoader()
	class := classLoader.GetPrimitiveClass(name)
	classObj := class.JClass()

	stack := frame.OperandStack()
	stack.PushRef(classObj)
}

// public native Class<? super T> getSuperclass();
// ()Ljava/lang/Class;
func getSuperclass(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	superClass := class.SuperClass()

	stack := frame.OperandStack()
	if superClass != nil {
		stack.PushRef(superClass.JClass())
	} else {
		stack.PushNull()
	}
}

// public native boolean isAssignableFrom(Class<?> cls);
// (Ljava/lang/Class;)Z
func isAssignableFrom(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()
	cls := vars.GetRef(1)

	thisClass := this.Extra().(*rtc.Class)
	clsClass := cls.Extra().(*rtc.Class)
	ok := thisClass.IsAssignableFrom(clsClass)
	frame.OperandStack().PushBoolean(ok)
}

// public native boolean isArray();
// ()Z
func isArray(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	stack := frame.OperandStack()
	stack.PushBoolean(class.IsArray())
}

// public native boolean isInterface();
// ()Z
func isInterface(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	stack := frame.OperandStack()
	stack.PushBoolean(class.IsInterface())
}

// public native boolean isPrimitive();
// ()Z
func isPrimitive(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	stack := frame.OperandStack()
	stack.PushBoolean(class.IsPrimitive())
}
<commit_msg>rename getInterfaces() to getInterfaces0()<commit_after>package lang

import (
	. "jvmgo/any"
	"jvmgo/jvm/rtda"
	rtc "jvmgo/jvm/rtda/class"
	"jvmgo/util"
	"strings"
)

func init() {
	_class(desiredAssertionStatus0, "desiredAssertionStatus0", "(Ljava/lang/Class;)Z")
	_class(forName0, "forName0", "(Ljava/lang/String;ZLjava/lang/ClassLoader;)Ljava/lang/Class;")
	_class(getClassLoader0, "getClassLoader0", "()Ljava/lang/ClassLoader;")
	_class(getComponentType, "getComponentType", "()Ljava/lang/Class;")
	_class(getConstantPool, "getConstantPool", "()Lsun/reflect/ConstantPool;")
	_class(getDeclaringClass, "getDeclaringClass", "()Ljava/lang/Class;")
	_class(getEnclosingMethod0, "getEnclosingMethod0", "()[Ljava/lang/Object;")
	_class(getInterfaces0, "getInterfaces0", "()[Ljava/lang/Class;")
	_class(getModifiers, "getModifiers", "()I")
	_class(getName0, "getName0", "()Ljava/lang/String;")
	_class(getPrimitiveClass, "getPrimitiveClass", "(Ljava/lang/String;)Ljava/lang/Class;")
	_class(getSuperclass, "getSuperclass", "()Ljava/lang/Class;")
	_class(isAssignableFrom, "isAssignableFrom", "(Ljava/lang/Class;)Z")
	_class(isArray, "isArray", "()Z")
	_class(isInterface, "isInterface", "()Z")
	_class(isPrimitive, "isPrimitive", "()Z")
}

func _class(method Any, name, desc string) {
	rtc.RegisterNativeMethod("java/lang/Class", name, desc, method)
}

// private static native boolean desiredAssertionStatus0(Class<?> clazz);
// (Ljava/lang/Class;)Z
func desiredAssertionStatus0(frame *rtda.Frame) {
	// todo
	stack := frame.OperandStack()
	//stack.PopRef() // this
	stack.PushBoolean(false)
}

// private static native Class<?> forName0(String name, boolean initialize, ClassLoader loader) throws ClassNotFoundException;
// (Ljava/lang/String;ZLjava/lang/ClassLoader;)Ljava/lang/Class;
func forName0(frame *rtda.Frame) {
	vars := frame.LocalVars()
	jName := vars.GetRef(0)
	initialize := vars.GetBoolean(1)
	//jLoader := vars.GetRef(2)

	goName := rtda.GoString(jName)
	goName = util.ReplaceAll(goName, ".", "/")
	goClass := frame.ClassLoader().LoadClass(goName)
	jClass := goClass.JClass()

	if initialize && goClass.InitializationNotStarted() {
		// undo forName0
		thread := frame.Thread()
		frame.SetNextPC(thread.PC())
		// init class
		thread.InitClass(goClass)
	} else {
		stack := frame.OperandStack()
		stack.PushRef(jClass)
	}
}

// native ClassLoader getClassLoader0();
// ()Ljava/lang/ClassLoader;
func getClassLoader0(frame *rtda.Frame) {
	// todo
	// _ = stack.PopRef() // this
	stack := frame.OperandStack()
	stack.PushRef(nil)
}

// public native Class<?> getComponentType();
// ()Ljava/lang/Class;
func getComponentType(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	componentClass := class.ComponentClass()
	componentClassObj := componentClass.JClass()

	stack := frame.OperandStack()
	stack.PushRef(componentClassObj)
}

// native ConstantPool getConstantPool();
// ()Lsun/reflect/ConstantPool;
func getConstantPool(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	cpClass := class.ClassLoader().LoadClass("sun/reflect/ConstantPool")
	if cpClass.InitializationNotStarted() {
		frame.RevertNextPC()
		frame.Thread().InitClass(cpClass)
		return
	}

	cp := class.ConstantPool()
	cpObj := cpClass.NewObjWithExtra(cp) // todo init cpObj
	frame.OperandStack().PushRef(cpObj)
}

// private native Class<?> getDeclaringClass();
// ()Ljava/lang/Class;
func getDeclaringClass(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	if class.IsArray() || class.IsPrimitive() {
		frame.OperandStack().PushRef(nil)
		return
	}

	lastDollerIndex := strings.LastIndex(class.Name(), "$")
	if lastDollerIndex < 0 {
		frame.OperandStack().PushRef(nil)
		return
	}

	// todo
	declaringClassName := class.Name()[:lastDollerIndex]
	declaringClass := frame.ClassLoader().LoadClass(declaringClassName)
	frame.OperandStack().PushRef(declaringClass.JClass())
}

// private native Object[] getEnclosingMethod0();
// ()[Ljava/lang/Object;
func getEnclosingMethod0(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	emInfo := class.Attributes().EnclosingMethod()
	emInfoObj := _createEnclosintMethodInfo(frame.ClassLoader(), emInfo)
	frame.OperandStack().PushRef(emInfoObj)
}

func _createEnclosintMethodInfo(classLoader *rtc.ClassLoader, emInfo *rtc.EnclosingMethod) *rtc.Obj {
	if emInfo == nil {
		return nil
	}

	enclosingClass := classLoader.LoadClass(emInfo.ClassName())
	enclosingClassObj := enclosingClass.JClass()
	var methodNameObj, methodDescriptorObj *rtc.Obj
	if emInfo.MethodName() != "" {
		methodNameObj = rtda.NewJString(emInfo.MethodName(), classLoader)
		methodDescriptorObj = rtda.NewJString(emInfo.MethodDescriptor(), classLoader)
	} else {
		methodNameObj, methodDescriptorObj = nil, nil
	}

	objs := []*rtc.Obj{enclosingClassObj, methodNameObj, methodDescriptorObj}
	return rtc.NewRefArray2(classLoader.JLObjectClass(), objs) // Object[]
}

// private native Class<?>[] getInterfaces0();
// ()[Ljava/lang/Class;
func getInterfaces0(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	interfaces := class.Interfaces()
	interfaceObjs := make([]*rtc.Obj, len(interfaces))
	for i, iface := range interfaces {
		interfaceObjs[i] = iface.JClass()
	}

	jlClassClass := class.ClassLoader().JLClassClass()
	interfaceArr := rtc.NewRefArray2(jlClassClass, interfaceObjs)

	stack := frame.OperandStack()
	stack.PushRef(interfaceArr)
}

// private native String getName0();
// ()Ljava/lang/String;
func getName0(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	name := class.JlsName()
	nameObj := rtda.NewJString(name, frame)

	stack := frame.OperandStack()
	stack.PushRef(nameObj)
}

// public native int getModifiers();
// ()I
func getModifiers(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	modifiers := class.GetAccessFlags()

	stack := frame.OperandStack()
	stack.PushInt(int32(modifiers))
}

// static native Class<?> getPrimitiveClass(String name);
// (Ljava/lang/String;)Ljava/lang/Class;
func getPrimitiveClass(frame *rtda.Frame) {
	vars := frame.LocalVars()
	nameObj := vars.GetRef(0)

	name := rtda.GoString(nameObj)
	classLoader := frame.ClassLoader()
	class := classLoader.GetPrimitiveClass(name)
	classObj := class.JClass()

	stack := frame.OperandStack()
	stack.PushRef(classObj)
}

// public native Class<? super T> getSuperclass();
// ()Ljava/lang/Class;
func getSuperclass(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	superClass := class.SuperClass()

	stack := frame.OperandStack()
	if superClass != nil {
		stack.PushRef(superClass.JClass())
	} else {
		stack.PushNull()
	}
}

// public native boolean isAssignableFrom(Class<?> cls);
// (Ljava/lang/Class;)Z
func isAssignableFrom(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()
	cls := vars.GetRef(1)

	thisClass := this.Extra().(*rtc.Class)
	clsClass := cls.Extra().(*rtc.Class)
	ok := thisClass.IsAssignableFrom(clsClass)
	frame.OperandStack().PushBoolean(ok)
}

// public native boolean isArray();
// ()Z
func isArray(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	stack := frame.OperandStack()
	stack.PushBoolean(class.IsArray())
}

// public native boolean isInterface();
// ()Z
func isInterface(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	stack := frame.OperandStack()
	stack.PushBoolean(class.IsInterface())
}

// public native boolean isPrimitive();
// ()Z
func isPrimitive(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	class := this.Extra().(*rtc.Class)
	stack := frame.OperandStack()
	stack.PushBoolean(class.IsPrimitive())
}
<|endoftext|>


<commit_before>// fetch is a wrapper around package adn
//
// Gets Global feed from ADN (App.Net) and prints username and post text
package main

import (
  "database/sql"
	"fetch/adn"
	"fmt"
	"github.com/boltdb/bolt"
_ "github.com/mattn/go-sqlite3"
	"log"
	"os"
)

var file = "data/blog.db"

func ToFromSqlite(r adn.Response, file string)([]string, error) {

  os.Remove(file)
	db, err := sql.Open("sqlite3", file)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sqlStmt := `
	create table posts (id integer not null primary key, user text, post text);
	delete from posts;
	`
	_, err = db.Exec(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
		return nil, err
	}

	tx, err := db.Begin()
	if err != nil {
		log.Fatal(err)
	}
	stmt, err := tx.Prepare("insert into posts(id, user, post) values(?, ?, ?)")
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()
	for it, p := range r.Data {
		_, err = stmt.Exec(it, p.User.UserName, p.Text)
		if err != nil {
			log.Fatal(err)
		}
	}
	tx.Commit()

	rows, err := db.Query("select id, user, post from posts")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var values []string
	var items = 0
	for rows.Next() {
		var id int
		var user string
		var post string
		rows.Scan(&id, &user, &post)
		values = append(values, string(id)+user+post)
		items++
	}
	rows.Close()
	fmt.Printf("%d items retrieved\n", items)

	return values, err
}

func check_error_status(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func ToFrom(r adn.Response, file string) ([]string, error) {
	// create new file (remove if exists)
	os.Remove(file)

	db, err := bolt.Open(file, 0600, nil)
	check_error_status(err)
	defer db.Close()

	// store posts in file
	var bName = "posts"
	var items int
	for it, p := range r.Data {
		db.Update(func(tx *bolt.Tx) error {
			b, err := tx.CreateBucketIfNotExists([]byte(bName))
			if err != nil {
				return err
			}
			return b.Put([]byte(p.Id), []byte(p.User.UserName+p.Text))
		})
		items = it + 1
	}
	fmt.Printf("%d items stored\n", items)

	var values []string
	items = 0

	// display again
	err = db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bName))
		c := b.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			values = append(values, string(v[:]))
			items++
		}
		return nil
	})
	fmt.Printf("%d items retrieved\n", items)

	return values, err
}

func main() {
	r, err := adn.GetGlobal()
	check_error_status(err)

	//results, err := ToFrom(r, file)
	results, err := ToFromSqlite(r, file)
	check_error_status(err)

	for _, v := range results {
		fmt.Printf("%s\n", v)
	}
}
<commit_msg>ran go fmt to format some code<commit_after>// fetch is a wrapper around package adn
//
// Gets Global feed from ADN (App.Net) and prints username and post text
package main

import (
	"database/sql"
	"fetch/adn"
	"fmt"
	"github.com/boltdb/bolt"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"os"
)

var file = "data/blog.db"

func ToFromSqlite(r adn.Response, file string) ([]string, error) {

	os.Remove(file)
	db, err := sql.Open("sqlite3", file)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sqlStmt := `
	create table posts (id integer not null primary key, user text, post text);
	delete from posts;
	`
	_, err = db.Exec(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
		return nil, err
	}

	tx, err := db.Begin()
	if err != nil {
		log.Fatal(err)
	}
	stmt, err := tx.Prepare("insert into posts(id, user, post) values(?, ?, ?)")
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()
	for it, p := range r.Data {
		_, err = stmt.Exec(it, p.User.UserName, p.Text)
		if err != nil {
			log.Fatal(err)
		}
	}
	tx.Commit()

	rows, err := db.Query("select id, user, post from posts")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var values []string
	var items = 0
	for rows.Next() {
		var id int
		var user string
		var post string
		rows.Scan(&id, &user, &post)
		values = append(values, string(id)+user+post)
		items++
	}
	rows.Close()
	fmt.Printf("%d items retrieved\n", items)

	return values, err
}

func check_error_status(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func ToFrom(r adn.Response, file string) ([]string, error) {
	// create new file (remove if exists)
	os.Remove(file)

	db, err := bolt.Open(file, 0600, nil)
	check_error_status(err)
	defer db.Close()

	// store posts in file
	var bName = "posts"
	var items int
	for it, p := range r.Data {
		db.Update(func(tx *bolt.Tx) error {
			b, err := tx.CreateBucketIfNotExists([]byte(bName))
			if err != nil {
				return err
			}
			return b.Put([]byte(p.Id), []byte(p.User.UserName+p.Text))
		})
		items = it + 1
	}
	fmt.Printf("%d items stored\n", items)

	var values []string
	items = 0

	// display again
	err = db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bName))
		c := b.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			values = append(values, string(v[:]))
			items++
		}
		return nil
	})
	fmt.Printf("%d items retrieved\n", items)

	return values, err
}

func main() {
	r, err := adn.GetGlobal()
	check_error_status(err)

	//results, err := ToFrom(r, file)
	results, err := ToFromSqlite(r, file)
	check_error_status(err)

	for _, v := range results {
		fmt.Printf("%s\n", v)
	}
}
<|endoftext|>


<commit_before>package application_test

import (
	"errors"
	"net/http"
	"net/http/httptest"
	"time"

	"code.cloudfoundry.org/cli/cf/api/apifakes"
	"code.cloudfoundry.org/cli/cf/commandregistry"
	"code.cloudfoundry.org/cli/cf/commands/commandsfakes"
	"code.cloudfoundry.org/cli/cf/configuration/coreconfig"
	"code.cloudfoundry.org/cli/cf/models"
	"code.cloudfoundry.org/cli/cf/net"
	"code.cloudfoundry.org/cli/cf/requirements"
	"code.cloudfoundry.org/cli/cf/requirements/requirementsfakes"
	"code.cloudfoundry.org/cli/cf/ssh/sshfakes"
	testcmd "code.cloudfoundry.org/cli/util/testhelpers/commands"
	testconfig "code.cloudfoundry.org/cli/util/testhelpers/configuration"
	testnet "code.cloudfoundry.org/cli/util/testhelpers/net"
	testterm "code.cloudfoundry.org/cli/util/testhelpers/terminal"

	"code.cloudfoundry.org/cli/cf/trace/tracefakes"
	. "code.cloudfoundry.org/cli/util/testhelpers/matchers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("SSH command", func() {
	var (
		ui *testterm.FakeUI

		sshCodeGetter         *commandsfakes.FakeSSHCodeGetter
		originalSSHCodeGetter commandregistry.Command

		requirementsFactory *requirementsfakes.FakeFactory
		configRepo          coreconfig.Repository
		deps                commandregistry.Dependency
		ccGateway           net.Gateway

		fakeSecureShell *sshfakes.FakeSecureShell
	)

	BeforeEach(func() {
		ui = &testterm.FakeUI{}
		configRepo = testconfig.NewRepositoryWithDefaults()
		requirementsFactory = new(requirementsfakes.FakeFactory)
		deps.Gateways = make(map[string]net.Gateway)

		//save original command and restore later
		originalSSHCodeGetter = commandregistry.Commands.FindCommand("ssh-code")

		sshCodeGetter = new(commandsfakes.FakeSSHCodeGetter)

		//setup fakes to correctly interact with commandregistry
		sshCodeGetter.SetDependencyStub = func(_ commandregistry.Dependency, _ bool) commandregistry.Command {
			return sshCodeGetter
		}
		sshCodeGetter.MetaDataReturns(commandregistry.CommandMetadata{Name: "ssh-code"})
	})

	AfterEach(func() {
		//restore original command
		commandregistry.Register(originalSSHCodeGetter)
	})

	updateCommandDependency := func(pluginCall bool) {
		deps.UI = ui
		deps.Config = configRepo

		//inject fake 'sshCodeGetter' into registry
		commandregistry.Register(sshCodeGetter)

		commandregistry.Commands.SetCommand(commandregistry.Commands.FindCommand("ssh").SetDependency(deps, pluginCall))
	}

	runCommand := func(args ...string) bool {
		return testcmd.RunCLICommand("ssh", args, requirementsFactory, updateCommandDependency, false, ui)
	}

	Describe("Requirements", func() {
		It("fails with usage when not provided exactly one arg", func() {
			requirementsFactory.NewLoginRequirementReturns(requirements.Passing{})

			runCommand()
			Expect(ui.Outputs()).To(ContainSubstrings(
				[]string{"Incorrect Usage", "Requires", "argument"},
			))

		})

		It("fails requirements when not logged in", func() {
			requirementsFactory.NewLoginRequirementReturns(requirements.Failing{Message: "not logged in"})
			Expect(runCommand("my-app")).To(BeFalse())
		})

		It("fails if a space is not targeted", func() {
			requirementsFactory.NewLoginRequirementReturns(requirements.Passing{})
			requirementsFactory.NewTargetedSpaceRequirementReturns(requirements.Failing{Message: "not targeting space"})
			Expect(runCommand("my-app")).To(BeFalse())
		})

		It("fails if a application is not found", func() {
			requirementsFactory.NewLoginRequirementReturns(requirements.Passing{})
			requirementsFactory.NewTargetedSpaceRequirementReturns(requirements.Passing{})
			applicationReq := new(requirementsfakes.FakeApplicationRequirement)
			applicationReq.ExecuteReturns(errors.New("no app"))
			requirementsFactory.NewApplicationRequirementReturns(applicationReq)

			Expect(runCommand("my-app")).To(BeFalse())
		})

		Describe("Flag options", func() {
			var args []string

			BeforeEach(func() {
				requirementsFactory.NewLoginRequirementReturns(requirements.Passing{})
				requirementsFactory.NewTargetedSpaceRequirementReturns(requirements.Passing{})
			})

			Context("when an -i flag is provided", func() {
				BeforeEach(func() {
					args = append(args, "app-name")
				})

				Context("with a negative integer argument", func() {
					BeforeEach(func() {
						args = append(args, "-i", "-3")
					})

					It("returns an error", func() {
						Expect(runCommand(args...)).To(BeFalse())
						Expect(ui.Outputs()).To(ContainSubstrings(
							[]string{"Incorrect Usage", "cannot be negative"},
						))

					})
				})

				Context("with a negative integer argument", func() {
					It("returns an error", func() {
						Expect(runCommand("my-app", "-i", "-3")).To(BeFalse())
						Expect(ui.Outputs()).To(ContainSubstrings(
							[]string{"Incorrect Usage", "cannot be negative"},
						))

					})
				})

				Context("with a value greater than the application's highest instance index", func() {
					BeforeEach(func() {
						var app models.Application

						app = models.Application{}
						app.Name = "my-app"
						app.State = "started"
						app.GUID = "my-app-guid"
						app.EnableSSH = true
						app.Diego = true
						app.InstanceCount = 3

						applicationReq := new(requirementsfakes.FakeApplicationRequirement)
						applicationReq.GetApplicationReturns(app)
						requirementsFactory.NewApplicationRequirementReturns(applicationReq)
					})

					It("returns an error", func() {
						Expect(runCommand("my-app", "-i", "3")).To(BeFalse())
						Expect(ui.Outputs()).To(ContainSubstrings(
							[]string{"Incorrect Usage", "specified application instance does not exist"},
						))
					})
				})
			})
		})

		Describe("SSHOptions", func() {
			Context("when an error is returned during initialization", func() {
				It("shows error and prints command usage", func() {
					Expect(runCommand("app_name", "-L", "[9999:localhost...")).To(BeFalse())
					Expect(ui.Outputs()).To(ContainSubstrings(
						[]string{"Incorrect Usage"},
						[]string{"USAGE:"},
					))

				})
			})
		})

	})

	Describe("ssh", func() {
		var (
			currentApp models.Application
		)

		BeforeEach(func() {
			requirementsFactory.NewLoginRequirementReturns(requirements.Passing{})
			requirementsFactory.NewTargetedSpaceRequirementReturns(requirements.Passing{})
			currentApp = models.Application{}
			currentApp.Name = "my-app"
			currentApp.State = "started"
			currentApp.GUID = "my-app-guid"
			currentApp.EnableSSH = true
			currentApp.Diego = true

			applicationReq := new(requirementsfakes.FakeApplicationRequirement)
			applicationReq.GetApplicationReturns(currentApp)
			requirementsFactory.NewApplicationRequirementReturns(applicationReq)
		})

		Describe("Error getting required info to run ssh", func() {
			var (
				testServer *httptest.Server
				handler    *testnet.TestHandler
			)

			AfterEach(func() {
				testServer.Close()
			})

			Context("error when getting SSH info from /v2/info", func() {
				BeforeEach(func() {
					getRequest := apifakes.NewCloudControllerTestRequest(testnet.TestRequest{
						Method: "GET",
						Path:   "/v2/info",
						Response: testnet.TestResponse{
							Status: http.StatusNotFound,
							Body:   `{}`,
						},
					})

					testServer, handler = testnet.NewServer([]testnet.TestRequest{getRequest})
					configRepo.SetAPIEndpoint(testServer.URL)
					ccGateway = net.NewCloudControllerGateway(configRepo, time.Now, &testterm.FakeUI{}, new(tracefakes.FakePrinter), "")
					deps.Gateways["cloud-controller"] = ccGateway
				})

				It("notifies users", func() {
					runCommand("my-app")

					Expect(handler).To(HaveAllRequestsCalled())
					Expect(ui.Outputs()).To(ContainSubstrings(
						[]string{"Error getting SSH info", "404"},
					))

				})
			})

			Context("error when getting oauth token", func() {
				BeforeEach(func() {
					sshCodeGetter.GetReturns("", errors.New("auth api error"))

					getRequest := apifakes.NewCloudControllerTestRequest(testnet.TestRequest{
						Method: "GET",
						Path:   "/v2/info",
						Response: testnet.TestResponse{
							Status: http.StatusOK,
							Body:   `{}`,
						},
					})

					testServer, handler = testnet.NewServer([]testnet.TestRequest{getRequest})
					configRepo.SetAPIEndpoint(testServer.URL)
					ccGateway = net.NewCloudControllerGateway(configRepo, time.Now, &testterm.FakeUI{}, new(tracefakes.FakePrinter), "")
					deps.Gateways["cloud-controller"] = ccGateway
				})

				It("notifies users", func() {
					runCommand("my-app")

					Expect(handler).To(HaveAllRequestsCalled())
					Expect(ui.Outputs()).To(ContainSubstrings(
						[]string{"Error getting one time auth code", "auth api error"},
					))

				})
			})
		})

		Describe("Connecting to ssh server", func() {
			var testServer *httptest.Server

			AfterEach(func() {
				testServer.Close()
			})

			BeforeEach(func() {
				fakeSecureShell = new(sshfakes.FakeSecureShell)

				deps.WildcardDependency = fakeSecureShell

				getRequest := apifakes.NewCloudControllerTestRequest(testnet.TestRequest{
					Method: "GET",
					Path:   "/v2/info",
					Response: testnet.TestResponse{
						Status: http.StatusOK,
						Body:   getInfoResponseBody,
					},
				})

				testServer, _ = testnet.NewServer([]testnet.TestRequest{getRequest})
				configRepo.SetAPIEndpoint(testServer.URL)
				ccGateway = net.NewCloudControllerGateway(configRepo, time.Now, &testterm.FakeUI{}, new(tracefakes.FakePrinter), "")
				deps.Gateways["cloud-controller"] = ccGateway
			})

			Context("Error when connecting", func() {
				It("notifies users", func() {
					fakeSecureShell.ConnectReturns(errors.New("dial errorrr"))

					runCommand("my-app")

					Expect(ui.Outputs()).To(ContainSubstrings(
						[]string{"Error opening SSH connection", "dial error"},
					))

				})
			})

			Context("Error port forwarding when -L is provided", func() {
				It("notifies users", func() {
					fakeSecureShell.LocalPortForwardReturns(errors.New("listen error"))

					runCommand("my-app", "-L", "8000:localhost:8000")

					Expect(ui.Outputs()).To(ContainSubstrings(
						[]string{"Error forwarding port", "listen error"},
					))

				})
			})

			Context("when -N is provided", func() {
				It("calls secureShell.Wait()", func() {
					fakeSecureShell.ConnectReturns(nil)
					fakeSecureShell.LocalPortForwardReturns(nil)

					runCommand("my-app", "-N")

					Expect(fakeSecureShell.WaitCallCount()).To(Equal(1))
				})
			})

			Context("when -N is provided", func() {
				It("calls secureShell.InteractiveSession()", func() {
					fakeSecureShell.ConnectReturns(nil)
					fakeSecureShell.LocalPortForwardReturns(nil)

					runCommand("my-app", "-k")

					Expect(fakeSecureShell.InteractiveSessionCallCount()).To(Equal(1))
				})
			})

			Context("when Wait() or InteractiveSession() returns error", func() {

				It("notifities users", func() {
					fakeSecureShell.ConnectReturns(nil)
					fakeSecureShell.LocalPortForwardReturns(nil)

					fakeSecureShell.InteractiveSessionReturns(errors.New("ssh exit error"))
					runCommand("my-app", "-k")

					Expect(ui.Outputs()).To(ContainSubstrings(
						[]string{"ssh exit error"},
					))

				})
			})
		})
	})
})

const getInfoResponseBody string = `
{
   "name": "vcap",
   "build": "2222",
   "support": "http://support.cloudfoundry.com",
   "version": 2,
   "description": "Cloud Foundry sponsored by ABC",
   "authorization_endpoint": "https://login.run.abc.com",
   "token_endpoint": "https://uaa.run.abc.com",
   "min_cli_version": null,
   "min_recommended_cli_version": null,
   "api_version": "2.35.0",
   "app_ssh_endpoint": "ssh.run.pivotal.io:2222",
   "app_ssh_host_key_fingerprint": "11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11",
   "logging_endpoint": "wss://loggregator.run.abc.com:443",
   "doppler_logging_endpoint": "wss://doppler.run.abc.com:443",
   "user": "6e477566-ac8d-4653-98c6-d319595ec7b0"
}`
<commit_msg>remove duplicate test in ssh_test<commit_after>package application_test

import (
	"errors"
	"net/http"
	"net/http/httptest"
	"time"

	"code.cloudfoundry.org/cli/cf/api/apifakes"
	"code.cloudfoundry.org/cli/cf/commandregistry"
	"code.cloudfoundry.org/cli/cf/commands/commandsfakes"
	"code.cloudfoundry.org/cli/cf/configuration/coreconfig"
	"code.cloudfoundry.org/cli/cf/models"
	"code.cloudfoundry.org/cli/cf/net"
	"code.cloudfoundry.org/cli/cf/requirements"
	"code.cloudfoundry.org/cli/cf/requirements/requirementsfakes"
	"code.cloudfoundry.org/cli/cf/ssh/sshfakes"
	testcmd "code.cloudfoundry.org/cli/util/testhelpers/commands"
	testconfig "code.cloudfoundry.org/cli/util/testhelpers/configuration"
	testnet "code.cloudfoundry.org/cli/util/testhelpers/net"
	testterm "code.cloudfoundry.org/cli/util/testhelpers/terminal"

	"code.cloudfoundry.org/cli/cf/trace/tracefakes"
	. "code.cloudfoundry.org/cli/util/testhelpers/matchers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("SSH command", func() {
	var (
		ui *testterm.FakeUI

		sshCodeGetter         *commandsfakes.FakeSSHCodeGetter
		originalSSHCodeGetter commandregistry.Command

		requirementsFactory *requirementsfakes.FakeFactory
		configRepo          coreconfig.Repository
		deps                commandregistry.Dependency
		ccGateway           net.Gateway

		fakeSecureShell *sshfakes.FakeSecureShell
	)

	BeforeEach(func() {
		ui = &testterm.FakeUI{}
		configRepo = testconfig.NewRepositoryWithDefaults()
		requirementsFactory = new(requirementsfakes.FakeFactory)
		deps.Gateways = make(map[string]net.Gateway)

		//save original command and restore later
		originalSSHCodeGetter = commandregistry.Commands.FindCommand("ssh-code")

		sshCodeGetter = new(commandsfakes.FakeSSHCodeGetter)

		//setup fakes to correctly interact with commandregistry
		sshCodeGetter.SetDependencyStub = func(_ commandregistry.Dependency, _ bool) commandregistry.Command {
			return sshCodeGetter
		}
		sshCodeGetter.MetaDataReturns(commandregistry.CommandMetadata{Name: "ssh-code"})
	})

	AfterEach(func() {
		//restore original command
		commandregistry.Register(originalSSHCodeGetter)
	})

	updateCommandDependency := func(pluginCall bool) {
		deps.UI = ui
		deps.Config = configRepo

		//inject fake 'sshCodeGetter' into registry
		commandregistry.Register(sshCodeGetter)

		commandregistry.Commands.SetCommand(commandregistry.Commands.FindCommand("ssh").SetDependency(deps, pluginCall))
	}

	runCommand := func(args ...string) bool {
		return testcmd.RunCLICommand("ssh", args, requirementsFactory, updateCommandDependency, false, ui)
	}

	Describe("Requirements", func() {
		It("fails with usage when not provided exactly one arg", func() {
			requirementsFactory.NewLoginRequirementReturns(requirements.Passing{})

			runCommand()
			Expect(ui.Outputs()).To(ContainSubstrings(
				[]string{"Incorrect Usage", "Requires", "argument"},
			))

		})

		It("fails requirements when not logged in", func() {
			requirementsFactory.NewLoginRequirementReturns(requirements.Failing{Message: "not logged in"})
			Expect(runCommand("my-app")).To(BeFalse())
		})

		It("fails if a space is not targeted", func() {
			requirementsFactory.NewLoginRequirementReturns(requirements.Passing{})
			requirementsFactory.NewTargetedSpaceRequirementReturns(requirements.Failing{Message: "not targeting space"})
			Expect(runCommand("my-app")).To(BeFalse())
		})

		It("fails if a application is not found", func() {
			requirementsFactory.NewLoginRequirementReturns(requirements.Passing{})
			requirementsFactory.NewTargetedSpaceRequirementReturns(requirements.Passing{})
			applicationReq := new(requirementsfakes.FakeApplicationRequirement)
			applicationReq.ExecuteReturns(errors.New("no app"))
			requirementsFactory.NewApplicationRequirementReturns(applicationReq)

			Expect(runCommand("my-app")).To(BeFalse())
		})

		Describe("Flag options", func() {
			BeforeEach(func() {
				requirementsFactory.NewLoginRequirementReturns(requirements.Passing{})
				requirementsFactory.NewTargetedSpaceRequirementReturns(requirements.Passing{})
			})

			Context("when an -i flag is provided", func() {
				Context("with a negative integer argument", func() {
					It("returns an error", func() {
						Expect(runCommand("my-app", "-i", "-3")).To(BeFalse())
						Expect(ui.Outputs()).To(ContainSubstrings(
							[]string{"Incorrect Usage", "cannot be negative"},
						))
					})
				})

				Context("with a value greater than the application's highest instance index", func() {
					BeforeEach(func() {
						var app models.Application

						app = models.Application{}
						app.Name = "my-app"
						app.State = "started"
						app.GUID = "my-app-guid"
						app.EnableSSH = true
						app.Diego = true
						app.InstanceCount = 3

						applicationReq := new(requirementsfakes.FakeApplicationRequirement)
						applicationReq.GetApplicationReturns(app)
						requirementsFactory.NewApplicationRequirementReturns(applicationReq)
					})

					It("returns an error", func() {
						Expect(runCommand("my-app", "-i", "3")).To(BeFalse())
						Expect(ui.Outputs()).To(ContainSubstrings(
							[]string{"Incorrect Usage", "specified application instance does not exist"},
						))
					})
				})
			})
		})

		Describe("SSHOptions", func() {
			Context("when an error is returned during initialization", func() {
				It("shows error and prints command usage", func() {
					Expect(runCommand("app_name", "-L", "[9999:localhost...")).To(BeFalse())
					Expect(ui.Outputs()).To(ContainSubstrings(
						[]string{"Incorrect Usage"},
						[]string{"USAGE:"},
					))

				})
			})
		})

	})

	Describe("ssh", func() {
		var (
			currentApp models.Application
		)

		BeforeEach(func() {
			requirementsFactory.NewLoginRequirementReturns(requirements.Passing{})
			requirementsFactory.NewTargetedSpaceRequirementReturns(requirements.Passing{})
			currentApp = models.Application{}
			currentApp.Name = "my-app"
			currentApp.State = "started"
			currentApp.GUID = "my-app-guid"
			currentApp.EnableSSH = true
			currentApp.Diego = true

			applicationReq := new(requirementsfakes.FakeApplicationRequirement)
			applicationReq.GetApplicationReturns(currentApp)
			requirementsFactory.NewApplicationRequirementReturns(applicationReq)
		})

		Describe("Error getting required info to run ssh", func() {
			var (
				testServer *httptest.Server
				handler    *testnet.TestHandler
			)

			AfterEach(func() {
				testServer.Close()
			})

			Context("error when getting SSH info from /v2/info", func() {
				BeforeEach(func() {
					getRequest := apifakes.NewCloudControllerTestRequest(testnet.TestRequest{
						Method: "GET",
						Path:   "/v2/info",
						Response: testnet.TestResponse{
							Status: http.StatusNotFound,
							Body:   `{}`,
						},
					})

					testServer, handler = testnet.NewServer([]testnet.TestRequest{getRequest})
					configRepo.SetAPIEndpoint(testServer.URL)
					ccGateway = net.NewCloudControllerGateway(configRepo, time.Now, &testterm.FakeUI{}, new(tracefakes.FakePrinter), "")
					deps.Gateways["cloud-controller"] = ccGateway
				})

				It("notifies users", func() {
					runCommand("my-app")

					Expect(handler).To(HaveAllRequestsCalled())
					Expect(ui.Outputs()).To(ContainSubstrings(
						[]string{"Error getting SSH info", "404"},
					))

				})
			})

			Context("error when getting oauth token", func() {
				BeforeEach(func() {
					sshCodeGetter.GetReturns("", errors.New("auth api error"))

					getRequest := apifakes.NewCloudControllerTestRequest(testnet.TestRequest{
						Method: "GET",
						Path:   "/v2/info",
						Response: testnet.TestResponse{
							Status: http.StatusOK,
							Body:   `{}`,
						},
					})

					testServer, handler = testnet.NewServer([]testnet.TestRequest{getRequest})
					configRepo.SetAPIEndpoint(testServer.URL)
					ccGateway = net.NewCloudControllerGateway(configRepo, time.Now, &testterm.FakeUI{}, new(tracefakes.FakePrinter), "")
					deps.Gateways["cloud-controller"] = ccGateway
				})

				It("notifies users", func() {
					runCommand("my-app")

					Expect(handler).To(HaveAllRequestsCalled())
					Expect(ui.Outputs()).To(ContainSubstrings(
						[]string{"Error getting one time auth code", "auth api error"},
					))

				})
			})
		})

		Describe("Connecting to ssh server", func() {
			var testServer *httptest.Server

			AfterEach(func() {
				testServer.Close()
			})

			BeforeEach(func() {
				fakeSecureShell = new(sshfakes.FakeSecureShell)

				deps.WildcardDependency = fakeSecureShell

				getRequest := apifakes.NewCloudControllerTestRequest(testnet.TestRequest{
					Method: "GET",
					Path:   "/v2/info",
					Response: testnet.TestResponse{
						Status: http.StatusOK,
						Body:   getInfoResponseBody,
					},
				})

				testServer, _ = testnet.NewServer([]testnet.TestRequest{getRequest})
				configRepo.SetAPIEndpoint(testServer.URL)
				ccGateway = net.NewCloudControllerGateway(configRepo, time.Now, &testterm.FakeUI{}, new(tracefakes.FakePrinter), "")
				deps.Gateways["cloud-controller"] = ccGateway
			})

			Context("Error when connecting", func() {
				It("notifies users", func() {
					fakeSecureShell.ConnectReturns(errors.New("dial errorrr"))

					runCommand("my-app")

					Expect(ui.Outputs()).To(ContainSubstrings(
						[]string{"Error opening SSH connection", "dial error"},
					))

				})
			})

			Context("Error port forwarding when -L is provided", func() {
				It("notifies users", func() {
					fakeSecureShell.LocalPortForwardReturns(errors.New("listen error"))

					runCommand("my-app", "-L", "8000:localhost:8000")

					Expect(ui.Outputs()).To(ContainSubstrings(
						[]string{"Error forwarding port", "listen error"},
					))

				})
			})

			Context("when -N is provided", func() {
				It("calls secureShell.Wait()", func() {
					fakeSecureShell.ConnectReturns(nil)
					fakeSecureShell.LocalPortForwardReturns(nil)

					runCommand("my-app", "-N")

					Expect(fakeSecureShell.WaitCallCount()).To(Equal(1))
				})
			})

			Context("when -N is provided", func() {
				It("calls secureShell.InteractiveSession()", func() {
					fakeSecureShell.ConnectReturns(nil)
					fakeSecureShell.LocalPortForwardReturns(nil)

					runCommand("my-app", "-k")

					Expect(fakeSecureShell.InteractiveSessionCallCount()).To(Equal(1))
				})
			})

			Context("when Wait() or InteractiveSession() returns error", func() {

				It("notifities users", func() {
					fakeSecureShell.ConnectReturns(nil)
					fakeSecureShell.LocalPortForwardReturns(nil)

					fakeSecureShell.InteractiveSessionReturns(errors.New("ssh exit error"))
					runCommand("my-app", "-k")

					Expect(ui.Outputs()).To(ContainSubstrings(
						[]string{"ssh exit error"},
					))

				})
			})
		})
	})
})

const getInfoResponseBody string = `
{
   "name": "vcap",
   "build": "2222",
   "support": "http://support.cloudfoundry.com",
   "version": 2,
   "description": "Cloud Foundry sponsored by ABC",
   "authorization_endpoint": "https://login.run.abc.com",
   "token_endpoint": "https://uaa.run.abc.com",
   "min_cli_version": null,
   "min_recommended_cli_version": null,
   "api_version": "2.35.0",
   "app_ssh_endpoint": "ssh.run.pivotal.io:2222",
   "app_ssh_host_key_fingerprint": "11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11",
   "logging_endpoint": "wss://loggregator.run.abc.com:443",
   "doppler_logging_endpoint": "wss://doppler.run.abc.com:443",
   "user": "6e477566-ac8d-4653-98c6-d319595ec7b0"
}`
<|endoftext|>


<commit_before>package config

//go:generate go run templates_gen.go
//go:generate gofmt -w templates.go

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"strings"
	"text/template"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/kms"

	"github.com/coreos/coreos-cloudinit/config/validate"
	yaml "gopkg.in/yaml.v2"
)

const (
	credentialsDir = "credentials"
	userDataDir    = "userdata"
)

func newDefaultCluster() *Cluster {
	return &Cluster{
		ClusterName:              "kubernetes",
		ReleaseChannel:           "alpha",
		VPCCIDR:                  "10.0.0.0/16",
		InstanceCIDR:             "10.0.0.0/24",
		ControllerIP:             "10.0.0.50",
		PodCIDR:                  "10.2.0.0/16",
		ServiceCIDR:              "10.3.0.0/24",
		DNSServiceIP:             "10.3.0.10",
		K8sVer:                   "v1.1.8_coreos.0",
		HyperkubeImageRepo:       "quay.io/coreos/hyperkube",
		ControllerInstanceType:   "m3.medium",
		ControllerRootVolumeSize: 30,
		WorkerCount:              1,
		WorkerInstanceType:       "m3.medium",
		WorkerRootVolumeSize:     30,
	}
}

func ClusterFromFile(filename string) (*Cluster, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	c, err := clusterFromBytes(data)
	if err != nil {
		return nil, fmt.Errorf("file %s: %v", filename, err)
	}

	return c, nil
}

//Necessary for unit tests, which store configs as hardcoded strings
func clusterFromBytes(data []byte) (*Cluster, error) {
	c := newDefaultCluster()
	if err := yaml.Unmarshal(data, c); err != nil {
		return nil, fmt.Errorf("failed to parse cluster: %v", err)
	}
	if err := c.valid(); err != nil {
		return nil, fmt.Errorf("invalid cluster: %v", err)
	}
	return c, nil
}

type Cluster struct {
	ClusterName              string `yaml:"clusterName"`
	ExternalDNSName          string `yaml:"externalDNSName"`
	KeyName                  string `yaml:"keyName"`
	Region                   string `yaml:"region"`
	AvailabilityZone         string `yaml:"availabilityZone"`
	ReleaseChannel           string `yaml:"releaseChannel"`
	ControllerInstanceType   string `yaml:"controllerInstanceType"`
	ControllerRootVolumeSize int    `yaml:"controllerRootVolumeSize"`
	WorkerCount              int    `yaml:"workerCount"`
	WorkerInstanceType       string `yaml:"workerInstanceType"`
	WorkerRootVolumeSize     int    `yaml:"workerRootVolumeSize"`
	WorkerSpotPrice          string `yaml:"workerSpotPrice"`
	VPCCIDR                  string `yaml:"vpcCIDR"`
	InstanceCIDR             string `yaml:"instanceCIDR"`
	ControllerIP             string `yaml:"controllerIP"`
	PodCIDR                  string `yaml:"podCIDR"`
	ServiceCIDR              string `yaml:"serviceCIDR"`
	DNSServiceIP             string `yaml:"dnsServiceIP"`
	K8sVer                   string `yaml:"kubernetesVersion"`
	HyperkubeImageRepo       string `yaml:"hyperkubeImageRepo"`
	KMSKeyARN                string `yaml:"kmsKeyArn"`
}

func (c Cluster) Config() (*Config, error) {
	config := Config{Cluster: c}
	config.ETCDEndpoints = fmt.Sprintf("http://%s:2379", c.ControllerIP)
	config.APIServers = fmt.Sprintf("http://%s:8080", c.ControllerIP)
	config.SecureAPIServers = fmt.Sprintf("https://%s:443", c.ControllerIP)
	config.APIServerEndpoint = fmt.Sprintf("https://%s", c.ExternalDNSName)

	var err error
	if config.AMI, err = getAMI(config.Region, config.ReleaseChannel); err != nil {
		return nil, fmt.Errorf("failed getting AMI for config: %v", err)
	}

	return &config, nil
}

type StackTemplateOptions struct {
	TLSAssetsDir          string
	ControllerTmplFile    string
	WorkerTmplFile        string
	StackTemplateTmplFile string
}

type stackConfig struct {
	*Config
	UserDataWorker     string
	UserDataController string
}

func execute(filename string, data interface{}, compress bool) (string, error) {
	raw, err := ioutil.ReadFile(filename)
	if err != nil {
		return "", err
	}
	tmpl, err := template.New(filename).Parse(string(raw))
	if err != nil {
		return "", err
	}
	var buff bytes.Buffer
	if err := tmpl.Execute(&buff, data); err != nil {
		return "", err
	}
	if compress {
		return compressData(buff.Bytes())
	}
	return buff.String(), nil
}

func (c Cluster) stackConfig(opts StackTemplateOptions, compressUserData bool) (*stackConfig, error) {
	assets, err := ReadTLSAssets(opts.TLSAssetsDir)
	if err != nil {
		return nil, err
	}
	stackConfig := stackConfig{}

	if stackConfig.Config, err = c.Config(); err != nil {
		return nil, err
	}

	awsConfig := aws.NewConfig()
	awsConfig = awsConfig.WithRegion(stackConfig.Config.Region)
	kmsSvc := kms.New(session.New(awsConfig))

	compactAssets, err := assets.compact(stackConfig.Config, kmsSvc)
	if err != nil {
		return nil, fmt.Errorf("failed to compress TLS assets: %v", err)
	}

	stackConfig.Config.TLSConfig = compactAssets

	if stackConfig.UserDataWorker, err = execute(opts.WorkerTmplFile, stackConfig.Config, compressUserData); err != nil {
		return nil, fmt.Errorf("failed to render worker cloud config: %v", err)
	}
	if stackConfig.UserDataController, err = execute(opts.ControllerTmplFile, stackConfig.Config, compressUserData); err != nil {
		return nil, fmt.Errorf("failed to render controller cloud config: %v", err)
	}

	return &stackConfig, nil
}

func (c Cluster) ValidateUserData(opts StackTemplateOptions) error {
	stackConfig, err := c.stackConfig(opts, false)
	if err != nil {
		return err
	}

	errors := []string{}

	for _, userData := range []struct {
		Name    string
		Content string
	}{
		{
			Content: stackConfig.UserDataWorker,
			Name:    "UserDataWorker",
		},
		{
			Content: stackConfig.UserDataController,
			Name:    "UserDataController",
		},
	} {
		report, err := validate.Validate([]byte(userData.Content))

		if err != nil {
			errors = append(
				errors,
				fmt.Sprintf("cloud-config %s could not be parsed: %v",
					userData.Name,
					err,
				),
			)
			continue
		}

		for _, entry := range report.Entries() {
			errors = append(errors, fmt.Sprintf("%s: %+v", userData.Name, entry))
		}
	}

	if len(errors) > 0 {
		reportString := strings.Join(errors, "\n")
		return fmt.Errorf("cloud-config validation errors:\n%s\n", reportString)
	}

	return nil
}

func (c Cluster) RenderStackTemplate(opts StackTemplateOptions) ([]byte, error) {
	stackConfig, err := c.stackConfig(opts, true)
	if err != nil {
		return nil, err
	}

	rendered, err := execute(opts.StackTemplateTmplFile, stackConfig, false)
	if err != nil {
		return nil, err
	}
	// minify JSON
	var buff bytes.Buffer
	if err := json.Compact(&buff, []byte(rendered)); err != nil {
		return nil, err
	}
	return buff.Bytes(), nil
}

type Config struct {
	Cluster

	ETCDEndpoints     string
	APIServers        string
	SecureAPIServers  string
	APIServerEndpoint string
	AMI               string

	// Encoded TLS assets
	TLSConfig *CompactTLSAssets
}

func (cfg Cluster) valid() error {
	if cfg.ExternalDNSName == "" {
		return errors.New("externalDNSName must be set")
	}
	if cfg.KeyName == "" {
		return errors.New("keyName must be set")
	}
	if cfg.Region == "" {
		return errors.New("region must be set")
	}
	if cfg.AvailabilityZone == "" {
		return errors.New("availabilityZone must be set")
	}
	if cfg.ClusterName == "" {
		return errors.New("clusterName must be set")
	}
	if cfg.KMSKeyARN == "" {
		return errors.New("kmsKeyArn must be set")
	}

	vpcNetIP, vpcNet, err := net.ParseCIDR(cfg.VPCCIDR)
	if err != nil {
		return fmt.Errorf("invalid vpcCIDR: %v", err)
	}

	instancesNetIP, instancesNet, err := net.ParseCIDR(cfg.InstanceCIDR)
	if err != nil {
		return fmt.Errorf("invalid instanceCIDR: %v", err)
	}
	if !vpcNet.Contains(instancesNetIP) {
		return fmt.Errorf("vpcCIDR (%s) does not contain instanceCIDR (%s)",
			cfg.VPCCIDR,
			cfg.InstanceCIDR,
		)
	}

	controllerIPAddr := net.ParseIP(cfg.ControllerIP)
	if controllerIPAddr == nil {
		return fmt.Errorf("invalid controllerIP: %s", cfg.ControllerIP)
	}
	if !instancesNet.Contains(controllerIPAddr) {
		return fmt.Errorf("instanceCIDR (%s) does not contain controllerIP (%s)",
			cfg.InstanceCIDR,
			cfg.ControllerIP,
		)
	}

	podNetIP, podNet, err := net.ParseCIDR(cfg.PodCIDR)
	if err != nil {
		return fmt.Errorf("invalid podCIDR: %v", err)
	}
	if vpcNet.Contains(podNetIP) {
		return fmt.Errorf("vpcCIDR (%s) overlaps with podCIDR (%s)", cfg.VPCCIDR, cfg.PodCIDR)
	}

	serviceNetIP, serviceNet, err := net.ParseCIDR(cfg.ServiceCIDR)
	if err != nil {
		return fmt.Errorf("invalid serviceCIDR: %v", err)
	}
	if vpcNet.Contains(serviceNetIP) || serviceNet.Contains(vpcNetIP) {
		return fmt.Errorf("vpcCIDR (%s) overlaps with serviceCIDR (%s)", cfg.VPCCIDR, cfg.ServiceCIDR)
	}
	if vpcNet.Contains(podNetIP) || podNet.Contains(vpcNetIP) {
		return fmt.Errorf("vpcCIDR (%s) overlaps with podCIDR (%s)", cfg.VPCCIDR, cfg.PodCIDR)
	}
	if podNet.Contains(serviceNetIP) || serviceNet.Contains(podNetIP) {
		return fmt.Errorf("serviceCIDR (%s) overlaps with podCIDR (%s)", cfg.ServiceCIDR, cfg.PodCIDR)
	}

	kubernetesServiceIPAddr := incrementIP(serviceNet.IP)
	if !serviceNet.Contains(kubernetesServiceIPAddr) {
		return fmt.Errorf("serviceCIDR (%s) does not contain kubernetesServiceIP (%s)", cfg.ServiceCIDR, kubernetesServiceIPAddr)
	}

	dnsServiceIPAddr := net.ParseIP(cfg.DNSServiceIP)
	if dnsServiceIPAddr == nil {
		return fmt.Errorf("Invalid dnsServiceIP: %s", cfg.DNSServiceIP)
	}
	if !serviceNet.Contains(dnsServiceIPAddr) {
		return fmt.Errorf("serviceCIDR (%s) does not contain dnsServiceIP (%s)", cfg.ServiceCIDR, cfg.DNSServiceIP)
	}

	if dnsServiceIPAddr.Equal(kubernetesServiceIPAddr) {
		return fmt.Errorf("dnsServiceIp conflicts with kubernetesServiceIp (%s)", dnsServiceIPAddr)
	}

	return nil
}

//Return next IP address in network range
func incrementIP(netIP net.IP) net.IP {
	ip := make(net.IP, len(netIP))
	copy(ip, netIP)

	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}

	return ip
}
<commit_msg>kube-aws: improve cloudformation json validation<commit_after>package config

//go:generate go run templates_gen.go
//go:generate gofmt -w templates.go

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"strings"
	"text/template"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/kms"

	"github.com/coreos/coreos-cloudinit/config/validate"
	yaml "gopkg.in/yaml.v2"
)

const (
	credentialsDir = "credentials"
	userDataDir    = "userdata"
)

func newDefaultCluster() *Cluster {
	return &Cluster{
		ClusterName:              "kubernetes",
		ReleaseChannel:           "alpha",
		VPCCIDR:                  "10.0.0.0/16",
		InstanceCIDR:             "10.0.0.0/24",
		ControllerIP:             "10.0.0.50",
		PodCIDR:                  "10.2.0.0/16",
		ServiceCIDR:              "10.3.0.0/24",
		DNSServiceIP:             "10.3.0.10",
		K8sVer:                   "v1.1.8_coreos.0",
		HyperkubeImageRepo:       "quay.io/coreos/hyperkube",
		ControllerInstanceType:   "m3.medium",
		ControllerRootVolumeSize: 30,
		WorkerCount:              1,
		WorkerInstanceType:       "m3.medium",
		WorkerRootVolumeSize:     30,
	}
}

func ClusterFromFile(filename string) (*Cluster, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	c, err := clusterFromBytes(data)
	if err != nil {
		return nil, fmt.Errorf("file %s: %v", filename, err)
	}

	return c, nil
}

//Necessary for unit tests, which store configs as hardcoded strings
func clusterFromBytes(data []byte) (*Cluster, error) {
	c := newDefaultCluster()
	if err := yaml.Unmarshal(data, c); err != nil {
		return nil, fmt.Errorf("failed to parse cluster: %v", err)
	}
	if err := c.valid(); err != nil {
		return nil, fmt.Errorf("invalid cluster: %v", err)
	}
	return c, nil
}

type Cluster struct {
	ClusterName              string `yaml:"clusterName"`
	ExternalDNSName          string `yaml:"externalDNSName"`
	KeyName                  string `yaml:"keyName"`
	Region                   string `yaml:"region"`
	AvailabilityZone         string `yaml:"availabilityZone"`
	ReleaseChannel           string `yaml:"releaseChannel"`
	ControllerInstanceType   string `yaml:"controllerInstanceType"`
	ControllerRootVolumeSize int    `yaml:"controllerRootVolumeSize"`
	WorkerCount              int    `yaml:"workerCount"`
	WorkerInstanceType       string `yaml:"workerInstanceType"`
	WorkerRootVolumeSize     int    `yaml:"workerRootVolumeSize"`
	WorkerSpotPrice          string `yaml:"workerSpotPrice"`
	VPCCIDR                  string `yaml:"vpcCIDR"`
	InstanceCIDR             string `yaml:"instanceCIDR"`
	ControllerIP             string `yaml:"controllerIP"`
	PodCIDR                  string `yaml:"podCIDR"`
	ServiceCIDR              string `yaml:"serviceCIDR"`
	DNSServiceIP             string `yaml:"dnsServiceIP"`
	K8sVer                   string `yaml:"kubernetesVersion"`
	HyperkubeImageRepo       string `yaml:"hyperkubeImageRepo"`
	KMSKeyARN                string `yaml:"kmsKeyArn"`
}

func (c Cluster) Config() (*Config, error) {
	config := Config{Cluster: c}
	config.ETCDEndpoints = fmt.Sprintf("http://%s:2379", c.ControllerIP)
	config.APIServers = fmt.Sprintf("http://%s:8080", c.ControllerIP)
	config.SecureAPIServers = fmt.Sprintf("https://%s:443", c.ControllerIP)
	config.APIServerEndpoint = fmt.Sprintf("https://%s", c.ExternalDNSName)

	var err error
	if config.AMI, err = getAMI(config.Region, config.ReleaseChannel); err != nil {
		return nil, fmt.Errorf("failed getting AMI for config: %v", err)
	}

	return &config, nil
}

type StackTemplateOptions struct {
	TLSAssetsDir          string
	ControllerTmplFile    string
	WorkerTmplFile        string
	StackTemplateTmplFile string
}

type stackConfig struct {
	*Config
	UserDataWorker     string
	UserDataController string
}

func execute(filename string, data interface{}, compress bool) (string, error) {
	raw, err := ioutil.ReadFile(filename)
	if err != nil {
		return "", err
	}
	tmpl, err := template.New(filename).Parse(string(raw))
	if err != nil {
		return "", err
	}
	var buff bytes.Buffer
	if err := tmpl.Execute(&buff, data); err != nil {
		return "", err
	}
	if compress {
		return compressData(buff.Bytes())
	}
	return buff.String(), nil
}

func (c Cluster) stackConfig(opts StackTemplateOptions, compressUserData bool) (*stackConfig, error) {
	assets, err := ReadTLSAssets(opts.TLSAssetsDir)
	if err != nil {
		return nil, err
	}
	stackConfig := stackConfig{}

	if stackConfig.Config, err = c.Config(); err != nil {
		return nil, err
	}

	awsConfig := aws.NewConfig()
	awsConfig = awsConfig.WithRegion(stackConfig.Config.Region)
	kmsSvc := kms.New(session.New(awsConfig))

	compactAssets, err := assets.compact(stackConfig.Config, kmsSvc)
	if err != nil {
		return nil, fmt.Errorf("failed to compress TLS assets: %v", err)
	}

	stackConfig.Config.TLSConfig = compactAssets

	if stackConfig.UserDataWorker, err = execute(opts.WorkerTmplFile, stackConfig.Config, compressUserData); err != nil {
		return nil, fmt.Errorf("failed to render worker cloud config: %v", err)
	}
	if stackConfig.UserDataController, err = execute(opts.ControllerTmplFile, stackConfig.Config, compressUserData); err != nil {
		return nil, fmt.Errorf("failed to render controller cloud config: %v", err)
	}

	return &stackConfig, nil
}

func (c Cluster) ValidateUserData(opts StackTemplateOptions) error {
	stackConfig, err := c.stackConfig(opts, false)
	if err != nil {
		return err
	}

	errors := []string{}

	for _, userData := range []struct {
		Name    string
		Content string
	}{
		{
			Content: stackConfig.UserDataWorker,
			Name:    "UserDataWorker",
		},
		{
			Content: stackConfig.UserDataController,
			Name:    "UserDataController",
		},
	} {
		report, err := validate.Validate([]byte(userData.Content))

		if err != nil {
			errors = append(
				errors,
				fmt.Sprintf("cloud-config %s could not be parsed: %v",
					userData.Name,
					err,
				),
			)
			continue
		}

		for _, entry := range report.Entries() {
			errors = append(errors, fmt.Sprintf("%s: %+v", userData.Name, entry))
		}
	}

	if len(errors) > 0 {
		reportString := strings.Join(errors, "\n")
		return fmt.Errorf("cloud-config validation errors:\n%s\n", reportString)
	}

	return nil
}

func (c Cluster) RenderStackTemplate(opts StackTemplateOptions) ([]byte, error) {
	stackConfig, err := c.stackConfig(opts, true)
	if err != nil {
		return nil, err
	}

	rendered, err := execute(opts.StackTemplateTmplFile, stackConfig, false)
	if err != nil {
		return nil, err
	}

	//Use unmarshal function to do syntax validation
	renderedBytes := []byte(rendered)
	var jsonHolder map[string]interface{}
	if err := json.Unmarshal(renderedBytes, &jsonHolder); err != nil {
		syntaxError, ok := err.(*json.SyntaxError)
		if ok {
			contextString := getContextString(renderedBytes, int(syntaxError.Offset), 3)
			return nil, fmt.Errorf("%v:\njson syntax error (offset=%d), in this region:\n-------\n%s\n-------\n", err, syntaxError.Offset, contextString)
		}
		return nil, err
	}

	// minify JSON
	var buff bytes.Buffer
	if err := json.Compact(&buff, renderedBytes); err != nil {
		return nil, err
	}
	return buff.Bytes(), nil
}

func getContextString(buf []byte, offset, lineCount int) string {

	linesSeen := 0
	var leftLimit int
	for leftLimit = offset; leftLimit > 0 && linesSeen <= lineCount; leftLimit-- {
		if buf[leftLimit] == '\n' {
			linesSeen++
		}
	}

	linesSeen = 0
	var rightLimit int
	for rightLimit = offset + 1; rightLimit < len(buf) && linesSeen <= lineCount; rightLimit++ {
		if buf[rightLimit] == '\n' {
			linesSeen++
		}
	}

	return string(buf[leftLimit:rightLimit])
}

type Config struct {
	Cluster

	ETCDEndpoints     string
	APIServers        string
	SecureAPIServers  string
	APIServerEndpoint string
	AMI               string

	// Encoded TLS assets
	TLSConfig *CompactTLSAssets
}

func (cfg Cluster) valid() error {
	if cfg.ExternalDNSName == "" {
		return errors.New("externalDNSName must be set")
	}
	if cfg.KeyName == "" {
		return errors.New("keyName must be set")
	}
	if cfg.Region == "" {
		return errors.New("region must be set")
	}
	if cfg.AvailabilityZone == "" {
		return errors.New("availabilityZone must be set")
	}
	if cfg.ClusterName == "" {
		return errors.New("clusterName must be set")
	}
	if cfg.KMSKeyARN == "" {
		return errors.New("kmsKeyArn must be set")
	}

	vpcNetIP, vpcNet, err := net.ParseCIDR(cfg.VPCCIDR)
	if err != nil {
		return fmt.Errorf("invalid vpcCIDR: %v", err)
	}

	instancesNetIP, instancesNet, err := net.ParseCIDR(cfg.InstanceCIDR)
	if err != nil {
		return fmt.Errorf("invalid instanceCIDR: %v", err)
	}
	if !vpcNet.Contains(instancesNetIP) {
		return fmt.Errorf("vpcCIDR (%s) does not contain instanceCIDR (%s)",
			cfg.VPCCIDR,
			cfg.InstanceCIDR,
		)
	}

	controllerIPAddr := net.ParseIP(cfg.ControllerIP)
	if controllerIPAddr == nil {
		return fmt.Errorf("invalid controllerIP: %s", cfg.ControllerIP)
	}
	if !instancesNet.Contains(controllerIPAddr) {
		return fmt.Errorf("instanceCIDR (%s) does not contain controllerIP (%s)",
			cfg.InstanceCIDR,
			cfg.ControllerIP,
		)
	}

	podNetIP, podNet, err := net.ParseCIDR(cfg.PodCIDR)
	if err != nil {
		return fmt.Errorf("invalid podCIDR: %v", err)
	}
	if vpcNet.Contains(podNetIP) {
		return fmt.Errorf("vpcCIDR (%s) overlaps with podCIDR (%s)", cfg.VPCCIDR, cfg.PodCIDR)
	}

	serviceNetIP, serviceNet, err := net.ParseCIDR(cfg.ServiceCIDR)
	if err != nil {
		return fmt.Errorf("invalid serviceCIDR: %v", err)
	}
	if vpcNet.Contains(serviceNetIP) || serviceNet.Contains(vpcNetIP) {
		return fmt.Errorf("vpcCIDR (%s) overlaps with serviceCIDR (%s)", cfg.VPCCIDR, cfg.ServiceCIDR)
	}
	if vpcNet.Contains(podNetIP) || podNet.Contains(vpcNetIP) {
		return fmt.Errorf("vpcCIDR (%s) overlaps with podCIDR (%s)", cfg.VPCCIDR, cfg.PodCIDR)
	}
	if podNet.Contains(serviceNetIP) || serviceNet.Contains(podNetIP) {
		return fmt.Errorf("serviceCIDR (%s) overlaps with podCIDR (%s)", cfg.ServiceCIDR, cfg.PodCIDR)
	}

	kubernetesServiceIPAddr := incrementIP(serviceNet.IP)
	if !serviceNet.Contains(kubernetesServiceIPAddr) {
		return fmt.Errorf("serviceCIDR (%s) does not contain kubernetesServiceIP (%s)", cfg.ServiceCIDR, kubernetesServiceIPAddr)
	}

	dnsServiceIPAddr := net.ParseIP(cfg.DNSServiceIP)
	if dnsServiceIPAddr == nil {
		return fmt.Errorf("Invalid dnsServiceIP: %s", cfg.DNSServiceIP)
	}
	if !serviceNet.Contains(dnsServiceIPAddr) {
		return fmt.Errorf("serviceCIDR (%s) does not contain dnsServiceIP (%s)", cfg.ServiceCIDR, cfg.DNSServiceIP)
	}

	if dnsServiceIPAddr.Equal(kubernetesServiceIPAddr) {
		return fmt.Errorf("dnsServiceIp conflicts with kubernetesServiceIp (%s)", dnsServiceIPAddr)
	}

	return nil
}

//Return next IP address in network range
func incrementIP(netIP net.IP) net.IP {
	ip := make(net.IP, len(netIP))
	copy(ip, netIP)

	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}

	return ip
}
<|endoftext|>


<commit_before>package metadata_manager

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/samuel/go-zookeeper/zk"
	"strings"
	"time"
	"github.com/satori/go.uuid"
)

type MetadataManager struct {
	frameworkName           string
	setTaskUUIDChan 		chan setTaskUUIDRequest
	getTaskUUIDChan  	    chan getTaskUUIDRequest
	zkConn                  *zk.Conn
}

type setTaskUUIDRequest struct {
	oldUUID			string
	taskName     	string
	replyChannel	chan string
}

func (msg *setTaskUUIDRequest) Reply(response string) {
	msg.replyChannel <- response
}

type getTaskUUIDRequest struct {
	taskName     	string
	replyChannel	chan string
}

func (msg *getTaskUUIDRequest) Reply(response string) {
	msg.replyChannel <- response
}

func NewMetadataManager(frameworkName string, zookeeperAddr string) *MetadataManager {
	conn, _, err := zk.Connect([]string{zookeeperAddr}, time.Second*10)
	if err != nil {
		panic(err)
	}
	manager := &MetadataManager{
		frameworkName:           frameworkName,
		setTaskUUIDChan:         make(chan setTaskUUIDRequest, 1),
		getTaskUUIDChan:         make(chan getTaskUUIDRequest, 1),
		zkConn:                  conn,
	}

	go manager.loop()
	return manager
}
func (mgr *MetadataManager) createPathIfNotExists(path string) {
	splitString := strings.Split(path, "/")
	for idx := range splitString {
		if idx == 0 { continue }
		mgr.createIfNotExists(strings.Join(splitString[0:idx+1], "/"))
	}
}
func (mgr *MetadataManager) createIfNotExists(path string) {
	exists, _, err := mgr.zkConn.Exists(path)
	if err != nil {
		log.Panic(err)
	}
	if !exists {
		_, err := mgr.zkConn.Create(path, nil, 0, zk.WorldACL(zk.PermAll))
		if err != nil {
			log.Panic(err)
		}
	}
}
func (mgr *MetadataManager) loop() {
	defer close(mgr.setTaskUUIDChan)
	defer close(mgr.getTaskUUIDChan)
	path := fmt.Sprintf("/bletchley/frameworks/%s/tasks", mgr.frameworkName)
	mgr.createPathIfNotExists(path)
	for {
		select {
		case rq := <-mgr.setTaskUUIDChan: mgr.setTaskUUID(rq)
		case rq := <-mgr.getTaskUUIDChan: mgr.getTaskUUID(rq)

		}
	}
}

func (mgr *MetadataManager) getTaskUUID(rq getTaskUUIDRequest) {
	defer close(rq.replyChannel)

	path := fmt.Sprintf("/bletchley/frameworks/%s/tasks/%s/uuid", mgr.frameworkName, rq.taskName)

	exists, _, err := mgr.zkConn.Exists(path)
	if err != nil {
		log.Panic(err)
	}

	if exists {
		data, _, err := mgr.zkConn.Get(path)
		if err != nil {
			log.Panic(err)
		}
		zkUUID, err := uuid.FromBytes(data)
		if err != nil {
			log.Panic(err)
		}
		rq.Reply(zkUUID.String())
	} else {
		taskBasePath := fmt.Sprintf("/bletchley/frameworks/%s/tasks/%s", mgr.frameworkName, rq.taskName)
		mgr.createPathIfNotExists(taskBasePath)
		uuid := uuid.NewV4()
		_, err := mgr.zkConn.Create(path, uuid.Bytes(), 0, zk.WorldACL(zk.PermAll))
		if err != nil {
			log.Panic(err)
		}
		rq.Reply(uuid.String())
	}
}

func (mgr *MetadataManager) setTaskUUID(rq setTaskUUIDRequest) {
	defer close(rq.replyChannel)
	path := fmt.Sprintf("/bletchley/frameworks/%s/tasks/%s/uuid", mgr.frameworkName, rq.taskName)
	data, stat, err := mgr.zkConn.Get(path)
	if err != nil {
		log.Panic(err)
	}
	oldZKUUID, err := uuid.FromBytes(data)
	if err != nil {
		log.Panic(err)
	}
	oldTaskUUID, err := uuid.FromString(rq.oldUUID)
	if err != nil {
		log.Panic(err)
	}

	if !uuid.Equal(oldZKUUID, oldTaskUUID) { log.Panic("UUIDs not equal") }

	newUUID := uuid.NewV4()
	_, err = mgr.zkConn.Set(path, newUUID.Bytes(), stat.Version)
	if err != nil {
		log.Panic(err)
	}
	rq.Reply(newUUID.String())
}

func (mgr *MetadataManager) GetTaskUUID(taskName string) string {
	rq := getTaskUUIDRequest{
		replyChannel: make(chan string),
		taskName:     taskName,
	}
	mgr.getTaskUUIDChan <- rq
	retval := <-rq.replyChannel
	return retval
}

func (mgr *MetadataManager) SetTaskUUID(taskName string, oldUUID string) string {
	rq := setTaskUUIDRequest{
		replyChannel: make(chan string),
		taskName:     taskName,
		oldUUID:	  oldUUID,
	}
	mgr.setTaskUUIDChan <- rq
	retval := <-rq.replyChannel
	return retval
}
<commit_msg>Add beginning of the ability to add 'clusters'<commit_after>package metadata_manager

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/samuel/go-zookeeper/zk"
	"strings"
	"time"
	"github.com/satori/go.uuid"
)

type MetadataManager struct {
	frameworkName           string
	setTaskUUIDChan 		chan setTaskUUIDRequest
	getTaskUUIDChan  	    chan getTaskUUIDRequest
	addClusterChan			chan addClusterRequest
	zkConn                  *zk.Conn
}

type addClusterRequest struct {
	clusterName				string
	replyChannel			chan bool
}

func (msg *addClusterRequest) Reply(response bool) {
	msg.replyChannel <- response
}

type setTaskUUIDRequest struct {
	oldUUID			string
	taskName     	string
	replyChannel	chan string
}

func (msg *setTaskUUIDRequest) Reply(response string) {
	msg.replyChannel <- response
}

type getTaskUUIDRequest struct {
	taskName     	string
	replyChannel	chan string
}

func (msg *getTaskUUIDRequest) Reply(response string) {
	msg.replyChannel <- response
}

func NewMetadataManager(frameworkName string, zookeeperAddr string) *MetadataManager {
	conn, _, err := zk.Connect([]string{zookeeperAddr}, time.Second*10)
	if err != nil {
		panic(err)
	}
	manager := &MetadataManager{
		frameworkName:           frameworkName,
		setTaskUUIDChan:         make(chan setTaskUUIDRequest, 1),
		getTaskUUIDChan:         make(chan getTaskUUIDRequest, 1),
		addClusterChan:			 make(chan addClusterRequest, 1),
		zkConn:                  conn,
	}

	go manager.loop()
	return manager
}
func (mgr *MetadataManager) createPathIfNotExists(path string) {
	splitString := strings.Split(path, "/")
	for idx := range splitString {
		if idx == 0 { continue }
		mgr.createIfNotExists(strings.Join(splitString[0:idx+1], "/"))
	}
}
func (mgr *MetadataManager) createIfNotExists(path string) {
	exists, _, err := mgr.zkConn.Exists(path)
	if err != nil {
		log.Panic(err)
	}
	if !exists {
		_, err := mgr.zkConn.Create(path, nil, 0, zk.WorldACL(zk.PermAll))
		if err != nil {
			log.Panic(err)
		}
	}
}
func (mgr *MetadataManager) loop() {
	defer close(mgr.setTaskUUIDChan)
	defer close(mgr.getTaskUUIDChan)
	tasksPath := fmt.Sprintf("/bletchley/frameworks/%s/tasks", mgr.frameworkName)
	mgr.createPathIfNotExists(tasksPath)
	clustersPath := fmt.Sprintf("/bletchley/frameworks/%s/clusters", mgr.frameworkName)
	mgr.createPathIfNotExists(clustersPath)
	children, _, clusterEventChannel, err := mgr.zkConn.ChildrenW("/bletchley/frameworks/%s/clusters")

	if err != nil {
		log.Panic(err)
	}
	for child := range children {
		log.Info("Saw child: ", child)
	}

	for {
		select {
		case rq := <-mgr.setTaskUUIDChan: mgr.setTaskUUID(rq)
		case rq := <-mgr.getTaskUUIDChan: mgr.getTaskUUID(rq)
		case event := <- clusterEventChannel: { log.Info("Got cluster event: ", event) }
		case rq := <-mgr.addClusterChan: { log.Panic("not yet implemented: ", rq) }
		}
	}
}

func (mgr *MetadataManager) getTaskUUID(rq getTaskUUIDRequest) {
	defer close(rq.replyChannel)

	path := fmt.Sprintf("/bletchley/frameworks/%s/tasks/%s/uuid", mgr.frameworkName, rq.taskName)

	exists, _, err := mgr.zkConn.Exists(path)
	if err != nil {
		log.Panic(err)
	}

	if exists {
		data, _, err := mgr.zkConn.Get(path)
		if err != nil {
			log.Panic(err)
		}
		zkUUID, err := uuid.FromBytes(data)
		if err != nil {
			log.Panic(err)
		}
		rq.Reply(zkUUID.String())
	} else {
		taskBasePath := fmt.Sprintf("/bletchley/frameworks/%s/tasks/%s", mgr.frameworkName, rq.taskName)
		mgr.createPathIfNotExists(taskBasePath)
		uuid := uuid.NewV4()
		_, err := mgr.zkConn.Create(path, uuid.Bytes(), 0, zk.WorldACL(zk.PermAll))
		if err != nil {
			log.Panic(err)
		}
		rq.Reply(uuid.String())
	}
}

func (mgr *MetadataManager) setTaskUUID(rq setTaskUUIDRequest) {
	defer close(rq.replyChannel)
	path := fmt.Sprintf("/bletchley/frameworks/%s/tasks/%s/uuid", mgr.frameworkName, rq.taskName)
	data, stat, err := mgr.zkConn.Get(path)
	if err != nil {
		log.Panic(err)
	}
	oldZKUUID, err := uuid.FromBytes(data)
	if err != nil {
		log.Panic(err)
	}
	oldTaskUUID, err := uuid.FromString(rq.oldUUID)
	if err != nil {
		log.Panic(err)
	}

	if !uuid.Equal(oldZKUUID, oldTaskUUID) { log.Panic("UUIDs not equal") }

	newUUID := uuid.NewV4()
	_, err = mgr.zkConn.Set(path, newUUID.Bytes(), stat.Version)
	if err != nil {
		log.Panic(err)
	}
	rq.Reply(newUUID.String())
}

func (mgr *MetadataManager) GetTaskUUID(taskName string) string {
	rq := getTaskUUIDRequest{
		replyChannel: make(chan string),
		taskName:     taskName,
	}
	mgr.getTaskUUIDChan <- rq
	retval := <-rq.replyChannel
	return retval
}

func (mgr *MetadataManager) SetTaskUUID(taskName string, oldUUID string) string {
	rq := setTaskUUIDRequest{
		replyChannel: make(chan string),
		taskName:     taskName,
		oldUUID:	  oldUUID,
	}
	mgr.setTaskUUIDChan <- rq
	retval := <-rq.replyChannel
	return retval
}

func (mgr *MetadataManager) AddCluster(clusterName string) bool {
	rq := addClusterRequest{
		replyChannel: 	make(chan bool),
		clusterName:	clusterName,
	}
	mgr.addClusterChan <- rq
	retval := <-rq.replyChannel
	return retval
}<|endoftext|>