<commit_before>//go:build !windows && !plan9 && !js

package main

import (
	"bytes"
	_ "embed"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/creack/pty"
	"src.elv.sh/pkg/sys/eunix"
	"src.elv.sh/pkg/ui"
)

const (
	terminalRows = 100
	terminalCols = 58
)

var promptMarker = "[PROMPT]"

//go:embed rc.elv
var rcElv string

// Creates a temporary home directory for running tmux and elvish in. The caller
// is responsible for removing the directory.
func setupHome() (string, error) {
	homePath, err := os.MkdirTemp("", "ttyshot-*")
	if err != nil {
		return "", fmt.Errorf("create temp home: %w", err)
	}

	// The temporary directory may include symlinks in the path. Expand them so
	// that commands like tilde-abbr behaves as expected.
	resolvedHomePath, err := filepath.EvalSymlinks(homePath)
	if err != nil {
		return homePath, fmt.Errorf("resolve symlinks in homePath: %w", err)
	}
	homePath = resolvedHomePath

	err = ApplyDir(Dir{
		// Directories to be used in navigation mode.
		"bash": Dir{},
		"elvish": Dir{
			"1.0-release.md":  "1.0 has not been released yet.",
			"CONTRIBUTING.md": "",
			"Dockerfile":      "",
			"LICENSE":         "",
			"Makefile":        "",
			"PACKAGING.md":    "",
			"README.md":       "",
			"SECURITY.md":     "",
			"cmd":             Dir{},
			"go.mod":          "",
			"go.sum":          "",
			"pkg":             Dir{},
			"syntaxes":        Dir{},
			"tools":           Dir{},
			"vscode":          Dir{},
			"website":         Dir{},
		},
		"zsh": Dir{},

		// Will keep tmux and elvish's sockets, and raw output of capture-pane
		".tmp": Dir{},

		".config": Dir{
			"elvish": Dir{
				"rc.elv": rcElv,
			},
		},
	}, homePath)
	return homePath, err
}

func createTtyshot(homePath string, script []op, saveRaw string) ([]byte, error) {
	ctrl, tty, err := pty.Open()
	if err != nil {
		return nil, err
	}
	defer ctrl.Close()
	defer tty.Close()
	winsize := pty.Winsize{Rows: terminalRows, Cols: terminalCols}
	pty.Setsize(ctrl, &winsize)

	rawPath := filepath.Join(homePath, ".tmp", "ttyshot.raw")
	if saveRaw != "" {
		saveRaw, err := filepath.Abs(saveRaw)
		if err != nil {
			return nil, fmt.Errorf("resolve path to raw dump file: %w", err)
		}
		os.Symlink(saveRaw, rawPath)
	}

	doneCh, err := spawnElvish(homePath, tty)
	if err != nil {
		return nil, err
	}
	executeScript(script, ctrl)

	err = <-doneCh
	if err != nil {
		return nil, err
	}

	rawBytes, err := os.ReadFile(rawPath)
	if err != nil {
		return nil, err
	}

	ttyshot := string(rawBytes)
	ttyshot = strings.TrimRight(ttyshot, "\n")
	ttyshot = strings.ReplaceAll(ttyshot, promptMarker+"\n", "")
	return []byte(sgrTextToHTML(ttyshot) + "\n"), nil
}

func spawnElvish(homePath string, tty *os.File) (<-chan error, error) {
	elvishPath, err := exec.LookPath("elvish")
	if err != nil {
		return nil, fmt.Errorf("find elvish: %w", err)
	}
	tmuxPath, err := exec.LookPath("tmux")
	if err != nil {
		return nil, fmt.Errorf("find tmux: %w", err)
	}

	tmuxSock := filepath.Join(homePath, ".tmp", "tmux.sock")
	elvSock := filepath.Join(homePath, ".tmp", "elv.sock")

	// Start tmux and have it start a hermetic Elvish session.
	tmuxCmd := exec.Cmd{
		Path: tmuxPath,
		Args: []string{
			tmuxPath,
			"-S", tmuxSock, "-f", "/dev/null", "-u", "-T", "256,RGB",
			"new-session", elvishPath, "-sock", elvSock},
		Dir: homePath,
		Env: []string{
			"HOME=" + homePath,
			"PATH=" + os.Getenv("PATH"),
			// The actual value doesn't matter here, as long as it can be looked
			// up in terminfo. We rely on the -T flag above to force tmux to
			// support certain terminal features.
			"TERM=xterm",
		},
		Stdin:  tty,
		Stdout: tty,
		Stderr: tty,
	}

	doneCh := make(chan error)
	go func() {
		doneCh <- tmuxCmd.Run()
	}()

	return doneCh, nil
}

func executeScript(script []op, ctrl *os.File) {
	implicitEnter := true
	for _, op := range script {
		switch op.typ {
		case opText:
			text := op.val.(string)
			ctrl.WriteString(text)
			if implicitEnter {
				ctrl.Write([]byte{'\r'})
			}
		case opAlt:
			ctrl.Write([]byte{'\033', op.val.(byte)})
		case opCtrl:
			ctrl.Write([]byte{op.val.(byte) & 0x1F})
		case opEnter:
			ctrl.Write([]byte{'\r'})
			implicitEnter = true
		case opUp:
			ctrl.Write([]byte{'\033', '[', 'A'})
		case opDown:
			ctrl.Write([]byte{'\033', '[', 'B'})
		case opRight:
			ctrl.Write([]byte{'\033', '[', 'C'})
		case opLeft:
			ctrl.Write([]byte{'\033', '[', 'D'})
		case opNoEnter:
			implicitEnter = false
		case opWaitForPrompt:
			waitForOutput(ctrl, promptMarker,
				func(bs []byte) bool { return bytes.HasSuffix(bs, []byte(promptMarker)) })
		default:
			panic("unhandled op")
		}
	}
	// Alt-q is bound to a function that captures the content of the pane and
	// exits
	ctrl.Write([]byte{'\033', 'q'})
}

func waitForOutput(f *os.File, expected string, matcher func([]byte) bool) error {
	var buf bytes.Buffer
	// It shouldn't take more than a couple of seconds to see the expected
	// output, so use a timeout an order of magnitude longer to allow for
	// overloaded systems.
	deadline := time.Now().Add(30 * time.Second)
	for {
		budget := time.Until(deadline)
		if budget <= 0 {
			break
		}
		ready, err := eunix.WaitForRead(budget, f)
		if err != nil {
			return fmt.Errorf("waiting for tmux output: %w", err)
		}
		if !ready[0] {
			break
		}
		_, err = io.CopyN(&buf, f, 1)
		if err != nil {
			return fmt.Errorf("reading tmux output: %w", err)
		}
		if matcher(buf.Bytes()) {
			return nil
		}
	}
	return fmt.Errorf("timed out waiting for %s in tmux output; output so far: %q", expected, buf)
}

var htmlEscaper = strings.NewReplacer("&", "&", "<", "<", ">", ">")

func sgrTextToHTML(ttyshot string) string {
	t := ui.ParseSGREscapedText(ttyshot)

	var sb strings.Builder
	for i, line := range t.SplitByRune('\n') {
		if i > 0 {
			sb.WriteRune('\n')
		}
		for j, seg := range line {
			style := seg.Style
			var classes []string
			if style.Inverse {
				// The inverse attribute means that the foreground and
				// background colors should be swapped, which cannot be
				// expressed in pure CSS. To work around this, this code swaps
				// the foreground and background colors, and uses two special
				// CSS classes to indicate that the foreground/background should
				// take the inverse of the default color.
				style.Inverse = false
				style.Foreground, style.Background = style.Background, style.Foreground
				if style.Foreground == nil {
					classes = append(classes, "sgr-7fg")
				}
				if style.Background == nil {
					classes = append(classes, "sgr-7bg")
				}
			}

			for _, c := range style.SGRValues() {
				classes = append(classes, "sgr-"+c)
			}
			text := seg.Text
			// We pass -N to tmux capture-pane in order to correctly preserve
			// trailing spaces that have background colors. However, this
			// preserves unstyled trailing spaces too, which makes the ttyshot
			// harder to copy-paste, so strip it.
			if len(classes) == 0 && j == len(line)-1 {
				text = strings.TrimRight(text, " ")
			}
			if text == "" {
				continue
			}
			escapedText := htmlEscaper.Replace(text)
			if len(classes) == 0 {
				sb.WriteString(escapedText)
			} else {
				fmt.Fprintf(&sb, `<span class="%s">%s</span>`, strings.Join(classes, " "), escapedText)
			}
		}
	}

	return sb.String()
}
<commit_msg>website/cmd/ttyshot: Drain tmux output before waiting for it to exit.<commit_after>//go:build !windows && !plan9 && !js

package main

import (
	"bytes"
	_ "embed"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/creack/pty"
	"src.elv.sh/pkg/sys/eunix"
	"src.elv.sh/pkg/ui"
)

const (
	terminalRows = 100
	terminalCols = 58
)

var promptMarker = "[PROMPT]"

//go:embed rc.elv
var rcElv string

// Creates a temporary home directory for running tmux and elvish in. The caller
// is responsible for removing the directory.
func setupHome() (string, error) {
	homePath, err := os.MkdirTemp("", "ttyshot-*")
	if err != nil {
		return "", fmt.Errorf("create temp home: %w", err)
	}

	// The temporary directory may include symlinks in the path. Expand them so
	// that commands like tilde-abbr behaves as expected.
	resolvedHomePath, err := filepath.EvalSymlinks(homePath)
	if err != nil {
		return homePath, fmt.Errorf("resolve symlinks in homePath: %w", err)
	}
	homePath = resolvedHomePath

	err = ApplyDir(Dir{
		// Directories to be used in navigation mode.
		"bash": Dir{},
		"elvish": Dir{
			"1.0-release.md":  "1.0 has not been released yet.",
			"CONTRIBUTING.md": "",
			"Dockerfile":      "",
			"LICENSE":         "",
			"Makefile":        "",
			"PACKAGING.md":    "",
			"README.md":       "",
			"SECURITY.md":     "",
			"cmd":             Dir{},
			"go.mod":          "",
			"go.sum":          "",
			"pkg":             Dir{},
			"syntaxes":        Dir{},
			"tools":           Dir{},
			"vscode":          Dir{},
			"website":         Dir{},
		},
		"zsh": Dir{},

		// Will keep tmux and elvish's sockets, and raw output of capture-pane
		".tmp": Dir{},

		".config": Dir{
			"elvish": Dir{
				"rc.elv": rcElv,
			},
		},
	}, homePath)
	return homePath, err
}

func createTtyshot(homePath string, script []op, saveRaw string) ([]byte, error) {
	ctrl, tty, err := pty.Open()
	if err != nil {
		return nil, err
	}
	defer ctrl.Close()
	defer tty.Close()
	winsize := pty.Winsize{Rows: terminalRows, Cols: terminalCols}
	pty.Setsize(ctrl, &winsize)

	rawPath := filepath.Join(homePath, ".tmp", "ttyshot.raw")
	if saveRaw != "" {
		saveRaw, err := filepath.Abs(saveRaw)
		if err != nil {
			return nil, fmt.Errorf("resolve path to raw dump file: %w", err)
		}
		os.Symlink(saveRaw, rawPath)
	}

	doneCh, err := spawnElvish(homePath, tty)
	if err != nil {
		return nil, err
	}
	executeScript(script, ctrl)

	// Drain outputs from the terminal. This is needed so that tmux can exit
	// properly without blocking on flushing outputs.
	go io.Copy(io.Discard, ctrl)

	err = <-doneCh
	if err != nil {
		return nil, err
	}

	rawBytes, err := os.ReadFile(rawPath)
	if err != nil {
		return nil, err
	}

	ttyshot := string(rawBytes)
	ttyshot = strings.TrimRight(ttyshot, "\n")
	ttyshot = strings.ReplaceAll(ttyshot, promptMarker+"\n", "")
	return []byte(sgrTextToHTML(ttyshot) + "\n"), nil
}

func spawnElvish(homePath string, tty *os.File) (<-chan error, error) {
	elvishPath, err := exec.LookPath("elvish")
	if err != nil {
		return nil, fmt.Errorf("find elvish: %w", err)
	}
	tmuxPath, err := exec.LookPath("tmux")
	if err != nil {
		return nil, fmt.Errorf("find tmux: %w", err)
	}

	tmuxSock := filepath.Join(homePath, ".tmp", "tmux.sock")
	elvSock := filepath.Join(homePath, ".tmp", "elv.sock")

	// Start tmux and have it start a hermetic Elvish session.
	tmuxCmd := exec.Cmd{
		Path: tmuxPath,
		Args: []string{
			tmuxPath,
			"-S", tmuxSock, "-f", "/dev/null", "-u", "-T", "256,RGB",
			"new-session", elvishPath, "-sock", elvSock},
		Dir: homePath,
		Env: []string{
			"HOME=" + homePath,
			"PATH=" + os.Getenv("PATH"),
			// The actual value doesn't matter here, as long as it can be looked
			// up in terminfo. We rely on the -T flag above to force tmux to
			// support certain terminal features.
			"TERM=xterm",
		},
		Stdin:  tty,
		Stdout: tty,
		Stderr: tty,
	}

	doneCh := make(chan error)
	go func() {
		doneCh <- tmuxCmd.Run()
	}()

	return doneCh, nil
}

func executeScript(script []op, ctrl *os.File) {
	implicitEnter := true
	for _, op := range script {
		switch op.typ {
		case opText:
			text := op.val.(string)
			ctrl.WriteString(text)
			if implicitEnter {
				ctrl.Write([]byte{'\r'})
			}
		case opAlt:
			ctrl.Write([]byte{'\033', op.val.(byte)})
		case opCtrl:
			ctrl.Write([]byte{op.val.(byte) & 0x1F})
		case opEnter:
			ctrl.Write([]byte{'\r'})
			implicitEnter = true
		case opUp:
			ctrl.Write([]byte{'\033', '[', 'A'})
		case opDown:
			ctrl.Write([]byte{'\033', '[', 'B'})
		case opRight:
			ctrl.Write([]byte{'\033', '[', 'C'})
		case opLeft:
			ctrl.Write([]byte{'\033', '[', 'D'})
		case opNoEnter:
			implicitEnter = false
		case opWaitForPrompt:
			waitForOutput(ctrl, promptMarker,
				func(bs []byte) bool { return bytes.HasSuffix(bs, []byte(promptMarker)) })
		default:
			panic("unhandled op")
		}
	}
	// Alt-q is bound to a function that captures the content of the pane and
	// exits
	ctrl.Write([]byte{'\033', 'q'})
}

func waitForOutput(f *os.File, expected string, matcher func([]byte) bool) error {
	var buf bytes.Buffer
	// It shouldn't take more than a couple of seconds to see the expected
	// output, so use a timeout an order of magnitude longer to allow for
	// overloaded systems.
	deadline := time.Now().Add(30 * time.Second)
	for {
		budget := time.Until(deadline)
		if budget <= 0 {
			break
		}
		ready, err := eunix.WaitForRead(budget, f)
		if err != nil {
			return fmt.Errorf("waiting for tmux output: %w", err)
		}
		if !ready[0] {
			break
		}
		_, err = io.CopyN(&buf, f, 1)
		if err != nil {
			return fmt.Errorf("reading tmux output: %w", err)
		}
		if matcher(buf.Bytes()) {
			return nil
		}
	}
	return fmt.Errorf("timed out waiting for %s in tmux output; output so far: %q", expected, buf)
}

var htmlEscaper = strings.NewReplacer("&", "&", "<", "<", ">", ">")

func sgrTextToHTML(ttyshot string) string {
	t := ui.ParseSGREscapedText(ttyshot)

	var sb strings.Builder
	for i, line := range t.SplitByRune('\n') {
		if i > 0 {
			sb.WriteRune('\n')
		}
		for j, seg := range line {
			style := seg.Style
			var classes []string
			if style.Inverse {
				// The inverse attribute means that the foreground and
				// background colors should be swapped, which cannot be
				// expressed in pure CSS. To work around this, this code swaps
				// the foreground and background colors, and uses two special
				// CSS classes to indicate that the foreground/background should
				// take the inverse of the default color.
				style.Inverse = false
				style.Foreground, style.Background = style.Background, style.Foreground
				if style.Foreground == nil {
					classes = append(classes, "sgr-7fg")
				}
				if style.Background == nil {
					classes = append(classes, "sgr-7bg")
				}
			}

			for _, c := range style.SGRValues() {
				classes = append(classes, "sgr-"+c)
			}
			text := seg.Text
			// We pass -N to tmux capture-pane in order to correctly preserve
			// trailing spaces that have background colors. However, this
			// preserves unstyled trailing spaces too, which makes the ttyshot
			// harder to copy-paste, so strip it.
			if len(classes) == 0 && j == len(line)-1 {
				text = strings.TrimRight(text, " ")
			}
			if text == "" {
				continue
			}
			escapedText := htmlEscaper.Replace(text)
			if len(classes) == 0 {
				sb.WriteString(escapedText)
			} else {
				fmt.Fprintf(&sb, `<span class="%s">%s</span>`, strings.Join(classes, " "), escapedText)
			}
		}
	}

	return sb.String()
}
<|endoftext|>


<commit_before>// Copyright 2018 The OpenPitrix Authors. All rights reserved.
// Use of this source code is governed by a Apache license
// that can be found in the LICENSE file.

// +build integration

package test

import (
	"net/url"
	"testing"

	"github.com/stretchr/testify/require"

	"openpitrix.io/openpitrix/pkg/constants"
	"openpitrix.io/openpitrix/pkg/util/idutil"
	"openpitrix.io/openpitrix/test/client/runtime_manager"
	"openpitrix.io/openpitrix/test/models"
)

func getRuntimeCredential(t *testing.T) string {
	return execCmd(t, "kubectl config view --flatten")
}

func TestRuntime(t *testing.T) {
	credential := getRuntimeCredential(t)

	client := GetClient(clientConfig)

	testRuntimeName := "e2e-test-runtime"
	describeParams := runtime_manager.NewDescribeRuntimesParams()
	describeParams.SetSearchWord(&testRuntimeName)
	describeParams.SetStatus([]string{constants.StatusActive})
	describeResp, err := client.RuntimeManager.DescribeRuntimes(describeParams, nil)
	require.NoError(t, err)
	runtimes := describeResp.Payload.RuntimeSet
	for _, runtime := range runtimes {
		deleteParams := runtime_manager.NewDeleteRuntimesParams()
		deleteParams.SetBody(
			&models.OpenpitrixDeleteRuntimesRequest{
				RuntimeID: []string{runtime.RuntimeID},
			})
		_, err := client.RuntimeManager.DeleteRuntimes(deleteParams, nil)
		require.NoError(t, err)
	}
	// create runtime
	createParams := runtime_manager.NewCreateRuntimeParams()
	createParams.SetBody(
		&models.OpenpitrixCreateRuntimeRequest{
			Name:              testRuntimeName,
			Description:       "description",
			Provider:          constants.ProviderKubernetes,
			RuntimeURL:        "",
			RuntimeCredential: credential,
			Zone:              "default",
		})
	createResp, err := client.RuntimeManager.CreateRuntime(createParams, nil)
	require.NoError(t, err)
	runtimeId := createResp.Payload.RuntimeID
	// modify runtime
	modifyParams := runtime_manager.NewModifyRuntimeParams()
	modifyParams.SetBody(
		&models.OpenpitrixModifyRuntimeRequest{
			RuntimeID:   runtimeId,
			Description: "cc",
		})
	modifyResp, err := client.RuntimeManager.ModifyRuntime(modifyParams, nil)
	require.NoError(t, err)
	t.Log(modifyResp)
	// describe runtime
	describeParams.WithRuntimeID([]string{runtimeId})
	describeResp, err = client.RuntimeManager.DescribeRuntimes(describeParams, nil)
	require.NoError(t, err)
	runtimes = describeResp.Payload.RuntimeSet
	if len(runtimes) != 1 {
		t.Fatalf("failed to describe runtimes with params [%+v]", describeParams)
	}
	if runtimes[0].Name != testRuntimeName || runtimes[0].Description != "cc" {
		t.Fatalf("failed to modify runtime [%+v]", runtimes[0])
	}
	// delete runtime
	deleteParams := runtime_manager.NewDeleteRuntimesParams()
	deleteParams.WithBody(&models.OpenpitrixDeleteRuntimesRequest{
		RuntimeID: []string{runtimeId},
	})
	deleteResp, err := client.RuntimeManager.DeleteRuntimes(deleteParams, nil)
	require.NoError(t, err)
	t.Log(deleteResp)
	// describe deleted runtime
	describeParams.WithRuntimeID([]string{runtimeId})
	describeParams.WithStatus([]string{constants.StatusDeleted})
	describeParams.WithSearchWord(nil)
	describeResp, err = client.RuntimeManager.DescribeRuntimes(describeParams, nil)
	require.NoError(t, err)
	runtimes = describeResp.Payload.RuntimeSet
	runtimes = describeResp.Payload.RuntimeSet
	if len(runtimes) != 1 {
		t.Fatalf("failed to describe runtimes with params [%+v]", describeParams)
	}
	runtime := runtimes[0]
	if runtime.RuntimeID != runtimeId {
		t.Fatalf("failed to describe runtime")
	}
	if runtime.Status != constants.StatusDeleted {
		t.Fatalf("failed to delete runtime, got runtime status [%s]", runtime.Status)
	}

	t.Log("test runtime finish, all test is ok")
}

func generateLabels() string {
	v := url.Values{}
	v.Add("key1", idutil.GetUuid(""))
	v.Add("key2", idutil.GetUuid(""))
	v.Add("key3", idutil.GetUuid(""))
	v.Add("key4", idutil.GetUuid(""))
	v.Add("key5", idutil.GetUuid(""))
	return v.Encode()
}

func TestRuntimeLabel(t *testing.T) {
	credential := getRuntimeCredential(t)
	client := GetClient(clientConfig)
	// Create a test runtime that can attach label on it
	testRuntimeName := "e2e-test-runtime"
	labels := generateLabels()
	createParams := runtime_manager.NewCreateRuntimeParams()
	createParams.SetBody(
		&models.OpenpitrixCreateRuntimeRequest{
			Name:              testRuntimeName,
			Description:       "description",
			Provider:          constants.ProviderKubernetes,
			RuntimeURL:        "",
			RuntimeCredential: credential,
			Zone:              "default",
			Labels:            labels,
		})
	createResp, err := client.RuntimeManager.CreateRuntime(createParams, nil)
	require.NoError(t, err)
	runtimeId := createResp.Payload.RuntimeID

	describeParams := runtime_manager.NewDescribeRuntimesParams()
	describeParams.Label = &labels
	describeParams.Status = []string{constants.StatusActive}
	describeResp, err := client.RuntimeManager.DescribeRuntimes(describeParams, nil)
	require.NoError(t, err)
	if len(describeResp.Payload.RuntimeSet) != 1 {
		t.Fatalf("describe runtime with filter failed")
	}
	if describeResp.Payload.RuntimeSet[0].RuntimeID != runtimeId {
		t.Fatalf("describe runtime with filter failed")
	}

	// delete runtime
	deleteParams := runtime_manager.NewDeleteRuntimesParams()
	deleteParams.WithBody(&models.OpenpitrixDeleteRuntimesRequest{
		RuntimeID: []string{runtimeId},
	})
	deleteResp, err := client.RuntimeManager.DeleteRuntimes(deleteParams, nil)
	require.NoError(t, err)
	t.Log(deleteResp)

	t.Log("test runtime label finish, all test is ok")
}
<commit_msg>fix ci-test error<commit_after>// Copyright 2018 The OpenPitrix Authors. All rights reserved.
// Use of this source code is governed by a Apache license
// that can be found in the LICENSE file.

// +build integration

package test

import (
	"net/url"
	"testing"

	"github.com/stretchr/testify/require"

	"openpitrix.io/openpitrix/pkg/constants"
	"openpitrix.io/openpitrix/pkg/util/idutil"
	"openpitrix.io/openpitrix/test/client/runtime_manager"
	"openpitrix.io/openpitrix/test/models"
)

func getRuntimeCredential(t *testing.T) string {
	return execCmd(t, "kubectl config view --flatten")
}

func TestRuntime(t *testing.T) {
	credential := getRuntimeCredential(t)

	client := GetClient(clientConfig)

	testRuntimeName := "e2e-test-runtime"
	describeParams := runtime_manager.NewDescribeRuntimesParams()
	describeParams.SetSearchWord(&testRuntimeName)
	describeParams.SetStatus([]string{constants.StatusActive})
	describeResp, err := client.RuntimeManager.DescribeRuntimes(describeParams, nil)
	require.NoError(t, err)
	runtimes := describeResp.Payload.RuntimeSet
	for _, runtime := range runtimes {
		deleteParams := runtime_manager.NewDeleteRuntimesParams()
		deleteParams.SetBody(
			&models.OpenpitrixDeleteRuntimesRequest{
				RuntimeID: []string{runtime.RuntimeID},
			})
		_, err := client.RuntimeManager.DeleteRuntimes(deleteParams, nil)
		require.NoError(t, err)
	}
	// create runtime
	createParams := runtime_manager.NewCreateRuntimeParams()
	createParams.SetBody(
		&models.OpenpitrixCreateRuntimeRequest{
			Name:              testRuntimeName,
			Description:       "description",
			Provider:          constants.ProviderKubernetes,
			RuntimeURL:        "",
			RuntimeCredential: credential,
			Zone:              idutil.GetUuid36("r-"),
		})
	createResp, err := client.RuntimeManager.CreateRuntime(createParams, nil)
	require.NoError(t, err)
	runtimeId := createResp.Payload.RuntimeID
	// modify runtime
	modifyParams := runtime_manager.NewModifyRuntimeParams()
	modifyParams.SetBody(
		&models.OpenpitrixModifyRuntimeRequest{
			RuntimeID:   runtimeId,
			Description: "cc",
		})
	modifyResp, err := client.RuntimeManager.ModifyRuntime(modifyParams, nil)
	require.NoError(t, err)
	t.Log(modifyResp)
	// describe runtime
	describeParams.WithRuntimeID([]string{runtimeId})
	describeResp, err = client.RuntimeManager.DescribeRuntimes(describeParams, nil)
	require.NoError(t, err)
	runtimes = describeResp.Payload.RuntimeSet
	if len(runtimes) != 1 {
		t.Fatalf("failed to describe runtimes with params [%+v]", describeParams)
	}
	if runtimes[0].Name != testRuntimeName || runtimes[0].Description != "cc" {
		t.Fatalf("failed to modify runtime [%+v]", runtimes[0])
	}
	// delete runtime
	deleteParams := runtime_manager.NewDeleteRuntimesParams()
	deleteParams.WithBody(&models.OpenpitrixDeleteRuntimesRequest{
		RuntimeID: []string{runtimeId},
	})
	deleteResp, err := client.RuntimeManager.DeleteRuntimes(deleteParams, nil)
	require.NoError(t, err)
	t.Log(deleteResp)
	// describe deleted runtime
	describeParams.WithRuntimeID([]string{runtimeId})
	describeParams.WithStatus([]string{constants.StatusDeleted})
	describeParams.WithSearchWord(nil)
	describeResp, err = client.RuntimeManager.DescribeRuntimes(describeParams, nil)
	require.NoError(t, err)
	runtimes = describeResp.Payload.RuntimeSet
	runtimes = describeResp.Payload.RuntimeSet
	if len(runtimes) != 1 {
		t.Fatalf("failed to describe runtimes with params [%+v]", describeParams)
	}
	runtime := runtimes[0]
	if runtime.RuntimeID != runtimeId {
		t.Fatalf("failed to describe runtime")
	}
	if runtime.Status != constants.StatusDeleted {
		t.Fatalf("failed to delete runtime, got runtime status [%s]", runtime.Status)
	}

	t.Log("test runtime finish, all test is ok")
}

func generateLabels() string {
	v := url.Values{}
	v.Add("key1", idutil.GetUuid(""))
	v.Add("key2", idutil.GetUuid(""))
	v.Add("key3", idutil.GetUuid(""))
	v.Add("key4", idutil.GetUuid(""))
	v.Add("key5", idutil.GetUuid(""))
	return v.Encode()
}

func TestRuntimeLabel(t *testing.T) {
	credential := getRuntimeCredential(t)
	client := GetClient(clientConfig)
	// Create a test runtime that can attach label on it
	testRuntimeName := "e2e-test-runtime"
	labels := generateLabels()
	createParams := runtime_manager.NewCreateRuntimeParams()
	createParams.SetBody(
		&models.OpenpitrixCreateRuntimeRequest{
			Name:              testRuntimeName,
			Description:       "description",
			Provider:          constants.ProviderKubernetes,
			RuntimeURL:        "",
			RuntimeCredential: credential,
			Zone:              idutil.GetUuid36("r-"),
			Labels:            labels,
		})
	createResp, err := client.RuntimeManager.CreateRuntime(createParams, nil)
	require.NoError(t, err)
	runtimeId := createResp.Payload.RuntimeID

	describeParams := runtime_manager.NewDescribeRuntimesParams()
	describeParams.Label = &labels
	describeParams.Status = []string{constants.StatusActive}
	describeResp, err := client.RuntimeManager.DescribeRuntimes(describeParams, nil)
	require.NoError(t, err)
	if len(describeResp.Payload.RuntimeSet) != 1 {
		t.Fatalf("describe runtime with filter failed")
	}
	if describeResp.Payload.RuntimeSet[0].RuntimeID != runtimeId {
		t.Fatalf("describe runtime with filter failed")
	}

	// delete runtime
	deleteParams := runtime_manager.NewDeleteRuntimesParams()
	deleteParams.WithBody(&models.OpenpitrixDeleteRuntimesRequest{
		RuntimeID: []string{runtimeId},
	})
	deleteResp, err := client.RuntimeManager.DeleteRuntimes(deleteParams, nil)
	require.NoError(t, err)
	t.Log(deleteResp)

	t.Log("test runtime label finish, all test is ok")
}
<|endoftext|>


<commit_before>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package fake

import (
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/discovery"
	fakediscovery "k8s.io/client-go/discovery/fake"
	"k8s.io/client-go/testing"
	clientset "k8s.io/metrics/pkg/client/clientset/versioned"
	metricsv1alpha1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1"
	fakemetricsv1alpha1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1/fake"
	metricsv1beta1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1"
	fakemetricsv1beta1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/fake"
)

// NewSimpleClientset returns a clientset that will respond with the provided objects.
// It's backed by a very simple object tracker that processes creates, updates and deletions as-is,
// without applying any validations and/or defaults. It shouldn't be considered a replacement
// for a real clientset and is mostly useful in simple unit tests.
func NewSimpleClientset(objects ...runtime.Object) *Clientset {
	o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder())
	for _, obj := range objects {
		if err := o.Add(obj); err != nil {
			panic(err)
		}
	}

	cs := &Clientset{tracker: o}
	cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake}
	cs.AddReactor("*", "*", testing.ObjectReaction(o))
	cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
		gvr := action.GetResource()
		ns := action.GetNamespace()
		watch, err := o.Watch(gvr, ns)
		if err != nil {
			return false, nil, err
		}
		return true, watch, nil
	})

	return cs
}

// Clientset implements clientset.Interface. Meant to be embedded into a
// struct to get a default implementation. This makes faking out just the method
// you want to test easier.
type Clientset struct {
	testing.Fake
	discovery *fakediscovery.FakeDiscovery
	tracker   testing.ObjectTracker
}

func (c *Clientset) Discovery() discovery.DiscoveryInterface {
	return c.discovery
}

func (c *Clientset) Tracker() testing.ObjectTracker {
	return c.tracker
}

var _ clientset.Interface = &Clientset{}

// MetricsV1alpha1 retrieves the MetricsV1alpha1Client
func (c *Clientset) MetricsV1alpha1() metricsv1alpha1.MetricsV1alpha1Interface {
	return &fakemetricsv1alpha1.FakeMetricsV1alpha1{Fake: &c.Fake}
}

// MetricsV1beta1 retrieves the MetricsV1beta1Client
func (c *Clientset) MetricsV1beta1() metricsv1beta1.MetricsV1beta1Interface {
	return &fakemetricsv1beta1.FakeMetricsV1beta1{Fake: &c.Fake}
}
<commit_msg>Implement a FakeClient interface<commit_after>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package fake

import (
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/discovery"
	fakediscovery "k8s.io/client-go/discovery/fake"
	"k8s.io/client-go/testing"
	clientset "k8s.io/metrics/pkg/client/clientset/versioned"
	metricsv1alpha1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1"
	fakemetricsv1alpha1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1/fake"
	metricsv1beta1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1"
	fakemetricsv1beta1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/fake"
)

// NewSimpleClientset returns a clientset that will respond with the provided objects.
// It's backed by a very simple object tracker that processes creates, updates and deletions as-is,
// without applying any validations and/or defaults. It shouldn't be considered a replacement
// for a real clientset and is mostly useful in simple unit tests.
func NewSimpleClientset(objects ...runtime.Object) *Clientset {
	o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder())
	for _, obj := range objects {
		if err := o.Add(obj); err != nil {
			panic(err)
		}
	}

	cs := &Clientset{tracker: o}
	cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake}
	cs.AddReactor("*", "*", testing.ObjectReaction(o))
	cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
		gvr := action.GetResource()
		ns := action.GetNamespace()
		watch, err := o.Watch(gvr, ns)
		if err != nil {
			return false, nil, err
		}
		return true, watch, nil
	})

	return cs
}

// Clientset implements clientset.Interface. Meant to be embedded into a
// struct to get a default implementation. This makes faking out just the method
// you want to test easier.
type Clientset struct {
	testing.Fake
	discovery *fakediscovery.FakeDiscovery
	tracker   testing.ObjectTracker
}

func (c *Clientset) Discovery() discovery.DiscoveryInterface {
	return c.discovery
}

func (c *Clientset) Tracker() testing.ObjectTracker {
	return c.tracker
}

var (
	_ clientset.Interface = &Clientset{}
	_ testing.FakeClient  = &Clientset{}
)

// MetricsV1alpha1 retrieves the MetricsV1alpha1Client
func (c *Clientset) MetricsV1alpha1() metricsv1alpha1.MetricsV1alpha1Interface {
	return &fakemetricsv1alpha1.FakeMetricsV1alpha1{Fake: &c.Fake}
}

// MetricsV1beta1 retrieves the MetricsV1beta1Client
func (c *Clientset) MetricsV1beta1() metricsv1beta1.MetricsV1beta1Interface {
	return &fakemetricsv1beta1.FakeMetricsV1beta1{Fake: &c.Fake}
}
<|endoftext|>


<commit_before>package generator

var ClusterTemplate = `{{ if ne .clusterTarget.CurrentTarget "none" }}

<source>
  @type  tail
  path  /var/lib/rancher/rke/log/*.log
  pos_file  /fluentd/etc/log/fluentd-rke-logging.pos
  time_format  %Y-%m-%dT%H:%M:%S
  tag  rke.*
  format  json
  read_from_head  true
</source>

<filter rke.**>
  @type record_transformer
  enable_ruby true
  <record>
    tag ${tag}
    log_type k8s_infrastructure_container
    driver rke
    component ${tag_suffix[6].split("_")[0]}
    container_id ${tag_suffix[6].split(".")[0]}
  </record>
</filter>

<source>
   @type  tail
   path  /var/log/containers/*.log
   pos_file  /fluentd/etc/log/fluentd-cluster-logging.pos
   time_format  %Y-%m-%dT%H:%M:%S
   tag  cluster.*
   format  json
   read_from_head  true
</source>

<filter  cluster.**>
   @type  kubernetes_metadata
   merge_json_log  true
   preserve_json_log  true
</filter>

<filter cluster.**>
  @type record_transformer
  <record>
    tag ${tag}
    log_type k8s_normal_container
    {{range $k, $val := .clusterTarget.OutputTags -}}
    {{$k}} {{$val}}
    {{end -}}
  </record>
</filter>

<match  cluster.** rke.**>
    {{ if eq .clusterTarget.CurrentTarget "embedded"}}
    @type elasticsearch
    include_tag_key  true
    hosts "elasticsearch.cattle-logging:9200"
    reload_connections "true"
    logstash_prefix {{.clusterTarget.ClusterName}}
    logstash_format true
    logstash_dateformat  {{.clusterTarget.WrapEmbedded.DateFormat}}
    type_name  "container_log"
    reload_connections false
    {{end -}}

    {{ if eq .clusterTarget.CurrentTarget "elasticsearch"}}
    @type elasticsearch
    include_tag_key  true
    {{ if and .clusterTarget.ElasticsearchConfig.AuthUserName .clusterTarget.ElasticsearchConfig.AuthPassword}}
    hosts {{.clusterTarget.WrapElasticsearch.Scheme}}://{{.clusterTarget.ElasticsearchConfig.AuthUserName}}:{{.clusterTarget.ElasticsearchConfig.AuthPassword}}@{{.clusterTarget.WrapElasticsearch.Host}}
    {{else -}}
    hosts {{.clusterTarget.ElasticsearchConfig.Endpoint}}
    {{end -}}

    reload_connections "true"
    logstash_prefix "{{.clusterTarget.ElasticsearchConfig.IndexPrefix}}"
    logstash_format true
    logstash_dateformat  {{.clusterTarget.WrapElasticsearch.DateFormat}}
    type_name  "container_log"
    reload_connections false
    {{end -}}

    {{ if eq .clusterTarget.CurrentTarget "splunk"}}
    @type splunk-http-eventcollector
    server  {{.clusterTarget.WrapSplunk.Server}}
    all_items true
    protocol {{.clusterTarget.WrapSplunk.Scheme}}
    sourcetype {{.clusterTarget.SplunkConfig.Source}}
    token {{.clusterTarget.SplunkConfig.Token}}
    format json
    reload_connections "true"
    {{end -}}

    {{ if eq .clusterTarget.CurrentTarget "kafka"}}
    @type kafka_buffered
    {{ if .clusterTarget.KafkaConfig.ZookeeperEndpoint }}
    zookeeper {{.clusterTarget.WrapKafka.Zookeeper}}
    {{else}}
    brokers {{.clusterTarget.WrapKafka.Brokers}}
    {{end}}
    default_topic {{.clusterTarget.KafkaConfig.Topic}}
    output_data_type  "json"
    output_include_tag true
    output_include_time true
    # get_kafka_client_log  true
    max_send_retries  3
    {{end -}}

    {{ if eq .clusterTarget.CurrentTarget "syslog"}}
    @type remote_syslog
    host {{.clusterTarget.WrapSyslog.Host}}
    port {{.clusterTarget.WrapSyslog.Port}}
    severity {{.clusterTarget.SyslogConfig.Severity}}
    program {{.clusterTarget.SyslogConfig.Program}}
    {{end -}}

    flush_interval 2s
    buffer_type file
    buffer_path /fluentd/etc/buffer/cluster.buffer
    buffer_queue_limit 128
    buffer_chunk_limit 256m
    max_retry_wait 30
    disable_retry_limit
    num_threads 8
    slow_flush_log_threshold 40.0
</match>
{{end -}}
`
<commit_msg>fix embedded index name<commit_after>package generator

var ClusterTemplate = `{{ if ne .clusterTarget.CurrentTarget "none" }}

<source>
  @type  tail
  path  /var/lib/rancher/rke/log/*.log
  pos_file  /fluentd/etc/log/fluentd-rke-logging.pos
  time_format  %Y-%m-%dT%H:%M:%S
  tag  rke.*
  format  json
  read_from_head  true
</source>

<filter rke.**>
  @type record_transformer
  enable_ruby true
  <record>
    tag ${tag}
    log_type k8s_infrastructure_container
    driver rke
    component ${tag_suffix[6].split("_")[0]}
    container_id ${tag_suffix[6].split(".")[0]}
  </record>
</filter>

<source>
   @type  tail
   path  /var/log/containers/*.log
   pos_file  /fluentd/etc/log/fluentd-cluster-logging.pos
   time_format  %Y-%m-%dT%H:%M:%S
   tag  cluster.*
   format  json
   read_from_head  true
</source>

<filter  cluster.**>
   @type  kubernetes_metadata
   merge_json_log  true
   preserve_json_log  true
</filter>

<filter cluster.**>
  @type record_transformer
  <record>
    tag ${tag}
    log_type k8s_normal_container
    {{range $k, $val := .clusterTarget.OutputTags -}}
    {{$k}} {{$val}}
    {{end -}}
  </record>
</filter>

<match  cluster.** rke.**>
    {{ if eq .clusterTarget.CurrentTarget "embedded"}}
    @type elasticsearch
    include_tag_key  true
    hosts "elasticsearch.cattle-logging:9200"
    reload_connections "true"
    logstash_prefix {{.clusterTarget.EmbeddedConfig.IndexPrefix}}
    logstash_format true
    logstash_dateformat  {{.clusterTarget.WrapEmbedded.DateFormat}}
    type_name  "container_log"
    reload_connections false
    {{end -}}

    {{ if eq .clusterTarget.CurrentTarget "elasticsearch"}}
    @type elasticsearch
    include_tag_key  true
    {{ if and .clusterTarget.ElasticsearchConfig.AuthUserName .clusterTarget.ElasticsearchConfig.AuthPassword}}
    hosts {{.clusterTarget.WrapElasticsearch.Scheme}}://{{.clusterTarget.ElasticsearchConfig.AuthUserName}}:{{.clusterTarget.ElasticsearchConfig.AuthPassword}}@{{.clusterTarget.WrapElasticsearch.Host}}
    {{else -}}
    hosts {{.clusterTarget.ElasticsearchConfig.Endpoint}}
    {{end -}}

    reload_connections "true"
    logstash_prefix "{{.clusterTarget.ElasticsearchConfig.IndexPrefix}}"
    logstash_format true
    logstash_dateformat  {{.clusterTarget.WrapElasticsearch.DateFormat}}
    type_name  "container_log"
    reload_connections false
    {{end -}}

    {{ if eq .clusterTarget.CurrentTarget "splunk"}}
    @type splunk-http-eventcollector
    server  {{.clusterTarget.WrapSplunk.Server}}
    all_items true
    protocol {{.clusterTarget.WrapSplunk.Scheme}}
    sourcetype {{.clusterTarget.SplunkConfig.Source}}
    token {{.clusterTarget.SplunkConfig.Token}}
    format json
    reload_connections "true"
    {{end -}}

    {{ if eq .clusterTarget.CurrentTarget "kafka"}}
    @type kafka_buffered
    {{ if .clusterTarget.KafkaConfig.ZookeeperEndpoint }}
    zookeeper {{.clusterTarget.WrapKafka.Zookeeper}}
    {{else}}
    brokers {{.clusterTarget.WrapKafka.Brokers}}
    {{end}}
    default_topic {{.clusterTarget.KafkaConfig.Topic}}
    output_data_type  "json"
    output_include_tag true
    output_include_time true
    # get_kafka_client_log  true
    max_send_retries  3
    {{end -}}

    {{ if eq .clusterTarget.CurrentTarget "syslog"}}
    @type remote_syslog
    host {{.clusterTarget.WrapSyslog.Host}}
    port {{.clusterTarget.WrapSyslog.Port}}
    severity {{.clusterTarget.SyslogConfig.Severity}}
    program {{.clusterTarget.SyslogConfig.Program}}
    {{end -}}

    flush_interval 2s
    buffer_type file
    buffer_path /fluentd/etc/buffer/cluster.buffer
    buffer_queue_limit 128
    buffer_chunk_limit 256m
    max_retry_wait 30
    disable_retry_limit
    num_threads 8
    slow_flush_log_threshold 40.0
</match>
{{end -}}
`
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package knativeserving

import (
	"context"

	mf "github.com/jcrossley3/manifestival"
	"go.uber.org/zap"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/errors"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/tools/cache"

	"knative.dev/pkg/controller"
	servingv1alpha1 "knative.dev/serving-operator/pkg/apis/serving/v1alpha1"
	listers "knative.dev/serving-operator/pkg/client/listers/serving/v1alpha1"
	"knative.dev/serving-operator/pkg/reconciler"
	"knative.dev/serving-operator/pkg/reconciler/knativeserving/common"
	"knative.dev/serving-operator/version"
)

var (
	// Platform-specific behavior to affect the installation
	platform common.Platforms
)

// Reconciler implements controller.Reconciler for Knativeserving resources.
type Reconciler struct {
	*reconciler.Base
	// Listers index properties about resources
	knativeServingLister listers.KnativeServingLister
	config               mf.Manifest
	servings             sets.String
}

// Check that our Reconciler implements controller.Reconciler
var _ controller.Reconciler = (*Reconciler)(nil)

// Reconcile compares the actual state with the desired, and attempts to
// converge the two. It then updates the Status block of the Knativeserving resource
// with the current status of the resource.
func (r *Reconciler) Reconcile(ctx context.Context, key string) error {
	// Convert the namespace/name string into a distinct namespace and name
	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		r.Logger.Errorf("invalid resource key: %s", key)
		return nil
	}
	// Get the KnativeServing resource with this namespace/name.
	original, err := r.knativeServingLister.KnativeServings(namespace).Get(name)
	if apierrs.IsNotFound(err) {
		// The resource was deleted
		r.servings.Delete(key)
		if r.servings.Len() == 0 {
			r.config.DeleteAll(&metav1.DeleteOptions{})
		}
		return nil

	} else if err != nil {
		r.Logger.Error(err, "Error getting KnativeServing")
		return err
	}
	// Keep track of the number of KnativeServings in the cluster
	r.servings.Insert(key)

	// Don't modify the informers copy.
	knativeServing := original.DeepCopy()

	// Reconcile this copy of the KnativeServing resource and then write back any status
	// updates regardless of whether the reconciliation errored out.
	reconcileErr := r.reconcile(ctx, knativeServing)
	if equality.Semantic.DeepEqual(original.Status, knativeServing.Status) {
		// If we didn't change anything then don't call updateStatus.
		// This is important because the copy we loaded from the informer's
		// cache may be stale and we don't want to overwrite a prior update
		// to status with this stale state.
	} else if err = r.updateStatus(knativeServing); err != nil {
		r.Logger.Warnw("Failed to update knativeServing status", zap.Error(err))
		r.Recorder.Eventf(knativeServing, corev1.EventTypeWarning, "UpdateFailed",
			"Failed to update status for KnativeServing %q: %v", knativeServing.Name, err)
		return err
	}
	if reconcileErr != nil {
		r.Recorder.Event(knativeServing, corev1.EventTypeWarning, "InternalError", reconcileErr.Error())
		return reconcileErr
	}
	return nil
}

func (r *Reconciler) reconcile(ctx context.Context, ks *servingv1alpha1.KnativeServing) error {
	reqLogger := r.Logger.With(zap.String("Request.Namespace", ks.Namespace)).With("Request.Name", ks.Name)
	reqLogger.Infow("Reconciling KnativeServing", "status", ks.Status)

	stages := []func(*mf.Manifest, *servingv1alpha1.KnativeServing) error{
		r.initStatus,
		r.install,
		r.checkDeployments,
		r.deleteObsoleteResources,
	}

	manifest, err := r.transform(ks)
	if err != nil {
		return err
	}

	for _, stage := range stages {
		if err := stage(manifest, ks); err != nil {
			return err
		}
	}
	reqLogger.Infow("Reconcile stages complete", "status", ks.Status)
	return nil
}

// Transform the resources
func (r *Reconciler) transform(instance *servingv1alpha1.KnativeServing) (*mf.Manifest, error) {
	r.Logger.Debug("Transforming manifest")
	transforms, err := platform.Transformers(r.KubeClientSet, instance, r.Logger)
	if err != nil {
		return nil, err
	}
	return r.config.Transform(transforms...)
}

// Update the status subresource
func (r *Reconciler) updateStatus(instance *servingv1alpha1.KnativeServing) error {
	afterUpdate, err := r.KnativeServingClientSet.OperatorV1alpha1().KnativeServings(instance.Namespace).UpdateStatus(instance)

	if err != nil {
		return err
	}
	// TODO: We shouldn't rely on mutability and return the updated entities from functions instead.
	afterUpdate.DeepCopyInto(instance)
	return nil
}

// Initialize status conditions
func (r *Reconciler) initStatus(_ *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	r.Logger.Debug("Initializing status")
	if len(instance.Status.Conditions) == 0 {
		instance.Status.InitializeConditions()
		if err := r.updateStatus(instance); err != nil {
			return err
		}
	}
	return nil
}

// Apply the manifest resources
func (r *Reconciler) install(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	r.Logger.Debug("Installing manifest")
	if err := manifest.ApplyAll(); err != nil {
		instance.Status.MarkInstallFailed(err.Error())
		return err
	}
	instance.Status.MarkInstallSucceeded()
	instance.Status.Version = version.Version
	return nil
}

// Check for all deployments available
func (r *Reconciler) checkDeployments(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	r.Logger.Debug("Checking deployments")
	defer r.updateStatus(instance)
	available := func(d *appsv1.Deployment) bool {
		for _, c := range d.Status.Conditions {
			if c.Type == appsv1.DeploymentAvailable && c.Status == corev1.ConditionTrue {
				return true
			}
		}
		return false
	}
	for _, u := range manifest.Resources {
		if u.GetKind() == "Deployment" {
			deployment, err := r.KubeClientSet.AppsV1().Deployments(u.GetNamespace()).Get(u.GetName(), metav1.GetOptions{})
			if err != nil {
				instance.Status.MarkDeploymentsNotReady()
				if errors.IsNotFound(err) {
					return nil
				}
				return err
			}
			if !available(deployment) {
				instance.Status.MarkDeploymentsNotReady()
				return nil
			}
		}
	}
	instance.Status.MarkDeploymentsAvailable()
	return nil
}

// Delete obsolete resources from previous versions
func (r *Reconciler) deleteObsoleteResources(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	// istio-system resources from 0.3
	resource := &unstructured.Unstructured{}
	resource.SetNamespace("istio-system")
	resource.SetName("knative-ingressgateway")
	resource.SetAPIVersion("v1")
	resource.SetKind("Service")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	resource.SetAPIVersion("apps/v1")
	resource.SetKind("Deployment")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	resource.SetAPIVersion("autoscaling/v1")
	resource.SetKind("HorizontalPodAutoscaler")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	// config-controller from 0.5
	resource.SetNamespace(instance.GetNamespace())
	resource.SetName("config-controller")
	resource.SetAPIVersion("v1")
	resource.SetKind("ConfigMap")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	return nil
}
<commit_msg>We need to fail the install if any transforms error (#212)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package knativeserving

import (
	"context"

	mf "github.com/jcrossley3/manifestival"
	"go.uber.org/zap"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/errors"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/tools/cache"

	"knative.dev/pkg/controller"
	servingv1alpha1 "knative.dev/serving-operator/pkg/apis/serving/v1alpha1"
	listers "knative.dev/serving-operator/pkg/client/listers/serving/v1alpha1"
	"knative.dev/serving-operator/pkg/reconciler"
	"knative.dev/serving-operator/pkg/reconciler/knativeserving/common"
	"knative.dev/serving-operator/version"
)

var (
	// Platform-specific behavior to affect the installation
	platform common.Platforms
)

// Reconciler implements controller.Reconciler for Knativeserving resources.
type Reconciler struct {
	*reconciler.Base
	// Listers index properties about resources
	knativeServingLister listers.KnativeServingLister
	config               mf.Manifest
	servings             sets.String
}

// Check that our Reconciler implements controller.Reconciler
var _ controller.Reconciler = (*Reconciler)(nil)

// Reconcile compares the actual state with the desired, and attempts to
// converge the two. It then updates the Status block of the Knativeserving resource
// with the current status of the resource.
func (r *Reconciler) Reconcile(ctx context.Context, key string) error {
	// Convert the namespace/name string into a distinct namespace and name
	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		r.Logger.Errorf("invalid resource key: %s", key)
		return nil
	}
	// Get the KnativeServing resource with this namespace/name.
	original, err := r.knativeServingLister.KnativeServings(namespace).Get(name)
	if apierrs.IsNotFound(err) {
		// The resource was deleted
		r.servings.Delete(key)
		if r.servings.Len() == 0 {
			r.config.DeleteAll(&metav1.DeleteOptions{})
		}
		return nil

	} else if err != nil {
		r.Logger.Error(err, "Error getting KnativeServing")
		return err
	}
	// Keep track of the number of KnativeServings in the cluster
	r.servings.Insert(key)

	// Don't modify the informers copy.
	knativeServing := original.DeepCopy()

	// Reconcile this copy of the KnativeServing resource and then write back any status
	// updates regardless of whether the reconciliation errored out.
	reconcileErr := r.reconcile(ctx, knativeServing)
	if equality.Semantic.DeepEqual(original.Status, knativeServing.Status) {
		// If we didn't change anything then don't call updateStatus.
		// This is important because the copy we loaded from the informer's
		// cache may be stale and we don't want to overwrite a prior update
		// to status with this stale state.
	} else if err = r.updateStatus(knativeServing); err != nil {
		r.Logger.Warnw("Failed to update knativeServing status", zap.Error(err))
		r.Recorder.Eventf(knativeServing, corev1.EventTypeWarning, "UpdateFailed",
			"Failed to update status for KnativeServing %q: %v", knativeServing.Name, err)
		return err
	}
	if reconcileErr != nil {
		r.Recorder.Event(knativeServing, corev1.EventTypeWarning, "InternalError", reconcileErr.Error())
		return reconcileErr
	}
	return nil
}

func (r *Reconciler) reconcile(ctx context.Context, ks *servingv1alpha1.KnativeServing) error {
	reqLogger := r.Logger.With(zap.String("Request.Namespace", ks.Namespace)).With("Request.Name", ks.Name)
	reqLogger.Infow("Reconciling KnativeServing", "status", ks.Status)

	stages := []func(*mf.Manifest, *servingv1alpha1.KnativeServing) error{
		r.initStatus,
		r.install,
		r.checkDeployments,
		r.deleteObsoleteResources,
	}

	manifest, err := r.transform(ks)
	if err != nil {
		ks.Status.MarkInstallFailed(err.Error())
		return err
	}

	for _, stage := range stages {
		if err := stage(manifest, ks); err != nil {
			return err
		}
	}
	reqLogger.Infow("Reconcile stages complete", "status", ks.Status)
	return nil
}

// Transform the resources
func (r *Reconciler) transform(instance *servingv1alpha1.KnativeServing) (*mf.Manifest, error) {
	r.Logger.Debug("Transforming manifest")
	transforms, err := platform.Transformers(r.KubeClientSet, instance, r.Logger)
	if err != nil {
		return nil, err
	}
	return r.config.Transform(transforms...)
}

// Update the status subresource
func (r *Reconciler) updateStatus(instance *servingv1alpha1.KnativeServing) error {
	afterUpdate, err := r.KnativeServingClientSet.OperatorV1alpha1().KnativeServings(instance.Namespace).UpdateStatus(instance)

	if err != nil {
		return err
	}
	// TODO: We shouldn't rely on mutability and return the updated entities from functions instead.
	afterUpdate.DeepCopyInto(instance)
	return nil
}

// Initialize status conditions
func (r *Reconciler) initStatus(_ *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	r.Logger.Debug("Initializing status")
	if len(instance.Status.Conditions) == 0 {
		instance.Status.InitializeConditions()
		if err := r.updateStatus(instance); err != nil {
			return err
		}
	}
	return nil
}

// Apply the manifest resources
func (r *Reconciler) install(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	r.Logger.Debug("Installing manifest")
	if err := manifest.ApplyAll(); err != nil {
		instance.Status.MarkInstallFailed(err.Error())
		return err
	}
	instance.Status.MarkInstallSucceeded()
	instance.Status.Version = version.Version
	return nil
}

// Check for all deployments available
func (r *Reconciler) checkDeployments(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	r.Logger.Debug("Checking deployments")
	available := func(d *appsv1.Deployment) bool {
		for _, c := range d.Status.Conditions {
			if c.Type == appsv1.DeploymentAvailable && c.Status == corev1.ConditionTrue {
				return true
			}
		}
		return false
	}
	for _, u := range manifest.Resources {
		if u.GetKind() == "Deployment" {
			deployment, err := r.KubeClientSet.AppsV1().Deployments(u.GetNamespace()).Get(u.GetName(), metav1.GetOptions{})
			if err != nil {
				instance.Status.MarkDeploymentsNotReady()
				if errors.IsNotFound(err) {
					return nil
				}
				return err
			}
			if !available(deployment) {
				instance.Status.MarkDeploymentsNotReady()
				return nil
			}
		}
	}
	instance.Status.MarkDeploymentsAvailable()
	return nil
}

// Delete obsolete resources from previous versions
func (r *Reconciler) deleteObsoleteResources(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	// istio-system resources from 0.3
	resource := &unstructured.Unstructured{}
	resource.SetNamespace("istio-system")
	resource.SetName("knative-ingressgateway")
	resource.SetAPIVersion("v1")
	resource.SetKind("Service")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	resource.SetAPIVersion("apps/v1")
	resource.SetKind("Deployment")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	resource.SetAPIVersion("autoscaling/v1")
	resource.SetKind("HorizontalPodAutoscaler")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	// config-controller from 0.5
	resource.SetNamespace(instance.GetNamespace())
	resource.SetName("config-controller")
	resource.SetAPIVersion("v1")
	resource.SetKind("ConfigMap")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>/*
                       WARNING WARNING WARNING

                Attention all potential contributors

   This testfile is not in the best state. We've been slowly transitioning
   from the built in "testing" package to using Ginkgo. As you can see, we've
   changed the format, but a lot of the setup, test body, descriptions, etc
   are either hardcoded, completely lacking, or misleading.

   For example:

   Describe("Testing with ginkgo"...)      // This is not a great description
   It("TestDoesSoemthing"...)              // This is a horrible description

   Describe("create-user command"...       // Describe the actual object under test
   It("creates a user when provided ..."   // this is more descriptive

   For good examples of writing Ginkgo tests for the cli, refer to

   src/cf/commands/application/delete_app_test.go
   src/cf/terminal/ui_test.go
   src/github.com/cloudfoundry/loggregator_consumer/consumer_test.go
*/

package application_test

import (
	. "cf/commands/application"
	"cf/errors"
	"cf/models"
	"code.google.com/p/gogoprotobuf/proto"
	"github.com/cloudfoundry/loggregatorlib/logmessage"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	testapi "testhelpers/api"
	testcmd "testhelpers/commands"
	testconfig "testhelpers/configuration"
	testlogs "testhelpers/logs"
	testreq "testhelpers/requirements"
	testterm "testhelpers/terminal"
	"time"

	. "testhelpers/matchers"
)

var _ = Describe("logs command", func() {
	It("fails with usage when called without one argument", func() {
		requirementsFactory, logsRepo := getLogsDependencies()

		ui := callLogs([]string{}, requirementsFactory, logsRepo)
		Expect(ui.FailedWithUsage).To(BeTrue())
	})

	It("fails requirements when not logged in", func() {
		requirementsFactory, logsRepo := getLogsDependencies()
		requirementsFactory.LoginSuccess = false

		callLogs([]string{"my-app"}, requirementsFactory, logsRepo)
		Expect(testcmd.CommandDidPassRequirements).To(BeFalse())
	})

	It("TestLogsOutputsRecentLogs", func() {
		app := models.Application{}
		app.Name = "my-app"
		app.Guid = "my-app-guid"

		currentTime := time.Now()

		recentLogs := []*logmessage.LogMessage{
			testlogs.NewLogMessage("Log Line 1", app.Guid, "DEA", currentTime),
			testlogs.NewLogMessage("Log Line 2", app.Guid, "DEA", currentTime),
		}

		requirementsFactory, logsRepo := getLogsDependencies()
		requirementsFactory.Application = app
		logsRepo.RecentLogs = recentLogs

		ui := callLogs([]string{"--recent", "my-app"}, requirementsFactory, logsRepo)

		Expect(requirementsFactory.ApplicationName).To(Equal("my-app"))
		Expect(app.Guid).To(Equal(logsRepo.AppLoggedGuid))
		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"Connected, dumping recent logs for app", "my-app", "my-org", "my-space", "my-user"},
			[]string{"Log Line 1"},
			[]string{"Log Line 2"},
		))
	})

	It("TestLogsEscapeFormattingVerbs", func() {
		app := models.Application{}
		app.Name = "my-app"
		app.Guid = "my-app-guid"

		recentLogs := []*logmessage.LogMessage{
			testlogs.NewLogMessage("hello%2Bworld%v", app.Guid, "DEA", time.Now()),
		}

		requirementsFactory, logsRepo := getLogsDependencies()
		requirementsFactory.Application = app
		logsRepo.RecentLogs = recentLogs

		ui := callLogs([]string{"--recent", "my-app"}, requirementsFactory, logsRepo)

		Expect(ui.Outputs).To(ContainSubstrings([]string{"hello%2Bworld%v"}))
	})

	It("TestLogsTailsTheAppLogs", func() {
		app := models.Application{}
		app.Name = "my-app"
		app.Guid = "my-app-guid"

		logs := []*logmessage.LogMessage{
			testlogs.NewLogMessage("Log Line 1", app.Guid, "DEA", time.Now()),
		}

		requirementsFactory, logsRepo := getLogsDependencies()
		requirementsFactory.Application = app
		logsRepo.TailLogMessages = logs

		ui := callLogs([]string{"my-app"}, requirementsFactory, logsRepo)

		Expect(requirementsFactory.ApplicationName).To(Equal("my-app"))
		Expect(app.Guid).To(Equal(logsRepo.AppLoggedGuid))
		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"Connected, tailing logs for app", "my-app", "my-org", "my-space", "my-user"},
			[]string{"Log Line 1"},
		))
	})

	Context("when the loggregator server has an invalid cert", func() {
		var (
			requirementsFactory *testreq.FakeReqFactory
			logsRepo            *testapi.FakeLogsRepository
		)

		BeforeEach(func() {
			requirementsFactory, logsRepo = getLogsDependencies()
		})

		Context("when the skip-ssl-validation flag is not set", func() {
			It("fails and informs the user about the skip-ssl-validation flag", func() {
				logsRepo.TailLogErr = errors.NewInvalidSSLCert("https://example.com", "it don't work good")
				ui := callLogs([]string{"my-app"}, requirementsFactory, logsRepo)

				Expect(ui.Outputs).To(ContainSubstrings(
					[]string{"Received invalid SSL certificate", "https://example.com"},
					[]string{"TIP"},
				))
			})

			It("informs the user of the error when they include the --recent flag", func() {
				logsRepo.RecentLogErr = errors.NewInvalidSSLCert("https://example.com", "how does SSL work???")
				ui := callLogs([]string{"--recent", "my-app"}, requirementsFactory, logsRepo)

				Expect(ui.Outputs).To(ContainSubstrings(
					[]string{"Received invalid SSL certificate", "https://example.com"},
					[]string{"TIP"},
				))
			})
		})
	})

	Context("when the loggregator server has a valid cert", func() {
		var (
			flags               []string
			requirementsFactory *testreq.FakeReqFactory
			logsRepo            *testapi.FakeLogsRepository
		)

		BeforeEach(func() {
			requirementsFactory, logsRepo = getLogsDependencies()
			flags = []string{"my-app"}
		})

		It("tails logs", func() {
			ui := callLogs(flags, requirementsFactory, logsRepo)

			Expect(ui.Outputs).To(ContainSubstrings(
				[]string{"Connected, tailing logs for app", "my-org", "my-space", "my-user"},
			))
		})
	})

	Describe("Helpers", func() {
		date := time.Date(2014, 4, 4, 11, 39, 20, 5, time.UTC)

		createMessage := func(sourceId string, sourceName string, msgType logmessage.LogMessage_MessageType, date time.Time) *logmessage.LogMessage {
			timestamp := date.UnixNano()
			return &logmessage.LogMessage{
				Message:     []byte("Hello World!\n\r\n\r"),
				AppId:       proto.String("my-app-guid"),
				MessageType: &msgType,
				SourceId:    &sourceId,
				Timestamp:   &timestamp,
				SourceName:  &sourceName,
			}
		}

		Context("when the message comes from an app", func() {
			It("includes the instance index", func() {
				msg := createMessage("4", "App", logmessage.LogMessage_OUT, date)
				Expect(LogMessageOutput(msg, time.UTC)).To(Equal("2014-04-04T11:39:20.00+0000 [App/4]   OUT Hello World!"))
			})
		})

		Context("when the message comes from a cloudfoundry component", func() {
			It("doesn't include the instance index", func() {
				msg := createMessage("4", "DEA", logmessage.LogMessage_OUT, date)
				Expect(LogMessageOutput(msg, time.UTC)).To(Equal("2014-04-04T11:39:20.00+0000 [DEA]     OUT Hello World!"))
			})
		})

		Context("when the message was written to stderr", func() {
			It("shows the log type as 'ERR'", func() {
				msg := createMessage("4", "DEA", logmessage.LogMessage_ERR, date)
				Expect(LogMessageOutput(msg, time.UTC)).To(Equal("2014-04-04T11:39:20.00+0000 [DEA]     ERR Hello World!"))
			})
		})

		It("formats the time in the given time zone", func() {
			msg := createMessage("4", "DEA", logmessage.LogMessage_ERR, date)
			Expect(LogMessageOutput(msg, time.FixedZone("the-zone", 3*60*60))).To(Equal("2014-04-04T14:39:20.00+0300 [DEA]     ERR Hello World!"))
		})
	})
})

func getLogsDependencies() (requirementsFactory *testreq.FakeReqFactory, logsRepo *testapi.FakeLogsRepository) {
	logsRepo = &testapi.FakeLogsRepository{}
	requirementsFactory = &testreq.FakeReqFactory{LoginSuccess: true}
	return
}

func callLogs(args []string, requirementsFactory *testreq.FakeReqFactory, logsRepo *testapi.FakeLogsRepository) (ui *testterm.FakeUI) {
	ui = new(testterm.FakeUI)
	ctxt := testcmd.NewContext("logs", args)

	configRepo := testconfig.NewRepositoryWithDefaults()
	cmd := NewLogs(ui, configRepo, logsRepo)
	testcmd.RunCommand(cmd, ctxt, requirementsFactory)
	return
}
<commit_msg>Decolorize log messages before asserting on contents<commit_after>/*
                       WARNING WARNING WARNING

                Attention all potential contributors

   This testfile is not in the best state. We've been slowly transitioning
   from the built in "testing" package to using Ginkgo. As you can see, we've
   changed the format, but a lot of the setup, test body, descriptions, etc
   are either hardcoded, completely lacking, or misleading.

   For example:

   Describe("Testing with ginkgo"...)      // This is not a great description
   It("TestDoesSoemthing"...)              // This is a horrible description

   Describe("create-user command"...       // Describe the actual object under test
   It("creates a user when provided ..."   // this is more descriptive

   For good examples of writing Ginkgo tests for the cli, refer to

   src/cf/commands/application/delete_app_test.go
   src/cf/terminal/ui_test.go
   src/github.com/cloudfoundry/loggregator_consumer/consumer_test.go
*/

package application_test

import (
	. "cf/commands/application"
	"cf/errors"
	"cf/models"
	"cf/terminal"
	"code.google.com/p/gogoprotobuf/proto"
	"github.com/cloudfoundry/loggregatorlib/logmessage"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	testapi "testhelpers/api"
	testcmd "testhelpers/commands"
	testconfig "testhelpers/configuration"
	testlogs "testhelpers/logs"
	testreq "testhelpers/requirements"
	testterm "testhelpers/terminal"
	"time"

	. "testhelpers/matchers"
)

var _ = Describe("logs command", func() {
	It("fails with usage when called without one argument", func() {
		requirementsFactory, logsRepo := getLogsDependencies()

		ui := callLogs([]string{}, requirementsFactory, logsRepo)
		Expect(ui.FailedWithUsage).To(BeTrue())
	})

	It("fails requirements when not logged in", func() {
		requirementsFactory, logsRepo := getLogsDependencies()
		requirementsFactory.LoginSuccess = false

		callLogs([]string{"my-app"}, requirementsFactory, logsRepo)
		Expect(testcmd.CommandDidPassRequirements).To(BeFalse())
	})

	It("TestLogsOutputsRecentLogs", func() {
		app := models.Application{}
		app.Name = "my-app"
		app.Guid = "my-app-guid"

		currentTime := time.Now()

		recentLogs := []*logmessage.LogMessage{
			testlogs.NewLogMessage("Log Line 1", app.Guid, "DEA", currentTime),
			testlogs.NewLogMessage("Log Line 2", app.Guid, "DEA", currentTime),
		}

		requirementsFactory, logsRepo := getLogsDependencies()
		requirementsFactory.Application = app
		logsRepo.RecentLogs = recentLogs

		ui := callLogs([]string{"--recent", "my-app"}, requirementsFactory, logsRepo)

		Expect(requirementsFactory.ApplicationName).To(Equal("my-app"))
		Expect(app.Guid).To(Equal(logsRepo.AppLoggedGuid))
		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"Connected, dumping recent logs for app", "my-app", "my-org", "my-space", "my-user"},
			[]string{"Log Line 1"},
			[]string{"Log Line 2"},
		))
	})

	It("TestLogsEscapeFormattingVerbs", func() {
		app := models.Application{}
		app.Name = "my-app"
		app.Guid = "my-app-guid"

		recentLogs := []*logmessage.LogMessage{
			testlogs.NewLogMessage("hello%2Bworld%v", app.Guid, "DEA", time.Now()),
		}

		requirementsFactory, logsRepo := getLogsDependencies()
		requirementsFactory.Application = app
		logsRepo.RecentLogs = recentLogs

		ui := callLogs([]string{"--recent", "my-app"}, requirementsFactory, logsRepo)

		Expect(ui.Outputs).To(ContainSubstrings([]string{"hello%2Bworld%v"}))
	})

	It("TestLogsTailsTheAppLogs", func() {
		app := models.Application{}
		app.Name = "my-app"
		app.Guid = "my-app-guid"

		logs := []*logmessage.LogMessage{
			testlogs.NewLogMessage("Log Line 1", app.Guid, "DEA", time.Now()),
		}

		requirementsFactory, logsRepo := getLogsDependencies()
		requirementsFactory.Application = app
		logsRepo.TailLogMessages = logs

		ui := callLogs([]string{"my-app"}, requirementsFactory, logsRepo)

		Expect(requirementsFactory.ApplicationName).To(Equal("my-app"))
		Expect(app.Guid).To(Equal(logsRepo.AppLoggedGuid))
		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"Connected, tailing logs for app", "my-app", "my-org", "my-space", "my-user"},
			[]string{"Log Line 1"},
		))
	})

	Context("when the loggregator server has an invalid cert", func() {
		var (
			requirementsFactory *testreq.FakeReqFactory
			logsRepo            *testapi.FakeLogsRepository
		)

		BeforeEach(func() {
			requirementsFactory, logsRepo = getLogsDependencies()
		})

		Context("when the skip-ssl-validation flag is not set", func() {
			It("fails and informs the user about the skip-ssl-validation flag", func() {
				logsRepo.TailLogErr = errors.NewInvalidSSLCert("https://example.com", "it don't work good")
				ui := callLogs([]string{"my-app"}, requirementsFactory, logsRepo)

				Expect(ui.Outputs).To(ContainSubstrings(
					[]string{"Received invalid SSL certificate", "https://example.com"},
					[]string{"TIP"},
				))
			})

			It("informs the user of the error when they include the --recent flag", func() {
				logsRepo.RecentLogErr = errors.NewInvalidSSLCert("https://example.com", "how does SSL work???")
				ui := callLogs([]string{"--recent", "my-app"}, requirementsFactory, logsRepo)

				Expect(ui.Outputs).To(ContainSubstrings(
					[]string{"Received invalid SSL certificate", "https://example.com"},
					[]string{"TIP"},
				))
			})
		})
	})

	Context("when the loggregator server has a valid cert", func() {
		var (
			flags               []string
			requirementsFactory *testreq.FakeReqFactory
			logsRepo            *testapi.FakeLogsRepository
		)

		BeforeEach(func() {
			requirementsFactory, logsRepo = getLogsDependencies()
			flags = []string{"my-app"}
		})

		It("tails logs", func() {
			ui := callLogs(flags, requirementsFactory, logsRepo)

			Expect(ui.Outputs).To(ContainSubstrings(
				[]string{"Connected, tailing logs for app", "my-org", "my-space", "my-user"},
			))
		})
	})

	Describe("Helpers", func() {
		date := time.Date(2014, 4, 4, 11, 39, 20, 5, time.UTC)

		createMessage := func(sourceId string, sourceName string, msgType logmessage.LogMessage_MessageType, date time.Time) *logmessage.LogMessage {
			timestamp := date.UnixNano()
			return &logmessage.LogMessage{
				Message:     []byte("Hello World!\n\r\n\r"),
				AppId:       proto.String("my-app-guid"),
				MessageType: &msgType,
				SourceId:    &sourceId,
				Timestamp:   &timestamp,
				SourceName:  &sourceName,
			}
		}

		Context("when the message comes from an app", func() {
			It("includes the instance index", func() {
				msg := createMessage("4", "App", logmessage.LogMessage_OUT, date)
				Expect(terminal.Decolorize(LogMessageOutput(msg, time.UTC))).To(Equal("2014-04-04T11:39:20.00+0000 [App/4]   OUT Hello World!"))
			})
		})

		Context("when the message comes from a cloudfoundry component", func() {
			It("doesn't include the instance index", func() {
				msg := createMessage("4", "DEA", logmessage.LogMessage_OUT, date)
				Expect(terminal.Decolorize(LogMessageOutput(msg, time.UTC))).To(Equal("2014-04-04T11:39:20.00+0000 [DEA]     OUT Hello World!"))
			})
		})

		Context("when the message was written to stderr", func() {
			It("shows the log type as 'ERR'", func() {
				msg := createMessage("4", "DEA", logmessage.LogMessage_ERR, date)
				Expect(terminal.Decolorize(LogMessageOutput(msg, time.UTC))).To(Equal("2014-04-04T11:39:20.00+0000 [DEA]     ERR Hello World!"))
			})
		})

		It("formats the time in the given time zone", func() {
			msg := createMessage("4", "DEA", logmessage.LogMessage_ERR, date)
			Expect(terminal.Decolorize(LogMessageOutput(msg, time.FixedZone("the-zone", 3*60*60)))).To(Equal("2014-04-04T14:39:20.00+0300 [DEA]     ERR Hello World!"))
		})
	})
})

func getLogsDependencies() (requirementsFactory *testreq.FakeReqFactory, logsRepo *testapi.FakeLogsRepository) {
	logsRepo = &testapi.FakeLogsRepository{}
	requirementsFactory = &testreq.FakeReqFactory{LoginSuccess: true}
	return
}

func callLogs(args []string, requirementsFactory *testreq.FakeReqFactory, logsRepo *testapi.FakeLogsRepository) (ui *testterm.FakeUI) {
	ui = new(testterm.FakeUI)
	ctxt := testcmd.NewContext("logs", args)

	configRepo := testconfig.NewRepositoryWithDefaults()
	cmd := NewLogs(ui, configRepo, logsRepo)
	testcmd.RunCommand(cmd, ctxt, requirementsFactory)
	return
}
<|endoftext|>


<commit_before>package allocrunnerv2

import (
	"context"
	"fmt"
	"time"

	multierror "github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/client/allocrunnerv2/interfaces"
	"github.com/hashicorp/nomad/nomad/structs"
)

// allocHealthSetter is a shim to allow the alloc health watcher hook to set
// and clear the alloc health without full access to the alloc runner state
type allocHealthSetter struct {
	ar *allocRunner
}

// ClearHealth allows the health watcher hook to clear the alloc's deployment
// health if the deployment id changes. It does not update the server as the
// status is only cleared when already receiving an update from the server.
//
// Only for use by health hook.
func (a *allocHealthSetter) ClearHealth() {
	a.ar.stateLock.Lock()
	a.ar.state.ClearDeploymentStatus()
	a.ar.stateLock.Unlock()
}

// SetHealth allows the health watcher hook to set the alloc's
// deployment/migration health and emit task events.
//
// Only for use by health hook.
func (a *allocHealthSetter) SetHealth(healthy, isDeploy bool, trackerTaskEvents map[string]*structs.TaskEvent) {
	// Updating alloc deployment state is tricky because it may be nil, but
	// if it's not then we need to maintain the values of Canary and
	// ModifyIndex as they're only mutated by the server.
	a.ar.stateLock.Lock()
	a.ar.state.SetDeploymentStatus(time.Now(), healthy)
	a.ar.stateLock.Unlock()

	// If deployment is unhealthy emit task events explaining why
	a.ar.tasksLock.RLock()
	if !healthy && isDeploy {
		for task, event := range trackerTaskEvents {
			if tr, ok := a.ar.tasks[task]; ok {
				tr.EmitEvent(event)
			}
		}
	}

	// Gather the state of the other tasks
	states := make(map[string]*structs.TaskState, len(a.ar.tasks))
	for name, tr := range a.ar.tasks {
		states[name] = tr.TaskState()
	}
	a.ar.tasksLock.RUnlock()

	// Build the client allocation
	calloc := a.ar.clientAlloc(states)

	// Update the server
	a.ar.stateUpdater.AllocStateUpdated(calloc)

	// Broadcast client alloc to listeners
	a.ar.allocBroadcaster.Send(calloc)
}

// initRunnerHooks intializes the runners hooks.
func (ar *allocRunner) initRunnerHooks() {
	hookLogger := ar.logger.Named("runner_hook")

	// create health setting shim
	hs := &allocHealthSetter{ar}

	// Create the alloc directory hook. This is run first to ensure the
	// directoy path exists for other hooks.
	ar.runnerHooks = []interfaces.RunnerHook{
		newAllocDirHook(hookLogger, ar.allocDir),
		newDiskMigrationHook(hookLogger, ar.prevAllocWatcher, ar.allocDir),
		newAllocHealthWatcherHook(hookLogger, ar.Alloc(), hs, ar.Listener(), ar.consulClient),
	}
}

// prerun is used to run the runners prerun hooks.
func (ar *allocRunner) prerun() error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running pre-run hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished pre-run hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	for _, hook := range ar.runnerHooks {
		pre, ok := hook.(interfaces.RunnerPrerunHook)
		if !ok {
			continue
		}

		//TODO Check hook state

		name := pre.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running pre-run hook", "name", name, "start", start)
		}

		if err := pre.Prerun(context.TODO()); err != nil {
			return fmt.Errorf("hook %q failed: %v", name, err)
		}

		//TODO Persist hook state locally

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished pre-run hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return nil
}

// update runs the alloc runner update hooks. Update hooks are run
// asynchronously with all other alloc runner operations.
func (ar *allocRunner) update(update *structs.Allocation) error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running update hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished update hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	req := &interfaces.RunnerUpdateRequest{
		Alloc: update,
	}

	for _, hook := range ar.runnerHooks {
		h, ok := hook.(interfaces.RunnerUpdateHook)
		if !ok {
			continue
		}

		name := h.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running pre-run hook", "name", name, "start", start)
		}

		if err := h.Update(req); err != nil {
			return fmt.Errorf("hook %q failed: %v", name, err)
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished update hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return nil
}

// postrun is used to run the runners postrun hooks.
func (ar *allocRunner) postrun() error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running post-run hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished post-run hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	for _, hook := range ar.runnerHooks {
		post, ok := hook.(interfaces.RunnerPostrunHook)
		if !ok {
			continue
		}

		name := post.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running post-run hook", "name", name, "start", start)
		}

		if err := post.Postrun(); err != nil {
			return fmt.Errorf("hook %q failed: %v", name, err)
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished post-run hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return nil
}

// destroy is used to run the runners destroy hooks. All hooks are run and
// errors are returned as a multierror.
func (ar *allocRunner) destroy() error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running destroy hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished destroy hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	var merr multierror.Error
	for _, hook := range ar.runnerHooks {
		h, ok := hook.(interfaces.RunnerDestroyHook)
		if !ok {
			continue
		}

		name := h.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running destroy hook", "name", name, "start", start)
		}

		if err := h.Destroy(); err != nil {
			merr.Errors = append(merr.Errors, fmt.Errorf("destroy hook %q failed: %v", name, err))
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished destroy hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return nil
}
<commit_msg>ar: use multierror in update hook loop<commit_after>package allocrunnerv2

import (
	"context"
	"fmt"
	"time"

	multierror "github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/client/allocrunnerv2/interfaces"
	"github.com/hashicorp/nomad/nomad/structs"
)

// allocHealthSetter is a shim to allow the alloc health watcher hook to set
// and clear the alloc health without full access to the alloc runner state
type allocHealthSetter struct {
	ar *allocRunner
}

// ClearHealth allows the health watcher hook to clear the alloc's deployment
// health if the deployment id changes. It does not update the server as the
// status is only cleared when already receiving an update from the server.
//
// Only for use by health hook.
func (a *allocHealthSetter) ClearHealth() {
	a.ar.stateLock.Lock()
	a.ar.state.ClearDeploymentStatus()
	a.ar.stateLock.Unlock()
}

// SetHealth allows the health watcher hook to set the alloc's
// deployment/migration health and emit task events.
//
// Only for use by health hook.
func (a *allocHealthSetter) SetHealth(healthy, isDeploy bool, trackerTaskEvents map[string]*structs.TaskEvent) {
	// Updating alloc deployment state is tricky because it may be nil, but
	// if it's not then we need to maintain the values of Canary and
	// ModifyIndex as they're only mutated by the server.
	a.ar.stateLock.Lock()
	a.ar.state.SetDeploymentStatus(time.Now(), healthy)
	a.ar.stateLock.Unlock()

	// If deployment is unhealthy emit task events explaining why
	a.ar.tasksLock.RLock()
	if !healthy && isDeploy {
		for task, event := range trackerTaskEvents {
			if tr, ok := a.ar.tasks[task]; ok {
				tr.EmitEvent(event)
			}
		}
	}

	// Gather the state of the other tasks
	states := make(map[string]*structs.TaskState, len(a.ar.tasks))
	for name, tr := range a.ar.tasks {
		states[name] = tr.TaskState()
	}
	a.ar.tasksLock.RUnlock()

	// Build the client allocation
	calloc := a.ar.clientAlloc(states)

	// Update the server
	a.ar.stateUpdater.AllocStateUpdated(calloc)

	// Broadcast client alloc to listeners
	a.ar.allocBroadcaster.Send(calloc)
}

// initRunnerHooks intializes the runners hooks.
func (ar *allocRunner) initRunnerHooks() {
	hookLogger := ar.logger.Named("runner_hook")

	// create health setting shim
	hs := &allocHealthSetter{ar}

	// Create the alloc directory hook. This is run first to ensure the
	// directoy path exists for other hooks.
	ar.runnerHooks = []interfaces.RunnerHook{
		newAllocDirHook(hookLogger, ar.allocDir),
		newDiskMigrationHook(hookLogger, ar.prevAllocWatcher, ar.allocDir),
		newAllocHealthWatcherHook(hookLogger, ar.Alloc(), hs, ar.Listener(), ar.consulClient),
	}
}

// prerun is used to run the runners prerun hooks.
func (ar *allocRunner) prerun() error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running pre-run hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished pre-run hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	for _, hook := range ar.runnerHooks {
		pre, ok := hook.(interfaces.RunnerPrerunHook)
		if !ok {
			continue
		}

		//TODO Check hook state

		name := pre.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running pre-run hook", "name", name, "start", start)
		}

		if err := pre.Prerun(context.TODO()); err != nil {
			return fmt.Errorf("pre-run hook %q failed: %v", name, err)
		}

		//TODO Persist hook state locally

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished pre-run hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return nil
}

// update runs the alloc runner update hooks. Update hooks are run
// asynchronously with all other alloc runner operations.
func (ar *allocRunner) update(update *structs.Allocation) error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running update hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished update hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	req := &interfaces.RunnerUpdateRequest{
		Alloc: update,
	}

	var merr multierror.Error
	for _, hook := range ar.runnerHooks {
		h, ok := hook.(interfaces.RunnerUpdateHook)
		if !ok {
			continue
		}

		name := h.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running pre-run hook", "name", name, "start", start)
		}

		if err := h.Update(req); err != nil {
			merr.Errors = append(merr.Errors, fmt.Errorf("update hook %q failed: %v", name, err))
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished update hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return merr.ErrorOrNil()
}

// postrun is used to run the runners postrun hooks.
func (ar *allocRunner) postrun() error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running post-run hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished post-run hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	for _, hook := range ar.runnerHooks {
		post, ok := hook.(interfaces.RunnerPostrunHook)
		if !ok {
			continue
		}

		name := post.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running post-run hook", "name", name, "start", start)
		}

		if err := post.Postrun(); err != nil {
			return fmt.Errorf("hook %q failed: %v", name, err)
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished post-run hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return nil
}

// destroy is used to run the runners destroy hooks. All hooks are run and
// errors are returned as a multierror.
func (ar *allocRunner) destroy() error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running destroy hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished destroy hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	var merr multierror.Error
	for _, hook := range ar.runnerHooks {
		h, ok := hook.(interfaces.RunnerDestroyHook)
		if !ok {
			continue
		}

		name := h.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running destroy hook", "name", name, "start", start)
		}

		if err := h.Destroy(); err != nil {
			merr.Errors = append(merr.Errors, fmt.Errorf("destroy hook %q failed: %v", name, err))
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished destroy hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package ghttp

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/gogf/gf/text/gregex"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/gogf/gf/os/gfile"
)

// Get send GET request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Get(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("GET", url, data...)
}

// Put send PUT request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Put(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("PUT", url, data...)
}

// Post sends request using HTTP method POST and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Post(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("POST", url, data...)
}

// Delete send DELETE request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Delete(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("DELETE", url, data...)
}

// Head send HEAD request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Head(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("HEAD", url, data...)
}

// Patch send PATCH request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Patch(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("PATCH", url, data...)
}

// Connect send CONNECT request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Connect(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("CONNECT", url, data...)
}

// Options send OPTIONS request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Options(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("OPTIONS", url, data...)
}

// Trace send TRACE request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Trace(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("TRACE", url, data...)
}

// DoRequest sends request with given HTTP method and data and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
//
// Note that it uses "multipart/form-data" as its Content-Type if it contains file uploading,
// else it uses "application/x-www-form-urlencoded". It also automatically detects the post
// content for JSON format, and for that it automatically sets the Content-Type as
// "application/json".
func (c *Client) DoRequest(method, url string, data ...interface{}) (resp *ClientResponse, err error) {
	method = strings.ToUpper(method)
	if len(c.prefix) > 0 {
		url = c.prefix + url
	}
	param := ""
	if len(data) > 0 {
		param = BuildParams(data[0])
	}
	req := (*http.Request)(nil)
	if strings.Contains(param, "@file:") {
		// File uploading request.
		buffer := new(bytes.Buffer)
		writer := multipart.NewWriter(buffer)
		defer writer.Close()
		for _, item := range strings.Split(param, "&") {
			array := strings.Split(item, "=")
			if len(array[1]) > 6 && strings.Compare(array[1][0:6], "@file:") == 0 {
				path := array[1][6:]
				if !gfile.Exists(path) {
					return nil, errors.New(fmt.Sprintf(`"%s" does not exist`, path))
				}
				if file, err := writer.CreateFormFile(array[0], path); err == nil {
					if f, err := os.Open(path); err == nil {
						defer f.Close()
						if _, err = io.Copy(file, f); err != nil {
							return nil, err
						}
					} else {
						return nil, err
					}
				} else {
					return nil, err
				}
			} else {
				if err := writer.WriteField(array[0], array[1]); err != nil {
					return nil, err
				}
			}
		}
		if req, err = http.NewRequest(method, url, buffer); err != nil {
			return nil, err
		} else {
			req.Header.Set("Content-Type", writer.FormDataContentType())
		}
	} else {
		// Normal request.
		paramBytes := []byte(param)
		if req, err = http.NewRequest(method, url, bytes.NewReader(paramBytes)); err != nil {
			return nil, err
		} else {
			if v, ok := c.header["Content-Type"]; ok {
				// Custom Content-Type.
				req.Header.Set("Content-Type", v)
			} else {
				if json.Valid(paramBytes) {
					// Auto detecting and setting the post content format: JSON.
					req.Header.Set("Content-Type", "application/json")
				} else if gregex.IsMatchString(`^[\w\[\]]+=.+`, param) {
					// If the parameters passed like "name=value", it then uses form type.
					req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
				}
			}
		}
	}
	// Custom header.
	if len(c.header) > 0 {
		for k, v := range c.header {
			req.Header.Set(k, v)
		}
	}
	// Custom Cookie.
	if len(c.cookies) > 0 {
		headerCookie := ""
		for k, v := range c.cookies {
			if len(headerCookie) > 0 {
				headerCookie += ";"
			}
			headerCookie += k + "=" + v
		}
		if len(headerCookie) > 0 {
			req.Header.Set("Cookie", headerCookie)
		}
	}
	// HTTP basic authentication.
	if len(c.authUser) > 0 {
		req.SetBasicAuth(c.authUser, c.authPass)
	}
	// Sending request.
	var r *http.Response
	for {
		if r, err = c.Do(req); err != nil {
			if c.retryCount > 0 {
				c.retryCount--
			} else {
				return nil, err
			}
		} else {
			break
		}
	}
	resp = &ClientResponse{
		Response: r,
	}
	// Auto saving cookie content.
	if c.browserMode {
		now := time.Now()
		for _, v := range r.Cookies() {
			if v.Expires.UnixNano() < now.UnixNano() {
				delete(c.cookies, v.Name)
			} else {
				c.cookies[v.Name] = v.Value
			}
		}
	}
	return resp, nil
}
<commit_msg>improve ghttp.Client<commit_after>// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package ghttp

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/gogf/gf/text/gregex"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/gogf/gf/os/gfile"
)

// Get send GET request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Get(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("GET", url, data...)
}

// Put send PUT request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Put(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("PUT", url, data...)
}

// Post sends request using HTTP method POST and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Post(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("POST", url, data...)
}

// Delete send DELETE request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Delete(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("DELETE", url, data...)
}

// Head send HEAD request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Head(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("HEAD", url, data...)
}

// Patch send PATCH request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Patch(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("PATCH", url, data...)
}

// Connect send CONNECT request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Connect(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("CONNECT", url, data...)
}

// Options send OPTIONS request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Options(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("OPTIONS", url, data...)
}

// Trace send TRACE request and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
func (c *Client) Trace(url string, data ...interface{}) (*ClientResponse, error) {
	return c.DoRequest("TRACE", url, data...)
}

// DoRequest sends request with given HTTP method and data and returns the response object.
// Note that the response object MUST be closed if it'll be never used.
//
// Note that it uses "multipart/form-data" as its Content-Type if it contains file uploading,
// else it uses "application/x-www-form-urlencoded". It also automatically detects the post
// content for JSON format, and for that it automatically sets the Content-Type as
// "application/json".
func (c *Client) DoRequest(method, url string, data ...interface{}) (resp *ClientResponse, err error) {
	method = strings.ToUpper(method)
	if len(c.prefix) > 0 {
		url = c.prefix + url
	}
	param := ""
	if len(data) > 0 {
		param = BuildParams(data[0])
	}
	req := (*http.Request)(nil)
	if strings.Contains(param, "@file:") {
		// File uploading request.
		buffer := new(bytes.Buffer)
		writer := multipart.NewWriter(buffer)
		for _, item := range strings.Split(param, "&") {
			array := strings.Split(item, "=")
			if len(array[1]) > 6 && strings.Compare(array[1][0:6], "@file:") == 0 {
				path := array[1][6:]
				if !gfile.Exists(path) {
					return nil, errors.New(fmt.Sprintf(`"%s" does not exist`, path))
				}
				if file, err := writer.CreateFormFile(array[0], path); err == nil {
					if f, err := os.Open(path); err == nil {
						if _, err = io.Copy(file, f); err != nil {
							f.Close()
							return nil, err
						}
						f.Close()
					} else {
						return nil, err
					}
				} else {
					return nil, err
				}
			} else {
				if err = writer.WriteField(array[0], array[1]); err != nil {
					return nil, err
				}
			}
		}
		// Close finishes the multipart message and writes the trailing
		// boundary end line to the output.
		if err = writer.Close(); err != nil {
			return nil, err
		}
		if req, err = http.NewRequest(method, url, buffer); err != nil {
			return nil, err
		} else {
			req.Header.Set("Content-Type", writer.FormDataContentType())
		}
	} else {
		// Normal request.
		paramBytes := []byte(param)
		if req, err = http.NewRequest(method, url, bytes.NewReader(paramBytes)); err != nil {
			return nil, err
		} else {
			if v, ok := c.header["Content-Type"]; ok {
				// Custom Content-Type.
				req.Header.Set("Content-Type", v)
			} else {
				if json.Valid(paramBytes) {
					// Auto detecting and setting the post content format: JSON.
					req.Header.Set("Content-Type", "application/json")
				} else if gregex.IsMatchString(`^[\w\[\]]+=.+`, param) {
					// If the parameters passed like "name=value", it then uses form type.
					req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
				}
			}
		}
	}
	// Custom header.
	if len(c.header) > 0 {
		for k, v := range c.header {
			req.Header.Set(k, v)
		}
	}
	// Custom Cookie.
	if len(c.cookies) > 0 {
		headerCookie := ""
		for k, v := range c.cookies {
			if len(headerCookie) > 0 {
				headerCookie += ";"
			}
			headerCookie += k + "=" + v
		}
		if len(headerCookie) > 0 {
			req.Header.Set("Cookie", headerCookie)
		}
	}
	// HTTP basic authentication.
	if len(c.authUser) > 0 {
		req.SetBasicAuth(c.authUser, c.authPass)
	}
	// Sending request.
	var r *http.Response
	for {
		if r, err = c.Do(req); err != nil {
			if c.retryCount > 0 {
				c.retryCount--
			} else {
				return nil, err
			}
		} else {
			break
		}
	}
	resp = &ClientResponse{
		Response: r,
	}
	// Auto saving cookie content.
	if c.browserMode {
		now := time.Now()
		for _, v := range r.Cookies() {
			if v.Expires.UnixNano() < now.UnixNano() {
				delete(c.cookies, v.Name)
			} else {
				c.cookies[v.Name] = v.Value
			}
		}
	}
	return resp, nil
}
<|endoftext|>