<commit_before>// Copyright ©2013 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat

import (
	"gonum.org/v1/gonum/lapack"
	"gonum.org/v1/gonum/lapack/lapack64"
)

const (
	badFact   = "mat: use without successful factorization"
	badNoVect = "mat: eigenvectors not computed"
)

// EigenSym is a type for creating and manipulating the Eigen decomposition of
// symmetric matrices.
type EigenSym struct {
	vectorsComputed bool

	values  []float64
	vectors *Dense
}

// Factorize computes the eigenvalue decomposition of the symmetric matrix a.
// The Eigen decomposition is defined as
//  A = P * D * P^-1
// where D is a diagonal matrix containing the eigenvalues of the matrix, and
// P is a matrix of the eigenvectors of A. Factorize computes the eigenvalues
// in ascending order. If the vectors input argument is false, the eigenvectors
// are not computed.
//
// Factorize returns whether the decomposition succeeded. If the decomposition
// failed, methods that require a successful factorization will panic.
func (e *EigenSym) Factorize(a Symmetric, vectors bool) (ok bool) {
	n := a.Symmetric()
	sd := NewSymDense(n, nil)
	sd.CopySym(a)

	jobz := lapack.EVNone
	if vectors {
		jobz = lapack.EVCompute
	}
	w := make([]float64, n)
	work := []float64{0}
	lapack64.Syev(jobz, sd.mat, w, work, -1)

	work = getFloats(int(work[0]), false)
	ok = lapack64.Syev(jobz, sd.mat, w, work, len(work))
	putFloats(work)
	if !ok {
		e.vectorsComputed = false
		e.values = nil
		e.vectors = nil
		return false
	}
	e.vectorsComputed = vectors
	e.values = w
	e.vectors = NewDense(n, n, sd.mat.Data)
	return true
}

// succFact returns whether the receiver contains a successful factorization.
func (e *EigenSym) succFact() bool {
	return len(e.values) != 0
}

// Values extracts the eigenvalues of the factorized matrix. If dst is
// non-nil, the values are stored in-place into dst. In this case
// dst must have length n, otherwise Values will panic. If dst is
// nil, then a new slice will be allocated of the proper length and filled
// with the eigenvalues.
//
// Values panics if the Eigen decomposition was not successful.
func (e *EigenSym) Values(dst []float64) []float64 {
	if !e.succFact() {
		panic(badFact)
	}
	if dst == nil {
		dst = make([]float64, len(e.values))
	}
	if len(dst) != len(e.values) {
		panic(ErrSliceLengthMismatch)
	}
	copy(dst, e.values)
	return dst
}

// EigenvectorsSym extracts the eigenvectors of the factorized matrix and stores
// them in the receiver. Each eigenvector is a column corresponding to the
// respective eigenvalue returned by e.Values.
//
// EigenvectorsSym panics if the factorization was not successful or if the
// decomposition did not compute the eigenvectors.
func (m *Dense) EigenvectorsSym(e *EigenSym) {
	if !e.succFact() {
		panic(badFact)
	}
	if !e.vectorsComputed {
		panic(badNoVect)
	}
	m.reuseAs(len(e.values), len(e.values))
	m.Copy(e.vectors)
}

// Eigen is a type for creating and using the eigenvalue decomposition of a dense matrix.
type Eigen struct {
	n int // The size of the factorized matrix.

	right bool // have the right eigenvectors been computed
	left  bool // have the left eigenvectors been computed

	values   []complex128
	rVectors *CDense
	lVectors *CDense
}

// succFact returns whether the receiver contains a successful factorization.
func (e *Eigen) succFact() bool {
	return len(e.values) != 0
}

// Factorize computes the eigenvalues of the square matrix a, and optionally
// the eigenvectors.
//
// A right eigenvalue/eigenvector combination is defined by
//  A * x_r = λ * x_r
// where x_r is the column vector called an eigenvector, and λ is the corresponding
// eigenvector.
//
// Similarly, a left eigenvalue/eigenvector combination is defined by
//  x_l * A = λ * x_l
// The eigenvalues, but not the eigenvectors, are the same for both decompositions.
//
// Typically eigenvectors refer to right eigenvectors.
//
// In all cases, Eigen computes the eigenvalues of the matrix. If right and left
// are true, then the right and left eigenvectors will be computed, respectively.
// Eigen panics if the input matrix is not square.
//
// Factorize returns whether the decomposition succeeded. If the decomposition
// failed, methods that require a successful factorization will panic.
func (e *Eigen) Factorize(a Matrix, left, right bool) (ok bool) {
	// Copy a because it is modified during the Lapack call.
	r, c := a.Dims()
	if r != c {
		panic(ErrShape)
	}
	var sd Dense
	sd.Clone(a)

	var vl, vr Dense
	jobvl := lapack.LeftEVNone
	jobvr := lapack.RightEVNone
	if left {
		vl = *NewDense(r, r, nil)
		jobvl = lapack.LeftEVCompute
	}
	if right {
		vr = *NewDense(c, c, nil)
		jobvr = lapack.RightEVCompute
	}

	wr := getFloats(c, false)
	defer putFloats(wr)
	wi := getFloats(c, false)
	defer putFloats(wi)

	work := []float64{0}
	lapack64.Geev(jobvl, jobvr, sd.mat, wr, wi, vl.mat, vr.mat, work, -1)
	work = getFloats(int(work[0]), false)
	first := lapack64.Geev(jobvl, jobvr, sd.mat, wr, wi, vl.mat, vr.mat, work, len(work))
	putFloats(work)

	if first != 0 {
		e.values = nil
		return false
	}
	e.n = r
	e.right = right
	e.left = left

	// Construct complex eigenvalues from float64 data.
	values := make([]complex128, r)
	for i, v := range wr {
		values[i] = complex(v, wi[i])
	}
	e.values = values

	// Construct complex eigenvectors from float64 data.
	var cvl, cvr CDense
	if left {
		cvl = *NewCDense(r, r, nil)
		e.complexEigenTo(&cvl, &vl)
		e.lVectors = &cvl
	} else {
		e.lVectors = nil
	}
	if right {
		cvr = *NewCDense(c, c, nil)
		e.complexEigenTo(&cvr, &vr)
		e.rVectors = &cvr
	} else {
		e.rVectors = nil
	}
	return true
}

// Values extracts the eigenvalues of the factorized matrix. If dst is
// non-nil, the values are stored in-place into dst. In this case
// dst must have length n, otherwise Values will panic. If dst is
// nil, then a new slice will be allocated of the proper length and
// filed with the eigenvalues.
//
// Values panics if the Eigen decomposition was not successful.
func (e *Eigen) Values(dst []complex128) []complex128 {
	if !e.succFact() {
		panic(badFact)
	}
	if dst == nil {
		dst = make([]complex128, e.n)
	}
	if len(dst) != e.n {
		panic(ErrSliceLengthMismatch)
	}
	copy(dst, e.values)
	return dst
}

// complexEigenTo extracts the complex eigenvectors from the Dense matrix r and
// stores them into the complex matrix c.
//
// The returned dense matrix contains the eigenvectors of the decomposition
// in the columns of the n×n matrix in the same order as their eigenvalues.
// If the j-th eigenvalue is real, then
//  dst_j = d[:,j],
// and if it is not real, then j and j+1 form a complex conjugate pair and the
// eigenvectors can be recovered as
//  dst_j     = d[:,j] + i*d[:,j+1],
//  dst_{j+1} = d[:,j] - i*d[:,j+1],
// where i is the imaginary unit.
func (e *Eigen) complexEigenTo(dst *CDense, d *Dense) {
	r, c := d.Dims()
	cr, cc := dst.Dims()
	if r != cr {
		panic("size mismatch")
	}
	if c != cc {
		panic("size mismatch")
	}
	for j := 0; j < c; j++ {
		if imag(e.values[j]) == 0 {
			for i := 0; i < r; i++ {
				dst.set(i, j, complex(d.at(i, j), 0))
			}
			continue
		}
		for i := 0; i < r; i++ {
			real := d.at(i, j)
			imag := d.at(i, j+1)
			dst.set(i, j, complex(real, imag))
			dst.set(i, j+1, complex(real, -imag))
		}
		j++
	}
}

// Vectors returns the right eigenvectors of the decomposition. Vectors
// will panic if the right eigenvectors were not computed during the factorization,
// or if the factorization was not successful.
//
// The computed eigenvectors are normalized to have Euclidean norm equal to 1
// and largest component real.
func (e *Eigen) VectorsTo(dst *CDense) *CDense {
	if !e.succFact() {
		panic(badFact)
	}
	if !e.right {
		panic(badNoVect)
	}
	if dst == nil {
		dst = NewCDense(e.n, e.n, nil)
	} else {
		dst.reuseAs(e.n, e.n)
	}
	dst.Copy(e.rVectors)
	return dst
}

// LeftVectors returns the left eigenvectors of the decomposition. Vectors
// will panic if the left eigenvectors were not computed during the factorization,
// or if the factorization was not successful.
//
// The computed eigenvectors are normalized to have Euclidean norm equal to 1
// and largest component real.
func (e *Eigen) LeftVectorsTo(dst *CDense) *CDense {
	if !e.succFact() {
		panic(badFact)
	}
	if !e.left {
		panic(badNoVect)
	}
	if dst == nil {
		dst = NewCDense(e.n, e.n, nil)
	} else {
		dst.reuseAs(e.n, e.n)
	}
	dst.Copy(e.lVectors)
	return dst
}
<commit_msg>mat: update documentation for Eigen<commit_after>// Copyright ©2013 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat

import (
	"gonum.org/v1/gonum/lapack"
	"gonum.org/v1/gonum/lapack/lapack64"
)

const (
	badFact   = "mat: use without successful factorization"
	badNoVect = "mat: eigenvectors not computed"
)

// EigenSym is a type for creating and manipulating the Eigen decomposition of
// symmetric matrices.
type EigenSym struct {
	vectorsComputed bool

	values  []float64
	vectors *Dense
}

// Factorize computes the eigenvalue decomposition of the symmetric matrix a.
// The Eigen decomposition is defined as
//  A = P * D * P^-1
// where D is a diagonal matrix containing the eigenvalues of the matrix, and
// P is a matrix of the eigenvectors of A. Factorize computes the eigenvalues
// in ascending order. If the vectors input argument is false, the eigenvectors
// are not computed.
//
// Factorize returns whether the decomposition succeeded. If the decomposition
// failed, methods that require a successful factorization will panic.
func (e *EigenSym) Factorize(a Symmetric, vectors bool) (ok bool) {
	n := a.Symmetric()
	sd := NewSymDense(n, nil)
	sd.CopySym(a)

	jobz := lapack.EVNone
	if vectors {
		jobz = lapack.EVCompute
	}
	w := make([]float64, n)
	work := []float64{0}
	lapack64.Syev(jobz, sd.mat, w, work, -1)

	work = getFloats(int(work[0]), false)
	ok = lapack64.Syev(jobz, sd.mat, w, work, len(work))
	putFloats(work)
	if !ok {
		e.vectorsComputed = false
		e.values = nil
		e.vectors = nil
		return false
	}
	e.vectorsComputed = vectors
	e.values = w
	e.vectors = NewDense(n, n, sd.mat.Data)
	return true
}

// succFact returns whether the receiver contains a successful factorization.
func (e *EigenSym) succFact() bool {
	return len(e.values) != 0
}

// Values extracts the eigenvalues of the factorized matrix. If dst is
// non-nil, the values are stored in-place into dst. In this case
// dst must have length n, otherwise Values will panic. If dst is
// nil, then a new slice will be allocated of the proper length and filled
// with the eigenvalues.
//
// Values panics if the Eigen decomposition was not successful.
func (e *EigenSym) Values(dst []float64) []float64 {
	if !e.succFact() {
		panic(badFact)
	}
	if dst == nil {
		dst = make([]float64, len(e.values))
	}
	if len(dst) != len(e.values) {
		panic(ErrSliceLengthMismatch)
	}
	copy(dst, e.values)
	return dst
}

// EigenvectorsSym extracts the eigenvectors of the factorized matrix and stores
// them in the receiver. Each eigenvector is a column corresponding to the
// respective eigenvalue returned by e.Values.
//
// EigenvectorsSym panics if the factorization was not successful or if the
// decomposition did not compute the eigenvectors.
func (m *Dense) EigenvectorsSym(e *EigenSym) {
	if !e.succFact() {
		panic(badFact)
	}
	if !e.vectorsComputed {
		panic(badNoVect)
	}
	m.reuseAs(len(e.values), len(e.values))
	m.Copy(e.vectors)
}

// Eigen is a type for creating and using the eigenvalue decomposition of a dense matrix.
type Eigen struct {
	n int // The size of the factorized matrix.

	right bool // have the right eigenvectors been computed
	left  bool // have the left eigenvectors been computed

	values   []complex128
	rVectors *CDense
	lVectors *CDense
}

// succFact returns whether the receiver contains a successful factorization.
func (e *Eigen) succFact() bool {
	return len(e.values) != 0
}

// Factorize computes the eigenvalues of the square matrix a, and optionally
// the eigenvectors.
//
// A right eigenvalue/eigenvector combination is defined by
//  A * x_r = λ * x_r
// where x_r is the column vector called an eigenvector, and λ is the corresponding
// eigenvalue.
//
// Similarly, a left eigenvalue/eigenvector combination is defined by
//  x_l * A = λ * x_l
// The eigenvalues, but not the eigenvectors, are the same for both decompositions.
//
// Typically eigenvectors refer to right eigenvectors.
//
// In all cases, Factorize computes the eigenvalues of the matrix. If right and left
// are true, then the right and left eigenvectors will be computed, respectively.
// Eigen panics if the input matrix is not square.
//
// Factorize returns whether the decomposition succeeded. If the decomposition
// failed, methods that require a successful factorization will panic.
func (e *Eigen) Factorize(a Matrix, left, right bool) (ok bool) {
	// Copy a because it is modified during the Lapack call.
	r, c := a.Dims()
	if r != c {
		panic(ErrShape)
	}
	var sd Dense
	sd.Clone(a)

	var vl, vr Dense
	jobvl := lapack.LeftEVNone
	jobvr := lapack.RightEVNone
	if left {
		vl = *NewDense(r, r, nil)
		jobvl = lapack.LeftEVCompute
	}
	if right {
		vr = *NewDense(c, c, nil)
		jobvr = lapack.RightEVCompute
	}

	wr := getFloats(c, false)
	defer putFloats(wr)
	wi := getFloats(c, false)
	defer putFloats(wi)

	work := []float64{0}
	lapack64.Geev(jobvl, jobvr, sd.mat, wr, wi, vl.mat, vr.mat, work, -1)
	work = getFloats(int(work[0]), false)
	first := lapack64.Geev(jobvl, jobvr, sd.mat, wr, wi, vl.mat, vr.mat, work, len(work))
	putFloats(work)

	if first != 0 {
		e.values = nil
		return false
	}
	e.n = r
	e.right = right
	e.left = left

	// Construct complex eigenvalues from float64 data.
	values := make([]complex128, r)
	for i, v := range wr {
		values[i] = complex(v, wi[i])
	}
	e.values = values

	// Construct complex eigenvectors from float64 data.
	var cvl, cvr CDense
	if left {
		cvl = *NewCDense(r, r, nil)
		e.complexEigenTo(&cvl, &vl)
		e.lVectors = &cvl
	} else {
		e.lVectors = nil
	}
	if right {
		cvr = *NewCDense(c, c, nil)
		e.complexEigenTo(&cvr, &vr)
		e.rVectors = &cvr
	} else {
		e.rVectors = nil
	}
	return true
}

// Values extracts the eigenvalues of the factorized matrix. If dst is
// non-nil, the values are stored in-place into dst. In this case
// dst must have length n, otherwise Values will panic. If dst is
// nil, then a new slice will be allocated of the proper length and
// filed with the eigenvalues.
//
// Values panics if the Eigen decomposition was not successful.
func (e *Eigen) Values(dst []complex128) []complex128 {
	if !e.succFact() {
		panic(badFact)
	}
	if dst == nil {
		dst = make([]complex128, e.n)
	}
	if len(dst) != e.n {
		panic(ErrSliceLengthMismatch)
	}
	copy(dst, e.values)
	return dst
}

// complexEigenTo extracts the complex eigenvectors from the real matrix d
// and stores them into the complex matrix dst.
//
// The columns of the returned n×n dense matrix contain the eigenvectors of the
// decomposition in the same order as the eigenvalues.
// If the j-th eigenvalue is real, then
//  dst[:,j] = d[:,j],
// and if it is not real, then the elements of the j-th and (j+1)-th columns of d
// form complex conjugate pairs and the eigenvectors are recovered as
//  dst[:,j]   = d[:,j] + i*d[:,j+1],
//  dst[:,j+1] = d[:,j] - i*d[:,j+1],
// where i is the imaginary unit.
func (e *Eigen) complexEigenTo(dst *CDense, d *Dense) {
	r, c := d.Dims()
	cr, cc := dst.Dims()
	if r != cr {
		panic("size mismatch")
	}
	if c != cc {
		panic("size mismatch")
	}
	for j := 0; j < c; j++ {
		if imag(e.values[j]) == 0 {
			for i := 0; i < r; i++ {
				dst.set(i, j, complex(d.at(i, j), 0))
			}
			continue
		}
		for i := 0; i < r; i++ {
			real := d.at(i, j)
			imag := d.at(i, j+1)
			dst.set(i, j, complex(real, imag))
			dst.set(i, j+1, complex(real, -imag))
		}
		j++
	}
}

// VectorsTo returns the right eigenvectors of the decomposition. VectorsTo
// will panic if the right eigenvectors were not computed during the factorization,
// or if the factorization was not successful.
//
// The computed eigenvectors are normalized to have Euclidean norm equal to 1
// and largest component real.
func (e *Eigen) VectorsTo(dst *CDense) *CDense {
	if !e.succFact() {
		panic(badFact)
	}
	if !e.right {
		panic(badNoVect)
	}
	if dst == nil {
		dst = NewCDense(e.n, e.n, nil)
	} else {
		dst.reuseAs(e.n, e.n)
	}
	dst.Copy(e.rVectors)
	return dst
}

// LeftVectorsTo returns the left eigenvectors of the decomposition. LeftVectorsTo
// will panic if the left eigenvectors were not computed during the factorization,
// or if the factorization was not successful.
//
// The computed eigenvectors are normalized to have Euclidean norm equal to 1
// and largest component real.
func (e *Eigen) LeftVectorsTo(dst *CDense) *CDense {
	if !e.succFact() {
		panic(badFact)
	}
	if !e.left {
		panic(badNoVect)
	}
	if dst == nil {
		dst = NewCDense(e.n, e.n, nil)
	} else {
		dst.reuseAs(e.n, e.n)
	}
	dst.Copy(e.lVectors)
	return dst
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"

	"github.com/juju/errors"
)

/*
 * Get Functions
 */

// GetSafes returns a list of safe
func (o *OnlineAPI) GetSafes(useCache bool) (safes []OnlineGetSafe, err error) {
	if safes, err = o.cache.CopySafes(); err == nil {
		return
	}
	if !useCache {
		if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/safe", APIUrl), &safes); err != nil {
			err = errors.Annotate(err, "GetSafes")
			return
		}
		for _, safe := range safes {
			if _, ok := o.cache.GetSafe(safe.UUIDRef); !ok {
				o.cache.InsertSafe(safe.UUIDRef, safe)
			}
		}
	}
	return
}

// GetSafe returns a safe
func (o *OnlineAPI) GetSafe(uuid string) (safe OnlineGetSafe, err error) {
	// TODO: enable to use the name instead of only the UUID
	var (
		ok bool
	)

	if safe, ok = o.cache.GetSafe(uuid); ok {
		return
	}
	if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/safe/%s", APIUrl, uuid), &safe); err != nil {
		err = errors.Annotate(err, "GetSafe")
		return
	}
	o.cache.InsertSafe(safe.UUIDRef, safe)
	return
}

// GetPlatforms returns a list of platform
func (o *OnlineAPI) GetPlatforms() (platform []OnlineGetPlatform, err error) {
	if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/platform", APIUrl), &platform); err != nil {
		err = errors.Annotate(err, "GetPlatforms")
	}
	return
}

// GetPlatform returns a platform
func (o *OnlineAPI) GetPlatform(uuid string) (platform OnlineGetPlatform, err error) {
	// TODO: enable to use the name instead of only the UUID
	if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/platform/%s", APIUrl, uuid), &platform); err != nil {
		err = errors.Annotate(err, "GetPlatform")
	}
	return
}

func (o *OnlineAPI) GetSSHKeys() (keys []OnlineGetSSHKey, err error) {
	if err = o.getWrapper(fmt.Sprintf("%s/user/key/ssh", APIUrl), &keys); err != nil {
		err = errors.Annotate(err, "GetSSHKeys")
	}
	return
}

func (o *OnlineAPI) GetSSHKey(uuid string) (key OnlineGetSSHKey, err error) {
	if err = o.getWrapper(fmt.Sprintf("%s/user/key/ssh/%s", APIUrl, uuid), &key); err != nil {
		err = errors.Annotate(err, "GetSSHKey")
	}
	return
}

func (o *OnlineAPI) GetArchives(uuidSafe string, useCache bool) (archives []OnlineGetArchive, err error) {
	if archives, err = o.cache.CopyArchives(uuidSafe); err == nil {
		return
	}
	if !useCache {
		if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/safe/%s/archive", APIUrl, uuidSafe), &archives); err != nil {
			err = errors.Annotate(err, "GetArchives")
			return
		}
		for _, archive := range archives {
			if _, ok := o.cache.GetArchive(uuidSafe, archive.UUIDRef); !ok {
				o.cache.InsertArchive(uuidSafe, archive.UUIDRef, archive)
			}
		}
	}
	return
}

func (o *OnlineAPI) GetArchive(uuidSafe, uuidArchive string) (archive OnlineGetArchive, err error) {
	var (
		ok bool
	)

	if archive, ok = o.cache.GetArchive(uuidSafe, uuidArchive); ok {
		return
	}
	if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/safe/%s/archive/%s", APIUrl, uuidSafe, uuidArchive), &archive); err != nil {
		err = errors.Annotate(err, "GetArchive")
		return
	}
	o.cache.InsertArchive(uuidSafe, uuidArchive, archive)
	return
}

func (o *OnlineAPI) GetBucket(uuidSafe, uuidArchive string) (bucket OnlineGetBucket, err error) {
	var (
		ok bool
	)

	if bucket, ok = o.cache.GetBucket(uuidSafe, uuidArchive); ok {
		return
	}
	if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/safe/%s/archive/%s/bucket", APIUrl, uuidSafe, uuidArchive), &bucket); err != nil {
		err = errors.Annotate(err, "GetBucket")
		return
	}
	o.cache.InsertBucket(uuidSafe, uuidArchive, bucket)
	return
}

/*
 * Create Functions
 */

func (o *OnlineAPI) CreateSafe(name, desc string) (uuid string, err error) {
	var (
		buff []byte
	)

	if buff, err = o.postWrapper(fmt.Sprintf("%s/storage/c14/safe", APIUrl), OnlinePostSafe{
		Name:        name,
		Description: desc,
	}); err != nil {
		err = errors.Annotate(err, "CreateSafe")
		return
	}
	uuid = string(buff)
	return
}

type ConfigCreateArchive struct {
	UUIDSafe  string
	Name      string
	Desc      string
	Parity    string
	Protocols []string
	SSHKeys   []string
	Platforms []string
	Days      int
}

func (o *OnlineAPI) CreateArchive(config ConfigCreateArchive) (uuid string, err error) {
	var (
		buff []byte
	)

	if buff, err = o.postWrapper(fmt.Sprintf("%s/storage/c14/safe/%s/archive", APIUrl, config.UUIDSafe), OnlinePostArchive{
		Name:        config.Name,
		Description: config.Desc,
		Protocols:   config.Protocols,
		SSHKeys:     config.SSHKeys,
		Platforms:   config.Platforms,
		Days:        config.Days,
	}); err != nil {
		err = errors.Annotate(err, "CreateArchive")
		return
	}
	uuid = string(buff)
	return
}

/*
 * Delete Functions
 */

func (o *OnlineAPI) DeleteSafe(uuid string) (err error) {
	// TODO: remove from cache
	if err = o.deleteWrapper(fmt.Sprintf("%s/storage/c14/safe/%s", APIUrl, uuid)); err != nil {
		err = errors.Annotate(err, "DeleteSafe")
	}
	return
}

func (o *OnlineAPI) DeleteArchive(uuidSafe, uuidArchive string) (err error) {
	// TODO: remove from cache
	if err = o.deleteWrapper(fmt.Sprintf("%s/storage/c14/safe/%s/archive/%s", APIUrl, uuidSafe, uuidArchive)); err != nil {
		err = errors.Annotate(err, "DeleteArchive")
	}
	return
}
<commit_msg>cache: register object during the create<commit_after>package api

import (
	"fmt"

	"github.com/juju/errors"
)

/*
 * Get Functions
 */

// GetSafes returns a list of safe
func (o *OnlineAPI) GetSafes(useCache bool) (safes []OnlineGetSafe, err error) {
	if safes, err = o.cache.CopySafes(); err == nil {
		return
	}
	if !useCache {
		if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/safe", APIUrl), &safes); err != nil {
			err = errors.Annotate(err, "GetSafes")
			return
		}
		for _, safe := range safes {
			if _, ok := o.cache.GetSafe(safe.UUIDRef); !ok {
				o.cache.InsertSafe(safe.UUIDRef, safe)
			}
		}
	}
	return
}

// GetSafe returns a safe
func (o *OnlineAPI) GetSafe(uuid string) (safe OnlineGetSafe, err error) {
	// TODO: enable to use the name instead of only the UUID
	var (
		ok bool
	)

	if safe, ok = o.cache.GetSafe(uuid); ok {
		return
	}
	if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/safe/%s", APIUrl, uuid), &safe); err != nil {
		err = errors.Annotate(err, "GetSafe")
		return
	}
	o.cache.InsertSafe(safe.UUIDRef, safe)
	return
}

// GetPlatforms returns a list of platform
func (o *OnlineAPI) GetPlatforms() (platform []OnlineGetPlatform, err error) {
	if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/platform", APIUrl), &platform); err != nil {
		err = errors.Annotate(err, "GetPlatforms")
	}
	return
}

// GetPlatform returns a platform
func (o *OnlineAPI) GetPlatform(uuid string) (platform OnlineGetPlatform, err error) {
	// TODO: enable to use the name instead of only the UUID
	if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/platform/%s", APIUrl, uuid), &platform); err != nil {
		err = errors.Annotate(err, "GetPlatform")
	}
	return
}

func (o *OnlineAPI) GetSSHKeys() (keys []OnlineGetSSHKey, err error) {
	if err = o.getWrapper(fmt.Sprintf("%s/user/key/ssh", APIUrl), &keys); err != nil {
		err = errors.Annotate(err, "GetSSHKeys")
	}
	return
}

func (o *OnlineAPI) GetSSHKey(uuid string) (key OnlineGetSSHKey, err error) {
	if err = o.getWrapper(fmt.Sprintf("%s/user/key/ssh/%s", APIUrl, uuid), &key); err != nil {
		err = errors.Annotate(err, "GetSSHKey")
	}
	return
}

func (o *OnlineAPI) GetArchives(uuidSafe string, useCache bool) (archives []OnlineGetArchive, err error) {
	if archives, err = o.cache.CopyArchives(uuidSafe); err == nil {
		return
	}
	if !useCache {
		if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/safe/%s/archive", APIUrl, uuidSafe), &archives); err != nil {
			err = errors.Annotate(err, "GetArchives")
			return
		}
		for _, archive := range archives {
			if _, ok := o.cache.GetArchive(uuidSafe, archive.UUIDRef); !ok {
				o.cache.InsertArchive(uuidSafe, archive.UUIDRef, archive)
			}
		}
	}
	return
}

func (o *OnlineAPI) GetArchive(uuidSafe, uuidArchive string) (archive OnlineGetArchive, err error) {
	var (
		ok bool
	)

	if archive, ok = o.cache.GetArchive(uuidSafe, uuidArchive); ok {
		return
	}
	if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/safe/%s/archive/%s", APIUrl, uuidSafe, uuidArchive), &archive); err != nil {
		err = errors.Annotate(err, "GetArchive")
		return
	}
	o.cache.InsertArchive(uuidSafe, uuidArchive, archive)
	return
}

func (o *OnlineAPI) GetBucket(uuidSafe, uuidArchive string) (bucket OnlineGetBucket, err error) {
	var (
		ok bool
	)

	if bucket, ok = o.cache.GetBucket(uuidSafe, uuidArchive); ok {
		return
	}
	if err = o.getWrapper(fmt.Sprintf("%s/storage/c14/safe/%s/archive/%s/bucket", APIUrl, uuidSafe, uuidArchive), &bucket); err != nil {
		err = errors.Annotate(err, "GetBucket")
		return
	}
	o.cache.InsertBucket(uuidSafe, uuidArchive, bucket)
	return
}

/*
 * Create Functions
 */

func (o *OnlineAPI) CreateSafe(name, desc string) (uuid string, err error) {
	var (
		buff []byte
	)

	if buff, err = o.postWrapper(fmt.Sprintf("%s/storage/c14/safe", APIUrl), OnlinePostSafe{
		Name:        name,
		Description: desc,
	}); err != nil {
		err = errors.Annotate(err, "CreateSafe")
		return
	}
	uuid = string(buff)
	_, err = o.GetSafe(uuid)
	return
}

type ConfigCreateArchive struct {
	UUIDSafe  string
	Name      string
	Desc      string
	Parity    string
	Protocols []string
	SSHKeys   []string
	Platforms []string
	Days      int
}

func (o *OnlineAPI) CreateArchive(config ConfigCreateArchive) (uuid string, err error) {
	var (
		buff []byte
	)

	if buff, err = o.postWrapper(fmt.Sprintf("%s/storage/c14/safe/%s/archive", APIUrl, config.UUIDSafe), OnlinePostArchive{
		Name:        config.Name,
		Description: config.Desc,
		Protocols:   config.Protocols,
		SSHKeys:     config.SSHKeys,
		Platforms:   config.Platforms,
		Days:        config.Days,
	}); err != nil {
		err = errors.Annotate(err, "CreateArchive")
		return
	}
	uuid = string(buff)
	_, err = o.GetArchive(config.UUIDSafe, uuid)
	return
}

/*
 * Delete Functions
 */

func (o *OnlineAPI) DeleteSafe(uuid string) (err error) {
	// TODO: remove from cache
	if err = o.deleteWrapper(fmt.Sprintf("%s/storage/c14/safe/%s", APIUrl, uuid)); err != nil {
		err = errors.Annotate(err, "DeleteSafe")
	}
	return
}

func (o *OnlineAPI) DeleteArchive(uuidSafe, uuidArchive string) (err error) {
	// TODO: remove from cache
	if err = o.deleteWrapper(fmt.Sprintf("%s/storage/c14/safe/%s/archive/%s", APIUrl, uuidSafe, uuidArchive)); err != nil {
		err = errors.Annotate(err, "DeleteArchive")
	}
	return
}
<|endoftext|>


<commit_before>package api

import (
	"errors"
	"net/http"
	"time"

	"github.com/grafana/grafana/pkg/tsdb/grafanads"

	"github.com/grafana/grafana-plugin-sdk-go/backend"
	"github.com/grafana/grafana/pkg/api/dtos"
	"github.com/grafana/grafana/pkg/api/response"
	"github.com/grafana/grafana/pkg/expr"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/plugins/adapters"
)

// QueryMetricsV2 returns query metrics.
// POST /api/ds/query   DataSource query w/ expressions
func (hs *HTTPServer) QueryMetricsV2(c *models.ReqContext, reqDTO dtos.MetricRequest) response.Response {
	if len(reqDTO.Queries) == 0 {
		return response.Error(http.StatusBadRequest, "No queries found in query", nil)
	}

	timeRange := plugins.NewDataTimeRange(reqDTO.From, reqDTO.To)
	request := plugins.DataQuery{
		TimeRange: &timeRange,
		Debug:     reqDTO.Debug,
		User:      c.SignedInUser,
		Queries:   make([]plugins.DataSubQuery, 0, len(reqDTO.Queries)),
	}

	// Loop to see if we have an expression.
	prevType := ""
	var ds *models.DataSource
	for _, query := range reqDTO.Queries {
		dsType := query.Get("datasource").MustString("")
		if dsType == expr.DatasourceName {
			return hs.handleExpressions(c, reqDTO)
		}
		if prevType != "" && prevType != dsType {
			// For mixed datasource case, each data source is sent in a single request.
			// So only the datasource from the first query is needed. As all requests
			// should be the same data source.
			hs.log.Debug("Can't process query since it's missing data source ID")
			return response.Error(http.StatusBadRequest, "All queries must use the same datasource", nil)
		}

		if ds == nil {
			// require ID for everything
			dsID, err := query.Get("datasourceId").Int64()
			if err != nil {
				hs.log.Debug("Can't process query since it's missing data source ID")
				return response.Error(http.StatusBadRequest, "Query missing data source ID", nil)
			}
			if dsID == grafanads.DatasourceID {
				ds = grafanads.DataSourceModel(c.OrgId)
			} else {
				ds, err = hs.DataSourceCache.GetDatasource(dsID, c.SignedInUser, c.SkipCache)
				if err != nil {
					return hs.handleGetDataSourceError(err, dsID)
				}
			}
		}
		prevType = dsType
	}

	for _, query := range reqDTO.Queries {
		hs.log.Debug("Processing metrics query", "query", query)

		request.Queries = append(request.Queries, plugins.DataSubQuery{
			RefID:         query.Get("refId").MustString("A"),
			MaxDataPoints: query.Get("maxDataPoints").MustInt64(100),
			IntervalMS:    query.Get("intervalMs").MustInt64(1000),
			QueryType:     query.Get("queryType").MustString(""),
			Model:         query,
			DataSource:    ds,
		})
	}

	err := hs.PluginRequestValidator.Validate(ds.Url, nil)
	if err != nil {
		return response.Error(http.StatusForbidden, "Access denied", err)
	}

	req, err := hs.createRequest(ds, request)
	if err != nil {
		return response.Error(http.StatusBadRequest, "Request formation error", err)
	}

	resp, err := hs.pluginClient.QueryData(c.Req.Context(), req)
	if err != nil {
		return response.Error(http.StatusInternalServerError, "Metric request error", err)
	}

	return toMacronResponse(resp)
}

func toMacronResponse(qdr *backend.QueryDataResponse) response.Response {
	statusCode := http.StatusOK
	for _, res := range qdr.Responses {
		if res.Error != nil {
			statusCode = http.StatusBadRequest
		}
	}

	return response.JSONStreaming(statusCode, qdr)
}

// handleExpressions handles POST /api/ds/query when there is an expression.
func (hs *HTTPServer) handleExpressions(c *models.ReqContext, reqDTO dtos.MetricRequest) response.Response {
	timeRange := plugins.NewDataTimeRange(reqDTO.From, reqDTO.To)
	request := plugins.DataQuery{
		TimeRange: &timeRange,
		Debug:     reqDTO.Debug,
		User:      c.SignedInUser,
		Queries:   make([]plugins.DataSubQuery, 0, len(reqDTO.Queries)),
	}

	for _, query := range reqDTO.Queries {
		hs.log.Debug("Processing metrics query", "query", query)
		name := query.Get("datasource").MustString("")

		datasourceID, err := query.Get("datasourceId").Int64()
		if err != nil {
			hs.log.Debug("Can't process query since it's missing data source ID")
			return response.Error(400, "Query missing data source ID", nil)
		}

		if name != expr.DatasourceName {
			// Expression requests have everything in one request, so need to check
			// all data source queries for possible permission / not found issues.
			if _, err = hs.DataSourceCache.GetDatasource(datasourceID, c.SignedInUser, c.SkipCache); err != nil {
				return hs.handleGetDataSourceError(err, datasourceID)
			}
		}

		request.Queries = append(request.Queries, plugins.DataSubQuery{
			RefID:         query.Get("refId").MustString("A"),
			MaxDataPoints: query.Get("maxDataPoints").MustInt64(100),
			IntervalMS:    query.Get("intervalMs").MustInt64(1000),
			QueryType:     query.Get("queryType").MustString(""),
			Model:         query,
		})
	}

	exprService := expr.Service{
		Cfg:         hs.Cfg,
		DataService: hs.DataService,
	}
	qdr, err := exprService.WrapTransformData(c.Req.Context(), request)
	if err != nil {
		return response.Error(500, "expression request error", err)
	}
	return toMacronResponse(qdr)
}

func (hs *HTTPServer) handleGetDataSourceError(err error, datasourceID int64) *response.NormalResponse {
	hs.log.Debug("Encountered error getting data source", "err", err, "id", datasourceID)
	if errors.Is(err, models.ErrDataSourceAccessDenied) {
		return response.Error(403, "Access denied to data source", err)
	}
	if errors.Is(err, models.ErrDataSourceNotFound) {
		return response.Error(400, "Invalid data source ID", err)
	}
	return response.Error(500, "Unable to load data source metadata", err)
}

// QueryMetrics returns query metrics
// POST /api/tsdb/query
func (hs *HTTPServer) QueryMetrics(c *models.ReqContext, reqDto dtos.MetricRequest) response.Response {
	if len(reqDto.Queries) == 0 {
		return response.Error(http.StatusBadRequest, "No queries found in query", nil)
	}

	datasourceId, err := reqDto.Queries[0].Get("datasourceId").Int64()
	if err != nil {
		return response.Error(http.StatusBadRequest, "Query missing datasourceId", nil)
	}

	ds, err := hs.DataSourceCache.GetDatasource(datasourceId, c.SignedInUser, c.SkipCache)
	if err != nil {
		return hs.handleGetDataSourceError(err, datasourceId)
	}

	err = hs.PluginRequestValidator.Validate(ds.Url, nil)
	if err != nil {
		return response.Error(http.StatusForbidden, "Access denied", err)
	}

	timeRange := plugins.NewDataTimeRange(reqDto.From, reqDto.To)
	request := plugins.DataQuery{
		TimeRange: &timeRange,
		Debug:     reqDto.Debug,
		User:      c.SignedInUser,
	}

	for _, query := range reqDto.Queries {
		request.Queries = append(request.Queries, plugins.DataSubQuery{
			RefID:         query.Get("refId").MustString("A"),
			MaxDataPoints: query.Get("maxDataPoints").MustInt64(100),
			IntervalMS:    query.Get("intervalMs").MustInt64(1000),
			Model:         query,
			DataSource:    ds,
		})
	}

	resp, err := hs.DataService.HandleRequest(c.Req.Context(), ds, request)
	if err != nil {
		return response.Error(http.StatusInternalServerError, "Metric request error", err)
	}

	statusCode := http.StatusOK
	for _, res := range resp.Results {
		if res.Error != nil {
			res.ErrorString = res.Error.Error()
			resp.Message = res.ErrorString
			statusCode = http.StatusBadRequest
		}
	}

	return response.JSON(statusCode, &resp)
}

// nolint:staticcheck // plugins.DataQueryResponse deprecated
func (hs *HTTPServer) createRequest(ds *models.DataSource, query plugins.DataQuery) (*backend.QueryDataRequest, error) {
	instanceSettings, err := adapters.ModelToInstanceSettings(ds, hs.decryptSecureJsonDataFn())
	if err != nil {
		return nil, err
	}

	req := &backend.QueryDataRequest{
		PluginContext: backend.PluginContext{
			OrgID:                      ds.OrgId,
			PluginID:                   ds.Type,
			User:                       adapters.BackendUserFromSignedInUser(query.User),
			DataSourceInstanceSettings: instanceSettings,
		},
		Queries: []backend.DataQuery{},
		Headers: query.Headers,
	}

	for _, q := range query.Queries {
		modelJSON, err := q.Model.MarshalJSON()
		if err != nil {
			return nil, err
		}
		req.Queries = append(req.Queries, backend.DataQuery{
			RefID:         q.RefID,
			Interval:      time.Duration(q.IntervalMS) * time.Millisecond,
			MaxDataPoints: q.MaxDataPoints,
			TimeRange: backend.TimeRange{
				From: query.TimeRange.GetFromAsTimeUTC(),
				To:   query.TimeRange.GetToAsTimeUTC(),
			},
			QueryType: q.QueryType,
			JSON:      modelJSON,
		})
	}

	return req, nil
}
<commit_msg>add oauth pass thru logic to api/ds/query (#41352)<commit_after>package api

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/grafana/grafana/pkg/tsdb/grafanads"

	"github.com/grafana/grafana-plugin-sdk-go/backend"
	"github.com/grafana/grafana/pkg/api/dtos"
	"github.com/grafana/grafana/pkg/api/response"
	"github.com/grafana/grafana/pkg/expr"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/plugins/adapters"
)

// QueryMetricsV2 returns query metrics.
// POST /api/ds/query   DataSource query w/ expressions
func (hs *HTTPServer) QueryMetricsV2(c *models.ReqContext, reqDTO dtos.MetricRequest) response.Response {
	if len(reqDTO.Queries) == 0 {
		return response.Error(http.StatusBadRequest, "No queries found in query", nil)
	}

	timeRange := plugins.NewDataTimeRange(reqDTO.From, reqDTO.To)
	request := plugins.DataQuery{
		TimeRange: &timeRange,
		Debug:     reqDTO.Debug,
		User:      c.SignedInUser,
		Queries:   make([]plugins.DataSubQuery, 0, len(reqDTO.Queries)),
	}

	// Loop to see if we have an expression.
	prevType := ""
	var ds *models.DataSource
	for _, query := range reqDTO.Queries {
		dsType := query.Get("datasource").MustString("")
		if dsType == expr.DatasourceName {
			return hs.handleExpressions(c, reqDTO)
		}
		if prevType != "" && prevType != dsType {
			// For mixed datasource case, each data source is sent in a single request.
			// So only the datasource from the first query is needed. As all requests
			// should be the same data source.
			hs.log.Debug("Can't process query since it's missing data source ID")
			return response.Error(http.StatusBadRequest, "All queries must use the same datasource", nil)
		}

		if ds == nil {
			// require ID for everything
			dsID, err := query.Get("datasourceId").Int64()
			if err != nil {
				hs.log.Debug("Can't process query since it's missing data source ID")
				return response.Error(http.StatusBadRequest, "Query missing data source ID", nil)
			}
			if dsID == grafanads.DatasourceID {
				ds = grafanads.DataSourceModel(c.OrgId)
			} else {
				ds, err = hs.DataSourceCache.GetDatasource(dsID, c.SignedInUser, c.SkipCache)
				if err != nil {
					return hs.handleGetDataSourceError(err, dsID)
				}
			}
		}
		prevType = dsType
	}

	for _, query := range reqDTO.Queries {
		hs.log.Debug("Processing metrics query", "query", query)

		request.Queries = append(request.Queries, plugins.DataSubQuery{
			RefID:         query.Get("refId").MustString("A"),
			MaxDataPoints: query.Get("maxDataPoints").MustInt64(100),
			IntervalMS:    query.Get("intervalMs").MustInt64(1000),
			QueryType:     query.Get("queryType").MustString(""),
			Model:         query,
			DataSource:    ds,
		})
	}

	err := hs.PluginRequestValidator.Validate(ds.Url, nil)
	if err != nil {
		return response.Error(http.StatusForbidden, "Access denied", err)
	}

	req, err := hs.createRequest(c.Req.Context(), ds, request)
	if err != nil {
		return response.Error(http.StatusBadRequest, "Request formation error", err)
	}

	resp, err := hs.pluginClient.QueryData(c.Req.Context(), req)
	if err != nil {
		return response.Error(http.StatusInternalServerError, "Metric request error", err)
	}

	return toMacronResponse(resp)
}

func toMacronResponse(qdr *backend.QueryDataResponse) response.Response {
	statusCode := http.StatusOK
	for _, res := range qdr.Responses {
		if res.Error != nil {
			statusCode = http.StatusBadRequest
		}
	}

	return response.JSONStreaming(statusCode, qdr)
}

// handleExpressions handles POST /api/ds/query when there is an expression.
func (hs *HTTPServer) handleExpressions(c *models.ReqContext, reqDTO dtos.MetricRequest) response.Response {
	timeRange := plugins.NewDataTimeRange(reqDTO.From, reqDTO.To)
	request := plugins.DataQuery{
		TimeRange: &timeRange,
		Debug:     reqDTO.Debug,
		User:      c.SignedInUser,
		Queries:   make([]plugins.DataSubQuery, 0, len(reqDTO.Queries)),
	}

	for _, query := range reqDTO.Queries {
		hs.log.Debug("Processing metrics query", "query", query)
		name := query.Get("datasource").MustString("")

		datasourceID, err := query.Get("datasourceId").Int64()
		if err != nil {
			hs.log.Debug("Can't process query since it's missing data source ID")
			return response.Error(400, "Query missing data source ID", nil)
		}

		if name != expr.DatasourceName {
			// Expression requests have everything in one request, so need to check
			// all data source queries for possible permission / not found issues.
			if _, err = hs.DataSourceCache.GetDatasource(datasourceID, c.SignedInUser, c.SkipCache); err != nil {
				return hs.handleGetDataSourceError(err, datasourceID)
			}
		}

		request.Queries = append(request.Queries, plugins.DataSubQuery{
			RefID:         query.Get("refId").MustString("A"),
			MaxDataPoints: query.Get("maxDataPoints").MustInt64(100),
			IntervalMS:    query.Get("intervalMs").MustInt64(1000),
			QueryType:     query.Get("queryType").MustString(""),
			Model:         query,
		})
	}

	exprService := expr.Service{
		Cfg:         hs.Cfg,
		DataService: hs.DataService,
	}
	qdr, err := exprService.WrapTransformData(c.Req.Context(), request)
	if err != nil {
		return response.Error(500, "expression request error", err)
	}
	return toMacronResponse(qdr)
}

func (hs *HTTPServer) handleGetDataSourceError(err error, datasourceID int64) *response.NormalResponse {
	hs.log.Debug("Encountered error getting data source", "err", err, "id", datasourceID)
	if errors.Is(err, models.ErrDataSourceAccessDenied) {
		return response.Error(403, "Access denied to data source", err)
	}
	if errors.Is(err, models.ErrDataSourceNotFound) {
		return response.Error(400, "Invalid data source ID", err)
	}
	return response.Error(500, "Unable to load data source metadata", err)
}

// QueryMetrics returns query metrics
// POST /api/tsdb/query
func (hs *HTTPServer) QueryMetrics(c *models.ReqContext, reqDto dtos.MetricRequest) response.Response {
	if len(reqDto.Queries) == 0 {
		return response.Error(http.StatusBadRequest, "No queries found in query", nil)
	}

	datasourceId, err := reqDto.Queries[0].Get("datasourceId").Int64()
	if err != nil {
		return response.Error(http.StatusBadRequest, "Query missing datasourceId", nil)
	}

	ds, err := hs.DataSourceCache.GetDatasource(datasourceId, c.SignedInUser, c.SkipCache)
	if err != nil {
		return hs.handleGetDataSourceError(err, datasourceId)
	}

	err = hs.PluginRequestValidator.Validate(ds.Url, nil)
	if err != nil {
		return response.Error(http.StatusForbidden, "Access denied", err)
	}

	timeRange := plugins.NewDataTimeRange(reqDto.From, reqDto.To)
	request := plugins.DataQuery{
		TimeRange: &timeRange,
		Debug:     reqDto.Debug,
		User:      c.SignedInUser,
	}

	for _, query := range reqDto.Queries {
		request.Queries = append(request.Queries, plugins.DataSubQuery{
			RefID:         query.Get("refId").MustString("A"),
			MaxDataPoints: query.Get("maxDataPoints").MustInt64(100),
			IntervalMS:    query.Get("intervalMs").MustInt64(1000),
			Model:         query,
			DataSource:    ds,
		})
	}

	resp, err := hs.DataService.HandleRequest(c.Req.Context(), ds, request)
	if err != nil {
		return response.Error(http.StatusInternalServerError, "Metric request error", err)
	}

	statusCode := http.StatusOK
	for _, res := range resp.Results {
		if res.Error != nil {
			res.ErrorString = res.Error.Error()
			resp.Message = res.ErrorString
			statusCode = http.StatusBadRequest
		}
	}

	return response.JSON(statusCode, &resp)
}

// nolint:staticcheck // plugins.DataQueryResponse deprecated
func (hs *HTTPServer) createRequest(ctx context.Context, ds *models.DataSource,
	query plugins.DataQuery) (*backend.QueryDataRequest, error) {
	instanceSettings, err := adapters.ModelToInstanceSettings(ds, hs.decryptSecureJsonDataFn())
	if err != nil {
		return nil, err
	}

	if query.Headers == nil {
		query.Headers = make(map[string]string)
	}

	if hs.OAuthTokenService.IsOAuthPassThruEnabled(ds) {
		if token := hs.OAuthTokenService.GetCurrentOAuthToken(ctx, query.User); token != nil {
			delete(query.Headers, "Authorization")
			query.Headers["Authorization"] = fmt.Sprintf("%s %s", token.Type(), token.AccessToken)
		}
	}

	req := &backend.QueryDataRequest{
		PluginContext: backend.PluginContext{
			OrgID:                      ds.OrgId,
			PluginID:                   ds.Type,
			User:                       adapters.BackendUserFromSignedInUser(query.User),
			DataSourceInstanceSettings: instanceSettings,
		},
		Queries: []backend.DataQuery{},
		Headers: query.Headers,
	}

	for _, q := range query.Queries {
		modelJSON, err := q.Model.MarshalJSON()
		if err != nil {
			return nil, err
		}
		req.Queries = append(req.Queries, backend.DataQuery{
			RefID:         q.RefID,
			Interval:      time.Duration(q.IntervalMS) * time.Millisecond,
			MaxDataPoints: q.MaxDataPoints,
			TimeRange: backend.TimeRange{
				From: query.TimeRange.GetFromAsTimeUTC(),
				To:   query.TimeRange.GetToAsTimeUTC(),
			},
			QueryType: q.QueryType,
			JSON:      modelJSON,
		})
	}

	return req, nil
}
<|endoftext|>


<commit_before>package cli

import (
	"errors"
	"fmt"
	"io"
	"os"

	"github.com/apprenda/kismatic/pkg/data"
	"github.com/apprenda/kismatic/pkg/install"
	"github.com/apprenda/kismatic/pkg/util"
	"github.com/spf13/cobra"
)

type upgradeOpts struct {
	generatedAssetsDir string
	verbose            bool
	outputFormat       string
	skipPreflight      bool
	online             bool
	planFile           string
	restartServices    bool
	partialAllowed     bool
	maxParallelWorkers int
	dryRun             bool
}

// NewCmdUpgrade returns the upgrade command
func NewCmdUpgrade(out io.Writer) *cobra.Command {
	var opts upgradeOpts
	cmd := &cobra.Command{
		Use:   "upgrade",
		Short: "Upgrade your Kubernetes cluster",
		Long: `Upgrade your Kubernetes cluster.

The upgrade process is applied to each node, one node at a time. If a private docker registry
is being used, the new container images will be pushed by Kismatic before starting to upgrade
nodes.

Nodes in the cluster are upgraded in the following order:

1. Etcd nodes
2. Master nodes
3. Worker nodes (regardless of specialization)
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return cmd.Help()
		},
	}

	cmd.PersistentFlags().StringVar(&opts.generatedAssetsDir, "generated-assets-dir", "generated", "path to the directory where assets generated during the installation process will be stored")
	cmd.PersistentFlags().BoolVar(&opts.verbose, "verbose", false, "enable verbose logging from the installation")
	cmd.PersistentFlags().StringVarP(&opts.outputFormat, "output", "o", "simple", "installation output format (options \"simple\"|\"raw\")")
	cmd.PersistentFlags().BoolVar(&opts.skipPreflight, "skip-preflight", false, "skip upgrade pre-flight checks")
	cmd.PersistentFlags().BoolVar(&opts.restartServices, "restart-services", false, "force restart cluster services (Use with care)")
	cmd.PersistentFlags().BoolVar(&opts.partialAllowed, "partial-ok", false, "allow the upgrade of ready nodes, and skip nodes that have been deemed unready for upgrade")
	cmd.PersistentFlags().BoolVar(&opts.dryRun, "dry-run", false, "simulate the upgrade, but don't actually upgrade the cluster")
	addPlanFileFlag(cmd.PersistentFlags(), &opts.planFile)

	// Subcommands
	cmd.AddCommand(NewCmdUpgradeOffline(out, &opts))
	cmd.AddCommand(NewCmdUpgradeOnline(out, &opts))
	return cmd
}

// NewCmdUpgradeOffline returns the command for running offline upgrades
func NewCmdUpgradeOffline(out io.Writer, opts *upgradeOpts) *cobra.Command {
	cmd := cobra.Command{
		Use:   "offline",
		Short: "Perform an offline upgrade of your Kubernetes cluster",
		Long: `Perform an offline upgrade of your Kubernetes cluster.

The offline upgrade is available for those clusters in which safety and availabilty are not a concern.
In this mode, the safety and availability checks will not be performed, nor will the nodes in the cluster
be drained of workloads.

Performing an offline upgrade could result in loss of critical data and reduced service
availability. For this reason, this method should not be used for clusters that are housing
production workloads.
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return doUpgrade(out, opts)
		},
	}
	cmd.Flags().IntVar(&opts.maxParallelWorkers, "max-parallel-workers", 1, "the maximum number of worker nodes to be upgraded in parallel")
	return &cmd
}

// NewCmdUpgradeOnline returns the command for running online upgrades
func NewCmdUpgradeOnline(out io.Writer, opts *upgradeOpts) *cobra.Command {
	cmd := cobra.Command{
		Use:   "online",
		Short: "Perform an online upgrade of your Kubernetes cluster",
		Long: `Perform an online upgrade of your Kubernetes cluster.

During an online upgrade, Kismatic will run safety and availability checks (see table below) against the
existing cluster before performing the upgrade. If any unsafe condition is detected, a report will
be printed, and the upgrade will not proceed.

If the node under upgrade is a Kubernetes node, it is cordoned and drained of workloads
before any changes are applied.
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			opts.online = true
			return doUpgrade(out, opts)
		},
	}
	return &cmd
}

func doUpgrade(out io.Writer, opts *upgradeOpts) error {
	if opts.maxParallelWorkers < 1 {
		return fmt.Errorf("max-parallel-workers must be greater or equal to 1, got: %d", opts.maxParallelWorkers)
	}

	planFile := opts.planFile
	planner := install.FilePlanner{File: planFile}
	executorOpts := install.ExecutorOptions{
		GeneratedAssetsDirectory: opts.generatedAssetsDir,
		RestartServices:          opts.restartServices,
		OutputFormat:             opts.outputFormat,
		Verbose:                  opts.verbose,
		DryRun:                   opts.dryRun,
	}
	executor, err := install.NewExecutor(out, os.Stderr, executorOpts)
	if err != nil {
		return err
	}
	preflightExecOpts := executorOpts
	preflightExecOpts.DryRun = false // We always want to run preflight, even if doing a dry-run
	preflightExec, err := install.NewPreFlightExecutor(out, os.Stderr, preflightExecOpts)
	if err != nil {
		return err
	}
	util.PrintHeader(out, "Computing upgrade plan", '=')

	// Read plan file
	if !planner.PlanExists() {
		util.PrettyPrintErr(out, "Reading plan file")
		return fmt.Errorf("plan file %q does not exist", planFile)
	}
	util.PrettyPrintOk(out, "Reading plan file")
	plan, err := planner.Read()
	if err != nil {
		util.PrettyPrintErr(out, "Reading plan file")
		return fmt.Errorf("error reading plan file %q: %v", planFile, err)
	}

	// Validate SSH connectivity to nodes
	if ok, errs := install.ValidatePlanSSHConnections(plan); !ok {
		util.PrettyPrintErr(out, "Validate SSH connectivity to nodes")
		util.PrintValidationErrors(out, errs)
		return fmt.Errorf("SSH connectivity validation errors found")
	}
	util.PrettyPrintOk(out, "Validate SSH connectivity to nodes")

	// Figure out which nodes to upgrade
	cv, err := install.ListVersions(plan)
	if err != nil {
		return fmt.Errorf("error listing cluster versions: %v", err)
	}
	var toUpgrade []install.ListableNode
	var toSkip []install.ListableNode
	for _, n := range cv.Nodes {
		if install.IsOlderVersion(n.Version) {
			toUpgrade = append(toUpgrade, n)
		} else {
			toSkip = append(toSkip, n)
		}
	}

	// Print the nodes that will be skipped
	if len(toSkip) > 0 {
		util.PrintHeader(out, "Skipping nodes", '=')
		for _, n := range toSkip {
			util.PrettyPrintOk(out, "- %q is at the target version %q", n.Node.Host, n.Version)
		}
		fmt.Fprintln(out)
	}

	if plan.ConfigureDockerRegistry() && plan.Cluster.DisconnectedInstallation {
		util.PrintHeader(out, "Upgrade: Docker Registry", '=')
		if err = executor.UpgradeDockerRegistry(*plan); err != nil {
			return fmt.Errorf("Failed to upgrade docker registry: %v", err)
		}
	}

	// Print message if there's no work to do
	if len(toUpgrade) == 0 {
		fmt.Fprintln(out, "All nodes are at the target version. Skipping node upgrades.")
	} else {
		if err = upgradeNodes(out, *plan, *opts, toUpgrade, executor, preflightExec); err != nil {
			return err
		}
	}

	if opts.partialAllowed {
		util.PrintColor(out, util.Green, `

Partial upgrade complete.

Cluster level services are still left to upgrade. These can only be upgraded
when performing a full upgrade. When you are ready, you may use "kismatic upgrade"
without the "--partial-ok" flag to perform a full upgrade.

`)
		return nil
	}

	// Upgrade the cluster services
	util.PrintHeader(out, "Upgrade: Cluster Services", '=')
	if err := executor.UpgradeClusterServices(*plan); err != nil {
		return fmt.Errorf("Failed to upgrade cluster services: %v", err)
	}

	if err := executor.RunSmokeTest(plan); err != nil {
		return fmt.Errorf("Smoke test failed: %v", err)
	}

	if !opts.dryRun {
		fmt.Fprintln(out)
		util.PrintColor(out, util.Green, "Upgrade complete\n")
		fmt.Fprintln(out)
	}
	return nil
}

func upgradeNodes(out io.Writer, plan install.Plan, opts upgradeOpts, nodesNeedUpgrade []install.ListableNode, executor install.Executor, preflightExec install.PreFlightExecutor) error {
	// Run safety checks if doing an online upgrade
	unsafeNodes := []install.ListableNode{}
	if opts.online {
		util.PrintHeader(out, "Validate Online Upgrade", '=')
		// Use the first master node for running kubectl
		client, err := plan.GetSSHClient(plan.Master.Nodes[0].Host)
		if err != nil {
			return fmt.Errorf("error getting SSH client: %v", err)
		}
		kubeClient := data.RemoteKubectl{SSHClient: client}
		for _, node := range nodesNeedUpgrade {
			util.PrettyPrint(out, "%s %v", node.Node.Host, node.Roles)
			errs := install.DetectNodeUpgradeSafety(plan, node.Node, kubeClient)
			if len(errs) != 0 {
				util.PrintError(out)
				fmt.Fprintln(out)
				for _, err := range errs {
					fmt.Println("-", err.Error())
				}
				unsafeNodes = append(unsafeNodes, node)
			} else {
				util.PrintOkln(out)
			}
		}
		// If we found any unsafe nodes, and we are not doing a partial upgrade, exit.
		if len(unsafeNodes) > 0 && !opts.partialAllowed {
			return errors.New("Unable to perform an online upgrade due to the unsafe conditions detected.")
		}
		// Block the upgrade if partial is allowed but there is an etcd or master node
		// that cannot be upgraded
		if opts.partialAllowed {
			for _, n := range unsafeNodes {
				for _, r := range n.Roles {
					if r == "master" || r == "etcd" {
						return errors.New("Unable to perform an online upgrade due to the unsafe conditions detected.")
					}
				}
			}
		}
	}

	// Run upgrade preflight on the nodes that are to be upgraded
	unreadyNodes := []install.ListableNode{}
	if !opts.skipPreflight {
		for _, node := range nodesNeedUpgrade {
			util.PrintHeader(out, fmt.Sprintf("Preflight Checks: %s %s", node.Node.Host, node.Roles), '=')
			if err := preflightExec.RunUpgradePreFlightCheck(&plan, node); err != nil {
				// return fmt.Errorf("Upgrade preflight check failed: %v", err)
				unreadyNodes = append(unreadyNodes, node)
			}
		}
	}

	// Block upgrade if we found unready nodes, and we are not doing a partial upgrade
	if len(unreadyNodes) > 0 && !opts.partialAllowed {
		return errors.New("Errors found during preflight checks")
	}

	// Block the upgrade if partial is allowed but there is an etcd or master node
	// that cannot be upgraded
	if opts.partialAllowed {
		for _, n := range unreadyNodes {
			for _, r := range n.Roles {
				if r == "master" || r == "etcd" {
					return errors.New("Errors found during preflight checks")
				}
			}
		}
	}

	// Filter out the nodes that are unsafe/unready
	toUpgrade := []install.ListableNode{}
	for _, n := range nodesNeedUpgrade {
		upgrade := true
		for _, unsafe := range unsafeNodes {
			if unsafe.Node == n.Node {
				upgrade = false
			}
		}
		for _, unready := range unreadyNodes {
			if unready.Node == n.Node {
				upgrade = false
			}
		}
		if upgrade {
			toUpgrade = append(toUpgrade, n)
		}
	}

	// get all etcd nodes
	etcdToUpgrade := make([]install.ListableNode, 0)
	for _, n := range install.NodesWithRoles(toUpgrade, "etcd") {
		// only transition nodes that are not 1.3.0...
		if install.IsLessThanVersion(n.Version, "v1.3.0-alpha.0") {
			etcdToUpgrade = append(etcdToUpgrade, n)
		}
	}
	if len(etcdToUpgrade) > 1 {
		// Run the upgrade on the nodes to Etcd v3.0.x
		if err := executor.UpgradeEtcd2Nodes(plan, etcdToUpgrade); err != nil {
			return fmt.Errorf("Failed to upgrade etcd2 nodes: %v", err)
		}
	}

	// All KET releases with v1.6+ do not need this to run
	var migrationNeeded bool
	for _, n := range install.NodesWithRoles(toUpgrade, "master") {
		if install.IsLessThanVersion(n.Version, "v1.3.0") {
			migrationNeeded = true
			break
		}
	}
	// Only upgrade if any of the masters are not already at KET 1.3
	if migrationNeeded {
		util.PrintHeader(out, "Migrate: Kubernetes Etcd Cluster", '=')
		if err := executor.MigrateEtcdCluster(plan); err != nil {
			return fmt.Errorf("Failed to migrate kubernetes etcd cluster: %v", err)
		}
	}

	// Run the upgrade on the nodes that need it
	if err := executor.UpgradeNodes(plan, toUpgrade, opts.online, opts.maxParallelWorkers); err != nil {
		return fmt.Errorf("Failed to upgrade nodes: %v", err)
	}
	return nil
}
<commit_msg>Updated comments about upgrades<commit_after>package cli

import (
	"errors"
	"fmt"
	"io"
	"os"

	"github.com/apprenda/kismatic/pkg/data"
	"github.com/apprenda/kismatic/pkg/install"
	"github.com/apprenda/kismatic/pkg/util"
	"github.com/spf13/cobra"
)

type upgradeOpts struct {
	generatedAssetsDir string
	verbose            bool
	outputFormat       string
	skipPreflight      bool
	online             bool
	planFile           string
	restartServices    bool
	partialAllowed     bool
	maxParallelWorkers int
	dryRun             bool
}

// NewCmdUpgrade returns the upgrade command
func NewCmdUpgrade(out io.Writer) *cobra.Command {
	var opts upgradeOpts
	cmd := &cobra.Command{
		Use:   "upgrade",
		Short: "Upgrade your Kubernetes cluster",
		Long: `Upgrade your Kubernetes cluster.

The upgrade process is applied to each node, one node at a time. If a private docker registry
is being used, the new container images will be pushed by Kismatic before starting to upgrade
nodes.

Nodes in the cluster are upgraded in the following order:

1. Etcd nodes
2. Master nodes
3. Worker nodes (regardless of specialization)
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return cmd.Help()
		},
	}

	cmd.PersistentFlags().StringVar(&opts.generatedAssetsDir, "generated-assets-dir", "generated", "path to the directory where assets generated during the installation process will be stored")
	cmd.PersistentFlags().BoolVar(&opts.verbose, "verbose", false, "enable verbose logging from the installation")
	cmd.PersistentFlags().StringVarP(&opts.outputFormat, "output", "o", "simple", "installation output format (options \"simple\"|\"raw\")")
	cmd.PersistentFlags().BoolVar(&opts.skipPreflight, "skip-preflight", false, "skip upgrade pre-flight checks")
	cmd.PersistentFlags().BoolVar(&opts.restartServices, "restart-services", false, "force restart cluster services (Use with care)")
	cmd.PersistentFlags().BoolVar(&opts.partialAllowed, "partial-ok", false, "allow the upgrade of ready nodes, and skip nodes that have been deemed unready for upgrade")
	cmd.PersistentFlags().BoolVar(&opts.dryRun, "dry-run", false, "simulate the upgrade, but don't actually upgrade the cluster")
	addPlanFileFlag(cmd.PersistentFlags(), &opts.planFile)

	// Subcommands
	cmd.AddCommand(NewCmdUpgradeOffline(out, &opts))
	cmd.AddCommand(NewCmdUpgradeOnline(out, &opts))
	return cmd
}

// NewCmdUpgradeOffline returns the command for running offline upgrades
func NewCmdUpgradeOffline(out io.Writer, opts *upgradeOpts) *cobra.Command {
	cmd := cobra.Command{
		Use:   "offline",
		Short: "Perform an offline upgrade of your Kubernetes cluster",
		Long: `Perform an offline upgrade of your Kubernetes cluster.

The offline upgrade is available for those clusters in which safety and availabilty are not a concern.
In this mode, the safety and availability checks will not be performed, nor will the nodes in the cluster
be drained of workloads.

Performing an offline upgrade could result in loss of critical data and reduced service
availability. For this reason, this method should not be used for clusters that are housing
production workloads.
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			return doUpgrade(out, opts)
		},
	}
	cmd.Flags().IntVar(&opts.maxParallelWorkers, "max-parallel-workers", 1, "the maximum number of worker nodes to be upgraded in parallel")
	return &cmd
}

// NewCmdUpgradeOnline returns the command for running online upgrades
func NewCmdUpgradeOnline(out io.Writer, opts *upgradeOpts) *cobra.Command {
	cmd := cobra.Command{
		Use:   "online",
		Short: "Perform an online upgrade of your Kubernetes cluster",
		Long: `Perform an online upgrade of your Kubernetes cluster.

During an online upgrade, Kismatic will run safety and availability checks (see table below) against the
existing cluster before performing the upgrade. If any unsafe condition is detected, a report will
be printed, and the upgrade will not proceed.

If the node under upgrade is a Kubernetes node, it is cordoned and drained of workloads
before any changes are applied.
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			opts.online = true
			return doUpgrade(out, opts)
		},
	}
	return &cmd
}

func doUpgrade(out io.Writer, opts *upgradeOpts) error {
	if opts.maxParallelWorkers < 1 {
		return fmt.Errorf("max-parallel-workers must be greater or equal to 1, got: %d", opts.maxParallelWorkers)
	}

	planFile := opts.planFile
	planner := install.FilePlanner{File: planFile}
	executorOpts := install.ExecutorOptions{
		GeneratedAssetsDirectory: opts.generatedAssetsDir,
		RestartServices:          opts.restartServices,
		OutputFormat:             opts.outputFormat,
		Verbose:                  opts.verbose,
		DryRun:                   opts.dryRun,
	}
	executor, err := install.NewExecutor(out, os.Stderr, executorOpts)
	if err != nil {
		return err
	}
	preflightExecOpts := executorOpts
	preflightExecOpts.DryRun = false // We always want to run preflight, even if doing a dry-run
	preflightExec, err := install.NewPreFlightExecutor(out, os.Stderr, preflightExecOpts)
	if err != nil {
		return err
	}
	util.PrintHeader(out, "Computing upgrade plan", '=')

	// Read plan file
	if !planner.PlanExists() {
		util.PrettyPrintErr(out, "Reading plan file")
		return fmt.Errorf("plan file %q does not exist", planFile)
	}
	util.PrettyPrintOk(out, "Reading plan file")
	plan, err := planner.Read()
	if err != nil {
		util.PrettyPrintErr(out, "Reading plan file")
		return fmt.Errorf("error reading plan file %q: %v", planFile, err)
	}

	// Validate SSH connectivity to nodes
	if ok, errs := install.ValidatePlanSSHConnections(plan); !ok {
		util.PrettyPrintErr(out, "Validate SSH connectivity to nodes")
		util.PrintValidationErrors(out, errs)
		return fmt.Errorf("SSH connectivity validation errors found")
	}
	util.PrettyPrintOk(out, "Validate SSH connectivity to nodes")

	// Figure out which nodes to upgrade
	cv, err := install.ListVersions(plan)
	if err != nil {
		return fmt.Errorf("error listing cluster versions: %v", err)
	}
	var toUpgrade []install.ListableNode
	var toSkip []install.ListableNode
	for _, n := range cv.Nodes {
		if install.IsOlderVersion(n.Version) {
			toUpgrade = append(toUpgrade, n)
		} else {
			toSkip = append(toSkip, n)
		}
	}

	// Print the nodes that will be skipped
	if len(toSkip) > 0 {
		util.PrintHeader(out, "Skipping nodes", '=')
		for _, n := range toSkip {
			util.PrettyPrintOk(out, "- %q is at the target version %q", n.Node.Host, n.Version)
		}
		fmt.Fprintln(out)
	}

	if plan.ConfigureDockerRegistry() && plan.Cluster.DisconnectedInstallation {
		util.PrintHeader(out, "Upgrade: Docker Registry", '=')
		if err = executor.UpgradeDockerRegistry(*plan); err != nil {
			return fmt.Errorf("Failed to upgrade docker registry: %v", err)
		}
	}

	// Print message if there's no work to do
	if len(toUpgrade) == 0 {
		fmt.Fprintln(out, "All nodes are at the target version. Skipping node upgrades.")
	} else {
		if err = upgradeNodes(out, *plan, *opts, toUpgrade, executor, preflightExec); err != nil {
			return err
		}
	}

	if opts.partialAllowed {
		util.PrintColor(out, util.Green, `

Partial upgrade complete.

Cluster level services are still left to upgrade. These can only be upgraded
when performing a full upgrade. When you are ready, you may use "kismatic upgrade"
without the "--partial-ok" flag to perform a full upgrade.

`)
		return nil
	}

	// Upgrade the cluster services
	util.PrintHeader(out, "Upgrade: Cluster Services", '=')
	if err := executor.UpgradeClusterServices(*plan); err != nil {
		return fmt.Errorf("Failed to upgrade cluster services: %v", err)
	}

	if err := executor.RunSmokeTest(plan); err != nil {
		return fmt.Errorf("Smoke test failed: %v", err)
	}

	if !opts.dryRun {
		fmt.Fprintln(out)
		util.PrintColor(out, util.Green, "Upgrade complete\n")
		fmt.Fprintln(out)
	}
	return nil
}

func upgradeNodes(out io.Writer, plan install.Plan, opts upgradeOpts, nodesNeedUpgrade []install.ListableNode, executor install.Executor, preflightExec install.PreFlightExecutor) error {
	// Run safety checks if doing an online upgrade
	unsafeNodes := []install.ListableNode{}
	if opts.online {
		util.PrintHeader(out, "Validate Online Upgrade", '=')
		// Use the first master node for running kubectl
		client, err := plan.GetSSHClient(plan.Master.Nodes[0].Host)
		if err != nil {
			return fmt.Errorf("error getting SSH client: %v", err)
		}
		kubeClient := data.RemoteKubectl{SSHClient: client}
		for _, node := range nodesNeedUpgrade {
			util.PrettyPrint(out, "%s %v", node.Node.Host, node.Roles)
			errs := install.DetectNodeUpgradeSafety(plan, node.Node, kubeClient)
			if len(errs) != 0 {
				util.PrintError(out)
				fmt.Fprintln(out)
				for _, err := range errs {
					fmt.Println("-", err.Error())
				}
				unsafeNodes = append(unsafeNodes, node)
			} else {
				util.PrintOkln(out)
			}
		}
		// If we found any unsafe nodes, and we are not doing a partial upgrade, exit.
		if len(unsafeNodes) > 0 && !opts.partialAllowed {
			return errors.New("Unable to perform an online upgrade due to the unsafe conditions detected.")
		}
		// Block the upgrade if partial is allowed but there is an etcd or master node
		// that cannot be upgraded
		if opts.partialAllowed {
			for _, n := range unsafeNodes {
				for _, r := range n.Roles {
					if r == "master" || r == "etcd" {
						return errors.New("Unable to perform an online upgrade due to the unsafe conditions detected.")
					}
				}
			}
		}
	}

	// Run upgrade preflight on the nodes that are to be upgraded
	unreadyNodes := []install.ListableNode{}
	if !opts.skipPreflight {
		for _, node := range nodesNeedUpgrade {
			util.PrintHeader(out, fmt.Sprintf("Preflight Checks: %s %s", node.Node.Host, node.Roles), '=')
			if err := preflightExec.RunUpgradePreFlightCheck(&plan, node); err != nil {
				// return fmt.Errorf("Upgrade preflight check failed: %v", err)
				unreadyNodes = append(unreadyNodes, node)
			}
		}
	}

	// Block upgrade if we found unready nodes, and we are not doing a partial upgrade
	if len(unreadyNodes) > 0 && !opts.partialAllowed {
		return errors.New("Errors found during preflight checks")
	}

	// Block the upgrade if partial is allowed but there is an etcd or master node
	// that cannot be upgraded
	if opts.partialAllowed {
		for _, n := range unreadyNodes {
			for _, r := range n.Roles {
				if r == "master" || r == "etcd" {
					return errors.New("Errors found during preflight checks")
				}
			}
		}
	}

	// Filter out the nodes that are unsafe/unready
	toUpgrade := []install.ListableNode{}
	for _, n := range nodesNeedUpgrade {
		upgrade := true
		for _, unsafe := range unsafeNodes {
			if unsafe.Node == n.Node {
				upgrade = false
			}
		}
		for _, unready := range unreadyNodes {
			if unready.Node == n.Node {
				upgrade = false
			}
		}
		if upgrade {
			toUpgrade = append(toUpgrade, n)
		}
	}

	// Get all etcd nodes
	// Nodes >=v1.3.0-alpha.0 do not need to be upgraded from Calico etcd v2
	// If any of the nodes fail during the Calico etcd v2 upgrade its safe to rerun on all nodes
	// Plays are idempotent and check etcd version, will skip if are at target
	etcdToUpgrade := make([]install.ListableNode, 0)
	for _, n := range install.NodesWithRoles(toUpgrade, "etcd") {
		// only transition nodes that are not 1.3.0...
		if install.IsLessThanVersion(n.Version, "v1.3.0-alpha.0") {
			etcdToUpgrade = append(etcdToUpgrade, n)
		}
	}
	if len(etcdToUpgrade) > 1 {
		// Run the upgrade on the nodes to Etcd v3.0.x
		if err := executor.UpgradeEtcd2Nodes(plan, etcdToUpgrade); err != nil {
			return fmt.Errorf("Failed to upgrade etcd2 nodes: %v", err)
		}
	}

	// Nodes >=v1.3.0 with k8s v1.6+ do not need to be migrated
	// Keep trying untill all etcd nodes have been migrated
	// This will rerun the migration on all nodes
	// Plays are idempotent and its safe to rerun `etcdctl migrate`
	var migrationNeeded bool
	for _, n := range install.NodesWithRoles(toUpgrade, "master") {
		if install.IsLessThanVersion(n.Version, "v1.3.0") {
			migrationNeeded = true
			break
		}
	}
	if migrationNeeded {
		util.PrintHeader(out, "Migrate: Kubernetes Etcd Cluster", '=')
		if err := executor.MigrateEtcdCluster(plan); err != nil {
			return fmt.Errorf("Failed to migrate kubernetes etcd cluster: %v", err)
		}
	}

	// Run the upgrade on the nodes that need it
	if err := executor.UpgradeNodes(plan, toUpgrade, opts.online, opts.maxParallelWorkers); err != nil {
		return fmt.Errorf("Failed to upgrade nodes: %v", err)
	}
	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright 2018 The original author or authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package core

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
	"bufio"
	"os"
	"strings"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	istioNamespace  = "istio-system"
	istioRelease    = "https://storage.googleapis.com/knative-releases/serving/previous/v20180809-6b01d8e/istio.yaml"
	servingRelease  = "https://storage.googleapis.com/knative-releases/serving/previous/v20180809-6b01d8e/release-no-mon.yaml"
	eventingRelease = "https://storage.googleapis.com/knative-releases/eventing/previous/v20180809-34ab480/release.yaml"
	stubBusRelease  = "https://storage.googleapis.com/knative-releases/eventing/previous/v20180809-34ab480/release-clusterbus-stub.yaml"
)

type SystemInstallOptions struct {
	NodePort bool
	Force bool
}

type SystemUninstallOptions struct {
	Istio bool
	Force bool
}

var (
	knativeNamespaces = []string{"knative-eventing", "knative-serving", "knative-build"}
	allNameSpaces     = append(knativeNamespaces, istioNamespace)
)

func (kc *kubectlClient) SystemInstall(options SystemInstallOptions) error {

	err := ensureNotTerminating(kc, allNameSpaces, "Please try again later.")
	if err != nil {
		return err
	}

	istioStatus, err := getNamespaceStatus(kc,istioNamespace)
	if istioStatus == "'NotFound'" {
		fmt.Print("Installing Istio Components\n")
		istioYaml, err := loadRelease(istioRelease)
		if err != nil {
			return err
		}
		if options.NodePort {
			istioYaml = bytes.Replace(istioYaml, []byte("LoadBalancer"), []byte("NodePort"), -1)
		}
		fmt.Printf("Applying resources defined in: %s\n", istioRelease)
		istioLog, err := kc.kubeCtl.ExecStdin([]string{"apply", "-f", "-"}, &istioYaml)
		if err != nil {
			fmt.Printf("%s\n", istioLog)
			return err
		}

		fmt.Print("Istio for riff installed\n\n")
	} else {
		if !options.Force {
			answer, err := confirm("Istio is already installed, do you want to install the Knative components for riff?")
			if err != nil {
				return err
			}
			if !answer {
				return nil
			}
		}
	}

	err = waitForIstioComponents(kc)
	if err != nil {
		return err
	}

	fmt.Print("Installing Knative Components\n")

	servingYaml, err := loadRelease(servingRelease)
	if err != nil {
		return err
	}
	if options.NodePort {
		servingYaml = bytes.Replace(servingYaml, []byte("LoadBalancer"), []byte("NodePort"), -1)
	}
	fmt.Printf("Applying resources defined in: %s\n", servingRelease)
	servingLog, err := kc.kubeCtl.ExecStdin([]string{"apply", "-f", "-"}, &servingYaml)
	if err != nil {
		fmt.Printf("%s\n", servingLog)
		return err
	}

	applyResources(kc, eventingRelease)

	applyResources(kc, stubBusRelease)

	fmt.Print("Knative for riff installed\n\n")
	return nil
}

func (kc *kubectlClient) SystemUninstall(options SystemUninstallOptions) error {

	err := ensureNotTerminating(kc, allNameSpaces, "This would indicate that the system was already uninstalled.")
	if err != nil {
		return err
	}
	knativeNsCount, err := checkNamespacesExists(kc, knativeNamespaces)
	istioNsCount, err := checkNamespacesExists(kc, []string{istioNamespace})
	if err != nil {
		return err
	}
	if knativeNsCount == 0 {
		fmt.Print("No Knative components for riff found\n")
	} else {
		if !options.Force {
			answer, err := confirm("Are you sure you want to uninstall the riff system?")
			if err != nil {
				return err
			}
			if !answer {
				return nil
			}
		}
		fmt.Print("Removing Knative for riff components\n")
		err = deleteCrds(kc, "knative.dev")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "knative-")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "build-controller-")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "eventing-controller-")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "clusterbus-controller-")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrole", "knative-")
		if err != nil {
			return err
		}
		err = deleteNamespaces(kc, knativeNamespaces)
		if err != nil {
			return err
		}
	}
	if istioNsCount == 0 {
		fmt.Print("No Istio components found\n")
	} else {
		if !options.Istio {
			if options.Force {
				return nil
			}
			answer, err := confirm("Do you also want to uninstall Istio components?")
			if err != nil {
				return err
			}
			if !answer {
				return nil
			}
		}
		fmt.Print("Removing Istio components\n")
		err = deleteCrds(kc, "istio.io")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "istio-")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrole", "istio-")
		if err != nil {
			return err
		}
		err = deleteNamespaces(kc, []string{istioNamespace})
		if err != nil {
			return err
		}
	}
	return nil
}

func resolveReleaseURLs(filename string) (url.URL, error) {
	u, err := url.Parse(filename)
	if err != nil {
		return url.URL{}, err
	}
	if u.Scheme == "http" || u.Scheme == "https" {
		return *u, nil
	}
	return *u, fmt.Errorf("filename must be file, http or https, got %s", u.Scheme)
}

func loadRelease(release string) ([]byte, error) {
	releaseUrl, err := resolveReleaseURLs(release)
	if err != nil {
		return nil, err
	}
	resp, err := http.Get(releaseUrl.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, nil
}

func waitForIstioComponents(kc *kubectlClient) error {
	fmt.Print("Waiting for the Istio components to start ")
	for i := 0; i < 36; i++ {
		fmt.Print(".")
		pods := kc.kubeClient.CoreV1().Pods(istioNamespace)
		podList, err := pods.List(metav1.ListOptions{})
		if err != nil {
			return err
		}
		waitLonger := false
		for _, pod := range podList.Items {
			if !strings.HasPrefix(pod.Name, "istio-") {
				continue
			}
			if pod.Status.Phase != "Running" && pod.Status.Phase != "Succeeded" {
				waitLonger = true
				break
			} else {
				if pod.Status.Phase == "Running" {
					containers := pod.Status.ContainerStatuses
					for _, cont := range containers {
						if !cont.Ready {
							waitLonger = true
							break
						}
					}
				}
			}
		}
		if !waitLonger {
			fmt.Print(" all components are 'Running'\n\n")
			return nil
		}
		time.Sleep(10 * time.Second) // wait for them to start
	}
	return errors.New("the Istio components did not start in time")
}

func applyResources(kc *kubectlClient, release string) error {
	releaseUrl, err := resolveReleaseURLs(release)
	if err != nil {
		return err
	}
	fmt.Printf("Applying resources defined in: %s\n", releaseUrl.String())
	releaseLog, err := kc.kubeCtl.Exec([]string{"apply", "-f", releaseUrl.String()})
	if err != nil {
		fmt.Printf("%s", releaseLog)
	}
	return nil
}

func deleteNamespaces(kc *kubectlClient, namespaces []string) error {
	for _, namespace := range namespaces {
		fmt.Printf("Deleting resources defined in: %s\n", namespace)
		deleteLog, err := kc.kubeCtl.Exec([]string{"delete", "namespace", namespace})
		if err != nil {
			fmt.Printf("%s", deleteLog)
		}
	}
	return nil
}

func deleteClusterResources(kc *kubectlClient, resourceType string, prefix string) error {
	fmt.Printf("Deleting %ss prefixed with %s\n", resourceType, prefix)
	resourceList, err := kc.kubeCtl.Exec([]string{"get", resourceType, "-ocustom-columns=name:metadata.name"})
	if err != nil {
		return err
	}
	resource := strings.Split(string(resourceList), "\n")
	var resourcesToDelete []string
	for _, resource := range resource {
		if strings.HasPrefix(resource, prefix) {
			resourcesToDelete = append(resourcesToDelete, resource)
		}
	}
	if len(resourcesToDelete) > 0 {
		resourceLog, err := kc.kubeCtl.Exec(append([]string{"delete", resourceType}, resourcesToDelete...))
		if err != nil {
			fmt.Printf("%s", resourceLog)
			return err
		}
	}
	return nil
}

func deleteCrds(kc *kubectlClient, suffix string) error {
	fmt.Printf("Deleting CRDs for %s\n", suffix)
	crdList, err := kc.kubeCtl.Exec([]string{"get", "customresourcedefinitions", "-ocustom-columns=name:metadata.name"})
	if err != nil {
		return err
	}
	crds := strings.Split(string(crdList), "\n")
	var crdsToDelete []string
	for _, crd := range crds {
		if strings.HasSuffix(crd, suffix) {
			crdsToDelete = append(crdsToDelete, crd)
		}
	}
	if len(crdsToDelete) > 0 {
		crdLog, err := kc.kubeCtl.Exec(append([]string{"delete", "customresourcedefinition"}, crdsToDelete...))
		if err != nil {
			fmt.Printf("%s", crdLog)
			return err
		}
	}
	return nil
}

func checkNamespacesExists (kc *kubectlClient, names []string) (int, error) {
	count := 0
	for _, name := range names {
		status, err := getNamespaceStatus(kc, name)
		if err != nil {
			return count, err
		}
		if status != "'NotFound'" {
			count =+ 1
		}
	}
	return count, nil
}

func ensureNotTerminating (kc *kubectlClient, names []string, message string) error {
	for _, name := range names {
		status, err := getNamespaceStatus(kc, name)
		if err != nil {
			return err
		}
		if status == "'Terminating'" {
			return errors.New(fmt.Sprintf("The %s namespace is currently 'Terminating'. %s", name, message))
		}
	}
	return nil
}

func getNamespaceStatus(kc *kubectlClient, name string) (string, error) {
	nsLog, err := kc.kubeCtl.Exec([]string{"get", "namespace", name, "-o", "jsonpath='{.status.phase}'"})
	if err != nil {
		if strings.Contains(nsLog, "NotFound") {
			return "'NotFound'", nil
		}
		return "", err
	}
	return nsLog, nil
}

func confirm(s string) (bool, error) {
	reader := bufio.NewReader(os.Stdin)
	fmt.Printf("%s [y/n]: ", s)
	res, err := reader.ReadString('\n')
	if err != nil {
		return false, err
	}
	if len(res) < 2 {
		return false, nil
	}
	answer := strings.ToLower(strings.TrimSpace(res))[0] == 'y'
	return answer, nil
}
<commit_msg>Use a riff Istio yaml file for installs (#661)<commit_after>/*
 * Copyright 2018 The original author or authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package core

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
	"bufio"
	"os"
	"strings"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	istioNamespace  = "istio-system"
	istioRelease    = "https://storage.googleapis.com/riff-releases/istio/istio-1.0.0-riff.yaml"
	servingRelease  = "https://storage.googleapis.com/knative-releases/serving/previous/v20180809-6b01d8e/release-no-mon.yaml"
	eventingRelease = "https://storage.googleapis.com/knative-releases/eventing/previous/v20180809-34ab480/release.yaml"
	stubBusRelease  = "https://storage.googleapis.com/knative-releases/eventing/previous/v20180809-34ab480/release-clusterbus-stub.yaml"
)

type SystemInstallOptions struct {
	NodePort bool
	Force bool
}

type SystemUninstallOptions struct {
	Istio bool
	Force bool
}

var (
	knativeNamespaces = []string{"knative-eventing", "knative-serving", "knative-build"}
	allNameSpaces     = append(knativeNamespaces, istioNamespace)
)

func (kc *kubectlClient) SystemInstall(options SystemInstallOptions) error {

	err := ensureNotTerminating(kc, allNameSpaces, "Please try again later.")
	if err != nil {
		return err
	}

	istioStatus, err := getNamespaceStatus(kc,istioNamespace)
	if istioStatus == "'NotFound'" {
		fmt.Print("Installing Istio Components\n")
		istioYaml, err := loadRelease(istioRelease)
		if err != nil {
			return err
		}
		if options.NodePort {
			istioYaml = bytes.Replace(istioYaml, []byte("LoadBalancer"), []byte("NodePort"), -1)
		}
		fmt.Printf("Applying resources defined in: %s\n", istioRelease)
		istioLog, err := kc.kubeCtl.ExecStdin([]string{"apply", "-f", "-"}, &istioYaml)
		if err != nil {
			fmt.Printf("%s\n", istioLog)
			return err
		}

		fmt.Print("Istio for riff installed\n\n")
	} else {
		if !options.Force {
			answer, err := confirm("Istio is already installed, do you want to install the Knative components for riff?")
			if err != nil {
				return err
			}
			if !answer {
				return nil
			}
		}
	}

	err = waitForIstioComponents(kc)
	if err != nil {
		return err
	}

	fmt.Print("Installing Knative Components\n")

	servingYaml, err := loadRelease(servingRelease)
	if err != nil {
		return err
	}
	if options.NodePort {
		servingYaml = bytes.Replace(servingYaml, []byte("LoadBalancer"), []byte("NodePort"), -1)
	}
	fmt.Printf("Applying resources defined in: %s\n", servingRelease)
	servingLog, err := kc.kubeCtl.ExecStdin([]string{"apply", "-f", "-"}, &servingYaml)
	if err != nil {
		fmt.Printf("%s\n", servingLog)
		return err
	}

	applyResources(kc, eventingRelease)

	applyResources(kc, stubBusRelease)

	fmt.Print("Knative for riff installed\n\n")
	return nil
}

func (kc *kubectlClient) SystemUninstall(options SystemUninstallOptions) error {

	err := ensureNotTerminating(kc, allNameSpaces, "This would indicate that the system was already uninstalled.")
	if err != nil {
		return err
	}
	knativeNsCount, err := checkNamespacesExists(kc, knativeNamespaces)
	istioNsCount, err := checkNamespacesExists(kc, []string{istioNamespace})
	if err != nil {
		return err
	}
	if knativeNsCount == 0 {
		fmt.Print("No Knative components for riff found\n")
	} else {
		if !options.Force {
			answer, err := confirm("Are you sure you want to uninstall the riff system?")
			if err != nil {
				return err
			}
			if !answer {
				return nil
			}
		}
		fmt.Print("Removing Knative for riff components\n")
		err = deleteCrds(kc, "knative.dev")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "knative-")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "build-controller-")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "eventing-controller-")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "clusterbus-controller-")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrole", "knative-")
		if err != nil {
			return err
		}
		err = deleteNamespaces(kc, knativeNamespaces)
		if err != nil {
			return err
		}
	}
	if istioNsCount == 0 {
		fmt.Print("No Istio components found\n")
	} else {
		if !options.Istio {
			if options.Force {
				return nil
			}
			answer, err := confirm("Do you also want to uninstall Istio components?")
			if err != nil {
				return err
			}
			if !answer {
				return nil
			}
		}
		fmt.Print("Removing Istio components\n")
		err = deleteCrds(kc, "istio.io")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrolebinding", "istio-")
		if err != nil {
			return err
		}
		err = deleteClusterResources(kc, "clusterrole", "istio-")
		if err != nil {
			return err
		}
		err = deleteNamespaces(kc, []string{istioNamespace})
		if err != nil {
			return err
		}
	}
	return nil
}

func resolveReleaseURLs(filename string) (url.URL, error) {
	u, err := url.Parse(filename)
	if err != nil {
		return url.URL{}, err
	}
	if u.Scheme == "http" || u.Scheme == "https" {
		return *u, nil
	}
	return *u, fmt.Errorf("filename must be file, http or https, got %s", u.Scheme)
}

func loadRelease(release string) ([]byte, error) {
	releaseUrl, err := resolveReleaseURLs(release)
	if err != nil {
		return nil, err
	}
	resp, err := http.Get(releaseUrl.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, nil
}

func waitForIstioComponents(kc *kubectlClient) error {
	fmt.Print("Waiting for the Istio components to start ")
	for i := 0; i < 36; i++ {
		fmt.Print(".")
		pods := kc.kubeClient.CoreV1().Pods(istioNamespace)
		podList, err := pods.List(metav1.ListOptions{})
		if err != nil {
			return err
		}
		waitLonger := false
		for _, pod := range podList.Items {
			if !strings.HasPrefix(pod.Name, "istio-") {
				continue
			}
			if pod.Status.Phase != "Running" && pod.Status.Phase != "Succeeded" {
				waitLonger = true
				break
			} else {
				if pod.Status.Phase == "Running" {
					containers := pod.Status.ContainerStatuses
					for _, cont := range containers {
						if !cont.Ready {
							waitLonger = true
							break
						}
					}
				}
			}
		}
		if !waitLonger {
			fmt.Print(" all components are 'Running'\n\n")
			return nil
		}
		time.Sleep(10 * time.Second) // wait for them to start
	}
	return errors.New("the Istio components did not start in time")
}

func applyResources(kc *kubectlClient, release string) error {
	releaseUrl, err := resolveReleaseURLs(release)
	if err != nil {
		return err
	}
	fmt.Printf("Applying resources defined in: %s\n", releaseUrl.String())
	releaseLog, err := kc.kubeCtl.Exec([]string{"apply", "-f", releaseUrl.String()})
	if err != nil {
		fmt.Printf("%s", releaseLog)
	}
	return nil
}

func deleteNamespaces(kc *kubectlClient, namespaces []string) error {
	for _, namespace := range namespaces {
		fmt.Printf("Deleting resources defined in: %s\n", namespace)
		deleteLog, err := kc.kubeCtl.Exec([]string{"delete", "namespace", namespace})
		if err != nil {
			fmt.Printf("%s", deleteLog)
		}
	}
	return nil
}

func deleteClusterResources(kc *kubectlClient, resourceType string, prefix string) error {
	fmt.Printf("Deleting %ss prefixed with %s\n", resourceType, prefix)
	resourceList, err := kc.kubeCtl.Exec([]string{"get", resourceType, "-ocustom-columns=name:metadata.name"})
	if err != nil {
		return err
	}
	resource := strings.Split(string(resourceList), "\n")
	var resourcesToDelete []string
	for _, resource := range resource {
		if strings.HasPrefix(resource, prefix) {
			resourcesToDelete = append(resourcesToDelete, resource)
		}
	}
	if len(resourcesToDelete) > 0 {
		resourceLog, err := kc.kubeCtl.Exec(append([]string{"delete", resourceType}, resourcesToDelete...))
		if err != nil {
			fmt.Printf("%s", resourceLog)
			return err
		}
	}
	return nil
}

func deleteCrds(kc *kubectlClient, suffix string) error {
	fmt.Printf("Deleting CRDs for %s\n", suffix)
	crdList, err := kc.kubeCtl.Exec([]string{"get", "customresourcedefinitions", "-ocustom-columns=name:metadata.name"})
	if err != nil {
		return err
	}
	crds := strings.Split(string(crdList), "\n")
	var crdsToDelete []string
	for _, crd := range crds {
		if strings.HasSuffix(crd, suffix) {
			crdsToDelete = append(crdsToDelete, crd)
		}
	}
	if len(crdsToDelete) > 0 {
		crdLog, err := kc.kubeCtl.Exec(append([]string{"delete", "customresourcedefinition"}, crdsToDelete...))
		if err != nil {
			fmt.Printf("%s", crdLog)
			return err
		}
	}
	return nil
}

func checkNamespacesExists (kc *kubectlClient, names []string) (int, error) {
	count := 0
	for _, name := range names {
		status, err := getNamespaceStatus(kc, name)
		if err != nil {
			return count, err
		}
		if status != "'NotFound'" {
			count =+ 1
		}
	}
	return count, nil
}

func ensureNotTerminating (kc *kubectlClient, names []string, message string) error {
	for _, name := range names {
		status, err := getNamespaceStatus(kc, name)
		if err != nil {
			return err
		}
		if status == "'Terminating'" {
			return errors.New(fmt.Sprintf("The %s namespace is currently 'Terminating'. %s", name, message))
		}
	}
	return nil
}

func getNamespaceStatus(kc *kubectlClient, name string) (string, error) {
	nsLog, err := kc.kubeCtl.Exec([]string{"get", "namespace", name, "-o", "jsonpath='{.status.phase}'"})
	if err != nil {
		if strings.Contains(nsLog, "NotFound") {
			return "'NotFound'", nil
		}
		return "", err
	}
	return nsLog, nil
}

func confirm(s string) (bool, error) {
	reader := bufio.NewReader(os.Stdin)
	fmt.Printf("%s [y/n]: ", s)
	res, err := reader.ReadString('\n')
	if err != nil {
		return false, err
	}
	if len(res) < 2 {
		return false, nil
	}
	answer := strings.ToLower(strings.TrimSpace(res))[0] == 'y'
	return answer, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// I heard you like shell...

//go:generate bash -c "echo -e '// AUTOGENERATED FROM executor/common_linux.h\npackage csource\nvar commonHeaderLinux = `' > linux_common.go; cat ../../executor/common_linux.h | sed -e '/#include \"common.h\"/ {' -e 'r ../../executor/common.h' -e 'd' -e '}' - | sed -e '/#include \"common_kvm_amd64.h\"/ {' -e 'r ../../executor/common_kvm_amd64.h' -e 'd' -e '}' - | sed -e '/#include \"common_kvm_arm64.h\"/ {' -e 'r ../../executor/common_kvm_arm64.h' -e 'd' -e '}' - | sed -e '/#include \"kvm.h\"/ {' -e 'r ../../executor/kvm.h' -e 'd' -e '}' - | sed -e '/#include \"kvm.S.h\"/ {' -e 'r ../../executor/kvm.S.h' -e 'd' -e '}' - | egrep -v '^[   ]*//' | sed '/^[ 	]*\\/\\/.*/d' | sed 's#[ 	]*//.*##g' >> linux_common.go; echo '`' >> linux_common.go"
//go:generate go fmt linux_common.go

//go:generate bash -c "echo -e '// AUTOGENERATED FROM executor/common_akaros.h\npackage csource\nvar commonHeaderAkaros = `' > akaros_common.go; cat ../../executor/common_akaros.h | sed -e '/#include \"common.h\"/ {' -e 'r ../../executor/common.h' -e 'd' -e '}' - | egrep -v '^[   ]*//' | sed '/^[ 	]*\\/\\/.*/d' | sed 's#[ 	]*//.*##g' >> akaros_common.go; echo '`' >> akaros_common.go"
//go:generate go fmt akaros_common.go

//go:generate bash -c "echo -e '// AUTOGENERATED FROM executor/common_bsd.h\npackage csource\nvar commonHeaderFreebsd = `' > freebsd_common.go; cat ../../executor/common_bsd.h | sed -e '/#include \"common.h\"/ {' -e 'r ../../executor/common.h' -e 'd' -e '}' - | egrep -v '^[   ]*//' | sed '/^[ 	]*\\/\\/.*/d' | sed 's#[ 	]*//.*##g' >> freebsd_common.go; echo '`' >> freebsd_common.go"
//go:generate go fmt freebsd_common.go

//go:generate bash -c "echo -e '// AUTOGENERATED FROM executor/common_bsd.h\npackage csource\nvar commonHeaderNetbsd = `' > netbsd_common.go; cat ../../executor/common_bsd.h | sed -e '/#include \"common.h\"/ {' -e 'r ../../executor/common.h' -e 'd' -e '}' - | egrep -v '^[   ]*//' | sed '/^[ 	]*\\/\\/.*/d' | sed 's#[ 	]*//.*##g' >> netbsd_common.go; echo '`' >> netbsd_common.go"
//go:generate go fmt netbsd_common.go

package csource
<commit_msg>csource: Fix sed(1) invocation<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// I heard you like shell...

//go:generate bash -c "echo -e '// AUTOGENERATED FROM executor/common_linux.h\npackage csource\nvar commonHeaderLinux = `' > linux_common.go; cat ../../executor/common_linux.h | sed -e '/#include \"common.h\"/ {' -e 'r ../../executor/common.h' -e 'd' -e '}' | sed -e '/#include \"common_kvm_amd64.h\"/ {' -e 'r ../../executor/common_kvm_amd64.h' -e 'd' -e '}' | sed -e '/#include \"common_kvm_arm64.h\"/ {' -e 'r ../../executor/common_kvm_arm64.h' -e 'd' -e '}' | sed -e '/#include \"kvm.h\"/ {' -e 'r ../../executor/kvm.h' -e 'd' -e '}' | sed -e '/#include \"kvm.S.h\"/ {' -e 'r ../../executor/kvm.S.h' -e 'd' -e '}' | egrep -v '^[   ]*//' | sed '/^[ 	]*\\/\\/.*/d' | sed 's#[ 	]*//.*##g' >> linux_common.go; echo '`' >> linux_common.go"
//go:generate go fmt linux_common.go

//go:generate bash -c "echo -e '// AUTOGENERATED FROM executor/common_akaros.h\npackage csource\nvar commonHeaderAkaros = `' > akaros_common.go; cat ../../executor/common_akaros.h | sed -e '/#include \"common.h\"/ {' -e 'r ../../executor/common.h' -e 'd' -e '}' | egrep -v '^[   ]*//' | sed '/^[ 	]*\\/\\/.*/d' | sed 's#[ 	]*//.*##g' >> akaros_common.go; echo '`' >> akaros_common.go"
//go:generate go fmt akaros_common.go

//go:generate bash -c "echo -e '// AUTOGENERATED FROM executor/common_bsd.h\npackage csource\nvar commonHeaderFreebsd = `' > freebsd_common.go; cat ../../executor/common_bsd.h | sed -e '/#include \"common.h\"/ {' -e 'r ../../executor/common.h' -e 'd' -e '}' | egrep -v '^[   ]*//' | sed '/^[ 	]*\\/\\/.*/d' | sed 's#[ 	]*//.*##g' >> freebsd_common.go; echo '`' >> freebsd_common.go"
//go:generate go fmt freebsd_common.go

//go:generate bash -c "echo -e '// AUTOGENERATED FROM executor/common_bsd.h\npackage csource\nvar commonHeaderNetbsd = `' > netbsd_common.go; cat ../../executor/common_bsd.h | sed -e '/#include \"common.h\"/ {' -e 'r ../../executor/common.h' -e 'd' -e '}' | egrep -v '^[   ]*//' | sed '/^[ 	]*\\/\\/.*/d' | sed 's#[ 	]*//.*##g' >> netbsd_common.go; echo '`' >> netbsd_common.go"
//go:generate go fmt netbsd_common.go

package csource
<|endoftext|>


<commit_before>package daemon

import (
	"github.com/pkg/errors"

	"github.com/havoc-io/mutagen/pkg/filesystem/locking"
)

// Lock represents a daemon lock instance.
type Lock struct {
	// locker is the daemon file lock, uniquely held by a single daemon
	// instance. Because the locking semantics vary by platform, hosting
	// processes should only attempt to create a single daemon lock at a time.
	locker *locking.Locker
}

// AcquireLock attempts to acquire the daemon lock. It is the only way to
// acquire a daemon Lock instance.
func AcquireLock() (*Lock, error) {
	// Compute the lock path.
	lockPath, err := subpath(lockName)
	if err != nil {
		return nil, errors.Wrap(err, "unable to compute daemon lock path")
	}

	// Create the daemon locker and attempt to acquire the lock.
	locker, err := locking.NewLocker(lockPath, 0600)
	if err != nil {
		return nil, errors.Wrap(err, "unable to create daemon locker")
	} else if err = locker.Lock(false); err != nil {
		locker.Close()
		return nil, err
	}

	// Create the lock.
	return &Lock{
		locker: locker,
	}, nil
}

// Release releases the daemon lock.
func (l *Lock) Release() error {
	// Release the lock.
	if err := l.locker.Unlock(); err != nil {
		l.locker.Close()
		return err
	}

	// Close the locker.
	return errors.Wrap(l.locker.Close(), "unable to close locker")
}
<commit_msg>Cleaned up comments for daemon.Lock.<commit_after>package daemon

import (
	"github.com/pkg/errors"

	"github.com/havoc-io/mutagen/pkg/filesystem/locking"
)

// Lock represents the global daemon lock. It is held by a single daemon
// instance at a time.
type Lock struct {
	// locker is the underlying file locker.
	locker *locking.Locker
}

// AcquireLock attempts to acquire the global daemon lock.
func AcquireLock() (*Lock, error) {
	// Compute the lock path.
	lockPath, err := subpath(lockName)
	if err != nil {
		return nil, errors.Wrap(err, "unable to compute daemon lock path")
	}

	// Create the daemon locker and attempt to acquire the lock.
	locker, err := locking.NewLocker(lockPath, 0600)
	if err != nil {
		return nil, errors.Wrap(err, "unable to create daemon file locker")
	} else if err = locker.Lock(false); err != nil {
		locker.Close()
		return nil, err
	}

	// Create the lock.
	return &Lock{
		locker: locker,
	}, nil
}

// Release releases the daemon lock.
func (l *Lock) Release() error {
	// Release the lock.
	if err := l.locker.Unlock(); err != nil {
		l.locker.Close()
		return err
	}

	// Close the locker.
	return errors.Wrap(l.locker.Close(), "unable to close locker")
}
<|endoftext|>