<commit_before>package cleanup

import (
	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

// SpecCleanup cleans the fields of the object which should be ignored.
// Each function is responsible for handling different versions of objects itself.
type SpecCleanup func(spec, actual *unstructured.Unstructured) (updatedSpec *unstructured.Unstructured, err error)

type SpecCleaner struct {
	KnownTypes map[schema.GroupKind]SpecCleanup
}

func New(kts ...map[schema.GroupKind]SpecCleanup) *SpecCleaner {
	kt := make(map[schema.GroupKind]SpecCleanup)
	for _, knownTypes := range kts {
		for knownGK, f := range knownTypes {
			if kt[knownGK] != nil {
				panic(errors.Errorf("GK specified more than once: %s", knownGK))
			}
			kt[knownGK] = f
		}
	}
	return &SpecCleaner{
		KnownTypes: kt,
	}
}

func (oc *SpecCleaner) Cleanup(spec, actual *unstructured.Unstructured) (updatedSpec *unstructured.Unstructured, err error) {
	gvk := spec.GroupVersionKind()
	gk := gvk.GroupKind()

	if gk.Kind == "" || gvk.Version == "" { // Group can be empty e.g. built-in objects like ConfigMap
		return nil, errors.Errorf("object has empty kind/version: %s", gvk)
	}

	if objCleanup, ok := oc.KnownTypes[gk]; ok {
		return objCleanup(spec, actual)
	}

	return spec, nil
}
<commit_msg>Remove extra unnecessary version check<commit_after>package cleanup

import (
	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

// SpecCleanup cleans the fields of the object which should be ignored.
// Each function is responsible for handling different versions of objects itself.
type SpecCleanup func(spec, actual *unstructured.Unstructured) (updatedSpec *unstructured.Unstructured, err error)

type SpecCleaner struct {
	KnownTypes map[schema.GroupKind]SpecCleanup
}

func New(kts ...map[schema.GroupKind]SpecCleanup) *SpecCleaner {
	kt := make(map[schema.GroupKind]SpecCleanup)
	for _, knownTypes := range kts {
		for knownGK, f := range knownTypes {
			if kt[knownGK] != nil {
				panic(errors.Errorf("GK specified more than once: %s", knownGK))
			}
			kt[knownGK] = f
		}
	}
	return &SpecCleaner{
		KnownTypes: kt,
	}
}

func (oc *SpecCleaner) Cleanup(spec, actual *unstructured.Unstructured) (updatedSpec *unstructured.Unstructured, err error) {
	gk := spec.GroupVersionKind().GroupKind()

	if objCleanup, ok := oc.KnownTypes[gk]; ok {
		return objCleanup(spec, actual)
	}

	return spec, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Scenarios:
// a) Stream - just copy into a big buffer as at present (... but if there is a MaxBof??)
// b) File
//    b i)   Satisifed with small read beginning
//    b ii)  Small enough for full read
//    b iii) Mmap
//    b iv) Too big for MMap - small buffers and expose ReaderAt

// Package siegreader implements multiple independent Readers (and ReverseReaders) from a single Buffer.
//
// Example:
//   buf := siegreader.New()
//	 err := buf.SetSource(io.Reader)
//   if err != nil {
//     log.Fatal(err)
//   }
//   rdr := buf.Reader()
//	 second_rdr := buf.Reader()
//   reverse_rdr, err := buf.ReverseReader()
//   if err != nil {
//	   log.Fatal(err)
//   }
//   i, _ := rdr.Read(slc)
//   i2, _ := second_rdr.Read(slc2)
//   i3, _ := reverse_rdr.Read(slc3)
package siegreader

import (
	"errors"
	"io"
	"log"
	"os"
	"sync"
)

type Buf interface {
	SetQuit(chan struct{})
	SetSource(io.Reader) error
	Size() int64
	SizeNow() int64
	Slice(off int64, length int, whence bool) ([]byte, error)
	canSeek(off int64, whence bool) (bool, error)
}

var (
	ErrQuit      = errors.New("siegreader: quit chan closed while awaiting EOF")
	ErrNilBuffer = errors.New("siegreader: attempt to SetSource on a nil buffer")
)

var (
	readSz      = 4096
	initialRead = readSz * 2
)

type protected struct {
	sync.Mutex
	val     int
	eofRead bool
}

// Buffer wraps an io.Reader, buffering its contents in byte slices that will keep growing until IO.EOF.
// It supports multiple concurrent Readers, including Readers reading from the end of the stream (ReverseReaders)
type Buffer struct {
	quit      chan struct{} // allows quittting - otherwise will block forever while awaiting EOF
	drain     bool          // Does this Buffer have a regular reader that we can expect will read to the EOF (and hence allow ReverseReaders to wait)
	src       io.Reader
	buf, eof  []byte
	eofc      chan struct{} // signals if EOF bytes are available. When EOF bytes are available, this chan is closed
	completec chan struct{} // signals when the file has been completely read, allows EOF scanning beyond the small buffer
	complete  bool          // marks that the file has been completely read
	sz        int64
	w         protected // index of latest write
}

// New instatatiates a new Buffer with a buf size of 4096*3, and an end-of-file buf size of 4096
func New() *Buffer {
	b := new(Buffer)
	b.buf, b.eof = make([]byte, initialRead), make([]byte, readSz)
	return b
}

func (b *Buffer) reset() {
	b.eofc, b.completec = make(chan struct{}), make(chan struct{})
	b.complete = false
	b.sz = 0
	b.w.Lock()
	b.w.val = 0
	b.w.eofRead = false
	b.w.Unlock()
}

// SetSource sets the buffer's source.
// Can be any io.Reader. If it is an os.File, will load EOF buffer early. Otherwise waits for a complete read.
// The source can be reset to recycle an existing Buffer.
// Siegreader blocks on EOF reads or Size() calls when the reader isn't a file or the stream isn't completely read. The quit channel overrides this block.
func (b *Buffer) SetSource(r io.Reader) error {
	if b == nil {
		return errors.New("Siegreader: attempt to SetSource on a nil buffer")
	}
	b.reset()
	b.src = r
	file, ok := r.(*os.File)
	if ok {
		info, err := file.Stat()
		if err != nil {
			return err
		}
		b.sz = info.Size()
		if b.sz > int64(initialRead) {
			b.eof = b.eof[:cap(b.eof)]
		} else {
			b.eof = b.eof[:0]
		}
	} else {
		b.sz = 0
		b.eof = b.eof[:0]
	}
	_, err := b.fill() // initial fill
	return err
}

func (b *Buffer) SetQuit(q chan struct{}) {
	b.quit = q
}

// Size returns the buffer's size, which is available immediately for files. Must wait for full read for streams.
func (b *Buffer) Size() int {
	if b.sz > 0 {
		return int(b.sz)
	}
	select {
	case <-b.eofc:
		return int(b.sz)
	case <-b.quit:
		return 0
	}
}

// non-blocking Size(), for use with zip reader
func (b *Buffer) SizeNow() int64 {
	if b.sz > 0 {
		return b.sz
	}
	b.w.Lock()
	defer b.w.Unlock()
	var err error
	for _, err = b.fill(); err == nil; _, err = b.fill() {
	}
	if err != io.EOF {
		log.Printf("SIEGREADER WARNING: FAILED TO READ FULL STREAM, ERROR MESSAGE %v", err)
		return 0
	}
	return b.sz
}

func (b *Buffer) grow() {
	// Rules for growing:
	// - if we need to grow, we have passed the initial read and can assume we will need whole file so, if we have a sz grow to it straight away
	// - otherwise, double capacity each time
	var buf []byte
	if b.sz > 0 {
		buf = make([]byte, int(b.sz))
	} else {
		buf = make([]byte, cap(b.buf)*2)
	}
	copy(buf, b.buf[:b.w.val]) // don't care about unlocking as grow() is only called by fill()
	b.buf = buf
}

// Rules for filling:
// - if we have a sz greater than 0, if there is stuff in the eof buffer, and if we are less than readSz from the end, copy across from the eof buffer
// - read readsz * 2 at a time
func (b *Buffer) fill() (int, error) {
	// if we've run out of room, grow the buffer
	if len(b.buf)-readSz < b.w.val {
		b.grow()
	}
	// if we have an eof buffer, and we are near the end of the file, avoid an extra read by copying straight into the main buffer
	if len(b.eof) > 0 && b.w.eofRead && b.w.val+readSz >= int(b.sz) {
		close(b.completec)
		b.complete = true
		lr := int(b.sz) - b.w.val
		b.w.val += copy(b.buf[b.w.val:b.w.val+lr], b.eof[readSz-lr:])
		return b.w.val, io.EOF
	}
	// otherwise, let's read
	e := b.w.val + readSz
	if e > len(b.buf) {
		e = len(b.buf)
	}
	i, err := b.src.Read(b.buf[b.w.val:e])
	if i < readSz {
		err = io.EOF // Readers can give EOF or nil here
	}
	if err != nil {
		close(b.completec)
		b.complete = true
		if err == io.EOF {
			b.w.val += i
			// if we haven't got an eof buf already
			if len(b.eof) < readSz {
				b.sz = int64(b.w.val)
				close(b.eofc)
			}
		}
		return b.w.val, err
	}
	b.w.val += i
	return b.w.val, nil
}

func (b *Buffer) fillEof() error {
	// return nil for a non-file or small file reader
	if len(b.eof) < readSz {
		return nil
	}
	b.w.Lock()
	defer b.w.Unlock()
	if b.w.eofRead {
		return nil // another reverse reader has already filled the buffer
	}
	rs := b.src.(io.ReadSeeker)
	_, err := rs.Seek(0-int64(readSz), 2)
	if err != nil {
		return err
	}
	_, err = rs.Read(b.eof)
	if err != nil {
		return err
	}
	_, err = rs.Seek(int64(b.w.val), 0)
	if err != nil {
		return err
	}
	close(b.eofc)
	b.w.eofRead = true
	return nil
}

// Return a slice from the buffer that begins at offset s and has length l
func (b *Buffer) Slice(s, l int) ([]byte, error) {
	b.w.Lock()
	defer b.w.Unlock()
	var err error
	var bound int
	if s+l > b.w.val && !b.complete {
		for bound, err = b.fill(); s+l > bound && err == nil; bound, err = b.fill() {
		}
	}
	if err == nil && !b.complete {
		return b.buf[s : s+l], nil
	}
	if err == io.EOF || b.complete {
		if s+l > b.w.val {
			if s > b.w.val {
				return nil, io.EOF
			}
			// in the case of an empty file
			if b.Size() == 0 {
				return nil, io.EOF
			}
			return b.buf[s:b.w.val], io.EOF
		} else {
			return b.buf[s : s+l], nil
		}
	}
	return nil, err
}

// Return a slice from the end of the buffer that begins at offset s and has length l.
// This will block until the slice is available (which may be until the full stream is read).
func (b *Buffer) EofSlice(s, l int) ([]byte, error) {
	// block until the EOF is available or we quit
	select {
	case <-b.quit:
		return nil, ErrQuit
	case <-b.eofc:
	}
	var buf []byte
	if len(b.eof) > 0 && s+l <= len(b.eof) {
		buf = b.eof
	} else {
		select {
		case <-b.quit:
			return nil, ErrQuit
		case <-b.completec:
		}
		buf = b.buf[:int(b.sz)]
		if s+l == len(buf) {
			return buf[:len(buf)-s], io.EOF
		}
	}
	if s+l > len(buf) {
		if s > len(buf) {
			return nil, io.EOF
		}
		return buf[:len(buf)-s], io.EOF
	}
	return buf[len(buf)-(s+l) : len(buf)-s], nil
}

// SafeSlice calls Slice or EofSlice (which one depends on the rev argument: true for EofSlice)
func (b *Buffer) SafeSlice(s, l int, rev bool) ([]byte, error) {
	if rev {
		return b.EofSlice(s, l)
	} else {
		return b.Slice(s, l)
	}
}

// MustSlice calls Slice or EofSlice (which one depends on the rev argument: true for EofSlice) and suppresses the error.
// If a non io.EOF error is encountered, it will be logged as a warning.
func (b *Buffer) MustSlice(s, l int, rev bool) []byte {
	var slc []byte
	var err error
	if rev {
		slc, err = b.EofSlice(s, l)
	} else {
		slc, err = b.Slice(s, l)
	}
	if err != nil && err != io.EOF {
		log.Printf("Siegreader warning: failed to slice %d for length %d; slice length is is %d; reverse is %b", s, l, len(slc), rev)
	}
	return slc
}

// fill until a seek to a particular offset is possible, then return true, if it is impossible return false
func (b *Buffer) canSeek(o int64, rev bool) (bool, error) {
	if rev {
		if b.sz > 0 {
			o = b.sz - o
			if o < 0 {
				return false, nil
			}
			// continue on to fill below
		} else {
			var err error
			for _, err = b.fill(); err == nil; _, err = b.fill() {
			}
			if err != io.EOF {
				return false, err
			}
			if b.sz-o < 0 {
				return false, nil
			}
			return true, nil
		}
	}
	b.w.Lock()
	defer b.w.Unlock()
	var err error
	var bound int
	if o > int64(b.w.val) {
		for bound, err = b.fill(); o > int64(bound) && err == nil; bound, err = b.fill() {
		}
	}
	if err == nil {
		return true, nil
	}
	if err == io.EOF {
		if o > int64(b.w.val) {
			return false, err
		}
		return true, nil
	}
	return false, err
}
<commit_msg>schoolboy error<commit_after>// Copyright 2014 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Scenarios:
// a) Stream - just copy into a big buffer as at present (... but if there is a MaxBof??)
// b) File
//    b i)   Satisifed with small read beginning
//    b ii)  Small enough for full read
//    b iii) Mmap
//    b iv) Too big for MMap - small buffers and expose ReaderAt

// Package siegreader implements multiple independent Readers (and ReverseReaders) from a single Buffer.
//
// Example:
//   buf := siegreader.New()
//	 err := buf.SetSource(io.Reader)
//   if err != nil {
//     log.Fatal(err)
//   }
//   rdr := buf.Reader()
//	 second_rdr := buf.Reader()
//   reverse_rdr, err := buf.ReverseReader()
//   if err != nil {
//	   log.Fatal(err)
//   }
//   i, _ := rdr.Read(slc)
//   i2, _ := second_rdr.Read(slc2)
//   i3, _ := reverse_rdr.Read(slc3)
package siegreader

import (
	"errors"
	"io"
	"log"
	"os"
	"sync"
)

type Buf interface {
	SetQuit(chan struct{})
	SetSource(io.Reader) error
	Size() int64
	SizeNow() int64
	Slice(off int64, length int, whence bool) ([]byte, error)
	canSeek(off int64, whence bool) (bool, error)
}

var (
	ErrQuit      = errors.New("siegreader: quit chan closed while awaiting EOF")
	ErrNilBuffer = errors.New("siegreader: attempt to SetSource on a nil buffer")
)

const (
	readSz      = 4096
	initialRead = readSz * 2
)

type protected struct {
	sync.Mutex
	val     int
	eofRead bool
}

// Buffer wraps an io.Reader, buffering its contents in byte slices that will keep growing until IO.EOF.
// It supports multiple concurrent Readers, including Readers reading from the end of the stream (ReverseReaders)
type Buffer struct {
	quit      chan struct{} // allows quittting - otherwise will block forever while awaiting EOF
	drain     bool          // Does this Buffer have a regular reader that we can expect will read to the EOF (and hence allow ReverseReaders to wait)
	src       io.Reader
	buf, eof  []byte
	eofc      chan struct{} // signals if EOF bytes are available. When EOF bytes are available, this chan is closed
	completec chan struct{} // signals when the file has been completely read, allows EOF scanning beyond the small buffer
	complete  bool          // marks that the file has been completely read
	sz        int64
	w         protected // index of latest write
}

// New instatatiates a new Buffer with a buf size of 4096*3, and an end-of-file buf size of 4096
func New() *Buffer {
	b := new(Buffer)
	b.buf, b.eof = make([]byte, initialRead), make([]byte, readSz)
	return b
}

func (b *Buffer) reset() {
	b.eofc, b.completec = make(chan struct{}), make(chan struct{})
	b.complete = false
	b.sz = 0
	b.w.Lock()
	b.w.val = 0
	b.w.eofRead = false
	b.w.Unlock()
}

// SetSource sets the buffer's source.
// Can be any io.Reader. If it is an os.File, will load EOF buffer early. Otherwise waits for a complete read.
// The source can be reset to recycle an existing Buffer.
// Siegreader blocks on EOF reads or Size() calls when the reader isn't a file or the stream isn't completely read. The quit channel overrides this block.
func (b *Buffer) SetSource(r io.Reader) error {
	if b == nil {
		return errors.New("Siegreader: attempt to SetSource on a nil buffer")
	}
	b.reset()
	b.src = r
	file, ok := r.(*os.File)
	if ok {
		info, err := file.Stat()
		if err != nil {
			return err
		}
		b.sz = info.Size()
		if b.sz > int64(initialRead) {
			b.eof = b.eof[:cap(b.eof)]
		} else {
			b.eof = b.eof[:0]
		}
	} else {
		b.sz = 0
		b.eof = b.eof[:0]
	}
	_, err := b.fill() // initial fill
	return err
}

func (b *Buffer) SetQuit(q chan struct{}) {
	b.quit = q
}

// Size returns the buffer's size, which is available immediately for files. Must wait for full read for streams.
func (b *Buffer) Size() int {
	if b.sz > 0 {
		return int(b.sz)
	}
	select {
	case <-b.eofc:
		return int(b.sz)
	case <-b.quit:
		return 0
	}
}

// non-blocking Size(), for use with zip reader
func (b *Buffer) SizeNow() int64 {
	if b.sz > 0 {
		return b.sz
	}
	b.w.Lock()
	defer b.w.Unlock()
	var err error
	for _, err = b.fill(); err == nil; _, err = b.fill() {
	}
	if err != io.EOF {
		log.Printf("SIEGREADER WARNING: FAILED TO READ FULL STREAM, ERROR MESSAGE %v", err)
		return 0
	}
	return b.sz
}

func (b *Buffer) grow() {
	// Rules for growing:
	// - if we need to grow, we have passed the initial read and can assume we will need whole file so, if we have a sz grow to it straight away
	// - otherwise, double capacity each time
	var buf []byte
	if b.sz > 0 {
		buf = make([]byte, int(b.sz))
	} else {
		buf = make([]byte, cap(b.buf)*2)
	}
	copy(buf, b.buf[:b.w.val]) // don't care about unlocking as grow() is only called by fill()
	b.buf = buf
}

// Rules for filling:
// - if we have a sz greater than 0, if there is stuff in the eof buffer, and if we are less than readSz from the end, copy across from the eof buffer
// - read readsz * 2 at a time
func (b *Buffer) fill() (int, error) {
	// if we've run out of room, grow the buffer
	if len(b.buf)-readSz < b.w.val {
		b.grow()
	}
	// if we have an eof buffer, and we are near the end of the file, avoid an extra read by copying straight into the main buffer
	if len(b.eof) > 0 && b.w.eofRead && b.w.val+readSz >= int(b.sz) {
		close(b.completec)
		b.complete = true
		lr := int(b.sz) - b.w.val
		b.w.val += copy(b.buf[b.w.val:b.w.val+lr], b.eof[readSz-lr:])
		return b.w.val, io.EOF
	}
	// otherwise, let's read
	e := b.w.val + readSz
	if e > len(b.buf) {
		e = len(b.buf)
	}
	i, err := b.src.Read(b.buf[b.w.val:e])
	if i < readSz {
		err = io.EOF // Readers can give EOF or nil here
	}
	if err != nil {
		close(b.completec)
		b.complete = true
		if err == io.EOF {
			b.w.val += i
			// if we haven't got an eof buf already
			if len(b.eof) < readSz {
				b.sz = int64(b.w.val)
				close(b.eofc)
			}
		}
		return b.w.val, err
	}
	b.w.val += i
	return b.w.val, nil
}

func (b *Buffer) fillEof() error {
	// return nil for a non-file or small file reader
	if len(b.eof) < readSz {
		return nil
	}
	b.w.Lock()
	defer b.w.Unlock()
	if b.w.eofRead {
		return nil // another reverse reader has already filled the buffer
	}
	rs := b.src.(io.ReadSeeker)
	_, err := rs.Seek(0-int64(readSz), 2)
	if err != nil {
		return err
	}
	_, err = rs.Read(b.eof)
	if err != nil {
		return err
	}
	_, err = rs.Seek(int64(b.w.val), 0)
	if err != nil {
		return err
	}
	close(b.eofc)
	b.w.eofRead = true
	return nil
}

// Return a slice from the buffer that begins at offset s and has length l
func (b *Buffer) Slice(s, l int) ([]byte, error) {
	b.w.Lock()
	defer b.w.Unlock()
	var err error
	var bound int
	if s+l > b.w.val && !b.complete {
		for bound, err = b.fill(); s+l > bound && err == nil; bound, err = b.fill() {
		}
	}
	if err == nil && !b.complete {
		return b.buf[s : s+l], nil
	}
	if err == io.EOF || b.complete {
		if s+l > b.w.val {
			if s > b.w.val {
				return nil, io.EOF
			}
			// in the case of an empty file
			if b.Size() == 0 {
				return nil, io.EOF
			}
			return b.buf[s:b.w.val], io.EOF
		} else {
			return b.buf[s : s+l], nil
		}
	}
	return nil, err
}

// Return a slice from the end of the buffer that begins at offset s and has length l.
// This will block until the slice is available (which may be until the full stream is read).
func (b *Buffer) EofSlice(s, l int) ([]byte, error) {
	// block until the EOF is available or we quit
	select {
	case <-b.quit:
		return nil, ErrQuit
	case <-b.eofc:
	}
	var buf []byte
	if len(b.eof) > 0 && s+l <= len(b.eof) {
		buf = b.eof
	} else {
		select {
		case <-b.quit:
			return nil, ErrQuit
		case <-b.completec:
		}
		buf = b.buf[:int(b.sz)]
		if s+l == len(buf) {
			return buf[:len(buf)-s], io.EOF
		}
	}
	if s+l > len(buf) {
		if s > len(buf) {
			return nil, io.EOF
		}
		return buf[:len(buf)-s], io.EOF
	}
	return buf[len(buf)-(s+l) : len(buf)-s], nil
}

// SafeSlice calls Slice or EofSlice (which one depends on the rev argument: true for EofSlice)
func (b *Buffer) SafeSlice(s, l int, rev bool) ([]byte, error) {
	if rev {
		return b.EofSlice(s, l)
	} else {
		return b.Slice(s, l)
	}
}

// MustSlice calls Slice or EofSlice (which one depends on the rev argument: true for EofSlice) and suppresses the error.
// If a non io.EOF error is encountered, it will be logged as a warning.
func (b *Buffer) MustSlice(s, l int, rev bool) []byte {
	var slc []byte
	var err error
	if rev {
		slc, err = b.EofSlice(s, l)
	} else {
		slc, err = b.Slice(s, l)
	}
	if err != nil && err != io.EOF {
		log.Printf("Siegreader warning: failed to slice %d for length %d; slice length is is %d; reverse is %b", s, l, len(slc), rev)
	}
	return slc
}

// fill until a seek to a particular offset is possible, then return true, if it is impossible return false
func (b *Buffer) canSeek(o int64, rev bool) (bool, error) {
	if rev {
		if b.sz > 0 {
			o = b.sz - o
			if o < 0 {
				return false, nil
			}
			// continue on to fill below
		} else {
			var err error
			for _, err = b.fill(); err == nil; _, err = b.fill() {
			}
			if err != io.EOF {
				return false, err
			}
			if b.sz-o < 0 {
				return false, nil
			}
			return true, nil
		}
	}
	b.w.Lock()
	defer b.w.Unlock()
	var err error
	var bound int
	if o > int64(b.w.val) {
		for bound, err = b.fill(); o > int64(bound) && err == nil; bound, err = b.fill() {
		}
	}
	if err == nil {
		return true, nil
	}
	if err == io.EOF {
		if o > int64(b.w.val) {
			return false, err
		}
		return true, nil
	}
	return false, err
}
<|endoftext|>


<commit_before>package array

import (
  "github.com/lestrrat/go-xslate/functions"
)

var depot = functions.NewFuncDepot("array")
func init () {
  depot.Set("Item", Item)
  depot.Set("Size", Size)
}

// Item returns the `i`-th item in the list
func Item(l []interface{}, i int) interface{} {
  return l[i]
}

// Size returns the size of the list
func Size(l []interface {}) int {
  return len(l)
}

// Depot returns the FuncDepot for "array"
func Depot() *functions.FuncDepot {
  return depot
}

<commit_msg>add functions<commit_after>package array

import (
  "github.com/lestrrat/go-xslate/functions"
)

var depot = functions.NewFuncDepot("array")
func init () {
  depot.Set("Item", Item)
  depot.Set("Size", Size)
}

// Item returns the `i`-th item in the list
func Item(l []interface{}, i int) interface{} {
  return l[i]
}

// Size returns the size of the list
func Size(l []interface {}) int {
  return len(l)
}

// First returns the first element
func First(l []interface{}) interface {} {
  return l[0]
}

// Last returns the last element
func Last(l []interface{}) interface {} {
  return l[len(l) - 1]
}

// Depot returns the FuncDepot for "array"
func Depot() *functions.FuncDepot {
  return depot
}

<|endoftext|>


<commit_before>// Copyright 2013 Jongmin Kim. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package korean

import (
	"bytes"
	"testing"
)

var basicTestCases = []struct {
	euckr string
	utf8  string
}{
	{
		euckr: "\xb9\xe6\xb0\xa1\xb9\xe6\xb0\xa1\x20\xb0\xed\xc6\xdb",
		utf8:  "방가방가 고퍼",
	},
}

type transFunc func([]byte) ([]byte, error)

func TestBasics(t *testing.T) {
	for _, tc := range basicTestCases {
		for _, direction := range []string{"UTF8", "EUCKR"} {
			newTransformer, want, src := (transFunc)(nil), "", ""
			if direction == "UTF8" {
				newTransformer, want, src = UTF8, tc.utf8, tc.euckr
			} else {
				newTransformer, want, src = EUCKR, tc.euckr, tc.utf8
			}

			result, err := newTransformer([]byte(src))
			if err != nil {
				t.Errorf("%s didn't match: %v", direction, err)
				continue
			}
			if !bytes.Equal(result, []byte(want)) {
				t.Errorf("%s didn't match: %v", direction, err)
				continue
			}
		}
	}
}
<commit_msg>add unnecessary test. I don't know what I do to increase the code coverage<commit_after>// Copyright 2013 Jongmin Kim. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package korean

import (
	"bytes"
	"fmt"
	"testing"

	"code.google.com/p/go.text/encoding/korean"
)

var basicTestCases = []struct {
	euckr string
	utf8  string
}{
	{
		euckr: "\xb9\xe6\xb0\xa1\xb9\xe6\xb0\xa1\x20\xb0\xed\xc6\xdb",
		utf8:  "방가방가 고퍼",
	},
}

type transFunc func([]byte) ([]byte, error)

func TestBasics(t *testing.T) {
	for _, tc := range basicTestCases {
		for _, direction := range []string{"UTF8", "EUCKR"} {
			newTransformer, want, src := (transFunc)(nil), "", ""
			if direction == "UTF8" {
				newTransformer, want, src = UTF8, tc.utf8, tc.euckr
			} else {
				newTransformer, want, src = EUCKR, tc.euckr, tc.utf8
			}

			result, err := newTransformer([]byte(src))
			if err != nil {
				t.Errorf("%s didn't match: %v", direction, err)
				continue
			}
			if !bytes.Equal(result, []byte(want)) {
				t.Errorf("%s didn't match: %v", direction, err)
				continue
			}
		}
	}
}

func TestTrans(t *testing.T) {
	trans(korean.EUCKR.NewDecoder(), []byte("고퍼"))
}

// TODO(atomaths): fill correct benchmark test case.
func BenchmarkTrans(b *testing.B) {
	for i := 0; i < b.N; i++ {
	}
}

func ExampleTest() {
	dst, _ := UTF8([]byte("\xb9\xe6\xb0\xa1\xb9\xe6\xb0\xa1\x20\xb0\xed\xc6\xdb"))
	fmt.Printf("%s", dst)
	// Output:
	// 방가방가 고퍼
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin freebsd netbsd openbsd

package fswatch

import (
	"syscall"
	"testing"
)

func TestNewKqueue(t *testing.T) {
	kq, err := newkqueue()
	assert(t, err)
	assert(t, kq.close())
}

func TestKqueueAddSocket(t *testing.T) {
	kq, err := newkqueue()
	assert(t, err)
	s, err := syscall.Socket(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
	assert(t, err)
	defer syscall.Close(s)
	assert(t, kq.add(uintptr(s), syscall.EVFILT_WRITE, 0, 0))
	assert(t, kq.close())
}

func TestKqueueAddFile(t *testing.T) {
	d, done := setup(t)
	defer done()
	kq, err := newkqueue()
	assert(t, err)
	f := newfile(t, d)
	defer f.Close()
	assert(t, kq.add(f.Fd(), syscall.EVFILT_WRITE, 0, 0))
	assert(t, kq.close())
}

func TestKqueueWaitSocketClose(t *testing.T) {
	kq, err := newkqueue()
	assert(t, err)
	s, err := syscall.Socket(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
	assert(t, err)
	assert(t, kq.add(uintptr(s), syscall.EVFILT_READ, syscall.EV_ONESHOT, 0))
	result := make(chan error)
	go func() {
		_, err := kq.wait(make([]syscall.Kevent_t, 1))
		result <- err
	}()
	syscall.Close(s)
	assert(t, <-result)
	assert(t, kq.close())
}
<commit_msg>pipe close test works<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin freebsd netbsd openbsd

package fswatch

import (
	"os"
	"syscall"
	"testing"
)

func TestNewKqueue(t *testing.T) {
	kq, err := newkqueue()
	assert(t, err)
	assert(t, kq.close())
}

func TestKqueueAddSocket(t *testing.T) {
	kq, err := newkqueue()
	assert(t, err)
	s, err := syscall.Socket(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
	assert(t, err)
	defer syscall.Close(s)
	assert(t, kq.add(uintptr(s), syscall.EVFILT_WRITE, 0, 0))
	assert(t, kq.close())
}

func TestKqueueAddFile(t *testing.T) {
	d, done := setup(t)
	defer done()
	kq, err := newkqueue()
	assert(t, err)
	f := newfile(t, d)
	defer f.Close()
	assert(t, kq.add(f.Fd(), syscall.EVFILT_WRITE, 0, 0))
	assert(t, kq.close())
}

func TestKqueueWaitPipeClose(t *testing.T) {
	kq, err := newkqueue()
	assert(t, err)
	pr, pw, err := os.Pipe()
	assert(t, err)
	defer pw.Close()
	assert(t, kq.add(uintptr(pr.Fd()), syscall.EVFILT_READ, syscall.EV_ONESHOT, 0))
	result := make(chan error)
	go func() {
		_, err := kq.wait(make([]syscall.Kevent_t, 1))
		result <- err
	}()
	assert(t, pw.Close())
	assert(t, <-result)
	assert(t, kq.close())
}
<|endoftext|>


<commit_before>package kube

import (
	"bytes"
	"fmt"
	"io"
	"strings"

	"crypto/tls"
	"crypto/x509"
	"encoding/base64"
	"encoding/json"
	"github.com/Sirupsen/logrus"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/http"
)

// KubeClient is the interface to access the kubernetes job API
type KubeClient interface {
	CreateJob(job *Job) error
	GetSecret(namespace string, secretName string) (map[string]string, error)
	DeployResourceFile(resourceFile []byte) error
}

// concrete implementation of a job client
type realKubeClient struct {
	*http.Client
	address string
	token   string
}

// NewClient returns a new KubeClient connecting to the address. This uses the service
// account credentials
func NewClient(address string) (KubeClient, error) {
	// create tls client
	cacertFile := "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
	capem, err := ioutil.ReadFile(cacertFile)
	if err != nil {
		return nil, err
	}
	cacert := x509.NewCertPool()
	if !cacert.AppendCertsFromPEM(capem) {
		return nil, fmt.Errorf("unable to load certificate authority")
	}
	config := &tls.Config{RootCAs: cacert}
	transport := &http.Transport{TLSClientConfig: config}

	// read token
	client := &http.Client{Transport: transport}
	tokenFile := "/var/run/secrets/kubernetes.io/serviceaccount/token"
	token, err := ioutil.ReadFile(tokenFile)
	if err != nil {
		return nil, err
	}
	return &realKubeClient{client, address, string(token)}, nil
}

// Create a new kubernetes Job with the given job
func (r *realKubeClient) CreateJob(job *Job) error {
	url := "/apis/extensions/v1beta1/namespaces/" + job.Metadata["namespace"].(string) + "/jobs"
	data, err := json.Marshal(job)
	if err != nil {
		return err
	}
	byteData := bytes.NewReader(data)
	return r.doPost(url, byteData)

}

// DeployResourceFile deploys a Kubernbetes YAML spec file as Kubernetes resources
func (r *realKubeClient) DeployResourceFile(resourceFile []byte) error {

	resources := strings.Split(string(resourceFile), "---")

	for _, resource := range resources {

		// if it's empty, skip
		if strings.TrimSpace(resource) == "" {
			continue
		}

		logrus.Info("deploying to kubernetes: ", resource)

		var out map[string]interface{}
		err := yaml.Unmarshal([]byte(resource), &out)
		if err != nil {
			return err
		}

		// if unmarshalled data is nil, skip
		if out == nil {
			continue
		}

		kind := strings.ToLower(out["kind"].(string)) + "s"
		metadata := out["metadata"]
		namespace := "default"
		if metadata != nil {
			namespace = metadata.(map[interface{}]interface{})["namespace"].(string)
		}

		// endpoint is /api/v1/namespaces/{namespace}/{resourceType}
		uri := fmt.Sprintf("/api/v1/namespaces/%s/%s", namespace, kind)
		data, err := json.Marshal(out)
		if err != nil {
			return err
		}

		err = r.doPost(uri, bytes.NewReader(data))
		if err != nil {
			return err
		}
	}

	return nil
}

//Get secret with a given namespace and secret name
func (r *realKubeClient) GetSecret(namespace string, secretName string) (map[string]string, error) {
	secret := &Secret{}
	secrets := make(map[string]string)

	uri := "/api/v1/namespaces/" + namespace + "/secrets/" + secretName

	err := r.doGet(uri, secret)
	if err != nil {
		return nil, err
	}

	for key, value := range secret.Data {
		decodedValue, err := base64.StdEncoding.DecodeString(value)

		if err != nil {
			logrus.WithError(err).Println("Unable to decode secret", key, value)
			continue
		}
		secrets[key] = string(decodedValue)
	}
	return secrets, nil
}

func (r *realKubeClient) doGet(uri string, response interface{}) error {
	req, err := r.createRequest("GET", uri, nil)
	if err != nil {
		return err
	}
	res, err := r.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}
	if res.StatusCode != 200 {
		return fmt.Errorf("%d: %s", res.StatusCode, string(body))
	}
	err = json.Unmarshal(body, response)
	if err != nil {
		return err
	}
	return nil
}

func (r *realKubeClient) doPost(uri string, data io.Reader) error {
	req, err := r.createRequest("POST", uri, data)
	if err != nil {
		return err
	}
	res, err := r.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode == http.StatusCreated || res.StatusCode == http.StatusOK {
		return nil
	}
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}
	return fmt.Errorf("%d: %s", res.StatusCode, string(body))
}

func (r *realKubeClient) createRequest(method, uri string, data io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, r.address+uri, data)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Authorization", "Bearer "+r.token)
	return req, nil
}
<commit_msg>more logs<commit_after>package kube

import (
	"bytes"
	"fmt"
	"io"
	"strings"

	"crypto/tls"
	"crypto/x509"
	"encoding/base64"
	"encoding/json"
	"github.com/Sirupsen/logrus"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/http"
)

// KubeClient is the interface to access the kubernetes job API
type KubeClient interface {
	CreateJob(job *Job) error
	GetSecret(namespace string, secretName string) (map[string]string, error)
	DeployResourceFile(resourceFile []byte) error
}

// concrete implementation of a job client
type realKubeClient struct {
	*http.Client
	address string
	token   string
}

// NewClient returns a new KubeClient connecting to the address. This uses the service
// account credentials
func NewClient(address string) (KubeClient, error) {
	// create tls client
	cacertFile := "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
	capem, err := ioutil.ReadFile(cacertFile)
	if err != nil {
		return nil, err
	}
	cacert := x509.NewCertPool()
	if !cacert.AppendCertsFromPEM(capem) {
		return nil, fmt.Errorf("unable to load certificate authority")
	}
	config := &tls.Config{RootCAs: cacert}
	transport := &http.Transport{TLSClientConfig: config}

	// read token
	client := &http.Client{Transport: transport}
	tokenFile := "/var/run/secrets/kubernetes.io/serviceaccount/token"
	token, err := ioutil.ReadFile(tokenFile)
	if err != nil {
		return nil, err
	}
	return &realKubeClient{client, address, string(token)}, nil
}

// Create a new kubernetes Job with the given job
func (r *realKubeClient) CreateJob(job *Job) error {
	url := "/apis/extensions/v1beta1/namespaces/" + job.Metadata["namespace"].(string) + "/jobs"
	data, err := json.Marshal(job)
	if err != nil {
		return err
	}
	byteData := bytes.NewReader(data)
	return r.doPost(url, byteData)

}

// DeployResourceFile deploys a Kubernbetes YAML spec file as Kubernetes resources
func (r *realKubeClient) DeployResourceFile(resourceFile []byte) error {

	resources := strings.Split(string(resourceFile), "---")

	for _, resource := range resources {

		// if it's empty, skip
		if strings.TrimSpace(resource) == "" {
			continue
		}

		logrus.Info("deploying to kubernetes: ", resource)

		var out map[string]interface{}

		err := yaml.Unmarshal([]byte(resource), &out)
		if err != nil {
			return err
		}

		logrus.Info("parsed: ", out)

		// if unmarshalled data is nil, skip
		if out == nil {
			continue
		}

		kind := strings.ToLower(out["kind"].(string)) + "s"
		metadata := out["metadata"]
		namespace := "default"
		if metadata != nil {
			namespace = metadata.(map[interface{}]interface{})["namespace"].(string)
		}

		// endpoint is /api/v1/namespaces/{namespace}/{resourceType}
		uri := fmt.Sprintf("/api/v1/namespaces/%s/%s", namespace, kind)
		data, err := json.Marshal(out)
		if err != nil {
			return err
		}

		err = r.doPost(uri, bytes.NewReader(data))
		if err != nil {
			return err
		}
	}

	return nil
}

//Get secret with a given namespace and secret name
func (r *realKubeClient) GetSecret(namespace string, secretName string) (map[string]string, error) {
	secret := &Secret{}
	secrets := make(map[string]string)

	uri := "/api/v1/namespaces/" + namespace + "/secrets/" + secretName

	err := r.doGet(uri, secret)
	if err != nil {
		return nil, err
	}

	for key, value := range secret.Data {
		decodedValue, err := base64.StdEncoding.DecodeString(value)

		if err != nil {
			logrus.WithError(err).Println("Unable to decode secret", key, value)
			continue
		}
		secrets[key] = string(decodedValue)
	}
	return secrets, nil
}

func (r *realKubeClient) doGet(uri string, response interface{}) error {
	req, err := r.createRequest("GET", uri, nil)
	if err != nil {
		return err
	}
	res, err := r.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}
	if res.StatusCode != 200 {
		return fmt.Errorf("%d: %s", res.StatusCode, string(body))
	}
	err = json.Unmarshal(body, response)
	if err != nil {
		return err
	}
	return nil
}

func (r *realKubeClient) doPost(uri string, data io.Reader) error {
	req, err := r.createRequest("POST", uri, data)
	if err != nil {
		return err
	}
	res, err := r.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode == http.StatusCreated || res.StatusCode == http.StatusOK {
		return nil
	}
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}
	return fmt.Errorf("%d: %s", res.StatusCode, string(body))
}

func (r *realKubeClient) createRequest(method, uri string, data io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, r.address+uri, data)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Authorization", "Bearer "+r.token)
	return req, nil
}
<|endoftext|>


<commit_before>package libcontainer

import (
	"github.com/dotcloud/docker/pkg/libcontainer/cgroups"
)

// Context is a generic key value pair that allows
// arbatrary data to be sent
type Context map[string]string

// Container defines configuration options for how a
// container is setup inside a directory and how a process should be executed
type Container struct {
	Hostname     string              `json:"hostname,omitempty"`      // hostname
	ReadonlyFs   bool                `json:"readonly_fs,omitempty"`   // set the containers rootfs as readonly
	NoPivotRoot  bool                `json:"no_pivot_root,omitempty"` // this can be enabled if you are running in ramdisk
	User         string              `json:"user,omitempty"`          // user to execute the process as
	WorkingDir   string              `json:"working_dir,omitempty"`   // current working directory
	Env          []string            `json:"environment,omitempty"`   // environment to set
	Tty          bool                `json:"tty,omitempty"`           // setup a proper tty or not
	Namespaces   map[string]bool     `json:"namespaces,omitempty"`    // namespaces to apply
	Capabilities []string            `json:"capabilities,omitempty"`  // capabilities given to the container
	Networks     []*Network          `json:"networks,omitempty"`      // nil for host's network stack
	Cgroups      *cgroups.Cgroup     `json:"cgroups,omitempty"`       // cgroups
	Context      Context             `json:"context,omitempty"`       // generic context for specific options (apparmor, selinux)
	Mounts       Mounts              `json:"mounts,omitempty"`
	DeviceNodes  map[string][]string `json:"device_nodes,omitempty"` // device nodes to add to the container's /dev
}

// Network defines configuration for a container's networking stack
//
// The network configuration can be omited from a container causing the
// container to be setup with the host's networking stack
type Network struct {
	Type    string  `json:"type,omitempty"`    // type of networking to setup i.e. veth, macvlan, etc
	Context Context `json:"context,omitempty"` // generic context for type specific networking options
	Address string  `json:"address,omitempty"`
	Gateway string  `json:"gateway,omitempty"`
	Mtu     int     `json:"mtu,omitempty"`
}
<commit_msg>Update documentation for container struct in libcontainer Docker-DCO-1.1-Signed-off-by: Michael Crosby <michael@crosbymichael.com> (github: crosbymichael)<commit_after>package libcontainer

import (
	"github.com/dotcloud/docker/pkg/libcontainer/cgroups"
)

// Context is a generic key value pair that allows arbatrary data to be sent
type Context map[string]string

// Container defines configuration options for executing a process inside a contained environment
type Container struct {
	// Hostname optionally sets the container's hostname if provided
	Hostname string `json:"hostname,omitempty"`

	// ReadonlyFs will remount the container's rootfs as readonly where only externally mounted
	// bind mounts are writtable
	ReadonlyFs bool `json:"readonly_fs,omitempty"`

	// NoPivotRoot will use MS_MOVE and a chroot to jail the process into the container's rootfs
	// This is a common option when the container is running in ramdisk
	NoPivotRoot bool `json:"no_pivot_root,omitempty"`

	// User will set the uid and gid of the executing process running inside the container
	User string `json:"user,omitempty"`

	// WorkingDir will change the processes current working directory inside the container's rootfs
	WorkingDir string `json:"working_dir,omitempty"`

	// Env will populate the processes environment with the provided values
	// Any values from the parent processes will be cleared before the values
	// provided in Env are provided to the process
	Env []string `json:"environment,omitempty"`

	// Tty when true will allocate a pty slave on the host for access by the container's process
	// and ensure that it is mounted inside the container's rootfs
	Tty bool `json:"tty,omitempty"`

	// Namespaces specifies the container's namespaces that it should setup when cloning the init process
	// If a namespace is not provided that namespace is shared from the container's parent process
	Namespaces map[string]bool `json:"namespaces,omitempty"`

	// Capabilities specify the capabilities to keep when executing the process inside the container
	// All capbilities not specified will be dropped from the processes capability mask
	Capabilities []string `json:"capabilities,omitempty"`

	// Networks specifies the container's network stop to be created
	Networks []*Network `json:"networks,omitempty"`

	// Cgroups specifies specific cgroup settings for the various subsystems that the container is
	// placed into to limit the resources the container has available
	Cgroups *cgroups.Cgroup `json:"cgroups,omitempty"`

	// Context is a generic key value format that allows for additional settings to be passed
	// on the container's creation
	// This is commonly used to specify apparmor profiles, selinux labels, and different restrictions
	// placed on the container's processes
	Context Context `json:"context,omitempty"`

	// Mounts specify additional source and destination paths that will be mounted inside the container's
	// rootfs and mount namespace if specified
	Mounts Mounts `json:"mounts,omitempty"`

	// DeviceNodes are a list of 'required' and 'additional' nodes that will be mknod into the container's
	// rootfs at /dev
	//
	// Required device nodes will return an error if the host system does not have this device available
	//
	// Additional device nodes are created but no error is returned if the host system does not have the
	// device avaliable for use by the container
	DeviceNodes map[string][]string `json:"device_nodes,omitempty"`
}

// Network defines configuration for a container's networking stack
//
// The network configuration can be omited from a container causing the
// container to be setup with the host's networking stack
type Network struct {
	// Type sets the networks type, commonly veth and loopback
	Type string `json:"type,omitempty"`

	// Context is a generic key value format for setting additional options that are specific to
	// the network type
	Context Context `json:"context,omitempty"`

	// Address contains the IP and mask to set on the network interface
	Address string `json:"address,omitempty"`

	// Gateway sets the gateway address that is used as the default for the interface
	Gateway string `json:"gateway,omitempty"`

	// Mtu sets the mtu value for the interface and will be mirrored on both the host and
	// container's interfaces if a pair is created, specifically in the case of type veth
	Mtu int `json:"mtu,omitempty"`
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"encoding/json"
	"fmt"
	"os/exec"
	"path"
	"path/filepath"
	"time"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/host"
	"github.com/golang/glog"
	"github.com/juju/mutex"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/registry"
	"k8s.io/minikube/pkg/minikube/sshutil"
	"k8s.io/minikube/pkg/minikube/vmpath"
	"k8s.io/minikube/pkg/util/lock"
)

var (
	// requiredDirectories are directories to create on the host during setup
	requiredDirectories = []string{
		vmpath.GuestAddonsDir,
		vmpath.GuestManifestsDir,
		vmpath.GuestEphemeralDir,
		vmpath.GuestPersistentDir,
		vmpath.GuestCertsDir,
		path.Join(vmpath.GuestPersistentDir, "images"),
		path.Join(vmpath.GuestPersistentDir, "binaries"),
	}
)

// StartHost starts a host VM.
func StartHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
	// Prevent machine-driver boot races, as well as our own certificate race
	releaser, err := acquireMachinesLock(cfg.Name)
	if err != nil {
		return nil, errors.Wrap(err, "boot lock")
	}
	start := time.Now()
	defer func() {
		glog.Infof("releasing machines lock for %q, held for %s", cfg.Name, time.Since(start))
		releaser.Release()
	}()

	exists, err := api.Exists(cfg.Name)
	if err != nil {
		return nil, errors.Wrapf(err, "exists: %s", cfg.Name)
	}
	if !exists {
		glog.Infof("Provisioning new machine with config: %+v", cfg)
		return createHost(api, cfg)
	}
	glog.Infoln("Skipping create...Using existing machine configuration")
	return fixHost(api, cfg)
}

func engineOptions(cfg config.MachineConfig) *engine.Options {
	o := engine.Options{
		Env:              cfg.DockerEnv,
		InsecureRegistry: append([]string{constants.DefaultServiceCIDR}, cfg.InsecureRegistry...),
		RegistryMirror:   cfg.RegistryMirror,
		ArbitraryFlags:   cfg.DockerOpt,
		InstallURL:       drivers.DefaultEngineInstallURL,
	}
	return &o
}

func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
	glog.Infof("createHost starting for %q (driver=%q)", cfg.Name, cfg.VMDriver)
	start := time.Now()
	defer func() {
		glog.Infof("createHost completed in %s", time.Since(start))
	}()

	if cfg.VMDriver == driver.VMwareFusion && viper.GetBool(config.ShowDriverDeprecationNotification) {
		out.WarningT(`The vmwarefusion driver is deprecated and support for it will be removed in a future release.
			Please consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.
			See https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information.
			To disable this message, run [minikube config set ShowDriverDeprecationNotification false]`)
	}
	showHostInfo(cfg)
	def := registry.Driver(cfg.VMDriver)
	if def.Empty() {
		return nil, fmt.Errorf("unsupported/missing driver: %s", cfg.VMDriver)
	}
	dd := def.Config(cfg)
	data, err := json.Marshal(dd)
	if err != nil {
		return nil, errors.Wrap(err, "marshal")
	}

	h, err := api.NewHost(cfg.VMDriver, data)
	if err != nil {
		return nil, errors.Wrap(err, "new host")
	}

	h.HostOptions.AuthOptions.CertDir = localpath.MiniPath()
	h.HostOptions.AuthOptions.StorePath = localpath.MiniPath()
	h.HostOptions.EngineOptions = engineOptions(cfg)

	cstart := time.Now()
	glog.Infof("libmachine.API.Create for %q (driver=%q)", cfg.Name, cfg.VMDriver)
	if err := api.Create(h); err != nil {
		// Wait for all the logs to reach the client
		time.Sleep(2 * time.Second)
		return nil, errors.Wrap(err, "create")
	}
	glog.Infof("libmachine.API.Create for %q took %s", cfg.Name, time.Since(cstart))

	if err := postStartSetup(h, cfg); err != nil {
		return h, errors.Wrap(err, "post-start")
	}

	if err := api.Save(h); err != nil {
		return nil, errors.Wrap(err, "save")
	}
	return h, nil
}

// postStart are functions shared between startHost and fixHost
func postStartSetup(h *host.Host, mc config.MachineConfig) error {
	glog.Infof("post-start starting for %q (driver=%q)", h.Name, h.DriverName)
	start := time.Now()
	defer func() {
		glog.Infof("post-start completed in %s", time.Since(start))
	}()

	if driver.IsMock(h.DriverName) {
		return nil
	}

	glog.Infof("creating required directories: %v", requiredDirectories)
	r, err := commandRunner(h)
	if err != nil {
		return errors.Wrap(err, "command runner")
	}

	args := append([]string{"mkdir", "-p"}, requiredDirectories...)
	if _, err := r.RunCmd(exec.Command("sudo", args...)); err != nil {
		return errors.Wrapf(err, "sudo mkdir (%s)", h.DriverName)
	}

	if driver.BareMetal(mc.VMDriver) {
		showLocalOsRelease()
	}
	if driver.IsVM(mc.VMDriver) {
		logRemoteOsRelease(h.Driver)
	}
	return syncLocalAssets(r)
}

// commandRunner returns best available command runner for this host
func commandRunner(h *host.Host) (command.Runner, error) {
	d := h.Driver.DriverName()
	glog.V(1).Infof("determining appropriate runner for %q", d)
	if driver.IsMock(d) {
		glog.Infof("returning FakeCommandRunner for %q driver", d)
		return &command.FakeCommandRunner{}, nil
	}

  	if driver.BareMetal(h.Driver.DriverName()) {
		glog.Infof("returning ExecRunner for %q driver", d)
    		return command.NewExecRunner(), nil
	}
	if driver.IsKIC(d) {
		glog.Infof("Returning KICRunner for %q driver", d)
		return command.NewKICRunner(h.Name, "docker"), nil
	}

	glog.Infof("Creating SSH client and returning SSHRunner for %q driver", d)
	client, err := sshutil.NewSSHClient(h.Driver)
	if err != nil {
		return nil, errors.Wrap(err, "ssh client")
	}
	return command.NewSSHRunner(client), nil
}

// acquireMachinesLock protects against code that is not parallel-safe (libmachine, cert setup)
func acquireMachinesLock(name string) (mutex.Releaser, error) {
	spec := lock.PathMutexSpec(filepath.Join(localpath.MiniPath(), "machines"))
	// NOTE: Provisioning generally completes within 60 seconds
	spec.Timeout = 15 * time.Minute

	glog.Infof("acquiring machines lock for %s: %+v", name, spec)
	start := time.Now()
	r, err := mutex.Acquire(spec)
	if err == nil {
		glog.Infof("acquired machines lock for %q in %s", name, time.Since(start))
	}
	return r, err
}
<commit_msg>goimports<commit_after>/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"encoding/json"
	"fmt"
	"os/exec"
	"path"
	"path/filepath"
	"time"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/host"
	"github.com/golang/glog"
	"github.com/juju/mutex"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/registry"
	"k8s.io/minikube/pkg/minikube/sshutil"
	"k8s.io/minikube/pkg/minikube/vmpath"
	"k8s.io/minikube/pkg/util/lock"
)

var (
	// requiredDirectories are directories to create on the host during setup
	requiredDirectories = []string{
		vmpath.GuestAddonsDir,
		vmpath.GuestManifestsDir,
		vmpath.GuestEphemeralDir,
		vmpath.GuestPersistentDir,
		vmpath.GuestCertsDir,
		path.Join(vmpath.GuestPersistentDir, "images"),
		path.Join(vmpath.GuestPersistentDir, "binaries"),
	}
)

// StartHost starts a host VM.
func StartHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
	// Prevent machine-driver boot races, as well as our own certificate race
	releaser, err := acquireMachinesLock(cfg.Name)
	if err != nil {
		return nil, errors.Wrap(err, "boot lock")
	}
	start := time.Now()
	defer func() {
		glog.Infof("releasing machines lock for %q, held for %s", cfg.Name, time.Since(start))
		releaser.Release()
	}()

	exists, err := api.Exists(cfg.Name)
	if err != nil {
		return nil, errors.Wrapf(err, "exists: %s", cfg.Name)
	}
	if !exists {
		glog.Infof("Provisioning new machine with config: %+v", cfg)
		return createHost(api, cfg)
	}
	glog.Infoln("Skipping create...Using existing machine configuration")
	return fixHost(api, cfg)
}

func engineOptions(cfg config.MachineConfig) *engine.Options {
	o := engine.Options{
		Env:              cfg.DockerEnv,
		InsecureRegistry: append([]string{constants.DefaultServiceCIDR}, cfg.InsecureRegistry...),
		RegistryMirror:   cfg.RegistryMirror,
		ArbitraryFlags:   cfg.DockerOpt,
		InstallURL:       drivers.DefaultEngineInstallURL,
	}
	return &o
}

func createHost(api libmachine.API, cfg config.MachineConfig) (*host.Host, error) {
	glog.Infof("createHost starting for %q (driver=%q)", cfg.Name, cfg.VMDriver)
	start := time.Now()
	defer func() {
		glog.Infof("createHost completed in %s", time.Since(start))
	}()

	if cfg.VMDriver == driver.VMwareFusion && viper.GetBool(config.ShowDriverDeprecationNotification) {
		out.WarningT(`The vmwarefusion driver is deprecated and support for it will be removed in a future release.
			Please consider switching to the new vmware unified driver, which is intended to replace the vmwarefusion driver.
			See https://minikube.sigs.k8s.io/docs/reference/drivers/vmware/ for more information.
			To disable this message, run [minikube config set ShowDriverDeprecationNotification false]`)
	}
	showHostInfo(cfg)
	def := registry.Driver(cfg.VMDriver)
	if def.Empty() {
		return nil, fmt.Errorf("unsupported/missing driver: %s", cfg.VMDriver)
	}
	dd := def.Config(cfg)
	data, err := json.Marshal(dd)
	if err != nil {
		return nil, errors.Wrap(err, "marshal")
	}

	h, err := api.NewHost(cfg.VMDriver, data)
	if err != nil {
		return nil, errors.Wrap(err, "new host")
	}

	h.HostOptions.AuthOptions.CertDir = localpath.MiniPath()
	h.HostOptions.AuthOptions.StorePath = localpath.MiniPath()
	h.HostOptions.EngineOptions = engineOptions(cfg)

	cstart := time.Now()
	glog.Infof("libmachine.API.Create for %q (driver=%q)", cfg.Name, cfg.VMDriver)
	if err := api.Create(h); err != nil {
		// Wait for all the logs to reach the client
		time.Sleep(2 * time.Second)
		return nil, errors.Wrap(err, "create")
	}
	glog.Infof("libmachine.API.Create for %q took %s", cfg.Name, time.Since(cstart))

	if err := postStartSetup(h, cfg); err != nil {
		return h, errors.Wrap(err, "post-start")
	}

	if err := api.Save(h); err != nil {
		return nil, errors.Wrap(err, "save")
	}
	return h, nil
}

// postStart are functions shared between startHost and fixHost
func postStartSetup(h *host.Host, mc config.MachineConfig) error {
	glog.Infof("post-start starting for %q (driver=%q)", h.Name, h.DriverName)
	start := time.Now()
	defer func() {
		glog.Infof("post-start completed in %s", time.Since(start))
	}()

	if driver.IsMock(h.DriverName) {
		return nil
	}

	glog.Infof("creating required directories: %v", requiredDirectories)
	r, err := commandRunner(h)
	if err != nil {
		return errors.Wrap(err, "command runner")
	}

	args := append([]string{"mkdir", "-p"}, requiredDirectories...)
	if _, err := r.RunCmd(exec.Command("sudo", args...)); err != nil {
		return errors.Wrapf(err, "sudo mkdir (%s)", h.DriverName)
	}

	if driver.BareMetal(mc.VMDriver) {
		showLocalOsRelease()
	}
	if driver.IsVM(mc.VMDriver) {
		logRemoteOsRelease(h.Driver)
	}
	return syncLocalAssets(r)
}

// commandRunner returns best available command runner for this host
func commandRunner(h *host.Host) (command.Runner, error) {
	d := h.Driver.DriverName()
	glog.V(1).Infof("determining appropriate runner for %q", d)
	if driver.IsMock(d) {
		glog.Infof("returning FakeCommandRunner for %q driver", d)
		return &command.FakeCommandRunner{}, nil
	}

	if driver.BareMetal(h.Driver.DriverName()) {
		glog.Infof("returning ExecRunner for %q driver", d)
		return command.NewExecRunner(), nil
	}
	if driver.IsKIC(d) {
		glog.Infof("Returning KICRunner for %q driver", d)
		return command.NewKICRunner(h.Name, "docker"), nil
	}

	glog.Infof("Creating SSH client and returning SSHRunner for %q driver", d)
	client, err := sshutil.NewSSHClient(h.Driver)
	if err != nil {
		return nil, errors.Wrap(err, "ssh client")
	}
	return command.NewSSHRunner(client), nil
}

// acquireMachinesLock protects against code that is not parallel-safe (libmachine, cert setup)
func acquireMachinesLock(name string) (mutex.Releaser, error) {
	spec := lock.PathMutexSpec(filepath.Join(localpath.MiniPath(), "machines"))
	// NOTE: Provisioning generally completes within 60 seconds
	spec.Timeout = 15 * time.Minute

	glog.Infof("acquiring machines lock for %s: %+v", name, spec)
	start := time.Now()
	r, err := mutex.Acquire(spec)
	if err == nil {
		glog.Infof("acquired machines lock for %q in %s", name, time.Since(start))
	}
	return r, err
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package registry

import (
	"fmt"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
	"github.com/golang/glog"
)

// TODO: Need to add a reconciler loop that makes sure that things in pods are reflected into
//       kubelet (and vice versa)

// EtcdRegistry implements PodRegistry, ControllerRegistry and ServiceRegistry with backed by etcd.
type EtcdRegistry struct {
	etcdClient      tools.EtcdClient
	machines        MinionRegistry
	manifestFactory ManifestFactory
}

// MakeEtcdRegistry creates an etcd registry.
// 'client' is the connection to etcd
// 'machines' is the list of machines
// 'scheduler' is the scheduling algorithm to use.
func MakeEtcdRegistry(client tools.EtcdClient, machines MinionRegistry) *EtcdRegistry {
	registry := &EtcdRegistry{
		etcdClient: client,
		machines:   machines,
	}
	registry.manifestFactory = &BasicManifestFactory{
		serviceRegistry: registry,
	}
	return registry
}

func makePodKey(machine, podID string) string {
	return "/registry/hosts/" + machine + "/pods/" + podID
}

func (registry *EtcdRegistry) helper() *tools.EtcdHelper {
	return &tools.EtcdHelper{registry.etcdClient}
}

// ListPods obtains a list of pods that match selector.
func (registry *EtcdRegistry) ListPods(selector labels.Selector) ([]api.Pod, error) {
	pods := []api.Pod{}
	machines, err := registry.machines.List()
	if err != nil {
		return nil, err
	}
	for _, machine := range machines {
		var machinePods []api.Pod
		err := registry.helper().ExtractList("/registry/hosts/"+machine+"/pods", &machinePods)
		if err != nil {
			return pods, err
		}
		for _, pod := range machinePods {
			if selector.Matches(labels.Set(pod.Labels)) {
				pod.CurrentState.Host = machine
				pods = append(pods, pod)
			}
		}
	}
	return pods, nil
}

// GetPod gets a specific pod specified by its ID.
func (registry *EtcdRegistry) GetPod(podID string) (*api.Pod, error) {
	pod, _, err := registry.findPod(podID)
	return &pod, err
}

func makeContainerKey(machine string) string {
	return "/registry/hosts/" + machine + "/kubelet"
}

// CreatePod creates a pod based on a specification, schedule it onto a specific machine.
func (registry *EtcdRegistry) CreatePod(machineIn string, pod api.Pod) error {
	podOut, machine, err := registry.findPod(pod.ID)
	if err == nil {
		// TODO: this error message looks racy.
		return fmt.Errorf("a pod named %s already exists on %s (%#v)", pod.ID, machine, podOut)
	}
	return registry.runPod(pod, machineIn)
}

func (registry *EtcdRegistry) runPod(pod api.Pod, machine string) error {
	podKey := makePodKey(machine, pod.ID)
	err := registry.helper().SetObj(podKey, pod)

	manifest, err := registry.manifestFactory.MakeManifest(machine, pod)
	if err != nil {
		return err
	}

	contKey := makeContainerKey(machine)
	err = registry.helper().AtomicUpdate(contKey, &[]api.ContainerManifest{}, func(in interface{}) (interface{}, error) {
		manifests := *in.(*[]api.ContainerManifest)
		return append(manifests, manifest), nil
	})
	if err != nil {
		// Don't strand stuff.
		_, err2 := registry.etcdClient.Delete(podKey, false)
		if err2 != nil {
			glog.Errorf("Probably stranding a pod, couldn't delete %v: %#v", podKey, err2)
		}
	}
	return err
}

func (registry *EtcdRegistry) UpdatePod(pod api.Pod) error {
	return fmt.Errorf("unimplemented!")
}

// DeletePod deletes an existing pod specified by its ID.
func (registry *EtcdRegistry) DeletePod(podID string) error {
	_, machine, err := registry.findPod(podID)
	if err != nil {
		return err
	}
	return registry.deletePodFromMachine(machine, podID)
}

func (registry *EtcdRegistry) deletePodFromMachine(machine, podID string) error {
	// First delete the pod, so a scheduler doesn't notice it getting removed from the
	// machine and attempt to put it somewhere.
	podKey := makePodKey(machine, podID)
	_, err := registry.etcdClient.Delete(podKey, true)
	if tools.IsEtcdNotFound(err) {
		return apiserver.NewNotFoundErr("pod", podID)
	}
	if err != nil {
		return err
	}

	// Next, remove the pod from the machine atomically.
	contKey := makeContainerKey(machine)
	return registry.helper().AtomicUpdate(contKey, &[]api.ContainerManifest{}, func(in interface{}) (interface{}, error) {
		manifests := *in.(*[]api.ContainerManifest)
		newManifests := make([]api.ContainerManifest, 0, len(manifests))
		found := false
		for _, manifest := range manifests {
			if manifest.ID != podID {
				newManifests = append(newManifests, manifest)
			} else {
				found = true
			}
		}
		if !found {
			// This really shouldn't happen, it indicates something is broken, and likely
			// there is a lost pod somewhere.
			// However it is "deleted" so log it and move on
			glog.Infof("Couldn't find: %s in %#v", podID, manifests)
		}
		return newManifests, nil
	})
}

func (registry *EtcdRegistry) getPodForMachine(machine, podID string) (pod api.Pod, err error) {
	key := makePodKey(machine, podID)
	err = registry.helper().ExtractObj(key, &pod, false)
	if err != nil {
		return
	}
	pod.CurrentState.Host = machine
	return
}

func (registry *EtcdRegistry) findPod(podID string) (api.Pod, string, error) {
	machines, err := registry.machines.List()
	if err != nil {
		return api.Pod{}, "", err
	}
	for _, machine := range machines {
		pod, err := registry.getPodForMachine(machine, podID)
		if err == nil {
			return pod, machine, nil
		}
	}
	return api.Pod{}, "", apiserver.NewNotFoundErr("pod", podID)
}

// ListControllers obtains a list of ReplicationControllers.
func (registry *EtcdRegistry) ListControllers() ([]api.ReplicationController, error) {
	var controllers []api.ReplicationController
	err := registry.helper().ExtractList("/registry/controllers", &controllers)
	return controllers, err
}

func makeControllerKey(id string) string {
	return "/registry/controllers/" + id
}

// GetController gets a specific ReplicationController specified by its ID.
func (registry *EtcdRegistry) GetController(controllerID string) (*api.ReplicationController, error) {
	var controller api.ReplicationController
	key := makeControllerKey(controllerID)
	err := registry.helper().ExtractObj(key, &controller, false)
	if tools.IsEtcdNotFound(err) {
		return nil, apiserver.NewNotFoundErr("replicationController", controllerID)
	}
	if err != nil {
		return nil, err
	}
	return &controller, nil
}

// CreateController creates a new ReplicationController.
func (registry *EtcdRegistry) CreateController(controller api.ReplicationController) error {
	// TODO : check for existence here and error.
	return registry.UpdateController(controller)
}

// UpdateController replaces an existing ReplicationController.
func (registry *EtcdRegistry) UpdateController(controller api.ReplicationController) error {
	return registry.helper().SetObj(makeControllerKey(controller.ID), controller)
}

// DeleteController deletes a ReplicationController specified by its ID.
func (registry *EtcdRegistry) DeleteController(controllerID string) error {
	key := makeControllerKey(controllerID)
	_, err := registry.etcdClient.Delete(key, false)
	if tools.IsEtcdNotFound(err) {
		return apiserver.NewNotFoundErr("replicationController", controllerID)
	}
	return err
}

func makeServiceKey(name string) string {
	return "/registry/services/specs/" + name
}

// ListServices obtains a list of Services.
func (registry *EtcdRegistry) ListServices() (api.ServiceList, error) {
	var list api.ServiceList
	err := registry.helper().ExtractList("/registry/services/specs", &list.Items)
	return list, err
}

// CreateService creates a new Service.
func (registry *EtcdRegistry) CreateService(svc api.Service) error {
	return registry.helper().SetObj(makeServiceKey(svc.ID), svc)
}

// GetService obtains a Service specified by its name.
func (registry *EtcdRegistry) GetService(name string) (*api.Service, error) {
	key := makeServiceKey(name)
	var svc api.Service
	err := registry.helper().ExtractObj(key, &svc, false)
	if tools.IsEtcdNotFound(err) {
		return nil, apiserver.NewNotFoundErr("service", name)
	}
	if err != nil {
		return nil, err
	}
	return &svc, nil
}

// DeleteService deletes a Service specified by its name.
func (registry *EtcdRegistry) DeleteService(name string) error {
	key := makeServiceKey(name)
	_, err := registry.etcdClient.Delete(key, true)
	if tools.IsEtcdNotFound(err) {
		return apiserver.NewNotFoundErr("service", name)
	}
	if err != nil {
		return err
	}
	key = "/registry/services/endpoints/" + name
	_, err = registry.etcdClient.Delete(key, true)
	return err
}

// UpdateService replaces an existing Service.
func (registry *EtcdRegistry) UpdateService(svc api.Service) error {
	// TODO : check for existence here and error.
	return registry.CreateService(svc)
}

// UpdateEndpoints update Endpoints of a Service.
func (registry *EtcdRegistry) UpdateEndpoints(e api.Endpoints) error {
	return registry.helper().SetObj("/registry/services/endpoints/"+e.Name, e)
}
<commit_msg>Cleanly delete service without endpoints key<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package registry

import (
	"fmt"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
	"github.com/golang/glog"
)

// TODO: Need to add a reconciler loop that makes sure that things in pods are reflected into
//       kubelet (and vice versa)

// EtcdRegistry implements PodRegistry, ControllerRegistry and ServiceRegistry with backed by etcd.
type EtcdRegistry struct {
	etcdClient      tools.EtcdClient
	machines        MinionRegistry
	manifestFactory ManifestFactory
}

// MakeEtcdRegistry creates an etcd registry.
// 'client' is the connection to etcd
// 'machines' is the list of machines
// 'scheduler' is the scheduling algorithm to use.
func MakeEtcdRegistry(client tools.EtcdClient, machines MinionRegistry) *EtcdRegistry {
	registry := &EtcdRegistry{
		etcdClient: client,
		machines:   machines,
	}
	registry.manifestFactory = &BasicManifestFactory{
		serviceRegistry: registry,
	}
	return registry
}

func makePodKey(machine, podID string) string {
	return "/registry/hosts/" + machine + "/pods/" + podID
}

func (registry *EtcdRegistry) helper() *tools.EtcdHelper {
	return &tools.EtcdHelper{registry.etcdClient}
}

// ListPods obtains a list of pods that match selector.
func (registry *EtcdRegistry) ListPods(selector labels.Selector) ([]api.Pod, error) {
	pods := []api.Pod{}
	machines, err := registry.machines.List()
	if err != nil {
		return nil, err
	}
	for _, machine := range machines {
		var machinePods []api.Pod
		err := registry.helper().ExtractList("/registry/hosts/"+machine+"/pods", &machinePods)
		if err != nil {
			return pods, err
		}
		for _, pod := range machinePods {
			if selector.Matches(labels.Set(pod.Labels)) {
				pod.CurrentState.Host = machine
				pods = append(pods, pod)
			}
		}
	}
	return pods, nil
}

// GetPod gets a specific pod specified by its ID.
func (registry *EtcdRegistry) GetPod(podID string) (*api.Pod, error) {
	pod, _, err := registry.findPod(podID)
	return &pod, err
}

func makeContainerKey(machine string) string {
	return "/registry/hosts/" + machine + "/kubelet"
}

// CreatePod creates a pod based on a specification, schedule it onto a specific machine.
func (registry *EtcdRegistry) CreatePod(machineIn string, pod api.Pod) error {
	podOut, machine, err := registry.findPod(pod.ID)
	if err == nil {
		// TODO: this error message looks racy.
		return fmt.Errorf("a pod named %s already exists on %s (%#v)", pod.ID, machine, podOut)
	}
	return registry.runPod(pod, machineIn)
}

func (registry *EtcdRegistry) runPod(pod api.Pod, machine string) error {
	podKey := makePodKey(machine, pod.ID)
	err := registry.helper().SetObj(podKey, pod)

	manifest, err := registry.manifestFactory.MakeManifest(machine, pod)
	if err != nil {
		return err
	}

	contKey := makeContainerKey(machine)
	err = registry.helper().AtomicUpdate(contKey, &[]api.ContainerManifest{}, func(in interface{}) (interface{}, error) {
		manifests := *in.(*[]api.ContainerManifest)
		return append(manifests, manifest), nil
	})
	if err != nil {
		// Don't strand stuff.
		_, err2 := registry.etcdClient.Delete(podKey, false)
		if err2 != nil {
			glog.Errorf("Probably stranding a pod, couldn't delete %v: %#v", podKey, err2)
		}
	}
	return err
}

func (registry *EtcdRegistry) UpdatePod(pod api.Pod) error {
	return fmt.Errorf("unimplemented!")
}

// DeletePod deletes an existing pod specified by its ID.
func (registry *EtcdRegistry) DeletePod(podID string) error {
	_, machine, err := registry.findPod(podID)
	if err != nil {
		return err
	}
	return registry.deletePodFromMachine(machine, podID)
}

func (registry *EtcdRegistry) deletePodFromMachine(machine, podID string) error {
	// First delete the pod, so a scheduler doesn't notice it getting removed from the
	// machine and attempt to put it somewhere.
	podKey := makePodKey(machine, podID)
	_, err := registry.etcdClient.Delete(podKey, true)
	if tools.IsEtcdNotFound(err) {
		return apiserver.NewNotFoundErr("pod", podID)
	}
	if err != nil {
		return err
	}

	// Next, remove the pod from the machine atomically.
	contKey := makeContainerKey(machine)
	return registry.helper().AtomicUpdate(contKey, &[]api.ContainerManifest{}, func(in interface{}) (interface{}, error) {
		manifests := *in.(*[]api.ContainerManifest)
		newManifests := make([]api.ContainerManifest, 0, len(manifests))
		found := false
		for _, manifest := range manifests {
			if manifest.ID != podID {
				newManifests = append(newManifests, manifest)
			} else {
				found = true
			}
		}
		if !found {
			// This really shouldn't happen, it indicates something is broken, and likely
			// there is a lost pod somewhere.
			// However it is "deleted" so log it and move on
			glog.Infof("Couldn't find: %s in %#v", podID, manifests)
		}
		return newManifests, nil
	})
}

func (registry *EtcdRegistry) getPodForMachine(machine, podID string) (pod api.Pod, err error) {
	key := makePodKey(machine, podID)
	err = registry.helper().ExtractObj(key, &pod, false)
	if err != nil {
		return
	}
	pod.CurrentState.Host = machine
	return
}

func (registry *EtcdRegistry) findPod(podID string) (api.Pod, string, error) {
	machines, err := registry.machines.List()
	if err != nil {
		return api.Pod{}, "", err
	}
	for _, machine := range machines {
		pod, err := registry.getPodForMachine(machine, podID)
		if err == nil {
			return pod, machine, nil
		}
	}
	return api.Pod{}, "", apiserver.NewNotFoundErr("pod", podID)
}

// ListControllers obtains a list of ReplicationControllers.
func (registry *EtcdRegistry) ListControllers() ([]api.ReplicationController, error) {
	var controllers []api.ReplicationController
	err := registry.helper().ExtractList("/registry/controllers", &controllers)
	return controllers, err
}

func makeControllerKey(id string) string {
	return "/registry/controllers/" + id
}

// GetController gets a specific ReplicationController specified by its ID.
func (registry *EtcdRegistry) GetController(controllerID string) (*api.ReplicationController, error) {
	var controller api.ReplicationController
	key := makeControllerKey(controllerID)
	err := registry.helper().ExtractObj(key, &controller, false)
	if tools.IsEtcdNotFound(err) {
		return nil, apiserver.NewNotFoundErr("replicationController", controllerID)
	}
	if err != nil {
		return nil, err
	}
	return &controller, nil
}

// CreateController creates a new ReplicationController.
func (registry *EtcdRegistry) CreateController(controller api.ReplicationController) error {
	// TODO : check for existence here and error.
	return registry.UpdateController(controller)
}

// UpdateController replaces an existing ReplicationController.
func (registry *EtcdRegistry) UpdateController(controller api.ReplicationController) error {
	return registry.helper().SetObj(makeControllerKey(controller.ID), controller)
}

// DeleteController deletes a ReplicationController specified by its ID.
func (registry *EtcdRegistry) DeleteController(controllerID string) error {
	key := makeControllerKey(controllerID)
	_, err := registry.etcdClient.Delete(key, false)
	if tools.IsEtcdNotFound(err) {
		return apiserver.NewNotFoundErr("replicationController", controllerID)
	}
	return err
}

func makeServiceKey(name string) string {
	return "/registry/services/specs/" + name
}

// ListServices obtains a list of Services.
func (registry *EtcdRegistry) ListServices() (api.ServiceList, error) {
	var list api.ServiceList
	err := registry.helper().ExtractList("/registry/services/specs", &list.Items)
	return list, err
}

// CreateService creates a new Service.
func (registry *EtcdRegistry) CreateService(svc api.Service) error {
	return registry.helper().SetObj(makeServiceKey(svc.ID), svc)
}

// GetService obtains a Service specified by its name.
func (registry *EtcdRegistry) GetService(name string) (*api.Service, error) {
	key := makeServiceKey(name)
	var svc api.Service
	err := registry.helper().ExtractObj(key, &svc, false)
	if tools.IsEtcdNotFound(err) {
		return nil, apiserver.NewNotFoundErr("service", name)
	}
	if err != nil {
		return nil, err
	}
	return &svc, nil
}

func makeServiceEndpointsKey(name string) string {
	return "/registry/services/endpoints/" + name
}

// DeleteService deletes a Service specified by its name.
func (registry *EtcdRegistry) DeleteService(name string) error {
	key := makeServiceKey(name)
	_, err := registry.etcdClient.Delete(key, true)
	if tools.IsEtcdNotFound(err) {
		return apiserver.NewNotFoundErr("service", name)
	}
	if err != nil {
		return err
	}
	key = makeServiceEndpointsKey(name)
	_, err = registry.etcdClient.Delete(key, true)
	if !tools.IsEtcdNotFound(err) {
		return err
	}
	return nil
}

// UpdateService replaces an existing Service.
func (registry *EtcdRegistry) UpdateService(svc api.Service) error {
	// TODO : check for existence here and error.
	return registry.CreateService(svc)
}

// UpdateEndpoints update Endpoints of a Service.
func (registry *EtcdRegistry) UpdateEndpoints(e api.Endpoints) error {
	return registry.helper().SetObj("/registry/services/endpoints/"+e.Name, e)
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/docker/docker/api"
	"github.com/docker/docker/client"
	"github.com/docker/go-connections/tlsconfig"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type APIClient interface {
	client.CommonAPIClient
}

var (
	dockerAPIClientOnce sync.Once
	dockerAPIClient     APIClient
	dockerAPIClientErr  error
	procVerDetect       bool
	isWsl               bool
)

// NewAPIClient guesses the docker client to use based on current kubernetes context.
func NewAPIClient() (APIClient, error) {
	procVerDetect = false
	isWsl = false

	dockerAPIClientOnce.Do(func() {
		kubeContext, err := kubernetes.CurrentContext()
		if err != nil {
			dockerAPIClientErr = errors.Wrap(err, "getting current cluster context")
			return
		}

		dockerAPIClient, dockerAPIClientErr = newAPIClient(kubeContext)
	})

	return dockerAPIClient, dockerAPIClientErr
}

// newAPIClient guesses the docker client to use based on current kubernetes context.
func newAPIClient(kubeContext string) (APIClient, error) {
	if kubeContext == constants.DefaultMinikubeContext {
		return newMinikubeAPIClient()
	}
	return newEnvAPIClient()
}

// newEnvAPIClient returns a docker client based on the environment variables set.
// It will "negotiate" the highest possible API version supported by both the client
// and the server if there is a mismatch.
func newEnvAPIClient() (APIClient, error) {
	cli, err := client.NewClientWithOpts(client.FromEnv)
	if err != nil {
		return nil, fmt.Errorf("Error getting docker client: %s", err)
	}
	cli.NegotiateAPIVersion(context.Background())

	return cli, nil
}

// newMinikubeAPIClient returns a docker client using the environment variables
// provided by minikube.
func newMinikubeAPIClient() (APIClient, error) {
	env, err := getMinikubeDockerEnv()
	if err != nil {
		logrus.Warnf("Could not get minikube docker env, falling back to local docker daemon: %s", err)
		return newEnvAPIClient()
	}

	var httpclient *http.Client
	if dockerCertPath := env["DOCKER_CERT_PATH"]; dockerCertPath != "" {
		options := tlsconfig.Options{
			CAFile:             filepath.Join(dockerCertPath, "ca.pem"),
			CertFile:           filepath.Join(dockerCertPath, "cert.pem"),
			KeyFile:            filepath.Join(dockerCertPath, "key.pem"),
			InsecureSkipVerify: env["DOCKER_TLS_VERIFY"] == "",
		}
		tlsc, err := tlsconfig.Client(options)
		if err != nil {
			return nil, err
		}

		httpclient = &http.Client{
			Transport: &http.Transport{
				TLSClientConfig: tlsc,
			},
			CheckRedirect: client.CheckRedirect,
		}
	}

	host := env["DOCKER_HOST"]
	if host == "" {
		host = client.DefaultDockerHost
	}
	version := env["DOCKER_API_VERSION"]
	if version == "" {
		version = api.DefaultVersion
	}

	return client.NewClient(host, version, httpclient, nil)
}

func detectWsl() (bool, error) {
	if !procVerDetect {
		procVerDetect = true
		if _, err := os.Stat("/proc/version"); err == nil {
			b, err := ioutil.ReadFile("/proc/version")
			if err != nil {
				return false, errors.Wrap(err, "read /proc/version")
			}

			if bytes.Contains(b, []byte("Microsoft")) {
				isWsl = true
			}
		}
	}
	return isWsl, nil
}

func getMiniKubeFilename() (string, error) {
	const winMiniKubeEnv = "SKAFFOLD_WINDOWS_MINIKUBE"
	if found, _ := detectWsl(); found {
		filename, envExists := os.LookupEnv(winMiniKubeEnv)
		if !envExists {
			return "", fmt.Errorf("Unable to find minikube.exe. Please set %s environment variable", winMiniKubeEnv)
		}
		if _, err := os.Stat(filename); os.IsNotExist(err) {
			return "", fmt.Errorf("Unable to find minikube.exe. File not found %s", filename)
		}
		return filename, nil
	}
	return "minikube", nil
}

func getMinikubeDockerEnv() (map[string]string, error) {
	miniKubeFilename, err := getMiniKubeFilename()
	if err != nil {
		return nil, errors.Wrap(err, "getting minikube filename")
	}
	cmd := exec.Command(miniKubeFilename, "docker-env", "--shell", "none")
	out, err := util.RunCmdOut(cmd)
	if err != nil {
		return nil, errors.Wrap(err, "getting minikube env")
	}

	env := map[string]string{}
	for _, line := range strings.Split(string(out), "\n") {
		if line == "" {
			continue
		}
		kv := strings.Split(line, "=")
		if len(kv) != 2 {
			return nil, fmt.Errorf("Unable to parse minikube docker-env keyvalue: %s, line: %s, output: %s", kv, line, string(out))
		}
		env[kv[0]] = kv[1]
	}

	if found, _ := detectWsl(); found {
		cmd := exec.Command("wslpath", env["DOCKER_CERT_PATH"])
		out, err := util.RunCmdOut(cmd)
		if err == nil {
			env["DOCKER_CERT_PATH"] = strings.TrimRight(string(out), "\n")
		} else {
			return nil, fmt.Errorf("Can't run wslpath: %s", err)
		}
	}

	return env, nil
}
<commit_msg>use of LookPath. Env removed. Static vars removed.<commit_after>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/docker/docker/api"
	"github.com/docker/docker/client"
	"github.com/docker/go-connections/tlsconfig"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type APIClient interface {
	client.CommonAPIClient
}

var (
	dockerAPIClientOnce sync.Once
	dockerAPIClient     APIClient
	dockerAPIClientErr  error
)

// NewAPIClient guesses the docker client to use based on current kubernetes context.
func NewAPIClient() (APIClient, error) {
	dockerAPIClientOnce.Do(func() {
		kubeContext, err := kubernetes.CurrentContext()
		if err != nil {
			dockerAPIClientErr = errors.Wrap(err, "getting current cluster context")
			return
		}

		dockerAPIClient, dockerAPIClientErr = newAPIClient(kubeContext)
	})

	return dockerAPIClient, dockerAPIClientErr
}

// newAPIClient guesses the docker client to use based on current kubernetes context.
func newAPIClient(kubeContext string) (APIClient, error) {
	if kubeContext == constants.DefaultMinikubeContext {
		return newMinikubeAPIClient()
	}
	return newEnvAPIClient()
}

// newEnvAPIClient returns a docker client based on the environment variables set.
// It will "negotiate" the highest possible API version supported by both the client
// and the server if there is a mismatch.
func newEnvAPIClient() (APIClient, error) {
	cli, err := client.NewClientWithOpts(client.FromEnv)
	if err != nil {
		return nil, fmt.Errorf("Error getting docker client: %s", err)
	}
	cli.NegotiateAPIVersion(context.Background())

	return cli, nil
}

// newMinikubeAPIClient returns a docker client using the environment variables
// provided by minikube.
func newMinikubeAPIClient() (APIClient, error) {
	env, err := getMinikubeDockerEnv()
	if err != nil {
		logrus.Warnf("Could not get minikube docker env, falling back to local docker daemon: %s", err)
		return newEnvAPIClient()
	}

	var httpclient *http.Client
	if dockerCertPath := env["DOCKER_CERT_PATH"]; dockerCertPath != "" {
		options := tlsconfig.Options{
			CAFile:             filepath.Join(dockerCertPath, "ca.pem"),
			CertFile:           filepath.Join(dockerCertPath, "cert.pem"),
			KeyFile:            filepath.Join(dockerCertPath, "key.pem"),
			InsecureSkipVerify: env["DOCKER_TLS_VERIFY"] == "",
		}
		tlsc, err := tlsconfig.Client(options)
		if err != nil {
			return nil, err
		}

		httpclient = &http.Client{
			Transport: &http.Transport{
				TLSClientConfig: tlsc,
			},
			CheckRedirect: client.CheckRedirect,
		}
	}

	host := env["DOCKER_HOST"]
	if host == "" {
		host = client.DefaultDockerHost
	}
	version := env["DOCKER_API_VERSION"]
	if version == "" {
		version = api.DefaultVersion
	}

	return client.NewClient(host, version, httpclient, nil)
}

func detectWsl() (bool, error) {
	if _, err := os.Stat("/proc/version"); err == nil {
		b, err := ioutil.ReadFile("/proc/version")
		if err != nil {
			return false, errors.Wrap(err, "read /proc/version")
		}

		if bytes.Contains(b, []byte("Microsoft")) {
			return true, nil
		}
	}
	return false, nil
}

func getMiniKubeFilename() (string, error) {
	if found, _ := detectWsl(); found {
		filename, err := exec.LookPath("minikube.exe")
		if err != nil {
			return "", fmt.Errorf("Unable to find minikube.exe. Please add it to PATH environment variable")
		}
		if _, err := os.Stat(filename); os.IsNotExist(err) {
			return "", fmt.Errorf("Unable to find minikube.exe. File not found %s", filename)
		}
		return filename, nil
	}
	return "minikube", nil
}

func getMinikubeDockerEnv() (map[string]string, error) {
	miniKubeFilename, err := getMiniKubeFilename()
	if err != nil {
		return nil, errors.Wrap(err, "getting minikube filename")
	}
	cmd := exec.Command(miniKubeFilename, "docker-env", "--shell", "none")
	out, err := util.RunCmdOut(cmd)
	if err != nil {
		return nil, errors.Wrap(err, "getting minikube env")
	}

	env := map[string]string{}
	for _, line := range strings.Split(string(out), "\n") {
		if line == "" {
			continue
		}
		kv := strings.Split(line, "=")
		if len(kv) != 2 {
			return nil, fmt.Errorf("Unable to parse minikube docker-env keyvalue: %s, line: %s, output: %s", kv, line, string(out))
		}
		env[kv[0]] = kv[1]
	}

	if found, _ := detectWsl(); found {
		cmd := exec.Command("wslpath", env["DOCKER_CERT_PATH"])
		out, err := util.RunCmdOut(cmd)
		if err == nil {
			env["DOCKER_CERT_PATH"] = strings.TrimRight(string(out), "\n")
		} else {
			return nil, fmt.Errorf("Can't run wslpath: %s", err)
		}
	}

	return env, nil
}
<|endoftext|>