<commit_before>package repofs

import (
	"encoding/binary"
	"errors"
	"fmt"
	"hash/fnv"
	"io"
	"log"
	"sync/atomic"

	"github.com/kopia/kopia/fs"
	"github.com/kopia/kopia/repo"
)

const (
	maxDirReadAheadCount = 256
	maxBundleFileSize    = 65536
)

// ErrUploadCancelled is returned when the upload gets cancelled.
var ErrUploadCancelled = errors.New("upload cancelled")

func hashEntryMetadata(w io.Writer, e *fs.EntryMetadata) {
	binary.Write(w, binary.LittleEndian, e.Name)
	binary.Write(w, binary.LittleEndian, e.ModTime.UnixNano())
	binary.Write(w, binary.LittleEndian, e.FileMode())
	binary.Write(w, binary.LittleEndian, e.FileSize)
	binary.Write(w, binary.LittleEndian, e.UserID)
	binary.Write(w, binary.LittleEndian, e.GroupID)
}

func metadataHash(e *fs.EntryMetadata) uint64 {
	h := fnv.New64a()
	hashEntryMetadata(h, e)
	return h.Sum64()
}

func bundleHash(b *bundle) uint64 {
	h := fnv.New64a()
	hashEntryMetadata(h, b.metadata)
	for i, f := range b.files {
		binary.Write(h, binary.LittleEndian, i)
		hashEntryMetadata(h, f.Metadata())
	}
	return h.Sum64()
}

// UploadResult stores results of an upload.
type UploadResult struct {
	ObjectID   repo.ObjectID
	ManifestID repo.ObjectID
	Cancelled  bool

	Stats struct {
		CachedDirectories    int
		CachedFiles          int
		NonCachedDirectories int
		NonCachedFiles       int
	}
}

// Uploader supports efficient uploading files and directories to repository.
type Uploader interface {
	UploadDir(dir fs.Directory, previousManifestID *repo.ObjectID) (*UploadResult, error)
	UploadFile(file fs.File) (*UploadResult, error)
	Cancel()
}

type uploader struct {
	repo           *repo.Repository
	enableBundling bool

	cancelled int32
}

func (u *uploader) isCancelled() bool {
	return atomic.LoadInt32(&u.cancelled) != 0
}

func (u *uploader) uploadFileInternal(f fs.File, relativePath string, forceStored bool) (*dirEntry, uint64, error) {
	log.Printf("Uploading file %v", relativePath)
	file, err := f.Open()
	if err != nil {
		return nil, 0, fmt.Errorf("unable to open file: %v", err)
	}
	defer file.Close()

	writer := u.repo.NewWriter(
		repo.WithDescription("FILE:" + f.Metadata().Name),
	)
	defer writer.Close()

	written, err := io.Copy(writer, file)
	if err != nil {
		return nil, 0, err
	}

	e2, err := file.EntryMetadata()
	if err != nil {
		return nil, 0, err
	}

	r, err := writer.Result(forceStored)
	if err != nil {
		return nil, 0, err
	}

	de := newDirEntry(e2, r)
	de.FileSize = written

	return de, metadataHash(&de.EntryMetadata), nil
}

func newDirEntry(md *fs.EntryMetadata, oid repo.ObjectID) *dirEntry {
	return &dirEntry{
		EntryMetadata: *md,
		ObjectID:      oid,
	}
}

func (u *uploader) uploadBundleInternal(b *bundle) (*dirEntry, uint64, error) {
	bundleMetadata := b.Metadata()

	log.Printf("uploading bundle %v (%v files)", bundleMetadata.Name, len(b.files))
	defer log.Printf("finished uploading bundle")

	writer := u.repo.NewWriter(
		repo.WithDescription("BUNDLE:" + bundleMetadata.Name),
	)
	defer writer.Close()

	var uploadedFiles []fs.File
	var err error

	de := newDirEntry(bundleMetadata, repo.NullObjectID)

	for _, fileEntry := range b.files {
		file, err := fileEntry.Open()
		if err != nil {
			return nil, 0, err
		}

		fileMetadata, err := file.EntryMetadata()
		if err != nil {
			return nil, 0, err
		}

		written, err := io.Copy(writer, file)
		if err != nil {
			return nil, 0, err
		}

		fileMetadata.FileSize = written
		de.BundledChildren = append(de.BundledChildren, newDirEntry(fileMetadata, repo.NullObjectID))

		uploadedFiles = append(uploadedFiles, &bundledFile{metadata: fileMetadata})
		file.Close()
	}

	b.files = uploadedFiles
	de.ObjectID, err = writer.Result(true)
	if err != nil {
		return nil, 0, err
	}

	return de, bundleHash(b), nil
}

func (u *uploader) UploadFile(file fs.File) (*UploadResult, error) {
	result := &UploadResult{}
	e, _, err := u.uploadFileInternal(file, file.Metadata().Name, true)
	result.ObjectID = e.ObjectID
	return result, err
}

func (u *uploader) UploadDir(dir fs.Directory, previousManifestID *repo.ObjectID) (*UploadResult, error) {
	var mr hashcacheReader
	var err error

	if previousManifestID != nil {
		if r, err := u.repo.Open(*previousManifestID); err == nil {
			mr.open(r)
		}
	}

	mw := u.repo.NewWriter(
		repo.WithDescription("HASHCACHE:"+dir.Metadata().Name),
		repo.WithBlockNamePrefix("H"),
	)
	defer mw.Close()
	hcw := newHashCacheWriter(mw)

	result := &UploadResult{}
	result.ObjectID, _, _, err = u.uploadDirInternal(result, dir, ".", hcw, &mr, true)
	if err != nil {
		return result, err
	}

	hcw.Finalize()

	result.ManifestID, err = mw.Result(true)
	return result, nil
}

func (u *uploader) uploadDirInternal(
	result *UploadResult,
	dir fs.Directory,
	relativePath string,
	hcw *hashcacheWriter,
	mr *hashcacheReader,
	forceStored bool,
) (repo.ObjectID, uint64, bool, error) {
	log.Printf("Uploading dir %v", relativePath)
	defer log.Printf("Finished uploading dir %v", relativePath)

	entries, err := dir.Readdir()
	if err != nil {
		return repo.NullObjectID, 0, false, err
	}

	entries = u.bundleEntries(entries)

	writer := u.repo.NewWriter(
		repo.WithDescription("DIR:" + relativePath),
	)

	dw := newDirWriter(writer)
	defer writer.Close()

	allCached := true

	dirHasher := fnv.New64a()
	dirHasher.Write([]byte(relativePath))
	dirHasher.Write([]byte{0})

	for _, entry := range entries {
		e := entry.Metadata()
		entryRelativePath := relativePath + "/" + e.Name

		var de *dirEntry

		var hash uint64

		switch entry := entry.(type) {
		case fs.Directory:
			oid, h, wasCached, err := u.uploadDirInternal(result, entry, entryRelativePath, hcw, mr, false)
			if err != nil {
				return repo.NullObjectID, 0, false, err
			}
			//log.Printf("dirHash: %v %v", fullPath, h)
			hash = h
			allCached = allCached && wasCached
			de = newDirEntry(e, oid)

		case fs.Symlink:
			l, err := entry.Readlink()
			if err != nil {
				return repo.NullObjectID, 0, false, err
			}

			de = newDirEntry(e, repo.InlineObjectID([]byte(l)))
			hash = metadataHash(e)

		case *bundle:
			// See if we had this name during previous pass.
			cachedEntry := mr.findEntry(entryRelativePath)

			// ... and whether file metadata is identical to the previous one.
			cacheMatches := (cachedEntry != nil) && cachedEntry.Hash == bundleHash(entry)

			allCached = allCached && cacheMatches
			childrenMetadata := make([]*dirEntry, len(entry.files))
			for i, f := range entry.files {
				childrenMetadata[i] = newDirEntry(f.Metadata(), repo.NullObjectID)
			}

			if cacheMatches {
				result.Stats.CachedFiles++
				// Avoid hashing by reusing previous object ID.
				de = newDirEntry(e, cachedEntry.ObjectID)
				de.BundledChildren = childrenMetadata
				hash = cachedEntry.Hash
			} else {
				result.Stats.NonCachedFiles++
				de, hash, err = u.uploadBundleInternal(entry)
				if err != nil {
					return repo.NullObjectID, 0, false, fmt.Errorf("unable to hash file: %s", err)
				}
			}

		case fs.File:
			// regular file
			// See if we had this name during previous pass.
			cachedEntry := mr.findEntry(entryRelativePath)

			// ... and whether file metadata is identical to the previous one.
			computedHash := metadataHash(e)
			cacheMatches := (cachedEntry != nil) && cachedEntry.Hash == computedHash

			allCached = allCached && cacheMatches

			if cacheMatches {
				result.Stats.CachedFiles++
				// Avoid hashing by reusing previous object ID.
				de = newDirEntry(e, cachedEntry.ObjectID)
				hash = cachedEntry.Hash
			} else {
				result.Stats.NonCachedFiles++
				de, hash, err = u.uploadFileInternal(entry, entryRelativePath, false)
				if err != nil {
					return repo.NullObjectID, 0, false, fmt.Errorf("unable to hash file: %s", err)
				}
			}

		default:
			return repo.NullObjectID, 0, false, fmt.Errorf("file type %v not supported", entry.Metadata().Type)
		}

		if hash != 0 {
			dirHasher.Write([]byte(de.Name))
			dirHasher.Write([]byte{0})
			binary.Write(dirHasher, binary.LittleEndian, hash)
		}

		if err := dw.WriteEntry(de); err != nil {
			return repo.NullObjectID, 0, false, err
		}

		if de.Type != fs.EntryTypeDirectory && de.ObjectID.StorageBlock != "" {
			if err := hcw.WriteEntry(hashCacheEntry{
				Name:     entryRelativePath,
				Hash:     hash,
				ObjectID: de.ObjectID,
			}); err != nil {
				return repo.NullObjectID, 0, false, err
			}
		}
	}

	dw.Finalize()

	var directoryOID repo.ObjectID
	dirHash := dirHasher.Sum64()

	cacheddirEntry := mr.findEntry(relativePath + "/")
	allCached = allCached && cacheddirEntry != nil && cacheddirEntry.Hash == dirHash

	if allCached {
		// Avoid hashing directory listing if every entry matched the cache.
		result.Stats.CachedDirectories++
		directoryOID = cacheddirEntry.ObjectID
	} else {
		result.Stats.NonCachedDirectories++
		directoryOID, err = writer.Result(forceStored)
		if err != nil {
			return directoryOID, 0, false, err
		}
	}

	if err := hcw.WriteEntry(hashCacheEntry{
		Name:     relativePath + "/",
		ObjectID: directoryOID,
		Hash:     dirHash,
	}); err != nil {
		return repo.NullObjectID, 0, false, err
	}

	// log.Printf("Dir: %v %v %v %v", relativePath, directoryOID.UIString(), dirHash, allCached)
	return directoryOID, dirHash, allCached, nil
}

func (u *uploader) bundleEntries(entries fs.Entries) fs.Entries {
	var bundleMap map[int]*bundle

	result := entries[:0]

	for _, e := range entries {
		switch e := e.(type) {
		case fs.File:
			md := e.Metadata()
			bundleNo := u.getBundleNumber(md)
			if bundleNo != 0 {
				// See if we already started this bundle.
				b := bundleMap[bundleNo]
				if b == nil {
					if bundleMap == nil {
						bundleMap = make(map[int]*bundle)
					}

					bundleMetadata := &fs.EntryMetadata{
						Name: fmt.Sprintf("bundle-%v", bundleNo),
						Type: entryTypeBundle,
					}

					b = &bundle{
						metadata: bundleMetadata,
					}
					bundleMap[bundleNo] = b

					// Add the bundle instead of an entry.
					result = append(result, b)
				}

				// Append entry to the bundle.
				b.append(e)

			} else {
				// Append original entry
				result = append(result, e)
			}

		default:
			// Append original entry
			result = append(result, e)
		}
	}

	return result
}

func (u *uploader) getBundleNumber(md *fs.EntryMetadata) int {
	if u.enableBundling {
		if md.FileMode().IsRegular() && md.FileSize < maxBundleFileSize {
			return md.ModTime.Year()*100 + int(md.ModTime.Month())
		}
	}

	return 0
}

func (u *uploader) Cancel() {
	atomic.StoreInt32(&u.cancelled, 1)
}

// UploadOption modifies the behavior of uploader.
type UploadOption func(u *uploader)

// EnableBundling allows uploader to create bundle objects.
func EnableBundling() UploadOption {
	return func(u *uploader) {
		u.enableBundling = true
	}
}

// NewUploader creates new Uploader object for the specified Repository
func NewUploader(repo *repo.Repository, options ...UploadOption) Uploader {
	u := &uploader{
		repo: repo,
	}

	for _, o := range options {
		o(u)
	}

	return u
}
<commit_msg>do not store inline directory ObjectIDs in hash cache<commit_after>package repofs

import (
	"encoding/binary"
	"errors"
	"fmt"
	"hash/fnv"
	"io"
	"log"
	"sync/atomic"

	"github.com/kopia/kopia/fs"
	"github.com/kopia/kopia/repo"
)

const (
	maxDirReadAheadCount = 256
	maxBundleFileSize    = 65536
)

// ErrUploadCancelled is returned when the upload gets cancelled.
var ErrUploadCancelled = errors.New("upload cancelled")

func hashEntryMetadata(w io.Writer, e *fs.EntryMetadata) {
	binary.Write(w, binary.LittleEndian, e.Name)
	binary.Write(w, binary.LittleEndian, e.ModTime.UnixNano())
	binary.Write(w, binary.LittleEndian, e.FileMode())
	binary.Write(w, binary.LittleEndian, e.FileSize)
	binary.Write(w, binary.LittleEndian, e.UserID)
	binary.Write(w, binary.LittleEndian, e.GroupID)
}

func metadataHash(e *fs.EntryMetadata) uint64 {
	h := fnv.New64a()
	hashEntryMetadata(h, e)
	return h.Sum64()
}

func bundleHash(b *bundle) uint64 {
	h := fnv.New64a()
	hashEntryMetadata(h, b.metadata)
	for i, f := range b.files {
		binary.Write(h, binary.LittleEndian, i)
		hashEntryMetadata(h, f.Metadata())
	}
	return h.Sum64()
}

// UploadResult stores results of an upload.
type UploadResult struct {
	ObjectID   repo.ObjectID
	ManifestID repo.ObjectID
	Cancelled  bool

	Stats struct {
		CachedDirectories    int
		CachedFiles          int
		NonCachedDirectories int
		NonCachedFiles       int
	}
}

// Uploader supports efficient uploading files and directories to repository.
type Uploader interface {
	UploadDir(dir fs.Directory, previousManifestID *repo.ObjectID) (*UploadResult, error)
	UploadFile(file fs.File) (*UploadResult, error)
	Cancel()
}

type uploader struct {
	repo           *repo.Repository
	enableBundling bool

	cancelled int32
}

func (u *uploader) isCancelled() bool {
	return atomic.LoadInt32(&u.cancelled) != 0
}

func (u *uploader) uploadFileInternal(f fs.File, relativePath string, forceStored bool) (*dirEntry, uint64, error) {
	log.Printf("Uploading file %v", relativePath)
	file, err := f.Open()
	if err != nil {
		return nil, 0, fmt.Errorf("unable to open file: %v", err)
	}
	defer file.Close()

	writer := u.repo.NewWriter(
		repo.WithDescription("FILE:" + f.Metadata().Name),
	)
	defer writer.Close()

	written, err := io.Copy(writer, file)
	if err != nil {
		return nil, 0, err
	}

	e2, err := file.EntryMetadata()
	if err != nil {
		return nil, 0, err
	}

	r, err := writer.Result(forceStored)
	if err != nil {
		return nil, 0, err
	}

	de := newDirEntry(e2, r)
	de.FileSize = written

	return de, metadataHash(&de.EntryMetadata), nil
}

func newDirEntry(md *fs.EntryMetadata, oid repo.ObjectID) *dirEntry {
	return &dirEntry{
		EntryMetadata: *md,
		ObjectID:      oid,
	}
}

func (u *uploader) uploadBundleInternal(b *bundle) (*dirEntry, uint64, error) {
	bundleMetadata := b.Metadata()

	log.Printf("uploading bundle %v (%v files)", bundleMetadata.Name, len(b.files))
	defer log.Printf("finished uploading bundle")

	writer := u.repo.NewWriter(
		repo.WithDescription("BUNDLE:" + bundleMetadata.Name),
	)
	defer writer.Close()

	var uploadedFiles []fs.File
	var err error

	de := newDirEntry(bundleMetadata, repo.NullObjectID)

	for _, fileEntry := range b.files {
		file, err := fileEntry.Open()
		if err != nil {
			return nil, 0, err
		}

		fileMetadata, err := file.EntryMetadata()
		if err != nil {
			return nil, 0, err
		}

		written, err := io.Copy(writer, file)
		if err != nil {
			return nil, 0, err
		}

		fileMetadata.FileSize = written
		de.BundledChildren = append(de.BundledChildren, newDirEntry(fileMetadata, repo.NullObjectID))

		uploadedFiles = append(uploadedFiles, &bundledFile{metadata: fileMetadata})
		file.Close()
	}

	b.files = uploadedFiles
	de.ObjectID, err = writer.Result(true)
	if err != nil {
		return nil, 0, err
	}

	return de, bundleHash(b), nil
}

func (u *uploader) UploadFile(file fs.File) (*UploadResult, error) {
	result := &UploadResult{}
	e, _, err := u.uploadFileInternal(file, file.Metadata().Name, true)
	result.ObjectID = e.ObjectID
	return result, err
}

func (u *uploader) UploadDir(dir fs.Directory, previousManifestID *repo.ObjectID) (*UploadResult, error) {
	var mr hashcacheReader
	var err error

	if previousManifestID != nil {
		if r, err := u.repo.Open(*previousManifestID); err == nil {
			mr.open(r)
		}
	}

	mw := u.repo.NewWriter(
		repo.WithDescription("HASHCACHE:"+dir.Metadata().Name),
		repo.WithBlockNamePrefix("H"),
	)
	defer mw.Close()
	hcw := newHashCacheWriter(mw)

	result := &UploadResult{}
	result.ObjectID, _, _, err = u.uploadDirInternal(result, dir, ".", hcw, &mr, true)
	if err != nil {
		return result, err
	}

	hcw.Finalize()

	result.ManifestID, err = mw.Result(true)
	return result, nil
}

func (u *uploader) uploadDirInternal(
	result *UploadResult,
	dir fs.Directory,
	relativePath string,
	hcw *hashcacheWriter,
	mr *hashcacheReader,
	forceStored bool,
) (repo.ObjectID, uint64, bool, error) {
	log.Printf("Uploading dir %v", relativePath)
	defer log.Printf("Finished uploading dir %v", relativePath)

	entries, err := dir.Readdir()
	if err != nil {
		return repo.NullObjectID, 0, false, err
	}

	entries = u.bundleEntries(entries)

	writer := u.repo.NewWriter(
		repo.WithDescription("DIR:" + relativePath),
	)

	dw := newDirWriter(writer)
	defer writer.Close()

	allCached := true

	dirHasher := fnv.New64a()
	dirHasher.Write([]byte(relativePath))
	dirHasher.Write([]byte{0})

	for _, entry := range entries {
		e := entry.Metadata()
		entryRelativePath := relativePath + "/" + e.Name

		var de *dirEntry

		var hash uint64

		switch entry := entry.(type) {
		case fs.Directory:
			oid, h, wasCached, err := u.uploadDirInternal(result, entry, entryRelativePath, hcw, mr, false)
			if err != nil {
				return repo.NullObjectID, 0, false, err
			}
			//log.Printf("dirHash: %v %v", fullPath, h)
			hash = h
			allCached = allCached && wasCached
			de = newDirEntry(e, oid)

		case fs.Symlink:
			l, err := entry.Readlink()
			if err != nil {
				return repo.NullObjectID, 0, false, err
			}

			de = newDirEntry(e, repo.InlineObjectID([]byte(l)))
			hash = metadataHash(e)

		case *bundle:
			// See if we had this name during previous pass.
			cachedEntry := mr.findEntry(entryRelativePath)

			// ... and whether file metadata is identical to the previous one.
			cacheMatches := (cachedEntry != nil) && cachedEntry.Hash == bundleHash(entry)

			allCached = allCached && cacheMatches
			childrenMetadata := make([]*dirEntry, len(entry.files))
			for i, f := range entry.files {
				childrenMetadata[i] = newDirEntry(f.Metadata(), repo.NullObjectID)
			}

			if cacheMatches {
				result.Stats.CachedFiles++
				// Avoid hashing by reusing previous object ID.
				de = newDirEntry(e, cachedEntry.ObjectID)
				de.BundledChildren = childrenMetadata
				hash = cachedEntry.Hash
			} else {
				result.Stats.NonCachedFiles++
				de, hash, err = u.uploadBundleInternal(entry)
				if err != nil {
					return repo.NullObjectID, 0, false, fmt.Errorf("unable to hash file: %s", err)
				}
			}

		case fs.File:
			// regular file
			// See if we had this name during previous pass.
			cachedEntry := mr.findEntry(entryRelativePath)

			// ... and whether file metadata is identical to the previous one.
			computedHash := metadataHash(e)
			cacheMatches := (cachedEntry != nil) && cachedEntry.Hash == computedHash

			allCached = allCached && cacheMatches

			if cacheMatches {
				result.Stats.CachedFiles++
				// Avoid hashing by reusing previous object ID.
				de = newDirEntry(e, cachedEntry.ObjectID)
				hash = cachedEntry.Hash
			} else {
				result.Stats.NonCachedFiles++
				de, hash, err = u.uploadFileInternal(entry, entryRelativePath, false)
				if err != nil {
					return repo.NullObjectID, 0, false, fmt.Errorf("unable to hash file: %s", err)
				}
			}

		default:
			return repo.NullObjectID, 0, false, fmt.Errorf("file type %v not supported", entry.Metadata().Type)
		}

		if hash != 0 {
			dirHasher.Write([]byte(de.Name))
			dirHasher.Write([]byte{0})
			binary.Write(dirHasher, binary.LittleEndian, hash)
		}

		if err := dw.WriteEntry(de); err != nil {
			return repo.NullObjectID, 0, false, err
		}

		if de.Type != fs.EntryTypeDirectory && de.ObjectID.StorageBlock != "" {
			if err := hcw.WriteEntry(hashCacheEntry{
				Name:     entryRelativePath,
				Hash:     hash,
				ObjectID: de.ObjectID,
			}); err != nil {
				return repo.NullObjectID, 0, false, err
			}
		}
	}

	dw.Finalize()

	var directoryOID repo.ObjectID
	dirHash := dirHasher.Sum64()

	cacheddirEntry := mr.findEntry(relativePath + "/")
	allCached = allCached && cacheddirEntry != nil && cacheddirEntry.Hash == dirHash

	if allCached {
		// Avoid hashing directory listing if every entry matched the cache.
		result.Stats.CachedDirectories++
		directoryOID = cacheddirEntry.ObjectID
	} else {
		result.Stats.NonCachedDirectories++
		directoryOID, err = writer.Result(forceStored)
		if err != nil {
			return directoryOID, 0, false, err
		}
	}

	if directoryOID.StorageBlock != "" {
		if err := hcw.WriteEntry(hashCacheEntry{
			Name:     relativePath + "/",
			ObjectID: directoryOID,
			Hash:     dirHash,
		}); err != nil {
			return repo.NullObjectID, 0, false, err
		}
	}

	// log.Printf("Dir: %v %v %v %v", relativePath, directoryOID.UIString(), dirHash, allCached)
	return directoryOID, dirHash, allCached, nil
}

func (u *uploader) bundleEntries(entries fs.Entries) fs.Entries {
	var bundleMap map[int]*bundle

	result := entries[:0]

	for _, e := range entries {
		switch e := e.(type) {
		case fs.File:
			md := e.Metadata()
			bundleNo := u.getBundleNumber(md)
			if bundleNo != 0 {
				// See if we already started this bundle.
				b := bundleMap[bundleNo]
				if b == nil {
					if bundleMap == nil {
						bundleMap = make(map[int]*bundle)
					}

					bundleMetadata := &fs.EntryMetadata{
						Name: fmt.Sprintf("bundle-%v", bundleNo),
						Type: entryTypeBundle,
					}

					b = &bundle{
						metadata: bundleMetadata,
					}
					bundleMap[bundleNo] = b

					// Add the bundle instead of an entry.
					result = append(result, b)
				}

				// Append entry to the bundle.
				b.append(e)

			} else {
				// Append original entry
				result = append(result, e)
			}

		default:
			// Append original entry
			result = append(result, e)
		}
	}

	return result
}

func (u *uploader) getBundleNumber(md *fs.EntryMetadata) int {
	if u.enableBundling {
		if md.FileMode().IsRegular() && md.FileSize < maxBundleFileSize {
			return md.ModTime.Year()*100 + int(md.ModTime.Month())
		}
	}

	return 0
}

func (u *uploader) Cancel() {
	atomic.StoreInt32(&u.cancelled, 1)
}

// UploadOption modifies the behavior of uploader.
type UploadOption func(u *uploader)

// EnableBundling allows uploader to create bundle objects.
func EnableBundling() UploadOption {
	return func(u *uploader) {
		u.enableBundling = true
	}
}

// NewUploader creates new Uploader object for the specified Repository
func NewUploader(repo *repo.Repository, options ...UploadOption) Uploader {
	u := &uploader{
		repo: repo,
	}

	for _, o := range options {
		o(u)
	}

	return u
}
<|endoftext|>


<commit_before>package docker

import (
	"fmt"
	"github.com/dotcloud/docker/utils"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"testing"
	"time"
)

const (
	unitTestImageName = "docker-unit-tests"
	unitTestImageID   = "e9aa60c60128cad1"
	unitTestStoreBase = "/var/lib/docker/unit-tests"
	testDaemonAddr    = "127.0.0.1:4270"
	testDaemonProto   = "tcp"
)

var globalRuntime *Runtime

func nuke(runtime *Runtime) error {
	var wg sync.WaitGroup
	for _, container := range runtime.List() {
		wg.Add(1)
		go func(c *Container) {
			c.Kill()
			wg.Done()
		}(container)
	}
	wg.Wait()
	return os.RemoveAll(runtime.root)
}

func cleanup(runtime *Runtime) error {
	for _, container := range runtime.List() {
		container.Kill()
		runtime.Destroy(container)
	}
	images, err := runtime.graph.All()
	if err != nil {
		return err
	}
	for _, image := range images {
		if image.ID != unitTestImageID {
			runtime.graph.Delete(image.ID)
		}
	}
	return nil
}

func layerArchive(tarfile string) (io.Reader, error) {
	// FIXME: need to close f somewhere
	f, err := os.Open(tarfile)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func init() {
	// Hack to run sys init during unit testing
	if utils.SelfPath() == "/sbin/init" {
		SysInit()
		return
	}

	if uid := syscall.Geteuid(); uid != 0 {
		log.Fatal("docker tests need to be run as root")
	}

	NetworkBridgeIface = "testdockbr0"

	// Make it our Store root
	runtime, err := NewRuntimeFromDirectory(unitTestStoreBase, false)
	if err != nil {
		panic(err)
	}
	globalRuntime = runtime

	// Create the "Server"
	srv := &Server{
		runtime:     runtime,
		enableCors:  false,
		pullingPool: make(map[string]struct{}),
		pushingPool: make(map[string]struct{}),
	}
	// Retrieve the Image
	if err := srv.ImagePull(unitTestImageName, "", "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil {
		panic(err)
	}
	// Spawn a Daemon
	go func() {
		if err := ListenAndServe(testDaemonProto, testDaemonAddr, srv, os.Getenv("DEBUG") != ""); err != nil {
			panic(err)
		}
	}()

	// Give some time to ListenAndServer to actually start
	time.Sleep(time.Second)
}

// FIXME: test that ImagePull(json=true) send correct json output

func newTestRuntime() (*Runtime, error) {
	root, err := ioutil.TempDir("", "docker-test")
	if err != nil {
		return nil, err
	}
	if err := os.Remove(root); err != nil {
		return nil, err
	}
	if err := utils.CopyDirectory(unitTestStoreBase, root); err != nil {
		return nil, err
	}

	runtime, err := NewRuntimeFromDirectory(root, false)
	if err != nil {
		return nil, err
	}
	runtime.UpdateCapabilities(true)
	return runtime, nil
}

func GetTestImage(runtime *Runtime) *Image {
	imgs, err := runtime.graph.All()
	if err != nil {
		panic(err)
	}
	for i := range imgs {
		if imgs[i].ID == unitTestImageID {
			return imgs[i]
		}
	}
	panic(fmt.Errorf("Test image %v not found", unitTestImageID))
}

func TestRuntimeCreate(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	// Make sure we start we 0 containers
	if len(runtime.List()) != 0 {
		t.Errorf("Expected 0 containers, %v found", len(runtime.List()))
	}

	builder := NewBuilder(runtime)

	container, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := runtime.Destroy(container); err != nil {
			t.Error(err)
		}
	}()

	// Make sure we can find the newly created container with List()
	if len(runtime.List()) != 1 {
		t.Errorf("Expected 1 container, %v found", len(runtime.List()))
	}

	// Make sure the container List() returns is the right one
	if runtime.List()[0].ID != container.ID {
		t.Errorf("Unexpected container %v returned by List", runtime.List()[0])
	}

	// Make sure we can get the container with Get()
	if runtime.Get(container.ID) == nil {
		t.Errorf("Unable to get newly created container")
	}

	// Make sure it is the right container
	if runtime.Get(container.ID) != container {
		t.Errorf("Get() returned the wrong container")
	}

	// Make sure Exists returns it as existing
	if !runtime.Exists(container.ID) {
		t.Errorf("Exists() returned false for a newly created container")
	}

	// Make sure crete with bad parameters returns an error
	_, err = builder.Create(
		&Config{
			Image: GetTestImage(runtime).ID,
		},
	)
	if err == nil {
		t.Fatal("Builder.Create should throw an error when Cmd is missing")
	}

	_, err = builder.Create(
		&Config{
			Image: GetTestImage(runtime).ID,
			Cmd:   []string{},
		},
	)
	if err == nil {
		t.Fatal("Builder.Create should throw an error when Cmd is empty")
	}
}

func TestDestroy(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)
	container, err := NewBuilder(runtime).Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	// Destroy
	if err := runtime.Destroy(container); err != nil {
		t.Error(err)
	}

	// Make sure runtime.Exists() behaves correctly
	if runtime.Exists("test_destroy") {
		t.Errorf("Exists() returned true")
	}

	// Make sure runtime.List() doesn't list the destroyed container
	if len(runtime.List()) != 0 {
		t.Errorf("Expected 0 container, %v found", len(runtime.List()))
	}

	// Make sure runtime.Get() refuses to return the unexisting container
	if runtime.Get(container.ID) != nil {
		t.Errorf("Unable to get newly created container")
	}

	// Make sure the container root directory does not exist anymore
	_, err = os.Stat(container.root)
	if err == nil || !os.IsNotExist(err) {
		t.Errorf("Container root directory still exists after destroy")
	}

	// Test double destroy
	if err := runtime.Destroy(container); err == nil {
		// It should have failed
		t.Errorf("Double destroy did not fail")
	}
}

func TestGet(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	builder := NewBuilder(runtime)

	container1, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container1)

	container2, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container2)

	container3, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container3)

	if runtime.Get(container1.ID) != container1 {
		t.Errorf("Get(test1) returned %v while expecting %v", runtime.Get(container1.ID), container1)
	}

	if runtime.Get(container2.ID) != container2 {
		t.Errorf("Get(test2) returned %v while expecting %v", runtime.Get(container2.ID), container2)
	}

	if runtime.Get(container3.ID) != container3 {
		t.Errorf("Get(test3) returned %v while expecting %v", runtime.Get(container3.ID), container3)
	}

}

func findAvailalblePort(runtime *Runtime, port int) (*Container, error) {
	strPort := strconv.Itoa(port)
	container, err := NewBuilder(runtime).Create(&Config{
		Image:     GetTestImage(runtime).ID,
		Cmd:       []string{"sh", "-c", "echo well hello there | nc -l -p " + strPort},
		PortSpecs: []string{strPort},
	},
	)
	if err != nil {
		return nil, err
	}
	hostConfig := &HostConfig{}
	if err := container.Start(hostConfig); err != nil {
		if strings.Contains(err.Error(), "address already in use") {
			return nil, nil
		}
		return nil, err
	}
	return container, nil
}

// Run a container with a TCP port allocated, and test that it can receive connections on localhost
func TestAllocatePortLocalhost(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	port := 5554

	var container *Container
	for {
		port += 1
		log.Println("Trying port", port)
		t.Log("Trying port", port)
		container, err = findAvailalblePort(runtime, port)
		if container != nil {
			break
		}
		if err != nil {
			t.Fatal(err)
		}
		log.Println("Port", port, "already in use")
		t.Log("Port", port, "already in use")
	}

	defer container.Kill()

	setTimeout(t, "Waiting for the container to be started timed out", 2*time.Second, func() {
		for !container.State.Running {
			time.Sleep(10 * time.Millisecond)
		}
	})

	// Even if the state is running, lets give some time to lxc to spawn the process
	container.WaitTimeout(500 * time.Millisecond)

	conn, err := net.Dial("tcp",
		fmt.Sprintf(
			"localhost:%s", container.NetworkSettings.PortMapping[strconv.Itoa(port)],
		),
	)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	output, err := ioutil.ReadAll(conn)
	if err != nil {
		t.Fatal(err)
	}
	if string(output) != "well hello there\n" {
		t.Fatalf("Received wrong output from network connection: should be '%s', not '%s'",
			"well hello there\n",
			string(output),
		)
	}
	container.Wait()
}

func TestRestore(t *testing.T) {

	root, err := ioutil.TempDir("", "docker-test")
	if err != nil {
		t.Fatal(err)
	}
	if err := os.Remove(root); err != nil {
		t.Fatal(err)
	}
	if err := utils.CopyDirectory(unitTestStoreBase, root); err != nil {
		t.Fatal(err)
	}

	runtime1, err := NewRuntimeFromDirectory(root, false)
	if err != nil {
		t.Fatal(err)
	}

	builder := NewBuilder(runtime1)

	// Create a container with one instance of docker
	container1, err := builder.Create(&Config{
		Image: GetTestImage(runtime1).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime1.Destroy(container1)

	// Create a second container meant to be killed
	container2, err := builder.Create(&Config{
		Image:     GetTestImage(runtime1).ID,
		Cmd:       []string{"/bin/cat"},
		OpenStdin: true,
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime1.Destroy(container2)

	// Start the container non blocking
	hostConfig := &HostConfig{}
	if err := container2.Start(hostConfig); err != nil {
		t.Fatal(err)
	}

	if !container2.State.Running {
		t.Fatalf("Container %v should appear as running but isn't", container2.ID)
	}

	// Simulate a crash/manual quit of dockerd: process dies, states stays 'Running'
	cStdin, _ := container2.StdinPipe()
	cStdin.Close()
	if err := container2.WaitTimeout(2 * time.Second); err != nil {
		t.Fatal(err)
	}
	container2.State.Running = true
	container2.ToDisk()

	if len(runtime1.List()) != 2 {
		t.Errorf("Expected 2 container, %v found", len(runtime1.List()))
	}
	if err := container1.Run(); err != nil {
		t.Fatal(err)
	}

	if !container2.State.Running {
		t.Fatalf("Container %v should appear as running but isn't", container2.ID)
	}

	// Here are are simulating a docker restart - that is, reloading all containers
	// from scratch
	runtime2, err := NewRuntimeFromDirectory(root, false)
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime2)
	if len(runtime2.List()) != 2 {
		t.Errorf("Expected 2 container, %v found", len(runtime2.List()))
	}
	runningCount := 0
	for _, c := range runtime2.List() {
		if c.State.Running {
			t.Errorf("Running container found: %v (%v)", c.ID, c.Path)
			runningCount++
		}
	}
	if runningCount != 0 {
		t.Fatalf("Expected 0 container alive, %d found", runningCount)
	}
	container3 := runtime2.Get(container1.ID)
	if container3 == nil {
		t.Fatal("Unable to Get container")
	}
	if err := container3.Run(); err != nil {
		t.Fatal(err)
	}
	container2.State.Running = false
}
<commit_msg>Fixed runtime_test (ImagePull prototyped changed)<commit_after>package docker

import (
	"fmt"
	"github.com/dotcloud/docker/utils"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"testing"
	"time"
)

const (
	unitTestImageName = "docker-unit-tests"
	unitTestImageID   = "e9aa60c60128cad1"
	unitTestStoreBase = "/var/lib/docker/unit-tests"
	testDaemonAddr    = "127.0.0.1:4270"
	testDaemonProto   = "tcp"
)

var globalRuntime *Runtime

func nuke(runtime *Runtime) error {
	var wg sync.WaitGroup
	for _, container := range runtime.List() {
		wg.Add(1)
		go func(c *Container) {
			c.Kill()
			wg.Done()
		}(container)
	}
	wg.Wait()
	return os.RemoveAll(runtime.root)
}

func cleanup(runtime *Runtime) error {
	for _, container := range runtime.List() {
		container.Kill()
		runtime.Destroy(container)
	}
	images, err := runtime.graph.All()
	if err != nil {
		return err
	}
	for _, image := range images {
		if image.ID != unitTestImageID {
			runtime.graph.Delete(image.ID)
		}
	}
	return nil
}

func layerArchive(tarfile string) (io.Reader, error) {
	// FIXME: need to close f somewhere
	f, err := os.Open(tarfile)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func init() {
	// Hack to run sys init during unit testing
	if utils.SelfPath() == "/sbin/init" {
		SysInit()
		return
	}

	if uid := syscall.Geteuid(); uid != 0 {
		log.Fatal("docker tests need to be run as root")
	}

	NetworkBridgeIface = "testdockbr0"

	// Make it our Store root
	runtime, err := NewRuntimeFromDirectory(unitTestStoreBase, false)
	if err != nil {
		panic(err)
	}
	globalRuntime = runtime

	// Create the "Server"
	srv := &Server{
		runtime:     runtime,
		enableCors:  false,
		pullingPool: make(map[string]struct{}),
		pushingPool: make(map[string]struct{}),
	}
	// Retrieve the Image
	if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil {
		panic(err)
	}
	// Spawn a Daemon
	go func() {
		if err := ListenAndServe(testDaemonProto, testDaemonAddr, srv, os.Getenv("DEBUG") != ""); err != nil {
			panic(err)
		}
	}()

	// Give some time to ListenAndServer to actually start
	time.Sleep(time.Second)
}

// FIXME: test that ImagePull(json=true) send correct json output

func newTestRuntime() (*Runtime, error) {
	root, err := ioutil.TempDir("", "docker-test")
	if err != nil {
		return nil, err
	}
	if err := os.Remove(root); err != nil {
		return nil, err
	}
	if err := utils.CopyDirectory(unitTestStoreBase, root); err != nil {
		return nil, err
	}

	runtime, err := NewRuntimeFromDirectory(root, false)
	if err != nil {
		return nil, err
	}
	runtime.UpdateCapabilities(true)
	return runtime, nil
}

func GetTestImage(runtime *Runtime) *Image {
	imgs, err := runtime.graph.All()
	if err != nil {
		panic(err)
	}
	for i := range imgs {
		if imgs[i].ID == unitTestImageID {
			return imgs[i]
		}
	}
	panic(fmt.Errorf("Test image %v not found", unitTestImageID))
}

func TestRuntimeCreate(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	// Make sure we start we 0 containers
	if len(runtime.List()) != 0 {
		t.Errorf("Expected 0 containers, %v found", len(runtime.List()))
	}

	builder := NewBuilder(runtime)

	container, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := runtime.Destroy(container); err != nil {
			t.Error(err)
		}
	}()

	// Make sure we can find the newly created container with List()
	if len(runtime.List()) != 1 {
		t.Errorf("Expected 1 container, %v found", len(runtime.List()))
	}

	// Make sure the container List() returns is the right one
	if runtime.List()[0].ID != container.ID {
		t.Errorf("Unexpected container %v returned by List", runtime.List()[0])
	}

	// Make sure we can get the container with Get()
	if runtime.Get(container.ID) == nil {
		t.Errorf("Unable to get newly created container")
	}

	// Make sure it is the right container
	if runtime.Get(container.ID) != container {
		t.Errorf("Get() returned the wrong container")
	}

	// Make sure Exists returns it as existing
	if !runtime.Exists(container.ID) {
		t.Errorf("Exists() returned false for a newly created container")
	}

	// Make sure crete with bad parameters returns an error
	_, err = builder.Create(
		&Config{
			Image: GetTestImage(runtime).ID,
		},
	)
	if err == nil {
		t.Fatal("Builder.Create should throw an error when Cmd is missing")
	}

	_, err = builder.Create(
		&Config{
			Image: GetTestImage(runtime).ID,
			Cmd:   []string{},
		},
	)
	if err == nil {
		t.Fatal("Builder.Create should throw an error when Cmd is empty")
	}
}

func TestDestroy(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)
	container, err := NewBuilder(runtime).Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	// Destroy
	if err := runtime.Destroy(container); err != nil {
		t.Error(err)
	}

	// Make sure runtime.Exists() behaves correctly
	if runtime.Exists("test_destroy") {
		t.Errorf("Exists() returned true")
	}

	// Make sure runtime.List() doesn't list the destroyed container
	if len(runtime.List()) != 0 {
		t.Errorf("Expected 0 container, %v found", len(runtime.List()))
	}

	// Make sure runtime.Get() refuses to return the unexisting container
	if runtime.Get(container.ID) != nil {
		t.Errorf("Unable to get newly created container")
	}

	// Make sure the container root directory does not exist anymore
	_, err = os.Stat(container.root)
	if err == nil || !os.IsNotExist(err) {
		t.Errorf("Container root directory still exists after destroy")
	}

	// Test double destroy
	if err := runtime.Destroy(container); err == nil {
		// It should have failed
		t.Errorf("Double destroy did not fail")
	}
}

func TestGet(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	builder := NewBuilder(runtime)

	container1, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container1)

	container2, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container2)

	container3, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container3)

	if runtime.Get(container1.ID) != container1 {
		t.Errorf("Get(test1) returned %v while expecting %v", runtime.Get(container1.ID), container1)
	}

	if runtime.Get(container2.ID) != container2 {
		t.Errorf("Get(test2) returned %v while expecting %v", runtime.Get(container2.ID), container2)
	}

	if runtime.Get(container3.ID) != container3 {
		t.Errorf("Get(test3) returned %v while expecting %v", runtime.Get(container3.ID), container3)
	}

}

func findAvailalblePort(runtime *Runtime, port int) (*Container, error) {
	strPort := strconv.Itoa(port)
	container, err := NewBuilder(runtime).Create(&Config{
		Image:     GetTestImage(runtime).ID,
		Cmd:       []string{"sh", "-c", "echo well hello there | nc -l -p " + strPort},
		PortSpecs: []string{strPort},
	},
	)
	if err != nil {
		return nil, err
	}
	hostConfig := &HostConfig{}
	if err := container.Start(hostConfig); err != nil {
		if strings.Contains(err.Error(), "address already in use") {
			return nil, nil
		}
		return nil, err
	}
	return container, nil
}

// Run a container with a TCP port allocated, and test that it can receive connections on localhost
func TestAllocatePortLocalhost(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	port := 5554

	var container *Container
	for {
		port += 1
		log.Println("Trying port", port)
		t.Log("Trying port", port)
		container, err = findAvailalblePort(runtime, port)
		if container != nil {
			break
		}
		if err != nil {
			t.Fatal(err)
		}
		log.Println("Port", port, "already in use")
		t.Log("Port", port, "already in use")
	}

	defer container.Kill()

	setTimeout(t, "Waiting for the container to be started timed out", 2*time.Second, func() {
		for !container.State.Running {
			time.Sleep(10 * time.Millisecond)
		}
	})

	// Even if the state is running, lets give some time to lxc to spawn the process
	container.WaitTimeout(500 * time.Millisecond)

	conn, err := net.Dial("tcp",
		fmt.Sprintf(
			"localhost:%s", container.NetworkSettings.PortMapping[strconv.Itoa(port)],
		),
	)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	output, err := ioutil.ReadAll(conn)
	if err != nil {
		t.Fatal(err)
	}
	if string(output) != "well hello there\n" {
		t.Fatalf("Received wrong output from network connection: should be '%s', not '%s'",
			"well hello there\n",
			string(output),
		)
	}
	container.Wait()
}

func TestRestore(t *testing.T) {

	root, err := ioutil.TempDir("", "docker-test")
	if err != nil {
		t.Fatal(err)
	}
	if err := os.Remove(root); err != nil {
		t.Fatal(err)
	}
	if err := utils.CopyDirectory(unitTestStoreBase, root); err != nil {
		t.Fatal(err)
	}

	runtime1, err := NewRuntimeFromDirectory(root, false)
	if err != nil {
		t.Fatal(err)
	}

	builder := NewBuilder(runtime1)

	// Create a container with one instance of docker
	container1, err := builder.Create(&Config{
		Image: GetTestImage(runtime1).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime1.Destroy(container1)

	// Create a second container meant to be killed
	container2, err := builder.Create(&Config{
		Image:     GetTestImage(runtime1).ID,
		Cmd:       []string{"/bin/cat"},
		OpenStdin: true,
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime1.Destroy(container2)

	// Start the container non blocking
	hostConfig := &HostConfig{}
	if err := container2.Start(hostConfig); err != nil {
		t.Fatal(err)
	}

	if !container2.State.Running {
		t.Fatalf("Container %v should appear as running but isn't", container2.ID)
	}

	// Simulate a crash/manual quit of dockerd: process dies, states stays 'Running'
	cStdin, _ := container2.StdinPipe()
	cStdin.Close()
	if err := container2.WaitTimeout(2 * time.Second); err != nil {
		t.Fatal(err)
	}
	container2.State.Running = true
	container2.ToDisk()

	if len(runtime1.List()) != 2 {
		t.Errorf("Expected 2 container, %v found", len(runtime1.List()))
	}
	if err := container1.Run(); err != nil {
		t.Fatal(err)
	}

	if !container2.State.Running {
		t.Fatalf("Container %v should appear as running but isn't", container2.ID)
	}

	// Here are are simulating a docker restart - that is, reloading all containers
	// from scratch
	runtime2, err := NewRuntimeFromDirectory(root, false)
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime2)
	if len(runtime2.List()) != 2 {
		t.Errorf("Expected 2 container, %v found", len(runtime2.List()))
	}
	runningCount := 0
	for _, c := range runtime2.List() {
		if c.State.Running {
			t.Errorf("Running container found: %v (%v)", c.ID, c.Path)
			runningCount++
		}
	}
	if runningCount != 0 {
		t.Fatalf("Expected 0 container alive, %d found", runningCount)
	}
	container3 := runtime2.Get(container1.ID)
	if container3 == nil {
		t.Fatal("Unable to Get container")
	}
	if err := container3.Run(); err != nil {
		t.Fatal(err)
	}
	container2.State.Running = false
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package safehttp

import (
	"fmt"
	"log"
	"net/http"
)

// The HTTP request methods defined by RFC.
const (
	MethodConnect = "CONNECT" // RFC 7231, 4.3.6
	MethodDelete  = "DELETE"  // RFC 7231, 4.3.5
	MethodGet     = "GET"     // RFC 7231, 4.3.1
	MethodHead    = "HEAD"    // RFC 7231, 4.3.2
	MethodOptions = "OPTIONS" // RFC 7231, 4.3.7
	MethodPatch   = "PATCH"   // RFC 5789
	MethodPost    = "POST"    // RFC 7231, 4.3.3
	MethodPut     = "PUT"     // RFC 7231, 4.3.4
	MethodTrace   = "TRACE"   // RFC 7231, 4.3.8
)

// ServeMux is an HTTP request multiplexer. It matches the URL of each incoming
// request against a list of registered patterns and calls the handler for
// the pattern that most closely matches the URL.
//
// Patterns names are fixed, rooted paths, like "/favicon.ico", or rooted
// subtrees like "/images/" (note the trailing slash). Longer patterns take
// precedence over shorter ones, so that if there are handlers registered for
// both "/images/" and "/images/thumbnails/", the latter handler will be called
// for paths beginning "/images/thumbnails/" and the former will receive
// requests for any other paths in the "/images/" subtree.
//
// Note that since a pattern ending in a slash names a rooted subtree, the
// pattern "/" matches all paths not matched by other registered patterns,
// not just the URL with Path == "/".
//
// If a subtree has been registered and a request is received naming the subtree
// root without its trailing slash, ServeMux redirects that request to
// the subtree root (adding the trailing slash). This behavior can be overridden
// with a separate registration for the path without the trailing slash. For
// example, registering "/images/" causes ServeMux to redirect a request for
// "/images" to "/images/", unless "/images" has been registered separately.
//
// Patterns may optionally begin with a host name, restricting matches to URLs
// on that host only.  Host-specific patterns take precedence over general
// patterns, so that a handler might register for the two patterns "/codesearch"
// and "codesearch.google.com/" without also taking over requests for
// "http://www.google.com/".
//
// ServeMux also takes care of sanitizing the URL request path and the Host
// header, stripping the port number and redirecting any request containing . or
// .. elements or repeated slashes to an equivalent, cleaner URL.
//
// Multiple handlers can be registered for a single pattern, as long as they
// handle different HTTP methods.
type ServeMux struct {
	mux      *http.ServeMux
	handlers map[string]*registeredHandler

	dispatcher       Dispatcher
	interceptors     []Interceptor
	methodNotAllowed handlerConfig
}

// ServeHTTP dispatches the request to the handler whose method matches the
// incoming request and whose pattern most closely matches the request URL.
//
//  For each incoming request:
//  - [Before Phase] Interceptor.Before methods are called for every installed
//    interceptor, until an interceptor writes to a ResponseWriter (including
//    errors) or panics,
//  - the handler is called after a [Before Phase] if no writes or panics occured,
//  - the handler triggers the [Commit Phase] by writing to the ResponseWriter,
//  - [Commit Phase] Interceptor.Commit methods run for every interceptor whose
//    Before method was called,
//  - [Dispatcher Phase] after the [Commit Phase], the Dispatcher's appropriate
//    write method is called; the Dispatcher is responsible for determining whether
//    the response is indeed safe and writing it,
//  - if the handler attempts to write more than once, it is treated as an
//    unrecoverable error; the request processing ends abrubptly with a panic and
//    nothing else happens (note: this will change as soon as [After Phase] is
//    introduced)
//
// Interceptors should NOT rely on the order they're run.
func (m *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	m.mux.ServeHTTP(w, r)
}

// Handle registers a handler for the given pattern and method. If a handler is
// registered twice for the same pattern and method, Build will panic.
//
// InterceptorConfigs can be passed in order to modify the behavior of the
// interceptors on a registered handler. Passing an InterceptorConfig whose
// corresponding Interceptor was not installed will produce no effect. If
// multiple configurations are passed for the same Interceptor, Mux will panic.
func (m *ServeMux) Handle(pattern string, method string, h Handler, cfgs ...InterceptorConfig) {
	if m.handlers[pattern] == nil {
		m.handlers[pattern] = &registeredHandler{
			pattern:          pattern,
			methodNotAllowed: m.methodNotAllowed,
			methods:          make(map[string]handlerConfig),
		}
		m.mux.Handle(pattern, m.handlers[pattern])
	}
	m.handlers[pattern].handleMethod(method,
		handlerConfig{
			Dispatcher:   m.dispatcher,
			Handler:      h,
			Interceptors: configureInterceptors(m.interceptors, cfgs),
		})
}

// ServeMuxConfig is a builder for ServeMux.
type ServeMuxConfig struct {
	dispatcher   Dispatcher
	interceptors []Interceptor

	methodNotAllowed     Handler
	methodNotAllowedCfgs []InterceptorConfig
}

// NewServeMuxConfig crates a ServeMuxConfig with the provided Dispatcher. If
// the provided Dispatcher is nil, the DefaultDispatcher is used.
func NewServeMuxConfig(disp Dispatcher) *ServeMuxConfig {
	if disp == nil {
		disp = &DefaultDispatcher{}
	}
	return &ServeMuxConfig{
		dispatcher:       disp,
		methodNotAllowed: HandlerFunc(defaultMethotNotAllowed),
	}
}

// HandleMethodNotAllowed registers a handler that runs when a given method is
// not allowed for a registered path.
func (s *ServeMuxConfig) HandleMethodNotAllowed(h Handler, cfgs ...InterceptorConfig) {
	s.methodNotAllowed = h
	s.methodNotAllowedCfgs = cfgs
}

var defaultMethotNotAllowed = HandlerFunc(func(w ResponseWriter, req *IncomingRequest) Result {
	return w.WriteError(StatusMethodNotAllowed)
})

// Intercept installs the given interceptors.
//
// Interceptors order is respected and interceptors are always run in the
// order they've been installed.
//
// Calling Intercept multiple times is valid. Interceptors that are added last
// will run last.
func (s *ServeMuxConfig) Intercept(is ...Interceptor) {
	s.interceptors = append(s.interceptors, is...)
}

// Mux returns the ServeMux with a copy of the current configuration.
func (s *ServeMuxConfig) Mux() *ServeMux {
	freezeLocalDev = true
	if IsLocalDev() {
		log.Println("Warning: creating safehttp.Mux in dev mode. This configuration is not valid for production use")
	}

	if s.dispatcher == nil {
		panic("Use NewServeMuxConfig instead of creating ServeMuxConfig using a composite literal.")
	}

	methodNotAllowed := handlerConfig{
		Dispatcher:   s.dispatcher,
		Handler:      s.methodNotAllowed,
		Interceptors: configureInterceptors(s.interceptors, s.methodNotAllowedCfgs),
	}

	m := &ServeMux{
		mux:              http.NewServeMux(),
		handlers:         make(map[string]*registeredHandler),
		dispatcher:       s.dispatcher,
		interceptors:     s.interceptors,
		methodNotAllowed: methodNotAllowed,
	}
	return m
}

// Clone creates a copy of the current config.
// This can be used to create several instances of Mux that share the same set of
// plugins and some common handlers.
func (s *ServeMuxConfig) Clone() *ServeMuxConfig {
	c := &ServeMuxConfig{
		dispatcher:       s.dispatcher,
		interceptors:     make([]Interceptor, len(s.interceptors)),
		methodNotAllowed: s.methodNotAllowed,
	}
	copy(c.interceptors, s.interceptors)
	copy(c.methodNotAllowedCfgs, s.methodNotAllowedCfgs)
	return c
}

type registeredHandler struct {
	pattern          string
	methods          map[string]handlerConfig
	methodNotAllowed handlerConfig
}

func (rh *registeredHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	cfg, ok := rh.methods[r.Method]
	if !ok {
		cfg = rh.methodNotAllowed
	}
	processRequest(cfg, w, r)
}

func (rh *registeredHandler) handleMethod(method string, cfg handlerConfig) {
	if _, exists := rh.methods[method]; exists {
		panic(fmt.Sprintf("double registration of (pattern = %q, method = %q)", rh.pattern, method))
	}
	rh.methods[method] = cfg
}

func configureInterceptors(interceptors []Interceptor, cfgs []InterceptorConfig) []configuredInterceptor {
	var its []configuredInterceptor
	for _, it := range interceptors {
		var matches []InterceptorConfig
		for _, c := range cfgs {
			if it.Match(c) {
				matches = append(matches, c)
			}
		}

		if len(matches) > 1 {
			msg := fmt.Sprintf("multiple configurations specified for interceptor %T: ", it)
			for _, match := range matches {
				msg += fmt.Sprintf("%#v", match)
			}
			panic(msg)
		}

		var cfg InterceptorConfig
		if len(matches) == 1 {
			cfg = matches[0]
		}
		its = append(its, configuredInterceptor{interceptor: it, config: cfg})
	}
	return its
}
<commit_msg>updated a comment<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package safehttp

import (
	"fmt"
	"log"
	"net/http"
)

// The HTTP request methods defined by RFC.
const (
	MethodConnect = "CONNECT" // RFC 7231, 4.3.6
	MethodDelete  = "DELETE"  // RFC 7231, 4.3.5
	MethodGet     = "GET"     // RFC 7231, 4.3.1
	MethodHead    = "HEAD"    // RFC 7231, 4.3.2
	MethodOptions = "OPTIONS" // RFC 7231, 4.3.7
	MethodPatch   = "PATCH"   // RFC 5789
	MethodPost    = "POST"    // RFC 7231, 4.3.3
	MethodPut     = "PUT"     // RFC 7231, 4.3.4
	MethodTrace   = "TRACE"   // RFC 7231, 4.3.8
)

// ServeMux is an HTTP request multiplexer. It matches the URL of each incoming
// request against a list of registered patterns and calls the handler for
// the pattern that most closely matches the URL.
//
// Patterns names are fixed, rooted paths, like "/favicon.ico", or rooted
// subtrees like "/images/" (note the trailing slash). Longer patterns take
// precedence over shorter ones, so that if there are handlers registered for
// both "/images/" and "/images/thumbnails/", the latter handler will be called
// for paths beginning "/images/thumbnails/" and the former will receive
// requests for any other paths in the "/images/" subtree.
//
// Note that since a pattern ending in a slash names a rooted subtree, the
// pattern "/" matches all paths not matched by other registered patterns,
// not just the URL with Path == "/".
//
// If a subtree has been registered and a request is received naming the subtree
// root without its trailing slash, ServeMux redirects that request to
// the subtree root (adding the trailing slash). This behavior can be overridden
// with a separate registration for the path without the trailing slash. For
// example, registering "/images/" causes ServeMux to redirect a request for
// "/images" to "/images/", unless "/images" has been registered separately.
//
// Patterns may optionally begin with a host name, restricting matches to URLs
// on that host only.  Host-specific patterns take precedence over general
// patterns, so that a handler might register for the two patterns "/codesearch"
// and "codesearch.google.com/" without also taking over requests for
// "http://www.google.com/".
//
// ServeMux also takes care of sanitizing the URL request path and the Host
// header, stripping the port number and redirecting any request containing . or
// .. elements or repeated slashes to an equivalent, cleaner URL.
//
// Multiple handlers can be registered for a single pattern, as long as they
// handle different HTTP methods.
type ServeMux struct {
	mux      *http.ServeMux
	handlers map[string]*registeredHandler

	dispatcher       Dispatcher
	interceptors     []Interceptor
	methodNotAllowed handlerConfig
}

// ServeHTTP dispatches the request to the handler whose method matches the
// incoming request and whose pattern most closely matches the request URL.
//
//  For each incoming request:
//  - [Before Phase] Interceptor.Before methods are called for every installed
//    interceptor, until an interceptor writes to a ResponseWriter (including
//    errors) or panics,
//  - the handler is called after a [Before Phase] if no writes or panics occured,
//  - the handler triggers the [Commit Phase] by writing to the ResponseWriter,
//  - [Commit Phase] Interceptor.Commit methods run for every interceptor whose
//    Before method was called,
//  - [Dispatcher Phase] after the [Commit Phase], the Dispatcher's appropriate
//    write method is called; the Dispatcher is responsible for determining whether
//    the response is indeed safe and writing it,
//  - if the handler attempts to write more than once, it is treated as an
//    unrecoverable error; the request processing ends abrubptly with a panic and
//    nothing else happens (note: this will change as soon as [After Phase] is
//    introduced)
//
// Interceptors should NOT rely on the order they're run.
func (m *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	m.mux.ServeHTTP(w, r)
}

// Handle registers a handler for the given pattern and method. If a handler is
// registered twice for the same pattern and method, Build will panic.
//
// InterceptorConfigs can be passed in order to modify the behavior of the
// interceptors on a registered handler. Passing an InterceptorConfig whose
// corresponding Interceptor was not installed will produce no effect. If
// multiple configurations are passed for the same Interceptor, Mux will panic.
func (m *ServeMux) Handle(pattern string, method string, h Handler, cfgs ...InterceptorConfig) {
	if m.handlers[pattern] == nil {
		m.handlers[pattern] = &registeredHandler{
			pattern:          pattern,
			methodNotAllowed: m.methodNotAllowed,
			methods:          make(map[string]handlerConfig),
		}
		m.mux.Handle(pattern, m.handlers[pattern])
	}
	m.handlers[pattern].handleMethod(method,
		handlerConfig{
			Dispatcher:   m.dispatcher,
			Handler:      h,
			Interceptors: configureInterceptors(m.interceptors, cfgs),
		})
}

// ServeMuxConfig is a builder for ServeMux.
type ServeMuxConfig struct {
	dispatcher   Dispatcher
	interceptors []Interceptor

	methodNotAllowed     Handler
	methodNotAllowedCfgs []InterceptorConfig
}

// NewServeMuxConfig crates a ServeMuxConfig with the provided Dispatcher. If
// the provided Dispatcher is nil, the DefaultDispatcher is used.
func NewServeMuxConfig(disp Dispatcher) *ServeMuxConfig {
	if disp == nil {
		disp = &DefaultDispatcher{}
	}
	return &ServeMuxConfig{
		dispatcher:       disp,
		methodNotAllowed: HandlerFunc(defaultMethotNotAllowed),
	}
}

// HandleMethodNotAllowed registers a handler that runs when a given method is
// not allowed for a registered path.
func (s *ServeMuxConfig) HandleMethodNotAllowed(h Handler, cfgs ...InterceptorConfig) {
	s.methodNotAllowed = h
	s.methodNotAllowedCfgs = cfgs
}

var defaultMethotNotAllowed = HandlerFunc(func(w ResponseWriter, req *IncomingRequest) Result {
	return w.WriteError(StatusMethodNotAllowed)
})

// Intercept installs the given interceptors.
//
// Interceptors order is respected and interceptors are always run in the
// order they've been installed.
//
// Calling Intercept multiple times is valid. Interceptors that are added last
// will run last.
func (s *ServeMuxConfig) Intercept(is ...Interceptor) {
	s.interceptors = append(s.interceptors, is...)
}

// Mux returns the ServeMux with a copy of the current configuration.
func (s *ServeMuxConfig) Mux() *ServeMux {
	freezeLocalDev = true
	if IsLocalDev() {
		log.Println("Warning: creating safehttp.Mux in dev mode. This configuration is not valid for production use")
	}

	if s.dispatcher == nil {
		panic("Use NewServeMuxConfig instead of creating ServeMuxConfig using a composite literal.")
	}

	methodNotAllowed := handlerConfig{
		Dispatcher:   s.dispatcher,
		Handler:      s.methodNotAllowed,
		Interceptors: configureInterceptors(s.interceptors, s.methodNotAllowedCfgs),
	}

	m := &ServeMux{
		mux:              http.NewServeMux(),
		handlers:         make(map[string]*registeredHandler),
		dispatcher:       s.dispatcher,
		interceptors:     s.interceptors,
		methodNotAllowed: methodNotAllowed,
	}
	return m
}

// Clone creates a copy of the current config.
// This can be used to create several instances of Mux that share the same set of
// plugins.
func (s *ServeMuxConfig) Clone() *ServeMuxConfig {
	c := &ServeMuxConfig{
		dispatcher:       s.dispatcher,
		interceptors:     make([]Interceptor, len(s.interceptors)),
		methodNotAllowed: s.methodNotAllowed,
	}
	copy(c.interceptors, s.interceptors)
	copy(c.methodNotAllowedCfgs, s.methodNotAllowedCfgs)
	return c
}

type registeredHandler struct {
	pattern          string
	methods          map[string]handlerConfig
	methodNotAllowed handlerConfig
}

func (rh *registeredHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	cfg, ok := rh.methods[r.Method]
	if !ok {
		cfg = rh.methodNotAllowed
	}
	processRequest(cfg, w, r)
}

func (rh *registeredHandler) handleMethod(method string, cfg handlerConfig) {
	if _, exists := rh.methods[method]; exists {
		panic(fmt.Sprintf("double registration of (pattern = %q, method = %q)", rh.pattern, method))
	}
	rh.methods[method] = cfg
}

func configureInterceptors(interceptors []Interceptor, cfgs []InterceptorConfig) []configuredInterceptor {
	var its []configuredInterceptor
	for _, it := range interceptors {
		var matches []InterceptorConfig
		for _, c := range cfgs {
			if it.Match(c) {
				matches = append(matches, c)
			}
		}

		if len(matches) > 1 {
			msg := fmt.Sprintf("multiple configurations specified for interceptor %T: ", it)
			for _, match := range matches {
				msg += fmt.Sprintf("%#v", match)
			}
			panic(msg)
		}

		var cfg InterceptorConfig
		if len(matches) == 1 {
			cfg = matches[0]
		}
		its = append(its, configuredInterceptor{interceptor: it, config: cfg})
	}
	return its
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"

	"github.com/gorilla/mux"
	_ "github.com/mattn/go-sqlite3"

	"github.com/lxc/lxd/shared"

	log "gopkg.in/inconshreveable/log15.v2"
)

/* This is used for both profiles post and profile put */
type profilesPostReq struct {
	Name        string            `json:"name"`
	Config      map[string]string `json:"config"`
	Description string            `json:"description"`
	Devices     shared.Devices    `json:"devices"`
}

func profilesGet(d *Daemon, r *http.Request) Response {
	results, err := dbProfiles(d.db)
	if err != nil {
		return SmartError(err)
	}

	recursion := d.isRecursionRequest(r)

	resultString := make([]string, len(results))
	resultMap := make([]*shared.ProfileConfig, len(results))
	i := 0
	for _, name := range results {
		if !recursion {
			url := fmt.Sprintf("/%s/profiles/%s", shared.APIVersion, name)
			resultString[i] = url
		} else {
			profile, err := doProfileGet(d, name)
			if err != nil {
				shared.Log.Error("Failed to get profile", log.Ctx{"profile": name})
				continue
			}
			resultMap[i] = profile
		}
		i++
	}

	if !recursion {
		return SyncResponse(true, resultString)
	}

	return SyncResponse(true, resultMap)
}

func profilesPost(d *Daemon, r *http.Request) Response {
	req := profilesPostReq{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return BadRequest(err)
	}

	// Sanity checks
	if req.Name == "" {
		return BadRequest(fmt.Errorf("No name provided"))
	}

	err := containerValidConfig(req.Config, true, false)
	if err != nil {
		return BadRequest(err)
	}

	err = containerValidDevices(req.Devices, true, false)
	if err != nil {
		return BadRequest(err)
	}

	// Update DB entry
	_, err = dbProfileCreate(d.db, req.Name, req.Description, req.Config, req.Devices)
	if err != nil {
		return InternalError(
			fmt.Errorf("Error inserting %s into database: %s", req.Name, err))
	}

	return EmptySyncResponse
}

var profilesCmd = Command{
	name: "profiles",
	get:  profilesGet,
	post: profilesPost}

func doProfileGet(d *Daemon, name string) (*shared.ProfileConfig, error) {
	_, profile, err := dbProfileGet(d.db, name)
	return profile, err
}

func profileGet(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	resp, err := doProfileGet(d, name)
	if err != nil {
		return SmartError(err)
	}

	return SyncResponse(true, resp)
}

func getRunningContainersWithProfile(d *Daemon, profile string) []container {
	results := []container{}

	output, err := dbProfileContainersGet(d.db, profile)
	if err != nil {
		return results
	}

	for _, name := range output {
		c, err := containerLoadByName(d, name)
		if err != nil {
			shared.Log.Error("Failed opening container", log.Ctx{"container": name})
			continue
		}
		results = append(results, c)
	}
	return results
}

func profilePut(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	req := profilesPostReq{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return BadRequest(err)
	}

	// Sanity checks
	err := containerValidConfig(req.Config, true, false)
	if err != nil {
		return BadRequest(err)
	}

	err = containerValidDevices(req.Devices, true, false)
	if err != nil {
		return BadRequest(err)
	}

	// Get the running container list
	clist := getRunningContainersWithProfile(d, name)
	var containers []container
	for _, c := range clist {
		if !c.IsRunning() {
			continue
		}

		containers = append(containers, c)
	}

	// Update the database
	id, profile, err := dbProfileGet(d.db, name)
	if err != nil {
		return InternalError(fmt.Errorf("Failed to retrieve profile='%s'", name))
	}

	tx, err := dbBegin(d.db)
	if err != nil {
		return InternalError(err)
	}

	if profile.Description != req.Description {
		err = dbProfileDescriptionUpdate(tx, id, req.Description)
		if err != nil {
			tx.Rollback()
			return InternalError(err)
		}
	}

	// Optimize for description-only changes
	if reflect.DeepEqual(profile.Config, req.Config) && reflect.DeepEqual(profile.Devices, req.Devices) {
		err = txCommit(tx)
		if err != nil {
			return InternalError(err)
		}

		return EmptySyncResponse
	}

	err = dbProfileConfigClear(tx, id)
	if err != nil {
		tx.Rollback()
		return InternalError(err)
	}

	err = dbProfileConfigAdd(tx, id, req.Config)
	if err != nil {
		tx.Rollback()
		return SmartError(err)
	}

	err = dbDevicesAdd(tx, "profile", id, req.Devices)
	if err != nil {
		tx.Rollback()
		return SmartError(err)
	}

	err = txCommit(tx)
	if err != nil {
		return InternalError(err)
	}

	// Update all the containers using the profile. Must be done after txCommit due to DB lock.
	failures := map[string]error{}
	for _, c := range containers {
		err = c.Update(containerArgs{
			Architecture: c.Architecture(),
			Ephemeral:    c.IsEphemeral(),
			Config:       c.LocalConfig(),
			Devices:      c.LocalDevices(),
			Profiles:     c.Profiles()}, true)

		if err != nil {
			failures[c.Name()] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return InternalError(fmt.Errorf("%s", msg))
	}

	return EmptySyncResponse
}

// The handler for the post operation.
func profilePost(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	req := profilesPostReq{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return BadRequest(err)
	}

	// Sanity checks
	if req.Name == "" {
		return BadRequest(fmt.Errorf("No name provided"))
	}

	err := dbProfileUpdate(d.db, name, req.Name)
	if err != nil {
		return InternalError(err)
	}

	return EmptySyncResponse
}

// The handler for the delete operation.
func profileDelete(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]
	err := dbProfileDelete(d.db, name)

	if err != nil {
		return InternalError(err)
	}

	return EmptySyncResponse
}

var profileCmd = Command{name: "profiles/{name}", get: profileGet, put: profilePut, delete: profileDelete, post: profilePost}
<commit_msg>Fail when removing non-existent profiles<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"

	"github.com/gorilla/mux"
	_ "github.com/mattn/go-sqlite3"

	"github.com/lxc/lxd/shared"

	log "gopkg.in/inconshreveable/log15.v2"
)

/* This is used for both profiles post and profile put */
type profilesPostReq struct {
	Name        string            `json:"name"`
	Config      map[string]string `json:"config"`
	Description string            `json:"description"`
	Devices     shared.Devices    `json:"devices"`
}

func profilesGet(d *Daemon, r *http.Request) Response {
	results, err := dbProfiles(d.db)
	if err != nil {
		return SmartError(err)
	}

	recursion := d.isRecursionRequest(r)

	resultString := make([]string, len(results))
	resultMap := make([]*shared.ProfileConfig, len(results))
	i := 0
	for _, name := range results {
		if !recursion {
			url := fmt.Sprintf("/%s/profiles/%s", shared.APIVersion, name)
			resultString[i] = url
		} else {
			profile, err := doProfileGet(d, name)
			if err != nil {
				shared.Log.Error("Failed to get profile", log.Ctx{"profile": name})
				continue
			}
			resultMap[i] = profile
		}
		i++
	}

	if !recursion {
		return SyncResponse(true, resultString)
	}

	return SyncResponse(true, resultMap)
}

func profilesPost(d *Daemon, r *http.Request) Response {
	req := profilesPostReq{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return BadRequest(err)
	}

	// Sanity checks
	if req.Name == "" {
		return BadRequest(fmt.Errorf("No name provided"))
	}

	err := containerValidConfig(req.Config, true, false)
	if err != nil {
		return BadRequest(err)
	}

	err = containerValidDevices(req.Devices, true, false)
	if err != nil {
		return BadRequest(err)
	}

	// Update DB entry
	_, err = dbProfileCreate(d.db, req.Name, req.Description, req.Config, req.Devices)
	if err != nil {
		return InternalError(
			fmt.Errorf("Error inserting %s into database: %s", req.Name, err))
	}

	return EmptySyncResponse
}

var profilesCmd = Command{
	name: "profiles",
	get:  profilesGet,
	post: profilesPost}

func doProfileGet(d *Daemon, name string) (*shared.ProfileConfig, error) {
	_, profile, err := dbProfileGet(d.db, name)
	return profile, err
}

func profileGet(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	resp, err := doProfileGet(d, name)
	if err != nil {
		return SmartError(err)
	}

	return SyncResponse(true, resp)
}

func getRunningContainersWithProfile(d *Daemon, profile string) []container {
	results := []container{}

	output, err := dbProfileContainersGet(d.db, profile)
	if err != nil {
		return results
	}

	for _, name := range output {
		c, err := containerLoadByName(d, name)
		if err != nil {
			shared.Log.Error("Failed opening container", log.Ctx{"container": name})
			continue
		}
		results = append(results, c)
	}
	return results
}

func profilePut(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	req := profilesPostReq{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return BadRequest(err)
	}

	// Sanity checks
	err := containerValidConfig(req.Config, true, false)
	if err != nil {
		return BadRequest(err)
	}

	err = containerValidDevices(req.Devices, true, false)
	if err != nil {
		return BadRequest(err)
	}

	// Get the running container list
	clist := getRunningContainersWithProfile(d, name)
	var containers []container
	for _, c := range clist {
		if !c.IsRunning() {
			continue
		}

		containers = append(containers, c)
	}

	// Update the database
	id, profile, err := dbProfileGet(d.db, name)
	if err != nil {
		return InternalError(fmt.Errorf("Failed to retrieve profile='%s'", name))
	}

	tx, err := dbBegin(d.db)
	if err != nil {
		return InternalError(err)
	}

	if profile.Description != req.Description {
		err = dbProfileDescriptionUpdate(tx, id, req.Description)
		if err != nil {
			tx.Rollback()
			return InternalError(err)
		}
	}

	// Optimize for description-only changes
	if reflect.DeepEqual(profile.Config, req.Config) && reflect.DeepEqual(profile.Devices, req.Devices) {
		err = txCommit(tx)
		if err != nil {
			return InternalError(err)
		}

		return EmptySyncResponse
	}

	err = dbProfileConfigClear(tx, id)
	if err != nil {
		tx.Rollback()
		return InternalError(err)
	}

	err = dbProfileConfigAdd(tx, id, req.Config)
	if err != nil {
		tx.Rollback()
		return SmartError(err)
	}

	err = dbDevicesAdd(tx, "profile", id, req.Devices)
	if err != nil {
		tx.Rollback()
		return SmartError(err)
	}

	err = txCommit(tx)
	if err != nil {
		return InternalError(err)
	}

	// Update all the containers using the profile. Must be done after txCommit due to DB lock.
	failures := map[string]error{}
	for _, c := range containers {
		err = c.Update(containerArgs{
			Architecture: c.Architecture(),
			Ephemeral:    c.IsEphemeral(),
			Config:       c.LocalConfig(),
			Devices:      c.LocalDevices(),
			Profiles:     c.Profiles()}, true)

		if err != nil {
			failures[c.Name()] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return InternalError(fmt.Errorf("%s", msg))
	}

	return EmptySyncResponse
}

// The handler for the post operation.
func profilePost(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	req := profilesPostReq{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return BadRequest(err)
	}

	// Sanity checks
	if req.Name == "" {
		return BadRequest(fmt.Errorf("No name provided"))
	}

	err := dbProfileUpdate(d.db, name, req.Name)
	if err != nil {
		return InternalError(err)
	}

	return EmptySyncResponse
}

// The handler for the delete operation.
func profileDelete(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	_, err := doProfileGet(d, name)
	if err != nil {
		return SmartError(err)
	}

	err = dbProfileDelete(d.db, name)
	if err != nil {
		return SmartError(err)
	}

	return EmptySyncResponse
}

var profileCmd = Command{name: "profiles/{name}", get: profileGet, put: profilePut, delete: profileDelete, post: profilePost}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package examples_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/yaml"
	"github.com/golang/glog"
)

func validateObject(obj runtime.Object) (errors []error) {
	switch t := obj.(type) {
	case *api.ReplicationController:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidateReplicationController(t)
	case *api.ReplicationControllerList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Service:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidateService(t)
	case *api.ServiceList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Pod:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidatePod(t)
	case *api.PodList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.PersistentVolume:
		errors = validation.ValidatePersistentVolume(t)
	case *api.PersistentVolumeClaim:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidatePersistentVolumeClaim(t)
	case *api.PodTemplate:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidatePodTemplate(t)
	case *api.Endpoints:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidateEndpoints(t)
	default:
		return []error{fmt.Errorf("no validation defined for %#v", obj)}
	}
	return errors
}

func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error {
	return filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.IsDir() && path != inDir {
			return filepath.SkipDir
		}

		file := filepath.Base(path)
		if ext := filepath.Ext(file); ext == ".json" || ext == ".yaml" {
			glog.Infof("Testing %s", path)
			data, err := ioutil.ReadFile(path)
			if err != nil {
				return err
			}
			name := strings.TrimSuffix(file, ext)

			if ext == ".yaml" {
				out, err := yaml.ToJSON(data)
				if err != nil {
					return err
				}
				data = out
			}

			fn(name, path, data)
		}
		return nil
	})
}

func TestExampleObjectSchemas(t *testing.T) {
	cases := map[string]map[string]runtime.Object{
		"../cmd/integration": {
			"v1beta1-controller": &api.ReplicationController{},
			"v1beta3-controller": &api.ReplicationController{},
		},
		"../examples/guestbook": {
			"frontend-controller":     &api.ReplicationController{},
			"redis-slave-controller":  &api.ReplicationController{},
			"redis-master-controller": &api.ReplicationController{},
			"frontend-service":        &api.Service{},
			"redis-master-service":    &api.Service{},
			"redis-slave-service":     &api.Service{},
		},
		"../examples/guestbook-go": {
			"guestbook-controller":    &api.ReplicationController{},
			"redis-slave-controller":  &api.ReplicationController{},
			"redis-master-controller": &api.ReplicationController{},
			"guestbook-service":       &api.Service{},
			"redis-master-service":    &api.Service{},
			"redis-slave-service":     &api.Service{},
		},
		"../examples/walkthrough": {
			"pod1": &api.Pod{},
			"pod2": &api.Pod{},
			"pod-with-http-healthcheck": &api.Pod{},
			"service":                   &api.Service{},
			"replication-controller":    &api.ReplicationController{},
			"podtemplate":               &api.PodTemplate{},
		},
		"../examples/update-demo": {
			"kitten-rc":   &api.ReplicationController{},
			"nautilus-rc": &api.ReplicationController{},
		},
		"../examples/persistent-volumes/volumes": {
			"local-01": &api.PersistentVolume{},
			"local-02": &api.PersistentVolume{},
			"gce":      &api.PersistentVolume{},
			"nfs":      &api.PersistentVolume{},
		},
		"../examples/persistent-volumes/claims": {
			"claim-01": &api.PersistentVolumeClaim{},
			"claim-02": &api.PersistentVolumeClaim{},
			"claim-03": &api.PersistentVolumeClaim{},
		},
		"../examples/iscsi": {
			"iscsi": &api.Pod{},
		},
		"../examples/glusterfs": {
			"glusterfs-pod":       &api.Pod{},
			"glusterfs-endpoints": &api.Endpoints{},
		},
		"../examples/liveness": {
			"exec-liveness": &api.Pod{},
			"http-liveness": &api.Pod{},
		},
		"../examples": {
			"pod":         &api.Pod{},
			"replication": &api.ReplicationController{},
		},
		"../examples/rbd/v1beta3": {
			"rbd":             &api.Pod{},
			"rbd-with-secret": &api.Pod{},
		},
	}

	for path, expected := range cases {
		tested := 0
		err := walkJSONFiles(path, func(name, path string, data []byte) {
			expectedType, found := expected[name]
			if !found {
				t.Errorf("%s: %s does not have a test case defined", path, name)
				return
			}
			tested += 1
			if err := latest.Codec.DecodeInto(data, expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
				return
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
		})
		if err != nil {
			t.Errorf("Expected no error, Got %v", err)
		}
		if tested != len(expected) {
			t.Errorf("Expected %d examples, Got %d", len(expected), tested)
		}
	}
}

// This regex is tricky, but it works.  For future me, here is the decode:
//
// Flags: (?ms) = multiline match, allow . to match \n
// 1) Look for a line that starts with ``` (a markdown code block)
// 2) (?: ... ) = non-capturing group
// 3) (P<name>) = capture group as "name"
// 4) Look for #1 followed by either:
// 4a)    "yaml" followed by any word-characters followed by a newline (e.g. ```yamlfoo\n)
// 4b)    "any word-characters followed by a newline (e.g. ```json\n)
// 5) Look for either:
// 5a)    #4a followed by one or more characters (non-greedy)
// 5b)    #4b followed by { followed by one or more characters (non-greedy) followed by }
// 6) Look for #5 followed by a newline followed by ``` (end of the code block)
//
// This could probably be simplified, but is already too delicate.  Before any
// real changes, we should have a testscase that just tests this regex.
var sampleRegexp = regexp.MustCompile("(?ms)^```(?:(?P<type>yaml)\\w*\\n(?P<content>.+?)|\\w*\\n(?P<content>\\{.+?\\}))\\n^```")
var subsetRegexp = regexp.MustCompile("(?ms)\\.{3}")

func TestReadme(t *testing.T) {
	paths := []struct {
		file         string
		expectedType []runtime.Object
	}{
		{"../README.md", []runtime.Object{&api.Pod{}}},
		{"../examples/walkthrough/README.md", []runtime.Object{&api.Pod{}}},
		{"../examples/iscsi/README.md", []runtime.Object{&api.Pod{}}},
		{"../examples/simple-yaml.md", []runtime.Object{&api.Pod{}, &api.ReplicationController{}}},
	}

	for _, path := range paths {
		data, err := ioutil.ReadFile(path.file)
		if err != nil {
			t.Errorf("Unable to read file %s: %v", path, err)
			continue
		}

		matches := sampleRegexp.FindAllStringSubmatch(string(data), -1)
		if matches == nil {
			continue
		}
		ix := 0
		for _, match := range matches {
			var content, subtype string
			for i, name := range sampleRegexp.SubexpNames() {
				if name == "type" {
					subtype = match[i]
				}
				if name == "content" && match[i] != "" {
					content = match[i]
				}
			}
			if subtype == "yaml" && subsetRegexp.FindString(content) != "" {
				t.Logf("skipping (%s): \n%s", subtype, content)
				continue
			}

			var expectedType runtime.Object
			if len(path.expectedType) == 1 {
				expectedType = path.expectedType[0]
			} else {
				expectedType = path.expectedType[ix]
				ix++
			}
			json, err := yaml.ToJSON([]byte(content))
			if err != nil {
				t.Errorf("%s could not be converted to JSON: %v\n%s", path, err, string(content))
			}
			if err := latest.Codec.DecodeInto(json, expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(content))
				continue
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
			_, err = latest.Codec.Encode(expectedType)
			if err != nil {
				t.Errorf("Could not encode object: %v", err)
				continue
			}
		}
	}
}
<commit_msg>validate all examples<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package examples_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/yaml"
	"github.com/golang/glog"
)

func validateObject(obj runtime.Object) (errors []error) {
	switch t := obj.(type) {
	case *api.ReplicationController:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidateReplicationController(t)
	case *api.ReplicationControllerList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Service:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidateService(t)
	case *api.ServiceList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Pod:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidatePod(t)
	case *api.PodList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.PersistentVolume:
		errors = validation.ValidatePersistentVolume(t)
	case *api.PersistentVolumeClaim:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidatePersistentVolumeClaim(t)
	case *api.PodTemplate:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidatePodTemplate(t)
	case *api.Endpoints:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidateEndpoints(t)
	case *api.Namespace:
		errors = validation.ValidateNamespace(t)
	case *api.Secret:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidateSecret(t)
	case *api.LimitRange:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidateLimitRange(t)
	case *api.ResourceQuota:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidateResourceQuota(t)
	default:
		return []error{fmt.Errorf("no validation defined for %#v", obj)}
	}
	return errors
}

func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error {
	return filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.IsDir() && path != inDir {
			return filepath.SkipDir
		}

		file := filepath.Base(path)
		if ext := filepath.Ext(file); ext == ".json" || ext == ".yaml" {
			glog.Infof("Testing %s", path)
			data, err := ioutil.ReadFile(path)
			if err != nil {
				return err
			}
			name := strings.TrimSuffix(file, ext)

			if ext == ".yaml" {
				out, err := yaml.ToJSON(data)
				if err != nil {
					return fmt.Errorf("%s: %v", path, err)
				}
				data = out
			}

			fn(name, path, data)
		}
		return nil
	})
}

func TestExampleObjectSchemas(t *testing.T) {
	cases := map[string]map[string]runtime.Object{
		"../cmd/integration": {
			"v1beta1-controller": &api.ReplicationController{},
			"v1beta3-controller": &api.ReplicationController{},
		},
		"../examples/guestbook": {
			"frontend-controller":     &api.ReplicationController{},
			"redis-slave-controller":  &api.ReplicationController{},
			"redis-master-controller": &api.ReplicationController{},
			"frontend-service":        &api.Service{},
			"redis-master-service":    &api.Service{},
			"redis-slave-service":     &api.Service{},
		},
		"../examples/guestbook-go": {
			"guestbook-controller":    &api.ReplicationController{},
			"redis-slave-controller":  &api.ReplicationController{},
			"redis-master-controller": &api.ReplicationController{},
			"guestbook-service":       &api.Service{},
			"redis-master-service":    &api.Service{},
			"redis-slave-service":     &api.Service{},
		},
		"../examples/walkthrough": {
			"pod1": &api.Pod{},
			"pod2": &api.Pod{},
			"pod-with-http-healthcheck": &api.Pod{},
			"service":                   &api.Service{},
			"replication-controller":    &api.ReplicationController{},
			"podtemplate":               &api.PodTemplate{},
		},
		"../examples/update-demo": {
			"kitten-rc":   &api.ReplicationController{},
			"nautilus-rc": &api.ReplicationController{},
		},
		"../examples/persistent-volumes/volumes": {
			"local-01": &api.PersistentVolume{},
			"local-02": &api.PersistentVolume{},
			"gce":      &api.PersistentVolume{},
			"nfs":      &api.PersistentVolume{},
		},
		"../examples/persistent-volumes/claims": {
			"claim-01": &api.PersistentVolumeClaim{},
			"claim-02": &api.PersistentVolumeClaim{},
			"claim-03": &api.PersistentVolumeClaim{},
		},
		"../examples/persistent-volumes/simpletest": {
			"namespace": &api.Namespace{},
			"pod":       &api.Pod{},
			"service":   &api.Service{},
		},
		"../examples/iscsi": {
			"iscsi": &api.Pod{},
		},
		"../examples/glusterfs": {
			"glusterfs-pod":       &api.Pod{},
			"glusterfs-endpoints": &api.Endpoints{},
		},
		"../examples/liveness": {
			"exec-liveness": &api.Pod{},
			"http-liveness": &api.Pod{},
		},
		"../examples": {
			"pod":         &api.Pod{},
			"replication": &api.ReplicationController{},
		},
		"../examples/rbd/secret": {
			"ceph-secret": &api.Secret{},
		},
		"../examples/rbd/v1beta3": {
			"rbd":             &api.Pod{},
			"rbd-with-secret": &api.Pod{},
		},
		"../examples/cassandra": {
			"cassandra-controller": &api.ReplicationController{},
			"cassandra-service":    &api.Service{},
			"cassandra":            &api.Pod{},
		},
		"../examples/celery-rabbitmq": {
			"celery-controller":   &api.ReplicationController{},
			"flower-controller":   &api.ReplicationController{},
			"rabbitmq-controller": &api.ReplicationController{},
			"rabbitmq-service":    &api.Service{},
		},
		"../examples/cluster-dns": {
			"dns-backend-rc":      &api.ReplicationController{},
			"dns-backend-service": &api.Service{},
			"dns-frontend-pod":    &api.Pod{},
			"namespace-dev":       &api.Namespace{},
			"namespace-prod":      &api.Namespace{},
		},
		"../examples/downward-api": {
			"dapi-pod": &api.Pod{},
		},
		"../examples/elasticsearch": {
			"apiserver-secret": nil,
			"music-rc":         &api.ReplicationController{},
			"music-service":    &api.Service{},
		},
		"../examples/explorer": {
			"pod": &api.Pod{},
		},
		"../examples/hazelcast": {
			"hazelcast-controller": &api.ReplicationController{},
			"hazelcast-service":    &api.Service{},
		},
		"../examples/kubernetes-namespaces": {
			"namespace-dev":  &api.Namespace{},
			"namespace-prod": &api.Namespace{},
		},
		"../examples/limitrange": {
			"invalid-pod": &api.Pod{},
			"limit-range": &api.LimitRange{},
			"valid-pod":   &api.Pod{},
		},
		"../examples/logging-demo": {
			"synthetic_0_25lps": &api.Pod{},
			"synthetic_10lps":   &api.Pod{},
		},
		"../examples/meteor": {
			"meteor-controller": &api.ReplicationController{},
			"meteor-service":    &api.Service{},
			"mongo-pod":         &api.Pod{},
			"mongo-service":     &api.Service{},
		},
		"../examples/mysql-wordpress-pd": {
			"mysql-service":     &api.Service{},
			"mysql":             &api.Pod{},
			"wordpress-service": &api.Service{},
			"wordpress":         &api.Pod{},
		},
		"../examples/nfs": {
			"nfs-server-pod":     &api.Pod{},
			"nfs-server-service": &api.Service{},
			"nfs-web-pod":        &api.Pod{},
		},
		"../examples/node-selection": {
			"pod": &api.Pod{},
		},
		"../examples/openshift-origin": {
			"openshift-controller": &api.ReplicationController{},
			"openshift-service":    &api.Service{},
		},
		"../examples/phabricator": {
			"authenticator-controller": &api.ReplicationController{},
			"phabricator-controller":   &api.ReplicationController{},
			"phabricator-service":      &api.Service{},
		},
		"../examples/redis": {
			"redis-controller":          &api.ReplicationController{},
			"redis-master":              &api.Pod{},
			"redis-proxy":               &api.Pod{},
			"redis-sentinel-controller": &api.ReplicationController{},
			"redis-sentinel-service":    &api.Service{},
		},
		"../examples/resourcequota": {
			"resource-quota": &api.ResourceQuota{},
		},
		"../examples/rethinkdb": {
			"admin-pod":      &api.Pod{},
			"admin-service":  &api.Service{},
			"driver-service": &api.Service{},
			"rc":             &api.ReplicationController{},
		},
		"../examples/secrets": {
			"secret-pod": &api.Pod{},
			"secret":     &api.Secret{},
		},
		"../examples/spark": {
			"spark-master-service":    &api.Service{},
			"spark-master":            &api.Pod{},
			"spark-worker-controller": &api.ReplicationController{},
		},
		"../examples/storm": {
			"storm-nimbus-service":    &api.Service{},
			"storm-nimbus":            &api.Pod{},
			"storm-worker-controller": &api.ReplicationController{},
			"zookeeper-service":       &api.Service{},
			"zookeeper":               &api.Pod{},
		},
	}

	capabilities.SetForTests(capabilities.Capabilities{
		AllowPrivileged: true,
	})

	for path, expected := range cases {
		tested := 0
		err := walkJSONFiles(path, func(name, path string, data []byte) {
			expectedType, found := expected[name]
			if !found {
				t.Errorf("%s: %s does not have a test case defined", path, name)
				return
			}
			tested++
			if expectedType == nil {
				t.Logf("skipping : %s/%s\n", path, name)
				return
			}
			if err := latest.Codec.DecodeInto(data, expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
				return
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
		})
		if err != nil {
			t.Errorf("Expected no error, Got %v", err)
		}
		if tested != len(expected) {
			t.Errorf("Expected %d examples, Got %d", len(expected), tested)
		}
	}
}

// This regex is tricky, but it works.  For future me, here is the decode:
//
// Flags: (?ms) = multiline match, allow . to match \n
// 1) Look for a line that starts with ``` (a markdown code block)
// 2) (?: ... ) = non-capturing group
// 3) (P<name>) = capture group as "name"
// 4) Look for #1 followed by either:
// 4a)    "yaml" followed by any word-characters followed by a newline (e.g. ```yamlfoo\n)
// 4b)    "any word-characters followed by a newline (e.g. ```json\n)
// 5) Look for either:
// 5a)    #4a followed by one or more characters (non-greedy)
// 5b)    #4b followed by { followed by one or more characters (non-greedy) followed by }
// 6) Look for #5 followed by a newline followed by ``` (end of the code block)
//
// This could probably be simplified, but is already too delicate.  Before any
// real changes, we should have a testscase that just tests this regex.
var sampleRegexp = regexp.MustCompile("(?ms)^```(?:(?P<type>yaml)\\w*\\n(?P<content>.+?)|\\w*\\n(?P<content>\\{.+?\\}))\\n^```")
var subsetRegexp = regexp.MustCompile("(?ms)\\.{3}")

func TestReadme(t *testing.T) {
	paths := []struct {
		file         string
		expectedType []runtime.Object
	}{
		{"../README.md", []runtime.Object{&api.Pod{}}},
		{"../examples/walkthrough/README.md", []runtime.Object{&api.Pod{}}},
		{"../examples/iscsi/README.md", []runtime.Object{&api.Pod{}}},
		{"../examples/simple-yaml.md", []runtime.Object{&api.Pod{}, &api.ReplicationController{}}},
	}

	for _, path := range paths {
		data, err := ioutil.ReadFile(path.file)
		if err != nil {
			t.Errorf("Unable to read file %s: %v", path, err)
			continue
		}

		matches := sampleRegexp.FindAllStringSubmatch(string(data), -1)
		if matches == nil {
			continue
		}
		ix := 0
		for _, match := range matches {
			var content, subtype string
			for i, name := range sampleRegexp.SubexpNames() {
				if name == "type" {
					subtype = match[i]
				}
				if name == "content" && match[i] != "" {
					content = match[i]
				}
			}
			if subtype == "yaml" && subsetRegexp.FindString(content) != "" {
				t.Logf("skipping (%s): \n%s", subtype, content)
				continue
			}

			var expectedType runtime.Object
			if len(path.expectedType) == 1 {
				expectedType = path.expectedType[0]
			} else {
				expectedType = path.expectedType[ix]
				ix++
			}
			json, err := yaml.ToJSON([]byte(content))
			if err != nil {
				t.Errorf("%s could not be converted to JSON: %v\n%s", path, err, string(content))
			}
			if err := latest.Codec.DecodeInto(json, expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(content))
				continue
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
			_, err = latest.Codec.Encode(expectedType)
			if err != nil {
				t.Errorf("Could not encode object: %v", err)
				continue
			}
		}
	}
}
<|endoftext|>