<commit_before>//  Copyright (c) 2015 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package objectserver

import (
	"crypto/md5"
	"encoding/hex"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	hummingbird "hummingbird/common"
)

const METADATA_CHUNK_SIZE = 65536

func GetXAttr(fileNameOrFd interface{}, attr string, value []byte) (int, error) {
	switch v := fileNameOrFd.(type) {
	case string:
		return syscall.Getxattr(v, attr, value)
	case uintptr:
		return hummingbird.FGetXattr(v, attr, value)
	}
	return 0, &hummingbird.BackendError{Err: errors.New("Invalid fileNameOrFd"), Code: hummingbird.UnhandledError}
}

func RawReadMetadata(fileNameOrFd interface{}) ([]byte, error) {
	var pickledMetadata []byte
	offset := 0
	for index := 0; ; index += 1 {
		var metadataName string
		// get name of next xattr
		if index == 0 {
			metadataName = "user.swift.metadata"
		} else {
			metadataName = "user.swift.metadata" + strconv.Itoa(index)
		}
		// get size of xattr
		length, _ := GetXAttr(fileNameOrFd, metadataName, nil)
		if length <= 0 {
			break
		}
		// grow buffer to hold xattr
		for cap(pickledMetadata) < offset+length {
			pickledMetadata = append(pickledMetadata, 0)
		}
		pickledMetadata = pickledMetadata[0 : offset+length]
		if _, err := GetXAttr(fileNameOrFd, metadataName, pickledMetadata[offset:]); err != nil {
			return nil, err
		}
		offset += length
	}
	return pickledMetadata, nil
}

func ReadMetadata(fileNameOrFd interface{}) (map[interface{}]interface{}, error) {
	pickledMetadata, err := RawReadMetadata(fileNameOrFd)
	if err != nil {
		return nil, err
	}
	v, err := hummingbird.PickleLoads(pickledMetadata)
	if err != nil {
		return nil, err
	}
	return v.(map[interface{}]interface{}), nil
}

func RawWriteMetadata(fd uintptr, buf []byte) error {
	for index := 0; len(buf) > 0; index++ {
		var metadataName string
		if index == 0 {
			metadataName = "user.swift.metadata"
		} else {
			metadataName = "user.swift.metadata" + strconv.Itoa(index)
		}
		writelen := METADATA_CHUNK_SIZE
		if len(buf) < writelen {
			writelen = len(buf)
		}
		if _, err := hummingbird.FSetXattr(fd, metadataName, buf[0:writelen]); err != nil {
			return err
		}
		buf = buf[writelen:len(buf)]
	}
	return nil
}

func WriteMetadata(fd uintptr, v map[string]string) error {
	return RawWriteMetadata(fd, hummingbird.PickleDumps(v))
}

func QuarantineHash(hashDir string) error {
	// FYI- this does not invalidate the hash like swift's version. Please
	// do that yourself
	hash := filepath.Base(hashDir)
	//          drive        objects      partition    suffix       hash
	driveDir := filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(hashDir))))
	// TODO: this will need to be slightly more complicated once policies
	quarantineDir := filepath.Join(driveDir, "quarantined", "objects")
	if err := os.MkdirAll(quarantineDir, 0770); err != nil {
		return err
	}
	destDir := filepath.Join(quarantineDir, hash+"-"+hummingbird.UUID())
	if err := os.Rename(hashDir, destDir); err != nil {
		return err
	}
	return nil
}

func InvalidateHash(hashDir string) {
	// TODO: return errors
	suffDir := filepath.Dir(hashDir)
	partitionDir := filepath.Dir(suffDir)
	partitionLock, err := hummingbird.LockPath(partitionDir, 10)
	if err != nil {
		return
	}
	defer partitionLock.Close()
	pklFile := partitionDir + "/hashes.pkl"
	data, err := ioutil.ReadFile(pklFile)
	if err != nil {
		return
	}
	v, _ := hummingbird.PickleLoads(data)
	suffixDirSplit := strings.Split(suffDir, "/")
	suffix := suffixDirSplit[len(suffixDirSplit)-1]
	if current, ok := v.(map[interface{}]interface{})[suffix]; ok && current == nil {
		return
	}
	v.(map[interface{}]interface{})[suffix] = nil
	hummingbird.WriteFileAtomic(pklFile, hummingbird.PickleDumps(v), 0600)
}

func HashCleanupListDir(hashDir string, logger hummingbird.LoggingContext) ([]string, *hummingbird.BackendError) {
	fileList, err := hummingbird.ReadDirNames(hashDir)
	returnList := []string{}
	if err != nil {

		if os.IsNotExist(err) {
			return returnList, nil
		}
		if hummingbird.IsNotDir(err) {
			return returnList, &hummingbird.BackendError{Err: err, Code: hummingbird.PathNotDirErrorCode}
		}
		return returnList, &hummingbird.BackendError{Err: err, Code: hummingbird.OsErrorCode}
	}
	deleteRest := false
	deleteRestMeta := false
	if len(fileList) == 1 {
		filename := fileList[0]
		if strings.HasSuffix(filename, ".ts") {
			withoutSuffix := strings.Split(filename, ".")[0]
			if strings.Contains(withoutSuffix, "_") {
				withoutSuffix = strings.Split(withoutSuffix, "_")[0]
			}
			timestamp, _ := strconv.ParseFloat(withoutSuffix, 64)
			if time.Now().Unix()-int64(timestamp) > int64(hummingbird.ONE_WEEK) {
				os.RemoveAll(hashDir + "/" + filename)
				return returnList, nil
			}
		}
		returnList = append(returnList, filename)
	} else {
		for index := len(fileList) - 1; index >= 0; index-- {
			filename := fileList[index]
			if deleteRest {
				os.RemoveAll(hashDir + "/" + filename)
			} else {
				if strings.HasSuffix(filename, ".meta") {
					if deleteRestMeta {
						os.RemoveAll(hashDir + "/" + filename)
						continue
					}
					deleteRestMeta = true
				}
				if strings.HasSuffix(filename, ".ts") || strings.HasSuffix(filename, ".data") {
					// TODO: check .ts time for expiration
					deleteRest = true
				}
				returnList = append(returnList, filename)
			}
		}
	}
	return returnList, nil
}

func RecalculateSuffixHash(suffixDir string, logger hummingbird.LoggingContext) (string, *hummingbird.BackendError) {
	// the is hash_suffix in swift
	h := md5.New()

	hashList, err := hummingbird.ReadDirNames(suffixDir)
	if err != nil {
		if hummingbird.IsNotDir(err) {
			return "", &hummingbird.BackendError{Err: err, Code: hummingbird.PathNotDirErrorCode}
		}
		return "", &hummingbird.BackendError{Err: err, Code: hummingbird.OsErrorCode}
	}
	for _, fullHash := range hashList {
		hashPath := suffixDir + "/" + fullHash
		fileList, err := HashCleanupListDir(hashPath, logger)
		if err != nil {
			if err.Code == hummingbird.PathNotDirErrorCode {
				if QuarantineHash(hashPath) == nil {
					InvalidateHash(hashPath)
				}
				continue
			}
			return "", err
		}
		if len(fileList) > 0 {
			for _, fileName := range fileList {
				io.WriteString(h, fileName)
			}
		} else {
			os.Remove(hashPath) // leaves the suffix (swift removes it but who cares)
		}
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}

func GetHashes(driveRoot string, device string, partition string, recalculate []string, logger hummingbird.LoggingContext) (map[string]string, *hummingbird.BackendError) {
	partitionDir := filepath.Join(driveRoot, device, "objects", partition)
	pklFile := filepath.Join(partitionDir, "hashes.pkl")

	modified := false
	mtime := int64(-1)
	hashes := make(map[string]string, 4096)
	lsForSuffixes := true
	if data, err := ioutil.ReadFile(pklFile); err == nil {
		if v, err := hummingbird.PickleLoads(data); err == nil {
			if pickledHashes, ok := v.(map[interface{}]interface{}); ok {
				if fileInfo, err := os.Stat(pklFile); err == nil {
					mtime = fileInfo.ModTime().Unix()
					lsForSuffixes = false
					for suff, hash := range pickledHashes {
						if hashes[suff.(string)], ok = hash.(string); !ok {
							hashes[suff.(string)] = ""
						}
					}
				}
			}
		}
	}
	if lsForSuffixes {
		// couldn't load hashes pickle, start building new one
		suffs, _ := hummingbird.ReadDirNames(partitionDir)

		for _, suffName := range suffs {
			if len(suffName) == 3 && hashes[suffName] == "" {
				hashes[suffName] = ""
			}
		}
	}
	for _, suffix := range recalculate {
		if len(suffix) == 3 {
			hashes[suffix] = ""
		}
	}
	for suffix, hash := range hashes {
		if hash == "" {
			modified = true
			suffixDir := driveRoot + "/" + device + "/objects/" + partition + "/" + suffix
			recalc_hash, err := RecalculateSuffixHash(suffixDir, logger)
			if err == nil {
				hashes[suffix] = recalc_hash
			} else {
				switch {
				case err.Code == hummingbird.PathNotDirErrorCode:
					delete(hashes, suffix)
				case err.Code == hummingbird.OsErrorCode:
					logger.LogError("Error hashing suffix: %s/%s (%s)", partitionDir, suffix, "asdf")
				}
			}
		}
	}
	if modified {
		partitionLock, err := hummingbird.LockPath(partitionDir, 10)
		defer partitionLock.Close()
		if err != nil {
			return nil, &hummingbird.BackendError{Err: err, Code: hummingbird.LockPathError}
		} else {
			fileInfo, err := os.Stat(pklFile)
			if lsForSuffixes || os.IsNotExist(err) || mtime == fileInfo.ModTime().Unix() {
				hummingbird.WriteFileAtomic(pklFile, hummingbird.PickleDumps(hashes), 0600)
				return hashes, nil
			}
			logger.LogError("Made recursive call to GetHashes: %s", partitionDir)
			return GetHashes(driveRoot, device, partition, recalculate, logger)
		}
	}
	return hashes, nil
}

func ObjHashDir(vars map[string]string, driveRoot string, hashPathPrefix string, hashPathSuffix string) string {
	h := md5.New()
	io.WriteString(h, hashPathPrefix+"/"+vars["account"]+"/"+vars["container"]+"/"+vars["obj"]+hashPathSuffix)
	hexHash := hex.EncodeToString(h.Sum(nil))
	suffix := hexHash[29:32]
	return driveRoot + "/" + vars["device"] + "/objects/" + vars["partition"] + "/" + suffix + "/" + hexHash
}

func ObjectFiles(directory string) (string, string) {
	fileList, err := hummingbird.ReadDirNames(directory)
	metaFile := ""
	if err != nil {
		return "", ""
	}
	for index := len(fileList) - 1; index >= 0; index-- {
		filename := fileList[index]
		if strings.HasSuffix(filename, ".meta") {
			metaFile = filename
		}
		if strings.HasSuffix(filename, ".ts") || strings.HasSuffix(filename, ".data") {
			if metaFile != "" {
				return filepath.Join(directory, filename), filepath.Join(directory, metaFile)
			} else {
				return filepath.Join(directory, filename), ""
			}
		}
	}
	return "", ""
}

func ObjTempFile(vars map[string]string, driveRoot, prefix string) (*os.File, error) {
	tempDir := driveRoot + "/" + vars["device"] + "/" + "tmp"
	if err := os.MkdirAll(tempDir, 0770); err != nil {
		return nil, err
	}
	return ioutil.TempFile(tempDir, prefix)
}

func applyMetaFile(metaFile string, datafileMetadata map[interface{}]interface{}) (map[interface{}]interface{}, error) {
	if metadata, err := ReadMetadata(metaFile); err != nil {
		return nil, err
	} else {
		for k, v := range datafileMetadata {
			if k == "Content-Length" || k == "Content-Type" || k == "deleted" || k == "ETag" || strings.HasPrefix(k.(string), "X-Object-Sysmeta-") {
				metadata[k] = v
			}
		}
		return metadata, nil
	}
}

func OpenObjectMetadata(fd uintptr, metaFile string) (map[interface{}]interface{}, error) {
	pickledMetadata, err := RawReadMetadata(fd)
	if err != nil {
		return nil, err
	}
	v, err := hummingbird.PickleLoads(pickledMetadata)
	if err != nil {
		return nil, err
	}
	datafileMetadata, ok := v.(map[interface{}]interface{})
	if !ok {
		return nil, errors.New("Metadata unpickled to wrong type.")
	}
	if metaFile != "" {
		return applyMetaFile(metaFile, datafileMetadata)
	}
	return datafileMetadata, nil
}

func ObjectMetadata(dataFile string, metaFile string) (map[interface{}]interface{}, error) {
	datafileMetadata, err := ReadMetadata(dataFile)
	if err != nil {
		return nil, err
	}
	if metaFile != "" {
		return applyMetaFile(metaFile, datafileMetadata)
	}
	return datafileMetadata, nil
}

func FreeDiskSpace(fd uintptr) (int64, error) {
	var st syscall.Statfs_t
	if err := syscall.Fstatfs(int(fd), &st); err != nil {
		return 0, err
	} else {
		return int64(st.Frsize) * int64(st.Bavail), nil
	}
}
<commit_msg>ls for suffixes 10% of the time<commit_after>//  Copyright (c) 2015 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package objectserver

import (
	"crypto/md5"
	"encoding/hex"
	"errors"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	hummingbird "hummingbird/common"
)

const METADATA_CHUNK_SIZE = 65536

func GetXAttr(fileNameOrFd interface{}, attr string, value []byte) (int, error) {
	switch v := fileNameOrFd.(type) {
	case string:
		return syscall.Getxattr(v, attr, value)
	case uintptr:
		return hummingbird.FGetXattr(v, attr, value)
	}
	return 0, &hummingbird.BackendError{Err: errors.New("Invalid fileNameOrFd"), Code: hummingbird.UnhandledError}
}

func RawReadMetadata(fileNameOrFd interface{}) ([]byte, error) {
	var pickledMetadata []byte
	offset := 0
	for index := 0; ; index += 1 {
		var metadataName string
		// get name of next xattr
		if index == 0 {
			metadataName = "user.swift.metadata"
		} else {
			metadataName = "user.swift.metadata" + strconv.Itoa(index)
		}
		// get size of xattr
		length, _ := GetXAttr(fileNameOrFd, metadataName, nil)
		if length <= 0 {
			break
		}
		// grow buffer to hold xattr
		for cap(pickledMetadata) < offset+length {
			pickledMetadata = append(pickledMetadata, 0)
		}
		pickledMetadata = pickledMetadata[0 : offset+length]
		if _, err := GetXAttr(fileNameOrFd, metadataName, pickledMetadata[offset:]); err != nil {
			return nil, err
		}
		offset += length
	}
	return pickledMetadata, nil
}

func ReadMetadata(fileNameOrFd interface{}) (map[interface{}]interface{}, error) {
	pickledMetadata, err := RawReadMetadata(fileNameOrFd)
	if err != nil {
		return nil, err
	}
	v, err := hummingbird.PickleLoads(pickledMetadata)
	if err != nil {
		return nil, err
	}
	return v.(map[interface{}]interface{}), nil
}

func RawWriteMetadata(fd uintptr, buf []byte) error {
	for index := 0; len(buf) > 0; index++ {
		var metadataName string
		if index == 0 {
			metadataName = "user.swift.metadata"
		} else {
			metadataName = "user.swift.metadata" + strconv.Itoa(index)
		}
		writelen := METADATA_CHUNK_SIZE
		if len(buf) < writelen {
			writelen = len(buf)
		}
		if _, err := hummingbird.FSetXattr(fd, metadataName, buf[0:writelen]); err != nil {
			return err
		}
		buf = buf[writelen:len(buf)]
	}
	return nil
}

func WriteMetadata(fd uintptr, v map[string]string) error {
	return RawWriteMetadata(fd, hummingbird.PickleDumps(v))
}

func QuarantineHash(hashDir string) error {
	// FYI- this does not invalidate the hash like swift's version. Please
	// do that yourself
	hash := filepath.Base(hashDir)
	//          drive        objects      partition    suffix       hash
	driveDir := filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(hashDir))))
	// TODO: this will need to be slightly more complicated once policies
	quarantineDir := filepath.Join(driveDir, "quarantined", "objects")
	if err := os.MkdirAll(quarantineDir, 0770); err != nil {
		return err
	}
	destDir := filepath.Join(quarantineDir, hash+"-"+hummingbird.UUID())
	if err := os.Rename(hashDir, destDir); err != nil {
		return err
	}
	return nil
}

func InvalidateHash(hashDir string) {
	// TODO: return errors
	suffDir := filepath.Dir(hashDir)
	partitionDir := filepath.Dir(suffDir)
	partitionLock, err := hummingbird.LockPath(partitionDir, 10)
	if err != nil {
		return
	}
	defer partitionLock.Close()
	pklFile := partitionDir + "/hashes.pkl"
	data, err := ioutil.ReadFile(pklFile)
	if err != nil {
		return
	}
	v, _ := hummingbird.PickleLoads(data)
	suffixDirSplit := strings.Split(suffDir, "/")
	suffix := suffixDirSplit[len(suffixDirSplit)-1]
	if current, ok := v.(map[interface{}]interface{})[suffix]; ok && (current == nil || current == "") {
		return
	}
	v.(map[interface{}]interface{})[suffix] = nil
	hummingbird.WriteFileAtomic(pklFile, hummingbird.PickleDumps(v), 0600)
}

func HashCleanupListDir(hashDir string, logger hummingbird.LoggingContext) ([]string, *hummingbird.BackendError) {
	fileList, err := hummingbird.ReadDirNames(hashDir)
	returnList := []string{}
	if err != nil {

		if os.IsNotExist(err) {
			return returnList, nil
		}
		if hummingbird.IsNotDir(err) {
			return returnList, &hummingbird.BackendError{Err: err, Code: hummingbird.PathNotDirErrorCode}
		}
		return returnList, &hummingbird.BackendError{Err: err, Code: hummingbird.OsErrorCode}
	}
	deleteRest := false
	deleteRestMeta := false
	if len(fileList) == 1 {
		filename := fileList[0]
		if strings.HasSuffix(filename, ".ts") {
			withoutSuffix := strings.Split(filename, ".")[0]
			if strings.Contains(withoutSuffix, "_") {
				withoutSuffix = strings.Split(withoutSuffix, "_")[0]
			}
			timestamp, _ := strconv.ParseFloat(withoutSuffix, 64)
			if time.Now().Unix()-int64(timestamp) > int64(hummingbird.ONE_WEEK) {
				os.RemoveAll(hashDir + "/" + filename)
				return returnList, nil
			}
		}
		returnList = append(returnList, filename)
	} else {
		for index := len(fileList) - 1; index >= 0; index-- {
			filename := fileList[index]
			if deleteRest {
				os.RemoveAll(hashDir + "/" + filename)
			} else {
				if strings.HasSuffix(filename, ".meta") {
					if deleteRestMeta {
						os.RemoveAll(hashDir + "/" + filename)
						continue
					}
					deleteRestMeta = true
				}
				if strings.HasSuffix(filename, ".ts") || strings.HasSuffix(filename, ".data") {
					// TODO: check .ts time for expiration
					deleteRest = true
				}
				returnList = append(returnList, filename)
			}
		}
	}
	return returnList, nil
}

func RecalculateSuffixHash(suffixDir string, logger hummingbird.LoggingContext) (string, *hummingbird.BackendError) {
	// the is hash_suffix in swift
	h := md5.New()

	hashList, err := hummingbird.ReadDirNames(suffixDir)
	if err != nil {
		if hummingbird.IsNotDir(err) {
			return "", &hummingbird.BackendError{Err: err, Code: hummingbird.PathNotDirErrorCode}
		}
		return "", &hummingbird.BackendError{Err: err, Code: hummingbird.OsErrorCode}
	}
	for _, fullHash := range hashList {
		hashPath := suffixDir + "/" + fullHash
		fileList, err := HashCleanupListDir(hashPath, logger)
		if err != nil {
			if err.Code == hummingbird.PathNotDirErrorCode {
				if QuarantineHash(hashPath) == nil {
					InvalidateHash(hashPath)
				}
				continue
			}
			return "", err
		}
		if len(fileList) > 0 {
			for _, fileName := range fileList {
				io.WriteString(h, fileName)
			}
		} else {
			os.Remove(hashPath) // leaves the suffix (swift removes it but who cares)
		}
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}

func GetHashes(driveRoot string, device string, partition string, recalculate []string, logger hummingbird.LoggingContext) (map[string]string, *hummingbird.BackendError) {
	partitionDir := filepath.Join(driveRoot, device, "objects", partition)
	pklFile := filepath.Join(partitionDir, "hashes.pkl")

	modified := false
	mtime := int64(-1)
	hashes := make(map[string]string, 4096)
	lsForSuffixes := true
	if data, err := ioutil.ReadFile(pklFile); err == nil {
		if v, err := hummingbird.PickleLoads(data); err == nil {
			if pickledHashes, ok := v.(map[interface{}]interface{}); ok {
				if fileInfo, err := os.Stat(pklFile); err == nil {
					mtime = fileInfo.ModTime().Unix()
					lsForSuffixes = false
					for suff, hash := range pickledHashes {
						if hashes[suff.(string)], ok = hash.(string); !ok {
							hashes[suff.(string)] = ""
						}
					}
				}
			}
		}
	}
	// check occasionally to see if there are any suffixes not in the hashes.pkl
	if !lsForSuffixes && len(hashes) < 4096 {
		lsForSuffixes = rand.Int31n(10) == 0
	}
	if lsForSuffixes {
		// couldn't load hashes pickle, start building new one
		suffs, _ := hummingbird.ReadDirNames(partitionDir)

		for _, suffName := range suffs {
			if len(suffName) == 3 && hashes[suffName] == "" {
				hashes[suffName] = ""
			}
		}
	}
	for _, suffix := range recalculate {
		if len(suffix) == 3 {
			hashes[suffix] = ""
		}
	}
	for suffix, hash := range hashes {
		if hash == "" {
			modified = true
			suffixDir := driveRoot + "/" + device + "/objects/" + partition + "/" + suffix
			recalc_hash, err := RecalculateSuffixHash(suffixDir, logger)
			if err == nil {
				hashes[suffix] = recalc_hash
			} else {
				switch {
				case err.Code == hummingbird.PathNotDirErrorCode:
					delete(hashes, suffix)
				case err.Code == hummingbird.OsErrorCode:
					logger.LogError("Error hashing suffix: %s/%s (%s)", partitionDir, suffix, "asdf")
				}
			}
		}
	}
	if modified {
		partitionLock, err := hummingbird.LockPath(partitionDir, 10)
		defer partitionLock.Close()
		if err != nil {
			return nil, &hummingbird.BackendError{Err: err, Code: hummingbird.LockPathError}
		} else {
			fileInfo, err := os.Stat(pklFile)
			if lsForSuffixes || os.IsNotExist(err) || mtime == fileInfo.ModTime().Unix() {
				hummingbird.WriteFileAtomic(pklFile, hummingbird.PickleDumps(hashes), 0600)
				return hashes, nil
			}
			logger.LogError("Made recursive call to GetHashes: %s", partitionDir)
			return GetHashes(driveRoot, device, partition, recalculate, logger)
		}
	}
	return hashes, nil
}

func ObjHashDir(vars map[string]string, driveRoot string, hashPathPrefix string, hashPathSuffix string) string {
	h := md5.New()
	io.WriteString(h, hashPathPrefix+"/"+vars["account"]+"/"+vars["container"]+"/"+vars["obj"]+hashPathSuffix)
	hexHash := hex.EncodeToString(h.Sum(nil))
	suffix := hexHash[29:32]
	return driveRoot + "/" + vars["device"] + "/objects/" + vars["partition"] + "/" + suffix + "/" + hexHash
}

func ObjectFiles(directory string) (string, string) {
	fileList, err := hummingbird.ReadDirNames(directory)
	metaFile := ""
	if err != nil {
		return "", ""
	}
	for index := len(fileList) - 1; index >= 0; index-- {
		filename := fileList[index]
		if strings.HasSuffix(filename, ".meta") {
			metaFile = filename
		}
		if strings.HasSuffix(filename, ".ts") || strings.HasSuffix(filename, ".data") {
			if metaFile != "" {
				return filepath.Join(directory, filename), filepath.Join(directory, metaFile)
			} else {
				return filepath.Join(directory, filename), ""
			}
		}
	}
	return "", ""
}

func ObjTempFile(vars map[string]string, driveRoot, prefix string) (*os.File, error) {
	tempDir := driveRoot + "/" + vars["device"] + "/" + "tmp"
	if err := os.MkdirAll(tempDir, 0770); err != nil {
		return nil, err
	}
	return ioutil.TempFile(tempDir, prefix)
}

func applyMetaFile(metaFile string, datafileMetadata map[interface{}]interface{}) (map[interface{}]interface{}, error) {
	if metadata, err := ReadMetadata(metaFile); err != nil {
		return nil, err
	} else {
		for k, v := range datafileMetadata {
			if k == "Content-Length" || k == "Content-Type" || k == "deleted" || k == "ETag" || strings.HasPrefix(k.(string), "X-Object-Sysmeta-") {
				metadata[k] = v
			}
		}
		return metadata, nil
	}
}

func OpenObjectMetadata(fd uintptr, metaFile string) (map[interface{}]interface{}, error) {
	pickledMetadata, err := RawReadMetadata(fd)
	if err != nil {
		return nil, err
	}
	v, err := hummingbird.PickleLoads(pickledMetadata)
	if err != nil {
		return nil, err
	}
	datafileMetadata, ok := v.(map[interface{}]interface{})
	if !ok {
		return nil, errors.New("Metadata unpickled to wrong type.")
	}
	if metaFile != "" {
		return applyMetaFile(metaFile, datafileMetadata)
	}
	return datafileMetadata, nil
}

func ObjectMetadata(dataFile string, metaFile string) (map[interface{}]interface{}, error) {
	datafileMetadata, err := ReadMetadata(dataFile)
	if err != nil {
		return nil, err
	}
	if metaFile != "" {
		return applyMetaFile(metaFile, datafileMetadata)
	}
	return datafileMetadata, nil
}

func FreeDiskSpace(fd uintptr) (int64, error) {
	var st syscall.Statfs_t
	if err := syscall.Fstatfs(int(fd), &st); err != nil {
		return 0, err
	} else {
		return int64(st.Frsize) * int64(st.Bavail), nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package render

import (
	"bytes"
	"fmt"
	"io"
	"path"
	"sort"
	"strconv"
	"strings"

	"github.com/googlecodelabs/tools/claat/types"
)

// MD renders nodes as markdown for the target env.
func MD(env string, nodes ...types.Node) (string, error) {
	var buf bytes.Buffer
	if err := WriteMD(&buf, env, nodes...); err != nil {
		return "", err
	}
	return buf.String(), nil
}

// WriteMD does the same as MD but outputs rendered markup to w.
func WriteMD(w io.Writer, env string, nodes ...types.Node) error {
	mw := mdWriter{w: w, env: env}
	return mw.write(nodes...)
}

type mdWriter struct {
	w         io.Writer // output writer
	env       string    // target environment
	err       error     // error during any writeXxx methods
	lineStart bool
}

func (mw *mdWriter) writeBytes(b []byte) {
	if mw.err != nil {
		return
	}
	mw.lineStart = len(b) > 0 && b[len(b)-1] == '\n'
	_, mw.err = mw.w.Write(b)
}

func (mw *mdWriter) writeString(s string) {
	mw.writeBytes([]byte(s))
}

func (mw *mdWriter) space() {
	if !mw.lineStart {
		mw.writeString(" ")
	}
}

func (mw *mdWriter) newBlock() {
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
	mw.writeBytes(newLine)
}

func (mw *mdWriter) matchEnv(v []string) bool {
	if len(v) == 0 || mw.env == "" {
		return true
	}
	i := sort.SearchStrings(v, mw.env)
	return i < len(v) && v[i] == mw.env
}

func (mw *mdWriter) write(nodes ...types.Node) error {
	for _, n := range nodes {
		if !mw.matchEnv(n.Env()) {
			continue
		}
		switch n := n.(type) {
		case *types.TextNode:
			mw.text(n)
		case *types.ImageNode:
			mw.image(n)
		case *types.URLNode:
			mw.url(n)
		case *types.ButtonNode:
			mw.write(n.Content.Nodes...)
		case *types.CodeNode:
			mw.code(n)
		case *types.ListNode:
			mw.list(n)
		case *types.ImportNode:
			if len(n.Content.Nodes) == 0 {
				break
			}
			mw.write(n.Content.Nodes...)
		case *types.ItemsListNode:
			mw.itemsList(n)
		//case *types.GridNode:
		//	mw.grid(n)
		case *types.InfoboxNode:
			mw.infobox(n)
		//case *types.SurveyNode:
		//	mw.survey(n)
		case *types.HeaderNode:
			mw.header(n)
			//case *types.YouTubeNode:
			//	mw.youtube(n)
		}
		if mw.err != nil {
			return mw.err
		}
	}
	return nil
}

func (mw *mdWriter) text(n *types.TextNode) {
	if n.Bold {
		mw.writeString("**")
	}
	if n.Italic {
		mw.writeString(" *")
	}
	if n.Code {
		mw.writeString("`")
	}
	mw.writeString(n.Value)
	if n.Code {
		mw.writeString("`")
	}
	if n.Italic {
		mw.writeString("* ")
	}
	if n.Bold {
		mw.writeString("**")
	}
}

func (mw *mdWriter) image(n *types.ImageNode) {
	mw.space()
	mw.writeString(" 0 {
		mw.writeString(fmt.Sprintf(" width=\"%.2f\" ", n.Width))
	}

	mw.writeString("/>")
}

func (mw *mdWriter) url(n *types.URLNode) {
	mw.space()
	if n.URL != "" {
		mw.writeString("[")
	}
	for _, cn := range n.Content.Nodes {
		if t, ok := cn.(*types.TextNode); ok {
			mw.writeString(t.Value)
		}
	}
	if n.URL != "" {
		mw.writeString("](")
		mw.writeString(n.URL)
		mw.writeString(")")
	}
}

func (mw *mdWriter) code(n *types.CodeNode) {
	mw.newBlock()
	defer mw.writeBytes(newLine)
	if n.Term {
		var buf bytes.Buffer
		const prefix = "    "
		lineStart := true
		for _, r := range n.Value {
			if lineStart {
				buf.WriteString(prefix)
			}
			buf.WriteRune(r)
			lineStart = r == '\n'
		}
		mw.writeBytes(buf.Bytes())
		return
	}
	mw.writeString("```")
	mw.writeString(n.Lang)
	mw.writeBytes(newLine)
	mw.writeString(n.Value)
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
	mw.writeString("```")
}

func (mw *mdWriter) list(n *types.ListNode) {
	if n.Block() == true {
		mw.newBlock()
	}
	mw.write(n.Nodes...)
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
}

func (mw *mdWriter) itemsList(n *types.ItemsListNode) {
	mw.newBlock()
	for i, item := range n.Items {
		s := "* "
		if n.Type() == types.NodeItemsList && n.Start > 0 {
			s = strconv.Itoa(i+n.Start) + ". "
		}
		mw.writeString(s)
		mw.write(item.Nodes...)
		if !mw.lineStart {
			mw.writeBytes(newLine)
		}
	}
}

func (mw *mdWriter) infobox(n *types.InfoboxNode) {
	// TODO: This should use the "detail item" syntax so that it can be pure MD and not HTML
	// kind
	// : <content>
	//
	// The main issue is that when you do write(n.Content.Nodes...) it always adds two newlines
	// at the beginning.
	mw.newBlock()
	mw.writeString(`<aside class="`)
	mw.writeString(string(n.Kind))
	mw.writeString(`">`)
	mw.write(n.Content.Nodes...)
	mw.writeString("")
}

func (mw *mdWriter) header(n *types.HeaderNode) {
	mw.newBlock()
	mw.writeString(strings.Repeat("#", n.Level+1))
	mw.writeString(" ")
	mw.write(n.Content.Nodes...)
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
}
<commit_msg>Fix youtube nodes in md renderer<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package render

import (
	"bytes"
	"fmt"
	"io"
	"path"
	"sort"
	"strconv"
	"strings"

	"github.com/googlecodelabs/tools/claat/types"
)

// MD renders nodes as markdown for the target env.
func MD(env string, nodes ...types.Node) (string, error) {
	var buf bytes.Buffer
	if err := WriteMD(&buf, env, nodes...); err != nil {
		return "", err
	}
	return buf.String(), nil
}

// WriteMD does the same as MD but outputs rendered markup to w.
func WriteMD(w io.Writer, env string, nodes ...types.Node) error {
	mw := mdWriter{w: w, env: env}
	return mw.write(nodes...)
}

type mdWriter struct {
	w         io.Writer // output writer
	env       string    // target environment
	err       error     // error during any writeXxx methods
	lineStart bool
}

func (mw *mdWriter) writeBytes(b []byte) {
	if mw.err != nil {
		return
	}
	mw.lineStart = len(b) > 0 && b[len(b)-1] == '\n'
	_, mw.err = mw.w.Write(b)
}

func (mw *mdWriter) writeString(s string) {
	mw.writeBytes([]byte(s))
}

func (mw *mdWriter) space() {
	if !mw.lineStart {
		mw.writeString(" ")
	}
}

func (mw *mdWriter) newBlock() {
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
	mw.writeBytes(newLine)
}

func (mw *mdWriter) matchEnv(v []string) bool {
	if len(v) == 0 || mw.env == "" {
		return true
	}
	i := sort.SearchStrings(v, mw.env)
	return i < len(v) && v[i] == mw.env
}

func (mw *mdWriter) write(nodes ...types.Node) error {
	for _, n := range nodes {
		if !mw.matchEnv(n.Env()) {
			continue
		}
		switch n := n.(type) {
		case *types.TextNode:
			mw.text(n)
		case *types.ImageNode:
			mw.image(n)
		case *types.URLNode:
			mw.url(n)
		case *types.ButtonNode:
			mw.write(n.Content.Nodes...)
		case *types.CodeNode:
			mw.code(n)
		case *types.ListNode:
			mw.list(n)
		case *types.ImportNode:
			if len(n.Content.Nodes) == 0 {
				break
			}
			mw.write(n.Content.Nodes...)
		case *types.ItemsListNode:
			mw.itemsList(n)
		//case *types.GridNode:
		//	mw.grid(n)
		case *types.InfoboxNode:
			mw.infobox(n)
		//case *types.SurveyNode:
		//	mw.survey(n)
		case *types.HeaderNode:
			mw.header(n)
		case *types.YouTubeNode:
			mw.youtube(n)
		}
		if mw.err != nil {
			return mw.err
		}
	}
	return nil
}

func (mw *mdWriter) text(n *types.TextNode) {
	if n.Bold {
		mw.writeString("**")
	}
	if n.Italic {
		mw.writeString(" *")
	}
	if n.Code {
		mw.writeString("`")
	}
	mw.writeString(n.Value)
	if n.Code {
		mw.writeString("`")
	}
	if n.Italic {
		mw.writeString("* ")
	}
	if n.Bold {
		mw.writeString("**")
	}
}

func (mw *mdWriter) image(n *types.ImageNode) {
	mw.space()
	mw.writeString(" 0 {
		mw.writeString(fmt.Sprintf(" width=\"%.2f\" ", n.Width))
	}

	mw.writeString("/>")
}

func (mw *mdWriter) url(n *types.URLNode) {
	mw.space()
	if n.URL != "" {
		mw.writeString("[")
	}
	for _, cn := range n.Content.Nodes {
		if t, ok := cn.(*types.TextNode); ok {
			mw.writeString(t.Value)
		}
	}
	if n.URL != "" {
		mw.writeString("](")
		mw.writeString(n.URL)
		mw.writeString(")")
	}
}

func (mw *mdWriter) code(n *types.CodeNode) {
	mw.newBlock()
	defer mw.writeBytes(newLine)
	if n.Term {
		var buf bytes.Buffer
		const prefix = "    "
		lineStart := true
		for _, r := range n.Value {
			if lineStart {
				buf.WriteString(prefix)
			}
			buf.WriteRune(r)
			lineStart = r == '\n'
		}
		mw.writeBytes(buf.Bytes())
		return
	}
	mw.writeString("```")
	mw.writeString(n.Lang)
	mw.writeBytes(newLine)
	mw.writeString(n.Value)
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
	mw.writeString("```")
}

func (mw *mdWriter) list(n *types.ListNode) {
	if n.Block() == true {
		mw.newBlock()
	}
	mw.write(n.Nodes...)
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
}

func (mw *mdWriter) itemsList(n *types.ItemsListNode) {
	mw.newBlock()
	for i, item := range n.Items {
		s := "* "
		if n.Type() == types.NodeItemsList && n.Start > 0 {
			s = strconv.Itoa(i+n.Start) + ". "
		}
		mw.writeString(s)
		mw.write(item.Nodes...)
		if !mw.lineStart {
			mw.writeBytes(newLine)
		}
	}
}

func (mw *mdWriter) infobox(n *types.InfoboxNode) {
	// TODO: This should use the "detail item" syntax so that it can be pure MD and not HTML
	// kind
	// : <content>
	//
	// The main issue is that when you do write(n.Content.Nodes...) it always adds two newlines
	// at the beginning.
	mw.newBlock()
	mw.writeString(`<aside class="`)
	mw.writeString(string(n.Kind))
	mw.writeString(`">`)
	mw.write(n.Content.Nodes...)
	mw.writeString("")
}

func (mw *mdWriter) header(n *types.HeaderNode) {
	mw.newBlock()
	mw.writeString(strings.Repeat("#", n.Level+1))
	mw.writeString(" ")
	mw.write(n.Content.Nodes...)
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
}

func (mw *mdWriter) youtube(n *types.YouTubeNode) {
	mw.newBlock()
	mw.writeString(`<video id="`)
	mw.writeString(n.VideoID)
	mw.writeString(`"></video>`)
}
<|endoftext|>


<commit_before>package private
<commit_msg>Remove obsolete private utils<commit_after><|endoftext|>


<commit_before>package client

import (
	"fmt"
	"strconv"
	"strings"
	"text/tabwriter"

	Cli "github.com/daolicloud/daolictl/cli"
)

// Usage: daolictl firewall <COMMAND> [OPTIONS]
func (cli *DaoliCli) CmdFirewall(args ...string) error {
	cmd := Cli.Subcmd("firewall", []string{"COMMAND [OPTIONS]"}, firewallUsage(), false)
	err := ParseFlags(cmd, args, true)
	cmd.Usage()
	return err
}

// Usage: daolictl firewall list
func (cli *DaoliCli) CmdFirewallList(args ...string) error {
	cmd := Cli.Subcmd("firewall list", nil, "Lists firewalls", true)
	if err := ParseFlags(cmd, args, true); err != nil {
		return err
	}

	firewalls, err := cli.client.FirewallList()
	if err != nil {
		return err
	}

	wr := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
	fmt.Fprintln(wr, "FIREWALL NAME\tCONTAINER\tGATEWAY IP\tGATEWAY PORT\tSERVICE PORT")
	for _, firewall := range firewalls {
		fmt.Fprintf(wr, "%s\t%s\t%s\t%d\t%d\t",
			firewall.Name,
			strings.TrimLeft(firewall.Container, "/"),
			firewall.GatewayIP,
			firewall.GatewayPort,
			firewall.ServicePort,
		)
		fmt.Fprint(wr, "\n")
	}
	wr.Flush()
	return nil
}

// Usage: daolictl firewall show <CONTAINER>
func (cli *DaoliCli) CmdFirewallShow(args ...string) error {
	cmd := Cli.Subcmd("firewall show", []string{"CONTAINER"}, "Displays detailed information on the container", false)
	if err := ParseFlags(cmd, args, true); err != nil {
		return err
	}

	if len(cmd.Args()) <= 0 {
		cmd.Usage()
		return nil
	}

	firewalls, err := cli.client.FirewallShow(cmd.Arg(0))
	if err != nil {
		return err
	}

	wr := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
	fmt.Fprintln(wr, "FIREWALL NAME\tGATEWAY IP\tGATEWAY PORT\tSERVICE PORT")
	for _, firewall := range firewalls {
		fmt.Fprintf(wr, "%s\t%s\t%d\t%d\t",
			firewall.Name,
			firewall.GatewayIP,
			firewall.GatewayPort,
			firewall.ServicePort,
		)
		fmt.Fprint(wr, "\n")
	}
	wr.Flush()
	return nil
}

// Usage: daolictl firewall create <NAME> --rule <GATEWAYPORT:SERVICEPORT> --container <CONTAINER>
func (cli *DaoliCli) CmdFirewallCreate(args ...string) error {
	cmd := Cli.Subcmd("firewall create", []string{"FIREWALL-NAME"},
		"Creates a firewall rule with a given name", false)
	rule := cmd.String("rule", "", "Add firewall mapping")
	container := cmd.String("container", "", "Add firewall target")

	if err := ParseFlags(cmd, args, true); err != nil {
		return err
	}

	parts := strings.Split(*rule, ":")
	if len(parts) != 2 {
		fmt.Fprintf(cli.err, "--rule %s format is <GATEWAYPORT:SERVICEPORT>", *rule)
		cmd.Usage()
		return nil
	}

	if len(cmd.Args()) <= 0 {
		cmd.Usage()
		return nil
	}

	gport, err := strconv.Atoi(parts[0])
	if err != nil {
		return fmt.Errorf("GATEWAYPORT (%s) must be available number", parts[0])
	}

	sport, err := strconv.Atoi(parts[1])
	if err != nil {
		return fmt.Errorf("SERVICEPORT (%s) must be available number", parts[1])
	}

	fw := Firewall{
		Name:        cmd.Arg(0),
		Container:   *container,
		GatewayPort: gport,
		ServicePort: sport,
	}

	firewall, err := cli.client.FirewallCreate(fw)
	if err != nil {
		return err
	}

	wr := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
	fmt.Fprintln(wr, "FIREWALL NAME\tGATEWAY IP\tGATEWAY PORT\tSERVICE PORT")
	fmt.Fprintf(wr, "%s\t%s\t%d\t%d\t",
		firewall.Name,
		firewall.GatewayIP,
		firewall.GatewayPort,
		firewall.ServicePort,
	)
	fmt.Fprintln(wr)
	wr.Flush()
	return nil
}

// Usage: daolictl firewall delete <FIREWALL-NAME>
func (cli *DaoliCli) CmdFirewallDelete(args ...string) error {
	cmd := Cli.Subcmd("firewall delete", []string{"FIREWALL-NAME"}, "Deletes one or more firewalls", false)
	if err := ParseFlags(cmd, args, true); err != nil {
		return err
	}

	if len(cmd.Args()) <= 0 {
		cmd.Usage()
		return nil
	}

	var err error
	for _, name := range cmd.Args() {
		if e := cli.client.FirewallDelete(name); e != nil {
			err = e
			continue
		}
	}

	return err
}

func firewallUsage() string {
	firewallCommands := map[string]string{
		"list":   "List all firewall rules",
		"show":   "Display detailed information on the container",
		"create": "Create a firewall rule",
		"delete": "Delete a firewall rule",
	}

	help := "Commands:\n"

	for cmd, description := range firewallCommands {
		help += fmt.Sprintf("  %-25.25s%s\n", cmd, description)
	}

	help += fmt.Sprintf("\nRun 'daolictl firewall COMMAND --help' for more information on a command.")
	return help
}
<commit_msg>modify name info<commit_after>package client

import (
	"fmt"
	"strconv"
	"strings"
	"text/tabwriter"

	Cli "github.com/daolicloud/daolictl/cli"
)

// Usage: daolictl firewall <COMMAND> [OPTIONS]
func (cli *DaoliCli) CmdFirewall(args ...string) error {
	cmd := Cli.Subcmd("firewall", []string{"COMMAND [OPTIONS]"}, firewallUsage(), false)
	err := ParseFlags(cmd, args, true)
	cmd.Usage()
	return err
}

// Usage: daolictl firewall list
func (cli *DaoliCli) CmdFirewallList(args ...string) error {
	cmd := Cli.Subcmd("firewall list", nil, "Lists firewalls", true)
	if err := ParseFlags(cmd, args, true); err != nil {
		return err
	}

	firewalls, err := cli.client.FirewallList()
	if err != nil {
		return err
	}

	wr := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
	fmt.Fprintln(wr, "FIREWALL NAME\tCONTAINER\tGATEWAY IP\tGATEWAY PORT\tSERVICE PORT")
	for _, firewall := range firewalls {
		fmt.Fprintf(wr, "%s\t%s\t%s\t%d\t%d\t",
			firewall.Name,
			strings.TrimLeft(firewall.Container, "/"),
			firewall.GatewayIP,
			firewall.GatewayPort,
			firewall.ServicePort,
		)
		fmt.Fprint(wr, "\n")
	}
	wr.Flush()
	return nil
}

// Usage: daolictl firewall show <CONTAINER>
func (cli *DaoliCli) CmdFirewallShow(args ...string) error {
	cmd := Cli.Subcmd("firewall show", []string{"CONTAINER"}, "Displays detailed information on the container", false)
	if err := ParseFlags(cmd, args, true); err != nil {
		return err
	}

	if len(cmd.Args()) <= 0 {
		cmd.Usage()
		return nil
	}

	firewalls, err := cli.client.FirewallShow(cmd.Arg(0))
	if err != nil {
		return err
	}

	wr := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
	fmt.Fprintln(wr, "FIREWALL NAME\tGATEWAY IP\tGATEWAY PORT\tSERVICE PORT")
	for _, firewall := range firewalls {
		fmt.Fprintf(wr, "%s\t%s\t%d\t%d\t",
			firewall.Name,
			firewall.GatewayIP,
			firewall.GatewayPort,
			firewall.ServicePort,
		)
		fmt.Fprint(wr, "\n")
	}
	wr.Flush()
	return nil
}

// Usage: daolictl firewall create <NAME> --rule <GATEWAYPORT:SERVICEPORT> --container <CONTAINER>
func (cli *DaoliCli) CmdFirewallCreate(args ...string) error {
	cmd := Cli.Subcmd("firewall create", []string{"FIREWALL-NAME"},
		"Creates a firewall rule with a given name", false)
	rule := cmd.String("rule", "", "Add firewall mapping(format: <GATEWAYPORT>:<SERVICEPORT>")
	container := cmd.String("container", "", "Add firewall target")

	if err := ParseFlags(cmd, args, true); err != nil {
		return err
	}

	parts := strings.Split(*rule, ":")
	if len(parts) != 2 {
		fmt.Fprintf(cli.err, "--rule %s format is <GATEWAYPORT:SERVICEPORT>", *rule)
		cmd.Usage()
		return nil
	}

	if len(cmd.Args()) <= 0 {
		cmd.Usage()
		return nil
	}

	gport, err := strconv.Atoi(parts[0])
	if err != nil {
		return fmt.Errorf("GATEWAYPORT (%s) must be available number", parts[0])
	}

	sport, err := strconv.Atoi(parts[1])
	if err != nil {
		return fmt.Errorf("SERVICEPORT (%s) must be available number", parts[1])
	}

	fw := Firewall{
		Name:        cmd.Arg(0),
		Container:   *container,
		GatewayPort: gport,
		ServicePort: sport,
	}

	firewall, err := cli.client.FirewallCreate(fw)
	if err != nil {
		return err
	}

	wr := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
	fmt.Fprintln(wr, "FIREWALL NAME\tGATEWAY IP\tGATEWAY PORT\tSERVICE PORT")
	fmt.Fprintf(wr, "%s\t%s\t%d\t%d\t",
		firewall.Name,
		firewall.GatewayIP,
		firewall.GatewayPort,
		firewall.ServicePort,
	)
	fmt.Fprintln(wr)
	wr.Flush()
	return nil
}

// Usage: daolictl firewall delete <FIREWALL-NAME>
func (cli *DaoliCli) CmdFirewallDelete(args ...string) error {
	cmd := Cli.Subcmd("firewall delete", []string{"FIREWALL-NAME"}, "Deletes one or more firewalls", false)
	if err := ParseFlags(cmd, args, true); err != nil {
		return err
	}

	if len(cmd.Args()) <= 0 {
		cmd.Usage()
		return nil
	}

	var err error
	for _, name := range cmd.Args() {
		if e := cli.client.FirewallDelete(name); e != nil {
			err = e
			continue
		}
	}

	return err
}

func firewallUsage() string {
	firewallCommands := map[string]string{
		"list":   "List all firewall rules",
		"show":   "Display detailed information on the container",
		"create": "Create a firewall rule",
		"delete": "Delete a firewall rule",
	}

	help := "Commands:\n"

	for cmd, description := range firewallCommands {
		help += fmt.Sprintf("  %-25.25s%s\n", cmd, description)
	}

	help += fmt.Sprintf("\nRun 'daolictl firewall COMMAND --help' for more information on a command.")
	return help
}
<|endoftext|>


<commit_before>package oidc

import (
	gotime "time"

	"github.com/dgrijalva/jwt-go"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/urlprefix"
	"github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/time"
)

type IDToken struct {
	jwt.StandardClaims
	Nonce string `json:"nonce,omitempty"`
}

type IDTokenIssuer struct {
	OIDCConfig config.OIDCConfiguration
	URLPrefix  urlprefix.Provider
	Time       time.Provider
}

const CodeGrantValidDuration = 5 * gotime.Minute

func (ti *IDTokenIssuer) IssueIDToken(client config.OAuthClientConfiguration, userID string, nonce string) (string, error) {
	now := ti.Time.NowUTC()
	token := &IDToken{
		StandardClaims: jwt.StandardClaims{
			Issuer:    ti.URLPrefix.Value().String(),
			Audience:  client.ClientID(),
			IssuedAt:  now.Unix(),
			ExpiresAt: now.Add(CodeGrantValidDuration).Unix(),
			Subject:   userID,
		},
		Nonce: nonce,
	}

	key := ti.OIDCConfig.Keys[0]
	privKey, err := jwt.ParseRSAPrivateKeyFromPEM([]byte(key.PrivateKey))
	if err != nil {
		return "", err
	}

	jwt := jwt.NewWithClaims(jwt.SigningMethodRS256, token)
	jwt.Header["kid"] = key.KID
	return jwt.SignedString(privKey)
}
<commit_msg>Clarify valid period of ID token<commit_after>package oidc

import (
	gotime "time"

	"github.com/dgrijalva/jwt-go"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/urlprefix"
	"github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/time"
)

type IDToken struct {
	jwt.StandardClaims
	Nonce string `json:"nonce,omitempty"`
}

type IDTokenIssuer struct {
	OIDCConfig config.OIDCConfiguration
	URLPrefix  urlprefix.Provider
	Time       time.Provider
}

// IDTokenValidDuration is the valid period of ID token.
// It can be short, since id_token_hint should accept expired ID tokens.
const IDTokenValidDuration = 5 * gotime.Minute

func (ti *IDTokenIssuer) IssueIDToken(client config.OAuthClientConfiguration, userID string, nonce string) (string, error) {
	now := ti.Time.NowUTC()
	token := &IDToken{
		StandardClaims: jwt.StandardClaims{
			Issuer:    ti.URLPrefix.Value().String(),
			Audience:  client.ClientID(),
			IssuedAt:  now.Unix(),
			ExpiresAt: now.Add(IDTokenValidDuration).Unix(),
			Subject:   userID,
		},
		Nonce: nonce,
	}

	key := ti.OIDCConfig.Keys[0]
	privKey, err := jwt.ParseRSAPrivateKeyFromPEM([]byte(key.PrivateKey))
	if err != nil {
		return "", err
	}

	jwt := jwt.NewWithClaims(jwt.SigningMethodRS256, token)
	jwt.Header["kid"] = key.KID
	return jwt.SignedString(privKey)
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package unversioned

import (
	"fmt"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apis/apps"
	"k8s.io/kubernetes/pkg/apis/batch"
	"k8s.io/kubernetes/pkg/apis/extensions"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/watch"
)

// ControllerHasDesiredReplicas returns a condition that will be true if and only if
// the desired replica count for a controller's ReplicaSelector equals the Replicas count.
func ControllerHasDesiredReplicas(c Interface, controller *api.ReplicationController) wait.ConditionFunc {

	// If we're given a controller where the status lags the spec, it either means that the controller is stale,
	// or that the rc manager hasn't noticed the update yet. Polling status.Replicas is not safe in the latter case.
	desiredGeneration := controller.Generation

	return func() (bool, error) {
		ctrl, err := c.ReplicationControllers(controller.Namespace).Get(controller.Name)
		if err != nil {
			return false, err
		}
		// There's a chance a concurrent update modifies the Spec.Replicas causing this check to pass,
		// or, after this check has passed, a modification causes the rc manager to create more pods.
		// This will not be an issue once we've implemented graceful delete for rcs, but till then
		// concurrent stop operations on the same rc might have unintended side effects.
		return ctrl.Status.ObservedGeneration >= desiredGeneration && ctrl.Status.Replicas == ctrl.Spec.Replicas, nil
	}
}

// ReplicaSetHasDesiredReplicas returns a condition that will be true if and only if
// the desired replica count for a ReplicaSet's ReplicaSelector equals the Replicas count.
func ReplicaSetHasDesiredReplicas(c ExtensionsInterface, replicaSet *extensions.ReplicaSet) wait.ConditionFunc {

	// If we're given a ReplicaSet where the status lags the spec, it either means that the
	// ReplicaSet is stale, or that the ReplicaSet manager hasn't noticed the update yet.
	// Polling status.Replicas is not safe in the latter case.
	desiredGeneration := replicaSet.Generation

	return func() (bool, error) {
		rs, err := c.ReplicaSets(replicaSet.Namespace).Get(replicaSet.Name)
		if err != nil {
			return false, err
		}
		// There's a chance a concurrent update modifies the Spec.Replicas causing this check to
		// pass, or, after this check has passed, a modification causes the ReplicaSet manager to
		// create more pods. This will not be an issue once we've implemented graceful delete for
		// ReplicaSets, but till then concurrent stop operations on the same ReplicaSet might have
		// unintended side effects.
		return rs.Status.ObservedGeneration >= desiredGeneration && rs.Status.Replicas == rs.Spec.Replicas, nil
	}
}

func PetSetHasDesiredPets(c AppsInterface, petset *apps.PetSet) wait.ConditionFunc {
	// TODO: Differentiate between 0 pets and a really quick scale down using generation.
	return func() (bool, error) {
		ps, err := c.PetSets(petset.Namespace).Get(petset.Name)
		if err != nil {
			return false, err
		}
		return ps.Status.Replicas == ps.Spec.Replicas, nil
	}
}

// JobHasDesiredParallelism returns a condition that will be true if the desired parallelism count
// for a job equals the current active counts or is less by an appropriate successful/unsuccessful count.
func JobHasDesiredParallelism(c BatchInterface, job *batch.Job) wait.ConditionFunc {

	return func() (bool, error) {
		job, err := c.Jobs(job.Namespace).Get(job.Name)
		if err != nil {
			return false, err
		}

		// desired parallelism can be either the exact number, in which case return immediately
		if job.Status.Active == *job.Spec.Parallelism {
			return true, nil
		}
		if job.Spec.Completions == nil {
			// A job without specified completions needs to wait for Active to reach Parallelism.
			return false, nil
		} else {
			// otherwise count successful
			progress := *job.Spec.Completions - job.Status.Active - job.Status.Succeeded
			return progress == 0, nil
		}
	}
}

// DeploymentHasDesiredReplicas returns a condition that will be true if and only if
// the desired replica count for a deployment equals its updated replicas count.
// (non-terminated pods that have the desired template spec).
func DeploymentHasDesiredReplicas(c ExtensionsInterface, deployment *extensions.Deployment) wait.ConditionFunc {
	// If we're given a deployment where the status lags the spec, it either
	// means that the deployment is stale, or that the deployment manager hasn't
	// noticed the update yet. Polling status.Replicas is not safe in the latter
	// case.
	desiredGeneration := deployment.Generation

	return func() (bool, error) {
		deployment, err := c.Deployments(deployment.Namespace).Get(deployment.Name)
		if err != nil {
			return false, err
		}
		return deployment.Status.ObservedGeneration >= desiredGeneration &&
			deployment.Status.UpdatedReplicas == deployment.Spec.Replicas, nil
	}
}

// ErrPodCompleted is returned by PodRunning or PodContainerRunning to indicate that
// the pod has already reached completed state.
var ErrPodCompleted = fmt.Errorf("pod ran to completion")

// ErrContainerTerminated is returned by PodContainerRunning in the intermediate
// state where the pod indicates it's still running, but its container is already terminated
var ErrContainerTerminated = fmt.Errorf("container terminated")

// PodRunning returns true if the pod is running, false if the pod has not yet reached running state,
// returns ErrPodCompleted if the pod has run to completion, or an error in any other case.
func PodRunning(event watch.Event) (bool, error) {
	switch event.Type {
	case watch.Deleted:
		return false, errors.NewNotFound(unversioned.GroupResource{Resource: "pods"}, "")
	}
	switch t := event.Object.(type) {
	case *api.Pod:
		switch t.Status.Phase {
		case api.PodRunning:
			return true, nil
		case api.PodFailed, api.PodSucceeded:
			return false, ErrPodCompleted
		}
	}
	return false, nil
}

// PodCompleted returns true if the pod has run to completion, false if the pod has not yet
// reached running state, or an error in any other case.
func PodCompleted(event watch.Event) (bool, error) {
	switch event.Type {
	case watch.Deleted:
		return false, errors.NewNotFound(unversioned.GroupResource{Resource: "pods"}, "")
	}
	switch t := event.Object.(type) {
	case *api.Pod:
		switch t.Status.Phase {
		case api.PodFailed, api.PodSucceeded:
			return true, nil
		}
	}
	return false, nil
}

// PodRunningAndReady returns true if the pod is running and ready, false if the pod has not
// yet reached those states, returns ErrPodCompleted if the pod has run to completion, or
// an error in any other case.
func PodRunningAndReady(event watch.Event) (bool, error) {
	switch event.Type {
	case watch.Deleted:
		return false, errors.NewNotFound(unversioned.GroupResource{Resource: "pods"}, "")
	}
	switch t := event.Object.(type) {
	case *api.Pod:
		switch t.Status.Phase {
		case api.PodFailed, api.PodSucceeded:
			return false, ErrPodCompleted
		case api.PodRunning:
			return api.IsPodReady(t), nil
		}
	}
	return false, nil
}

// PodNotPending returns true if the pod has left the pending state, false if it has not,
// or an error in any other case (such as if the pod was deleted).
func PodNotPending(event watch.Event) (bool, error) {
	switch event.Type {
	case watch.Deleted:
		return false, errors.NewNotFound(unversioned.GroupResource{Resource: "pods"}, "")
	}
	switch t := event.Object.(type) {
	case *api.Pod:
		switch t.Status.Phase {
		case api.PodPending:
			return false, nil
		default:
			return true, nil
		}
	}
	return false, nil
}

// PodContainerRunning returns false until the named container has ContainerStatus running (at least once),
// and will return an error if the pod is deleted, runs to completion, or the container pod is not available.
func PodContainerRunning(containerName string) watch.ConditionFunc {
	return func(event watch.Event) (bool, error) {
		switch event.Type {
		case watch.Deleted:
			return false, errors.NewNotFound(unversioned.GroupResource{Resource: "pods"}, "")
		}
		switch t := event.Object.(type) {
		case *api.Pod:
			switch t.Status.Phase {
			case api.PodRunning, api.PodPending:
			case api.PodFailed, api.PodSucceeded:
				return false, ErrPodCompleted
			default:
				return false, nil
			}
			for _, s := range t.Status.ContainerStatuses {
				if s.Name != containerName {
					continue
				}
				if s.State.Terminated != nil {
					return false, ErrContainerTerminated
				}
				return s.State.Running != nil, nil
			}
			return false, nil
		}
		return false, nil
	}
}

// ServiceAccountHasSecrets returns true if the service account has at least one secret,
// false if it does not, or an error.
func ServiceAccountHasSecrets(event watch.Event) (bool, error) {
	switch event.Type {
	case watch.Deleted:
		return false, errors.NewNotFound(unversioned.GroupResource{Resource: "serviceaccounts"}, "")
	}
	switch t := event.Object.(type) {
	case *api.ServiceAccount:
		return len(t.Secrets) > 0, nil
	}
	return false, nil
}
<commit_msg>Check init containers in PodContainerRunning<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package unversioned

import (
	"fmt"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apis/apps"
	"k8s.io/kubernetes/pkg/apis/batch"
	"k8s.io/kubernetes/pkg/apis/extensions"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/watch"
)

// ControllerHasDesiredReplicas returns a condition that will be true if and only if
// the desired replica count for a controller's ReplicaSelector equals the Replicas count.
func ControllerHasDesiredReplicas(c Interface, controller *api.ReplicationController) wait.ConditionFunc {

	// If we're given a controller where the status lags the spec, it either means that the controller is stale,
	// or that the rc manager hasn't noticed the update yet. Polling status.Replicas is not safe in the latter case.
	desiredGeneration := controller.Generation

	return func() (bool, error) {
		ctrl, err := c.ReplicationControllers(controller.Namespace).Get(controller.Name)
		if err != nil {
			return false, err
		}
		// There's a chance a concurrent update modifies the Spec.Replicas causing this check to pass,
		// or, after this check has passed, a modification causes the rc manager to create more pods.
		// This will not be an issue once we've implemented graceful delete for rcs, but till then
		// concurrent stop operations on the same rc might have unintended side effects.
		return ctrl.Status.ObservedGeneration >= desiredGeneration && ctrl.Status.Replicas == ctrl.Spec.Replicas, nil
	}
}

// ReplicaSetHasDesiredReplicas returns a condition that will be true if and only if
// the desired replica count for a ReplicaSet's ReplicaSelector equals the Replicas count.
func ReplicaSetHasDesiredReplicas(c ExtensionsInterface, replicaSet *extensions.ReplicaSet) wait.ConditionFunc {

	// If we're given a ReplicaSet where the status lags the spec, it either means that the
	// ReplicaSet is stale, or that the ReplicaSet manager hasn't noticed the update yet.
	// Polling status.Replicas is not safe in the latter case.
	desiredGeneration := replicaSet.Generation

	return func() (bool, error) {
		rs, err := c.ReplicaSets(replicaSet.Namespace).Get(replicaSet.Name)
		if err != nil {
			return false, err
		}
		// There's a chance a concurrent update modifies the Spec.Replicas causing this check to
		// pass, or, after this check has passed, a modification causes the ReplicaSet manager to
		// create more pods. This will not be an issue once we've implemented graceful delete for
		// ReplicaSets, but till then concurrent stop operations on the same ReplicaSet might have
		// unintended side effects.
		return rs.Status.ObservedGeneration >= desiredGeneration && rs.Status.Replicas == rs.Spec.Replicas, nil
	}
}

func PetSetHasDesiredPets(c AppsInterface, petset *apps.PetSet) wait.ConditionFunc {
	// TODO: Differentiate between 0 pets and a really quick scale down using generation.
	return func() (bool, error) {
		ps, err := c.PetSets(petset.Namespace).Get(petset.Name)
		if err != nil {
			return false, err
		}
		return ps.Status.Replicas == ps.Spec.Replicas, nil
	}
}

// JobHasDesiredParallelism returns a condition that will be true if the desired parallelism count
// for a job equals the current active counts or is less by an appropriate successful/unsuccessful count.
func JobHasDesiredParallelism(c BatchInterface, job *batch.Job) wait.ConditionFunc {

	return func() (bool, error) {
		job, err := c.Jobs(job.Namespace).Get(job.Name)
		if err != nil {
			return false, err
		}

		// desired parallelism can be either the exact number, in which case return immediately
		if job.Status.Active == *job.Spec.Parallelism {
			return true, nil
		}
		if job.Spec.Completions == nil {
			// A job without specified completions needs to wait for Active to reach Parallelism.
			return false, nil
		} else {
			// otherwise count successful
			progress := *job.Spec.Completions - job.Status.Active - job.Status.Succeeded
			return progress == 0, nil
		}
	}
}

// DeploymentHasDesiredReplicas returns a condition that will be true if and only if
// the desired replica count for a deployment equals its updated replicas count.
// (non-terminated pods that have the desired template spec).
func DeploymentHasDesiredReplicas(c ExtensionsInterface, deployment *extensions.Deployment) wait.ConditionFunc {
	// If we're given a deployment where the status lags the spec, it either
	// means that the deployment is stale, or that the deployment manager hasn't
	// noticed the update yet. Polling status.Replicas is not safe in the latter
	// case.
	desiredGeneration := deployment.Generation

	return func() (bool, error) {
		deployment, err := c.Deployments(deployment.Namespace).Get(deployment.Name)
		if err != nil {
			return false, err
		}
		return deployment.Status.ObservedGeneration >= desiredGeneration &&
			deployment.Status.UpdatedReplicas == deployment.Spec.Replicas, nil
	}
}

// ErrPodCompleted is returned by PodRunning or PodContainerRunning to indicate that
// the pod has already reached completed state.
var ErrPodCompleted = fmt.Errorf("pod ran to completion")

// ErrContainerTerminated is returned by PodContainerRunning in the intermediate
// state where the pod indicates it's still running, but its container is already terminated
var ErrContainerTerminated = fmt.Errorf("container terminated")

// PodRunning returns true if the pod is running, false if the pod has not yet reached running state,
// returns ErrPodCompleted if the pod has run to completion, or an error in any other case.
func PodRunning(event watch.Event) (bool, error) {
	switch event.Type {
	case watch.Deleted:
		return false, errors.NewNotFound(unversioned.GroupResource{Resource: "pods"}, "")
	}
	switch t := event.Object.(type) {
	case *api.Pod:
		switch t.Status.Phase {
		case api.PodRunning:
			return true, nil
		case api.PodFailed, api.PodSucceeded:
			return false, ErrPodCompleted
		}
	}
	return false, nil
}

// PodCompleted returns true if the pod has run to completion, false if the pod has not yet
// reached running state, or an error in any other case.
func PodCompleted(event watch.Event) (bool, error) {
	switch event.Type {
	case watch.Deleted:
		return false, errors.NewNotFound(unversioned.GroupResource{Resource: "pods"}, "")
	}
	switch t := event.Object.(type) {
	case *api.Pod:
		switch t.Status.Phase {
		case api.PodFailed, api.PodSucceeded:
			return true, nil
		}
	}
	return false, nil
}

// PodRunningAndReady returns true if the pod is running and ready, false if the pod has not
// yet reached those states, returns ErrPodCompleted if the pod has run to completion, or
// an error in any other case.
func PodRunningAndReady(event watch.Event) (bool, error) {
	switch event.Type {
	case watch.Deleted:
		return false, errors.NewNotFound(unversioned.GroupResource{Resource: "pods"}, "")
	}
	switch t := event.Object.(type) {
	case *api.Pod:
		switch t.Status.Phase {
		case api.PodFailed, api.PodSucceeded:
			return false, ErrPodCompleted
		case api.PodRunning:
			return api.IsPodReady(t), nil
		}
	}
	return false, nil
}

// PodNotPending returns true if the pod has left the pending state, false if it has not,
// or an error in any other case (such as if the pod was deleted).
func PodNotPending(event watch.Event) (bool, error) {
	switch event.Type {
	case watch.Deleted:
		return false, errors.NewNotFound(unversioned.GroupResource{Resource: "pods"}, "")
	}
	switch t := event.Object.(type) {
	case *api.Pod:
		switch t.Status.Phase {
		case api.PodPending:
			return false, nil
		default:
			return true, nil
		}
	}
	return false, nil
}

// PodContainerRunning returns false until the named container has ContainerStatus running (at least once),
// and will return an error if the pod is deleted, runs to completion, or the container pod is not available.
func PodContainerRunning(containerName string) watch.ConditionFunc {
	return func(event watch.Event) (bool, error) {
		switch event.Type {
		case watch.Deleted:
			return false, errors.NewNotFound(unversioned.GroupResource{Resource: "pods"}, "")
		}
		switch t := event.Object.(type) {
		case *api.Pod:
			switch t.Status.Phase {
			case api.PodRunning, api.PodPending:
			case api.PodFailed, api.PodSucceeded:
				return false, ErrPodCompleted
			default:
				return false, nil
			}
			for _, s := range t.Status.ContainerStatuses {
				if s.Name != containerName {
					continue
				}
				if s.State.Terminated != nil {
					return false, ErrContainerTerminated
				}
				return s.State.Running != nil, nil
			}
			for _, s := range t.Status.InitContainerStatuses {
				if s.Name != containerName {
					continue
				}
				if s.State.Terminated != nil {
					return false, ErrContainerTerminated
				}
				return s.State.Running != nil, nil
			}
			return false, nil
		}
		return false, nil
	}
}

// ServiceAccountHasSecrets returns true if the service account has at least one secret,
// false if it does not, or an error.
func ServiceAccountHasSecrets(event watch.Event) (bool, error) {
	switch event.Type {
	case watch.Deleted:
		return false, errors.NewNotFound(unversioned.GroupResource{Resource: "serviceaccounts"}, "")
	}
	switch t := event.Object.(type) {
	case *api.ServiceAccount:
		return len(t.Secrets) > 0, nil
	}
	return false, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controller

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	appsv1beta1 "k8s.io/api/apps/v1beta1"
	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"

	crv1 "github.com/kinvolk/habitat-operator/pkg/habitat/apis/cr/v1"
)

const (
	resyncPeriod = 1 * time.Minute
	peerFile     = "peer-watch-file"
)

type HabitatController struct {
	config Config
	logger log.Logger
}

type Config struct {
	HabitatClient       *rest.RESTClient
	KubernetesClientset *kubernetes.Clientset
	Scheme              *runtime.Scheme
}

func New(config Config, logger log.Logger) (*HabitatController, error) {
	if config.HabitatClient == nil {
		return nil, errors.New("invalid controller config: no HabitatClient")
	}
	if config.KubernetesClientset == nil {
		return nil, errors.New("invalid controller config: no KubernetesClientset")
	}
	if config.Scheme == nil {
		return nil, errors.New("invalid controller config: no Schema")
	}
	if logger == nil {
		return nil, errors.New("invalid controller config: no logger")
	}

	hc := &HabitatController{
		config: config,
		logger: logger,
	}

	return hc, nil
}

// Run starts a Habitat resource controller.
func (hc *HabitatController) Run(ctx context.Context) error {
	level.Info(hc.logger).Log("msg", "Watching Service Group objects")

	hc.watchCustomResources(ctx)

	hc.watchPods(ctx)

	// This channel is closed when the context is canceled or times out.
	<-ctx.Done()

	// Err() contains the error, if any.
	return ctx.Err()
}

func (hc *HabitatController) watchCustomResources(ctx context.Context) {
	source := cache.NewListWatchFromClient(
		hc.config.HabitatClient,
		crv1.ServiceGroupResourcePlural,
		apiv1.NamespaceAll,
		fields.Everything())

	_, k8sController := cache.NewInformer(
		source,

		// The object type.
		&crv1.ServiceGroup{},

		// resyncPeriod
		// Every resyncPeriod, all resources in the cache will retrigger events.
		// Set to 0 to disable the resync.
		resyncPeriod,

		// Your custom resource event handlers.
		cache.ResourceEventHandlerFuncs{
			AddFunc:    hc.onAdd,
			UpdateFunc: hc.onUpdate,
			DeleteFunc: hc.onDelete,
		})

	// The k8sController will start processing events from the API.
	go k8sController.Run(ctx.Done())
}

func (hc *HabitatController) onAdd(obj interface{}) {
	sg, ok := obj.(*crv1.ServiceGroup)
	if !ok {
		level.Error(hc.logger).Log("msg", "unknown event type")
		return
	}

	level.Debug(hc.logger).Log("function", "onAdd", "msg", sg.ObjectMeta.SelfLink)

	// Validate object.
	if err := validateCustomObject(*sg); err != nil {
		if vErr, ok := err.(validationError); ok {
			level.Error(hc.logger).Log("type", "validation error", "msg", err, "key", vErr.Key)
			return
		}

		level.Error(hc.logger).Log("msg", err)
		return
	}

	level.Debug(hc.logger).Log("msg", "validated object")

	group := "default"
	if sg.Spec.Habitat.Group != "" {
		group = sg.Spec.Habitat.Group
	}

	// This value needs to be passed as a *int32, so we convert it, assign it to a
	// variable and afterwards pass a pointer to it.
	count := int32(sg.Spec.Count)

	// Create a deployment.
	deployment := &appsv1beta1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Name: sg.Name,
		},
		Spec: appsv1beta1.DeploymentSpec{
			Replicas: &count,
			Template: apiv1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						"habitat":       "true",
						"service-group": sg.Name,
					},
				},
				Spec: apiv1.PodSpec{
					Containers: []apiv1.Container{
						{
							Name:  "habitat-service",
							Image: sg.Spec.Image,
							Args: []string{
								"--group", group,
							},
							VolumeMounts: []apiv1.VolumeMount{
								{
									Name:      "config",
									MountPath: "/habitat-operator",
									ReadOnly:  true,
								},
							},
						},
					},
					// Define the volume for the ConfigMap.
					Volumes: []apiv1.Volume{
						{
							Name: "config",
							VolumeSource: apiv1.VolumeSource{
								ConfigMap: &apiv1.ConfigMapVolumeSource{
									LocalObjectReference: apiv1.LocalObjectReference{
										Name: configMapName(sg.Name),
									},
									Items: []apiv1.KeyToPath{
										{
											Key:  peerFile,
											Path: "peer-ip",
										},
									},
								},
							},
						},
					},
				},
			},
		},
	}

	d, err := hc.config.KubernetesClientset.AppsV1beta1Client.Deployments(apiv1.NamespaceDefault).Create(deployment)
	if err != nil {
		level.Error(hc.logger).Log("msg", err)
		return
	}

	level.Info(hc.logger).Log("msg", "created deployment", "name", d.GetObjectMeta().GetName())

	// Create the ConfigMap for the peer watch file.
	configMap := newConfigMap(sg.Name, d.UID, "")
	_, err = hc.config.KubernetesClientset.CoreV1Client.ConfigMaps(apiv1.NamespaceDefault).Create(configMap)
	if err != nil {
		level.Error(hc.logger).Log("msg", err)
		return
	}

	level.Debug(hc.logger).Log("msg", "created ConfigMap with peer IP", "object", configMap.Data["peer-ip"])
}

func (hc *HabitatController) onUpdate(oldObj, newObj interface{}) {
	oldServiceGroup := oldObj.(*crv1.ServiceGroup)
	newServiceGroup := newObj.(*crv1.ServiceGroup)
	level.Info(hc.logger).Log("function", "onUpdate", "msg", fmt.Sprintf("oldObj: %s, newObj: %s", oldServiceGroup.ObjectMeta.SelfLink, newServiceGroup.ObjectMeta.SelfLink))
}

func (hc *HabitatController) onDelete(obj interface{}) {
	sg, ok := obj.(*crv1.ServiceGroup)
	if !ok {
		level.Error(hc.logger).Log("msg", "unknown event type")
		return
	}

	level.Debug(hc.logger).Log("function", "onDelete", "msg", sg.ObjectMeta.SelfLink)

	deploymentsClient := hc.config.KubernetesClientset.AppsV1beta1Client.Deployments(sg.ObjectMeta.Namespace)
	deploymentName := sg.Name

	// With this policy, dependent resources will be deleted, but we don't wait
	// for that to happen.
	deletePolicy := metav1.DeletePropagationBackground
	deleteOptions := &metav1.DeleteOptions{
		PropagationPolicy: &deletePolicy,
	}

	err := deploymentsClient.Delete(deploymentName, deleteOptions)
	if err != nil {
		level.Error(hc.logger).Log("msg", err)
		return
	}

	level.Info(hc.logger).Log("msg", "deleted deployment", "name", deploymentName)
}

func (hc *HabitatController) watchPods(ctx context.Context) {
	ls := labels.SelectorFromSet(labels.Set(map[string]string{"habitat": "true"}))
	clw := newListWatchFromClientWithLabels(
		hc.config.KubernetesClientset.CoreV1().RESTClient(),
		"pods",
		apiv1.NamespaceAll,
		ls)

	_, c := cache.NewInformer(
		clw,
		&apiv1.Pod{},
		resyncPeriod,
		cache.ResourceEventHandlerFuncs{
			AddFunc:    hc.onPodAdd,
			UpdateFunc: hc.onPodUpdate,
			DeleteFunc: hc.onPodDelete,
		})

	go c.Run(ctx.Done())
}

func (hc *HabitatController) onPodAdd(obj interface{}) {
}

func (hc *HabitatController) onPodUpdate(oldObj, newObj interface{}) {
	// TODO: Do not retrieve or write IP if we are deploying a standalone topology.
	pod, ok := newObj.(*apiv1.Pod)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to cast pod.")
		return
	}
	if pod == nil {
		return
	}
	if pod.Status.Phase != apiv1.PodRunning {
		return
	}
	err := hc.writeIP(pod)
	if err != nil {
		level.Error(hc.logger).Log("msg", err)
		return
	}
}

func (hc *HabitatController) onPodDelete(obj interface{}) {
	// TODO: Make sure the pod that was deleted was not the same pod
	// whose IP was in the ConfigMap.
}

func (hc *HabitatController) writeIP(pod *apiv1.Pod) error {
	sgName := pod.ObjectMeta.Labels["service-group"]
	cmName := configMapName(sgName)
	ip := pod.Status.PodIP

	cm, err := hc.config.KubernetesClientset.CoreV1().ConfigMaps(apiv1.NamespaceDefault).Get(cmName, metav1.GetOptions{})
	if err != nil {
		return err
	}
	oldIP := cm.Data[peerFile]
	if oldIP != "" {
		// Do not overwrite IP with itself.
		if ip == oldIP {
			return nil
		}
		podList, err := hc.config.KubernetesClientset.CoreV1().Pods(apiv1.NamespaceDefault).List(metav1.ListOptions{})
		if err != nil {
			return err
		}
		for i := range podList.Items {
			oldPod := &podList.Items[i]
			// Do not write a new IP if pod with the IP in the CM is still running.
			if oldPod.Status.PodIP == oldIP && oldPod.Status.Phase == apiv1.PodRunning {
				return nil
			}
		}
	}
	updatedCM := newConfigMap(sgName, cm.UID, ip)
	_, err = hc.config.KubernetesClientset.CoreV1().ConfigMaps(apiv1.NamespaceDefault).Update(updatedCM)
	if err != nil {
		return err
	}
	return nil
}

func newConfigMap(sgName string, uid types.UID, ip string) *apiv1.ConfigMap {
	return &apiv1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name: configMapName(sgName),
			OwnerReferences: []metav1.OwnerReference{
				{
					APIVersion: "extensions/v1beta1",
					Kind:       "Deployment",
					Name:       sgName,
					UID:        uid,
				},
			},
		},
		Data: map[string]string{
			peerFile: ip,
		},
	}
}

func configMapName(sgName string) string {
	return fmt.Sprintf("%s-peer-file", sgName)
}
<commit_msg>handle onPodDelete<commit_after>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controller

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	appsv1beta1 "k8s.io/api/apps/v1beta1"
	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"

	crv1 "github.com/kinvolk/habitat-operator/pkg/habitat/apis/cr/v1"
)

const (
	resyncPeriod = 1 * time.Minute
	peerFile     = "peer-watch-file"
)

type HabitatController struct {
	config Config
	logger log.Logger
}

type Config struct {
	HabitatClient       *rest.RESTClient
	KubernetesClientset *kubernetes.Clientset
	Scheme              *runtime.Scheme
}

func New(config Config, logger log.Logger) (*HabitatController, error) {
	if config.HabitatClient == nil {
		return nil, errors.New("invalid controller config: no HabitatClient")
	}
	if config.KubernetesClientset == nil {
		return nil, errors.New("invalid controller config: no KubernetesClientset")
	}
	if config.Scheme == nil {
		return nil, errors.New("invalid controller config: no Schema")
	}
	if logger == nil {
		return nil, errors.New("invalid controller config: no logger")
	}

	hc := &HabitatController{
		config: config,
		logger: logger,
	}

	return hc, nil
}

// Run starts a Habitat resource controller.
func (hc *HabitatController) Run(ctx context.Context) error {
	level.Info(hc.logger).Log("msg", "Watching Service Group objects")

	hc.watchCustomResources(ctx)

	hc.watchPods(ctx)

	// This channel is closed when the context is canceled or times out.
	<-ctx.Done()

	// Err() contains the error, if any.
	return ctx.Err()
}

func (hc *HabitatController) watchCustomResources(ctx context.Context) {
	source := cache.NewListWatchFromClient(
		hc.config.HabitatClient,
		crv1.ServiceGroupResourcePlural,
		apiv1.NamespaceAll,
		fields.Everything())

	_, k8sController := cache.NewInformer(
		source,

		// The object type.
		&crv1.ServiceGroup{},

		// resyncPeriod
		// Every resyncPeriod, all resources in the cache will retrigger events.
		// Set to 0 to disable the resync.
		resyncPeriod,

		// Your custom resource event handlers.
		cache.ResourceEventHandlerFuncs{
			AddFunc:    hc.onAdd,
			UpdateFunc: hc.onUpdate,
			DeleteFunc: hc.onDelete,
		})

	// The k8sController will start processing events from the API.
	go k8sController.Run(ctx.Done())
}

func (hc *HabitatController) onAdd(obj interface{}) {
	sg, ok := obj.(*crv1.ServiceGroup)
	if !ok {
		level.Error(hc.logger).Log("msg", "unknown event type")
		return
	}

	level.Debug(hc.logger).Log("function", "onAdd", "msg", sg.ObjectMeta.SelfLink)

	// Validate object.
	if err := validateCustomObject(*sg); err != nil {
		if vErr, ok := err.(validationError); ok {
			level.Error(hc.logger).Log("type", "validation error", "msg", err, "key", vErr.Key)
			return
		}

		level.Error(hc.logger).Log("msg", err)
		return
	}

	level.Debug(hc.logger).Log("msg", "validated object")

	group := "default"
	if sg.Spec.Habitat.Group != "" {
		group = sg.Spec.Habitat.Group
	}

	// This value needs to be passed as a *int32, so we convert it, assign it to a
	// variable and afterwards pass a pointer to it.
	count := int32(sg.Spec.Count)

	// Create a deployment.
	deployment := &appsv1beta1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Name: sg.Name,
		},
		Spec: appsv1beta1.DeploymentSpec{
			Replicas: &count,
			Template: apiv1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						"habitat":       "true",
						"service-group": sg.Name,
					},
				},
				Spec: apiv1.PodSpec{
					Containers: []apiv1.Container{
						{
							Name:  "habitat-service",
							Image: sg.Spec.Image,
							Args: []string{
								"--group", group,
							},
							VolumeMounts: []apiv1.VolumeMount{
								{
									Name:      "config",
									MountPath: "/habitat-operator",
									ReadOnly:  true,
								},
							},
						},
					},
					// Define the volume for the ConfigMap.
					Volumes: []apiv1.Volume{
						{
							Name: "config",
							VolumeSource: apiv1.VolumeSource{
								ConfigMap: &apiv1.ConfigMapVolumeSource{
									LocalObjectReference: apiv1.LocalObjectReference{
										Name: configMapName(sg.Name),
									},
									Items: []apiv1.KeyToPath{
										{
											Key:  peerFile,
											Path: "peer-ip",
										},
									},
								},
							},
						},
					},
				},
			},
		},
	}

	d, err := hc.config.KubernetesClientset.AppsV1beta1Client.Deployments(apiv1.NamespaceDefault).Create(deployment)
	if err != nil {
		level.Error(hc.logger).Log("msg", err)
		return
	}

	level.Info(hc.logger).Log("msg", "created deployment", "name", d.GetObjectMeta().GetName())

	// Create the ConfigMap for the peer watch file.
	configMap := newConfigMap(sg.Name, d.UID, "")
	_, err = hc.config.KubernetesClientset.CoreV1Client.ConfigMaps(apiv1.NamespaceDefault).Create(configMap)
	if err != nil {
		level.Error(hc.logger).Log("msg", err)
		return
	}

	level.Debug(hc.logger).Log("msg", "created ConfigMap with peer IP", "object", configMap.Data["peer-ip"])
}

func (hc *HabitatController) onUpdate(oldObj, newObj interface{}) {
	oldServiceGroup := oldObj.(*crv1.ServiceGroup)
	newServiceGroup := newObj.(*crv1.ServiceGroup)
	level.Info(hc.logger).Log("function", "onUpdate", "msg", fmt.Sprintf("oldObj: %s, newObj: %s", oldServiceGroup.ObjectMeta.SelfLink, newServiceGroup.ObjectMeta.SelfLink))
}

func (hc *HabitatController) onDelete(obj interface{}) {
	sg, ok := obj.(*crv1.ServiceGroup)
	if !ok {
		level.Error(hc.logger).Log("msg", "unknown event type")
		return
	}

	level.Debug(hc.logger).Log("function", "onDelete", "msg", sg.ObjectMeta.SelfLink)

	deploymentsClient := hc.config.KubernetesClientset.AppsV1beta1Client.Deployments(sg.ObjectMeta.Namespace)
	deploymentName := sg.Name

	// With this policy, dependent resources will be deleted, but we don't wait
	// for that to happen.
	deletePolicy := metav1.DeletePropagationBackground
	deleteOptions := &metav1.DeleteOptions{
		PropagationPolicy: &deletePolicy,
	}

	err := deploymentsClient.Delete(deploymentName, deleteOptions)
	if err != nil {
		level.Error(hc.logger).Log("msg", err)
		return
	}

	level.Info(hc.logger).Log("msg", "deleted deployment", "name", deploymentName)
}

func (hc *HabitatController) watchPods(ctx context.Context) {
	ls := labels.SelectorFromSet(labels.Set(map[string]string{"habitat": "true"}))
	clw := newListWatchFromClientWithLabels(
		hc.config.KubernetesClientset.CoreV1().RESTClient(),
		"pods",
		apiv1.NamespaceAll,
		ls)

	_, c := cache.NewInformer(
		clw,
		&apiv1.Pod{},
		resyncPeriod,
		cache.ResourceEventHandlerFuncs{
			AddFunc:    hc.onPodAdd,
			UpdateFunc: hc.onPodUpdate,
			DeleteFunc: hc.onPodDelete,
		})

	go c.Run(ctx.Done())
}

func (hc *HabitatController) onPodAdd(obj interface{}) {
}

func (hc *HabitatController) onPodUpdate(oldObj, newObj interface{}) {
	// TODO: Do not retrieve or write IP if we are deploying a standalone topology.
	pod, ok := newObj.(*apiv1.Pod)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to cast pod.")
		return
	}
	if pod == nil {
		return
	}
	if pod.Status.Phase != apiv1.PodRunning {
		return
	}
	err := hc.writeIP(pod)
	if err != nil {
		level.Error(hc.logger).Log("msg", err)
		return
	}
}

func (hc *HabitatController) onPodDelete(obj interface{}) {
	pod, ok := obj.(*apiv1.Pod)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to cast pod.")
		return
	}
	if pod == nil {
		return
	}
	sgName := pod.ObjectMeta.Labels["service-group"]
	cmName := configMapName(sgName)
	cm, err := hc.config.KubernetesClientset.CoreV1().ConfigMaps(apiv1.NamespaceDefault).Get(cmName, metav1.GetOptions{})
	if err != nil {
		level.Error(hc.logger).Log("msg", err)
		return
	}
	currIP := cm.Data[peerFile]
	deletedPodIP := pod.Status.PodIP
	if currIP != deletedPodIP {
		return
	}
	// Get only those pods that are running.
	fs := fields.SelectorFromSet(fields.Set{
		"status.phase": "Running",
	})
	podList, err := hc.config.KubernetesClientset.CoreV1().Pods(apiv1.NamespaceDefault).List(metav1.ListOptions{FieldSelector: fs.String()})
	if err != nil {
		level.Error(hc.logger).Log("msg", err)
		return
	}
	for i := range podList.Items {
		newPod := &podList.Items[i]
		if newPod.Status.Phase == apiv1.PodRunning {
			// Replace our IP in the CM file with a new IP of a running pod.
			err := hc.writeIP(newPod)
			if err != nil {
				level.Error(hc.logger).Log("msg", err)
			}
			return
		}
	}
}

func (hc *HabitatController) writeIP(pod *apiv1.Pod) error {
	sgName := pod.ObjectMeta.Labels["service-group"]
	cmName := configMapName(sgName)
	ip := pod.Status.PodIP

	cm, err := hc.config.KubernetesClientset.CoreV1().ConfigMaps(apiv1.NamespaceDefault).Get(cmName, metav1.GetOptions{})
	if err != nil {
		return err
	}
	oldIP := cm.Data[peerFile]
	if oldIP != "" {
		// Do not overwrite IP with itself.
		if ip == oldIP {
			return nil
		}
		podList, err := hc.config.KubernetesClientset.CoreV1().Pods(apiv1.NamespaceDefault).List(metav1.ListOptions{})
		if err != nil {
			return err
		}
		for i := range podList.Items {
			oldPod := &podList.Items[i]
			// Do not write a new IP if pod with the IP in the CM is still running.
			if oldPod.Status.PodIP == oldIP && oldPod.Status.Phase == apiv1.PodRunning {
				return nil
			}
		}
	}
	updatedCM := newConfigMap(sgName, cm.UID, ip)
	_, err = hc.config.KubernetesClientset.CoreV1().ConfigMaps(apiv1.NamespaceDefault).Update(updatedCM)
	if err != nil {
		return err
	}
	return nil
}

func newConfigMap(sgName string, uid types.UID, ip string) *apiv1.ConfigMap {
	return &apiv1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name: configMapName(sgName),
			OwnerReferences: []metav1.OwnerReference{
				{
					APIVersion: "extensions/v1beta1",
					Kind:       "Deployment",
					Name:       sgName,
					UID:        uid,
				},
			},
		},
		Data: map[string]string{
			peerFile: ip,
		},
	}
}

func configMapName(sgName string) string {
	return fmt.Sprintf("%s-peer-file", sgName)
}
<|endoftext|>