<commit_before>package main

import (
	"flag"
	"fmt"
	fs "github.com/tiglabs/containerfs/fs"
	"github.com/tiglabs/containerfs/logger"
	"github.com/tiglabs/containerfs/utils"
	"os"
	"strconv"
	"strings"
)

func main() {

	var loglevel string
	var logpath string
	var peers string
	var cmd string

	flag.StringVar(&peers, "volmgr", "127.0.0.1:7703,127.0.0.1:7713,127.0.0.1:7723", "ContainerFS VolMgr Host")
	flag.StringVar(&logpath, "logpath", "/export/Logs/containerfs/logs/", "ContainerFS Log Path")
	flag.StringVar(&loglevel, "loglevel", "error", "ContainerFS Log Level")
	flag.StringVar(&cmd, "action", "", `
		createvol [volumename ] size [sata/sas/ssd/nvme...]
		expandvol [volumeuuid ] size
		migrate [host]
		deletevol [volumeuuid ]
		snapshootvol [volumeuuid ]
		getvolmetaleader [volumeuuid ]
		getvolmgrleader
		getvolinfo [volumeuuid ]
		getvols
		getinodeinfo [volumeuuid ] parentinodeid name
		getdatanodes
		deldatanode ip port`)

	flag.Parse()

	fs.VolMgrHosts = strings.Split(peers, ",")
	fs.BufferSize = 1024 * 1024

	logger.SetConsole(true)
	logger.SetRollingFile(logpath, "CLI.log", 10, 100, logger.MB) //each 100M rolling
	switch loglevel {
	case "error":
		logger.SetLevel(logger.ERROR)
	case "debug":
		logger.SetLevel(logger.DEBUG)
	case "info":
		logger.SetLevel(logger.INFO)
	default:
		logger.SetLevel(logger.ERROR)
	}

	//switch flag.Arg(0) {
	switch cmd {

	case "createvol":
		argNum := flag.NArg()
		if argNum != 2 && argNum != 3 {
			fmt.Println("createvol [volname] [space GB] [sata/sas/ssd/nvme...]")
			os.Exit(1)
		}

		var ret int32
		if argNum == 2 {
			ret = fs.CreateVol(flag.Arg(0), flag.Arg(1), "sas")
		} else if argNum == 3 {
			ret = fs.CreateVol(flag.Arg(0), flag.Arg(1), flag.Arg(2))
		}
		if ret != 0 {
			fmt.Println("failed")
		}
		/*
			case "expandvol":
				argNum := flag.NArg()
				if argNum != 2 {
					fmt.Println("expandvol [volUUID] [space GB]")
					os.Exit(1)
				}
				ret := fs.ExpandVolTS(flag.Arg(0), flag.Arg(1))
				if ret != 0 {
					fmt.Println("failed")
				}
		*/
	case "migrate":
		argNum := flag.NArg()
		if argNum != 1 {
			fmt.Println("migrate host")
			os.Exit(1)
		}
		ret := fs.Migrate(flag.Arg(0))
		if ret != 0 {
			fmt.Println("Migrate failed")
		} else {
			fmt.Println("Migrate process on background")
		}

	case "deletevol":
		argNum := flag.NArg()
		if argNum != 1 {
			fmt.Println("deletevol [voluuid]")
			os.Exit(1)
		}
		ret := fs.DeleteVol(flag.Arg(0))
		if ret != 0 {
			fmt.Println("failed")
		}
		/*
			case "snapshootvol":
				argNum := flag.NArg()
				if argNum != 1 {
					fmt.Println("snapshootvol [voluuid]")
					os.Exit(1)
				}
				ret := fs.SnapShotVol(flag.Arg(0))
				if ret != 0 {
					fmt.Println("failed")
				}
		*/
	case "getvolmetaleader":
		argNum := flag.NArg()
		if argNum != 1 {
			fmt.Println("getvolmetaleader [voluuid]")
			os.Exit(1)
		}
		leader, _ := fs.GetVolMetaLeader(flag.Arg(0))
		fmt.Println(leader)
	case "getvolmgrleader":
		leader, _ := utils.GetVolMgrLeader(fs.VolMgrHosts)
		fmt.Println(leader)
	case "getvolinfo":
		argNum := flag.NArg()
		if argNum != 1 {
			fmt.Println("getvolinfo [volUUID]")
			os.Exit(1)
		}
		ret, vi := fs.GetVolInfo(flag.Arg(0))
		if ret == 0 {
			fmt.Println(vi)
		} else {
			fmt.Printf("get volume info failed , ret :%d", ret)
		}
	case "getvols":
		argNum := flag.NArg()
		if argNum != 0 {
			fmt.Println("getvols")
			os.Exit(1)
		}
		ret, vi := fs.GetAllVolumeInfos()
		if ret == 0 {
			for _, v := range vi {
				fmt.Println(v)
			}
		} else {
			fmt.Printf("get all volumes info failed , ret :%d", ret)
		}

	case "getinodeinfo":
		argNum := flag.NArg()
		if argNum != 3 {
			fmt.Println("getinodeinfo [volUUID] parentinodeid name")
			os.Exit(1)
		}
		cfs := fs.OpenFileSystem(flag.Arg(0))
		if cfs == nil {
			fmt.Println("no such volume")
			os.Exit(1)
		}

		pinode, err := strconv.ParseUint(flag.Arg(1), 10, 64)
		if err == nil {
			ret, inode, inodeinfo := cfs.GetInodeInfoDirect(pinode, flag.Arg(2))
			if ret != 0 {
				fmt.Println("no such inode")
			} else {
				fmt.Printf("inode:%v\ninodeinfo:%v\n", inode, inodeinfo)
				ret, chunkinfo, _ := cfs.GetFileChunksDirect(pinode, flag.Arg(2))
				if ret != 0 {
					fmt.Printf("no chunkinfo")
				} else {
					fmt.Printf("chunkinfo:%v\n", chunkinfo)

				}
			}
		} else {
			fmt.Println("err parent inode", err)
		}
	case "getdatanodes":
		argNum := flag.NArg()
		if argNum != 0 {
			fmt.Println("getdatanodes")
			os.Exit(1)
		}
		ret, vi := fs.GetAllDatanode()
		if ret == 0 {
			for _, v := range vi {
				fmt.Println(v)
			}
		} else {
			fmt.Printf("get all datanodes info failed , ret :%d", ret)
		}

	case "deldatanode":
		argNum := flag.NArg()
		if argNum != 2 {
			fmt.Println("deldatanode ip port")
			os.Exit(1)
		}
		ret := fs.DelDatanode(flag.Arg(0))
		if ret == 0 {
			fmt.Printf("del success")
		} else {
			fmt.Printf("del failed , ret :%d", ret)
		}
	default:
		fmt.Println("wrong action operation")
	}

}
<commit_msg>Update main.go<commit_after>package main

import (
	"flag"
	"fmt"
	fs "github.com/tiglabs/containerfs/fs"
	"github.com/tiglabs/containerfs/logger"
	"github.com/tiglabs/containerfs/utils"
	"os"
	"strconv"
	"strings"
)

func main() {

	var loglevel string
	var logpath string
	var peers string
	var cmd string

	flag.StringVar(&peers, "volmgr", "127.0.0.1:7703,127.0.0.1:7713,127.0.0.1:7723", "ContainerFS VolMgr Host")
	flag.StringVar(&logpath, "logpath", "/export/Logs/containerfs/logs/", "ContainerFS Log Path")
	flag.StringVar(&loglevel, "loglevel", "error", "ContainerFS Log Level")
	flag.StringVar(&cmd, "action", "", `
		createvol [volumename ] size [sata/sas/ssd/nvme...]
		expandvol [volumeuuid ] size
		migrate [host]
		deletevol [volumeuuid ]
		snapshootvol [volumeuuid ]
		getvolmetaleader [volumeuuid ]
		getvolmgrleader
		getvolinfo [volumeuuid ]
		getvols
		getinodeinfo [volumeuuid ] parentinodeid name
		getdatanodes
		deldatanode ip port`)

	flag.Parse()

	fs.VolMgrHosts = strings.Split(peers, ",")
	fs.BufferSize = 1024 * 1024

	logger.SetConsole(true)
	logger.SetRollingFile(logpath, "CLI.log", 10, 100, logger.MB) //each 100M rolling
	switch loglevel {
	case "error":
		logger.SetLevel(logger.ERROR)
	case "debug":
		logger.SetLevel(logger.DEBUG)
	case "info":
		logger.SetLevel(logger.INFO)
	default:
		logger.SetLevel(logger.ERROR)
	}

	//switch flag.Arg(0) {
	switch cmd {

	case "createvol":
		argNum := flag.NArg()
		if argNum != 2 && argNum != 3 {
			fmt.Println("createvol [volname] [space GB] [sata/sas/ssd/nvme...]")
			os.Exit(1)
		}

		var ret int32
		if argNum == 2 {
			ret = fs.CreateVol(flag.Arg(0), flag.Arg(1), "sas")
		} else if argNum == 3 {
			ret = fs.CreateVol(flag.Arg(0), flag.Arg(1), flag.Arg(2))
		}
		if ret != 0 {
			fmt.Println("failed")
		}
		/*
			case "expandvol":
				argNum := flag.NArg()
				if argNum != 2 {
					fmt.Println("expandvol [volUUID] [space GB]")
					os.Exit(1)
				}
				ret := fs.ExpandVolTS(flag.Arg(0), flag.Arg(1))
				if ret != 0 {
					fmt.Println("failed")
				}
		*/
	case "migrate":
		argNum := flag.NArg()
		if argNum != 1 {
			fmt.Println("migrate host")
			os.Exit(1)
		}
		ret := fs.Migrate(flag.Arg(0))
		if ret != 0 {
			fmt.Println("Migrate failed")
		} else {
			fmt.Println("Migrate process on background")
		}

	case "deletevol":
		argNum := flag.NArg()
		if argNum != 1 {
			fmt.Println("deletevol [voluuid]")
			os.Exit(1)
		}
		ret := fs.DeleteVol(flag.Arg(0))
		if ret != 0 {
			fmt.Println("failed")
		}
	case "snapshotvol":
		argNum := flag.NArg()
		if argNum != 1 {
			fmt.Println("snapshotvol [voluuid]")
			os.Exit(1)
		}
		ret := fs.SnapShotVol(flag.Arg(0))
		if ret != 0 {
			fmt.Println("failed")
		}
	case "getvolmetaleader":
		argNum := flag.NArg()
		if argNum != 1 {
			fmt.Println("getvolmetaleader [voluuid]")
			os.Exit(1)
		}
		leader, _ := fs.GetVolMetaLeader(flag.Arg(0))
		fmt.Println(leader)
	case "getvolmgrleader":
		leader, _ := utils.GetVolMgrLeader(fs.VolMgrHosts)
		fmt.Println(leader)
	case "getvolinfo":
		argNum := flag.NArg()
		if argNum != 1 {
			fmt.Println("getvolinfo [volUUID]")
			os.Exit(1)
		}
		ret, vi := fs.GetVolInfo(flag.Arg(0))
		if ret == 0 {
			fmt.Println(vi)
		} else {
			fmt.Printf("get volume info failed , ret :%d", ret)
		}
	case "getvols":
		argNum := flag.NArg()
		if argNum != 0 {
			fmt.Println("getvols")
			os.Exit(1)
		}
		ret, vi := fs.GetAllVolumeInfos()
		if ret == 0 {
			for _, v := range vi {
				fmt.Println(v)
			}
		} else {
			fmt.Printf("get all volumes info failed , ret :%d", ret)
		}

	case "getinodeinfo":
		argNum := flag.NArg()
		if argNum != 3 {
			fmt.Println("getinodeinfo [volUUID] parentinodeid name")
			os.Exit(1)
		}
		cfs := fs.OpenFileSystem(flag.Arg(0))
		if cfs == nil {
			fmt.Println("no such volume")
			os.Exit(1)
		}

		pinode, err := strconv.ParseUint(flag.Arg(1), 10, 64)
		if err == nil {
			ret, inode, inodeinfo := cfs.GetInodeInfoDirect(pinode, flag.Arg(2))
			if ret != 0 {
				fmt.Println("no such inode")
			} else {
				fmt.Printf("inode:%v\ninodeinfo:%v\n", inode, inodeinfo)
				ret, chunkinfo, _ := cfs.GetFileChunksDirect(pinode, flag.Arg(2))
				if ret != 0 {
					fmt.Printf("no chunkinfo")
				} else {
					fmt.Printf("chunkinfo:%v\n", chunkinfo)

				}
			}
		} else {
			fmt.Println("err parent inode", err)
		}
	case "getdatanodes":
		argNum := flag.NArg()
		if argNum != 0 {
			fmt.Println("getdatanodes")
			os.Exit(1)
		}
		ret, vi := fs.GetAllDatanode()
		if ret == 0 {
			for _, v := range vi {
				fmt.Println(v)
			}
		} else {
			fmt.Printf("get all datanodes info failed , ret :%d", ret)
		}

	case "deldatanode":
		argNum := flag.NArg()
		if argNum != 2 {
			fmt.Println("deldatanode ip port")
			os.Exit(1)
		}
		ret := fs.DelDatanode(flag.Arg(0))
		if ret == 0 {
			fmt.Printf("del success")
		} else {
			fmt.Printf("del failed , ret :%d", ret)
		}
	default:
		fmt.Println("wrong action operation")
	}

}
<|endoftext|>


<commit_before>package mcclient

import (
	"bytes"
	"fmt"
	"image/png"
	"strings"
	"time"

	"github.com/minotar/imgd/pkg/cache"
	"github.com/minotar/imgd/pkg/mcclient/mcuser"
	"github.com/minotar/imgd/pkg/mcclient/uuid"
	"github.com/minotar/imgd/pkg/util/log"
	"github.com/minotar/minecraft"
)

const (
	skinTTL = 1 * time.Hour
)

// Todo: metrics / tracing / timing

// Todo: Also, add function context to logger
// User cache name and UUID in logger.With()?

// CacheRetrieveUUIDEntry searches the cache based on Username, expecting a UUID in response
func (mc *McClient) CacheRetrieveUUIDEntry(logger log.Logger, username string) (uuidEntry uuid.UUIDEntry, err error) {
	// logger should already be With() the username
	username = strings.ToLower(username)
	// Metrics timer / tracing
	// Though - is this useless when using a TieredCache which is inherentantly varied?
	uuidBytes, err := mc.Caches.UUID.Retrieve(username)
	// Observe Cache retrieve
	if err != nil {
		// Return an error (and log based on severity)
		if err == cache.ErrNotFound {
			// Metrics stat "Miss"
			logger.Debugf("Did not find username in %s", mc.Caches.UUID.Name())
			return
		} else {
			// Metrics stat Cache RetrieveError
			logger.Errorf("Failed to lookup up username in %s: %v", mc.Caches.UUID.Name(), err)
			return
		}
	}

	if len(uuidBytes) < 5 {
		// 4 bytes or less would be an invalid status/timestamp
		// Metrics stats Cache Empty Error
		logger.Errorf("Null UUID returned for username in %s: %v", mc.Caches.UUID.Name(), uuidBytes)
		return uuid.UUIDEntry{}, fmt.Errorf("cache returned null UUID")
	}

	uuidEntry = uuid.DecodeUUIDEntry(uuidBytes)

	// Metrics stat Hit
	logger.Debugf("Found username in %s", mc.Caches.UUID.Name())
	return
}

// CacheInsertUUIDEntry takes a valid UUIDEntry and caches it
// There is no sanity checking on the input
// The item is cached witha TTL assuming it's brand-new
func (mc *McClient) CacheInsertUUIDEntry(logger log.Logger, username string, uuidEntry uuid.UUIDEntry) (err error) {
	// logger should already be With() the username and UUID
	username = strings.ToLower(username)
	// Technically this could be empty / nil???
	uuidBytes := uuidEntry.Encode()

	// Metrics timer / tracing
	// Though - is this useless when using a TieredCache which is inherentantly varied?
	err = mc.Caches.UUID.InsertTTL(username, uuidBytes, uuidEntry.TTL())
	// Observe Cache insert
	if err != nil {
		// stats.CacheUUID("error")
		logger.Errorf("Failed Insert into cache %s: %v", mc.Caches.UUID.Name(), err)
	}
	return
}

func (mc *McClient) CacheRetrieveMcUser(logger log.Logger, uuid string) (user mcuser.McUser, err error) {
	// logger should already be With() the UUID (and maybe username)
	uuid = strings.ToLower(uuid)
	// Metrics timer / tracing
	// Though - is this useless when using a TieredCache which is inherentantly varied?
	userBytes, err := mc.Caches.UserData.Retrieve(uuid)
	// Observe Cache retrieve
	if err != nil {
		// Return an error (and log based on severity)
		if err == cache.ErrNotFound {
			// Metrics stat "Miss"
			logger.Debugf("Did not find uuid in %s", mc.Caches.UserData.Name())
			return
		} else {
			// Metrics stat Cache RetrieveError
			logger.Errorf("Failed to lookup up uuid in %s: %v", mc.Caches.UserData.Name(), err)
			return
		}
	}

	user, err = mcuser.DecompressMcUser(userBytes)
	if err != nil {
		logger.Errorf("Failed to decode McUser from uuid in %s: %v", mc.Caches.UserData.Name(), err)
		// Metrics stats Cache Decode Error
		return
	}
	// Metrics stat Hit
	logger.Debugf("Found uuid in %s", mc.Caches.UserData.Name())
	return
}

func (mc *McClient) CacheInsertMcUser(logger log.Logger, uuid string, user mcuser.McUser) (err error) {
	// logger should already be With() the UUID (and maybe username)
	uuid = strings.ToLower(uuid)
	packedUserBytes, err := user.Compress()
	if err != nil {
		// stats.CacheUser("pack_error")
		logger.Errorf("Failed to pack UUID:user ready to cache: %v", err)
	}

	// Metrics timer / tracing
	// Though - is this useless when using a TieredCache which is inherentantly varied?
	err = mc.Caches.UUID.InsertTTL(uuid, packedUserBytes, user.TTL())
	// Observe Cache insert
	if err != nil {
		// stats.CacheUser("insert_error")
		logger.Errorf("Failed Insert UUID:user into cache %s: %v", mc.Caches.UserData.Name(), err)
	}
	return
}

//

func (mc *McClient) CacheRetrieveTexture(logger log.Logger, textureKey string) (texture minecraft.Texture, err error) {
	// logger should already be With() the skinPath/texturePath (and UUID and Username)
	textureKey = strings.ToLower(textureKey)
	// Metrics timer / tracing
	// Though - is this useless when using a TieredCache which is inherentantly varied?
	textureBytes, err := mc.Caches.Textures.Retrieve(textureKey)
	// Observe Cache retrieve
	if err != nil {
		// Return an error (and log based on severity)
		if err == cache.ErrNotFound {
			// Metrics stat "Miss"
			logger.Debugf("Did not find texture in %s", mc.Caches.Textures.Name())
			return
		} else {
			// Metrics stat Cache RetrieveError
			logger.Errorf("Failed to lookup up texture in %s: %v", mc.Caches.Textures.Name(), err)
			return
		}
	}

	textureReader := bytes.NewReader(textureBytes)

	err = texture.Decode(textureReader)
	if err != nil {
		logger.Errorf("Failed to decode texture from %s: %v", mc.Caches.Textures.Name(), err)
		// Metrics stats Cache Decode Error
		return
	}
	// Metrics stat Hit
	logger.Debugf("Found texture in %s", mc.Caches.Textures.Name())
	return
}

func (mc *McClient) CacheInsertTexture(logger log.Logger, textureKey string, texture minecraft.Texture) (err error) {
	// logger should already be With() the skinPath/texturePath (and UUID and Username)
	textureKey = strings.ToLower(textureKey)

	textureBuf := new(bytes.Buffer)
	// Todo: research better lossless encoding - or just try the png.Encoder with higher compression
	// bimg/vips ? Maybe cache as a different format if more efficient
	err = png.Encode(textureBuf, texture.Image)
	if err != nil {
		// stats.CacheUser("pack_error")
		logger.Errorf("Failed to PNG encode texture for cache: %v", err)
	}

	// Metrics timer / tracing
	// Though - is this useless when using a TieredCache which is inherentantly varied?
	err = mc.Caches.UUID.InsertTTL(textureKey, textureBuf.Bytes(), skinTTL)
	// Observe Cache insert
	if err != nil {
		// stats.CacheUser("insert_error")
		logger.Errorf("Failed Insert texture PNG into cache %s: %v", mc.Caches.Textures.Name(), err)
	}
	return
}
<commit_msg>Fix used cache for insert<commit_after>package mcclient

import (
	"bytes"
	"fmt"
	"image/png"
	"strings"
	"time"

	"github.com/minotar/imgd/pkg/cache"
	"github.com/minotar/imgd/pkg/mcclient/mcuser"
	"github.com/minotar/imgd/pkg/mcclient/uuid"
	"github.com/minotar/imgd/pkg/util/log"
	"github.com/minotar/minecraft"
)

const (
	skinTTL = 1 * time.Hour
)

// Todo: metrics / tracing / timing

// Todo: Also, add function context to logger
// User cache name and UUID in logger.With()?

// CacheRetrieveUUIDEntry searches the cache based on Username, expecting a UUID in response
func (mc *McClient) CacheRetrieveUUIDEntry(logger log.Logger, username string) (uuidEntry uuid.UUIDEntry, err error) {
	// logger should already be With() the username
	username = strings.ToLower(username)
	// Metrics timer / tracing
	// Though - is this useless when using a TieredCache which is inherentantly varied?
	uuidBytes, err := mc.Caches.UUID.Retrieve(username)
	// Observe Cache retrieve
	if err != nil {
		// Return an error (and log based on severity)
		if err == cache.ErrNotFound {
			// Metrics stat "Miss"
			logger.Debugf("Did not find username in %s", mc.Caches.UUID.Name())
			return
		} else {
			// Metrics stat Cache RetrieveError
			logger.Errorf("Failed to lookup up username in %s: %v", mc.Caches.UUID.Name(), err)
			return
		}
	}

	if len(uuidBytes) < 5 {
		// 4 bytes or less would be an invalid status/timestamp
		// Metrics stats Cache Empty Error
		logger.Errorf("Null UUID returned for username in %s: %v", mc.Caches.UUID.Name(), uuidBytes)
		return uuid.UUIDEntry{}, fmt.Errorf("cache returned null UUID")
	}

	uuidEntry = uuid.DecodeUUIDEntry(uuidBytes)

	// Metrics stat Hit
	logger.Debugf("Found username in %s", mc.Caches.UUID.Name())
	return
}

// CacheInsertUUIDEntry takes a valid UUIDEntry and caches it
// There is no sanity checking on the input
// The item is cached witha TTL assuming it's brand-new
func (mc *McClient) CacheInsertUUIDEntry(logger log.Logger, username string, uuidEntry uuid.UUIDEntry) (err error) {
	// logger should already be With() the username and UUID
	username = strings.ToLower(username)
	// Technically this could be empty / nil???
	uuidBytes := uuidEntry.Encode()

	// Metrics timer / tracing
	// Though - is this useless when using a TieredCache which is inherentantly varied?
	err = mc.Caches.UUID.InsertTTL(username, uuidBytes, uuidEntry.TTL())
	// Observe Cache insert
	if err != nil {
		// stats.CacheUUID("error")
		logger.Errorf("Failed Insert into cache %s: %v", mc.Caches.UUID.Name(), err)
	}
	return
}

func (mc *McClient) CacheRetrieveMcUser(logger log.Logger, uuid string) (user mcuser.McUser, err error) {
	// logger should already be With() the UUID (and maybe username)
	uuid = strings.ToLower(uuid)
	// Metrics timer / tracing
	// Though - is this useless when using a TieredCache which is inherentantly varied?
	userBytes, err := mc.Caches.UserData.Retrieve(uuid)
	// Observe Cache retrieve
	if err != nil {
		// Return an error (and log based on severity)
		if err == cache.ErrNotFound {
			// Metrics stat "Miss"
			logger.Debugf("Did not find uuid in %s", mc.Caches.UserData.Name())
			return
		} else {
			// Metrics stat Cache RetrieveError
			logger.Errorf("Failed to lookup up uuid in %s: %v", mc.Caches.UserData.Name(), err)
			return
		}
	}

	user, err = mcuser.DecompressMcUser(userBytes)
	if err != nil {
		logger.Errorf("Failed to decode McUser from uuid in %s: %v", mc.Caches.UserData.Name(), err)
		// Metrics stats Cache Decode Error
		return
	}
	// Metrics stat Hit
	logger.Debugf("Found uuid in %s", mc.Caches.UserData.Name())
	return
}

func (mc *McClient) CacheInsertMcUser(logger log.Logger, uuid string, user mcuser.McUser) (err error) {
	// logger should already be With() the UUID (and maybe username)
	uuid = strings.ToLower(uuid)
	packedUserBytes, err := user.Compress()
	if err != nil {
		// stats.CacheUser("pack_error")
		logger.Errorf("Failed to pack UUID:user ready to cache: %v", err)
	}

	// Metrics timer / tracing
	// Though - is this useless when using a TieredCache which is inherentantly varied?
	err = mc.Caches.UserData.InsertTTL(uuid, packedUserBytes, user.TTL())
	// Observe Cache insert
	if err != nil {
		// stats.CacheUser("insert_error")
		logger.Errorf("Failed Insert UUID:user into cache %s: %v", mc.Caches.UserData.Name(), err)
	}
	return
}

//

func (mc *McClient) CacheRetrieveTexture(logger log.Logger, textureKey string) (texture minecraft.Texture, err error) {
	// logger should already be With() the skinPath/texturePath (and UUID and Username)
	textureKey = strings.ToLower(textureKey)
	// Metrics timer / tracing
	// Though - is this useless when using a TieredCache which is inherentantly varied?
	textureBytes, err := mc.Caches.Textures.Retrieve(textureKey)
	// Observe Cache retrieve
	if err != nil {
		// Return an error (and log based on severity)
		if err == cache.ErrNotFound {
			// Metrics stat "Miss"
			logger.Debugf("Did not find texture in %s", mc.Caches.Textures.Name())
			return
		} else {
			// Metrics stat Cache RetrieveError
			logger.Errorf("Failed to lookup up texture in %s: %v", mc.Caches.Textures.Name(), err)
			return
		}
	}

	textureReader := bytes.NewReader(textureBytes)

	err = texture.Decode(textureReader)
	if err != nil {
		logger.Errorf("Failed to decode texture from %s: %v", mc.Caches.Textures.Name(), err)
		// Metrics stats Cache Decode Error
		return
	}
	// Metrics stat Hit
	logger.Debugf("Found texture in %s", mc.Caches.Textures.Name())
	return
}

func (mc *McClient) CacheInsertTexture(logger log.Logger, textureKey string, texture minecraft.Texture) (err error) {
	// logger should already be With() the skinPath/texturePath (and UUID and Username)
	textureKey = strings.ToLower(textureKey)

	textureBuf := new(bytes.Buffer)
	// Todo: research better lossless encoding - or just try the png.Encoder with higher compression
	// bimg/vips ? Maybe cache as a different format if more efficient
	err = png.Encode(textureBuf, texture.Image)
	if err != nil {
		// stats.CacheUser("pack_error")
		logger.Errorf("Failed to PNG encode texture for cache: %v", err)
	}

	// Metrics timer / tracing
	// Though - is this useless when using a TieredCache which is inherentantly varied?
	err = mc.Caches.Textures.InsertTTL(textureKey, textureBuf.Bytes(), skinTTL)
	// Observe Cache insert
	if err != nil {
		// stats.CacheUser("insert_error")
		logger.Errorf("Failed Insert texture PNG into cache %s: %v", mc.Caches.Textures.Name(), err)
	}
	return
}
<|endoftext|>


<commit_before>package heroku

import (
	"context"
	"log"

	heroku "github.com/cyberdelia/heroku-go/v3"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceHerokuSpace() *schema.Resource {
	return &schema.Resource{
		Create: resourceHerokuSpaceCreate,
		Read:   resourceHerokuSpaceRead,
		Update: resourceHerokuSpaceUpdate,
		Delete: resourceHerokuSpaceDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
			},

			"organization": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"region": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
		},
	}
}

func resourceHerokuSpaceCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*heroku.Service)

	opts := heroku.SpaceCreateOpts{}
	opts.Name = d.Get("name").(string)
	opts.Organization = d.Get("organization").(string)

	if v, ok := d.GetOk("region"); ok {
		vs := v.(string)
		opts.Region = &vs
	}

	space, err := client.SpaceCreate(context.TODO(), opts)
	if err != nil {
		return err
	}

	d.SetId(space.ID)
	log.Printf("[INFO] Space ID: %s", d.Id())

	setSpaceAttributes(d, (*heroku.Space)(space))
	return nil
}

func resourceHerokuSpaceRead(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*heroku.Service)

	space, err := client.SpaceInfo(context.TODO(), d.Id())
	if err != nil {
		return err
	}

	setSpaceAttributes(d, (*heroku.Space)(space))
	return nil
}

func resourceHerokuSpaceUpdate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*heroku.Service)

	if !d.HasChange("name") {
		return nil
	}

	name := d.Get("name").(string)
	opts := heroku.SpaceUpdateOpts{Name: &name}

	space, err := client.SpaceUpdate(context.TODO(), d.Id(), opts)
	if err != nil {
		return err
	}

	setSpaceAttributes(d, (*heroku.Space)(space))
	return nil
}

func setSpaceAttributes(d *schema.ResourceData, space *heroku.Space) {
	d.Set("name", space.Name)
	d.Set("organization", space.Organization.Name)
	d.Set("region", space.Region.Name)
}

func resourceHerokuSpaceDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*heroku.Service)

	log.Printf("[INFO] Deleting space: %s", d.Id())
	_, err := client.SpaceDelete(context.TODO(), d.Id())
	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}
<commit_msg>Add note about type conversion<commit_after>package heroku

import (
	"context"
	"log"

	heroku "github.com/cyberdelia/heroku-go/v3"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceHerokuSpace() *schema.Resource {
	return &schema.Resource{
		Create: resourceHerokuSpaceCreate,
		Read:   resourceHerokuSpaceRead,
		Update: resourceHerokuSpaceUpdate,
		Delete: resourceHerokuSpaceDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
			},

			"organization": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"region": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
		},
	}
}

func resourceHerokuSpaceCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*heroku.Service)

	opts := heroku.SpaceCreateOpts{}
	opts.Name = d.Get("name").(string)
	opts.Organization = d.Get("organization").(string)

	if v, ok := d.GetOk("region"); ok {
		vs := v.(string)
		opts.Region = &vs
	}

	space, err := client.SpaceCreate(context.TODO(), opts)
	if err != nil {
		return err
	}

	d.SetId(space.ID)
	log.Printf("[INFO] Space ID: %s", d.Id())

	// The type conversion here can be dropped when the vendored version of
	// heroku-go is updated.
	setSpaceAttributes(d, (*heroku.Space)(space))
	return nil
}

func resourceHerokuSpaceRead(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*heroku.Service)

	space, err := client.SpaceInfo(context.TODO(), d.Id())
	if err != nil {
		return err
	}

	// The type conversion here can be dropped when the vendored version of
	// heroku-go is updated.
	setSpaceAttributes(d, (*heroku.Space)(space))
	return nil
}

func resourceHerokuSpaceUpdate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*heroku.Service)

	if !d.HasChange("name") {
		return nil
	}

	name := d.Get("name").(string)
	opts := heroku.SpaceUpdateOpts{Name: &name}

	space, err := client.SpaceUpdate(context.TODO(), d.Id(), opts)
	if err != nil {
		return err
	}

	// The type conversion here can be dropped when the vendored version of
	// heroku-go is updated.
	setSpaceAttributes(d, (*heroku.Space)(space))
	return nil
}

func setSpaceAttributes(d *schema.ResourceData, space *heroku.Space) {
	d.Set("name", space.Name)
	d.Set("organization", space.Organization.Name)
	d.Set("region", space.Region.Name)
}

func resourceHerokuSpaceDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*heroku.Service)

	log.Printf("[INFO] Deleting space: %s", d.Id())
	_, err := client.SpaceDelete(context.TODO(), d.Id())
	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}
<|endoftext|>


<commit_before>// Copyright © 2016 Thomas Rabaix <thomas.rabaix@gmail.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package composer

import (
	"encoding/json"
	"fmt"
	"time"
)

type ProviderInclude map[string]struct {
	Sha256 string `json:"sha256"`
}

type PackagesResult struct {
	Packages         json.RawMessage `json:"packages"`
	Notify           string          `json:"notify"`
	NotifyBatch      string          `json:"notify-batch"`
	ProvidersURL     string          `json:"providers-url"`
	Search           string          `json:"search"`
	ProviderIncludes ProviderInclude `json:"provider-includes"`
}

type ProvidersResult struct {
	Providers map[string]struct {
		Sha256 string `json:"sha256"`
	} `json:"providers"`
	Code string `json:"-"`
}

// package description
type Package struct {
	Name              string           `json:"name"`
	Abandoned         *json.RawMessage `json:"abandoned"`
	Description       string           `json:"description"`
	Keywords          []string         `json:"keywords"`
	Homepage          string           `json:"homepage"`
	Version           string           `json:"version"`
	VersionNormalized string           `json:"version_normalized"`
	License           []string         `json:"license"`
	Bin               []string         `json:"bin"`
	Authors           []struct {
		Name     string `json:"name"`
		Email    string `json:"email"`
		Homepage string `json:"homepage"`
		Role     string `json:"role"`
	} `json:"authors"`
	Source struct {
		Type      string `json:"type"`
		URL       string `json:"url"`
		Reference string `json:"reference"`
	} `json:"source"`
	Dist struct {
		Type      string `json:"type"`
		URL       string `json:"url"`
		Reference string `json:"reference"`
		Shasum    string `json:"shasum"`
	} `json:"dist"`
	Extra      *json.RawMessage `json:"extra"`
	TargetDir  string           `json:"target-dir"`
	Type       string           `json:"type"`
	Time       time.Time        `json:"time"`
	Autoload   *json.RawMessage `json:"autoload"`
	Replace    *json.RawMessage `json:"replace"`
	Conflict   *json.RawMessage `json:"conflict"`
	Provide    *json.RawMessage `json:"provide"`
	Require    *json.RawMessage `json:"require"`
	RequireDev *json.RawMessage `json:"require-dev"`
	Suggest    *json.RawMessage `json:"suggest"`
	UID        int              `json:"uid"`
}

// used to load the packages.json file
type PackageResult struct {
	Packages map[string]map[string]*Package `json:"packages"`
}

type PackageInformation struct {
	Server        string        `json:"server"`
	PackageResult PackageResult `json:"-"`
	Package       string        `json:"package"`
	Exist         bool          `json:"-"`
	HashSource    string        `json:"hash_source"`
	HashTarget    string        `json:"hash_target"`
}

func (pi *PackageInformation) GetSourceKey() string {
	return fmt.Sprintf("%s$%s.json", pi.Package, pi.HashSource)
}

func (pi *PackageInformation) GetTargetKey() string {
	return fmt.Sprintf("%s$%s", pi.Package, pi.HashTarget)
}
<commit_msg>fix(composer): remove empty field from serialization<commit_after>// Copyright © 2016 Thomas Rabaix <thomas.rabaix@gmail.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package composer

import (
	"encoding/json"
	"fmt"
	"time"
)

type ProviderInclude map[string]struct {
	Sha256 string `json:"sha256"`
}

type PackagesResult struct {
	Packages         json.RawMessage `json:"packages"`
	Notify           string          `json:"notify"`
	NotifyBatch      string          `json:"notify-batch"`
	ProvidersURL     string          `json:"providers-url"`
	Search           string          `json:"search"`
	ProviderIncludes ProviderInclude `json:"provider-includes"`
}

type ProvidersResult struct {
	Providers map[string]struct {
		Sha256 string `json:"sha256"`
	} `json:"providers"`
	Code string `json:"-"`
}

// package description
type Package struct {
	Name              string           `json:"name,omitempty"`
	Abandoned         *json.RawMessage `json:"abandoned,omitempty"`
	Description       string           `json:"description,omitempty"`
	Keywords          []string         `json:"keywords,omitempty"`
	Homepage          string           `json:"homepage,omitempty"`
	Version           string           `json:"version,omitempty"`
	VersionNormalized string           `json:"version_normalized,omitempty"`
	License           []string         `json:"license,omitempty"`
	Bin               []string         `json:"bin,omitempty"`
	Authors           []struct {
		Name     string `json:"name"`
		Email    string `json:"email"`
		Homepage string `json:"homepage"`
		Role     string `json:"role"`
	} `json:"authors,omitempty"`
	Source struct {
		Type      string `json:"type"`
		URL       string `json:"url"`
		Reference string `json:"reference"`
	} `json:"source,omitempty"`
	Dist struct {
		Type      string `json:"type"`
		URL       string `json:"url"`
		Reference string `json:"reference"`
		Shasum    string `json:"shasum"`
	} `json:"dist,omitempty"`
	Extra      *json.RawMessage `json:"extra,omitempty"`
	TargetDir  string           `json:"target-dir,omitempty"`
	Type       string           `json:"type,omitempty"`
	Time       time.Time        `json:"time,omitempty"`
	Autoload   *json.RawMessage `json:"autoload,omitempty"`
	Replace    *json.RawMessage `json:"replace,omitempty"`
	Conflict   *json.RawMessage `json:"conflict,omitempty"`
	Provide    *json.RawMessage `json:"provide,omitempty"`
	Require    *json.RawMessage `json:"require,omitempty"`
	RequireDev *json.RawMessage `json:"require-dev,omitempty"`
	Suggest    *json.RawMessage `json:"suggest,omitempty"`
	UID        int              `json:"uid,omitempty"`
}

// used to load the packages.json file
type PackageResult struct {
	Packages map[string]map[string]*Package `json:"packages"`
}

type PackageInformation struct {
	Server        string        `json:"server"`
	PackageResult PackageResult `json:"-"`
	Package       string        `json:"package"`
	Exist         bool          `json:"-"`
	HashSource    string        `json:"hash_source"`
	HashTarget    string        `json:"hash_target"`
}

func (pi *PackageInformation) GetSourceKey() string {
	return fmt.Sprintf("%s$%s.json", pi.Package, pi.HashSource)
}

func (pi *PackageInformation) GetTargetKey() string {
	return fmt.Sprintf("%s$%s", pi.Package, pi.HashTarget)
}
<|endoftext|>


<commit_before>package rancher

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"gopkg.in/yaml.v2"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/machine/drivers/amazonec2"
	"github.com/docker/machine/drivers/digitalocean"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/drivers/rpc"
	"github.com/docker/machine/libmachine/mcnflag"
	"github.com/docker/machine/libmachine/state"
	"github.com/packethost/docker-machine-driver-packet"
)

const (
	flagPrefix = "rancher-"
)

var (
	apiKeyFlagNames = []string{
		// Amazon
		"access-key",
		"secret-key",

		// Digital Ocean
		"access-token",

		// Packet
		"api-key",
		"project-id",
	}
)

type Flavor struct {
	Provider      string
	DriverOptions map[string]interface{}
}

var _ drivers.Driver = &Driver{}

type Driver struct {
	*drivers.BaseDriver

	ProviderDriverOptions map[string]interface{}

	AvailableFlavors map[string]Flavor
	flavor           Flavor

	AmazonEC2Driver    *amazonec2.Driver
	DigitalOceanDriver *digitalocean.Driver
	PacketDriver       *packet.Driver
	Driver             drivers.Driver
}

func NewDriver(hostName, storePath string) *Driver {
	return &Driver{
		BaseDriver: &drivers.BaseDriver{
			MachineName: hostName,
			StorePath:   storePath,
		},
	}
}

func (d *Driver) DriverName() string {
	return "rancher"
}

// Transforms a list of flags to add rancher- as a prefix
func addPrefixToFlags(flags []mcnflag.Flag) []mcnflag.Flag {
	var newFlags []mcnflag.Flag
	for _, flag := range flags {
		switch flag := flag.(type) {
		case mcnflag.BoolFlag:
			flag.Name = flagPrefix + flag.Name
			newFlags = append(newFlags, flag)
		case mcnflag.IntFlag:
			flag.Name = flagPrefix + flag.Name
			newFlags = append(newFlags, flag)
		case mcnflag.StringFlag:
			flag.Name = flagPrefix + flag.Name
			newFlags = append(newFlags, flag)
		case mcnflag.StringSliceFlag:
			flag.Name = flagPrefix + flag.Name
			newFlags = append(newFlags, flag)
		}
	}
	return newFlags
}

func (d *Driver) GetCreateFlags() []mcnflag.Flag {
	// Rancher specific flags
	flags := []mcnflag.Flag{
		mcnflag.StringFlag{
			Name: "flavor",
		},
	}

	// Borrow API key flags from inner drivers
	var innerFlags []mcnflag.Flag
	innerFlags = append(innerFlags, d.AmazonEC2Driver.GetCreateFlags()...)
	innerFlags = append(innerFlags, d.DigitalOceanDriver.GetCreateFlags()...)
	innerFlags = append(innerFlags, d.PacketDriver.GetCreateFlags()...)
	for _, innerFlag := range innerFlags {
		for _, name := range apiKeyFlagNames {
			if strings.Contains(innerFlag.String(), name) {
				flags = append(flags, innerFlag)
			}
		}
	}

	// Docker Machine requires all driver flags to be prefixed with rancher-
	return addPrefixToFlags(flags)
}

func (d *Driver) setupInnerDriver() error {
	if d.AmazonEC2Driver == nil {
		d.AmazonEC2Driver = amazonec2.NewDriver(d.MachineName, d.StorePath)
	}
	if d.DigitalOceanDriver == nil {
		d.DigitalOceanDriver = digitalocean.NewDriver(d.MachineName, d.StorePath)
	}
	if d.PacketDriver == nil {
		d.PacketDriver = packet.NewDriver(d.MachineName, d.StorePath)
	}

	if d.flavor.Provider == "amazonec2" {
		d.Driver = d.AmazonEC2Driver
	} else if d.flavor.Provider == "digitalocean" {
		d.Driver = d.DigitalOceanDriver
	} else if d.flavor.Provider == "packet" {
		d.Driver = d.PacketDriver
	}

	return nil
}

func (d *Driver) readProviderAndFlavorInfo(selectedFlavor string) error {
	flavorsDir := os.Getenv("FLAVORS_DIR")
	log.Debugf("Reading flavor configs from directory %s", flavorsDir)
	providersDir := os.Getenv("PROVIDERS_DIR")
	log.Debugf("Reading provider configs from directory %s", providersDir)

	files, err := ioutil.ReadDir(flavorsDir)
	if err != nil {
		return fmt.Errorf("Failed to read flavors directory %s: %v", flavorsDir, err)
	}

	flavorFound := false
	for _, file := range files {
		if file.Name() == selectedFlavor+".yml" {
			bytes, err := ioutil.ReadFile(path.Join(flavorsDir, file.Name()))
			if err != nil {
				return err
			}
			if err = yaml.Unmarshal(bytes, &d.flavor); err != nil {
				return err
			}
			flavorFound = true
			break
		}
	}
	if !flavorFound {
		return fmt.Errorf("Invalid flavor %s", selectedFlavor)
	}

	files, err = ioutil.ReadDir(providersDir)
	if err != nil {
		return fmt.Errorf("Failed to read providers directory: %v", err)
	}

	providerFound := false
	for _, file := range files {
		if file.Name() == d.flavor.Provider+".yml" {
			bytes, err := ioutil.ReadFile(path.Join(providersDir, file.Name()))
			if err != nil {
				return err
			}
			if err = yaml.Unmarshal(bytes, &d.ProviderDriverOptions); err != nil {
				return err
			}
			providerFound = true
			break
		}
	}
	if !providerFound {
		return fmt.Errorf("Invalid provider %s", d.flavor.Provider)
	}

	return nil
}

// Strips off the rancher- prefix from flags
func stripPrefixFromFlags(flags rpcdriver.RPCFlags) rpcdriver.RPCFlags {
	for k, v := range flags.Values {
		s := fmt.Sprint(k)
		if strings.Contains(s, flagPrefix) {
			delete(flags.Values, k)
			flags.Values[strings.Replace(s, flagPrefix, "", -1)] = v
		}
	}
	return flags
}

// Merge the four sources of flag values, from lowest priority to highest
// Defaults from inner driver
// Values determined by the provider
// Values determined by flavor
// Values passed in via CLI (likely API keys)
func getDriverOpts(mcnflags []mcnflag.Flag, providerDriverOptions, flavorDriverOptions, cliDriverOptions map[string]interface{}) rpcdriver.RPCFlags {
	driverOpts := rpcdriver.RPCFlags{
		Values: make(map[string]interface{}),
	}
	for _, f := range mcnflags {
		driverOpts.Values[f.String()] = f.Default()
		if f.Default() == nil {
			driverOpts.Values[f.String()] = false
		}
	}
	for k, v := range providerDriverOptions {
		driverOpts.Values[k] = v
	}
	for k, v := range flavorDriverOptions {
		driverOpts.Values[k] = v
	}
	for k, v := range cliDriverOptions {
		driverOpts.Values[k] = v
	}

	// Strip off the rancher- prefix since inner drivers won't recognize it
	return stripPrefixFromFlags(driverOpts)
}

func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error {
	if err := d.readProviderAndFlavorInfo(flags.String("rancher-flavor")); err != nil {
		return err
	}

	if err := d.setupInnerDriver(); err != nil {
		return err
	}

	// TODO: try to avoid this type assertion
	cliDriverOptions := flags.(*rpcdriver.RPCFlags)
	driverOptions := getDriverOpts(d.Driver.GetCreateFlags(), d.ProviderDriverOptions, cliDriverOptions.Values, d.flavor.DriverOptions)

	if err := d.Driver.SetConfigFromFlags(driverOptions); err != nil {
		return err
	}

	if d.flavor.Provider == "amazonec2" {
		if err := d.setupAmazon(); err != nil {
			return err
		}
	}

	return nil
}

func (d *Driver) GetSSHHostname() (string, error) {
	return d.Driver.GetSSHHostname()
}

func (d *Driver) PreCreateCheck() error {
	return d.Driver.PreCreateCheck()
}

func (d *Driver) Create() error {
	return d.Driver.Create()
}

func (d *Driver) GetURL() (string, error) {
	return d.Driver.GetURL()
}

func (d *Driver) GetIP() (string, error) {
	return d.Driver.GetIP()
}

func (d *Driver) GetState() (state.State, error) {
	return d.Driver.GetState()
}

func (d *Driver) Start() error {
	return d.Driver.Start()
}

func (d *Driver) Stop() error {
	return d.Driver.Stop()
}

func (d *Driver) Remove() error {
	return d.Driver.Remove()
}

func (d *Driver) Restart() error {
	return d.Driver.Restart()
}

func (d *Driver) Kill() error {
	return d.Driver.Kill()
}
<commit_msg>Small tweaks to provider/flavor file handling<commit_after>package rancher

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"gopkg.in/yaml.v2"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/machine/drivers/amazonec2"
	"github.com/docker/machine/drivers/digitalocean"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/drivers/rpc"
	"github.com/docker/machine/libmachine/mcnflag"
	"github.com/docker/machine/libmachine/state"
	"github.com/packethost/docker-machine-driver-packet"
)

const (
	flagPrefix = "rancher-"
)

var (
	flavorsDirDefault   = "/machine/flavors"
	providersDirDefault = "/machine/providers"
)

func init() {
	home := os.Getenv("CATTLE_HOME")
	if home == "" {
		home = "/var/lib/cattle"
	}
	flavorsDirDefault = home + flavorsDirDefault
	providersDirDefault = home + providersDirDefault
}

var (
	apiKeyFlagNames = []string{
		// Amazon
		"access-key",
		"secret-key",

		// Digital Ocean
		"access-token",

		// Packet
		"api-key",
		"project-id",
	}
)

type Flavor struct {
	Provider      string
	DriverOptions map[string]interface{} `yaml:"driver_options,omitempty"`
}

var _ drivers.Driver = &Driver{}

type Driver struct {
	*drivers.BaseDriver

	ProviderDriverOptions map[string]interface{}

	AvailableFlavors map[string]Flavor
	flavor           Flavor

	AmazonEC2Driver    *amazonec2.Driver
	DigitalOceanDriver *digitalocean.Driver
	PacketDriver       *packet.Driver
	Driver             drivers.Driver
}

func NewDriver(hostName, storePath string) *Driver {
	return &Driver{
		BaseDriver: &drivers.BaseDriver{
			MachineName: hostName,
			StorePath:   storePath,
		},
	}
}

func (d *Driver) DriverName() string {
	return "rancher"
}

// Transforms a list of flags to add rancher- as a prefix
func addPrefixToFlags(flags []mcnflag.Flag) []mcnflag.Flag {
	var newFlags []mcnflag.Flag
	for _, flag := range flags {
		switch flag := flag.(type) {
		case mcnflag.BoolFlag:
			flag.Name = flagPrefix + flag.Name
			newFlags = append(newFlags, flag)
		case mcnflag.IntFlag:
			flag.Name = flagPrefix + flag.Name
			newFlags = append(newFlags, flag)
		case mcnflag.StringFlag:
			flag.Name = flagPrefix + flag.Name
			newFlags = append(newFlags, flag)
		case mcnflag.StringSliceFlag:
			flag.Name = flagPrefix + flag.Name
			newFlags = append(newFlags, flag)
		}
	}
	return newFlags
}

func (d *Driver) GetCreateFlags() []mcnflag.Flag {
	// Rancher specific flags
	flags := []mcnflag.Flag{
		mcnflag.StringFlag{
			Name: "flavor",
		},
	}

	// Borrow API key flags from inner drivers
	var innerFlags []mcnflag.Flag
	innerFlags = append(innerFlags, d.AmazonEC2Driver.GetCreateFlags()...)
	innerFlags = append(innerFlags, d.DigitalOceanDriver.GetCreateFlags()...)
	innerFlags = append(innerFlags, d.PacketDriver.GetCreateFlags()...)
	for _, innerFlag := range innerFlags {
		for _, name := range apiKeyFlagNames {
			if strings.Contains(innerFlag.String(), name) {
				flags = append(flags, innerFlag)
			}
		}
	}

	// Docker Machine requires all driver flags to be prefixed with rancher-
	return addPrefixToFlags(flags)
}

func (d *Driver) setupInnerDriver() error {
	if d.AmazonEC2Driver == nil {
		d.AmazonEC2Driver = amazonec2.NewDriver(d.MachineName, d.StorePath)
	}
	if d.DigitalOceanDriver == nil {
		d.DigitalOceanDriver = digitalocean.NewDriver(d.MachineName, d.StorePath)
	}
	if d.PacketDriver == nil {
		d.PacketDriver = packet.NewDriver(d.MachineName, d.StorePath)
	}

	if d.flavor.Provider == "amazonec2" {
		d.Driver = d.AmazonEC2Driver
	} else if d.flavor.Provider == "digitalocean" {
		d.Driver = d.DigitalOceanDriver
	} else if d.flavor.Provider == "packet" {
		d.Driver = d.PacketDriver
	}

	return nil
}

func getenv(name, def string) string {
	v := os.Getenv(name)
	if v == "" {
		return def
	}
	return v
}

func (d *Driver) readProviderAndFlavorInfo(selectedFlavor string) error {
	flavorsDir := getenv("FLAVORS_DIR", flavorsDirDefault)
	log.Debugf("Reading flavor configs from directory %s", flavorsDir)
	providersDir := getenv("PROVIDERS_DIR", providersDirDefault)
	log.Debugf("Reading provider configs from directory %s", providersDir)

	files, err := ioutil.ReadDir(flavorsDir)
	if err != nil {
		return fmt.Errorf("Failed to read flavors directory %s: %v", flavorsDir, err)
	}

	flavorFound := false
	selectedFlavorFile := selectedFlavor + ".yaml"
	for _, file := range files {
		if file.Name() == selectedFlavorFile {
			bytes, err := ioutil.ReadFile(path.Join(flavorsDir, file.Name()))
			if err != nil {
				return err
			}
			if err = yaml.Unmarshal(bytes, &d.flavor); err != nil {
				return err
			}
			flavorFound = true
			break
		}
	}
	if !flavorFound {
		return fmt.Errorf("Invalid flavor %s, did not file %s", selectedFlavor, selectedFlavorFile)
	}

	files, err = ioutil.ReadDir(providersDir)
	if os.IsNotExist(err) {
		return nil
	} else if err != nil {
		return fmt.Errorf("Failed to read providers directory: %v", err)
	}

	providerFile := d.flavor.Provider + ".yaml"
	for _, file := range files {
		if file.Name() == providerFile {
			bytes, err := ioutil.ReadFile(path.Join(providersDir, file.Name()))
			if err != nil {
				return err
			}
			if err = yaml.Unmarshal(bytes, &d.ProviderDriverOptions); err != nil {
				return err
			}
			break
		}
	}

	return nil
}

// Strips off the rancher- prefix from flags
func stripPrefixFromFlags(flags rpcdriver.RPCFlags) rpcdriver.RPCFlags {
	for k, v := range flags.Values {
		s := fmt.Sprint(k)
		if strings.Contains(s, flagPrefix) {
			delete(flags.Values, k)
			flags.Values[strings.Replace(s, flagPrefix, "", -1)] = v
		}
	}
	return flags
}

// Merge the four sources of flag values, from lowest priority to highest
// Defaults from inner driver
// Values determined by the provider
// Values determined by flavor
// Values passed in via CLI (likely API keys)
func getDriverOpts(mcnflags []mcnflag.Flag, providerDriverOptions, flavorDriverOptions, cliDriverOptions map[string]interface{}) rpcdriver.RPCFlags {
	driverOpts := rpcdriver.RPCFlags{
		Values: make(map[string]interface{}),
	}
	for _, f := range mcnflags {
		driverOpts.Values[f.String()] = f.Default()
		if f.Default() == nil {
			driverOpts.Values[f.String()] = false
		}
	}
	for k, v := range providerDriverOptions {
		driverOpts.Values[k] = v
	}
	for k, v := range flavorDriverOptions {
		driverOpts.Values[k] = v
	}
	for k, v := range cliDriverOptions {
		driverOpts.Values[k] = v
	}

	// Strip off the rancher- prefix since inner drivers won't recognize it
	return stripPrefixFromFlags(driverOpts)
}

func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error {
	if err := d.readProviderAndFlavorInfo(flags.String("rancher-flavor")); err != nil {
		return err
	}

	if err := d.setupInnerDriver(); err != nil {
		return err
	}

	// TODO: try to avoid this type assertion
	cliDriverOptions := flags.(*rpcdriver.RPCFlags)
	driverOptions := getDriverOpts(d.Driver.GetCreateFlags(), d.ProviderDriverOptions, cliDriverOptions.Values, d.flavor.DriverOptions)

	if err := d.Driver.SetConfigFromFlags(driverOptions); err != nil {
		return err
	}

	if d.flavor.Provider == "amazonec2" {
		if err := d.setupAmazon(); err != nil {
			return err
		}
	}

	return nil
}

func (d *Driver) GetSSHHostname() (string, error) {
	return d.Driver.GetSSHHostname()
}

func (d *Driver) PreCreateCheck() error {
	return d.Driver.PreCreateCheck()
}

func (d *Driver) Create() error {
	return d.Driver.Create()
}

func (d *Driver) GetURL() (string, error) {
	return d.Driver.GetURL()
}

func (d *Driver) GetIP() (string, error) {
	return d.Driver.GetIP()
}

func (d *Driver) GetState() (state.State, error) {
	return d.Driver.GetState()
}

func (d *Driver) Start() error {
	return d.Driver.Start()
}

func (d *Driver) Stop() error {
	return d.Driver.Stop()
}

func (d *Driver) Remove() error {
	return d.Driver.Remove()
}

func (d *Driver) Restart() error {
	return d.Driver.Restart()
}

func (d *Driver) Kill() error {
	return d.Driver.Kill()
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package images

import (
	"errors"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/clock"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/flowcontrol"
	. "k8s.io/kubernetes/pkg/kubelet/container"
	ctest "k8s.io/kubernetes/pkg/kubelet/container/testing"
)

type pullerExpects struct {
	calls []string
	err   error
}

type pullerTestCase struct {
	containerImage string
	policy         v1.PullPolicy
	inspectErr     error
	pullerErr      error
	expected       []pullerExpects
}

func pullerTestCases() []pullerTestCase {
	return []pullerTestCase{
		{ // pull missing image
			containerImage: "missing_image",
			policy:         v1.PullIfNotPresent,
			inspectErr:     nil,
			pullerErr:      nil,
			expected: []pullerExpects{
				{[]string{"GetImageRef", "PullImage"}, nil},
			}},

		{ // image present, don't pull
			containerImage: "present_image",
			policy:         v1.PullIfNotPresent,
			inspectErr:     nil,
			pullerErr:      nil,
			expected: []pullerExpects{
				{[]string{"GetImageRef"}, nil},
				{[]string{"GetImageRef"}, nil},
				{[]string{"GetImageRef"}, nil},
			}},
		// image present, pull it
		{containerImage: "present_image",
			policy:     v1.PullAlways,
			inspectErr: nil,
			pullerErr:  nil,
			expected: []pullerExpects{
				{[]string{"GetImageRef", "PullImage"}, nil},
				{[]string{"GetImageRef", "PullImage"}, nil},
				{[]string{"GetImageRef", "PullImage"}, nil},
			}},
		// missing image, error PullNever
		{containerImage: "missing_image",
			policy:     v1.PullNever,
			inspectErr: nil,
			pullerErr:  nil,
			expected: []pullerExpects{
				{[]string{"GetImageRef"}, ErrImageNeverPull},
				{[]string{"GetImageRef"}, ErrImageNeverPull},
				{[]string{"GetImageRef"}, ErrImageNeverPull},
			}},
		// missing image, unable to inspect
		{containerImage: "missing_image",
			policy:     v1.PullIfNotPresent,
			inspectErr: errors.New("unknown inspectError"),
			pullerErr:  nil,
			expected: []pullerExpects{
				{[]string{"GetImageRef"}, ErrImageInspect},
				{[]string{"GetImageRef"}, ErrImageInspect},
				{[]string{"GetImageRef"}, ErrImageInspect},
			}},
		// missing image, unable to fetch
		{containerImage: "typo_image",
			policy:     v1.PullIfNotPresent,
			inspectErr: nil,
			pullerErr:  errors.New("404"),
			expected: []pullerExpects{
				{[]string{"GetImageRef", "PullImage"}, ErrImagePull},
				{[]string{"GetImageRef", "PullImage"}, ErrImagePull},
				{[]string{"GetImageRef"}, ErrImagePullBackOff},
				{[]string{"GetImageRef", "PullImage"}, ErrImagePull},
				{[]string{"GetImageRef"}, ErrImagePullBackOff},
				{[]string{"GetImageRef"}, ErrImagePullBackOff},
			}},
	}
}

func pullerTestEnv(c pullerTestCase, serialized bool) (puller ImageManager, fakeClock *clock.FakeClock, fakeRuntime *ctest.FakeRuntime, container *v1.Container) {
	container = &v1.Container{
		Name:            "container_name",
		Image:           c.containerImage,
		ImagePullPolicy: c.policy,
	}

	backOff := flowcontrol.NewBackOff(time.Second, time.Minute)
	fakeClock = clock.NewFakeClock(time.Now())
	backOff.Clock = fakeClock

	fakeRuntime = &ctest.FakeRuntime{}
	fakeRecorder := &record.FakeRecorder{}

	fakeRuntime.ImageList = []Image{{ID: "present_image:latest"}}
	fakeRuntime.Err = c.pullerErr
	fakeRuntime.InspectErr = c.inspectErr

	puller = NewImageManager(fakeRecorder, fakeRuntime, backOff, serialized, 0, 0)
	return
}

func TestParallelPuller(t *testing.T) {
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:            "test_pod",
			Namespace:       "test-ns",
			UID:             "bar",
			ResourceVersion: "42",
			SelfLink:        "/api/v1/pods/foo",
		}}

	cases := pullerTestCases()

	useSerializedEnv := false
	for i, c := range cases {
		puller, fakeClock, fakeRuntime, container := pullerTestEnv(c, useSerializedEnv)

		for tick, expected := range c.expected {
			fakeRuntime.CalledFunctions = nil
			fakeClock.Step(time.Second)
			_, _, err := puller.EnsureImageExists(pod, container, nil, nil)
			assert.NoError(t, fakeRuntime.AssertCalls(expected.calls), "in test %d tick=%d", i, tick)
			assert.Equal(t, expected.err, err, "in test %d tick=%d", i, tick)
		}
	}
}

func TestSerializedPuller(t *testing.T) {
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:            "test_pod",
			Namespace:       "test-ns",
			UID:             "bar",
			ResourceVersion: "42",
			SelfLink:        "/api/v1/pods/foo",
		}}

	cases := pullerTestCases()

	useSerializedEnv := true
	for i, c := range cases {
		puller, fakeClock, fakeRuntime, container := pullerTestEnv(c, useSerializedEnv)

		for tick, expected := range c.expected {
			fakeRuntime.CalledFunctions = nil
			fakeClock.Step(time.Second)
			_, _, err := puller.EnsureImageExists(pod, container, nil, nil)
			assert.NoError(t, fakeRuntime.AssertCalls(expected.calls), "in test %d tick=%d", i, tick)
			assert.Equal(t, expected.err, err, "in test %d tick=%d", i, tick)
		}
	}
}

func TestApplyDefaultImageTag(t *testing.T) {
	for _, testCase := range []struct {
		Input  string
		Output string
	}{
		{Input: "root", Output: "root:latest"},
		{Input: "root:tag", Output: "root:tag"},
		{Input: "root@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", Output: "root@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},
	} {
		image, err := applyDefaultImageTag(testCase.Input)
		if err != nil {
			t.Errorf("applyDefaultImageTag(%s) failed: %v", testCase.Input, err)
		} else if image != testCase.Output {
			t.Errorf("Expected image reference: %q, got %q", testCase.Output, image)
		}
	}
}

func TestPullAndListImageWithPodAnnotations(t *testing.T) {
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:            "test_pod",
			Namespace:       "test-ns",
			UID:             "bar",
			ResourceVersion: "42",
			SelfLink:        "/api/v1/pods/foo",
			Annotations: map[string]string{
				"kubernetes.io/runtimehandler": "handler_name",
			},
		}}
	c := pullerTestCase{ // pull missing image
		containerImage: "missing_image",
		policy:         v1.PullIfNotPresent,
		inspectErr:     nil,
		pullerErr:      nil,
		expected: []pullerExpects{
			{[]string{"GetImageRef", "PullImage"}, nil},
		}}

	useSerializedEnv := true
	puller, fakeClock, fakeRuntime, container := pullerTestEnv(c, useSerializedEnv)
	fakeRuntime.CalledFunctions = nil
	fakeRuntime.ImageList = []Image{}
	fakeClock.Step(time.Second)

	_, _, err := puller.EnsureImageExists(pod, container, nil, nil)
	assert.NoError(t, fakeRuntime.AssertCalls(c.expected[0].calls), "tick=%d", 0)
	assert.Equal(t, c.expected[0].err, err, "tick=%d", 0)

	images, _ := fakeRuntime.ListImages()
	assert.Equal(t, 1, len(images), "ListImages() count")

	image := images[0]
	assert.Equal(t, "missing_image:latest", image.ID, "Image ID")

	expectedAnnotations := []Annotation{
		{
			Name:  "kubernetes.io/runtimehandler",
			Value: "handler_name",
		}}
	assert.Equal(t, expectedAnnotations, image.Spec.Annotations, "image spec annotations")
}
<commit_msg>Add test coverage for throttledImageService<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package images

import (
	"errors"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/clock"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/flowcontrol"
	. "k8s.io/kubernetes/pkg/kubelet/container"
	ctest "k8s.io/kubernetes/pkg/kubelet/container/testing"
)

type pullerExpects struct {
	calls []string
	err   error
}

type pullerTestCase struct {
	testName       string
	containerImage string
	policy         v1.PullPolicy
	inspectErr     error
	pullerErr      error
	qps            float32
	burst          int
	expected       []pullerExpects
}

func pullerTestCases() []pullerTestCase {
	return []pullerTestCase{
		{ // pull missing image
			testName:       "image missing, pull",
			containerImage: "missing_image",
			policy:         v1.PullIfNotPresent,
			inspectErr:     nil,
			pullerErr:      nil,
			qps:            0.0,
			burst:          0,
			expected: []pullerExpects{
				{[]string{"GetImageRef", "PullImage"}, nil},
			}},

		{ // image present, don't pull
			testName:       "image present, don't pull ",
			containerImage: "present_image",
			policy:         v1.PullIfNotPresent,
			inspectErr:     nil,
			pullerErr:      nil,
			qps:            0.0,
			burst:          0,
			expected: []pullerExpects{
				{[]string{"GetImageRef"}, nil},
				{[]string{"GetImageRef"}, nil},
				{[]string{"GetImageRef"}, nil},
			}},
		// image present, pull it
		{containerImage: "present_image",
			testName:   "image present, pull ",
			policy:     v1.PullAlways,
			inspectErr: nil,
			pullerErr:  nil,
			qps:        0.0,
			burst:      0,
			expected: []pullerExpects{
				{[]string{"GetImageRef", "PullImage"}, nil},
				{[]string{"GetImageRef", "PullImage"}, nil},
				{[]string{"GetImageRef", "PullImage"}, nil},
			}},
		// missing image, error PullNever
		{containerImage: "missing_image",
			testName:   "image missing, never pull",
			policy:     v1.PullNever,
			inspectErr: nil,
			pullerErr:  nil,
			qps:        0.0,
			burst:      0,
			expected: []pullerExpects{
				{[]string{"GetImageRef"}, ErrImageNeverPull},
				{[]string{"GetImageRef"}, ErrImageNeverPull},
				{[]string{"GetImageRef"}, ErrImageNeverPull},
			}},
		// missing image, unable to inspect
		{containerImage: "missing_image",
			testName:   "image missing, pull if not present",
			policy:     v1.PullIfNotPresent,
			inspectErr: errors.New("unknown inspectError"),
			pullerErr:  nil,
			qps:        0.0,
			burst:      0,
			expected: []pullerExpects{
				{[]string{"GetImageRef"}, ErrImageInspect},
				{[]string{"GetImageRef"}, ErrImageInspect},
				{[]string{"GetImageRef"}, ErrImageInspect},
			}},
		// missing image, unable to fetch
		{containerImage: "typo_image",
			testName:   "image missing, unable to fetch",
			policy:     v1.PullIfNotPresent,
			inspectErr: nil,
			pullerErr:  errors.New("404"),
			qps:        0.0,
			burst:      0,
			expected: []pullerExpects{
				{[]string{"GetImageRef", "PullImage"}, ErrImagePull},
				{[]string{"GetImageRef", "PullImage"}, ErrImagePull},
				{[]string{"GetImageRef"}, ErrImagePullBackOff},
				{[]string{"GetImageRef", "PullImage"}, ErrImagePull},
				{[]string{"GetImageRef"}, ErrImagePullBackOff},
				{[]string{"GetImageRef"}, ErrImagePullBackOff},
			}},
		// image present, non-zero qps, try to pull
		{containerImage: "present_image",
			testName:   "image present and qps>0, pull",
			policy:     v1.PullAlways,
			inspectErr: nil,
			pullerErr:  nil,
			qps:        400.0,
			burst:      600,
			expected: []pullerExpects{
				{[]string{"GetImageRef", "PullImage"}, nil},
				{[]string{"GetImageRef", "PullImage"}, nil},
				{[]string{"GetImageRef", "PullImage"}, nil},
			}},
		// image present, non-zero qps, try to pull when qps exceeded
		{containerImage: "present_image",
			testName:   "image present and excessive qps rate, pull",
			policy:     v1.PullAlways,
			inspectErr: nil,
			pullerErr:  nil,
			qps:        2000.0,
			burst:      0,
			expected: []pullerExpects{
				{[]string{"GetImageRef"}, ErrImagePull},
				{[]string{"GetImageRef"}, ErrImagePull},
				{[]string{"GetImageRef"}, ErrImagePullBackOff},
			}},
	}
}

func pullerTestEnv(c pullerTestCase, serialized bool) (puller ImageManager, fakeClock *clock.FakeClock, fakeRuntime *ctest.FakeRuntime, container *v1.Container) {
	container = &v1.Container{
		Name:            "container_name",
		Image:           c.containerImage,
		ImagePullPolicy: c.policy,
	}

	backOff := flowcontrol.NewBackOff(time.Second, time.Minute)
	fakeClock = clock.NewFakeClock(time.Now())
	backOff.Clock = fakeClock

	fakeRuntime = &ctest.FakeRuntime{}
	fakeRecorder := &record.FakeRecorder{}

	fakeRuntime.ImageList = []Image{{ID: "present_image:latest"}}
	fakeRuntime.Err = c.pullerErr
	fakeRuntime.InspectErr = c.inspectErr

	puller = NewImageManager(fakeRecorder, fakeRuntime, backOff, serialized, c.qps, c.burst)
	return
}

func TestParallelPuller(t *testing.T) {
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:            "test_pod",
			Namespace:       "test-ns",
			UID:             "bar",
			ResourceVersion: "42",
			SelfLink:        "/api/v1/pods/foo",
		}}

	cases := pullerTestCases()

	useSerializedEnv := false
	for _, c := range cases {
		puller, fakeClock, fakeRuntime, container := pullerTestEnv(c, useSerializedEnv)

		t.Run(c.testName, func(t *testing.T) {
			for _, expected := range c.expected {
				fakeRuntime.CalledFunctions = nil
				fakeClock.Step(time.Second)
				_, _, err := puller.EnsureImageExists(pod, container, nil, nil)
				assert.NoError(t, fakeRuntime.AssertCalls(expected.calls))
				assert.Equal(t, expected.err, err)
			}
		})
	}
}

func TestSerializedPuller(t *testing.T) {
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:            "test_pod",
			Namespace:       "test-ns",
			UID:             "bar",
			ResourceVersion: "42",
			SelfLink:        "/api/v1/pods/foo",
		}}

	cases := pullerTestCases()

	useSerializedEnv := true
	for _, c := range cases {
		puller, fakeClock, fakeRuntime, container := pullerTestEnv(c, useSerializedEnv)

		t.Run(c.testName, func(t *testing.T) {
			for _, expected := range c.expected {
				fakeRuntime.CalledFunctions = nil
				fakeClock.Step(time.Second)
				_, _, err := puller.EnsureImageExists(pod, container, nil, nil)
				assert.NoError(t, fakeRuntime.AssertCalls(expected.calls))
				assert.Equal(t, expected.err, err)
			}
		})
	}
}

func TestApplyDefaultImageTag(t *testing.T) {
	for _, testCase := range []struct {
		testName string
		Input    string
		Output   string
	}{
		{testName: "root", Input: "root", Output: "root:latest"},
		{testName: "root:tag", Input: "root:tag", Output: "root:tag"},
		{testName: "root@sha", Input: "root@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", Output: "root@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},
	} {
		t.Run(testCase.testName, func(t *testing.T) {
			image, err := applyDefaultImageTag(testCase.Input)
			if err != nil {
				t.Errorf("applyDefaultImageTag(%s) failed: %v", testCase.Input, err)
			} else if image != testCase.Output {
				t.Errorf("Expected image reference: %q, got %q", testCase.Output, image)
			}
		})
	}
}

func TestPullAndListImageWithPodAnnotations(t *testing.T) {
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:            "test_pod",
			Namespace:       "test-ns",
			UID:             "bar",
			ResourceVersion: "42",
			SelfLink:        "/api/v1/pods/foo",
			Annotations: map[string]string{
				"kubernetes.io/runtimehandler": "handler_name",
			},
		}}
	c := pullerTestCase{ // pull missing image
		testName:       "test pull and list image with pod annotations",
		containerImage: "missing_image",
		policy:         v1.PullIfNotPresent,
		inspectErr:     nil,
		pullerErr:      nil,
		expected: []pullerExpects{
			{[]string{"GetImageRef", "PullImage"}, nil},
		}}

	useSerializedEnv := true
	puller, fakeClock, fakeRuntime, container := pullerTestEnv(c, useSerializedEnv)
	fakeRuntime.CalledFunctions = nil
	fakeRuntime.ImageList = []Image{}
	fakeClock.Step(time.Second)

	t.Run(c.testName, func(t *testing.T) {
		_, _, err := puller.EnsureImageExists(pod, container, nil, nil)
		assert.NoError(t, fakeRuntime.AssertCalls(c.expected[0].calls), "tick=%d", 0)
		assert.Equal(t, c.expected[0].err, err, "tick=%d", 0)

		images, _ := fakeRuntime.ListImages()
		assert.Equal(t, 1, len(images), "ListImages() count")

		image := images[0]
		assert.Equal(t, "missing_image:latest", image.ID, "Image ID")

		expectedAnnotations := []Annotation{
			{
				Name:  "kubernetes.io/runtimehandler",
				Value: "handler_name",
			}}
		assert.Equal(t, expectedAnnotations, image.Spec.Annotations, "image spec annotations")
	})
}
<|endoftext|>


<commit_before>// Copyright (c) 2019 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// docs_test is a tool to run sanity tests on the documentation before publishing.
package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/m3db/m3/src/x/docs"

	"github.com/pborman/getopt"
	"gopkg.in/russross/blackfriday.v2"
)

func main() {
	// Default link excludes.
	linkExcludes := []string{
		// Youtube has some problematic public rate limits.
		"youtu.be",
		"youtube.com",
	}

	var (
		docsDirArg   = getopt.StringLong("docsdir", 'd', "docs", "The documents directory to test")
		linkSleepArg = getopt.DurationLong("linksleep", 's', time.Second, "Amount to sleep between checking links to avoid 429 rate limits from github")
	)
	getopt.ListVarLong(&linkExcludes, "linkexcludes", 'e', "Exclude strings to check links against due to rate limiting/flakiness")
	getopt.Parse()
	if len(*docsDirArg) == 0 {
		getopt.Usage()
		return
	}

	docsDir := *docsDirArg
	linkSleep := *linkSleepArg

	var (
		excluded           []string
		repoPathsValidated int
	)
	resultErr := filepath.Walk(docsDir, func(filePath string, info os.FileInfo, err error) error {
		if err != nil {
			return fmt.Errorf("failed to walk path=%s: %v", filePath, err)
		}
		if !strings.HasSuffix(filePath, ".md") {
			return nil
		}

		data, err := ioutil.ReadFile(filePath)
		if err != nil {
			return fmt.Errorf("failed to read path=%s: %v", filePath, err)
		}

		var validateErr error
		abort := func(err error) blackfriday.WalkStatus {
			validateErr = err
			return blackfriday.Terminate
		}

		md := blackfriday.New()
		md.Parse(data).Walk(func(node *blackfriday.Node, entering bool) blackfriday.WalkStatus {
			switch node.Type {
			case blackfriday.Link:
				link := node.LinkData
				linkURL := strings.TrimSpace(string(link.Destination))

				resolvedPath := ""
				if parse, ok := docs.ParseRepoPathURL(linkURL); ok {
					// Internal relative local repo path
					resolvedPath = parse.RepoPath
				} else if isHTTPOrHTTPS(linkURL) {
					// External link
					resolvedPath = linkURL
				} else {
					// Otherwise should be a direct relative link
					// (not repo URL prefixed)
					if v := strings.Index(linkURL, "#"); v != -1 {
						// Remove links to subsections of docs
						linkURL = linkURL[:v]
					}
					if len(linkURL) > 0 && linkURL[0] == '/' {
						// This is an absolute link to within the docs directory
						resolvedPath = path.Join(docsDir, linkURL[1:])
					} else {
						// We assume this is a relative path from the current doc
						resolvedPath = path.Join(path.Dir(filePath), linkURL)
					}
				}

				checked := checkedLink{
					document:     filePath,
					url:          linkURL,
					resolvedPath: resolvedPath,
				}
				if isHTTPOrHTTPS(resolvedPath) {
					excludeCheck := false
					for _, str := range linkExcludes {
						if strings.Contains(resolvedPath, str) {
							excludeCheck = true
							break
						}
					}
					if excludeCheck {
						// Exclude this link and walk to next.
						excluded = append(excluded, resolvedPath)
						return blackfriday.GoToNext
					}

					// Check external link using HEAD request
					client := &http.Client{
						Transport: &http.Transport{
							// Some sites have invalid certs, like some kubernetes docs
							TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
						},
					}
					resp, err := client.Head(resolvedPath)
					if err == nil {
						class := resp.StatusCode / 100
						// Ignore 5xx errors since they are indicative of a problem with the external server
						// and not our documentation.
						if class != 2 && class != 5 {
							err = fmt.Errorf("unexpected status code: status=%v", resp.StatusCode)
						}
					}
					if err != nil {
						return abort(checkedLinkError(checked, err))
					}
					// Sleep to avoid rate limits.
					time.Sleep(linkSleep)
				} else {
					// Check relative path
					if _, err := os.Stat(resolvedPath); err != nil {
						return abort(checkedLinkError(checked, err))
					}
				}

				// Valid
				repoPathsValidated++
			}
			return blackfriday.GoToNext
		})

		return validateErr
	})
	if resultErr != nil {
		log.Fatalf("failed validation: %v", resultErr)
	}

	log.Printf("validated docs (repoPathsValidated=%d, excluded=%d)\n",
		repoPathsValidated, len(excluded))

	for _, linkURL := range excluded {
		log.Printf("excluded: %s\n", linkURL)
	}
}

func isHTTPOrHTTPS(url string) bool {
	return strings.HasPrefix(strings.ToLower(url), "http://") ||
		strings.HasPrefix(strings.ToLower(url), "https://")
}

type checkedLink struct {
	document     string
	url          string
	resolvedPath string
}

func checkedLinkError(l checkedLink, err error) error {
	return fmt.Errorf(
		"could not validate URL link: document=%s, link=%s, resolved_path=%s, err=%v",
		l.document, l.url, l.resolvedPath, err)
}
<commit_msg>[CI] ignores 429s on documentation tests. (#2746)<commit_after>// Copyright (c) 2019 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// docs_test is a tool to run sanity tests on the documentation before publishing.
package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/m3db/m3/src/x/docs"

	"github.com/pborman/getopt"
	"gopkg.in/russross/blackfriday.v2"
)

func main() {
	// Default link excludes.
	linkExcludes := []string{
		// Youtube has some problematic public rate limits.
		"youtu.be",
		"youtube.com",
		// cassandra has been flakey recently.
		"cassandra.apache.org",
	}

	var (
		docsDirArg   = getopt.StringLong("docsdir", 'd', "docs", "The documents directory to test")
		linkSleepArg = getopt.DurationLong("linksleep", 's', time.Second, "Amount to sleep between checking links to avoid 429 rate limits from github")
	)
	getopt.ListVarLong(&linkExcludes, "linkexcludes", 'e', "Exclude strings to check links against due to rate limiting/flakiness")
	getopt.Parse()
	if len(*docsDirArg) == 0 {
		getopt.Usage()
		return
	}

	docsDir := *docsDirArg
	linkSleep := *linkSleepArg

	var (
		excluded           []string
		repoPathsValidated int
	)
	resultErr := filepath.Walk(docsDir, func(filePath string, info os.FileInfo, err error) error {
		if err != nil {
			return fmt.Errorf("failed to walk path=%s: %v", filePath, err)
		}
		if !strings.HasSuffix(filePath, ".md") {
			return nil
		}

		data, err := ioutil.ReadFile(filePath)
		if err != nil {
			return fmt.Errorf("failed to read path=%s: %v", filePath, err)
		}

		var validateErr error
		abort := func(err error) blackfriday.WalkStatus {
			validateErr = err
			return blackfriday.Terminate
		}

		md := blackfriday.New()
		md.Parse(data).Walk(func(node *blackfriday.Node, entering bool) blackfriday.WalkStatus {
			switch node.Type {
			case blackfriday.Link:
				link := node.LinkData
				linkURL := strings.TrimSpace(string(link.Destination))

				resolvedPath := ""
				if parse, ok := docs.ParseRepoPathURL(linkURL); ok {
					// Internal relative local repo path
					resolvedPath = parse.RepoPath
				} else if isHTTPOrHTTPS(linkURL) {
					// External link
					resolvedPath = linkURL
				} else {
					// Otherwise should be a direct relative link
					// (not repo URL prefixed)
					if v := strings.Index(linkURL, "#"); v != -1 {
						// Remove links to subsections of docs
						linkURL = linkURL[:v]
					}
					if len(linkURL) > 0 && linkURL[0] == '/' {
						// This is an absolute link to within the docs directory
						resolvedPath = path.Join(docsDir, linkURL[1:])
					} else {
						// We assume this is a relative path from the current doc
						resolvedPath = path.Join(path.Dir(filePath), linkURL)
					}
				}

				checked := checkedLink{
					document:     filePath,
					url:          linkURL,
					resolvedPath: resolvedPath,
				}
				if isHTTPOrHTTPS(resolvedPath) {
					excludeCheck := false
					for _, str := range linkExcludes {
						if strings.Contains(resolvedPath, str) {
							excludeCheck = true
							break
						}
					}
					if excludeCheck {
						// Exclude this link and walk to next.
						excluded = append(excluded, resolvedPath)
						return blackfriday.GoToNext
					}

					// Check external link using HEAD request
					client := &http.Client{
						Transport: &http.Transport{
							// Some sites have invalid certs, like some kubernetes docs
							TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
						},
					}
					resp, err := client.Head(resolvedPath)
					if err == nil {
						class := resp.StatusCode / 100
						// Ignore 5xx errors since they are indicative of a problem with the external server
						// and not our documentation.
						if class != 2 && class != 5 && resp.StatusCode != http.StatusTooManyRequests {
							err = fmt.Errorf("unexpected status code: status=%v", resp.StatusCode)
						}
					}
					if err != nil {
						return abort(checkedLinkError(checked, err))
					}
					// Sleep to avoid rate limits.
					time.Sleep(linkSleep)
				} else {
					// Check relative path
					if _, err := os.Stat(resolvedPath); err != nil {
						return abort(checkedLinkError(checked, err))
					}
				}

				// Valid
				repoPathsValidated++
			}
			return blackfriday.GoToNext
		})

		return validateErr
	})
	if resultErr != nil {
		log.Fatalf("failed validation: %v", resultErr)
	}

	log.Printf("validated docs (repoPathsValidated=%d, excluded=%d)\n",
		repoPathsValidated, len(excluded))

	for _, linkURL := range excluded {
		log.Printf("excluded: %s\n", linkURL)
	}
}

func isHTTPOrHTTPS(url string) bool {
	return strings.HasPrefix(strings.ToLower(url), "http://") ||
		strings.HasPrefix(strings.ToLower(url), "https://")
}

type checkedLink struct {
	document     string
	url          string
	resolvedPath string
}

func checkedLinkError(l checkedLink, err error) error {
	return fmt.Errorf(
		"could not validate URL link: document=%s, link=%s, resolved_path=%s, err=%v",
		l.document, l.url, l.resolvedPath, err)
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datastore

import (
	"bytes"
	"encoding/base64"
	"encoding/gob"
	"errors"
	"strconv"
	"strings"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	pb "google.golang.org/cloud/internal/datastore"
)

// Key represents the datastore key for a stored entity, and is immutable.
type Key struct {
	kind   string
	id     int64
	name   string
	parent *Key

	namespace string
}

func (k *Key) Kind() string {
	return k.kind
}

func (k *Key) ID() int64 {
	return k.id
}

func (k *Key) Name() string {
	return k.name
}

func (k *Key) Parent() *Key {
	return k.parent
}

func (k *Key) SetParent(v *Key) {
	if v.Incomplete() {
		panic("can't set an incomplete key as parent")
	}
	k.parent = v
}

func (k *Key) Namespace() string {
	return k.namespace
}

// Complete returns whether the key does not refer to a stored entity.
func (k *Key) Incomplete() bool {
	return k.name == "" && k.id == 0
}

// valid returns whether the key is valid.
func (k *Key) valid() bool {
	if k == nil {
		return false
	}
	for ; k != nil; k = k.parent {
		if k.kind == "" {
			return false
		}
		if k.name != "" && k.id != 0 {
			return false
		}
		if k.parent != nil {
			if k.parent.Incomplete() {
				return false
			}
			if k.parent.namespace != k.namespace {
				return false
			}
		}
	}
	return true
}

func (k *Key) Equal(o *Key) bool {
	for {
		if k == nil || o == nil {
			return k == o // if either is nil, both must be nil
		}
		if k.namespace != o.namespace || k.name != o.name || k.id != o.id || k.kind != o.kind {
			return false
		}
		if k.parent == nil && o.parent == nil {
			return true
		}
		k = k.parent
		o = o.parent
	}
}

// marshal marshals the key's string representation to the buffer.
func (k *Key) marshal(b *bytes.Buffer) {
	if k.parent != nil {
		k.parent.marshal(b)
	}
	b.WriteByte('/')
	b.WriteString(k.kind)
	b.WriteByte(',')
	if k.name != "" {
		b.WriteString(k.name)
	} else {
		b.WriteString(strconv.FormatInt(k.id, 10))
	}
}

// String returns a string representation of the key.
func (k *Key) String() string {
	if k == nil {
		return ""
	}
	b := bytes.NewBuffer(make([]byte, 0, 512))
	k.marshal(b)
	return b.String()
}

// Note: Fields not renamed compared to appengine gobKey struct
// This ensures gobs created by appengine can be read here, and vice/versa
type gobKey struct {
	Kind      string
	StringID  string
	IntID     int64
	Parent    *gobKey
	AppID     string
	Namespace string
}

func keyToGobKey(k *Key) *gobKey {
	if k == nil {
		return nil
	}
	return &gobKey{
		Kind:      k.kind,
		StringID:  k.name,
		IntID:     k.id,
		Parent:    keyToGobKey(k.parent),
		Namespace: k.namespace,
	}
}

func gobKeyToKey(gk *gobKey) *Key {
	if gk == nil {
		return nil
	}
	return &Key{
		kind:      gk.Kind,
		name:      gk.StringID,
		id:        gk.IntID,
		parent:    gobKeyToKey(gk.Parent),
		namespace: gk.Namespace,
	}
}

func (k *Key) GobEncode() ([]byte, error) {
	buf := new(bytes.Buffer)
	if err := gob.NewEncoder(buf).Encode(keyToGobKey(k)); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (k *Key) GobDecode(buf []byte) error {
	gk := new(gobKey)
	if err := gob.NewDecoder(bytes.NewBuffer(buf)).Decode(gk); err != nil {
		return err
	}
	*k = *gobKeyToKey(gk)
	return nil
}

func (k *Key) MarshalJSON() ([]byte, error) {
	return []byte(`"` + k.Encode() + `"`), nil
}

func (k *Key) UnmarshalJSON(buf []byte) error {
	if len(buf) < 2 || buf[0] != '"' || buf[len(buf)-1] != '"' {
		return errors.New("datastore: bad JSON key")
	}
	k2, err := DecodeKey(string(buf[1 : len(buf)-1]))
	if err != nil {
		return err
	}
	*k = *k2
	return nil
}

// Encode returns an opaque representation of the key
// suitable for use in HTML and URLs.
// This is compatible with the Python and Java runtimes.
func (k *Key) Encode() string {
	pKey := keyToProto(k)

	b, err := proto.Marshal(pKey)
	if err != nil {
		panic(err)
	}

	// Trailing padding is stripped.
	return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=")
}

// DecodeKey decodes a key from the opaque representation returned by Encode.
func DecodeKey(encoded string) (*Key, error) {
	// Re-add padding.
	if m := len(encoded) % 4; m != 0 {
		encoded += strings.Repeat("=", 4-m)
	}

	b, err := base64.URLEncoding.DecodeString(encoded)
	if err != nil {
		return nil, err
	}

	pKey := new(pb.Key)
	if err := proto.Unmarshal(b, pKey); err != nil {
		return nil, err
	}

	return protoToKey(pKey), nil
}

// NewIncompleteKey creates a new incomplete key.
// kind cannot be empty.
func NewIncompleteKey(ctx context.Context, kind string, parent *Key) *Key {
	return NewKey(ctx, kind, "", 0, parent)
}

// NewKey creates a new key.
// kind cannot be empty.
// Either one or both of stringID and intID must be zero. If both are zero,
// the key returned is incomplete.
// parent must either be a complete key or nil.
func NewKey(ctx context.Context, kind, name string, id int64, parent *Key) *Key {
	return &Key{
		kind:      kind,
		name:      name,
		id:        id,
		parent:    parent,
		namespace: ctxNamespace(ctx),
	}
}

// AllocateIDs accepts a slice of incomplete keys and returns a
// slice of complete keys that are guaranteed to be valid in the datastore
func (c *Client) AllocateIDs(ctx context.Context, keys []*Key) ([]*Key, error) {
	if keys == nil {
		return nil, nil
	}

	req := &pb.AllocateIdsRequest{Key: multiKeyToProto(keys)}
	res := &pb.AllocateIdsResponse{}
	if err := c.call(ctx, "allocateIds", req, res); err != nil {
		return nil, err
	}

	return multiProtoToKey(res.Key), nil
}
<commit_msg>datastore: Update param names used in NewKey doc.<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datastore

import (
	"bytes"
	"encoding/base64"
	"encoding/gob"
	"errors"
	"strconv"
	"strings"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	pb "google.golang.org/cloud/internal/datastore"
)

// Key represents the datastore key for a stored entity, and is immutable.
type Key struct {
	kind   string
	id     int64
	name   string
	parent *Key

	namespace string
}

func (k *Key) Kind() string {
	return k.kind
}

func (k *Key) ID() int64 {
	return k.id
}

func (k *Key) Name() string {
	return k.name
}

func (k *Key) Parent() *Key {
	return k.parent
}

func (k *Key) SetParent(v *Key) {
	if v.Incomplete() {
		panic("can't set an incomplete key as parent")
	}
	k.parent = v
}

func (k *Key) Namespace() string {
	return k.namespace
}

// Complete returns whether the key does not refer to a stored entity.
func (k *Key) Incomplete() bool {
	return k.name == "" && k.id == 0
}

// valid returns whether the key is valid.
func (k *Key) valid() bool {
	if k == nil {
		return false
	}
	for ; k != nil; k = k.parent {
		if k.kind == "" {
			return false
		}
		if k.name != "" && k.id != 0 {
			return false
		}
		if k.parent != nil {
			if k.parent.Incomplete() {
				return false
			}
			if k.parent.namespace != k.namespace {
				return false
			}
		}
	}
	return true
}

func (k *Key) Equal(o *Key) bool {
	for {
		if k == nil || o == nil {
			return k == o // if either is nil, both must be nil
		}
		if k.namespace != o.namespace || k.name != o.name || k.id != o.id || k.kind != o.kind {
			return false
		}
		if k.parent == nil && o.parent == nil {
			return true
		}
		k = k.parent
		o = o.parent
	}
}

// marshal marshals the key's string representation to the buffer.
func (k *Key) marshal(b *bytes.Buffer) {
	if k.parent != nil {
		k.parent.marshal(b)
	}
	b.WriteByte('/')
	b.WriteString(k.kind)
	b.WriteByte(',')
	if k.name != "" {
		b.WriteString(k.name)
	} else {
		b.WriteString(strconv.FormatInt(k.id, 10))
	}
}

// String returns a string representation of the key.
func (k *Key) String() string {
	if k == nil {
		return ""
	}
	b := bytes.NewBuffer(make([]byte, 0, 512))
	k.marshal(b)
	return b.String()
}

// Note: Fields not renamed compared to appengine gobKey struct
// This ensures gobs created by appengine can be read here, and vice/versa
type gobKey struct {
	Kind      string
	StringID  string
	IntID     int64
	Parent    *gobKey
	AppID     string
	Namespace string
}

func keyToGobKey(k *Key) *gobKey {
	if k == nil {
		return nil
	}
	return &gobKey{
		Kind:      k.kind,
		StringID:  k.name,
		IntID:     k.id,
		Parent:    keyToGobKey(k.parent),
		Namespace: k.namespace,
	}
}

func gobKeyToKey(gk *gobKey) *Key {
	if gk == nil {
		return nil
	}
	return &Key{
		kind:      gk.Kind,
		name:      gk.StringID,
		id:        gk.IntID,
		parent:    gobKeyToKey(gk.Parent),
		namespace: gk.Namespace,
	}
}

func (k *Key) GobEncode() ([]byte, error) {
	buf := new(bytes.Buffer)
	if err := gob.NewEncoder(buf).Encode(keyToGobKey(k)); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (k *Key) GobDecode(buf []byte) error {
	gk := new(gobKey)
	if err := gob.NewDecoder(bytes.NewBuffer(buf)).Decode(gk); err != nil {
		return err
	}
	*k = *gobKeyToKey(gk)
	return nil
}

func (k *Key) MarshalJSON() ([]byte, error) {
	return []byte(`"` + k.Encode() + `"`), nil
}

func (k *Key) UnmarshalJSON(buf []byte) error {
	if len(buf) < 2 || buf[0] != '"' || buf[len(buf)-1] != '"' {
		return errors.New("datastore: bad JSON key")
	}
	k2, err := DecodeKey(string(buf[1 : len(buf)-1]))
	if err != nil {
		return err
	}
	*k = *k2
	return nil
}

// Encode returns an opaque representation of the key
// suitable for use in HTML and URLs.
// This is compatible with the Python and Java runtimes.
func (k *Key) Encode() string {
	pKey := keyToProto(k)

	b, err := proto.Marshal(pKey)
	if err != nil {
		panic(err)
	}

	// Trailing padding is stripped.
	return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=")
}

// DecodeKey decodes a key from the opaque representation returned by Encode.
func DecodeKey(encoded string) (*Key, error) {
	// Re-add padding.
	if m := len(encoded) % 4; m != 0 {
		encoded += strings.Repeat("=", 4-m)
	}

	b, err := base64.URLEncoding.DecodeString(encoded)
	if err != nil {
		return nil, err
	}

	pKey := new(pb.Key)
	if err := proto.Unmarshal(b, pKey); err != nil {
		return nil, err
	}

	return protoToKey(pKey), nil
}

// NewIncompleteKey creates a new incomplete key.
// kind cannot be empty.
func NewIncompleteKey(ctx context.Context, kind string, parent *Key) *Key {
	return NewKey(ctx, kind, "", 0, parent)
}

// NewKey creates a new key.
// kind cannot be empty.
// Either one or both of name and id must be zero. If both are zero,
// the key returned is incomplete.
// parent must either be a complete key or nil.
func NewKey(ctx context.Context, kind, name string, id int64, parent *Key) *Key {
	return &Key{
		kind:      kind,
		name:      name,
		id:        id,
		parent:    parent,
		namespace: ctxNamespace(ctx),
	}
}

// AllocateIDs accepts a slice of incomplete keys and returns a
// slice of complete keys that are guaranteed to be valid in the datastore
func (c *Client) AllocateIDs(ctx context.Context, keys []*Key) ([]*Key, error) {
	if keys == nil {
		return nil, nil
	}

	req := &pb.AllocateIdsRequest{Key: multiKeyToProto(keys)}
	res := &pb.AllocateIdsResponse{}
	if err := c.call(ctx, "allocateIds", req, res); err != nil {
		return nil, err
	}

	return multiProtoToKey(res.Key), nil
}
<|endoftext|>


<commit_before>package mongo

import (
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type Recorder struct {
	session    *mgo.Session
	set        string
	collection string
}

func NewRecorder(host string, set string, collection string) (*Recorder, error) {
	sess, err := mgo.Dial(host)
	if err != nil {
		return nil, err
	}

	return &Recorder{
		session:    sess,
		set:        set,
		collection: collection,
	}
}

func (r *Recorder) Insert(args interface{}) error {
	dbHandler := r.session.DB(r.set).C(r.collection)
	err := dbHandler.Insert(args)
	if err != nil {
		return err
	}

	return nil
}

func (r *Recorder) FindLatest(deviceid uint64, records interface{}) error {
	dbHandler := r.session.DB(r.set).C(r.collection)
	err := dbHandler.Find(bson.M{
		"$query":   bson.M{"deviceid": deviceid},
		"$orderby": bson.M{"timestamp": -1},
	}).Limit(1).One(records)

	return nil
}

func (r *Recorder) FindByTimestamp(deviceid uint64, start uint64, end uint64, records interface{}) error {
	dbHandler := r.session.DB(r.set).C(r.collection)
	err := dbHandler.Find(bson.M{
		"$query":    bson.M{"deviceid": deviceid},
		"timestamp": bson.M{"$gte": start, "$lte": end},
	}).All(records)

	return nil
}
<commit_msg>fix redis build failed. build #33<commit_after>package mongo

import (
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type Recorder struct {
	session    *mgo.Session
	set        string
	collection string
}

func NewRecorder(host string, set string, collection string) (*Recorder, error) {
	sess, err := mgo.Dial(host)
	if err != nil {
		return nil, err
	}

	return &Recorder{
		session:    sess,
		set:        set,
		collection: collection,
	}, nil
}

func (r *Recorder) Insert(args interface{}) error {
	dbHandler := r.session.DB(r.set).C(r.collection)
	err := dbHandler.Insert(args)
	if err != nil {
		return err
	}

	return nil
}

func (r *Recorder) FindLatest(deviceid uint64, records interface{}) error {
	dbHandler := r.session.DB(r.set).C(r.collection)
	err := dbHandler.Find(bson.M{
		"$query":   bson.M{"deviceid": deviceid},
		"$orderby": bson.M{"timestamp": -1},
	}).Limit(1).One(records)

	return err
}

func (r *Recorder) FindByTimestamp(deviceid uint64, start uint64, end uint64, records interface{}) error {
	dbHandler := r.session.DB(r.set).C(r.collection)
	err := dbHandler.Find(bson.M{
		"$query":    bson.M{"deviceid": deviceid},
		"timestamp": bson.M{"$gte": start, "$lte": end},
	}).All(records)

	return err
}
<|endoftext|>