<commit_before>/*
 * Minio Cloud Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package fs

import (
	"bytes"
	"io"
	"os"
	"path/filepath"
	"strings"

	"crypto/md5"
	"encoding/base64"
	"encoding/hex"
	"runtime"

	"github.com/minio/minio/pkg/atomic"
	"github.com/minio/minio/pkg/crypto/sha256"
	"github.com/minio/minio/pkg/disk"
	"github.com/minio/minio/pkg/mimedb"
	"github.com/minio/minio/pkg/probe"
	"github.com/minio/minio/pkg/s3/signature4"
)

/// Object Operations

// GetObject - GET object
func (fs Filesystem) GetObject(w io.Writer, bucket, object string, start, length int64) (int64, *probe.Error) {
	// Critical region requiring read lock.
	fs.rwLock.RLock()
	defer fs.rwLock.RUnlock()

	// Input validation.
	if !IsValidBucketName(bucket) {
		return 0, probe.NewError(BucketNameInvalid{Bucket: bucket})
	}
	if !IsValidObjectName(object) {
		return 0, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
	}

	// normalize buckets.
	bucket = fs.denormalizeBucket(bucket)
	bucketPath := filepath.Join(fs.path, bucket)
	if _, e := os.Stat(bucketPath); e != nil {
		if os.IsNotExist(e) {
			return 0, probe.NewError(BucketNotFound{Bucket: bucket})
		}
		return 0, probe.NewError(e)
	}

	objectPath := filepath.Join(bucketPath, object)
	filestat, err := os.Stat(objectPath)
	switch err := err.(type) {
	case nil:
		if filestat.IsDir() {
			return 0, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
		}
	default:
		if os.IsNotExist(err) {
			return 0, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
		}
		return 0, probe.NewError(err)
	}
	file, err := os.Open(objectPath)
	if err != nil {
		return 0, probe.NewError(err)
	}
	defer file.Close()

	_, err = file.Seek(start, os.SEEK_SET)
	if err != nil {
		return 0, probe.NewError(err)
	}

	var count int64
	if length > 0 {
		count, err = io.CopyN(w, file, length)
		if err != nil {
			return count, probe.NewError(err)
		}
	} else {
		count, err = io.Copy(w, file)
		if err != nil {
			return count, probe.NewError(err)
		}
	}
	return count, nil
}

// GetObjectInfo - get object info.
func (fs Filesystem) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error) {
	// Input validation.
	if !IsValidBucketName(bucket) {
		return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
	}

	if !IsValidObjectName(object) {
		return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: bucket})
	}

	// Normalize buckets.
	bucket = fs.denormalizeBucket(bucket)
	bucketPath := filepath.Join(fs.path, bucket)
	if _, e := os.Stat(bucketPath); e != nil {
		if os.IsNotExist(e) {
			return ObjectInfo{}, probe.NewError(BucketNotFound{Bucket: bucket})
		}
		return ObjectInfo{}, probe.NewError(e)
	}

	info, err := getObjectInfo(fs.path, bucket, object)
	if err != nil {
		return ObjectInfo{}, err.Trace(bucket, object)
	}
	return info, nil
}

// getObjectInfo - get object stat info.
func getObjectInfo(rootPath, bucket, object string) (ObjectInfo, *probe.Error) {
	// Do not use filepath.Join() since filepath.Join strips off any
	// object names with '/', use them as is in a static manner so
	// that we can send a proper 'ObjectNotFound' reply back upon
	// os.Stat().
	var objectPath string
	// For windows use its special os.PathSeparator == "\\"
	if runtime.GOOS == "windows" {
		objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
	} else {
		objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
	}
	stat, err := os.Stat(objectPath)
	if err != nil {
		if os.IsNotExist(err) {
			return ObjectInfo{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
		}
		return ObjectInfo{}, probe.NewError(err)
	}
	contentType := "application/octet-stream"
	if runtime.GOOS == "windows" {
		object = filepath.ToSlash(object)
	}

	if objectExt := filepath.Ext(object); objectExt != "" {
		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
		if ok {
			contentType = content.ContentType
		}
	}
	metadata := ObjectInfo{
		Bucket:       bucket,
		Name:         object,
		ModifiedTime: stat.ModTime(),
		Size:         stat.Size(),
		ContentType:  contentType,
		IsDir:        stat.Mode().IsDir(),
	}
	return metadata, nil
}

// isMD5SumEqual - returns error if md5sum mismatches, success its `nil`
func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) bool {
	// Verify the md5sum.
	if expectedMD5Sum != "" && actualMD5Sum != "" {
		// Decode md5sum to bytes from their hexadecimal
		// representations.
		expectedMD5SumBytes, err := hex.DecodeString(expectedMD5Sum)
		if err != nil {
			return false
		}
		actualMD5SumBytes, err := hex.DecodeString(actualMD5Sum)
		if err != nil {
			return false
		}
		// Verify md5sum bytes are equal after successful decoding.
		if !bytes.Equal(expectedMD5SumBytes, actualMD5SumBytes) {
			return false
		}
		return true
	}
	return false
}

// CreateObject - create an object.
func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size int64, data io.Reader, sig *signature4.Sign) (ObjectInfo, *probe.Error) {
	di, e := disk.GetInfo(fs.path)
	if e != nil {
		return ObjectInfo{}, probe.NewError(e)
	}

	// Remove 5% from total space for cumulative disk space used for
	// journalling, inodes etc.
	availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
	if int64(availableDiskSpace) <= fs.minFreeDisk {
		return ObjectInfo{}, probe.NewError(RootPathFull{Path: fs.path})
	}

	// Check bucket name valid.
	if !IsValidBucketName(bucket) {
		return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
	}

	bucket = fs.denormalizeBucket(bucket)
	bucketPath := filepath.Join(fs.path, bucket)
	if _, e = os.Stat(bucketPath); e != nil {
		if os.IsNotExist(e) {
			return ObjectInfo{}, probe.NewError(BucketNotFound{Bucket: bucket})
		}
		return ObjectInfo{}, probe.NewError(e)
	}

	// Verify object path legal.
	if !IsValidObjectName(object) {
		return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
	}

	// Get object path.
	objectPath := filepath.Join(bucketPath, object)
	if expectedMD5Sum != "" {
		var expectedMD5SumBytes []byte
		expectedMD5SumBytes, e = base64.StdEncoding.DecodeString(expectedMD5Sum)
		if e != nil {
			// Pro-actively close the connection.
			return ObjectInfo{}, probe.NewError(InvalidDigest{MD5: expectedMD5Sum})
		}
		expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
	}

	// Write object.
	file, e := atomic.FileCreateWithPrefix(objectPath, expectedMD5Sum+"$tmpobject")
	if e != nil {
		switch e := e.(type) {
		case *os.PathError:
			if e.Op == "mkdir" {
				if strings.Contains(e.Error(), "not a directory") {
					return ObjectInfo{}, probe.NewError(ObjectExistsAsPrefix{Bucket: bucket, Prefix: object})
				}
			}
			return ObjectInfo{}, probe.NewError(e)
		default:
			return ObjectInfo{}, probe.NewError(e)
		}
	}

	// Instantiate checksum hashers and create a multiwriter.
	md5Hasher := md5.New()
	sha256Hasher := sha256.New()
	objectWriter := io.MultiWriter(file, md5Hasher, sha256Hasher)

	if size > 0 {
		if _, e = io.CopyN(objectWriter, data, size); e != nil {
			file.CloseAndPurge()
			return ObjectInfo{}, probe.NewError(e)
		}
	} else {
		if _, e = io.Copy(objectWriter, data); e != nil {
			file.CloseAndPurge()
			return ObjectInfo{}, probe.NewError(e)
		}
	}

	md5Sum := hex.EncodeToString(md5Hasher.Sum(nil))
	// Verify if the written object is equal to what is expected, only
	// if it is requested as such.
	if expectedMD5Sum != "" {
		if !isMD5SumEqual(expectedMD5Sum, md5Sum) {
			file.CloseAndPurge()
			return ObjectInfo{}, probe.NewError(BadDigest{MD5: expectedMD5Sum, Bucket: bucket, Object: object})
		}
	}
	sha256Sum := hex.EncodeToString(sha256Hasher.Sum(nil))
	if sig != nil {
		ok, err := sig.DoesSignatureMatch(sha256Sum)
		if err != nil {
			file.CloseAndPurge()
			return ObjectInfo{}, err.Trace()
		}
		if !ok {
			file.CloseAndPurge()
			return ObjectInfo{}, probe.NewError(SignDoesNotMatch{})
		}
	}
	file.Close()

	// Set stat again to get the latest metadata.
	st, e := os.Stat(objectPath)
	if e != nil {
		return ObjectInfo{}, probe.NewError(e)
	}
	contentType := "application/octet-stream"
	if objectExt := filepath.Ext(objectPath); objectExt != "" {
		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
		if ok {
			contentType = content.ContentType
		}
	}
	newObject := ObjectInfo{
		Bucket:       bucket,
		Name:         object,
		ModifiedTime: st.ModTime(),
		Size:         st.Size(),
		ContentType:  contentType,
		MD5Sum:       md5Sum,
	}
	return newObject, nil
}

// deleteObjectPath - delete object path if its empty.
func deleteObjectPath(basePath, deletePath, bucket, object string) *probe.Error {
	if basePath == deletePath {
		return nil
	}
	// Verify if the path exists.
	pathSt, e := os.Stat(deletePath)
	if e != nil {
		if os.IsNotExist(e) {
			return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
		}
		return probe.NewError(e)
	}
	if pathSt.IsDir() {
		// Verify if directory is empty.
		empty, e := isDirEmpty(deletePath)
		if e != nil {
			return probe.NewError(e)
		}
		if !empty {
			return nil
		}
	}
	// Attempt to remove path.
	if e := os.Remove(deletePath); e != nil {
		return probe.NewError(e)
	}
	// Recursively go down the next path and delete again.
	if err := deleteObjectPath(basePath, filepath.Dir(deletePath), bucket, object); err != nil {
		return err.Trace(basePath, deletePath, bucket, object)
	}
	return nil
}

// DeleteObject - delete object.
func (fs Filesystem) DeleteObject(bucket, object string) *probe.Error {
	// Check bucket name valid
	if !IsValidBucketName(bucket) {
		return probe.NewError(BucketNameInvalid{Bucket: bucket})
	}

	bucket = fs.denormalizeBucket(bucket)
	bucketPath := filepath.Join(fs.path, bucket)
	// Check bucket exists
	if _, e := os.Stat(bucketPath); e != nil {
		if os.IsNotExist(e) {
			return probe.NewError(BucketNotFound{Bucket: bucket})
		}
		return probe.NewError(e)
	}

	// Verify object path legal
	if !IsValidObjectName(object) {
		return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
	}

	// Do not use filepath.Join() since filepath.Join strips off any
	// object names with '/', use them as is in a static manner so
	// that we can send a proper 'ObjectNotFound' reply back upon
	// os.Stat().
	var objectPath string
	if runtime.GOOS == "windows" {
		objectPath = fs.path + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
	} else {
		objectPath = fs.path + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
	}
	// Delete object path if its empty.
	err := deleteObjectPath(bucketPath, objectPath, bucket, object)
	if err != nil {
		if os.IsNotExist(err.ToGoError()) {
			return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
		}
		return err.Trace(bucketPath, objectPath, bucket, object)
	}
	return nil
}
<commit_msg>pkg/fs: optimize GetObject syscalls for common case<commit_after>/*
 * Minio Cloud Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package fs

import (
	"bytes"
	"io"
	"os"
	"path/filepath"
	"strings"

	"crypto/md5"
	"encoding/base64"
	"encoding/hex"
	"runtime"

	"github.com/minio/minio/pkg/atomic"
	"github.com/minio/minio/pkg/crypto/sha256"
	"github.com/minio/minio/pkg/disk"
	"github.com/minio/minio/pkg/mimedb"
	"github.com/minio/minio/pkg/probe"
	"github.com/minio/minio/pkg/s3/signature4"
)

/// Object Operations

// GetObject - GET object
func (fs Filesystem) GetObject(w io.Writer, bucket, object string, start, length int64) (int64, *probe.Error) {
	// Critical region requiring read lock.
	fs.rwLock.RLock()
	defer fs.rwLock.RUnlock()

	// Input validation.
	if !IsValidBucketName(bucket) {
		return 0, probe.NewError(BucketNameInvalid{Bucket: bucket})
	}
	if !IsValidObjectName(object) {
		return 0, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
	}

	// normalize buckets.
	bucket = fs.denormalizeBucket(bucket)
	objectPath := filepath.Join(fs.path, bucket, object)

	file, e := os.Open(objectPath)
	if e != nil {
		// If the object doesn't exist, the bucket might not exist either. Stat for
		// the bucket and give a better error message if that is true.
		if os.IsNotExist(e) {
			_, e = os.Stat(filepath.Join(fs.path, bucket))
			if os.IsNotExist(e) {
				return 0, probe.NewError(BucketNotFound{Bucket: bucket})
			}

			return 0, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
		}
		return 0, probe.NewError(e)
	}
	defer file.Close()

	_, e = file.Seek(start, os.SEEK_SET)
	if e != nil {
		// When the "handle is invalid", the file might be a directory on Windows.
		if runtime.GOOS == "windows" && strings.Contains(e.Error(), "handle is invalid") {
			return 0, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
		}

		return 0, probe.NewError(e)
	}

	var count int64
	// Copy over the whole file if the length is non-positive.
	if length > 0 {
		count, e = io.CopyN(w, file, length)
	} else {
		count, e = io.Copy(w, file)
	}

	if e != nil {
		// This call will fail if the object is a directory. Stat the file to see if
		// this is true, if so, return an ObjectNotFound error.
		stat, e := os.Stat(objectPath)
		if e == nil && stat.IsDir() {
			return count, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
		}

		return count, probe.NewError(e)
	}

	return count, nil
}

// GetObjectInfo - get object info.
func (fs Filesystem) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error) {
	// Input validation.
	if !IsValidBucketName(bucket) {
		return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
	}

	if !IsValidObjectName(object) {
		return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: bucket})
	}

	// Normalize buckets.
	bucket = fs.denormalizeBucket(bucket)
	bucketPath := filepath.Join(fs.path, bucket)
	if _, e := os.Stat(bucketPath); e != nil {
		if os.IsNotExist(e) {
			return ObjectInfo{}, probe.NewError(BucketNotFound{Bucket: bucket})
		}
		return ObjectInfo{}, probe.NewError(e)
	}

	info, err := getObjectInfo(fs.path, bucket, object)
	if err != nil {
		return ObjectInfo{}, err.Trace(bucket, object)
	}
	return info, nil
}

// getObjectInfo - get object stat info.
func getObjectInfo(rootPath, bucket, object string) (ObjectInfo, *probe.Error) {
	// Do not use filepath.Join() since filepath.Join strips off any
	// object names with '/', use them as is in a static manner so
	// that we can send a proper 'ObjectNotFound' reply back upon
	// os.Stat().
	var objectPath string
	// For windows use its special os.PathSeparator == "\\"
	if runtime.GOOS == "windows" {
		objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
	} else {
		objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
	}
	stat, err := os.Stat(objectPath)
	if err != nil {
		if os.IsNotExist(err) {
			return ObjectInfo{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
		}
		return ObjectInfo{}, probe.NewError(err)
	}
	contentType := "application/octet-stream"
	if runtime.GOOS == "windows" {
		object = filepath.ToSlash(object)
	}

	if objectExt := filepath.Ext(object); objectExt != "" {
		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
		if ok {
			contentType = content.ContentType
		}
	}
	metadata := ObjectInfo{
		Bucket:       bucket,
		Name:         object,
		ModifiedTime: stat.ModTime(),
		Size:         stat.Size(),
		ContentType:  contentType,
		IsDir:        stat.Mode().IsDir(),
	}
	return metadata, nil
}

// isMD5SumEqual - returns error if md5sum mismatches, success its `nil`
func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) bool {
	// Verify the md5sum.
	if expectedMD5Sum != "" && actualMD5Sum != "" {
		// Decode md5sum to bytes from their hexadecimal
		// representations.
		expectedMD5SumBytes, err := hex.DecodeString(expectedMD5Sum)
		if err != nil {
			return false
		}
		actualMD5SumBytes, err := hex.DecodeString(actualMD5Sum)
		if err != nil {
			return false
		}
		// Verify md5sum bytes are equal after successful decoding.
		if !bytes.Equal(expectedMD5SumBytes, actualMD5SumBytes) {
			return false
		}
		return true
	}
	return false
}

// CreateObject - create an object.
func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size int64, data io.Reader, sig *signature4.Sign) (ObjectInfo, *probe.Error) {
	di, e := disk.GetInfo(fs.path)
	if e != nil {
		return ObjectInfo{}, probe.NewError(e)
	}

	// Remove 5% from total space for cumulative disk space used for
	// journalling, inodes etc.
	availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
	if int64(availableDiskSpace) <= fs.minFreeDisk {
		return ObjectInfo{}, probe.NewError(RootPathFull{Path: fs.path})
	}

	// Check bucket name valid.
	if !IsValidBucketName(bucket) {
		return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
	}

	bucket = fs.denormalizeBucket(bucket)
	bucketPath := filepath.Join(fs.path, bucket)
	if _, e = os.Stat(bucketPath); e != nil {
		if os.IsNotExist(e) {
			return ObjectInfo{}, probe.NewError(BucketNotFound{Bucket: bucket})
		}
		return ObjectInfo{}, probe.NewError(e)
	}

	// Verify object path legal.
	if !IsValidObjectName(object) {
		return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
	}

	// Get object path.
	objectPath := filepath.Join(bucketPath, object)
	if expectedMD5Sum != "" {
		var expectedMD5SumBytes []byte
		expectedMD5SumBytes, e = base64.StdEncoding.DecodeString(expectedMD5Sum)
		if e != nil {
			// Pro-actively close the connection.
			return ObjectInfo{}, probe.NewError(InvalidDigest{MD5: expectedMD5Sum})
		}
		expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
	}

	// Write object.
	file, e := atomic.FileCreateWithPrefix(objectPath, expectedMD5Sum+"$tmpobject")
	if e != nil {
		switch e := e.(type) {
		case *os.PathError:
			if e.Op == "mkdir" {
				if strings.Contains(e.Error(), "not a directory") {
					return ObjectInfo{}, probe.NewError(ObjectExistsAsPrefix{Bucket: bucket, Prefix: object})
				}
			}
			return ObjectInfo{}, probe.NewError(e)
		default:
			return ObjectInfo{}, probe.NewError(e)
		}
	}

	// Instantiate checksum hashers and create a multiwriter.
	md5Hasher := md5.New()
	sha256Hasher := sha256.New()
	objectWriter := io.MultiWriter(file, md5Hasher, sha256Hasher)

	if size > 0 {
		if _, e = io.CopyN(objectWriter, data, size); e != nil {
			file.CloseAndPurge()
			return ObjectInfo{}, probe.NewError(e)
		}
	} else {
		if _, e = io.Copy(objectWriter, data); e != nil {
			file.CloseAndPurge()
			return ObjectInfo{}, probe.NewError(e)
		}
	}

	md5Sum := hex.EncodeToString(md5Hasher.Sum(nil))
	// Verify if the written object is equal to what is expected, only
	// if it is requested as such.
	if expectedMD5Sum != "" {
		if !isMD5SumEqual(expectedMD5Sum, md5Sum) {
			file.CloseAndPurge()
			return ObjectInfo{}, probe.NewError(BadDigest{MD5: expectedMD5Sum, Bucket: bucket, Object: object})
		}
	}
	sha256Sum := hex.EncodeToString(sha256Hasher.Sum(nil))
	if sig != nil {
		ok, err := sig.DoesSignatureMatch(sha256Sum)
		if err != nil {
			file.CloseAndPurge()
			return ObjectInfo{}, err.Trace()
		}
		if !ok {
			file.CloseAndPurge()
			return ObjectInfo{}, probe.NewError(SignDoesNotMatch{})
		}
	}
	file.Close()

	// Set stat again to get the latest metadata.
	st, e := os.Stat(objectPath)
	if e != nil {
		return ObjectInfo{}, probe.NewError(e)
	}
	contentType := "application/octet-stream"
	if objectExt := filepath.Ext(objectPath); objectExt != "" {
		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
		if ok {
			contentType = content.ContentType
		}
	}
	newObject := ObjectInfo{
		Bucket:       bucket,
		Name:         object,
		ModifiedTime: st.ModTime(),
		Size:         st.Size(),
		ContentType:  contentType,
		MD5Sum:       md5Sum,
	}
	return newObject, nil
}

// deleteObjectPath - delete object path if its empty.
func deleteObjectPath(basePath, deletePath, bucket, object string) *probe.Error {
	if basePath == deletePath {
		return nil
	}
	// Verify if the path exists.
	pathSt, e := os.Stat(deletePath)
	if e != nil {
		if os.IsNotExist(e) {
			return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
		}
		return probe.NewError(e)
	}
	if pathSt.IsDir() {
		// Verify if directory is empty.
		empty, e := isDirEmpty(deletePath)
		if e != nil {
			return probe.NewError(e)
		}
		if !empty {
			return nil
		}
	}
	// Attempt to remove path.
	if e := os.Remove(deletePath); e != nil {
		return probe.NewError(e)
	}
	// Recursively go down the next path and delete again.
	if err := deleteObjectPath(basePath, filepath.Dir(deletePath), bucket, object); err != nil {
		return err.Trace(basePath, deletePath, bucket, object)
	}
	return nil
}

// DeleteObject - delete object.
func (fs Filesystem) DeleteObject(bucket, object string) *probe.Error {
	// Check bucket name valid
	if !IsValidBucketName(bucket) {
		return probe.NewError(BucketNameInvalid{Bucket: bucket})
	}

	bucket = fs.denormalizeBucket(bucket)
	bucketPath := filepath.Join(fs.path, bucket)
	// Check bucket exists
	if _, e := os.Stat(bucketPath); e != nil {
		if os.IsNotExist(e) {
			return probe.NewError(BucketNotFound{Bucket: bucket})
		}
		return probe.NewError(e)
	}

	// Verify object path legal
	if !IsValidObjectName(object) {
		return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
	}

	// Do not use filepath.Join() since filepath.Join strips off any
	// object names with '/', use them as is in a static manner so
	// that we can send a proper 'ObjectNotFound' reply back upon
	// os.Stat().
	var objectPath string
	if runtime.GOOS == "windows" {
		objectPath = fs.path + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
	} else {
		objectPath = fs.path + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
	}
	// Delete object path if its empty.
	err := deleteObjectPath(bucketPath, objectPath, bucket, object)
	if err != nil {
		if os.IsNotExist(err.ToGoError()) {
			return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
		}
		return err.Trace(bucketPath, objectPath, bucket, object)
	}
	return nil
}
<|endoftext|>


<commit_before>package kafka

import (
	"crypto/rand"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
)

// Config is the configuration of kafka pkg.
type Config struct {
	async bool

	Sarama *sarama.Config
}

// DefaultConfig creates a default Config as async producer.
func DefaultConfig() *Config {
	cf := sarama.NewConfig()
	clientID, err := generateClientID()
	if err != nil {
		panic(err)
	}
	cf.ClientID = clientID

	// network
	cf.Net.DialTimeout = time.Second * 30
	cf.Net.ReadTimeout = time.Second * 30
	cf.Net.WriteTimeout = time.Second * 30
	cf.Net.MaxOpenRequests = 5

	// common for sync and async
	cf.ChannelBufferSize = 256 * 4        // default was 256
	cf.Producer.MaxMessageBytes = 1000000 // 1MB
	cf.Producer.Timeout = time.Second * 30
	cf.Producer.Compression = sarama.CompressionNone
	cf.Producer.Retry.Max = 5
	cf.Producer.Retry.Backoff = time.Millisecond * 350
	cf.Producer.RequiredAcks = sarama.WaitForLocal // default ack
	cf.Metadata.RefreshFrequency = time.Minute * 10
	cf.Metadata.Retry.Max = 5
	cf.Metadata.Retry.Backoff = time.Second

	// async specific
	cf.Producer.Return.Errors = true
	cf.Producer.Return.Successes = true
	cf.Producer.Flush.Frequency = time.Second / 2
	cf.Producer.Flush.Messages = 1024 // TODO
	cf.Producer.Flush.MaxMessages = 0 // unlimited
	//cf.Producer.Flush.Bytes = 64 << 10

	return &Config{
		Sarama: cf,
		async:  true,
	}
}

// Ack sets the kafka producer required ack parameter.
func (c *Config) Ack(ack sarama.RequiredAcks) *Config {
	c.Sarama.Producer.RequiredAcks = ack
	return c
}

// SyncMode will switch the kafka producer to sync mode.
func (c *Config) SyncMode() *Config {
	c.async = false

	// explicitly zero batch
	c.Sarama.Producer.Flush.Frequency = 0
	c.Sarama.Producer.Flush.Bytes = 0
	c.Sarama.Producer.Flush.Messages = 0
	return c
}

// SyncMode will switch the kafka producer to async mode.
func (c *Config) AsyncMode() *Config {
	c.async = true
	return c
}

func generateClientID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(rand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}

	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40

	host, err := os.Hostname()
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%s:%s", host, fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:])), nil
}

func init() {
	ctx.LoadFromHome()
	log.SetFlags(log.LstdFlags | log.Lshortfile) // for sarama
}
<commit_msg>ClientID fmt was not accepted by sarama<commit_after>package kafka

import (
	"fmt"
	"log"
	"os"
	"sync/atomic"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
)

var clientID uint32

// Config is the configuration of kafka pkg.
type Config struct {
	async bool

	Sarama *sarama.Config
}

// DefaultConfig creates a default Config as async producer.
func DefaultConfig() *Config {
	cf := sarama.NewConfig()
	clientID, err := generateClientID()
	if err != nil {
		panic(err)
	}
	cf.ClientID = clientID

	// network
	cf.Net.DialTimeout = time.Second * 30
	cf.Net.ReadTimeout = time.Second * 30
	cf.Net.WriteTimeout = time.Second * 30
	cf.Net.MaxOpenRequests = 5

	// common for sync and async
	cf.ChannelBufferSize = 256 * 4        // default was 256
	cf.Producer.MaxMessageBytes = 1000000 // 1MB
	cf.Producer.Timeout = time.Second * 30
	cf.Producer.Compression = sarama.CompressionNone
	cf.Producer.Retry.Max = 5
	cf.Producer.Retry.Backoff = time.Millisecond * 350
	cf.Producer.RequiredAcks = sarama.WaitForLocal // default ack
	cf.Metadata.RefreshFrequency = time.Minute * 10
	cf.Metadata.Retry.Max = 5
	cf.Metadata.Retry.Backoff = time.Second

	// async specific
	cf.Producer.Return.Errors = true
	cf.Producer.Return.Successes = true
	cf.Producer.Flush.Frequency = time.Second / 2
	cf.Producer.Flush.Messages = 1024 // TODO
	cf.Producer.Flush.MaxMessages = 0 // unlimited
	//cf.Producer.Flush.Bytes = 64 << 10

	return &Config{
		Sarama: cf,
		async:  true,
	}
}

// Ack sets the kafka producer required ack parameter.
func (c *Config) Ack(ack sarama.RequiredAcks) *Config {
	c.Sarama.Producer.RequiredAcks = ack
	return c
}

// SyncMode will switch the kafka producer to sync mode.
func (c *Config) SyncMode() *Config {
	c.async = false

	// explicitly zero batch
	c.Sarama.Producer.Flush.Frequency = 0
	c.Sarama.Producer.Flush.Bytes = 0
	c.Sarama.Producer.Flush.Messages = 0
	return c
}

// SyncMode will switch the kafka producer to async mode.
func (c *Config) AsyncMode() *Config {
	c.async = true
	return c
}

func generateClientID() (string, error) {
	host, err := os.Hostname()
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%s-%s", host, fmt.Sprintf("%d", atomic.AddUint32(&clientID, 1))), nil
}

func init() {
	ctx.LoadFromHome()
	log.SetFlags(log.LstdFlags | log.Lshortfile) // for sarama
}
<|endoftext|>


<commit_before>package library

import (
	"fmt"
	"os"
	"path"
	"syscall"
	"time"

	"github.com/facette/facette/pkg/logger"
	"github.com/facette/facette/pkg/utils"
	"github.com/fatih/set"
	uuid "github.com/nu7hatch/gouuid"
)

// Item represents the base structure of a library item.
type Item struct {
	path        string
	ID          string    `json:"id"`
	Name        string    `json:"name"`
	Description string    `json:"description"`
	Modified    time.Time `json:"-"`
}

func (item *Item) String() string {
	return fmt.Sprintf(
		"Item{path:%q ID:%q Name:%q Description:%q Modified:%s}",
		item.path,
		item.ID,
		item.Name,
		item.Description,
		item.Modified,
	)
}

// GetItem returns the base structure of a library item.
func (item *Item) GetItem() *Item {
	return item
}

// DeleteItem removes an existing item from the library.
func (library *Library) DeleteItem(id string, itemType int) error {
	if !library.ItemExists(id, itemType) {
		return os.ErrNotExist
	}

	// Delete sub-collections
	if itemType == LibraryItemCollection {
		for _, child := range library.Collections[id].Children {
			library.DeleteItem(child.ID, LibraryItemCollection)
		}
	}

	// Remove stored JSON
	if err := syscall.Unlink(library.getFilePath(id, itemType)); err != nil {
		return err
	}

	// Delete item from library
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		delete(library.Groups, id)

	case LibraryItemScale:
		delete(library.Scales, id)

	case LibraryItemUnit:
		delete(library.Units, id)

	case LibraryItemGraph:
		delete(library.Graphs, id)

	case LibraryItemCollection:
		delete(library.Collections, id)
	}

	return nil
}

// GetItem gets an item from the library by its identifier.
func (library *Library) GetItem(id string, itemType int) (interface{}, error) {
	if !library.ItemExists(id, itemType) {
		return nil, os.ErrNotExist
	}

	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		return library.Groups[id], nil

	case LibraryItemScale:
		return library.Scales[id], nil

	case LibraryItemUnit:
		return library.Units[id], nil

	case LibraryItemGraph:
		return library.Graphs[id], nil

	case LibraryItemCollection:
		return library.Collections[id], nil
	}

	return nil, fmt.Errorf("no item found")
}

// GetItemByName gets an item from the library by its name.
func (library *Library) GetItemByName(name string, itemType int) (interface{}, error) {
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		for _, item := range library.Groups {
			if item.Type != itemType || item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemScale:
		for _, item := range library.Scales {
			if item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemUnit:
		for _, item := range library.Units {
			if item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemGraph:
		for _, item := range library.Graphs {
			if item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemCollection:
		for _, item := range library.Collections {
			if item.Name != name {
				continue
			}

			return item, nil
		}
	}

	return nil, os.ErrNotExist
}

// ItemExists returns whether an item exists the library or not.
func (library *Library) ItemExists(id string, itemType int) bool {
	exists := false

	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		if _, ok := library.Groups[id]; ok && library.Groups[id].Type == itemType {
			exists = true
		}

	case LibraryItemScale:
		_, exists = library.Scales[id]

	case LibraryItemUnit:
		_, exists = library.Units[id]

	case LibraryItemGraph:
		_, exists = library.Graphs[id]

	case LibraryItemCollection:
		_, exists = library.Collections[id]
	}

	return exists
}

// LoadItem loads an item by its identifier.
func (library *Library) LoadItem(id string, itemType int) error {
	// Load item from file
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		tmpGroup := &Group{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpGroup)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Groups[id] = tmpGroup
		library.Groups[id].Type = itemType
		library.Groups[id].Modified = fileInfo.ModTime()

	case LibraryItemScale:
		tmpScale := &Scale{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpScale)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Scales[id] = tmpScale
		library.Scales[id].Modified = fileInfo.ModTime()

	case LibraryItemUnit:
		tmpUnit := &Unit{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpUnit)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Units[id] = tmpUnit
		library.Units[id].Modified = fileInfo.ModTime()

	case LibraryItemGraph:
		tmpGraph := &Graph{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpGraph)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Graphs[id] = tmpGraph
		library.Graphs[id].Modified = fileInfo.ModTime()

	case LibraryItemCollection:
		var tmpCollection *struct {
			*Collection
			Parent string `json:"parent"`
		}

		filePath := library.getFilePath(id, LibraryItemCollection)

		fileInfo, err := utils.JSONLoad(filePath, &tmpCollection)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		if !library.ItemExists(id, LibraryItemCollection) {
			library.Collections[id] = &Collection{}
		}

		*library.Collections[id] = *tmpCollection.Collection

		if tmpCollection.Parent != "" {
			library.Collections[id].ParentID = tmpCollection.Parent
		}

		library.Collections[id].Modified = fileInfo.ModTime()
	}

	return nil
}

// StoreItem stores an item into the library.
func (library *Library) StoreItem(item interface{}, itemType int, foreignIdAllowd bool) error {
	var itemStruct *Item

	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		itemStruct = item.(*Group).GetItem()

	case LibraryItemScale:
		itemStruct = item.(*Scale).GetItem()

	case LibraryItemUnit:
		itemStruct = item.(*Unit).GetItem()

	case LibraryItemGraph:
		itemStruct = item.(*Graph).GetItem()

	case LibraryItemCollection:
		itemStruct = item.(*Collection).GetItem()

	default:
		return os.ErrInvalid
	}

	// If item has no ID specified, generate one
	if itemStruct.ID == "" {
		uuidTemp, err := uuid.NewV4()
		if err != nil {
			return err
		}

		itemStruct.ID = uuidTemp.String()
	} else if !library.ItemExists(itemStruct.ID, itemType) && !foreignIdAllowd {
		return os.ErrNotExist
	}

	// Check for name field presence/duplicates
	if itemStruct.Name == "" {
		logger.Log(logger.LevelError, "library", "item missing `name' field")
		return os.ErrInvalid
	}

	itemTemp, err := library.GetItemByName(itemStruct.Name, itemType)

	// Item exists, check for duplicates
	if err == nil {
		switch itemType {
		case LibraryItemSourceGroup, LibraryItemMetricGroup:
			if itemTemp.(*Group).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate group identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemScale:
			if itemTemp.(*Scale).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate scale identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemUnit:
			if itemTemp.(*Unit).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate unit identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemGraph:
			if itemTemp.(*Graph).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate graph identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemCollection:
			if itemTemp.(*Collection).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate collection identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}
		}
	}

	// Item does not exist, store it into library
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		library.Groups[itemStruct.ID] = item.(*Group)
		library.Groups[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemScale:
		library.Scales[itemStruct.ID] = item.(*Scale)
		library.Scales[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemUnit:
		library.Units[itemStruct.ID] = item.(*Unit)
		library.Units[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemGraph:
		// Check for definition names duplicates
		groupSet := set.New(set.ThreadSafe)
		seriesSet := set.New(set.ThreadSafe)

		for _, group := range item.(*Graph).Groups {
			if group == nil {
				logger.Log(logger.LevelError, "library", "found null group")
				return os.ErrInvalid
			} else if groupSet.Has(group.Name) {
				logger.Log(logger.LevelError, "library", "duplicate group name `%s'", group.Name)
				return os.ErrExist
			}

			groupSet.Add(group.Name)

			for _, series := range group.Series {
				if series == nil {
					logger.Log(logger.LevelError, "library", "found null series in group `%s'", group.Name)
					return os.ErrInvalid
				} else if seriesSet.Has(series.Name) {
					logger.Log(logger.LevelError, "library", "duplicate series name `%s'", series.Name)
					return os.ErrExist
				}

				seriesSet.Add(series.Name)
			}
		}

		library.Graphs[itemStruct.ID] = item.(*Graph)
		library.Graphs[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemCollection:
		library.Collections[itemStruct.ID] = item.(*Collection)
		library.Collections[itemStruct.ID].ID = itemStruct.ID
	}

	itemStruct.Modified = time.Now()

	// Store JSON data
	if err := utils.JSONDump(library.getFilePath(itemStruct.ID, itemType), item, itemStruct.Modified); err != nil {
		return err
	}

	return nil
}

func (library *Library) getDirPath(itemType int) string {
	var dirName string

	switch itemType {
	case LibraryItemSourceGroup:
		dirName = "sourcegroups"

	case LibraryItemMetricGroup:
		dirName = "metricgroups"

	case LibraryItemScale:
		dirName = "scales"

	case LibraryItemUnit:
		dirName = "units"

	case LibraryItemGraph:
		dirName = "graphs"

	case LibraryItemCollection:
		dirName = "collections"
	}

	return path.Join(library.Config.DataDir, dirName)
}

func (library *Library) getFilePath(id string, itemType int) string {
	return path.Join(library.getDirPath(itemType), id[0:2], id[2:4], id+".json")
}
<commit_msg>allow own UUIDs for library creation ops<commit_after>package library

import (
	"fmt"
	"os"
	"path"
	"syscall"
	"time"

	"github.com/facette/facette/pkg/logger"
	"github.com/facette/facette/pkg/utils"
	"github.com/fatih/set"
	uuid "github.com/nu7hatch/gouuid"
)

// Item represents the base structure of a library item.
type Item struct {
	path        string
	ID          string    `json:"id"`
	Name        string    `json:"name"`
	Description string    `json:"description"`
	Modified    time.Time `json:"-"`
}

func (item *Item) String() string {
	return fmt.Sprintf(
		"Item{path:%q ID:%q Name:%q Description:%q Modified:%s}",
		item.path,
		item.ID,
		item.Name,
		item.Description,
		item.Modified,
	)
}

// GetItem returns the base structure of a library item.
func (item *Item) GetItem() *Item {
	return item
}

// DeleteItem removes an existing item from the library.
func (library *Library) DeleteItem(id string, itemType int) error {
	if !library.ItemExists(id, itemType) {
		return os.ErrNotExist
	}

	// Delete sub-collections
	if itemType == LibraryItemCollection {
		for _, child := range library.Collections[id].Children {
			library.DeleteItem(child.ID, LibraryItemCollection)
		}
	}

	// Remove stored JSON
	if err := syscall.Unlink(library.getFilePath(id, itemType)); err != nil {
		return err
	}

	// Delete item from library
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		delete(library.Groups, id)

	case LibraryItemScale:
		delete(library.Scales, id)

	case LibraryItemUnit:
		delete(library.Units, id)

	case LibraryItemGraph:
		delete(library.Graphs, id)

	case LibraryItemCollection:
		delete(library.Collections, id)
	}

	return nil
}

// GetItem gets an item from the library by its identifier.
func (library *Library) GetItem(id string, itemType int) (interface{}, error) {
	if !library.ItemExists(id, itemType) {
		return nil, os.ErrNotExist
	}

	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		return library.Groups[id], nil

	case LibraryItemScale:
		return library.Scales[id], nil

	case LibraryItemUnit:
		return library.Units[id], nil

	case LibraryItemGraph:
		return library.Graphs[id], nil

	case LibraryItemCollection:
		return library.Collections[id], nil
	}

	return nil, fmt.Errorf("no item found")
}

// GetItemByName gets an item from the library by its name.
func (library *Library) GetItemByName(name string, itemType int) (interface{}, error) {
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		for _, item := range library.Groups {
			if item.Type != itemType || item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemScale:
		for _, item := range library.Scales {
			if item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemUnit:
		for _, item := range library.Units {
			if item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemGraph:
		for _, item := range library.Graphs {
			if item.Name != name {
				continue
			}

			return item, nil
		}

	case LibraryItemCollection:
		for _, item := range library.Collections {
			if item.Name != name {
				continue
			}

			return item, nil
		}
	}

	return nil, os.ErrNotExist
}

// ItemExists returns whether an item exists the library or not.
func (library *Library) ItemExists(id string, itemType int) bool {
	exists := false

	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		if _, ok := library.Groups[id]; ok && library.Groups[id].Type == itemType {
			exists = true
		}

	case LibraryItemScale:
		_, exists = library.Scales[id]

	case LibraryItemUnit:
		_, exists = library.Units[id]

	case LibraryItemGraph:
		_, exists = library.Graphs[id]

	case LibraryItemCollection:
		_, exists = library.Collections[id]
	}

	return exists
}

// LoadItem loads an item by its identifier.
func (library *Library) LoadItem(id string, itemType int) error {
	// Load item from file
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		tmpGroup := &Group{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpGroup)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Groups[id] = tmpGroup
		library.Groups[id].Type = itemType
		library.Groups[id].Modified = fileInfo.ModTime()

	case LibraryItemScale:
		tmpScale := &Scale{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpScale)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Scales[id] = tmpScale
		library.Scales[id].Modified = fileInfo.ModTime()

	case LibraryItemUnit:
		tmpUnit := &Unit{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpUnit)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Units[id] = tmpUnit
		library.Units[id].Modified = fileInfo.ModTime()

	case LibraryItemGraph:
		tmpGraph := &Graph{}

		filePath := library.getFilePath(id, itemType)

		fileInfo, err := utils.JSONLoad(filePath, &tmpGraph)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		library.Graphs[id] = tmpGraph
		library.Graphs[id].Modified = fileInfo.ModTime()

	case LibraryItemCollection:
		var tmpCollection *struct {
			*Collection
			Parent string `json:"parent"`
		}

		filePath := library.getFilePath(id, LibraryItemCollection)

		fileInfo, err := utils.JSONLoad(filePath, &tmpCollection)
		if err != nil {
			return fmt.Errorf("in %s, %s", filePath, err)
		}

		if !library.ItemExists(id, LibraryItemCollection) {
			library.Collections[id] = &Collection{}
		}

		*library.Collections[id] = *tmpCollection.Collection

		if tmpCollection.Parent != "" {
			library.Collections[id].ParentID = tmpCollection.Parent
		}

		library.Collections[id].Modified = fileInfo.ModTime()
	}

	return nil
}

// StoreItem stores an item into the library.
func (library *Library) StoreItem(item interface{}, itemType int, foreignIdAllowd bool) error {
	var itemStruct *Item

	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		itemStruct = item.(*Group).GetItem()

	case LibraryItemScale:
		itemStruct = item.(*Scale).GetItem()

	case LibraryItemUnit:
		itemStruct = item.(*Unit).GetItem()

	case LibraryItemGraph:
		itemStruct = item.(*Graph).GetItem()

	case LibraryItemCollection:
		itemStruct = item.(*Collection).GetItem()

	default:
		return os.ErrInvalid
	}

	// If item has no ID specified, generate one
	if itemStruct.ID == "" {
		uuidTemp, err := uuid.NewV4()
		if err != nil {
			return err
		}

		itemStruct.ID = uuidTemp.String()
	}
        if library.ItemExists(itemStruct.ID, itemType) {
                return os.ErrExist
        } else if !foreignIdAllowd {
                return os.ErrNotExist
        }

	// Check for name field presence/duplicates
	if itemStruct.Name == "" {
		logger.Log(logger.LevelError, "library", "item missing `name' field")
		return os.ErrInvalid
	}

	itemTemp, err := library.GetItemByName(itemStruct.Name, itemType)

	// Item exists, check for duplicates
	if err == nil {
		switch itemType {
		case LibraryItemSourceGroup, LibraryItemMetricGroup:
			if itemTemp.(*Group).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate group identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemScale:
			if itemTemp.(*Scale).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate scale identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemUnit:
			if itemTemp.(*Unit).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate unit identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemGraph:
			if itemTemp.(*Graph).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate graph identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}

		case LibraryItemCollection:
			if itemTemp.(*Collection).ID != itemStruct.ID {
				logger.Log(logger.LevelError, "library", "duplicate collection identifier `%s'", itemStruct.ID)
				return os.ErrExist
			}
		}
	}

	// Item does not exist, store it into library
	switch itemType {
	case LibraryItemSourceGroup, LibraryItemMetricGroup:
		library.Groups[itemStruct.ID] = item.(*Group)
		library.Groups[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemScale:
		library.Scales[itemStruct.ID] = item.(*Scale)
		library.Scales[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemUnit:
		library.Units[itemStruct.ID] = item.(*Unit)
		library.Units[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemGraph:
		// Check for definition names duplicates
		groupSet := set.New(set.ThreadSafe)
		seriesSet := set.New(set.ThreadSafe)

		for _, group := range item.(*Graph).Groups {
			if group == nil {
				logger.Log(logger.LevelError, "library", "found null group")
				return os.ErrInvalid
			} else if groupSet.Has(group.Name) {
				logger.Log(logger.LevelError, "library", "duplicate group name `%s'", group.Name)
				return os.ErrExist
			}

			groupSet.Add(group.Name)

			for _, series := range group.Series {
				if series == nil {
					logger.Log(logger.LevelError, "library", "found null series in group `%s'", group.Name)
					return os.ErrInvalid
				} else if seriesSet.Has(series.Name) {
					logger.Log(logger.LevelError, "library", "duplicate series name `%s'", series.Name)
					return os.ErrExist
				}

				seriesSet.Add(series.Name)
			}
		}

		library.Graphs[itemStruct.ID] = item.(*Graph)
		library.Graphs[itemStruct.ID].ID = itemStruct.ID

	case LibraryItemCollection:
		library.Collections[itemStruct.ID] = item.(*Collection)
		library.Collections[itemStruct.ID].ID = itemStruct.ID
	}

	itemStruct.Modified = time.Now()

	// Store JSON data
	if err := utils.JSONDump(library.getFilePath(itemStruct.ID, itemType), item, itemStruct.Modified); err != nil {
		return err
	}

	return nil
}

func (library *Library) getDirPath(itemType int) string {
	var dirName string

	switch itemType {
	case LibraryItemSourceGroup:
		dirName = "sourcegroups"

	case LibraryItemMetricGroup:
		dirName = "metricgroups"

	case LibraryItemScale:
		dirName = "scales"

	case LibraryItemUnit:
		dirName = "units"

	case LibraryItemGraph:
		dirName = "graphs"

	case LibraryItemCollection:
		dirName = "collections"
	}

	return path.Join(library.Config.DataDir, dirName)
}

func (library *Library) getFilePath(id string, itemType int) string {
	return path.Join(library.getDirPath(itemType), id[0:2], id[2:4], id+".json")
}
<|endoftext|>


<commit_before>/*
 * Mini Object Storage, (C) 2014,2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package minio implements a generic Minio client
package minio

import (
	"bytes"
	"encoding/base64"
	"encoding/hex"
	"encoding/xml"
	"errors"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	MAX_OBJECT_LIST = 1000
)

// Client is an Minio MINIO client.
type Client struct {
	Hostname  string
	Transport http.RoundTripper // or nil for the default
}

type Bucket struct {
	Name         string
	CreationDate string // 2006-02-03T16:45:09.000Z
}

func (c *Client) transport() http.RoundTripper {
	if c.Transport != nil {
		return c.Transport
	}
	return http.DefaultTransport
}

func (c *Client) hostname() string {
	if c.Hostname != "" {
		return c.Hostname
	}
	return "localhost"
}

// bucketURL returns the URL prefix of the bucket, with trailing slash
func (c *Client) bucketURL(bucket string) string {
	return fmt.Sprintf("http://%s/%s/", c.hostname(), bucket)
}

func (c *Client) keyURL(bucket, key string) string {
	return c.bucketURL(bucket) + key
}

func newReq(url_ string) *http.Request {
	req, err := http.NewRequest("GET", url_, nil)
	if err != nil {
		panic(fmt.Sprintf("minio client; invalid URL: %v", err))
	}
	req.Header.Set("User-Agent", "minio")
	return req
}

func (c *Client) Buckets() ([]*Bucket, error) {
	req := newReq("http://" + c.hostname() + "/")
	res, err := c.transport().RoundTrip(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("minio: Unexpected status code %d fetching bucket list", res.StatusCode)
	}
	return parseListAllMyBuckets(res.Body)
}

func parseListAllMyBuckets(r io.Reader) ([]*Bucket, error) {
	type allMyBuckets struct {
		Buckets struct {
			Bucket []*Bucket
		}
	}
	var res allMyBuckets
	if err := xml.NewDecoder(r).Decode(&res); err != nil {
		return nil, err
	}
	return res.Buckets.Bucket, nil
}

// Returns 0, os.ErrNotExist if not on MINIO, otherwise reterr is real.
func (c *Client) Stat(key, bucket string) (size int64, reterr error) {
	req := newReq(c.keyURL(bucket, key))
	req.Method = "HEAD"
	res, err := c.transport().RoundTrip(req)
	if err != nil {
		return 0, err
	}
	if res.Body != nil {
		defer res.Body.Close()
	}
	switch res.StatusCode {
	case http.StatusNotFound:
		return 0, os.ErrNotExist
	case http.StatusOK:
		return strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64)
	}
	return 0, fmt.Errorf("minio: Unexpected status code %d statting object %v", res.StatusCode, key)
}

func (c *Client) PutBucket(bucket string) error {
	req := newReq(c.bucketURL(bucket))
	req.Method = "PUT"
	res, err := c.transport().RoundTrip(req)
	if res != nil && res.Body != nil {
		defer res.Body.Close()
	}

	if err != nil {
		return err
	}
	if res.StatusCode != http.StatusOK {
		// res.Write(os.Stderr)
		return fmt.Errorf("Got response code %d from minio", res.StatusCode)
	}
	return nil

}

func (c *Client) Put(bucket, key string, md5 hash.Hash, size int64, body io.Reader) error {
	req := newReq(c.keyURL(bucket, key))
	req.Method = "PUT"
	req.ContentLength = size
	if md5 != nil {
		b64 := new(bytes.Buffer)
		encoder := base64.NewEncoder(base64.StdEncoding, b64)
		encoder.Write(md5.Sum(nil))
		encoder.Close()
		req.Header.Set("Content-MD5", b64.String())
	}
	req.Body = ioutil.NopCloser(body)

	res, err := c.transport().RoundTrip(req)
	if res != nil && res.Body != nil {
		defer res.Body.Close()
	}

	if err != nil {
		return err
	}
	if res.StatusCode != http.StatusOK {
		// res.Write(os.Stderr)
		return fmt.Errorf("Got response code %d from minio", res.StatusCode)
	}
	return nil
}

type Item struct {
	Key          string
	LastModified string
	Size         int64
}

type listBucketResults struct {
	Contents    []*Item
	IsTruncated bool
	MaxKeys     int
	Name        string // bucket name
	Marker      string
}

// BucketLocation returns the Minio hostname to be used with the given bucket.
func (c *Client) BucketLocation(bucket string, hostname string) (location string, err error) {
	url_ := fmt.Sprintf("http://%s/%s/?location", hostname, url.QueryEscape(bucket))
	req := newReq(url_)
	res, err := c.transport().RoundTrip(req)
	if err != nil {
		return
	}
	var xres xmlLocationConstraint
	if err := xml.NewDecoder(res.Body).Decode(&xres); err != nil {
		return "", err
	}
	if xres.Location == "" {
		return "localhost", nil
	}
	return "minio-" + xres.Location + "." + hostname, nil
}

// GetBucket (List Objects) returns 0 to maxKeys (inclusive) items from the
// provided bucket. Keys before startAt will be skipped. (This is the MINIO
// 'marker' value). If the length of the returned items is equal to
// maxKeys, there is no indication whether or not the returned list is truncated.
func (c *Client) GetBucket(bucket string, startAt string, maxKeys int) (items []*Item, err error) {
	if maxKeys < 0 {
		return nil, errors.New("invalid negative maxKeys")
	}
	marker := startAt
	for len(items) < maxKeys {
		fetchN := maxKeys - len(items)
		if fetchN > MAX_OBJECT_LIST {
			fetchN = MAX_OBJECT_LIST
		}
		var bres listBucketResults

		url_ := fmt.Sprintf("%s?marker=%s&max-keys=%d",
			c.bucketURL(bucket), url.QueryEscape(marker), fetchN)

		// Try the enumerate three times, since s3 likes to close
		// https connections a lot, and Go sucks at dealing with it:
		// https://code.google.com/p/go/issues/detail?id=3514
		const maxTries = 5
		for try := 1; try <= maxTries; try++ {
			time.Sleep(time.Duration(try-1) * 100 * time.Millisecond)
			req := newReq(url_)
			res, err := c.transport().RoundTrip(req)
			if err != nil {
				if try < maxTries {
					continue
				}
				return nil, err
			}
			if res.StatusCode != http.StatusOK {
				if res.StatusCode < 500 {
					body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
					aerr := &Error{
						Op:     "ListBucket",
						Code:   res.StatusCode,
						Body:   body,
						Header: res.Header,
					}
					aerr.parseXML()
					res.Body.Close()
					return nil, aerr
				}
			} else {
				bres = listBucketResults{}
				var logbuf bytes.Buffer
				err = xml.NewDecoder(io.TeeReader(res.Body, &logbuf)).Decode(&bres)
				if err != nil {
					log.Printf("Error parsing minio XML response: %v for %q", err, logbuf.Bytes())
				} else if bres.MaxKeys != fetchN || bres.Name != bucket || bres.Marker != marker {
					err = fmt.Errorf("Unexpected parse from server: %#v from: %s", bres, logbuf.Bytes())
					log.Print(err)
				}
			}
			res.Body.Close()
			if err != nil {
				if try < maxTries-1 {
					continue
				}
				log.Print(err)
				return nil, err
			}
			break
		}
		for _, it := range bres.Contents {
			if it.Key == marker && it.Key != startAt {
				// Skip first dup on pages 2 and higher.
				continue
			}
			if it.Key < startAt {
				return nil, fmt.Errorf("Unexpected response from Minio: item key %q but wanted greater than %q", it.Key, startAt)
			}
			items = append(items, it)
			marker = it.Key
		}
		if !bres.IsTruncated {
			// log.Printf("Not truncated. so breaking. items = %d; len Contents = %d, url = %s", len(items), len(bres.Contents), url_)
			break
		}
	}
	return items, nil
}

func (c *Client) Get(bucket, key string) (body io.ReadCloser, size int64, err error) {
	req := newReq(c.keyURL(bucket, key))
	res, err := c.transport().RoundTrip(req)
	if err != nil {
		return
	}
	switch res.StatusCode {
	case http.StatusOK:
		return res.Body, res.ContentLength, nil
	case http.StatusNotFound:
		res.Body.Close()
		return nil, 0, os.ErrNotExist
	default:
		res.Body.Close()
		return nil, 0, fmt.Errorf("Minio HTTP error on GET: %d", res.StatusCode)
	}
}

// GetPartial fetches part of the minio key object in bucket.
// If length is negative, the rest of the object is returned.
// The caller must close rc.
func (c *Client) GetPartial(bucket, key string, offset, length int64) (rc io.ReadCloser, err error) {
	if offset < 0 {
		return nil, errors.New("invalid negative length")
	}

	req := newReq(c.keyURL(bucket, key))
	if length >= 0 {
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1))
	} else {
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
	}

	res, err := c.transport().RoundTrip(req)
	if err != nil {
		return
	}
	switch res.StatusCode {
	case http.StatusOK, http.StatusPartialContent:
		return res.Body, nil
	case http.StatusNotFound:
		res.Body.Close()
		return nil, os.ErrNotExist
	default:
		res.Body.Close()
		return nil, fmt.Errorf("Minio HTTP error on GET: %d", res.StatusCode)
	}
}

func NewMinioClient(hostname string) (client *Client) {
	client = &Client{hostname, http.DefaultTransport}
	return
}

// Error is the type returned by some API operations.
type Error struct {
	Op     string
	Code   int         // HTTP status code
	Body   []byte      // response body
	Header http.Header // response headers

	// UsedEndpoint and MinioCode are the XML response's Endpoint and
	// Code fields, respectively.
	UseEndpoint string // if a temporary redirect (wrong hostname)
	MinioCode   string
}

func (e *Error) Error() string {
	if bytes.Contains(e.Body, []byte("<Error>")) {
		return fmt.Sprintf("minio.%s: status %d: %s", e.Op, e.Code, e.Body)
	}
	return fmt.Sprintf("minio.%s: status %d", e.Op, e.Code)
}

func (e *Error) parseXML() {
	var xe xmlError
	_ = xml.NewDecoder(bytes.NewReader(e.Body)).Decode(&xe)
	e.MinioCode = xe.Code
	if xe.Code == "TemporaryRedirect" {
		e.UseEndpoint = xe.Endpoint
	}
	if xe.Code == "SignatureDoesNotMatch" {
		want, _ := hex.DecodeString(strings.Replace(xe.StringToSignBytes, " ", "", -1))
		log.Printf("MINIO SignatureDoesNotMatch. StringToSign should be %d bytes: %q (%x)", len(want), want, want)
	}

}

// xmlError is the Error response from Minio.
type xmlError struct {
	XMLName           xml.Name `xml:"Error"`
	Code              string
	Message           string
	RequestId         string
	Bucket            string
	Endpoint          string
	StringToSignBytes string
}

// xmlLocationConstraint is the LocationConstraint returned from BucketLocation.
type xmlLocationConstraint struct {
	XMLName  xml.Name `xml:"LocationConstraint"`
	Location string   `xml:",chardata"`
}
<commit_msg>Remove BucketLocation() function, minio does not implement such a resource<commit_after>/*
 * Mini Object Storage, (C) 2014,2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package minio implements a generic Minio client
package minio

import (
	"bytes"
	"encoding/base64"
	"encoding/hex"
	"encoding/xml"
	"errors"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	MAX_OBJECT_LIST = 1000
)

// Client is an Minio MINIO client.
type Client struct {
	Hostname  string
	Transport http.RoundTripper // or nil for the default
}

type Bucket struct {
	Name         string
	CreationDate string // 2006-02-03T16:45:09.000Z
}

func (c *Client) transport() http.RoundTripper {
	if c.Transport != nil {
		return c.Transport
	}
	return http.DefaultTransport
}

func (c *Client) hostname() string {
	if c.Hostname != "" {
		return c.Hostname
	}
	return "localhost"
}

// bucketURL returns the URL prefix of the bucket, with trailing slash
func (c *Client) bucketURL(bucket string) string {
	return fmt.Sprintf("http://%s/%s/", c.hostname(), bucket)
}

func (c *Client) keyURL(bucket, key string) string {
	return c.bucketURL(bucket) + key
}

func newReq(url_ string) *http.Request {
	req, err := http.NewRequest("GET", url_, nil)
	if err != nil {
		panic(fmt.Sprintf("minio client; invalid URL: %v", err))
	}
	req.Header.Set("User-Agent", "Minio")
	return req
}

func (c *Client) Buckets() ([]*Bucket, error) {
	req := newReq("http://" + c.hostname() + "/")
	res, err := c.transport().RoundTrip(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("minio: Unexpected status code %d fetching bucket list", res.StatusCode)
	}
	return parseListAllMyBuckets(res.Body)
}

func parseListAllMyBuckets(r io.Reader) ([]*Bucket, error) {
	type allMyBuckets struct {
		Buckets struct {
			Bucket []*Bucket
		}
	}
	var res allMyBuckets
	if err := xml.NewDecoder(r).Decode(&res); err != nil {
		return nil, err
	}
	return res.Buckets.Bucket, nil
}

// Returns 0, os.ErrNotExist if not on MINIO, otherwise reterr is real.
func (c *Client) Stat(key, bucket string) (size int64, reterr error) {
	req := newReq(c.keyURL(bucket, key))
	req.Method = "HEAD"
	res, err := c.transport().RoundTrip(req)
	if err != nil {
		return 0, err
	}
	if res.Body != nil {
		defer res.Body.Close()
	}
	switch res.StatusCode {
	case http.StatusNotFound:
		return 0, os.ErrNotExist
	case http.StatusOK:
		return strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64)
	}
	return 0, fmt.Errorf("minio: Unexpected status code %d statting object %v", res.StatusCode, key)
}

func (c *Client) PutBucket(bucket string) error {
	req := newReq(c.bucketURL(bucket))
	req.Method = "PUT"
	res, err := c.transport().RoundTrip(req)
	if res != nil && res.Body != nil {
		defer res.Body.Close()
	}

	if err != nil {
		return err
	}
	if res.StatusCode != http.StatusOK {
		// res.Write(os.Stderr)
		return fmt.Errorf("Got response code %d from minio", res.StatusCode)
	}
	return nil

}

func (c *Client) Put(bucket, key string, md5 hash.Hash, size int64, body io.Reader) error {
	req := newReq(c.keyURL(bucket, key))
	req.Method = "PUT"
	req.ContentLength = size
	if md5 != nil {
		b64 := new(bytes.Buffer)
		encoder := base64.NewEncoder(base64.StdEncoding, b64)
		encoder.Write(md5.Sum(nil))
		encoder.Close()
		req.Header.Set("Content-MD5", b64.String())
	}
	req.Body = ioutil.NopCloser(body)

	res, err := c.transport().RoundTrip(req)
	if res != nil && res.Body != nil {
		defer res.Body.Close()
	}

	if err != nil {
		return err
	}
	if res.StatusCode != http.StatusOK {
		// res.Write(os.Stderr)
		return fmt.Errorf("Got response code %d from minio", res.StatusCode)
	}
	return nil
}

type Item struct {
	Key          string
	LastModified string
	Size         int64
}

type listBucketResults struct {
	Contents    []*Item
	IsTruncated bool
	MaxKeys     int
	Name        string // bucket name
	Marker      string
}

// GetBucket (List Objects) returns 0 to maxKeys (inclusive) items from the
// provided bucket. Keys before startAt will be skipped. (This is the MINIO
// 'marker' value). If the length of the returned items is equal to
// maxKeys, there is no indication whether or not the returned list is truncated.
func (c *Client) GetBucket(bucket string, startAt string, maxKeys int) (items []*Item, err error) {
	if maxKeys < 0 {
		return nil, errors.New("invalid negative maxKeys")
	}
	marker := startAt
	for len(items) < maxKeys {
		fetchN := maxKeys - len(items)
		if fetchN > MAX_OBJECT_LIST {
			fetchN = MAX_OBJECT_LIST
		}
		var bres listBucketResults

		url_ := fmt.Sprintf("%s?marker=%s&max-keys=%d",
			c.bucketURL(bucket), url.QueryEscape(marker), fetchN)

		// Try the enumerate three times, since s3 likes to close
		// https connections a lot, and Go sucks at dealing with it:
		// https://code.google.com/p/go/issues/detail?id=3514
		const maxTries = 5
		for try := 1; try <= maxTries; try++ {
			time.Sleep(time.Duration(try-1) * 100 * time.Millisecond)
			req := newReq(url_)
			res, err := c.transport().RoundTrip(req)
			if err != nil {
				if try < maxTries {
					continue
				}
				return nil, err
			}
			if res.StatusCode != http.StatusOK {
				if res.StatusCode < 500 {
					body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
					aerr := &Error{
						Op:     "ListBucket",
						Code:   res.StatusCode,
						Body:   body,
						Header: res.Header,
					}
					aerr.parseXML()
					res.Body.Close()
					return nil, aerr
				}
			} else {
				bres = listBucketResults{}
				var logbuf bytes.Buffer
				err = xml.NewDecoder(io.TeeReader(res.Body, &logbuf)).Decode(&bres)
				if err != nil {
					log.Printf("Error parsing minio XML response: %v for %q", err, logbuf.Bytes())
				} else if bres.MaxKeys != fetchN || bres.Name != bucket || bres.Marker != marker {
					err = fmt.Errorf("Unexpected parse from server: %#v from: %s", bres, logbuf.Bytes())
					log.Print(err)
				}
			}
			res.Body.Close()
			if err != nil {
				if try < maxTries-1 {
					continue
				}
				log.Print(err)
				return nil, err
			}
			break
		}
		for _, it := range bres.Contents {
			if it.Key == marker && it.Key != startAt {
				// Skip first dup on pages 2 and higher.
				continue
			}
			if it.Key < startAt {
				return nil, fmt.Errorf("Unexpected response from Minio: item key %q but wanted greater than %q", it.Key, startAt)
			}
			items = append(items, it)
			marker = it.Key
		}
		if !bres.IsTruncated {
			// log.Printf("Not truncated. so breaking. items = %d; len Contents = %d, url = %s", len(items), len(bres.Contents), url_)
			break
		}
	}
	return items, nil
}

func (c *Client) Get(bucket, key string) (body io.ReadCloser, size int64, err error) {
	req := newReq(c.keyURL(bucket, key))
	res, err := c.transport().RoundTrip(req)
	if err != nil {
		return
	}
	switch res.StatusCode {
	case http.StatusOK:
		return res.Body, res.ContentLength, nil
	case http.StatusNotFound:
		res.Body.Close()
		return nil, 0, os.ErrNotExist
	default:
		res.Body.Close()
		return nil, 0, fmt.Errorf("Minio HTTP error on GET: %d", res.StatusCode)
	}
}

// GetPartial fetches part of the minio key object in bucket.
// If length is negative, the rest of the object is returned.
// The caller must close rc.
func (c *Client) GetPartial(bucket, key string, offset, length int64) (rc io.ReadCloser, err error) {
	if offset < 0 {
		return nil, errors.New("invalid negative length")
	}

	req := newReq(c.keyURL(bucket, key))
	if length >= 0 {
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1))
	} else {
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
	}

	res, err := c.transport().RoundTrip(req)
	if err != nil {
		return
	}
	switch res.StatusCode {
	case http.StatusOK, http.StatusPartialContent:
		return res.Body, nil
	case http.StatusNotFound:
		res.Body.Close()
		return nil, os.ErrNotExist
	default:
		res.Body.Close()
		return nil, fmt.Errorf("Minio HTTP error on GET: %d", res.StatusCode)
	}
}

func NewMinioClient(hostname string) (client *Client) {
	client = &Client{hostname, http.DefaultTransport}
	return
}

// Error is the type returned by some API operations.
type Error struct {
	Op     string
	Code   int         // HTTP status code
	Body   []byte      // response body
	Header http.Header // response headers

	// UsedEndpoint and MinioCode are the XML response's Endpoint and
	// Code fields, respectively.
	UseEndpoint string // if a temporary redirect (wrong hostname)
	MinioCode   string
}

func (e *Error) Error() string {
	if bytes.Contains(e.Body, []byte("<Error>")) {
		return fmt.Sprintf("minio.%s: status %d: %s", e.Op, e.Code, e.Body)
	}
	return fmt.Sprintf("minio.%s: status %d", e.Op, e.Code)
}

func (e *Error) parseXML() {
	var xe xmlError
	_ = xml.NewDecoder(bytes.NewReader(e.Body)).Decode(&xe)
	e.MinioCode = xe.Code
	if xe.Code == "TemporaryRedirect" {
		e.UseEndpoint = xe.Endpoint
	}
	if xe.Code == "SignatureDoesNotMatch" {
		want, _ := hex.DecodeString(strings.Replace(xe.StringToSignBytes, " ", "", -1))
		log.Printf("Minio SignatureDoesNotMatch. StringToSign should be %d bytes: %q (%x)", len(want), want, want)
	}

}

// xmlError is the Error response from Minio.
type xmlError struct {
	XMLName           xml.Name `xml:"Error"`
	Code              string
	Message           string
	RequestId         string
	Bucket            string
	Endpoint          string
	StringToSignBytes string
}
<|endoftext|>


<commit_before>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package node

import (
	"bytes"
	"fmt"
	"net"
	"os/exec"
	"strings"

	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/maps/tunnel"

	log "github.com/sirupsen/logrus"
	"github.com/vishvananda/netlink"
)

// RouteType represents the route type to be configured when adding the node
// routes
type RouteType int

const (
	// TunnelRoute is the route type to set up the BPF tunnel maps
	TunnelRoute RouteType = 1 << iota
	// DirectRoute is the route type to set up the L3 route using iproute
	DirectRoute
)

var (
	mutex lock.RWMutex
	nodes = map[Identity]*Node{}
)

// GetNode returns the node with the given identity, if exists, from the nodes
// map.
func GetNode(ni Identity) *Node {
	mutex.RLock()
	n := nodes[ni]
	mutex.RUnlock()
	return n
}

func deleteNodeCIDR(ip *net.IPNet) {
	if ip == nil {
		return
	}

	if err := tunnel.DeleteTunnelEndpoint(ip.IP); err != nil {
		log.Errorf("bpf: Unable to delete %s in tunnel endpoint map: %s", ip, err)
	}
}

func updateNodeCIDR(ip *net.IPNet, host net.IP) {
	if ip == nil {
		return
	}

	if err := tunnel.SetTunnelEndpoint(ip.IP, host); err != nil {
		log.Errorf("bpf: Unable to update %s in tunnel endpoint map: %s", ip, err)
	}
}

// UpdateNode updates the new node in the nodes' map with the given identity.
// When using DirectRoute RouteType the field ownAddr should contain the IPv6
// address of the interface that can reach the other nodes.
func UpdateNode(ni Identity, n *Node, routesTypes RouteType, ownAddr net.IP) {
	mutex.Lock()
	defer mutex.Unlock()

	oldNode, oldNodeExists := nodes[ni]
	if (routesTypes & TunnelRoute) != 0 {
		if oldNodeExists {
			deleteNodeCIDR(oldNode.IPv4AllocCIDR)
			deleteNodeCIDR(oldNode.IPv6AllocCIDR)
		}
		// FIXME if PodCIDR is empty retrieve the CIDR from the KVStore
		log.Debugf("bpf: Setting tunnel endpoint %+v: %+v %+v",
			n.GetNodeIP(false), n.IPv4AllocCIDR, n.IPv6AllocCIDR)

		nodeIP := n.GetNodeIP(false)
		updateNodeCIDR(n.IPv4AllocCIDR, nodeIP)
		updateNodeCIDR(n.IPv6AllocCIDR, nodeIP)
	}
	if (routesTypes & DirectRoute) != 0 {
		updateIPRoute(oldNode, n, ownAddr)
	}

	nodes[ni] = n
}

// DeleteNode remove the node from the nodes' maps and / or the L3 routes to
// reach that node.
func DeleteNode(ni Identity, routesTypes RouteType) {
	var err1, err2 error
	mutex.Lock()
	if n, ok := nodes[ni]; ok {
		if (routesTypes & TunnelRoute) != 0 {
			log.Debugf("bpf: Removing tunnel endpoint %+v: %+v %+v",
				n.GetNodeIP(false), n.IPv4AllocCIDR, n.IPv6AllocCIDR)

			if n.IPv4AllocCIDR != nil {
				err1 = tunnel.DeleteTunnelEndpoint(n.IPv4AllocCIDR.IP)
				if err1 == nil {
					n.IPv4AllocCIDR = nil
				}
			}

			if n.IPv6AllocCIDR != nil {
				err2 = tunnel.DeleteTunnelEndpoint(n.IPv6AllocCIDR.IP)
				if err2 == nil {
					n.IPv6AllocCIDR = nil
				}
			}
		}
		if (routesTypes & DirectRoute) != 0 {
			deleteIPRoute(n)
		}
	}

	// Keep node around
	if err1 == nil && err2 == nil {
		delete(nodes, ni)
	}

	mutex.Unlock()
}

// updateIPRoute updates the IP routing entry for the given node n via the
// network interface that as ownAddr.
func updateIPRoute(oldNode, n *Node, ownAddr net.IP) {
	nodeIPv6 := n.GetNodeIP(true)
	log.Debugf("iproute: Setting endpoint v6 route for %s via %s",
		n.IPv6AllocCIDR, nodeIPv6)

	nl, err := firstLinkWithv6(ownAddr)
	if err != nil {
		log.WithError(err).Errorf("iproute: Unable to get v6 interface with IP %s", ownAddr)
		return
	}
	dev := nl.Attrs().Name
	if dev == "" {
		log.Errorf("iproute: Unable to get v6 interface for %s: empty interface name", ownAddr)
		return
	}

	if oldNode != nil {
		oldNodeIPv6 := oldNode.GetNodeIP(true)
		if oldNode.IPv6AllocCIDR.String() != n.IPv6AllocCIDR.String() ||
			!oldNodeIPv6.Equal(nodeIPv6) ||
			oldNode.dev != n.dev {
			// If any of the routing components changed, then remove the old entries

			err = routeDel(oldNodeIPv6.String(), oldNode.IPv6AllocCIDR.String(), oldNode.dev)
			if err != nil {
				log.Warning(err)
			}
		}
	} else {
		n.dev = dev
	}

	// Always re add
	err = routeAdd(nodeIPv6.String(), n.IPv6AllocCIDR.String(), dev)
	if err != nil {
		log.Warning(err)
		return
	}
}

// deleteIPRoute deletes the routing entries previously created for the given
// node.
func deleteIPRoute(node *Node) {
	oldNodeIPv6 := node.GetNodeIP(true)

	err := routeDel(oldNodeIPv6.String(), node.IPv6AllocCIDR.String(), node.dev)
	if err != nil {
		log.Warning(err)
	}
}

// firstLinkWithv6 returns the first network interface that contains the given
// IPv6 address.
func firstLinkWithv6(ip net.IP) (netlink.Link, error) {
	links, err := netlink.LinkList()
	if err != nil {
		return nil, err
	}
	for _, l := range links {
		addrs, _ := netlink.AddrList(l, netlink.FAMILY_V6)
		for _, a := range addrs {
			if ip.Equal(a.IP) {
				return l, nil
			}
		}
	}

	return nil, fmt.Errorf("No address found")
}

func routeAdd(dstNode, podCIDR, dev string) error {
	prog := "ip"

	// for example: ip -6 r a fd00::b dev eth0
	// TODO: don't add direct route if a subnet of that IP is already present
	// in the routing table
	args := []string{"-6", "route", "add", dstNode, "dev", dev}
	out, err := exec.Command(prog, args...).CombinedOutput()
	// Ignore file exists in case the route already exists
	if err != nil && !bytes.Contains(out, []byte("File exists")) {
		return fmt.Errorf("unable to add routing entry, command %s %s failed: %s: %s", prog,
			strings.Join(args, " "), err, out)
	}

	// now we can add the pods cidr route via the other's node IP
	// for example: ip -6 r a f00d::ac1f:32:0:0/96 via fd00::b
	args = []string{"-6", "route", "add", podCIDR, "via", dstNode}
	out, err = exec.Command(prog, args...).CombinedOutput()
	// Ignore file exists in case the route already exists
	if err != nil && !bytes.Contains(out, []byte("File exists")) {
		return fmt.Errorf("unable to add routing entry, command %s %s failed: %s: %s", prog,
			strings.Join(args, " "), err, out)
	}
	return nil
}

func routeDel(dstNode, podCIDR, dev string) error {
	prog := "ip"

	args := []string{"-6", "route", "del", podCIDR, "via", dstNode}
	out, err := exec.Command(prog, args...).CombinedOutput()
	if err != nil {
		return fmt.Errorf("unable to clean up old routing entry, command %s %s failed: %s: %s", prog,
			strings.Join(args, " "), err, out)
	}

	args = []string{"-6", "route", "del", dstNode, "dev", dev}
	out, err = exec.Command(prog, args...).CombinedOutput()
	if err != nil {
		return fmt.Errorf("unable to clean up old routing entry, command %s %s failed: %s: %s", prog,
			strings.Join(args, " "), err, out)
	}
	return nil
}
<commit_msg>pkg/node: Consistently use logrus fields<commit_after>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package node

import (
	"bytes"
	"fmt"
	"net"
	"os/exec"
	"strings"

	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/logfields"
	"github.com/cilium/cilium/pkg/maps/tunnel"

	log "github.com/sirupsen/logrus"
	"github.com/vishvananda/netlink"
)

// RouteType represents the route type to be configured when adding the node
// routes
type RouteType int

const (
	// TunnelRoute is the route type to set up the BPF tunnel maps
	TunnelRoute RouteType = 1 << iota
	// DirectRoute is the route type to set up the L3 route using iproute
	DirectRoute
)

var (
	mutex lock.RWMutex
	nodes = map[Identity]*Node{}
)

// GetNode returns the node with the given identity, if exists, from the nodes
// map.
func GetNode(ni Identity) *Node {
	mutex.RLock()
	n := nodes[ni]
	mutex.RUnlock()
	return n
}

func deleteNodeCIDR(ip *net.IPNet) {
	if ip == nil {
		return
	}

	if err := tunnel.DeleteTunnelEndpoint(ip.IP); err != nil {
		log.WithError(err).WithField(logfields.IPAddr, ip).Error("bpf: Unable to delete in tunnel endpoint map")
	}
}

func updateNodeCIDR(ip *net.IPNet, host net.IP) {
	if ip == nil {
		return
	}

	if err := tunnel.SetTunnelEndpoint(ip.IP, host); err != nil {
		log.WithError(err).WithField(logfields.IPAddr, ip).Error("bpf: Unable to update in tunnel endpoint map")
	}
}

// UpdateNode updates the new node in the nodes' map with the given identity.
// When using DirectRoute RouteType the field ownAddr should contain the IPv6
// address of the interface that can reach the other nodes.
func UpdateNode(ni Identity, n *Node, routesTypes RouteType, ownAddr net.IP) {
	mutex.Lock()
	defer mutex.Unlock()

	oldNode, oldNodeExists := nodes[ni]
	if (routesTypes & TunnelRoute) != 0 {
		if oldNodeExists {
			deleteNodeCIDR(oldNode.IPv4AllocCIDR)
			deleteNodeCIDR(oldNode.IPv6AllocCIDR)
		}
		// FIXME if PodCIDR is empty retrieve the CIDR from the KVStore
		log.WithFields(log.Fields{
			logfields.IPAddr:   n.GetNodeIP(false),
			logfields.V4Prefix: n.IPv4AllocCIDR,
			logfields.V6Prefix: n.IPv6AllocCIDR,
		}).Debug("bpf: Setting tunnel endpoint")

		nodeIP := n.GetNodeIP(false)
		updateNodeCIDR(n.IPv4AllocCIDR, nodeIP)
		updateNodeCIDR(n.IPv6AllocCIDR, nodeIP)
	}
	if (routesTypes & DirectRoute) != 0 {
		updateIPRoute(oldNode, n, ownAddr)
	}

	nodes[ni] = n
}

// DeleteNode remove the node from the nodes' maps and / or the L3 routes to
// reach that node.
func DeleteNode(ni Identity, routesTypes RouteType) {
	var err1, err2 error
	mutex.Lock()
	if n, ok := nodes[ni]; ok {
		if (routesTypes & TunnelRoute) != 0 {
			log.WithFields(log.Fields{
				logfields.IPAddr:   n.GetNodeIP(false),
				logfields.V4Prefix: n.IPv4AllocCIDR,
				logfields.V6Prefix: n.IPv6AllocCIDR,
			}).Debug("bpf: Removing tunnel endpoint")

			if n.IPv4AllocCIDR != nil {
				err1 = tunnel.DeleteTunnelEndpoint(n.IPv4AllocCIDR.IP)
				if err1 == nil {
					n.IPv4AllocCIDR = nil
				}
			}

			if n.IPv6AllocCIDR != nil {
				err2 = tunnel.DeleteTunnelEndpoint(n.IPv6AllocCIDR.IP)
				if err2 == nil {
					n.IPv6AllocCIDR = nil
				}
			}
		}
		if (routesTypes & DirectRoute) != 0 {
			deleteIPRoute(n)
		}
	}

	// Keep node around
	if err1 == nil && err2 == nil {
		delete(nodes, ni)
	}

	mutex.Unlock()
}

// updateIPRoute updates the IP routing entry for the given node n via the
// network interface that as ownAddr.
func updateIPRoute(oldNode, n *Node, ownAddr net.IP) {
	nodeIPv6 := n.GetNodeIP(true)
	scopedLog := log.WithField(logfields.V6Prefix, n.IPv6AllocCIDR)
	scopedLog.WithField(logfields.IPAddr, nodeIPv6).Debug("iproute: Setting endpoint v6 route for prefix via IP")

	nl, err := firstLinkWithv6(ownAddr)
	if err != nil {
		scopedLog.WithError(err).WithField(logfields.IPAddr, ownAddr).Error("iproute: Unable to get v6 interface with IP")
		return
	}
	dev := nl.Attrs().Name
	if dev == "" {
		scopedLog.WithField(logfields.IPAddr, ownAddr).Error("iproute: Unable to get v6 interface for address: empty interface name")
		return
	}

	if oldNode != nil {
		oldNodeIPv6 := oldNode.GetNodeIP(true)
		if oldNode.IPv6AllocCIDR.String() != n.IPv6AllocCIDR.String() ||
			!oldNodeIPv6.Equal(nodeIPv6) ||
			oldNode.dev != n.dev {
			// If any of the routing components changed, then remove the old entries

			err = routeDel(oldNodeIPv6.String(), oldNode.IPv6AllocCIDR.String(), oldNode.dev)
			if err != nil {
				log.WithError(err).WithFields(log.Fields{
					logfields.IPAddr:   oldNodeIPv6,
					logfields.V6Prefix: oldNode.IPv6AllocCIDR,
					"device":           oldNode.dev,
				}).Warn("Cannot delete old route during update")
			}
		}
	} else {
		n.dev = dev
	}

	// Always re add
	err = routeAdd(nodeIPv6.String(), n.IPv6AllocCIDR.String(), dev)
	if err != nil {
		log.WithError(err).WithFields(log.Fields{
			logfields.IPAddr:   nodeIPv6,
			logfields.V6Prefix: n.IPv6AllocCIDR,
			"device":           dev,
		}).Warn("Cannot re-add route")
		return
	}
}

// deleteIPRoute deletes the routing entries previously created for the given
// node.
func deleteIPRoute(node *Node) {
	oldNodeIPv6 := node.GetNodeIP(true)

	err := routeDel(oldNodeIPv6.String(), node.IPv6AllocCIDR.String(), node.dev)
	if err != nil {
		log.WithError(err).WithFields(log.Fields{
			logfields.IPAddr:   oldNodeIPv6,
			logfields.V6Prefix: node.IPv6AllocCIDR,
			"device":           node.dev,
		}).Warn("Cannot delete route")
	}
}

// firstLinkWithv6 returns the first network interface that contains the given
// IPv6 address.
func firstLinkWithv6(ip net.IP) (netlink.Link, error) {
	links, err := netlink.LinkList()
	if err != nil {
		return nil, err
	}
	for _, l := range links {
		addrs, _ := netlink.AddrList(l, netlink.FAMILY_V6)
		for _, a := range addrs {
			if ip.Equal(a.IP) {
				return l, nil
			}
		}
	}

	return nil, fmt.Errorf("No address found")
}

func routeAdd(dstNode, podCIDR, dev string) error {
	prog := "ip"

	// for example: ip -6 r a fd00::b dev eth0
	// TODO: don't add direct route if a subnet of that IP is already present
	// in the routing table
	args := []string{"-6", "route", "add", dstNode, "dev", dev}
	out, err := exec.Command(prog, args...).CombinedOutput()
	// Ignore file exists in case the route already exists
	if err != nil && !bytes.Contains(out, []byte("File exists")) {
		return fmt.Errorf("unable to add routing entry, command %s %s failed: %s: %s", prog,
			strings.Join(args, " "), err, out)
	}

	// now we can add the pods cidr route via the other's node IP
	// for example: ip -6 r a f00d::ac1f:32:0:0/96 via fd00::b
	args = []string{"-6", "route", "add", podCIDR, "via", dstNode}
	out, err = exec.Command(prog, args...).CombinedOutput()
	// Ignore file exists in case the route already exists
	if err != nil && !bytes.Contains(out, []byte("File exists")) {
		return fmt.Errorf("unable to add routing entry, command %s %s failed: %s: %s", prog,
			strings.Join(args, " "), err, out)
	}
	return nil
}

func routeDel(dstNode, podCIDR, dev string) error {
	prog := "ip"

	args := []string{"-6", "route", "del", podCIDR, "via", dstNode}
	out, err := exec.Command(prog, args...).CombinedOutput()
	if err != nil {
		return fmt.Errorf("unable to clean up old routing entry, command %s %s failed: %s: %s", prog,
			strings.Join(args, " "), err, out)
	}

	args = []string{"-6", "route", "del", dstNode, "dev", dev}
	out, err = exec.Command(prog, args...).CombinedOutput()
	if err != nil {
		return fmt.Errorf("unable to clean up old routing entry, command %s %s failed: %s: %s", prog,
			strings.Join(args, " "), err, out)
	}
	return nil
}
<|endoftext|>


<commit_before>package qemu

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"strings"
	"time"

	"code.cloudfoundry.org/bytefmt"
	"github.com/digitalocean/go-qemu/qmp"
	"github.com/digitalocean/go-qemu/qmp/raw"
	uuid "github.com/satori/go.uuid"
	"github.com/shirou/gopsutil/process"
)

type Qemu struct {
	proc *process.Process

	// args
	id      *uuid.UUID
	qmpPath string
	isKVM   bool

	qmp qmp.Monitor
	m   *raw.Monitor
}

func OpenQemu(id *uuid.UUID) (*Qemu, error) {
	q := &Qemu{
		id:    id,
		isKVM: true,
	}

	if err := q.init(); err != nil {
		return nil, err
	}

	return q, nil
}

func (q *Qemu) Close() error {
	if q.m == nil {
		return nil
	}

	return q.qmp.Disconnect()
}

func (q Qemu) Reset() error {
	return q.m.SystemReset()
}

func (q Qemu) Shutdown() error {
	return q.m.SystemPowerdown()
}

func (q Qemu) Boot() error {
	if err := q.m.Cont(); err != nil {
		return err
	}

	return q.m.SystemWakeup()
}

type Status int

const (
	StatusDebug         Status = Status(raw.RunStateDebug)
	StatusFinishMigrate Status = Status(raw.RunStateFinishMigrate)
	StatusGuestPanicked Status = Status(raw.RunStateGuestPanicked)
	StatusIOError       Status = Status(raw.RunStateIOError)
	StatusInMigrate     Status = Status(raw.RunStateInmigrate)
	StatusInternalError Status = Status(raw.RunStateInternalError)
	StatusPaused        Status = Status(raw.RunStatePaused)
	StatusPostMigrate   Status = Status(raw.RunStatePostmigrate)
	StatusPreLaunch     Status = Status(raw.RunStatePrelaunch)
	StatusRestoreVM     Status = Status(raw.RunStateRestoreVM)
	StatusRunning       Status = Status(raw.RunStateRunning)
	StatusSaveVM        Status = Status(raw.RunStateSaveVM)
	StatusShutdown      Status = Status(raw.RunStateShutdown)
	StatusSuspended     Status = Status(raw.RunStateSuspended)
	StatusWatchdog      Status = Status(raw.RunStateWatchdog)
)

func (q Qemu) Status() (Status, error) {
	s, err := q.m.QueryStatus()
	if err != nil {
		if strings.Contains(err.Error(), "not running") {
			return StatusShutdown, nil
		}

		return 0, err
	}

	return Status(s.Status), nil
}

func (q Qemu) IsRunning() bool {
	if q.proc == nil {
		return false
	}

	return true
}

func (q *Qemu) Kill() error { // 統一感を持たせるためにDeleteにしたほうがいいかもしれない
	// if err := q.Close(); err != nil {
	// 	return fmt.Errorf("Failed to close ")
	// }

	if q.proc != nil {
		if err := q.proc.Kill(); err != nil {
			return fmt.Errorf("Failed to kill process: err='%s'", err.Error())
		}

		q.proc = nil
	}

	if err := os.Remove(q.qmpPath); err != nil {
		return fmt.Errorf("Failed to delete QMP socket: err='%s'", err.Error())
	}

	// delete monitor.sock

	return nil
}

func (q Qemu) getVNCPort() uint16 {
	for p := 5900; p < 65536; p++ {
		l, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", p))
		if err == nil {
			defer l.Close()

			return uint16(p)
		}
	}

	return 0
}

func (q *Qemu) StartProcess(name, qmpPath string, vncWebsocketPort, vcpus uint32, memory uint64) error {
	args := []string{
		"qemu-system-x86_64",

		// -- QEMU metadata --
		"-uuid",
		q.id.String(),
		"-name",
		fmt.Sprintf("guest=%s,debug-threads=on", name),
		"-msg",
		"timestamp=on",

		// Config
		"-daemonize",
		"-nodefaults",     // Don't create default devices
		"-no-user-config", // The "-no-user-config" option makes QEMU not load any of the user-provided config files on sysconfdir
		"-S",              // Do not start CPU at startup
		"-no-shutdown",    // Don't exit QEMU on guest shutdown

		// QMP
		"-chardev",
		fmt.Sprintf("socket,id=charmonitor,path=%s,server,nowait", qmpPath),
		"-mon",
		"chardev=charmonitor,id=monitor,mode=control",

		// -- BIOS --
		// boot priority
		"-boot",
		"menu=on,strict=on",

		// keyboard
		"-k",
		"en-us",

		// VNC
		"-vnc",
		fmt.Sprintf("127.0.0.1:%d,websocket=%d", q.getVNCPort()-5900, vncWebsocketPort), // TODO: ぶつからないようにポートを設定する必要がある、現状一台しか立たない

		// clock
		"-rtc",
		"base=utc,driftfix=slew",
		"-global",
		"kvm-pit.lost_tick_policy=delay",
		"-no-hpet",

		// CPU
		// TODO: 必要があればmonitorを操作してhotaddできるようにする
		// TODO: スケジューリングが可能かどうか調べる
		"-smp",
		fmt.Sprintf("%d,sockets=1,cores=%d,threads=1", vcpus, vcpus),
		"-cpu",
		"host",
		"-enable-kvm",

		// Memory
		"-m",
		fmt.Sprintf("%s", bytefmt.ByteSize(memory)),
		// "-device",
		// "virtio-balloon-pci,id=balloon0,bus=pci.0", // dynamic configurations
		"-realtime",
		"mlock=off",

		// VGA controller
		"-device",
		"VGA,id=video0,bus=pci.0",

		// SCSI controller
		"-device",
		"lsi53c895a,bus=pci.0,id=scsi0",
	}

	if !q.isKVM {
		args = append(args[:29], args[32:]...)
	}

	cmd := exec.Command(args[0], args[1:]...)
	if err := cmd.Start(); err != nil { // TODO: combine でもいいかもしれない
		return fmt.Errorf("Failed to start process: args='%s', err='%s'", args, err.Error())
	}

	done := make(chan error)
	go func() {
		done <- cmd.Wait()
	}()

	select {
	case <-time.After(3 * time.Second):
		return nil
	case err := <-done:
		if err != nil {
			return fmt.Errorf("Failed to run process: args='%s', err='%s'", args, err.Error()) // stderrを表示できるようにする必要がある
		}

		if err := q.init(); err != nil {
			return fmt.Errorf("Failed to initialize: args='%s', err='%s'", args, err.Error())
		}

		return nil
	}
}
<commit_msg>add comment about disabling kvm on module/qemu<commit_after>package qemu

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"strings"
	"time"

	"code.cloudfoundry.org/bytefmt"
	"github.com/digitalocean/go-qemu/qmp"
	"github.com/digitalocean/go-qemu/qmp/raw"
	uuid "github.com/satori/go.uuid"
	"github.com/shirou/gopsutil/process"
)

type Qemu struct {
	proc *process.Process

	// args
	id      *uuid.UUID
	qmpPath string
	isKVM   bool

	qmp qmp.Monitor
	m   *raw.Monitor
}

func OpenQemu(id *uuid.UUID) (*Qemu, error) {
	q := &Qemu{
		id:    id,
		isKVM: true,
	}

	if err := q.init(); err != nil {
		return nil, err
	}

	return q, nil
}

func (q *Qemu) Close() error {
	if q.m == nil {
		return nil
	}

	return q.qmp.Disconnect()
}

func (q Qemu) Reset() error {
	return q.m.SystemReset()
}

func (q Qemu) Shutdown() error {
	return q.m.SystemPowerdown()
}

func (q Qemu) Boot() error {
	if err := q.m.Cont(); err != nil {
		return err
	}

	return q.m.SystemWakeup()
}

type Status int

const (
	StatusDebug         Status = Status(raw.RunStateDebug)
	StatusFinishMigrate Status = Status(raw.RunStateFinishMigrate)
	StatusGuestPanicked Status = Status(raw.RunStateGuestPanicked)
	StatusIOError       Status = Status(raw.RunStateIOError)
	StatusInMigrate     Status = Status(raw.RunStateInmigrate)
	StatusInternalError Status = Status(raw.RunStateInternalError)
	StatusPaused        Status = Status(raw.RunStatePaused)
	StatusPostMigrate   Status = Status(raw.RunStatePostmigrate)
	StatusPreLaunch     Status = Status(raw.RunStatePrelaunch)
	StatusRestoreVM     Status = Status(raw.RunStateRestoreVM)
	StatusRunning       Status = Status(raw.RunStateRunning)
	StatusSaveVM        Status = Status(raw.RunStateSaveVM)
	StatusShutdown      Status = Status(raw.RunStateShutdown)
	StatusSuspended     Status = Status(raw.RunStateSuspended)
	StatusWatchdog      Status = Status(raw.RunStateWatchdog)
)

func (q Qemu) Status() (Status, error) {
	s, err := q.m.QueryStatus()
	if err != nil {
		if strings.Contains(err.Error(), "not running") {
			return StatusShutdown, nil
		}

		return 0, err
	}

	return Status(s.Status), nil
}

func (q Qemu) IsRunning() bool {
	if q.proc == nil {
		return false
	}

	return true
}

func (q *Qemu) Kill() error { // 統一感を持たせるためにDeleteにしたほうがいいかもしれない
	// if err := q.Close(); err != nil {
	// 	return fmt.Errorf("Failed to close ")
	// }

	if q.proc != nil {
		if err := q.proc.Kill(); err != nil {
			return fmt.Errorf("Failed to kill process: err='%s'", err.Error())
		}

		q.proc = nil
	}

	if err := os.Remove(q.qmpPath); err != nil {
		return fmt.Errorf("Failed to delete QMP socket: err='%s'", err.Error())
	}

	// delete monitor.sock

	return nil
}

func (q Qemu) getVNCPort() uint16 {
	for p := 5900; p < 65536; p++ {
		l, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", p))
		if err == nil {
			defer l.Close()

			return uint16(p)
		}
	}

	return 0
}

func (q *Qemu) StartProcess(name, qmpPath string, vncWebsocketPort, vcpus uint32, memory uint64) error {
	args := []string{
		"qemu-system-x86_64",

		// -- QEMU metadata --
		"-uuid",
		q.id.String(),
		"-name",
		fmt.Sprintf("guest=%s,debug-threads=on", name),
		"-msg",
		"timestamp=on",

		// Config
		"-daemonize",
		"-nodefaults",     // Don't create default devices
		"-no-user-config", // The "-no-user-config" option makes QEMU not load any of the user-provided config files on sysconfdir
		"-S",              // Do not start CPU at startup
		"-no-shutdown",    // Don't exit QEMU on guest shutdown

		// QMP
		"-chardev",
		fmt.Sprintf("socket,id=charmonitor,path=%s,server,nowait", qmpPath),
		"-mon",
		"chardev=charmonitor,id=monitor,mode=control",

		// -- BIOS --
		// boot priority
		"-boot",
		"menu=on,strict=on",

		// keyboard
		"-k",
		"en-us",

		// VNC
		"-vnc",
		fmt.Sprintf("127.0.0.1:%d,websocket=%d", q.getVNCPort()-5900, vncWebsocketPort), // TODO: ぶつからないようにポートを設定する必要がある、現状一台しか立たない

		// clock
		"-rtc",
		"base=utc,driftfix=slew",
		"-global",
		"kvm-pit.lost_tick_policy=delay",
		"-no-hpet",

		// CPU
		// TODO: 必要があればmonitorを操作してhotaddできるようにする
		// TODO: スケジューリングが可能かどうか調べる
		"-smp",
		fmt.Sprintf("%d,sockets=1,cores=%d,threads=1", vcpus, vcpus),
		"-cpu",
		"host",
		"-enable-kvm",

		// Memory
		"-m",
		fmt.Sprintf("%s", bytefmt.ByteSize(memory)),
		// "-device",
		// "virtio-balloon-pci,id=balloon0,bus=pci.0", // dynamic configurations
		"-realtime",
		"mlock=off",

		// VGA controller
		"-device",
		"VGA,id=video0,bus=pci.0",

		// SCSI controller
		"-device",
		"lsi53c895a,bus=pci.0,id=scsi0",
	}

	if !q.isKVM {
		// remove "-cpu", "host" and "-enable-kvm", because kvm is disable
		args = append(args[:29], args[32:]...)
	}

	cmd := exec.Command(args[0], args[1:]...)
	if err := cmd.Start(); err != nil { // TODO: combine でもいいかもしれない
		return fmt.Errorf("Failed to start process: args='%s', err='%s'", args, err.Error())
	}

	done := make(chan error)
	go func() {
		done <- cmd.Wait()
	}()

	select {
	case <-time.After(3 * time.Second):
		return nil
	case err := <-done:
		if err != nil {
			return fmt.Errorf("Failed to run process: args='%s', err='%s'", args, err.Error()) // stderrを表示できるようにする必要がある
		}

		if err := q.init(); err != nil {
			return fmt.Errorf("Failed to initialize: args='%s', err='%s'", args, err.Error())
		}

		return nil
	}
}
<|endoftext|>