<commit_before>/*
 * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"bytes"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"reflect"
	"runtime"
	"strings"
)

// getReaderSize - Determine the size of Reader if available.
func getReaderSize(reader io.Reader) (size int64, err error) {
	var result []reflect.Value
	size = -1
	if reader != nil {
		// Verify if there is a method by name 'Size'.
		lenFn := reflect.ValueOf(reader).MethodByName("Size")
		if lenFn.IsValid() {
			if lenFn.Kind() == reflect.Func {
				// Call the 'Size' function and save its return value.
				result = lenFn.Call([]reflect.Value{})
				if len(result) == 1 {
					lenValue := result[0]
					if lenValue.IsValid() {
						switch lenValue.Kind() {
						case reflect.Int:
							fallthrough
						case reflect.Int8:
							fallthrough
						case reflect.Int16:
							fallthrough
						case reflect.Int32:
							fallthrough
						case reflect.Int64:
							size = lenValue.Int()
						}
					}
				}
			}
		} else {
			// Fallback to Stat() method, two possible Stat() structs
			// exist.
			switch v := reader.(type) {
			case *os.File:
				var st os.FileInfo
				st, err = v.Stat()
				if err != nil {
					// Handle this case specially for "windows",
					// certain files for example 'Stdin', 'Stdout' and
					// 'Stderr' it is not allowed to fetch file information.
					if runtime.GOOS == "windows" {
						if strings.Contains(err.Error(), "GetFileInformationByHandle") {
							return -1, nil
						}
					}
					return
				}
				// Ignore if input is a directory, throw an error.
				if st.Mode().IsDir() {
					return -1, ErrInvalidArgument("Input file cannot be a directory.")
				}
				// Ignore 'Stdin', 'Stdout' and 'Stderr', since they
				// represent *os.File type but internally do not
				// implement Seekable calls. Ignore them and treat
				// them like a stream with unknown length.
				switch st.Name() {
				case "stdin":
					fallthrough
				case "stdout":
					fallthrough
				case "stderr":
					return
				}
				size = st.Size()
			case *Object:
				var st ObjectInfo
				st, err = v.Stat()
				if err != nil {
					return
				}
				size = st.Size
			}
		}
	}
	// Returns the size here.
	return size, err
}

// completedParts is a collection of parts sortable by their part numbers.
// used for sorting the uploaded parts before completing the multipart request.
type completedParts []completePart

func (a completedParts) Len() int           { return len(a) }
func (a completedParts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }

// PutObject creates an object in a bucket.
//
// You must have WRITE permissions on a bucket to create an object.
//
//  - For size smaller than 5MiB PutObject automatically does a single atomic Put operation.
//  - For size larger than 5MiB PutObject automatically does a resumable multipart Put operation.
//  - For size input as -1 PutObject does a multipart Put operation until input stream reaches EOF.
//    Maximum object size that can be uploaded through this operation will be 5TiB.
//
// NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT.
// So we fall back to single PUT operation with the maximum limit of 5GiB.
//
// NOTE: For anonymous requests Amazon S3 doesn't allow multipart upload. So we fall back to single PUT operation.
func (c Client) PutObject(bucketName, objectName string, reader io.Reader, contentType string) (n int64, err error) {
	return c.PutObjectWithProgress(bucketName, objectName, reader, contentType, nil)
}

// putObjectNoChecksum special function used Google Cloud Storage. This special function
// is used for Google Cloud Storage since Google's multipart API is not S3 compatible.
func (c Client) putObjectNoChecksum(bucketName, objectName string, reader io.Reader, size int64, contentType string, progress io.Reader) (n int64, err error) {
	// Input validation.
	if err := isValidBucketName(bucketName); err != nil {
		return 0, err
	}
	if err := isValidObjectName(objectName); err != nil {
		return 0, err
	}
	if size > maxSinglePutObjectSize {
		return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
	}

	// Update progress reader appropriately to the latest offset as we
	// read from the source.
	readSeeker := newHook(reader, progress)

	// This function does not calculate sha256 and md5sum for payload.
	// Execute put object.
	st, err := c.putObjectDo(bucketName, objectName, readSeeker, nil, nil, size, contentType)
	if err != nil {
		return 0, err
	}
	if st.Size != size {
		return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
	}
	return size, nil
}

// putObjectSingle is a special function for uploading single put object request.
// This special function is used as a fallback when multipart upload fails.
func (c Client) putObjectSingle(bucketName, objectName string, reader io.Reader, size int64, contentType string, progress io.Reader) (n int64, err error) {
	// Input validation.
	if err := isValidBucketName(bucketName); err != nil {
		return 0, err
	}
	if err := isValidObjectName(objectName); err != nil {
		return 0, err
	}
	if size > maxSinglePutObjectSize {
		return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
	}
	// If size is a stream, upload up to 5GiB.
	if size <= -1 {
		size = maxSinglePutObjectSize
	}
	var md5Sum, sha256Sum []byte
	if size <= minPartSize {
		// Initialize a new temporary buffer.
		tmpBuffer := new(bytes.Buffer)
		md5Sum, sha256Sum, size, err = c.hashCopyN(tmpBuffer, reader, size)
		reader = bytes.NewReader(tmpBuffer.Bytes())
		tmpBuffer.Reset()
	} else {
		// Initialize a new temporary file.
		var tmpFile *tempFile
		tmpFile, err = newTempFile("single$-putobject-single")
		if err != nil {
			return 0, err
		}
		defer tmpFile.Close()
		md5Sum, sha256Sum, size, err = c.hashCopyN(tmpFile, reader, size)
		// Seek back to beginning of the temporary file.
		if _, err = tmpFile.Seek(0, 0); err != nil {
			return 0, err
		}
		reader = tmpFile
	}
	// Return error if its not io.EOF.
	if err != nil {
		if err != io.EOF {
			return 0, err
		}
	}
	// Execute put object.
	st, err := c.putObjectDo(bucketName, objectName, reader, md5Sum, sha256Sum, size, contentType)
	if err != nil {
		return 0, err
	}
	if st.Size != size {
		return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
	}
	// Progress the reader to the size if putObjectDo is successful.
	if progress != nil {
		if _, err = io.CopyN(ioutil.Discard, progress, size); err != nil {
			return size, err
		}
	}
	return size, nil
}

// putObjectDo - executes the put object http operation.
// NOTE: You must have WRITE permissions on a bucket to add an object to it.
func (c Client) putObjectDo(bucketName, objectName string, reader io.Reader, md5Sum []byte, sha256Sum []byte, size int64, contentType string) (ObjectInfo, error) {
	// Input validation.
	if err := isValidBucketName(bucketName); err != nil {
		return ObjectInfo{}, err
	}
	if err := isValidObjectName(objectName); err != nil {
		return ObjectInfo{}, err
	}

	if size <= -1 {
		return ObjectInfo{}, ErrEntityTooSmall(size, bucketName, objectName)
	}

	if size > maxSinglePutObjectSize {
		return ObjectInfo{}, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
	}

	if strings.TrimSpace(contentType) == "" {
		contentType = "application/octet-stream"
	}

	// Set headers.
	customHeader := make(http.Header)
	customHeader.Set("Content-Type", contentType)

	// Populate request metadata.
	reqMetadata := requestMetadata{
		bucketName:         bucketName,
		objectName:         objectName,
		customHeader:       customHeader,
		contentBody:        reader,
		contentLength:      size,
		contentMD5Bytes:    md5Sum,
		contentSHA256Bytes: sha256Sum,
	}

	// Execute PUT an objectName.
	resp, err := c.executeMethod("PUT", reqMetadata)
	defer closeResponse(resp)
	if err != nil {
		return ObjectInfo{}, err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName)
		}
	}

	var metadata ObjectInfo
	// Trim off the odd double quotes from ETag in the beginning and end.
	metadata.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
	metadata.ETag = strings.TrimSuffix(metadata.ETag, "\"")
	// A success here means data was written to server successfully.
	metadata.Size = size

	// Return here.
	return metadata, nil
}
<commit_msg>api: getReaderSize - Supports Len() function<commit_after>/*
 * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"bytes"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"reflect"
	"runtime"
	"strings"
)

// toInt - converts go value to its integer representation based
// on the value kind if it is an integer.
func toInt(value reflect.Value) (size int64) {
	size = -1
	if value.IsValid() {
		switch value.Kind() {
		case reflect.Int:
			fallthrough
		case reflect.Int8:
			fallthrough
		case reflect.Int16:
			fallthrough
		case reflect.Int32:
			fallthrough
		case reflect.Int64:
			size = value.Int()
		}
	}
	return size
}

// getReaderSize - Determine the size of Reader if available.
func getReaderSize(reader io.Reader) (size int64, err error) {
	size = -1
	if reader == nil {
		return -1, nil
	}
	// Verify if there is a method by name 'Size'.
	sizeFn := reflect.ValueOf(reader).MethodByName("Size")
	// Verify if there is a method by name 'Len'.
	lenFn := reflect.ValueOf(reader).MethodByName("Len")
	if sizeFn.IsValid() {
		if sizeFn.Kind() == reflect.Func {
			// Call the 'Size' function and save its return value.
			result := sizeFn.Call([]reflect.Value{})
			if len(result) == 1 {
				size = toInt(result[0])
			}
		}
	} else if lenFn.IsValid() {
		if lenFn.Kind() == reflect.Func {
			// Call the 'Len' function and save its return value.
			result := lenFn.Call([]reflect.Value{})
			if len(result) == 1 {
				size = toInt(result[0])
			}
		}
	} else {
		// Fallback to Stat() method, two possible Stat() structs exist.
		switch v := reader.(type) {
		case *os.File:
			var st os.FileInfo
			st, err = v.Stat()
			if err != nil {
				// Handle this case specially for "windows",
				// certain files for example 'Stdin', 'Stdout' and
				// 'Stderr' it is not allowed to fetch file information.
				if runtime.GOOS == "windows" {
					if strings.Contains(err.Error(), "GetFileInformationByHandle") {
						return -1, nil
					}
				}
				return
			}
			// Ignore if input is a directory, throw an error.
			if st.Mode().IsDir() {
				return -1, ErrInvalidArgument("Input file cannot be a directory.")
			}
			// Ignore 'Stdin', 'Stdout' and 'Stderr', since they
			// represent *os.File type but internally do not
			// implement Seekable calls. Ignore them and treat
			// them like a stream with unknown length.
			switch st.Name() {
			case "stdin":
				fallthrough
			case "stdout":
				fallthrough
			case "stderr":
				return
			}
			size = st.Size()
		case *Object:
			var st ObjectInfo
			st, err = v.Stat()
			if err != nil {
				return
			}
			size = st.Size
		}
	}
	// Returns the size here.
	return size, err
}

// completedParts is a collection of parts sortable by their part numbers.
// used for sorting the uploaded parts before completing the multipart request.
type completedParts []completePart

func (a completedParts) Len() int           { return len(a) }
func (a completedParts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }

// PutObject creates an object in a bucket.
//
// You must have WRITE permissions on a bucket to create an object.
//
//  - For size smaller than 5MiB PutObject automatically does a single atomic Put operation.
//  - For size larger than 5MiB PutObject automatically does a resumable multipart Put operation.
//  - For size input as -1 PutObject does a multipart Put operation until input stream reaches EOF.
//    Maximum object size that can be uploaded through this operation will be 5TiB.
//
// NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT.
// So we fall back to single PUT operation with the maximum limit of 5GiB.
//
// NOTE: For anonymous requests Amazon S3 doesn't allow multipart upload. So we fall back to single PUT operation.
func (c Client) PutObject(bucketName, objectName string, reader io.Reader, contentType string) (n int64, err error) {
	return c.PutObjectWithProgress(bucketName, objectName, reader, contentType, nil)
}

// putObjectNoChecksum special function used Google Cloud Storage. This special function
// is used for Google Cloud Storage since Google's multipart API is not S3 compatible.
func (c Client) putObjectNoChecksum(bucketName, objectName string, reader io.Reader, size int64, contentType string, progress io.Reader) (n int64, err error) {
	// Input validation.
	if err := isValidBucketName(bucketName); err != nil {
		return 0, err
	}
	if err := isValidObjectName(objectName); err != nil {
		return 0, err
	}
	if size > maxSinglePutObjectSize {
		return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
	}

	// Update progress reader appropriately to the latest offset as we
	// read from the source.
	readSeeker := newHook(reader, progress)

	// This function does not calculate sha256 and md5sum for payload.
	// Execute put object.
	st, err := c.putObjectDo(bucketName, objectName, readSeeker, nil, nil, size, contentType)
	if err != nil {
		return 0, err
	}
	if st.Size != size {
		return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
	}
	return size, nil
}

// putObjectSingle is a special function for uploading single put object request.
// This special function is used as a fallback when multipart upload fails.
func (c Client) putObjectSingle(bucketName, objectName string, reader io.Reader, size int64, contentType string, progress io.Reader) (n int64, err error) {
	// Input validation.
	if err := isValidBucketName(bucketName); err != nil {
		return 0, err
	}
	if err := isValidObjectName(objectName); err != nil {
		return 0, err
	}
	if size > maxSinglePutObjectSize {
		return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
	}
	// If size is a stream, upload up to 5GiB.
	if size <= -1 {
		size = maxSinglePutObjectSize
	}
	var md5Sum, sha256Sum []byte
	if size <= minPartSize {
		// Initialize a new temporary buffer.
		tmpBuffer := new(bytes.Buffer)
		md5Sum, sha256Sum, size, err = c.hashCopyN(tmpBuffer, reader, size)
		reader = bytes.NewReader(tmpBuffer.Bytes())
		tmpBuffer.Reset()
	} else {
		// Initialize a new temporary file.
		var tmpFile *tempFile
		tmpFile, err = newTempFile("single$-putobject-single")
		if err != nil {
			return 0, err
		}
		defer tmpFile.Close()
		md5Sum, sha256Sum, size, err = c.hashCopyN(tmpFile, reader, size)
		// Seek back to beginning of the temporary file.
		if _, err = tmpFile.Seek(0, 0); err != nil {
			return 0, err
		}
		reader = tmpFile
	}
	// Return error if its not io.EOF.
	if err != nil {
		if err != io.EOF {
			return 0, err
		}
	}
	// Execute put object.
	st, err := c.putObjectDo(bucketName, objectName, reader, md5Sum, sha256Sum, size, contentType)
	if err != nil {
		return 0, err
	}
	if st.Size != size {
		return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName)
	}
	// Progress the reader to the size if putObjectDo is successful.
	if progress != nil {
		if _, err = io.CopyN(ioutil.Discard, progress, size); err != nil {
			return size, err
		}
	}
	return size, nil
}

// putObjectDo - executes the put object http operation.
// NOTE: You must have WRITE permissions on a bucket to add an object to it.
func (c Client) putObjectDo(bucketName, objectName string, reader io.Reader, md5Sum []byte, sha256Sum []byte, size int64, contentType string) (ObjectInfo, error) {
	// Input validation.
	if err := isValidBucketName(bucketName); err != nil {
		return ObjectInfo{}, err
	}
	if err := isValidObjectName(objectName); err != nil {
		return ObjectInfo{}, err
	}

	if size <= -1 {
		return ObjectInfo{}, ErrEntityTooSmall(size, bucketName, objectName)
	}

	if size > maxSinglePutObjectSize {
		return ObjectInfo{}, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
	}

	if strings.TrimSpace(contentType) == "" {
		contentType = "application/octet-stream"
	}

	// Set headers.
	customHeader := make(http.Header)
	customHeader.Set("Content-Type", contentType)

	// Populate request metadata.
	reqMetadata := requestMetadata{
		bucketName:         bucketName,
		objectName:         objectName,
		customHeader:       customHeader,
		contentBody:        reader,
		contentLength:      size,
		contentMD5Bytes:    md5Sum,
		contentSHA256Bytes: sha256Sum,
	}

	// Execute PUT an objectName.
	resp, err := c.executeMethod("PUT", reqMetadata)
	defer closeResponse(resp)
	if err != nil {
		return ObjectInfo{}, err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName)
		}
	}

	var metadata ObjectInfo
	// Trim off the odd double quotes from ETag in the beginning and end.
	metadata.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
	metadata.ETag = strings.TrimSuffix(metadata.ETag, "\"")
	// A success here means data was written to server successfully.
	metadata.Size = size

	// Return here.
	return metadata, nil
}
<|endoftext|>


<commit_before>package instructions

import (
    . "jvmgo/any"
    "jvmgo/rtda"
    "jvmgo/rtda/class"
)

// Store into reference array
type aastore struct {NoOperandsInstruction}
func (self *aastore) Execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    refArr := arrRef.Fields().([]*class.Obj)
    checkArrIndex(index, len(refArr))
    // todo
    ref := val.(*class.Obj)
    refArr[index] = ref
}

// Store into byte or boolean array
type bastore struct {NoOperandsInstruction}
func (self *bastore) Execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    byteArr := arrRef.Fields().([]int8)
    checkArrIndex(index, len(byteArr))
    byteArr[index] = int8(val.(int32))
}

// Store into char array
type castore struct {NoOperandsInstruction}
func (self *castore) Execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    charArr := arrRef.Fields().([]uint16)
    checkArrIndex(index, len(charArr))
    charArr[index] = uint16(val.(int32))
}

// Store into double array
type dastore struct {NoOperandsInstruction}
func (self *dastore) Execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    doubleArr := arrRef.Fields().([]float64)
    checkArrIndex(index, len(doubleArr))
    doubleArr[index] = val.(float64)
}

// Store into float array
type fastore struct {NoOperandsInstruction}
func (self *fastore) Execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    floatArr := arrRef.Fields().([]float32)
    checkArrIndex(index, len(floatArr))
    floatArr[index] = val.(float32)
}

// Store into int array
type iastore struct {NoOperandsInstruction}
func (self *iastore) Execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    intArr := arrRef.Fields().([]int32)
    checkArrIndex(index, len(intArr))
    intArr[index] = val.(int32)
}

// Store into long array
type lastore struct {NoOperandsInstruction}
func (self *lastore) Execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    longArr := arrRef.Fields().([]int64)
    checkArrIndex(index, len(longArr))
    longArr[index] = val.(int64)
}

// Store into short array
type sastore struct {NoOperandsInstruction}
func (self *sastore) Execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    shortArr := arrRef.Fields().([]int16)
    checkArrIndex(index, len(shortArr))
    shortArr[index] = int16(val.(int32))
}

func popOperands(thread *rtda.Thread) (*class.Obj, int, Any) {
    stack := thread.CurrentFrame().OperandStack()
    val := stack.Pop()
    index := int(stack.PopInt())
    arrRef := stack.PopRef()
    if arrRef == nil {
        // todo
        panic("NullPointerException")
    }
    return arrRef, index, val
}
<commit_msg>refactor xastore<commit_after>package instructions

import (
    . "jvmgo/any"
    "jvmgo/rtda"
    "jvmgo/rtda/class"
)

// Store into reference array
type aastore struct {NoOperandsInstruction}
func (self *aastore) Execute(frame *rtda.Frame) {
    arrRef, index, val := popOperands(frame)
    refArr := arrRef.Fields().([]*class.Obj)
    checkArrIndex(index, len(refArr))
    // todo
    ref := val.(*class.Obj)
    refArr[index] = ref
}

// Store into byte or boolean array
type bastore struct {NoOperandsInstruction}
func (self *bastore) Execute(frame *rtda.Frame) {
    arrRef, index, val := popOperands(frame)
    byteArr := arrRef.Fields().([]int8)
    checkArrIndex(index, len(byteArr))
    byteArr[index] = int8(val.(int32))
}

// Store into char array
type castore struct {NoOperandsInstruction}
func (self *castore) Execute(frame *rtda.Frame) {
    arrRef, index, val := popOperands(frame)
    charArr := arrRef.Fields().([]uint16)
    checkArrIndex(index, len(charArr))
    charArr[index] = uint16(val.(int32))
}

// Store into double array
type dastore struct {NoOperandsInstruction}
func (self *dastore) Execute(frame *rtda.Frame) {
    arrRef, index, val := popOperands(frame)
    doubleArr := arrRef.Fields().([]float64)
    checkArrIndex(index, len(doubleArr))
    doubleArr[index] = val.(float64)
}

// Store into float array
type fastore struct {NoOperandsInstruction}
func (self *fastore) Execute(frame *rtda.Frame) {
    arrRef, index, val := popOperands(frame)
    floatArr := arrRef.Fields().([]float32)
    checkArrIndex(index, len(floatArr))
    floatArr[index] = val.(float32)
}

// Store into int array
type iastore struct {NoOperandsInstruction}
func (self *iastore) Execute(frame *rtda.Frame) {
    arrRef, index, val := popOperands(frame)
    intArr := arrRef.Fields().([]int32)
    checkArrIndex(index, len(intArr))
    intArr[index] = val.(int32)
}

// Store into long array
type lastore struct {NoOperandsInstruction}
func (self *lastore) Execute(frame *rtda.Frame) {
    arrRef, index, val := popOperands(frame)
    longArr := arrRef.Fields().([]int64)
    checkArrIndex(index, len(longArr))
    longArr[index] = val.(int64)
}

// Store into short array
type sastore struct {NoOperandsInstruction}
func (self *sastore) Execute(frame *rtda.Frame) {
    arrRef, index, val := popOperands(frame)
    shortArr := arrRef.Fields().([]int16)
    checkArrIndex(index, len(shortArr))
    shortArr[index] = int16(val.(int32))
}

func popOperands(frame *rtda.Frame) (*class.Obj, int, Any) {
    stack := frame.OperandStack()
    val := stack.Pop()
    index := int(stack.PopInt())
    arrRef := stack.PopRef()
    if arrRef == nil {
        // todo
        panic("NullPointerException")
    }
    return arrRef, index, val
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"flag"
	"fmt"
	"io"
	"math"
	"os"
	"sync/atomic"
	"time"

	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse/fsutil"
	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

var fTempDir = flag.String(
	"gcsproxy.temp_dir", "",
	"The temporary directory in which to store local copies of GCS objects. "+
		"If empty, the system default (probably /tmp) will be used.")

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization around the methods where it is not otherwise noted.
type MutableObject struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket
	clock  timeutil.Clock

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A record for the specific generation of the object from which our local
	// state is branched. If we have no local state, the contents of this
	// generation are exactly our contents.
	src gcs.Object

	// The current generation number. Must be accessed using sync/atomic.
	//
	// INVARIANT: atomic.LoadInt64(&sourceGeneration) == src.Generation
	sourceGeneration int64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// 'src' above.
	localFile *os.File

	// The time at which a method that modifies our contents was last called, or
	// nil if never.
	mtime *time.Time

	// true if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If dirty, then localFile != nil
	// INVARIANT: If dirty, then mtime != nil
	dirty bool
}

type StatResult struct {
	// The current size in bytes of the content, including any local
	// modifications that have not been Sync'd.
	Size int64

	// The time at which the contents were last updated, or the creation time of
	// the source object if they never have been.
	Mtime time.Time

	// Has the object changed out from under us in GCS? If so, Sync will fail.
	Clobbered bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation.
//
// REQUIRES: o != nil
func NewMutableObject(
	clock timeutil.Clock,
	bucket gcs.Bucket,
	o *gcs.Object) (mo *MutableObject) {
	// Set up the basic struct.
	mo = &MutableObject{
		clock:            clock,
		bucket:           bucket,
		src:              *o,
		sourceGeneration: o.Generation,
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket, depending on whether the backing object has
// been deleted.
//
// May be called concurrently with any method.
func (mo *MutableObject) Name() string {
	return mo.src.Name
}

// Return the generation of the object from which the current contents of this
// proxy were branched. If Sync has been successfully called, this is the
// generation most recently returned by Sync. Otherwise it is the generation
// from which the proxy was created.
//
// May be called concurrently with any method, but note that without excluding
// concurrent calls to Sync this may change spontaneously.
func (mo *MutableObject) SourceGeneration() int64 {
	return atomic.LoadInt64(&mo.sourceGeneration)
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (mo *MutableObject) CheckInvariants() {
	// INVARIANT: atomic.LoadInt64(&sourceGeneration) == src.Generation
	{
		g := atomic.LoadInt64(&mo.sourceGeneration)
		if g != mo.src.Generation {
			panic(fmt.Sprintf("Generation mismatch: %v vs. %v", g, mo.src.Generation))
		}
	}

	// INVARIANT: If dirty, then localFile != nil
	if mo.dirty && mo.localFile == nil {
		panic("Expected non-nil localFile.")
	}

	// INVARIANT: If dirty, then mtime != nil
	if mo.dirty && mo.mtime == nil {
		panic("Expected non-nil mtime.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (mo *MutableObject) Destroy() (err error) {
	// Make sure that when we exit no invariants are violated.
	defer func() {
		mo.localFile = nil
		mo.dirty = false
	}()

	// If we have no local file, there's nothing to do.
	if mo.localFile == nil {
		return
	}

	// Close the local file.
	if err = mo.localFile.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
//
// sr.Clobbered will be set only if needClobbered is true. Otherwise a round
// trip to GCS can be saved.
func (mo *MutableObject) Stat(
	ctx context.Context,
	needClobbered bool) (sr StatResult, err error) {
	// If we have ever been modified, our mtime field is authoritative (even if
	// we've been Sync'd, because Sync is not supposed to affect the mtime).
	// Otherwise our source object's creation time is our mtime.
	if mo.mtime != nil {
		sr.Mtime = *mo.mtime
	} else {
		sr.Mtime = mo.src.Updated
	}

	// If we have a file, it is authoritative for our size. Otherwise our source
	// size is authoritative.
	if mo.localFile != nil {
		var fi os.FileInfo
		if fi, err = mo.localFile.Stat(); err != nil {
			err = fmt.Errorf("Stat: %v", err)
			return
		}

		sr.Size = fi.Size()
	} else {
		sr.Size = int64(mo.src.Size)
	}

	// Figure out whether we were clobbered iff the user asked us to.
	if needClobbered {
		sr.Clobbered, err = mo.clobbered(ctx)
		if err != nil {
			err = fmt.Errorf("clobbered: %v", err)
			return
		}
	}

	return
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (mo *MutableObject) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = mo.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = mo.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (mo *MutableObject) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = mo.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	newMtime := mo.clock.Now()

	mo.dirty = true
	mo.mtime = &newMtime
	n, err = mo.localFile.WriteAt(buf, offset)

	return
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (mo *MutableObject) Truncate(ctx context.Context, n int64) (err error) {
	// Make sure we have a local file.
	if err = mo.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Convert to signed, which is what os.File wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	newMtime := mo.clock.Now()

	mo.dirty = true
	mo.mtime = &newMtime
	err = mo.localFile.Truncate(int64(n))

	return
}

// If the proxy is dirty due to having been modified, save its current contents
// to GCS, creating a generation with exactly those contents. Do so with a
// precondition such that the creation will fail if the source generation is
// not current. In that case, return an error of type *gcs.PreconditionError.
// If the proxy is not dirty, simply return nil.
//
// After this method successfully returns, SourceGeneration returns the
// generation at which the contents are current.
func (mo *MutableObject) Sync(ctx context.Context) (err error) {
	// Do we need to do anything?
	if !mo.dirty {
		return
	}

	// Seek the file to the start so that it can be used as a reader for its full
	// contents below.
	_, err = mo.localFile.Seek(0, 0)
	if err != nil {
		err = fmt.Errorf("Seek: %v", err)
		return
	}

	// Write a new generation of the object with the appropriate contents, using
	// an appropriate precondition.
	req := &gcs.CreateObjectRequest{
		Name:                   mo.src.Name,
		Contents:               mo.localFile,
		GenerationPrecondition: &mo.src.Generation,
	}

	o, err := mo.bucket.CreateObject(ctx, req)

	// Special case: handle precondition errors.
	if _, ok := err.(*gcs.PreconditionError); ok {
		err = &gcs.PreconditionError{
			Err: fmt.Errorf("CreateObject: %v", err),
		}

		return
	}

	// Propagate other errors more directly.
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	// Update our state.
	mo.src = *o
	mo.dirty = false
	atomic.StoreInt64(&mo.sourceGeneration, mo.src.Generation)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up an unlinked local temporary file for the given generation of the
// given object.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation int64) (f *os.File, err error) {
	// Create the file.
	f, err = fsutil.AnonymousFile(*fTempDir)
	if err != nil {
		err = fmt.Errorf("AnonymousFile: %v", err)
		return
	}

	// Ensure that we clean up the file if we return in error from this method.
	defer func() {
		if err != nil {
			f.Close()
			f = nil
		}
	}()

	// Open the object for reading.
	req := &gcs.ReadObjectRequest{
		Name:       name,
		Generation: generation,
	}

	var rc io.ReadCloser
	if rc, err = bucket.NewReader(ctx, req); err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	// Copy to the file.
	if _, err = io.Copy(f, rc); err != nil {
		err = fmt.Errorf("Copy: %v", err)
		return
	}

	// Close.
	if err = rc.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Ensure that mo.localFile is non-nil with an authoritative view of mo's
// contents.
func (mo *MutableObject) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if mo.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, mo.bucket, mo.Name(), mo.src.Generation)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	mo.localFile = f
	return
}

func (mo *MutableObject) clobbered(
	ctx context.Context) (clobbered bool, err error) {
	// Stat the object in GCS.
	req := &gcs.StatObjectRequest{Name: mo.Name()}
	o, err := mo.bucket.StatObject(ctx, req)

	// Special case: "not found" means we have been clobbered.
	if _, ok := err.(*gcs.NotFoundError); ok {
		err = nil
		clobbered = true
		return
	}

	// Propagate other errors.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	// We are clobbered iff the generation doesn't match our source generation.
	clobbered = (o.Generation != mo.src.Generation)

	return
}
<commit_msg>Added a concurrency TODO.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"flag"
	"fmt"
	"io"
	"math"
	"os"
	"sync/atomic"
	"time"

	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse/fsutil"
	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

var fTempDir = flag.String(
	"gcsproxy.temp_dir", "",
	"The temporary directory in which to store local copies of GCS objects. "+
		"If empty, the system default (probably /tmp) will be used.")

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization around the methods where it is not otherwise noted.
type MutableObject struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket
	clock  timeutil.Clock

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A record for the specific generation of the object from which our local
	// state is branched. If we have no local state, the contents of this
	// generation are exactly our contents.
	src gcs.Object

	// The current generation number. Must be accessed using sync/atomic.
	//
	// INVARIANT: atomic.LoadInt64(&sourceGeneration) == src.Generation
	sourceGeneration int64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// 'src' above.
	localFile *os.File

	// The time at which a method that modifies our contents was last called, or
	// nil if never.
	mtime *time.Time

	// true if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If dirty, then localFile != nil
	// INVARIANT: If dirty, then mtime != nil
	dirty bool
}

type StatResult struct {
	// The current size in bytes of the content, including any local
	// modifications that have not been Sync'd.
	Size int64

	// The time at which the contents were last updated, or the creation time of
	// the source object if they never have been.
	Mtime time.Time

	// Has the object changed out from under us in GCS? If so, Sync will fail.
	Clobbered bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation.
//
// REQUIRES: o != nil
func NewMutableObject(
	clock timeutil.Clock,
	bucket gcs.Bucket,
	o *gcs.Object) (mo *MutableObject) {
	// Set up the basic struct.
	mo = &MutableObject{
		clock:            clock,
		bucket:           bucket,
		src:              *o,
		sourceGeneration: o.Generation,
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket, depending on whether the backing object has
// been deleted.
//
// May be called concurrently with any method.
//
// TODO(jacobsa): I think there is a race on reading src.Name with the write in
// Sync. Do we actually need this guarantee?
func (mo *MutableObject) Name() string {
	return mo.src.Name
}

// Return the generation of the object from which the current contents of this
// proxy were branched. If Sync has been successfully called, this is the
// generation most recently returned by Sync. Otherwise it is the generation
// from which the proxy was created.
//
// May be called concurrently with any method, but note that without excluding
// concurrent calls to Sync this may change spontaneously.
func (mo *MutableObject) SourceGeneration() int64 {
	return atomic.LoadInt64(&mo.sourceGeneration)
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (mo *MutableObject) CheckInvariants() {
	// INVARIANT: atomic.LoadInt64(&sourceGeneration) == src.Generation
	{
		g := atomic.LoadInt64(&mo.sourceGeneration)
		if g != mo.src.Generation {
			panic(fmt.Sprintf("Generation mismatch: %v vs. %v", g, mo.src.Generation))
		}
	}

	// INVARIANT: If dirty, then localFile != nil
	if mo.dirty && mo.localFile == nil {
		panic("Expected non-nil localFile.")
	}

	// INVARIANT: If dirty, then mtime != nil
	if mo.dirty && mo.mtime == nil {
		panic("Expected non-nil mtime.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (mo *MutableObject) Destroy() (err error) {
	// Make sure that when we exit no invariants are violated.
	defer func() {
		mo.localFile = nil
		mo.dirty = false
	}()

	// If we have no local file, there's nothing to do.
	if mo.localFile == nil {
		return
	}

	// Close the local file.
	if err = mo.localFile.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
//
// sr.Clobbered will be set only if needClobbered is true. Otherwise a round
// trip to GCS can be saved.
func (mo *MutableObject) Stat(
	ctx context.Context,
	needClobbered bool) (sr StatResult, err error) {
	// If we have ever been modified, our mtime field is authoritative (even if
	// we've been Sync'd, because Sync is not supposed to affect the mtime).
	// Otherwise our source object's creation time is our mtime.
	if mo.mtime != nil {
		sr.Mtime = *mo.mtime
	} else {
		sr.Mtime = mo.src.Updated
	}

	// If we have a file, it is authoritative for our size. Otherwise our source
	// size is authoritative.
	if mo.localFile != nil {
		var fi os.FileInfo
		if fi, err = mo.localFile.Stat(); err != nil {
			err = fmt.Errorf("Stat: %v", err)
			return
		}

		sr.Size = fi.Size()
	} else {
		sr.Size = int64(mo.src.Size)
	}

	// Figure out whether we were clobbered iff the user asked us to.
	if needClobbered {
		sr.Clobbered, err = mo.clobbered(ctx)
		if err != nil {
			err = fmt.Errorf("clobbered: %v", err)
			return
		}
	}

	return
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (mo *MutableObject) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = mo.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = mo.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (mo *MutableObject) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = mo.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	newMtime := mo.clock.Now()

	mo.dirty = true
	mo.mtime = &newMtime
	n, err = mo.localFile.WriteAt(buf, offset)

	return
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (mo *MutableObject) Truncate(ctx context.Context, n int64) (err error) {
	// Make sure we have a local file.
	if err = mo.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Convert to signed, which is what os.File wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	newMtime := mo.clock.Now()

	mo.dirty = true
	mo.mtime = &newMtime
	err = mo.localFile.Truncate(int64(n))

	return
}

// If the proxy is dirty due to having been modified, save its current contents
// to GCS, creating a generation with exactly those contents. Do so with a
// precondition such that the creation will fail if the source generation is
// not current. In that case, return an error of type *gcs.PreconditionError.
// If the proxy is not dirty, simply return nil.
//
// After this method successfully returns, SourceGeneration returns the
// generation at which the contents are current.
func (mo *MutableObject) Sync(ctx context.Context) (err error) {
	// Do we need to do anything?
	if !mo.dirty {
		return
	}

	// Seek the file to the start so that it can be used as a reader for its full
	// contents below.
	_, err = mo.localFile.Seek(0, 0)
	if err != nil {
		err = fmt.Errorf("Seek: %v", err)
		return
	}

	// Write a new generation of the object with the appropriate contents, using
	// an appropriate precondition.
	req := &gcs.CreateObjectRequest{
		Name:                   mo.src.Name,
		Contents:               mo.localFile,
		GenerationPrecondition: &mo.src.Generation,
	}

	o, err := mo.bucket.CreateObject(ctx, req)

	// Special case: handle precondition errors.
	if _, ok := err.(*gcs.PreconditionError); ok {
		err = &gcs.PreconditionError{
			Err: fmt.Errorf("CreateObject: %v", err),
		}

		return
	}

	// Propagate other errors more directly.
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	// Update our state.
	mo.src = *o
	mo.dirty = false
	atomic.StoreInt64(&mo.sourceGeneration, mo.src.Generation)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up an unlinked local temporary file for the given generation of the
// given object.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation int64) (f *os.File, err error) {
	// Create the file.
	f, err = fsutil.AnonymousFile(*fTempDir)
	if err != nil {
		err = fmt.Errorf("AnonymousFile: %v", err)
		return
	}

	// Ensure that we clean up the file if we return in error from this method.
	defer func() {
		if err != nil {
			f.Close()
			f = nil
		}
	}()

	// Open the object for reading.
	req := &gcs.ReadObjectRequest{
		Name:       name,
		Generation: generation,
	}

	var rc io.ReadCloser
	if rc, err = bucket.NewReader(ctx, req); err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	// Copy to the file.
	if _, err = io.Copy(f, rc); err != nil {
		err = fmt.Errorf("Copy: %v", err)
		return
	}

	// Close.
	if err = rc.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Ensure that mo.localFile is non-nil with an authoritative view of mo's
// contents.
func (mo *MutableObject) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if mo.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, mo.bucket, mo.Name(), mo.src.Generation)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	mo.localFile = f
	return
}

func (mo *MutableObject) clobbered(
	ctx context.Context) (clobbered bool, err error) {
	// Stat the object in GCS.
	req := &gcs.StatObjectRequest{Name: mo.Name()}
	o, err := mo.bucket.StatObject(ctx, req)

	// Special case: "not found" means we have been clobbered.
	if _, ok := err.(*gcs.NotFoundError); ok {
		err = nil
		clobbered = true
		return
	}

	// Propagate other errors.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	// We are clobbered iff the generation doesn't match our source generation.
	clobbered = (o.Generation != mo.src.Generation)

	return
}
<|endoftext|>


<commit_before>package hackernews

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Client struct{}
type Story struct {
	Title string `json:"title"`
	Url   string `json:"url"`
}
type storyRequest struct {
	id       int
	response *Story
	err      error
}

const (
	urlBase    = "https://hacker-news.firebaseio.com/v0"
	TopStories = iota
	NewStories
)

var (
	urlTopStories = fmt.Sprintf("%s/topstories.json", urlBase)
	urlNewStories = fmt.Sprintf("%s/newstories.json", urlBase)
	urlStoryBase  = fmt.Sprintf("%s/item", urlBase)
)

func getStoryUrl(id int) string {
	return fmt.Sprintf("%s/%d.json", urlStoryBase, id)
}

func getStoryIds(url string) ([]int, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	ids := []int{}
	if err = json.Unmarshal(body, &ids); err != nil {
		return nil, err
	}
	return ids, nil
}

func getStory(id int) (*Story, error) {
	url := getStoryUrl(id)

	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var story Story

	if err = json.Unmarshal(body, &story); err != nil {
		return nil, err
	}
	return &story, nil
}

func getUrl(storyType int) string {
	switch storyType {
	case TopStories:
		return urlTopStories
	default:
		return urlNewStories
	}
}

func (c *Client) GetStories(storyType, limit int) ([]*Story, error) {
	stories := []*Story{}
	url := getUrl(storyType)
	ids, err := getStoryIds(url)
	targetIds := ids[:limit]
	if err != nil {
		return nil, err
	}

	storyResponses := make(chan *storyRequest, limit)
	for _, id := range targetIds {
		go func(id int) {
			story, err := getStory(id)
			storyResponses <- &storyRequest{id, story, err}
		}(id)
	}

	storiesMap := make(map[int]*Story)
	for i := 0; i < limit; i++ {
		request := <-storyResponses
		if request.err != nil {
			return nil, err
		}
		storiesMap[request.id] = request.response
	}
	close(storyResponses)

	// To keep order
	for _, id := range targetIds {
		stories = append(stories, storiesMap[id])
	}
	return stories, nil
}

func New() *Client {
	return &Client{}
}
<commit_msg>apply Generator and Future concurrency pattern to get stories from hackernews<commit_after>package hackernews

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Client struct{}
type Story struct {
	Title string `json:"title"`
	Url   string `json:"url"`
}
type storyRequest struct {
	id       int
	response *Story
	err      error
}

const (
	urlBase    = "https://hacker-news.firebaseio.com/v0"
	TopStories = iota
	NewStories
)

var (
	urlTopStories = fmt.Sprintf("%s/topstories.json", urlBase)
	urlNewStories = fmt.Sprintf("%s/newstories.json", urlBase)
	urlStoryBase  = fmt.Sprintf("%s/item", urlBase)
)

func getStoryUrl(id int) string {
	return fmt.Sprintf("%s/%d.json", urlStoryBase, id)
}

func getStoryIds(url string) ([]int, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	ids := []int{}
	if err = json.Unmarshal(body, &ids); err != nil {
		return nil, err
	}
	return ids, nil
}

func getStory(id int) (*Story, error) {
	url := getStoryUrl(id)

	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var story Story

	if err = json.Unmarshal(body, &story); err != nil {
		return nil, err
	}
	return &story, nil
}

func getUrl(storyType int) string {
	switch storyType {
	case TopStories:
		return urlTopStories
	default:
		return urlNewStories
	}
}

func storiesGenerator(targetIds []int) chan chan *storyRequest {
	generator := make(chan chan *storyRequest, len(targetIds))

	go func() {
		for _, id := range targetIds {
			generator <- func(id int) chan *storyRequest {
				future := make(chan *storyRequest, 1)
				go func() {
					story, err := getStory(id)
					future <- &storyRequest{id, story, err}
					close(future)
				}()
				return future
			}(id)
		}
		close(generator)
	}()

	return generator
}

func (c *Client) GetStories(storyType, limit int) ([]*Story, error) {
	stories := []*Story{}
	url := getUrl(storyType)
	ids, err := getStoryIds(url)
	if err != nil {
		return nil, err
	}
	targetIds := ids[:limit]

	storiesMap := make(map[int]*Story)
	for future := range storiesGenerator(targetIds) {
		request := <-future
		if request.err != nil {
			return nil, err
		}
		storiesMap[request.id] = request.response
	}

	// To keep order
	for _, id := range targetIds {
		stories = append(stories, storiesMap[id])
	}
	return stories, nil
}

func New() *Client {
	return &Client{}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"reflect"
	"runtime"
)

type RebalanceConfig struct {
	AllowLeaderRebalancing    bool
	MinReplicasForRebalancing int
	MinUnbalance              float64

	Brokers []BrokerID
}

func DefaultRebalanceConfig() RebalanceConfig {
	return RebalanceConfig{
		AllowLeaderRebalancing:    false,
		MinReplicasForRebalancing: 2,
		MinUnbalance:              0.00001,
	}
}

var steps = []func(*PartitionList, RebalanceConfig) (*PartitionList, error){
	ValidateWeights,
	ValidateReplicas,
	FillDefaults,
	RemoveExtraReplicas,
	AddMissingReplicas,
	MoveDisallowedReplicas,
	MoveNonLeaders,
	MoveLeaders,
}

func emptypl() *PartitionList {
	return &PartitionList{Version: 1}
}

func singlepl(p Partition) *PartitionList {
	return &PartitionList{Version: 1, Partitions: []Partition{p}}
}

func replacepl(p Partition, orig BrokerID, repl BrokerID) *PartitionList {
	for idx, id := range p.Replicas {
		if id == orig {
			if repl == -1 {
				p.Replicas = append(p.Replicas[:idx], p.Replicas[idx+1:]...)
			} else {
				p.Replicas[idx] = repl
			}
			return singlepl(p)
		}
	}
	panic(fmt.Sprintf("partition %v replicas don't contain %d", p, orig))
}

func addpl(p Partition, b BrokerID) *PartitionList {
	p.Replicas = append(p.Replicas, b)
	return singlepl(p)
}

// Balance analyzes the workload distribution among brokers for the
// partitions listed in the argument. It returns a PartitionList with 0 or more
// partition reassignments.
func Balance(pl *PartitionList, cfg RebalanceConfig) (*PartitionList, error) {
	log.Printf("rebalance config: %+v", cfg)

	for _, step := range steps {
		stepName := runtime.FuncForPC(reflect.ValueOf(step).Pointer()).Name()
		ppl, err := step(pl, cfg)
		if err != nil {
			return nil, fmt.Errorf("%s: %s", stepName, err)
		}
		if ppl != nil {
			log.Printf("%s: %v", stepName, ppl)
			return ppl, nil
		}
	}

	log.Print("no candidate changes")
	return emptypl(), nil
}

// ValidateWeights make sure that either all partitions have an explicit,
// strictly positive weight or that all partitions have no weight
func ValidateWeights(pl *PartitionList, _ RebalanceConfig) (*PartitionList, error) {
	hasWeights := pl.Partitions[0].Weight != 0

	for _, p := range pl.Partitions {
		if hasWeights && p.Weight == 0 {
			return nil, fmt.Errorf("partition %v has no weight", p)
		}
		if !hasWeights && p.Weight != 0 {
			return nil, fmt.Errorf("partition %v has weight", p)
		}
		if p.Weight < 0 {
			return nil, fmt.Errorf("partition %v has negative weight", p)
		}
	}

	return nil, nil
}

// ValidateReplicas checks that partitions don't have more than one replica per
// broker
func ValidateReplicas(pl *PartitionList, _ RebalanceConfig) (*PartitionList, error) {
	for _, p := range pl.Partitions {
		replicaset := toBrokerSet(p.Replicas)
		if len(replicaset) != len(p.Replicas) {
			return nil, fmt.Errorf("partition %v has duplicated replicas", p)
		}
	}

	return nil, nil
}

// FillDefaults fills in default values for Weight, Brokers and NumReplicas
func FillDefaults(pl *PartitionList, cfg RebalanceConfig) (*PartitionList, error) {
	// if the weights are 0, set them to 1
	if pl.Partitions[0].Weight == 0 {
		for idx := range pl.Partitions {
			pl.Partitions[idx].Weight = 1.0
		}
	}

	// if the set of candidate brokers is empty, fill it with the default set
	brokers := cfg.Brokers
	if brokers == nil {
		brokers = getBrokerList(pl)
	}
	for idx := range pl.Partitions {
		if pl.Partitions[idx].Brokers == nil {
			pl.Partitions[idx].Brokers = brokers
		}
	}

	// if the desired number of replicas is 0, fill it with the current number
	for idx := range pl.Partitions {
		if pl.Partitions[idx].NumReplicas == 0 {
			pl.Partitions[idx].NumReplicas = len(pl.Partitions[idx].Replicas)
		}
	}

	return nil, nil
}

// RemoveExtraReplicas removes replicas from partitions having lower NumReplicas
// than the current number of replicas
func RemoveExtraReplicas(pl *PartitionList, _ RebalanceConfig) (*PartitionList, error) {
	loads := getBrokerLoad(pl)

	for _, p := range pl.Partitions {
		if p.NumReplicas >= len(p.Replicas) {
			continue
		}

		brokersByLoad := getBrokerListByLoad(loads, p.Brokers)
		replicaset := toBrokerSet(p.Replicas)
		for _, b := range brokersByLoad {
			if _, found := replicaset[b]; found {
				return replacepl(p, b, -1), nil
			}
		}

		return nil, fmt.Errorf("partition %v unable to pick replica to remove", p)
	}

	return nil, nil
}

// AddMissingReplicas adds replicas to partitions having NumReplicas greater
// than the current number of replicas
func AddMissingReplicas(pl *PartitionList, _ RebalanceConfig) (*PartitionList, error) {
	loads := getBrokerLoad(pl)
	// add missing replicas
	for _, p := range pl.Partitions {
		if p.NumReplicas <= len(p.Replicas) {
			continue
		}

		brokersByLoad := getBrokerListByLoad(loads, p.Brokers)
		replicaset := toBrokerSet(p.Replicas)
		for idx := len(brokersByLoad) - 1; idx >= 0; idx-- {
			b := brokersByLoad[idx]
			if _, found := replicaset[b]; !found {
				return addpl(p, b), nil
			}
		}

		return nil, fmt.Errorf("partition %v unable to pick replica to add", p)
	}

	return nil, nil
}

// MoveDisallowedReplicas moves replicas from non-allowed brokers to the least
// loaded ones
func MoveDisallowedReplicas(pl *PartitionList, _ RebalanceConfig) (*PartitionList, error) {
	loads := getBrokerLoad(pl)

	for _, p := range pl.Partitions {
		brokersByLoad := getBrokerListByLoad(loads, p.Brokers)
		brokersset := toBrokerSet(brokersByLoad)

		for _, id := range p.Replicas {
			if _, allowed := brokersset[id]; allowed {
				continue
			}

			replicaset := toBrokerSet(p.Replicas)

			for idx := len(brokersByLoad) - 1; idx >= 0; idx-- {
				b := brokersByLoad[idx]
				if _, alreadyInReplicas := replicaset[b]; alreadyInReplicas {
					continue
				}

				return replacepl(p, id, b), nil
			}

			return nil, fmt.Errorf("partition %v unable to pick replica to replace broker %d", p, id)
		}
	}

	return nil, nil
}

func move(pl *PartitionList, cfg RebalanceConfig, leaders bool) (*PartitionList, error) {
	var cp Partition
	var cr, cb BrokerID

	loads := getBrokerLoad(pl)
	for _, id := range cfg.Brokers {
		if _, found := loads[id]; !found {
			loads[id] = 0
		}
	}

	su := getUnbalance(loads)
	cu := su

	for _, p := range pl.Partitions {
		if p.NumReplicas < cfg.MinReplicasForRebalancing {
			continue
		}

		replicaset := toBrokerSet(p.Replicas)
		replicas := p.Replicas[1:]
		if leaders {
			replicas = p.Replicas[0:1]
		}

		for _, r := range replicas {
			rload := loads[r]
			loads[r] -= p.Weight

			for _, b := range p.Brokers {
				if _, alreadyInReplicas := replicaset[b]; alreadyInReplicas {
					continue
				}

				bload := loads[b]
				loads[b] += p.Weight
				u := getUnbalance(loads)
				if u < cu {
					cu, cp, cr, cb = u, p, r, b
				}

				loads[b] = bload
			}

			loads[r] = rload
		}
	}

	if cu < su-cfg.MinUnbalance {
		return replacepl(cp, cr, cb), nil
	}

	return nil, nil
}

// MoveNonLeaders moves non-leader replicas from overloaded brokers to
// underloaded brokers
func MoveNonLeaders(pl *PartitionList, cfg RebalanceConfig) (*PartitionList, error) {
	return move(pl, cfg, false)
}

// MoveLeaders moves leader replicas from overloaded brokers to underloaded
// brokers
func MoveLeaders(pl *PartitionList, cfg RebalanceConfig) (*PartitionList, error) {
	if !cfg.AllowLeaderRebalancing {
		return nil, nil
	}

	return move(pl, cfg, true)
}
<commit_msg>Add some comments<commit_after>package main

import (
	"fmt"
	"log"
	"reflect"
	"runtime"
)

// RebalanceConfig contains the configuration that drives the rebalancing.
type RebalanceConfig struct {
	AllowLeaderRebalancing    bool
	MinReplicasForRebalancing int
	MinUnbalance              float64

	Brokers []BrokerID
}

// DefaultRebalanceConfig returns the default RebalanceConfig. These values are
// also the one used as the default values for the CLI flags.
func DefaultRebalanceConfig() RebalanceConfig {
	return RebalanceConfig{
		AllowLeaderRebalancing:    false,
		MinReplicasForRebalancing: 2,
		MinUnbalance:              0.00001,
	}
}

var steps = []func(*PartitionList, RebalanceConfig) (*PartitionList, error){
	ValidateWeights,
	ValidateReplicas,
	FillDefaults,
	RemoveExtraReplicas,
	AddMissingReplicas,
	MoveDisallowedReplicas,
	MoveNonLeaders,
	MoveLeaders,
}

func emptypl() *PartitionList {
	return &PartitionList{Version: 1}
}

func singlepl(p Partition) *PartitionList {
	return &PartitionList{Version: 1, Partitions: []Partition{p}}
}

func replacepl(p Partition, orig BrokerID, repl BrokerID) *PartitionList {
	for idx, id := range p.Replicas {
		if id == orig {
			if repl == -1 {
				p.Replicas = append(p.Replicas[:idx], p.Replicas[idx+1:]...)
			} else {
				p.Replicas[idx] = repl
			}
			return singlepl(p)
		}
	}
	panic(fmt.Sprintf("partition %v replicas don't contain %d", p, orig))
}

func addpl(p Partition, b BrokerID) *PartitionList {
	p.Replicas = append(p.Replicas, b)
	return singlepl(p)
}

// Balance analyzes the workload distribution among brokers for the
// partitions listed in the argument. It returns a PartitionList with 0 or more
// partition reassignments.
func Balance(pl *PartitionList, cfg RebalanceConfig) (*PartitionList, error) {
	log.Printf("rebalance config: %+v", cfg)

	for _, step := range steps {
		stepName := runtime.FuncForPC(reflect.ValueOf(step).Pointer()).Name()
		ppl, err := step(pl, cfg)
		if err != nil {
			return nil, fmt.Errorf("%s: %s", stepName, err)
		}
		if ppl != nil {
			log.Printf("%s: %v", stepName, ppl)
			return ppl, nil
		}
	}

	log.Print("no candidate changes")
	return emptypl(), nil
}

// ValidateWeights make sure that either all partitions have an explicit,
// strictly positive weight or that all partitions have no weight
func ValidateWeights(pl *PartitionList, _ RebalanceConfig) (*PartitionList, error) {
	hasWeights := pl.Partitions[0].Weight != 0

	for _, p := range pl.Partitions {
		if hasWeights && p.Weight == 0 {
			return nil, fmt.Errorf("partition %v has no weight", p)
		}
		if !hasWeights && p.Weight != 0 {
			return nil, fmt.Errorf("partition %v has weight", p)
		}
		if p.Weight < 0 {
			return nil, fmt.Errorf("partition %v has negative weight", p)
		}
	}

	return nil, nil
}

// ValidateReplicas checks that partitions don't have more than one replica per
// broker
func ValidateReplicas(pl *PartitionList, _ RebalanceConfig) (*PartitionList, error) {
	for _, p := range pl.Partitions {
		replicaset := toBrokerSet(p.Replicas)
		if len(replicaset) != len(p.Replicas) {
			return nil, fmt.Errorf("partition %v has duplicated replicas", p)
		}
	}

	return nil, nil
}

// FillDefaults fills in default values for Weight, Brokers and NumReplicas
func FillDefaults(pl *PartitionList, cfg RebalanceConfig) (*PartitionList, error) {
	// if the weights are 0, set them to 1
	if pl.Partitions[0].Weight == 0 {
		for idx := range pl.Partitions {
			pl.Partitions[idx].Weight = 1.0
		}
	}

	// if the set of candidate brokers is empty, fill it with the default set
	brokers := cfg.Brokers
	if brokers == nil {
		brokers = getBrokerList(pl)
	}
	for idx := range pl.Partitions {
		if pl.Partitions[idx].Brokers == nil {
			pl.Partitions[idx].Brokers = brokers
		}
	}

	// if the desired number of replicas is 0, fill it with the current number
	for idx := range pl.Partitions {
		if pl.Partitions[idx].NumReplicas == 0 {
			pl.Partitions[idx].NumReplicas = len(pl.Partitions[idx].Replicas)
		}
	}

	return nil, nil
}

// RemoveExtraReplicas removes replicas from partitions having lower NumReplicas
// than the current number of replicas
func RemoveExtraReplicas(pl *PartitionList, _ RebalanceConfig) (*PartitionList, error) {
	loads := getBrokerLoad(pl)

	for _, p := range pl.Partitions {
		if p.NumReplicas >= len(p.Replicas) {
			continue
		}

		brokersByLoad := getBrokerListByLoad(loads, p.Brokers)
		replicaset := toBrokerSet(p.Replicas)
		for _, b := range brokersByLoad {
			if _, found := replicaset[b]; found {
				return replacepl(p, b, -1), nil
			}
		}

		return nil, fmt.Errorf("partition %v unable to pick replica to remove", p)
	}

	return nil, nil
}

// AddMissingReplicas adds replicas to partitions having NumReplicas greater
// than the current number of replicas
func AddMissingReplicas(pl *PartitionList, _ RebalanceConfig) (*PartitionList, error) {
	loads := getBrokerLoad(pl)
	// add missing replicas
	for _, p := range pl.Partitions {
		if p.NumReplicas <= len(p.Replicas) {
			continue
		}

		brokersByLoad := getBrokerListByLoad(loads, p.Brokers)
		replicaset := toBrokerSet(p.Replicas)
		for idx := len(brokersByLoad) - 1; idx >= 0; idx-- {
			b := brokersByLoad[idx]
			if _, found := replicaset[b]; !found {
				return addpl(p, b), nil
			}
		}

		return nil, fmt.Errorf("partition %v unable to pick replica to add", p)
	}

	return nil, nil
}

// MoveDisallowedReplicas moves replicas from non-allowed brokers to the least
// loaded ones
func MoveDisallowedReplicas(pl *PartitionList, _ RebalanceConfig) (*PartitionList, error) {
	loads := getBrokerLoad(pl)

	for _, p := range pl.Partitions {
		brokersByLoad := getBrokerListByLoad(loads, p.Brokers)
		brokersset := toBrokerSet(brokersByLoad)

		for _, id := range p.Replicas {
			if _, allowed := brokersset[id]; allowed {
				continue
			}

			replicaset := toBrokerSet(p.Replicas)

			for idx := len(brokersByLoad) - 1; idx >= 0; idx-- {
				b := brokersByLoad[idx]
				if _, alreadyInReplicas := replicaset[b]; alreadyInReplicas {
					continue
				}

				return replacepl(p, id, b), nil
			}

			return nil, fmt.Errorf("partition %v unable to pick replica to replace broker %d", p, id)
		}
	}

	return nil, nil
}

func move(pl *PartitionList, cfg RebalanceConfig, leaders bool) (*PartitionList, error) {
	var cp Partition
	var cr, cb BrokerID

	loads := getBrokerLoad(pl)
	for _, id := range cfg.Brokers {
		if _, found := loads[id]; !found {
			loads[id] = 0
		}
	}

	su := getUnbalance(loads)
	cu := su

	for _, p := range pl.Partitions {
		if p.NumReplicas < cfg.MinReplicasForRebalancing {
			continue
		}

		replicaset := toBrokerSet(p.Replicas)
		replicas := p.Replicas[1:]
		if leaders {
			replicas = p.Replicas[0:1]
		}

		for _, r := range replicas {
			rload := loads[r]
			loads[r] -= p.Weight

			for _, b := range p.Brokers {
				if _, alreadyInReplicas := replicaset[b]; alreadyInReplicas {
					continue
				}

				bload := loads[b]
				loads[b] += p.Weight
				u := getUnbalance(loads)
				if u < cu {
					cu, cp, cr, cb = u, p, r, b
				}

				loads[b] = bload
			}

			loads[r] = rload
		}
	}

	if cu < su-cfg.MinUnbalance {
		return replacepl(cp, cr, cb), nil
	}

	return nil, nil
}

// MoveNonLeaders moves non-leader replicas from overloaded brokers to
// underloaded brokers
func MoveNonLeaders(pl *PartitionList, cfg RebalanceConfig) (*PartitionList, error) {
	return move(pl, cfg, false)
}

// MoveLeaders moves leader replicas from overloaded brokers to underloaded
// brokers
func MoveLeaders(pl *PartitionList, cfg RebalanceConfig) (*PartitionList, error) {
	if !cfg.AllowLeaderRebalancing {
		return nil, nil
	}

	return move(pl, cfg, true)
}
<|endoftext|>


<commit_before>// Copyright 2015-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// bbramfs builds a simple initramfs given an existing built bb; see bb.go
// You have to run bb first, which creates cmds/bb/bbsh. cd to that directory,
// and run bbramfs, and you have a single binary which does all u-root commands.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/u-root/u-root/pkg/cpio"
	_ "github.com/u-root/u-root/pkg/cpio/newc"
	"github.com/u-root/u-root/pkg/ldd"
)

var (
	// Paths contains the paths to put into the initramfs.
	// The index is a root directory, and the value is the place from which to
	// walk. The only required root is the bbsh dir itself, and the starting
	// walk is init -- i.e. we grab only one file. Should you wish to bring in,
	// e.g., /lib/modules/4.04, you would do add the root as / and the
	// starting point for the walk as lib/modules/4.04. That way we only preserve
	// as much of the path as we need, but we can preserve it all.
	paths      = map[string][]string{}
	extraPaths = flag.String("extra", "", `Extra paths to add in the form root:start, e.g. /:etc/hosts.
The path before the : is used as a starting point for a walk; the path after the : selects what things to put
into the initramfs starting at /. E.g., /tmp/prototype:/ will install the prototype file system into / of the initramfs`)
	extraCmds = flag.String("cmds", "", "Extra commands to add (full path, comma-separated string)")
)

func sanity() {
	goBinGo := filepath.Join(config.Goroot, "bin/go")
	_, err := os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	// but does the one in go/bin/OS_ARCH exist too?
	goBinGo = filepath.Join(config.Goroot, fmt.Sprintf("bin/%s_%s/go", config.Goos, config.Arch))
	_, err = os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	if config.Go == "" {
		log.Fatalf("Can't find a go binary! Is GOROOT set correctly?")
	}
}

// dirComponents takes a string and returns an array of strings,
// such that we can create the directory records for intermediate
// directories.
func dirComponents(dir string) []string {
	var dirlist []string
	if filepath.Dir(dir) == "." {
		return []string{}
	}
	for d := filepath.Dir(dir); d != "/"; d = filepath.Dir(d) {
		dirlist = append([]string{d}, dirlist...)
	}
	dirlist = append(dirlist, dir)
	return dirlist
}

// copyCommands takes a list of commands, generates the list of libs,
// and creates cpio records, including directory records.
func copyCommands(w cpio.Writer, cmds []string) {
	debug("copyCommands: start with %v", cmds)
	var recs []cpio.Record
	libs, err := uroot.LddList(cmds)
	if err != nil {
		log.Fatalf("%v\n", err)
	}
	cmds = append(cmds, libs...)
	for _, n := range cmds {
		debug("copyCommands: file %v", n)
		for _, n := range dirComponents(n) {
			debug("copyCommands: %v", n)
			r, err := cpio.GetRecord(n)
			if err != nil {
				log.Fatalf("%v: %v", n, err)
			}
			recs = append(recs, r)
		}
	}
	cpio.MakeReproducible(recs)
	if err := w.WriteRecords(recs); err != nil {
		log.Fatalf("%v\n", err)
	}
}

func ramfs() {
	archiver, err := cpio.Format("newc")
	if err != nil {
		log.Fatalf("Creating newc archiver: %v", err)
	}

	oname := fmt.Sprintf("/tmp/initramfs.%v_%v.cpio", config.Goos, config.Arch)
	f, err := os.Create(oname)
	if err != nil {
		log.Fatalf("%v\n", err)
	}

	w := archiver.Writer(f)
	cpio.MakeReproducible(devCPIO[:])
	if err := w.WriteRecords(devCPIO[:]); err != nil {
		log.Fatalf("%v\n", err)
	}

	paths[filepath.Join(config.Gopath, "src/github.com/u-root/u-root/bb/bbsh")] = []string{"init", "ubin"}

	if *extraPaths != "" {
		extras := strings.Split(*extraPaths, " ")
		for _, x := range extras {
			p := strings.Split(x, ":")
			if len(p) != 2 {
				p = append([]string{"/"}, p...)
			}
			paths[p[0]] = append(paths[p[0]], p[1])
		}
	}

	if *extraCmds != "" {
		copyCommands(w, strings.Split(*extraCmds, " "))
	}

	// For all the 'roots' in paths, start walking at the name.
	debug("PATHS: %v", paths)
	for r, list := range paths {
		debug("PATHS: root %v", r)
		// we need to make all the path prefix directories.
		for _, n := range list {
			debug("\troot %v, name %v", r, n)
			for _, d := range dirComponents(n) {
				debug("\t\troot %v, name %v, component %v", r, n, d)
				rec, err := cpio.GetRecord(d)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", d, err)
				}
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v\n", err)
				}
			}
		}
		for _, n := range list {
			if err := filepath.Walk(filepath.Join(r, n), func(name string, fi os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				cn, err := filepath.Rel(r, name)
				if err != nil {
					log.Fatalf("filepath.Rel(%v, %v): %v", r, name, err)
				}
				debug("%v\n", cn)
				rec, err := cpio.GetRecord(name)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", cn, err)
				}
				// the name in the cpio is relative to our starting point.
				rec.Name = cn
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v\n", err)
				}
				return nil
			}); err != nil {
				log.Fatalf("bbsh walk failed: %v", err)
			}
		}
	}

	if err := w.WriteTrailer(); err != nil {
		log.Fatalf("Error writing trailer record: %v", err)
	}
	fmt.Printf("Output file is in %v\n", oname)
}
<commit_msg>bb: allow additional cpio archives<commit_after>// Copyright 2015-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// bbramfs builds a simple initramfs given an existing built bb; see bb.go
// You have to run bb first, which creates cmds/bb/bbsh. cd to that directory,
// and run bbramfs, and you have a single binary which does all u-root commands.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/u-root/u-root/pkg/cpio"
	_ "github.com/u-root/u-root/pkg/cpio/newc"
	"github.com/u-root/u-root/pkg/ldd"
)

var (
	// Paths contains the paths to put into the initramfs.
	// The index is a root directory, and the value is the place from which to
	// walk. The only required root is the bbsh dir itself, and the starting
	// walk is init -- i.e. we grab only one file. Should you wish to bring in,
	// e.g., /lib/modules/4.04, you would do add the root as / and the
	// starting point for the walk as lib/modules/4.04. That way we only preserve
	// as much of the path as we need, but we can preserve it all.
	paths      = map[string][]string{}
	extraPaths = flag.String("extra", "", `Extra paths to add in the form root:start, e.g. /:etc/hosts.
The path before the : is used as a starting point for a walk; the path after the : selects what things to put
into the initramfs starting at /. E.g., /tmp/prototype:/ will install the prototype file system into / of the initramfs`)
	extraCmds = flag.String("cmds", "", "Extra commands to add (full path, comma-separated string)")
	extraCpio = flag.String("cpio", "", "A list of cpio archives to include in the output")
)

func sanity() {
	goBinGo := filepath.Join(config.Goroot, "bin/go")
	_, err := os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	// but does the one in go/bin/OS_ARCH exist too?
	goBinGo = filepath.Join(config.Goroot, fmt.Sprintf("bin/%s_%s/go", config.Goos, config.Arch))
	_, err = os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	if config.Go == "" {
		log.Fatalf("Can't find a go binary! Is GOROOT set correctly?")
	}
}

// dirComponents takes a string and returns an array of strings,
// such that we can create the directory records for intermediate
// directories.
func dirComponents(dir string) []string {
	var dirlist []string
	if filepath.Dir(dir) == "." {
		return []string{}
	}
	for d := filepath.Dir(dir); d != "/"; d = filepath.Dir(d) {
		dirlist = append([]string{d}, dirlist...)
	}
	dirlist = append(dirlist, dir)
	return dirlist
}

// copyCommands takes a list of commands, generates the list of libs,
// and creates cpio records, including directory records.
func copyCommands(w cpio.Writer, cmds []string) {
	debug("copyCommands: start with %v", cmds)
	var recs []cpio.Record
	libs, err := uroot.LddList(cmds)
	if err != nil {
		log.Fatalf("%v\n", err)
	}
	cmds = append(cmds, libs...)
	for _, n := range cmds {
		debug("copyCommands: file %v", n)
		for _, n := range dirComponents(n) {
			debug("copyCommands: %v", n)
			r, err := cpio.GetRecord(n)
			if err != nil {
				log.Fatalf("%v: %v", n, err)
			}
			recs = append(recs, r)
		}
	}
	cpio.MakeReproducible(recs)
	if err := w.WriteRecords(recs); err != nil {
		log.Fatalf("%v\n", err)
	}
}

func ramfs() {
	archiver, err := cpio.Format("newc")
	if err != nil {
		log.Fatalf("Creating newc archiver: %v", err)
	}

	oname := fmt.Sprintf("/tmp/initramfs.%v_%v.cpio", config.Goos, config.Arch)
	f, err := os.Create(oname)
	if err != nil {
		log.Fatalf("%v\n", err)
	}

	w := archiver.Writer(f)
	cpio.MakeReproducible(devCPIO[:])
	if err := w.WriteRecords(devCPIO[:]); err != nil {
		log.Fatalf("%v\n", err)
	}

	paths[filepath.Join(config.Gopath, "src/github.com/u-root/u-root/bb/bbsh")] = []string{"init", "ubin"}

	if *extraPaths != "" {
		extras := strings.Split(*extraPaths, " ")
		for _, x := range extras {
			p := strings.Split(x, ":")
			if len(p) != 2 {
				p = append([]string{"/"}, p...)
			}
			paths[p[0]] = append(paths[p[0]], p[1])
		}
	}

	if *extraCmds != "" {
		copyCommands(w, strings.Split(*extraCmds, " "))
	}

	if *extraCpio != "" {
		extras := strings.Split(*extraCpio, " ")
		for _, x := range extras {
			a, err := cpio.Format("newc")
			if err != nil {
				log.Fatalf("Creating archiver: %v", err)
			}
			f, err := os.Open(x)
			if err != nil {
				log.Fatalf("%v: %v", x, err)
			}
			defer f.Close()
			rr := a.Reader(f)
			recs, err := rr.ReadRecords()
			if err != nil {
				log.Fatalf("read records: %v", err)
			}
			cpio.MakeReproducible(recs)
			if err := w.WriteRecords(recs); err != nil {
				log.Fatalf("%v\n", err)
			}
		}
	}

	// For all the 'roots' in paths, start walking at the name.
	debug("PATHS: %v", paths)
	for r, list := range paths {
		debug("PATHS: root %v", r)
		// we need to make all the path prefix directories.
		for _, n := range list {
			debug("\troot %v, name %v", r, n)
			for _, d := range dirComponents(n) {
				debug("\t\troot %v, name %v, component %v", r, n, d)
				rec, err := cpio.GetRecord(d)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", d, err)
				}
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v\n", err)
				}
			}
		}
		for _, n := range list {
			if err := filepath.Walk(filepath.Join(r, n), func(name string, fi os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				cn, err := filepath.Rel(r, name)
				if err != nil {
					log.Fatalf("filepath.Rel(%v, %v): %v", r, name, err)
				}
				debug("%v\n", cn)
				rec, err := cpio.GetRecord(name)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", cn, err)
				}
				// the name in the cpio is relative to our starting point.
				rec.Name = cn
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v\n", err)
				}
				return nil
			}); err != nil {
				log.Fatalf("bbsh walk failed: %v", err)
			}
		}
	}

	if err := w.WriteTrailer(); err != nil {
		log.Fatalf("Error writing trailer record: %v", err)
	}
	fmt.Printf("Output file is in %v\n", oname)
}
<|endoftext|>


<commit_before>package farmer

import (
	"github.com/fsouza/go-dockerclient"
	"os"
)

var dockerClient *docker.Client

func init() {
	dockerClient, _ = docker.NewClient(os.Getenv("FARMER_DOCKER_API"))
}

func dockerCreateContainer(box *Box) error {
	container, err := dockerClient.CreateContainer(
		dockerCreateContainerOptions(box),
	)

	if err != nil {
		return err
	}

	box.ContainerID = container.ID
	dockerInspectContainer(box)

	return err
}

func dockerInspectContainer(box *Box) error {
	container, err := dockerClient.InspectContainer(box.ContainerID)

	if err != nil {
		return err
	}

	box.Hostname = container.Config.Hostname
	box.CgroupParent = container.HostConfig.CgroupParent
	box.Image = container.Config.Image
	box.IP = container.NetworkSettings.IPAddress

	box.Ports = dockerExtractPortBindings(container.NetworkSettings.Ports)
	box.Status = dockerTranslateContainerState(container.State)

	return nil
}

func dockerStartContainer(box *Box) error {
	err := dockerClient.StartContainer(box.ContainerID, dockerCreateContainerOptions(box).HostConfig)

	if err != nil {
		return err
	}

	return dockerInspectContainer(box)
}

func dockerRunContainer(box *Box) error {
	if err := dockerCreateContainer(box); err != nil {
		return err
	}

	return dockerStartContainer(box)
}

func dockerExecOnContainer(box *Box, commands []string) error {
	exec, err := dockerClient.CreateExec(docker.CreateExecOptions{
		Container:    box.ContainerID,
		AttachStdin:  false,
		AttachStdout: true,
		AttachStderr: true,
		Tty:          false,
		Cmd:          commands,
	})

	if err != nil {
		return err
	}

	return dockerClient.StartExec(exec.ID, docker.StartExecOptions{
		Detach:       false,
		Tty:          false,
		OutputStream: box.OutputStream,
		ErrorStream:  box.ErrorStream,
	})
}

func dockerDeleteContainer(box *Box) error {
	return dockerClient.RemoveContainer(docker.RemoveContainerOptions{
		ID:            box.ContainerID,
		RemoveVolumes: false,
		Force:         true,
	})
}

func dockerRestartContainer(box *Box) error {
	return dockerClient.RestartContainer(box.ContainerID, 1)
}

func dockerCreateContainerOptions(box *Box) docker.CreateContainerOptions {
	dockerConfig := &docker.Config{
		Hostname:     box.Name,
		Image:        box.Image,
		ExposedPorts: dockerReversePortBindings(box.Ports),
		Env: 		  box.Env,
	}

	dockerHostConfig := &docker.HostConfig{
		Binds:           []string{box.CodeDirectory + ":/app"},
		CgroupParent:    box.CgroupParent,
		PublishAllPorts: true,
	}

	return docker.CreateContainerOptions{
		Name:       box.Name,
		Config:     dockerConfig,
		HostConfig: dockerHostConfig,
	}
}

func dockerReversePortBindings(ports []string) map[docker.Port]struct{} {
	portBindings := make(map[docker.Port]struct{})

	for _, portAndProtocol := range ports {
		var pp docker.Port
		pp = docker.Port(portAndProtocol)
		portBindings[pp] = struct{}{}
	}

	return portBindings
}

func dockerExtractPortBindings(ports map[docker.Port][]docker.PortBinding) []string {
	var portBindings []string
	for port, _ := range ports {
		portBindings = append(portBindings, string(port))
	}

	return portBindings
}

func dockerTranslateContainerState(state docker.State) string {
	switch {
	case state.Running:
		return "running"

	case state.Paused:
		return "paused"

	case state.Restarting:
		return "restarting"

	default:
		return "created"
	}
}
<commit_msg>feature(docker): Auto pull image when it's not available localy<commit_after>package farmer

import (
	"github.com/fsouza/go-dockerclient"
	"os"
	"strings"
)

var dockerClient *docker.Client

func init() {
	dockerClient, _ = docker.NewClient(os.Getenv("FARMER_DOCKER_API"))
}

func dockerCreateContainer(box *Box) error {
	container, err := dockerClient.CreateContainer(
		dockerCreateContainerOptions(box),
	)

	if err == docker.ErrNoSuchImage {
		if err := dockerPullImage(box); err != nil {
			return err
		}
		return dockerCreateContainer(box)

	} else if err != nil {
		return err
	}

	box.ContainerID = container.ID
	dockerInspectContainer(box)

	return err
}

func dockerInspectContainer(box *Box) error {
	container, err := dockerClient.InspectContainer(box.ContainerID)

	if err != nil {
		return err
	}

	box.Hostname = container.Config.Hostname
	box.CgroupParent = container.HostConfig.CgroupParent
	box.Image = container.Config.Image
	box.IP = container.NetworkSettings.IPAddress

	box.Ports = dockerExtractPortBindings(container.NetworkSettings.Ports)
	box.Status = dockerTranslateContainerState(container.State)

	return nil
}

func dockerStartContainer(box *Box) error {
	err := dockerClient.StartContainer(box.ContainerID, dockerCreateContainerOptions(box).HostConfig)

	if err != nil {
		return err
	}

	return dockerInspectContainer(box)
}

func dockerRunContainer(box *Box) error {
	if err := dockerCreateContainer(box); err != nil {
		return err
	}

	return dockerStartContainer(box)
}

func dockerExecOnContainer(box *Box, commands []string) error {
	exec, err := dockerClient.CreateExec(docker.CreateExecOptions{
		Container:    box.ContainerID,
		AttachStdin:  false,
		AttachStdout: true,
		AttachStderr: true,
		Tty:          false,
		Cmd:          commands,
	})

	if err != nil {
		return err
	}

	return dockerClient.StartExec(exec.ID, docker.StartExecOptions{
		Detach:       false,
		Tty:          false,
		OutputStream: box.OutputStream,
		ErrorStream:  box.ErrorStream,
	})
}

func dockerDeleteContainer(box *Box) error {
	return dockerClient.RemoveContainer(docker.RemoveContainerOptions{
		ID:            box.ContainerID,
		RemoveVolumes: false,
		Force:         true,
	})
}

func dockerRestartContainer(box *Box) error {
	return dockerClient.RestartContainer(box.ContainerID, 1)
}

func dockerCreateContainerOptions(box *Box) docker.CreateContainerOptions {
	dockerConfig := &docker.Config{
		Hostname:     box.Name,
		Image:        box.Image,
		ExposedPorts: dockerReversePortBindings(box.Ports),
		Env:          box.Env,
	}

	dockerHostConfig := &docker.HostConfig{
		Binds:           []string{box.CodeDirectory + ":/app"},
		CgroupParent:    box.CgroupParent,
		PublishAllPorts: true,
	}

	return docker.CreateContainerOptions{
		Name:       box.Name,
		Config:     dockerConfig,
		HostConfig: dockerHostConfig,
	}
}

func dockerPullImage(box *Box) error {
	repository := strings.Split(box.Image, ":")[0]
	tag := "latest"
	if s := strings.Split(box.Image, ":"); len(s) > 1 {
		tag = s[1]
	}

	return dockerClient.PullImage(docker.PullImageOptions{
		OutputStream: box.OutputStream,
		Repository:   repository,
		Tag:          tag,
	}, docker.AuthConfiguration{})
}

func dockerReversePortBindings(ports []string) map[docker.Port]struct{} {
	portBindings := make(map[docker.Port]struct{})

	for _, portAndProtocol := range ports {
		var pp docker.Port
		pp = docker.Port(portAndProtocol)
		portBindings[pp] = struct{}{}
	}

	return portBindings
}

func dockerExtractPortBindings(ports map[docker.Port][]docker.PortBinding) []string {
	var portBindings []string
	for port, _ := range ports {
		portBindings = append(portBindings, string(port))
	}

	return portBindings
}

func dockerTranslateContainerState(state docker.State) string {
	switch {
	case state.Running:
		return "running"

	case state.Paused:
		return "paused"

	case state.Restarting:
		return "restarting"

	default:
		return "created"
	}
}
<|endoftext|>