<commit_before>/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package sdk

import (
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth"
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials"
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/endpoints"
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors"
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses"
	"net"
	"net/http"
)

// this value will be replaced while build: -ldflags="-X sdk.version=x.x.x"
var Version = "0.0.1"

type Client struct {
	regionId       string
	config         *Config
	signer         auth.Signer
	httpClient     *http.Client
	asyncTaskQueue chan func()

	debug     bool
	isRunning bool
}

func (client *Client) Init() (err error) {
	panic("not support yet")
}

func (client *Client) InitWithOptions(regionId string, config *Config, credential auth.Credential) (err error) {
	client.isRunning = true
	client.regionId = regionId
	client.config = config
	if err != nil {
		return
	}
	client.httpClient = &http.Client{}

	if config.HttpTransport != nil {
		client.httpClient.Transport = config.HttpTransport
	}

	if config.Timeout > 0 {
		client.httpClient.Timeout = config.Timeout
	}

	if config.EnableAsync {
		client.EnableAsync(config.GoRoutinePoolSize, config.MaxTaskQueueSize)
	}

	client.signer, err = auth.NewSignerWithCredential(credential, client.ProcessCommonRequestWithSigner)

	return
}

func (client *Client) EnableAsync(routinePoolSize, maxTaskQueueSize int) {
	client.asyncTaskQueue = make(chan func(), maxTaskQueueSize)
	for i := 0; i < routinePoolSize; i++ {
		go func() {
			for client.isRunning {
				select {
				case task := <-client.asyncTaskQueue:
					task()
				}
			}
		}()
	}
}

func (client *Client) InitWithAccessKey(regionId, accessKeyId, accessKeySecret string) (err error) {
	config := client.InitClientConfig()
	credential := &credentials.BaseCredential{
		AccessKeyId:     accessKeyId,
		AccessKeySecret: accessKeySecret,
	}
	return client.InitWithOptions(regionId, config, credential)
}

func (client *Client) InitWithRoleArn(regionId, accessKeyId, accessKeySecret, roleArn, roleSessionName string) (err error) {
	config := client.InitClientConfig()
	credential := &credentials.StsAssumeRoleCredential{
		AccessKeyId:     accessKeyId,
		AccessKeySecret: accessKeySecret,
		RoleArn:         roleArn,
		RoleSessionName: roleSessionName,
	}
	return client.InitWithOptions(regionId, config, credential)
}

func (client *Client) InitWithKeyPair(regionId, publicKeyId, privateKey string, sessionExpiration int) (err error) {
	config := client.InitClientConfig()
	credential := &credentials.KeyPairCredential{
		PrivateKey:        privateKey,
		PublicKeyId:       publicKeyId,
		SessionExpiration: sessionExpiration,
	}
	return client.InitWithOptions(regionId, config, credential)
}

func (client *Client) InitWithEcsInstance(regionId, roleName string) (err error) {
	config := client.InitClientConfig()
	credential := &credentials.EcsInstanceCredential{
		RoleName: roleName,
	}
	return client.InitWithOptions(regionId, config, credential)
}

func (client *Client) InitClientConfig() (config *Config) {
	config = NewConfig()
	if client.config != nil {
		config = client.config
	}

	return
}

func (client *Client) DoAction(request requests.AcsRequest, response responses.AcsResponse) (err error) {
	return client.DoActionWithSigner(request, response, nil)
}

func (client *Client) DoActionWithSigner(request requests.AcsRequest, response responses.AcsResponse, signer auth.Signer) (err error) {

	// add clientVersion
	request.GetHeaders()["x-sdk-core-version"] = Version

	regionId := client.regionId
	if len(request.GetRegionId()) > 0 {
		regionId = request.GetRegionId()
	}

	// resolve endpoint
	resolveParam := &endpoints.ResolveParam{
		Domain:           request.GetDomain(),
		Product:          request.GetProduct(),
		RegionId:         client.regionId,
		LocationProduct:  request.GetLocationServiceCode(),
		LocationEndpoint: request.GetLocationEndpointType(),
		CommonApi:        client.ProcessCommonRequest,
	}
	endpoint, err := endpoints.Resolve(resolveParam)
	if err != nil {
		return
	}
	request.SetDomain(endpoint)

	// init request params
	err = requests.InitParams(request)
	if err != nil {
		return
	}

	// signature
	if signer != nil {
		err = auth.Sign(request, signer, regionId)
	} else {
		err = auth.Sign(request, client.signer, regionId)
	}

	if err != nil {
		return
	}

	requestMethod := request.GetMethod()
	requestUrl := request.GetUrl()
	body := request.GetBodyReader()
	httpRequest, err := http.NewRequest(requestMethod, requestUrl, body)
	if err != nil {
		return
	}
	for key, value := range request.GetHeaders() {
		httpRequest.Header[key] = []string{value}
	}
	var httpResponse *http.Response
	for retryTimes := 0; retryTimes <= client.config.MaxRetryTime; retryTimes++ {
		httpResponse, err = client.httpClient.Do(httpRequest)

		// retry params
		var timeout bool
		var serverError bool

		// receive error
		if err != nil {
			// if not timeout error, return
			if timeout = isTimeout(err); !timeout {
				return
			}
		}
		serverError = isServerError(httpResponse)
		//  if status code >= 500 or timeout, will trigger retry
		if client.config.AutoRetry && (timeout || serverError) {
			continue
		}
		break
	}
	err = responses.Unmarshal(response, httpResponse, request.GetAcceptFormat())
	return
}

func isTimeout(err error) bool {
	if err == nil {
		return false
	}
	netErr, isNetError := err.(net.Error)
	return isNetError && netErr.Timeout()
}

func isServerError(httpResponse *http.Response) bool {
	return httpResponse.StatusCode >= http.StatusInternalServerError
}

func (client *Client) AddAsyncTask(task func()) (err error) {
	if client.asyncTaskQueue != nil {
		client.asyncTaskQueue <- task
	} else {
		err = errors.NewClientError(errors.AsyncFunctionNotEnabledCode, errors.AsyncFunctionNotEnabledMessage, nil)
	}
	return
}

func NewClient() (client *Client, err error) {
	client = &Client{}
	err = client.Init()
	return
}

func NewClientWithOptions(regionId string, config *Config, credential auth.Credential) (client *Client, err error) {
	client = &Client{}
	err = client.InitWithOptions(regionId, config, credential)
	return
}

func NewClientWithAccessKey(regionId, accessKeyId, accessKeySecret string) (client *Client, err error) {
	client = &Client{}
	err = client.InitWithAccessKey(regionId, accessKeyId, accessKeySecret)
	return
}

func NewClientWithKeyPair(regionId string, config *Config, publicKeyId, privateKey string, sessionExpiration int) (client *Client, err error) {
	client = &Client{}
	client.config = config
	err = client.InitWithKeyPair(regionId, publicKeyId, privateKey, sessionExpiration)
	return
}

func NewClientWithEcsInstance(regionId string, config *Config, roleName string) (client *Client, err error) {
	client = &Client{}
	client.config = config
	err = client.InitWithEcsInstance(regionId, roleName)
	return
}

func NewClientWithRoleArn(regionId string, config *Config, accessKeyId, accessKeySecret, roleArn, roleSessionName string) (client *Client, err error) {
	client = &Client{}
	client.config = config
	err = client.InitWithRoleArn(regionId, accessKeyId, accessKeySecret, roleArn, roleSessionName)
	return
}

func (client *Client) ProcessCommonRequest(request *requests.CommonRequest) (response *responses.CommonResponse, err error) {
	request.TransToAcsRequest()
	response = responses.NewCommonResponse()
	err = client.DoAction(request, response)
	return
}

func (client *Client) ProcessCommonRequestWithSigner(request *requests.CommonRequest, signerInterface interface{}) (response *responses.CommonResponse, err error) {
	if signer, isSigner := signerInterface.(auth.Signer); isSigner {
		request.TransToAcsRequest()
		response = responses.NewCommonResponse()
		err = client.DoActionWithSigner(request, response, signer)
		return
	} else {
		panic("should not be here")
	}
}

func (client *Client) Shutdown() {
	client.signer.Shutdown()
	close(client.asyncTaskQueue)
	client.isRunning = false
}
<commit_msg>fix null pointer while timeout<commit_after>/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package sdk

import (
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth"
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials"
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/endpoints"
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors"
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses"
	"net"
	"net/http"
)

// this value will be replaced while build: -ldflags="-X sdk.version=x.x.x"
var Version = "0.0.1"

type Client struct {
	regionId       string
	config         *Config
	signer         auth.Signer
	httpClient     *http.Client
	asyncTaskQueue chan func()

	debug     bool
	isRunning bool
}

func (client *Client) Init() (err error) {
	panic("not support yet")
}

func (client *Client) InitWithOptions(regionId string, config *Config, credential auth.Credential) (err error) {
	client.isRunning = true
	client.regionId = regionId
	client.config = config
	if err != nil {
		return
	}
	client.httpClient = &http.Client{}

	if config.HttpTransport != nil {
		client.httpClient.Transport = config.HttpTransport
	}

	if config.Timeout > 0 {
		client.httpClient.Timeout = config.Timeout
	}

	if config.EnableAsync {
		client.EnableAsync(config.GoRoutinePoolSize, config.MaxTaskQueueSize)
	}

	client.signer, err = auth.NewSignerWithCredential(credential, client.ProcessCommonRequestWithSigner)

	return
}

func (client *Client) EnableAsync(routinePoolSize, maxTaskQueueSize int) {
	client.asyncTaskQueue = make(chan func(), maxTaskQueueSize)
	for i := 0; i < routinePoolSize; i++ {
		go func() {
			for client.isRunning {
				select {
				case task := <-client.asyncTaskQueue:
					task()
				}
			}
		}()
	}
}

func (client *Client) InitWithAccessKey(regionId, accessKeyId, accessKeySecret string) (err error) {
	config := client.InitClientConfig()
	credential := &credentials.BaseCredential{
		AccessKeyId:     accessKeyId,
		AccessKeySecret: accessKeySecret,
	}
	return client.InitWithOptions(regionId, config, credential)
}

func (client *Client) InitWithRoleArn(regionId, accessKeyId, accessKeySecret, roleArn, roleSessionName string) (err error) {
	config := client.InitClientConfig()
	credential := &credentials.StsAssumeRoleCredential{
		AccessKeyId:     accessKeyId,
		AccessKeySecret: accessKeySecret,
		RoleArn:         roleArn,
		RoleSessionName: roleSessionName,
	}
	return client.InitWithOptions(regionId, config, credential)
}

func (client *Client) InitWithKeyPair(regionId, publicKeyId, privateKey string, sessionExpiration int) (err error) {
	config := client.InitClientConfig()
	credential := &credentials.KeyPairCredential{
		PrivateKey:        privateKey,
		PublicKeyId:       publicKeyId,
		SessionExpiration: sessionExpiration,
	}
	return client.InitWithOptions(regionId, config, credential)
}

func (client *Client) InitWithEcsInstance(regionId, roleName string) (err error) {
	config := client.InitClientConfig()
	credential := &credentials.EcsInstanceCredential{
		RoleName: roleName,
	}
	return client.InitWithOptions(regionId, config, credential)
}

func (client *Client) InitClientConfig() (config *Config) {
	config = NewConfig()
	if client.config != nil {
		config = client.config
	}

	return
}

func (client *Client) DoAction(request requests.AcsRequest, response responses.AcsResponse) (err error) {
	return client.DoActionWithSigner(request, response, nil)
}

func (client *Client) DoActionWithSigner(request requests.AcsRequest, response responses.AcsResponse, signer auth.Signer) (err error) {

	// add clientVersion
	request.GetHeaders()["x-sdk-core-version"] = Version

	regionId := client.regionId
	if len(request.GetRegionId()) > 0 {
		regionId = request.GetRegionId()
	}

	// resolve endpoint
	resolveParam := &endpoints.ResolveParam{
		Domain:           request.GetDomain(),
		Product:          request.GetProduct(),
		RegionId:         client.regionId,
		LocationProduct:  request.GetLocationServiceCode(),
		LocationEndpoint: request.GetLocationEndpointType(),
		CommonApi:        client.ProcessCommonRequest,
	}
	endpoint, err := endpoints.Resolve(resolveParam)
	if err != nil {
		return
	}
	request.SetDomain(endpoint)

	// init request params
	err = requests.InitParams(request)
	if err != nil {
		return
	}

	// signature
	if signer != nil {
		err = auth.Sign(request, signer, regionId)
	} else {
		err = auth.Sign(request, client.signer, regionId)
	}

	if err != nil {
		return
	}

	requestMethod := request.GetMethod()
	requestUrl := request.GetUrl()
	body := request.GetBodyReader()
	httpRequest, err := http.NewRequest(requestMethod, requestUrl, body)
	if err != nil {
		return
	}
	for key, value := range request.GetHeaders() {
		httpRequest.Header[key] = []string{value}
	}
	var httpResponse *http.Response
	for retryTimes := 0; retryTimes <= client.config.MaxRetryTime; retryTimes++ {
		httpResponse, err = client.httpClient.Do(httpRequest)

		var timeout bool
		// receive error
		if err != nil {
			// if not timeout error, return
			if timeout = isTimeout(err); !timeout {
				return
			}
		}
		//  if status code >= 500 or timeout, will trigger retry
		if client.config.AutoRetry && (timeout || isServerError(httpResponse)) {
			continue
		}
		break
	}
	err = responses.Unmarshal(response, httpResponse, request.GetAcceptFormat())
	return
}

func isTimeout(err error) bool {
	if err == nil {
		return false
	}
	netErr, isNetError := err.(net.Error)
	return isNetError && netErr.Timeout()
}

func isServerError(httpResponse *http.Response) bool {
	return httpResponse.StatusCode >= http.StatusInternalServerError
}

func (client *Client) AddAsyncTask(task func()) (err error) {
	if client.asyncTaskQueue != nil {
		client.asyncTaskQueue <- task
	} else {
		err = errors.NewClientError(errors.AsyncFunctionNotEnabledCode, errors.AsyncFunctionNotEnabledMessage, nil)
	}
	return
}

func NewClient() (client *Client, err error) {
	client = &Client{}
	err = client.Init()
	return
}

func NewClientWithOptions(regionId string, config *Config, credential auth.Credential) (client *Client, err error) {
	client = &Client{}
	err = client.InitWithOptions(regionId, config, credential)
	return
}

func NewClientWithAccessKey(regionId, accessKeyId, accessKeySecret string) (client *Client, err error) {
	client = &Client{}
	err = client.InitWithAccessKey(regionId, accessKeyId, accessKeySecret)
	return
}

func NewClientWithKeyPair(regionId string, config *Config, publicKeyId, privateKey string, sessionExpiration int) (client *Client, err error) {
	client = &Client{}
	client.config = config
	err = client.InitWithKeyPair(regionId, publicKeyId, privateKey, sessionExpiration)
	return
}

func NewClientWithEcsInstance(regionId string, config *Config, roleName string) (client *Client, err error) {
	client = &Client{}
	client.config = config
	err = client.InitWithEcsInstance(regionId, roleName)
	return
}

func NewClientWithRoleArn(regionId string, config *Config, accessKeyId, accessKeySecret, roleArn, roleSessionName string) (client *Client, err error) {
	client = &Client{}
	client.config = config
	err = client.InitWithRoleArn(regionId, accessKeyId, accessKeySecret, roleArn, roleSessionName)
	return
}

func (client *Client) ProcessCommonRequest(request *requests.CommonRequest) (response *responses.CommonResponse, err error) {
	request.TransToAcsRequest()
	response = responses.NewCommonResponse()
	err = client.DoAction(request, response)
	return
}

func (client *Client) ProcessCommonRequestWithSigner(request *requests.CommonRequest, signerInterface interface{}) (response *responses.CommonResponse, err error) {
	if signer, isSigner := signerInterface.(auth.Signer); isSigner {
		request.TransToAcsRequest()
		response = responses.NewCommonResponse()
		err = client.DoActionWithSigner(request, response, signer)
		return
	} else {
		panic("should not be here")
	}
}

func (client *Client) Shutdown() {
	client.signer.Shutdown()
	close(client.asyncTaskQueue)
	client.isRunning = false
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"github.com/github/git-lfs/lfs"
	"io"
	"net"
	"os"
	"path/filepath"
	"testing"

	"github.com/bmizerany/assert"
)

// Here we use a real client SSH context to talk to the real Serve() function
// However a Pipe is used to connect the two, no real SSH at this point
func TestServe(t *testing.T) {

	testcontentsz := int64(634)

	config := NewConfig()
	config.BasePath = filepath.Join(os.TempDir(), "git-lfs-serve-test")
	os.MkdirAll(config.BasePath, 0755)
	repopath := "test/repo"

	testcontent := make([]byte, testcontentsz)
	// put something interesting in it so we can detect it at each end
	testcontent[0] = '2'
	testcontent[1] = 'Z'
	testcontent[2] = '>'
	testcontent[3] = 'Q'
	testcontent[testcontentsz-1] = '#'
	testcontent[testcontentsz-2] = 'y'
	testcontent[testcontentsz-3] = 'L'
	testcontent[testcontentsz-4] = 'A'

	// Defer cleanup
	defer os.RemoveAll(config.BasePath)

	hasher := sha256.New()
	inbuf := bytes.NewReader(testcontent)
	io.Copy(hasher, inbuf)
	testoid := hex.EncodeToString(hasher.Sum(nil))

	cli, srv := net.Pipe()
	var outerr bytes.Buffer

	// 'Serve' is the real server function, usually connected to stdin/stdout but to pipe for test
	go Serve(srv, srv, &outerr, config, repopath)
	defer cli.Close()

	ctx := lfs.NewManualSSHApiContext(cli, cli)

	rdr := bytes.NewReader(testcontent)
	obj, wrerr := ctx.UploadCheck(testoid, int64(len(testcontent)))
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)
	assert.NotEqual(t, (*lfs.ObjectResource)(nil), wrerr)
	wrerr = ctx.UploadObject(obj, rdr)
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)
	assert.Equal(t, 0, rdr.Len()) // server should have read all bytes
	uploadDestPath, _ := mediaPath(testoid, config)
	s, err := os.Stat(uploadDestPath)
	assert.Equal(t, nil, err)
	assert.Equal(t, int64(len(testcontent)), s.Size())

	// Prove that it fails safely when trying to upload duplicate content
	rdr = bytes.NewReader(testcontent)
	obj, wrerr = ctx.UploadCheck(testoid, int64(len(testcontent)))
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)
	assert.Equal(t, (*lfs.ObjectResource)(nil), obj)

	// Now try to download same data
	var dlbuf bytes.Buffer
	dlrdr, sz, wrerr := ctx.Download(testoid)
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)
	assert.Equal(t, testcontentsz, sz)
	_, err = io.CopyN(&dlbuf, dlrdr, sz)
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)

	downloadedbytes := dlbuf.Bytes()
	assert.Equal(t, testcontent, downloadedbytes)

	// Now separate DownloadCheck/DownloadObject
	dlbuf.Reset()
	obj, wrerr = ctx.DownloadCheck(testoid)
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)
	assert.NotEqual(t, (*lfs.ObjectResource)(nil), obj)
	assert.Equal(t, testoid, obj.Oid)
	assert.Equal(t, testcontentsz, obj.Size)
	assert.Equal(t, true, obj.CanDownload())
	assert.Equal(t, false, obj.CanUpload())

	dlrdr, sz, wrerr = ctx.DownloadObject(obj)
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)
	assert.Equal(t, testcontentsz, sz)
	_, err = io.CopyN(&dlbuf, dlrdr, sz)
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)

	downloadedbytes = dlbuf.Bytes()
	assert.Equal(t, testcontent, downloadedbytes)

	// Now test safe fail state with DownloadCheck
	obj, wrerr = ctx.DownloadCheck("99999999999999999999999999999999999")
	assert.Equal(t, (*lfs.ObjectResource)(nil), obj)
	assert.NotEqual(t, (*lfs.WrappedError)(nil), wrerr)
	ctx.Close()

}
<commit_msg>Test batch method implementation<commit_after>package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"github.com/github/git-lfs/lfs"
	"io"
	"net"
	"os"
	"path/filepath"
	"testing"

	"github.com/bmizerany/assert"
)

// Here we use a real client SSH context to talk to the real Serve() function
// However a Pipe is used to connect the two, no real SSH at this point
func TestServe(t *testing.T) {

	testcontentsz := int64(634)

	config := NewConfig()
	config.BasePath = filepath.Join(os.TempDir(), "git-lfs-serve-test")
	os.MkdirAll(config.BasePath, 0755)
	repopath := "test/repo"

	testcontent := make([]byte, testcontentsz)
	// put something interesting in it so we can detect it at each end
	testcontent[0] = '2'
	testcontent[1] = 'Z'
	testcontent[2] = '>'
	testcontent[3] = 'Q'
	testcontent[testcontentsz-1] = '#'
	testcontent[testcontentsz-2] = 'y'
	testcontent[testcontentsz-3] = 'L'
	testcontent[testcontentsz-4] = 'A'

	// Defer cleanup
	defer os.RemoveAll(config.BasePath)

	hasher := sha256.New()
	inbuf := bytes.NewReader(testcontent)
	io.Copy(hasher, inbuf)
	testoid := hex.EncodeToString(hasher.Sum(nil))

	cli, srv := net.Pipe()
	var outerr bytes.Buffer

	// 'Serve' is the real server function, usually connected to stdin/stdout but to pipe for test
	go Serve(srv, srv, &outerr, config, repopath)
	defer cli.Close()

	ctx := lfs.NewManualSSHApiContext(cli, cli)

	rdr := bytes.NewReader(testcontent)
	obj, wrerr := ctx.UploadCheck(testoid, int64(len(testcontent)))
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)
	assert.NotEqual(t, (*lfs.ObjectResource)(nil), wrerr)
	wrerr = ctx.UploadObject(obj, rdr)
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)
	assert.Equal(t, 0, rdr.Len()) // server should have read all bytes
	uploadDestPath, _ := mediaPath(testoid, config)
	s, err := os.Stat(uploadDestPath)
	assert.Equal(t, nil, err)
	assert.Equal(t, int64(len(testcontent)), s.Size())

	// Prove that it fails safely when trying to upload duplicate content
	rdr = bytes.NewReader(testcontent)
	obj, wrerr = ctx.UploadCheck(testoid, int64(len(testcontent)))
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)
	assert.Equal(t, (*lfs.ObjectResource)(nil), obj)

	// Now try to download same data
	var dlbuf bytes.Buffer
	dlrdr, sz, wrerr := ctx.Download(testoid)
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)
	assert.Equal(t, testcontentsz, sz)
	_, err = io.CopyN(&dlbuf, dlrdr, sz)
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)

	downloadedbytes := dlbuf.Bytes()
	assert.Equal(t, testcontent, downloadedbytes)

	// Now separate DownloadCheck/DownloadObject
	dlbuf.Reset()
	obj, wrerr = ctx.DownloadCheck(testoid)
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)
	assert.NotEqual(t, (*lfs.ObjectResource)(nil), obj)
	assert.Equal(t, testoid, obj.Oid)
	assert.Equal(t, testcontentsz, obj.Size)
	assert.Equal(t, true, obj.CanDownload())
	assert.Equal(t, false, obj.CanUpload())

	dlrdr, sz, wrerr = ctx.DownloadObject(obj)
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)
	assert.Equal(t, testcontentsz, sz)
	_, err = io.CopyN(&dlbuf, dlrdr, sz)
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)

	downloadedbytes = dlbuf.Bytes()
	assert.Equal(t, testcontent, downloadedbytes)

	// Now test safe fail state with DownloadCheck
	garbageoid := "99999999999999999999999999999999999"
	obj, wrerr = ctx.DownloadCheck(garbageoid)
	assert.Equal(t, (*lfs.ObjectResource)(nil), obj)
	assert.NotEqual(t, (*lfs.WrappedError)(nil), wrerr)

	// Now batch test
	var inobjs []*lfs.ObjectResource
	inobjs = append(inobjs, &lfs.ObjectResource{Oid: testoid})
	inobjs = append(inobjs, &lfs.ObjectResource{Oid: garbageoid, Size: 500})
	retobjs, wrerr := ctx.Batch(inobjs)
	assert.Equal(t, (*lfs.WrappedError)(nil), wrerr)
	assert.Equal(t, 2, len(retobjs))
	for i, ro := range retobjs {
		switch i {
		case 0:
			assert.Equal(t, testoid, ro.Oid)
			assert.Equal(t, testcontentsz, ro.Size)
			assert.Equal(t, true, ro.CanDownload())
			assert.Equal(t, false, ro.CanUpload())
		case 1:
			assert.Equal(t, garbageoid, ro.Oid)
			assert.Equal(t, int64(500), ro.Size)
			assert.Equal(t, false, ro.CanDownload())
			assert.Equal(t, true, ro.CanUpload())
		}
	}

	ctx.Close()

}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/bouk/httprouter"
	"github.com/influxdata/chronograf" // When julienschmidt/httprouter v2 w/ context is out, switch
	"github.com/influxdata/chronograf/jwt"
)

const (
	// JSONType the mimetype for a json request
	JSONType = "application/json"
)

// MuxOpts are the options for the router.  Mostly related to auth.
type MuxOpts struct {
	Logger             chronograf.Logger
	Develop            bool     // Develop loads assets from filesystem instead of bindata
	UseAuth            bool     // UseAuth turns on Github OAuth and JWT
	TokenSecret        string   // TokenSecret is the JWT secret
	GithubClientID     string   // GithubClientID is the GH OAuth id
	GithubClientSecret string   // GithubClientSecret is the GH OAuth secret
	GithubOrgs         []string // GithubOrgs is the list of organizations a user my be a member of
}

// NewMux attaches all the route handlers; handler returned servers chronograf.
func NewMux(opts MuxOpts, service Service) http.Handler {
	router := httprouter.New()

	/* React Application */
	assets := Assets(AssetsOpts{
		Develop: opts.Develop,
		Logger:  opts.Logger,
	})
	// The react application handles all the routing if the server does not
	// know about the route.  This means that we never have unknown
	// routes on the server.
	router.NotFound = assets

	/* Documentation */
	router.GET("/swagger.json", Spec())
	router.GET("/docs", Redoc("/swagger.json"))

	/* API */
	// Root Routes returns all top-level routes in the API
	router.GET("/chronograf/v1/", AllRoutes(opts.Logger))

	// Sources
	router.GET("/chronograf/v1/sources", service.Sources)
	router.POST("/chronograf/v1/sources", service.NewSource)

	router.GET("/chronograf/v1/sources/:id", service.SourcesID)
	router.PATCH("/chronograf/v1/sources/:id", service.UpdateSource)
	router.DELETE("/chronograf/v1/sources/:id", service.RemoveSource)

	// Source Proxy
	router.POST("/chronograf/v1/sources/:id/proxy", service.Proxy)

	// Kapacitor
	router.GET("/chronograf/v1/sources/:id/kapacitors", service.Kapacitors)
	router.POST("/chronograf/v1/sources/:id/kapacitors", service.NewKapacitor)

	router.GET("/chronograf/v1/sources/:id/kapacitors/:kid", service.KapacitorsID)
	router.PATCH("/chronograf/v1/sources/:id/kapacitors/:kid", service.UpdateKapacitor)
	router.DELETE("/chronograf/v1/sources/:id/kapacitors/:kid", service.RemoveKapacitor)

	// Kapacitor rules
	router.GET("/chronograf/v1/sources/:id/kapacitors/:kid/rules", service.KapacitorRulesGet)
	router.POST("/chronograf/v1/sources/:id/kapacitors/:kid/rules", service.KapacitorRulesPost)

	router.GET("/chronograf/v1/sources/:id/kapacitors/:kid/rules/:tid", service.KapacitorRulesID)
	router.PUT("/chronograf/v1/sources/:id/kapacitors/:kid/rules/:tid", service.KapacitorRulesPut)
	router.DELETE("/chronograf/v1/sources/:id/kapacitors/:kid/rules/:tid", service.KapacitorRulesDelete)

	// Kapacitor Proxy
	router.GET("/chronograf/v1/sources/:id/kapacitors/:kid/proxy", service.KapacitorProxyGet)
	router.POST("/chronograf/v1/sources/:id/kapacitors/:kid/proxy", service.KapacitorProxyPost)
	router.PATCH("/chronograf/v1/sources/:id/kapacitors/:kid/proxy", service.KapacitorProxyPatch)
	router.DELETE("/chronograf/v1/sources/:id/kapacitors/:kid/proxy", service.KapacitorProxyDelete)

	// Mappings
	router.GET("/chronograf/v1/mappings", service.GetMappings)

	// Layouts
	router.GET("/chronograf/v1/layouts", service.Layouts)
	router.POST("/chronograf/v1/layouts", service.NewLayout)

	router.GET("/chronograf/v1/layouts/:id", service.LayoutsID)
	router.PUT("/chronograf/v1/layouts/:id", service.UpdateLayout)
	router.DELETE("/chronograf/v1/layouts/:id", service.RemoveLayout)

	// Users
	router.GET("/chronograf/v1/me", service.Me)
	router.POST("/chronograf/v1/users", service.NewUser)

	router.GET("/chronograf/v1/users/:id", service.UserID)
	router.PATCH("/chronograf/v1/users/:id", service.UpdateUser)
	router.DELETE("/chronograf/v1/users/:id", service.RemoveUser)

	// Explorations
	router.GET("/chronograf/v1/users/:id/explorations", service.Explorations)
	router.POST("/chronograf/v1/users/:id/explorations", service.NewExploration)

	router.GET("/chronograf/v1/users/:id/explorations/:eid", service.ExplorationsID)
	router.PATCH("/chronograf/v1/users/:id/explorations/:eid", service.UpdateExploration)
	router.DELETE("/chronograf/v1/users/:id/explorations/:eid", service.RemoveExploration)

	// Dashboards
	router.GET("/chronograf/v1/dashboards", service.Dashboards)
	router.POST("/chronograf/v1/dashboards", service.NewDashboard)

	router.GET("/chronograf/v1/dashboards/:id", service.DashboardID)
	router.DELETE("/chronograf/v1/dashboard/:id", service.RemoveDashboard)
	router.PUT("/chronograf/v1/dashboard/:id", service.UpdateDashboard)

	/* Authentication */
	if opts.UseAuth {
		auth := AuthAPI(opts, router)
		return Logger(opts.Logger, auth)
	}
	return Logger(opts.Logger, router)
}

// AuthAPI adds the OAuth routes if auth is enabled.
func AuthAPI(opts MuxOpts, router *httprouter.Router) http.Handler {
	auth := jwt.NewJWT(opts.TokenSecret)

	successURL := "/"
	failureURL := "/login"
	gh := NewGithub(
		opts.GithubClientID,
		opts.GithubClientSecret,
		successURL,
		failureURL,
		opts.GithubOrgs,
		&auth,
		opts.Logger,
	)

	router.GET("/oauth/github", gh.Login())
	router.GET("/oauth/logout", gh.Logout())
	router.GET("/oauth/github/callback", gh.Callback())

	tokenMiddleware := AuthorizedToken(&auth, &CookieExtractor{Name: "session"}, opts.Logger, router)
	// Wrap the API with token validation middleware.
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.URL.Path, "/chronograf/v1/") {
			tokenMiddleware.ServeHTTP(w, r)
			return
		}
		router.ServeHTTP(w, r)
	})
}

func encodeJSON(w http.ResponseWriter, status int, v interface{}, logger chronograf.Logger) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	if err := json.NewEncoder(w).Encode(v); err != nil {
		unknownErrorWithMessage(w, err, logger)
	}
}

// Error writes an JSON message
func Error(w http.ResponseWriter, code int, msg string, logger chronograf.Logger) {
	e := ErrorMessage{
		Code:    code,
		Message: msg,
	}
	b, err := json.Marshal(e)
	if err != nil {
		code = http.StatusInternalServerError
		b = []byte(`{"code": 500, "message":"server_error"}`)
	}

	logger.
		WithField("component", "server").
		WithField("http_status ", code).
		Error("Error message ", msg)
	w.Header().Set("Content-Type", JSONType)
	w.WriteHeader(code)
	_, _ = w.Write(b)
}

func invalidData(w http.ResponseWriter, err error, logger chronograf.Logger) {
	Error(w, http.StatusUnprocessableEntity, fmt.Sprintf("%v", err), logger)
}

func invalidJSON(w http.ResponseWriter, logger chronograf.Logger) {
	Error(w, http.StatusBadRequest, "Unparsable JSON", logger)
}

func unknownErrorWithMessage(w http.ResponseWriter, err error, logger chronograf.Logger) {
	Error(w, http.StatusInternalServerError, fmt.Sprintf("Unknown error: %v", err), logger)
}

func notFound(w http.ResponseWriter, id int, logger chronograf.Logger) {
	Error(w, http.StatusNotFound, fmt.Sprintf("ID %d not found", id), logger)
}

func paramID(key string, r *http.Request) (int, error) {
	ctx := r.Context()
	param := httprouter.GetParamFromContext(ctx, key)
	id, err := strconv.Atoi(param)
	if err != nil {
		return -1, fmt.Errorf("Error converting ID %s", param)
	}
	return id, nil
}
<commit_msg>fix dashboards routes<commit_after>package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/bouk/httprouter"
	"github.com/influxdata/chronograf" // When julienschmidt/httprouter v2 w/ context is out, switch
	"github.com/influxdata/chronograf/jwt"
)

const (
	// JSONType the mimetype for a json request
	JSONType = "application/json"
)

// MuxOpts are the options for the router.  Mostly related to auth.
type MuxOpts struct {
	Logger             chronograf.Logger
	Develop            bool     // Develop loads assets from filesystem instead of bindata
	UseAuth            bool     // UseAuth turns on Github OAuth and JWT
	TokenSecret        string   // TokenSecret is the JWT secret
	GithubClientID     string   // GithubClientID is the GH OAuth id
	GithubClientSecret string   // GithubClientSecret is the GH OAuth secret
	GithubOrgs         []string // GithubOrgs is the list of organizations a user my be a member of
}

// NewMux attaches all the route handlers; handler returned servers chronograf.
func NewMux(opts MuxOpts, service Service) http.Handler {
	router := httprouter.New()

	/* React Application */
	assets := Assets(AssetsOpts{
		Develop: opts.Develop,
		Logger:  opts.Logger,
	})
	// The react application handles all the routing if the server does not
	// know about the route.  This means that we never have unknown
	// routes on the server.
	router.NotFound = assets

	/* Documentation */
	router.GET("/swagger.json", Spec())
	router.GET("/docs", Redoc("/swagger.json"))

	/* API */
	// Root Routes returns all top-level routes in the API
	router.GET("/chronograf/v1/", AllRoutes(opts.Logger))

	// Sources
	router.GET("/chronograf/v1/sources", service.Sources)
	router.POST("/chronograf/v1/sources", service.NewSource)

	router.GET("/chronograf/v1/sources/:id", service.SourcesID)
	router.PATCH("/chronograf/v1/sources/:id", service.UpdateSource)
	router.DELETE("/chronograf/v1/sources/:id", service.RemoveSource)

	// Source Proxy
	router.POST("/chronograf/v1/sources/:id/proxy", service.Proxy)

	// Kapacitor
	router.GET("/chronograf/v1/sources/:id/kapacitors", service.Kapacitors)
	router.POST("/chronograf/v1/sources/:id/kapacitors", service.NewKapacitor)

	router.GET("/chronograf/v1/sources/:id/kapacitors/:kid", service.KapacitorsID)
	router.PATCH("/chronograf/v1/sources/:id/kapacitors/:kid", service.UpdateKapacitor)
	router.DELETE("/chronograf/v1/sources/:id/kapacitors/:kid", service.RemoveKapacitor)

	// Kapacitor rules
	router.GET("/chronograf/v1/sources/:id/kapacitors/:kid/rules", service.KapacitorRulesGet)
	router.POST("/chronograf/v1/sources/:id/kapacitors/:kid/rules", service.KapacitorRulesPost)

	router.GET("/chronograf/v1/sources/:id/kapacitors/:kid/rules/:tid", service.KapacitorRulesID)
	router.PUT("/chronograf/v1/sources/:id/kapacitors/:kid/rules/:tid", service.KapacitorRulesPut)
	router.DELETE("/chronograf/v1/sources/:id/kapacitors/:kid/rules/:tid", service.KapacitorRulesDelete)

	// Kapacitor Proxy
	router.GET("/chronograf/v1/sources/:id/kapacitors/:kid/proxy", service.KapacitorProxyGet)
	router.POST("/chronograf/v1/sources/:id/kapacitors/:kid/proxy", service.KapacitorProxyPost)
	router.PATCH("/chronograf/v1/sources/:id/kapacitors/:kid/proxy", service.KapacitorProxyPatch)
	router.DELETE("/chronograf/v1/sources/:id/kapacitors/:kid/proxy", service.KapacitorProxyDelete)

	// Mappings
	router.GET("/chronograf/v1/mappings", service.GetMappings)

	// Layouts
	router.GET("/chronograf/v1/layouts", service.Layouts)
	router.POST("/chronograf/v1/layouts", service.NewLayout)

	router.GET("/chronograf/v1/layouts/:id", service.LayoutsID)
	router.PUT("/chronograf/v1/layouts/:id", service.UpdateLayout)
	router.DELETE("/chronograf/v1/layouts/:id", service.RemoveLayout)

	// Users
	router.GET("/chronograf/v1/me", service.Me)
	router.POST("/chronograf/v1/users", service.NewUser)

	router.GET("/chronograf/v1/users/:id", service.UserID)
	router.PATCH("/chronograf/v1/users/:id", service.UpdateUser)
	router.DELETE("/chronograf/v1/users/:id", service.RemoveUser)

	// Explorations
	router.GET("/chronograf/v1/users/:id/explorations", service.Explorations)
	router.POST("/chronograf/v1/users/:id/explorations", service.NewExploration)

	router.GET("/chronograf/v1/users/:id/explorations/:eid", service.ExplorationsID)
	router.PATCH("/chronograf/v1/users/:id/explorations/:eid", service.UpdateExploration)
	router.DELETE("/chronograf/v1/users/:id/explorations/:eid", service.RemoveExploration)

	// Dashboards
	router.GET("/chronograf/v1/dashboards", service.Dashboards)
	router.POST("/chronograf/v1/dashboards", service.NewDashboard)

	router.GET("/chronograf/v1/dashboards/:id", service.DashboardID)
	router.DELETE("/chronograf/v1/dashboards/:id", service.RemoveDashboard)
	router.PUT("/chronograf/v1/dashboards/:id", service.UpdateDashboard)

	/* Authentication */
	if opts.UseAuth {
		auth := AuthAPI(opts, router)
		return Logger(opts.Logger, auth)
	}
	return Logger(opts.Logger, router)
}

// AuthAPI adds the OAuth routes if auth is enabled.
func AuthAPI(opts MuxOpts, router *httprouter.Router) http.Handler {
	auth := jwt.NewJWT(opts.TokenSecret)

	successURL := "/"
	failureURL := "/login"
	gh := NewGithub(
		opts.GithubClientID,
		opts.GithubClientSecret,
		successURL,
		failureURL,
		opts.GithubOrgs,
		&auth,
		opts.Logger,
	)

	router.GET("/oauth/github", gh.Login())
	router.GET("/oauth/logout", gh.Logout())
	router.GET("/oauth/github/callback", gh.Callback())

	tokenMiddleware := AuthorizedToken(&auth, &CookieExtractor{Name: "session"}, opts.Logger, router)
	// Wrap the API with token validation middleware.
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.URL.Path, "/chronograf/v1/") {
			tokenMiddleware.ServeHTTP(w, r)
			return
		}
		router.ServeHTTP(w, r)
	})
}

func encodeJSON(w http.ResponseWriter, status int, v interface{}, logger chronograf.Logger) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	if err := json.NewEncoder(w).Encode(v); err != nil {
		unknownErrorWithMessage(w, err, logger)
	}
}

// Error writes an JSON message
func Error(w http.ResponseWriter, code int, msg string, logger chronograf.Logger) {
	e := ErrorMessage{
		Code:    code,
		Message: msg,
	}
	b, err := json.Marshal(e)
	if err != nil {
		code = http.StatusInternalServerError
		b = []byte(`{"code": 500, "message":"server_error"}`)
	}

	logger.
		WithField("component", "server").
		WithField("http_status ", code).
		Error("Error message ", msg)
	w.Header().Set("Content-Type", JSONType)
	w.WriteHeader(code)
	_, _ = w.Write(b)
}

func invalidData(w http.ResponseWriter, err error, logger chronograf.Logger) {
	Error(w, http.StatusUnprocessableEntity, fmt.Sprintf("%v", err), logger)
}

func invalidJSON(w http.ResponseWriter, logger chronograf.Logger) {
	Error(w, http.StatusBadRequest, "Unparsable JSON", logger)
}

func unknownErrorWithMessage(w http.ResponseWriter, err error, logger chronograf.Logger) {
	Error(w, http.StatusInternalServerError, fmt.Sprintf("Unknown error: %v", err), logger)
}

func notFound(w http.ResponseWriter, id int, logger chronograf.Logger) {
	Error(w, http.StatusNotFound, fmt.Sprintf("ID %d not found", id), logger)
}

func paramID(key string, r *http.Request) (int, error) {
	ctx := r.Context()
	param := httprouter.GetParamFromContext(ctx, key)
	id, err := strconv.Atoi(param)
	if err != nil {
		return -1, fmt.Errorf("Error converting ID %s", param)
	}
	return id, nil
}
<|endoftext|>


<commit_before>// This file definse the logOuter interface and several types of logOuter.
//
// emptyOuter = logOuter where both Out and Outf are noops
// lineOuter = logOuter where a newline is inserted after every call to
//			   Out and Outf
// fatalLineOuter = logOuter that logs message with inserted newline then
//					exits with call to os.EXIT(1)

package golog

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"strconv"
	"time"
)

// The location in the source of a log message. Any fields set to their zero
// values will be assumed to be absent. (The empty string is not a valid
// package/function/file, and 0 is not a valid line number).
type LogLocation struct {
	Package  string
	Function string
	File     string
	Line     int
}

type LogMessage struct {
	Level       int
	Nanoseconds int64
	Message     string
	Location    *LogLocation
}

type LogOuter interface {
	// Output a LogMessage (to a file, to stderr, to a tester, etc). Output
	// must be safe to call from multiple threads.
	Output(*LogMessage)
}

// Render a formatted LogLocation to the buffer.
func renderLogLocation(buf *bytes.Buffer, l *LogLocation) {
	if l == nil {
		return
	}
	packPresent := len(l.Package) > 0
	funcPresent := len(l.Function) > 0
	filePresent := len(l.Function) > 0
	linePresent := l.Line > 0

	// TODO(awreece) This logic is terrifying.
	if packPresent {
		buf.WriteString(l.Package)
	}
	if funcPresent {
		if packPresent {
			buf.WriteString(".")
		}
		buf.WriteString(l.Function)
	}
	if (packPresent || funcPresent) && (filePresent || linePresent) {
		buf.WriteString("/")
	}
	if filePresent {
		buf.WriteString(l.File)
	}
	if linePresent {
		if filePresent {
			buf.WriteString(":")
		}
		buf.WriteString(strconv.Itoa(l.Line))
	}
}

// Format the message as a string, optionally inserting a newline.
func formatLogMessage(m *LogMessage, insertNewline bool) string {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("L%d", m.Level))
	t := time.NanosecondsToLocalTime(m.Nanoseconds)
	buf.WriteString(t.Format(" 15:04:05.000000"))
	if m.Location != nil {
		buf.WriteString(" ")
		renderLogLocation(&buf, m.Location)
	}
	buf.WriteString("] ")
	buf.WriteString(m.Message)
	if insertNewline {
		buf.WriteString("\n")
	}
	return buf.String()
}

type writerLogOuter struct {
	// TODO Insert mutex?
	io.Writer
}

func (f *writerLogOuter) Output(m *LogMessage) {
	// TODO Grab mutex?
	// Make sure to insert a newline.
	f.Write([]byte(formatLogMessage(m, true)))
}

func NewWriterLogOuter(f io.Writer) LogOuter {
	return &writerLogOuter{f}
}

func NewFileLogOuter(filename string) (LogOuter, os.Error) {
	if file, err := os.Create(filename); err != nil {
		return nil, err
	} else {
		return NewWriterLogOuter(file), nil
	}

	panic("Code never reaches here, this mollifies the compiler.")
}

// We want to allow an abitrary testing framework.
type TestController interface {
	// We will assume that testers insert newlines in manner similar to
	// the FEATURE of testing.T where it inserts extra newlines. >.<
	Log(...interface{})
	FailNow()
}

type testLogOuter struct {
	TestController
}

func (t *testLogOuter) Output(m *LogMessage) {
	// Don't insert an additional log message since the tester inserts them
	// for us.
	t.Log(formatLogMessage(m, false))
}

func NewTestLogOuter(t TestController) LogOuter {
	return &testLogOuter{t}
}
<commit_msg>assing TODOs<commit_after>// This file definse the logOuter interface and several types of logOuter.
//
// emptyOuter = logOuter where both Out and Outf are noops
// lineOuter = logOuter where a newline is inserted after every call to
//			   Out and Outf
// fatalLineOuter = logOuter that logs message with inserted newline then
//					exits with call to os.EXIT(1)

package golog

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"strconv"
	"time"
)

// The location in the source of a log message. Any fields set to their zero
// values will be assumed to be absent. (The empty string is not a valid
// package/function/file, and 0 is not a valid line number).
type LogLocation struct {
	Package  string
	Function string
	File     string
	Line     int
}

type LogMessage struct {
	Level       int
	Nanoseconds int64
	Message     string
	Location    *LogLocation
}

type LogOuter interface {
	// Output a LogMessage (to a file, to stderr, to a tester, etc). Output
	// must be safe to call from multiple threads.
	Output(*LogMessage)
}

// Render a formatted LogLocation to the buffer.
func renderLogLocation(buf *bytes.Buffer, l *LogLocation) {
	if l == nil {
		return
	}
	packPresent := len(l.Package) > 0
	funcPresent := len(l.Function) > 0
	filePresent := len(l.Function) > 0
	linePresent := l.Line > 0

	// TODO(awreece) This logic is terrifying.
	if packPresent {
		buf.WriteString(l.Package)
	}
	if funcPresent {
		if packPresent {
			buf.WriteString(".")
		}
		buf.WriteString(l.Function)
	}
	if (packPresent || funcPresent) && (filePresent || linePresent) {
		buf.WriteString("/")
	}
	if filePresent {
		buf.WriteString(l.File)
	}
	if linePresent {
		if filePresent {
			buf.WriteString(":")
		}
		buf.WriteString(strconv.Itoa(l.Line))
	}
}

// Format the message as a string, optionally inserting a newline.
func formatLogMessage(m *LogMessage, insertNewline bool) string {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("L%d", m.Level))
	t := time.NanosecondsToLocalTime(m.Nanoseconds)
	buf.WriteString(t.Format(" 15:04:05.000000"))
	if m.Location != nil {
		buf.WriteString(" ")
		renderLogLocation(&buf, m.Location)
	}
	buf.WriteString("] ")
	buf.WriteString(m.Message)
	if insertNewline {
		buf.WriteString("\n")
	}
	return buf.String()
}

type writerLogOuter struct {
	// TODO(awreece) Insert mutex?
	io.Writer
}

func (f *writerLogOuter) Output(m *LogMessage) {
	// TODO(awreece) Grab mutex?
	// Make sure to insert a newline.
	f.Write([]byte(formatLogMessage(m, true)))
}

func NewWriterLogOuter(f io.Writer) LogOuter {
	return &writerLogOuter{f}
}

func NewFileLogOuter(filename string) (LogOuter, os.Error) {
	if file, err := os.Create(filename); err != nil {
		return nil, err
	} else {
		return NewWriterLogOuter(file), nil
	}

	panic("Code never reaches here, this mollifies the compiler.")
}

// We want to allow an abitrary testing framework.
type TestController interface {
	// We will assume that testers insert newlines in manner similar to
	// the FEATURE of testing.T where it inserts extra newlines. >.<
	Log(...interface{})
	FailNow()
}

type testLogOuter struct {
	TestController
}

func (t *testLogOuter) Output(m *LogMessage) {
	// Don't insert an additional log message since the tester inserts them
	// for us.
	t.Log(formatLogMessage(m, false))
}

func NewTestLogOuter(t TestController) LogOuter {
	return &testLogOuter{t}
}
<|endoftext|>


<commit_before>package implements

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
)

var (
	// CastXMLBin is the name/location of the castxml binary.
	CastXMLBin = "castxml"

	// Add a stub C function that goes nowhere and does nothing. Just
	// to give castxml something to chew on. It may not be strictly
	// needed but it worked for me.
	// TODO Cleanup - The macro part is probably totally unnecessary but I wanted
	// to see how much "extra stuff" castxml picked up.

	gndn = `

#define GNDN
GNDN int foo(int x) {
    return x;
}
`

	// Individual "package" stubs. Add the needed headers to pick up the
	// ceph lib<whatever> content plus the code stub for castxml.

	cephfsCStub = `
#define FILE_OFFSET_BITS 64
#include <stdlib.h>
#define __USE_FILE_OFFSET64
#include <cephfs/libcephfs.h>
` + gndn
	radosCStub = `
#include <rados/librados.h>
` + gndn
	rbdCStub = `
#include <rbd/librbd.h>
#include <rbd/features.h>
` + gndn

	stubs = map[string]string{
		"cephfs": cephfsCStub,
		"rados":  radosCStub,
		"rbd":    rbdCStub,
	}
	funcPrefix = map[string]string{
		"cephfs": "ceph_",
		"rados":  "rados_",
		"rbd":    "rbd_",
	}
)

type allCFunctions struct {
	Functions CFunctions `xml:"Function"`
}

func parseCFunctions(xmlData []byte) ([]CFunction, error) {
	cf := allCFunctions{}
	if err := xml.Unmarshal(xmlData, &cf); err != nil {
		return nil, err
	}
	return cf.Functions.ensure()
}

func parseCFunctionsFromFile(fname string) ([]CFunction, error) {
	cf := allCFunctions{}

	f, err := os.Open(fname)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	xdec := xml.NewDecoder(f)
	err = xdec.Decode(&cf)
	if err != nil {
		return nil, err
	}
	return cf.Functions.ensure()
}

func parseCFunctionsFromCmd(args []string) (CFunctions, error) {
	cf := allCFunctions{}

	cmd := exec.Command(args[0], args[1:]...)
	logger.Printf("will call: %v", cmd)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	err = cmd.Start()
	if err != nil {
		return nil, err
	}

	xdec := xml.NewDecoder(stdout)
	parseErr := xdec.Decode(&cf)

	err = cmd.Wait()
	if err != nil {
		return nil, err
	}

	if parseErr != nil {
		return nil, parseErr
	}
	return cf.Functions.ensure()
}

func stubCFunctions(libname string) (CFunctions, error) {
	cstub := stubs[libname]
	if cstub == "" {
		return nil, fmt.Errorf("no C stub available for '%s'", libname)
	}

	tfile, err := ioutil.TempFile("", "*-"+libname+".c")
	if err != nil {
		return nil, err
	}
	defer os.Remove(tfile.Name())

	_, err = tfile.Write([]byte(cstub))
	if err != nil {
		return nil, err
	}

	cmd := []string{
		CastXMLBin,
		"--castxml-output=1",
		"-o", "-",
		tfile.Name(),
	}
	return parseCFunctionsFromCmd(cmd)
}

// CephCFunctions will extract C functions from the supplied package name
// and update the results within the code inspector.
func CephCFunctions(pkg string, ii *Inspector) error {
	logger.Printf("getting C AST for %s", pkg)
	f, err := stubCFunctions(pkg)
	if err != nil {
		return err
	}
	return ii.SetExpected(funcPrefix[pkg], f)
}
<commit_msg>contrib: implements tool prints stderr if castxml fails<commit_after>package implements

import (
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
)

var (
	// CastXMLBin is the name/location of the castxml binary.
	CastXMLBin = "castxml"

	// Add a stub C function that goes nowhere and does nothing. Just
	// to give castxml something to chew on. It may not be strictly
	// needed but it worked for me.
	// TODO Cleanup - The macro part is probably totally unnecessary but I wanted
	// to see how much "extra stuff" castxml picked up.

	gndn = `

#define GNDN
GNDN int foo(int x) {
    return x;
}
`

	// Individual "package" stubs. Add the needed headers to pick up the
	// ceph lib<whatever> content plus the code stub for castxml.

	cephfsCStub = `
#define FILE_OFFSET_BITS 64
#include <stdlib.h>
#define __USE_FILE_OFFSET64
#include <cephfs/libcephfs.h>
` + gndn
	radosCStub = `
#include <rados/librados.h>
` + gndn
	rbdCStub = `
#include <rbd/librbd.h>
#include <rbd/features.h>
` + gndn

	stubs = map[string]string{
		"cephfs": cephfsCStub,
		"rados":  radosCStub,
		"rbd":    rbdCStub,
	}
	funcPrefix = map[string]string{
		"cephfs": "ceph_",
		"rados":  "rados_",
		"rbd":    "rbd_",
	}
)

type allCFunctions struct {
	Functions CFunctions `xml:"Function"`
}

func parseCFunctions(xmlData []byte) ([]CFunction, error) {
	cf := allCFunctions{}
	if err := xml.Unmarshal(xmlData, &cf); err != nil {
		return nil, err
	}
	return cf.Functions.ensure()
}

func parseCFunctionsFromFile(fname string) ([]CFunction, error) {
	cf := allCFunctions{}

	f, err := os.Open(fname)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	xdec := xml.NewDecoder(f)
	err = xdec.Decode(&cf)
	if err != nil {
		return nil, err
	}
	return cf.Functions.ensure()
}

func parseCFunctionsFromCmd(args []string) (CFunctions, error) {
	cf := allCFunctions{}

	cmd := exec.Command(args[0], args[1:]...)
	logger.Printf("will call: %v", cmd)
	stdout, err := cmd.Output()
	if err != nil {
		var ee *exec.ExitError
		if errors.As(err, &ee) {
			err = fmt.Errorf("%w, stderr:\n%s", err, ee.Stderr)
		}
		return nil, err
	}

	parseErr := xml.Unmarshal(stdout, &cf)
	if parseErr != nil {
		return nil, parseErr
	}
	return cf.Functions.ensure()
}

func stubCFunctions(libname string) (CFunctions, error) {
	cstub := stubs[libname]
	if cstub == "" {
		return nil, fmt.Errorf("no C stub available for '%s'", libname)
	}

	tfile, err := ioutil.TempFile("", "*-"+libname+".c")
	if err != nil {
		return nil, err
	}
	defer os.Remove(tfile.Name())

	_, err = tfile.Write([]byte(cstub))
	if err != nil {
		return nil, err
	}

	cmd := []string{
		CastXMLBin,
		"--castxml-output=1",
		"-o", "-",
		tfile.Name(),
	}
	return parseCFunctionsFromCmd(cmd)
}

// CephCFunctions will extract C functions from the supplied package name
// and update the results within the code inspector.
func CephCFunctions(pkg string, ii *Inspector) error {
	logger.Printf("getting C AST for %s", pkg)
	f, err := stubCFunctions(pkg)
	if err != nil {
		return err
	}
	return ii.SetExpected(funcPrefix[pkg], f)
}
<|endoftext|>


<commit_before>package gonx

import (
	"fmt"
	"strconv"
	"strings"
)

// Shortcut for the map of strings
type Fields map[string]string

// Parsed log record. Use Get method to retrieve a value by name instead of
// threating this as a map, because inner representation is in design.
type Entry struct {
	fields Fields
}

// Creates an empty Entry to be filled later
func NewEmptyEntry() *Entry {
	return &Entry{make(Fields)}
}

// Creates an Entry with fiven fields
func NewEntry(fields Fields) *Entry {
	return &Entry{fields}
}

// Return entry field value by name or empty string and error if it
// does not exist.
func (entry *Entry) Field(name string) (value string, err error) {
	value, ok := entry.fields[name]
	if !ok {
		err = fmt.Errorf("field '%v' does not found in record %+v", name, *entry)
	}
	return
}

// Return entry field value as float64. Rutuen nil if field does not exists
// and convertion error if cannot cast a type.
func (entry *Entry) FloatField(name string) (value float64, err error) {
	tmp, err := entry.Field(name)
	if err == nil {
		value, err = strconv.ParseFloat(tmp, 64)
	}
	return
}

// Field value setter
func (entry *Entry) SetField(name string, value string) {
	entry.fields[name] = value
}

// Float field value setter. It accepts float64, but still store it as a
// string in the same fields map. The precision is 2, its enough for log
// parsing task
func (entry *Entry) SetFloatField(name string, value float64) {
	entry.SetField(name, strconv.FormatFloat(value, 'f', 2, 64))
}

// Integer field value setter. It accepts float64, but still store it as a
// string in the same fields map.
func (entry *Entry) SetUintField(name string, value uint64) {
	entry.SetField(name, strconv.FormatUint(uint64(value), 10))
}

// Merge two entries by updating values for master entry with given.
func (master *Entry) Merge(entry *Entry) {
	for name, value := range entry.fields {
		master.SetField(name, value)
	}
}

func (entry *Entry) FieldsHash(fields []string) string {
	var key []string
	for _, name := range fields {
		value, err := entry.Field(name)
		if err != nil {
			value = "NULL"
		}
		key = append(key, fmt.Sprintf("'%v'=%v", name, value))
	}
	return strings.Join(key, ";")
}

func (entry *Entry) Partial(fields []string) *Entry {
	partial := NewEmptyEntry()
	for _, name := range fields {
		value, _ := entry.Field(name)
		partial.SetField(name, value)
	}
	return partial
}
<commit_msg>Add Entry.ToJson() method<commit_after>package gonx

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
)

// Shortcut for the map of strings
type Fields map[string]string

// Parsed log record. Use Get method to retrieve a value by name instead of
// threating this as a map, because inner representation is in design.
type Entry struct {
	fields Fields
}

// Creates an empty Entry to be filled later
func NewEmptyEntry() *Entry {
	return &Entry{make(Fields)}
}

// Creates an Entry with fiven fields
func NewEntry(fields Fields) *Entry {
	return &Entry{fields}
}

// Return entry field value by name or empty string and error if it
// does not exist.
func (entry *Entry) Field(name string) (value string, err error) {
	value, ok := entry.fields[name]
	if !ok {
		err = fmt.Errorf("field '%v' does not found in record %+v", name, *entry)
	}
	return
}

// Return entry field value as float64. Rutuen nil if field does not exists
// and convertion error if cannot cast a type.
func (entry *Entry) FloatField(name string) (value float64, err error) {
	tmp, err := entry.Field(name)
	if err == nil {
		value, err = strconv.ParseFloat(tmp, 64)
	}
	return
}

// Field value setter
func (entry *Entry) SetField(name string, value string) {
	entry.fields[name] = value
}

// Float field value setter. It accepts float64, but still store it as a
// string in the same fields map. The precision is 2, its enough for log
// parsing task
func (entry *Entry) SetFloatField(name string, value float64) {
	entry.SetField(name, strconv.FormatFloat(value, 'f', 2, 64))
}

// Integer field value setter. It accepts float64, but still store it as a
// string in the same fields map.
func (entry *Entry) SetUintField(name string, value uint64) {
	entry.SetField(name, strconv.FormatUint(uint64(value), 10))
}

// Merge two entries by updating values for master entry with given.
func (master *Entry) Merge(entry *Entry) {
	for name, value := range entry.fields {
		master.SetField(name, value)
	}
}

func (entry *Entry) FieldsHash(fields []string) string {
	var key []string
	for _, name := range fields {
		value, err := entry.Field(name)
		if err != nil {
			value = "NULL"
		}
		key = append(key, fmt.Sprintf("'%v'=%v", name, value))
	}
	return strings.Join(key, ";")
}

func (entry *Entry) Partial(fields []string) *Entry {
	partial := NewEmptyEntry()
	for _, name := range fields {
		value, _ := entry.Field(name)
		partial.SetField(name, value)
	}
	return partial
}

func (entry *Entry) ToJson() ([]byte, error) {
	return json.Marshal(entry.fields)
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"net"
	"reflect"
	"strconv"

	"github.com/juju/errors"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"

	"github.com/juju/juju/network"
)

// controllerAddresses returns the list of internal addresses of the state
// server machines.
func (st *State) controllerAddresses() ([]string, error) {
	ssState := st
	model, err := st.ControllerModel()
	if err != nil {
		return nil, errors.Trace(err)
	}
	if st.ModelTag() != model.ModelTag() {
		// We are not using the controller model, so get one.
		logger.Debugf("getting a controller state connection, current env: %s", st.ModelTag())
		ssState, err = st.ForModel(model.ModelTag())
		if err != nil {
			return nil, errors.Trace(err)
		}
		defer ssState.Close()
		logger.Debugf("ssState env: %s", ssState.ModelTag())
	}

	type addressMachine struct {
		Addresses []address
	}
	var allAddresses []addressMachine
	// TODO(rog) 2013/10/14 index machines on jobs.
	machines, closer := ssState.getCollection(machinesC)
	defer closer()
	err = machines.Find(bson.D{{"jobs", JobManageModel}}).All(&allAddresses)
	if err != nil {
		return nil, err
	}
	if len(allAddresses) == 0 {
		return nil, errors.New("no controller machines found")
	}
	apiAddrs := make([]string, 0, len(allAddresses))
	for _, addrs := range allAddresses {
		naddrs := networkAddresses(addrs.Addresses)
		addr, ok := network.SelectControllerAddress(naddrs, false)
		if ok {
			apiAddrs = append(apiAddrs, addr.Value)
		}
	}
	if len(apiAddrs) == 0 {
		return nil, errors.New("no controller machines with addresses found")
	}
	return apiAddrs, nil
}

func appendPort(addrs []string, port int) []string {
	newAddrs := make([]string, len(addrs))
	for i, addr := range addrs {
		newAddrs[i] = net.JoinHostPort(addr, strconv.Itoa(port))
	}
	return newAddrs
}

// Addresses returns the list of cloud-internal addresses that
// can be used to connect to the state.
func (st *State) Addresses() ([]string, error) {
	addrs, err := st.controllerAddresses()
	if err != nil {
		return nil, errors.Trace(err)
	}
	config, err := st.ModelConfig()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return appendPort(addrs, config.StatePort()), nil
}

// APIAddressesFromMachines returns the list of cloud-internal addresses that
// can be used to connect to the state API server.
// This method will be deprecated when API addresses are
// stored independently in their own document.
func (st *State) APIAddressesFromMachines() ([]string, error) {
	addrs, err := st.controllerAddresses()
	if err != nil {
		return nil, errors.Trace(err)
	}
	config, err := st.ModelConfig()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return appendPort(addrs, config.APIPort()), nil
}

const apiHostPortsKey = "apiHostPorts"

type apiHostPortsDoc struct {
	APIHostPorts [][]hostPort `bson:"apihostports"`
}

// SetAPIHostPorts sets the addresses of the API server instances.
// Each server is represented by one element in the top level slice.
func (st *State) SetAPIHostPorts(netHostsPorts [][]network.HostPort) error {
	doc := apiHostPortsDoc{
		APIHostPorts: fromNetworkHostsPorts(netHostsPorts),
	}
	buildTxn := func(attempt int) ([]txn.Op, error) {
		existing, err := st.APIHostPorts()
		if err != nil {
			return nil, err
		}
		op := txn.Op{
			C:  controllersC,
			Id: apiHostPortsKey,
			Assert: bson.D{{
				"apihostports", fromNetworkHostsPorts(existing),
			}},
		}
		if !hostsPortsEqual(netHostsPorts, existing) {
			op.Update = bson.D{{
				"$set", bson.D{{"apihostports", doc.APIHostPorts}},
			}}
		}
		return []txn.Op{op}, nil
	}
	if err := st.run(buildTxn); err != nil {
		return errors.Annotate(err, "cannot set API addresses")
	}
	logger.Debugf("setting API hostPorts: %v", netHostsPorts)
	return nil
}

// APIHostPorts returns the API addresses as set by SetAPIHostPorts.
func (st *State) APIHostPorts() ([][]network.HostPort, error) {
	var doc apiHostPortsDoc
	controllers, closer := st.getCollection(controllersC)
	defer closer()
	err := controllers.Find(bson.D{{"_id", apiHostPortsKey}}).One(&doc)
	if err != nil {
		return nil, err
	}
	return networkHostsPorts(doc.APIHostPorts), nil
}

type DeployerConnectionValues struct {
	StateAddresses []string
	APIAddresses   []string
}

// DeployerConnectionInfo returns the address information necessary for the deployer.
// The function does the expensive operations (getting stuff from mongo) just once.
func (st *State) DeployerConnectionInfo() (*DeployerConnectionValues, error) {
	addrs, err := st.controllerAddresses()
	if err != nil {
		return nil, errors.Trace(err)
	}
	config, err := st.ModelConfig()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return &DeployerConnectionValues{
		StateAddresses: appendPort(addrs, config.StatePort()),
		APIAddresses:   appendPort(addrs, config.APIPort()),
	}, nil
}

// address represents the location of a machine, including metadata
// about what kind of location the address describes.
//
// TODO(dimitern) Make sure we integrate this with other networking
// stuff at some point. We want to use juju-specific network names
// that point to existing documents in the networks collection.
type address struct {
	Value       string `bson:"value"`
	AddressType string `bson:"addresstype"`
	NetworkName string `bson:"networkname,omitempty"`
	Scope       string `bson:"networkscope,omitempty"`
	Origin      string `bson:"origin,omitempty"`
	SpaceName   string `bson:"spacename,omitempty"`
}

// Origin specifies where an address comes from, whether it was reported by a
// provider or by a machine.
type Origin string

const (
	// Address origin unknown.
	OriginUnknown Origin = ""
	// Address comes from a provider.
	OriginProvider Origin = "provider"
	// Address comes from a machine.
	OriginMachine Origin = "machine"
)

// fromNetworkAddress is a convenience helper to create a state type
// out of the network type, here for Address with a given Origin.
func fromNetworkAddress(netAddr network.Address, origin Origin) address {
	return address{
		Value:       netAddr.Value,
		AddressType: string(netAddr.Type),
		NetworkName: netAddr.NetworkName,
		Scope:       string(netAddr.Scope),
		Origin:      string(origin),
		SpaceName:   string(netAddr.SpaceName),
	}
}

// networkAddress is a convenience helper to return the state type
// as network type, here for Address.
func (addr *address) networkAddress() network.Address {
	return network.Address{
		Value:       addr.Value,
		Type:        network.AddressType(addr.AddressType),
		NetworkName: addr.NetworkName,
		Scope:       network.Scope(addr.Scope),
		SpaceName:   network.SpaceName(addr.SpaceName),
	}
}

// fromNetworkAddresses is a convenience helper to create a state type
// out of the network type, here for a slice of Address with a given origin.
func fromNetworkAddresses(netAddrs []network.Address, origin Origin) []address {
	addrs := make([]address, len(netAddrs))
	for i, netAddr := range netAddrs {
		addrs[i] = fromNetworkAddress(netAddr, origin)
	}
	return addrs
}

// networkAddresses is a convenience helper to return the state type
// as network type, here for a slice of Address.
func networkAddresses(addrs []address) []network.Address {
	netAddrs := make([]network.Address, len(addrs))
	for i, addr := range addrs {
		netAddrs[i] = addr.networkAddress()
	}
	return netAddrs
}

// hostPort associates an address with a port. See also network.HostPort,
// from/to which this is transformed.
//
// TODO(dimitern) Make sure we integrate this with other networking
// stuff at some point. We want to use juju-specific network names
// that point to existing documents in the networks collection.
type hostPort struct {
	Value       string `bson:"value"`
	AddressType string `bson:"addresstype"`
	NetworkName string `bson:"networkname,omitempty"`
	Scope       string `bson:"networkscope,omitempty"`
	Port        int    `bson:"port"`
	SpaceName   string `bson:"spacename,omitempty"`
}

// fromNetworkHostPort is a convenience helper to create a state type
// out of the network type, here for HostPort.
func fromNetworkHostPort(netHostPort network.HostPort) hostPort {
	return hostPort{
		Value:       netHostPort.Value,
		AddressType: string(netHostPort.Type),
		NetworkName: netHostPort.NetworkName,
		Scope:       string(netHostPort.Scope),
		Port:        netHostPort.Port,
		SpaceName:   string(netHostPort.SpaceName),
	}
}

// networkHostPort is a convenience helper to return the state type
// as network type, here for HostPort.
func (hp *hostPort) networkHostPort() network.HostPort {
	return network.HostPort{
		Address: network.Address{
			Value:       hp.Value,
			Type:        network.AddressType(hp.AddressType),
			NetworkName: hp.NetworkName,
			Scope:       network.Scope(hp.Scope),
			SpaceName:   network.SpaceName(hp.SpaceName),
		},
		Port: hp.Port,
	}
}

// fromNetworkHostsPorts is a helper to create a state type
// out of the network type, here for a nested slice of HostPort.
func fromNetworkHostsPorts(netHostsPorts [][]network.HostPort) [][]hostPort {
	hsps := make([][]hostPort, len(netHostsPorts))
	for i, netHostPorts := range netHostsPorts {
		hsps[i] = make([]hostPort, len(netHostPorts))
		for j, netHostPort := range netHostPorts {
			hsps[i][j] = fromNetworkHostPort(netHostPort)
		}
	}
	return hsps
}

// networkHostsPorts is a convenience helper to return the state type
// as network type, here for a nested slice of HostPort.
func networkHostsPorts(hsps [][]hostPort) [][]network.HostPort {
	netHostsPorts := make([][]network.HostPort, len(hsps))
	for i, hps := range hsps {
		netHostsPorts[i] = make([]network.HostPort, len(hps))
		for j, hp := range hps {
			netHostsPorts[i][j] = hp.networkHostPort()
		}
	}
	return netHostsPorts
}

// addressEqual checks that two slices of network addresses are equal.
func addressesEqual(a, b []network.Address) bool {
	return reflect.DeepEqual(a, b)
}

// hostsPortsEqual checks that two arrays of network hostports are equal.
func hostsPortsEqual(a, b [][]network.HostPort) bool {
	return reflect.DeepEqual(a, b)
}
<commit_msg>Remove unused state.DeployerConnectionInfo<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"net"
	"reflect"
	"strconv"

	"github.com/juju/errors"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"

	"github.com/juju/juju/network"
)

// controllerAddresses returns the list of internal addresses of the state
// server machines.
func (st *State) controllerAddresses() ([]string, error) {
	ssState := st
	model, err := st.ControllerModel()
	if err != nil {
		return nil, errors.Trace(err)
	}
	if st.ModelTag() != model.ModelTag() {
		// We are not using the controller model, so get one.
		logger.Debugf("getting a controller state connection, current env: %s", st.ModelTag())
		ssState, err = st.ForModel(model.ModelTag())
		if err != nil {
			return nil, errors.Trace(err)
		}
		defer ssState.Close()
		logger.Debugf("ssState env: %s", ssState.ModelTag())
	}

	type addressMachine struct {
		Addresses []address
	}
	var allAddresses []addressMachine
	// TODO(rog) 2013/10/14 index machines on jobs.
	machines, closer := ssState.getCollection(machinesC)
	defer closer()
	err = machines.Find(bson.D{{"jobs", JobManageModel}}).All(&allAddresses)
	if err != nil {
		return nil, err
	}
	if len(allAddresses) == 0 {
		return nil, errors.New("no controller machines found")
	}
	apiAddrs := make([]string, 0, len(allAddresses))
	for _, addrs := range allAddresses {
		naddrs := networkAddresses(addrs.Addresses)
		addr, ok := network.SelectControllerAddress(naddrs, false)
		if ok {
			apiAddrs = append(apiAddrs, addr.Value)
		}
	}
	if len(apiAddrs) == 0 {
		return nil, errors.New("no controller machines with addresses found")
	}
	return apiAddrs, nil
}

func appendPort(addrs []string, port int) []string {
	newAddrs := make([]string, len(addrs))
	for i, addr := range addrs {
		newAddrs[i] = net.JoinHostPort(addr, strconv.Itoa(port))
	}
	return newAddrs
}

// Addresses returns the list of cloud-internal addresses that
// can be used to connect to the state.
func (st *State) Addresses() ([]string, error) {
	addrs, err := st.controllerAddresses()
	if err != nil {
		return nil, errors.Trace(err)
	}
	config, err := st.ModelConfig()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return appendPort(addrs, config.StatePort()), nil
}

// APIAddressesFromMachines returns the list of cloud-internal addresses that
// can be used to connect to the state API server.
// This method will be deprecated when API addresses are
// stored independently in their own document.
func (st *State) APIAddressesFromMachines() ([]string, error) {
	addrs, err := st.controllerAddresses()
	if err != nil {
		return nil, errors.Trace(err)
	}
	config, err := st.ModelConfig()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return appendPort(addrs, config.APIPort()), nil
}

const apiHostPortsKey = "apiHostPorts"

type apiHostPortsDoc struct {
	APIHostPorts [][]hostPort `bson:"apihostports"`
}

// SetAPIHostPorts sets the addresses of the API server instances.
// Each server is represented by one element in the top level slice.
func (st *State) SetAPIHostPorts(netHostsPorts [][]network.HostPort) error {
	doc := apiHostPortsDoc{
		APIHostPorts: fromNetworkHostsPorts(netHostsPorts),
	}
	buildTxn := func(attempt int) ([]txn.Op, error) {
		existing, err := st.APIHostPorts()
		if err != nil {
			return nil, err
		}
		op := txn.Op{
			C:  controllersC,
			Id: apiHostPortsKey,
			Assert: bson.D{{
				"apihostports", fromNetworkHostsPorts(existing),
			}},
		}
		if !hostsPortsEqual(netHostsPorts, existing) {
			op.Update = bson.D{{
				"$set", bson.D{{"apihostports", doc.APIHostPorts}},
			}}
		}
		return []txn.Op{op}, nil
	}
	if err := st.run(buildTxn); err != nil {
		return errors.Annotate(err, "cannot set API addresses")
	}
	logger.Debugf("setting API hostPorts: %v", netHostsPorts)
	return nil
}

// APIHostPorts returns the API addresses as set by SetAPIHostPorts.
func (st *State) APIHostPorts() ([][]network.HostPort, error) {
	var doc apiHostPortsDoc
	controllers, closer := st.getCollection(controllersC)
	defer closer()
	err := controllers.Find(bson.D{{"_id", apiHostPortsKey}}).One(&doc)
	if err != nil {
		return nil, err
	}
	return networkHostsPorts(doc.APIHostPorts), nil
}

// address represents the location of a machine, including metadata
// about what kind of location the address describes.
//
// TODO(dimitern) Make sure we integrate this with other networking
// stuff at some point. We want to use juju-specific network names
// that point to existing documents in the networks collection.
type address struct {
	Value       string `bson:"value"`
	AddressType string `bson:"addresstype"`
	NetworkName string `bson:"networkname,omitempty"`
	Scope       string `bson:"networkscope,omitempty"`
	Origin      string `bson:"origin,omitempty"`
	SpaceName   string `bson:"spacename,omitempty"`
}

// Origin specifies where an address comes from, whether it was reported by a
// provider or by a machine.
type Origin string

const (
	// Address origin unknown.
	OriginUnknown Origin = ""
	// Address comes from a provider.
	OriginProvider Origin = "provider"
	// Address comes from a machine.
	OriginMachine Origin = "machine"
)

// fromNetworkAddress is a convenience helper to create a state type
// out of the network type, here for Address with a given Origin.
func fromNetworkAddress(netAddr network.Address, origin Origin) address {
	return address{
		Value:       netAddr.Value,
		AddressType: string(netAddr.Type),
		NetworkName: netAddr.NetworkName,
		Scope:       string(netAddr.Scope),
		Origin:      string(origin),
		SpaceName:   string(netAddr.SpaceName),
	}
}

// networkAddress is a convenience helper to return the state type
// as network type, here for Address.
func (addr *address) networkAddress() network.Address {
	return network.Address{
		Value:       addr.Value,
		Type:        network.AddressType(addr.AddressType),
		NetworkName: addr.NetworkName,
		Scope:       network.Scope(addr.Scope),
		SpaceName:   network.SpaceName(addr.SpaceName),
	}
}

// fromNetworkAddresses is a convenience helper to create a state type
// out of the network type, here for a slice of Address with a given origin.
func fromNetworkAddresses(netAddrs []network.Address, origin Origin) []address {
	addrs := make([]address, len(netAddrs))
	for i, netAddr := range netAddrs {
		addrs[i] = fromNetworkAddress(netAddr, origin)
	}
	return addrs
}

// networkAddresses is a convenience helper to return the state type
// as network type, here for a slice of Address.
func networkAddresses(addrs []address) []network.Address {
	netAddrs := make([]network.Address, len(addrs))
	for i, addr := range addrs {
		netAddrs[i] = addr.networkAddress()
	}
	return netAddrs
}

// hostPort associates an address with a port. See also network.HostPort,
// from/to which this is transformed.
//
// TODO(dimitern) Make sure we integrate this with other networking
// stuff at some point. We want to use juju-specific network names
// that point to existing documents in the networks collection.
type hostPort struct {
	Value       string `bson:"value"`
	AddressType string `bson:"addresstype"`
	NetworkName string `bson:"networkname,omitempty"`
	Scope       string `bson:"networkscope,omitempty"`
	Port        int    `bson:"port"`
	SpaceName   string `bson:"spacename,omitempty"`
}

// fromNetworkHostPort is a convenience helper to create a state type
// out of the network type, here for HostPort.
func fromNetworkHostPort(netHostPort network.HostPort) hostPort {
	return hostPort{
		Value:       netHostPort.Value,
		AddressType: string(netHostPort.Type),
		NetworkName: netHostPort.NetworkName,
		Scope:       string(netHostPort.Scope),
		Port:        netHostPort.Port,
		SpaceName:   string(netHostPort.SpaceName),
	}
}

// networkHostPort is a convenience helper to return the state type
// as network type, here for HostPort.
func (hp *hostPort) networkHostPort() network.HostPort {
	return network.HostPort{
		Address: network.Address{
			Value:       hp.Value,
			Type:        network.AddressType(hp.AddressType),
			NetworkName: hp.NetworkName,
			Scope:       network.Scope(hp.Scope),
			SpaceName:   network.SpaceName(hp.SpaceName),
		},
		Port: hp.Port,
	}
}

// fromNetworkHostsPorts is a helper to create a state type
// out of the network type, here for a nested slice of HostPort.
func fromNetworkHostsPorts(netHostsPorts [][]network.HostPort) [][]hostPort {
	hsps := make([][]hostPort, len(netHostsPorts))
	for i, netHostPorts := range netHostsPorts {
		hsps[i] = make([]hostPort, len(netHostPorts))
		for j, netHostPort := range netHostPorts {
			hsps[i][j] = fromNetworkHostPort(netHostPort)
		}
	}
	return hsps
}

// networkHostsPorts is a convenience helper to return the state type
// as network type, here for a nested slice of HostPort.
func networkHostsPorts(hsps [][]hostPort) [][]network.HostPort {
	netHostsPorts := make([][]network.HostPort, len(hsps))
	for i, hps := range hsps {
		netHostsPorts[i] = make([]network.HostPort, len(hps))
		for j, hp := range hps {
			netHostsPorts[i][j] = hp.networkHostPort()
		}
	}
	return netHostsPorts
}

// addressEqual checks that two slices of network addresses are equal.
func addressesEqual(a, b []network.Address) bool {
	return reflect.DeepEqual(a, b)
}

// hostsPortsEqual checks that two arrays of network hostports are equal.
func hostsPortsEqual(a, b [][]network.HostPort) bool {
	return reflect.DeepEqual(a, b)
}
<|endoftext|>


<commit_before>// Copyright (c) 2012-2013 Matt Nunogawa @amattn
// This source code is release under the MIT License, http://opensource.org/licenses/MIT

package deeperror

import (
	"fmt"
	"log"
	"net/http"
	"runtime"
	"strings"
)

var gERROR_LOGGING_ENABLED bool

func init() {
	gERROR_LOGGING_ENABLED = false
}

type DeepError struct {
	Num           int64
	Filename      string
	CallingMethod string
	Line          int
	EndUserMsg    string
	DebugMsg      string
	Err           error // inner or source error
	StatusCode    int
	StackTrace    string
}

func New(num int64, endUserMsg string, parentErr error) *DeepError {
	e := new(DeepError)
	e.Num = num
	e.EndUserMsg = endUserMsg
	e.Err = parentErr
	e.StatusCode = http.StatusInternalServerError // default status code...

	gerr, ok := parentErr.(*DeepError)
	if ok {
		if gerr != nil {
			e.StatusCode = gerr.StatusCode
		}
	}

	pc, file, line, ok := runtime.Caller(1)

	if ok {
		e.Line = line
		components := strings.Split(file, "/")
		e.Filename = components[(len(components) - 1)]
		f := runtime.FuncForPC(pc)
		e.CallingMethod = f.Name()
	}

	const size = 1 << 12
	buf := make([]byte, size)
	n := runtime.Stack(buf, false)

	e.StackTrace = string(buf[:n])

	if gERROR_LOGGING_ENABLED {
		log.Print(e)
	}
	return e
}

func NewHTTPError(num int64, endUserMsg string, err error, statusCode int) *DeepError {
	grunwayErrorPtr := New(num, endUserMsg, err)
	grunwayErrorPtr.StatusCode = statusCode
	return grunwayErrorPtr
}

func prependToLines(para, prefix string) string {
	lines := strings.Split(para, "\n")
	for i, line := range lines {
		lines[i] = prefix + line
	}
	return strings.Join(lines, "\n")
}

func (e *DeepError) Error() string {

	parentError := "nil"

	// fmt.Println("THISERR", e.Num, "PARENT ERR", e.Err)

	if e.Err != nil {
		parentError = prependToLines(e.Err.Error(), "-- ")
	}

	return fmt.Sprintln(
		"\n\n-- DeepError",
		e.Num,
		e.StatusCode,
		e.Filename,
		e.CallingMethod,
		"line:", e.Line,
		"\n-- EndUserMsg: ", e.EndUserMsg,
		"\n-- DebugMsg: ", e.DebugMsg,
		"\n-- StackTrace:",
		strings.TrimLeft(prependToLines(e.StackTrace, "-- "), " "),
		"\n-- ParentError:", parentError,
	)
}

func ErrorLoggingEnabled() bool {
	return gERROR_LOGGING_ENABLED
}

type NoErrorsLoggingAction func()

// you can use this method to temporarily disable
func ExecWithoutErrorLogging(action NoErrorsLoggingAction) {
	// this is racy...  I feel ashamed.
	original := gERROR_LOGGING_ENABLED
	gERROR_LOGGING_ENABLED = false
	action()
	gERROR_LOGGING_ENABLED = original
}
<commit_msg>new convenience TODO func<commit_after>// Copyright (c) 2012-2013 Matt Nunogawa @amattn
// This source code is release under the MIT License, http://opensource.org/licenses/MIT

package deeperror

import (
	"fmt"
	"log"
	"net/http"
	"runtime"
	"strings"
)

var gERROR_LOGGING_ENABLED bool

func init() {
	gERROR_LOGGING_ENABLED = false
}

type DeepError struct {
	Num           int64
	Filename      string
	CallingMethod string
	Line          int
	EndUserMsg    string
	DebugMsg      string
	Err           error // inner or source error
	StatusCode    int
	StackTrace    string
}

func New(num int64, endUserMsg string, parentErr error) *DeepError {
	e := new(DeepError)
	e.Num = num
	e.EndUserMsg = endUserMsg
	e.Err = parentErr
	e.StatusCode = http.StatusInternalServerError // default status code...

	gerr, ok := parentErr.(*DeepError)
	if ok {
		if gerr != nil {
			e.StatusCode = gerr.StatusCode
		}
	}

	pc, file, line, ok := runtime.Caller(1)

	if ok {
		e.Line = line
		components := strings.Split(file, "/")
		e.Filename = components[(len(components) - 1)]
		f := runtime.FuncForPC(pc)
		e.CallingMethod = f.Name()
	}

	const size = 1 << 12
	buf := make([]byte, size)
	n := runtime.Stack(buf, false)

	e.StackTrace = string(buf[:n])

	if gERROR_LOGGING_ENABLED {
		log.Print(e)
	}
	return e
}

func NewHTTPError(num int64, endUserMsg string, err error, statusCode int) *DeepError {
	grunwayErrorPtr := New(num, endUserMsg, err)
	grunwayErrorPtr.StatusCode = statusCode
	return grunwayErrorPtr
}

func NewTODOError(num int64) *DeepError {
	grunwayErrorPtr := New(num, "TODO", nil)
	return grunwayErrorPtr
}

func prependToLines(para, prefix string) string {
	lines := strings.Split(para, "\n")
	for i, line := range lines {
		lines[i] = prefix + line
	}
	return strings.Join(lines, "\n")
}

func (e *DeepError) Error() string {

	parentError := "nil"

	// fmt.Println("THISERR", e.Num, "PARENT ERR", e.Err)

	if e.Err != nil {
		parentError = prependToLines(e.Err.Error(), "-- ")
	}

	return fmt.Sprintln(
		"\n\n-- DeepError",
		e.Num,
		e.StatusCode,
		e.Filename,
		e.CallingMethod,
		"line:", e.Line,
		"\n-- EndUserMsg: ", e.EndUserMsg,
		"\n-- DebugMsg: ", e.DebugMsg,
		"\n-- StackTrace:",
		strings.TrimLeft(prependToLines(e.StackTrace, "-- "), " "),
		"\n-- ParentError:", parentError,
	)
}

func ErrorLoggingEnabled() bool {
	return gERROR_LOGGING_ENABLED
}

type NoErrorsLoggingAction func()

// you can use this method to temporarily disable
func ExecWithoutErrorLogging(action NoErrorsLoggingAction) {
	// this is racy...  I feel ashamed.
	original := gERROR_LOGGING_ENABLED
	gERROR_LOGGING_ENABLED = false
	action()
	gERROR_LOGGING_ENABLED = original
}
<|endoftext|>


<commit_before>package worker

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path"
	"time"

	"github.com/cloudfoundry-incubator/garden"
	"github.com/concourse/atc"
	"github.com/concourse/baggageclaim"
	"github.com/pivotal-golang/lager"
)

const ephemeralPropertyName = "concourse:ephemeral"
const volumePropertyName = "concourse:volumes"
const volumeMountsPropertyName = "concourse:volume-mounts"

type releasable interface {
	Release(*time.Duration)
}

type gardenContainerSpecFactory struct {
	logger             lager.Logger
	baggageclaimClient baggageclaim.Client
	imageFetcher       ImageFetcher
	releaseAfterCreate []releasable
	db                 GardenWorkerDB
}

func NewGardenContainerSpecFactory(logger lager.Logger, baggageclaimClient baggageclaim.Client, imageFetcher ImageFetcher, db GardenWorkerDB) gardenContainerSpecFactory {
	return gardenContainerSpecFactory{
		logger:             logger,
		baggageclaimClient: baggageclaimClient,
		imageFetcher:       imageFetcher,
		releaseAfterCreate: []releasable{},
		db:                 db,
	}
}

func (factory *gardenContainerSpecFactory) BuildContainerSpec(
	spec ContainerSpec,
	resourceTypes []atc.WorkerResourceType,
	workerTags atc.Tags,
	cancel <-chan os.Signal,
	delegate ImageFetchingDelegate,
	id Identifier,
	metadata Metadata,
	workerClient Client,
	customTypes atc.ResourceTypes,
) (garden.ContainerSpec, error) {
	resourceTypeContainerSpec, ok := spec.(ResourceTypeContainerSpec)
	if ok {
		for _, customType := range customTypes {
			if customType.Name == resourceTypeContainerSpec.Type {
				customTypes = customTypes.Without(resourceTypeContainerSpec.Type)

				resourceTypeContainerSpec.ImageResourcePointer = &atc.TaskImageConfig{
					Source: customType.Source,
					Type:   customType.Type,
				}

				spec = resourceTypeContainerSpec
			}
		}
	}

	var volumeHandles []string
	var user string

	imageResourceConfig, hasImageResource := spec.ImageResource()
	var gardenSpec garden.ContainerSpec
	if hasImageResource {
		image, err := factory.imageFetcher.FetchImage(
			factory.logger,
			imageResourceConfig,
			cancel,
			id,
			metadata,
			delegate,
			workerClient,
			workerTags,
			customTypes,
		)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		imageVolume := image.Volume()

		volumeHandles = append(volumeHandles, imageVolume.Handle())
		factory.releaseAfterCreate = append(factory.releaseAfterCreate, image)
		user = image.Metadata().User

		gardenSpec = garden.ContainerSpec{
			Properties: garden.Properties{},
			RootFSPath: path.Join(imageVolume.Path(), "rootfs"),
			Env:        image.Metadata().Env,
		}
	} else {
		gardenSpec = garden.ContainerSpec{
			Properties: garden.Properties{},
		}
	}

	volumeMounts := map[string]string{}

dance:
	switch s := spec.(type) {
	case ResourceTypeContainerSpec:
		if len(s.Mounts) > 0 && s.Cache.Volume != nil {
			return gardenSpec, errors.New("a container may not have mounts and a cache")
		}

		gardenSpec.Privileged = true
		gardenSpec.Env = append(gardenSpec.Env, s.Env...)

		if s.Ephemeral {
			gardenSpec.Properties[ephemeralPropertyName] = "true"
		}

		if s.Cache.Volume != nil && s.Cache.MountPath != "" {
			gardenSpec.BindMounts = []garden.BindMount{
				{
					SrcPath: s.Cache.Volume.Path(),
					DstPath: s.Cache.MountPath,
					Mode:    garden.BindMountModeRW,
				},
			}

			volumeHandles = append(volumeHandles, s.Cache.Volume.Handle())
			volumeMounts[s.Cache.Volume.Handle()] = s.Cache.MountPath
		}

		var err error
		var newVolumeHandles []string
		var newVolumeMounts map[string]string
		gardenSpec, newVolumeHandles, newVolumeMounts, err = factory.createVolumes(gardenSpec, s.Mounts)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		for _, h := range newVolumeHandles {
			volumeHandles = append(volumeHandles, h)
		}

		for k, v := range newVolumeMounts {
			volumeMounts[k] = v
		}

		if s.ImageResourcePointer == nil {
			for _, t := range resourceTypes {
				if t.Type == s.Type {
					gardenSpec.RootFSPath = t.Image
					break dance
				}
			}

			return garden.ContainerSpec{}, ErrUnsupportedResourceType
		}

		break dance
	case TaskContainerSpec:
		if s.ImageResourcePointer == nil {
			gardenSpec.RootFSPath = s.Image
		}

		gardenSpec.Privileged = s.Privileged

		var err error
		var newVolumeHandles []string
		var newVolumeMounts map[string]string
		gardenSpec, newVolumeHandles, newVolumeMounts, err = factory.createVolumes(gardenSpec, s.Inputs)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		for _, h := range newVolumeHandles {
			volumeHandles = append(volumeHandles, h)
		}

		for k, v := range newVolumeMounts {
			volumeMounts[k] = v
		}

		for _, mount := range s.Outputs {
			volume := mount.Volume
			gardenSpec.BindMounts = append(gardenSpec.BindMounts, garden.BindMount{
				SrcPath: volume.Path(),
				DstPath: mount.MountPath,
				Mode:    garden.BindMountModeRW,
			})

			volumeHandles = append(volumeHandles, volume.Handle())
			volumeMounts[volume.Handle()] = mount.MountPath
		}

		break dance
	default:
		return garden.ContainerSpec{}, fmt.Errorf("unknown container spec type: %T (%#v)", s, s)
	}

	if len(volumeHandles) > 0 {
		volumesJSON, err := json.Marshal(volumeHandles)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		gardenSpec.Properties[volumePropertyName] = string(volumesJSON)

		mountsJSON, err := json.Marshal(volumeMounts)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		gardenSpec.Properties[volumeMountsPropertyName] = string(mountsJSON)
	}

	gardenSpec.Properties["user"] = user

	return gardenSpec, nil
}

func (factory *gardenContainerSpecFactory) ReleaseVolumes() {
	for _, cow := range factory.releaseAfterCreate {
		cow.Release(nil)
	}
}

func (factory *gardenContainerSpecFactory) createVolumes(containerSpec garden.ContainerSpec, mounts []VolumeMount) (garden.ContainerSpec, []string, map[string]string, error) {
	var volumeHandles []string
	volumeMounts := map[string]string{}

	for _, mount := range mounts {
		cowVolume, err := factory.baggageclaimClient.CreateVolume(factory.logger, baggageclaim.VolumeSpec{
			Strategy: baggageclaim.COWStrategy{
				Parent: mount.Volume,
			},
			Privileged: containerSpec.Privileged,
			TTL:        VolumeTTL,
		})
		if err != nil {
			return containerSpec, []string{}, map[string]string{}, err
		}

		factory.releaseAfterCreate = append(factory.releaseAfterCreate, cowVolume)

		err = factory.db.InsertCOWVolume(mount.Volume.Handle(), cowVolume.Handle(), VolumeTTL)
		if err != nil {
			return containerSpec, []string{}, map[string]string{}, err
		}

		containerSpec.BindMounts = append(containerSpec.BindMounts, garden.BindMount{
			SrcPath: cowVolume.Path(),
			DstPath: mount.MountPath,
			Mode:    garden.BindMountModeRW,
		})

		volumeHandles = append(volumeHandles, cowVolume.Handle())
		volumeMounts[cowVolume.Handle()] = mount.MountPath

		factory.logger.Info("created-cow-volume", lager.Data{
			"original-volume-handle": mount.Volume.Handle(),
			"cow-volume-handle":      cowVolume.Handle(),
		})
	}

	return containerSpec, volumeHandles, volumeMounts, nil
}
<commit_msg>cow -> releasable<commit_after>package worker

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path"
	"time"

	"github.com/cloudfoundry-incubator/garden"
	"github.com/concourse/atc"
	"github.com/concourse/baggageclaim"
	"github.com/pivotal-golang/lager"
)

const ephemeralPropertyName = "concourse:ephemeral"
const volumePropertyName = "concourse:volumes"
const volumeMountsPropertyName = "concourse:volume-mounts"

type releasable interface {
	Release(*time.Duration)
}

type gardenContainerSpecFactory struct {
	logger             lager.Logger
	baggageclaimClient baggageclaim.Client
	imageFetcher       ImageFetcher
	releaseAfterCreate []releasable
	db                 GardenWorkerDB
}

func NewGardenContainerSpecFactory(logger lager.Logger, baggageclaimClient baggageclaim.Client, imageFetcher ImageFetcher, db GardenWorkerDB) gardenContainerSpecFactory {
	return gardenContainerSpecFactory{
		logger:             logger,
		baggageclaimClient: baggageclaimClient,
		imageFetcher:       imageFetcher,
		releaseAfterCreate: []releasable{},
		db:                 db,
	}
}

func (factory *gardenContainerSpecFactory) BuildContainerSpec(
	spec ContainerSpec,
	resourceTypes []atc.WorkerResourceType,
	workerTags atc.Tags,
	cancel <-chan os.Signal,
	delegate ImageFetchingDelegate,
	id Identifier,
	metadata Metadata,
	workerClient Client,
	customTypes atc.ResourceTypes,
) (garden.ContainerSpec, error) {
	resourceTypeContainerSpec, ok := spec.(ResourceTypeContainerSpec)
	if ok {
		for _, customType := range customTypes {
			if customType.Name == resourceTypeContainerSpec.Type {
				customTypes = customTypes.Without(resourceTypeContainerSpec.Type)

				resourceTypeContainerSpec.ImageResourcePointer = &atc.TaskImageConfig{
					Source: customType.Source,
					Type:   customType.Type,
				}

				spec = resourceTypeContainerSpec
			}
		}
	}

	var volumeHandles []string
	var user string

	imageResourceConfig, hasImageResource := spec.ImageResource()
	var gardenSpec garden.ContainerSpec
	if hasImageResource {
		image, err := factory.imageFetcher.FetchImage(
			factory.logger,
			imageResourceConfig,
			cancel,
			id,
			metadata,
			delegate,
			workerClient,
			workerTags,
			customTypes,
		)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		imageVolume := image.Volume()

		volumeHandles = append(volumeHandles, imageVolume.Handle())
		factory.releaseAfterCreate = append(factory.releaseAfterCreate, image)
		user = image.Metadata().User

		gardenSpec = garden.ContainerSpec{
			Properties: garden.Properties{},
			RootFSPath: path.Join(imageVolume.Path(), "rootfs"),
			Env:        image.Metadata().Env,
		}
	} else {
		gardenSpec = garden.ContainerSpec{
			Properties: garden.Properties{},
		}
	}

	volumeMounts := map[string]string{}

dance:
	switch s := spec.(type) {
	case ResourceTypeContainerSpec:
		if len(s.Mounts) > 0 && s.Cache.Volume != nil {
			return gardenSpec, errors.New("a container may not have mounts and a cache")
		}

		gardenSpec.Privileged = true
		gardenSpec.Env = append(gardenSpec.Env, s.Env...)

		if s.Ephemeral {
			gardenSpec.Properties[ephemeralPropertyName] = "true"
		}

		if s.Cache.Volume != nil && s.Cache.MountPath != "" {
			gardenSpec.BindMounts = []garden.BindMount{
				{
					SrcPath: s.Cache.Volume.Path(),
					DstPath: s.Cache.MountPath,
					Mode:    garden.BindMountModeRW,
				},
			}

			volumeHandles = append(volumeHandles, s.Cache.Volume.Handle())
			volumeMounts[s.Cache.Volume.Handle()] = s.Cache.MountPath
		}

		var err error
		var newVolumeHandles []string
		var newVolumeMounts map[string]string
		gardenSpec, newVolumeHandles, newVolumeMounts, err = factory.createVolumes(gardenSpec, s.Mounts)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		for _, h := range newVolumeHandles {
			volumeHandles = append(volumeHandles, h)
		}

		for k, v := range newVolumeMounts {
			volumeMounts[k] = v
		}

		if s.ImageResourcePointer == nil {
			for _, t := range resourceTypes {
				if t.Type == s.Type {
					gardenSpec.RootFSPath = t.Image
					break dance
				}
			}

			return garden.ContainerSpec{}, ErrUnsupportedResourceType
		}

		break dance
	case TaskContainerSpec:
		if s.ImageResourcePointer == nil {
			gardenSpec.RootFSPath = s.Image
		}

		gardenSpec.Privileged = s.Privileged

		var err error
		var newVolumeHandles []string
		var newVolumeMounts map[string]string
		gardenSpec, newVolumeHandles, newVolumeMounts, err = factory.createVolumes(gardenSpec, s.Inputs)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		for _, h := range newVolumeHandles {
			volumeHandles = append(volumeHandles, h)
		}

		for k, v := range newVolumeMounts {
			volumeMounts[k] = v
		}

		for _, mount := range s.Outputs {
			volume := mount.Volume
			gardenSpec.BindMounts = append(gardenSpec.BindMounts, garden.BindMount{
				SrcPath: volume.Path(),
				DstPath: mount.MountPath,
				Mode:    garden.BindMountModeRW,
			})

			volumeHandles = append(volumeHandles, volume.Handle())
			volumeMounts[volume.Handle()] = mount.MountPath
		}

		break dance
	default:
		return garden.ContainerSpec{}, fmt.Errorf("unknown container spec type: %T (%#v)", s, s)
	}

	if len(volumeHandles) > 0 {
		volumesJSON, err := json.Marshal(volumeHandles)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		gardenSpec.Properties[volumePropertyName] = string(volumesJSON)

		mountsJSON, err := json.Marshal(volumeMounts)
		if err != nil {
			return garden.ContainerSpec{}, err
		}

		gardenSpec.Properties[volumeMountsPropertyName] = string(mountsJSON)
	}

	gardenSpec.Properties["user"] = user

	return gardenSpec, nil
}

func (factory *gardenContainerSpecFactory) ReleaseVolumes() {
	for _, releasable := range factory.releaseAfterCreate {
		releasable.Release(nil)
	}
}

func (factory *gardenContainerSpecFactory) createVolumes(containerSpec garden.ContainerSpec, mounts []VolumeMount) (garden.ContainerSpec, []string, map[string]string, error) {
	var volumeHandles []string
	volumeMounts := map[string]string{}

	for _, mount := range mounts {
		cowVolume, err := factory.baggageclaimClient.CreateVolume(factory.logger, baggageclaim.VolumeSpec{
			Strategy: baggageclaim.COWStrategy{
				Parent: mount.Volume,
			},
			Privileged: containerSpec.Privileged,
			TTL:        VolumeTTL,
		})
		if err != nil {
			return containerSpec, []string{}, map[string]string{}, err
		}

		factory.releaseAfterCreate = append(factory.releaseAfterCreate, cowVolume)

		err = factory.db.InsertCOWVolume(mount.Volume.Handle(), cowVolume.Handle(), VolumeTTL)
		if err != nil {
			return containerSpec, []string{}, map[string]string{}, err
		}

		containerSpec.BindMounts = append(containerSpec.BindMounts, garden.BindMount{
			SrcPath: cowVolume.Path(),
			DstPath: mount.MountPath,
			Mode:    garden.BindMountModeRW,
		})

		volumeHandles = append(volumeHandles, cowVolume.Handle())
		volumeMounts[cowVolume.Handle()] = mount.MountPath

		factory.logger.Info("created-cow-volume", lager.Data{
			"original-volume-handle": mount.Volume.Handle(),
			"cow-volume-handle":      cowVolume.Handle(),
		})
	}

	return containerSpec, volumeHandles, volumeMounts, nil
}
<|endoftext|>