<commit_before>package lib

import (
	"fmt"
	"os"
	"time"

	oss "github.com/aliyun/aliyun-oss-go-sdk/oss"
	. "gopkg.in/check.v1"
)

func (s *OssutilCommandSuite) TestRemoveObject(c *C) {
	bucket := bucketNameMB

	// put object
	object := "TestRemoveObject"
	s.putObject(bucket, object, uploadFileName, c)
	time.Sleep(2 * sleepTime)

	// list object
	objects := s.listObjects(bucket, object, "ls - ", c)
	c.Assert(len(objects), Equals, 1)
	c.Assert(objects[0], Equals, object)

	// remove object
	s.removeObjects(bucket, object, false, true, c)

	// list object
	objects = s.listObjects(bucket, object, "ls - ", c)
	c.Assert(len(objects), Equals, 0)
}

func (s *OssutilCommandSuite) TestRemoveObjects(c *C) {
	bucket := bucketNamePrefix + "rmb1"
	s.putBucket(bucket, c)
	//time.Sleep(14 * time.Second)

	// put object
	num := 2
	objectNames := []string{}
	for i := 0; i < num; i++ {
		object := fmt.Sprintf("remove%d", i)
		s.putObject(bucket, object, uploadFileName, c)
		objectNames = append(objectNames, object)
	}
	time.Sleep(2 * sleepTime)

	command := "rm"
	args := []string{CloudURLToString(bucket, "")}
	str := ""
	ok := true
	options := OptionMapType{
		"endpoint":        &str,
		"accessKeyID":     &str,
		"accessKeySecret": &str,
		"stsToken":        &str,
		"configFile":      &configFile,
		"bucket":          &ok,
		"force":           &ok,
	}
	_, err := cm.RunCommand(command, args, options)
	c.Assert(err, NotNil)

	// list object
	objects := s.listObjects(bucket, "", "ls - ", c)
	c.Assert(len(objects), Equals, num)

	// "rm oss://bucket/ -r"
	// remove object
	s.removeObjects(bucket, "", true, false, c)

	objects = s.listObjects(bucket, "", "ls - ", c)
	c.Assert(len(objects), Equals, num)

	// "rm oss://bucket/prefix -r -f"
	// remove object
	s.removeObjects(bucket, "re", true, true, c)
	time.Sleep(3 * sleepTime)

	// list object
	objects = s.listObjects(bucket, "", "ls - ", c)
	c.Assert(len(objects), Equals, 0)

	//reput objects and delete bucket
	for i := 0; i < num; i++ {
		object := fmt.Sprintf("remove%d", i)
		s.putObject(bucket, object, uploadFileName, c)
	}

	// list buckets
	buckets := s.listBuckets(false, c)
	c.Assert(FindPos(bucket, buckets) != -1, Equals, true)

	// error remove bucket with config
	cfile := "ossutil_test.config_boto"
	data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\n[Bucket-Endpoint]\n%s=%s[Bucket-Cname]\n%s=%s", "abc", "def", "ghi", bucket, "abc", bucket, "abc")
	s.createFile(cfile, data, c)

	options = OptionMapType{
		"endpoint":        &str,
		"accessKeyID":     &str,
		"accessKeySecret": &str,
		"stsToken":        &str,
		"configFile":      &cfile,
		"recursive":       &ok,
		"bucket":          &ok,
		"force":           &ok,
	}
	showElapse, err := cm.RunCommand(command, args, options)
	c.Assert(err, NotNil)

	options = OptionMapType{
		"endpoint":        &endpoint,
		"accessKeyID":     &accessKeyID,
		"accessKeySecret": &accessKeySecret,
		"stsToken":        &str,
		"configFile":      &cfile,
		"recursive":       &ok,
		"bucket":          &ok,
		"force":           &ok,
	}
	showElapse, err = cm.RunCommand(command, args, options)
	c.Assert(err, IsNil)
	c.Assert(showElapse, Equals, true)

	_ = os.Remove(cfile)
	time.Sleep(2 * 7 * time.Second)

	// list buckets
	buckets = s.listBuckets(false, c)
	c.Assert(FindPos(bucket, buckets) == -1, Equals, true)
}

func (s *OssutilCommandSuite) TestRemoveObjectBucketOption(c *C) {
	bucket := bucketNameExist

	object := "test_object"
	command := "rm"
	args := []string{CloudURLToString(bucket, object)}
	str := ""
	ok := true
	options := OptionMapType{
		"endpoint":        &str,
		"accessKeyID":     &str,
		"accessKeySecret": &str,
		"stsToken":        &str,
		"configFile":      &configFile,
		"bucket":          &ok,
		"force":           &ok,
	}
	_, err := cm.RunCommand(command, args, options)
	c.Assert(err, NotNil)

	// list buckets
	buckets := s.listBuckets(false, c)
	c.Assert(FindPos(bucket, buckets) != -1, Equals, true)
}

func (s *OssutilCommandSuite) TestErrRemove(c *C) {
	bucket := bucketNameExist

	showElapse, err := s.rawRemove([]string{"oss://"}, false, true, true)
	c.Assert(err, NotNil)
	c.Assert(showElapse, Equals, false)

	showElapse, err = s.rawRemove([]string{"./"}, false, true, true)
	c.Assert(err, NotNil)
	c.Assert(showElapse, Equals, false)

	showElapse, err = s.rawRemove([]string{CloudURLToString(bucket, "")}, false, true, false)
	c.Assert(err, NotNil)
	c.Assert(showElapse, Equals, false)

    showElapse, err = s.rawRemove([]string{"oss:///object"}, false, true, false)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)

    // remove bucket without force
	showElapse, err = s.rawRemove([]string{CloudURLToString(bucket, "")}, false, false, true)
	c.Assert(err, IsNil)
	c.Assert(showElapse, Equals, true)

	bucketStat := s.getStat(bucket, "", c)
	c.Assert(bucketStat[StatName], Equals, bucket)

	// batch delete not exist objects
    object := "batch_delete_notexst_object"
    showElapse, err = s.rawRemove([]string{CloudURLToString(bucket, object)}, true, true, false)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    // clear not exist bucket
    bucketName := bucketNamePrefix + "rmnotexist"
    showElapse, err = s.rawRemove([]string{CloudURLToString(bucketName, "")}, true, true, false)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)

    // test oss batch delete not exist objects
    objects := []string{}
    ossBucket, err := removeCommand.command.ossBucket(bucket)
    c.Assert(err, IsNil)
    num, err := removeCommand.ossBatchDeleteObjectsRetry(ossBucket, objects)
    c.Assert(err, IsNil)
    c.Assert(num, Equals, 0)
}

func (s *OssutilCommandSuite) TestErrDeleteObject(c *C) {
	bucketName := bucketNameNotExist

	bucket, err := removeCommand.command.ossBucket(bucketName)
	c.Assert(err, IsNil)

	object := "object"
	err = removeCommand.ossDeleteObjectRetry(bucket, object)
	c.Assert(err, NotNil)

	_, err = removeCommand.ossBatchDeleteObjectsRetry(bucket, []string{object})
	c.Assert(err, NotNil)
}

func (s *OssutilCommandSuite) TestAllTypeObject(c *C) {
	bucketName := bucketNameMB

	s.clearAllMultipartInBucket(bucketName, c)

	normal_object := "TestAllTypeObject"
	s.putObject(bucketName, normal_object, uploadFileName, c)
	//time.Sleep(2 * sleepTime)

	object := "TestMultipartObjectRm"
	s.putObject(bucketName, object, uploadFileName, c)
	//time.Sleep(2 * sleepTime)

	objects := s.listObjects(bucketName, object, "ls - ", c)
	c.Assert(len(objects), Equals, 1)
	c.Assert(objects[0], Equals, object)

	bucket, err := copyCommand.command.ossBucket(bucketName)
	for i := 0; i < 20; i++ {
		_, err = bucket.InitiateMultipartUpload(object)
		c.Assert(err, IsNil)
	}

	lmr, e := bucket.ListMultipartUploads(oss.Prefix(object))
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 20)

	_, e = s.removeWrapper("rm -arf", bucketName, object, c)
	c.Assert(e, IsNil)

	lmr, e = bucket.ListMultipartUploads(oss.Prefix(object))
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 0)

	// list normal_object
	objects = s.listObjects(bucketName, normal_object, "ls - ", c)
	c.Assert(len(objects), Equals, 1)
	c.Assert(objects[0], Equals, normal_object)
}

func (s *OssutilCommandSuite) TestMultipartUpload(c *C) {
	bucketName := bucketNameMB

	s.clearAllMultipartInBucket(bucketName, c)

	// put object
	object := "TestMultipartObject"
	s.putObject(bucketName, object, uploadFileName, c)
	time.Sleep(2 * sleepTime)

	// list object
	objects := s.listObjects(bucketName, object, "ls - ", c)
	c.Assert(len(objects), Equals, 1)
	c.Assert(objects[0], Equals, object)

	bucket, err := copyCommand.command.ossBucket(bucketName)
	for i := 0; i < 20; i++ {
		_, err = bucket.InitiateMultipartUpload(object)
		c.Assert(err, IsNil)
	}

	lmr, e := bucket.ListMultipartUploads(oss.Prefix(object))
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 20)

	_, e = s.removeWrapper("rm -mrf", bucketName, object, c)
	c.Assert(e, IsNil)

	lmr, e = bucket.ListMultipartUploads(oss.Prefix(object))
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 0)

	obj := "TestMultipartObjectUploads"
	s.putObject(bucketName, obj, uploadFileName, c)
	time.Sleep(4 * sleepTime)

	for i := 0; i < 20; i++ {
		_, err = bucket.InitiateMultipartUpload(obj)
		c.Assert(err, IsNil)
	}
	_, e = s.removeWrapper("rm -mrf", bucketName, "", c)
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 0)
}

func (s *OssutilCommandSuite) TestMultipartUpload_Prefix(c *C) {
	bucketName := bucketNameMB
	bucket, err := copyCommand.command.ossBucket(bucketName)
	c.Assert(err, IsNil)

	s.clearAllMultipartInBucket(bucketName, c)

	object := "TestMultipartObject"
	s.putObject(bucketName, object, uploadFileName, c)
	time.Sleep(2 * sleepTime)

	object1 := "TestMultipartObject" + "prefix"
	s.putObject(bucketName, object1, uploadFileName, c)
	time.Sleep(2 * sleepTime)

	object2 := "TestMultipartObject" + "/dir/test"
	s.putObject(bucketName, object2, uploadFileName, c)
	time.Sleep(2 * sleepTime)

	// list object
	objects := s.listObjects(bucketName, object, "ls - ", c)
	c.Assert(len(objects), Equals, 3)

	for i := 0; i < 20; i++ {
		_, err = bucket.InitiateMultipartUpload(object)
		c.Assert(err, IsNil)
	}

	for i := 0; i < 20; i++ {
		_, err = bucket.InitiateMultipartUpload(object1)
		c.Assert(err, IsNil)
	}

	for i := 0; i < 20; i++ {
		_, err = bucket.InitiateMultipartUpload(object2)
		c.Assert(err, IsNil)
	}

	lmr, e := bucket.ListMultipartUploads(oss.Prefix(object))
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 20*3)

	_, e = s.removeWrapper("rm -mrf", bucketName, "", c)
	c.Assert(e, IsNil)

	lmr, e = bucket.ListMultipartUploads(oss.Prefix(object))
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 0)
}

func (s *OssutilCommandSuite) TestMultipartError(c *C) {
	bucketName := bucketNameMB
	object := "TestMultipartError"

	_, e := s.removeWrapper("rm -mb", bucketName, object, c)
	c.Assert(e, NotNil)

	_, e = s.removeWrapper("rm -mf", bucketName, "", c)
	c.Assert(e, NotNil)
}

func (s *OssutilCommandSuite) TestAllTypeError(c *C) {
	bucketName := bucketNameMB
	object := "random"

	_, e := s.removeWrapper("rm -ab", bucketName, object, c)
	c.Assert(e, NotNil)
}
<commit_msg>modify case<commit_after>package lib

import (
	"fmt"
	"os"
	"time"

	oss "github.com/aliyun/aliyun-oss-go-sdk/oss"
	. "gopkg.in/check.v1"
)

func (s *OssutilCommandSuite) TestRemoveObject(c *C) {
	bucket := bucketNameMB

	// put object
	object := "TestRemoveObject"
	s.putObject(bucket, object, uploadFileName, c)
	time.Sleep(2 * sleepTime)

	// list object
	objects := s.listObjects(bucket, object, "ls - ", c)
	c.Assert(len(objects), Equals, 1)
	c.Assert(objects[0], Equals, object)

	// remove object
	s.removeObjects(bucket, object, false, true, c)

	// list object
	objects = s.listObjects(bucket, object, "ls - ", c)
	c.Assert(len(objects), Equals, 0)
}

func (s *OssutilCommandSuite) TestRemoveObjects(c *C) {
	bucket := bucketNamePrefix + "rmb1"
	s.putBucket(bucket, c)
	//time.Sleep(14 * time.Second)

	// put object
	num := 2
	objectNames := []string{}
	for i := 0; i < num; i++ {
		object := fmt.Sprintf("remove%d", i)
		s.putObject(bucket, object, uploadFileName, c)
		objectNames = append(objectNames, object)
	}
	time.Sleep(2 * sleepTime)

	command := "rm"
	args := []string{CloudURLToString(bucket, "")}
	str := ""
	ok := true
	options := OptionMapType{
		"endpoint":        &str,
		"accessKeyID":     &str,
		"accessKeySecret": &str,
		"stsToken":        &str,
		"configFile":      &configFile,
		"bucket":          &ok,
		"force":           &ok,
	}
	_, err := cm.RunCommand(command, args, options)
	c.Assert(err, NotNil)

	// list object
	objects := s.listObjects(bucket, "", "ls - ", c)
	c.Assert(len(objects), Equals, num)

	// "rm oss://bucket/ -r"
	// remove object
	s.removeObjects(bucket, "", true, false, c)

	objects = s.listObjects(bucket, "", "ls - ", c)
	c.Assert(len(objects), Equals, num)

	// "rm oss://bucket/prefix -r -f"
	// remove object
	s.removeObjects(bucket, "re", true, true, c)
	time.Sleep(3 * sleepTime)

	// list object
	objects = s.listObjects(bucket, "", "ls - ", c)
	c.Assert(len(objects), Equals, 0)

	//reput objects and delete bucket
	for i := 0; i < num; i++ {
		object := fmt.Sprintf("remove%d", i)
		s.putObject(bucket, object, uploadFileName, c)
	}

	// list buckets
	buckets := s.listBuckets(false, c)
	c.Assert(FindPos(bucket, buckets) != -1, Equals, true)

	// error remove bucket with config
	cfile := "ossutil_test.config_boto"
	data := fmt.Sprintf("[Credentials]\nendpoint=%s\naccessKeyID=%s\naccessKeySecret=%s\n[Bucket-Endpoint]\n%s=%s[Bucket-Cname]\n%s=%s", "abc", "def", "ghi", bucket, "abc", bucket, "abc")
	s.createFile(cfile, data, c)

	options = OptionMapType{
		"endpoint":        &str,
		"accessKeyID":     &str,
		"accessKeySecret": &str,
		"stsToken":        &str,
		"configFile":      &cfile,
		"recursive":       &ok,
		"bucket":          &ok,
		"force":           &ok,
	}
	showElapse, err := cm.RunCommand(command, args, options)
	c.Assert(err, NotNil)

	options = OptionMapType{
		"endpoint":        &endpoint,
		"accessKeyID":     &accessKeyID,
		"accessKeySecret": &accessKeySecret,
		"stsToken":        &str,
		"configFile":      &cfile,
		"recursive":       &ok,
		"bucket":          &ok,
		"force":           &ok,
	}
	showElapse, err = cm.RunCommand(command, args, options)
	c.Assert(err, IsNil)
	c.Assert(showElapse, Equals, true)

	_ = os.Remove(cfile)
	time.Sleep(2 * 7 * time.Second)

	// list buckets
	buckets = s.listBuckets(false, c)
	c.Assert(FindPos(bucket, buckets) == -1, Equals, true)
}

func (s *OssutilCommandSuite) TestRemoveObjectBucketOption(c *C) {
	bucket := bucketNameExist

	object := "test_object"
	command := "rm"
	args := []string{CloudURLToString(bucket, object)}
	str := ""
	ok := true
	options := OptionMapType{
		"endpoint":        &str,
		"accessKeyID":     &str,
		"accessKeySecret": &str,
		"stsToken":        &str,
		"configFile":      &configFile,
		"bucket":          &ok,
		"force":           &ok,
	}
	_, err := cm.RunCommand(command, args, options)
	c.Assert(err, NotNil)

	// list buckets
	buckets := s.listBuckets(false, c)
	c.Assert(FindPos(bucket, buckets) != -1, Equals, true)
}

func (s *OssutilCommandSuite) TestErrRemove(c *C) {
	bucket := bucketNameExist

	showElapse, err := s.rawRemove([]string{"oss://"}, false, true, true)
	c.Assert(err, NotNil)
	c.Assert(showElapse, Equals, false)

	showElapse, err = s.rawRemove([]string{"./"}, false, true, true)
	c.Assert(err, NotNil)
	c.Assert(showElapse, Equals, false)

	showElapse, err = s.rawRemove([]string{CloudURLToString(bucket, "")}, false, true, false)
	c.Assert(err, NotNil)
	c.Assert(showElapse, Equals, false)

    showElapse, err = s.rawRemove([]string{"oss:///object"}, false, true, false)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)

    // remove bucket without force
	showElapse, err = s.rawRemove([]string{CloudURLToString(bucket, "")}, false, false, true)
	c.Assert(err, IsNil)
	c.Assert(showElapse, Equals, true)

	bucketStat := s.getStat(bucket, "", c)
	c.Assert(bucketStat[StatName], Equals, bucket)

	// batch delete not exist objects
    object := "batch_delete_notexst_object"
    showElapse, err = s.rawRemove([]string{CloudURLToString(bucket, object)}, true, true, false)
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)

    // clear not exist bucket
    bucketName := bucketNamePrefix + "rmnotexist"
    showElapse, err = s.rawRemove([]string{CloudURLToString(bucketName, "")}, true, true, false)
    c.Assert(err, NotNil)
    c.Assert(showElapse, Equals, false)

    // test oss batch delete not exist objects
    objects := []string{}
    ossBucket, err := removeCommand.command.ossBucket(bucket)
    c.Assert(err, IsNil)
    num, err := removeCommand.ossBatchDeleteObjectsRetry(ossBucket, objects)
    c.Assert(err, IsNil)
    c.Assert(num, Equals, 0)
}

func (s *OssutilCommandSuite) TestErrDeleteObject(c *C) {
	bucketName := bucketNameNotExist

	bucket, err := removeCommand.command.ossBucket(bucketName)
	c.Assert(err, IsNil)

	object := "object"
	err = removeCommand.ossDeleteObjectRetry(bucket, object)
	c.Assert(err, NotNil)

	_, err = removeCommand.ossBatchDeleteObjectsRetry(bucket, []string{object})
	c.Assert(err, NotNil)
}

func (s *OssutilCommandSuite) TestAllTypeObject(c *C) {
	bucketName := bucketNameMB

	s.clearAllMultipartInBucket(bucketName, c)
    time.Sleep(2*time.Second)

	normal_object := "TestAllTypeObject"
	s.putObject(bucketName, normal_object, uploadFileName, c)

	object := "TestMultipartObjectRm"
	s.putObject(bucketName, object, uploadFileName, c)

	objects := s.listObjects(bucketName, object, "ls - ", c)
	c.Assert(len(objects), Equals, 1)
	c.Assert(objects[0], Equals, object)

	bucket, err := copyCommand.command.ossBucket(bucketName)
	for i := 0; i < 20; i++ {
		_, err = bucket.InitiateMultipartUpload(object)
		c.Assert(err, IsNil)
	}

	lmr, e := bucket.ListMultipartUploads(oss.Prefix(object))
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 20)

	_, e = s.removeWrapper("rm -arf", bucketName, object, c)
	c.Assert(e, IsNil)

	lmr, e = bucket.ListMultipartUploads(oss.Prefix(object))
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 0)

	// list normal_object
	objects = s.listObjects(bucketName, normal_object, "ls - ", c)
	c.Assert(len(objects), Equals, 1)
	c.Assert(objects[0], Equals, normal_object)
}

func (s *OssutilCommandSuite) TestMultipartUpload(c *C) {
	bucketName := bucketNameMB

	s.clearAllMultipartInBucket(bucketName, c)

	// put object
	object := "TestMultipartObject"
	s.putObject(bucketName, object, uploadFileName, c)
	time.Sleep(2 * sleepTime)

	// list object
	objects := s.listObjects(bucketName, object, "ls - ", c)
	c.Assert(len(objects), Equals, 1)
	c.Assert(objects[0], Equals, object)

	bucket, err := copyCommand.command.ossBucket(bucketName)
	for i := 0; i < 20; i++ {
		_, err = bucket.InitiateMultipartUpload(object)
		c.Assert(err, IsNil)
	}

	lmr, e := bucket.ListMultipartUploads(oss.Prefix(object))
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 20)

	_, e = s.removeWrapper("rm -mrf", bucketName, object, c)
	c.Assert(e, IsNil)

	lmr, e = bucket.ListMultipartUploads(oss.Prefix(object))
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 0)

	obj := "TestMultipartObjectUploads"
	s.putObject(bucketName, obj, uploadFileName, c)
	time.Sleep(4 * sleepTime)

	for i := 0; i < 20; i++ {
		_, err = bucket.InitiateMultipartUpload(obj)
		c.Assert(err, IsNil)
	}
	_, e = s.removeWrapper("rm -mrf", bucketName, "", c)
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 0)
}

func (s *OssutilCommandSuite) TestMultipartUpload_Prefix(c *C) {
	bucketName := bucketNameMB
	bucket, err := copyCommand.command.ossBucket(bucketName)
	c.Assert(err, IsNil)

	s.clearAllMultipartInBucket(bucketName, c)

	object := "TestMultipartObject"
	s.putObject(bucketName, object, uploadFileName, c)
	time.Sleep(2 * sleepTime)

	object1 := "TestMultipartObject" + "prefix"
	s.putObject(bucketName, object1, uploadFileName, c)
	time.Sleep(2 * sleepTime)

	object2 := "TestMultipartObject" + "/dir/test"
	s.putObject(bucketName, object2, uploadFileName, c)
	time.Sleep(2 * sleepTime)

	// list object
	objects := s.listObjects(bucketName, object, "ls - ", c)
	c.Assert(len(objects), Equals, 3)

	for i := 0; i < 20; i++ {
		_, err = bucket.InitiateMultipartUpload(object)
		c.Assert(err, IsNil)
	}

	for i := 0; i < 20; i++ {
		_, err = bucket.InitiateMultipartUpload(object1)
		c.Assert(err, IsNil)
	}

	for i := 0; i < 20; i++ {
		_, err = bucket.InitiateMultipartUpload(object2)
		c.Assert(err, IsNil)
	}

	lmr, e := bucket.ListMultipartUploads(oss.Prefix(object))
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 20*3)

	_, e = s.removeWrapper("rm -mrf", bucketName, "", c)
	c.Assert(e, IsNil)

	lmr, e = bucket.ListMultipartUploads(oss.Prefix(object))
	c.Assert(e, IsNil)
	c.Assert(len(lmr.Uploads), Equals, 0)
}

func (s *OssutilCommandSuite) TestMultipartError(c *C) {
	bucketName := bucketNameMB
	object := "TestMultipartError"

	_, e := s.removeWrapper("rm -mb", bucketName, object, c)
	c.Assert(e, NotNil)

	_, e = s.removeWrapper("rm -mf", bucketName, "", c)
	c.Assert(e, NotNil)
}

func (s *OssutilCommandSuite) TestAllTypeError(c *C) {
	bucketName := bucketNameMB
	object := "random"

	_, e := s.removeWrapper("rm -ab", bucketName, object, c)
	c.Assert(e, NotNil)
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"runtime"
	"time"

	"github.com/bitmark-inc/logger"
)

type ConfigReader interface {
	Initialise(string)
	OptimalThreadCount() uint32
	SetCalendar(JobCalendar)
	Refresh() error
	GetConfig() (*Configuration, string, error)
	SetLog(*logger.L) error
	Start()
	FirstTimeRun()
	SetProofer(Proofer)
}

const (
	oneMinute          = time.Duration(1) * time.Minute
	minThreadCount     = 1
	ReaderLoggerPrefix = "config-reader"
)

var (
	totalCPUCount = uint32(runtime.NumCPU())
)

type ConfigReaderData struct {
	fileName             string
	refreshByMinute      time.Duration
	log                  *logger.L
	currentConfiguration *Configuration
	initialized          bool
	threadCount          uint32
	calendar             JobCalendar
	proofer              Proofer
	watcherChannel       WatcherChannel
}

func newConfigReader(ch WatcherChannel) ConfigReader {
	return &ConfigReaderData{
		log:                  nil,
		currentConfiguration: nil,
		threadCount:          1,
		initialized:          false,
		refreshByMinute:      oneMinute,
		watcherChannel:       ch,
	}
}

// configuration needs read first to know logger file location
func (c *ConfigReaderData) Initialise(fileName string) {
	c.fileName = fileName
}

func (c *ConfigReaderData) SetCalendar(calendar JobCalendar) {
	c.calendar = calendar
}

func (c *ConfigReaderData) SetProofer(proofer Proofer) {
	c.proofer = proofer
}

func (c *ConfigReaderData) FirstTimeRun() {
	err := c.Refresh()
	if nil != err {
		return
	}
	c.notify()
}

func (c *ConfigReaderData) Start() {
	go func() {
		for {
			select {
			case <-c.watcherChannel.change:
				c.log.Info("receive file change event, wait for 1 minute to adapt")
				<-time.After(c.refreshByMinute)
				err := c.Refresh()
				if nil != err {
					c.log.Errorf("failed to read configuration from :%s error %s",
						c.fileName, err)
				}
				c.notify()
			case <-c.watcherChannel.remove:
				c.log.Warn("config file removed")
			}
		}

	}()
}

func (c *ConfigReaderData) UpdatePeriodically() {
	c.log.Info("star to update config perioditically")

	go func() {
		for {
			select {
			case <-time.After(c.refreshByMinute):
				err := c.Refresh()
				if nil != err {
					c.log.Errorf("failed to read configuration from :%s error %s",
						c.fileName, err)
				}
				c.notify()
			}
		}
	}()
}

func (c *ConfigReaderData) Refresh() error {
	configuration, err := c.parse()
	if nil != err {
		return err
	}
	c.update(configuration)
	return nil
}

func (c *ConfigReaderData) notify() {
	c.calendar.Refresh(c.currentConfiguration.Calendar)
	c.proofer.Refresh()
}

func (c *ConfigReaderData) parse() (*Configuration, error) {
	configuration, err := getConfiguration(c.fileName)
	if nil != err {
		return nil, err
	}
	return configuration, nil
}

func (c *ConfigReaderData) GetConfig() (*Configuration, string, error) {
	if nil == c.currentConfiguration {
		return nil, "", fmt.Errorf("configuration is empty")
	}
	return c.currentConfiguration, c.fileName, nil
}

func (c *ConfigReaderData) SetLog(log *logger.L) error {
	if nil == log {
		return fmt.Errorf("logger %v is nil", log)
	}
	c.log = log
	c.initialized = true
	return nil
}

func (c *ConfigReaderData) update(newConfiguration *Configuration) {
	c.currentConfiguration = newConfiguration
	c.threadCount = c.OptimalThreadCount()
	if c.initialized {
		c.log.Debugf("Updating configuration, target thread count %d, working: %t",
			c.threadCount,
			c.proofer.IsWorking(),
		)
	}
}

func (c *ConfigReaderData) updateCpuCount(count uint32) {
	if count > 0 {
		totalCPUCount = count
	}
}

func (c *ConfigReaderData) cpuCount() uint32 {
	return totalCPUCount
}

func (c *ConfigReaderData) OptimalThreadCount() uint32 {
	if !c.initialized {
		return uint32(minThreadCount)
	}
	percentage := float32(c.currentConfiguration.maxCPUUsage()) / 100
	threadCount := uint32(float32(totalCPUCount) * percentage)

	if threadCount <= minThreadCount {
		return minThreadCount
	}

	if threadCount > totalCPUCount {
		return totalCPUCount
	}

	return threadCount
}
<commit_msg>[recorderd] remove unused method<commit_after>// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"runtime"
	"time"

	"github.com/bitmark-inc/logger"
)

type ConfigReader interface {
	Initialise(string)
	OptimalThreadCount() uint32
	SetCalendar(JobCalendar)
	Refresh() error
	GetConfig() (*Configuration, string, error)
	SetLog(*logger.L) error
	Start()
	FirstTimeRun()
	SetProofer(Proofer)
}

const (
	oneMinute          = time.Duration(1) * time.Minute
	minThreadCount     = 1
	ReaderLoggerPrefix = "config-reader"
)

var (
	totalCPUCount = uint32(runtime.NumCPU())
)

type ConfigReaderData struct {
	fileName             string
	refreshByMinute      time.Duration
	log                  *logger.L
	currentConfiguration *Configuration
	initialized          bool
	threadCount          uint32
	calendar             JobCalendar
	proofer              Proofer
	watcherChannel       WatcherChannel
}

func newConfigReader(ch WatcherChannel) ConfigReader {
	return &ConfigReaderData{
		log:                  nil,
		currentConfiguration: nil,
		threadCount:          1,
		initialized:          false,
		refreshByMinute:      oneMinute,
		watcherChannel:       ch,
	}
}

// configuration needs read first to know logger file location
func (c *ConfigReaderData) Initialise(fileName string) {
	c.fileName = fileName
}

func (c *ConfigReaderData) SetCalendar(calendar JobCalendar) {
	c.calendar = calendar
}

func (c *ConfigReaderData) SetProofer(proofer Proofer) {
	c.proofer = proofer
}

func (c *ConfigReaderData) FirstTimeRun() {
	err := c.Refresh()
	if nil != err {
		return
	}
	c.notify()
}

func (c *ConfigReaderData) Start() {
	go func() {
		for {
			select {
			case <-c.watcherChannel.change:
				c.log.Info("receive file change event, wait for 1 minute to adapt")
				<-time.After(c.refreshByMinute)
				err := c.Refresh()
				if nil != err {
					c.log.Errorf("failed to read configuration from :%s error %s",
						c.fileName, err)
				}
				c.notify()
			case <-c.watcherChannel.remove:
				c.log.Warn("config file removed")
			}
		}

	}()
}

func (c *ConfigReaderData) Refresh() error {
	configuration, err := c.parse()
	if nil != err {
		return err
	}
	c.update(configuration)
	return nil
}

func (c *ConfigReaderData) notify() {
	c.calendar.Refresh(c.currentConfiguration.Calendar)
	c.proofer.Refresh()
}

func (c *ConfigReaderData) parse() (*Configuration, error) {
	configuration, err := getConfiguration(c.fileName)
	if nil != err {
		return nil, err
	}
	return configuration, nil
}

func (c *ConfigReaderData) GetConfig() (*Configuration, string, error) {
	if nil == c.currentConfiguration {
		return nil, "", fmt.Errorf("configuration is empty")
	}
	return c.currentConfiguration, c.fileName, nil
}

func (c *ConfigReaderData) SetLog(log *logger.L) error {
	if nil == log {
		return fmt.Errorf("logger %v is nil", log)
	}
	c.log = log
	c.initialized = true
	return nil
}

func (c *ConfigReaderData) update(newConfiguration *Configuration) {
	c.currentConfiguration = newConfiguration
	c.threadCount = c.OptimalThreadCount()
	if c.initialized {
		c.log.Debugf("Updating configuration, target thread count %d, working: %t",
			c.threadCount,
			c.proofer.IsWorking(),
		)
	}
}

func (c *ConfigReaderData) updateCpuCount(count uint32) {
	if count > 0 {
		totalCPUCount = count
	}
}

func (c *ConfigReaderData) cpuCount() uint32 {
	return totalCPUCount
}

func (c *ConfigReaderData) OptimalThreadCount() uint32 {
	if !c.initialized {
		return uint32(minThreadCount)
	}
	percentage := float32(c.currentConfiguration.maxCPUUsage()) / 100
	threadCount := uint32(float32(totalCPUCount) * percentage)

	if threadCount <= minThreadCount {
		return minThreadCount
	}

	if threadCount > totalCPUCount {
		return totalCPUCount
	}

	return threadCount
}
<|endoftext|>


<commit_before>package executor

import (
	"encoding/json"
	"io"
	"os/exec"
	"strings"
	"testing"
	"time"

	"github.com/intelsdi-x/athena/integration_tests/test_helpers"
	"github.com/intelsdi-x/athena/pkg/executor"
	"github.com/intelsdi-x/athena/pkg/kubernetes"
	"github.com/nu7hatch/gouuid"
	. "github.com/smartystreets/goconvey/convey"
	"k8s.io/kubernetes/pkg/api"
)

const (
	clusterSpawnTimeout = 20 * time.Second
	podSpawnTimeout     = 10 * time.Second
	podFinishedTimeout  = 90 * time.Second
)

func TestKubernetesExecutor(t *testing.T) {
	// Create Kubernetes configuration.
	config := kubernetes.DefaultConfig()
	executorConfig := executor.DefaultKubernetesConfig()

	// Create kubectl helper for communicate with Kubernetes cluster.
	kubectl, err := testhelpers.NewKubeClient(executorConfig)
	if err != nil {
		// NewKubeClient() returns error only when kubernetes configuration is
		// invalid.
		t.Errorf("Requested configuration is invalid: %q", err)
	}

	// Create Kubernetes launcher and spawn Kubernetes cluster.
	local := executor.NewLocal()
	k8sLauncher := kubernetes.New(local, local, config)
	k8sHandle, err := k8sLauncher.Launch()
	if err != nil {
		t.Errorf("Cannot start k8s cluster: %q", err)
	}

	// Wait for at least one node is up and running in cluster.
	if err := kubectl.WaitForCluster(clusterSpawnTimeout); err != nil {
		t.Errorf("Cannot launch K8s cluster: %q", err)
	}

	// Make sure cluster is shut down and cleaned up when test ends.
	defer func() {
		errs := executor.StopCleanAndErase(k8sHandle)

		errs.Add(exec.Command("etcdctl", "rm", "--recursive", "--dir", "/registry").Run())

		if err := errs.GetErrIfAny(); err != nil {
			t.Errorf("Cannot stop cluster: %q", err)
		}
	}()

	Convey("Creating a kubernetes executor _with_ a kubernetes cluster available", t, func() {

		// Generate random pod name. This pod name should be unique for each
		// test case inside this Convey.
		podName, err := uuid.NewV4()
		So(err, ShouldBeNil)
		executorConfig.PodNamePrefix = podName.String()

		// Create Kubernetes executor, which should be passed to following
		// Conveys.
		k8sexecutor, err := executor.NewKubernetes(executorConfig)
		So(err, ShouldBeNil)

		// Make sure no pods are running. GetPods() returns running pods and
		// finished pods. We are expecting that there is no running pods on
		// cluster.
		pods, _, err := kubectl.GetPods()
		So(err, ShouldBeNil)
		So(len(pods), ShouldEqual, 0)

		Convey("The generic Executor test should pass", func() {
			testExecutor(t, k8sexecutor)
		})

		Convey("Running a command with a successful exit status should leave one pod running", func() {
			// Start Kubernetes pod which should die after 3 seconds. ExitCode
			// should pass to taskHandle object.
			taskHandle, err := k8sexecutor.Execute("sleep 3 && exit 0")
			defer executor.StopCleanAndErase(taskHandle)
			So(err, ShouldBeNil)

			// Spawning pods on Kubernetes is a complex process which consists
			// of i.a.: scheduling and pulling image. Test should wait for
			// processing executing request.
			err = kubectl.WaitForPod(podSpawnTimeout)
			So(err, ShouldBeNil)

			Convey("And after few seconds", func() {
				// Pod should end after three seconds, but propagation of
				// status information can take longer time. To reduce number
				// of false-positive assertion fails, Wait() timeout is much
				// longer then time withing pod should shutdown.
				So(taskHandle.Wait(podFinishedTimeout), ShouldBeTrue)

				Convey("The exit status should be zero", func() {
					// ExitCode should appears in TaskHandle object after pod
					// termination.
					exitCode, err := taskHandle.ExitCode()
					So(err, ShouldBeNil)
					So(exitCode, ShouldEqual, 0)

					Convey("And there should be zero pods", func() {
						// There shouldn't be any running pods after test
						// executing.
						pods, _, err = kubectl.GetPods()
						So(err, ShouldBeNil)
						So(len(pods), ShouldEqual, 0)
					})
				})
			})
		})

		Convey("Running a command with an unsuccessful exit status should leave one pod running", func() {
			taskHandle, err := k8sexecutor.Execute("sleep 3 && exit 5")
			defer executor.StopCleanAndErase(taskHandle)
			So(err, ShouldBeNil)

			err = kubectl.WaitForPod(podSpawnTimeout)
			So(err, ShouldBeNil)

			Convey("And after few seconds", func() {
				So(taskHandle.Wait(podFinishedTimeout), ShouldBeTrue)

				Convey("The exit status should be 5", func() {
					exitCode, err := taskHandle.ExitCode()
					So(err, ShouldBeNil)
					So(exitCode, ShouldEqual, 5)

					Convey("And there should be zero pods", func() {
						pods, _, err = kubectl.GetPods()
						So(err, ShouldBeNil)
						So(len(pods), ShouldEqual, 0)
					})
				})
			})
		})

		Convey("Running a command and calling Clean() on task handle should not cause a data race", func() {
			taskHandle, err := k8sexecutor.Execute("sleep 3 && exit 0")
			defer executor.StopCleanAndErase(taskHandle)
			So(err, ShouldBeNil)
			taskHandle.Clean()
		})

		Convey("Logs should be available and non-empty", func() {
			taskHandle, err := k8sexecutor.Execute("echo \"This is Sparta\" && (echo \"This is England\" 1>&2) && exit 0")
			defer executor.StopCleanAndErase(taskHandle)

			So(err, ShouldBeNil)
			So(taskHandle.Wait(podFinishedTimeout), ShouldBeTrue)

			exitCode, err := taskHandle.ExitCode()
			So(exitCode, ShouldEqual, 0)

			stdout, err := taskHandle.StdoutFile()
			So(err, ShouldBeNil)
			defer stdout.Close()
			buffer := make([]byte, 31)
			n, err := stdout.Read(buffer)

			So(err, ShouldBeNil)
			So(n, ShouldEqual, 31)
			output := strings.Split(string(buffer), "\n")
			So(output, ShouldHaveLength, 3)
			So(output, ShouldContain, "This is Sparta")
			So(output, ShouldContain, "This is England")

			stderr, err := taskHandle.StderrFile()
			So(err, ShouldBeNil)
			defer stderr.Close()
			buffer = make([]byte, 10)
			n, err = stderr.Read(buffer)

			// stderr will always be empty as we are not able to fetch it from K8s.
			// stdout includes both stderr and stdout of the application run in the pod.
			So(err, ShouldEqual, io.EOF)
			So(n, ShouldEqual, 0)
		})

		Convey("Timeout should not block execution because of files being unavailable", func() {
			nodes, err := kubectl.Nodes().List(api.ListOptions{})
			So(err, ShouldBeNil)
			node := nodes.Items[0]
			newTaint := api.Taint{
				Key: "hponly", Value: "true", Effect: api.TaintEffectNoSchedule,
			}
			taintsInJSON, err := json.Marshal([]api.Taint{newTaint})
			So(err, ShouldBeNil)
			node.Annotations[api.TaintsAnnotationKey] = string(taintsInJSON)
			_, err = kubectl.Client.Nodes().Update(&node)
			So(err, ShouldBeNil)

			executorConfig.LaunchTimeout = 1 * time.Second
			k8sexecutor, err = executor.NewKubernetes(executorConfig)
			So(err, ShouldBeNil)
			taskHandle, err := k8sexecutor.Execute("sleep inf")
			defer executor.StopCleanAndErase(taskHandle)
			So(err, ShouldBeNil)

			stopped := taskHandle.Wait(5 * time.Second)
			So(stopped, ShouldBeTrue)
		})

	})
}
<commit_msg>Extend timeout for launching kubernetes cluster in integration tests. (#29)<commit_after>package executor

import (
	"encoding/json"
	"io"
	"os/exec"
	"strings"
	"testing"
	"time"

	"github.com/intelsdi-x/athena/integration_tests/test_helpers"
	"github.com/intelsdi-x/athena/pkg/executor"
	"github.com/intelsdi-x/athena/pkg/kubernetes"
	"github.com/nu7hatch/gouuid"
	. "github.com/smartystreets/goconvey/convey"
	"k8s.io/kubernetes/pkg/api"
)

const (
	clusterSpawnTimeout = 2 * time.Minute
	podSpawnTimeout     = 10 * time.Second
	podFinishedTimeout  = 90 * time.Second
)

func TestKubernetesExecutor(t *testing.T) {
	// Create Kubernetes configuration.
	config := kubernetes.DefaultConfig()
	executorConfig := executor.DefaultKubernetesConfig()

	// Create kubectl helper for communicate with Kubernetes cluster.
	kubectl, err := testhelpers.NewKubeClient(executorConfig)
	if err != nil {
		// NewKubeClient() returns error only when kubernetes configuration is
		// invalid.
		t.Errorf("Requested configuration is invalid: %q", err)
	}

	// Create Kubernetes launcher and spawn Kubernetes cluster.
	local := executor.NewLocal()
	k8sLauncher := kubernetes.New(local, local, config)
	k8sHandle, err := k8sLauncher.Launch()
	if err != nil {
		t.Errorf("Cannot start k8s cluster: %q", err)
	}

	// Wait for at least one node is up and running in cluster.
	if err := kubectl.WaitForCluster(clusterSpawnTimeout); err != nil {
		t.Errorf("Cannot launch K8s cluster: %q", err)
	}

	// Make sure cluster is shut down and cleaned up when test ends.
	defer func() {
		errs := executor.StopCleanAndErase(k8sHandle)

		errs.Add(exec.Command("etcdctl", "rm", "--recursive", "--dir", "/registry").Run())

		if err := errs.GetErrIfAny(); err != nil {
			t.Errorf("Cannot stop cluster: %q", err)
		}
	}()

	Convey("Creating a kubernetes executor _with_ a kubernetes cluster available", t, func() {

		// Generate random pod name. This pod name should be unique for each
		// test case inside this Convey.
		podName, err := uuid.NewV4()
		So(err, ShouldBeNil)
		executorConfig.PodNamePrefix = podName.String()

		// Create Kubernetes executor, which should be passed to following
		// Conveys.
		k8sexecutor, err := executor.NewKubernetes(executorConfig)
		So(err, ShouldBeNil)

		// Make sure no pods are running. GetPods() returns running pods and
		// finished pods. We are expecting that there is no running pods on
		// cluster.
		pods, _, err := kubectl.GetPods()
		So(err, ShouldBeNil)
		So(len(pods), ShouldEqual, 0)

		Convey("The generic Executor test should pass", func() {
			testExecutor(t, k8sexecutor)
		})

		Convey("Running a command with a successful exit status should leave one pod running", func() {
			// Start Kubernetes pod which should die after 3 seconds. ExitCode
			// should pass to taskHandle object.
			taskHandle, err := k8sexecutor.Execute("sleep 3 && exit 0")
			defer executor.StopCleanAndErase(taskHandle)
			So(err, ShouldBeNil)

			// Spawning pods on Kubernetes is a complex process which consists
			// of i.a.: scheduling and pulling image. Test should wait for
			// processing executing request.
			err = kubectl.WaitForPod(podSpawnTimeout)
			So(err, ShouldBeNil)

			Convey("And after few seconds", func() {
				// Pod should end after three seconds, but propagation of
				// status information can take longer time. To reduce number
				// of false-positive assertion fails, Wait() timeout is much
				// longer then time withing pod should shutdown.
				So(taskHandle.Wait(podFinishedTimeout), ShouldBeTrue)

				Convey("The exit status should be zero", func() {
					// ExitCode should appears in TaskHandle object after pod
					// termination.
					exitCode, err := taskHandle.ExitCode()
					So(err, ShouldBeNil)
					So(exitCode, ShouldEqual, 0)

					Convey("And there should be zero pods", func() {
						// There shouldn't be any running pods after test
						// executing.
						pods, _, err = kubectl.GetPods()
						So(err, ShouldBeNil)
						So(len(pods), ShouldEqual, 0)
					})
				})
			})
		})

		Convey("Running a command with an unsuccessful exit status should leave one pod running", func() {
			taskHandle, err := k8sexecutor.Execute("sleep 3 && exit 5")
			defer executor.StopCleanAndErase(taskHandle)
			So(err, ShouldBeNil)

			err = kubectl.WaitForPod(podSpawnTimeout)
			So(err, ShouldBeNil)

			Convey("And after few seconds", func() {
				So(taskHandle.Wait(podFinishedTimeout), ShouldBeTrue)

				Convey("The exit status should be 5", func() {
					exitCode, err := taskHandle.ExitCode()
					So(err, ShouldBeNil)
					So(exitCode, ShouldEqual, 5)

					Convey("And there should be zero pods", func() {
						pods, _, err = kubectl.GetPods()
						So(err, ShouldBeNil)
						So(len(pods), ShouldEqual, 0)
					})
				})
			})
		})

		Convey("Running a command and calling Clean() on task handle should not cause a data race", func() {
			taskHandle, err := k8sexecutor.Execute("sleep 3 && exit 0")
			defer executor.StopCleanAndErase(taskHandle)
			So(err, ShouldBeNil)
			taskHandle.Clean()
		})

		Convey("Logs should be available and non-empty", func() {
			taskHandle, err := k8sexecutor.Execute("echo \"This is Sparta\" && (echo \"This is England\" 1>&2) && exit 0")
			defer executor.StopCleanAndErase(taskHandle)

			So(err, ShouldBeNil)
			So(taskHandle.Wait(podFinishedTimeout), ShouldBeTrue)

			exitCode, err := taskHandle.ExitCode()
			So(exitCode, ShouldEqual, 0)

			stdout, err := taskHandle.StdoutFile()
			So(err, ShouldBeNil)
			defer stdout.Close()
			buffer := make([]byte, 31)
			n, err := stdout.Read(buffer)

			So(err, ShouldBeNil)
			So(n, ShouldEqual, 31)
			output := strings.Split(string(buffer), "\n")
			So(output, ShouldHaveLength, 3)
			So(output, ShouldContain, "This is Sparta")
			So(output, ShouldContain, "This is England")

			stderr, err := taskHandle.StderrFile()
			So(err, ShouldBeNil)
			defer stderr.Close()
			buffer = make([]byte, 10)
			n, err = stderr.Read(buffer)

			// stderr will always be empty as we are not able to fetch it from K8s.
			// stdout includes both stderr and stdout of the application run in the pod.
			So(err, ShouldEqual, io.EOF)
			So(n, ShouldEqual, 0)
		})

		Convey("Timeout should not block execution because of files being unavailable", func() {
			nodes, err := kubectl.Nodes().List(api.ListOptions{})
			So(err, ShouldBeNil)
			node := nodes.Items[0]
			newTaint := api.Taint{
				Key: "hponly", Value: "true", Effect: api.TaintEffectNoSchedule,
			}
			taintsInJSON, err := json.Marshal([]api.Taint{newTaint})
			So(err, ShouldBeNil)
			node.Annotations[api.TaintsAnnotationKey] = string(taintsInJSON)
			_, err = kubectl.Client.Nodes().Update(&node)
			So(err, ShouldBeNil)

			executorConfig.LaunchTimeout = 1 * time.Second
			k8sexecutor, err = executor.NewKubernetes(executorConfig)
			So(err, ShouldBeNil)
			taskHandle, err := k8sexecutor.Execute("sleep inf")
			defer executor.StopCleanAndErase(taskHandle)
			So(err, ShouldBeNil)

			stopped := taskHandle.Wait(5 * time.Second)
			So(stopped, ShouldBeTrue)
		})

	})
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "testing"

func TestParseFormulaLiteral(t *testing.T) {
	result := parseFormula("(a)")
	if result.value != "(a)" {
		t.Error("Test_parseFormulaLiteral failed: " + result.value)
	}
	fmt.Println(result.value)
}

func TestParseFormulaConjunction(t *testing.T) {
	result := parseFormula("((a)^(b))")
	if result.value != "^" {
		t.Error("Test_parseFormulaConjunction failed for connective: " + result.value)
	}
	if result.left == nil || result.left.value != "(a)" {
		t.Error("Test_parseFormulaConjunction failed for left formula.")
	}
	if result.right == nil || result.right.value != "(b)" {
		t.Error("Test_parseFormulaConjunction failed for right formula.")
	}
	fmt.Println(result.value)
}
<commit_msg>Renamed test functions in error messages as well<commit_after>package main

import "fmt"
import "testing"

func TestParseFormulaLiteral(t *testing.T) {
	result := parseFormula("(a)")
	if result.value != "(a)" {
		t.Error("TestParseFormulaLiteral failed: " + result.value)
	}
	fmt.Println(result.value)
}

func TestParseFormulaConjunction(t *testing.T) {
	result := parseFormula("((a)^(b))")
	if result.value != "^" {
		t.Error("TestParseFormulaConjunction failed for connective: " + result.value)
	}
	if result.left == nil || result.left.value != "(a)" {
		t.Error("TestParseFormulaConjunction failed for left formula.")
	}
	if result.right == nil || result.right.value != "(b)" {
		t.Error("TestParseFormulaConjunction failed for right formula.")
	}
	fmt.Println(result.value)
}
<|endoftext|>


<commit_before>package forward

import (
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/http-proxy/utils"
	"github.com/getlantern/idletiming"
)

var log = golog.LoggerFor("forward")

type Forwarder struct {
	errHandler   utils.ErrorHandler
	roundTripper http.RoundTripper
	rewriter     RequestRewriter
	next         http.Handler

	idleTimeout time.Duration
}

type optSetter func(f *Forwarder) error

func RoundTripper(r http.RoundTripper) optSetter {
	return func(f *Forwarder) error {
		f.roundTripper = r
		return nil
	}
}

type RequestRewriter interface {
	Rewrite(r *http.Request)
}

func Rewriter(r RequestRewriter) optSetter {
	return func(f *Forwarder) error {
		f.rewriter = r
		return nil
	}
}

func IdleTimeoutSetter(i time.Duration) optSetter {
	return func(f *Forwarder) error {
		f.idleTimeout = i
		return nil
	}
}

func New(next http.Handler, setters ...optSetter) (*Forwarder, error) {
	var dialerFunc func(string, string) (net.Conn, error)

	var timeoutTransport http.RoundTripper = &http.Transport{
		Dial:                dialerFunc,
		TLSHandshakeTimeout: 10 * time.Second,
	}
	f := &Forwarder{
		errHandler:   utils.DefaultHandler,
		roundTripper: timeoutTransport,
		next:         next,
		idleTimeout:  30,
	}
	for _, s := range setters {
		if err := s(f); err != nil {
			return nil, err
		}
	}
	if f.rewriter == nil {
		f.rewriter = &HeaderRewriter{
			TrustForwardHeader: true,
			Hostname:           "",
		}
	}

	dialerFunc = func(network, addr string) (conn net.Conn, err error) {
		conn, err = (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial(network, addr)
		if err != nil {
			return
		}

		idleConn := idletiming.Conn(conn, f.idleTimeout, func() {
			if conn != nil {
				conn.Close()
			}
		})
		return idleConn, err
	}

	return f, nil
}

func (f *Forwarder) ServeHTTP(w http.ResponseWriter, req *http.Request) {

	// Create a copy of the request suitable for our needs
	reqClone, err := f.cloneRequest(req, req.URL)
	if err != nil {
		log.Errorf("Error forwarding to %v, error: %v", req.Host, err)
		f.errHandler.ServeHTTP(w, req, err)
		return
	}
	f.rewriter.Rewrite(reqClone)

	if log.IsTraceEnabled() {
		reqStr, _ := httputil.DumpRequest(req, false)
		log.Tracef("Forwarder Middleware received request:\n%s", reqStr)

		reqStr2, _ := httputil.DumpRequest(reqClone, false)
		log.Tracef("Forwarder Middleware forwarding rewritten request:\n%s", reqStr2)
	}

	// Forward the request and get a response
	start := time.Now().UTC()
	response, err := f.roundTripper.RoundTrip(reqClone)
	if err != nil {
		log.Errorf("Error forwarding to %v, error: %v", req.Host, err)
		f.errHandler.ServeHTTP(w, req, err)
		return
	}
	log.Debugf("Round trip: %v, code: %v, duration: %v",
		req.URL, response.StatusCode, time.Now().UTC().Sub(start))

	if log.IsTraceEnabled() {
		respStr, _ := httputil.DumpResponse(response, true)
		log.Tracef("Forward Middleware received response:\n%s", respStr)
	}

	// Forward the response to the origin
	copyHeadersForForwarding(w.Header(), response.Header)
	w.WriteHeader(response.StatusCode)

	// It became nil in a Co-Advisor test though the doc says it will never be nil
	if response.Body != nil {
		_, err = io.Copy(w, response.Body)
		if err != nil {
			log.Error(err)
		}

		response.Body.Close()
	}
}

func (f *Forwarder) cloneRequest(req *http.Request, u *url.URL) (*http.Request, error) {
	outReq := new(http.Request)
	// Beware, this will make a shallow copy. We have to copy all maps
	*outReq = *req

	outReq.Proto = "HTTP/1.1"
	outReq.ProtoMajor = 1
	outReq.ProtoMinor = 1
	// Overwrite close flag: keep persistent connection for the backend servers
	outReq.Close = false

	// Request Header
	outReq.Header = make(http.Header)
	copyHeadersForForwarding(outReq.Header, req.Header)

	// Request URL
	outReq.URL = cloneURL(req.URL)
	// We know that is going to be HTTP always because HTTPS isn't forwarded.
	// We need to hardcode it here because req.URL.Scheme can be undefined, since
	// client request don't need to use absolute URIs
	outReq.URL.Scheme = "http"
	// We need to make sure the host is defined in the URL (not the actual URI)
	outReq.URL.Host = req.Host
	// Make sure we define an opaque URL, so the URI is just the path
	outReq.URL.Opaque = req.URL.Path
	outReq.URL.RawQuery = req.URL.RawQuery

	userAgent := req.UserAgent()
	if userAgent == "" {
		outReq.Header.Del("User-Agent")
	} else {
		outReq.Header.Set("User-Agent", userAgent)
	}

	/*
		// Trailer support
		// We are forced to do this because Go's server won't allow us to read the trailers otherwise
		_, err := httputil.DumpRequestOut(req, true)
		if err != nil {
			log.Errorf("Error: %v", err)
			return outReq, err
		}

		rcloser := ioutil.NopCloser(req.Body)
		outReq.Body = rcloser

		chunkedTransfer := false
		for _, enc := range req.TransferEncoding {
			if enc == "chunked" {
				chunkedTransfer = true
				break
			}
		}

		// Append Trailer
		if chunkedTransfer && len(req.Trailer) > 0 {
			outReq.Trailer = http.Header{}
			for k, vv := range req.Trailer {
				for _, v := range vv {
					outReq.Trailer.Add(k, v)
				}
			}
		}
	*/

	return outReq, nil
}
<commit_msg>Actually use dialer function in forwarder<commit_after>package forward

import (
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/http-proxy/utils"
	"github.com/getlantern/idletiming"
)

var log = golog.LoggerFor("forward")

type Forwarder struct {
	errHandler   utils.ErrorHandler
	roundTripper http.RoundTripper
	rewriter     RequestRewriter
	next         http.Handler

	idleTimeout time.Duration
}

type optSetter func(f *Forwarder) error

func RoundTripper(r http.RoundTripper) optSetter {
	return func(f *Forwarder) error {
		f.roundTripper = r
		return nil
	}
}

type RequestRewriter interface {
	Rewrite(r *http.Request)
}

func Rewriter(r RequestRewriter) optSetter {
	return func(f *Forwarder) error {
		f.rewriter = r
		return nil
	}
}

func IdleTimeoutSetter(i time.Duration) optSetter {
	return func(f *Forwarder) error {
		f.idleTimeout = i
		return nil
	}
}

func New(next http.Handler, setters ...optSetter) (*Forwarder, error) {
	idleTimeoutPtr := new(time.Duration)
	dialerFunc := func(network, addr string) (conn net.Conn, err error) {
		conn, err = net.DialTimeout(network, addr, time.Second*30)
		if err != nil {
			return
		}

		idleConn := idletiming.Conn(conn, *idleTimeoutPtr, func() {
			if conn != nil {
				conn.Close()
			}
		})
		return idleConn, err
	}

	var timeoutTransport http.RoundTripper = &http.Transport{
		Dial:                dialerFunc,
		TLSHandshakeTimeout: 10 * time.Second,
	}
	f := &Forwarder{
		errHandler:   utils.DefaultHandler,
		roundTripper: timeoutTransport,
		next:         next,
		idleTimeout:  30 * time.Second,
	}
	for _, s := range setters {
		if err := s(f); err != nil {
			return nil, err
		}
	}

	// Make sure we update the timeout that dialer is going to use
	*idleTimeoutPtr = f.idleTimeout

	if f.rewriter == nil {
		f.rewriter = &HeaderRewriter{
			TrustForwardHeader: true,
			Hostname:           "",
		}
	}

	return f, nil
}

func (f *Forwarder) ServeHTTP(w http.ResponseWriter, req *http.Request) {

	// Create a copy of the request suitable for our needs
	reqClone, err := f.cloneRequest(req, req.URL)
	if err != nil {
		log.Errorf("Error forwarding to %v, error: %v", req.Host, err)
		f.errHandler.ServeHTTP(w, req, err)
		return
	}
	f.rewriter.Rewrite(reqClone)

	if log.IsTraceEnabled() {
		reqStr, _ := httputil.DumpRequest(req, false)
		log.Tracef("Forwarder Middleware received request:\n%s", reqStr)

		reqStr2, _ := httputil.DumpRequest(reqClone, false)
		log.Tracef("Forwarder Middleware forwarding rewritten request:\n%s", reqStr2)
	}

	// Forward the request and get a response
	start := time.Now().UTC()
	response, err := f.roundTripper.RoundTrip(reqClone)
	if err != nil {
		log.Errorf("Error forwarding to %v, error: %v", req.Host, err)
		f.errHandler.ServeHTTP(w, req, err)
		return
	}
	log.Debugf("Round trip: %v, code: %v, duration: %v",
		req.URL, response.StatusCode, time.Now().UTC().Sub(start))

	if log.IsTraceEnabled() {
		respStr, _ := httputil.DumpResponse(response, true)
		log.Tracef("Forward Middleware received response:\n%s", respStr)
	}

	// Forward the response to the origin
	copyHeadersForForwarding(w.Header(), response.Header)
	w.WriteHeader(response.StatusCode)

	// It became nil in a Co-Advisor test though the doc says it will never be nil
	if response.Body != nil {
		_, err = io.Copy(w, response.Body)
		if err != nil {
			log.Error(err)
		}

		response.Body.Close()
	}
}

func (f *Forwarder) cloneRequest(req *http.Request, u *url.URL) (*http.Request, error) {
	outReq := new(http.Request)
	// Beware, this will make a shallow copy. We have to copy all maps
	*outReq = *req

	outReq.Proto = "HTTP/1.1"
	outReq.ProtoMajor = 1
	outReq.ProtoMinor = 1
	// Overwrite close flag: keep persistent connection for the backend servers
	outReq.Close = false

	// Request Header
	outReq.Header = make(http.Header)
	copyHeadersForForwarding(outReq.Header, req.Header)

	// Request URL
	outReq.URL = cloneURL(req.URL)
	// We know that is going to be HTTP always because HTTPS isn't forwarded.
	// We need to hardcode it here because req.URL.Scheme can be undefined, since
	// client request don't need to use absolute URIs
	outReq.URL.Scheme = "http"
	// We need to make sure the host is defined in the URL (not the actual URI)
	outReq.URL.Host = req.Host
	// Make sure we define an opaque URL, so the URI is just the path
	outReq.URL.Opaque = req.URL.Path
	outReq.URL.RawQuery = req.URL.RawQuery

	userAgent := req.UserAgent()
	if userAgent == "" {
		outReq.Header.Del("User-Agent")
	} else {
		outReq.Header.Set("User-Agent", userAgent)
	}

	/*
		// Trailer support
		// We are forced to do this because Go's server won't allow us to read the trailers otherwise
		_, err := httputil.DumpRequestOut(req, true)
		if err != nil {
			log.Errorf("Error: %v", err)
			return outReq, err
		}

		rcloser := ioutil.NopCloser(req.Body)
		outReq.Body = rcloser

		chunkedTransfer := false
		for _, enc := range req.TransferEncoding {
			if enc == "chunked" {
				chunkedTransfer = true
				break
			}
		}

		// Append Trailer
		if chunkedTransfer && len(req.Trailer) > 0 {
			outReq.Trailer = http.Header{}
			for k, vv := range req.Trailer {
				for _, v := range vv {
					outReq.Trailer.Add(k, v)
				}
			}
		}
	*/

	return outReq, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2020 Shivaram Lingamneni <slingamn@cs.stanford.edu>
// released under the MIT license

package irc

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"strconv"

	"github.com/tidwall/buntdb"

	"github.com/oragono/oragono/irc/utils"
)

const (
	// produce a hardcoded version of the database schema
	// XXX instead of referencing, e.g., keyAccountExists, we should write in the string literal
	// (to ensure that no matter what code changes happen elsewhere, we're still producing a
	// db of the hardcoded version)
	importDBSchemaVersion = 19
)

type userImport struct {
	Name            string
	Hash            string
	Email           string
	RegisteredAt    int64 `json:"registeredAt"`
	Vhost           string
	AdditionalNicks []string `json:"additionalNicks"`
	Certfps         []string
}

type channelImport struct {
	Name         string
	Founder      string
	RegisteredAt int64 `json:"registeredAt"`
	Topic        string
	TopicSetBy   string `json:"topicSetBy"`
	TopicSetAt   int64  `json:"topicSetAt"`
	Amode        map[string]string
	Modes        string
	Key          string
	Limit        int
}

type databaseImport struct {
	Version  int
	Source   string
	Users    map[string]userImport
	Channels map[string]channelImport
}

func serializeAmodes(raw map[string]string, validCfUsernames utils.StringSet) (result []byte, err error) {
	processed := make(map[string]int, len(raw))
	for accountName, mode := range raw {
		if len(mode) != 1 {
			return nil, fmt.Errorf("invalid mode %s for account %s", mode, accountName)
		}
		cfname, err := CasefoldName(accountName)
		if err != nil || !validCfUsernames.Has(cfname) {
			log.Printf("skipping invalid amode recipient %s\n", accountName)
		} else {
			processed[cfname] = int(mode[0])
		}
	}
	result, err = json.Marshal(processed)
	return
}

func doImportDBGeneric(config *Config, dbImport databaseImport, credsType CredentialsVersion, tx *buntdb.Tx) (err error) {
	requiredVersion := 1
	if dbImport.Version != requiredVersion {
		return fmt.Errorf("unsupported version of the db for import: version %d is required", requiredVersion)
	}

	tx.Set(keySchemaVersion, strconv.Itoa(importDBSchemaVersion), nil)
	tx.Set(keyCloakSecret, utils.GenerateSecretKey(), nil)

	cfUsernames := make(utils.StringSet)

	for username, userInfo := range dbImport.Users {
		cfUsername, err := CasefoldName(username)
		if err != nil {
			log.Printf("invalid username %s: %v", username, err)
			continue
		}
		var certfps []string
		for _, certfp := range userInfo.Certfps {
			normalizedCertfp, err := utils.NormalizeCertfp(certfp)
			if err == nil {
				certfps = append(certfps, normalizedCertfp)
			} else {
				log.Printf("invalid certfp %s for %s\n", username, certfp)
			}
		}
		credentials := AccountCredentials{
			Version:        credsType,
			PassphraseHash: []byte(userInfo.Hash),
			Certfps:        certfps,
		}
		marshaledCredentials, err := json.Marshal(&credentials)
		if err != nil {
			log.Printf("invalid credentials for %s: %v", username, err)
			continue
		}
		tx.Set(fmt.Sprintf(keyAccountExists, cfUsername), "1", nil)
		tx.Set(fmt.Sprintf(keyAccountVerified, cfUsername), "1", nil)
		tx.Set(fmt.Sprintf(keyAccountName, cfUsername), userInfo.Name, nil)
		tx.Set(fmt.Sprintf(keyAccountCallback, cfUsername), "mailto:"+userInfo.Email, nil)
		tx.Set(fmt.Sprintf(keyAccountCredentials, cfUsername), string(marshaledCredentials), nil)
		tx.Set(fmt.Sprintf(keyAccountRegTime, cfUsername), strconv.FormatInt(userInfo.RegisteredAt, 10), nil)
		if userInfo.Vhost != "" {
			vhinfo := VHostInfo{
				Enabled:       true,
				ApprovedVHost: userInfo.Vhost,
			}
			vhBytes, err := json.Marshal(vhinfo)
			if err == nil {
				tx.Set(fmt.Sprintf(keyAccountVHost, cfUsername), string(vhBytes), nil)
			} else {
				log.Printf("couldn't serialize vhost for %s: %v\n", username, err)
			}
		}
		if len(userInfo.AdditionalNicks) != 0 {
			tx.Set(fmt.Sprintf(keyAccountAdditionalNicks, cfUsername), marshalReservedNicks(userInfo.AdditionalNicks), nil)
		}
		for _, certfp := range certfps {
			tx.Set(fmt.Sprintf(keyCertToAccount, certfp), cfUsername, nil)
		}
		cfUsernames.Add(cfUsername)
	}

	for chname, chInfo := range dbImport.Channels {
		cfchname, err := CasefoldChannel(chname)
		if err != nil {
			log.Printf("invalid channel name %s: %v", chname, err)
			continue
		}
		cffounder, err := CasefoldName(chInfo.Founder)
		if err != nil {
			log.Printf("invalid founder %s for channel %s: %v", chInfo.Founder, chname, err)
			continue
		}
		tx.Set(fmt.Sprintf(keyChannelExists, cfchname), "1", nil)
		tx.Set(fmt.Sprintf(keyChannelName, cfchname), chname, nil)
		tx.Set(fmt.Sprintf(keyChannelRegTime, cfchname), strconv.FormatInt(chInfo.RegisteredAt, 10), nil)
		tx.Set(fmt.Sprintf(keyChannelFounder, cfchname), cffounder, nil)
		accountChannelsKey := fmt.Sprintf(keyAccountChannels, cffounder)
		founderChannels, fcErr := tx.Get(accountChannelsKey)
		if fcErr != nil || founderChannels == "" {
			founderChannels = cfchname
		} else {
			founderChannels = fmt.Sprintf("%s,%s", founderChannels, cfchname)
		}
		tx.Set(accountChannelsKey, founderChannels, nil)
		if chInfo.Topic != "" {
			tx.Set(fmt.Sprintf(keyChannelTopic, cfchname), chInfo.Topic, nil)
			tx.Set(fmt.Sprintf(keyChannelTopicSetTime, cfchname), strconv.FormatInt(chInfo.TopicSetAt, 10), nil)
			tx.Set(fmt.Sprintf(keyChannelTopicSetBy, cfchname), chInfo.TopicSetBy, nil)
		}
		if len(chInfo.Amode) != 0 {
			m, err := serializeAmodes(chInfo.Amode, cfUsernames)
			if err == nil {
				tx.Set(fmt.Sprintf(keyChannelAccountToUMode, cfchname), string(m), nil)
			} else {
				log.Printf("couldn't serialize amodes for %s: %v", chname, err)
			}
		}
		tx.Set(fmt.Sprintf(keyChannelModes, cfchname), chInfo.Modes, nil)
		if chInfo.Key != "" {
			tx.Set(fmt.Sprintf(keyChannelPassword, cfchname), chInfo.Key, nil)
		}
		if chInfo.Limit > 0 {
			tx.Set(fmt.Sprintf(keyChannelUserLimit, cfchname), strconv.Itoa(chInfo.Limit), nil)
		}
	}

	return nil
}

func doImportDB(config *Config, dbImport databaseImport, tx *buntdb.Tx) (err error) {
	switch dbImport.Source {
	case "atheme":
		return doImportDBGeneric(config, dbImport, CredentialsAtheme, tx)
	case "anope":
		return doImportDBGeneric(config, dbImport, CredentialsAnope, tx)
	default:
		return fmt.Errorf("unsupported import source: %s", dbImport.Source)
	}
}

func ImportDB(config *Config, infile string) (err error) {
	data, err := ioutil.ReadFile(infile)
	if err != nil {
		return
	}

	var dbImport databaseImport
	err = json.Unmarshal(data, &dbImport)
	if err != nil {
		return err
	}

	err = checkDBReadyForInit(config.Datastore.Path)
	if err != nil {
		return err
	}

	db, err := buntdb.Open(config.Datastore.Path)
	if err != nil {
		return err
	}

	performImport := func(tx *buntdb.Tx) (err error) {
		return doImportDB(config, dbImport, tx)
	}

	return db.Update(performImport)
}
<commit_msg>fix #1439<commit_after>// Copyright (c) 2020 Shivaram Lingamneni <slingamn@cs.stanford.edu>
// released under the MIT license

package irc

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"strconv"

	"github.com/tidwall/buntdb"

	"github.com/oragono/oragono/irc/utils"
)

const (
	// produce a hardcoded version of the database schema
	// XXX instead of referencing, e.g., keyAccountExists, we should write in the string literal
	// (to ensure that no matter what code changes happen elsewhere, we're still producing a
	// db of the hardcoded version)
	importDBSchemaVersion = 19
)

type userImport struct {
	Name            string
	Hash            string
	Email           string
	RegisteredAt    int64 `json:"registeredAt"`
	Vhost           string
	AdditionalNicks []string `json:"additionalNicks"`
	Certfps         []string
}

type channelImport struct {
	Name         string
	Founder      string
	RegisteredAt int64 `json:"registeredAt"`
	Topic        string
	TopicSetBy   string `json:"topicSetBy"`
	TopicSetAt   int64  `json:"topicSetAt"`
	Amode        map[string]string
	Modes        string
	Key          string
	Limit        int
}

type databaseImport struct {
	Version  int
	Source   string
	Users    map[string]userImport
	Channels map[string]channelImport
}

func serializeAmodes(raw map[string]string, validCfUsernames utils.StringSet) (result []byte, err error) {
	processed := make(map[string]int, len(raw))
	for accountName, mode := range raw {
		if len(mode) != 1 {
			return nil, fmt.Errorf("invalid mode %s for account %s", mode, accountName)
		}
		cfname, err := CasefoldName(accountName)
		if err != nil || !validCfUsernames.Has(cfname) {
			log.Printf("skipping invalid amode recipient %s\n", accountName)
		} else {
			processed[cfname] = int(mode[0])
		}
	}
	result, err = json.Marshal(processed)
	return
}

func doImportDBGeneric(config *Config, dbImport databaseImport, credsType CredentialsVersion, tx *buntdb.Tx) (err error) {
	requiredVersion := 1
	if dbImport.Version != requiredVersion {
		return fmt.Errorf("unsupported version of the db for import: version %d is required", requiredVersion)
	}

	tx.Set(keySchemaVersion, strconv.Itoa(importDBSchemaVersion), nil)
	tx.Set(keyCloakSecret, utils.GenerateSecretKey(), nil)

	cfUsernames := make(utils.StringSet)
	skeletonToUsername := make(map[string]string)
	warnSkeletons := false

	for username, userInfo := range dbImport.Users {
		cfUsername, err := CasefoldName(username)
		skeleton, skErr := Skeleton(username)
		if err != nil || skErr != nil {
			log.Printf("invalid username %s: %v\n", username, err)
			continue
		}

		if existingSkelUser, ok := skeletonToUsername[skeleton]; ok {
			log.Printf("Users %s and %s have confusable nicknames; this may render one or both accounts unusable\n", username, existingSkelUser)
			warnSkeletons = true
		} else {
			skeletonToUsername[skeleton] = username
		}

		var certfps []string
		for _, certfp := range userInfo.Certfps {
			normalizedCertfp, err := utils.NormalizeCertfp(certfp)
			if err == nil {
				certfps = append(certfps, normalizedCertfp)
			} else {
				log.Printf("invalid certfp %s for %s\n", username, certfp)
			}
		}
		credentials := AccountCredentials{
			Version:        credsType,
			PassphraseHash: []byte(userInfo.Hash),
			Certfps:        certfps,
		}
		marshaledCredentials, err := json.Marshal(&credentials)
		if err != nil {
			log.Printf("invalid credentials for %s: %v\n", username, err)
			continue
		}
		tx.Set(fmt.Sprintf(keyAccountExists, cfUsername), "1", nil)
		tx.Set(fmt.Sprintf(keyAccountVerified, cfUsername), "1", nil)
		tx.Set(fmt.Sprintf(keyAccountName, cfUsername), userInfo.Name, nil)
		tx.Set(fmt.Sprintf(keyAccountCallback, cfUsername), "mailto:"+userInfo.Email, nil)
		tx.Set(fmt.Sprintf(keyAccountCredentials, cfUsername), string(marshaledCredentials), nil)
		tx.Set(fmt.Sprintf(keyAccountRegTime, cfUsername), strconv.FormatInt(userInfo.RegisteredAt, 10), nil)
		if userInfo.Vhost != "" {
			vhinfo := VHostInfo{
				Enabled:       true,
				ApprovedVHost: userInfo.Vhost,
			}
			vhBytes, err := json.Marshal(vhinfo)
			if err == nil {
				tx.Set(fmt.Sprintf(keyAccountVHost, cfUsername), string(vhBytes), nil)
			} else {
				log.Printf("couldn't serialize vhost for %s: %v\n", username, err)
			}
		}
		if len(userInfo.AdditionalNicks) != 0 {
			tx.Set(fmt.Sprintf(keyAccountAdditionalNicks, cfUsername), marshalReservedNicks(userInfo.AdditionalNicks), nil)
		}
		for _, certfp := range certfps {
			tx.Set(fmt.Sprintf(keyCertToAccount, certfp), cfUsername, nil)
		}
		cfUsernames.Add(cfUsername)
	}

	for chname, chInfo := range dbImport.Channels {
		cfchname, err := CasefoldChannel(chname)
		if err != nil {
			log.Printf("invalid channel name %s: %v", chname, err)
			continue
		}
		cffounder, err := CasefoldName(chInfo.Founder)
		if err != nil {
			log.Printf("invalid founder %s for channel %s: %v", chInfo.Founder, chname, err)
			continue
		}
		tx.Set(fmt.Sprintf(keyChannelExists, cfchname), "1", nil)
		tx.Set(fmt.Sprintf(keyChannelName, cfchname), chname, nil)
		tx.Set(fmt.Sprintf(keyChannelRegTime, cfchname), strconv.FormatInt(chInfo.RegisteredAt, 10), nil)
		tx.Set(fmt.Sprintf(keyChannelFounder, cfchname), cffounder, nil)
		accountChannelsKey := fmt.Sprintf(keyAccountChannels, cffounder)
		founderChannels, fcErr := tx.Get(accountChannelsKey)
		if fcErr != nil || founderChannels == "" {
			founderChannels = cfchname
		} else {
			founderChannels = fmt.Sprintf("%s,%s", founderChannels, cfchname)
		}
		tx.Set(accountChannelsKey, founderChannels, nil)
		if chInfo.Topic != "" {
			tx.Set(fmt.Sprintf(keyChannelTopic, cfchname), chInfo.Topic, nil)
			tx.Set(fmt.Sprintf(keyChannelTopicSetTime, cfchname), strconv.FormatInt(chInfo.TopicSetAt, 10), nil)
			tx.Set(fmt.Sprintf(keyChannelTopicSetBy, cfchname), chInfo.TopicSetBy, nil)
		}
		if len(chInfo.Amode) != 0 {
			m, err := serializeAmodes(chInfo.Amode, cfUsernames)
			if err == nil {
				tx.Set(fmt.Sprintf(keyChannelAccountToUMode, cfchname), string(m), nil)
			} else {
				log.Printf("couldn't serialize amodes for %s: %v", chname, err)
			}
		}
		tx.Set(fmt.Sprintf(keyChannelModes, cfchname), chInfo.Modes, nil)
		if chInfo.Key != "" {
			tx.Set(fmt.Sprintf(keyChannelPassword, cfchname), chInfo.Key, nil)
		}
		if chInfo.Limit > 0 {
			tx.Set(fmt.Sprintf(keyChannelUserLimit, cfchname), strconv.Itoa(chInfo.Limit), nil)
		}
	}

	if warnSkeletons {
		log.Printf("NOTE: you may be able to avoid confusability issues by changing the server casemapping setting to `ascii`\n")
		log.Printf("However, this will prevent the use of non-ASCII Unicode characters in nicknames\n")
	}

	return nil
}

func doImportDB(config *Config, dbImport databaseImport, tx *buntdb.Tx) (err error) {
	switch dbImport.Source {
	case "atheme":
		return doImportDBGeneric(config, dbImport, CredentialsAtheme, tx)
	case "anope":
		return doImportDBGeneric(config, dbImport, CredentialsAnope, tx)
	default:
		return fmt.Errorf("unsupported import source: %s", dbImport.Source)
	}
}

func ImportDB(config *Config, infile string) (err error) {
	data, err := ioutil.ReadFile(infile)
	if err != nil {
		return
	}

	var dbImport databaseImport
	err = json.Unmarshal(data, &dbImport)
	if err != nil {
		return err
	}

	err = checkDBReadyForInit(config.Datastore.Path)
	if err != nil {
		return err
	}

	db, err := buntdb.Open(config.Datastore.Path)
	if err != nil {
		return err
	}

	performImport := func(tx *buntdb.Tx) (err error) {
		return doImportDB(config, dbImport, tx)
	}

	return db.Update(performImport)
}
<|endoftext|>


<commit_before>package jetpack

import (
	"crypto/sha512"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
	"github.com/juju/errors"

	"lib/run"
)

// Save image to an ACI file, return its hash.
//
// If path is an empty string, don't save the image, just return the
// hash. If path is "-", print image to stdout.
func (img *Image) saveACI(path string, packlist *os.File, perm os.FileMode) (*types.Hash, error) {
	tarArgs := []string{"-C", img.Path(), "-c", "--null", "-f", "-"}
	if packlist != nil {
		tarArgs = append(tarArgs, "-n", "-T", "-")
	} else {
		// no packlist -> flat ACI
		manifest := img.Manifest
		manifest.Dependencies = nil
		manifest.PathWhitelist = nil

		manifestF, err := ioutil.TempFile(img.Path(), "manifest.flat.")
		if err != nil {
			return nil, errors.Trace(err)
		}
		defer os.Remove(manifestF.Name())

		if manifestB, err := json.Marshal(manifest); err != nil {
			manifestF.Close()
			return nil, errors.Trace(err)
		} else {
			_, err := manifestF.Write(manifestB)
			manifestF.Close()
			if err != nil {
				return nil, errors.Trace(err)
			}
		}

		manifestN := filepath.Base(manifestF.Name())
		tarArgs = append(tarArgs, "-s", "/^"+manifestN+"$/manifest/", manifestN, "rootfs")
	}
	archiver := run.Command("tar", tarArgs...).ReadFrom(packlist)
	if archive, err := archiver.StdoutPipe(); err != nil {
		return nil, errors.Trace(err)
	} else {
		hash := sha512.New()
		faucet := io.TeeReader(archive, hash)
		sink := ioutil.Discard
		var compressor *run.Cmd = nil

		if path != "" {
			if path == "-" {
				sink = os.Stdout
			} else {
				if f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm); err != nil {
					return nil, errors.Trace(err)
				} else {
					defer f.Close()
					sink = f
				}
			}

			if compression := img.Host.Properties.GetString("images.aci.compression", "no"); compression != "none" {
				switch compression {
				case "xz":
					compressor = run.Command("xz", "-z", "-c")
				case "bzip2":
					compressor = run.Command("bzip2", "-z", "-c")
				case "gz":
				case "gzip":
					compressor = run.Command("gzip", "-c")
				default:
					return nil, errors.Errorf("Invalid setting images.aci.compression=%#v (allowed values: xz, bzip2, gzip, none)", compression)
				}

				compressor.Cmd.Stdout = sink
				if cin, err := compressor.StdinPipe(); err != nil {
					return nil, errors.Trace(err)
				} else {
					sink = cin
				}
			}
		}

		if err := archiver.Start(); err != nil {
			return nil, errors.Trace(err)
		}

		if compressor != nil {
			if err := compressor.Start(); err != nil {
				archiver.Cmd.Process.Kill()
				return nil, errors.Trace(err)
			}
		}

		if _, err := io.Copy(sink, faucet); err != nil {
			return nil, errors.Trace(err)
		}

		if hash, err := types.NewHash(fmt.Sprintf("sha512-%x", hash.Sum(nil))); err != nil {
			// CAN'T HAPPEN, srsly
			return nil, errors.Trace(err)
		} else {
			return hash, nil
		}
	}
}

func (img *Image) SaveFlatACI(path string, perm os.FileMode) (*types.Hash, error) {
	return img.saveACI(path, nil, perm)
}

func (img *Image) buildPodManifest(exec []string) *schema.PodManifest {
	bpm := schema.BlankPodManifest()

	// Figure out working path that doesn't exist in the image's rootfs
	workDir := ".jetpack.build."
	for {
		if _, err := os.Stat(img.getRootfs().Path(workDir)); err != nil {
			if os.IsNotExist(err) {
				break
			}
			panic(err)
		}
		workDir = fmt.Sprintf(".jetpack.build.%v", uuid.NewRandom())
	}

	bprta := img.RuntimeApp()
	bprta.Name.Set("jetpack/build")
	bprta.App = &types.App{
		Exec:             exec,
		WorkingDirectory: "/" + workDir,
		User:             "0",
		Group:            "0",
	}
	bpm.Apps = append(bpm.Apps, bprta)

	// This is needed by freebsd-update at least, should be okay to
	// allow this in builders.
	bpm.Annotations.Set("jetpack/jail.conf/allow.chflags", "true")

	return bpm
}

func (img *Image) Build(buildDir string, addFiles []string, buildExec []string) (*Image, error) {
	buildPod, err := img.Host.CreatePod(img.buildPodManifest(buildExec))
	if err != nil {
		return nil, errors.Trace(err)
	}

	ds, err := img.Host.Dataset.GetDataset(path.Join("pods", buildPod.UUID.String(), "rootfs.0"))
	if err != nil {
		return nil, errors.Trace(err)
	}

	parentSnap, err := ds.Snapshot("parent")
	if err != nil {
		return nil, errors.Trace(err)
	}

	fullWorkDir := buildPod.Path("rootfs/0", buildPod.Manifest.Apps[0].App.WorkingDirectory)
	if err := os.Mkdir(fullWorkDir, 0700); err != nil {
		return nil, errors.Trace(err)
	}

	if buildDir[len(buildDir)-1] != '/' {
		buildDir += "/"
	}

	cpArgs := []string{"-R", buildDir}
	if addFiles != nil {
		cpArgs = append(cpArgs, addFiles...)
	}
	cpArgs = append(cpArgs, fullWorkDir)

	if err := run.Command("cp", cpArgs...).Run(); err != nil {
		return nil, errors.Trace(err)
	}

	if err := buildPod.RunApp(buildPod.Manifest.Apps[0].Name); err != nil {
		return nil, errors.Trace(err)
	}

	if err := buildPod.Kill(); err != nil {
		return nil, errors.Trace(err)
	}

	manifestBytes, err := ioutil.ReadFile(filepath.Join(fullWorkDir, "manifest.json"))
	if err != nil {
		return nil, errors.Trace(err)
	}

	if err := os.RemoveAll(fullWorkDir); err != nil {
		return nil, errors.Trace(err)
	}

	if err := os.Remove(buildPod.Path("rootfs/0/etc/resolv.conf")); err != nil && !os.IsNotExist(err) {
		return nil, errors.Trace(err)
	}

	// Get packing list while parentSnap's name haven't changed

	packlist, err := ioutil.TempFile("", "aci.packlist.")
	if err != nil {
		return nil, errors.Trace(err)
	}
	os.Remove(packlist.Name())
	defer packlist.Close()

	haveDeletions := false
	if diffs, err := parentSnap.ZfsFields("diff"); err != nil {
		return nil, errors.Trace(err)
	} else {
		for _, diff := range diffs {
			switch diff[0] {
			case "+", "M":
				fmt.Fprintln(packlist, filepath.Join("rootfs", diff[1][len(ds.Mountpoint):]))
			case "R":
				fmt.Fprintln(packlist, filepath.Join("rootfs", diff[2][len(ds.Mountpoint):]))
				fallthrough
			case "-":
				haveDeletions = true
			default:
				return nil, errors.Errorf("Unknown `zfs diff` line: %v", diff)
			}
		}
	}
	packlist.Seek(0, os.SEEK_SET)

	// Pivot pod into an image
	childImage := NewImage(img.Host, buildPod.UUID)

	if err := ds.Set("mountpoint", childImage.Path("rootfs")); err != nil {
		return nil, errors.Trace(err)
	}

	if err := ds.Rename(img.Host.Dataset.ChildName(path.Join("images", childImage.UUID.String()))); err != nil {
		return nil, errors.Trace(err)
	}

	// Construct the child image's manifest

	if err := json.Unmarshal(manifestBytes, &childImage.Manifest); err != nil {
		return nil, errors.Trace(err)
	}

	// We don't need build pod anymore
	if err := buildPod.Destroy(); err != nil {
		return nil, errors.Trace(err)
	}
	buildPod = nil

	if _, ok := childImage.Manifest.Annotations.Get("timestamp"); !ok {
		if ts, err := time.Now().MarshalText(); err != nil {
			return nil, errors.Trace(err)
		} else {
			childImage.Manifest.Annotations.Set("timestamp", string(ts))
		}
	}

	for _, label := range []string{"os", "arch"} {
		if childValue, ok := childImage.Manifest.GetLabel(label); !ok {
			// if child has no os/arch, copy from parent
			if parentValue, ok := img.Manifest.GetLabel(label); ok {
				childImage.Manifest.Labels = append(childImage.Manifest.Labels,
					types.Label{Name: types.ACName(label), Value: parentValue})
			}
		} else if childValue == "" {
			// if child explicitly set to nil or empty string, remove the
			// label
			for i, l := range childImage.Manifest.Labels {
				if string(l.Name) == label {
					childImage.Manifest.Labels = append(
						childImage.Manifest.Labels[:i],
						childImage.Manifest.Labels[i+1:]...)
					break
				}
			}
		}
	}

	// TODO: option to create a flat image
	childImage.Manifest.Dependencies = append(types.Dependencies{
		types.Dependency{
			App:     img.Manifest.Name,
			ImageID: img.Hash,
			Labels:  img.Manifest.Labels,
		}}, childImage.Manifest.Dependencies...)

	if haveDeletions {
		prefixLen := len(ds.Mountpoint)
		if err := filepath.Walk(ds.Mountpoint, func(path string, _ os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if len(path) == prefixLen {
				// All paths are prefixed with ds.Mountpoint. Cheaper to compare lengths than whole string.
				return nil
			}
			childImage.Manifest.PathWhitelist = append(childImage.Manifest.PathWhitelist, path[prefixLen:])
			return nil
		}); err != nil {
			return nil, errors.Trace(err)
		}
		sort.Strings(childImage.Manifest.PathWhitelist)
	}

	if err := img.saveManifest(); err != nil {
		return nil, errors.Trace(err)
	}

	// Save the ACI
	if hash, err := childImage.saveACI(childImage.Path("aci"), packlist, 0440); err != nil {
		return nil, errors.Trace(err)
	} else {
		childImage.Hash = hash
	}

	if err := childImage.sealImage(); err != nil {
		return nil, errors.Trace(err)
	}

	return childImage, nil
}
<commit_msg>packlist should be NULL-, not newline-separated list<commit_after>package jetpack

import (
	"crypto/sha512"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
	"github.com/juju/errors"

	"lib/run"
)

// Save image to an ACI file, return its hash.
//
// If path is an empty string, don't save the image, just return the
// hash. If path is "-", print image to stdout.
func (img *Image) saveACI(path string, packlist *os.File, perm os.FileMode) (*types.Hash, error) {
	tarArgs := []string{"-C", img.Path(), "-c", "--null", "-f", "-"}
	if packlist != nil {
		tarArgs = append(tarArgs, "-n", "-T", "-")
	} else {
		// no packlist -> flat ACI
		manifest := img.Manifest
		manifest.Dependencies = nil
		manifest.PathWhitelist = nil

		manifestF, err := ioutil.TempFile(img.Path(), "manifest.flat.")
		if err != nil {
			return nil, errors.Trace(err)
		}
		defer os.Remove(manifestF.Name())

		if manifestB, err := json.Marshal(manifest); err != nil {
			manifestF.Close()
			return nil, errors.Trace(err)
		} else {
			_, err := manifestF.Write(manifestB)
			manifestF.Close()
			if err != nil {
				return nil, errors.Trace(err)
			}
		}

		manifestN := filepath.Base(manifestF.Name())
		tarArgs = append(tarArgs, "-s", "/^"+manifestN+"$/manifest/", manifestN, "rootfs")
	}
	archiver := run.Command("tar", tarArgs...).ReadFrom(packlist)
	if archive, err := archiver.StdoutPipe(); err != nil {
		return nil, errors.Trace(err)
	} else {
		hash := sha512.New()
		faucet := io.TeeReader(archive, hash)
		sink := ioutil.Discard
		var compressor *run.Cmd = nil

		if path != "" {
			if path == "-" {
				sink = os.Stdout
			} else {
				if f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm); err != nil {
					return nil, errors.Trace(err)
				} else {
					defer f.Close()
					sink = f
				}
			}

			if compression := img.Host.Properties.GetString("images.aci.compression", "no"); compression != "none" {
				switch compression {
				case "xz":
					compressor = run.Command("xz", "-z", "-c")
				case "bzip2":
					compressor = run.Command("bzip2", "-z", "-c")
				case "gz":
				case "gzip":
					compressor = run.Command("gzip", "-c")
				default:
					return nil, errors.Errorf("Invalid setting images.aci.compression=%#v (allowed values: xz, bzip2, gzip, none)", compression)
				}

				compressor.Cmd.Stdout = sink
				if cin, err := compressor.StdinPipe(); err != nil {
					return nil, errors.Trace(err)
				} else {
					sink = cin
				}
			}
		}

		if err := archiver.Start(); err != nil {
			return nil, errors.Trace(err)
		}

		if compressor != nil {
			if err := compressor.Start(); err != nil {
				archiver.Cmd.Process.Kill()
				return nil, errors.Trace(err)
			}
		}

		if _, err := io.Copy(sink, faucet); err != nil {
			return nil, errors.Trace(err)
		}

		if hash, err := types.NewHash(fmt.Sprintf("sha512-%x", hash.Sum(nil))); err != nil {
			// CAN'T HAPPEN, srsly
			return nil, errors.Trace(err)
		} else {
			return hash, nil
		}
	}
}

func (img *Image) SaveFlatACI(path string, perm os.FileMode) (*types.Hash, error) {
	return img.saveACI(path, nil, perm)
}

func (img *Image) buildPodManifest(exec []string) *schema.PodManifest {
	bpm := schema.BlankPodManifest()

	// Figure out working path that doesn't exist in the image's rootfs
	workDir := ".jetpack.build."
	for {
		if _, err := os.Stat(img.getRootfs().Path(workDir)); err != nil {
			if os.IsNotExist(err) {
				break
			}
			panic(err)
		}
		workDir = fmt.Sprintf(".jetpack.build.%v", uuid.NewRandom())
	}

	bprta := img.RuntimeApp()
	bprta.Name.Set("jetpack/build")
	bprta.App = &types.App{
		Exec:             exec,
		WorkingDirectory: "/" + workDir,
		User:             "0",
		Group:            "0",
	}
	bpm.Apps = append(bpm.Apps, bprta)

	// This is needed by freebsd-update at least, should be okay to
	// allow this in builders.
	bpm.Annotations.Set("jetpack/jail.conf/allow.chflags", "true")

	return bpm
}

func (img *Image) Build(buildDir string, addFiles []string, buildExec []string) (*Image, error) {
	buildPod, err := img.Host.CreatePod(img.buildPodManifest(buildExec))
	if err != nil {
		return nil, errors.Trace(err)
	}

	ds, err := img.Host.Dataset.GetDataset(path.Join("pods", buildPod.UUID.String(), "rootfs.0"))
	if err != nil {
		return nil, errors.Trace(err)
	}

	parentSnap, err := ds.Snapshot("parent")
	if err != nil {
		return nil, errors.Trace(err)
	}

	fullWorkDir := buildPod.Path("rootfs/0", buildPod.Manifest.Apps[0].App.WorkingDirectory)
	if err := os.Mkdir(fullWorkDir, 0700); err != nil {
		return nil, errors.Trace(err)
	}

	if buildDir[len(buildDir)-1] != '/' {
		buildDir += "/"
	}

	cpArgs := []string{"-R", buildDir}
	if addFiles != nil {
		cpArgs = append(cpArgs, addFiles...)
	}
	cpArgs = append(cpArgs, fullWorkDir)

	if err := run.Command("cp", cpArgs...).Run(); err != nil {
		return nil, errors.Trace(err)
	}

	if err := buildPod.RunApp(buildPod.Manifest.Apps[0].Name); err != nil {
		return nil, errors.Trace(err)
	}

	if err := buildPod.Kill(); err != nil {
		return nil, errors.Trace(err)
	}

	manifestBytes, err := ioutil.ReadFile(filepath.Join(fullWorkDir, "manifest.json"))
	if err != nil {
		return nil, errors.Trace(err)
	}

	if err := os.RemoveAll(fullWorkDir); err != nil {
		return nil, errors.Trace(err)
	}

	if err := os.Remove(buildPod.Path("rootfs/0/etc/resolv.conf")); err != nil && !os.IsNotExist(err) {
		return nil, errors.Trace(err)
	}

	// Get packing list while parentSnap's name haven't changed

	packlist, err := ioutil.TempFile("", "aci.packlist.")
	if err != nil {
		return nil, errors.Trace(err)
	}
	os.Remove(packlist.Name())
	defer packlist.Close()
	io.WriteString(packlist, "manifest")

	haveDeletions := false
	if diffs, err := parentSnap.ZfsFields("diff"); err != nil {
		return nil, errors.Trace(err)
	} else {
		for _, diff := range diffs {
			switch diff[0] {
			case "+", "M":
				io.WriteString(packlist, filepath.Join("\000rootfs", diff[1][len(ds.Mountpoint):]))
			case "R":
				io.WriteString(packlist, filepath.Join("\000rootfs", diff[2][len(ds.Mountpoint):]))
				fallthrough
			case "-":
				haveDeletions = true
			default:
				return nil, errors.Errorf("Unknown `zfs diff` line: %v", diff)
			}
		}
	}
	packlist.Seek(0, os.SEEK_SET)

	// Pivot pod into an image
	childImage := NewImage(img.Host, buildPod.UUID)

	if err := ds.Set("mountpoint", childImage.Path("rootfs")); err != nil {
		return nil, errors.Trace(err)
	}

	if err := ds.Rename(img.Host.Dataset.ChildName(path.Join("images", childImage.UUID.String()))); err != nil {
		return nil, errors.Trace(err)
	}

	// Construct the child image's manifest

	if err := json.Unmarshal(manifestBytes, &childImage.Manifest); err != nil {
		return nil, errors.Trace(err)
	}

	// We don't need build pod anymore
	if err := buildPod.Destroy(); err != nil {
		return nil, errors.Trace(err)
	}
	buildPod = nil

	if _, ok := childImage.Manifest.Annotations.Get("timestamp"); !ok {
		if ts, err := time.Now().MarshalText(); err != nil {
			return nil, errors.Trace(err)
		} else {
			childImage.Manifest.Annotations.Set("timestamp", string(ts))
		}
	}

	for _, label := range []string{"os", "arch"} {
		if childValue, ok := childImage.Manifest.GetLabel(label); !ok {
			// if child has no os/arch, copy from parent
			if parentValue, ok := img.Manifest.GetLabel(label); ok {
				childImage.Manifest.Labels = append(childImage.Manifest.Labels,
					types.Label{Name: types.ACName(label), Value: parentValue})
			}
		} else if childValue == "" {
			// if child explicitly set to nil or empty string, remove the
			// label
			for i, l := range childImage.Manifest.Labels {
				if string(l.Name) == label {
					childImage.Manifest.Labels = append(
						childImage.Manifest.Labels[:i],
						childImage.Manifest.Labels[i+1:]...)
					break
				}
			}
		}
	}

	// TODO: option to create a flat image
	childImage.Manifest.Dependencies = append(types.Dependencies{
		types.Dependency{
			App:     img.Manifest.Name,
			ImageID: img.Hash,
			Labels:  img.Manifest.Labels,
		}}, childImage.Manifest.Dependencies...)

	if haveDeletions {
		prefixLen := len(ds.Mountpoint)
		if err := filepath.Walk(ds.Mountpoint, func(path string, _ os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if len(path) == prefixLen {
				// All paths are prefixed with ds.Mountpoint. Cheaper to compare lengths than whole string.
				return nil
			}
			childImage.Manifest.PathWhitelist = append(childImage.Manifest.PathWhitelist, path[prefixLen:])
			return nil
		}); err != nil {
			return nil, errors.Trace(err)
		}
		sort.Strings(childImage.Manifest.PathWhitelist)
	}

	if err := childImage.saveManifest(); err != nil {
		return nil, errors.Trace(err)
	}

	// Save the ACI
	if hash, err := childImage.saveACI(childImage.Path("aci"), packlist, 0440); err != nil {
		return nil, errors.Trace(err)
	} else {
		childImage.Hash = hash
	}

	if err := childImage.sealImage(); err != nil {
		return nil, errors.Trace(err)
	}

	return childImage, nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"reflect"
	"testing"

	"github.com/google/cel-go/cel"
	"github.com/google/cel-go/common/types"
	exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
)

func TestTypes_ListType(t *testing.T) {
	list := NewListType(StringType)
	if !list.IsList() {
		t.Error("list type not identifiable as list")
	}
	if list.TypeName() != "list" {
		t.Errorf("got %s, wanted list", list.TypeName())
	}
	if list.DefaultValue() == nil {
		t.Error("got nil zero value for list type")
	}
	if list.ElemType.TypeName() != "string" {
		t.Errorf("got %s, wanted elem type of string", list.ElemType.TypeName())
	}
	if list.ExprType().GetListType() == nil {
		t.Errorf("got %v, wanted CEL list type", list.ExprType())
	}
}

func TestTypes_MapType(t *testing.T) {
	mp := NewMapType(StringType, IntType)
	if !mp.IsMap() {
		t.Error("map type not identifiable as map")
	}
	if mp.TypeName() != "map" {
		t.Errorf("got %s, wanted map", mp.TypeName())
	}
	if mp.DefaultValue() == nil {
		t.Error("got nil zero value for map type")
	}
	if mp.KeyType.TypeName() != "string" {
		t.Errorf("got %s, wanted key type of string", mp.KeyType.TypeName())
	}
	if mp.ElemType.TypeName() != "int" {
		t.Errorf("got %s, wanted elem type of int", mp.ElemType.TypeName())
	}
	if mp.ExprType().GetMapType() == nil {
		t.Errorf("got %v, wanted CEL map type", mp.ExprType())
	}
}

func TestTypes_SchemaDeclTypes(t *testing.T) {
	ts := testSchema()
	cust, typeMap, err := ts.DeclTypes("mock_template")
	if err != nil {
		t.Fatalf("ts.DeclTypes('mock_template') failed: %v", err)
	}
	nested, _ := cust.FindField("nested")
	dates, _ := nested.Type.FindField("dates")
	flags, _ := nested.Type.FindField("flags")
	// This is the type name that is assigned by the NewRuleTypes call, which may be informed
	// by the template name itself and of which the schema should not know directly.
	nested.Type.MaybeAssignTypeName("CustomObject.nested")
	expectedTypeMap := map[string]*DeclType{
		"CustomObject":              cust,
		"CustomObject.nested":       nested.Type,
		"CustomObject.nested.dates": dates.Type,
		"CustomObject.nested.flags": flags.Type,
	}
	if len(typeMap) != len(expectedTypeMap) {
		t.Errorf("got different type set. got=%v, wanted=%v", typeMap, expectedTypeMap)
	}
	for exp := range expectedTypeMap {
		found := false
		for act := range typeMap {
			if act == exp {
				found = true
				break
			}
		}
		if !found {
			t.Errorf("missing expected type: %s", exp)
		}
	}
	for exp, expType := range expectedTypeMap {
		actType, found := typeMap[exp]
		if !found {
			t.Errorf("missing type in rule types: %s", exp)
		}
		if !reflect.DeepEqual(expType, actType) {
			t.Errorf("incompatible CEL types. got=%v, wanted=%v", actType, expType)
		}
	}
}

func TestTypes_RuleTypesFieldMapping(t *testing.T) {
	stdEnv, _ := cel.NewEnv()
	reg := NewRegistry(stdEnv)
	rt, err := NewRuleTypes("mock_template", testSchema(), reg)
	if err != nil {
		t.Fatal(err)
	}
	nestedFieldType, found := rt.FindFieldType("CustomObject", "nested")
	if !found {
		t.Fatal("got field not found for 'CustomObject.nested', wanted found")
	}
	if nestedFieldType.Type.GetMessageType() != "CustomObject.nested" {
		t.Errorf("got field type %v, wanted mock_template.nested", nestedFieldType.Type)
	}
	subnameFieldType, found := rt.FindFieldType("CustomObject.nested", "subname")
	if !found {
		t.Fatal("got field not found for 'CustomObject.nested.subname', wanted found")
	}
	if subnameFieldType.Type.GetPrimitive() != exprpb.Type_STRING {
		t.Errorf("got field type %v, wanted string", subnameFieldType.Type)
	}
	flagsFieldType, found := rt.FindFieldType("CustomObject.nested", "flags")
	if !found {
		t.Fatal("got field not found for 'CustomObject.nested.flags', wanted found")
	}
	if flagsFieldType.Type.GetMapType() == nil {
		t.Errorf("got field type %v, wanted map", flagsFieldType.Type)
	}
	flagFieldType, found := rt.FindFieldType("CustomObject.nested.flags", "my_flag")
	if !found {
		t.Fatal("got field not found for 'CustomObject.nested.flags.my_flag', wanted found")
	}
	if flagFieldType.Type.GetPrimitive() != exprpb.Type_BOOL {
		t.Errorf("got field type %v, wanted bool", flagFieldType.Type)
	}

	// Manually constructed instance of the schema.
	name := NewField(1, "name")
	name.Ref = NewDynValue(2, "test-instance")
	nestedVal := NewMapValue()
	flags := NewField(5, "flags")
	flagsVal := NewMapValue()
	myFlag := NewField(6, "my_flag")
	myFlag.Ref = NewDynValue(7, true)
	flagsVal.AddField(myFlag)
	flags.Ref = NewDynValue(8, flagsVal)
	dates := NewField(9, "dates")
	dates.Ref = NewDynValue(10, NewListValue())
	nestedVal.AddField(flags)
	nestedVal.AddField(dates)
	nested := NewField(3, "nested")
	nested.Ref = NewDynValue(4, nestedVal)
	mapVal := NewMapValue()
	mapVal.AddField(name)
	mapVal.AddField(nested)
	rule := rt.ConvertToRule(NewDynValue(11, mapVal))
	if rule == nil {
		t.Error("map could not be converted to rule")
	}
	if rule.GetID() != 11 {
		t.Errorf("got %d as the rule id, wanted 11", rule.GetID())
	}
	ruleVal := rt.NativeToValue(rule)
	if ruleVal == nil {
		t.Error("got CEL rule value of nil, wanted non-nil")
	}

	opts, err := rt.EnvOptions(stdEnv.TypeProvider())
	if err != nil {
		t.Fatal(err)
	}
	ruleEnv, err := stdEnv.Extend(opts...)
	if err != nil {
		t.Fatal(err)
	}
	helloVal := ruleEnv.TypeAdapter().NativeToValue("hello")
	if helloVal.Equal(types.String("hello")) != types.True {
		t.Errorf("got %v, wanted types.String('hello')", helloVal)
	}
}
<commit_msg>Switch to proto.Equal to avoid common comparison bug<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"testing"

	"github.com/google/cel-go/cel"
	"github.com/google/cel-go/common/types"

	"github.com/golang/protobuf/proto"

	exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
)

func TestTypes_ListType(t *testing.T) {
	list := NewListType(StringType)
	if !list.IsList() {
		t.Error("list type not identifiable as list")
	}
	if list.TypeName() != "list" {
		t.Errorf("got %s, wanted list", list.TypeName())
	}
	if list.DefaultValue() == nil {
		t.Error("got nil zero value for list type")
	}
	if list.ElemType.TypeName() != "string" {
		t.Errorf("got %s, wanted elem type of string", list.ElemType.TypeName())
	}
	if list.ExprType().GetListType() == nil {
		t.Errorf("got %v, wanted CEL list type", list.ExprType())
	}
}

func TestTypes_MapType(t *testing.T) {
	mp := NewMapType(StringType, IntType)
	if !mp.IsMap() {
		t.Error("map type not identifiable as map")
	}
	if mp.TypeName() != "map" {
		t.Errorf("got %s, wanted map", mp.TypeName())
	}
	if mp.DefaultValue() == nil {
		t.Error("got nil zero value for map type")
	}
	if mp.KeyType.TypeName() != "string" {
		t.Errorf("got %s, wanted key type of string", mp.KeyType.TypeName())
	}
	if mp.ElemType.TypeName() != "int" {
		t.Errorf("got %s, wanted elem type of int", mp.ElemType.TypeName())
	}
	if mp.ExprType().GetMapType() == nil {
		t.Errorf("got %v, wanted CEL map type", mp.ExprType())
	}
}

func TestTypes_SchemaDeclTypes(t *testing.T) {
	ts := testSchema()
	cust, typeMap, err := ts.DeclTypes("mock_template")
	if err != nil {
		t.Fatalf("ts.DeclTypes('mock_template') failed: %v", err)
	}
	nested, _ := cust.FindField("nested")
	dates, _ := nested.Type.FindField("dates")
	flags, _ := nested.Type.FindField("flags")
	// This is the type name that is assigned by the NewRuleTypes call, which may be informed
	// by the template name itself and of which the schema should not know directly.
	nested.Type.MaybeAssignTypeName("CustomObject.nested")
	expectedTypeMap := map[string]*DeclType{
		"CustomObject":              cust,
		"CustomObject.nested":       nested.Type,
		"CustomObject.nested.dates": dates.Type,
		"CustomObject.nested.flags": flags.Type,
	}
	if len(typeMap) != len(expectedTypeMap) {
		t.Errorf("got different type set. got=%v, wanted=%v", typeMap, expectedTypeMap)
	}
	for exp := range expectedTypeMap {
		found := false
		for act := range typeMap {
			if act == exp {
				found = true
				break
			}
		}
		if !found {
			t.Errorf("missing expected type: %s", exp)
		}
	}
	for exp, expType := range expectedTypeMap {
		actType, found := typeMap[exp]
		if !found {
			t.Errorf("missing type in rule types: %s", exp)
		}
		if !proto.Equal(expType.ExprType(), actType.ExprType()) {
			t.Errorf("incompatible CEL types. got=%v, wanted=%v", actType.ExprType(), expType.ExprType())
		}
	}
}

func TestTypes_RuleTypesFieldMapping(t *testing.T) {
	stdEnv, _ := cel.NewEnv()
	reg := NewRegistry(stdEnv)
	rt, err := NewRuleTypes("mock_template", testSchema(), reg)
	if err != nil {
		t.Fatal(err)
	}
	nestedFieldType, found := rt.FindFieldType("CustomObject", "nested")
	if !found {
		t.Fatal("got field not found for 'CustomObject.nested', wanted found")
	}
	if nestedFieldType.Type.GetMessageType() != "CustomObject.nested" {
		t.Errorf("got field type %v, wanted mock_template.nested", nestedFieldType.Type)
	}
	subnameFieldType, found := rt.FindFieldType("CustomObject.nested", "subname")
	if !found {
		t.Fatal("got field not found for 'CustomObject.nested.subname', wanted found")
	}
	if subnameFieldType.Type.GetPrimitive() != exprpb.Type_STRING {
		t.Errorf("got field type %v, wanted string", subnameFieldType.Type)
	}
	flagsFieldType, found := rt.FindFieldType("CustomObject.nested", "flags")
	if !found {
		t.Fatal("got field not found for 'CustomObject.nested.flags', wanted found")
	}
	if flagsFieldType.Type.GetMapType() == nil {
		t.Errorf("got field type %v, wanted map", flagsFieldType.Type)
	}
	flagFieldType, found := rt.FindFieldType("CustomObject.nested.flags", "my_flag")
	if !found {
		t.Fatal("got field not found for 'CustomObject.nested.flags.my_flag', wanted found")
	}
	if flagFieldType.Type.GetPrimitive() != exprpb.Type_BOOL {
		t.Errorf("got field type %v, wanted bool", flagFieldType.Type)
	}

	// Manually constructed instance of the schema.
	name := NewField(1, "name")
	name.Ref = NewDynValue(2, "test-instance")
	nestedVal := NewMapValue()
	flags := NewField(5, "flags")
	flagsVal := NewMapValue()
	myFlag := NewField(6, "my_flag")
	myFlag.Ref = NewDynValue(7, true)
	flagsVal.AddField(myFlag)
	flags.Ref = NewDynValue(8, flagsVal)
	dates := NewField(9, "dates")
	dates.Ref = NewDynValue(10, NewListValue())
	nestedVal.AddField(flags)
	nestedVal.AddField(dates)
	nested := NewField(3, "nested")
	nested.Ref = NewDynValue(4, nestedVal)
	mapVal := NewMapValue()
	mapVal.AddField(name)
	mapVal.AddField(nested)
	rule := rt.ConvertToRule(NewDynValue(11, mapVal))
	if rule == nil {
		t.Error("map could not be converted to rule")
	}
	if rule.GetID() != 11 {
		t.Errorf("got %d as the rule id, wanted 11", rule.GetID())
	}
	ruleVal := rt.NativeToValue(rule)
	if ruleVal == nil {
		t.Error("got CEL rule value of nil, wanted non-nil")
	}

	opts, err := rt.EnvOptions(stdEnv.TypeProvider())
	if err != nil {
		t.Fatal(err)
	}
	ruleEnv, err := stdEnv.Extend(opts...)
	if err != nil {
		t.Fatal(err)
	}
	helloVal := ruleEnv.TypeAdapter().NativeToValue("hello")
	if helloVal.Equal(types.String("hello")) != types.True {
		t.Errorf("got %v, wanted types.String('hello')", helloVal)
	}
}
<|endoftext|>