<commit_before>package version

import (
	"os"
	"strconv"
)

// APIVersion contains the API base version. Only bumped for backward incompatible changes.
var APIVersion = "1.0"

// APIExtensions is the list of all API extensions in the order they were added.
//
// The following kind of changes come with a new extensions:
//
// - New configuration key
// - New valid values for a configuration key
// - New REST API endpoint
// - New argument inside an existing REST API call
// - New HTTPs authentication mechanisms or protocols
//
// This list is used mainly by the LXD server code, but it's in the shared
// package as well for reference.
var APIExtensions = []string{
	"storage_zfs_remove_snapshots",
	"container_host_shutdown_timeout",
	"container_stop_priority",
	"container_syscall_filtering",
	"auth_pki",
	"container_last_used_at",
	"etag",
	"patch",
	"usb_devices",
	"https_allowed_credentials",
	"image_compression_algorithm",
	"directory_manipulation",
	"container_cpu_time",
	"storage_zfs_use_refquota",
	"storage_lvm_mount_options",
	"network",
	"profile_usedby",
	"container_push",
	"container_exec_recording",
	"certificate_update",
	"container_exec_signal_handling",
	"gpu_devices",
	"container_image_properties",
	"migration_progress",
	"id_map",
	"network_firewall_filtering",
	"network_routes",
	"storage",
	"file_delete",
	"file_append",
	"network_dhcp_expiry",
	"storage_lvm_vg_rename",
	"storage_lvm_thinpool_rename",
	"network_vlan",
	"image_create_aliases",
	"container_stateless_copy",
	"container_only_migration",
	"storage_zfs_clone_copy",
	"unix_device_rename",
	"storage_lvm_use_thinpool",
	"storage_rsync_bwlimit",
	"network_vxlan_interface",
	"storage_btrfs_mount_options",
	"entity_description",
	"image_force_refresh",
	"storage_lvm_lv_resizing",
	"id_map_base",
	"file_symlinks",
	"container_push_target",
	"network_vlan_physical",
	"storage_images_delete",
	"container_edit_metadata",
	"container_snapshot_stateful_migration",
	"storage_driver_ceph",
	"storage_ceph_user_name",
	"resource_limits",
	"storage_volatile_initial_source",
	"storage_ceph_force_osd_reuse",
	"storage_block_filesystem_btrfs",
	"resources",
	"kernel_limits",
	"storage_api_volume_rename",
	"macaroon_authentication",
	"network_sriov",
	"console",
	"restrict_devlxd",
	"migration_pre_copy",
	"infiniband",
	"maas_network",
	"devlxd_events",
	"proxy",
	"network_dhcp_gateway",
	"file_get_symlink",
	"network_leases",
	"unix_device_hotplug",
	"storage_api_local_volume_handling",
	"operation_description",
	"clustering",
	"event_lifecycle",
	"storage_api_remote_volume_handling",
	"nvidia_runtime",
	"container_mount_propagation",
	"container_backup",
	"devlxd_images",
	"container_local_cross_pool_handling",
	"proxy_unix",
	"proxy_udp",
	"clustering_join",
	"proxy_tcp_udp_multi_port_handling",
	"network_state",
	"proxy_unix_dac_properties",
	"container_protection_delete",
	"unix_priv_drop",
	"pprof_http",
	"proxy_haproxy_protocol",
	"network_hwaddr",
	"proxy_nat",
	"network_nat_order",
	"container_full",
	"candid_authentication",
	"backup_compression",
	"candid_config",
	"nvidia_runtime_config",
	"storage_api_volume_snapshots",
	"storage_unmapped",
	"projects",
	"candid_config_key",
	"network_vxlan_ttl",
	"container_incremental_copy",
	"usb_optional_vendorid",
	"snapshot_scheduling",
	"container_copy_project",
	"clustering_server_address",
	"clustering_image_replication",
	"container_protection_shift",
	"snapshot_expiry",
	"container_backup_override_pool",
	"snapshot_expiry_creation",
	"network_leases_location",
	"resources_cpu_socket",
	"resources_gpu",
	"resources_numa",
	"kernel_features",
	"id_map_current",
	"event_location",
	"storage_api_remote_volume_snapshots",
	"network_nat_address",
	"container_nic_routes",
	"rbac",
	"cluster_internal_copy",
	"seccomp_notify",
	"lxc_features",
	"container_nic_ipvlan",
	"network_vlan_sriov",
	"storage_cephfs",
	"container_nic_ipfilter",
	"resources_v2",
	"container_exec_user_group_cwd",
	"container_syscall_intercept",
	"container_disk_shift",
	"storage_shifted",
	"resources_infiniband",
	"daemon_storage",
	"instances",
	"image_types",
	"resources_disk_sata",
	"clustering_roles",
	"images_expiry",
	"resources_network_firmware",
	"backup_compression_algorithm",
	"ceph_data_pool_name",
	"container_syscall_intercept_mount",
	"compression_squashfs",
	"container_raw_mount",
	"container_nic_routed",
	"container_syscall_intercept_mount_fuse",
	"container_disk_ceph",
	"virtual-machines",
	"image_profiles",
	"clustering_architecture",
	"resources_disk_id",
	"storage_lvm_stripes",
	"vm_boot_priority",
	"unix_hotplug_devices",
	"api_filtering",
	"instance_nic_network",
	"clustering_sizing",
	"firewall_driver",
	"projects_limits",
	"container_syscall_intercept_hugetlbfs",
	"limits_hugepages",
	"container_nic_routed_gateway",
	"projects_restrictions",
	"custom_volume_snapshot_expiry",
	"volume_snapshot_scheduling",
	"trust_ca_certificates",
	"snapshot_disk_usage",
	"clustering_edit_roles",
	"container_nic_routed_host_address",
	"container_nic_ipvlan_gateway",
	"resources_usb_pci",
	"resources_cpu_threads_numa",
	"resources_cpu_core_die",
	"api_os",
	"container_nic_routed_host_table",
	"container_nic_ipvlan_host_table",
	"container_nic_ipvlan_mode",
}

// APIExtensionsCount returns the number of available API extensions.
func APIExtensionsCount() int {
	count := len(APIExtensions)

	// This environment variable is an internal one to force the code
	// to believe that we have an API extensions count greater than we
	// actually have. It's used by integration tests to exercise the
	// cluster upgrade process.
	artificialBump := os.Getenv("LXD_ARTIFICIALLY_BUMP_API_EXTENSIONS")
	if artificialBump != "" {
		n, err := strconv.Atoi(artificialBump)
		if err == nil {
			count += n
		}
	}

	return count
}
<commit_msg>shared/version/api: Add resources_system API extension<commit_after>package version

import (
	"os"
	"strconv"
)

// APIVersion contains the API base version. Only bumped for backward incompatible changes.
var APIVersion = "1.0"

// APIExtensions is the list of all API extensions in the order they were added.
//
// The following kind of changes come with a new extensions:
//
// - New configuration key
// - New valid values for a configuration key
// - New REST API endpoint
// - New argument inside an existing REST API call
// - New HTTPs authentication mechanisms or protocols
//
// This list is used mainly by the LXD server code, but it's in the shared
// package as well for reference.
var APIExtensions = []string{
	"storage_zfs_remove_snapshots",
	"container_host_shutdown_timeout",
	"container_stop_priority",
	"container_syscall_filtering",
	"auth_pki",
	"container_last_used_at",
	"etag",
	"patch",
	"usb_devices",
	"https_allowed_credentials",
	"image_compression_algorithm",
	"directory_manipulation",
	"container_cpu_time",
	"storage_zfs_use_refquota",
	"storage_lvm_mount_options",
	"network",
	"profile_usedby",
	"container_push",
	"container_exec_recording",
	"certificate_update",
	"container_exec_signal_handling",
	"gpu_devices",
	"container_image_properties",
	"migration_progress",
	"id_map",
	"network_firewall_filtering",
	"network_routes",
	"storage",
	"file_delete",
	"file_append",
	"network_dhcp_expiry",
	"storage_lvm_vg_rename",
	"storage_lvm_thinpool_rename",
	"network_vlan",
	"image_create_aliases",
	"container_stateless_copy",
	"container_only_migration",
	"storage_zfs_clone_copy",
	"unix_device_rename",
	"storage_lvm_use_thinpool",
	"storage_rsync_bwlimit",
	"network_vxlan_interface",
	"storage_btrfs_mount_options",
	"entity_description",
	"image_force_refresh",
	"storage_lvm_lv_resizing",
	"id_map_base",
	"file_symlinks",
	"container_push_target",
	"network_vlan_physical",
	"storage_images_delete",
	"container_edit_metadata",
	"container_snapshot_stateful_migration",
	"storage_driver_ceph",
	"storage_ceph_user_name",
	"resource_limits",
	"storage_volatile_initial_source",
	"storage_ceph_force_osd_reuse",
	"storage_block_filesystem_btrfs",
	"resources",
	"kernel_limits",
	"storage_api_volume_rename",
	"macaroon_authentication",
	"network_sriov",
	"console",
	"restrict_devlxd",
	"migration_pre_copy",
	"infiniband",
	"maas_network",
	"devlxd_events",
	"proxy",
	"network_dhcp_gateway",
	"file_get_symlink",
	"network_leases",
	"unix_device_hotplug",
	"storage_api_local_volume_handling",
	"operation_description",
	"clustering",
	"event_lifecycle",
	"storage_api_remote_volume_handling",
	"nvidia_runtime",
	"container_mount_propagation",
	"container_backup",
	"devlxd_images",
	"container_local_cross_pool_handling",
	"proxy_unix",
	"proxy_udp",
	"clustering_join",
	"proxy_tcp_udp_multi_port_handling",
	"network_state",
	"proxy_unix_dac_properties",
	"container_protection_delete",
	"unix_priv_drop",
	"pprof_http",
	"proxy_haproxy_protocol",
	"network_hwaddr",
	"proxy_nat",
	"network_nat_order",
	"container_full",
	"candid_authentication",
	"backup_compression",
	"candid_config",
	"nvidia_runtime_config",
	"storage_api_volume_snapshots",
	"storage_unmapped",
	"projects",
	"candid_config_key",
	"network_vxlan_ttl",
	"container_incremental_copy",
	"usb_optional_vendorid",
	"snapshot_scheduling",
	"container_copy_project",
	"clustering_server_address",
	"clustering_image_replication",
	"container_protection_shift",
	"snapshot_expiry",
	"container_backup_override_pool",
	"snapshot_expiry_creation",
	"network_leases_location",
	"resources_cpu_socket",
	"resources_gpu",
	"resources_numa",
	"kernel_features",
	"id_map_current",
	"event_location",
	"storage_api_remote_volume_snapshots",
	"network_nat_address",
	"container_nic_routes",
	"rbac",
	"cluster_internal_copy",
	"seccomp_notify",
	"lxc_features",
	"container_nic_ipvlan",
	"network_vlan_sriov",
	"storage_cephfs",
	"container_nic_ipfilter",
	"resources_v2",
	"container_exec_user_group_cwd",
	"container_syscall_intercept",
	"container_disk_shift",
	"storage_shifted",
	"resources_infiniband",
	"daemon_storage",
	"instances",
	"image_types",
	"resources_disk_sata",
	"clustering_roles",
	"images_expiry",
	"resources_network_firmware",
	"backup_compression_algorithm",
	"ceph_data_pool_name",
	"container_syscall_intercept_mount",
	"compression_squashfs",
	"container_raw_mount",
	"container_nic_routed",
	"container_syscall_intercept_mount_fuse",
	"container_disk_ceph",
	"virtual-machines",
	"image_profiles",
	"clustering_architecture",
	"resources_disk_id",
	"storage_lvm_stripes",
	"vm_boot_priority",
	"unix_hotplug_devices",
	"api_filtering",
	"instance_nic_network",
	"clustering_sizing",
	"firewall_driver",
	"projects_limits",
	"container_syscall_intercept_hugetlbfs",
	"limits_hugepages",
	"container_nic_routed_gateway",
	"projects_restrictions",
	"custom_volume_snapshot_expiry",
	"volume_snapshot_scheduling",
	"trust_ca_certificates",
	"snapshot_disk_usage",
	"clustering_edit_roles",
	"container_nic_routed_host_address",
	"container_nic_ipvlan_gateway",
	"resources_usb_pci",
	"resources_cpu_threads_numa",
	"resources_cpu_core_die",
	"api_os",
	"container_nic_routed_host_table",
	"container_nic_ipvlan_host_table",
	"container_nic_ipvlan_mode",
	"resources_system",
}

// APIExtensionsCount returns the number of available API extensions.
func APIExtensionsCount() int {
	count := len(APIExtensions)

	// This environment variable is an internal one to force the code
	// to believe that we have an API extensions count greater than we
	// actually have. It's used by integration tests to exercise the
	// cluster upgrade process.
	artificialBump := os.Getenv("LXD_ARTIFICIALLY_BUMP_API_EXTENSIONS")
	if artificialBump != "" {
		n, err := strconv.Atoi(artificialBump)
		if err == nil {
			count += n
		}
	}

	return count
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"common"
	"compress/gzip"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"time"
)

type NoOrchestratorFound struct{}

func (t *NoOrchestratorFound) Error() string {
	return "No Orchestrator Found"
}

// loadBonesFile does initial bones file loading and does some quick and dirty
// data sanitization
func loadBonesFile() *common.SkeletonDeployment {
	log.Print("Loading bonesFile")
	config, err := os.Open("bonesFile")
	if err != nil {
		log.Fatal(err)
	}

	configslice, err := ioutil.ReadAll(config)
	if err != nil {
		log.Fatal(err)
	}

	deploy := new(common.SkeletonDeployment)

	err = json.Unmarshal(configslice, deploy)
	if err != nil {
		log.Fatal(err)
	}

	for k, v := range deploy.Containers {
		if len(v.Granularity) == 0 {
			v.Granularity = "deployment"
			deploy.Containers[k] = v
		}
		if len(v.Mode) == 0 {
			v.Mode = "default"
			deploy.Containers[k] = v
		}
	}

	if len(deploy.Machines.Provider) == 0 {
		log.Fatal("Machine Provider must be specified")
	}

	log.Print("bonesFile loaded")
	return deploy
}

// findOrchestrator finds a running orchestrator by scanning port 900 on all
// machines it knows about
func findOrchestrator(config *common.SkeletonDeployment) (string, error) {
	log.Print("Finding orchestrator")
	client := common.MakeHttpClient()
	for _, v := range config.Machines.Ip {
		_, err := net.DialTimeout("tcp", v+":900", 1000*time.Millisecond)
		if err != nil {
			continue
		}
		_, err = client.Get("http://" + v + ":900/version")
		if err == nil {
			log.Print("Orchestrator Found")
			return v, nil
		}
	}

	log.Print("No Orchestrator Found")
	return "", new(NoOrchestratorFound)
}

// bootstrapOrchestrator starts up the orchestrator on a machine
func bootstrapOrchestrator(ip string) string {
	log.Print("Bootstrapping Orchestrator")
	tar := tarDir("../../containers/orchestrator")
	err := common.BuildImage(ip, tar, "orchestrator")
	if err != nil {
		log.Fatal(err)
	}
	err = common.RunImage(ip, "orchestrator", true)
	if err != nil {
		log.Fatal(err)
	}
	log.Print("Orchestrator bootstrapped")
	return ip
}

// deploy pushes our new deploy configuration to the orchestrator
func deploy(ip string, config *common.SkeletonDeployment) {

	log.Print("Pushing images to Orchestrator")
	h := common.MakeHttpClient()

	for k, _ := range config.Containers {
		image := tarDir(k)
		resp, err := h.Post("http://"+ip+":900/image?name="+k, "application/tar",
			image)
		if err != nil {
			log.Fatal(err)
		}
		defer resp.Body.Close()
		common.LogReader(resp.Body)
	}

	log.Print("Pushing configuration to Orchestrator")

	barr, err := json.Marshal(config)
	if err != nil {
		log.Fatal(err)
	}

	b := bytes.NewBuffer(barr)

	resp, err := h.Post("http://"+ip+":900/deploy", "application/json", b)

	if err != nil {
		log.Fatal(err)
	}

	defer resp.Body.Close()

	log.Print("Deploy Pushed")

	common.LogReader(resp.Body)

	return
}

//helper tarDir function taking in path and a *Writer object (no idea if this actually does what I want)
func tardir(path string, totar map[*tar.Header][]byte) {
	// Find subdirectories
	ifd, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}

	fi, err := ifd.Readdir(-1)
	if err != nil {
		log.Fatal(err)
	}

	// Put them in tarfile
	for _, f := range fi {
		//need to check f.isDir()
		//		if true		call tarDir(path+f.name(), w)
		if f.IsDir() {
			tardir(path + "/" + f.Name(), totar)
		}else{
			h, err := tar.FileInfoHeader(f, "")
			if err != nil {
				log.Fatal(err)
			}

			//w.WriteHeader(h)

			ffd, err := os.Open(path + "/" + f.Name())

			if err != nil {
				log.Fatal(err)
			}

			c, err := ioutil.ReadAll(ffd)
			if err != nil {
				log.Fatal(err)
			}
			//w.Write(c)
			totar[h] = c
		}
	}
}
// tarDir takes a directory path and produces a reader which is all of its
// contents tarred up and compressed with gzip
func tarDir(path string) io.Reader {
	log.Print("compressing ", path)
	// check this is a directory
	i, err := os.Stat(path)
	if err != nil {
		log.Fatal(err)
	}
	if !i.IsDir() {
		log.Fatal("Directory to tar up is not a directory")
	}

	//Make a buffer to hold the file
	b := bytes.NewBuffer(nil)
	g := gzip.NewWriter(b)
	w := tar.NewWriter(g)

	//recursiveness needs to happen here
	//fi is a 'slice' of all of the files/subdirectories at path
	totar := map[*tar.Header][]byte{}
	tardir(path, totar)

	//fmt.Println(totar)
	for k,v := range totar {
		w.WriteHeader(k)
		w.Write(v)
	}

	/*
	// Find subdirectories
	ifd, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}

	fi, err := ifd.Readdir(-1)
	if err != nil {
		log.Fatal(err)
	}

	// Put them in tarfile
	for _, f := range fi {
		//need to check f.isDir()
		//		if true		call tarDir(path+f.name(), w)
		h, err := tar.FileInfoHeader(f, "")
		if err != nil {
			log.Fatal(err)
		}

		w.WriteHeader(h)

		ffd, err := os.Open(path + "/" + f.Name())

		if err != nil {
			log.Fatal(err)
		}

		c, err := ioutil.ReadAll(ffd)
		if err != nil {
			log.Fatal(err)
		}
		w.Write(c)
	}
	*/
	w.Close()
	g.Close()
	return b
}

func main() {

	config := loadBonesFile()

	orch, err := findOrchestrator(config)
	switch err.(type) {

	// Initial Setup
	case *NoOrchestratorFound:
		orch = bootstrapOrchestrator(config.Machines.Ip[0])
		deploy(orch, config)

	// Update Deploy
	case nil:
		common.StopImage(orch, "orchestrator")
		orch = bootstrapOrchestrator(config.Machines.Ip[0])
		deploy(orch, config)

	// Error contacting orchestrator
	default:
		log.Fatal(err)
	}
}
<commit_msg>Remove duplicate tarTest folder<commit_after><|endoftext|>


<commit_before><commit_msg>store/storecache: fix cleanup on close error for saveToCacheFile<commit_after><|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/types"
)

// storageRenter unlocks the wallet, mines some currency, sets an allowance
// using that currency, and uploads some files.  It will periodically try to
// download those files, printing any errors that occur.
func (j *JobRunner) storageRenter() {
	j.tg.Add()
	defer j.tg.Done()

	err := j.client.Post("/wallet/unlock", fmt.Sprintf("encryptionpassword=%s&dictionary=%s", j.walletPassword, "english"), nil)
	if err != nil {
		log.Printf("[%v jobStorageRenter ERROR]: %v\n", j.siaDirectory, err)
		return
	}

	err = j.client.Get("/miner/start", nil)
	if err != nil {
		log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
		return
	}

	// Mine at least 100,000 SC
	desiredbalance := types.NewCurrency64(100000).Mul(types.SiacoinPrecision)
	success := false
	for start := time.Now(); time.Since(start) < 5*time.Minute; {
		select {
		case <-j.tg.StopChan():
			return
		case <-time.After(time.Second):
		}

		var walletInfo api.WalletGET
		err = j.client.Get("/wallet", &walletInfo)
		if err != nil {
			log.Printf("[%v jobStorageRenter ERROR]: %v\n", j.siaDirectory, err)
			return
		}
		if walletInfo.ConfirmedSiacoinBalance.Cmp(desiredbalance) > 0 {
			success = true
			break
		}
	}
	if !success {
		log.Printf("[%v jobStorageRenter ERROR]: timeout: could not mine enough currency after 5 minutes\n", j.siaDirectory)
		return
	}

	// Set an initial 50ksc allowance
	allowance := types.NewCurrency64(50000).Mul(types.SiacoinPrecision)
	if err := j.client.Post("/renter", fmt.Sprintf("funds=%v&period=100", allowance), nil); err != nil {
		log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
	}

	// Every 1000 seconds, set a new allowance.
	go func() {
		for {
			select {
			case <-j.tg.StopChan():
				return
			case <-time.After(time.Second * 1000):
			}
			func() {
				j.tg.Add()
				defer j.tg.Done()

				// set an allowance of 50k SC
				allowance := types.NewCurrency64(50000).Mul(types.SiacoinPrecision)
				if err := j.client.Post("/renter", fmt.Sprintf("funds=%v&period=100", allowance), nil); err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}
			}()
		}
	}()

	// Every 120 seconds, upload a 500MB file.  After ten files, delete one file
	// at random each iteration.
	go func() {
		var files []string

		// Clean up by deleting any created files when this goroutine returns.
		defer func() {
			for _, file := range files {
				os.Remove(file)
			}
		}()

		for i := 0; ; i++ {
			select {
			case <-j.tg.StopChan():
				return
			case <-time.After(time.Second * 120):
			}
			func() {
				j.tg.Add()
				defer j.tg.Done()

				if i >= 10 {
					randindex, err := crypto.RandIntn(len(files))
					if err != nil {
						log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
						return
					}
					if err = j.client.Post(fmt.Sprintf("/renter/delete/%v", files[randindex]), "", nil); err != nil {
						log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
					}
					files = append(files[:randindex], files[randindex+1:]...)
				}

				// Generate some random data to upload
				f, err := ioutil.TempFile("", "antfarm-renter")
				if err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}

				_, err = io.CopyN(f, rand.Reader, 500e6)
				if err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}

				// Upload the random data
				if err = j.client.Post(fmt.Sprintf("/renter/upload/%v", f.Name()), fmt.Sprintf("source=%v", f.Name()), nil); err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}

				files = append(files, f.Name())
			}()
		}
	}()

	// Every 200 seconds, verify that not more than the allowance has been spent.
	go func() {
		var renterInfo api.RenterGET
		if err := j.client.Get("/renter", &renterInfo); err != nil {
			log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
		}

		var walletInfo api.WalletGET
		if err := j.client.Get("/wallet", &walletInfo); err != nil {
			log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
		}

		initialBalance := walletInfo.ConfirmedSiacoinBalance

		for {
			select {
			case <-j.tg.StopChan():
				return
			case <-time.After(time.Second * 200):
			}
			func() {
				j.tg.Add()
				defer j.tg.Done()

				if err = j.client.Get("/wallet", &walletInfo); err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}

				spent := initialBalance.Sub(walletInfo.ConfirmedSiacoinBalance)
				if spent.Cmp(renterInfo.Settings.Allowance.Funds) > 0 {
					log.Printf("[%v jobStorageRenter ERROR: spent more than allowance: spent %v, allowance %v\n", j.siaDirectory, spent, renterInfo.Settings.Allowance.Funds)
				}
			}()
		}
	}()
}
<commit_msg>append `files` immediately after file creation<commit_after>package main

import (
	"crypto/rand"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/types"
)

// storageRenter unlocks the wallet, mines some currency, sets an allowance
// using that currency, and uploads some files.  It will periodically try to
// download those files, printing any errors that occur.
func (j *JobRunner) storageRenter() {
	j.tg.Add()
	defer j.tg.Done()

	err := j.client.Post("/wallet/unlock", fmt.Sprintf("encryptionpassword=%s&dictionary=%s", j.walletPassword, "english"), nil)
	if err != nil {
		log.Printf("[%v jobStorageRenter ERROR]: %v\n", j.siaDirectory, err)
		return
	}

	err = j.client.Get("/miner/start", nil)
	if err != nil {
		log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
		return
	}

	// Mine at least 100,000 SC
	desiredbalance := types.NewCurrency64(100000).Mul(types.SiacoinPrecision)
	success := false
	for start := time.Now(); time.Since(start) < 5*time.Minute; {
		select {
		case <-j.tg.StopChan():
			return
		case <-time.After(time.Second):
		}

		var walletInfo api.WalletGET
		err = j.client.Get("/wallet", &walletInfo)
		if err != nil {
			log.Printf("[%v jobStorageRenter ERROR]: %v\n", j.siaDirectory, err)
			return
		}
		if walletInfo.ConfirmedSiacoinBalance.Cmp(desiredbalance) > 0 {
			success = true
			break
		}
	}
	if !success {
		log.Printf("[%v jobStorageRenter ERROR]: timeout: could not mine enough currency after 5 minutes\n", j.siaDirectory)
		return
	}

	// Set an initial 50ksc allowance
	allowance := types.NewCurrency64(50000).Mul(types.SiacoinPrecision)
	if err := j.client.Post("/renter", fmt.Sprintf("funds=%v&period=100", allowance), nil); err != nil {
		log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
	}

	// Every 1000 seconds, set a new allowance.
	go func() {
		for {
			select {
			case <-j.tg.StopChan():
				return
			case <-time.After(time.Second * 1000):
			}
			func() {
				j.tg.Add()
				defer j.tg.Done()

				// set an allowance of 50k SC
				allowance := types.NewCurrency64(50000).Mul(types.SiacoinPrecision)
				if err := j.client.Post("/renter", fmt.Sprintf("funds=%v&period=100", allowance), nil); err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}
			}()
		}
	}()

	// Every 120 seconds, upload a 500MB file.  After ten files, delete one file
	// at random each iteration.
	go func() {
		var files []string

		// Clean up by deleting any created files when this goroutine returns.
		defer func() {
			for _, file := range files {
				os.Remove(file)
			}
		}()

		for i := 0; ; i++ {
			select {
			case <-j.tg.StopChan():
				return
			case <-time.After(time.Second * 120):
			}
			func() {
				j.tg.Add()
				defer j.tg.Done()

				if i >= 10 {
					randindex, err := crypto.RandIntn(len(files))
					if err != nil {
						log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
						return
					}
					if err = j.client.Post(fmt.Sprintf("/renter/delete/%v", files[randindex]), "", nil); err != nil {
						log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
					}
					files = append(files[:randindex], files[randindex+1:]...)
				}

				// Generate some random data to upload
				f, err := ioutil.TempFile("", "antfarm-renter")
				if err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}
				files = append(files, f.Name())

				_, err = io.CopyN(f, rand.Reader, 500e6)
				if err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}

				// Upload the random data
				if err = j.client.Post(fmt.Sprintf("/renter/upload/%v", f.Name()), fmt.Sprintf("source=%v", f.Name()), nil); err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}
			}()
		}
	}()

	// Every 200 seconds, verify that not more than the allowance has been spent.
	go func() {
		var renterInfo api.RenterGET
		if err := j.client.Get("/renter", &renterInfo); err != nil {
			log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
		}

		var walletInfo api.WalletGET
		if err := j.client.Get("/wallet", &walletInfo); err != nil {
			log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
		}

		initialBalance := walletInfo.ConfirmedSiacoinBalance

		for {
			select {
			case <-j.tg.StopChan():
				return
			case <-time.After(time.Second * 200):
			}
			func() {
				j.tg.Add()
				defer j.tg.Done()

				if err = j.client.Get("/wallet", &walletInfo); err != nil {
					log.Printf("[%v jobStorageRenter ERROR: %v\n", j.siaDirectory, err)
				}

				spent := initialBalance.Sub(walletInfo.ConfirmedSiacoinBalance)
				if spent.Cmp(renterInfo.Settings.Allowance.Funds) > 0 {
					log.Printf("[%v jobStorageRenter ERROR: spent more than allowance: spent %v, allowance %v\n", j.siaDirectory, spent, renterInfo.Settings.Allowance.Funds)
				}
			}()
		}
	}()
}
<|endoftext|>


<commit_before>package personal

import (
	"Yearning-go/src/engine"
	"Yearning-go/src/lib"
	"Yearning-go/src/model"
	"encoding/hex"
	"errors"
	"fmt"
	"github.com/cookieY/sqlx"
	"strings"
)

const (
	ORDER_POST_SUCCESS      = "工单已提交,请等待审核人审核！"
	ER_DB_CONNENT           = "数据库实例连接失败！请检查相关配置是否正确！"
	CUSTOM_INFO_SUCCESS     = "邮箱/真实姓名修改成功！刷新后显示最新信息!"
	CUSTOM_PASSWORD_SUCCESS = "密码修改成功！"
	ER_SQL_EMPTY            = "无查询语句！"
	BUF                     = 1<<20 - 1
	ER_RPC                  = "rpc调用失败"
)

type queryBind struct {
	Table    string `json:"table"`
	DataBase string `json:"data_base"`
	Source   string `json:"source"`
}

type QueryDeal struct {
	Ref struct {
		Type     int    `msgpack:"type"` //0 conn 1 close
		Sql      string `msgpack:"sql"`
		Schema   string `msgpack:"schema"`
		SourceId string `msgpack:"source_id"`
	}
	MultiSQLRunner []MultiSQLRunner
}

type MultiSQLRunner struct {
	SQL              string
	InsulateWordList map[string]struct{}
}

type Query struct {
	Field []map[string]interface{} `msgpack:"field"`
	Data  []map[string]interface{} `msgpack:"data"`
}

type QueryArgs struct {
	SQL              string
	Limit            uint64
	InsulateWordList string
}

func (q *QueryDeal) PreCheck(insulateWordList string) error {
	var rs []engine.Record
	if client := lib.NewRpc(); client != nil {
		if err := client.Call("Engine.Query", &QueryArgs{
			SQL:              q.Ref.Sql,
			Limit:            model.GloOther.Limit,
			InsulateWordList: insulateWordList,
		}, &rs); err != nil {
			return err
		}
		for _, i := range rs {
			if i.Error != "" {
				return errors.New(i.Error)
			}
			q.MultiSQLRunner = append(q.MultiSQLRunner, MultiSQLRunner{SQL: i.SQL, InsulateWordList: lib.MapOn(i.InsulateWordList)})
		}
		return nil
	}
	return errors.New(ER_RPC)
}

func (m *MultiSQLRunner) Run(db *sqlx.DB, schema string) (*Query, error) {
	query := new(Query)
	if db == nil {
		return nil, errors.New("数据库连接失败！")
	}
	db.Exec(fmt.Sprintf("use %s", schema))
	rows, err := db.Queryx(m.SQL)

	if err != nil {
		return nil, err
	}

	cols, err := rows.Columns()

	if err != nil {
		return nil, err
	}
	defer rows.Close()
	for rows.Next() {
		results := make(map[string]interface{})
		_ = rows.MapScan(results)
		for key := range results {
			switch r := results[key].(type) {
			case []uint8:
				if len(r) > BUF {
					results[key] = "blob字段无法显示"
				} else {
					switch hex.EncodeToString(r) {
					case "01":
						results[key] = "true"
					case "00":
						results[key] = "false"
					default:
						results[key] = string(r)
					}
					if m.excludeFieldContext(key) {
						results[key] = "****脱敏字段"
					}
				}
			}
		}
		query.Data = append(query.Data, results)
	}

	ele := removeDuplicateElement(cols)

	for cv := range ele {
		query.Field = append(query.Field, map[string]interface{}{"title": ele[cv], "dataIndex": ele[cv], "width": 200, "resizable": true, "ellipsis": true})
	}
	query.Field[0]["fixed"] = "left"

	return query, nil
}

func (m *MultiSQLRunner) excludeFieldContext(field string) bool {
	_, ok := m.InsulateWordList[strings.ToLower(field)]
	return ok
}

func removeDuplicateElement(addrs []string) []string {
	result := make([]string, 0, len(addrs))
	temp := map[string]struct{}{}
	idx := 0
	for _, item := range addrs {
		if _, ok := temp[item]; !ok {
			temp[item] = struct{}{}
			result = append(result, item)
		} else {
			idx++
			item += fmt.Sprintf("(%v)", idx)
			result = append(result, item)
		}
	}
	return result
}
<commit_msg>1.修复数据库名称含有中横线选择失败 2.修复消息推送没有工单说明的问题<commit_after>package personal

import (
	"Yearning-go/src/engine"
	"Yearning-go/src/lib"
	"Yearning-go/src/model"
	"encoding/hex"
	"errors"
	"fmt"
	"github.com/cookieY/sqlx"
	"strings"
)

const (
	ORDER_POST_SUCCESS      = "工单已提交,请等待审核人审核！"
	ER_DB_CONNENT           = "数据库实例连接失败！请检查相关配置是否正确！"
	CUSTOM_INFO_SUCCESS     = "邮箱/真实姓名修改成功！刷新后显示最新信息!"
	CUSTOM_PASSWORD_SUCCESS = "密码修改成功！"
	ER_SQL_EMPTY            = "无查询语句！"
	BUF                     = 1<<20 - 1
	ER_RPC                  = "rpc调用失败"
)

type queryBind struct {
	Table    string `json:"table"`
	DataBase string `json:"data_base"`
	Source   string `json:"source"`
}

type QueryDeal struct {
	Ref struct {
		Type     int    `msgpack:"type"` //0 conn 1 close
		Sql      string `msgpack:"sql"`
		Schema   string `msgpack:"schema"`
		SourceId string `msgpack:"source_id"`
	}
	MultiSQLRunner []MultiSQLRunner
}

type MultiSQLRunner struct {
	SQL              string
	InsulateWordList map[string]struct{}
}

type Query struct {
	Field []map[string]interface{} `msgpack:"field"`
	Data  []map[string]interface{} `msgpack:"data"`
}

type QueryArgs struct {
	SQL              string
	Limit            uint64
	InsulateWordList string
}

func (q *QueryDeal) PreCheck(insulateWordList string) error {
	var rs []engine.Record
	if client := lib.NewRpc(); client != nil {
		if err := client.Call("Engine.Query", &QueryArgs{
			SQL:              q.Ref.Sql,
			Limit:            model.GloOther.Limit,
			InsulateWordList: insulateWordList,
		}, &rs); err != nil {
			return err
		}
		for _, i := range rs {
			if i.Error != "" {
				return errors.New(i.Error)
			}
			q.MultiSQLRunner = append(q.MultiSQLRunner, MultiSQLRunner{SQL: i.SQL, InsulateWordList: lib.MapOn(i.InsulateWordList)})
		}
		return nil
	}
	return errors.New(ER_RPC)
}

func (m *MultiSQLRunner) Run(db *sqlx.DB, schema string) (*Query, error) {
	query := new(Query)
	if db == nil {
		return nil, errors.New("数据库连接失败！")
	}
	db.Exec(fmt.Sprintf("use `%s`", schema))
	rows, err := db.Queryx(m.SQL)

	if err != nil {
		return nil, err
	}

	cols, err := rows.Columns()

	if err != nil {
		return nil, err
	}
	defer rows.Close()
	for rows.Next() {
		results := make(map[string]interface{})
		_ = rows.MapScan(results)
		for key := range results {
			switch r := results[key].(type) {
			case []uint8:
				if len(r) > BUF {
					results[key] = "blob字段无法显示"
				} else {
					switch hex.EncodeToString(r) {
					case "01":
						results[key] = "true"
					case "00":
						results[key] = "false"
					default:
						results[key] = string(r)
					}
					if m.excludeFieldContext(key) {
						results[key] = "****脱敏字段"
					}
				}
			}
		}
		query.Data = append(query.Data, results)
	}

	ele := removeDuplicateElement(cols)

	for cv := range ele {
		query.Field = append(query.Field, map[string]interface{}{"title": ele[cv], "dataIndex": ele[cv], "width": 200, "resizable": true, "ellipsis": true})
	}
	query.Field[0]["fixed"] = "left"

	return query, nil
}

func (m *MultiSQLRunner) excludeFieldContext(field string) bool {
	_, ok := m.InsulateWordList[strings.ToLower(field)]
	return ok
}

func removeDuplicateElement(addrs []string) []string {
	result := make([]string, 0, len(addrs))
	temp := map[string]struct{}{}
	idx := 0
	for _, item := range addrs {
		if _, ok := temp[item]; !ok {
			temp[item] = struct{}{}
			result = append(result, item)
		} else {
			idx++
			item += fmt.Sprintf("(%v)", idx)
			result = append(result, item)
		}
	}
	return result
}
<|endoftext|>


<commit_before>package slurp

import (
	"sync"
	"time"
)

// Item is nothing more then a time and map[string]interface{} pair.
type Item struct {
	At   time.Time
	Data map[string]interface{}
}

// NewItem creates a new Item and returns a pointer to it.
func NewItem(at time.Time) *Item {
	return &Item{
		At:   at,
		Data: make(map[string]interface{}),
	}
}

// ItemChannelStatWrapper is used to monitor the throughput rate.
type ItemChannelStatWrapper struct {
	in     <-chan *Item
	Out    chan *Item
	itemAt *time.Time
	rate   float64
	count  int64
	mutex  sync.RWMutex
}

// Stat returns information sbout the item channel.
func (c *ItemChannelStatWrapper) Stat() ItemChannelStat {
	c.mutex.RLock()
	defer c.mutex.RUnlock()
	return ItemChannelStat{
		ItemAt:   c.itemAt,
		Rate:     c.rate,
		Count:    c.count,
		Length:   len(c.in),
		Capacity: cap(c.in),
	}
}

// ItemChannelStat provides statistic about the item channel.
type ItemChannelStat struct {
	ItemAt   *time.Time `json:"itemAt,omitempty"`
	Rate     float64    `json:"rate"`
	Count    int64      `json:"count"`
	Length   int        `json:"length"`
	Capacity int        `json:"capacity"`
}

// NewItemChannelStatWrapper creates a new ItemChannelStatWrapper
func NewItemChannelStatWrapper(items <-chan *Item) *ItemChannelStatWrapper {
	c := &ItemChannelStatWrapper{
		in:  items,
		Out: make(chan *Item, 0), // We want the Out chan to be unbuffered.
	}
	go func() {
		var (
			count  int64
			itemAt *time.Time
		)
		times := make([]int64, 100, 100)
		timesIdx := len(times)
		rateReport := time.NewTicker(200 * time.Millisecond)
		for {
			select {
			case <-rateReport.C:
				c.mutex.Lock()
				if timesIdx == len(times) {
					c.rate = 0
				} else {
					c.rate = 1000000000.0 * float64(len(times)-timesIdx) / float64(time.Now().UnixNano()-times[timesIdx])
				}
				c.count = count
				c.itemAt = itemAt
				c.mutex.Unlock()
			case i, ok := <-c.in:
				if !ok {
					rateReport.Stop()
					close(c.Out)
					c.mutex.Lock()
					c.rate = 0
					c.count = count
					c.itemAt = itemAt
					c.mutex.Unlock()
					return
				}
				c.Out <- i
				times = times[1:]
				times = append(times, time.Now().UnixNano())
				if timesIdx > 0 {
					timesIdx--
				}
				count++
				itemAt = &i.At
			}
		}
	}()
	return c
}

// Slurper is responsible for consuming a channel of items.
type Slurper interface {
	Slurp(<-chan *Item)
}

// SlurperFunc is an adapter that allow you to use
// an ordinary function as a Slurper.
type SlurperFunc func(<-chan *Item)

// Slurp calls f(items)
func (f SlurperFunc) Slurp(items <-chan *Item) {
	f(items)
}

// CompositionSlurper can be used to present multiple slurpers as a single
// slurper.
type CompositionSlurper struct {
	slurpers      []Slurper
	slurpFunction func(in <-chan *Item, out []chan *Item)
}

// Slurp coordinates the slurp for multiple slurpers. Responsability for the
// reading and writing to the channel is defered to the slurpFunction.
func (s *CompositionSlurper) Slurp(in <-chan *Item) {
	wg := sync.WaitGroup{}
	out := make([]chan *Item, len(s.slurpers))
	for i, slurper := range s.slurpers {
		out[i] = make(chan *Item, cap(in))
		wg.Add(1)
		go func(s Slurper, ch <-chan *Item) {
			defer wg.Done()
			s.Slurp(ch)
		}(slurper, out[i])
	}
	wg.Add(1)
	go func() {
		defer wg.Done()
		s.slurpFunction(in, out)
		// in has been consumed so lets ensure that
		// we close all of our out chans.
		for _, o := range out {
			close(o)
		}
	}()
	wg.Wait()
}

// NewFanOutSlurper will fan out all items to all slurpers.
func NewFanOutSlurper(slurpers ...Slurper) *CompositionSlurper {
	return &CompositionSlurper{
		slurpers: slurpers,
		slurpFunction: func(in <-chan *Item, out []chan *Item) {
			for i := range in {
				for _, o := range out {
					o <- i
				}
			}
		},
	}
}
<commit_msg>Fix typo.<commit_after>package slurp

import (
	"sync"
	"time"
)

// Item is nothing more then a time and map[string]interface{} pair.
type Item struct {
	At   time.Time
	Data map[string]interface{}
}

// NewItem creates a new Item and returns a pointer to it.
func NewItem(at time.Time) *Item {
	return &Item{
		At:   at,
		Data: make(map[string]interface{}),
	}
}

// ItemChannelStatWrapper is used to monitor the throughput rate.
type ItemChannelStatWrapper struct {
	in     <-chan *Item
	Out    chan *Item
	itemAt *time.Time
	rate   float64
	count  int64
	mutex  sync.RWMutex
}

// Stat returns information about the item channel.
func (c *ItemChannelStatWrapper) Stat() ItemChannelStat {
	c.mutex.RLock()
	defer c.mutex.RUnlock()
	return ItemChannelStat{
		ItemAt:   c.itemAt,
		Rate:     c.rate,
		Count:    c.count,
		Length:   len(c.in),
		Capacity: cap(c.in),
	}
}

// ItemChannelStat provides statistic about the item channel.
type ItemChannelStat struct {
	ItemAt   *time.Time `json:"itemAt,omitempty"`
	Rate     float64    `json:"rate"`
	Count    int64      `json:"count"`
	Length   int        `json:"length"`
	Capacity int        `json:"capacity"`
}

// NewItemChannelStatWrapper creates a new ItemChannelStatWrapper
func NewItemChannelStatWrapper(items <-chan *Item) *ItemChannelStatWrapper {
	c := &ItemChannelStatWrapper{
		in:  items,
		Out: make(chan *Item, 0), // We want the Out chan to be unbuffered.
	}
	go func() {
		var (
			count  int64
			itemAt *time.Time
		)
		times := make([]int64, 100, 100)
		timesIdx := len(times)
		rateReport := time.NewTicker(200 * time.Millisecond)
		for {
			select {
			case <-rateReport.C:
				c.mutex.Lock()
				if timesIdx == len(times) {
					c.rate = 0
				} else {
					c.rate = 1000000000.0 * float64(len(times)-timesIdx) / float64(time.Now().UnixNano()-times[timesIdx])
				}
				c.count = count
				c.itemAt = itemAt
				c.mutex.Unlock()
			case i, ok := <-c.in:
				if !ok {
					rateReport.Stop()
					close(c.Out)
					c.mutex.Lock()
					c.rate = 0
					c.count = count
					c.itemAt = itemAt
					c.mutex.Unlock()
					return
				}
				c.Out <- i
				times = times[1:]
				times = append(times, time.Now().UnixNano())
				if timesIdx > 0 {
					timesIdx--
				}
				count++
				itemAt = &i.At
			}
		}
	}()
	return c
}

// Slurper is responsible for consuming a channel of items.
type Slurper interface {
	Slurp(<-chan *Item)
}

// SlurperFunc is an adapter that allow you to use
// an ordinary function as a Slurper.
type SlurperFunc func(<-chan *Item)

// Slurp calls f(items)
func (f SlurperFunc) Slurp(items <-chan *Item) {
	f(items)
}

// CompositionSlurper can be used to present multiple slurpers as a single
// slurper.
type CompositionSlurper struct {
	slurpers      []Slurper
	slurpFunction func(in <-chan *Item, out []chan *Item)
}

// Slurp coordinates the slurp for multiple slurpers. Responsability for the
// reading and writing to the channel is defered to the slurpFunction.
func (s *CompositionSlurper) Slurp(in <-chan *Item) {
	wg := sync.WaitGroup{}
	out := make([]chan *Item, len(s.slurpers))
	for i, slurper := range s.slurpers {
		out[i] = make(chan *Item, cap(in))
		wg.Add(1)
		go func(s Slurper, ch <-chan *Item) {
			defer wg.Done()
			s.Slurp(ch)
		}(slurper, out[i])
	}
	wg.Add(1)
	go func() {
		defer wg.Done()
		s.slurpFunction(in, out)
		// in has been consumed so lets ensure that
		// we close all of our out chans.
		for _, o := range out {
			close(o)
		}
	}()
	wg.Wait()
}

// NewFanOutSlurper will fan out all items to all slurpers.
func NewFanOutSlurper(slurpers ...Slurper) *CompositionSlurper {
	return &CompositionSlurper{
		slurpers: slurpers,
		slurpFunction: func(in <-chan *Item, out []chan *Item) {
			for i := range in {
				for _, o := range out {
					o <- i
				}
			}
		},
	}
}
<|endoftext|>


<commit_before>package signed

import (
	"testing"

	"github.com/agl/ed25519"
	"github.com/flynn/go-tuf/data"
	"github.com/flynn/go-tuf/keys"

	. "gopkg.in/check.v1"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

type VerifySuite struct{}

var _ = Suite(&VerifySuite{})

func (VerifySuite) Test(c *C) {
	type test struct {
		name  string
		keys  []*data.Key
		roles map[string]*data.Role
		s     *data.Signed
		ver   int
		typ   string
		role  string
		err   error
		mut   func(*test)
	}

	minVer := 10
	tests := []test{
		{
			name: "no signatures",
			mut:  func(t *test) { t.s.Signatures = []data.Signature{} },
			err:  ErrNoSignatures,
		},
		{
			name: "unknown role",
			role: "foo",
			err:  ErrUnknownRole,
		},
		{
			name: "wrong signature method",
			mut:  func(t *test) { t.s.Signatures[0].Method = "foo" },
			err:  ErrWrongMethod,
		},
		{
			name: "signature wrong length",
			mut:  func(t *test) { t.s.Signatures[0].Signature = []byte{0} },
			err:  ErrInvalid,
		},
		{
			name: "key missing from role",
			mut:  func(t *test) { t.roles["root"].KeyIDs = nil },
			err:  ErrRoleThreshold,
		},
		{
			name: "invalid signature",
			mut:  func(t *test) { t.s.Signatures[0].Signature = make([]byte, ed25519.SignatureSize) },
			err:  ErrInvalid,
		},
		{
			name: "not enough signatures",
			mut:  func(t *test) { t.roles["root"].Threshold = 2 },
			err:  ErrRoleThreshold,
		},
		{
			name: "exactly enough signatures",
		},
		{
			name: "more than enough signatures",
			mut: func(t *test) {
				k, _ := keys.NewKey()
				Sign(t.s, k)
				t.keys = append(t.keys, k.Serialize())
				t.roles["root"].KeyIDs = append(t.roles["root"].KeyIDs, k.ID)
			},
		},
		{
			name: "duplicate key id",
			mut: func(t *test) {
				t.roles["root"].Threshold = 2
				t.s.Signatures = append(t.s.Signatures, t.s.Signatures[0])
			},
			err: ErrRoleThreshold,
		},
		{
			name: "unknown key",
			mut: func(t *test) {
				k, _ := keys.NewKey()
				Sign(t.s, k)
			},
		},
		{
			name: "unknown key below threshold",
			mut: func(t *test) {
				k, _ := keys.NewKey()
				Sign(t.s, k)
				t.roles["root"].Threshold = 2
			},
			err: ErrRoleThreshold,
		},
		{
			name: "unknown keys in db",
			mut: func(t *test) {
				k, _ := keys.NewKey()
				Sign(t.s, k)
				t.keys = append(t.keys, k.Serialize())
			},
		},
		{
			name: "unknown keys in db below threshold",
			mut: func(t *test) {
				k, _ := keys.NewKey()
				Sign(t.s, k)
				t.keys = append(t.keys, k.Serialize())
				t.roles["root"].Threshold = 2
			},
			err: ErrRoleThreshold,
		},
		{
			name: "wrong type",
			typ:  "bar",
			err:  ErrWrongType,
		},
		{
			name: "low version",
			ver:  minVer - 1,
			err:  ErrLowVersion,
		},
	}
	for _, t := range tests {
		if t.role == "" {
			t.role = "root"
		}
		if t.ver == 0 {
			t.ver = minVer
		}
		if t.typ == "" {
			t.typ = t.role
		}
		if t.keys == nil && t.s == nil {
			k, _ := keys.NewKey()
			t.s, _ = Marshal(&signedMeta{Type: t.typ, Version: t.ver}, k)
			t.keys = []*data.Key{k.Serialize()}
		}
		if t.roles == nil {
			t.roles = map[string]*data.Role{
				"root": &data.Role{
					KeyIDs:    []string{t.keys[0].ID()},
					Threshold: 1,
				},
			}
		}
		if t.mut != nil {
			t.mut(&t)
		}

		db := keys.NewDB()
		for _, k := range t.keys {
			err := db.AddKey(k.ID(), k)
			c.Assert(err, IsNil)
		}
		for n, r := range t.roles {
			err := db.AddRole(n, r)
			c.Assert(err, IsNil)
		}

		err := Verify(t.s, t.role, minVer, db)
		c.Assert(err, Equals, t.err, Commentf("name = %s", t.name))
	}
}
<commit_msg>signed: Fix tests<commit_after>package signed

import (
	"testing"

	"github.com/agl/ed25519"
	"github.com/flynn/go-tuf/data"
	"github.com/flynn/go-tuf/keys"

	. "gopkg.in/check.v1"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

type VerifySuite struct{}

var _ = Suite(&VerifySuite{})

func (VerifySuite) Test(c *C) {
	type test struct {
		name  string
		keys  []*data.Key
		roles map[string]*data.Role
		s     *data.Signed
		ver   int
		typ   string
		role  string
		err   error
		mut   func(*test)
	}

	minVer := 10
	tests := []test{
		{
			name: "no signatures",
			mut:  func(t *test) { t.s.Signatures = []data.Signature{} },
			err:  ErrNoSignatures,
		},
		{
			name: "unknown role",
			role: "foo",
			err:  ErrUnknownRole,
		},
		{
			name: "wrong signature method",
			mut:  func(t *test) { t.s.Signatures[0].Method = "foo" },
			err:  ErrWrongMethod,
		},
		{
			name: "signature wrong length",
			mut:  func(t *test) { t.s.Signatures[0].Signature = []byte{0} },
			err:  ErrInvalid,
		},
		{
			name: "key missing from role",
			mut:  func(t *test) { t.roles["root"].KeyIDs = nil },
			err:  ErrRoleThreshold,
		},
		{
			name: "invalid signature",
			mut:  func(t *test) { t.s.Signatures[0].Signature = make([]byte, ed25519.SignatureSize) },
			err:  ErrInvalid,
		},
		{
			name: "not enough signatures",
			mut:  func(t *test) { t.roles["root"].Threshold = 2 },
			err:  ErrRoleThreshold,
		},
		{
			name: "exactly enough signatures",
		},
		{
			name: "more than enough signatures",
			mut: func(t *test) {
				k, _ := keys.NewKey()
				key := k.Serialize()
				Sign(t.s, key)
				t.keys = append(t.keys, key)
				t.roles["root"].KeyIDs = append(t.roles["root"].KeyIDs, k.ID)
			},
		},
		{
			name: "duplicate key id",
			mut: func(t *test) {
				t.roles["root"].Threshold = 2
				t.s.Signatures = append(t.s.Signatures, t.s.Signatures[0])
			},
			err: ErrRoleThreshold,
		},
		{
			name: "unknown key",
			mut: func(t *test) {
				k, _ := keys.NewKey()
				Sign(t.s, k.Serialize())
			},
		},
		{
			name: "unknown key below threshold",
			mut: func(t *test) {
				k, _ := keys.NewKey()
				Sign(t.s, k.Serialize())
				t.roles["root"].Threshold = 2
			},
			err: ErrRoleThreshold,
		},
		{
			name: "unknown keys in db",
			mut: func(t *test) {
				k, _ := keys.NewKey()
				Sign(t.s, k.Serialize())
				t.keys = append(t.keys, k.Serialize())
			},
		},
		{
			name: "unknown keys in db below threshold",
			mut: func(t *test) {
				k, _ := keys.NewKey()
				Sign(t.s, k.Serialize())
				t.keys = append(t.keys, k.Serialize())
				t.roles["root"].Threshold = 2
			},
			err: ErrRoleThreshold,
		},
		{
			name: "wrong type",
			typ:  "bar",
			err:  ErrWrongType,
		},
		{
			name: "low version",
			ver:  minVer - 1,
			err:  ErrLowVersion,
		},
	}
	for _, t := range tests {
		if t.role == "" {
			t.role = "root"
		}
		if t.ver == 0 {
			t.ver = minVer
		}
		if t.typ == "" {
			t.typ = t.role
		}
		if t.keys == nil && t.s == nil {
			k, _ := keys.NewKey()
			key := k.Serialize()
			t.s, _ = Marshal(&signedMeta{Type: t.typ, Version: t.ver}, key)
			t.keys = []*data.Key{key}
		}
		if t.roles == nil {
			t.roles = map[string]*data.Role{
				"root": &data.Role{
					KeyIDs:    []string{t.keys[0].ID()},
					Threshold: 1,
				},
			}
		}
		if t.mut != nil {
			t.mut(&t)
		}

		db := keys.NewDB()
		for _, k := range t.keys {
			err := db.AddKey(k.ID(), k)
			c.Assert(err, IsNil)
		}
		for n, r := range t.roles {
			err := db.AddRole(n, r)
			c.Assert(err, IsNil)
		}

		err := Verify(t.s, t.role, minVer, db)
		c.Assert(err, Equals, t.err, Commentf("name = %s", t.name))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/dpapathanasiou/go-recaptcha"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"text/template"
)

const maxCommentNameLength = 254
const maxCommentEmailLength = 254
const maxCommentBodyLength = 5000

func showSinglePost(b *BlogPost, w http.ResponseWriter, req *http.Request) {

	if b == nil {
		http.NotFound(w, req)
		return
	}

	page := PostPage{}
	page.Post = b
	page.Config = SharedConfig
	page.CommentName = ""
	page.CommentEmail = ""
	page.CommentBody = ""
	page.CommentNameError = ""
	page.CommentEmailError = ""
	page.CommentBodyError = ""

	t, _ := template.ParseFiles(SharedConfig.FullThemePath() + "/templates/post.html")
	t.Execute(w, page)
}

func postWithQuery(query url.Values) (*BlogPost, error) {

	title := query.Get(":title")

	year, err := strconv.Atoi(query.Get(":year"))

	if err != nil {
		log.Println("Invalid year supplied")
		return nil, err
	}

	month, err := strconv.Atoi(query.Get(":month"))

	if err != nil {
		log.Println("Invalid month supplied")
		return nil, err
	}

	day, err := strconv.Atoi(query.Get(":day"))

	if err != nil {
		log.Println("Invalid day supplied")
		return nil, err
	}

	url := fmt.Sprintf("%04d/%02d/%02d/%s", year, month, day, title)

	post, err := blog.PostWithUrl(url)

	return post, err
}

func post(w http.ResponseWriter, req *http.Request) {
	post, _ := postWithQuery(req.URL.Query())
	showSinglePost(post, w, req)
}

func createComment(w http.ResponseWriter, req *http.Request) {

	post, err := postWithQuery(req.URL.Query())

	if err != nil {
		log.Println("Could not load post")
		return
	}

	if post.Metadata.DisallowComments {
		return
	}

	author := strings.TrimSpace(req.FormValue("name"))
	email := strings.TrimSpace(req.FormValue("email"))
	body := strings.TrimSpace(req.FormValue("comment"))

	hasErrors := false
	commentNameError := ""
	commentEmailError := ""
	commentBodyError := ""
	commentRecaptchaError := ""

	if len(author) == 0 {
		hasErrors = true
		commentNameError = "Name cannot be blank"
	} else if len(author) > 254 {
		hasErrors = true
		commentNameError = fmt.Sprintf("Name must be less than %v characters", +maxCommentNameLength)
	}

	if len(email) < 5 {
		hasErrors = true
		commentEmailError = "Email must be a valid address"
	} else if len(email) > maxCommentEmailLength {
		hasErrors = true
		commentEmailError = fmt.Sprintf("Email must be less than %v characters", maxCommentEmailLength)
	} else if !strings.Contains(email, "@") {

		// Since regex is useless for validating emails, we'll just check for
		// the @ symbol.

		hasErrors = true
		commentEmailError = "Email must be a valid address"
	}

	if len(body) == 0 {
		hasErrors = true
		commentBodyError = "Comment cannot be blank"
	} else if len(body) > maxCommentBodyLength {
		hasErrors = true
		commentBodyError = fmt.Sprintf("Comment must be less than %v characters", maxCommentBodyLength)
	}

	if len(SharedConfig.RecaptchaPrivateKey) > 0 {
		recaptcha.Init(SharedConfig.RecaptchaPrivateKey)
		ip := strings.Split(req.RemoteAddr, ":")[0]
		if !recaptcha.Confirm(ip, req.FormValue("g-recaptcha-response")) {
			hasErrors = true
			commentRecaptchaError = "Incorrect reCAPTCHA entered"
		}
	}

	if !hasErrors {
		post.SaveComment(SharedConfig.AkismetAPIKey, SharedConfig.Address, getIpAddress(req), req.UserAgent(), req.Referer(), author, email, body)
		http.Redirect(w, req, "/posts/"+post.Url+"#comments", http.StatusFound)

		return
	} else {

		page := PostPage{}
		page.Post = post
		page.Config = SharedConfig
		page.CommentName = author
		page.CommentEmail = email
		page.CommentBody = body
		page.CommentNameError = commentNameError
		page.CommentEmailError = commentEmailError
		page.CommentBodyError = commentBodyError
		page.CommentRecaptchaError = commentRecaptchaError

		t, _ := template.ParseFiles(SharedConfig.FullThemePath() + "/templates/post.html")
		t.Execute(w, page)
	}
}

func ipAddrFromRemoteAddr(s string) string {
	idx := strings.LastIndex(s, ":")
	if idx == -1 {
		return s
	}
	return s[:idx]
}

func getIpAddress(r *http.Request) string {
	hdr := r.Header
	hdrRealIp := hdr.Get("X-Real-Ip")
	hdrForwardedFor := hdr.Get("X-Forwarded-For")
	if hdrRealIp == "" && hdrForwardedFor == "" {
		return ipAddrFromRemoteAddr(r.RemoteAddr)
	}
	if hdrForwardedFor != "" {
		// X-Forwarded-For is potentially a list of addresses separated with ","
		parts := strings.Split(hdrForwardedFor, ",")
		for i, p := range parts {
			parts[i] = strings.TrimSpace(p)
		}
		// TODO: should return first non-local address
		return parts[0]
	}
	return hdrRealIp
}
<commit_msg>Updated to deal with recaptcha now returning an error as well as a result boolean.<commit_after>package main

import (
	"fmt"
	"github.com/dpapathanasiou/go-recaptcha"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"text/template"
)

const maxCommentNameLength = 254
const maxCommentEmailLength = 254
const maxCommentBodyLength = 5000

func showSinglePost(b *BlogPost, w http.ResponseWriter, req *http.Request) {

	if b == nil {
		http.NotFound(w, req)
		return
	}

	page := PostPage{}
	page.Post = b
	page.Config = SharedConfig
	page.CommentName = ""
	page.CommentEmail = ""
	page.CommentBody = ""
	page.CommentNameError = ""
	page.CommentEmailError = ""
	page.CommentBodyError = ""

	t, _ := template.ParseFiles(SharedConfig.FullThemePath() + "/templates/post.html")
	t.Execute(w, page)
}

func postWithQuery(query url.Values) (*BlogPost, error) {

	title := query.Get(":title")

	year, err := strconv.Atoi(query.Get(":year"))

	if err != nil {
		log.Println("Invalid year supplied")
		return nil, err
	}

	month, err := strconv.Atoi(query.Get(":month"))

	if err != nil {
		log.Println("Invalid month supplied")
		return nil, err
	}

	day, err := strconv.Atoi(query.Get(":day"))

	if err != nil {
		log.Println("Invalid day supplied")
		return nil, err
	}

	url := fmt.Sprintf("%04d/%02d/%02d/%s", year, month, day, title)

	post, err := blog.PostWithUrl(url)

	return post, err
}

func post(w http.ResponseWriter, req *http.Request) {
	post, _ := postWithQuery(req.URL.Query())
	showSinglePost(post, w, req)
}

func createComment(w http.ResponseWriter, req *http.Request) {

	post, err := postWithQuery(req.URL.Query())

	if err != nil {
		log.Println("Could not load post")
		return
	}

	if post.Metadata.DisallowComments {
		return
	}

	author := strings.TrimSpace(req.FormValue("name"))
	email := strings.TrimSpace(req.FormValue("email"))
	body := strings.TrimSpace(req.FormValue("comment"))

	hasErrors := false
	commentNameError := ""
	commentEmailError := ""
	commentBodyError := ""
	commentRecaptchaError := ""

	if len(author) == 0 {
		hasErrors = true
		commentNameError = "Name cannot be blank"
	} else if len(author) > 254 {
		hasErrors = true
		commentNameError = fmt.Sprintf("Name must be less than %v characters", +maxCommentNameLength)
	}

	if len(email) < 5 {
		hasErrors = true
		commentEmailError = "Email must be a valid address"
	} else if len(email) > maxCommentEmailLength {
		hasErrors = true
		commentEmailError = fmt.Sprintf("Email must be less than %v characters", maxCommentEmailLength)
	} else if !strings.Contains(email, "@") {

		// Since regex is useless for validating emails, we'll just check for
		// the @ symbol.

		hasErrors = true
		commentEmailError = "Email must be a valid address"
	}

	if len(body) == 0 {
		hasErrors = true
		commentBodyError = "Comment cannot be blank"
	} else if len(body) > maxCommentBodyLength {
		hasErrors = true
		commentBodyError = fmt.Sprintf("Comment must be less than %v characters", maxCommentBodyLength)
	}

	if len(SharedConfig.RecaptchaPrivateKey) > 0 {
		recaptcha.Init(SharedConfig.RecaptchaPrivateKey)
		ip := strings.Split(req.RemoteAddr, ":")[0]

		success, _ := recaptcha.Confirm(ip, req.FormValue("g-recaptcha-response"))

		if (!success) {
			hasErrors = true
			commentRecaptchaError = "Incorrect reCAPTCHA entered"
		}
	}

	if !hasErrors {
		post.SaveComment(SharedConfig.AkismetAPIKey, SharedConfig.Address, getIpAddress(req), req.UserAgent(), req.Referer(), author, email, body)
		http.Redirect(w, req, "/posts/"+post.Url+"#comments", http.StatusFound)

		return
	} else {

		page := PostPage{}
		page.Post = post
		page.Config = SharedConfig
		page.CommentName = author
		page.CommentEmail = email
		page.CommentBody = body
		page.CommentNameError = commentNameError
		page.CommentEmailError = commentEmailError
		page.CommentBodyError = commentBodyError
		page.CommentRecaptchaError = commentRecaptchaError

		t, _ := template.ParseFiles(SharedConfig.FullThemePath() + "/templates/post.html")
		t.Execute(w, page)
	}
}

func ipAddrFromRemoteAddr(s string) string {
	idx := strings.LastIndex(s, ":")
	if idx == -1 {
		return s
	}
	return s[:idx]
}

func getIpAddress(r *http.Request) string {
	hdr := r.Header
	hdrRealIp := hdr.Get("X-Real-Ip")
	hdrForwardedFor := hdr.Get("X-Forwarded-For")
	if hdrRealIp == "" && hdrForwardedFor == "" {
		return ipAddrFromRemoteAddr(r.RemoteAddr)
	}
	if hdrForwardedFor != "" {
		// X-Forwarded-For is potentially a list of addresses separated with ","
		parts := strings.Split(hdrForwardedFor, ",")
		for i, p := range parts {
			parts[i] = strings.TrimSpace(p)
		}
		// TODO: should return first non-local address
		return parts[0]
	}
	return hdrRealIp
}
<|endoftext|>


<commit_before>// Copyright © 2014 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package source

import (
	"io"
	"path"
	"path/filepath"
	"strings"

	"github.com/spf13/hugo/helpers"
)

type File struct {
	relpath     string // Original Full Path eg. /Users/Home/Hugo/foo.txt
	logicalName string // foo.txt
	Contents    io.Reader
	section     string // The first directory
	dir         string // The full directory Path (minus file name)
	ext         string // Just the ext (eg txt)
	uniqueId    string // MD5 of the filename
}

func (f *File) UniqueId() string {
	if f.uniqueId == "" {
		f.uniqueId = helpers.Md5String(f.LogicalName())
	}
	return f.uniqueId
}

// Filename without extension
func (f *File) BaseFileName() string {
	return helpers.Filename(f.LogicalName())
}

func (f *File) Section() string {
	if f.section != "" {
		return f.section
	} else {
		f.section = helpers.GuessSection(f.Dir())
		return f.section
	}
}

func (f *File) LogicalName() string {
	if f.logicalName != "" {
		return f.logicalName
	} else {
		_, f.logicalName = path.Split(f.relpath)
		return f.logicalName
	}
}

//func (f *File) SetDir(dir string) {
//f.dir = dir
//}

func (f *File) Dir() string {
	if f.dir != "" {
		return f.dir
	} else {
		f.dir, _ = path.Split(f.relpath)
		return f.dir
	}
}

func (f *File) Extension() string {
	if f.ext != "" {
		return f.ext
	} else {
		f.ext = strings.TrimPrefix(filepath.Ext(f.LogicalName()), ".")
		return f.ext
	}
}

func (f *File) Ext() string {
	return f.Extension()
}

func (f *File) Path() string {
	return f.relpath
}

func NewFileWithContents(relpath string, content io.Reader) *File {
	file := NewFile(relpath)
	file.Contents = content
	return file
}

func NewFile(relpath string) *File {
	return &File{
		relpath: relpath,
	}
}

func NewFileFromAbs(base, fullpath string, content io.Reader) (f *File, err error) {
	var name string
	if name, err = helpers.GetRelativePath(fullpath, base); err != nil {
		return nil, err
	}

	return NewFileWithContents(name, content), nil
}
<commit_msg>Source files can provide content as String or Bytes or Reader<commit_after>// Copyright © 2014 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package source

import (
	"io"
	"path"
	"path/filepath"
	"strings"

	"github.com/spf13/hugo/helpers"
)

type File struct {
	relpath     string // Original Full Path eg. /Users/Home/Hugo/foo.txt
	logicalName string // foo.txt
	Contents    io.Reader
	section     string // The first directory
	dir         string // The full directory Path (minus file name)
	ext         string // Just the ext (eg txt)
	uniqueId    string // MD5 of the filename
}

func (f *File) UniqueId() string {
	if f.uniqueId == "" {
		f.uniqueId = helpers.Md5String(f.LogicalName())
	}
	return f.uniqueId
}

func (f *File) String() string {
	return helpers.ReaderToString(f.Contents)
}

func (f *File) Bytes() []byte {
	return helpers.ReaderToBytes(f.Contents)
}

// Filename without extension
func (f *File) BaseFileName() string {
	return helpers.Filename(f.LogicalName())
}

func (f *File) Section() string {
	if f.section != "" {
		return f.section
	} else {
		f.section = helpers.GuessSection(f.Dir())
		return f.section
	}
}

func (f *File) LogicalName() string {
	if f.logicalName != "" {
		return f.logicalName
	} else {
		_, f.logicalName = path.Split(f.relpath)
		return f.logicalName
	}
}

//func (f *File) SetDir(dir string) {
//f.dir = dir
//}

func (f *File) Dir() string {
	if f.dir != "" {
		return f.dir
	} else {
		f.dir, _ = path.Split(f.relpath)
		return f.dir
	}
}

func (f *File) Extension() string {
	if f.ext != "" {
		return f.ext
	} else {
		f.ext = strings.TrimPrefix(filepath.Ext(f.LogicalName()), ".")
		return f.ext
	}
}

func (f *File) Ext() string {
	return f.Extension()
}

func (f *File) Path() string {
	return f.relpath
}

func NewFileWithContents(relpath string, content io.Reader) *File {
	file := NewFile(relpath)
	file.Contents = content
	return file
}

func NewFile(relpath string) *File {
	return &File{
		relpath: relpath,
	}
}

func NewFileFromAbs(base, fullpath string, content io.Reader) (f *File, err error) {
	var name string
	if name, err = helpers.GetRelativePath(fullpath, base); err != nil {
		return nil, err
	}

	return NewFileWithContents(name, content), nil
}
<|endoftext|>


<commit_before>package sparta

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"testing"

	"github.com/Sirupsen/logrus"
)

type StructHandler1 struct {
}

func (handler *StructHandler1) handler(event *json.RawMessage,
	context *LambdaContext,
	w http.ResponseWriter,
	logger *logrus.Logger) {
	fmt.Fprintf(w, "StructHandler1 handler")
}

type StructHandler2 struct {
}

func (handler *StructHandler2) handler(event *json.RawMessage,
	context *LambdaContext,
	w http.ResponseWriter,
	logger *logrus.Logger) {
	fmt.Fprintf(w, "StructHandler1 handler")
}

func testLambdaStructData() []*LambdaAWSInfo {
	var lambdaFunctions []*LambdaAWSInfo

	handler1 := &StructHandler1{}
	lambdaFn1 := NewLambda(LambdaExecuteARN, handler1.handler, nil)
	lambdaFunctions = append(lambdaFunctions, lambdaFn1)

	handler2 := &StructHandler2{}
	lambdaFn2 := NewLambda(LambdaExecuteARN, handler2.handler, nil)
	lambdaFunctions = append(lambdaFunctions, lambdaFn2)

	return lambdaFunctions
}

func testLambdaDoubleStructPtrData() []*LambdaAWSInfo {
	var lambdaFunctions []*LambdaAWSInfo

	handler1 := &StructHandler1{}
	lambdaFn1 := NewLambda(LambdaExecuteARN, handler1.handler, nil)
	lambdaFunctions = append(lambdaFunctions, lambdaFn1)

	handler2 := &StructHandler1{}
	lambdaFn2 := NewLambda(LambdaExecuteARN, handler2.handler, nil)
	lambdaFunctions = append(lambdaFunctions, lambdaFn2)

	return lambdaFunctions
}

func userDefinedCustomResource1(requestType string,
	stackID string,
	properties map[string]interface{},
	logger *logrus.Logger) (map[string]interface{}, error) {
	return nil, nil
}

func userDefinedCustomResource2(requestType string,
	stackID string,
	properties map[string]interface{},
	logger *logrus.Logger) (map[string]interface{}, error) {
	return nil, nil
}

func TestStruct(t *testing.T) {
	logger, err := NewLogger("info")
	var templateWriter bytes.Buffer
	err = Provision(true,
		"SampleProvision",
		"",
		testLambdaStructData(),
		nil,
		nil,
		"S3Bucket",
		&templateWriter,
		nil,
		logger)
	if nil != err {
		t.Fatal(err.Error())
	}
}

func TestDoubleRefStruct(t *testing.T) {
	logger, err := NewLogger("info")
	var templateWriter bytes.Buffer
	err = Provision(true,
		"SampleProvision",
		"",
		testLambdaDoubleStructPtrData(),
		nil,
		nil,
		"S3Bucket",
		&templateWriter,
		nil,
		logger)

	if nil == err {
		t.Fatal("Failed to enforce lambda function uniqueness")
	}
}

func TestCustomResource(t *testing.T) {
	logger, err := NewLogger("info")
	lambdaFuncs := testLambdaStructData()
	lambdaFuncs[0].RequireCustomResource(IAMRoleDefinition{},
		userDefinedCustomResource1,
		nil,
		nil)

	lambdaFuncs[1].RequireCustomResource(IAMRoleDefinition{},
		userDefinedCustomResource2,
		nil,
		nil)

	var templateWriter bytes.Buffer
	err = Provision(true,
		"SampleProvision",
		"",
		lambdaFuncs,
		nil,
		nil,
		"S3Bucket",
		&templateWriter,
		nil,
		logger)

	if nil != err {
		t.Fatal("Failed to accept unique user CustomResource functions")
	}
}

func TestDoubleRefCustomResource(t *testing.T) {
	logger, err := NewLogger("info")
	lambdaFuncs := testLambdaStructData()

	for _, eachLambda := range lambdaFuncs {
		eachLambda.RequireCustomResource(IAMRoleDefinition{},
			userDefinedCustomResource1,
			nil,
			nil)
	}
	var templateWriter bytes.Buffer
	err = Provision(true,
		"SampleProvision",
		"",
		lambdaFuncs,
		nil,
		nil,
		"S3Bucket",
		&templateWriter,
		nil,
		logger)

	if nil == err {
		t.Fatal("Failed to reject duplicate user CustomResource functions")
	}
}
<commit_msg>Add `buildID` to signature<commit_after>package sparta

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"testing"

	"github.com/Sirupsen/logrus"
)

type StructHandler1 struct {
}

func (handler *StructHandler1) handler(event *json.RawMessage,
	context *LambdaContext,
	w http.ResponseWriter,
	logger *logrus.Logger) {
	fmt.Fprintf(w, "StructHandler1 handler")
}

type StructHandler2 struct {
}

func (handler *StructHandler2) handler(event *json.RawMessage,
	context *LambdaContext,
	w http.ResponseWriter,
	logger *logrus.Logger) {
	fmt.Fprintf(w, "StructHandler1 handler")
}

func testLambdaStructData() []*LambdaAWSInfo {
	var lambdaFunctions []*LambdaAWSInfo

	handler1 := &StructHandler1{}
	lambdaFn1 := NewLambda(LambdaExecuteARN, handler1.handler, nil)
	lambdaFunctions = append(lambdaFunctions, lambdaFn1)

	handler2 := &StructHandler2{}
	lambdaFn2 := NewLambda(LambdaExecuteARN, handler2.handler, nil)
	lambdaFunctions = append(lambdaFunctions, lambdaFn2)

	return lambdaFunctions
}

func testLambdaDoubleStructPtrData() []*LambdaAWSInfo {
	var lambdaFunctions []*LambdaAWSInfo

	handler1 := &StructHandler1{}
	lambdaFn1 := NewLambda(LambdaExecuteARN, handler1.handler, nil)
	lambdaFunctions = append(lambdaFunctions, lambdaFn1)

	handler2 := &StructHandler1{}
	lambdaFn2 := NewLambda(LambdaExecuteARN, handler2.handler, nil)
	lambdaFunctions = append(lambdaFunctions, lambdaFn2)

	return lambdaFunctions
}

func userDefinedCustomResource1(requestType string,
	stackID string,
	properties map[string]interface{},
	logger *logrus.Logger) (map[string]interface{}, error) {
	return nil, nil
}

func userDefinedCustomResource2(requestType string,
	stackID string,
	properties map[string]interface{},
	logger *logrus.Logger) (map[string]interface{}, error) {
	return nil, nil
}

func TestStruct(t *testing.T) {
	logger, err := NewLogger("info")
	var templateWriter bytes.Buffer
	err = Provision(true,
		"SampleProvision",
		"",
		testLambdaStructData(),
		nil,
		nil,
		"testBuildID",
		"S3Bucket",
		&templateWriter,
		nil,
		logger)
	if nil != err {
		t.Fatal(err.Error())
	}
}

func TestDoubleRefStruct(t *testing.T) {
	logger, err := NewLogger("info")
	var templateWriter bytes.Buffer
	err = Provision(true,
		"SampleProvision",
		"",
		testLambdaDoubleStructPtrData(),
		nil,
		nil,
		"testBuildID",
		"S3Bucket",
		&templateWriter,
		nil,
		logger)

	if nil == err {
		t.Fatal("Failed to enforce lambda function uniqueness")
	}
}

func TestCustomResource(t *testing.T) {
	logger, err := NewLogger("info")
	lambdaFuncs := testLambdaStructData()
	lambdaFuncs[0].RequireCustomResource(IAMRoleDefinition{},
		userDefinedCustomResource1,
		nil,
		nil)

	lambdaFuncs[1].RequireCustomResource(IAMRoleDefinition{},
		userDefinedCustomResource2,
		nil,
		nil)

	var templateWriter bytes.Buffer
	err = Provision(true,
		"SampleProvision",
		"",
		lambdaFuncs,
		nil,
		nil,
		"testBuildID",
		"S3Bucket",
		&templateWriter,
		nil,
		logger)

	if nil != err {
		t.Fatal("Failed to accept unique user CustomResource functions")
	}
}

func TestDoubleRefCustomResource(t *testing.T) {
	logger, err := NewLogger("info")
	lambdaFuncs := testLambdaStructData()

	for _, eachLambda := range lambdaFuncs {
		eachLambda.RequireCustomResource(IAMRoleDefinition{},
			userDefinedCustomResource1,
			nil,
			nil)
	}
	var templateWriter bytes.Buffer
	err = Provision(true,
		"SampleProvision",
		"",
		lambdaFuncs,
		nil,
		nil,
		"testBuildID",
		"S3Bucket",
		&templateWriter,
		nil,
		logger)

	if nil == err {
		t.Fatal("Failed to reject duplicate user CustomResource functions")
	}
}
<|endoftext|>


<commit_before>package donut

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"path"
	"strconv"
	"time"

	"crypto/md5"
	"encoding/hex"
	"encoding/json"

	"github.com/minio-io/minio/pkg/utils/split"
)

type bucket struct {
	name      string
	donutName string
	nodes     map[string]Node
	objects   map[string]Object
}

// NewBucket - instantiate a new bucket
func NewBucket(bucketName, donutName string, nodes map[string]Node) (Bucket, error) {
	if bucketName == "" {
		return nil, errors.New("invalid argument")
	}
	b := bucket{}
	b.name = bucketName
	b.donutName = donutName
	b.objects = make(map[string]Object)
	b.nodes = nodes
	return b, nil
}

func (b bucket) ListNodes() (map[string]Node, error) {
	return b.nodes, nil
}

func (b bucket) GetBucketName() string {
	return b.name
}

func (b bucket) ListObjects() (map[string]Object, error) {
	nodeSlice := 0
	for _, node := range b.nodes {
		disks, err := node.ListDisks()
		if err != nil {
			return nil, err
		}
		for _, disk := range disks {
			bucketSlice := fmt.Sprintf("%s$%d$%d", b.name, nodeSlice, disk.GetOrder())
			bucketPath := path.Join(b.donutName, bucketSlice)
			objects, err := disk.ListDir(bucketPath)
			if err != nil {
				return nil, err
			}
			for _, object := range objects {
				b.objects[object.Name()], err = NewObject(object.Name(), path.Join(disk.GetName(), bucketPath))
				if err != nil {
					return nil, err
				}
			}
		}
		nodeSlice = nodeSlice + 1
	}
	return b.objects, nil
}

func (b bucket) decodeData(curBlockSize int, readers []io.ReadCloser, encoder Encoder, writer *io.PipeWriter) ([]byte, error) {
	curChunkSize, err := encoder.GetEncodedBlockLen(curBlockSize)
	if err != nil {
		return nil, err
	}
	encodedBytes := make([][]byte, 16)
	for i, reader := range readers {
		var bytesBuffer bytes.Buffer
		_, err := io.CopyN(&bytesBuffer, reader, int64(curChunkSize))
		if err != nil {
			return nil, err
		}
		encodedBytes[i] = bytesBuffer.Bytes()
	}
	decodedData, err := encoder.Decode(encodedBytes, curBlockSize)
	if err != nil {
		return nil, err
	}
	return decodedData, nil
}

func (b bucket) GetObject(objectName string, writer *io.PipeWriter, donutObjectMetadata map[string]string) {
	if objectName == "" || writer == nil || len(donutObjectMetadata) == 0 {
		writer.CloseWithError(errors.New("invalid argument"))
		return
	}
	totalChunks, err := strconv.Atoi(donutObjectMetadata["chunkCount"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	totalLeft, err := strconv.ParseInt(donutObjectMetadata["size"], 10, 64)
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	blockSize, err := strconv.Atoi(donutObjectMetadata["blockSize"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	k, err := strconv.ParseUint(donutObjectMetadata["erasureK"], 10, 8)
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	m, err := strconv.ParseUint(donutObjectMetadata["erasureM"], 10, 8)
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	expectedMd5sum, err := hex.DecodeString(donutObjectMetadata["md5"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	technique, ok := donutObjectMetadata["erasureTechnique"]
	if !ok {
		writer.CloseWithError(errors.New("missing erasure Technique"))
		return
	}
	hasher := md5.New()
	mwriter := io.MultiWriter(writer, hasher)
	encoder, err := NewEncoder(uint8(k), uint8(m), technique)
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	readers, err := b.getDiskReaders(objectName, "data")
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	for i := 0; i < totalChunks; i++ {
		curBlockSize := 0
		if int64(blockSize) < totalLeft {
			curBlockSize = blockSize
		} else {
			curBlockSize = int(totalLeft) // cast is safe, blockSize in if protects
		}
		decodedData, err := b.decodeData(curBlockSize, readers, encoder, writer)
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		_, err = io.Copy(mwriter, bytes.NewBuffer(decodedData))
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		totalLeft = totalLeft - int64(blockSize)
	}
	actualMd5sum := hasher.Sum(nil)
	if bytes.Compare(expectedMd5sum, actualMd5sum) != 0 {
		writer.CloseWithError(errors.New("checksum mismatch"))
		return
	}
	writer.Close()
	return
}

func (b bucket) getDiskReaders(objectName, objectMeta string) ([]io.ReadCloser, error) {
	readers := make([]io.ReadCloser, 16)
	nodeSlice := 0
	for _, node := range b.nodes {
		disks, err := node.ListDisks()
		if err != nil {
			return nil, err
		}
		for _, disk := range disks {
			bucketSlice := fmt.Sprintf("%s$%d$%d", b.name, nodeSlice, disk.GetOrder())
			objectPath := path.Join(b.donutName, bucketSlice, objectName, objectMeta)
			objectSlice, err := disk.OpenFile(objectPath)
			if err != nil {
				return nil, err
			}
			readers[disk.GetOrder()] = objectSlice
		}
		nodeSlice = nodeSlice + 1
	}
	return readers, nil
}

func (b bucket) SetObjectMetadata(objectName string, objectMetadata map[string]string) error {
	if len(objectMetadata) == 0 {
		return errors.New("invalid argument")
	}
	objectMetadataWriters, err := b.getDiskWriters(objectName, "objectMetadata.json")
	if err != nil {
		return err
	}
	for _, objectMetadataWriter := range objectMetadataWriters {
		defer objectMetadataWriter.Close()
	}
	for _, objectMetadataWriter := range objectMetadataWriters {
		jenc := json.NewEncoder(objectMetadataWriter)
		if err := jenc.Encode(objectMetadata); err != nil {
			return err
		}
	}
	return nil
}

func (b bucket) SetDonutObjectMetadata(objectName string, donutObjectMetadata map[string]string) error {
	if len(donutObjectMetadata) == 0 {
		return errors.New("invalid argument")
	}
	donutObjectMetadataWriters, err := b.getDiskWriters(objectName, "donutObjectMetadata.json")
	if err != nil {
		return err
	}
	for _, donutObjectMetadataWriter := range donutObjectMetadataWriters {
		defer donutObjectMetadataWriter.Close()
	}
	for _, donutObjectMetadataWriter := range donutObjectMetadataWriters {
		jenc := json.NewEncoder(donutObjectMetadataWriter)
		if err := jenc.Encode(donutObjectMetadata); err != nil {
			return err
		}
	}
	return nil
}

func (b bucket) getDiskWriters(objectName, objectMeta string) ([]io.WriteCloser, error) {
	writers := make([]io.WriteCloser, 16)
	nodeSlice := 0
	for _, node := range b.nodes {
		disks, err := node.ListDisks()
		if err != nil {
			return nil, err
		}
		for _, disk := range disks {
			bucketSlice := fmt.Sprintf("%s$%d$%d", b.name, nodeSlice, disk.GetOrder())
			objectPath := path.Join(b.donutName, bucketSlice, objectName, objectMeta)
			objectSlice, err := disk.MakeFile(objectPath)
			if err != nil {
				return nil, err
			}
			writers[disk.GetOrder()] = objectSlice
		}
		nodeSlice = nodeSlice + 1
	}
	return writers, nil
}

func (b bucket) PutObject(objectName string, contents io.ReadCloser) error {
	if objectName == "" {
		return errors.New("invalid argument")
	}
	if contents == nil {
		return errors.New("invalid argument")
	}
	writers, err := b.getDiskWriters(objectName, "data")
	if err != nil {
		return err
	}
	for _, writer := range writers {
		defer writer.Close()
	}

	chunks := split.Stream(contents, 10*1024*1024)
	encoder, err := NewEncoder(8, 8, "Cauchy")
	if err != nil {
		return err
	}
	chunkCount := 0
	totalLength := 0
	summer := md5.New()
	for chunk := range chunks {
		if chunk.Err == nil {
			totalLength = totalLength + len(chunk.Data)
			encodedBlocks, _ := encoder.Encode(chunk.Data)
			summer.Write(chunk.Data)
			for blockIndex, block := range encodedBlocks {
				io.Copy(writers[blockIndex], bytes.NewBuffer(block))
			}
		}
		chunkCount = chunkCount + 1
	}

	dataMd5sum := summer.Sum(nil)
	donutObjectMetadata := make(map[string]string)
	donutObjectMetadata["blockSize"] = strconv.Itoa(10 * 1024 * 1024)
	donutObjectMetadata["chunkCount"] = strconv.Itoa(chunkCount)
	donutObjectMetadata["created"] = time.Now().Format(time.RFC3339Nano)
	donutObjectMetadata["erasureK"] = "8"
	donutObjectMetadata["erasureM"] = "8"
	donutObjectMetadata["erasureTechnique"] = "Cauchy"
	donutObjectMetadata["md5"] = hex.EncodeToString(dataMd5sum)
	donutObjectMetadata["size"] = strconv.Itoa(totalLength)
	if err := b.SetDonutObjectMetadata(objectName, donutObjectMetadata); err != nil {
		return err
	}
	objectMetadata := make(map[string]string)
	objectMetadata["bucket"] = b.name
	objectMetadata["object"] = objectName
	objectMetadata["contentType"] = "application/octet-stream"
	if err := b.SetObjectMetadata(objectName, objectMetadata); err != nil {
		return err
	}
	return nil
}
<commit_msg>Fix gocyclo issues<commit_after>package donut

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"path"
	"strconv"
	"time"

	"crypto/md5"
	"encoding/hex"
	"encoding/json"

	"github.com/minio-io/minio/pkg/utils/split"
)

type bucket struct {
	name      string
	donutName string
	nodes     map[string]Node
	objects   map[string]Object
}

// NewBucket - instantiate a new bucket
func NewBucket(bucketName, donutName string, nodes map[string]Node) (Bucket, error) {
	if bucketName == "" {
		return nil, errors.New("invalid argument")
	}
	b := bucket{}
	b.name = bucketName
	b.donutName = donutName
	b.objects = make(map[string]Object)
	b.nodes = nodes
	return b, nil
}

func (b bucket) ListNodes() (map[string]Node, error) {
	return b.nodes, nil
}

func (b bucket) GetBucketName() string {
	return b.name
}

func (b bucket) ListObjects() (map[string]Object, error) {
	nodeSlice := 0
	for _, node := range b.nodes {
		disks, err := node.ListDisks()
		if err != nil {
			return nil, err
		}
		for _, disk := range disks {
			bucketSlice := fmt.Sprintf("%s$%d$%d", b.name, nodeSlice, disk.GetOrder())
			bucketPath := path.Join(b.donutName, bucketSlice)
			objects, err := disk.ListDir(bucketPath)
			if err != nil {
				return nil, err
			}
			for _, object := range objects {
				b.objects[object.Name()], err = NewObject(object.Name(), path.Join(disk.GetName(), bucketPath))
				if err != nil {
					return nil, err
				}
			}
		}
		nodeSlice = nodeSlice + 1
	}
	return b.objects, nil
}

func (b bucket) decodeData(totalLeft, blockSize int64, readers []io.ReadCloser, encoder Encoder, writer *io.PipeWriter) ([]byte, error) {
	var curBlockSize int64
	if blockSize < totalLeft {
		curBlockSize = blockSize
	} else {
		curBlockSize = totalLeft // cast is safe, blockSize in if protects
	}
	curChunkSize, err := encoder.GetEncodedBlockLen(int(curBlockSize))
	if err != nil {
		return nil, err
	}
	encodedBytes := make([][]byte, 16)
	for i, reader := range readers {
		var bytesBuffer bytes.Buffer
		_, err := io.CopyN(&bytesBuffer, reader, int64(curChunkSize))
		if err != nil {
			return nil, err
		}
		encodedBytes[i] = bytesBuffer.Bytes()
	}
	decodedData, err := encoder.Decode(encodedBytes, int(curBlockSize))
	if err != nil {
		return nil, err
	}
	return decodedData, nil
}

func (b bucket) map2Ints(donutObjectMetadata map[string]string) (totalChunks int, totalLeft, blockSize int64, k, m uint64, err error) {
	totalChunks, err = strconv.Atoi(donutObjectMetadata["chunkCount"])
	totalLeft, err = strconv.ParseInt(donutObjectMetadata["size"], 10, 64)
	blockSize, err = strconv.ParseInt(donutObjectMetadata["blockSize"], 10, 64)
	k, err = strconv.ParseUint(donutObjectMetadata["erasureK"], 10, 8)
	m, err = strconv.ParseUint(donutObjectMetadata["erasureM"], 10, 8)
	return
}

func (b bucket) GetObject(objectName string, writer *io.PipeWriter, donutObjectMetadata map[string]string) {
	if objectName == "" || writer == nil || len(donutObjectMetadata) == 0 {
		writer.CloseWithError(errors.New("invalid argument"))
		return
	}
	expectedMd5sum, err := hex.DecodeString(donutObjectMetadata["md5"])
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	totalChunks, totalLeft, blockSize, k, m, err := b.map2Ints(donutObjectMetadata)
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	technique, ok := donutObjectMetadata["erasureTechnique"]
	if !ok {
		writer.CloseWithError(errors.New("missing erasure Technique"))
		return
	}
	hasher := md5.New()
	mwriter := io.MultiWriter(writer, hasher)
	encoder, err := NewEncoder(uint8(k), uint8(m), technique)
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	readers, err := b.getDiskReaders(objectName, "data")
	if err != nil {
		writer.CloseWithError(err)
		return
	}
	for i := 0; i < totalChunks; i++ {
		decodedData, err := b.decodeData(totalLeft, blockSize, readers, encoder, writer)
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		_, err = io.Copy(mwriter, bytes.NewBuffer(decodedData))
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		totalLeft = totalLeft - int64(blockSize)
	}
	actualMd5sum := hasher.Sum(nil)
	if bytes.Compare(expectedMd5sum, actualMd5sum) != 0 {
		writer.CloseWithError(errors.New("checksum mismatch"))
		return
	}
	writer.Close()
	return
}

func (b bucket) getDiskReaders(objectName, objectMeta string) ([]io.ReadCloser, error) {
	readers := make([]io.ReadCloser, 16)
	nodeSlice := 0
	for _, node := range b.nodes {
		disks, err := node.ListDisks()
		if err != nil {
			return nil, err
		}
		for _, disk := range disks {
			bucketSlice := fmt.Sprintf("%s$%d$%d", b.name, nodeSlice, disk.GetOrder())
			objectPath := path.Join(b.donutName, bucketSlice, objectName, objectMeta)
			objectSlice, err := disk.OpenFile(objectPath)
			if err != nil {
				return nil, err
			}
			readers[disk.GetOrder()] = objectSlice
		}
		nodeSlice = nodeSlice + 1
	}
	return readers, nil
}

func (b bucket) SetObjectMetadata(objectName string, objectMetadata map[string]string) error {
	if len(objectMetadata) == 0 {
		return errors.New("invalid argument")
	}
	objectMetadataWriters, err := b.getDiskWriters(objectName, "objectMetadata.json")
	if err != nil {
		return err
	}
	for _, objectMetadataWriter := range objectMetadataWriters {
		defer objectMetadataWriter.Close()
	}
	for _, objectMetadataWriter := range objectMetadataWriters {
		jenc := json.NewEncoder(objectMetadataWriter)
		if err := jenc.Encode(objectMetadata); err != nil {
			return err
		}
	}
	return nil
}

func (b bucket) SetDonutObjectMetadata(objectName string, donutObjectMetadata map[string]string) error {
	if len(donutObjectMetadata) == 0 {
		return errors.New("invalid argument")
	}
	donutObjectMetadataWriters, err := b.getDiskWriters(objectName, "donutObjectMetadata.json")
	if err != nil {
		return err
	}
	for _, donutObjectMetadataWriter := range donutObjectMetadataWriters {
		defer donutObjectMetadataWriter.Close()
	}
	for _, donutObjectMetadataWriter := range donutObjectMetadataWriters {
		jenc := json.NewEncoder(donutObjectMetadataWriter)
		if err := jenc.Encode(donutObjectMetadata); err != nil {
			return err
		}
	}
	return nil
}

func (b bucket) getDiskWriters(objectName, objectMeta string) ([]io.WriteCloser, error) {
	writers := make([]io.WriteCloser, 16)
	nodeSlice := 0
	for _, node := range b.nodes {
		disks, err := node.ListDisks()
		if err != nil {
			return nil, err
		}
		for _, disk := range disks {
			bucketSlice := fmt.Sprintf("%s$%d$%d", b.name, nodeSlice, disk.GetOrder())
			objectPath := path.Join(b.donutName, bucketSlice, objectName, objectMeta)
			objectSlice, err := disk.MakeFile(objectPath)
			if err != nil {
				return nil, err
			}
			writers[disk.GetOrder()] = objectSlice
		}
		nodeSlice = nodeSlice + 1
	}
	return writers, nil
}

func (b bucket) PutObject(objectName string, contents io.ReadCloser) error {
	if objectName == "" {
		return errors.New("invalid argument")
	}
	if contents == nil {
		return errors.New("invalid argument")
	}
	writers, err := b.getDiskWriters(objectName, "data")
	if err != nil {
		return err
	}
	for _, writer := range writers {
		defer writer.Close()
	}

	chunks := split.Stream(contents, 10*1024*1024)
	encoder, err := NewEncoder(8, 8, "Cauchy")
	if err != nil {
		return err
	}
	chunkCount := 0
	totalLength := 0
	summer := md5.New()
	for chunk := range chunks {
		if chunk.Err == nil {
			totalLength = totalLength + len(chunk.Data)
			encodedBlocks, _ := encoder.Encode(chunk.Data)
			summer.Write(chunk.Data)
			for blockIndex, block := range encodedBlocks {
				io.Copy(writers[blockIndex], bytes.NewBuffer(block))
			}
		}
		chunkCount = chunkCount + 1
	}

	dataMd5sum := summer.Sum(nil)
	donutObjectMetadata := make(map[string]string)
	donutObjectMetadata["blockSize"] = strconv.Itoa(10 * 1024 * 1024)
	donutObjectMetadata["chunkCount"] = strconv.Itoa(chunkCount)
	donutObjectMetadata["created"] = time.Now().Format(time.RFC3339Nano)
	donutObjectMetadata["erasureK"] = "8"
	donutObjectMetadata["erasureM"] = "8"
	donutObjectMetadata["erasureTechnique"] = "Cauchy"
	donutObjectMetadata["md5"] = hex.EncodeToString(dataMd5sum)
	donutObjectMetadata["size"] = strconv.Itoa(totalLength)
	if err := b.SetDonutObjectMetadata(objectName, donutObjectMetadata); err != nil {
		return err
	}
	objectMetadata := make(map[string]string)
	objectMetadata["bucket"] = b.name
	objectMetadata["object"] = objectName
	objectMetadata["contentType"] = "application/octet-stream"
	if err := b.SetObjectMetadata(objectName, objectMetadata); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package sphere

import (
	"fmt"
	"net"
	"net/http"
	"net/url"
	"testing"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/gorilla/websocket"
)

var (
	address = func() string {
		l, _ := net.Listen("tcp", ":0")
		defer l.Close()
		return fmt.Sprintf("127.0.0.1:%d", l.Addr().(*net.TCPAddr).Port)
	}()
)

type TestSphereModel struct {
	*ChannelModel
}

func (m *TestSphereModel) Subscribe(room string, connection *Connection) bool {
	return true
}

func (m *TestSphereModel) Disconnect(room string, connection *Connection) bool {
	return true
}

func (m *TestSphereModel) Receive(event string, message string) (string, error) {
	return "you_got_me", nil
}

func init() {
	gin.SetMode(gin.ReleaseMode)
	s, r := Default(), gin.New()
	s.ChannelModels(&TestSphereModel{ExtendChannelModel("test")})
	r.GET("/sync", func(c *gin.Context) {
		s.Handler(c.Writer, c.Request)
	})
	go r.Run(address)
}

func CreateConnection() (c *websocket.Conn, response *http.Response, err error) {
	u, err := url.Parse("ws://" + address + "/sync")
	if err != nil {
		return nil, nil, err
	}
	rawConn, err := net.Dial("tcp", u.Host)
	if err != nil {
		return nil, nil, err
	}
	wsHeaders := http.Header{
		"Origin": {"http://" + address},
	}
	return websocket.NewClient(rawConn, u, wsHeaders, 1024, 1024)
}

func TestSphereReady(t *testing.T) {
	count := 0
	ticker := time.NewTicker(time.Millisecond * 60)
	for {
		select {
		case <-ticker.C:
			s, err := net.Dial("tcp", address)
			defer s.Close()
			if err == nil || count > 3 {
				defer ticker.Stop()
				if err != nil {
					t.Fatal("Server could not get started")
				}
				return
			}
		}
		count++
	}
}

func TestSphereConnection(t *testing.T) {
	c, _, err := CreateConnection()
	if err != nil {
		t.Fatal(err.Error())
	}
	defer c.Close()
}

func TestSphereSendMessage(t *testing.T) {
	c, _, err := CreateConnection()
	if err != nil {
		t.Fatal(err.Error())
	}
	defer c.Close()
	p := &Packet{Type: PacketTypePing}
	res, err := p.toJSON()
	if err != nil {
		t.Fatal(err.Error())
	}
	if err := c.WriteMessage(TextMessage, res); err != nil {
		t.Fatal(err.Error())
	}
}

func TestSphereMessagePingPong(t *testing.T) {
	done := make(chan error)
	c, _, err := CreateConnection()
	if err != nil {
		t.Fatal(err.Error())
	}
	defer c.Close()
	go func() {
		for {
			_, _, err := c.ReadMessage()
			if err != nil {
				done <- err
				return
			}
			done <- nil
			return
		}
	}()
	p := &Packet{Type: PacketTypePing}
	res, err := p.toJSON()
	if err != nil {
		t.Fatal(err.Error())
	}
	if err := c.WriteMessage(TextMessage, res); err != nil {
		t.Fatal(err.Error())
	}
	<-done
}

func TestSphereSubscribeChannel(t *testing.T) {
	done := make(chan error)
	c, _, err := CreateConnection()
	if err != nil {
		t.Fatal(err.Error())
	}
	defer c.Close()
	go func() {
		for {
			_, _, err := c.ReadMessage()
			if err != nil {
				done <- err
				return
			}
			done <- nil
			return
		}
	}()
	p := &Packet{Type: PacketTypeSubscribe, Namespace: "test", Room: "helloworld"}
	res, err := p.toJSON()
	if err != nil {
		t.Fatal(err.Error())
	}
	if err := c.WriteMessage(TextMessage, res); err != nil {
		t.Fatal(err.Error())
	}
	if err := <-done; err != nil {
		t.Fatal(err.Error())
	}
}

func TestSphereChannelMessage(t *testing.T) {
	done := make(chan error)
	c, _, err := CreateConnection()
	if err != nil {
		t.Fatal(err.Error())
	}
	defer c.Close()
	go func() {
		for {
			_, msg, err := c.ReadMessage()
			if err != nil {
				done <- err
				return
			}
			p, err := ParsePacket(msg)
			if err != nil {
				done <- err
			} else {
				switch {
				case p.Type == PacketTypeSubscribed && p.Error == nil:
					m := &Packet{Type: PacketTypeChannel, Namespace: "test", Room: "helloworld", Message: &Message{Event: "HelloEvent", Data: "HelloWorld"}}
					if json, err := m.toJSON(); err == nil {
						if err := c.WriteMessage(TextMessage, json); err != nil {
							done <- err
						}
					} else {
						done <- err
					}
				case p.Type == PacketTypeChannel && p.Error == nil:
					done <- nil
				}
			}
		}
	}()
	p := &Packet{Type: PacketTypeSubscribe, Namespace: "test", Room: "helloworld"}
	res, err := p.toJSON()
	if err != nil {
		t.Fatal(err.Error())
	}
	if err := c.WriteMessage(TextMessage, res); err != nil {
		t.Fatal(err.Error())
	}
	if err := <-done; err != nil {
		t.Fatal(err.Error())
	}
}
<commit_msg>replace toJSON to ToJSON<commit_after>package sphere

import (
	"fmt"
	"net"
	"net/http"
	"net/url"
	"testing"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/gorilla/websocket"
)

var (
	address = func() string {
		l, _ := net.Listen("tcp", ":0")
		defer l.Close()
		return fmt.Sprintf("127.0.0.1:%d", l.Addr().(*net.TCPAddr).Port)
	}()
)

type TestSphereModel struct {
	*ChannelModel
}

func (m *TestSphereModel) Subscribe(room string, connection *Connection) bool {
	return true
}

func (m *TestSphereModel) Disconnect(room string, connection *Connection) bool {
	return true
}

func (m *TestSphereModel) Receive(event string, message string) (string, error) {
	return "you_got_me", nil
}

func init() {
	gin.SetMode(gin.ReleaseMode)
	s, r := Default(), gin.New()
	s.ChannelModels(&TestSphereModel{ExtendChannelModel("test")})
	r.GET("/sync", func(c *gin.Context) {
		s.Handler(c.Writer, c.Request)
	})
	go r.Run(address)
}

func CreateConnection() (c *websocket.Conn, response *http.Response, err error) {
	u, err := url.Parse("ws://" + address + "/sync")
	if err != nil {
		return nil, nil, err
	}
	rawConn, err := net.Dial("tcp", u.Host)
	if err != nil {
		return nil, nil, err
	}
	wsHeaders := http.Header{
		"Origin": {"http://" + address},
	}
	return websocket.NewClient(rawConn, u, wsHeaders, 1024, 1024)
}

func TestSphereReady(t *testing.T) {
	count := 0
	ticker := time.NewTicker(time.Millisecond * 60)
	for {
		select {
		case <-ticker.C:
			s, err := net.Dial("tcp", address)
			defer s.Close()
			if err == nil || count > 3 {
				defer ticker.Stop()
				if err != nil {
					t.Fatal("Server could not get started")
				}
				return
			}
		}
		count++
	}
}

func TestSphereConnection(t *testing.T) {
	c, _, err := CreateConnection()
	if err != nil {
		t.Fatal(err.Error())
	}
	defer c.Close()
}

func TestSphereSendMessage(t *testing.T) {
	c, _, err := CreateConnection()
	if err != nil {
		t.Fatal(err.Error())
	}
	defer c.Close()
	p := &Packet{Type: PacketTypePing}
	res, err := p.ToJSON()
	if err != nil {
		t.Fatal(err.Error())
	}
	if err := c.WriteMessage(TextMessage, res); err != nil {
		t.Fatal(err.Error())
	}
}

func TestSphereMessagePingPong(t *testing.T) {
	done := make(chan error)
	c, _, err := CreateConnection()
	if err != nil {
		t.Fatal(err.Error())
	}
	defer c.Close()
	go func() {
		for {
			_, _, err := c.ReadMessage()
			if err != nil {
				done <- err
				return
			}
			done <- nil
			return
		}
	}()
	p := &Packet{Type: PacketTypePing}
	res, err := p.ToJSON()
	if err != nil {
		t.Fatal(err.Error())
	}
	if err := c.WriteMessage(TextMessage, res); err != nil {
		t.Fatal(err.Error())
	}
	<-done
}

func TestSphereSubscribeChannel(t *testing.T) {
	done := make(chan error)
	c, _, err := CreateConnection()
	if err != nil {
		t.Fatal(err.Error())
	}
	defer c.Close()
	go func() {
		for {
			_, _, err := c.ReadMessage()
			if err != nil {
				done <- err
				return
			}
			done <- nil
			return
		}
	}()
	p := &Packet{Type: PacketTypeSubscribe, Namespace: "test", Room: "helloworld"}
	res, err := p.ToJSON()
	if err != nil {
		t.Fatal(err.Error())
	}
	if err := c.WriteMessage(TextMessage, res); err != nil {
		t.Fatal(err.Error())
	}
	if err := <-done; err != nil {
		t.Fatal(err.Error())
	}
}

func TestSphereChannelMessage(t *testing.T) {
	done := make(chan error)
	c, _, err := CreateConnection()
	if err != nil {
		t.Fatal(err.Error())
	}
	defer c.Close()
	go func() {
		for {
			_, msg, err := c.ReadMessage()
			if err != nil {
				done <- err
				return
			}
			p, err := ParsePacket(msg)
			if err != nil {
				done <- err
			} else {
				switch {
				case p.Type == PacketTypeSubscribed && p.Error == nil:
					m := &Packet{Type: PacketTypeChannel, Namespace: "test", Room: "helloworld", Message: &Message{Event: "HelloEvent", Data: "HelloWorld"}}
					if json, err := m.ToJSON(); err == nil {
						if err := c.WriteMessage(TextMessage, json); err != nil {
							done <- err
						}
					} else {
						done <- err
					}
				case p.Type == PacketTypeChannel && p.Error == nil:
					done <- nil
				}
			}
		}
	}()
	p := &Packet{Type: PacketTypeSubscribe, Namespace: "test", Room: "helloworld"}
	res, err := p.ToJSON()
	if err != nil {
		t.Fatal(err.Error())
	}
	if err := c.WriteMessage(TextMessage, res); err != nil {
		t.Fatal(err.Error())
	}
	if err := <-done; err != nil {
		t.Fatal(err.Error())
	}
}
<|endoftext|>