<commit_before>package standard_test

import (
	"testing"

	"og/helpers"
)

const testDirectory = "operations"

var standardTests = map[string]helpers.OpsFileTestParams{
	"aws.yml":                        {},
	"azure.yml":                      {},
	"bosh-lite.yml":                  {},
	"cf-syslog-skip-cert-verify.yml": {},
	"configure-confab-timeout.yml": {
		Vars: []string{"confab_timeout=60"},
	},
	"configure-default-router-group.yml": {
		Vars: []string{"default_router_group_reservable_ports=1234-2024"},
	},
	"disable-log-cache.yml":              {},
	"disable-router-tls-termination.yml": {},
	"enable-cc-rate-limiting.yml": {
		Vars: []string{"cc_rate_limiter_general_limit=blah", "cc_rate_limiter_unauthenticated_limit=something"},
	},
	"enable-nfs-ldap.yml": {
		Ops:       []string{"enable-nfs-volume-service.yml", "enable-nfs-ldap.yml"},
		VarsFiles: []string{"example-vars-files/vars-enable-nfs-ldap.yml"},
	},
	"enable-nfs-volume-service.yml":           {},
	"enable-privileged-container-support.yml": {},
	"enable-service-discovery.yml":            {},
	"enable-uniq-consul-node-name.yml":        {},
	"migrate-cf-mysql-to-pxc.yml":             {},
	"openstack.yml":                           {},
	"override-app-domains.yml": {
		VarsFiles: []string{"example-vars-files/vars-override-app-domains.yml"},
	},
	"rename-network-and-deployment.yml": {
		Vars: []string{"deployment_name=renamed_deployment", "network_name=renamed_network"},
	},
	"scale-database-cluster.yml": {},
	"scale-to-one-az.yml":        {},
	"set-bbs-active-key.yml": {
		Vars: []string{"diego_bbs_active_key_label=my_key_name"},
	},
	"set-router-static-ips.yml": {
		Vars: []string{"router_static_ips=[10.0.16.4,10.0.47.5]"},
	},
	"stop-skipping-tls-validation.yml": {},
	"use-alicloud-oss-blobstore.yml": {
		Ops:       []string{"use-external-blobstore.yml", "use-alicloud-oss-blobstore.yml"},
		VarsFiles: []string{"example-vars-files/vars-use-alicloud-oss-blobstore.yml"},
	},
	"use-azure-storage-blobstore.yml": {
		Ops:       []string{"use-external-blobstore.yml", "use-azure-storage-blobstore.yml"},
		VarsFiles: []string{"example-vars-files/vars-use-azure-storage-blobstore.yml"},
	},
	"use-blobstore-cdn.yml": {
		VarsFiles: []string{"example-vars-files/vars-use-blobstore-cdn.yml"},
	},
	"use-compiled-releases.yml": {},
	"use-external-blobstore.yml": {
		VarsFiles: []string{"example-vars-files/vars-use-external-blobstore.yml"},
	},
	"use-external-dbs.yml": {
		VarsFiles: []string{"example-vars-files/vars-use-external-dbs.yml"},
	},
	"use-gcs-blobstore-access-key.yml": {
		Ops:       []string{"use-external-blobstore.yml", "use-gcs-blobstore-access-key.yml"},
		Vars:      []string{"blobstore_access_key_id=TEST_ACCESS_KEY", "blobstore_secret_access_key=TEST_SECRET_ACCESS_KEY"},
		VarsFiles: []string{"example-vars-files/vars-use-gcs-blobstore-access-key.yml"},
	},
	"use-gcs-blobstore-service-account.yml": {
		Ops:       []string{"use-external-blobstore.yml", "use-gcs-blobstore-service-account.yml"},
		VarsFiles: []string{"example-vars-files/vars-use-gcs-blobstore-service-account.yml"},
	},
	"use-haproxy-public-network.yml": {
		Ops:  []string{"use-haproxy.yml", "use-haproxy-public-network.yml"},
		Vars: []string{"haproxy_public_network_name=public", "haproxy_public_ip=6.7.8.9"},
	},
	"use-haproxy.yml": {
		Vars: []string{"haproxy_private_ip=10.0.10.10"},
	},
	"use-latest-stemcell.yml": {
		PathValidator: helpers.PathValidator{
			Path: "/stemcells/alias=default/version", ExpectedValue: "latest",
		},
	},
	"use-latest-windows-stemcell.yml": {
		Ops: []string{"windows2012R2-cell.yml", "use-latest-windows-stemcell.yml"},
		PathValidator: helpers.PathValidator{
			Path: "/stemcells/alias=windows2012R2/version", ExpectedValue: "latest",
		},
	},
	"use-latest-windows2012R2-stemcell.yml": {
		Ops: []string{"windows2012R2-cell.yml", "use-latest-windows2012R2-stemcell.yml"},
		PathValidator: helpers.PathValidator{
			Path: "/stemcells/alias=windows2012R2/version", ExpectedValue: "latest",
		},
	},
	"use-latest-windows2016-stemcell.yml": {
		Ops: []string{"windows2016-cell.yml", "use-latest-windows2016-stemcell.yml"},
		PathValidator: helpers.PathValidator{
			Path: "/stemcells/alias=windows2016/version", ExpectedValue: "latest",
		},
	},
	"use-offline-windows2016fs.yml": {
		Ops: []string{"windows2016-cell.yml", "use-offline-windows2016fs.yml"},
	},
	"use-postgres.yml": {},
	"use-pxc-for-nfs-volume-service.yml": {
		Ops: []string{"enable-nfs-volume-service.yml", "use-pxc.yml", "use-pxc-for-nfs-volume-service.yml"},
	},
	"use-pxc.yml": {},
	"use-s3-blobstore.yml": {
		Ops:       []string{"use-external-blobstore.yml", "use-s3-blobstore.yml"},
		VarsFiles: []string{"example-vars-files/vars-use-s3-blobstore.yml"},
	},
	"use-swift-blobstore.yml": {
		Ops:       []string{"use-external-blobstore.yml", "use-swift-blobstore.yml"},
		VarsFiles: []string{"example-vars-files/vars-use-swift-blobstore.yml"},
	},
	"use-trusted-ca-cert-for-apps.yml": {
		VarsFiles: []string{"example-vars-files/vars-use-trusted-ca-cert-for-apps.yml"},
	},
	"windows-cell.yml": {},
	"windows2012R2-cell.yml windows2016-cell.yml": {
		Ops: []string{"windows2012R2-cell.yml", "windows2016-cell.yml"},
	},
	"windows2012R2-cell.yml": {},
	"windows2016-cell.yml":   {},
}

func TestStandard(t *testing.T) {
	cfDeploymentHome, err := helpers.SetPath()
	if err != nil {
		t.Fatalf("setup: %v", err)
	}

	suite := helpers.NewSuiteTest(cfDeploymentHome, testDirectory, standardTests)
	suite.EnsureTestCoverage(t)
	suite.ReadmeTest(t)
	suite.InterpolateTest(t)
}
<commit_msg>Add missing golang tests to the standard suite [#162678972]<commit_after>package standard_test

import (
	"testing"

	"og/helpers"
)

const testDirectory = "operations"

var standardTests = map[string]helpers.OpsFileTestParams{
	"aws.yml":                        {},
	"azure.yml":                      {},
	"bosh-lite.yml":                  {},
	"cf-syslog-skip-cert-verify.yml": {},
	"configure-default-router-group.yml": {
		Vars: []string{"default_router_group_reservable_ports=1234-2024"},
	},
	"disable-router-tls-termination.yml": {},
	"enable-cc-rate-limiting.yml": {
		Vars: []string{"cc_rate_limiter_general_limit=blah", "cc_rate_limiter_unauthenticated_limit=something"},
	},
	"enable-nfs-ldap.yml": {
		Ops:       []string{"enable-nfs-volume-service.yml", "enable-nfs-ldap.yml"},
		VarsFiles: []string{"example-vars-files/vars-enable-nfs-ldap.yml"},
	},
	"enable-nfs-volume-service.yml":           {},
	"enable-privileged-container-support.yml": {},
	"enable-routing-integrity.yml":            {},
	"enable-service-discovery.yml":            {},
	"migrate-cf-mysql-to-pxc.yml":             {},
	"openstack.yml":                           {},
	"override-app-domains.yml": {
		VarsFiles: []string{"example-vars-files/vars-override-app-domains.yml"},
	},
	"rename-network-and-deployment.yml": {
		Vars: []string{"deployment_name=renamed_deployment", "network_name=renamed_network"},
	},
	"scale-database-cluster.yml": {},
	"scale-to-one-az.yml":        {},
	"set-bbs-active-key.yml": {
		Vars: []string{"diego_bbs_active_key_label=my_key_name"},
	},
	"set-router-static-ips.yml": {
		Vars: []string{"router_static_ips=[10.0.16.4,10.0.47.5]"},
	},
	"stop-skipping-tls-validation.yml": {},
	"use-alicloud-oss-blobstore.yml": {
		Ops:       []string{"use-external-blobstore.yml", "use-alicloud-oss-blobstore.yml"},
		VarsFiles: []string{"example-vars-files/vars-use-alicloud-oss-blobstore.yml"},
	},
	"use-azure-storage-blobstore.yml": {
		Ops:       []string{"use-external-blobstore.yml", "use-azure-storage-blobstore.yml"},
		VarsFiles: []string{"example-vars-files/vars-use-azure-storage-blobstore.yml"},
	},
	"use-blobstore-cdn.yml": {
		VarsFiles: []string{"example-vars-files/vars-use-blobstore-cdn.yml"},
	},
	"use-compiled-releases.yml": {},
	"use-external-blobstore.yml": {
		VarsFiles: []string{"example-vars-files/vars-use-external-blobstore.yml"},
	},
	"use-external-dbs.yml": {
		VarsFiles: []string{"example-vars-files/vars-use-external-dbs.yml"},
	},
	"use-gcs-blobstore-access-key.yml": {
		Ops:       []string{"use-external-blobstore.yml", "use-gcs-blobstore-access-key.yml"},
		Vars:      []string{"blobstore_access_key_id=TEST_ACCESS_KEY", "blobstore_secret_access_key=TEST_SECRET_ACCESS_KEY"},
		VarsFiles: []string{"example-vars-files/vars-use-gcs-blobstore-access-key.yml"},
	},
	"use-gcs-blobstore-service-account.yml": {
		Ops:       []string{"use-external-blobstore.yml", "use-gcs-blobstore-service-account.yml"},
		VarsFiles: []string{"example-vars-files/vars-use-gcs-blobstore-service-account.yml"},
	},
	"use-haproxy-public-network.yml": {
		Ops:  []string{"use-haproxy.yml", "use-haproxy-public-network.yml"},
		Vars: []string{"haproxy_public_network_name=public", "haproxy_public_ip=6.7.8.9"},
	},
	"use-haproxy.yml": {
		Vars: []string{"haproxy_private_ip=10.0.10.10"},
	},
	"use-latest-stemcell.yml": {
		PathValidator: helpers.PathValidator{
			Path: "/stemcells/alias=default/version", ExpectedValue: "latest",
		},
	},
	"use-latest-windows2012R2-stemcell.yml": {
		Ops: []string{"windows2012R2-cell.yml", "use-latest-windows2012R2-stemcell.yml"},
		PathValidator: helpers.PathValidator{
			Path: "/stemcells/alias=windows2012R2/version", ExpectedValue: "latest",
		},
	},
	"use-latest-windows2016-stemcell.yml": {
		Ops: []string{"windows2016-cell.yml", "use-latest-windows2016-stemcell.yml"},
		PathValidator: helpers.PathValidator{
			Path: "/stemcells/alias=windows2016/version", ExpectedValue: "latest",
		},
	},
	"use-latest-windows1803-stemcell.yml": {
		Ops: []string{"windows1803-cell.yml", "use-latest-windows1803-stemcell.yml"},
		PathValidator: helpers.PathValidator{
			Path: "/stemcells/alias=windows1803/version", ExpectedValue: "latest",
		},
	},
	"use-offline-windows2016fs.yml": {
		Ops: []string{"windows2016-cell.yml", "use-offline-windows2016fs.yml"},
	},
	"use-postgres.yml": {},
	"use-pxc-for-nfs-volume-service.yml": {
		Ops: []string{"enable-nfs-volume-service.yml", "use-pxc.yml", "use-pxc-for-nfs-volume-service.yml"},
	},
	"use-pxc.yml": {},
	"use-s3-blobstore.yml": {
		Ops:       []string{"use-external-blobstore.yml", "use-s3-blobstore.yml"},
		VarsFiles: []string{"example-vars-files/vars-use-s3-blobstore.yml"},
	},
	"use-swift-blobstore.yml": {
		Ops:       []string{"use-external-blobstore.yml", "use-swift-blobstore.yml"},
		VarsFiles: []string{"example-vars-files/vars-use-swift-blobstore.yml"},
	},
	"use-trusted-ca-cert-for-apps.yml": {
		VarsFiles: []string{"example-vars-files/vars-use-trusted-ca-cert-for-apps.yml"},
	},
	"windows2012R2-cell.yml windows2016-cell.yml": {
		Ops: []string{"windows2012R2-cell.yml", "windows2016-cell.yml"},
	},
	"windows2012R2-cell.yml": {},
	"windows2016-cell.yml":   {},
	"windows1803-cell.yml":   {},
}

func TestStandard(t *testing.T) {
	cfDeploymentHome, err := helpers.SetPath()
	if err != nil {
		t.Fatalf("setup: %v", err)
	}

	suite := helpers.NewSuiteTest(cfDeploymentHome, testDirectory, standardTests)
	suite.EnsureTestCoverage(t)
	suite.ReadmeTest(t)
	suite.InterpolateTest(t)
}
<|endoftext|>


<commit_before>package es

import (
	"encoding/json"
	"github.com/tiaguinho/esmsync/mongo"
	"gopkg.in/mgo.v2/bson"
	"io/ioutil"
	"reflect"
	"strings"
	"unicode"
)

//Mapping struct
type Node struct {
	MongoField string `json:"mongo"`
	Type       string `json:"type"`
	EsField    string `json:"es"`
	ConvertIso string `json:"convert_iso"`
}

func getNodesFile() []Node {
	var nodes []Node

	content, err := ioutil.ReadFile("./config/mapping.json")
	if err == nil {
		json.Unmarshal(content, &nodes)
	}

	return nodes
}

//map a struct to the model defined in mapping.json
func Mapping(oplog interface{}) (object Elasticsearch) {
	nodes := getNodesFile()

	var data map[string]interface{}
	switch reflect.ValueOf(oplog).Field(0).FieldByName("Op").String() {
	case "i":
		s := oplog.(mongo.OplogInsert)
		data = s.O
		object = Elasticsearch{
			Id:        s.O["_id"].(bson.ObjectId).Hex(),
			Operation: "i",
		}
	case "u":
		s := oplog.(mongo.OplogUpdate)
		data = s.O
		object = Elasticsearch{
			Id:        s.O2["_id"].Hex(),
			Operation: "u",
		}
	case "d":
		s := oplog.(mongo.OplogDelete)
		object = Elasticsearch{
			Id:        s.O["_id"].Hex(),
			Operation: "d",
		}
	}

	if len(data) != 0 {
		object.Data = make(map[string]interface{}, len(nodes))

		for _, node := range nodes {
			rs := getValue(node.MongoField, node.Type, data)
			if rs != nil {
				object.Data[node.EsField] = rs

				if node.ConvertIso != "" {
					object.Data[node.ConvertIso] = removeSpecialChar(rs)
				}
			}
		}
	}

	return object
}

//return a value of the field
func getValue(key, data_type string, data map[string]interface{}) (resp interface{}) {
	if data[key] == nil {
		fields := strings.Split(key, ">")

		r := extractValue(fields, data)
		if r != nil {
			resp = r
		}
	} else {
		resp = data[key]
	}

	if resp != nil {
		if data_type != reflect.TypeOf(resp).Kind().String() {
			temps := make([]interface{}, 1)
			temps[0] = resp

			resp = temps
		}
	}

	return
}

func extractValue(fields []string, data interface{}) (result interface{}) {
	var index int
	if len(fields) > 1 {
		index = 1
	}

	if data != nil {
		if reflect.TypeOf(data).Kind() == reflect.Map {
			result = extractValue(fields[index:], data.(map[string]interface{})[fields[0]])
		} else if reflect.TypeOf(data).Kind() == reflect.Slice {
			length := reflect.ValueOf(data).Len()

			results := make([]interface{}, length)
			for i := 0; i < length; i++ {
				results[i] = extractValue(fields[index:], reflect.ValueOf(data).Index(i).Interface())
			}

			result = results
		} else if data != nil {
			result = data
		}
	}

	return
}

var special_caracters string = "áâãàäéêẽèëíîĩìïóôõòöúûũùüç"
var normal_caracters string = "aaaaaeeeeeiiiiiooooouuuuuc"

func removeSpecialChar(value interface{}) (resp interface{}) {
	special_runes := make([]rune, len(normal_caracters))
	for index, letter := range special_caracters {
		if letter != 0 {
			special_runes[index/2] = letter
		}
	}

	normal_runes := make([]rune, len(normal_caracters))
	for index, letter := range normal_caracters {
		normal_runes[index] = letter
	}

	runes := map[rune]rune{}
	for index, letter := range special_runes {
		runes[letter] = normal_runes[index]
	}

	name := strings.Map(func(r rune) rune {
		switch {
		case r == ' ':
			return ' '
		case unicode.IsLetter(r), unicode.IsDigit(r):
			if _, ok := runes[r]; ok {
				return runes[r]
			}

			return r
		default:
			return -1
		}
		return -1
	}, value.(string))

	return name
}
<commit_msg>lower words<commit_after>package es

import (
	"encoding/json"
	"github.com/tiaguinho/esmsync/mongo"
	"gopkg.in/mgo.v2/bson"
	"io/ioutil"
	"reflect"
	"strings"
	"unicode"
)

//Mapping struct
type Node struct {
	MongoField string `json:"mongo"`
	Type       string `json:"type"`
	EsField    string `json:"es"`
	ConvertIso string `json:"convert_iso"`
}

func getNodesFile() []Node {
	var nodes []Node

	content, err := ioutil.ReadFile("./config/mapping.json")
	if err == nil {
		json.Unmarshal(content, &nodes)
	}

	return nodes
}

//map a struct to the model defined in mapping.json
func Mapping(oplog interface{}) (object Elasticsearch) {
	nodes := getNodesFile()

	var data map[string]interface{}
	switch reflect.ValueOf(oplog).Field(0).FieldByName("Op").String() {
	case "i":
		s := oplog.(mongo.OplogInsert)
		data = s.O
		object = Elasticsearch{
			Id:        s.O["_id"].(bson.ObjectId).Hex(),
			Operation: "i",
		}
	case "u":
		s := oplog.(mongo.OplogUpdate)
		data = s.O
		object = Elasticsearch{
			Id:        s.O2["_id"].Hex(),
			Operation: "u",
		}
	case "d":
		s := oplog.(mongo.OplogDelete)
		object = Elasticsearch{
			Id:        s.O["_id"].Hex(),
			Operation: "d",
		}
	}

	if len(data) != 0 {
		object.Data = make(map[string]interface{}, len(nodes))

		for _, node := range nodes {
			rs := getValue(node.MongoField, node.Type, data)
			if rs != nil {
				object.Data[node.EsField] = rs

				if node.ConvertIso != "" {
					object.Data[node.ConvertIso] = removeSpecialChar(rs)
				}
			}
		}
	}

	return object
}

//return a value of the field
func getValue(key, data_type string, data map[string]interface{}) (resp interface{}) {
	if data[key] == nil {
		fields := strings.Split(key, ">")

		r := extractValue(fields, data)
		if r != nil {
			resp = r
		}
	} else {
		resp = data[key]
	}

	if resp != nil {
		if data_type != reflect.TypeOf(resp).Kind().String() {
			temps := make([]interface{}, 1)
			temps[0] = resp

			resp = temps
		}
	}

	return
}

func extractValue(fields []string, data interface{}) (result interface{}) {
	var index int
	if len(fields) > 1 {
		index = 1
	}

	if data != nil {
		if reflect.TypeOf(data).Kind() == reflect.Map {
			result = extractValue(fields[index:], data.(map[string]interface{})[fields[0]])
		} else if reflect.TypeOf(data).Kind() == reflect.Slice {
			length := reflect.ValueOf(data).Len()

			results := make([]interface{}, length)
			for i := 0; i < length; i++ {
				results[i] = extractValue(fields[index:], reflect.ValueOf(data).Index(i).Interface())
			}

			result = results
		} else if data != nil {
			result = data
		}
	}

	return
}

var special_caracters string = "áâãàäéêẽèëíîĩìïóôõòöúûũùüç"
var normal_caracters string = "aaaaaeeeeeiiiiiooooouuuuuc"

func removeSpecialChar(value interface{}) (resp interface{}) {
	special_runes := make([]rune, len(normal_caracters))
	for index, letter := range special_caracters {
		if letter != 0 {
			special_runes[index/2] = letter
		}
	}

	normal_runes := make([]rune, len(normal_caracters))
	for index, letter := range normal_caracters {
		normal_runes[index] = letter
	}

	runes := map[rune]rune{}
	for index, letter := range special_runes {
		runes[letter] = normal_runes[index]
	}

	name := strings.Map(func(r rune) rune {
		switch {
		case r == ' ':
			return ' '
		case unicode.IsLetter(r), unicode.IsDigit(r):
			if _, ok := runes[r]; ok {
				return runes[r]
			}

			return r
		default:
			return -1
		}
		return -1
	}, strings.ToLower(value.(string)))

	return name
}
<|endoftext|>


<commit_before>package util

import (
	"compress/gzip"
	"encoding/gob"
	"fmt"
	"io"
	"os"
	path "path/filepath"
	"strconv"
	"strings"

	"github.com/TuftsBCB/fragbag"
	"github.com/TuftsBCB/fragbag/bow"
	"github.com/TuftsBCB/fragbag/bowdb"
	"github.com/TuftsBCB/hhfrag"
	"github.com/TuftsBCB/io/msa"
	"github.com/TuftsBCB/io/pdb"
	"github.com/TuftsBCB/seq"
)

func Library(fpath string) fragbag.Library {
	libPath := os.Getenv("FRAGLIB_PATH")
	if path.Dir(fpath) == "." && !Exists(fpath) && len(libPath) > 0 {
		fpath = path.Join(libPath, fpath)
		if !strings.HasSuffix(fpath, ".json") {
			fpath += ".json"
		}
	}
	lib, err := fragbag.Open(OpenFile(fpath))
	Assert(err, "Could not open fragment library '%s'", fpath)
	return lib
}

func StructureLibrary(path string) fragbag.StructureLibrary {
	lib, ok := Library(path).(fragbag.StructureLibrary)
	if !ok {
		Fatalf("%s (%T) is not a structure library.", path, lib)
	}
	return lib
}

func SequenceLibrary(path string) fragbag.SequenceLibrary {
	lib, ok := Library(path).(fragbag.SequenceLibrary)
	if !ok {
		Fatalf("%s (%T) is not a sequence library.", path, lib)
	}
	return lib
}

func MSA(path string) seq.MSA {
	if strings.HasSuffix(path, "a2m") || strings.HasSuffix(path, "a3m") {
		aligned, err := msa.Read(OpenFile(path))
		Assert(err, "Could not read MSA (a2m/a3m) from '%s'", path)
		return aligned
	}
	aligned, err := msa.ReadFasta(OpenFile(path))
	Assert(err, "Could not read MSA (fasta) from '%s'", path)
	return aligned
}

func OpenBOWDB(path string) *bowdb.DB {
	db, err := bowdb.OpenDB(path)
	Assert(err, "Could not open BOW database '%s'", path)
	return db
}

func PDBRead(path string) *pdb.Entry {
	entry, err := pdb.ReadPDB(path)
	Assert(err, "Could not open PDB file '%s'", path)
	return entry
}

// PDBPath takes a PDB identifier (e.g., "1ctf" or "1ctfA") and returns
// the full path to the PDB file on the file system.
//
// The PDB_PATH environment variable must be set.
func PDBPath(pid string) string {
	if !IsPDBID(pid) && !IsChainID(pid) {
		Fatalf("PDB ids must contain 4 or 5 characters, but '%s' has %d.",
			pid, len(pid))
	}
	pdbPath := os.Getenv("PDB_PATH")
	if len(pdbPath) == 0 || !IsDir(pdbPath) {
		Fatalf("The PDB_PATH environment variable must be set to open " +
			"PDB chains by just their ID.\n" +
			"PDB_PATH should be set to the directory containing a full " +
			"copy of the PDB database.")
	}

	pdbid := strings.ToLower(pid[0:4])
	group := pdbid[1:3]
	basename := fmt.Sprintf("pdb%s.ent.gz", pdbid)
	return path.Join(pdbPath, group, basename)
}

func PDBReadId(pid string) (*pdb.Entry, *pdb.Chain) {
	e := PDBRead(PDBPath(pid))
	if IsChainID(pid) {
		chain := e.Chain(pid[4])
		if chain == nil {
			Fatalf("Could not find chain '%s' in PDB entry '%s'.", pid[4], pid)
		}
		return e, chain
	}
	return e, nil
}

func GetFmap(fpath string) *hhfrag.FragmentMap {
	var fmap *hhfrag.FragmentMap
	var err error

	switch {
	case IsFasta(fpath):
		fmap, err = HHfragConf.MapFromFasta(FlagPdbHhmDB, FlagSeqDB, fpath)
		Assert(err, "Could not generate map from '%s'", fpath)
	case IsFmap(fpath):
		fmap = FmapRead(fpath)
	default:
		Fatalf("File '%s' is not a fasta or fmap file.", fpath)
	}

	return fmap
}

func FmapRead(path string) *hhfrag.FragmentMap {
	var fmap *hhfrag.FragmentMap
	f := OpenFile(path)
	defer f.Close()

	r := gob.NewDecoder(f)
	Assert(r.Decode(&fmap), "Could not GOB decode fragment map '%s'", path)
	return fmap
}

func FmapWrite(w io.Writer, fmap *hhfrag.FragmentMap) {
	encoder := gob.NewEncoder(w)
	Assert(encoder.Encode(fmap), "Could not GOB encode fragment map")
}

func BOWRead(path string) bow.BOW {
	var bow bow.BOW
	f := OpenFile(path)
	defer f.Close()

	r := gob.NewDecoder(f)
	Assert(r.Decode(&bow), "Could not GOB decode BOW '%s'", path)
	return bow
}

func BOWWrite(w io.Writer, bow bow.BOW) {
	encoder := gob.NewEncoder(w)
	Assert(encoder.Encode(bow), "Could not GOB encode BOW")
}

func OpenFile(path string) *os.File {
	f, err := os.Open(path)
	Assert(err, "Could not open file '%s'", path)
	return f
}

func CreateFile(path string) *os.File {
	f, err := os.Create(path)
	Assert(err, "Could not create file '%s'", path)
	return f
}

func ParseInt(str string) int {
	num, err := strconv.ParseInt(str, 10, 32)
	Assert(err, "Could not parse '%s' as an integer", str)
	return int(num)
}

func IsFasta(fpath string) bool {
	suffix := func(ext string) bool {
		return strings.HasSuffix(fpath, ext)
	}
	return suffix(".fasta") || suffix(".fas") ||
		suffix(".fasta.gz") || suffix(".fas.gz")
}

func OpenFasta(fpath string) io.Reader {
	if strings.HasSuffix(fpath, ".gz") {
		r, err := gzip.NewReader(OpenFile(fpath))
		Assert(err, "Could not open '%s'", fpath)
		return r
	}
	return OpenFile(fpath)
}

func IsFmap(fpath string) bool {
	return strings.HasSuffix(fpath, ".fmap")
}

func IsPDB(fpath string) bool {
	suffix := func(ext string) bool {
		return strings.HasSuffix(fpath, ext)
	}
	return suffix(".ent.gz") || suffix(".pdb") || suffix(".ent")
}

func IsChainID(s string) bool {
	return len(s) == 5
}

func IsPDBID(s string) bool {
	return len(s) == 4
}
<commit_msg>Allow easy opening of SCOP PDB files using SCOP domain identifiers.<commit_after>package util

import (
	"compress/gzip"
	"encoding/gob"
	"fmt"
	"io"
	"os"
	path "path/filepath"
	"strconv"
	"strings"

	"github.com/TuftsBCB/fragbag"
	"github.com/TuftsBCB/fragbag/bow"
	"github.com/TuftsBCB/fragbag/bowdb"
	"github.com/TuftsBCB/hhfrag"
	"github.com/TuftsBCB/io/msa"
	"github.com/TuftsBCB/io/pdb"
	"github.com/TuftsBCB/seq"
)

func Library(fpath string) fragbag.Library {
	libPath := os.Getenv("FRAGLIB_PATH")
	if path.Dir(fpath) == "." && !Exists(fpath) && len(libPath) > 0 {
		fpath = path.Join(libPath, fpath)
		if !strings.HasSuffix(fpath, ".json") {
			fpath += ".json"
		}
	}
	lib, err := fragbag.Open(OpenFile(fpath))
	Assert(err, "Could not open fragment library '%s'", fpath)
	return lib
}

func StructureLibrary(path string) fragbag.StructureLibrary {
	lib, ok := Library(path).(fragbag.StructureLibrary)
	if !ok {
		Fatalf("%s (%T) is not a structure library.", path, lib)
	}
	return lib
}

func SequenceLibrary(path string) fragbag.SequenceLibrary {
	lib, ok := Library(path).(fragbag.SequenceLibrary)
	if !ok {
		Fatalf("%s (%T) is not a sequence library.", path, lib)
	}
	return lib
}

func MSA(path string) seq.MSA {
	if strings.HasSuffix(path, "a2m") || strings.HasSuffix(path, "a3m") {
		aligned, err := msa.Read(OpenFile(path))
		Assert(err, "Could not read MSA (a2m/a3m) from '%s'", path)
		return aligned
	}
	aligned, err := msa.ReadFasta(OpenFile(path))
	Assert(err, "Could not read MSA (fasta) from '%s'", path)
	return aligned
}

func OpenBOWDB(path string) *bowdb.DB {
	db, err := bowdb.OpenDB(path)
	Assert(err, "Could not open BOW database '%s'", path)
	return db
}

func PDBRead(path string) *pdb.Entry {
	entry, err := pdb.ReadPDB(path)
	Assert(err, "Could not open PDB file '%s'", path)
	return entry
}

// PDBPath takes a PDB identifier (e.g., "1ctf" or "1ctfA") and returns
// the full path to the PDB file on the file system.
//
// The PDB_PATH environment variable must be set.
func PDBPath(pid string) string {
	if !IsPDBID(pid) && !IsChainID(pid) {
		Fatalf("PDB ids must contain 4 or 5 characters, but '%s' has %d.",
			pid, len(pid))
	}
	pdbPath := os.Getenv("PDB_PATH")
	if len(pdbPath) == 0 || !IsDir(pdbPath) {
		Fatalf("The PDB_PATH environment variable must be set to open " +
			"PDB chains by just their ID.\n" +
			"PDB_PATH should be set to the directory containing a full " +
			"copy of the PDB database.")
	}

	pdbid := strings.ToLower(pid[0:4])
	group := pdbid[1:3]
	basename := fmt.Sprintf("pdb%s.ent.gz", pdbid)
	return path.Join(pdbPath, group, basename)
}

// ScopPath takes a SCOP identifier (e.g., "d3ciua1" or "d1g09c_") and returns
// the full path to the PDB file on the file system.
//
// The SCOP_PDB_PATH environment variable must be set.
func ScopPath(pid string) string {
	if len(pid) != 7 {
		Fatalf("SCOP domain ids must contain 7 characters, but '%s' has %d.",
			pid, len(pid))
	}
	pdbPath := os.Getenv("SCOP_PDB_PATH")
	if len(pdbPath) == 0 || !IsDir(pdbPath) {
		Fatalf("The SCOP_PDB_PATH environment variable must be set to open " +
			"PDB files of SCOP domain by just their ID.\n" +
			"SCOP_PDB_PATH should be set to the directory containing a full " +
			"copy of the SCOP database as PDB formatted files.")
	}

	group := pid[2:4]
	basename := fmt.Sprintf("%s.ent", pid)
	return path.Join(pdbPath, group, basename)
}

func PDBReadId(pid string) (*pdb.Entry, *pdb.Chain) {
	e := PDBRead(PDBPath(pid))
	if IsChainID(pid) {
		chain := e.Chain(pid[4])
		if chain == nil {
			Fatalf("Could not find chain '%s' in PDB entry '%s'.", pid[4], pid)
		}
		return e, chain
	}
	return e, nil
}

func GetFmap(fpath string) *hhfrag.FragmentMap {
	var fmap *hhfrag.FragmentMap
	var err error

	switch {
	case IsFasta(fpath):
		fmap, err = HHfragConf.MapFromFasta(FlagPdbHhmDB, FlagSeqDB, fpath)
		Assert(err, "Could not generate map from '%s'", fpath)
	case IsFmap(fpath):
		fmap = FmapRead(fpath)
	default:
		Fatalf("File '%s' is not a fasta or fmap file.", fpath)
	}

	return fmap
}

func FmapRead(path string) *hhfrag.FragmentMap {
	var fmap *hhfrag.FragmentMap
	f := OpenFile(path)
	defer f.Close()

	r := gob.NewDecoder(f)
	Assert(r.Decode(&fmap), "Could not GOB decode fragment map '%s'", path)
	return fmap
}

func FmapWrite(w io.Writer, fmap *hhfrag.FragmentMap) {
	encoder := gob.NewEncoder(w)
	Assert(encoder.Encode(fmap), "Could not GOB encode fragment map")
}

func BOWRead(path string) bow.BOW {
	var bow bow.BOW
	f := OpenFile(path)
	defer f.Close()

	r := gob.NewDecoder(f)
	Assert(r.Decode(&bow), "Could not GOB decode BOW '%s'", path)
	return bow
}

func BOWWrite(w io.Writer, bow bow.BOW) {
	encoder := gob.NewEncoder(w)
	Assert(encoder.Encode(bow), "Could not GOB encode BOW")
}

func OpenFile(path string) *os.File {
	f, err := os.Open(path)
	Assert(err, "Could not open file '%s'", path)
	return f
}

func CreateFile(path string) *os.File {
	f, err := os.Create(path)
	Assert(err, "Could not create file '%s'", path)
	return f
}

func ParseInt(str string) int {
	num, err := strconv.ParseInt(str, 10, 32)
	Assert(err, "Could not parse '%s' as an integer", str)
	return int(num)
}

func IsFasta(fpath string) bool {
	suffix := func(ext string) bool {
		return strings.HasSuffix(fpath, ext)
	}
	return suffix(".fasta") || suffix(".fas") ||
		suffix(".fasta.gz") || suffix(".fas.gz")
}

func OpenFasta(fpath string) io.Reader {
	if strings.HasSuffix(fpath, ".gz") {
		r, err := gzip.NewReader(OpenFile(fpath))
		Assert(err, "Could not open '%s'", fpath)
		return r
	}
	return OpenFile(fpath)
}

func IsFmap(fpath string) bool {
	return strings.HasSuffix(fpath, ".fmap")
}

func IsPDB(fpath string) bool {
	suffix := func(ext string) bool {
		return strings.HasSuffix(fpath, ext)
	}
	return suffix(".ent.gz") || suffix(".pdb") || suffix(".ent")
}

func IsChainID(s string) bool {
	return len(s) == 5
}

func IsPDBID(s string) bool {
	return len(s) == 4
}
<|endoftext|>


<commit_before>package util

import (
	"encoding/json"
)

func String(v interface{}) string {
	e, _ := json.Marshal(v)
	return string(e[:])
}
<commit_msg>Add a string prettify method which formats the json nicely<commit_after>package util

import (
	"encoding/json"
)

func String(v interface{}) string {
	e, _ := json.Marshal(v)
	return string(e[:])
}

func StringPrettify(v interface{}) string {
	e, _ := json.MarshalIndent(v, "", "  ")
	return string(e[:])
}
<|endoftext|>


<commit_before>package gamerules

import (
	"rand"
	"time"
	. "chunkymonkey/types"
	"log"
)

// Behaviour of a sapling block, takes care of growing or dying depending on
// world conditions.
func makeSaplingAspect() (aspect IBlockAspect) {
	return &SaplingAspect{
		&StandardAspect{},
	}
}

type SaplingAspect struct {
	*StandardAspect
}

func (aspect *SaplingAspect) Name() string {
	return "Sapling"
}

var rnd = rand.New(rand.NewSource(time.Nanoseconds()))

func (aspect *SaplingAspect) Tick(instance *BlockInstance) bool {
	if rnd.Intn(1e4) >= 1e4-1 {
		// Turn this block into a tree
		return aspect.makeTree(instance)
	}
	return true
}

func (aspect *SaplingAspect) makeTree(instance *BlockInstance) bool {
	loc := instance.SubLoc
	minheight := 3
	maxheight := 6
	height := minheight + rand.Intn(maxheight-minheight)
	maxy := loc.Y + SubChunkCoord(height)

	for y := loc.Y; y < maxy; y++ {
		loc.Y = y
		index, ok := loc.BlockIndex()
		if !ok {
			// TODO: Can't place a block outside chunk boundaries
			log.Printf("Couldn't place a tree block (%v,%v,%v)", loc.X, loc.Y, loc.Z)
		} else {
			instance.Chunk.SetBlockByIndex(index, BlockId(17), byte(0))
		}
	}

	// Store the location at the top block of the tree
	treex, treey, treez := int(loc.X), int(loc.Y), int(loc.Z)
	cradius := height / 2

	// Start one block above the tree and move down
	for y := treey + 1; y >= treey-1; y-- {
		var radius int

		// Slightly round out the canopy
		if y > treey {
			radius = cradius - 1
		} else if y == treey {
			radius = cradius
		} else if y < treey {
			radius = cradius - 1
		}

		for x := treex - radius; x <= treex+radius; x++ {
			for z := treez - radius; z <= treez+radius; z++ {
				if y > treey || x != treex || z != treez {
					loc.X = SubChunkCoord(x)
					loc.Y = SubChunkCoord(y)
					loc.Z = SubChunkCoord(z)
					index, ok := loc.BlockIndex()
					if !ok {
						// TODO: Can't place a block outside chunk boundaries
						log.Printf("Couldn't place a leaf block (%v,%v,%v)", loc.X, loc.Y, loc.Z)
					} else {
						instance.Chunk.SetBlockByIndex(index, BlockId(18), byte(0))
					}
				}
			}
		}
	}

	return true
}
<commit_msg>Simplify embedding of StandardAspect<commit_after>package gamerules

import (
	"rand"
	"time"
	. "chunkymonkey/types"
	"log"
)

// Behaviour of a sapling block, takes care of growing or dying depending on
// world conditions.
func makeSaplingAspect() (aspect IBlockAspect) {
	return &SaplingAspect{}
}

type SaplingAspect struct {
	StandardAspect
}

func (aspect *SaplingAspect) Name() string {
	return "Sapling"
}

var rnd = rand.New(rand.NewSource(time.Nanoseconds()))

func (aspect *SaplingAspect) Tick(instance *BlockInstance) bool {
	if rnd.Intn(1e4) >= 1e4-1 {
		// Turn this block into a tree
		return aspect.makeTree(instance)
	}
	return true
}

func (aspect *SaplingAspect) makeTree(instance *BlockInstance) bool {
	loc := instance.SubLoc
	minheight := 3
	maxheight := 6
	height := minheight + rand.Intn(maxheight-minheight)
	maxy := loc.Y + SubChunkCoord(height)

	for y := loc.Y; y < maxy; y++ {
		loc.Y = y
		index, ok := loc.BlockIndex()
		if !ok {
			// TODO: Can't place a block outside chunk boundaries
			log.Printf("Couldn't place a tree block (%v,%v,%v)", loc.X, loc.Y, loc.Z)
		} else {
			instance.Chunk.SetBlockByIndex(index, BlockId(17), byte(0))
		}
	}

	// Store the location at the top block of the tree
	treex, treey, treez := int(loc.X), int(loc.Y), int(loc.Z)
	cradius := height / 2

	// Start one block above the tree and move down
	for y := treey + 1; y >= treey-1; y-- {
		var radius int

		// Slightly round out the canopy
		if y > treey {
			radius = cradius - 1
		} else if y == treey {
			radius = cradius
		} else if y < treey {
			radius = cradius - 1
		}

		for x := treex - radius; x <= treex+radius; x++ {
			for z := treez - radius; z <= treez+radius; z++ {
				if y > treey || x != treex || z != treez {
					loc.X = SubChunkCoord(x)
					loc.Y = SubChunkCoord(y)
					loc.Z = SubChunkCoord(z)
					index, ok := loc.BlockIndex()
					if !ok {
						// TODO: Can't place a block outside chunk boundaries
						log.Printf("Couldn't place a leaf block (%v,%v,%v)", loc.X, loc.Y, loc.Z)
					} else {
						instance.Chunk.SetBlockByIndex(index, BlockId(18), byte(0))
					}
				}
			}
		}
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// package dashapi defines data structures used in dashboard communication
// and provides client interface.
package dashapi

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"strings"
)

type Dashboard struct {
	Client string
	Addr   string
	Key    string
}

func New(client, addr, key string) *Dashboard {
	return &Dashboard{
		Client: client,
		Addr:   addr,
		Key:    key,
	}
}

// Build describes all aspects of a kernel build.
type Build struct {
	Manager         string
	ID              string
	SyzkallerCommit string
	CompilerID      string
	KernelRepo      string
	KernelBranch    string
	KernelCommit    string
	KernelConfig    []byte
}

func (dash *Dashboard) UploadBuild(build *Build) error {
	return dash.query("upload_build", build, nil)
}

// Crash describes a single kernel crash (potentially with repro).
type Crash struct {
	BuildID     string // refers to Build.ID
	Title       string
	Maintainers []string
	Log         []byte
	Report      []byte
	// The following is optional and is filled only after repro.
	ReproOpts []byte
	ReproSyz  []byte
	ReproC    []byte
}

func (dash *Dashboard) ReportCrash(crash *Crash) error {
	return dash.query("report_crash", crash, nil)
}

// FailedRepro describes a failed repro attempt.
type FailedRepro struct {
	Manager string
	BuildID string
	Title   string
}

func (dash *Dashboard) ReportFailedRepro(repro *FailedRepro) error {
	return dash.query("report_failed_repro", repro, nil)
}

type LogEntry struct {
	Name string
	Text string
}

// Centralized logging on dashboard.
func (dash *Dashboard) LogError(name, msg string, args ...interface{}) {
	req := &LogEntry{
		Name: name,
		Text: fmt.Sprintf(msg, args...),
	}
	dash.query("log_error", req, nil)
}

// BugReport describes a single bug.
// Used by dashboard external reporting.
type BugReport struct {
	Config       []byte
	ID           string
	Title        string
	Maintainers  []string
	CompilerID   string
	KernelRepo   string
	KernelBranch string
	KernelCommit string
	KernelConfig []byte
	Log          []byte
	Report       []byte
	ReproC       []byte
	ReproSyz     []byte
}

type BugUpdate struct {
	ID         string
	Link       string
	Status     BugStatus
	ReproLevel ReproLevel
	DupOf      string
}

type BugUpdateReply struct {
	OK   bool
	Text string
}

type PollRequest struct {
	Type string
}

type PollResponse struct {
	Reports []*BugReport
}

type (
	BugStatus  int
	ReproLevel int
)

const (
	BugStatusOpen BugStatus = iota
	BugStatusUpstream
	BugStatusInvalid
	BugStatusDup
)

const (
	ReproLevelNone ReproLevel = iota
	ReproLevelSyz
	ReproLevelC
)

func (dash *Dashboard) query(method string, req, reply interface{}) error {
	return Query(dash.Client, dash.Addr, dash.Key, method,
		http.NewRequest, http.DefaultClient.Do, req, reply)
}

type (
	RequestCtor func(method, url string, body io.Reader) (*http.Request, error)
	RequestDoer func(req *http.Request) (*http.Response, error)
)

func Query(client, addr, key, method string, ctor RequestCtor, doer RequestDoer, req, reply interface{}) error {
	if reply != nil {
		// json decoding behavior is somewhat surprising
		// (see // https://github.com/golang/go/issues/21092).
		// To avoid any surprises, we zero the reply.
		typ := reflect.TypeOf(reply)
		if typ.Kind() != reflect.Ptr {
			return fmt.Errorf("resp must be a pointer")
		}
		reflect.ValueOf(reply).Elem().Set(reflect.New(typ.Elem()).Elem())
	}
	values := make(url.Values)
	values.Add("client", client)
	values.Add("key", key)
	values.Add("method", method)
	var body io.Reader
	gzipped := false
	if req != nil {
		data, err := json.Marshal(req)
		if err != nil {
			return fmt.Errorf("failed to marshal request: %v", err)
		}
		if len(data) < 100 || addr == "" || strings.HasPrefix(addr, "http://localhost:") {
			// Don't bother compressing tiny requests.
			// Don't compress for dev_appserver which does not support gzip.
			body = bytes.NewReader(data)
		} else {
			buf := new(bytes.Buffer)
			gz := gzip.NewWriter(buf)
			if _, err := gz.Write(data); err != nil {
				return err
			}
			if err := gz.Close(); err != nil {
				return err
			}
			body = buf
			gzipped = true
		}
	}
	url := fmt.Sprintf("%v/api?%v", addr, values.Encode())
	r, err := ctor("POST", url, body)
	if err != nil {
		return err
	}
	if body != nil {
		r.Header.Set("Content-Type", "application/json")
		if gzipped {
			r.Header.Set("Content-Encoding", "gzip")
		}
	}
	resp, err := doer(r)
	if err != nil {
		return fmt.Errorf("http request failed: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		data, _ := ioutil.ReadAll(resp.Body)
		return fmt.Errorf("request failed with %v: %s", resp.Status, data)
	}
	if reply != nil {
		if err := json.NewDecoder(resp.Body).Decode(reply); err != nil {
			return fmt.Errorf("failed to unmarshal response: %v", err)
		}
	}
	return nil
}
<commit_msg>dashboard/dashapi: add support for bug fixing commits<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// package dashapi defines data structures used in dashboard communication
// and provides client interface.
package dashapi

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"strings"
)

type Dashboard struct {
	Client string
	Addr   string
	Key    string
}

func New(client, addr, key string) *Dashboard {
	return &Dashboard{
		Client: client,
		Addr:   addr,
		Key:    key,
	}
}

// Build describes all aspects of a kernel build.
type Build struct {
	Manager         string
	ID              string
	SyzkallerCommit string
	CompilerID      string
	KernelRepo      string
	KernelBranch    string
	KernelCommit    string
	KernelConfig    []byte
	Commits         []string // see BuilderPoll
}

func (dash *Dashboard) UploadBuild(build *Build) error {
	return dash.query("upload_build", build, nil)
}

// BuilderPoll request is done by kernel builder before uploading a new build
// with UploadBuild request. Response contains list of commits that dashboard
// is interested in (i.e. commits that fix open bugs). When uploading a new
// build builder should pass subset of the commits that are present in the build
// in Build.Commits field.

type BuilderPollReq struct {
	Manager string
}

type BuilderPollResp struct {
	PendingCommits []string
}

func (dash *Dashboard) BuilderPoll(manager string) (*BuilderPollResp, error) {
	req := &BuilderPollReq{
		Manager: manager,
	}
	resp := new(BuilderPollResp)
	err := dash.query("builder_poll", req, resp)
	return resp, err
}

// Crash describes a single kernel crash (potentially with repro).
type Crash struct {
	BuildID     string // refers to Build.ID
	Title       string
	Maintainers []string
	Log         []byte
	Report      []byte
	// The following is optional and is filled only after repro.
	ReproOpts []byte
	ReproSyz  []byte
	ReproC    []byte
}

func (dash *Dashboard) ReportCrash(crash *Crash) error {
	return dash.query("report_crash", crash, nil)
}

// FailedRepro describes a failed repro attempt.
type FailedRepro struct {
	Manager string
	BuildID string
	Title   string
}

func (dash *Dashboard) ReportFailedRepro(repro *FailedRepro) error {
	return dash.query("report_failed_repro", repro, nil)
}

type LogEntry struct {
	Name string
	Text string
}

// Centralized logging on dashboard.
func (dash *Dashboard) LogError(name, msg string, args ...interface{}) {
	req := &LogEntry{
		Name: name,
		Text: fmt.Sprintf(msg, args...),
	}
	dash.query("log_error", req, nil)
}

// BugReport describes a single bug.
// Used by dashboard external reporting.
type BugReport struct {
	Config       []byte
	ID           string
	Title        string
	Maintainers  []string
	CompilerID   string
	KernelRepo   string
	KernelBranch string
	KernelCommit string
	KernelConfig []byte
	Log          []byte
	Report       []byte
	ReproC       []byte
	ReproSyz     []byte
}

type BugUpdate struct {
	ID         string
	Link       string
	Status     BugStatus
	ReproLevel ReproLevel
	DupOf      string
	FixCommits []string // Titles of commits that fix this bug.
}

type BugUpdateReply struct {
	OK   bool
	Text string
}

type PollRequest struct {
	Type string
}

type PollResponse struct {
	Reports []*BugReport
}

type (
	BugStatus  int
	ReproLevel int
)

const (
	BugStatusOpen BugStatus = iota
	BugStatusUpstream
	BugStatusInvalid
	BugStatusDup
)

const (
	ReproLevelNone ReproLevel = iota
	ReproLevelSyz
	ReproLevelC
)

func (dash *Dashboard) query(method string, req, reply interface{}) error {
	return Query(dash.Client, dash.Addr, dash.Key, method,
		http.NewRequest, http.DefaultClient.Do, req, reply)
}

type (
	RequestCtor func(method, url string, body io.Reader) (*http.Request, error)
	RequestDoer func(req *http.Request) (*http.Response, error)
)

func Query(client, addr, key, method string, ctor RequestCtor, doer RequestDoer, req, reply interface{}) error {
	if reply != nil {
		// json decoding behavior is somewhat surprising
		// (see // https://github.com/golang/go/issues/21092).
		// To avoid any surprises, we zero the reply.
		typ := reflect.TypeOf(reply)
		if typ.Kind() != reflect.Ptr {
			return fmt.Errorf("resp must be a pointer")
		}
		reflect.ValueOf(reply).Elem().Set(reflect.New(typ.Elem()).Elem())
	}
	values := make(url.Values)
	values.Add("client", client)
	values.Add("key", key)
	values.Add("method", method)
	var body io.Reader
	gzipped := false
	if req != nil {
		data, err := json.Marshal(req)
		if err != nil {
			return fmt.Errorf("failed to marshal request: %v", err)
		}
		if len(data) < 100 || addr == "" || strings.HasPrefix(addr, "http://localhost:") {
			// Don't bother compressing tiny requests.
			// Don't compress for dev_appserver which does not support gzip.
			body = bytes.NewReader(data)
		} else {
			buf := new(bytes.Buffer)
			gz := gzip.NewWriter(buf)
			if _, err := gz.Write(data); err != nil {
				return err
			}
			if err := gz.Close(); err != nil {
				return err
			}
			body = buf
			gzipped = true
		}
	}
	url := fmt.Sprintf("%v/api?%v", addr, values.Encode())
	r, err := ctor("POST", url, body)
	if err != nil {
		return err
	}
	if body != nil {
		r.Header.Set("Content-Type", "application/json")
		if gzipped {
			r.Header.Set("Content-Encoding", "gzip")
		}
	}
	resp, err := doer(r)
	if err != nil {
		return fmt.Errorf("http request failed: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		data, _ := ioutil.ReadAll(resp.Body)
		return fmt.Errorf("request failed with %v: %s", resp.Status, data)
	}
	if reply != nil {
		if err := json.NewDecoder(resp.Body).Decode(reply); err != nil {
			return fmt.Errorf("failed to unmarshal response: %v", err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package jsonutil
<commit_msg>完善framework/json单元测试<commit_after>package jsonutil

import (
	"testing"
)

// 以下是功能测试

type (
	colorGroup struct {
		ID     int
		Name   string
		Colors []string
	}
	mymap map[string]interface{}
)

func Test_GetJsonString_1(t *testing.T) {
	g := newGroup()
	json := GetJsonString(g)
	t.Log(json)
}

func Test_Marshal_1(t *testing.T) {
	g := newGroup()
	json, err := Marshal(g)
	if err != nil {
		t.Error(err)
	} else {
		t.Log(json)
	}

}

func Test_Unmarshal_1(t *testing.T) {
	var group colorGroup
	g := newGroup()
	json := GetJsonString(g)
	err := Unmarshal(json, &group)
	if err != nil {
		t.Error(err)
	} else {
		t.Log(group)
	}

}

func newGroup() colorGroup {
	group := colorGroup{
		ID:     1,
		Name:   "Reds",
		Colors: []string{"Crimson", "Red", "Ruby", "Maroon"},
	}
	return group
}
<|endoftext|>


<commit_before>package websocket

import (
	"bytes"
	"io/ioutil"
	"testing"
	"testing/iotest"

	ma "github.com/multiformats/go-multiaddr"
)

func TestWebsocketListen(t *testing.T) {
	zero, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0/ws")
	if err != nil {
		t.Fatal(err)
	}

	tpt := &WebsocketTransport{}
	l, err := tpt.Listen(zero)
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()

	msg := []byte("HELLO WORLD")

	go func() {
		d, _ := tpt.Dialer(nil)
		c, err := d.Dial(l.Multiaddr())
		if err != nil {
			t.Error(err)
			return
		}

		c.Write(msg)
		c.Close()
	}()

	c, err := l.Accept()
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	obr := iotest.OneByteReader(c)

	out, err := ioutil.ReadAll(obr)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(out, msg) {
		t.Fatal("got wrong message", out, msg)
	}
}
<commit_msg>test concurrent connection closing<commit_after>package websocket

import (
	"bytes"
	"io/ioutil"
	"testing"
	"testing/iotest"

	ma "github.com/multiformats/go-multiaddr"
)

func TestWebsocketListen(t *testing.T) {
	zero, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0/ws")
	if err != nil {
		t.Fatal(err)
	}

	tpt := &WebsocketTransport{}
	l, err := tpt.Listen(zero)
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()

	msg := []byte("HELLO WORLD")

	go func() {
		d, _ := tpt.Dialer(nil)
		c, err := d.Dial(l.Multiaddr())
		if err != nil {
			t.Error(err)
			return
		}

		c.Write(msg)
		c.Close()
	}()

	c, err := l.Accept()
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	obr := iotest.OneByteReader(c)

	out, err := ioutil.ReadAll(obr)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(out, msg) {
		t.Fatal("got wrong message", out, msg)
	}
}

func TestConcurrentClose(t *testing.T) {
	zero, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0/ws")
	if err != nil {
		t.Fatal(err)
	}

	tpt := &WebsocketTransport{}
	l, err := tpt.Listen(zero)
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()

	msg := []byte("HELLO WORLD")

	go func() {
		d, _ := tpt.Dialer(nil)
		for i := 0; i < 100; i++ {
			c, err := d.Dial(l.Multiaddr())
			if err != nil {
				t.Error(err)
				return
			}

			go c.Write(msg)
			go c.Close()
		}
	}()

	for i := 0; i < 100; i++ {
		c, err := l.Accept()
		if err != nil {
			t.Fatal(err)
		}
		c.Close()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"math/rand"
	"net/http"
	"reflect"
	"time"

	"golang.org/x/net/context"

	"github.com/pressly/chi"
	"github.com/pressly/chi/middleware"
	"github.com/pressly/chi/render"
)

type RuntimeObject struct {
	ID       int      `json:"id"`
	AuthData string   `json:"auth_data"`
	Data     []string `json:"data"`
}

type PresenterObject struct {
	*RuntimeObject `json:",inline"`

	URL   string `json:"url"`
	False bool   `json:"false"`

	// Omit by default. Show explicitly for auth'd users only.
	AuthData interface{} `json:"auth_data,omitempty"`
}

type PresenterObjectV2 struct {
	*PresenterObject `json:",inline"`

	ResourceURL string `json:"resource_url"`

	// Omit.
	URL interface{} `json:"url,omitempty"`
}

type PresenterObjectV1 struct {
	*PresenterObjectV2 `json:",inline"`

	Data map[string]bool `json:"data"`
}

var (
	ErrUnauthorized = errors.New("Unauthorized")
	ErrForbidden    = errors.New("Forbidden")
	ErrNotFound     = errors.New("Resource not found")
)

func main() {
	r := chi.NewRouter()

	r.Use(middleware.RequestID)
	r.Use(middleware.Logger)
	r.Use(middleware.Recoverer)
	r.Use(render.ParseContentType)

	render.DefaultPresenter.Register(func(ctx context.Context, from *RuntimeObject) (*PresenterObject, error) {
		to := &PresenterObject{
			RuntimeObject: from,
			URL:           fmt.Sprintf("https://api.example.com/objects/%v", from.ID),
		}
		// Only show to auth'd user.
		if _, ok := ctx.Value("auth").(bool); ok {
			to.AuthData = from.AuthData
		}
		return to, nil
	})

	v2 := render.NewPresenter()
	v2.RegisterFrom(render.DefaultPresenter)
	v2.Register(func(ctx context.Context, from *PresenterObject) (*PresenterObjectV2, error) {
		return &PresenterObjectV2{PresenterObject: from, ResourceURL: from.URL}, nil
	})

	v1 := render.NewPresenter()
	v1.RegisterFrom(v2)
	v1.Register(func(ctx context.Context, from *PresenterObjectV2) (*PresenterObjectV1, error) {
		to := &PresenterObjectV1{
			PresenterObjectV2: from,
			Data:              map[string]bool{},
		}
		for _, item := range from.Data {
			to.Data[item] = true
		}
		return to, nil
	})

	r.Get("/", objectHandler)
	r.Get("/v2", render.UsePresenter(v2), objectHandler)
	r.Get("/v1", render.UsePresenter(v1), objectHandler)

	r.Get("/error", randomErrorHandler)

	http.ListenAndServe(":3333", r)
}

func objectHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	obj := &RuntimeObject{
		ID:       1,
		AuthData: "secret data for auth'd users only",
		Data:     []string{"one", "two", "three", "four"},
	}

	// Simulate some context values (copy over from query params).
	if r.URL.Query().Get("auth") != "" {
		ctx = context.WithValue(ctx, "auth", true)
	}
	if r.URL.Query().Get("error") != "" {
		render.Respond(ctx, w, errors.New("error"))
		return
	}

	render.Respond(ctx, w, obj)
}

func randomErrorHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	errors := []error{ErrUnauthorized, ErrForbidden, ErrNotFound}

	rand.Seed(time.Now().Unix())
	render.Respond(ctx, w, errors[rand.Intn(len(errors))])
}

func init() {
	render.Respond = customRespond
}

func customRespond(ctx context.Context, w http.ResponseWriter, v interface{}) {
	// Set response status based on Error value/type.
	val := reflect.ValueOf(v)
	if err, ok := val.Interface().(error); ok {
		switch err {
		case ErrUnauthorized:
			ctx = render.Status(ctx, 401)
		case ErrForbidden:
			ctx = render.Status(ctx, 403)
		case ErrNotFound:
			ctx = render.Status(ctx, 404)
		default:
			ctx = render.Status(ctx, 500)
		}
		render.DefaultRespond(ctx, w, map[string]string{"error": err.Error()})
		return
	}

	render.DefaultRespond(ctx, w, v)
}
<commit_msg>Refactor render example<commit_after>package main

import (
	"errors"
	"fmt"
	"math/rand"
	"net/http"
	"reflect"
	"time"

	"golang.org/x/net/context"

	"github.com/pressly/chi"
	"github.com/pressly/chi/middleware"
	"github.com/pressly/chi/render"
)

type RuntimeObject struct {
	ID       int      `json:"id"`
	AuthData string   `json:"auth_data"`
	Data     []string `json:"data"`
}

type PresenterObject struct {
	*RuntimeObject `json:",inline"`

	URL   string `json:"url"`
	False bool   `json:"false"`

	// Omit by default. Show explicitly for auth'd users only.
	AuthData interface{} `json:"auth_data,omitempty"`
}

type PresenterObjectV2 struct {
	*PresenterObject `json:",inline"`

	ResourceURL string `json:"resource_url"`

	// Omit.
	URL interface{} `json:"url,omitempty"`
}

type PresenterObjectV1 struct {
	*PresenterObjectV2 `json:",inline"`

	Data map[string]bool `json:"data"`
}

var (
	ErrUnauthorized = errors.New("Unauthorized")
	ErrForbidden    = errors.New("Forbidden")
	ErrNotFound     = errors.New("Resource not found")

	v2 = render.NewPresenter()
	v1 = render.NewPresenter()
)

func init() {
	render.Respond = customRespond

	render.DefaultPresenter.Register(func(ctx context.Context, from *RuntimeObject) (*PresenterObject, error) {
		to := &PresenterObject{
			RuntimeObject: from,
			URL:           fmt.Sprintf("https://api.example.com/objects/%v", from.ID),
		}
		// Only show to auth'd user.
		if _, ok := ctx.Value("auth").(bool); ok {
			to.AuthData = from.AuthData
		}
		return to, nil
	})

	v2.RegisterFrom(render.DefaultPresenter)
	v2.Register(func(ctx context.Context, from *PresenterObject) (*PresenterObjectV2, error) {
		return &PresenterObjectV2{PresenterObject: from, ResourceURL: from.URL}, nil
	})

	v1.RegisterFrom(v2)
	v1.Register(func(ctx context.Context, from *PresenterObjectV2) (*PresenterObjectV1, error) {
		to := &PresenterObjectV1{
			PresenterObjectV2: from,
			Data:              map[string]bool{},
		}
		for _, item := range from.Data {
			to.Data[item] = true
		}
		return to, nil
	})
}

// customRespond sets response status code based on Error value/type.
func customRespond(ctx context.Context, w http.ResponseWriter, v interface{}) {
	val := reflect.ValueOf(v)
	if err, ok := val.Interface().(error); ok {
		switch err {
		case ErrUnauthorized:
			ctx = render.Status(ctx, 401)
		case ErrForbidden:
			ctx = render.Status(ctx, 403)
		case ErrNotFound:
			ctx = render.Status(ctx, 404)
		default:
			ctx = render.Status(ctx, 500)
		}
		render.DefaultRespond(ctx, w, map[string]string{"error": err.Error()})
		return
	}

	render.DefaultRespond(ctx, w, v)
}

func main() {
	r := chi.NewRouter()

	r.Use(middleware.RequestID)
	r.Use(middleware.Logger)
	r.Use(middleware.Recoverer)
	r.Use(render.ParseContentType)

	r.Get("/", objectHandler)
	r.Get("/v2", render.UsePresenter(v2), objectHandler)
	r.Get("/v1", render.UsePresenter(v1), objectHandler)

	r.Get("/error", randomErrorHandler)

	http.ListenAndServe(":3333", r)
}

func objectHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	obj := &RuntimeObject{
		ID:       1,
		AuthData: "secret data for auth'd users only",
		Data:     []string{"one", "two", "three", "four"},
	}

	// Simulate some context values (copy over from query params).
	if r.URL.Query().Get("auth") != "" {
		ctx = context.WithValue(ctx, "auth", true)
	}
	if r.URL.Query().Get("error") != "" {
		render.Respond(ctx, w, errors.New("error"))
		return
	}

	render.Respond(ctx, w, obj)
}

func randomErrorHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	errors := []error{ErrUnauthorized, ErrForbidden, ErrNotFound}

	rand.Seed(time.Now().Unix())
	render.Respond(ctx, w, errors[rand.Intn(len(errors))])
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"

	samplecontroller "k8s.io/sample-controller/pkg/apis/samplecontroller"
)

// SchemeGroupVersion is group version used to register these objects
var SchemeGroupVersion = schema.GroupVersion{Group: samplecontroller.GroupName, Version: "v1alpha1"}

// Kind takes an unqualified kind and returns back a Group qualified GroupKind
func Kind(kind string) schema.GroupKind {
	return SchemeGroupVersion.WithKind(kind).GroupKind()
}

// Resource takes an unqualified resource and returns a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}

var (
	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
	AddToScheme   = SchemeBuilder.AddToScheme
)

// Adds the list of known types to Scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Foo{},
		&FooList{},
	)
	return nil
}
<commit_msg>Register metav1 types into samplecontroller api scheme<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"

	samplecontroller "k8s.io/sample-controller/pkg/apis/samplecontroller"
)

// SchemeGroupVersion is group version used to register these objects
var SchemeGroupVersion = schema.GroupVersion{Group: samplecontroller.GroupName, Version: "v1alpha1"}

// Kind takes an unqualified kind and returns back a Group qualified GroupKind
func Kind(kind string) schema.GroupKind {
	return SchemeGroupVersion.WithKind(kind).GroupKind()
}

// Resource takes an unqualified resource and returns a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}

var (
	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
	AddToScheme   = SchemeBuilder.AddToScheme
)

// Adds the list of known types to Scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Foo{},
		&FooList{},
	)
	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
	return nil
}
<|endoftext|>


<commit_before>//
// Copyright 2016 Gregory Trubetskoy. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// NOTE: No locking!
package aggregator

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"math"
	"sort"
	"time"
)

type dataPointQueuer interface {
	QueueDataPoint(string, time.Time, float64)
}

type aggKind int

const (
	aggKindValue aggKind = iota
	aggKindGauge
	aggKindList
)

type aggregation struct {
	kind  aggKind
	value float64
	list  []float64
}

type Aggregator struct {
	t          dataPointQueuer
	m          map[string]*aggregation
	lastFlush  time.Time
	thresholds []int
}

func NewAggregator(t dataPointQueuer) *Aggregator {
	return &Aggregator{
		t:          t,
		m:          make(map[string]*aggregation),
		lastFlush:  time.Now(),
		thresholds: []int{90}, // TODO make me configurable?
	}
}

// Add to an already existing value at key name, created as
// 0.0/aggKindValue if not existing.
func (a *Aggregator) add(name string, value float64) {
	if a.m[name] == nil {
		a.m[name] = &aggregation{kind: aggKindValue}
	}
	a.m[name].value += value
}

// Add to an already existing value at key name, created as
// 0.0/aggKindGauge if not existing.
func (a *Aggregator) addGauge(name string, value float64) {
	if a.m[name] == nil {
		a.m[name] = &aggregation{kind: aggKindGauge}
	}
	a.m[name].value += value
}

// Set the value at key name overwriting any previous, created as
// 0.0./aggKindGauge if not existing
func (a *Aggregator) setGauge(name string, value float64) {
	if a.m[name] == nil {
		a.m[name] = &aggregation{kind: aggKindGauge, value: value}
	} else {
		a.m[name].value = value
	}
}

// Append to values at key name, created as aggKindList if not
// existing.
func (a *Aggregator) append(name string, value float64) {
	if a.m[name] == nil {
		a.m[name] = &aggregation{kind: aggKindList, list: make([]float64, 0, 2)}
	}
	if a.m[name].list != nil {
		a.m[name].list = append(a.m[name].list, value)
	}
}

func (a *Aggregator) ProcessCmd(cmd *Command) {
	switch cmd.cmd {
	case CmdAdd:
		a.add(cmd.name, cmd.value)
	case CmdAddGauge:
		a.addGauge(cmd.name, cmd.value)
	case CmdSetGauge:
		a.setGauge(cmd.name, cmd.value)
	case CmdAppend:
		a.append(cmd.name, cmd.value)
	}
}

func (a *Aggregator) Flush(now time.Time) {
	if now.IsZero() {
		now = time.Now()
	}

	for name, agg := range a.m {

		switch agg.kind {
		case aggKindValue:
			// store rate
			if now.After(a.lastFlush) {
				a.t.QueueDataPoint(name, now, agg.value/now.Sub(a.lastFlush).Seconds())
			}

		case aggKindGauge:
			// store as is
			a.t.QueueDataPoint(name, now, agg.value)

		case aggKindList:
			list := agg.list

			// count
			a.t.QueueDataPoint(name+".count", now, float64(len(list)))

			// lower, upper, sum, mean
			if len(list) > 0 {
				sort.Float64s(list)

				cumul := make([]float64, len(list))
				for n, v := range list {
					cumul[n] += v
				}

				a.t.QueueDataPoint(name+".lower", now, list[0])
				a.t.QueueDataPoint(name+".upper", now, list[len(list)-1])
				a.t.QueueDataPoint(name+".sum", now, cumul[len(list)-1])
				a.t.QueueDataPoint(name+".mean", now, cumul[len(list)-1]/float64(len(list)))

				// make a little round() since Go doesn't have one...
				round := func(f float64) int {
					return int(math.Floor(f + .5))
				}

				// TODO may be add "median" and "std"?
				for _, threshold := range a.thresholds {
					idx := round(float64(threshold)/100*float64(len(list))) - 1
					a.t.QueueDataPoint(name+fmt.Sprintf(".sum_%02d", threshold), now, cumul[idx])
					a.t.QueueDataPoint(name+fmt.Sprintf(".mean_%02d", threshold), now, cumul[idx]/float64(idx+1))
					a.t.QueueDataPoint(name+fmt.Sprintf(".upper_%02d", threshold), now, list[idx])
				}
			}
		}

		// clear the map
		a.m = make(map[string]*aggregation)
		a.lastFlush = now
	}
}

type AggCmd int

const (
	CmdAdd AggCmd = iota
	CmdAddGauge
	CmdSetGauge
	CmdAppend
)

type Command struct {
	cmd   AggCmd
	name  string
	value float64
	Hops  int
}

func (ac *Command) GobEncode() ([]byte, error) {
	buf := bytes.Buffer{}
	enc := gob.NewEncoder(&buf)
	var err error
	check := func(er error) {
		if er != nil && err == nil {
			err = er
		}
	}
	check(enc.Encode(ac.cmd))
	check(enc.Encode(ac.name))
	check(enc.Encode(ac.value))
	check(enc.Encode(ac.Hops))
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (ac *Command) GobDecode(b []byte) error {
	dec := gob.NewDecoder(bytes.NewBuffer(b))
	var err error
	check := func(er error) {
		if er != nil && err == nil {
			err = er
		}
	}
	check(dec.Decode(&ac.cmd))
	check(dec.Decode(&ac.name))
	check(dec.Decode(&ac.value))
	check(dec.Decode(&ac.Hops))
	return err
}

func NewCommand(cmd AggCmd, name string, value float64) *Command {
	return &Command{cmd: cmd, name: name, value: value}
}
<commit_msg>Ignore stale aggregator commands to avoid spikes on transitions when commands queue up in channels and then get processed all at once.<commit_after>//
// Copyright 2016 Gregory Trubetskoy. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// NOTE: No locking!
package aggregator

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"math"
	"sort"
	"time"
)

type dataPointQueuer interface {
	QueueDataPoint(string, time.Time, float64)
}

type aggKind int

const (
	aggKindValue aggKind = iota
	aggKindGauge
	aggKindList
)

type aggregation struct {
	kind  aggKind
	value float64
	list  []float64
}

type Aggregator struct {
	t          dataPointQueuer
	m          map[string]*aggregation
	lastFlush  time.Time
	thresholds []int
}

func NewAggregator(t dataPointQueuer) *Aggregator {
	return &Aggregator{
		t:          t,
		m:          make(map[string]*aggregation),
		lastFlush:  time.Now(),
		thresholds: []int{90}, // TODO make me configurable?
	}
}

// Add to an already existing value at key name, created as
// 0.0/aggKindValue if not existing.
func (a *Aggregator) add(name string, value float64) {
	if a.m[name] == nil {
		a.m[name] = &aggregation{kind: aggKindValue}
	}
	a.m[name].value += value
}

// Add to an already existing value at key name, created as
// 0.0/aggKindGauge if not existing.
func (a *Aggregator) addGauge(name string, value float64) {
	if a.m[name] == nil {
		a.m[name] = &aggregation{kind: aggKindGauge}
	}
	a.m[name].value += value
}

// Set the value at key name overwriting any previous, created as
// 0.0./aggKindGauge if not existing
func (a *Aggregator) setGauge(name string, value float64) {
	if a.m[name] == nil {
		a.m[name] = &aggregation{kind: aggKindGauge, value: value}
	} else {
		a.m[name].value = value
	}
}

// Append to values at key name, created as aggKindList if not
// existing.
func (a *Aggregator) append(name string, value float64) {
	if a.m[name] == nil {
		a.m[name] = &aggregation{kind: aggKindList, list: make([]float64, 0, 2)}
	}
	if a.m[name].list != nil {
		a.m[name].list = append(a.m[name].list, value)
	}
}

func (a *Aggregator) ProcessCmd(cmd *Command) {
	if !cmd.ts.IsZero() && cmd.ts.Before(a.lastFlush) {
		return // this command is too old for this aggregator, ignore it
	}
	switch cmd.cmd {
	case CmdAdd:
		a.add(cmd.name, cmd.value)
	case CmdAddGauge:
		a.addGauge(cmd.name, cmd.value)
	case CmdSetGauge:
		a.setGauge(cmd.name, cmd.value)
	case CmdAppend:
		a.append(cmd.name, cmd.value)
	}
}

func (a *Aggregator) Flush(now time.Time) {
	if now.IsZero() {
		now = time.Now()
	}

	for name, agg := range a.m {

		switch agg.kind {
		case aggKindValue:
			// store rate
			if now.After(a.lastFlush) {
				a.t.QueueDataPoint(name, now, agg.value/now.Sub(a.lastFlush).Seconds())
			}

		case aggKindGauge:
			// store as is
			a.t.QueueDataPoint(name, now, agg.value)

		case aggKindList:
			list := agg.list

			// count
			a.t.QueueDataPoint(name+".count", now, float64(len(list)))

			// lower, upper, sum, mean
			if len(list) > 0 {
				sort.Float64s(list)

				cumul := make([]float64, len(list))
				for n, v := range list {
					cumul[n] += v
				}

				a.t.QueueDataPoint(name+".lower", now, list[0])
				a.t.QueueDataPoint(name+".upper", now, list[len(list)-1])
				a.t.QueueDataPoint(name+".sum", now, cumul[len(list)-1])
				a.t.QueueDataPoint(name+".mean", now, cumul[len(list)-1]/float64(len(list)))

				// make a little round() since Go doesn't have one...
				round := func(f float64) int {
					return int(math.Floor(f + .5))
				}

				// TODO may be add "median" and "std"?
				for _, threshold := range a.thresholds {
					idx := round(float64(threshold)/100*float64(len(list))) - 1
					a.t.QueueDataPoint(name+fmt.Sprintf(".sum_%02d", threshold), now, cumul[idx])
					a.t.QueueDataPoint(name+fmt.Sprintf(".mean_%02d", threshold), now, cumul[idx]/float64(idx+1))
					a.t.QueueDataPoint(name+fmt.Sprintf(".upper_%02d", threshold), now, list[idx])
				}
			}
		}

		// clear the map
		a.m = make(map[string]*aggregation)
		a.lastFlush = now
	}
}

type AggCmd int

const (
	CmdAdd AggCmd = iota
	CmdAddGauge
	CmdSetGauge
	CmdAppend
)

type Command struct {
	cmd   AggCmd
	name  string
	value float64
	ts    time.Time
	Hops  int
}

func (ac *Command) GobEncode() ([]byte, error) {
	buf := bytes.Buffer{}
	enc := gob.NewEncoder(&buf)
	var err error
	check := func(er error) {
		if er != nil && err == nil {
			err = er
		}
	}
	check(enc.Encode(ac.cmd))
	check(enc.Encode(ac.name))
	check(enc.Encode(ac.value))
	check(enc.Encode(ac.ts))
	check(enc.Encode(ac.Hops))
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (ac *Command) GobDecode(b []byte) error {
	dec := gob.NewDecoder(bytes.NewBuffer(b))
	var err error
	check := func(er error) {
		if er != nil && err == nil {
			err = er
		}
	}
	check(dec.Decode(&ac.cmd))
	check(dec.Decode(&ac.name))
	check(dec.Decode(&ac.value))
	check(dec.Decode(&ac.ts))
	check(dec.Decode(&ac.Hops))
	return err
}

func NewCommand(cmd AggCmd, name string, value float64) *Command {
	return &Command{cmd: cmd, name: name, value: value, ts: time.Now()}
}
<|endoftext|>


<commit_before>// Package ancestrymanager provides an interface to query the ancestry information for a project.
package ancestrymanager

import (
	"context"
	"fmt"
	"log"
	"strings"

	"github.com/pkg/errors"
	"google.golang.org/api/cloudresourcemanager/v1"
	"google.golang.org/api/option"

	converter "github.com/GoogleCloudPlatform/terraform-google-conversion/google"
)

// AncestryManager is the interface that wraps the GetAncestry method.
type AncestryManager interface {
	// GetAncestry takes a project name to return an ancestry path
	GetAncestry(project string) (string, error)
	// GetAncestry takes a project name and resource data to return an ancestry path
	GetAncestryWithResource(project string, tfData converter.TerraformResourceData, cai converter.Asset) (string, error)
}

// resourceAncestryManager provides common methods for retrieving ancestry from resources
type resourceAncestryManager struct {
}

func (m *resourceAncestryManager) getFolderAncestry(folder_id string) ([]*cloudresourcemanager.Ancestor, error) {
	// TODO(morgantep): Incorporate folders.GetAncestry from v2alpha1 API
	log.Printf("[INFO] Retrieve ancestry for folder: %s", folder_id)

	return []*cloudresourcemanager.Ancestor{
		&cloudresourcemanager.Ancestor{
			ResourceId: &cloudresourcemanager.ResourceId{
				Type: "folder",
				Id:   folder_id,
			},
		},
		&cloudresourcemanager.Ancestor{
			ResourceId: &cloudresourcemanager.ResourceId{
				Type: "organization",
				Id:   "unknown",
			},
		},
	}, nil
}

func (m *resourceAncestryManager) getAncestryFromResource(tfData converter.TerraformResourceData, cai converter.Asset) ([]*cloudresourcemanager.Ancestor, bool) {
	log.Printf("[INFO] Retrieving ancestry from resource (type=%s)", cai.Type)

	switch cai.Type {
	case "cloudresourcemanager.googleapis.com/Project", "cloudbilling.googleapis.com/ProjectBillingInfo":
		ancestry := []*cloudresourcemanager.Ancestor{
			&cloudresourcemanager.Ancestor{
				ResourceId: &cloudresourcemanager.ResourceId{
					Type: "project",
					Id:   cai.Resource.Data["projectId"].(string),
				},
			},
		}

		org_id, ok := tfData.GetOk("org_id")
		if ok && org_id != "" {
			s := strings.Split(org_id.(string), "/")
			return append(ancestry, &cloudresourcemanager.Ancestor{
				ResourceId: &cloudresourcemanager.ResourceId{
					Type: "organization",
					Id:   s[len(s)-1],
				},
			}), true
		}

		folder_id, ok := tfData.GetOk("folder_id")
		if ok && folder_id != "" {
			folderAncestry, err := m.getFolderAncestry(folder_id.(string))
			if err != nil {
				log.Printf("[ERROR] Failed to retrieve folder ancestry: %s", err)
				return nil, false
			}
			return append(ancestry, folderAncestry...), true
		}
		return nil, true
	default:
		log.Printf("[INFO] Resource of type %s does not include sufficient data for ancestry retrieval", cai.Type)
		return nil, false
	}
}

type onlineAncestryManager struct {
	resourceAncestryManager
	// Talk to GCP resource manager. This field would be nil in offline mode.
	resourceManager *cloudresourcemanager.Service
	// Cache to prevent multiple network calls for looking up the same project's ancestry
	// map[project]ancestryPath
	ancestryCache map[string]string
}

// GetAncestry uses the resource manager API to get ancestry paths for
// projects. It implements a cache because many resources share the same
// project.
func (m *onlineAncestryManager) GetAncestry(project string) (string, error) {
	if path, ok := m.ancestryCache[project]; ok {
		return path, nil
	}
	ancestry, err := m.resourceManager.Projects.GetAncestry(project, &cloudresourcemanager.GetAncestryRequest{}).Do()
	if err != nil {
		return "", err
	}
	path := ancestryPath(ancestry.Ancestor)
	m.store(project, path)
	return path, nil
}

// GetAncestryWithResource first attempts to get Ancestry from the API
// If that fails, it falls back to inspecting the resource.
func (m *onlineAncestryManager) GetAncestryWithResource(project string, tfData converter.TerraformResourceData, cai converter.Asset) (string, error) {
	path, err := m.GetAncestry(project)
	if path != "" {
		return path, nil
	}

	ancestry, ok := m.getAncestryFromResource(tfData, cai)
	if !ok {
		return "", err
	}
	path = ancestryPath(ancestry)
	log.Printf("[INFO] Retrieved ancestry for %s: %s", project, path)
	m.store(project, path)
	return path, nil
}

func (m *onlineAncestryManager) store(project, ancestry string) {
	if project != "" && ancestry != "" {
		m.ancestryCache[project] = ancestry
	}
}

type offlineAncestryManager struct {
	resourceAncestryManager
	project  string
	ancestry string
}

// GetAncestry returns the ancestry for the project. It returns an error if
// the project does not equal to the one provided during initialization.
func (m *offlineAncestryManager) GetAncestry(project string) (string, error) {
	if project != m.project {
		return "", fmt.Errorf("cannot fetch ancestry in offline mode")
	}
	return m.ancestry, nil
}

// GetAncestryWithResource first attempts to get Ancestry from the offline cache
// If that fails, it falls back to inspecting the resource.
func (m *offlineAncestryManager) GetAncestryWithResource(project string, tfData converter.TerraformResourceData, cai converter.Asset) (string, error) {
	path, err := m.GetAncestry(project)
	if path != "" {
		return path, nil
	}

	ancestry, ok := m.getAncestryFromResource(tfData, cai)
	if !ok {
		return "", err
	}
	path = ancestryPath(ancestry)
	log.Printf("[INFO] Retrieved ancestry for %s: %s", project, path)
	return path, nil
}

// New returns AncestryManager that can be used to fetch ancestry information for a project.
func New(ctx context.Context, project, ancestry string, offline bool, opts ...option.ClientOption) (AncestryManager, error) {
	if ancestry != "" {
		ancestry = fmt.Sprintf("%s/project/%s", ancestry, project)
	}
	if offline {
		return &offlineAncestryManager{project: project, ancestry: ancestry}, nil
	}
	am := &onlineAncestryManager{ancestryCache: map[string]string{}}
	am.store(project, ancestry)
	rm, err := cloudresourcemanager.NewService(ctx, opts...)
	if err != nil {
		return nil, errors.Wrap(err, "constructing resource manager client")
	}
	am.resourceManager = rm
	return am, nil
}

// ancestryPath composes a path containing organization/folder/project
// (i.e. "organization/my-org/folder/my-folder/project/my-prj").
func ancestryPath(as []*cloudresourcemanager.Ancestor) string {
	var path []string
	for i := len(as) - 1; i >= 0; i-- {
		path = append(path, as[i].ResourceId.Type, as[i].ResourceId.Id)
	}
	return strings.Join(path, "/")
}
<commit_msg>Fix nil pointer exception<commit_after>// Package ancestrymanager provides an interface to query the ancestry information for a project.
package ancestrymanager

import (
	"context"
	"fmt"
	"log"
	"strings"

	"github.com/pkg/errors"
	"google.golang.org/api/cloudresourcemanager/v1"
	"google.golang.org/api/option"

	converter "github.com/GoogleCloudPlatform/terraform-google-conversion/google"
)

// AncestryManager is the interface that wraps the GetAncestry method.
type AncestryManager interface {
	// GetAncestry takes a project name to return an ancestry path
	GetAncestry(project string) (string, error)
	// GetAncestry takes a project name and resource data to return an ancestry path
	GetAncestryWithResource(project string, tfData converter.TerraformResourceData, cai converter.Asset) (string, error)
}

// resourceAncestryManager provides common methods for retrieving ancestry from resources
type resourceAncestryManager struct {
}

func (m *resourceAncestryManager) getFolderAncestry(folder_id string) ([]*cloudresourcemanager.Ancestor, error) {
	// TODO(morgantep): Incorporate folders.GetAncestry from v2alpha1 API
	log.Printf("[INFO] Retrieve ancestry for folder: %s", folder_id)

	return []*cloudresourcemanager.Ancestor{
		&cloudresourcemanager.Ancestor{
			ResourceId: &cloudresourcemanager.ResourceId{
				Type: "folder",
				Id:   folder_id,
			},
		},
		&cloudresourcemanager.Ancestor{
			ResourceId: &cloudresourcemanager.ResourceId{
				Type: "organization",
				Id:   "unknown",
			},
		},
	}, nil
}

func (m *resourceAncestryManager) getAncestryFromResource(tfData converter.TerraformResourceData, cai converter.Asset) ([]*cloudresourcemanager.Ancestor, bool) {
	log.Printf("[INFO] Retrieving ancestry from resource (type=%s)", cai.Type)

	switch cai.Type {
	case "cloudresourcemanager.googleapis.com/Project", "cloudbilling.googleapis.com/ProjectBillingInfo":
		ancestry := []*cloudresourcemanager.Ancestor{
			&cloudresourcemanager.Ancestor{
				ResourceId: &cloudresourcemanager.ResourceId{
					Type: "project",
					Id:   cai.Resource.Data["projectId"].(string),
				},
			},
		}

		org_id, ok := tfData.GetOk("org_id")
		if ok && org_id != "" {
			s := strings.Split(org_id.(string), "/")
			return append(ancestry, &cloudresourcemanager.Ancestor{
				ResourceId: &cloudresourcemanager.ResourceId{
					Type: "organization",
					Id:   s[len(s)-1],
				},
			}), true
		}

		folder_id, ok := tfData.GetOk("folder_id")
		if ok && folder_id != "" {
			folderAncestry, err := m.getFolderAncestry(folder_id.(string))
			if err != nil {
				log.Printf("[ERROR] Failed to retrieve folder ancestry: %s", err)
				return nil, false
			}
			return append(ancestry, folderAncestry...), true
		}
		return nil, true
	default:
		log.Printf("[INFO] Resource of type %s does not include sufficient data for ancestry retrieval", cai.Type)
		return nil, false
	}
}

type onlineAncestryManager struct {
	resourceAncestryManager
	// Talk to GCP resource manager. This field would be nil in offline mode.
	resourceManager *cloudresourcemanager.Service
	// Cache to prevent multiple network calls for looking up the same project's ancestry
	// map[project]ancestryPath
	ancestryCache map[string]string
}

// GetAncestry uses the resource manager API to get ancestry paths for
// projects. It implements a cache because many resources share the same
// project.
func (m *onlineAncestryManager) GetAncestry(project string) (string, error) {
	if path, ok := m.ancestryCache[project]; ok {
		return path, nil
	}
	ancestry, err := m.resourceManager.Projects.GetAncestry(project, &cloudresourcemanager.GetAncestryRequest{}).Do()
	if err != nil {
		return "", err
	}
	path := ancestryPath(ancestry.Ancestor)
	m.store(project, path)
	return path, nil
}

// GetAncestryWithResource first attempts to get Ancestry from the API
// If that fails, it falls back to inspecting the resource.
func (m *onlineAncestryManager) GetAncestryWithResource(project string, tfData converter.TerraformResourceData, cai converter.Asset) (string, error) {
	path, err := m.GetAncestry(project)
	if path != "" {
		return path, nil
	}

	ancestry, ok := m.getAncestryFromResource(tfData, cai)
	if !ok {
		return "", err
	}
	path = ancestryPath(ancestry)
	log.Printf("[INFO] Retrieved ancestry for %s: %s", project, path)
	m.store(project, path)
	return path, nil
}

func (m *onlineAncestryManager) store(project, ancestry string) {
	if project != "" && ancestry != "" {
		m.ancestryCache[project] = ancestry
	}
}

type offlineAncestryManager struct {
	resourceAncestryManager
	project  string
	ancestry string
}

// GetAncestry returns the ancestry for the project. It returns an error if
// the project does not equal to the one provided during initialization.
func (m *offlineAncestryManager) GetAncestry(project string) (string, error) {
	if project != m.project {
		return "", fmt.Errorf("cannot fetch ancestry in offline mode")
	}
	return m.ancestry, nil
}

// GetAncestryWithResource first attempts to get Ancestry from the resource
// If that fails, it falls back to the offline cache.
func (m *offlineAncestryManager) GetAncestryWithResource(project string, tfData converter.TerraformResourceData, cai converter.Asset) (string, error) {
	ancestry, ok := m.getAncestryFromResource(tfData, cai)
	if ok {
		path := ancestryPath(ancestry)
		log.Printf("[INFO] Retrieved ancestry for %s: %s", project, path)
		return path, nil
	}

	path, err := m.GetAncestry(project)
	if err != nil {
		return "", err
	}

	return path, nil
}

// New returns AncestryManager that can be used to fetch ancestry information for a project.
func New(ctx context.Context, project, ancestry string, offline bool, opts ...option.ClientOption) (AncestryManager, error) {
	if ancestry != "" {
		ancestry = fmt.Sprintf("%s/project/%s", ancestry, project)
	}
	if offline {
		return &offlineAncestryManager{project: project, ancestry: ancestry}, nil
	}
	am := &onlineAncestryManager{ancestryCache: map[string]string{}}
	am.store(project, ancestry)
	rm, err := cloudresourcemanager.NewService(ctx, opts...)
	if err != nil {
		return nil, errors.Wrap(err, "constructing resource manager client")
	}
	am.resourceManager = rm
	return am, nil
}

// ancestryPath composes a path containing organization/folder/project
// (i.e. "organization/my-org/folder/my-folder/project/my-prj").
func ancestryPath(as []*cloudresourcemanager.Ancestor) string {
	var path []string
	for i := len(as) - 1; i >= 0; i-- {
		path = append(path, as[i].ResourceId.Type, as[i].ResourceId.Id)
	}
	return strings.Join(path, "/")
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"testing"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/open-lambda/open-lambda/worker/config"
	"github.com/open-lambda/open-lambda/worker/dockerutil"
)

var server *Server
var docker_client *docker.Client

func init() {
	server = RunServer()
	var err error
	docker_client, err = docker.NewClientFromEnv()
	if err != nil {
		log.Fatal("failed to get docker client: ", err)
	}
}

func RunServer() *Server {
	conf, err := config.ParseConfig(os.Getenv("WORKER_CONFIG"))
	if err != nil {
		log.Fatal(err)
	}

	log.Printf(conf.Registry)
	log.Printf("Set skip_pull_existing = true\n")
	conf.Skip_pull_existing = true

	server, err := NewServer(conf)
	if err != nil {
		log.Fatal(err)
	}

	listener, err := net.Listen("tcp", ":8080")
	if err != nil {
		log.Fatal(err)
	}
	mux := http.NewServeMux()
	mux.HandleFunc("/runLambda/", server.RunLambda)
	s := &http.Server{Addr: ":8080", Handler: mux}
	go func() {
		log.Fatal(s.Serve(listener))
	}()

	return server
}

func testReq(lambda_name string, post string) (string, error) {
	url := "http://localhost:8080/runLambda/" + lambda_name
	req, err := http.NewRequest("POST", url, strings.NewReader(post))
	if err != nil {
		return "", err
	}
	req.Header.Set("Content-Type", "application/json")
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

func kill() {
	containers, err := docker_client.ListContainers(docker.ListContainersOptions{})
	if err != nil {
		log.Fatal("failed to get docker container list: ", err)
	}

	for _, container := range containers {
		if container.Labels[dockerutil.DOCKER_LABEL_CLUSTER] == server.config.Cluster_name {
			cid := container.ID
			typ := server.config.Cluster_name

			container_insp, err := docker_client.InspectContainer(cid)
			if err != nil {
				log.Fatalf("failed to get inspect docker container ID %v: ", cid, err)
			}

			if container_insp.State.Paused {
				fmt.Printf("Unpause container %v (%s)\n", cid, typ)
				if err := docker_client.UnpauseContainer(cid); err != nil {
					fmt.Printf("%s\n", err.Error())
					fmt.Printf("Failed to unpause container %v (%s).  May require manual cleanup.\n", cid, typ)
				}
			}

			fmt.Printf("Kill container %v (%s)\n", cid, typ)
			opts := docker.KillContainerOptions{ID: cid}
			if err := docker_client.KillContainer(opts); err != nil {
				fmt.Printf("%s\n", err.Error())
				fmt.Printf("Failed to kill container %v (%s).  May require manual cleanup.\n", cid, typ)
			}
		}
	}

}

func TestMain(m *testing.M) {
	ret_val := m.Run()
	fmt.Printf("\n========Cleaning========\n")
	kill()
	fmt.Printf("========================\n\n")
	os.Exit(ret_val)
}

func TestHello(t *testing.T) {
	recv, err := testReq("hello", "{}")
	if err != nil {
		t.Fatal(err)
	}
	expected := "\"hello\""
	if recv != expected {
		t.Fatalf("Expected '%v' from hello but got back '%v'\n", expected, recv)
	}
}

func TestEcho(t *testing.T) {
	values := []string{
		"{}",
		"{\"one\": 1}",
		"1",
		"\"test\"",
	}
	for _, send := range values {
		recv, err := testReq("echo", send)
		if err != nil {
			t.Fatal(err)
		}
		if recv != send {
			t.Fatalf("Sent '%v' to echo but got back '%v'\n", send, recv)
		}
	}
}

func last_count(img string) (int, error) {
	logs, err := server.handlers.Get(img).Sandbox().Logs()
	if err != nil {
		return 0, err
	}

	lines := strings.Split(logs, "\n")
	for i := range lines {
		line := lines[len(lines)-i-1]
		parts := strings.Split(line, "=")
		if parts[0] == "counter" {
			if n, err := strconv.Atoi(parts[1]); err == nil {
				return n, nil
			} else {
				return 0, err
			}
		}
	}

	return 0, nil
}

// thread_counter starts a backup thread that runs forever,
// incrementing a counter between 10ms sleeps.  If pausing works, the
// counter won't tick many times between requests, even if wait
// between them.
func TestThreadPausing(t *testing.T) {
	img := "thread_counter"
	testReq(img, "null")

	count1, err := last_count(img)
	if err != nil {
		t.Fatalf("Failed to get first count with: %v", err)
	}

	time.Sleep(100 * time.Millisecond)

	count2, err := last_count(img)
	if err != nil {
		t.Fatalf("Failed to get second count with: %v", err)
	}

	if count1 <= 0 {
		t.Fatalf("count1 isn't positive (%d) - logs working?", count1)
	}
	if count2 != count1 {
		t.Fatalf("count1 (%d) != count2 (%d)", count1, count2)
	}
}

func BenchmarkEcho(b *testing.B) {
	values := []string{
		"{\"one\": 1}",
	}
	for i := 0; i < b.N; i++ {
		recv, err := testReq("echo", values[0])
		if err != nil {
			b.Fatal(err)
		}
		if recv != values[0] {
			b.Fatalf("Sent '%v' to echo but got back '%v'\n", values[0], recv)
		}
	}
}

func BenchmarkEchoParallel(b *testing.B) {
	values := []string{
		"{\"one\": 1}",
	}
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			recv, err := testReq("echo", values[0])
			if err != nil {
				b.Fatal(err)
			}
			if recv != values[0] {
				b.Fatalf("Sent '%v' to echo but got back '%v'\n", values[0], recv)
			}
		}
	})
}
<commit_msg>remove containers during cleanup<commit_after>package server

import (
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"testing"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/open-lambda/open-lambda/worker/config"
	"github.com/open-lambda/open-lambda/worker/dockerutil"
)

var server *Server
var docker_client *docker.Client

func init() {
	server = RunServer()
	var err error
	docker_client, err = docker.NewClientFromEnv()
	if err != nil {
		log.Fatal("failed to get docker client: ", err)
	}
}

func RunServer() *Server {
	conf, err := config.ParseConfig(os.Getenv("WORKER_CONFIG"))
	if err != nil {
		log.Fatal(err)
	}

	log.Printf(conf.Registry)
	log.Printf("Set skip_pull_existing = true\n")
	conf.Skip_pull_existing = true

	server, err := NewServer(conf)
	if err != nil {
		log.Fatal(err)
	}

	listener, err := net.Listen("tcp", ":8080")
	if err != nil {
		log.Fatal(err)
	}
	mux := http.NewServeMux()
	mux.HandleFunc("/runLambda/", server.RunLambda)
	s := &http.Server{Addr: ":8080", Handler: mux}
	go func() {
		log.Fatal(s.Serve(listener))
	}()

	return server
}

func testReq(lambda_name string, post string) (string, error) {
	url := "http://localhost:8080/runLambda/" + lambda_name
	req, err := http.NewRequest("POST", url, strings.NewReader(post))
	if err != nil {
		return "", err
	}
	req.Header.Set("Content-Type", "application/json")
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

func kill() {
	containers, err := docker_client.ListContainers(docker.ListContainersOptions{})
	if err != nil {
		log.Fatal("failed to get docker container list: ", err)
	}

	for _, container := range containers {
		if container.Labels[dockerutil.DOCKER_LABEL_CLUSTER] == server.config.Cluster_name {
			cid := container.ID
			typ := server.config.Cluster_name

			container_insp, err := docker_client.InspectContainer(cid)
			if err != nil {
				log.Fatalf("failed to get inspect docker container ID %v: ", cid, err)
			}

			if container_insp.State.Paused {
				fmt.Printf("Unpause container %v (%s)\n", cid, typ)
				if err := docker_client.UnpauseContainer(cid); err != nil {
					fmt.Printf("%s\n", err.Error())
					fmt.Printf("Failed to unpause container %v (%s).  May require manual cleanup.\n", cid, typ)
				}
			}

			fmt.Printf("Kill container %v (%s)\n", cid, typ)
			killopts := docker.KillContainerOptions{ID: cid}
			if err := docker_client.KillContainer(killopts); err != nil {
				fmt.Printf("%s\n", err.Error())
				fmt.Printf("Failed to kill container %v (%s).  May require manual cleanup.\n", cid, typ)
			}

			fmt.Printf("Remove container %v (%s)\n", cid, typ)
			rmopts := docker.RemoveContainerOptions{ID: cid}
			if err := docker_client.RemoveContainer(rmopts); err != nil {
				fmt.Printf("%s\n", err.Error())
				fmt.Printf("Failed to remove container %v (%s).  May require manual cleanup.\n", cid, typ)
			}
		}
	}

}

func TestMain(m *testing.M) {
	ret_val := m.Run()
	fmt.Printf("\n========Cleaning========\n")
	kill()
	fmt.Printf("========================\n\n")
	os.Exit(ret_val)
}

func TestHello(t *testing.T) {
	recv, err := testReq("hello", "{}")
	if err != nil {
		t.Fatal(err)
	}
	expected := "\"hello\""
	if recv != expected {
		t.Fatalf("Expected '%v' from hello but got back '%v'\n", expected, recv)
	}
}

func TestEcho(t *testing.T) {
	values := []string{
		"{}",
		"{\"one\": 1}",
		"1",
		"\"test\"",
	}
	for _, send := range values {
		recv, err := testReq("echo", send)
		if err != nil {
			t.Fatal(err)
		}
		if recv != send {
			t.Fatalf("Sent '%v' to echo but got back '%v'\n", send, recv)
		}
	}
}

func last_count(img string) (int, error) {
	logs, err := server.handlers.Get(img).Sandbox().Logs()
	if err != nil {
		return 0, err
	}

	lines := strings.Split(logs, "\n")
	for i := range lines {
		line := lines[len(lines)-i-1]
		parts := strings.Split(line, "=")
		if parts[0] == "counter" {
			if n, err := strconv.Atoi(parts[1]); err == nil {
				return n, nil
			} else {
				return 0, err
			}
		}
	}

	return 0, nil
}

// thread_counter starts a backup thread that runs forever,
// incrementing a counter between 10ms sleeps.  If pausing works, the
// counter won't tick many times between requests, even if wait
// between them.
func TestThreadPausing(t *testing.T) {
	img := "thread_counter"
	testReq(img, "null")

	count1, err := last_count(img)
	if err != nil {
		t.Fatalf("Failed to get first count with: %v", err)
	}

	time.Sleep(100 * time.Millisecond)

	count2, err := last_count(img)
	if err != nil {
		t.Fatalf("Failed to get second count with: %v", err)
	}

	if count1 <= 0 {
		t.Fatalf("count1 isn't positive (%d) - logs working?", count1)
	}
	if count2 != count1 {
		t.Fatalf("count1 (%d) != count2 (%d)", count1, count2)
	}
}

func BenchmarkEcho(b *testing.B) {
	values := []string{
		"{\"one\": 1}",
	}
	for i := 0; i < b.N; i++ {
		recv, err := testReq("echo", values[0])
		if err != nil {
			b.Fatal(err)
		}
		if recv != values[0] {
			b.Fatalf("Sent '%v' to echo but got back '%v'\n", values[0], recv)
		}
	}
}

func BenchmarkEchoParallel(b *testing.B) {
	values := []string{
		"{\"one\": 1}",
	}
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			recv, err := testReq("echo", values[0])
			if err != nil {
				b.Fatal(err)
			}
			if recv != values[0] {
				b.Fatalf("Sent '%v' to echo but got back '%v'\n", values[0], recv)
			}
		}
	})
}
<|endoftext|>