<commit_before>package gps

import (
	"crypto/sha256"
	"fmt"
	"sort"
)

// HashInputs computes a hash digest of all data in a SolveOpts that are as
// function inputs to Solve().
//
// The digest returned from this function is the same as the digest that would
// be included with a Solve() Result. As such, it's appropriate for comparison
// against the digest stored in a lock file, generated by a previous Solve(): if
// the digests match, then manifest and lock are in sync, and a Solve() is
// unnecessary.
//
// (Basically, this is for memoization.)
func (s *solver) HashInputs() ([]byte, error) {
	// Do these checks up front before any other work is needed, as they're the
	// only things that can cause errors
	// Pass in magic root values, and the bridge will analyze the right thing
	ptree, err := s.b.listPackages(ProjectIdentifier{ProjectRoot: s.params.ImportRoot}, nil)
	if err != nil {
		return nil, badOptsFailure(fmt.Sprintf("Error while parsing packages under %s: %s", s.params.RootDir, err.Error()))
	}

	d, dd := s.params.Manifest.DependencyConstraints(), s.params.Manifest.TestDependencyConstraints()
	p := make(sortedDeps, len(d))
	copy(p, d)
	p = append(p, dd...)

	sort.Stable(p)

	// We have everything we need; now, compute the hash.
	h := sha256.New()
	for _, pd := range p {
		h.Write([]byte(pd.Ident.ProjectRoot))
		h.Write([]byte(pd.Ident.NetworkName))
		// FIXME Constraint.String() is a surjective-only transformation - tags
		// and branches with the same name are written out as the same string.
		// This could, albeit rarely, result in input collisions when a real
		// change has occurred.
		h.Write([]byte(pd.Constraint.String()))
	}

	// The stdlib and old appengine packages play the same functional role in
	// solving as ignores. Because they change, albeit quite infrequently, we
	// have to include them in the hash.
	h.Write([]byte(stdlibPkgs))
	h.Write([]byte(appenginePkgs))

	// Write each of the packages, or the errors that were found for a
	// particular subpath, into the hash.
	for _, perr := range ptree.Packages {
		if perr.Err != nil {
			h.Write([]byte(perr.Err.Error()))
		} else {
			h.Write([]byte(perr.P.Name))
			h.Write([]byte(perr.P.CommentPath))
			h.Write([]byte(perr.P.ImportPath))
			for _, imp := range perr.P.Imports {
				h.Write([]byte(imp))
			}
			for _, imp := range perr.P.TestImports {
				h.Write([]byte(imp))
			}
		}
	}

	// Add the package ignores, if any.
	if len(s.ig) > 0 {
		// Dump and sort the ignores
		ig := make([]string, len(s.ig))
		k := 0
		for pkg := range s.ig {
			ig[k] = pkg
			k++
		}
		sort.Strings(ig)

		for _, igp := range ig {
			h.Write([]byte(igp))
		}
	}

	an, av := s.b.analyzerInfo()
	h.Write([]byte(an))
	h.Write([]byte(av.String()))

	// TODO(sdboyer) overrides
	// TODO(sdboyer) aliases
	return h.Sum(nil), nil
}

type sortedDeps []ProjectConstraint

func (s sortedDeps) Len() int {
	return len(s)
}

func (s sortedDeps) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s sortedDeps) Less(i, j int) bool {
	return s[i].Ident.less(s[j].Ident)
}
<commit_msg>s/sortedDeps/sortedConstraints/<commit_after>package gps

import (
	"crypto/sha256"
	"fmt"
	"sort"
)

// HashInputs computes a hash digest of all data in a SolveOpts that are as
// function inputs to Solve().
//
// The digest returned from this function is the same as the digest that would
// be included with a Solve() Result. As such, it's appropriate for comparison
// against the digest stored in a lock file, generated by a previous Solve(): if
// the digests match, then manifest and lock are in sync, and a Solve() is
// unnecessary.
//
// (Basically, this is for memoization.)
func (s *solver) HashInputs() ([]byte, error) {
	// Do these checks up front before any other work is needed, as they're the
	// only things that can cause errors
	// Pass in magic root values, and the bridge will analyze the right thing
	ptree, err := s.b.listPackages(ProjectIdentifier{ProjectRoot: s.params.ImportRoot}, nil)
	if err != nil {
		return nil, badOptsFailure(fmt.Sprintf("Error while parsing packages under %s: %s", s.params.RootDir, err.Error()))
	}

	d, dd := s.params.Manifest.DependencyConstraints(), s.params.Manifest.TestDependencyConstraints()
	p := make(sortedConstraints, len(d))
	copy(p, d)
	p = append(p, dd...)

	sort.Stable(p)

	// We have everything we need; now, compute the hash.
	h := sha256.New()
	for _, pd := range p {
		h.Write([]byte(pd.Ident.ProjectRoot))
		h.Write([]byte(pd.Ident.NetworkName))
		// FIXME Constraint.String() is a surjective-only transformation - tags
		// and branches with the same name are written out as the same string.
		// This could, albeit rarely, result in input collisions when a real
		// change has occurred.
		h.Write([]byte(pd.Constraint.String()))
	}

	// The stdlib and old appengine packages play the same functional role in
	// solving as ignores. Because they change, albeit quite infrequently, we
	// have to include them in the hash.
	h.Write([]byte(stdlibPkgs))
	h.Write([]byte(appenginePkgs))

	// Write each of the packages, or the errors that were found for a
	// particular subpath, into the hash.
	for _, perr := range ptree.Packages {
		if perr.Err != nil {
			h.Write([]byte(perr.Err.Error()))
		} else {
			h.Write([]byte(perr.P.Name))
			h.Write([]byte(perr.P.CommentPath))
			h.Write([]byte(perr.P.ImportPath))
			for _, imp := range perr.P.Imports {
				h.Write([]byte(imp))
			}
			for _, imp := range perr.P.TestImports {
				h.Write([]byte(imp))
			}
		}
	}

	// Add the package ignores, if any.
	if len(s.ig) > 0 {
		// Dump and sort the ignores
		ig := make([]string, len(s.ig))
		k := 0
		for pkg := range s.ig {
			ig[k] = pkg
			k++
		}
		sort.Strings(ig)

		for _, igp := range ig {
			h.Write([]byte(igp))
		}
	}

	an, av := s.b.analyzerInfo()
	h.Write([]byte(an))
	h.Write([]byte(av.String()))

	// TODO(sdboyer) overrides
	// TODO(sdboyer) aliases
	return h.Sum(nil), nil
}

type sortedConstraints []ProjectConstraint

func (s sortedConstraints) Len() int {
	return len(s)
}

func (s sortedConstraints) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s sortedConstraints) Less(i, j int) bool {
	return s[i].Ident.less(s[j].Ident)
}
<|endoftext|>


<commit_before>package gps

import (
	"bytes"
	"crypto/sha256"
	"io"
	"sort"
)

// HashInputs computes a hash digest of all data in SolveParams and the
// RootManifest that act as function inputs to Solve().
//
// The digest returned from this function is the same as the digest that would
// be included with a Solve() Result. As such, it's appropriate for comparison
// against the digest stored in a lock file, generated by a previous Solve(): if
// the digests match, then manifest and lock are in sync, and a Solve() is
// unnecessary.
//
// (Basically, this is for memoization.)
func (s *solver) HashInputs() (digest []byte) {
	h := sha256.New()
	s.writeHashingInputs(h)

	hd := h.Sum(nil)
	digest = hd[:]
	return
}

func (s *solver) writeHashingInputs(w io.Writer) {
	writeString := func(s string) {
		// Skip zero-length string writes; it doesn't affect the real hash
		// calculation, and keeps misleading newlines from showing up in the
		// debug output.
		if s != "" {
			// All users of writeHashingInputs cannot error on Write(), so just
			// ignore it
			w.Write([]byte(s))
		}
	}

	// Apply overrides to the constraints from the root. Otherwise, the hash
	// would be computed on the basis of a constraint from root that doesn't
	// actually affect solving.
	wc := s.ovr.overrideAll(s.rm.DependencyConstraints().merge(s.rm.TestDependencyConstraints()))

	for _, pd := range wc {
		writeString(string(pd.Ident.ProjectRoot))
		writeString(pd.Ident.Source)
		// FIXME Constraint.String() is a surjective-only transformation - tags
		// and branches with the same name are written out as the same string.
		// This could, albeit rarely, result in erroneously identical inputs
		// when a real change has occurred.
		writeString(pd.Constraint.String())
	}

	// Get the external reach list

	// Write each of the packages, or the errors that were found for a
	// particular subpath, into the hash. We need to do this in a
	// deterministic order, so expand and sort the map.
	var pkgs []PackageOrErr
	for _, perr := range s.rpt.Packages {
		pkgs = append(pkgs, perr)
	}
	sort.Sort(sortPackageOrErr(pkgs))
	for _, perr := range pkgs {
		if perr.Err != nil {
			writeString(perr.Err.Error())
		} else {
			writeString(perr.P.Name)
			writeString(perr.P.CommentPath)
			writeString(perr.P.ImportPath)
			for _, imp := range perr.P.Imports {
				if !isStdLib(imp) {
					writeString(imp)
				}
			}
			for _, imp := range perr.P.TestImports {
				if !isStdLib(imp) {
					writeString(imp)
				}
			}
		}
	}

	// Write any require packages given in the root manifest.
	if len(s.req) > 0 {
		// Dump and sort the reqnores
		req := make([]string, 0, len(s.req))
		for pkg := range s.req {
			req = append(req, pkg)
		}
		sort.Strings(req)

		for _, reqp := range req {
			writeString(reqp)
		}
	}

	// Add the ignored packages, if any.
	if len(s.ig) > 0 {
		// Dump and sort the ignores
		ig := make([]string, 0, len(s.ig))
		for pkg := range s.ig {
			ig = append(ig, pkg)
		}
		sort.Strings(ig)

		for _, igp := range ig {
			writeString(igp)
		}
	}

	for _, pc := range s.ovr.asSortedSlice() {
		writeString(string(pc.Ident.ProjectRoot))
		if pc.Ident.Source != "" {
			writeString(pc.Ident.Source)
		}
		if pc.Constraint != nil {
			writeString(pc.Constraint.String())
		}
	}

	an, av := s.b.AnalyzerInfo()
	writeString(an)
	writeString(av.String())
}

// bytes.Buffer wrapper that injects newlines after each call to Write().
type nlbuf bytes.Buffer

func (buf *nlbuf) Write(p []byte) (n int, err error) {
	n, _ = (*bytes.Buffer)(buf).Write(p)
	(*bytes.Buffer)(buf).WriteByte('\n')
	return n + 1, nil
}

// HashingInputsAsString returns the raw input data used by Solver.HashInputs()
// as a string.
//
// This is primarily intended for debugging purposes.
func HashingInputsAsString(s Solver) string {
	ts := s.(*solver)
	buf := new(nlbuf)
	ts.writeHashingInputs(buf)

	return (*bytes.Buffer)(buf).String()
}

type sortPackageOrErr []PackageOrErr

func (s sortPackageOrErr) Len() int      { return len(s) }
func (s sortPackageOrErr) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

func (s sortPackageOrErr) Less(i, j int) bool {
	a, b := s[i], s[j]
	if a.Err != nil || b.Err != nil {
		// Sort errors last.
		if b.Err == nil {
			return false
		}
		if a.Err == nil {
			return true
		}
		// And then by string.
		return a.Err.Error() < b.Err.Error()
	}
	// And finally, sort by import path.
	return a.P.ImportPath < b.P.ImportPath
}
<commit_msg>Remove pointless ifs<commit_after>package gps

import (
	"bytes"
	"crypto/sha256"
	"io"
	"sort"
)

// HashInputs computes a hash digest of all data in SolveParams and the
// RootManifest that act as function inputs to Solve().
//
// The digest returned from this function is the same as the digest that would
// be included with a Solve() Result. As such, it's appropriate for comparison
// against the digest stored in a lock file, generated by a previous Solve(): if
// the digests match, then manifest and lock are in sync, and a Solve() is
// unnecessary.
//
// (Basically, this is for memoization.)
func (s *solver) HashInputs() (digest []byte) {
	h := sha256.New()
	s.writeHashingInputs(h)

	hd := h.Sum(nil)
	digest = hd[:]
	return
}

func (s *solver) writeHashingInputs(w io.Writer) {
	writeString := func(s string) {
		// Skip zero-length string writes; it doesn't affect the real hash
		// calculation, and keeps misleading newlines from showing up in the
		// debug output.
		if s != "" {
			// All users of writeHashingInputs cannot error on Write(), so just
			// ignore it
			w.Write([]byte(s))
		}
	}

	// Apply overrides to the constraints from the root. Otherwise, the hash
	// would be computed on the basis of a constraint from root that doesn't
	// actually affect solving.
	wc := s.ovr.overrideAll(s.rm.DependencyConstraints().merge(s.rm.TestDependencyConstraints()))

	for _, pd := range wc {
		writeString(string(pd.Ident.ProjectRoot))
		writeString(pd.Ident.Source)
		// FIXME Constraint.String() is a surjective-only transformation - tags
		// and branches with the same name are written out as the same string.
		// This could, albeit rarely, result in erroneously identical inputs
		// when a real change has occurred.
		writeString(pd.Constraint.String())
	}

	// Get the external reach list

	// Write each of the packages, or the errors that were found for a
	// particular subpath, into the hash. We need to do this in a
	// deterministic order, so expand and sort the map.
	var pkgs []PackageOrErr
	for _, perr := range s.rpt.Packages {
		pkgs = append(pkgs, perr)
	}
	sort.Sort(sortPackageOrErr(pkgs))
	for _, perr := range pkgs {
		if perr.Err != nil {
			writeString(perr.Err.Error())
		} else {
			writeString(perr.P.Name)
			writeString(perr.P.CommentPath)
			writeString(perr.P.ImportPath)
			for _, imp := range perr.P.Imports {
				if !isStdLib(imp) {
					writeString(imp)
				}
			}
			for _, imp := range perr.P.TestImports {
				if !isStdLib(imp) {
					writeString(imp)
				}
			}
		}
	}

	// Write any required packages given in the root manifest.
	req := make([]string, 0, len(s.req))
	for pkg := range s.req {
		req = append(req, pkg)
	}
	sort.Strings(req)

	for _, reqp := range req {
		writeString(reqp)
	}

	// Add the ignored packages, if any.
	ig := make([]string, 0, len(s.ig))
	for pkg := range s.ig {
		ig = append(ig, pkg)
	}
	sort.Strings(ig)

	for _, igp := range ig {
		writeString(igp)
	}

	for _, pc := range s.ovr.asSortedSlice() {
		writeString(string(pc.Ident.ProjectRoot))
		if pc.Ident.Source != "" {
			writeString(pc.Ident.Source)
		}
		if pc.Constraint != nil {
			writeString(pc.Constraint.String())
		}
	}

	an, av := s.b.AnalyzerInfo()
	writeString(an)
	writeString(av.String())
}

// bytes.Buffer wrapper that injects newlines after each call to Write().
type nlbuf bytes.Buffer

func (buf *nlbuf) Write(p []byte) (n int, err error) {
	n, _ = (*bytes.Buffer)(buf).Write(p)
	(*bytes.Buffer)(buf).WriteByte('\n')
	return n + 1, nil
}

// HashingInputsAsString returns the raw input data used by Solver.HashInputs()
// as a string.
//
// This is primarily intended for debugging purposes.
func HashingInputsAsString(s Solver) string {
	ts := s.(*solver)
	buf := new(nlbuf)
	ts.writeHashingInputs(buf)

	return (*bytes.Buffer)(buf).String()
}

type sortPackageOrErr []PackageOrErr

func (s sortPackageOrErr) Len() int      { return len(s) }
func (s sortPackageOrErr) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

func (s sortPackageOrErr) Less(i, j int) bool {
	a, b := s[i], s[j]
	if a.Err != nil || b.Err != nil {
		// Sort errors last.
		if b.Err == nil {
			return false
		}
		if a.Err == nil {
			return true
		}
		// And then by string.
		return a.Err.Error() < b.Err.Error()
	}
	// And finally, sort by import path.
	return a.P.ImportPath < b.P.ImportPath
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package examples_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/golang/glog"
)

func validateObject(obj runtime.Object) (errors []error) {
	ctx := api.NewDefaultContext()
	switch t := obj.(type) {
	case *api.ReplicationController:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidateReplicationController(t)
	case *api.ReplicationControllerList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Service:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		api.ValidNamespace(ctx, &t.ObjectMeta)
		errors = validation.ValidateService(t)
	case *api.ServiceList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Pod:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		api.ValidNamespace(ctx, &t.ObjectMeta)
		errors = validation.ValidatePod(t)
	case *api.PodList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	default:
		return []error{fmt.Errorf("no validation defined for %#v", obj)}
	}
	return errors
}

func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error {
	err := filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() && path != inDir {
			return filepath.SkipDir
		}
		name := filepath.Base(path)
		ext := filepath.Ext(name)
		if ext != "" {
			name = name[:len(name)-len(ext)]
		}
		if !(ext == ".json" || ext == ".yaml") {
			return nil
		}
		glog.Infof("Testing %s", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		fn(name, path, data)
		return nil
	})
	return err
}

func TestExampleObjectSchemas(t *testing.T) {
	cases := map[string]map[string]runtime.Object{
		"../api/examples": {
			"controller":       &api.ReplicationController{},
			"controller-list":  &api.ReplicationControllerList{},
			"pod":              &api.Pod{},
			"pod-list":         &api.PodList{},
			"service":          &api.Service{},
			"external-service": &api.Service{},
			"service-list":     &api.ServiceList{},
		},
		"../examples/guestbook": {
			"frontend-controller":    &api.ReplicationController{},
			"redis-slave-controller": &api.ReplicationController{},
			"redis-master":           &api.Pod{},
			"frontend-service":       &api.Service{},
			"redis-master-service":   &api.Service{},
			"redis-slave-service":    &api.Service{},
		},
		"../examples/guestbook/v3_json_files": {
			"frontend-controller":    &api.ReplicationController{},
			"redis-slave-controller": &api.ReplicationController{},
			"redis-master":           &api.ReplicationController{},
			"frontend-service":       &api.Service{},
			"redis-master-service":   &api.Service{},
			"redis-slave-service":    &api.Service{},
		},
		"../examples/walkthrough": {
			"pod1": &api.Pod{},
			"pod2": &api.Pod{},
			"pod-with-http-healthcheck": &api.Pod{},
			"service":                   &api.Service{},
			"replication-controller":    &api.ReplicationController{},
		},
		"../examples/update-demo": {
			"kitten-rc":   &api.ReplicationController{},
			"nautilus-rc": &api.ReplicationController{},
		},
	}

	for path, expected := range cases {
		tested := 0
		err := walkJSONFiles(path, func(name, path string, data []byte) {
			expectedType, found := expected[name]
			if !found {
				t.Errorf("%s does not have a test case defined", path)
				return
			}
			tested += 1
			if err := latest.Codec.DecodeInto(data, expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
				return
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
		})
		if err != nil {
			t.Errorf("Expected no error, Got %v", err)
		}
		if tested != len(expected) {
			t.Errorf("Expected %d examples, Got %d", len(expected), tested)
		}
	}
}

var sampleRegexp = regexp.MustCompile("(?ms)^```(?:(?P<type>yaml)\\w*\\n(?P<content>.+?)|\\w*\\n(?P<content>\\{.+?\\}))\\w*\\n^```")
var subsetRegexp = regexp.MustCompile("(?ms)\\.{3}")

func TestReadme(t *testing.T) {
	paths := []string{
		"../README.md",
		"../examples/walkthrough/README.md",
	}

	for _, path := range paths {
		data, err := ioutil.ReadFile(path)
		if err != nil {
			t.Errorf("Unable to read file %s: %v", path, err)
			continue
		}

		matches := sampleRegexp.FindAllStringSubmatch(string(data), -1)
		if matches == nil {
			continue
		}
		for _, match := range matches {
			var content, subtype string
			for i, name := range sampleRegexp.SubexpNames() {
				if name == "type" {
					subtype = match[i]
				}
				if name == "content" && match[i] != "" {
					content = match[i]
				}
			}
			if subtype == "yaml" && subsetRegexp.FindString(content) != "" {
				t.Logf("skipping (%s): \n%s", subtype, content)
				continue
			}

			//t.Logf("testing (%s): \n%s", subtype, content)
			expectedType := &api.Pod{}
			if err := latest.Codec.DecodeInto([]byte(content), expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(content))
				continue
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
			_, err := latest.Codec.Encode(expectedType)
			if err != nil {
				t.Errorf("Could not encode object: %v", err)
				continue
			}
		}
	}
}
<commit_msg>updating testcases<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package examples_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/golang/glog"
)

func validateObject(obj runtime.Object) (errors []error) {
	ctx := api.NewDefaultContext()
	switch t := obj.(type) {
	case *api.ReplicationController:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidateReplicationController(t)
	case *api.ReplicationControllerList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Service:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		api.ValidNamespace(ctx, &t.ObjectMeta)
		errors = validation.ValidateService(t)
	case *api.ServiceList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Pod:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		api.ValidNamespace(ctx, &t.ObjectMeta)
		errors = validation.ValidatePod(t)
	case *api.PodList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	default:
		return []error{fmt.Errorf("no validation defined for %#v", obj)}
	}
	return errors
}

func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error {
	err := filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() && path != inDir {
			return filepath.SkipDir
		}
		name := filepath.Base(path)
		ext := filepath.Ext(name)
		if ext != "" {
			name = name[:len(name)-len(ext)]
		}
		if !(ext == ".json" || ext == ".yaml") {
			return nil
		}
		glog.Infof("Testing %s", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		fn(name, path, data)
		return nil
	})
	return err
}

func TestExampleObjectSchemas(t *testing.T) {
	cases := map[string]map[string]runtime.Object{
		"../api/examples": {
			"controller":       &api.ReplicationController{},
			"controller-list":  &api.ReplicationControllerList{},
			"pod":              &api.Pod{},
			"pod-list":         &api.PodList{},
			"service":          &api.Service{},
			"external-service": &api.Service{},
			"service-list":     &api.ServiceList{},
		},
		"../examples/guestbook": {
			"frontend-controller":    &api.ReplicationController{},
			"redis-slave-controller": &api.ReplicationController{},
			"redis-master":           &api.Pod{},
			"frontend-service":       &api.Service{},
			"redis-master-service":   &api.Service{},
			"redis-slave-service":    &api.Service{},
		},
		"../examples/guestbook/v3_json_files": {
			"frontend-controller":    &api.ReplicationController{},
			"redis-slave-controller": &api.ReplicationController{},
			"redis-master":           &api.ReplicationController{},
			"frontend-service":       &api.Service{},
			"redis-master-service":   &api.Service{},
			"redis-slave-service":    &api.Service{},
		},
		"../examples/guestbook-go": {
			"guestbook-controller":    &api.ReplicationController{},
			"redis-slave-controller":  &api.ReplicationController{},
			"redis-master-controller": &api.ReplicationController{},
			"guestbook-service":       &api.Service{},
			"redis-master-service":    &api.Service{},
			"redis-slave-service":     &api.Service{},
		},
		"../examples/guestbook-go/v3_json_files": {
			"guestbook-controller":    &api.ReplicationController{},
			"redis-slave-controller":  &api.ReplicationController{},
			"redis-master-controller": &api.ReplicationController{},
			"guestbook-service":       &api.Service{},
			"redis-master-service":    &api.Service{},
			"redis-slave-service":     &api.Service{},
		},
		"../examples/walkthrough": {
			"pod1": &api.Pod{},
			"pod2": &api.Pod{},
			"pod-with-http-healthcheck": &api.Pod{},
			"service":                   &api.Service{},
			"replication-controller":    &api.ReplicationController{},
		},
		"../examples/update-demo": {
			"kitten-rc":   &api.ReplicationController{},
			"nautilus-rc": &api.ReplicationController{},
		},
	}

	for path, expected := range cases {
		tested := 0
		err := walkJSONFiles(path, func(name, path string, data []byte) {
			expectedType, found := expected[name]
			if !found {
				t.Errorf("%s does not have a test case defined", path)
				return
			}
			tested += 1
			if err := latest.Codec.DecodeInto(data, expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
				return
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
		})
		if err != nil {
			t.Errorf("Expected no error, Got %v", err)
		}
		if tested != len(expected) {
			t.Errorf("Expected %d examples, Got %d", len(expected), tested)
		}
	}
}

var sampleRegexp = regexp.MustCompile("(?ms)^```(?:(?P<type>yaml)\\w*\\n(?P<content>.+?)|\\w*\\n(?P<content>\\{.+?\\}))\\w*\\n^```")
var subsetRegexp = regexp.MustCompile("(?ms)\\.{3}")

func TestReadme(t *testing.T) {
	paths := []string{
		"../README.md",
		"../examples/walkthrough/README.md",
	}

	for _, path := range paths {
		data, err := ioutil.ReadFile(path)
		if err != nil {
			t.Errorf("Unable to read file %s: %v", path, err)
			continue
		}

		matches := sampleRegexp.FindAllStringSubmatch(string(data), -1)
		if matches == nil {
			continue
		}
		for _, match := range matches {
			var content, subtype string
			for i, name := range sampleRegexp.SubexpNames() {
				if name == "type" {
					subtype = match[i]
				}
				if name == "content" && match[i] != "" {
					content = match[i]
				}
			}
			if subtype == "yaml" && subsetRegexp.FindString(content) != "" {
				t.Logf("skipping (%s): \n%s", subtype, content)
				continue
			}

			//t.Logf("testing (%s): \n%s", subtype, content)
			expectedType := &api.Pod{}
			if err := latest.Codec.DecodeInto([]byte(content), expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(content))
				continue
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
			_, err := latest.Codec.Encode(expectedType)
			if err != nil {
				t.Errorf("Could not encode object: %v", err)
				continue
			}
		}
	}
}
<|endoftext|>


<commit_before>package sexp

func DontPanic(f func() error) (err error) {
	defer func() {
		if e := recover(); e != nil {
			if ue, ok := e.(*UnmarshalError); ok {
				err = ue
				return
			}
			panic(e)
		}
	}()
	return f()
}

// A simple helper structure inspired by the simplejson-go API. Use Help
// function to actually acquire it from the given *Node.
type Helper struct {
	node *Node
	err *UnmarshalError
}

func Help(node *Node) Helper {
	return Helper{node, nil}
}

func (h Helper) IsValid() bool {
	return h.node != nil
}

func (h Helper) Next() Helper {
	if h.node == nil {
		return h
	}
	if h.node.Next == nil {
		err := NewUnmarshalError(h.node, nil,
			"a sibling of the node was requested, but it has none")
		return Helper{nil, err}
	}
	return Helper{h.node.Next, nil}
}

func (h Helper) Child(n int) Helper {
	if h.node == nil {
		return h
	}
	c := h.node.Children
	if c == nil {
		err := NewUnmarshalError(h.node, nil,
			"cannot retrieve %d%s child node, node is not a list",
			n+1, number_suffix(n+1))
		return Helper{nil, err}
	}
	for i := 0; i < n; i++ {
		c = c.Next
		if c == nil {
			err := NewUnmarshalError(h.node, nil,
				"cannot retrieve %d%s child node, %s",
				n+1, number_suffix(n+1),
				the_list_has_n_children(h.node.NumChildren()))
			return Helper{nil, err}
		}
	}
	return Helper{c, nil}
}

func (h Helper) IsList() bool {
	if h.node == nil {
		return false
	}
	return h.node.IsList()
}

func (h Helper) IsScalar() bool {
	if h.node == nil {
		return false
	}
	return h.node.IsScalar()
}

func (h Helper) Bool() (bool, error) {
	if h.node == nil {
		return false, h.err
	}
	var v bool
	err := h.node.Unmarshal(&v)
	if err != nil {
		return false, err
	}
	return v, nil
}

func (h Helper) Int() (int, error) {
	if h.node == nil {
		return 0, h.err
	}
	var v int
	err := h.node.Unmarshal(&v)
	if err != nil {
		return 0, err
	}
	return v, nil
}

func (h Helper) Float64() (float64, error) {
	if h.node == nil {
		return 0, h.err
	}
	var v float64
	err := h.node.Unmarshal(&v)
	if err != nil {
		return 0, err
	}
	return v, nil
}

func (h Helper) String() (string, error) {
	if h.node == nil {
		return "", h.err
	}
	var v string
	err := h.node.Unmarshal(&v)
	if err != nil {
		return "", err
	}
	return v, nil
}

func (h Helper) MustBool() bool {
	v, err := h.Bool()
	if err != nil {
		panic(err)
	}
	return v
}

func (h Helper) MustInt() int {
	v, err := h.Int()
	if err != nil {
		panic(err)
	}
	return v
}

func (h Helper) MustFloat64() float64 {
	v, err := h.Float64()
	if err != nil {
		panic(err)
	}
	return v
}

func (h Helper) MustString() string {
	v, err := h.String()
	if err != nil {
		panic(err)
	}
	return v
}
<commit_msg>Add Node and MustNode accessors for Helper.<commit_after>package sexp

func DontPanic(f func() error) (err error) {
	defer func() {
		if e := recover(); e != nil {
			if ue, ok := e.(*UnmarshalError); ok {
				err = ue
				return
			}
			panic(e)
		}
	}()
	return f()
}

// A simple helper structure inspired by the simplejson-go API. Use Help
// function to actually acquire it from the given *Node.
type Helper struct {
	node *Node
	err *UnmarshalError
}

func Help(node *Node) Helper {
	return Helper{node, nil}
}

func (h Helper) IsValid() bool {
	return h.node != nil
}

func (h Helper) Next() Helper {
	if h.node == nil {
		return h
	}
	if h.node.Next == nil {
		err := NewUnmarshalError(h.node, nil,
			"a sibling of the node was requested, but it has none")
		return Helper{nil, err}
	}
	return Helper{h.node.Next, nil}
}

func (h Helper) Child(n int) Helper {
	if h.node == nil {
		return h
	}
	c := h.node.Children
	if c == nil {
		err := NewUnmarshalError(h.node, nil,
			"cannot retrieve %d%s child node, node is not a list",
			n+1, number_suffix(n+1))
		return Helper{nil, err}
	}
	for i := 0; i < n; i++ {
		c = c.Next
		if c == nil {
			err := NewUnmarshalError(h.node, nil,
				"cannot retrieve %d%s child node, %s",
				n+1, number_suffix(n+1),
				the_list_has_n_children(h.node.NumChildren()))
			return Helper{nil, err}
		}
	}
	return Helper{c, nil}
}

func (h Helper) IsList() bool {
	if h.node == nil {
		return false
	}
	return h.node.IsList()
}

func (h Helper) IsScalar() bool {
	if h.node == nil {
		return false
	}
	return h.node.IsScalar()
}

func (h Helper) Bool() (bool, error) {
	if h.node == nil {
		return false, h.err
	}
	var v bool
	err := h.node.Unmarshal(&v)
	if err != nil {
		return false, err
	}
	return v, nil
}

func (h Helper) Int() (int, error) {
	if h.node == nil {
		return 0, h.err
	}
	var v int
	err := h.node.Unmarshal(&v)
	if err != nil {
		return 0, err
	}
	return v, nil
}

func (h Helper) Float64() (float64, error) {
	if h.node == nil {
		return 0, h.err
	}
	var v float64
	err := h.node.Unmarshal(&v)
	if err != nil {
		return 0, err
	}
	return v, nil
}

func (h Helper) String() (string, error) {
	if h.node == nil {
		return "", h.err
	}
	var v string
	err := h.node.Unmarshal(&v)
	if err != nil {
		return "", err
	}
	return v, nil
}

func (h Helper) Node() (*Node, error) {
	if h.node == nil {
		return nil, h.err
	}
	return h.node, nil
}

func (h Helper) MustBool() bool {
	v, err := h.Bool()
	if err != nil {
		panic(err)
	}
	return v
}

func (h Helper) MustInt() int {
	v, err := h.Int()
	if err != nil {
		panic(err)
	}
	return v
}

func (h Helper) MustFloat64() float64 {
	v, err := h.Float64()
	if err != nil {
		panic(err)
	}
	return v
}

func (h Helper) MustString() string {
	v, err := h.String()
	if err != nil {
		panic(err)
	}
	return v
}

func (h Helper) MustNode() *Node {
	v, err := h.Node()
	if err != nil {
		panic(err)
	}
	return v
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package expect

import(`fmt`; `runtime`; `path/filepath`; `strings`; `testing`)

func Eq(t *testing.T, actual, expected interface{}) {
	log(t, actual, expected, equal(actual, expected))
}

func True(t *testing.T, actual interface{}) {
	log(t, actual, true, equal(actual, true))
}

func False(t *testing.T, actual interface{}) {
	log(t, actual, false, equal(actual, false))
}

func Contain(t *testing.T, actual interface{}, expected string) {
	match(t, fmt.Sprintf(`%+v`, actual), expected, true)
}

func NotContain(t *testing.T, actual interface{}, expected string) {
	match(t, fmt.Sprintf(`%+v`, actual), expected, false)
}

func equal(actual, expected interface{}) (passed bool) {
	switch expected.(type) {
	case bool:
		if assertion, ok := actual.(bool); ok {
			passed = (assertion == expected)
		}
	case int:
		if assertion, ok := actual.(int); ok {
			passed = (assertion == expected)
		}
	case uint64:
		if assertion, ok := actual.(uint64); ok {
			passed = (assertion == expected)
		}
	default:
		passed = (fmt.Sprintf(`%v`, actual) == fmt.Sprintf(`%v`, expected))
	}
	return
}

// Simple success/failure logger that assumes source test file is at Caller(2).
func log(t *testing.T, actual, expected interface{}, passed bool) {
	_, file, line, _ := runtime.Caller(2) 	// Get the calling file path and line number.
	file = filepath.Base(file) 		// Keep file name only.

	if !passed {
		t.Errorf("\r\t\x1B[31m%s line %d\nExpected: %v\n  Actual: %v\x1B[0m", file, line, expected, actual)
	} else if (testing.Verbose()) {
		t.Logf("\r\t\x1B[32m%s line %d: %v\x1B[0m", file, line, actual)
	}
}

func match(t *testing.T, actual, expected string, contains bool) {
	passed := (contains == strings.Contains(actual, expected))

	_, file, line, _ := runtime.Caller(2)
	file = filepath.Base(file)

	if !passed {
		t.Errorf("\r\t\x1B[31m%s line %d\nContains: %s\n  Actual: %s\x1B[0m", file, line, expected, actual)
	} else if (testing.Verbose()) {
		t.Logf("\r\t\x1B[32m%s line %d: %v\x1B[0m", file, line, actual)
	}
}
<commit_msg>Added expect.Ne(...) for not equal tests<commit_after>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package expect

import(`fmt`; `runtime`; `path/filepath`; `strings`; `testing`)

func Eq(t *testing.T, actual, expected interface{}) {
	log(t, actual, expected, equal(actual, expected))
}

func Ne(t *testing.T, actual, expected interface{}) {
	log(t, actual, expected, !equal(actual, expected))
}

func True(t *testing.T, actual interface{}) {
	log(t, actual, true, equal(actual, true))
}

func False(t *testing.T, actual interface{}) {
	log(t, actual, false, equal(actual, false))
}

func Contain(t *testing.T, actual interface{}, expected string) {
	match(t, fmt.Sprintf(`%+v`, actual), expected, true)
}

func NotContain(t *testing.T, actual interface{}, expected string) {
	match(t, fmt.Sprintf(`%+v`, actual), expected, false)
}

func equal(actual, expected interface{}) (passed bool) {
	switch expected.(type) {
	case bool:
		if assertion, ok := actual.(bool); ok {
			passed = (assertion == expected)
		}
	case int:
		if assertion, ok := actual.(int); ok {
			passed = (assertion == expected)
		}
	case uint64:
		if assertion, ok := actual.(uint64); ok {
			passed = (assertion == expected)
		}
	default:
		passed = (fmt.Sprintf(`%v`, actual) == fmt.Sprintf(`%v`, expected))
	}
	return
}

// Simple success/failure logger that assumes source test file is at Caller(2).
func log(t *testing.T, actual, expected interface{}, passed bool) {
	_, file, line, _ := runtime.Caller(2) 	// Get the calling file path and line number.
	file = filepath.Base(file) 		// Keep file name only.

	if !passed {
		t.Errorf("\r\t\x1B[31m%s line %d\nExpected: %v\n  Actual: %v\x1B[0m", file, line, expected, actual)
	} else if (testing.Verbose()) {
		t.Logf("\r\t\x1B[32m%s line %d: %v\x1B[0m", file, line, actual)
	}
}

func match(t *testing.T, actual, expected string, contains bool) {
	passed := (contains == strings.Contains(actual, expected))

	_, file, line, _ := runtime.Caller(2)
	file = filepath.Base(file)

	if !passed {
		t.Errorf("\r\t\x1B[31m%s line %d\nContains: %s\n  Actual: %s\x1B[0m", file, line, expected, actual)
	} else if (testing.Verbose()) {
		t.Logf("\r\t\x1B[32m%s line %d: %v\x1B[0m", file, line, actual)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"testing"
	"time"
)

// TestNegativeCounter validates when we send a negative
// number to a counter that we no longer panic the Exporter Listener.
func TestNegativeCounter(t *testing.T) {
	defer func() {
		if e := recover(); e != nil {
			err := e.(error)
			if err.Error() == "counter cannot decrease in value" {
				t.Fatalf("Counter was negative and causes a panic.")
			} else {
				t.Fatalf("Unknown panic and error: %q", err.Error())
			}
		}
	}()

	events := make(chan Events, 1)
	c := Events{
		&CounterEvent{
			metricName: "foo",
			value:      -1,
		},
	}
	events <- c
	ex := NewExporter(&metricMapper{})

	// Close channel to signify we are done with the listener after a short period.
	go func() {
		time.Sleep(time.Millisecond * 100)
		close(events)
	}()

	ex.Listen(events)
}
<commit_msg>Fix parameter to NewExporter in test<commit_after>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"testing"
	"time"
)

// TestNegativeCounter validates when we send a negative
// number to a counter that we no longer panic the Exporter Listener.
func TestNegativeCounter(t *testing.T) {
	defer func() {
		if e := recover(); e != nil {
			err := e.(error)
			if err.Error() == "counter cannot decrease in value" {
				t.Fatalf("Counter was negative and causes a panic.")
			} else {
				t.Fatalf("Unknown panic and error: %q", err.Error())
			}
		}
	}()

	events := make(chan Events, 1)
	c := Events{
		&CounterEvent{
			metricName: "foo",
			value:      -1,
		},
	}
	events <- c
	ex := NewExporter(&metricMapper{}, true)

	// Close channel to signify we are done with the listener after a short period.
	go func() {
		time.Sleep(time.Millisecond * 100)
		close(events)
	}()

	ex.Listen(events)
}
<|endoftext|>


<commit_before>package goat

func DbManager() {
	// channels
	SqlRequestChan := make(chan Request)
	MapRequestChan := make(chan Request, 100)

	// launch databases
	if Static.Config.Map {
		go new(MapDb).HandleDb(Static.RequestChan)
		Static.LogChan <- "MapDb instance launched"
	}
	if Static.Config.Sql {
		go new(SqlDb).HandleDb(Static.RequestChan)
		Static.LogChan <- "SqlDb instance launched"
	}

	if Static.Config.Map && Static.Config.Sql {
		for {
			select {
			case hold := <-Static.RequestChan:
				if hold.Data == nil {
					MapRequestChan <- hold
				} else {
					MapRequestChan <- hold
					SqlRequestChan <- hold
				}
			case hold := <-Static.PersistentChan:
				SqlRequestChan <- hold
			}
		}
	} else if Static.Config.Map {
		for {
			select {
			case hold := <-Static.RequestChan:
				MapRequestChan <- hold
			}
		}
	} else if Static.Config.Sql {
		for {
			select {
			case hold := <-Static.RequestChan:
				SqlRequestChan <- hold
			}
		}
	} else {
		Static.LogChan <- "No database in use."
	}
}

// Holds information for request from database
type Request struct {
	Id           string
	Data         interface{}
	ResponseChan chan Response
}

// Holds information for response from database
type Response struct {
	Id   string
	Db   string
	Data interface{}
}
type WriteResponse struct {
	Complete bool
}

// DbHandler interface method HandleDb defines a database handler which handles requests
type DbHandler interface {
	HandleDb(RequestChan chan Request)
}

// MapDb is a key value storage database
// Id will be an identification for sharding
type MapDb struct {
	Id      string
	Db      map[string]map[string]interface{}
	Workers map[string]MapWorker
}

// Handle data MapDb requests
func (db MapDb) HandleDb(RequestChan chan Request) {
	db.Db = make(map[string]map[string]interface{})
	for {
		select {
		case hold := <-RequestChan:
			l := len(hold.Id)
			s := Static.Config.CacheSize
			key := hold.Id[l-s : l-1]
			switch {
			// This logic needs to be refactored so that the number of shards can be
			// determined via the config file, which will define the number of digits
			// used from the ID for the shard name
			case hold.Data == nil:
				_, ok := db.Db[key]
				if !ok {
					db.Db[key] = make(map[string]interface{})
				}
				go new(MapWorker).Read(hold, db.Db[key])
			case hold.Data != nil:
				_, ok := db.Db[key]
				if !ok {
					db.Db[key] = make(map[string]interface{})
				}
				go new(MapWorker).Write(hold, db.Db[key])
			}
		}
	}
}

// SqlDb is a Sql based database
type SqlDb struct {
}

// Handle Sql based requests
func (s SqlDb) HandleDb(RequestChan chan Request) {
}
<commit_msg>fixed blockign request chan<commit_after>package goat

func DbManager() {
	// channels
	SqlRequestChan := make(chan Request)
	MapRequestChan := make(chan Request, 100)

	// launch databases
	if Static.Config.Map {
		go new(MapDb).HandleDb(MapRequestChan)
		Static.LogChan <- "MapDb instance launched"
	}
	if Static.Config.Sql {
		go new(SqlDb).HandleDb(SqlRequestChan)
		Static.LogChan <- "SqlDb instance launched"
	}

	if Static.Config.Map && Static.Config.Sql {
		for {
			select {
			case hold := <-Static.RequestChan:
				if hold.Data == nil {
					MapRequestChan <- hold
				} else {
					MapRequestChan <- hold
					SqlRequestChan <- hold
				}
			case hold := <-Static.PersistentChan:
				SqlRequestChan <- hold
			}
		}
	} else if Static.Config.Map {
		for {
			select {
			case hold := <-Static.RequestChan:
				MapRequestChan <- hold
			}
		}
	} else if Static.Config.Sql {
		for {
			select {
			case hold := <-Static.RequestChan:
				SqlRequestChan <- hold
			}
		}
	} else {
		Static.LogChan <- "No database in use."
	}
}

// Holds information for request from database
type Request struct {
	Id           string
	Data         interface{}
	ResponseChan chan Response
}

// Holds information for response from database
type Response struct {
	Id   string
	Db   string
	Data interface{}
}
type WriteResponse struct {
	Complete bool
}

// DbHandler interface method HandleDb defines a database handler which handles requests
type DbHandler interface {
	HandleDb(RequestChan chan Request)
}

// MapDb is a key value storage database
// Id will be an identification for sharding
type MapDb struct {
	Id      string
	Db      map[string]map[string]interface{}
	Workers map[string]MapWorker
}

// Handle data MapDb requests
func (db MapDb) HandleDb(RequestChan chan Request) {
	db.Db = make(map[string]map[string]interface{})
	for {
		select {
		case hold := <-RequestChan:
			l := len(hold.Id)
			s := Static.Config.CacheSize
			key := hold.Id[l-s : l-1]
			switch {
			// This logic needs to be refactored so that the number of shards can be
			// determined via the config file, which will define the number of digits
			// used from the ID for the shard name
			case hold.Data == nil:
				// check if key exits
				_, ok := db.Db[key]
				if !ok {
					// if key does not exist, make a new map with the given key
					db.Db[key] = make(map[string]interface{})
				}
				go new(MapWorker).Read(hold, db.Db[key])
			case hold.Data != nil:
				// check if key exits
				_, ok := db.Db[key]
				if !ok {
					// if key does not exist, make a new map with the given key
					db.Db[key] = make(map[string]interface{})
				}
				go new(MapWorker).Write(hold, db.Db[key])
			}
		}
	}
}

// SqlDb is a Sql based database
type SqlDb struct {
}

// Handle Sql based requests
func (s SqlDb) HandleDb(RequestChan chan Request) {
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	auth "github.com/heroku/lumbermill/Godeps/_workspace/src/github.com/heroku/authenticater"
	influx "github.com/heroku/lumbermill/Godeps/_workspace/src/github.com/influxdb/influxdb-go"
)

var influxDbStaleTimeout = 24 * time.Minute // Would be nice to make this smaller, but it lags due to continuous queries.
var influxDbSeriesCheckQueries = []string{
	"select * from MaxMean1mLoad.10m.dyno.dyno.load.%s limit 1",
	"select * from MaxMeanRssSwapMemory.10m.dyno.mem.%s limit 1",
}

var healthCheckClientsLock = new(sync.Mutex)
var healthCheckClients = make(map[string]*influx.Client)

type server struct {
	sync.WaitGroup
	connectionCloser chan struct{}
	hashRing         *hashRing
	http             *http.Server
	shutdownChan     shutdownChan
	isShuttingDown   bool
	credStore        map[string]string

	// scheduler based sampling lock for writing to recentTokens
	tokenLock        *int32
	recentTokensLock *sync.RWMutex
	recentTokens     map[string]string
}

func newServer(httpServer *http.Server, ath auth.Authenticater, hashRing *hashRing) *server {
	s := &server{
		connectionCloser: make(chan struct{}),
		shutdownChan:     make(chan struct{}),
		http:             httpServer,
		hashRing:         hashRing,
		credStore:        make(map[string]string),
		tokenLock:        new(int32),
		recentTokensLock: new(sync.RWMutex),
		recentTokens:     make(map[string]string),
	}

	mux := http.NewServeMux()

	mux.HandleFunc("/drain", auth.WrapAuth(ath,
		func(w http.ResponseWriter, r *http.Request) {
			s.serveDrain(w, r)
			s.recycleConnection(w)
		}))

	mux.HandleFunc("/health", s.serveHealth)
	mux.HandleFunc("/health/influxdb", s.serveInfluxDBHealth)
	mux.HandleFunc("/target/", auth.WrapAuth(ath, s.serveTarget))

	s.http.Handler = mux

	return s
}

func (s *server) Close() error {
	s.shutdownChan <- struct{}{}
	return nil
}

func (s *server) scheduleConnectionRecycling(after time.Duration) {
	for !s.isShuttingDown {
		time.Sleep(after)
		s.connectionCloser <- struct{}{}
	}
}

func (s *server) recycleConnection(w http.ResponseWriter) {
	select {
	case <-s.connectionCloser:
		w.Header().Set("Connection", "close")
	default:
		if s.isShuttingDown {
			w.Header().Set("Connection", "close")
		}
	}
}

func (s *server) Run(connRecycle time.Duration) {
	go s.awaitShutdown()
	go s.scheduleConnectionRecycling(connRecycle)

	if err := s.http.ListenAndServe(); err != nil {
		log.Fatalln("Unable to start HTTP server: ", err)
	}
}

// Serves a 200 OK, unless shutdown has been requested.
// Shutting down serves a 503 since that's how ELBs implement connection draining.
func (s *server) serveHealth(w http.ResponseWriter, r *http.Request) {
	if s.isShuttingDown {
		http.Error(w, "Shutting Down", 503)
	}

	w.WriteHeader(http.StatusOK)
}

func getHealthCheckClient(host string, f clientFunc) (*influx.Client, error) {
	var client *influx.Client
	var exists bool
	healthCheckClientsLock.Lock()
	defer healthCheckClientsLock.Unlock()

	if client, exists = healthCheckClients[host]; !exists {
		var err error

		clientConfig := createInfluxDBClient(host, f)
		client, err = influx.NewClient(&clientConfig)
		if err != nil {
			log.Printf("err=%q at=getHealthCheckClient host=%q", err, host)
			return nil, err
		}

		healthCheckClients[host] = client
		return client, nil
	}

	return client, nil
}

func checkRecentToken(client *influx.Client, token, host string, errors chan error) {
	for _, qfmt := range influxDbSeriesCheckQueries {
		query := fmt.Sprintf(qfmt, token)
		results, err := client.Query(query, influx.Second)
		if err != nil || len(results) == 0 {
			errors <- fmt.Errorf("at=influxdb-health err=%q result_length=%d host=%q query=%q", err, len(results), host, query)
			continue
		}

		t, ok := results[0].Points[0][0].(float64)
		if !ok {
			errors <- fmt.Errorf("at=influxdb-health err=\"time column was not a number\" host=%q query=%q", host, query)
			continue
		}

		ts := time.Unix(int64(t), int64(0)).UTC()
		now := time.Now().UTC()
		if now.Sub(ts) > influxDbStaleTimeout {
			errors <- fmt.Errorf("at=influxdb-health err=\"stale data\" host=%q ts=%q now=%q query=%q", host, ts, now, query)
		}
	}
}

func (s *server) checkRecentTokens() []error {
	var errSlice []error

	wg := new(sync.WaitGroup)

	s.recentTokensLock.RLock()
	tokenMap := make(map[string]string)
	for host, token := range s.recentTokens {
		tokenMap[host] = token
	}
	s.recentTokensLock.RUnlock()

	errors := make(chan error, len(tokenMap)*len(influxDbSeriesCheckQueries))

	for host, token := range tokenMap {
		wg.Add(1)
		go func(token, host string) {
			client, err := getHealthCheckClient(host, nil)
			if err != nil {
				return
			}
			checkRecentToken(client, token, host, errors)
			wg.Done()
		}(token, host)
	}

	wg.Wait()
	close(errors)

	for err := range errors {
		errSlice = append(errSlice, err)
	}

	return errSlice
}

func (s *server) serveInfluxDBHealth(w http.ResponseWriter, r *http.Request) {
	errors := s.checkRecentTokens()

	if len(errors) > 0 {
		w.WriteHeader(http.StatusServiceUnavailable)
		for _, err := range errors {
			w.Write([]byte(err.Error() + "\n"))
			log.Println(err)
		}
		return
	}
	w.WriteHeader(http.StatusOK)
}

func (s *server) awaitShutdown() {
	<-s.shutdownChan
	log.Printf("Shutting down.")
	s.isShuttingDown = true
}
<commit_msg>Cleanup getHealthCheckClient helper<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	auth "github.com/heroku/lumbermill/Godeps/_workspace/src/github.com/heroku/authenticater"
	influx "github.com/heroku/lumbermill/Godeps/_workspace/src/github.com/influxdb/influxdb-go"
)

var influxDbStaleTimeout = 24 * time.Minute // Would be nice to make this smaller, but it lags due to continuous queries.
var influxDbSeriesCheckQueries = []string{
	"select * from MaxMean1mLoad.10m.dyno.dyno.load.%s limit 1",
	"select * from MaxMeanRssSwapMemory.10m.dyno.mem.%s limit 1",
}

var healthCheckClientsLock = new(sync.Mutex)
var healthCheckClients = make(map[string]*influx.Client)

type server struct {
	sync.WaitGroup
	connectionCloser chan struct{}
	hashRing         *hashRing
	http             *http.Server
	shutdownChan     shutdownChan
	isShuttingDown   bool
	credStore        map[string]string

	// scheduler based sampling lock for writing to recentTokens
	tokenLock        *int32
	recentTokensLock *sync.RWMutex
	recentTokens     map[string]string
}

func newServer(httpServer *http.Server, ath auth.Authenticater, hashRing *hashRing) *server {
	s := &server{
		connectionCloser: make(chan struct{}),
		shutdownChan:     make(chan struct{}),
		http:             httpServer,
		hashRing:         hashRing,
		credStore:        make(map[string]string),
		tokenLock:        new(int32),
		recentTokensLock: new(sync.RWMutex),
		recentTokens:     make(map[string]string),
	}

	mux := http.NewServeMux()

	mux.HandleFunc("/drain", auth.WrapAuth(ath,
		func(w http.ResponseWriter, r *http.Request) {
			s.serveDrain(w, r)
			s.recycleConnection(w)
		}))

	mux.HandleFunc("/health", s.serveHealth)
	mux.HandleFunc("/health/influxdb", s.serveInfluxDBHealth)
	mux.HandleFunc("/target/", auth.WrapAuth(ath, s.serveTarget))

	s.http.Handler = mux

	return s
}

func (s *server) Close() error {
	s.shutdownChan <- struct{}{}
	return nil
}

func (s *server) scheduleConnectionRecycling(after time.Duration) {
	for !s.isShuttingDown {
		time.Sleep(after)
		s.connectionCloser <- struct{}{}
	}
}

func (s *server) recycleConnection(w http.ResponseWriter) {
	select {
	case <-s.connectionCloser:
		w.Header().Set("Connection", "close")
	default:
		if s.isShuttingDown {
			w.Header().Set("Connection", "close")
		}
	}
}

func (s *server) Run(connRecycle time.Duration) {
	go s.awaitShutdown()
	go s.scheduleConnectionRecycling(connRecycle)

	if err := s.http.ListenAndServe(); err != nil {
		log.Fatalln("Unable to start HTTP server: ", err)
	}
}

// Serves a 200 OK, unless shutdown has been requested.
// Shutting down serves a 503 since that's how ELBs implement connection draining.
func (s *server) serveHealth(w http.ResponseWriter, r *http.Request) {
	if s.isShuttingDown {
		http.Error(w, "Shutting Down", 503)
	}

	w.WriteHeader(http.StatusOK)
}

func getHealthCheckClient(host string, f clientFunc) (*influx.Client, error) {
	healthCheckClientsLock.Lock()
	defer healthCheckClientsLock.Unlock()

	client, exists := healthCheckClients[host]
	if !exists {
		var err error
		clientConfig := createInfluxDBClient(host, f)
		client, err = influx.NewClient(&clientConfig)
		if err != nil {
			log.Printf("err=%q at=getHealthCheckClient host=%q", err, host)
			return nil, err
		}

		healthCheckClients[host] = client
	}

	return client, nil
}

func checkRecentToken(client *influx.Client, token, host string, errors chan error) {
	for _, qfmt := range influxDbSeriesCheckQueries {
		query := fmt.Sprintf(qfmt, token)
		results, err := client.Query(query, influx.Second)
		if err != nil || len(results) == 0 {
			errors <- fmt.Errorf("at=influxdb-health err=%q result_length=%d host=%q query=%q", err, len(results), host, query)
			continue
		}

		t, ok := results[0].Points[0][0].(float64)
		if !ok {
			errors <- fmt.Errorf("at=influxdb-health err=\"time column was not a number\" host=%q query=%q", host, query)
			continue
		}

		ts := time.Unix(int64(t), int64(0)).UTC()
		now := time.Now().UTC()
		if now.Sub(ts) > influxDbStaleTimeout {
			errors <- fmt.Errorf("at=influxdb-health err=\"stale data\" host=%q ts=%q now=%q query=%q", host, ts, now, query)
		}
	}
}

func (s *server) checkRecentTokens() []error {
	var errSlice []error

	wg := new(sync.WaitGroup)

	s.recentTokensLock.RLock()
	tokenMap := make(map[string]string)
	for host, token := range s.recentTokens {
		tokenMap[host] = token
	}
	s.recentTokensLock.RUnlock()

	errors := make(chan error, len(tokenMap)*len(influxDbSeriesCheckQueries))

	for host, token := range tokenMap {
		wg.Add(1)
		go func(token, host string) {
			client, err := getHealthCheckClient(host, nil)
			if err != nil {
				return
			}
			checkRecentToken(client, token, host, errors)
			wg.Done()
		}(token, host)
	}

	wg.Wait()
	close(errors)

	for err := range errors {
		errSlice = append(errSlice, err)
	}

	return errSlice
}

func (s *server) serveInfluxDBHealth(w http.ResponseWriter, r *http.Request) {
	errors := s.checkRecentTokens()

	if len(errors) > 0 {
		w.WriteHeader(http.StatusServiceUnavailable)
		for _, err := range errors {
			w.Write([]byte(err.Error() + "\n"))
			log.Println(err)
		}
		return
	}
	w.WriteHeader(http.StatusOK)
}

func (s *server) awaitShutdown() {
	<-s.shutdownChan
	log.Printf("Shutting down.")
	s.isShuttingDown = true
}
<|endoftext|>


<commit_before>package oauth

import (
    "bufio"
    //"bytes"
    "crypto/tls"
    "net"
    "fmt"
    "http"
    "io"
    "os"
    "strings"
)

type badStringError struct {
    what string
    str string
}

func (e *badStringError) String() string {
    return fmt.Sprintf("%s %q", e.what, e.str)
}

type readClose struct {
    io.Reader
    io.Closer
}

type nopCloser struct {
    io.Reader
}
func (nopCloser) Close() os.Error { return nil }

func hasPort(s string) bool {
    return strings.LastIndex(s, ":") > strings.LastIndex(s, "]")
}

func send(req *http.Request) (resp *http.Response, err os.Error) {
    //dump, _ := http.DumpRequest(req, true)
    //fmt.Fprintf(os.Stderr, "%s", dump)
    //fmt.Fprintf(os.Stderr, "\n--- body:\n%s\n---", bodyString(req.Body))
    if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
        return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme}
    }

    addr := req.URL.Host
    var conn net.Conn
    switch(req.URL.Scheme) {
    case "http":
        if !hasPort(addr) {
            addr += ":http"
        }

        conn, err = net.Dial("tcp", "", addr)
    case "https":
        if !hasPort(addr) {
            addr += ":https"
        }

        conn, err = tls.Dial("tcp", "", addr)
    }
    if err != nil {
        return nil, err
    }

    err = req.Write(conn)
    if err != nil {
        conn.Close()
        return nil, err
    }

    reader := bufio.NewReader(conn)
    resp, err = http.ReadResponse(reader, req.Method)
    if err != nil {
        conn.Close()
        return nil, err
    }

    resp.Body = readClose{resp.Body, conn}

    return
}

func post(url string, oauthHeaders map[string]string) (r *http.Response, err os.Error) {
    var req http.Request
    req.Method = "POST"
    req.ProtoMajor = 1
    req.ProtoMinor = 1
    req.Close = true
    req.Header = map[string]string{
        "Authorization": "OAuth ",
    }
    req.TransferEncoding = []string{"chunked"}

    first := true
    for k, v := range oauthHeaders {
        if first {
            first = false
        } else {
            req.Header["Authorization"] += ",\n    "
        }
        req.Header["Authorization"] += k+"=\""+v+"\""
    }

    req.URL, err = http.ParseURL(url)
    if err != nil {
        return nil, err
    }

    return send(&req)
}

func get(url string, oauthHeaders map[string]string) (r *http.Response, err os.Error) {
    var req http.Request
    req.Method = "GET"
    req.ProtoMajor = 1
    req.ProtoMinor = 1
    req.Close = true
    req.Header = map[string]string{
        "Authorization": "OAuth ",
    }
    req.TransferEncoding = []string{"chunked"}

    first := true
    for k, v := range oauthHeaders {
        if first {
            first = false
        } else {
            req.Header["Authorization"] += ",\n    "
        }
        req.Header["Authorization"] += k+"=\""+v+"\""
    }

    req.URL, err = http.ParseURL(url)
    if err != nil {
        return nil, err
    }

    return send(&req)
}

<commit_msg>Account for updates in networking code.<commit_after>package oauth

import (
    "bufio"
    //"bytes"
    "crypto/tls"
    "net"
    "fmt"
    "http"
    "io"
    "os"
    "strings"
)

type badStringError struct {
    what string
    str string
}

func (e *badStringError) String() string {
    return fmt.Sprintf("%s %q", e.what, e.str)
}

type readClose struct {
    io.Reader
    io.Closer
}

type nopCloser struct {
    io.Reader
}
func (nopCloser) Close() os.Error { return nil }

func hasPort(s string) bool {
    return strings.LastIndex(s, ":") > strings.LastIndex(s, "]")
}

func send(req *http.Request) (resp *http.Response, err os.Error) {
    //dump, _ := http.DumpRequest(req, true)
    //fmt.Fprintf(os.Stderr, "%s", dump)
    //fmt.Fprintf(os.Stderr, "\n--- body:\n%s\n---", bodyString(req.Body))
    if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
        return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme}
    }

    addr := req.URL.Host
    var conn net.Conn
    switch(req.URL.Scheme) {
    case "http":
        if !hasPort(addr) {
            addr += ":http"
        }

        conn, err = net.Dial("tcp", addr)
    case "https":
        if !hasPort(addr) {
            addr += ":https"
        }

        conn, err = tls.Dial("tcp", addr, nil)
    }
    if err != nil {
        return nil, err
    }

    err = req.Write(conn)
    if err != nil {
        conn.Close()
        return nil, err
    }

    reader := bufio.NewReader(conn)
    resp, err = http.ReadResponse(reader, req.Method)
    if err != nil {
        conn.Close()
        return nil, err
    }

    resp.Body = readClose{resp.Body, conn}

    return
}

func post(url string, oauthHeaders map[string]string) (r *http.Response, err os.Error) {
    var req http.Request
    req.Method = "POST"
    req.ProtoMajor = 1
    req.ProtoMinor = 1
    req.Close = true
    req.Header = map[string][]string{
        "Authorization": {"OAuth "},
    }
    req.TransferEncoding = []string{"chunked"}

    first := true
    for k, v := range oauthHeaders {
        if first {
            first = false
        } else {
            req.Header["Authorization"][0] += ",\n    "
        }
        req.Header["Authorization"][0] += k+"=\""+v+"\""
    }

    req.URL, err = http.ParseURL(url)
    if err != nil {
        return nil, err
    }

    return send(&req)
}

func get(url string, oauthHeaders map[string]string) (r *http.Response, err os.Error) {
    var req http.Request
    req.Method = "GET"
    req.ProtoMajor = 1
    req.ProtoMinor = 1
    req.Close = true
    req.Header = map[string][]string{
        "Authorization": {"OAuth "},
    }
    req.TransferEncoding = []string{"chunked"}

    first := true
    for k, v := range oauthHeaders {
        if first {
            first = false
        } else {
            req.Header["Authorization"][0] += ",\n    "
        }
        req.Header["Authorization"][0] += k+"=\""+v+"\""
    }

    req.URL, err = http.ParseURL(url)
    if err != nil {
        return nil, err
    }

    return send(&req)
}

<|endoftext|>


<commit_before>package extend
import (
	"reflect"
)

const initialSize = 4

// Pusher must be passed a pointer to a slice. It returns
// a function that pushes a new value onto the end of the
// slice, reallocating the slice if necessary.
func Pusher(ap interface{}) func(interface{}) {
	v := reflect.NewValue(ap).(*reflect.PtrValue).Elem().(*reflect.SliceValue)
	t := v.Type().(*reflect.SliceType)
	return func(x interface{}) {
		len, cap := v.Len(), v.Cap()
		if len < cap {
			v.SetLen(len+1)
		}else{
			if cap == 0 {
				cap = initialSize
			}else{
				cap *= 2
			}
			b := reflect.MakeSlice(t, len+1, cap)
			reflect.ArrayCopy(b, v)
			v.SetValue(b)
		}

		v.Elem(len).SetValue(reflect.NewValue(x))
	}
}
<commit_msg>enabled unsafe hackery for 80% speed increase.<commit_after>package extend
import (
	"reflect"
	"unsafe"
)

const initialSize = 4

type interfaceHeader struct {
	t uintptr
	data uintptr
}

// Pusher must be passed a pointer to a slice. It returns
// a function that pushes a new value onto the end of the
// slice, reallocating the slice if necessary.
func Pusher(ap interface{}) func(interface{}) {
	v := reflect.NewValue(ap).(*reflect.PtrValue).Elem().(*reflect.SliceValue)
	h := (*reflect.SliceHeader)(unsafe.Pointer(v.Addr()))
	t := v.Type().(*reflect.SliceType)
	esize := t.Elem().Size()
	_, isInterface := t.Elem().(*reflect.InterfaceType)
	unsafeCopy := !isInterface && esize <= uintptr(unsafe.Sizeof(uintptr(0)))
	if !unsafeCopy {
		return func(x interface{}) {
			len, cap := h.Len, h.Cap
			if len < cap {
				h.Len++
			}else{
				if cap == 0 {
					cap = initialSize
				}else{
					cap *= 2
				}
				b := reflect.MakeSlice(t, len+1, cap)
				reflect.ArrayCopy(b, v)
				v.SetValue(b)
			}
			v.Elem(len).SetValue(reflect.NewValue(x))
		}
	}
	// Nasty unsafe hackery:
	//
	// We know that the size of the type fits in a pointer,
	// so the value is held directly inside the interface value.
	// We copy each element to icopy, so that we can take
	// the address of it without triggering the allocator.
	// We set up e1 as a []byte alias to the data,
	// and do the actual copy by setting up e0 as a []byte
	// alias to the array element and invoking copy().
	// TODO: enable this code for type with size > sizeof(uintptr).
	var e0 []byte
	var e1 []byte
	he0 := (*reflect.SliceHeader)(unsafe.Pointer(&e0))
	he1 := (*reflect.SliceHeader)(unsafe.Pointer(&e1))
	var icopy interface{}
	he1.Data = uintptr(unsafe.Pointer(&icopy)) + uintptr(unsafe.Offsetof(interfaceHeader{}.data))
	he1.Len = int(esize)
	he1.Cap = int(esize)

	return func(x interface{}) {
		len, cap := h.Len, h.Cap
		if len < cap {
			h.Len++
		}else{
			if cap == 0 {
				cap = initialSize
			}else{
				cap *= 2
			}
			b := reflect.MakeSlice(t, len+1, cap)
			reflect.ArrayCopy(b, v)
			v.SetValue(b)
		}
		icopy = x
		he0.Data = h.Data + esize * uintptr(len)
		he0.Len = int(esize)
		he0.Cap = int(esize)
		copy(e0, e1)
		e0 = nil
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 ENDOH takanao.
<https://github.com/MiCHiLU/go-lru-cache-stats>

Copyright 2013 Google Inc.
<https://github.com/golang/groupcache>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lru

import (
	"bytes"
	"net/http"
	"sync"
)

var httpPoolMade bool

type httpGetter struct {
	transport func(Context) http.RoundTripper
	baseURL   string
}

var bufferPool = sync.Pool{
	New: func() interface{} { return new(bytes.Buffer) },
}
<commit_msg>remove vars<commit_after>/*
Copyright 2015 ENDOH takanao.
<https://github.com/MiCHiLU/go-lru-cache-stats>

Copyright 2013 Google Inc.
<https://github.com/golang/groupcache>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lru

import (
	"net/http"
)

type httpGetter struct {
	transport func(Context) http.RoundTripper
	baseURL   string
}
<|endoftext|>


<commit_before>package lily

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/cookiejar"
	netUrl "net/url"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func HTTPStatusCodeIsOk(statusCode int) bool {
	return statusCode > 199 && statusCode < 300
}

func HTTPSetContentTypeJSON(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
}

func HTTPSetContentTypeHTML(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "text/html; charset=UTF-8")
}

func HTTPRespondStr(w http.ResponseWriter, code int, body string) {
	if len(body) == 0 {
		panic("body must be not empty")
	}
	w.WriteHeader(code)
	fmt.Fprint(w, body)
}

func HTTPRespondJSONObj(w http.ResponseWriter, code int, obj interface{}) {
	HTTPSetContentTypeJSON(w)
	w.WriteHeader(code)
	ErrPanic(json.NewEncoder(w).Encode(obj))
}

func HTTPRespondJSONParseError(w http.ResponseWriter) {
	HTTPRespond400(w, "bad_json", "Fail to parse JSON")
}

func HTTPSendRequest(withJar bool, method, url string, urlParams map[string]string,
	data []byte, timeout time.Duration, headers ...string) (*http.Response, error) {
	var err error
	var req *http.Request
	var jar http.CookieJar

	if data != nil {
		req, err = http.NewRequest(method, url, bytes.NewBuffer(data))
		ErrPanic(err)
	} else {
		req, err = http.NewRequest(method, url, nil)
		ErrPanic(err)
	}

	if urlParams != nil {
		q := netUrl.Values{}
		for k, v := range urlParams {
			q.Add(k, v)
		}
		req.URL.RawQuery = q.Encode()
	}

	for i := 0; (i + 1) < len(headers); i += 2 {
		req.Header.Set(headers[i], headers[i+1])
	}

	if withJar {
		jar, err = cookiejar.New(nil)
		ErrPanic(err)
	}

	client := http.Client{
		Timeout: timeout,
		Jar:     jar,
	}

	return client.Do(req)
}

func HTTPSendRequestReceiveBytes(withJar, errSCode bool, method, url string, urlParams map[string]string,
	data []byte, timeout time.Duration, headers ...string) (int, []byte, error) {
	var res []byte

	resp, err := HTTPSendRequest(withJar, method, url, urlParams, data, timeout, headers...)
	if err != nil {
		return 0, nil, err
	}
	defer resp.Body.Close()

	res, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return 0, nil, err
	}

	if !HTTPStatusCodeIsOk(resp.StatusCode) {
		if errSCode {
			return resp.StatusCode, res, errors.New(fmt.Sprintf("bad_http_status_code - %d\nbody: %s", resp.StatusCode, string(res)))
		}
		return resp.StatusCode, res, nil
	}

	return resp.StatusCode, res, nil
}

func HTTPSendRequestReceiveString(withJar, errSCode bool, method, url string, urlParams map[string]string,
	data []byte, timeout time.Duration, headers ...string) (int, string, error) {
	sCode, resBytes, err := HTTPSendRequestReceiveBytes(withJar, errSCode, method, url, urlParams, data, timeout, headers...)

	return sCode, string(resBytes), err
}

func HTTPSendRequestReceiveJSONObj(withJar, errSCode bool, method, url string, urlParams map[string]string,
	data []byte, rObj interface{}, timeout time.Duration, headers ...string) (int, []byte, error) {
	sCode, rBytes, err := HTTPSendRequestReceiveBytes(
		withJar, errSCode, method, url, urlParams, data, timeout, headers...)
	if err != nil || !HTTPStatusCodeIsOk(sCode) {
		return sCode, rBytes, err
	}

	err = json.Unmarshal(rBytes, rObj)
	if err != nil {
		return sCode, rBytes, errors.New(fmt.Sprintf("fail_to_parse_json - %s\nbody: %s", err.Error(), string(rBytes)))
	}

	return sCode, rBytes, nil
}

func HTTPSendJSONObjRequest(withJar bool, method, url string, urlParams map[string]string,
	sObj interface{}, timeout time.Duration, headers ...string) (*http.Response, error) {
	sBytes, err := json.Marshal(sObj)
	if err != nil {
		return nil, err
	}

	return HTTPSendRequest(withJar, method, url, urlParams, sBytes, timeout, headers...)
}

func HTTPSendJSONObjRequestReceiveBytes(withJar, errSCode bool, method, url string, urlParams map[string]string,
	sObj interface{}, timeout time.Duration, headers ...string) (int, []byte, error) {
	sBytes, err := json.Marshal(sObj)
	if err != nil {
		return 0, nil, err
	}

	return HTTPSendRequestReceiveBytes(withJar, errSCode, method, url, urlParams, sBytes, timeout, headers...)
}

func HTTPSendJSONObjRequestReceiveString(withJar, errSCode bool, method, url string, urlParams map[string]string,
	sObj interface{}, timeout time.Duration, headers ...string) (int, string, error) {
	sBytes, err := json.Marshal(sObj)
	if err != nil {
		return 0, "", err
	}

	return HTTPSendRequestReceiveString(withJar, errSCode, method, url, urlParams, sBytes, timeout, headers...)
}

func HTTPSendJSONObjRequestReceiveJSONObj(withJar, errSCode bool, method, url string, urlParams map[string]string,
	sObj interface{}, rObj interface{}, timeout time.Duration, headers ...string) (int, []byte, error) {
	sBytes, err := json.Marshal(sObj)
	if err != nil {
		return 0, nil, err
	}

	return HTTPSendRequestReceiveJSONObj(withJar, errSCode, method, url, urlParams, sBytes, rObj, timeout, headers...)
}

func HTTPRetrieveRequestHostURL(r *http.Request) string {
	scheme := r.Header.Get("X-Forwarded-Proto")
	if scheme == "" {
		if r.TLS == nil {
			scheme = "http"
		} else {
			scheme = "https"
		}
	}
	return scheme + "://" + r.Host
}

func HTTPRetrieveRemoteIP(r *http.Request) (result string) {
	result = ""
	if parts := strings.Split(r.RemoteAddr, ":"); len(parts) == 2 {
		result = parts[0]
	}
	// If we have a forwarded-for header, take the address from there
	if xff := strings.Trim(r.Header.Get("X-Forwarded-For"), ","); len(xff) > 0 {
		addrs := strings.Split(xff, ",")
		lastFwd := addrs[len(addrs)-1]
		if ip := net.ParseIP(lastFwd); ip != nil {
			result = ip.String()
		}
		// parse X-Real-Ip header
	} else if xri := r.Header.Get("X-Real-Ip"); len(xri) > 0 {
		if ip := net.ParseIP(xri); ip != nil {
			result = ip.String()
		}
	}
	return
}

func HTTPUploadFileFromRequestForm(r *http.Request, key, dirPath, dir string, filename string) (string, error) {
	var err error

	finalDirPath := filepath.Join(dirPath, dir)

	err = os.MkdirAll(finalDirPath, os.ModePerm)
	if err != nil {
		return "", err
	}

	srcFile, header, err := r.FormFile(key)
	if err != nil {
		return "", err
	}
	defer srcFile.Close()

	fileExt := filepath.Ext(header.Filename)
	if fileExt == "" {
		return "", errors.New("bad_extension")
	}

	dstFile, err := TempFile(finalDirPath, filename+"_*"+fileExt)
	if err != nil {
		return "", err
	}
	defer dstFile.Close()

	_, err = io.Copy(dstFile, srcFile)
	if err != nil {
		return "", err
	}

	err = os.Chmod(dstFile.Name(), 0644)
	if err != nil {
		return "", err
	}

	newName, err := filepath.Rel(dirPath, dstFile.Name())
	if err != nil {
		return "", err
	}

	return newName, nil
}

func HTTPRespondError(w http.ResponseWriter, code int, err string, detail string, extras ...interface{}) {
	obj := map[string]interface{}{}
	obj["error"] = err
	obj["error_dsc"] = detail
	for i := 0; (i + 1) < len(extras); i += 2 {
		obj[extras[i].(string)] = extras[i+1]
	}
	HTTPRespondJSONObj(w, code, obj)
}

func HTTPRespond400(w http.ResponseWriter, err, detail string, extras ...interface{}) {
	HTTPRespondError(w, 400, err, detail, extras...)
}

func HTTPRespond401(w http.ResponseWriter, detail string) {
	HTTPRespondError(w, 401, "unauthorized", detail)
}

func HTTPRespond403(w http.ResponseWriter, detail string) {
	HTTPRespondError(w, 403, "permission_denied", detail)
}

func HTTPRespond404(w http.ResponseWriter, detail string) {
	HTTPRespondError(w, 404, "not_found", detail)
}
<commit_msg>upgrade http<commit_after>package lily

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/cookiejar"
	netUrl "net/url"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func HTTPMwCORSAllowAll(h http.Handler, maxAge string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Vary", "Origin")
		w.Header().Set("Access-Control-Allow-Origin", "*")
		if r.Method == "OPTIONS" {
			w.Header().Set("Access-Control-Allow-Methods", "GET, HEAD, POST, PUT, DELETE, TRACE, CONNECT, OPTIONS")
			// headers
			requestHeaders := strings.Split(r.Header.Get("Access-Control-Request-Headers"), ",")
			var allowedHeaders []string
			for _, v := range requestHeaders {
				allowedHeaders = append(allowedHeaders, http.CanonicalHeaderKey(strings.TrimSpace(v)))
			}
			if len(allowedHeaders) > 0 {
				w.Header().Set("Access-Control-Allow-Headers", strings.Join(allowedHeaders, ","))
			}
			w.Header().Set("Access-Control-Max-Age", maxAge)
		} else {
			h.ServeHTTP(w, r)
		}
	})
}

func HTTPStatusCodeIsOk(statusCode int) bool {
	return statusCode > 199 && statusCode < 300
}

func HTTPSetContentTypeJSON(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
}

func HTTPSetContentTypeHTML(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "text/html; charset=UTF-8")
}

func HTTPRespondStr(w http.ResponseWriter, code int, body string) {
	if len(body) == 0 {
		panic("body must be not empty")
	}
	w.WriteHeader(code)
	fmt.Fprint(w, body)
}

func HTTPRespondJSONObj(w http.ResponseWriter, code int, obj interface{}) {
	HTTPSetContentTypeJSON(w)
	w.WriteHeader(code)
	ErrPanic(json.NewEncoder(w).Encode(obj))
}

func HTTPRespondJSONParseError(w http.ResponseWriter) {
	HTTPRespond400(w, "bad_json", "Fail to parse JSON")
}

func HTTPSendRequest(withJar bool, method, url string, urlParams map[string]string,
	data []byte, timeout time.Duration, headers ...string) (*http.Response, error) {
	var err error
	var req *http.Request
	var jar http.CookieJar

	if data != nil {
		req, err = http.NewRequest(method, url, bytes.NewBuffer(data))
		ErrPanic(err)
	} else {
		req, err = http.NewRequest(method, url, nil)
		ErrPanic(err)
	}

	if urlParams != nil {
		q := netUrl.Values{}
		for k, v := range urlParams {
			q.Add(k, v)
		}
		req.URL.RawQuery = q.Encode()
	}

	for i := 0; (i + 1) < len(headers); i += 2 {
		req.Header.Set(headers[i], headers[i+1])
	}

	if withJar {
		jar, err = cookiejar.New(nil)
		ErrPanic(err)
	}

	client := http.Client{
		Timeout: timeout,
		Jar:     jar,
	}

	return client.Do(req)
}

func HTTPSendRequestReceiveBytes(withJar, errSCode bool, method, url string, urlParams map[string]string,
	data []byte, timeout time.Duration, headers ...string) (int, []byte, error) {
	var res []byte

	resp, err := HTTPSendRequest(withJar, method, url, urlParams, data, timeout, headers...)
	if err != nil {
		return 0, nil, err
	}
	defer resp.Body.Close()

	res, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return 0, nil, err
	}

	if !HTTPStatusCodeIsOk(resp.StatusCode) {
		if errSCode {
			return resp.StatusCode, res, errors.New(fmt.Sprintf("bad_http_status_code - %d\nbody: %s", resp.StatusCode, string(res)))
		}
		return resp.StatusCode, res, nil
	}

	return resp.StatusCode, res, nil
}

func HTTPSendRequestReceiveString(withJar, errSCode bool, method, url string, urlParams map[string]string,
	data []byte, timeout time.Duration, headers ...string) (int, string, error) {
	sCode, resBytes, err := HTTPSendRequestReceiveBytes(withJar, errSCode, method, url, urlParams, data, timeout, headers...)

	return sCode, string(resBytes), err
}

func HTTPSendRequestReceiveJSONObj(withJar, errSCode bool, method, url string, urlParams map[string]string,
	data []byte, rObj interface{}, timeout time.Duration, headers ...string) (int, []byte, error) {
	sCode, rBytes, err := HTTPSendRequestReceiveBytes(
		withJar, errSCode, method, url, urlParams, data, timeout, headers...)
	if err != nil || !HTTPStatusCodeIsOk(sCode) {
		return sCode, rBytes, err
	}

	err = json.Unmarshal(rBytes, rObj)
	if err != nil {
		return sCode, rBytes, errors.New(fmt.Sprintf("fail_to_parse_json - %s\nbody: %s", err.Error(), string(rBytes)))
	}

	return sCode, rBytes, nil
}

func HTTPSendJSONObjRequest(withJar bool, method, url string, urlParams map[string]string,
	sObj interface{}, timeout time.Duration, headers ...string) (*http.Response, error) {
	sBytes, err := json.Marshal(sObj)
	if err != nil {
		return nil, err
	}

	return HTTPSendRequest(withJar, method, url, urlParams, sBytes, timeout, headers...)
}

func HTTPSendJSONObjRequestReceiveBytes(withJar, errSCode bool, method, url string, urlParams map[string]string,
	sObj interface{}, timeout time.Duration, headers ...string) (int, []byte, error) {
	sBytes, err := json.Marshal(sObj)
	if err != nil {
		return 0, nil, err
	}

	return HTTPSendRequestReceiveBytes(withJar, errSCode, method, url, urlParams, sBytes, timeout, headers...)
}

func HTTPSendJSONObjRequestReceiveString(withJar, errSCode bool, method, url string, urlParams map[string]string,
	sObj interface{}, timeout time.Duration, headers ...string) (int, string, error) {
	sBytes, err := json.Marshal(sObj)
	if err != nil {
		return 0, "", err
	}

	return HTTPSendRequestReceiveString(withJar, errSCode, method, url, urlParams, sBytes, timeout, headers...)
}

func HTTPSendJSONObjRequestReceiveJSONObj(withJar, errSCode bool, method, url string, urlParams map[string]string,
	sObj interface{}, rObj interface{}, timeout time.Duration, headers ...string) (int, []byte, error) {
	sBytes, err := json.Marshal(sObj)
	if err != nil {
		return 0, nil, err
	}

	return HTTPSendRequestReceiveJSONObj(withJar, errSCode, method, url, urlParams, sBytes, rObj, timeout, headers...)
}

func HTTPRetrieveRequestHostURL(r *http.Request) string {
	scheme := r.Header.Get("X-Forwarded-Proto")
	if scheme == "" {
		if r.TLS == nil {
			scheme = "http"
		} else {
			scheme = "https"
		}
	}
	return scheme + "://" + r.Host
}

func HTTPRetrieveRemoteIP(r *http.Request) (result string) {
	result = ""
	if parts := strings.Split(r.RemoteAddr, ":"); len(parts) == 2 {
		result = parts[0]
	}
	// If we have a forwarded-for header, take the address from there
	if xff := strings.Trim(r.Header.Get("X-Forwarded-For"), ","); len(xff) > 0 {
		addrs := strings.Split(xff, ",")
		lastFwd := addrs[len(addrs)-1]
		if ip := net.ParseIP(lastFwd); ip != nil {
			result = ip.String()
		}
		// parse X-Real-Ip header
	} else if xri := r.Header.Get("X-Real-Ip"); len(xri) > 0 {
		if ip := net.ParseIP(xri); ip != nil {
			result = ip.String()
		}
	}
	return
}

func HTTPUploadFileFromRequestForm(r *http.Request, key, dirPath, dir string, filename string) (string, error) {
	var err error

	finalDirPath := filepath.Join(dirPath, dir)

	err = os.MkdirAll(finalDirPath, os.ModePerm)
	if err != nil {
		return "", err
	}

	srcFile, header, err := r.FormFile(key)
	if err != nil {
		return "", err
	}
	defer srcFile.Close()

	fileExt := filepath.Ext(header.Filename)
	if fileExt == "" {
		return "", errors.New("bad_extension")
	}

	dstFile, err := TempFile(finalDirPath, filename+"_*"+fileExt)
	if err != nil {
		return "", err
	}
	defer dstFile.Close()

	_, err = io.Copy(dstFile, srcFile)
	if err != nil {
		return "", err
	}

	err = os.Chmod(dstFile.Name(), 0644)
	if err != nil {
		return "", err
	}

	newName, err := filepath.Rel(dirPath, dstFile.Name())
	if err != nil {
		return "", err
	}

	return newName, nil
}

func HTTPRespondError(w http.ResponseWriter, code int, err string, detail string, extras ...interface{}) {
	obj := map[string]interface{}{}
	obj["error"] = err
	obj["error_dsc"] = detail
	for i := 0; (i + 1) < len(extras); i += 2 {
		obj[extras[i].(string)] = extras[i+1]
	}
	HTTPRespondJSONObj(w, code, obj)
}

func HTTPRespond400(w http.ResponseWriter, err, detail string, extras ...interface{}) {
	HTTPRespondError(w, 400, err, detail, extras...)
}

func HTTPRespond401(w http.ResponseWriter, detail string) {
	HTTPRespondError(w, 401, "unauthorized", detail)
}

func HTTPRespond403(w http.ResponseWriter, detail string) {
	HTTPRespondError(w, 403, "permission_denied", detail)
}

func HTTPRespond404(w http.ResponseWriter, detail string) {
	HTTPRespondError(w, 404, "not_found", detail)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 ENDOH takanao.
<https://github.com/MiCHiLU/go-lru-cache-stats>

Copyright 2013 Google Inc.
<https://github.com/golang/groupcache>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lru

import (
	"bytes"
	"net/http"
	"sync"
)

const defaultBasePath = "/_groupcache/"

const defaultReplicas = 50

// HTTPPoolOptions are the configurations of a HTTPPool.
type HTTPPoolOptions struct {
	// BasePath specifies the HTTP path that will serve groupcache requests.
	// If blank, it defaults to "/_groupcache/".
	BasePath string

	// Replicas specifies the number of key replicas on the consistent hash.
	// If blank, it defaults to 50.
	Replicas int
}

var httpPoolMade bool

type httpGetter struct {
	transport func(Context) http.RoundTripper
	baseURL   string
}

var bufferPool = sync.Pool{
	New: func() interface{} { return new(bytes.Buffer) },
}
<commit_msg>remove consts<commit_after>/*
Copyright 2015 ENDOH takanao.
<https://github.com/MiCHiLU/go-lru-cache-stats>

Copyright 2013 Google Inc.
<https://github.com/golang/groupcache>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lru

import (
	"bytes"
	"net/http"
	"sync"
)

// HTTPPoolOptions are the configurations of a HTTPPool.
type HTTPPoolOptions struct {
	// BasePath specifies the HTTP path that will serve groupcache requests.
	// If blank, it defaults to "/_groupcache/".
	BasePath string

	// Replicas specifies the number of key replicas on the consistent hash.
	// If blank, it defaults to 50.
	Replicas int
}

var httpPoolMade bool

type httpGetter struct {
	transport func(Context) http.RoundTripper
	baseURL   string
}

var bufferPool = sync.Pool{
	New: func() interface{} { return new(bytes.Buffer) },
}
<|endoftext|>