<commit_before>// Copyright 2019 xgfone
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package execution executes a command line program in a new process and returns a output.
package execution

import (
	"bytes"
	"context"
	"errors"
	"os/exec"
	"time"
)

// ErrDeny is returned when the hook denies the cmd.
var ErrDeny = errors.New("the cmd is denied")

// Hook is used to filter or handle the cmd `name` with the arguments `args`.
//
// If returning true, it will continue to run it, or do nothing.
type Hook func(name string, args ...string) bool

// Cmd represents a command executor.
type Cmd struct {
	hooks []Hook

	// Timeout is used to produce the timeout context based on the context
	// argument if not 0 when executing the command.
	Timeout time.Duration

	// SetCmd allows the user to customize exec.Cmd.
	//
	// Notice: You should not modify the fields `Stdout` and `Stderr`.
	SetCmd func(*exec.Cmd)
}

// NewCmd returns a new executor Cmd.
func NewCmd() *Cmd {
	return new(Cmd)
}

// AppendHooks appends some hooks.
func (c *Cmd) AppendHooks(hooks ...Hook) *Cmd {
	for _, hook := range hooks {
		if hook != nil {
			c.hooks = append(c.hooks, hook)
		}
	}
	return c
}

func geterr(stdout, stderr []byte, err error) error {
	if err != nil {
		if len(stderr) > 0 {
			err = errors.New(string(stderr))
		} else if len(stdout) > 0 {
			err = errors.New(string(stdout))
		}
	}
	return err
}

// RunCmd executes the command, name, with its arguments, args,
// then returns stdout, stderr and error.
func (c *Cmd) RunCmd(cxt context.Context, name string, args ...string) (
	stdout, stderr []byte, err error) {

	for _, hook := range c.hooks {
		if ok := hook(name, args...); !ok {
			return nil, nil, ErrDeny
		}
	}

	var cancel func()
	if c.Timeout > 0 {
		cxt, cancel = context.WithTimeout(cxt, c.Timeout)
		defer cancel()
	}

	cmd := exec.CommandContext(cxt, name, args...)
	if c.SetCmd != nil {
		c.SetCmd(cmd)
	}

	var output bytes.Buffer
	var errput bytes.Buffer
	cmd.Stdout = &output
	cmd.Stderr = &errput
	err = cmd.Run()
	stdout = output.Bytes()
	stderr = errput.Bytes()
	err = geterr(stdout, stderr, err)
	return
}

// Run is the alias of RunCmd.
func (c *Cmd) Run(ctx context.Context, name string, args ...string) (
	stdout, stderr []byte, err error) {
	return c.RunCmd(ctx, name, args...)
}

// RetryRunCmd is the same as RunCmd, but try to run once again if failed.
func (c *Cmd) RetryRunCmd(ctx context.Context, name string, args ...string) (
	stdout, stderr []byte, err error) {
	stdout, stderr, err = c.RunCmd(ctx, name, args...)
	if err != nil {
		stdout, stderr, err = c.RunCmd(ctx, name, args...)
	}
	return
}

// Execute is the same as RunCmd, but only returns the error.
func (c *Cmd) Execute(cxt context.Context, name string, args ...string) error {
	_, _, err := c.RunCmd(cxt, name, args...)
	return err
}

// Output is the same as RunCmd, but only returns the stdout and the error.
func (c *Cmd) Output(cxt context.Context, name string, args ...string) (string, error) {
	stdout, _, err := c.RunCmd(cxt, name, args...)
	return string(stdout), err
}

// Executes is equal to Execute(cxt, cmds[0], cmds[1:]...)
func (c *Cmd) Executes(cxt context.Context, cmds []string) error {
	_, _, err := c.RunCmd(cxt, cmds[0], cmds[1:]...)
	return err
}

// Outputs is equal to Output(cxt, cmds[0], cmds[1:]...).
func (c *Cmd) Outputs(cxt context.Context, cmds []string) (string, error) {
	stdout, _, err := c.RunCmd(cxt, cmds[0], cmds[1:]...)
	return string(stdout), err
}
<commit_msg>export the field Hooks of Cmd<commit_after>// Copyright 2019 xgfone
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package execution executes a command line program in a new process and returns a output.
package execution

import (
	"bytes"
	"context"
	"errors"
	"os/exec"
	"time"
)

// ErrDeny is returned when the hook denies the cmd.
var ErrDeny = errors.New("the cmd is denied")

// Hook is used to filter or handle the cmd `name` with the arguments `args`.
//
// If returning true, it will continue to run it, or do nothing.
type Hook func(name string, args ...string) bool

// Cmd represents a command executor.
type Cmd struct {
	Hooks []Hook

	// Timeout is used to produce the timeout context based on the context
	// argument if not 0 when executing the command.
	Timeout time.Duration

	// SetCmd allows the user to customize exec.Cmd.
	//
	// Notice: You should not modify the fields `Stdout` and `Stderr`.
	SetCmd func(*exec.Cmd)
}

// NewCmd returns a new executor Cmd.
func NewCmd() *Cmd {
	return new(Cmd)
}

// AppendHooks appends some hooks.
func (c *Cmd) AppendHooks(hooks ...Hook) *Cmd {
	for _, hook := range hooks {
		if hook != nil {
			c.Hooks = append(c.Hooks, hook)
		}
	}
	return c
}

func geterr(stdout, stderr []byte, err error) error {
	if err != nil {
		if len(stderr) > 0 {
			err = errors.New(string(stderr))
		} else if len(stdout) > 0 {
			err = errors.New(string(stdout))
		}
	}
	return err
}

// RunCmd executes the command, name, with its arguments, args,
// then returns stdout, stderr and error.
func (c *Cmd) RunCmd(cxt context.Context, name string, args ...string) (
	stdout, stderr []byte, err error) {

	for _, hook := range c.Hooks {
		if ok := hook(name, args...); !ok {
			return nil, nil, ErrDeny
		}
	}

	var cancel func()
	if c.Timeout > 0 {
		cxt, cancel = context.WithTimeout(cxt, c.Timeout)
		defer cancel()
	}

	cmd := exec.CommandContext(cxt, name, args...)
	if c.SetCmd != nil {
		c.SetCmd(cmd)
	}

	var output bytes.Buffer
	var errput bytes.Buffer
	cmd.Stdout = &output
	cmd.Stderr = &errput
	err = cmd.Run()
	stdout = output.Bytes()
	stderr = errput.Bytes()
	err = geterr(stdout, stderr, err)
	return
}

// Run is the alias of RunCmd.
func (c *Cmd) Run(ctx context.Context, name string, args ...string) (
	stdout, stderr []byte, err error) {
	return c.RunCmd(ctx, name, args...)
}

// RetryRunCmd is the same as RunCmd, but try to run once again if failed.
func (c *Cmd) RetryRunCmd(ctx context.Context, name string, args ...string) (
	stdout, stderr []byte, err error) {
	stdout, stderr, err = c.RunCmd(ctx, name, args...)
	if err != nil {
		stdout, stderr, err = c.RunCmd(ctx, name, args...)
	}
	return
}

// Execute is the same as RunCmd, but only returns the error.
func (c *Cmd) Execute(cxt context.Context, name string, args ...string) error {
	_, _, err := c.RunCmd(cxt, name, args...)
	return err
}

// Output is the same as RunCmd, but only returns the stdout and the error.
func (c *Cmd) Output(cxt context.Context, name string, args ...string) (string, error) {
	stdout, _, err := c.RunCmd(cxt, name, args...)
	return string(stdout), err
}

// Executes is equal to Execute(cxt, cmds[0], cmds[1:]...)
func (c *Cmd) Executes(cxt context.Context, cmds []string) error {
	_, _, err := c.RunCmd(cxt, cmds[0], cmds[1:]...)
	return err
}

// Outputs is equal to Output(cxt, cmds[0], cmds[1:]...).
func (c *Cmd) Outputs(cxt context.Context, cmds []string) (string, error) {
	stdout, _, err := c.RunCmd(cxt, cmds[0], cmds[1:]...)
	return string(stdout), err
}
<|endoftext|>


<commit_before>// Copyright 2017, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package grpc-stats-collection<commit_msg>Deleted not used directories<commit_after><|endoftext|>


<commit_before>package nca

import (
	log "gopkg.in/inconshreveable/log15.v2"
	"os"
	"os/signal"
)

// Runner is the main component of the application. It runs the Checker and
// Publisher subcomponents and facilitates communication between them.
type Runner struct {
	config      Config
	checker     *checker
	publishers  map[string]Publisher
	publishChan chan *CheckResult // The cchannel check results are received from
	log         log.Logger
	done        chan struct{} // Used for signalling goroutines that we're shutting down
}

// NewRunner creates a new Runner with the given configuration.
func NewRunner(cfg Config) (*Runner, error) {
	r := &Runner{}
	if err := r.Init(cfg); err != nil {
		return nil, err
	}
	return r, nil
}

// NewRunnerFromFile creates a new Runner using the configuration loaded
// from the given file.
func NewRunnerFromFile(filename string) (*Runner, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	cfg, err := ReadConfig(f)
	if err != nil {
		return nil, err
	}
	r, err := NewRunner(*cfg)
	if err != nil {
		return nil, err
	}
	return r, nil
}

// Init initializes the Runner with the given configuration.
func (r *Runner) Init(cfg Config) error {
	r.config = cfg
	r.log = Log.New("component", "runner")
	r.checker = &checker{}
	r.checker.RegisterChecks(r.config.Checks)

	r.publishers = make(map[string]Publisher)
	for label, config := range cfg.Publishers {
		ptype := config["type"].(string)
		p := newPublisher(ptype)
		if err := p.Configure(config); err != nil {
			r.log.Error("Invalid publisher configuration", "publisher", label, "error", err)
			return err
		}
		r.publishers[label] = p
	}

	return nil
}

// Start starts the Runner and begins running checks, publishing them as they
// complete.
func (r *Runner) Start() error {
	r.log.Info("Runner starting")
	r.done = make(chan struct{})

	r.publishChan, _ = r.checker.Start()
	for _, publisher := range r.publishers {
		publisher.Start()
	}
	go r.process()
	return nil
}

// process reads results produced by the checker and distributes them
// to the publishers.
func (r *Runner) process() {
	r.log.Debug("process() loop start")
	for {
		select {
		case result := <-r.publishChan:
			if result == nil {
				continue
			}
			for label, publisher := range r.publishers {
				l := r.log.New("check", result.Name, "publisher", label)
				l.Debug("Publishing check result to publisher")
				if err := publisher.Publish(result); err != nil {
					l.Warn("Error publishing check result", "error", err)
				} else {
					l.Debug("Check result published")
				}
			}
		case <-r.done:
			return
		}
	}
	r.log.Debug("process() loop end")
}

// Stop stops and shuts down the Runner.
func (r *Runner) Stop() error {
	r.log.Info("Runner stopping")
	r.checker.Stop()
	for _, publisher := range r.publishers {
		publisher.Stop()
	}
	return nil
}

// Run starts the runner and blocks until an interrupt signal is received.
func (r *Runner) Run() error {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)

	r.Start()
	for sig := range c {
		if sig == os.Interrupt {
			r.log.Info("Received interrupt, shutting down")
			r.Stop()
			break
		}
	}

	return nil
}
<commit_msg>Publish results in seperate goroutines<commit_after>package nca

import (
	log "gopkg.in/inconshreveable/log15.v2"
	"os"
	"os/signal"
)

// Runner is the main component of the application. It runs the Checker and
// Publisher subcomponents and facilitates communication between them.
type Runner struct {
	config      Config
	checker     *checker
	publishers  map[string]Publisher
	publishChan chan *CheckResult // The cchannel check results are received from
	log         log.Logger
	done        chan struct{} // Used for signalling goroutines that we're shutting down
}

// NewRunner creates a new Runner with the given configuration.
func NewRunner(cfg Config) (*Runner, error) {
	r := &Runner{}
	if err := r.Init(cfg); err != nil {
		return nil, err
	}
	return r, nil
}

// NewRunnerFromFile creates a new Runner using the configuration loaded
// from the given file.
func NewRunnerFromFile(filename string) (*Runner, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	cfg, err := ReadConfig(f)
	if err != nil {
		return nil, err
	}
	r, err := NewRunner(*cfg)
	if err != nil {
		return nil, err
	}
	return r, nil
}

// Init initializes the Runner with the given configuration.
func (r *Runner) Init(cfg Config) error {
	r.config = cfg
	r.log = Log.New("component", "runner")
	r.checker = &checker{}
	r.checker.RegisterChecks(r.config.Checks)

	r.publishers = make(map[string]Publisher)
	for label, config := range cfg.Publishers {
		ptype := config["type"].(string)
		p := newPublisher(ptype)
		if err := p.Configure(config); err != nil {
			r.log.Error("Invalid publisher configuration", "publisher", label, "error", err)
			return err
		}
		r.publishers[label] = p
	}

	return nil
}

// Start starts the Runner and begins running checks, publishing them as they
// complete.
func (r *Runner) Start() error {
	r.log.Info("Runner starting")
	r.done = make(chan struct{})

	r.publishChan, _ = r.checker.Start()
	for _, publisher := range r.publishers {
		publisher.Start()
	}
	go r.process()
	return nil
}

// process reads results produced by the checker and distributes them
// to the publishers.
func (r *Runner) process() {
	r.log.Debug("process() loop start")
	for {
		select {
		case result := <-r.publishChan:
			if result == nil {
				continue
			}
			for label, publisher := range r.publishers {
				go func() {
					l := r.log.New("check", result.Name, "publisher", label)
					l.Debug("Publishing check result to publisher")
					if err := publisher.Publish(result); err != nil {
						l.Warn("Error publishing check result", "error", err)
					} else {
						l.Debug("Check result published")
					}
				}()
			}
		case <-r.done:
			return
		}
	}
	r.log.Debug("process() loop end")
}

// Stop stops and shuts down the Runner.
func (r *Runner) Stop() error {
	r.log.Info("Runner stopping")
	r.checker.Stop()
	for _, publisher := range r.publishers {
		publisher.Stop()
	}
	return nil
}

// Run starts the runner and blocks until an interrupt signal is received.
func (r *Runner) Run() error {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)

	r.Start()
	for sig := range c {
		if sig == os.Interrupt {
			r.log.Info("Received interrupt, shutting down")
			r.Stop()
			break
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package server

import (
	. "github.com/smartystreets/goconvey/convey"
	"gopkg.in/h2non/gock.v1"
	"net/http"
	"testing"
)

func TestRouter(t *testing.T) {
	app := App()

	Convey("the index page", t, func() {

		Convey("should return OK", func() {
			req, _ := http.NewRequest("GET", "/", nil)
			res := executeRequest(app, req)

			So(res.Code, ShouldEqual, http.StatusOK)
			So(res.Body.String(), ShouldEqual, "Welcome!")
		})

	})

	Convey("CAS authentication", t, func() {

		Convey("should redirect to CAS when not logged in", func() {
			gock.New("https://fed.princeton.edu").
				Get("/cas/validate").
				Reply(200).
				BodyString("no\n\n")
			defer gock.Off()

			req, _ := http.NewRequest("GET", "/login", nil)
			res := executeRequest(app, req)

			So(res.Code, ShouldEqual, http.StatusFound)
		})

		// TODO: Figure out why this test doesn't work
		SkipConvey("should return the proper user when logged in", func() {
			gock.New("https://fed.princeton.edu").
				Get("/cas/validate").
				Reply(200).
				BodyString("yes\ntestuser\n")
			defer gock.Off()

			req, _ := http.NewRequest("GET", "/login", nil)
			res := executeRequest(app, req)

			So(res.Code, ShouldEqual, http.StatusOK)
			So(res.Body.String(), ShouldEqual, "testuser")
		})

	})
}
<commit_msg>add CORS tests<commit_after>package server

import (
	. "github.com/smartystreets/goconvey/convey"
	"gopkg.in/h2non/gock.v1"
	"net/http"
	"os"
	"testing"
)

func TestRouter(t *testing.T) {
	app := App()

	Convey("the index page", t, func() {

		Convey("should return OK", func() {
			req, _ := http.NewRequest("GET", "/", nil)
			res := executeRequest(app, req)

			So(res.Code, ShouldEqual, http.StatusOK)
			So(res.Body.String(), ShouldEqual, "Welcome!")
		})

		Convey("should have CORS headers", func() {
			origin := os.Getenv("CLIENT_ROOT")
			req, _ := http.NewRequest("GET", "/", nil)
			req.Header.Add("Origin", origin)
			res := executeRequest(app, req)

			So(res.Header().Get("Access-Control-Allow-Origin"), ShouldEqual, origin)
		})

		Convey("should not have global CORS headers", func() {
			req, _ := http.NewRequest("GET", "/", nil)
			req.Header.Add("Origin", "http://example.com/")
			res := executeRequest(app, req)

			So(res.Header().Get("Access-Control-Allow-Origin"), ShouldEqual, "")
		})

	})

	Convey("CAS authentication", t, func() {

		Convey("should redirect to CAS when not logged in", func() {
			gock.New("https://fed.princeton.edu").
				Get("/cas/validate").
				Reply(200).
				BodyString("no\n\n")
			defer gock.Off()

			req, _ := http.NewRequest("GET", "/login", nil)
			res := executeRequest(app, req)

			So(res.Code, ShouldEqual, http.StatusFound)
		})

		// TODO: Figure out why this test doesn't work
		SkipConvey("should return the proper user when logged in", func() {
			gock.New("https://fed.princeton.edu").
				Get("/cas/validate").
				Reply(200).
				BodyString("yes\ntestuser\n")
			defer gock.Off()

			req, _ := http.NewRequest("GET", "/login", nil)
			res := executeRequest(app, req)

			So(res.Code, ShouldEqual, http.StatusOK)
			So(res.Body.String(), ShouldEqual, "testuser")
		})

	})
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
  "bytes"
  "strings"

	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/pebbe/util"
	"github.com/intervention-engine/fhir/models"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
  . "gopkg.in/check.v1"
)

type ServerSuite struct {
	Session   *mgo.Session
	Router    *mux.Router
	Server    *httptest.Server
	FixtureId string
}

func Test(t *testing.T) { TestingT(t) }

var _ = Suite(&ServerSuite{})

func (s *ServerSuite) SetUpSuite(c *C) {

	// Set up the database
  var err error
	s.Session, err = mgo.Dial("localhost")
	util.CheckErr(err)
	Database = s.Session.DB("fhir-test")

	// Build routes for testing
	s.Router = mux.NewRouter()
	s.Router.StrictSlash(true)
	s.Router.KeepContext = true
	RegisterRoutes(s.Router, make(map[string][]negroni.Handler))

	// Create httptest server
	s.Server = httptest.NewServer(s.Router)

	// Add patient fixture
	patientCollection := Database.C("patients")
	patient := LoadPatientFromFixture("../fixtures/patient-example-a.json")
	i := bson.NewObjectId()
	s.FixtureId = i.Hex()
	patient.Id = s.FixtureId
	err = patientCollection.Insert(patient)
	util.CheckErr(err)
}

func (s *ServerSuite) TearDownSuite(c *C) {
  Database.DropDatabase()
  s.Session.Close()
	s.Server.Close()
}

func (s *ServerSuite) TestGetPatient(c *C) {
	res, err := http.Get(s.Server.URL + "/Patient/" + s.FixtureId)
	util.CheckErr(err)

  decoder := json.NewDecoder(res.Body)
  patient := &models.Patient{}
  err = decoder.Decode(patient)
  util.CheckErr(err)
  c.Assert(patient.Name[0].Family[0], Equals, "Donald")
}

func (s *ServerSuite) TestShowPatient(c *C) {
  res, err := http.Get(s.Server.URL + "/Patient")
  util.CheckErr(err)

  decoder := json.NewDecoder(res.Body)
  patientBundle := &models.PatientBundle{}
  err = decoder.Decode(patientBundle)
  util.CheckErr(err)

  var result []models.Patient
  collection := Database.C("patients")
  iter := collection.Find(nil).Iter()
  err = iter.All(&result)
  util.CheckErr(err)

  c.Assert(patientBundle.TotalResults, Equals, len(result))
  c.Assert(patientBundle.Title, Equals, "Patient Index")
}

func (s *ServerSuite) TestCreatePatient(c *C) {
  createPatient := LoadPatientFromFixture("../fixtures/patient-example-b.json")
  var buf bytes.Buffer
  encoder := json.NewEncoder(&buf)
  encoder.Encode(createPatient)
  res, err := http.Post(s.Server.URL + "/Patient", "application/json", &buf)
  util.CheckErr(err)

  splitLocation := strings.Split(res.Header["Location"][0], "/")
  createdPatientId := splitLocation[len(splitLocation)-1]

	patientCollection := Database.C("patients")
	patient := models.Patient{}
	err = patientCollection.Find(bson.M{"_id": createdPatientId}).One(&patient)
	util.CheckErr(err)
  c.Assert(patient.Name[0].Family[0], Equals, "Daffy")
}

func (s *ServerSuite) TestUpdatePatient(c *C) {
  updatePatient := LoadPatientFromFixture("../fixtures/patient-example-c.json")
  var buf bytes.Buffer
  encoder := json.NewEncoder(&buf)
  encoder.Encode(updatePatient)

  client := &http.Client{}
  req, err := http.NewRequest("PUT", s.Server.URL + "/Patient/" + s.FixtureId, &buf)
  util.CheckErr(err)
  _, err = client.Do(req)

	patientCollection := Database.C("patients")
	patient := models.Patient{}
	err = patientCollection.Find(bson.M{"_id": s.FixtureId}).One(&patient)
	util.CheckErr(err)
  c.Assert(patient.Name[0].Family[0], Equals, "Darkwing")
}

func (s *ServerSuite) TestDeletePatient(c *C) {

  createPatient := LoadPatientFromFixture("../fixtures/patient-example-d.json")
  var buf bytes.Buffer
  encoder := json.NewEncoder(&buf)
  encoder.Encode(createPatient)
  res, err := http.Post(s.Server.URL + "/Patient", "application/json", &buf)
  util.CheckErr(err)

  splitLocation := strings.Split(res.Header["Location"][0], "/")
  createdPatientId := splitLocation[len(splitLocation)-1]

  client := &http.Client{}
  req, err := http.NewRequest("DELETE", s.Server.URL + "/Patient/" + createdPatientId, nil)
  util.CheckErr(err)
  _, err = client.Do(req)

  patientCollection := Database.C("patients")

  count, err := patientCollection.Find(bson.M{"_id": createdPatientId}).Count()
  c.Assert(count, Equals, 0)
}

func LoadPatientFromFixture(fileName string) *models.Patient {
	data, err := os.Open(fileName)
	defer data.Close()
	util.CheckErr(err)
	decoder := json.NewDecoder(data)
	patient := &models.Patient{}
	err = decoder.Decode(patient)
	util.CheckErr(err)
	return patient
}
<commit_msg>refactored test suite to avoid creating patient structs and re-encoding them to json. http tests now use raw json from fixtures<commit_after>package server

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
  "strings"

	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/pebbe/util"
	"github.com/intervention-engine/fhir/models"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
  . "gopkg.in/check.v1"
)

type ServerSuite struct {
	Session   *mgo.Session
	Router    *mux.Router
	Server    *httptest.Server
	FixtureId string
}

func Test(t *testing.T) { TestingT(t) }

var _ = Suite(&ServerSuite{})

func (s *ServerSuite) SetUpSuite(c *C) {

	// Set up the database
  var err error
	s.Session, err = mgo.Dial("localhost")
	util.CheckErr(err)
	Database = s.Session.DB("fhir-test")

	// Build routes for testing
	s.Router = mux.NewRouter()
	s.Router.StrictSlash(true)
	s.Router.KeepContext = true
	RegisterRoutes(s.Router, make(map[string][]negroni.Handler))

	// Create httptest server
	s.Server = httptest.NewServer(s.Router)

	// Add patient fixture
	patientCollection := Database.C("patients")
	patient := LoadPatientFromFixture("../fixtures/patient-example-a.json")
	i := bson.NewObjectId()
	s.FixtureId = i.Hex()
	patient.Id = s.FixtureId
	err = patientCollection.Insert(patient)
	util.CheckErr(err)
}

func (s *ServerSuite) TearDownSuite(c *C) {
  Database.DropDatabase()
  s.Session.Close()
	s.Server.Close()
}

func (s *ServerSuite) TestGetPatient(c *C) {
	res, err := http.Get(s.Server.URL + "/Patient/" + s.FixtureId)
	util.CheckErr(err)

  decoder := json.NewDecoder(res.Body)
  patient := &models.Patient{}
  err = decoder.Decode(patient)
  util.CheckErr(err)
  c.Assert(patient.Name[0].Family[0], Equals, "Donald")
}

func (s *ServerSuite) TestShowPatient(c *C) {
  res, err := http.Get(s.Server.URL + "/Patient")
  util.CheckErr(err)

  decoder := json.NewDecoder(res.Body)
  patientBundle := &models.PatientBundle{}
  err = decoder.Decode(patientBundle)
  util.CheckErr(err)

  var result []models.Patient
  collection := Database.C("patients")
  iter := collection.Find(nil).Iter()
  err = iter.All(&result)
  util.CheckErr(err)

  c.Assert(patientBundle.TotalResults, Equals, len(result))
  c.Assert(patientBundle.Title, Equals, "Patient Index")
}

func (s *ServerSuite) TestCreatePatient(c *C) {
	data, err := os.Open("../fixtures/patient-example-b.json")
	defer data.Close()
	util.CheckErr(err)
  res, err := http.Post(s.Server.URL + "/Patient", "application/json", data)
  util.CheckErr(err)

  splitLocation := strings.Split(res.Header["Location"][0], "/")
  createdPatientId := splitLocation[len(splitLocation)-1]

	patientCollection := Database.C("patients")
	patient := models.Patient{}
	err = patientCollection.Find(bson.M{"_id": createdPatientId}).One(&patient)
	util.CheckErr(err)
  c.Assert(patient.Name[0].Family[0], Equals, "Daffy")
}

func (s *ServerSuite) TestUpdatePatient(c *C) {
	data, err := os.Open("../fixtures/patient-example-c.json")
	defer data.Close()
	util.CheckErr(err)

  client := &http.Client{}
  req, err := http.NewRequest("PUT", s.Server.URL + "/Patient/" + s.FixtureId, data)
  util.CheckErr(err)
  _, err = client.Do(req)

	patientCollection := Database.C("patients")
	patient := models.Patient{}
	err = patientCollection.Find(bson.M{"_id": s.FixtureId}).One(&patient)
	util.CheckErr(err)
  c.Assert(patient.Name[0].Family[0], Equals, "Darkwing")
}

func (s *ServerSuite) TestDeletePatient(c *C) {

	data, err := os.Open("../fixtures/patient-example-d.json")
	defer data.Close()
	util.CheckErr(err)
  res, err := http.Post(s.Server.URL + "/Patient", "application/json", data)
  util.CheckErr(err)

  splitLocation := strings.Split(res.Header["Location"][0], "/")
  createdPatientId := splitLocation[len(splitLocation)-1]

  client := &http.Client{}
  req, err := http.NewRequest("DELETE", s.Server.URL + "/Patient/" + createdPatientId, nil)
  util.CheckErr(err)
  _, err = client.Do(req)

  patientCollection := Database.C("patients")

  count, err := patientCollection.Find(bson.M{"_id": createdPatientId}).Count()
  c.Assert(count, Equals, 0)
}

func LoadPatientFromFixture(fileName string) *models.Patient {
	data, err := os.Open(fileName)
	defer data.Close()
	util.CheckErr(err)
	decoder := json.NewDecoder(data)
	patient := &models.Patient{}
	err = decoder.Decode(patient)
	util.CheckErr(err)
	return patient
}
<|endoftext|>


<commit_before>// Copyright 2013 Xing Xing <mikespook@gmail.com>.
// All rights reserved.
// Use of this source code is governed by a commercial
// license that can be found in the LICENSE file.

package iptpool

import (
	"sync"
)

var (
	DefaultMaxIdle = 32
)

type CreateFunc func() ScriptIpt
type EventFunc func(ScriptIpt) error

type ScriptIpt interface {
	Exec(name string, params interface{}) error
	Init(path string) error
	Final() error
	Bind(name string, item interface{}) error
	State() interface{}
}

type IptPool struct {
	p        sync.Pool
	OnCreate EventFunc
}

func NewIptPool(create CreateFunc) *IptPool {
	iptPool := &IptPool{}
	f := func() interface{} {
		ipt := create()
		if iptPool.OnCreate != nil {
			iptPool.OnCreate(ipt)
		}
		return ipt
	}
	iptPool.p = sync.Pool{New: f}
	return iptPool
}

func (pool *IptPool) Get() (ipt ScriptIpt) {
	return pool.p.Get().(ScriptIpt)
}

func (pool *IptPool) Put(ipt ScriptIpt) {
	pool.p.Put(ipt)
}
<commit_msg>no need to expose<commit_after>// Copyright 2013 Xing Xing <mikespook@gmail.com>.
// All rights reserved.
// Use of this source code is governed by a commercial
// license that can be found in the LICENSE file.

package iptpool

import (
	"sync"
)

var (
	DefaultMaxIdle = 32
)

type CreateFunc func() ScriptIpt
type EventFunc func(ScriptIpt) error

type ScriptIpt interface {
	Exec(name string, params interface{}) error
	Init(path string) error
	Final() error
	Bind(name string, item interface{}) error
}

type IptPool struct {
	p        sync.Pool
	OnCreate EventFunc
}

func NewIptPool(create CreateFunc) *IptPool {
	iptPool := &IptPool{}
	f := func() interface{} {
		ipt := create()
		if iptPool.OnCreate != nil {
			iptPool.OnCreate(ipt)
		}
		return ipt
	}
	iptPool.p = sync.Pool{New: f}
	return iptPool
}

func (pool *IptPool) Get() (ipt ScriptIpt) {
	return pool.p.Get().(ScriptIpt)
}

func (pool *IptPool) Put(ipt ScriptIpt) {
	pool.p.Put(ipt)
}
<|endoftext|>


<commit_before>// Copyright 2012 Google, Inc. All rights reserved.
// Copyright 2009-2011 Andreas Krennmair. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

package layers

import (
	"encoding/binary"
	"errors"
	"fmt"
	"github.com/google/gopacket"
	"net"
)

// EthernetBroadcast is the broadcast MAC address used by Ethernet.
var EthernetBroadcast = net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}

// Ethernet is the layer for Ethernet frame headers.
type Ethernet struct {
	BaseLayer
	SrcMAC, DstMAC net.HardwareAddr
	EthernetType   EthernetType
	// Length is only set if a length field exists within this header.  Ethernet
	// headers follow two different standards, one that uses an EthernetType, the
	// other which defines a length the follows with a LLC header (802.3).  If the
	// former is the case, we set EthernetType and Length stays 0.  In the latter
	// case, we set Length and EthernetType = EthernetTypeLLC.
	Length uint16
}

// LayerType returns LayerTypeEthernet
func (e *Ethernet) LayerType() gopacket.LayerType { return LayerTypeEthernet }

func (e *Ethernet) LinkFlow() gopacket.Flow {
	return gopacket.NewFlow(EndpointMAC, e.SrcMAC, e.DstMAC)
}

func (eth *Ethernet) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
	if len(data) < 14 {
		return errors.New("Ethernet packet too small")
	}
	eth.DstMAC = net.HardwareAddr(data[0:6])
	eth.SrcMAC = net.HardwareAddr(data[6:12])
	eth.EthernetType = EthernetType(binary.BigEndian.Uint16(data[12:14]))
	eth.BaseLayer = BaseLayer{data[:14], data[14:]}
	if eth.EthernetType < 0x0600 {
		eth.Length = uint16(eth.EthernetType)
		eth.EthernetType = EthernetTypeLLC
		if cmp := len(eth.Payload) - int(eth.Length); cmp < 0 {
			df.SetTruncated()
		} else if cmp > 0 {
			// Strip off bytes at the end, since we have too many bytes
			eth.Payload = eth.Payload[:len(eth.Payload)-cmp]
		}
		//	fmt.Println(eth)
	}
	return nil
}

// SerializeTo writes the serialized form of this layer into the
// SerializationBuffer, implementing gopacket.SerializableLayer.
// See the docs for gopacket.SerializableLayer for more info.
func (eth *Ethernet) SerializeTo(b gopacket.SerializeBuffer, opts gopacket.SerializeOptions) error {
	if len(eth.DstMAC) != 6 {
		return fmt.Errorf("invalid dst MAC: %v", eth.DstMAC)
	}
	if len(eth.SrcMAC) != 6 {
		return fmt.Errorf("invalid src MAC: %v", eth.SrcMAC)
	}
	payload := b.Bytes()
	bytes, err := b.PrependBytes(14)
	if err != nil {
		return err
	}
	copy(bytes, eth.DstMAC)
	copy(bytes[6:], eth.SrcMAC)
	if eth.Length != 0 || eth.EthernetType == EthernetTypeLLC {
		if opts.FixLengths {
			eth.Length = uint16(len(payload))
		}
		if eth.EthernetType != EthernetTypeLLC {
			return fmt.Errorf("ethernet type %v not compatible with length value %v", eth.EthernetType, eth.Length)
		} else if eth.Length > 0x0600 {
			return fmt.Errorf("invalid ethernet length %v", eth.Length)
		}
		binary.BigEndian.PutUint16(bytes[12:], eth.Length)
	} else {
		binary.BigEndian.PutUint16(bytes[12:], uint16(eth.EthernetType))
	}
	length := len(b.Bytes())
	if length < 60 {
		// Pad out to 60 bytes.
		padding, err := b.AppendBytes(60 - length)
		if err != nil {
			return err
		}
		copy(padding, lotsOfZeros[:])
	}
	return nil
}

func (eth *Ethernet) CanDecode() gopacket.LayerClass {
	return LayerTypeEthernet
}

func (eth *Ethernet) NextLayerType() gopacket.LayerType {
	return eth.EthernetType.LayerType()
}

func decodeEthernet(data []byte, p gopacket.PacketBuilder) error {
	eth := &Ethernet{}
	err := eth.DecodeFromBytes(data, p)
	if err != nil {
		return err
	}
	p.AddLayer(eth)
	p.SetLinkLayer(eth)
	return p.NextDecoder(eth.EthernetType)
}
<commit_msg>Reset Length field in Ethernet.DecodeFromBytes<commit_after>// Copyright 2012 Google, Inc. All rights reserved.
// Copyright 2009-2011 Andreas Krennmair. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

package layers

import (
	"encoding/binary"
	"errors"
	"fmt"
	"github.com/google/gopacket"
	"net"
)

// EthernetBroadcast is the broadcast MAC address used by Ethernet.
var EthernetBroadcast = net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}

// Ethernet is the layer for Ethernet frame headers.
type Ethernet struct {
	BaseLayer
	SrcMAC, DstMAC net.HardwareAddr
	EthernetType   EthernetType
	// Length is only set if a length field exists within this header.  Ethernet
	// headers follow two different standards, one that uses an EthernetType, the
	// other which defines a length the follows with a LLC header (802.3).  If the
	// former is the case, we set EthernetType and Length stays 0.  In the latter
	// case, we set Length and EthernetType = EthernetTypeLLC.
	Length uint16
}

// LayerType returns LayerTypeEthernet
func (e *Ethernet) LayerType() gopacket.LayerType { return LayerTypeEthernet }

func (e *Ethernet) LinkFlow() gopacket.Flow {
	return gopacket.NewFlow(EndpointMAC, e.SrcMAC, e.DstMAC)
}

func (eth *Ethernet) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
	if len(data) < 14 {
		return errors.New("Ethernet packet too small")
	}
	eth.DstMAC = net.HardwareAddr(data[0:6])
	eth.SrcMAC = net.HardwareAddr(data[6:12])
	eth.EthernetType = EthernetType(binary.BigEndian.Uint16(data[12:14]))
	eth.BaseLayer = BaseLayer{data[:14], data[14:]}
	eth.Length = 0
	if eth.EthernetType < 0x0600 {
		eth.Length = uint16(eth.EthernetType)
		eth.EthernetType = EthernetTypeLLC
		if cmp := len(eth.Payload) - int(eth.Length); cmp < 0 {
			df.SetTruncated()
		} else if cmp > 0 {
			// Strip off bytes at the end, since we have too many bytes
			eth.Payload = eth.Payload[:len(eth.Payload)-cmp]
		}
		//	fmt.Println(eth)
	}
	return nil
}

// SerializeTo writes the serialized form of this layer into the
// SerializationBuffer, implementing gopacket.SerializableLayer.
// See the docs for gopacket.SerializableLayer for more info.
func (eth *Ethernet) SerializeTo(b gopacket.SerializeBuffer, opts gopacket.SerializeOptions) error {
	if len(eth.DstMAC) != 6 {
		return fmt.Errorf("invalid dst MAC: %v", eth.DstMAC)
	}
	if len(eth.SrcMAC) != 6 {
		return fmt.Errorf("invalid src MAC: %v", eth.SrcMAC)
	}
	payload := b.Bytes()
	bytes, err := b.PrependBytes(14)
	if err != nil {
		return err
	}
	copy(bytes, eth.DstMAC)
	copy(bytes[6:], eth.SrcMAC)
	if eth.Length != 0 || eth.EthernetType == EthernetTypeLLC {
		if opts.FixLengths {
			eth.Length = uint16(len(payload))
		}
		if eth.EthernetType != EthernetTypeLLC {
			return fmt.Errorf("ethernet type %v not compatible with length value %v", eth.EthernetType, eth.Length)
		} else if eth.Length > 0x0600 {
			return fmt.Errorf("invalid ethernet length %v", eth.Length)
		}
		binary.BigEndian.PutUint16(bytes[12:], eth.Length)
	} else {
		binary.BigEndian.PutUint16(bytes[12:], uint16(eth.EthernetType))
	}
	length := len(b.Bytes())
	if length < 60 {
		// Pad out to 60 bytes.
		padding, err := b.AppendBytes(60 - length)
		if err != nil {
			return err
		}
		copy(padding, lotsOfZeros[:])
	}
	return nil
}

func (eth *Ethernet) CanDecode() gopacket.LayerClass {
	return LayerTypeEthernet
}

func (eth *Ethernet) NextLayerType() gopacket.LayerType {
	return eth.EthernetType.LayerType()
}

func decodeEthernet(data []byte, p gopacket.PacketBuilder) error {
	eth := &Ethernet{}
	err := eth.DecodeFromBytes(data, p)
	if err != nil {
		return err
	}
	p.AddLayer(eth)
	p.SetLinkLayer(eth)
	return p.NextDecoder(eth.EthernetType)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"sort"

	"github.com/sdboyer/gps"
)

type lock struct {
	Memo []byte
	P    []gps.LockedProject
}

type rawLock struct {
	Memo string      `json:"memo"`
	P    []lockedDep `json:"projects"`
}

type lockedDep struct {
	Name       string   `json:"name"`
	Version    string   `json:"version,omitempty"`
	Branch     string   `json:"branch,omitempty"`
	Revision   string   `json:"revision"`
	Repository string   `json:"repo,omitempty"`
	Packages   []string `json:"packages"`
}

func readLock(r io.Reader) (*lock, error) {
	rl := rawLock{}
	err := json.NewDecoder(r).Decode(&rl)
	if err != nil {
		return nil, err
	}

	b, err := hex.DecodeString(rl.Memo)
	if err != nil {
		return nil, fmt.Errorf("invalid hash digest in lock's memo field")
	}
	l := &lock{
		Memo: b,
		P:    make([]gps.LockedProject, len(rl.P)),
	}

	for i, ld := range rl.P {
		r := gps.Revision(ld.Revision)

		var v gps.Version
		if ld.Version != "" {
			if ld.Branch != "" {
				return nil, fmt.Errorf("lock file specified both a branch (%s) and version (%s) for %s", ld.Branch, ld.Version, ld.Name)
			}
			v = gps.NewVersion(ld.Version).Is(r)
		} else if ld.Branch != "" {
			v = gps.NewBranch(ld.Branch).Is(r)
		} else if r == "" {
			return nil, fmt.Errorf("lock file has entry for %s, but specifies no version", ld.Name)
		} else {
			v = r
		}

		id := gps.ProjectIdentifier{
			ProjectRoot: gps.ProjectRoot(ld.Name),
			Source: ld.Repository,
		}
		l.P[i] = gps.NewLockedProject(id, v, ld.Packages)
	}

	return l, nil
}

func (l *lock) InputHash() []byte {
	return l.Memo
}

func (l *lock) Projects() []gps.LockedProject {
	return l.P
}

func (l *lock) MarshalJSON() ([]byte, error) {
	raw := rawLock{
		Memo: hex.EncodeToString(l.Memo),
		P:    make([]lockedDep, len(l.P)),
	}

	sort.Sort(sortedLockedProjects(l.P))

	for k, lp := range l.P {
		id := lp.Ident()
		ld := lockedDep{
			Name:       string(id.ProjectRoot),
			Repository: id.Source,
			Packages:   lp.Packages(),
		}

		v := lp.Version()
		// Figure out how to get the underlying revision
		switch tv := v.(type) {
		case gps.UnpairedVersion:
			// TODO we could error here, if we want to be very defensive about not
			// allowing a lock to be written if without an immmutable revision
		case gps.Revision:
			ld.Revision = tv.String()
		case gps.PairedVersion:
			ld.Revision = tv.Underlying().String()
		}

		switch v.Type() {
		case gps.IsBranch:
			ld.Branch = v.String()
		case gps.IsSemver, gps.IsVersion:
			ld.Version = v.String()
		}

		raw.P[k] = ld
	}

	// TODO sort output - #15

	var buf bytes.Buffer
	enc := json.NewEncoder(&buf)
	enc.SetIndent("", "    ")
	enc.SetEscapeHTML(false)
	err := enc.Encode(raw)

	return buf.Bytes(), err
}

// lockFromInterface converts an arbitrary gps.Lock to dep's representation of a
// lock. If the input is already dep's *lock, the input is returned directly.
//
// Data is defensively copied wherever necessary to ensure the resulting *lock
// shares no memory with the original lock.
//
// As gps.Solution is a superset of gps.Lock, this can also be used to convert
// solutions to dep's lock form.
func lockFromInterface(in gps.Lock) *lock {
	if in == nil {
		return nil
	} else if l, ok := in.(*lock); ok {
		return l
	}

	h, p := in.InputHash(), in.Projects()

	l := &lock{
		Memo: make([]byte, len(h)),
		P:    make([]gps.LockedProject, len(p)),
	}

	copy(l.Memo, h)
	copy(l.P, p)
	return l
}

type sortedLockedProjects []gps.LockedProject

func (s sortedLockedProjects) Len() int      { return len(s) }
func (s sortedLockedProjects) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s sortedLockedProjects) Less(i, j int) bool {
	l, r := s[i].Ident(), s[j].Ident()

	if l.ProjectRoot < r.ProjectRoot {
		return true
	}
	if r.ProjectRoot < l.ProjectRoot {
		return false
	}

	return l.Source < r.Source
}
<commit_msg>Add locksAreEquivalent()<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"sort"

	"github.com/sdboyer/gps"
)

type lock struct {
	Memo []byte
	P    []gps.LockedProject
}

type rawLock struct {
	Memo string      `json:"memo"`
	P    []lockedDep `json:"projects"`
}

type lockedDep struct {
	Name       string   `json:"name"`
	Version    string   `json:"version,omitempty"`
	Branch     string   `json:"branch,omitempty"`
	Revision   string   `json:"revision"`
	Repository string   `json:"repo,omitempty"`
	Packages   []string `json:"packages"`
}

func readLock(r io.Reader) (*lock, error) {
	rl := rawLock{}
	err := json.NewDecoder(r).Decode(&rl)
	if err != nil {
		return nil, err
	}

	b, err := hex.DecodeString(rl.Memo)
	if err != nil {
		return nil, fmt.Errorf("invalid hash digest in lock's memo field")
	}
	l := &lock{
		Memo: b,
		P:    make([]gps.LockedProject, len(rl.P)),
	}

	for i, ld := range rl.P {
		r := gps.Revision(ld.Revision)

		var v gps.Version
		if ld.Version != "" {
			if ld.Branch != "" {
				return nil, fmt.Errorf("lock file specified both a branch (%s) and version (%s) for %s", ld.Branch, ld.Version, ld.Name)
			}
			v = gps.NewVersion(ld.Version).Is(r)
		} else if ld.Branch != "" {
			v = gps.NewBranch(ld.Branch).Is(r)
		} else if r == "" {
			return nil, fmt.Errorf("lock file has entry for %s, but specifies no version", ld.Name)
		} else {
			v = r
		}

		id := gps.ProjectIdentifier{
			ProjectRoot: gps.ProjectRoot(ld.Name),
			Source: ld.Repository,
		}
		l.P[i] = gps.NewLockedProject(id, v, ld.Packages)
	}

	return l, nil
}

func (l *lock) InputHash() []byte {
	return l.Memo
}

func (l *lock) Projects() []gps.LockedProject {
	return l.P
}

func (l *lock) MarshalJSON() ([]byte, error) {
	raw := rawLock{
		Memo: hex.EncodeToString(l.Memo),
		P:    make([]lockedDep, len(l.P)),
	}

	sort.Sort(sortedLockedProjects(l.P))

	for k, lp := range l.P {
		id := lp.Ident()
		ld := lockedDep{
			Name:       string(id.ProjectRoot),
			Repository: id.Source,
			Packages:   lp.Packages(),
		}

		v := lp.Version()
		// Figure out how to get the underlying revision
		switch tv := v.(type) {
		case gps.UnpairedVersion:
			// TODO we could error here, if we want to be very defensive about not
			// allowing a lock to be written if without an immmutable revision
		case gps.Revision:
			ld.Revision = tv.String()
		case gps.PairedVersion:
			ld.Revision = tv.Underlying().String()
		}

		switch v.Type() {
		case gps.IsBranch:
			ld.Branch = v.String()
		case gps.IsSemver, gps.IsVersion:
			ld.Version = v.String()
		}

		raw.P[k] = ld
	}

	// TODO sort output - #15

	var buf bytes.Buffer
	enc := json.NewEncoder(&buf)
	enc.SetIndent("", "    ")
	enc.SetEscapeHTML(false)
	err := enc.Encode(raw)

	return buf.Bytes(), err
}

// lockFromInterface converts an arbitrary gps.Lock to dep's representation of a
// lock. If the input is already dep's *lock, the input is returned directly.
//
// Data is defensively copied wherever necessary to ensure the resulting *lock
// shares no memory with the original lock.
//
// As gps.Solution is a superset of gps.Lock, this can also be used to convert
// solutions to dep's lock form.
func lockFromInterface(in gps.Lock) *lock {
	if in == nil {
		return nil
	} else if l, ok := in.(*lock); ok {
		return l
	}

	h, p := in.InputHash(), in.Projects()

	l := &lock{
		Memo: make([]byte, len(h)),
		P:    make([]gps.LockedProject, len(p)),
	}

	copy(l.Memo, h)
	copy(l.P, p)
	return l
}

type sortedLockedProjects []gps.LockedProject

func (s sortedLockedProjects) Len() int      { return len(s) }
func (s sortedLockedProjects) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s sortedLockedProjects) Less(i, j int) bool {
	l, r := s[i].Ident(), s[j].Ident()

	if l.ProjectRoot < r.ProjectRoot {
		return true
	}
	if r.ProjectRoot < l.ProjectRoot {
		return false
	}

	return l.Source < r.Source
}

// locksAreEquivalent compares two locks to see if they differ. If EITHER lock
// is nil, or their memos do not match, or any projects differ, then false is
// returned.
func locksAreEquivalent(l, r *lock) bool {
	if l == nil || r == nil {
		return false
	}

	if !bytes.Equal(l.Memo, r.Memo) {
		return false
	}

	if len(l.P) != len(r.P) {
		return false
	}

	sort.Sort(sortedLockedProjects(l.P))
	sort.Sort(sortedLockedProjects(r.P))

	for k, lp := range l.P {
		// TODO(sdboyer) gps will be adding a func to compare LockedProjects in the next
		// version; we can swap that in here when it lands
		rp := r.P[k]
		if lp.Ident() != rp.Ident() {
			return false
		}

		lpkg, rpkg := lp.Packages(), rp.Packages()
		if !reflect.DeepEqual(lpkg, rpkg) {
			return false
		}
	}

	return true
}
<|endoftext|>


<commit_before>package export

import (
	cc "github.com/pivotalservices/cf-mgmt/cloudcontroller"
	"github.com/pivotalservices/cf-mgmt/config"
	"github.com/pivotalservices/cf-mgmt/organization"
	"github.com/pivotalservices/cf-mgmt/space"
	"github.com/pivotalservices/cf-mgmt/uaac"
	"github.com/xchapter7x/lo"
)

//NewExportManager Creates a new instance of the ImportConfig manager
func NewExportManager(
	configDir string,
	uaacMgr uaac.Manager,
	cloudController cc.Manager) Manager {
	return &DefaultImportManager{
		ConfigDir:       configDir,
		UAACMgr:         uaacMgr,
		CloudController: cloudController,
	}
}

//ExportConfig Imports org and space configuration from an existing CF instance
//Entries part of excludedOrgs and excludedSpaces are not included in the import
func (im *DefaultImportManager) ExportConfig(excludedOrgs map[string]string, excludedSpaces map[string]string) error {
	//Get all the users from the foundation
	userIDToUserMap, err := im.UAACMgr.UsersByID()
	if err != nil {
		lo.G.Error("Unable to retrieve users")
		return err
	}
	//Get all the orgs
	orgs, err := im.CloudController.ListOrgs()
	if err != nil {
		lo.G.Errorf("Unable to retrieve orgs. Error : %s", err)
		return err
	}
	configMgr := config.NewManager(im.ConfigDir)
	lo.G.Info("Trying to delete existing config directory")
	//Delete existing config directory
	err = configMgr.DeleteConfigIfExists()
	if err != nil {
		return err
	}
	//Create a brand new directory
	lo.G.Info("Trying to create new config folder")

	var uaaUserOrigin string
	for _, usr := range userIDToUserMap {
		if usr.Origin != "" {
			uaaUserOrigin = usr.Origin
			break
		}
	}
	lo.G.Infof("Using UAA user origin: %s", uaaUserOrigin)
	err = configMgr.CreateConfigIfNotExists(uaaUserOrigin)
	if err != nil {
		return err
	}
	lo.G.Debugf("Orgs to process: %s", orgs)

	for _, org := range orgs {
		if _, ok := excludedOrgs[org.Entity.Name]; ok {
			lo.G.Infof("Skipping org: %s as it is ignored from import", org.Entity.Name)
			continue
		}
		lo.G.Infof("Processing org: %s ", org.Entity.Name)
		orgConfig := &config.OrgConfig{Org: org.Entity.Name}
		//Add users
		addOrgUsers(orgConfig, im.CloudController, userIDToUserMap, org.MetaData.GUID)
		//Add Quota definition if applicable
		if org.Entity.QuotaDefinitionGUID != "" {
			quota := quotaDefinition(im.CloudController, org.Entity.QuotaDefinitionGUID, organization.ORGS)
			orgConfig.EnableOrgQuota = quota.IsQuotaEnabled()
			orgConfig.MemoryLimit = quota.GetMemoryLimit()
			orgConfig.InstanceMemoryLimit = quota.GetInstanceMemoryLimit()
			orgConfig.TotalRoutes = quota.GetTotalRoutes()
			orgConfig.TotalServices = quota.GetTotalServices()
			orgConfig.PaidServicePlansAllowed = quota.IsPaidServicesAllowed()
			orgConfig.TotalPrivateDomains = quota.TotalPrivateDomains
			orgConfig.TotalReservedRoutePorts = quota.TotalReservedRoutePorts
			orgConfig.TotalServiceKeys = quota.TotalServiceKeys
			orgConfig.AppInstanceLimit = quota.AppInstanceLimit
		}
		configMgr.AddOrgToConfig(orgConfig)

		lo.G.Infof("Done creating org %s", orgConfig.Org)
		lo.G.Infof("Listing spaces for org %s", orgConfig.Org)
		spaces, _ := im.CloudController.ListSpaces(org.MetaData.GUID)
		lo.G.Infof("Found %d Spaces for org %s", len(spaces), orgConfig.Org)
		for _, orgSpace := range spaces {
			if _, ok := excludedSpaces[orgSpace.Entity.Name]; ok {
				lo.G.Infof("Skipping space: %s as it is ignored from import", orgSpace.Entity.Name)
				continue
			}
			lo.G.Infof("Processing space: %s", orgSpace.Entity.Name)

			spaceConfig := &config.SpaceConfig{Org: org.Entity.Name, Space: orgSpace.Entity.Name}
			//Add users
			addSpaceUsers(spaceConfig, im.CloudController, userIDToUserMap, orgSpace.MetaData.GUID)
			//Add Quota definition if applicable
			if orgSpace.Entity.QuotaDefinitionGUID != "" {
				quota := quotaDefinition(im.CloudController, orgSpace.Entity.QuotaDefinitionGUID, space.SPACES)
				spaceConfig.EnableSpaceQuota = quota.IsQuotaEnabled()
				spaceConfig.MemoryLimit = quota.GetMemoryLimit()
				spaceConfig.InstanceMemoryLimit = quota.GetInstanceMemoryLimit()
				spaceConfig.TotalRoutes = quota.GetTotalRoutes()
				spaceConfig.TotalServices = quota.GetTotalServices()
				spaceConfig.PaidServicePlansAllowed = quota.IsPaidServicesAllowed()
				spaceConfig.TotalPrivateDomains = quota.TotalPrivateDomains
				spaceConfig.TotalReservedRoutePorts = quota.TotalReservedRoutePorts
				spaceConfig.TotalServiceKeys = quota.TotalServiceKeys
				spaceConfig.AppInstanceLimit = quota.AppInstanceLimit
			}
			if orgSpace.Entity.AllowSSH {
				spaceConfig.AllowSSH = true
			}
			configMgr.AddSpaceToConfig(spaceConfig)
		}
	}
	return nil
}

func quotaDefinition(controller cc.Manager, quotaDefinitionGUID, entityType string) cc.QuotaEntity {
	quotaDef, _ := controller.QuotaDef(quotaDefinitionGUID, entityType)
	if quotaDef.Entity.Name != "default" {
		return quotaDef.Entity
	}
	return cc.QuotaEntity{}
}

func addOrgUsers(orgConfig *config.OrgConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, orgGUID string) {
	addOrgManagers(orgConfig, controller, userIDToUserMap, orgGUID)
	addBillingManagers(orgConfig, controller, userIDToUserMap, orgGUID)
	addOrgAuditors(orgConfig, controller, userIDToUserMap, orgGUID)
}

func addSpaceUsers(spaceConfig *config.SpaceConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, spaceGUID string) {
	addSpaceDevelopers(spaceConfig, controller, userIDToUserMap, spaceGUID)
	addSpaceManagers(spaceConfig, controller, userIDToUserMap, spaceGUID)
	addSpaceAuditors(spaceConfig, controller, userIDToUserMap, spaceGUID)
}

func addOrgManagers(orgConfig *config.OrgConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, orgGUID string) {
	orgMgrs, _ := getCFUsers(controller, orgGUID, organization.ORGS, organization.ROLE_ORG_MANAGERS)
	lo.G.Debugf("Found %d Org Managers for Org: %s", len(orgMgrs), orgConfig.Org)
	doAddUsers(orgMgrs, &orgConfig.Manager.Users, &orgConfig.Manager.LDAPUsers, userIDToUserMap)
}

func addBillingManagers(orgConfig *config.OrgConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, orgGUID string) {
	orgBillingMgrs, _ := getCFUsers(controller, orgGUID, organization.ORGS, organization.ROLE_ORG_BILLING_MANAGERS)
	lo.G.Debugf("Found %d Org Billing Managers for Org: %s", len(orgBillingMgrs), orgConfig.Org)
	doAddUsers(orgBillingMgrs, &orgConfig.BillingManager.Users, &orgConfig.BillingManager.LDAPUsers, userIDToUserMap)
}

func addOrgAuditors(orgConfig *config.OrgConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, orgGUID string) {
	orgAuditors, _ := getCFUsers(controller, orgGUID, organization.ORGS, organization.ROLE_ORG_AUDITORS)
	lo.G.Debugf("Found %d Org Auditors for Org: %s", len(orgAuditors), orgConfig.Org)
	doAddUsers(orgAuditors, &orgConfig.Auditor.Users, &orgConfig.Auditor.LDAPUsers, userIDToUserMap)
}

func addSpaceManagers(spaceConfig *config.SpaceConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, spaceGUID string) {
	spaceMgrs, _ := getCFUsers(controller, spaceGUID, space.SPACES, space.ROLE_SPACE_MANAGERS)
	lo.G.Debugf("Found %d Space Managers for Org: %s and  Space:  %s", len(spaceMgrs), spaceConfig.Org, spaceConfig.Space)
	doAddUsers(spaceMgrs, &spaceConfig.Manager.Users, &spaceConfig.Manager.LDAPUsers, userIDToUserMap)
}

func addSpaceDevelopers(spaceConfig *config.SpaceConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, spaceGUID string) {
	spaceDevs, _ := getCFUsers(controller, spaceGUID, space.SPACES, space.ROLE_SPACE_DEVELOPERS)
	lo.G.Debugf("Found %d Space Developers for Org: %s and  Space:  %s", len(spaceDevs), spaceConfig.Org, spaceConfig.Space)
	doAddUsers(spaceDevs, &spaceConfig.Developer.Users, &spaceConfig.Developer.LDAPUsers, userIDToUserMap)
}

func addSpaceAuditors(spaceConfig *config.SpaceConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, spaceGUID string) {
	spaceAuditors, _ := getCFUsers(controller, spaceGUID, space.SPACES, space.ROLE_SPACE_AUDITORS)
	lo.G.Debugf("Found %d Space Auditors for Org: %s and  Space:  %s", len(spaceAuditors), spaceConfig.Org, spaceConfig.Space)
	doAddUsers(spaceAuditors, &spaceConfig.Auditor.Users, &spaceConfig.Auditor.LDAPUsers, userIDToUserMap)
}

func doAddUsers(cfUsers map[string]string, uaaUsers *[]string, ldapUsers *[]string, userIDToUserMap map[string]uaac.User) {
	for cfUser := range cfUsers {
		if usr, ok := userIDToUserMap[cfUser]; ok {
			if usr.Origin == "uaa" {
				*uaaUsers = append(*uaaUsers, usr.UserName)
			} else {
				*ldapUsers = append(*ldapUsers, usr.UserName)
			}
		} else {
			lo.G.Infof("CFUser [%s] not found in uaa user list", cfUser)
		}
	}
}

func getCFUsers(cc cc.Manager, entityGUID, entityType, role string) (map[string]string, error) {
	return cc.GetCFUsers(entityGUID, entityType, role)
}
<commit_msg>adding logging to help triage issue with exportConfig not populating all users<commit_after>package export

import (
	cc "github.com/pivotalservices/cf-mgmt/cloudcontroller"
	"github.com/pivotalservices/cf-mgmt/config"
	"github.com/pivotalservices/cf-mgmt/organization"
	"github.com/pivotalservices/cf-mgmt/space"
	"github.com/pivotalservices/cf-mgmt/uaac"
	"github.com/xchapter7x/lo"
)

//NewExportManager Creates a new instance of the ImportConfig manager
func NewExportManager(
	configDir string,
	uaacMgr uaac.Manager,
	cloudController cc.Manager) Manager {
	return &DefaultImportManager{
		ConfigDir:       configDir,
		UAACMgr:         uaacMgr,
		CloudController: cloudController,
	}
}

//ExportConfig Imports org and space configuration from an existing CF instance
//Entries part of excludedOrgs and excludedSpaces are not included in the import
func (im *DefaultImportManager) ExportConfig(excludedOrgs map[string]string, excludedSpaces map[string]string) error {
	//Get all the users from the foundation
	userIDToUserMap, err := im.UAACMgr.UsersByID()
	if err != nil {
		lo.G.Error("Unable to retrieve users")
		return err
	}
	lo.G.Debugf("uaa user id map %v", userIDToUserMap)
	//Get all the orgs
	orgs, err := im.CloudController.ListOrgs()
	if err != nil {
		lo.G.Errorf("Unable to retrieve orgs. Error : %s", err)
		return err
	}
	configMgr := config.NewManager(im.ConfigDir)
	lo.G.Info("Trying to delete existing config directory")
	//Delete existing config directory
	err = configMgr.DeleteConfigIfExists()
	if err != nil {
		return err
	}
	//Create a brand new directory
	lo.G.Info("Trying to create new config folder")

	var uaaUserOrigin string
	for _, usr := range userIDToUserMap {
		if usr.Origin != "" {
			uaaUserOrigin = usr.Origin
			break
		}
	}
	lo.G.Infof("Using UAA user origin: %s", uaaUserOrigin)
	err = configMgr.CreateConfigIfNotExists(uaaUserOrigin)
	if err != nil {
		return err
	}
	lo.G.Debugf("Orgs to process: %s", orgs)

	for _, org := range orgs {
		if _, ok := excludedOrgs[org.Entity.Name]; ok {
			lo.G.Infof("Skipping org: %s as it is ignored from import", org.Entity.Name)
			continue
		}
		lo.G.Infof("Processing org: %s ", org.Entity.Name)
		orgConfig := &config.OrgConfig{Org: org.Entity.Name}
		//Add users
		addOrgUsers(orgConfig, im.CloudController, userIDToUserMap, org.MetaData.GUID)
		//Add Quota definition if applicable
		if org.Entity.QuotaDefinitionGUID != "" {
			quota := quotaDefinition(im.CloudController, org.Entity.QuotaDefinitionGUID, organization.ORGS)
			orgConfig.EnableOrgQuota = quota.IsQuotaEnabled()
			orgConfig.MemoryLimit = quota.GetMemoryLimit()
			orgConfig.InstanceMemoryLimit = quota.GetInstanceMemoryLimit()
			orgConfig.TotalRoutes = quota.GetTotalRoutes()
			orgConfig.TotalServices = quota.GetTotalServices()
			orgConfig.PaidServicePlansAllowed = quota.IsPaidServicesAllowed()
			orgConfig.TotalPrivateDomains = quota.TotalPrivateDomains
			orgConfig.TotalReservedRoutePorts = quota.TotalReservedRoutePorts
			orgConfig.TotalServiceKeys = quota.TotalServiceKeys
			orgConfig.AppInstanceLimit = quota.AppInstanceLimit
		}
		configMgr.AddOrgToConfig(orgConfig)

		lo.G.Infof("Done creating org %s", orgConfig.Org)
		lo.G.Infof("Listing spaces for org %s", orgConfig.Org)
		spaces, _ := im.CloudController.ListSpaces(org.MetaData.GUID)
		lo.G.Infof("Found %d Spaces for org %s", len(spaces), orgConfig.Org)
		for _, orgSpace := range spaces {
			if _, ok := excludedSpaces[orgSpace.Entity.Name]; ok {
				lo.G.Infof("Skipping space: %s as it is ignored from import", orgSpace.Entity.Name)
				continue
			}
			lo.G.Infof("Processing space: %s", orgSpace.Entity.Name)

			spaceConfig := &config.SpaceConfig{Org: org.Entity.Name, Space: orgSpace.Entity.Name}
			//Add users
			addSpaceUsers(spaceConfig, im.CloudController, userIDToUserMap, orgSpace.MetaData.GUID)
			//Add Quota definition if applicable
			if orgSpace.Entity.QuotaDefinitionGUID != "" {
				quota := quotaDefinition(im.CloudController, orgSpace.Entity.QuotaDefinitionGUID, space.SPACES)
				spaceConfig.EnableSpaceQuota = quota.IsQuotaEnabled()
				spaceConfig.MemoryLimit = quota.GetMemoryLimit()
				spaceConfig.InstanceMemoryLimit = quota.GetInstanceMemoryLimit()
				spaceConfig.TotalRoutes = quota.GetTotalRoutes()
				spaceConfig.TotalServices = quota.GetTotalServices()
				spaceConfig.PaidServicePlansAllowed = quota.IsPaidServicesAllowed()
				spaceConfig.TotalPrivateDomains = quota.TotalPrivateDomains
				spaceConfig.TotalReservedRoutePorts = quota.TotalReservedRoutePorts
				spaceConfig.TotalServiceKeys = quota.TotalServiceKeys
				spaceConfig.AppInstanceLimit = quota.AppInstanceLimit
			}
			if orgSpace.Entity.AllowSSH {
				spaceConfig.AllowSSH = true
			}
			configMgr.AddSpaceToConfig(spaceConfig)
		}
	}
	return nil
}

func quotaDefinition(controller cc.Manager, quotaDefinitionGUID, entityType string) cc.QuotaEntity {
	quotaDef, _ := controller.QuotaDef(quotaDefinitionGUID, entityType)
	if quotaDef.Entity.Name != "default" {
		return quotaDef.Entity
	}
	return cc.QuotaEntity{}
}

func addOrgUsers(orgConfig *config.OrgConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, orgGUID string) {
	addOrgManagers(orgConfig, controller, userIDToUserMap, orgGUID)
	addBillingManagers(orgConfig, controller, userIDToUserMap, orgGUID)
	addOrgAuditors(orgConfig, controller, userIDToUserMap, orgGUID)
}

func addSpaceUsers(spaceConfig *config.SpaceConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, spaceGUID string) {
	addSpaceDevelopers(spaceConfig, controller, userIDToUserMap, spaceGUID)
	addSpaceManagers(spaceConfig, controller, userIDToUserMap, spaceGUID)
	addSpaceAuditors(spaceConfig, controller, userIDToUserMap, spaceGUID)
}

func addOrgManagers(orgConfig *config.OrgConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, orgGUID string) {
	orgMgrs, _ := getCFUsers(controller, orgGUID, organization.ORGS, organization.ROLE_ORG_MANAGERS)
	lo.G.Debugf("Found %d Org Managers for Org: %s", len(orgMgrs), orgConfig.Org)
	doAddUsers(orgMgrs, &orgConfig.Manager.Users, &orgConfig.Manager.LDAPUsers, userIDToUserMap)
}

func addBillingManagers(orgConfig *config.OrgConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, orgGUID string) {
	orgBillingMgrs, _ := getCFUsers(controller, orgGUID, organization.ORGS, organization.ROLE_ORG_BILLING_MANAGERS)
	lo.G.Debugf("Found %d Org Billing Managers for Org: %s", len(orgBillingMgrs), orgConfig.Org)
	doAddUsers(orgBillingMgrs, &orgConfig.BillingManager.Users, &orgConfig.BillingManager.LDAPUsers, userIDToUserMap)
}

func addOrgAuditors(orgConfig *config.OrgConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, orgGUID string) {
	orgAuditors, _ := getCFUsers(controller, orgGUID, organization.ORGS, organization.ROLE_ORG_AUDITORS)
	lo.G.Debugf("Found %d Org Auditors for Org: %s", len(orgAuditors), orgConfig.Org)
	doAddUsers(orgAuditors, &orgConfig.Auditor.Users, &orgConfig.Auditor.LDAPUsers, userIDToUserMap)
}

func addSpaceManagers(spaceConfig *config.SpaceConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, spaceGUID string) {
	spaceMgrs, _ := getCFUsers(controller, spaceGUID, space.SPACES, space.ROLE_SPACE_MANAGERS)
	lo.G.Debugf("Found %d Space Managers for Org: %s and  Space:  %s", len(spaceMgrs), spaceConfig.Org, spaceConfig.Space)
	doAddUsers(spaceMgrs, &spaceConfig.Manager.Users, &spaceConfig.Manager.LDAPUsers, userIDToUserMap)
}

func addSpaceDevelopers(spaceConfig *config.SpaceConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, spaceGUID string) {
	spaceDevs, _ := getCFUsers(controller, spaceGUID, space.SPACES, space.ROLE_SPACE_DEVELOPERS)
	lo.G.Debugf("Found %d Space Developers for Org: %s and  Space:  %s", len(spaceDevs), spaceConfig.Org, spaceConfig.Space)
	doAddUsers(spaceDevs, &spaceConfig.Developer.Users, &spaceConfig.Developer.LDAPUsers, userIDToUserMap)
}

func addSpaceAuditors(spaceConfig *config.SpaceConfig, controller cc.Manager, userIDToUserMap map[string]uaac.User, spaceGUID string) {
	spaceAuditors, _ := getCFUsers(controller, spaceGUID, space.SPACES, space.ROLE_SPACE_AUDITORS)
	lo.G.Debugf("Found %d Space Auditors for Org: %s and  Space:  %s", len(spaceAuditors), spaceConfig.Org, spaceConfig.Space)
	doAddUsers(spaceAuditors, &spaceConfig.Auditor.Users, &spaceConfig.Auditor.LDAPUsers, userIDToUserMap)
}

func doAddUsers(cfUsers map[string]string, uaaUsers *[]string, ldapUsers *[]string, userIDToUserMap map[string]uaac.User) {
	for cfUser := range cfUsers {
		if usr, ok := userIDToUserMap[cfUser]; ok {
			if usr.Origin == "uaa" {
				*uaaUsers = append(*uaaUsers, usr.UserName)
			} else {
				*ldapUsers = append(*ldapUsers, usr.UserName)
			}
		} else {
			lo.G.Infof("CFUser [%s] not found in uaa user list", cfUser)
		}
	}
}

func getCFUsers(cc cc.Manager, entityGUID, entityType, role string) (map[string]string, error) {
	return cc.GetCFUsers(entityGUID, entityType, role)
}
<|endoftext|>


<commit_before>package ts

import "io"
import "bytes"
import "github.com/32bitkid/mpeg/util"

type PayloadReader interface {
	io.Reader
	TransportStreamControl
}

func NewPayloadReader(source io.Reader, where PacketTester) PayloadReader {
	return &payloadReader{
		currentPacket: new(Packet),
		br:            util.NewBitReader(source),
		where:         where,
		closed:        false,
		isAligned:     false,
		skipUntil:     alwaysTrueTester,
		takeWhile:     alwaysTrueTester,
	}
}

type payloadReader struct {
	currentPacket *Packet
	br            util.BitReader32
	where         PacketTester
	skipUntil     PacketTester
	takeWhile     PacketTester
	remainder     bytes.Buffer
	closed        bool
	isAligned     bool
}

func (r *payloadReader) SkipUntil(skipUntil PacketTester) {
	r.skipUntil = r.where.And(skipUntil)
}

func (r *payloadReader) TakeWhile(takeWhile PacketTester) {
	r.takeWhile = r.where.And(takeWhile)
}

func (r *payloadReader) Read(p []byte) (n int, err error) {

	if r.closed == true {
		return 0, io.EOF
	}

	if r.isAligned == false {
		err = r.realign()
		if err != nil {
			return
		}
	}

	var remainder []byte

	if r.remainder.Len() > 0 {
		copied, err := r.remainder.Read(p)
		if err != nil {
			return copied, err
		}
		n = n + copied
		p = p[copied:]
	}

	for len(p) > 0 {
		err = r.next()
		if err != nil {
			return
		}

		if r.where(r.currentPacket) {
			copied := copy(p, r.currentPacket.Payload)
			n = n + copied
			p = p[copied:]
			remainder = r.currentPacket.Payload[copied:]
		}

		cont := r.takeWhile(r.currentPacket)
		if cont == false {
			r.closed = true
			return n, io.EOF
		}
	}

	_, err = r.remainder.Write(remainder)

	if err != nil {
		return
	}

	return
}

func (r *payloadReader) next() error {
	return r.currentPacket.ReadFrom(r.br)
}

func (r *payloadReader) realign() (err error) {
	for {
		r.next()
		if err != nil {
			return err
		}
		done := r.skipUntil(r.currentPacket)
		if done {
			r.isAligned = true
			r.remainder.Write(r.currentPacket.Payload)
			return nil
		}
	}
}
<commit_msg>Make sure to drain the remainder before moving on.<commit_after>package ts

import "io"
import "bytes"
import "github.com/32bitkid/mpeg/util"

type PayloadReader interface {
	io.Reader
	TransportStreamControl
}

func NewPayloadReader(source io.Reader, where PacketTester) PayloadReader {
	return &payloadReader{
		currentPacket: new(Packet),
		br:            util.NewBitReader(source),
		where:         where,
		closed:        false,
		isAligned:     false,
		skipUntil:     alwaysTrueTester,
		takeWhile:     alwaysTrueTester,
	}
}

type payloadReader struct {
	currentPacket *Packet
	br            util.BitReader32
	where         PacketTester
	skipUntil     PacketTester
	takeWhile     PacketTester
	remainder     bytes.Buffer
	closed        bool
	isAligned     bool
}

func (r *payloadReader) SkipUntil(skipUntil PacketTester) {
	r.skipUntil = r.where.And(skipUntil)
}

func (r *payloadReader) TakeWhile(takeWhile PacketTester) {
	r.takeWhile = r.where.And(takeWhile)
}

func (r *payloadReader) Read(p []byte) (n int, err error) {

	if r.closed == true {
		return 0, io.EOF
	}

	if r.isAligned == false {
		err = r.realign()
		if err != nil {
			return
		}
	}

	var remainder []byte

	// Drain remainder
	for len(p) > 0 {
		cn, err := r.remainder.Read(p)
		n = n + cn
		p = p[cn:]
		if err == io.EOF {
			break
		} else if err != nil {
			return n, err
		}
	}

	for len(p) > 0 {
		err = r.next()
		if err != nil {
			return
		}

		if r.where(r.currentPacket) {
			copied := copy(p, r.currentPacket.Payload)
			n = n + copied
			p = p[copied:]
			remainder = r.currentPacket.Payload[copied:]
		}

		cont := r.takeWhile(r.currentPacket)
		if cont == false {
			r.closed = true
			return n, io.EOF
		}
	}

	_, err = r.remainder.Write(remainder)

	if err != nil {
		return
	}

	return
}

func (r *payloadReader) next() error {
	return r.currentPacket.ReadFrom(r.br)
}

func (r *payloadReader) realign() (err error) {
	for {
		r.next()
		if err != nil {
			return err
		}
		done := r.skipUntil(r.currentPacket)
		if done {
			r.isAligned = true
			r.remainder.Write(r.currentPacket.Payload)
			return nil
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>Add comment to generated Msgsize function (#161)<commit_after><|endoftext|>


<commit_before>package main

import (
	"math"
	"math/rand"
	"time"
)

var (
	MemoryByteString = []byte("mem") // heap optimization

	// Choices for modeling a host's memory capacity.
	MemoryMaxBytesChoices = []int64{8 << 30, 12 << 30, 16 << 30}

	// Field keys for 'mem' points.
	MemoryFieldKeys = [][]byte{
		[]byte("total"),
		[]byte("available"),
		[]byte("used"),
		[]byte("free"),
		[]byte("cached"),
		[]byte("buffered"),
		[]byte("used_percent"),
		[]byte("available_percent"),
	}
)

type MemMeasurement struct {
	// this doesn't change:
	bytesTotal int64

	// these change:
	timestamp                                         time.Time
	bytesUsedDist, bytesCachedDist, bytesBufferedDist Distribution
}

func NewMemMeasurement(start time.Time) *MemMeasurement {
	bytesTotal := MemoryMaxBytesChoices[rand.Intn(len(MemoryMaxBytesChoices))]
	bytesUsedDist := &ClampedRandomWalkDistribution{
		State: rand.Float64() * float64(bytesTotal),
		Min:   0.0,
		Max:   float64(bytesTotal),
		Step: &NormalDistribution{
			Mean:   0.0,
			StdDev: float64(bytesTotal) / 64,
		},
	}
	bytesCachedDist := &ClampedRandomWalkDistribution{
		State: rand.Float64() * float64(bytesTotal),
		Min:   0.0,
		Max:   float64(bytesTotal),
		Step: &NormalDistribution{
			Mean:   0.0,
			StdDev: float64(bytesTotal) / 64,
		},
	}
	bytesBufferedDist := &ClampedRandomWalkDistribution{
		State: rand.Float64() * float64(bytesTotal),
		Min:   0.0,
		Max:   float64(bytesTotal),
		Step: &NormalDistribution{
			Mean:   0.0,
			StdDev: float64(bytesTotal) / 64,
		},
	}
	return &MemMeasurement{
		timestamp:   start,

		bytesTotal:        bytesTotal,
		bytesUsedDist:     bytesUsedDist,
		bytesCachedDist:   bytesCachedDist,
		bytesBufferedDist: bytesBufferedDist,
	}
}

func (m *MemMeasurement) Tick(d time.Duration) {
	m.timestamp = m.timestamp.Add(d)

	m.bytesUsedDist.Advance()
	m.bytesCachedDist.Advance()
	m.bytesBufferedDist.Advance()
}

func (m *MemMeasurement) ToPoint(p *Point) {
	p.SetMeasurementName(MemoryByteString)
	p.SetTimestamp(&m.timestamp)

	total := m.bytesTotal
	used := m.bytesUsedDist.Get()
	cached := m.bytesCachedDist.Get()
	buffered := m.bytesBufferedDist.Get()

	p.AppendField(MemoryFieldKeys[0], total)
	p.AppendField(MemoryFieldKeys[1], int(math.Floor(float64(total)-used)))
	p.AppendField(MemoryFieldKeys[2], int(math.Floor(used)))
	p.AppendField(MemoryFieldKeys[3], int(math.Floor(cached)))
	p.AppendField(MemoryFieldKeys[4], int(math.Floor(buffered)))
	p.AppendField(MemoryFieldKeys[5], int(math.Floor(used)))
	p.AppendField(MemoryFieldKeys[6], 100.0*(used/float64(total)))
	p.AppendField(MemoryFieldKeys[7], 100.0*(float64(total)-used)/float64(total))
}
<commit_msg>add one more metric<commit_after>package main

import (
	"math"
	"math/rand"
	"time"
)

var (
	MemoryByteString = []byte("mem") // heap optimization

	// Choices for modeling a host's memory capacity.
	MemoryMaxBytesChoices = []int64{8 << 30, 12 << 30, 16 << 30}

	// Field keys for 'mem' points.
	MemoryFieldKeys = [][]byte{
		[]byte("total"),
		[]byte("available"),
		[]byte("used"),
		[]byte("free"),
		[]byte("cached"),
		[]byte("buffered"),
		[]byte("used_percent"),
		[]byte("available_percent"),
		[]byte("buffered_percent"),
	}
)

type MemMeasurement struct {
	// this doesn't change:
	bytesTotal int64

	// these change:
	timestamp                                         time.Time
	bytesUsedDist, bytesCachedDist, bytesBufferedDist Distribution
}

func NewMemMeasurement(start time.Time) *MemMeasurement {
	bytesTotal := MemoryMaxBytesChoices[rand.Intn(len(MemoryMaxBytesChoices))]
	bytesUsedDist := &ClampedRandomWalkDistribution{
		State: rand.Float64() * float64(bytesTotal),
		Min:   0.0,
		Max:   float64(bytesTotal),
		Step: &NormalDistribution{
			Mean:   0.0,
			StdDev: float64(bytesTotal) / 64,
		},
	}
	bytesCachedDist := &ClampedRandomWalkDistribution{
		State: rand.Float64() * float64(bytesTotal),
		Min:   0.0,
		Max:   float64(bytesTotal),
		Step: &NormalDistribution{
			Mean:   0.0,
			StdDev: float64(bytesTotal) / 64,
		},
	}
	bytesBufferedDist := &ClampedRandomWalkDistribution{
		State: rand.Float64() * float64(bytesTotal),
		Min:   0.0,
		Max:   float64(bytesTotal),
		Step: &NormalDistribution{
			Mean:   0.0,
			StdDev: float64(bytesTotal) / 64,
		},
	}
	return &MemMeasurement{
		timestamp:   start,

		bytesTotal:        bytesTotal,
		bytesUsedDist:     bytesUsedDist,
		bytesCachedDist:   bytesCachedDist,
		bytesBufferedDist: bytesBufferedDist,
	}
}

func (m *MemMeasurement) Tick(d time.Duration) {
	m.timestamp = m.timestamp.Add(d)

	m.bytesUsedDist.Advance()
	m.bytesCachedDist.Advance()
	m.bytesBufferedDist.Advance()
}

func (m *MemMeasurement) ToPoint(p *Point) {
	p.SetMeasurementName(MemoryByteString)
	p.SetTimestamp(&m.timestamp)

	total := m.bytesTotal
	used := m.bytesUsedDist.Get()
	cached := m.bytesCachedDist.Get()
	buffered := m.bytesBufferedDist.Get()

	p.AppendField(MemoryFieldKeys[0], total)
	p.AppendField(MemoryFieldKeys[1], int(math.Floor(float64(total)-used)))
	p.AppendField(MemoryFieldKeys[2], int(math.Floor(used)))
	p.AppendField(MemoryFieldKeys[3], int(math.Floor(cached)))
	p.AppendField(MemoryFieldKeys[4], int(math.Floor(buffered)))
	p.AppendField(MemoryFieldKeys[5], int(math.Floor(used)))
	p.AppendField(MemoryFieldKeys[6], 100.0*(used/float64(total)))
	p.AppendField(MemoryFieldKeys[7], 100.0*(float64(total)-used)/float64(total))
	p.AppendField(MemoryFieldKeys[8], 100.0*(float64(total)-buffered)/float64(total))
}
<|endoftext|>


<commit_before>package redis

import (
	"crypto/sha1"
	"encoding/hex"
	"io"
	"strings"
)

type scripter interface {
	Eval(script string, keys []string, args ...interface{}) *Cmd
	EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd
	ScriptExists(scripts ...string) *BoolSliceCmd
	ScriptLoad(script string) *StringCmd
}

var _ scripter = (*Client)(nil)
var _ scripter = (*Ring)(nil)
var _ scripter = (*ClusterClient)(nil)

type Script struct {
	src, hash string
}

func NewScript(src string) *Script {
	h := sha1.New()
	io.WriteString(h, src)
	return &Script{
		src:  src,
		hash: hex.EncodeToString(h.Sum(nil)),
	}
}

func (s *Script) Load(c scripter) *StringCmd {
	return c.ScriptLoad(s.src)
}

func (s *Script) Exists(c scripter) *BoolSliceCmd {
	return c.ScriptExists(s.src)
}

func (s *Script) Eval(c scripter, keys []string, args ...interface{}) *Cmd {
	return c.Eval(s.src, keys, args...)
}

func (s *Script) EvalSha(c scripter, keys []string, args ...interface{}) *Cmd {
	return c.EvalSha(s.hash, keys, args...)
}

// Run optimistically uses EVALSHA to run the script. If script does not exist
// it is retried using EVAL.
func (s *Script) Run(c scripter, keys []string, args ...interface{}) *Cmd {
	r := s.EvalSha(c, keys, args...)
	if err := r.Err(); err != nil && strings.HasPrefix(err.Error(), "NOSCRIPT ") {
		return s.Eval(c, keys, args...)
	}
	return r
}
<commit_msg>Add Script.Hash<commit_after>package redis

import (
	"crypto/sha1"
	"encoding/hex"
	"io"
	"strings"
)

type scripter interface {
	Eval(script string, keys []string, args ...interface{}) *Cmd
	EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd
	ScriptExists(scripts ...string) *BoolSliceCmd
	ScriptLoad(script string) *StringCmd
}

var _ scripter = (*Client)(nil)
var _ scripter = (*Ring)(nil)
var _ scripter = (*ClusterClient)(nil)

type Script struct {
	src, hash string
}

func NewScript(src string) *Script {
	h := sha1.New()
	io.WriteString(h, src)
	return &Script{
		src:  src,
		hash: hex.EncodeToString(h.Sum(nil)),
	}
}

func (s *Script) Hash() string {
	return s.hash
}

func (s *Script) Load(c scripter) *StringCmd {
	return c.ScriptLoad(s.src)
}

func (s *Script) Exists(c scripter) *BoolSliceCmd {
	return c.ScriptExists(s.src)
}

func (s *Script) Eval(c scripter, keys []string, args ...interface{}) *Cmd {
	return c.Eval(s.src, keys, args...)
}

func (s *Script) EvalSha(c scripter, keys []string, args ...interface{}) *Cmd {
	return c.EvalSha(s.hash, keys, args...)
}

// Run optimistically uses EVALSHA to run the script. If script does not exist
// it is retried using EVAL.
func (s *Script) Run(c scripter, keys []string, args ...interface{}) *Cmd {
	r := s.EvalSha(c, keys, args...)
	if err := r.Err(); err != nil && strings.HasPrefix(err.Error(), "NOSCRIPT ") {
		return s.Eval(c, keys, args...)
	}
	return r
}
<|endoftext|>


<commit_before><commit_msg>Always add date to slug<commit_after><|endoftext|>


<commit_before>package collector

import (
	"fmt"
	"sync"

	"github.com/henrylee2cn/pholcus/common/kafka"
	"github.com/henrylee2cn/pholcus/common/util"
)

/************************ Kafka 输出 ***************************/
func init() {
	var (
		kafkaSenders    = map[string]*kafka.KafkaSender{}
		kafkaSenderLock sync.RWMutex
	)

	var getKafkaSender = func(name string) (*kafka.KafkaSender, bool) {
		kafkaSenderLock.RLock()
		tab, ok := kafkaSenders[name]
		kafkaSenderLock.RUnlock()
		return tab, ok
	}

	var setKafkaSender = func(name string, tab *kafka.KafkaSender) {
		kafkaSenderLock.Lock()
		kafkaSenders[name] = tab
		kafkaSenderLock.Unlock()
	}

	DataOutput["kafka"] = func(self *Collector) error {
		_, err := kafka.GetProducer()
		if err != nil {
			return fmt.Errorf("kafka producer失败: %v", err)
		}
		var (
			kafkas    = make(map[string]*kafka.KafkaSender)
			namespace = util.FileNameReplace(self.namespace())
		)
		for _, datacell := range self.dataDocker {
			subNamespace := util.FileNameReplace(self.subNamespace(datacell))
			topicName := joinNamespaces(namespace, subNamespace)
			sender, ok := kafkas[topicName]
			if !ok {
				sender, ok = getKafkaSender(topicName)
				if ok {
					kafkas[topicName] = sender
				} else {
					sender = kafka.New()
					sender.SetTopic(topicName)
					setKafkaSender(topicName, sender)
					kafkas[topicName] = sender
				}
			}
			data := make(map[string]interface{})
			for _, title := range self.MustGetRule(datacell["RuleName"].(string)).ItemFields {
				vd := datacell["Data"].(map[string]interface{})
				if v, ok := vd[title].(string); ok || vd[title] == nil {
					data[title] = v
				} else {
					data[title] = util.JsonString(vd[title])
				}
			}
			if self.Spider.OutDefaultField() {
				data["url"] = datacell["Url"].(string)
				data["parent_url"] = datacell["ParentUrl"].(string)
				data["download_time"] = datacell["DownloadTime"].(string)
			}
			err := sender.Push(data)
			util.CheckErr(err)
		}
		kafkas = nil
		return nil
	}
}
<commit_msg>Compatible with Kafka v0.11.0.0<commit_after>package collector

import (
	"fmt"
	"regexp"
	"sync"

	"github.com/henrylee2cn/pholcus/common/kafka"
	"github.com/henrylee2cn/pholcus/common/util"
	"github.com/henrylee2cn/pholcus/logs"
)

/************************ Kafka 输出 ***************************/
func init() {
	var (
		kafkaSenders    = map[string]*kafka.KafkaSender{}
		kafkaSenderLock sync.RWMutex
	)

	var getKafkaSender = func(name string) (*kafka.KafkaSender, bool) {
		kafkaSenderLock.RLock()
		tab, ok := kafkaSenders[name]
		kafkaSenderLock.RUnlock()
		return tab, ok
	}

	var setKafkaSender = func(name string, tab *kafka.KafkaSender) {
		kafkaSenderLock.Lock()
		kafkaSenders[name] = tab
		kafkaSenderLock.Unlock()
	}

	var topic = regexp.MustCompile("^[0-9a-zA-Z_-]+$")

	DataOutput["kafka"] = func(self *Collector) error {
		_, err := kafka.GetProducer()
		if err != nil {
			return fmt.Errorf("kafka producer失败: %v", err)
		}
		var (
			kafkas    = make(map[string]*kafka.KafkaSender)
			namespace = util.FileNameReplace(self.namespace())
		)
		for _, datacell := range self.dataDocker {
			subNamespace := util.FileNameReplace(self.subNamespace(datacell))
			topicName := joinNamespaces(namespace, subNamespace)
			if !topic.MatchString(topicName) {
				logs.Log.Error("topic格式要求'^[0-9a-zA-Z_-]+$'，当前为：%s", topicName)
				continue
			}
			sender, ok := kafkas[topicName]
			if !ok {
				sender, ok = getKafkaSender(topicName)
				if ok {
					kafkas[topicName] = sender
				} else {
					sender = kafka.New()
					sender.SetTopic(topicName)
					setKafkaSender(topicName, sender)
					kafkas[topicName] = sender
				}
			}
			data := make(map[string]interface{})
			for _, title := range self.MustGetRule(datacell["RuleName"].(string)).ItemFields {
				vd := datacell["Data"].(map[string]interface{})
				if v, ok := vd[title].(string); ok || vd[title] == nil {
					data[title] = v
				} else {
					data[title] = util.JsonString(vd[title])
				}
			}
			if self.Spider.OutDefaultField() {
				data["url"] = datacell["Url"].(string)
				data["parent_url"] = datacell["ParentUrl"].(string)
				data["download_time"] = datacell["DownloadTime"].(string)
			}
			err := sender.Push(data)
			util.CheckErr(err)
		}
		kafkas = nil
		return nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestTargetWorker(t *testing.T) {
	type D1 struct {
		Error string
	}
	f := func(bucket, object, localPath string) error {
		log.Infof("f(%v, %v, %v)\n", bucket, object, localPath)
		if localPath == "" {
			log.Infof("localPath is blank\n")
			return fmt.Errorf("localPath is blank")
		}
		log.Infof("localPath is NOT blank\n")
		time.Sleep(100 * time.Millisecond)
		return nil
	}

	workers := TargetWorkers{
		&TargetWorker{name: "w1", impl: f, maxTries: 1, interval: 1 * time.Second},
		&TargetWorker{name: "w2", impl: f, maxTries: 1, interval: 1 * time.Second},
		&TargetWorker{name: "w3", impl: f, maxTries: 1, interval: 1 * time.Second},
	}

	// Empty targets
	targets0 := Targets{}
	err := workers.process(targets0)
	assert.NoError(t, err)

	// 1 target
	targets1 := Targets{
		&Target{LocalPath: "foo"},
	}
	err = workers.process(targets1)
	assert.NoError(t, err)

	// 1 error target
	targets1Error := Targets{
		&Target{},
	}
	err = workers.process(targets1Error)
	assert.Equal(t, "localPath is blank", err.Error())

	// 1 success and 1 error
	targets1Success1Error := Targets{
		&Target{LocalPath: "foo"},
		&Target{},
	}
	err = workers.process(targets1Success1Error)
	assert.Equal(t, "localPath is blank", err.Error())

	// 3 success and 2 error
	targets3Success2Error := Targets{
		&Target{},
		&Target{LocalPath: "foo"},
		&Target{LocalPath: "foo"},
		&Target{},
		&Target{LocalPath: "foo"},
	}
	err = workers.process(targets3Success2Error)
	assert.Equal(t, "localPath is blank\nlocalPath is blank", err.Error())

	// 3 success
	targets3Success := Targets{
		&Target{LocalPath: "foo"},
		&Target{LocalPath: "foo"},
		&Target{LocalPath: "foo"},
	}
	err = workers.process(targets3Success)
	assert.NoError(t, err)
}
<commit_msg>:shower:  Remove test for TargetWorker<commit_after><|endoftext|>