<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package operation_test

import (
	"github.com/juju/errors"
	"github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"
	"gopkg.in/juju/charm.v6-unstable/hooks"

	"github.com/juju/juju/worker/uniter/hook"
	"github.com/juju/juju/worker/uniter/operation"
	"github.com/juju/juju/worker/uniter/runner"
	"github.com/juju/juju/worker/uniter/runner/context"
)

type FailActionSuite struct {
	testing.IsolationSuite
}

var _ = gc.Suite(&FailActionSuite{})

func (s *FailActionSuite) TestPrepareErrorBadActionAndFailSucceeds(c *gc.C) {
	errBadAction := runner.NewBadActionError("some-action-id", "splat")
	runnerFactory := &MockRunnerFactory{
		MockNewActionRunner: &MockNewActionRunner{err: errBadAction},
	}
	callbacks := &RunActionCallbacks{
		MockFailAction: &MockFailAction{err: errors.New("squelch")},
	}
	factory := operation.NewFactory(operation.FactoryParams{
		RunnerFactory: runnerFactory,
		Callbacks:     callbacks,
	})
	op, err := factory.NewFailAction(someActionId)
	c.Assert(err, jc.ErrorIsNil)

	newState, err := op.Prepare(operation.State{})
	c.Assert(newState, gc.IsNil)
	c.Assert(err, gc.ErrorMatches, "squelch")
	c.Assert(*runnerFactory.MockNewActionRunner.gotActionId, gc.Equals, someActionId)
	c.Assert(*callbacks.MockFailAction.gotActionId, gc.Equals, someActionId)
	c.Assert(*callbacks.MockFailAction.gotMessage, gc.Equals, errBadAction.Error())
}

func (s *FailActionSuite) TestPrepareErrorBadActionAndFailErrors(c *gc.C) {
	errBadAction := runner.NewBadActionError("some-action-id", "foof")
	runnerFactory := &MockRunnerFactory{
		MockNewActionRunner: &MockNewActionRunner{err: errBadAction},
	}
	callbacks := &RunActionCallbacks{
		MockFailAction: &MockFailAction{},
	}
	factory := operation.NewFactory(operation.FactoryParams{
		RunnerFactory: runnerFactory,
		Callbacks:     callbacks,
	})
	op, err := factory.NewAction(someActionId)
	c.Assert(err, jc.ErrorIsNil)

	newState, err := op.Prepare(operation.State{})
	c.Assert(newState, gc.IsNil)
	c.Assert(err, gc.Equals, operation.ErrSkipExecute)
	c.Assert(*runnerFactory.MockNewActionRunner.gotActionId, gc.Equals, someActionId)
	c.Assert(*callbacks.MockFailAction.gotActionId, gc.Equals, someActionId)
	c.Assert(*callbacks.MockFailAction.gotMessage, gc.Equals, errBadAction.Error())
}

func (s *FailActionSuite) TestPrepareErrorActionNotAvailable(c *gc.C) {
	runnerFactory := &MockRunnerFactory{
		MockNewActionRunner: &MockNewActionRunner{err: runner.ErrActionNotAvailable},
	}
	factory := operation.NewFactory(operation.FactoryParams{
		RunnerFactory: runnerFactory,
	})
	op, err := factory.NewAction(someActionId)
	c.Assert(err, jc.ErrorIsNil)

	newState, err := op.Prepare(operation.State{})
	c.Assert(newState, gc.IsNil)
	c.Assert(err, gc.Equals, operation.ErrSkipExecute)
	c.Assert(*runnerFactory.MockNewActionRunner.gotActionId, gc.Equals, someActionId)
}

func (s *FailActionSuite) TestPrepareErrorOther(c *gc.C) {
	runnerFactory := &MockRunnerFactory{
		MockNewActionRunner: &MockNewActionRunner{err: errors.New("foop")},
	}
	factory := operation.NewFactory(operation.FactoryParams{
		RunnerFactory: runnerFactory,
	})
	op, err := factory.NewAction(someActionId)
	c.Assert(err, jc.ErrorIsNil)

	newState, err := op.Prepare(operation.State{})
	c.Assert(newState, gc.IsNil)
	c.Assert(err, gc.ErrorMatches, `cannot create runner for action ".*": foop`)
	c.Assert(*runnerFactory.MockNewActionRunner.gotActionId, gc.Equals, someActionId)
}

func (s *FailActionSuite) TestPrepareCtxCalled(c *gc.C) {
	ctx := &MockContext{actionData: &context.ActionData{Name: "some-action-name"}}
	runnerFactory := &MockRunnerFactory{
		MockNewActionRunner: &MockNewActionRunner{
			runner: &MockRunner{
				context: ctx,
			},
		},
	}
	factory := operation.NewFactory(operation.FactoryParams{
		RunnerFactory: runnerFactory,
	})
	op, err := factory.NewAction(someActionId)
	c.Assert(err, jc.ErrorIsNil)

	newState, err := op.Prepare(operation.State{})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(newState, gc.NotNil)
	ctx.CheckCall(c, 0, "Prepare")
}

func (s *FailActionSuite) TestPrepareCtxError(c *gc.C) {
	ctx := &MockContext{actionData: &context.ActionData{Name: "some-action-name"}}
	ctx.SetErrors(errors.New("ctx prepare error"))
	runnerFactory := &MockRunnerFactory{
		MockNewActionRunner: &MockNewActionRunner{
			runner: &MockRunner{
				context: ctx,
			},
		},
	}
	factory := operation.NewFactory(operation.FactoryParams{
		RunnerFactory: runnerFactory,
	})
	op, err := factory.NewAction(someActionId)
	c.Assert(err, jc.ErrorIsNil)

	newState, err := op.Prepare(operation.State{})
	c.Assert(err, gc.ErrorMatches, `ctx prepare error`)
	c.Assert(newState, gc.IsNil)
	ctx.CheckCall(c, 0, "Prepare")
}

func (s *FailActionSuite) TestPrepareSuccessCleanState(c *gc.C) {
	runnerFactory := NewRunActionRunnerFactory(errors.New("should not call"))
	factory := operation.NewFactory(operation.FactoryParams{
		RunnerFactory: runnerFactory,
	})
	op, err := factory.NewAction(someActionId)
	c.Assert(err, jc.ErrorIsNil)

	newState, err := op.Prepare(operation.State{})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(newState, jc.DeepEquals, &operation.State{
		Kind:     operation.RunAction,
		Step:     operation.Pending,
		ActionId: &someActionId,
	})
	c.Assert(*runnerFactory.MockNewActionRunner.gotActionId, gc.Equals, someActionId)
}

func (s *FailActionSuite) TestPrepareSuccessDirtyState(c *gc.C) {
	runnerFactory := NewRunActionRunnerFactory(errors.New("should not call"))
	factory := operation.NewFactory(operation.FactoryParams{
		RunnerFactory: runnerFactory,
	})
	op, err := factory.NewAction(someActionId)
	c.Assert(err, jc.ErrorIsNil)

	newState, err := op.Prepare(overwriteState)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(newState, jc.DeepEquals, &operation.State{
		Kind:     operation.RunAction,
		Step:     operation.Pending,
		ActionId: &someActionId,
		Started:  true,
		Hook:     &hook.Info{Kind: hooks.Install},
	})
	c.Assert(*runnerFactory.MockNewActionRunner.gotActionId, gc.Equals, someActionId)
}

func (s *FailActionSuite) TestExecuteSuccess(c *gc.C) {
	var stateChangeTests = []struct {
		description string
		before      operation.State
		after       operation.State
	}{{
		description: "empty state",
		after: operation.State{
			Kind:     operation.RunAction,
			Step:     operation.Done,
			ActionId: &someActionId,
		},
	}, {
		description: "preserves appropriate fields",
		before:      overwriteState,
		after: operation.State{
			Kind:     operation.RunAction,
			Step:     operation.Done,
			ActionId: &someActionId,
			Hook:     &hook.Info{Kind: hooks.Install},
			Started:  true,
		},
	}}

	for i, test := range stateChangeTests {
		c.Logf("test %d: %s", i, test.description)
		runnerFactory := NewRunActionRunnerFactory(nil)
		callbacks := &RunActionCallbacks{}
		factory := operation.NewFactory(operation.FactoryParams{
			RunnerFactory: runnerFactory,
			Callbacks:     callbacks,
		})
		op, err := factory.NewAction(someActionId)
		c.Assert(err, jc.ErrorIsNil)
		midState, err := op.Prepare(test.before)
		c.Assert(midState, gc.NotNil)
		c.Assert(err, jc.ErrorIsNil)

		newState, err := op.Execute(*midState)
		c.Assert(err, jc.ErrorIsNil)
		c.Assert(newState, jc.DeepEquals, &test.after)
		c.Assert(callbacks.executingMessage, gc.Equals, "running action some-action-name")
		c.Assert(*runnerFactory.MockNewActionRunner.runner.MockRunAction.gotName, gc.Equals, "some-action-name")
	}
}

func (s *FailActionSuite) TestCommit(c *gc.C) {
	var stateChangeTests = []struct {
		description string
		before      operation.State
		after       operation.State
	}{{
		description: "empty state",
		after: operation.State{
			Kind: operation.Continue,
			Step: operation.Pending,
		},
	}, {
		description: "preserves only appropriate fields, no hook",
		before: operation.State{
			Kind:     operation.Continue,
			Step:     operation.Pending,
			Started:  true,
			CharmURL: curl("cs:quantal/wordpress-2"),
			ActionId: &randomActionId,
		},
		after: operation.State{
			Kind:    operation.Continue,
			Step:    operation.Pending,
			Started: true,
		},
	}, {
		description: "preserves only appropriate fields, with hook",
		before: operation.State{
			Kind:     operation.Continue,
			Step:     operation.Pending,
			Started:  true,
			CharmURL: curl("cs:quantal/wordpress-2"),
			ActionId: &randomActionId,
			Hook:     &hook.Info{Kind: hooks.Install},
		},
		after: operation.State{
			Kind:    operation.RunHook,
			Step:    operation.Pending,
			Hook:    &hook.Info{Kind: hooks.Install},
			Started: true,
		},
	}}

	for i, test := range stateChangeTests {
		c.Logf("test %d: %s", i, test.description)
		factory := operation.NewFactory(operation.FactoryParams{})
		op, err := factory.NewAction(someActionId)
		c.Assert(err, jc.ErrorIsNil)

		newState, err := op.Commit(test.before)
		c.Assert(newState, jc.DeepEquals, &test.after)
	}
}

func (s *FailActionSuite) TestNeedsGlobalMachineLock(c *gc.C) {
	factory := operation.NewFactory(operation.FactoryParams{})
	op, err := factory.NewAction(someActionId)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(op.NeedsGlobalMachineLock(), jc.IsTrue)
}
<commit_msg>Test changes<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package operation_test

import (
	"github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"
	"gopkg.in/juju/charm.v6-unstable/hooks"

	"github.com/juju/juju/worker/uniter/hook"
	"github.com/juju/juju/worker/uniter/operation"
)

type FailActionSuite struct {
	testing.IsolationSuite
}

var _ = gc.Suite(&FailActionSuite{})

func (s *FailActionSuite) TestPrepare(c *gc.C) {
	factory := operation.NewFactory(operation.FactoryParams{})
	op, err := factory.NewFailAction(someActionId)
	c.Assert(err, jc.ErrorIsNil)

	newState, err := op.Prepare(operation.State{})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(newState, jc.DeepEquals, &operation.State{
		Kind:     operation.RunAction,
		Step:     operation.Pending,
		ActionId: &someActionId,
	})
}

func (s *FailActionSuite) TestExecuteSuccess(c *gc.C) {
	var stateChangeTests = []struct {
		description string
		before      operation.State
		after       operation.State
	}{{
		description: "empty state",
		after: operation.State{
			Kind:     operation.RunAction,
			Step:     operation.Done,
			ActionId: &someActionId,
		},
	}, {
		description: "preserves appropriate fields",
		before:      overwriteState,
		after: operation.State{
			Kind:     operation.RunAction,
			Step:     operation.Done,
			ActionId: &someActionId,
			Hook:     &hook.Info{Kind: hooks.Install},
			Started:  true,
		},
	}}

	for i, test := range stateChangeTests {
		c.Logf("test %d: %s", i, test.description)
		runnerFactory := NewRunActionRunnerFactory(nil)
		callbacks := &RunActionCallbacks{}
		factory := operation.NewFactory(operation.FactoryParams{
			RunnerFactory: runnerFactory,
			Callbacks:     callbacks,
		})
		op, err := factory.NewFailAction(someActionId)
		c.Assert(err, jc.ErrorIsNil)
		midState, err := op.Prepare(test.before)
		c.Assert(midState, gc.NotNil)
		c.Assert(err, jc.ErrorIsNil)

		newState, err := op.Execute(*midState)
		c.Assert(err, jc.ErrorIsNil)
		c.Assert(newState, jc.DeepEquals, &test.after)
		c.Assert(callbacks.executingMessage, gc.Equals, "running action some-action-name")
		c.Assert(*runnerFactory.MockNewActionRunner.runner.MockRunAction.gotName, gc.Equals, "some-action-name")
	}
}

func (s *FailActionSuite) TestCommit(c *gc.C) {
	var stateChangeTests = []struct {
		description string
		before      operation.State
		after       operation.State
	}{{
		description: "empty state",
		after: operation.State{
			Kind: operation.Continue,
			Step: operation.Pending,
		},
	}, {
		description: "preserves only appropriate fields, no hook",
		before: operation.State{
			Kind:     operation.Continue,
			Step:     operation.Pending,
			Started:  true,
			CharmURL: curl("cs:quantal/wordpress-2"),
			ActionId: &randomActionId,
		},
		after: operation.State{
			Kind:    operation.Continue,
			Step:    operation.Pending,
			Started: true,
		},
	}, {
		description: "preserves only appropriate fields, with hook",
		before: operation.State{
			Kind:     operation.Continue,
			Step:     operation.Pending,
			Started:  true,
			CharmURL: curl("cs:quantal/wordpress-2"),
			ActionId: &randomActionId,
			Hook:     &hook.Info{Kind: hooks.Install},
		},
		after: operation.State{
			Kind:    operation.RunHook,
			Step:    operation.Pending,
			Hook:    &hook.Info{Kind: hooks.Install},
			Started: true,
		},
	}}

	for i, test := range stateChangeTests {
		c.Logf("test %d: %s", i, test.description)
		factory := operation.NewFactory(operation.FactoryParams{})
		op, err := factory.NewFailAction(someActionId)
		c.Assert(err, jc.ErrorIsNil)

		newState, err := op.Commit(test.before)
		c.Assert(newState, jc.DeepEquals, &test.after)
	}
}

func (s *FailActionSuite) TestNeedsGlobalMachineLock(c *gc.C) {
	factory := operation.NewFactory(operation.FactoryParams{})
	op, err := factory.NewFailAction(someActionId)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(op.NeedsGlobalMachineLock(), jc.IsTrue)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"time"

	mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric"
	"go.opencensus.io/tag"

	"go.opentelemetry.io/otel/label"
	"go.opentelemetry.io/otel/metric"
	"go.opentelemetry.io/otel/sdk/metric/controller/push"
	"go.opentelemetry.io/otel/sdk/resource"

	"os"
)

var (
	env       = os.Getenv("ENV")
	projectID = os.Getenv("PROJECT_ID")
)

func main() {

	// Initialization. In order to pass the credentials to the exporter,
	// prepare credential file following the instruction described in this doc.
	// https://pkg.go.dev/golang.org/x/oauth2/google?tab=doc#FindDefaultCredentials
	opts := []mexporter.Option{
		mexporter.WithProjectID(projectID),
	}

	// NOTE: In current implementation of exporter, this resource is ignored
	resOpt := push.WithResource(resource.NewWithAttributes(
		label.String("instance_id", "abc123"),
		label.String("application", "example-app"),
	))
	pusher, err := mexporter.InstallNewPipeline(opts, resOpt)
	if err != nil {
		log.Fatalf("Failed to establish pipeline: %v", err)
	}
	defer pusher.Stop()

	meter := pusher.MeterProvider().Meter("cloudmonitoring/example")
	// Register request counter
	totalRequestsCounter := metric.Must(meter).NewInt64Counter("total_requests")
	// Register error counter
	errorsCounter := metric.Must(meter).NewInt64Counter("failed_requests")

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		ctx, _ := tag.New(context.Background())
		// requestReceived := time.Now()
		// count the request
		totalRequestsCounter.Add(ctx, 1)

		// fail 10% of the time
		if rand.Intn(100) > 90 {
			// count the failed request
			errorsCounter.Add(ctx, 1)
			fmt.Fprintf(w, "error!")
			// record latency for failure
			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
			return
		} else {
			delay := time.Duration(rand.Intn(1000)) * time.Millisecond
			time.Sleep(delay)
			// record latency for success
			fmt.Fprintf(w, "Responded after "+delay.String())
		}
	})
	log.Fatal(http.ListenAndServe(":8080", nil))

}
<commit_msg>latency works in OT/go<commit_after>package main

import (
	"context"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"time"

	mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric"
	"go.opencensus.io/tag"

	"go.opentelemetry.io/otel/label"
	"go.opentelemetry.io/otel/metric"
	"go.opentelemetry.io/otel/sdk/metric/controller/push"
	"go.opentelemetry.io/otel/sdk/resource"

	"os"
)

var (
	env          = os.Getenv("ENV")
	projectID    = os.Getenv("PROJECT_ID")
	latencyValue = 0.0
)

func main() {

	// Initialization. In order to pass the credentials to the exporter,
	// prepare credential file following the instruction described in this doc.
	// https://pkg.go.dev/golang.org/x/oauth2/google?tab=doc#FindDefaultCredentials
	opts := []mexporter.Option{
		mexporter.WithProjectID(projectID),
	}

	// NOTE: In current implementation of exporter, this resource is ignored
	resOpt := push.WithResource(resource.NewWithAttributes(
		label.String("instance_id", "abc123"),
		label.String("application", "example-app"),
	))
	pusher, err := mexporter.InstallNewPipeline(opts, resOpt)
	if err != nil {
		log.Fatalf("Failed to establish pipeline: %v", err)
	}
	defer pusher.Stop()

	meter := pusher.MeterProvider().Meter("cloudmonitoring/example")
	// Register request counter
	totalRequestsCounter := metric.Must(meter).NewInt64Counter("total_requests")
	// Register error counter
	errorsCounter := metric.Must(meter).NewInt64Counter("failed_requests")
	// Register latency observer
	olabels := []label.KeyValue{}
	callback := func(_ context.Context, result metric.Float64ObserverResult) {
		v := latencyValue
		result.Observe(v, olabels...)
	}
	metric.Must(meter).NewFloat64ValueObserver("response_latency", callback)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		ctx, _ := tag.New(context.Background())
		requestReceived := time.Now()
		// count the request
		totalRequestsCounter.Add(ctx, 1)

		// fail 10% of the time
		if rand.Intn(100) > 90 {
			// count the failed request
			errorsCounter.Add(ctx, 1)
			fmt.Fprintf(w, "error!")
			// record latency for failure
			latencyValue = float64(time.Since(requestReceived).Seconds())
			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
			return
		} else {
			delay := time.Duration(rand.Intn(1000)) * time.Millisecond
			time.Sleep(delay)
			// record latency for success
			latencyValue = float64(time.Since(requestReceived).Seconds())
			fmt.Fprintf(w, "Responded after "+delay.String())
		}
	})
	log.Fatal(http.ListenAndServe(":8080", nil))

}
<|endoftext|>


<commit_before>// Instance of the server.
// Copyright © 2015 - Rémy MATHIEU

package server

import (
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"os"
	"time"

	"github.com/boltdb/bolt"
	"github.com/gorilla/mux"
)

const (
	LAST_UPLOADED_KEY = "LastUploaded"
)

type Server struct {
	Config   Config   // Configuration
	Database *bolt.DB // opened bolt db
	Storage  string   // Storage used with this metadata file.
}

func NewServer(config Config) *Server {
	// init the random
	rand.Seed(time.Now().Unix())

	return &Server{
		Config:  config,
		Storage: config.Storage,
	}
}

// Starts the listening daemon.
func (s *Server) Start() {
	router := s.prepareRouter()

	// Setup the router on the net/http stack
	http.Handle("/", router)

	// Open the database
	s.openBoltDatabase()

	go s.StartCleanJob()

	// Listen
	if len(s.Config.CertificateFile) != 0 && len(s.Config.CertificateKey) != 0 {
		log.Println("[info] Start secure listening on", s.Config.Addr)
		err := http.ListenAndServeTLS(s.Config.Addr, s.Config.CertificateFile, s.Config.CertificateKey, nil)
		log.Println("[err]", err.Error())
	} else {
		log.Println("[info] Start listening on", s.Config.Addr)
		err := http.ListenAndServe(s.Config.Addr, nil)
		log.Println("[err]", err.Error())
	}
}

// Starts the Clean Job
func (s *Server) StartCleanJob() {
	timer := time.NewTicker(60 * time.Second)
	for _ = range timer.C {
		job := CleanJob{s}
		job.Run()
	}
}

// writeBoltMetadata stores the metadata in a BoltDB file.
func (s *Server) openBoltDatabase() {
	db, err := bolt.Open(s.Config.RuntimeDir+"/metadata.db", 0600, nil)
	if err != nil {
		log.Println("[err] Can't open the metadata.db file in :", s.Config.RuntimeDir)
		log.Println(err)
		os.Exit(1)
	}

	log.Printf("[info] %s opened.", s.Config.RuntimeDir+"/metadata.db")

	s.Database = db

	// creates the bucket if needed
	db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists([]byte("Metadata"))
		if err != nil {
			log.Println("Can't create the bucket 'Metadata'")
			log.Println(err)
		}
		_, err = tx.CreateBucketIfNotExists([]byte("Runtime"))
		if err != nil {
			log.Println("Can't create the bucket 'Runtime'")
			log.Println(err)
		}
		_, err = tx.CreateBucketIfNotExists([]byte("Config"))
		if err != nil {
			log.Println("Can't create the bucket 'LastUploaded'")
			log.Println(err)
		}
		return err
	})

	// test that the storage is still the same
	var mustSave bool
	s.Database.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("Config"))
		v := bucket.Get([]byte("storage"))
		if v == nil {
			mustSave = true
			return nil
		}

		if string(v) != s.Config.Storage {
			log.Printf("The database use the storage %s, can't start with the storage %s\n", string(v), s.Config.Storage)
			os.Exit(1)
		}
		return nil
	})

	// save the storage
	if mustSave {
		s.Database.Update(func(tx *bolt.Tx) error {
			bucket := tx.Bucket([]byte("Config"))
			bucket.Put([]byte("storage"), []byte(s.Config.Storage))
			return nil
		})
	}
}

func (s *Server) deleteMetadata(name string) error {
	// remove from last uploaded
	entry, err := s.GetEntry(name)
	if err != nil {
		return err
	}
	err = s.Database.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("LastUploaded"))
		return bucket.Delete([]byte(entry.CreationTime.String()))
	})

	err = s.Database.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("Metadata"))
		return bucket.Delete([]byte(name))
	})
	if err != nil {
		log.Println("Can't delete some metadata from the database:")
		log.Println(err)
	}

	return err
}

// getEntry looks in the Bolt DB whether this entry exists and returns it
// if found, otherwise, nil is returned.
func (s *Server) GetEntry(id string) (*Metadata, error) {
	var metadata Metadata
	err := s.Database.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("Metadata"))
		v := bucket.Get([]byte(id))
		if v == nil {
			return nil
		}

		// unmarshal the bytes
		err := json.Unmarshal(v, &metadata)
		if err != nil {
			return err
		}

		return nil
	})

	return &metadata, err
}

// GetLastUploaded reads into BoltDB the array
// of last uploaded entries.
func (s *Server) GetLastUploaded() ([]string, error) {
	values := make([]string, 0)
	err := s.Database.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("Runtime"))
		v := bucket.Get([]byte(LAST_UPLOADED_KEY))

		if v == nil {
			return nil
		}

		err := json.Unmarshal(v, &values)
		return err
	})

	return values, err
}

func (s *Server) SetLastUploaded(lastUploaded []string) error {
	return s.Database.Update(func(tx *bolt.Tx) error {
		d, err := json.Marshal(lastUploaded)
		if err != nil {
			return err
		}

		bucket := tx.Bucket([]byte("Runtime"))
		return bucket.Put([]byte(LAST_UPLOADED_KEY), d)
	})
}

// Prepares the route
func (s *Server) prepareRouter() *mux.Router {
	r := mux.NewRouter()

	println(s.Config.Route)
	sendHandler := &SendHandler{s}
	r.Handle(s.Config.Route+"/1.0/send", sendHandler)

	lastUploadeHandler := &LastUploadedHandler{s}
	r.Handle(s.Config.Route+"/1.0/list", lastUploadeHandler)

	searchTagsHandler := &SearchTagsHandler{s}
	r.Handle(s.Config.Route+"/1.0/search_tags", searchTagsHandler)

	deleteHandler := &DeleteHandler{s}
	r.Handle(s.Config.Route+"/{file}/{key}", deleteHandler)

	sh := &ServingHandler{s}
	r.Handle(s.Config.Route+"/{file}", sh) // Serving route.

	return r
}
<commit_msg>Unused import.<commit_after>// Instance of the server.
// Copyright © 2015 - Rémy MATHIEU

package server

import (
	"encoding/json"
	"log"
	"math/rand"
	"net/http"
	"os"
	"time"

	"github.com/boltdb/bolt"
	"github.com/gorilla/mux"
)

const (
	LAST_UPLOADED_KEY = "LastUploaded"
)

type Server struct {
	Config   Config   // Configuration
	Database *bolt.DB // opened bolt db
	Storage  string   // Storage used with this metadata file.
}

func NewServer(config Config) *Server {
	// init the random
	rand.Seed(time.Now().Unix())

	return &Server{
		Config:  config,
		Storage: config.Storage,
	}
}

// Starts the listening daemon.
func (s *Server) Start() {
	router := s.prepareRouter()

	// Setup the router on the net/http stack
	http.Handle("/", router)

	// Open the database
	s.openBoltDatabase()

	go s.StartCleanJob()

	// Listen
	if len(s.Config.CertificateFile) != 0 && len(s.Config.CertificateKey) != 0 {
		log.Println("[info] Start secure listening on", s.Config.Addr)
		err := http.ListenAndServeTLS(s.Config.Addr, s.Config.CertificateFile, s.Config.CertificateKey, nil)
		log.Println("[err]", err.Error())
	} else {
		log.Println("[info] Start listening on", s.Config.Addr)
		err := http.ListenAndServe(s.Config.Addr, nil)
		log.Println("[err]", err.Error())
	}
}

// Starts the Clean Job
func (s *Server) StartCleanJob() {
	timer := time.NewTicker(60 * time.Second)
	for _ = range timer.C {
		job := CleanJob{s}
		job.Run()
	}
}

// writeBoltMetadata stores the metadata in a BoltDB file.
func (s *Server) openBoltDatabase() {
	db, err := bolt.Open(s.Config.RuntimeDir+"/metadata.db", 0600, nil)
	if err != nil {
		log.Println("[err] Can't open the metadata.db file in :", s.Config.RuntimeDir)
		log.Println(err)
		os.Exit(1)
	}

	log.Printf("[info] %s opened.", s.Config.RuntimeDir+"/metadata.db")

	s.Database = db

	// creates the bucket if needed
	db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists([]byte("Metadata"))
		if err != nil {
			log.Println("Can't create the bucket 'Metadata'")
			log.Println(err)
		}
		_, err = tx.CreateBucketIfNotExists([]byte("Runtime"))
		if err != nil {
			log.Println("Can't create the bucket 'Runtime'")
			log.Println(err)
		}
		_, err = tx.CreateBucketIfNotExists([]byte("Config"))
		if err != nil {
			log.Println("Can't create the bucket 'LastUploaded'")
			log.Println(err)
		}
		return err
	})

	// test that the storage is still the same
	var mustSave bool
	s.Database.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("Config"))
		v := bucket.Get([]byte("storage"))
		if v == nil {
			mustSave = true
			return nil
		}

		if string(v) != s.Config.Storage {
			log.Printf("The database use the storage %s, can't start with the storage %s\n", string(v), s.Config.Storage)
			os.Exit(1)
		}
		return nil
	})

	// save the storage
	if mustSave {
		s.Database.Update(func(tx *bolt.Tx) error {
			bucket := tx.Bucket([]byte("Config"))
			bucket.Put([]byte("storage"), []byte(s.Config.Storage))
			return nil
		})
	}
}

func (s *Server) deleteMetadata(name string) error {
	// remove from last uploaded
	entry, err := s.GetEntry(name)
	if err != nil {
		return err
	}
	err = s.Database.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("LastUploaded"))
		return bucket.Delete([]byte(entry.CreationTime.String()))
	})

	err = s.Database.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("Metadata"))
		return bucket.Delete([]byte(name))
	})
	if err != nil {
		log.Println("Can't delete some metadata from the database:")
		log.Println(err)
	}

	return err
}

// getEntry looks in the Bolt DB whether this entry exists and returns it
// if found, otherwise, nil is returned.
func (s *Server) GetEntry(id string) (*Metadata, error) {
	var metadata Metadata
	err := s.Database.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("Metadata"))
		v := bucket.Get([]byte(id))
		if v == nil {
			return nil
		}

		// unmarshal the bytes
		err := json.Unmarshal(v, &metadata)
		if err != nil {
			return err
		}

		return nil
	})

	return &metadata, err
}

// GetLastUploaded reads into BoltDB the array
// of last uploaded entries.
func (s *Server) GetLastUploaded() ([]string, error) {
	values := make([]string, 0)
	err := s.Database.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("Runtime"))
		v := bucket.Get([]byte(LAST_UPLOADED_KEY))

		if v == nil {
			return nil
		}

		err := json.Unmarshal(v, &values)
		return err
	})

	return values, err
}

func (s *Server) SetLastUploaded(lastUploaded []string) error {
	return s.Database.Update(func(tx *bolt.Tx) error {
		d, err := json.Marshal(lastUploaded)
		if err != nil {
			return err
		}

		bucket := tx.Bucket([]byte("Runtime"))
		return bucket.Put([]byte(LAST_UPLOADED_KEY), d)
	})
}

// Prepares the route
func (s *Server) prepareRouter() *mux.Router {
	r := mux.NewRouter()

	println(s.Config.Route)
	sendHandler := &SendHandler{s}
	r.Handle(s.Config.Route+"/1.0/send", sendHandler)

	lastUploadeHandler := &LastUploadedHandler{s}
	r.Handle(s.Config.Route+"/1.0/list", lastUploadeHandler)

	searchTagsHandler := &SearchTagsHandler{s}
	r.Handle(s.Config.Route+"/1.0/search_tags", searchTagsHandler)

	deleteHandler := &DeleteHandler{s}
	r.Handle(s.Config.Route+"/{file}/{key}", deleteHandler)

	sh := &ServingHandler{s}
	r.Handle(s.Config.Route+"/{file}", sh) // Serving route.

	return r
}
<|endoftext|>


<commit_before>package organisations

import (
	"encoding/json"
	"fmt"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
	"sort"
)

type service struct {
	cypherRunner neoutils.CypherRunner
	indexManager neoutils.IndexManager
}

//NewCypherOrganisationService returns a new service responsible for writing organisations in Neo4j
func NewCypherOrganisationService(cypherRunner neoutils.CypherRunner, indexManager neoutils.IndexManager) service {
	return service{cypherRunner, indexManager}
}

func (cd service) Initialise() error {
	return neoutils.EnsureConstraints(cd.indexManager, map[string]string{
		"Thing":             "uuid",
		"Concept":           "uuid",
		"Organisation":      "uuid",
		"FactsetIdentifier": "value",
		"TMEIdentifier":     "value",
		"UPPIdentifier":     "value"})
}

func setProps(props *map[string]interface{}, item *string, propName string) {
	if *item != "" {
		(*props)[propName] = *item
	}
}

func setListProps(props *map[string]interface{}, itemList *[]string, propName string) {
	var items []string

	for _, item := range *itemList {
		items = append(items, item)
	}

	if len(items) > 0 {
		(*props)[propName] = items
	}
}

//Write - Writes an Organisation node
func (cd service) Write(thing interface{}) error {

	o := thing.(organisation)
	props := constructOrganisationProperties(o)

	deleteEntityRelationshipsQuery := constructDeleteEntityRelationshipQuery(o.UUID)
	resetOrgQuery := constructResetOrganisationQuery(o.UUID, props)

	queries := []*neoism.CypherQuery{deleteEntityRelationshipsQuery, resetOrgQuery}

	mergingQueriesForOldNodes, err := cd.constructMergingOldOrganisationNodesQueries(o.UUID, o.AlternativeIdentifiers.UUIDS)
	if err != nil {
		return err
	}

	if len(mergingQueriesForOldNodes) != 0 {
		queries = append(queries, mergingQueriesForOldNodes...)
	}

	//ADD all the IDENTIFIER nodes and IDENTIFIES relationships
	for _, alternativeUUID := range o.AlternativeIdentifiers.TME {
		alternativeIdentifierQuery := createNewIdentifierQuery(o.UUID, tmeIdentifierLabel, alternativeUUID)
		queries = append(queries, alternativeIdentifierQuery)
	}

	for _, alternativeUUID := range o.AlternativeIdentifiers.UUIDS {
		alternativeIdentifierQuery := createNewIdentifierQuery(o.UUID, uppIdentifierLabel, alternativeUUID)
		queries = append(queries, alternativeIdentifierQuery)
	}

	if o.AlternativeIdentifiers.FactsetIdentifier != "" {
		queries = append(queries, createNewIdentifierQuery(o.UUID, factsetIdentifierLabel, o.AlternativeIdentifiers.FactsetIdentifier))
	}

	if o.AlternativeIdentifiers.LeiCode != "" {
		queries = append(queries, createNewIdentifierQuery(o.UUID, leiIdentifierLabel, o.AlternativeIdentifiers.LeiCode))
	}

	//add type
	err, stringType := o.Type.String()
	if err == nil {
		setTypeStatement := fmt.Sprintf(`MERGE (o:Thing {uuid: {uuid}})  set o : %s `, stringType)
		setTypeQuery := &neoism.CypherQuery{
			Statement: setTypeStatement,
			Parameters: map[string]interface{}{
				"uuid": o.UUID,
			},
		}
		queries = append(queries, setTypeQuery)

	} else {
		return err
	}

	if o.IndustryClassification != "" {
		industryClassQuery := constructCreateIndustryClassificationQuery(o.UUID, o.IndustryClassification)
		queries = append(queries, industryClassQuery)
	}

	if o.ParentOrganisation != "" {
		parentQuery := constructCreateParentOrganisationQuery(o.UUID, o.ParentOrganisation)
		queries = append(queries, parentQuery)
	}
	logrus.Infof("Query being run: %v", queries)
	return cd.cypherRunner.CypherBatch(queries)
}

func (cd service) constructMergingOldOrganisationNodesQueries(canonicalUUID string, possibleOldNodes []string) ([]*neoism.CypherQuery, error) {

	queries := []*neoism.CypherQuery{}

	for _, identifier := range possibleOldNodes {
		// only nodes with uppAuthority can be older organisation nodes
		if identifier != canonicalUUID {
			nodeExists, err := cd.checkNodeExistence(identifier)
			if err != nil {
				return nil, err
			}
			if nodeExists {
				deleteEntityRelationshipsForDeprecatedOrgNodeQuery := constructDeleteEntityRelationshipQuery(identifier)
				queries = append(queries, deleteEntityRelationshipsForDeprecatedOrgNodeQuery)

				// re-point the remaining relationships from previous node to the canonical/actual one
				transferQueries, err := CreateTransferRelationshipsQueries(cd.cypherRunner, canonicalUUID, identifier)
				if err != nil {
					return nil, err
				}
				if len(transferQueries) != 0 {
					queries = append(queries, transferQueries...)
				}

				// delete oldOrg
				deleteOldOrganisationQuery := constructDeleteEmptyNodeQuery(identifier)
				queries = append(queries, deleteOldOrganisationQuery)
			}
		}
	}

	return queries, nil
}

func (cd service) checkNodeExistence(uuid string) (bool, error) {
	type result []struct {
		Count int `json:"nr"`
	}
	res := result{}

	checkNodeExistenceQuery := &neoism.CypherQuery{
		Statement: `match (a:Thing{uuid:{uuid}})
			           return count(a) as nr`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
		Result: &res,
	}

	readQueries := []*neoism.CypherQuery{checkNodeExistenceQuery}
	err := cd.cypherRunner.CypherBatch(readQueries)

	if err != nil {
		return false, err
	}

	if len(res) != 1 {
		return false, fmt.Errorf("DB inconsistence: one count result should be returned for node with UUID %s", uuid)
	}

	if res[0].Count == 0 {
		return false, nil
	} else if res[0].Count == 1 {
		return true, nil
	} else {
		return false, fmt.Errorf("DB inconsistence: %d node (instead of max 1) exists with UUID %s", res[0].Count, uuid)
	}
}

//Read - Internal Read of an Organisation
func (cd service) Read(uuid string) (interface{}, bool, error) {

	results := []struct {
		UUID                   string                 `json:"uuid"`
		Type                   []string               `json:"type"`
		ProperName             string                 `json:"properName"`
		PrefLabel              string                 `json:"prefLabel"`
		LegalName              string                 `json:"legalName"`
		ShortName              string                 `json:"shortName"`
		HiddenLabel            string                 `json:"hiddenLabel"`
		AlternativeIdentifiers alternativeIdentifiers `json:"alternativeIdentifiers"`
		TradeNames             []string               `json:"tradeNames"`
		LocalNames             []string               `json:"localNames"`
		FormerNames            []string               `json:"formerNames"`
		Aliases                []string               `json:"aliases"`
		IndustryClassification string                 `json:"industryClassification"`
		ParentOrganisation     string                 `json:"parentOrganisation"`
	}{}

	readQuery := &neoism.CypherQuery{
		Statement: `MATCH (o:Organisation:Concept{uuid:{uuid}})
            			OPTIONAL MATCH (o)-[:SUB_ORGANISATION_OF]->(par:Thing)
            			OPTIONAL MATCH (o)-[:HAS_CLASSIFICATION]->(ind:Thing)
           			OPTIONAL MATCH (upp:UPPIdentifier)-[:IDENTIFIES]->(o)
	    			OPTIONAL MATCH (factset:FactsetIdentifier)-[:IDENTIFIES]->(o)
	   			OPTIONAL MATCH (tme:TMEIdentifier)-[:IDENTIFIES]->(o)
	    			OPTIONAL MATCH (lei:LegalEntityIdentifier)-[:IDENTIFIES]->(o)
            		 	RETURN o.uuid as uuid,
					o.properName as properName,
					labels(o) as Type,
					o.prefLabel as prefLabel,
					o.legalName as legalName,
					o.shortName as shortName,
					o.hiddenLabel as hiddenLabel,
					o.formerNames as formerNames,
					o.tradeNames as tradeNames,
					o.localNames as localNames,
					o.aliases as aliases,
					ind.uuid as industryClassification,
					par.uuid as parentOrganisation,
					{uuids:collect(distinct upp.value),
					 TME:collect(distinct tme.value),
					 factsetIdentifier:factset.value,
					 leiCode:lei.value} as alternativeIdentifiers`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
		Result: &results,
	}

	if err := cd.cypherRunner.CypherBatch([]*neoism.CypherQuery{readQuery}); err != nil || len(results) == 0 {
		return organisation{}, false, err
	}

	result := results[0]

	o := organisation{
		UUID:                   result.UUID,
		ProperName:             result.ProperName,
		PrefLabel:              result.PrefLabel,
		LegalName:              result.LegalName,
		ShortName:              result.ShortName,
		HiddenLabel:            result.HiddenLabel,
		TradeNames:             result.TradeNames,
		LocalNames:             result.LocalNames,
		FormerNames:            result.FormerNames,
		AlternativeIdentifiers: result.AlternativeIdentifiers,
		Aliases:                result.Aliases,
		ParentOrganisation:     result.ParentOrganisation,
		IndustryClassification: result.IndustryClassification,
	}

	addType(&o.Type, &result.Type)
	sort.Strings(o.AlternativeIdentifiers.TME)
	sort.Strings(o.AlternativeIdentifiers.UUIDS)

	return o, true, nil
}

func addType(orgType *OrgType, types *[]string) {
	i := len(*types)
	if i == 3 {
		*orgType = Organisation
	}
	if i == 4 {
		*orgType = Company
	}
	if i == 5 {
		*orgType = PublicCompany
	}
}

//Delete - Deletes an Organisation
func (cd service) Delete(uuid string) (bool, error) {
	clearNode := &neoism.CypherQuery{
		Statement: `
			MATCH (org:Thing {uuid: {uuid}})
			OPTIONAL MATCH (org)<-[iden:IDENTIFIES]-(i:Identifier)
			REMOVE org:Concept:Organisation:Company:PublicCompany
			DELETE iden, i
			SET org={uuid: {uuid}}
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
		IncludeStats: true,
	}
	qs := []*neoism.CypherQuery{
		clearNode,
		{
			Statement: `
		MATCH (org:Thing {uuid: {uuid}})
		OPTIONAL MATCH (org)-[a]-(x) WITH org, count(a) AS relCount WHERE relCount = 0
		DELETE org
		`,
			Parameters: map[string]interface{}{
				"uuid": uuid,
			},
		},
	}

	err := cd.cypherRunner.CypherBatch(qs)

	s1, err := clearNode.Stats()

	if err != nil {
		return false, err
	}

	if s1.ContainsUpdates && s1.LabelsRemoved > 0 {
		return true, err
	}

	return false, err
}

func (cd service) Check() error {
	return neoutils.Check(cd.cypherRunner)
}

type countResult []struct {
	Count int `json:"c"`
}

func (cd service) Count() (int, error) {

	results := countResult{}

	err := cd.cypherRunner.CypherBatch([]*neoism.CypherQuery{{
		Statement: `MATCH (n:Organisation) return count(n) as c`,
		Result:    &results,
	}})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

func (cd service) DecodeJSON(dec *json.Decoder) (interface{}, string, error) {
	org := organisation{}
	err := dec.Decode(&org)
	return org, org.UUID, err
}

type requestError struct {
	details string
}

func (re requestError) Error() string {
	return "Invalid Request"
}

func (re requestError) InvalidRequestDetails() string {
	return re.details
}
<commit_msg>Adding even more logging for the frustrating ciiircleci issue<commit_after>package organisations

import (
	"encoding/json"
	"fmt"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
	"sort"
)

type service struct {
	cypherRunner neoutils.CypherRunner
	indexManager neoutils.IndexManager
}

//NewCypherOrganisationService returns a new service responsible for writing organisations in Neo4j
func NewCypherOrganisationService(cypherRunner neoutils.CypherRunner, indexManager neoutils.IndexManager) service {
	return service{cypherRunner, indexManager}
}

func (cd service) Initialise() error {
	return neoutils.EnsureConstraints(cd.indexManager, map[string]string{
		"Thing":             "uuid",
		"Concept":           "uuid",
		"Organisation":      "uuid",
		"FactsetIdentifier": "value",
		"TMEIdentifier":     "value",
		"UPPIdentifier":     "value"})
}

func setProps(props *map[string]interface{}, item *string, propName string) {
	if *item != "" {
		(*props)[propName] = *item
	}
}

func setListProps(props *map[string]interface{}, itemList *[]string, propName string) {
	var items []string

	for _, item := range *itemList {
		items = append(items, item)
	}

	if len(items) > 0 {
		(*props)[propName] = items
	}
}

//Write - Writes an Organisation node
func (cd service) Write(thing interface{}) error {

	o := thing.(organisation)
	props := constructOrganisationProperties(o)

	deleteEntityRelationshipsQuery := constructDeleteEntityRelationshipQuery(o.UUID)
	resetOrgQuery := constructResetOrganisationQuery(o.UUID, props)

	queries := []*neoism.CypherQuery{deleteEntityRelationshipsQuery, resetOrgQuery}

	mergingQueriesForOldNodes, err := cd.constructMergingOldOrganisationNodesQueries(o.UUID, o.AlternativeIdentifiers.UUIDS)
	if err != nil {
		return err
	}

	if len(mergingQueriesForOldNodes) != 0 {
		queries = append(queries, mergingQueriesForOldNodes...)
	}

	//ADD all the IDENTIFIER nodes and IDENTIFIES relationships
	for _, alternativeUUID := range o.AlternativeIdentifiers.TME {
		alternativeIdentifierQuery := createNewIdentifierQuery(o.UUID, tmeIdentifierLabel, alternativeUUID)
		queries = append(queries, alternativeIdentifierQuery)
	}

	for _, alternativeUUID := range o.AlternativeIdentifiers.UUIDS {
		alternativeIdentifierQuery := createNewIdentifierQuery(o.UUID, uppIdentifierLabel, alternativeUUID)
		queries = append(queries, alternativeIdentifierQuery)
	}

	if o.AlternativeIdentifiers.FactsetIdentifier != "" {
		queries = append(queries, createNewIdentifierQuery(o.UUID, factsetIdentifierLabel, o.AlternativeIdentifiers.FactsetIdentifier))
	}

	if o.AlternativeIdentifiers.LeiCode != "" {
		queries = append(queries, createNewIdentifierQuery(o.UUID, leiIdentifierLabel, o.AlternativeIdentifiers.LeiCode))
	}

	//add type
	err, stringType := o.Type.String()
	if err == nil {
		setTypeStatement := fmt.Sprintf(`MERGE (o:Thing {uuid: {uuid}})  set o : %s `, stringType)
		setTypeQuery := &neoism.CypherQuery{
			Statement: setTypeStatement,
			Parameters: map[string]interface{}{
				"uuid": o.UUID,
			},
		}
		queries = append(queries, setTypeQuery)

	} else {
		return err
	}

	if o.IndustryClassification != "" {
		industryClassQuery := constructCreateIndustryClassificationQuery(o.UUID, o.IndustryClassification)
		queries = append(queries, industryClassQuery)
	}

	if o.ParentOrganisation != "" {
		parentQuery := constructCreateParentOrganisationQuery(o.UUID, o.ParentOrganisation)
		queries = append(queries, parentQuery)
	}

	// Temporary while trying to fix circle ci
	for _, queryPointer := range queries {
		logrus.Infof("QUERY: %v", *queryPointer)
	}

	return cd.cypherRunner.CypherBatch(queries)
}

func (cd service) constructMergingOldOrganisationNodesQueries(canonicalUUID string, possibleOldNodes []string) ([]*neoism.CypherQuery, error) {

	queries := []*neoism.CypherQuery{}

	for _, identifier := range possibleOldNodes {
		// only nodes with uppAuthority can be older organisation nodes
		if identifier != canonicalUUID {
			nodeExists, err := cd.checkNodeExistence(identifier)
			if err != nil {
				return nil, err
			}
			if nodeExists {
				deleteEntityRelationshipsForDeprecatedOrgNodeQuery := constructDeleteEntityRelationshipQuery(identifier)
				queries = append(queries, deleteEntityRelationshipsForDeprecatedOrgNodeQuery)

				// re-point the remaining relationships from previous node to the canonical/actual one
				transferQueries, err := CreateTransferRelationshipsQueries(cd.cypherRunner, canonicalUUID, identifier)
				if err != nil {
					return nil, err
				}
				if len(transferQueries) != 0 {
					queries = append(queries, transferQueries...)
				}

				// delete oldOrg
				deleteOldOrganisationQuery := constructDeleteEmptyNodeQuery(identifier)
				queries = append(queries, deleteOldOrganisationQuery)
			}
		}
	}

	return queries, nil
}

func (cd service) checkNodeExistence(uuid string) (bool, error) {
	type result []struct {
		Count int `json:"nr"`
	}
	res := result{}

	checkNodeExistenceQuery := &neoism.CypherQuery{
		Statement: `match (a:Thing{uuid:{uuid}})
			           return count(a) as nr`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
		Result: &res,
	}

	readQueries := []*neoism.CypherQuery{checkNodeExistenceQuery}
	err := cd.cypherRunner.CypherBatch(readQueries)

	if err != nil {
		return false, err
	}

	if len(res) != 1 {
		return false, fmt.Errorf("DB inconsistence: one count result should be returned for node with UUID %s", uuid)
	}

	if res[0].Count == 0 {
		return false, nil
	} else if res[0].Count == 1 {
		return true, nil
	} else {
		return false, fmt.Errorf("DB inconsistence: %d node (instead of max 1) exists with UUID %s", res[0].Count, uuid)
	}
}

//Read - Internal Read of an Organisation
func (cd service) Read(uuid string) (interface{}, bool, error) {

	results := []struct {
		UUID                   string                 `json:"uuid"`
		Type                   []string               `json:"type"`
		ProperName             string                 `json:"properName"`
		PrefLabel              string                 `json:"prefLabel"`
		LegalName              string                 `json:"legalName"`
		ShortName              string                 `json:"shortName"`
		HiddenLabel            string                 `json:"hiddenLabel"`
		AlternativeIdentifiers alternativeIdentifiers `json:"alternativeIdentifiers"`
		TradeNames             []string               `json:"tradeNames"`
		LocalNames             []string               `json:"localNames"`
		FormerNames            []string               `json:"formerNames"`
		Aliases                []string               `json:"aliases"`
		IndustryClassification string                 `json:"industryClassification"`
		ParentOrganisation     string                 `json:"parentOrganisation"`
	}{}

	readQuery := &neoism.CypherQuery{
		Statement: `MATCH (o:Organisation:Concept{uuid:{uuid}})
            			OPTIONAL MATCH (o)-[:SUB_ORGANISATION_OF]->(par:Thing)
            			OPTIONAL MATCH (o)-[:HAS_CLASSIFICATION]->(ind:Thing)
           			OPTIONAL MATCH (upp:UPPIdentifier)-[:IDENTIFIES]->(o)
	    			OPTIONAL MATCH (factset:FactsetIdentifier)-[:IDENTIFIES]->(o)
	   			OPTIONAL MATCH (tme:TMEIdentifier)-[:IDENTIFIES]->(o)
	    			OPTIONAL MATCH (lei:LegalEntityIdentifier)-[:IDENTIFIES]->(o)
            		 	RETURN o.uuid as uuid,
					o.properName as properName,
					labels(o) as Type,
					o.prefLabel as prefLabel,
					o.legalName as legalName,
					o.shortName as shortName,
					o.hiddenLabel as hiddenLabel,
					o.formerNames as formerNames,
					o.tradeNames as tradeNames,
					o.localNames as localNames,
					o.aliases as aliases,
					ind.uuid as industryClassification,
					par.uuid as parentOrganisation,
					{uuids:collect(distinct upp.value),
					 TME:collect(distinct tme.value),
					 factsetIdentifier:factset.value,
					 leiCode:lei.value} as alternativeIdentifiers`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
		Result: &results,
	}

	if err := cd.cypherRunner.CypherBatch([]*neoism.CypherQuery{readQuery}); err != nil || len(results) == 0 {
		return organisation{}, false, err
	}

	result := results[0]

	o := organisation{
		UUID:                   result.UUID,
		ProperName:             result.ProperName,
		PrefLabel:              result.PrefLabel,
		LegalName:              result.LegalName,
		ShortName:              result.ShortName,
		HiddenLabel:            result.HiddenLabel,
		TradeNames:             result.TradeNames,
		LocalNames:             result.LocalNames,
		FormerNames:            result.FormerNames,
		AlternativeIdentifiers: result.AlternativeIdentifiers,
		Aliases:                result.Aliases,
		ParentOrganisation:     result.ParentOrganisation,
		IndustryClassification: result.IndustryClassification,
	}

	addType(&o.Type, &result.Type)
	sort.Strings(o.AlternativeIdentifiers.TME)
	sort.Strings(o.AlternativeIdentifiers.UUIDS)

	return o, true, nil
}

func addType(orgType *OrgType, types *[]string) {
	i := len(*types)
	if i == 3 {
		*orgType = Organisation
	}
	if i == 4 {
		*orgType = Company
	}
	if i == 5 {
		*orgType = PublicCompany
	}
}

//Delete - Deletes an Organisation
func (cd service) Delete(uuid string) (bool, error) {
	clearNode := &neoism.CypherQuery{
		Statement: `
			MATCH (org:Thing {uuid: {uuid}})
			OPTIONAL MATCH (org)<-[iden:IDENTIFIES]-(i:Identifier)
			REMOVE org:Concept:Organisation:Company:PublicCompany
			DELETE iden, i
			SET org={uuid: {uuid}}
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
		IncludeStats: true,
	}
	qs := []*neoism.CypherQuery{
		clearNode,
		{
			Statement: `
		MATCH (org:Thing {uuid: {uuid}})
		OPTIONAL MATCH (org)-[a]-(x) WITH org, count(a) AS relCount WHERE relCount = 0
		DELETE org
		`,
			Parameters: map[string]interface{}{
				"uuid": uuid,
			},
		},
	}

	err := cd.cypherRunner.CypherBatch(qs)

	s1, err := clearNode.Stats()

	if err != nil {
		return false, err
	}

	if s1.ContainsUpdates && s1.LabelsRemoved > 0 {
		return true, err
	}

	return false, err
}

func (cd service) Check() error {
	return neoutils.Check(cd.cypherRunner)
}

type countResult []struct {
	Count int `json:"c"`
}

func (cd service) Count() (int, error) {

	results := countResult{}

	err := cd.cypherRunner.CypherBatch([]*neoism.CypherQuery{{
		Statement: `MATCH (n:Organisation) return count(n) as c`,
		Result:    &results,
	}})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

func (cd service) DecodeJSON(dec *json.Decoder) (interface{}, string, error) {
	org := organisation{}
	err := dec.Decode(&org)
	return org, org.UUID, err
}

type requestError struct {
	details string
}

func (re requestError) Error() string {
	return "Invalid Request"
}

func (re requestError) InvalidRequestDetails() string {
	return re.details
}
<|endoftext|>


<commit_before>package collector

import (
	"encoding/json"
	"fullerite/metric"

	"reflect"
	"testing"

	l "github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/stretchr/testify/assert"
)

func getSUT() *DockerStats {
	expected_chan := make(chan metric.Metric)
	var expected_logger = defaultLog.WithFields(l.Fields{"collector": "fullerite"})

	return NewDockerStats(expected_chan, 10, expected_logger)
}

func TestDockerStatsNewDockerStats(t *testing.T) {
	expected_chan := make(chan metric.Metric)
	var expected_logger = defaultLog.WithFields(l.Fields{"collector": "fullerite"})
	expected_type := make(map[string]*CPUValues)

	d := NewDockerStats(expected_chan, 10, expected_logger)

	assert.Equal(t, d.log, expected_logger)
	assert.Equal(t, d.channel, expected_chan)
	assert.Equal(t, d.interval, 10)
	assert.Equal(t, d.name, "DockerStats")
	assert.Equal(t, reflect.TypeOf(d.previousCPUValues), reflect.TypeOf(expected_type))
	assert.Equal(t, len(d.previousCPUValues), 0)
	assert.Equal(t, d.dockerClient.Endpoint(), endpoint)
}

func TestDockerStatsConfigureEmptyConfig(t *testing.T) {
	config := make(map[string]interface{})

	d := NewDockerStats(nil, 123, nil)
	d.Configure(config)

	assert.Equal(t, 123, d.Interval())
}

func TestDockerStatsConfigure(t *testing.T) {
	config := make(map[string]interface{})
	config["interval"] = 9999

	d := NewDockerStats(nil, 123, nil)
	d.Configure(config)

	assert.Equal(t, 9999, d.Interval())
}

func TestDockerStatsBuildMetrics(t *testing.T) {
	stats := new(docker.Stats)
	stats.Network.RxBytes = 10
	stats.Network.TxBytes = 20
	stats.MemoryStats.Usage = 50
	stats.MemoryStats.Limit = 70

	containerJson := []byte(`
	{
		"ID": "test-id",
		"Name": "test-container",
		"Config": {
			"Env": [
				"MESOS_TASK_ID=my--service.main.blablagit6bdsadnoise"
			]
		}
	}`)
	var container *docker.Container
	err := json.Unmarshal(containerJson, &container)
	assert.Equal(t, err, nil)

	expected_dims := map[string]string{
		"container_id":   "test-id",
		"container_name": "test-container",
		"service_name":   "my_service",
		"instance_name":  "main",
		"collector":      "DockerStats",
	}
	expected_metrics := []metric.Metric{
		metric.Metric{"DockerRxBytes", "gauge", 10, expected_dims},
		metric.Metric{"DockerTxBytes", "gauge", 20, expected_dims},
		metric.Metric{"DockerMemoryUsed", "gauge", 50, expected_dims},
		metric.Metric{"DockerMemoryLimit", "gauge", 70, expected_dims},
		metric.Metric{"DockerCpuPercentage", "gauge", 0.5, expected_dims},
	}

	d := getSUT()
	ret := d.buildMetrics(container, stats, 0.5)

	assert.Equal(t, ret, expected_metrics)
}

func TestDockerStatsCalculateCPUPercent(t *testing.T) {
	var previousTotalUsage = uint64(0)
	var previousSystem = uint64(0)

	stats := new(docker.Stats)
	stats.CPUStats.CPUUsage.PercpuUsage = make([]uint64, 24)
	stats.CPUStats.CPUUsage.TotalUsage = 1261158030354
	stats.CPUStats.SystemCPUUsage = 108086414700000000

	assert.Equal(t, 0.02800332753427522, calculateCPUPercent(previousTotalUsage, previousSystem, stats))

	previousTotalUsage = stats.CPUStats.CPUUsage.TotalUsage
	previousSystem = stats.CPUStats.SystemCPUUsage
	stats.CPUStats.CPUUsage.TotalUsage = 1261164064229
	stats.CPUStats.SystemCPUUsage = 108086652820000000

	assert.Equal(t, 0.060815135225936505, calculateCPUPercent(previousTotalUsage, previousSystem, stats))
}

func TestGetInfoFromMesosTaskID(t *testing.T) {
	mesosID := "my--service.main.blablagit6bdsadnoise"
	serviceName, instanceName := getInfoFromMesosTaskID(mesosID)

	assert.Equal(t, serviceName, "my_service")
	assert.Equal(t, instanceName, "main")
}
<commit_msg>Clean up lint warnings on docker_stats_test<commit_after>package collector

import (
	"encoding/json"
	"fullerite/metric"

	"reflect"
	"testing"

	l "github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/stretchr/testify/assert"
)

func getSUT() *DockerStats {
	expectedChan := make(chan metric.Metric)
	var expectedLogger = defaultLog.WithFields(l.Fields{"collector": "fullerite"})

	return NewDockerStats(expectedChan, 10, expectedLogger)
}

func TestDockerStatsNewDockerStats(t *testing.T) {
	expectedChan := make(chan metric.Metric)
	var expectedLogger = defaultLog.WithFields(l.Fields{"collector": "fullerite"})
	expectedType := make(map[string]*CPUValues)

	d := NewDockerStats(expectedChan, 10, expectedLogger)

	assert.Equal(t, d.log, expectedLogger)
	assert.Equal(t, d.channel, expectedChan)
	assert.Equal(t, d.interval, 10)
	assert.Equal(t, d.name, "DockerStats")
	assert.Equal(t, reflect.TypeOf(d.previousCPUValues), reflect.TypeOf(expectedType))
	assert.Equal(t, len(d.previousCPUValues), 0)
	assert.Equal(t, d.dockerClient.Endpoint(), endpoint)
}

func TestDockerStatsConfigureEmptyConfig(t *testing.T) {
	config := make(map[string]interface{})

	d := NewDockerStats(nil, 123, nil)
	d.Configure(config)

	assert.Equal(t, 123, d.Interval())
}

func TestDockerStatsConfigure(t *testing.T) {
	config := make(map[string]interface{})
	config["interval"] = 9999

	d := NewDockerStats(nil, 123, nil)
	d.Configure(config)

	assert.Equal(t, 9999, d.Interval())
}

func TestDockerStatsBuildMetrics(t *testing.T) {
	stats := new(docker.Stats)
	stats.Network.RxBytes = 10
	stats.Network.TxBytes = 20
	stats.MemoryStats.Usage = 50
	stats.MemoryStats.Limit = 70

	containerJSON := []byte(`
	{
		"ID": "test-id",
		"Name": "test-container",
		"Config": {
			"Env": [
				"MESOS_TASK_ID=my--service.main.blablagit6bdsadnoise"
			]
		}
	}`)
	var container *docker.Container
	err := json.Unmarshal(containerJSON, &container)
	assert.Equal(t, err, nil)

	expectedDims := map[string]string{
		"container_id":   "test-id",
		"container_name": "test-container",
		"service_name":   "my_service",
		"instance_name":  "main",
		"collector":      "DockerStats",
	}
	expectedMetrics := []metric.Metric{
		metric.Metric{"DockerRxBytes", "gauge", 10, expectedDims},
		metric.Metric{"DockerTxBytes", "gauge", 20, expectedDims},
		metric.Metric{"DockerMemoryUsed", "gauge", 50, expectedDims},
		metric.Metric{"DockerMemoryLimit", "gauge", 70, expectedDims},
		metric.Metric{"DockerCpuPercentage", "gauge", 0.5, expectedDims},
	}

	d := getSUT()
	ret := d.buildMetrics(container, stats, 0.5)

	assert.Equal(t, ret, expectedMetrics)
}

func TestDockerStatsCalculateCPUPercent(t *testing.T) {
	var previousTotalUsage = uint64(0)
	var previousSystem = uint64(0)

	stats := new(docker.Stats)
	stats.CPUStats.CPUUsage.PercpuUsage = make([]uint64, 24)
	stats.CPUStats.CPUUsage.TotalUsage = 1261158030354
	stats.CPUStats.SystemCPUUsage = 108086414700000000

	assert.Equal(t, 0.02800332753427522, calculateCPUPercent(previousTotalUsage, previousSystem, stats))

	previousTotalUsage = stats.CPUStats.CPUUsage.TotalUsage
	previousSystem = stats.CPUStats.SystemCPUUsage
	stats.CPUStats.CPUUsage.TotalUsage = 1261164064229
	stats.CPUStats.SystemCPUUsage = 108086652820000000

	assert.Equal(t, 0.060815135225936505, calculateCPUPercent(previousTotalUsage, previousSystem, stats))
}

func TestGetInfoFromMesosTaskID(t *testing.T) {
	mesosID := "my--service.main.blablagit6bdsadnoise"
	serviceName, instanceName := getInfoFromMesosTaskID(mesosID)

	assert.Equal(t, serviceName, "my_service")
	assert.Equal(t, instanceName, "main")
}
<|endoftext|>


<commit_before>package controller

import (
	"fmt"
	"github.com/MG-RAST/AWE/lib/core"
	e "github.com/MG-RAST/AWE/lib/errors"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/AWE/lib/logger/event"
	"github.com/MG-RAST/golib/goweb"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
)

type WorkController struct{}

// OPTIONS: /work
func (cr *WorkController) Options(cx *goweb.Context) {
	LogRequest(cx.Request)
	cx.RespondWithOK()
	return
}

// GET: /work/{id}
// get a workunit by id, read-only
func (cr *WorkController) Read(id string, cx *goweb.Context) {
	LogRequest(cx.Request)

	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	if query.Has("datatoken") && query.Has("client") { //a client is requesting data token for this job
		//***insert code to authenticate and check ACL***
		clientid := query.Value("client")
		token, err := core.QMgr.FetchDataToken(id, clientid)
		if err != nil {
			cx.RespondWithErrorMessage("error in getting token for job "+id, http.StatusBadRequest)
		}
		cx.RespondWithData(token)
		return
	}

	if query.Has("report") { //retrieve report: stdout or stderr
		reportmsg, err := core.QMgr.GetReportMsg(id, query.Value("report"))
		if err != nil {
			cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
			return
		}
		cx.RespondWithData(reportmsg)
		return
	}

	// Load workunit by id
	workunit, err := core.QMgr.GetWorkById(id)

	if err != nil {
		if err.Error() != e.QueueEmpty {
			logger.Error("Err@work_Read:core.QMgr.GetWorkById(): " + err.Error())
		}
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}
	// Base case respond with workunit in json
	cx.RespondWithData(workunit)
	return
}

// GET: /work
// checkout a workunit with earliest submission time
// to-do: to support more options for workunit checkout
func (cr *WorkController) ReadMany(cx *goweb.Context) {

	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	if !query.Has("client") { //view workunits
		var workunits []*core.Workunit
		if query.Has("state") {
			workunits = core.QMgr.ShowWorkunits(query.Value("state"))
		} else {
			workunits = core.QMgr.ShowWorkunits("")
		}
		cx.RespondWithData(workunits)
		return
	}

	if core.Service == "proxy" { //drive proxy workStealer to checkout work from server
		core.ProxyWorkChan <- true
	}

	//checkout a workunit in FCFS order
	clientid := query.Value("client")
	workunits, err := core.QMgr.CheckoutWorkunits("FCFS", clientid, 1)

	if err != nil {
		if err.Error() != e.QueueEmpty && err.Error() != e.NoEligibleWorkunitFound && err.Error() != e.ClientNotFound {
			logger.Error("Err@work_ReadMany:core.QMgr.GetWorkByFCFS(): " + err.Error() + ";client=" + clientid)
		}
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	//log access info only when the queue is not empty, save some log
	LogRequest(cx.Request)

	//log event about workunit checkout (WO)
	workids := []string{}
	for _, work := range workunits {
		workids = append(workids, work.Id)
	}

	logger.Event(event.WORK_CHECKOUT,
		"workids="+strings.Join(workids, ","),
		"clientid="+clientid)

	// Base case respond with node in json
	cx.RespondWithData(workunits[0])
	return
}

// PUT: /work/{id} -> status update
func (cr *WorkController) Update(id string, cx *goweb.Context) {
	// Log Request and check for Auth
	LogRequest(cx.Request)
	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	fmt.Printf("query=%#v\n", query)

	if query.Has("status") && query.Has("client") { //notify execution result: "done" or "fail"
		notice := core.Notice{WorkId: id, Status: query.Value("status"), ClientId: query.Value("client"), Notes: ""}
		if query.Has("computetime") {
			if comptime, err := strconv.Atoi(query.Value("computetime")); err == nil {
				notice.ComputeTime = comptime
			}
		}
		if query.Has("report") { // if "report" is specified in query, parse performance statistics or errlog
			if _, files, err := ParseMultipartForm(cx.Request); err == nil {
				if _, ok := files["perf"]; ok {
					core.QMgr.FinalizeWorkPerf(id, files["perf"].Path)
				}
				if _, ok := files["notes"]; ok {
					if notes, err := ioutil.ReadFile(files["notes"].Path); err == nil {
						notice.Notes = string(notes)
					}
				}
				if _, ok := files["stdout"]; ok {
					core.QMgr.SaveStdLog(id, "stdout", files["stdout"].Path)
				}
				if _, ok := files["stderr"]; ok {
					core.QMgr.SaveStdLog(id, "stderr", files["stderr"].Path)
				}
			}
		}
		core.QMgr.NotifyWorkStatus(notice)
	}
	cx.RespondWithData("ok")
	return
}
<commit_msg>remove debug msg<commit_after>package controller

import (
	"github.com/MG-RAST/AWE/lib/core"
	e "github.com/MG-RAST/AWE/lib/errors"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/AWE/lib/logger/event"
	"github.com/MG-RAST/golib/goweb"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
)

type WorkController struct{}

// OPTIONS: /work
func (cr *WorkController) Options(cx *goweb.Context) {
	LogRequest(cx.Request)
	cx.RespondWithOK()
	return
}

// GET: /work/{id}
// get a workunit by id, read-only
func (cr *WorkController) Read(id string, cx *goweb.Context) {
	LogRequest(cx.Request)

	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	if query.Has("datatoken") && query.Has("client") { //a client is requesting data token for this job
		//***insert code to authenticate and check ACL***
		clientid := query.Value("client")
		token, err := core.QMgr.FetchDataToken(id, clientid)
		if err != nil {
			cx.RespondWithErrorMessage("error in getting token for job "+id, http.StatusBadRequest)
		}
		cx.RespondWithData(token)
		return
	}

	if query.Has("report") { //retrieve report: stdout or stderr
		reportmsg, err := core.QMgr.GetReportMsg(id, query.Value("report"))
		if err != nil {
			cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
			return
		}
		cx.RespondWithData(reportmsg)
		return
	}

	// Load workunit by id
	workunit, err := core.QMgr.GetWorkById(id)

	if err != nil {
		if err.Error() != e.QueueEmpty {
			logger.Error("Err@work_Read:core.QMgr.GetWorkById(): " + err.Error())
		}
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}
	// Base case respond with workunit in json
	cx.RespondWithData(workunit)
	return
}

// GET: /work
// checkout a workunit with earliest submission time
// to-do: to support more options for workunit checkout
func (cr *WorkController) ReadMany(cx *goweb.Context) {

	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	if !query.Has("client") { //view workunits
		var workunits []*core.Workunit
		if query.Has("state") {
			workunits = core.QMgr.ShowWorkunits(query.Value("state"))
		} else {
			workunits = core.QMgr.ShowWorkunits("")
		}
		cx.RespondWithData(workunits)
		return
	}

	if core.Service == "proxy" { //drive proxy workStealer to checkout work from server
		core.ProxyWorkChan <- true
	}

	//checkout a workunit in FCFS order
	clientid := query.Value("client")
	workunits, err := core.QMgr.CheckoutWorkunits("FCFS", clientid, 1)

	if err != nil {
		if err.Error() != e.QueueEmpty && err.Error() != e.NoEligibleWorkunitFound && err.Error() != e.ClientNotFound {
			logger.Error("Err@work_ReadMany:core.QMgr.GetWorkByFCFS(): " + err.Error() + ";client=" + clientid)
		}
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	//log access info only when the queue is not empty, save some log
	LogRequest(cx.Request)

	//log event about workunit checkout (WO)
	workids := []string{}
	for _, work := range workunits {
		workids = append(workids, work.Id)
	}

	logger.Event(event.WORK_CHECKOUT,
		"workids="+strings.Join(workids, ","),
		"clientid="+clientid)

	// Base case respond with node in json
	cx.RespondWithData(workunits[0])
	return
}

// PUT: /work/{id} -> status update
func (cr *WorkController) Update(id string, cx *goweb.Context) {
	// Log Request and check for Auth
	LogRequest(cx.Request)
	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	if query.Has("status") && query.Has("client") { //notify execution result: "done" or "fail"
		notice := core.Notice{WorkId: id, Status: query.Value("status"), ClientId: query.Value("client"), Notes: ""}
		if query.Has("computetime") {
			if comptime, err := strconv.Atoi(query.Value("computetime")); err == nil {
				notice.ComputeTime = comptime
			}
		}
		if query.Has("report") { // if "report" is specified in query, parse performance statistics or errlog
			if _, files, err := ParseMultipartForm(cx.Request); err == nil {
				if _, ok := files["perf"]; ok {
					core.QMgr.FinalizeWorkPerf(id, files["perf"].Path)
				}
				if _, ok := files["notes"]; ok {
					if notes, err := ioutil.ReadFile(files["notes"].Path); err == nil {
						notice.Notes = string(notes)
					}
				}
				if _, ok := files["stdout"]; ok {
					core.QMgr.SaveStdLog(id, "stdout", files["stdout"].Path)
				}
				if _, ok := files["stderr"]; ok {
					core.QMgr.SaveStdLog(id, "stderr", files["stderr"].Path)
				}
			}
		}
		core.QMgr.NotifyWorkStatus(notice)
	}
	cx.RespondWithData("ok")
	return
}
<|endoftext|>


<commit_before>package mounter

import (
	"os"
	fp "path/filepath"
	"strings"

	"github.com/docker/docker/pkg/fileutils"
	"github.com/docker/docker/pkg/mount"
	"github.com/pkg/errors"

	"github.com/yuuki/droot/log"
	"github.com/yuuki/droot/osutil"
)

type Mounter struct {
	rootDir string
}

func NewMounter(rootDir string) *Mounter {
	return &Mounter{rootDir: rootDir}
}

func ResolveRootDir(dir string) (string, error) {
	var err error

	if !osutil.ExistsDir(dir) {
		return dir, errors.Errorf("No such directory %s:", dir)
	}

	dir, err = fp.Abs(dir)
	if err != nil {
		return dir, err
	}

	if osutil.IsSymlink(dir) {
		dir, err = os.Readlink(dir)
		if err != nil {
			return dir, err
		}
	}

	return fp.Clean(dir), nil
}

func (m *Mounter) MountSysProc() error {
	// mount -t proc proc {{rootDir}}/proc
	if err := osutil.MountIfNotMounted("/proc", fp.Join(m.rootDir, "/proc"), "proc", ""); err != nil {
		return errors.Errorf("Failed to mount /proc: %s", err)
	}
	// mount --rbind /sys {{rootDir}}/sys
	if err := osutil.MountIfNotMounted("/sys", fp.Join(m.rootDir, "/sys"), "none", "rbind"); err != nil {
		return errors.Errorf("Failed to mount /sys: %s", err)
	}
	// mount --make-rslave /sys {{rootDir}}/sys
	if err := osutil.ForceMount("", fp.Join(m.rootDir, "/sys"), "none", "rslave"); err != nil {
		return errors.Errorf("Failed to mount --make-rslave /sys: %s", err)
	}

	return nil
}

func (m *Mounter) BindMount(hostDir, containerDir string) error {
	containerDir = fp.Join(m.rootDir, containerDir)

	if ok := osutil.IsDirEmpty(hostDir); ok {
		if _, err := os.Create(fp.Join(hostDir, ".droot.keep")); err != nil {
			return err
		}
	}

	if err := fileutils.CreateIfNotExists(containerDir, true); err != nil { // mkdir -p
		return err
	}

	if err := osutil.MountIfNotMounted(hostDir, containerDir, "none", "bind,rw"); err != nil {
		return err
	}

	return nil
}

func (m *Mounter) RoBindMount(hostDir, containerDir string) error {
	if err := m.BindMount(hostDir, containerDir); err != nil {
		return err
	}

	containerDir = fp.Join(m.rootDir, containerDir)

	if err := osutil.ForceMount(hostDir, containerDir, "none", "remount,ro,bind"); err != nil {
		return err
	}

	return nil
}

func (m *Mounter) GetMountsRoot() ([]*mount.Info, error) {
	mounts, err := mount.GetMounts()
	if err != nil {
		return nil, err
	}

	targets := make([]*mount.Info, 0)
	for _, mo := range mounts {
		if strings.HasPrefix(mo.Mountpoint, m.rootDir) {
			targets = append(targets, mo)
		}
	}

	return targets, nil
}

func (m *Mounter) UmountRoot() error {
	mounts, err := m.GetMountsRoot()
	if err != nil {
		return err
	}

	for _, mo := range mounts {
		if err := mount.Unmount(mo.Mountpoint); err != nil {
			return err
		}
		log.Debug("umount:", mo.Mountpoint)
	}

	return nil
}
<commit_msg>Fix that it failed to mount /proc when droot run twice<commit_after>package mounter

import (
	"os"
	fp "path/filepath"
	"strings"

	"github.com/docker/docker/pkg/fileutils"
	"github.com/docker/docker/pkg/mount"
	"github.com/pkg/errors"

	"github.com/yuuki/droot/log"
	"github.com/yuuki/droot/osutil"
)

type Mounter struct {
	rootDir string
}

func NewMounter(rootDir string) *Mounter {
	return &Mounter{rootDir: rootDir}
}

func ResolveRootDir(dir string) (string, error) {
	var err error

	if !osutil.ExistsDir(dir) {
		return dir, errors.Errorf("No such directory %s:", dir)
	}

	dir, err = fp.Abs(dir)
	if err != nil {
		return dir, err
	}

	if osutil.IsSymlink(dir) {
		dir, err = os.Readlink(dir)
		if err != nil {
			return dir, err
		}
	}

	return fp.Clean(dir), nil
}

func (m *Mounter) MountSysProc() error {
	// mount -t proc proc {{rootDir}}/proc
	if err := osutil.MountIfNotMounted("/proc", fp.Join(m.rootDir, "/proc"), "proc", "remount"); err != nil {
		return errors.Errorf("Failed to mount /proc: %s", err)
	}
	// mount --rbind /sys {{rootDir}}/sys
	if err := osutil.MountIfNotMounted("/sys", fp.Join(m.rootDir, "/sys"), "none", "rbind"); err != nil {
		return errors.Errorf("Failed to mount /sys: %s", err)
	}
	// mount --make-rslave /sys {{rootDir}}/sys
	if err := osutil.ForceMount("", fp.Join(m.rootDir, "/sys"), "none", "rslave"); err != nil {
		return errors.Errorf("Failed to mount --make-rslave /sys: %s", err)
	}

	return nil
}

func (m *Mounter) BindMount(hostDir, containerDir string) error {
	containerDir = fp.Join(m.rootDir, containerDir)

	if ok := osutil.IsDirEmpty(hostDir); ok {
		if _, err := os.Create(fp.Join(hostDir, ".droot.keep")); err != nil {
			return err
		}
	}

	if err := fileutils.CreateIfNotExists(containerDir, true); err != nil { // mkdir -p
		return err
	}

	if err := osutil.MountIfNotMounted(hostDir, containerDir, "none", "bind,rw"); err != nil {
		return err
	}

	return nil
}

func (m *Mounter) RoBindMount(hostDir, containerDir string) error {
	if err := m.BindMount(hostDir, containerDir); err != nil {
		return err
	}

	containerDir = fp.Join(m.rootDir, containerDir)

	if err := osutil.ForceMount(hostDir, containerDir, "none", "remount,ro,bind"); err != nil {
		return err
	}

	return nil
}

func (m *Mounter) GetMountsRoot() ([]*mount.Info, error) {
	mounts, err := mount.GetMounts()
	if err != nil {
		return nil, err
	}

	targets := make([]*mount.Info, 0)
	for _, mo := range mounts {
		if strings.HasPrefix(mo.Mountpoint, m.rootDir) {
			targets = append(targets, mo)
		}
	}

	return targets, nil
}

func (m *Mounter) UmountRoot() error {
	mounts, err := m.GetMountsRoot()
	if err != nil {
		return err
	}

	for _, mo := range mounts {
		if err := mount.Unmount(mo.Mountpoint); err != nil {
			return err
		}
		log.Debug("umount:", mo.Mountpoint)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017-2021 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package proxy_test

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/client-go/kubernetes/fake"

	proxy "github.com/projectcalico/felix/bpf/proxy"
)

var _ = Describe("BPF Proxy healthCheckNodeport", func() {
	var p proxy.Proxy
	k8s := fake.NewSimpleClientset()

	testNodeName := "testnode"
	testNodeNameOther := "anothertestnode"

	BeforeEach(func() {
		By("creating proxy with fake client and mock syncer", func() {
			var err error

			p, err = proxy.New(k8s, &mockDummySyncer{},
				testNodeName, proxy.WithMinSyncPeriod(200*time.Millisecond))
			Expect(err).NotTo(HaveOccurred())
		})
	})

	AfterEach(func() {
		By("stopping the proxy", func() {
			p.Stop()
		})
	})

	It("should expose health check enpoint", func() {
		healthCheckNodePort := 1212

		By("adding a LoadBalancer", func() {
			err := k8s.Tracker().Add(&v1.Service{
				TypeMeta:   typeMetaV1("Service"),
				ObjectMeta: objectMeataV1("LB"),
				Spec: v1.ServiceSpec{
					ClusterIP: "10.1.0.1",
					Type:      v1.ServiceTypeLoadBalancer,
					Selector: map[string]string{
						"app": "test",
					},
					ExternalTrafficPolicy: "Local",
					HealthCheckNodePort:   int32(healthCheckNodePort),
					Ports: []v1.ServicePort{
						{
							Protocol:   v1.ProtocolTCP,
							Port:       4321,
							TargetPort: intstr.FromInt(32678),
						},
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())
		})

		By("adding its endpoints", func() {
			err := k8s.Tracker().Add(&v1.Endpoints{
				TypeMeta:   typeMetaV1("Endpoints"),
				ObjectMeta: objectMeataV1("LB"),
				Subsets: []v1.EndpointSubset{
					{
						Addresses: []v1.EndpointAddress{
							{
								IP: "10.1.2.1",
							},
						},
						Ports: []v1.EndpointPort{
							{
								Port: 1234,
							},
						},
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())
		})

		By("checking that the healthCheckNodePort is accessable", func() {
			Eventually(func() bool {
				result, err := http.Get(fmt.Sprintf("http://localhost:%d", healthCheckNodePort))
				if err != nil {
					return false
				}
				if result.StatusCode != 503 {
					return false
				}
				return true
			}).Should(BeTrue())
		})

		By("checking that there is no local endpoint", func() {
			result, err := http.Get(fmt.Sprintf("http://localhost:%d", healthCheckNodePort))
			Expect(err).NotTo(HaveOccurred())
			Expect(result.StatusCode).Should(Equal(503))

			var status map[string]interface{}

			decoder := json.NewDecoder(result.Body)
			err = decoder.Decode(&status)
			Expect(err).NotTo(HaveOccurred())

			Expect(status).To(HaveKey("localEndpoints"))
			Expect(int(status["localEndpoints"].(float64))).To(Equal(0))
		})

		By("adding a local and a non-local endpoint", func() {
			err := k8s.Tracker().Update(v1.SchemeGroupVersion.WithResource("endpoints"),
				&v1.Endpoints{
					TypeMeta:   typeMetaV1("Endpoints"),
					ObjectMeta: objectMeataV1("LB"),
					Subsets: []v1.EndpointSubset{
						{
							Addresses: []v1.EndpointAddress{
								{
									IP:       "10.1.2.1",
									NodeName: &testNodeName,
								},
								{
									IP:       "10.1.2.2",
									NodeName: &testNodeNameOther,
								},
							},
							Ports: []v1.EndpointPort{
								{
									Port: 1234,
								},
							},
						},
					},
				},
				"default")
			Expect(err).NotTo(HaveOccurred())
		})

		By("checking that there is a local endpoint", func() {
			Eventually(func() bool {
				result, err := http.Get(fmt.Sprintf("http://localhost:%d", healthCheckNodePort))
				if err != nil {
					return false
				}
				if result.StatusCode != 200 {
					return false
				}

				var status map[string]interface{}

				decoder := json.NewDecoder(result.Body)
				err = decoder.Decode(&status)
				if err != nil {
					return false
				}

				return int(status["localEndpoints"].(float64)) == 1
			}).Should(BeTrue())

			By("making non-local a local endpoint", func() {
				err := k8s.Tracker().Update(v1.SchemeGroupVersion.WithResource("endpoints"),
					&v1.Endpoints{
						TypeMeta:   typeMetaV1("Endpoints"),
						ObjectMeta: objectMeataV1("LB"),
						Subsets: []v1.EndpointSubset{
							{
								Addresses: []v1.EndpointAddress{
									{
										IP:       "10.1.2.1",
										NodeName: &testNodeName,
									},
									{
										IP:       "10.1.2.2",
										NodeName: &testNodeName,
									},
								},
								Ports: []v1.EndpointPort{
									{
										Port: 1234,
									},
								},
							},
						},
					},
					"default")
				Expect(err).NotTo(HaveOccurred())
			})

			By("checking that there is a local endpoint", func() {
				Eventually(func() bool {
					result, err := http.Get(fmt.Sprintf("http://localhost:%d", healthCheckNodePort))
					if err != nil {
						return false
					}
					if result.StatusCode != 200 {
						return false
					}

					var status map[string]interface{}

					decoder := json.NewDecoder(result.Body)
					err = decoder.Decode(&status)
					if err != nil {
						return false
					}

					return int(status["localEndpoints"].(float64)) == 2
				}).Should(BeTrue())
			})
		})
	})
})

type mockDummySyncer struct {
	syncerConntrackAPIDummy
}

func (s *mockDummySyncer) SetTriggerFn(f func()) {
}

func (*mockDummySyncer) Stop() {}

func (*mockDummySyncer) Apply(state proxy.DPSyncerState) error {
	log("state = %+v\n", state)
	return nil
}
<commit_msg>bpf/proxy: Increase timeouts and log the last error<commit_after>// Copyright (c) 2017-2021 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package proxy_test

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/client-go/kubernetes/fake"

	proxy "github.com/projectcalico/felix/bpf/proxy"
)

var _ = Describe("BPF Proxy healthCheckNodeport", func() {
	var p proxy.Proxy
	k8s := fake.NewSimpleClientset()

	testNodeName := "testnode"
	testNodeNameOther := "anothertestnode"

	BeforeEach(func() {
		By("creating proxy with fake client and mock syncer", func() {
			var err error

			p, err = proxy.New(k8s, &mockDummySyncer{},
				testNodeName, proxy.WithMinSyncPeriod(200*time.Millisecond))
			Expect(err).NotTo(HaveOccurred())
		})
	})

	AfterEach(func() {
		By("stopping the proxy", func() {
			p.Stop()
		})
	})

	It("should expose health check enpoint", func() {
		healthCheckNodePort := 1212

		By("adding a LoadBalancer", func() {
			err := k8s.Tracker().Add(&v1.Service{
				TypeMeta:   typeMetaV1("Service"),
				ObjectMeta: objectMeataV1("LB"),
				Spec: v1.ServiceSpec{
					ClusterIP: "10.1.0.1",
					Type:      v1.ServiceTypeLoadBalancer,
					Selector: map[string]string{
						"app": "test",
					},
					ExternalTrafficPolicy: "Local",
					HealthCheckNodePort:   int32(healthCheckNodePort),
					Ports: []v1.ServicePort{
						{
							Protocol:   v1.ProtocolTCP,
							Port:       4321,
							TargetPort: intstr.FromInt(32678),
						},
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())
		})

		By("adding its endpoints", func() {
			err := k8s.Tracker().Add(&v1.Endpoints{
				TypeMeta:   typeMetaV1("Endpoints"),
				ObjectMeta: objectMeataV1("LB"),
				Subsets: []v1.EndpointSubset{
					{
						Addresses: []v1.EndpointAddress{
							{
								IP: "10.1.2.1",
							},
						},
						Ports: []v1.EndpointPort{
							{
								Port: 1234,
							},
						},
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())
		})

		By("checking that the healthCheckNodePort is accessable", func() {
			Eventually(func() error {
				result, err := http.Get(fmt.Sprintf("http://localhost:%d", healthCheckNodePort))
				if err != nil {
					return err
				}
				if result.StatusCode != 503 {
					return fmt.Errorf("Unexpected status code %d; expected 503", result.StatusCode)
				}
				return nil
			}, "5s", "200ms").Should(Succeed())
		})

		By("checking that there is no local endpoint", func() {
			result, err := http.Get(fmt.Sprintf("http://localhost:%d", healthCheckNodePort))
			Expect(err).NotTo(HaveOccurred())
			Expect(result.StatusCode).Should(Equal(503))

			var status map[string]interface{}

			decoder := json.NewDecoder(result.Body)
			err = decoder.Decode(&status)
			Expect(err).NotTo(HaveOccurred())

			Expect(status).To(HaveKey("localEndpoints"))
			Expect(int(status["localEndpoints"].(float64))).To(Equal(0))
		})

		By("adding a local and a non-local endpoint", func() {
			err := k8s.Tracker().Update(v1.SchemeGroupVersion.WithResource("endpoints"),
				&v1.Endpoints{
					TypeMeta:   typeMetaV1("Endpoints"),
					ObjectMeta: objectMeataV1("LB"),
					Subsets: []v1.EndpointSubset{
						{
							Addresses: []v1.EndpointAddress{
								{
									IP:       "10.1.2.1",
									NodeName: &testNodeName,
								},
								{
									IP:       "10.1.2.2",
									NodeName: &testNodeNameOther,
								},
							},
							Ports: []v1.EndpointPort{
								{
									Port: 1234,
								},
							},
						},
					},
				},
				"default")
			Expect(err).NotTo(HaveOccurred())
		})

		By("checking that there is a local endpoint", func() {
			Eventually(func() error {
				result, err := http.Get(fmt.Sprintf("http://localhost:%d", healthCheckNodePort))
				if err != nil {
					return err
				}
				if result.StatusCode != 200 {
					return fmt.Errorf("Unexpected status code %d; expected 200", result.StatusCode)
				}

				var status map[string]interface{}

				decoder := json.NewDecoder(result.Body)
				err = decoder.Decode(&status)
				if err != nil {
					return err
				}

				if int(status["localEndpoints"].(float64)) != 1 {
					return fmt.Errorf("Expected 1 endpoint no %d", int(status["localEndpoints"].(float64)))
				}

				return nil
			}, "5s", "200ms").Should(Succeed())

			By("making non-local a local endpoint", func() {
				err := k8s.Tracker().Update(v1.SchemeGroupVersion.WithResource("endpoints"),
					&v1.Endpoints{
						TypeMeta:   typeMetaV1("Endpoints"),
						ObjectMeta: objectMeataV1("LB"),
						Subsets: []v1.EndpointSubset{
							{
								Addresses: []v1.EndpointAddress{
									{
										IP:       "10.1.2.1",
										NodeName: &testNodeName,
									},
									{
										IP:       "10.1.2.2",
										NodeName: &testNodeName,
									},
								},
								Ports: []v1.EndpointPort{
									{
										Port: 1234,
									},
								},
							},
						},
					},
					"default")
				Expect(err).NotTo(HaveOccurred())
			})

			By("checking that there is a local endpoint", func() {
				Eventually(func() error {
					result, err := http.Get(fmt.Sprintf("http://localhost:%d", healthCheckNodePort))
					if err != nil {
						return err
					}
					if result.StatusCode != 200 {
						return fmt.Errorf("Unexpected status code %d; expected 200", result.StatusCode)
					}

					var status map[string]interface{}

					decoder := json.NewDecoder(result.Body)
					err = decoder.Decode(&status)
					if err != nil {
						return err
					}

					if int(status["localEndpoints"].(float64)) != 2 {
						return fmt.Errorf("Expected 1 endpoint no %d", int(status["localEndpoints"].(float64)))
					}

					return nil
				}, "5s", "200ms").Should(Succeed())
			})
		})
	})
})

type mockDummySyncer struct {
	syncerConntrackAPIDummy
}

func (s *mockDummySyncer) SetTriggerFn(f func()) {
}

func (*mockDummySyncer) Stop() {}

func (*mockDummySyncer) Apply(state proxy.DPSyncerState) error {
	log("state = %+v\n", state)
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package examples_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/golang/glog"
)

func validateObject(obj runtime.Object) (errors []error) {
	ctx := api.NewDefaultContext()
	switch t := obj.(type) {
	case *api.ReplicationController:
		errors = validation.ValidateManifest(&t.DesiredState.PodTemplate.DesiredState.Manifest)
	case *api.ReplicationControllerList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Service:
		api.ValidNamespace(ctx, &t.TypeMeta)
		errors = validation.ValidateService(t)
	case *api.ServiceList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Pod:
		api.ValidNamespace(ctx, &t.TypeMeta)
		errors = validation.ValidateManifest(&t.DesiredState.Manifest)
	case *api.PodList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	default:
		return []error{fmt.Errorf("no validation defined for %#v", obj)}
	}
	return errors
}

func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error {
	err := filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() && path != inDir {
			return filepath.SkipDir
		}
		name := filepath.Base(path)
		ext := filepath.Ext(name)
		if ext != "" {
			name = name[:len(name)-len(ext)]
		}
		if !(ext == ".json" || ext == ".yaml") {
			return nil
		}
		glog.Infof("Testing %s", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		fn(name, path, data)
		return nil
	})
	return err
}

func TestExampleObjectSchemas(t *testing.T) {
	cases := map[string]map[string]runtime.Object{
		"../api/examples": {
			"controller":       &api.ReplicationController{},
			"controller-list":  &api.ReplicationControllerList{},
			"pod":              &api.Pod{},
			"pod-list":         &api.PodList{},
			"service":          &api.Service{},
			"external-service": &api.Service{},
			"service-list":     &api.ServiceList{},
		},
		"../examples/guestbook": {
			"frontend-controller":    &api.ReplicationController{},
			"redis-slave-controller": &api.ReplicationController{},
			"redis-master":           &api.Pod{},
			"frontend-service":       &api.Service{},
			"redis-master-service":   &api.Service{},
			"redis-slave-service":    &api.Service{},
		},
		"../examples/walkthrough": {
			"pod1": &api.Pod{},
			"pod2": &api.Pod{},
			"pod-with-http-healthcheck": &api.Pod{},
			"service":                   &api.Service{},
			"replication-controller":    &api.ReplicationController{},
		},
	}

	for path, expected := range cases {
		tested := 0
		err := walkJSONFiles(path, func(name, path string, data []byte) {
			expectedType, found := expected[name]
			if !found {
				t.Errorf("%s does not have a test case defined", path)
				return
			}
			tested += 1
			if err := latest.Codec.DecodeInto(data, expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
				return
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
		})
		if err != nil {
			t.Errorf("Expected no error, Got %v", err)
		}
		if tested != len(expected) {
			t.Errorf("Expected %d examples, Got %d", len(expected), tested)
		}
	}
}

var sampleRegexp = regexp.MustCompile("(?ms)^```(?:(?P<type>yaml)\\w*\\n(?P<content>.+?)|\\w*\\n(?P<content>\\{.+?\\}))\\w*\\n^```")
var subsetRegexp = regexp.MustCompile("(?ms)\\.{3}")

func TestReadme(t *testing.T) {
	paths := []string{
		"../README.md",
		"../examples/walkthrough/README.md",
	}

	for _, path := range paths {
		data, err := ioutil.ReadFile(path)
		if err != nil {
			t.Errorf("Unable to read file %s: %v", path, err)
			continue
		}

		matches := sampleRegexp.FindAllStringSubmatch(string(data), -1)
		if matches == nil {
			continue
		}
		for _, match := range matches {
			var content, subtype string
			for i, name := range sampleRegexp.SubexpNames() {
				if name == "type" {
					subtype = match[i]
				}
				if name == "content" && match[i] != "" {
					content = match[i]
				}
			}
			if subtype == "yaml" && subsetRegexp.FindString(content) != "" {
				t.Logf("skipping (%s): \n%s", subtype, content)
				continue
			}

			//t.Logf("testing (%s): \n%s", subtype, content)
			expectedType := &api.Pod{}
			if err := latest.Codec.DecodeInto([]byte(content), expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(content))
				continue
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
			_, err := latest.Codec.Encode(expectedType)
			if err != nil {
				t.Errorf("Could not encode object: %v", err)
				continue
			}
		}
	}
}
<commit_msg>Refactor tests to split ObjectMeta from TypeMeta<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package examples_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/golang/glog"
)

func validateObject(obj runtime.Object) (errors []error) {
	ctx := api.NewDefaultContext()
	switch t := obj.(type) {
	case *api.ReplicationController:
		errors = validation.ValidateManifest(&t.DesiredState.PodTemplate.DesiredState.Manifest)
	case *api.ReplicationControllerList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Service:
		api.ValidNamespace(ctx, &t.ObjectMeta)
		errors = validation.ValidateService(t)
	case *api.ServiceList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Pod:
		api.ValidNamespace(ctx, &t.ObjectMeta)
		errors = validation.ValidateManifest(&t.DesiredState.Manifest)
	case *api.PodList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	default:
		return []error{fmt.Errorf("no validation defined for %#v", obj)}
	}
	return errors
}

func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error {
	err := filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() && path != inDir {
			return filepath.SkipDir
		}
		name := filepath.Base(path)
		ext := filepath.Ext(name)
		if ext != "" {
			name = name[:len(name)-len(ext)]
		}
		if !(ext == ".json" || ext == ".yaml") {
			return nil
		}
		glog.Infof("Testing %s", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		fn(name, path, data)
		return nil
	})
	return err
}

func TestExampleObjectSchemas(t *testing.T) {
	cases := map[string]map[string]runtime.Object{
		"../api/examples": {
			"controller":       &api.ReplicationController{},
			"controller-list":  &api.ReplicationControllerList{},
			"pod":              &api.Pod{},
			"pod-list":         &api.PodList{},
			"service":          &api.Service{},
			"external-service": &api.Service{},
			"service-list":     &api.ServiceList{},
		},
		"../examples/guestbook": {
			"frontend-controller":    &api.ReplicationController{},
			"redis-slave-controller": &api.ReplicationController{},
			"redis-master":           &api.Pod{},
			"frontend-service":       &api.Service{},
			"redis-master-service":   &api.Service{},
			"redis-slave-service":    &api.Service{},
		},
		"../examples/walkthrough": {
			"pod1": &api.Pod{},
			"pod2": &api.Pod{},
			"pod-with-http-healthcheck": &api.Pod{},
			"service":                   &api.Service{},
			"replication-controller":    &api.ReplicationController{},
		},
	}

	for path, expected := range cases {
		tested := 0
		err := walkJSONFiles(path, func(name, path string, data []byte) {
			expectedType, found := expected[name]
			if !found {
				t.Errorf("%s does not have a test case defined", path)
				return
			}
			tested += 1
			if err := latest.Codec.DecodeInto(data, expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
				return
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
		})
		if err != nil {
			t.Errorf("Expected no error, Got %v", err)
		}
		if tested != len(expected) {
			t.Errorf("Expected %d examples, Got %d", len(expected), tested)
		}
	}
}

var sampleRegexp = regexp.MustCompile("(?ms)^```(?:(?P<type>yaml)\\w*\\n(?P<content>.+?)|\\w*\\n(?P<content>\\{.+?\\}))\\w*\\n^```")
var subsetRegexp = regexp.MustCompile("(?ms)\\.{3}")

func TestReadme(t *testing.T) {
	paths := []string{
		"../README.md",
		"../examples/walkthrough/README.md",
	}

	for _, path := range paths {
		data, err := ioutil.ReadFile(path)
		if err != nil {
			t.Errorf("Unable to read file %s: %v", path, err)
			continue
		}

		matches := sampleRegexp.FindAllStringSubmatch(string(data), -1)
		if matches == nil {
			continue
		}
		for _, match := range matches {
			var content, subtype string
			for i, name := range sampleRegexp.SubexpNames() {
				if name == "type" {
					subtype = match[i]
				}
				if name == "content" && match[i] != "" {
					content = match[i]
				}
			}
			if subtype == "yaml" && subsetRegexp.FindString(content) != "" {
				t.Logf("skipping (%s): \n%s", subtype, content)
				continue
			}

			//t.Logf("testing (%s): \n%s", subtype, content)
			expectedType := &api.Pod{}
			if err := latest.Codec.DecodeInto([]byte(content), expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(content))
				continue
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
			_, err := latest.Codec.Encode(expectedType)
			if err != nil {
				t.Errorf("Could not encode object: %v", err)
				continue
			}
		}
	}
}
<|endoftext|>