<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strconv"

	"github.com/codegangsta/cli"
	"github.com/drone/drone-go/drone"
)

var MachineCmd = cli.Command{
	Name:  "node",
	Usage: "manage build nodes",
	Subcommands: []cli.Command{
		// Node List
		{
			Name:  "ls",
			Usage: "list all nodes",
			Action: func(c *cli.Context) {
				handle(c, NodeListCmd)
			},
		},
		// Node Info
		{
			Name:  "info",
			Usage: "show node details",
			Action: func(c *cli.Context) {
				handle(c, NodeInfoCmd)
			},
		},
		// Node Add
		{
			Name:  "create",
			Usage: "creates a node",
			Action: func(c *cli.Context) {
				handle(c, NodeCreateCmd)
			},
			Flags: []cli.Flag{
				cli.StringFlag{
					EnvVar: "DOCKER_HOST",
					Name:   "docker-host",
					Usage:  "docker deamon address",
					Value:  "",
				},
				cli.BoolFlag{
					EnvVar: "DOCKER_TLS_VERIFY",
					Name:   "docker-tls-verify",
					Usage:  "docker daemon supports tlsverify",
				},
				cli.StringFlag{
					EnvVar: "DOCKER_CERT_PATH",
					Name:   "docker-cert-path",
					Usage:  "docker certificate directory",
					Value:  "",
				},
			},
		},
		// Node Delete
		{
			Name:  "rm",
			Usage: "remove a node",
			Action: func(c *cli.Context) {
				handle(c, NodeDelCmd)
			},
		},
	},
}

func NodeInfoCmd(c *cli.Context, client drone.Client) error {
	id, err := strconv.ParseInt(c.Args().Get(0), 0, 64)
	if err != nil {
		return fmt.Errorf("Invalid or missing node id. Must be an integer")
	}

	node, err := client.Node(id)
	if err != nil {
		return fmt.Errorf("Endpoint is not yet supported")
	}
	fmt.Println(node.Addr)
	return nil
}

func NodeListCmd(c *cli.Context, client drone.Client) error {
	nodes, err := client.NodeList()
	if err != nil {
		return err
	}

	for _, node := range nodes {
		fmt.Println(node.ID, node.Addr)
	}

	return nil
}

func NodeDelCmd(c *cli.Context, client drone.Client) error {
	id, err := strconv.ParseInt(c.Args().Get(0), 0, 64)
	if err != nil {
		return fmt.Errorf("Invalid or missing node id. Must be an integer")
	}

	err = client.NodeDel(id)
	if err != nil {
		return err
	}

	fmt.Printf("Successfully removed node %d\n", id)
	return nil
}

func NodeCreateCmd(c *cli.Context, client drone.Client) error {
	node := drone.Node{
		Addr: c.String("docker-host"),
		Arch: "linux_amd64",
	}

	cert, _ := ioutil.ReadFile(filepath.Join(
		c.String("docker-cert-path"),
		"cert.pem",
	))

	key, _ := ioutil.ReadFile(filepath.Join(
		c.String("docker-cert-path"),
		"key.pem",
	))

	ca, _ := ioutil.ReadFile(filepath.Join(
		c.String("docker-cert-path"),
		"ca.pem",
	))

	if len(cert) == 0 || len(key) == 0 {
		return fmt.Errorf("Error reading cert.pem or key.pem from %s",
			c.String("docker-cert-path"))
	}

	node.Cert = string(cert)
	node.Key = string(key)

	// only use the certificate authority if tls verify
	// is enabled for this docker host.
	if c.Bool("docker-tls-verify") {
		node.CA = string(ca)
	}

	_, err := client.NodePost(&node)
	if err != nil {
		return err
	}

	fmt.Printf("Successfully added %s\n", node.Addr)
	return nil
}
<commit_msg>Quick spelling fix so I don't get all OCD<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strconv"

	"github.com/codegangsta/cli"
	"github.com/drone/drone-go/drone"
)

var MachineCmd = cli.Command{
	Name:  "node",
	Usage: "manage build nodes",
	Subcommands: []cli.Command{
		// Node List
		{
			Name:  "ls",
			Usage: "list all nodes",
			Action: func(c *cli.Context) {
				handle(c, NodeListCmd)
			},
		},
		// Node Info
		{
			Name:  "info",
			Usage: "show node details",
			Action: func(c *cli.Context) {
				handle(c, NodeInfoCmd)
			},
		},
		// Node Add
		{
			Name:  "create",
			Usage: "creates a node",
			Action: func(c *cli.Context) {
				handle(c, NodeCreateCmd)
			},
			Flags: []cli.Flag{
				cli.StringFlag{
					EnvVar: "DOCKER_HOST",
					Name:   "docker-host",
					Usage:  "docker daemon address",
					Value:  "",
				},
				cli.BoolFlag{
					EnvVar: "DOCKER_TLS_VERIFY",
					Name:   "docker-tls-verify",
					Usage:  "docker daemon supports tlsverify",
				},
				cli.StringFlag{
					EnvVar: "DOCKER_CERT_PATH",
					Name:   "docker-cert-path",
					Usage:  "docker certificate directory",
					Value:  "",
				},
			},
		},
		// Node Delete
		{
			Name:  "rm",
			Usage: "remove a node",
			Action: func(c *cli.Context) {
				handle(c, NodeDelCmd)
			},
		},
	},
}

func NodeInfoCmd(c *cli.Context, client drone.Client) error {
	id, err := strconv.ParseInt(c.Args().Get(0), 0, 64)
	if err != nil {
		return fmt.Errorf("Invalid or missing node id. Must be an integer")
	}

	node, err := client.Node(id)
	if err != nil {
		return fmt.Errorf("Endpoint is not yet supported")
	}
	fmt.Println(node.Addr)
	return nil
}

func NodeListCmd(c *cli.Context, client drone.Client) error {
	nodes, err := client.NodeList()
	if err != nil {
		return err
	}

	for _, node := range nodes {
		fmt.Println(node.ID, node.Addr)
	}

	return nil
}

func NodeDelCmd(c *cli.Context, client drone.Client) error {
	id, err := strconv.ParseInt(c.Args().Get(0), 0, 64)
	if err != nil {
		return fmt.Errorf("Invalid or missing node id. Must be an integer")
	}

	err = client.NodeDel(id)
	if err != nil {
		return err
	}

	fmt.Printf("Successfully removed node %d\n", id)
	return nil
}

func NodeCreateCmd(c *cli.Context, client drone.Client) error {
	node := drone.Node{
		Addr: c.String("docker-host"),
		Arch: "linux_amd64",
	}

	cert, _ := ioutil.ReadFile(filepath.Join(
		c.String("docker-cert-path"),
		"cert.pem",
	))

	key, _ := ioutil.ReadFile(filepath.Join(
		c.String("docker-cert-path"),
		"key.pem",
	))

	ca, _ := ioutil.ReadFile(filepath.Join(
		c.String("docker-cert-path"),
		"ca.pem",
	))

	if len(cert) == 0 || len(key) == 0 {
		return fmt.Errorf("Error reading cert.pem or key.pem from %s",
			c.String("docker-cert-path"))
	}

	node.Cert = string(cert)
	node.Key = string(key)

	// only use the certificate authority if tls verify
	// is enabled for this docker host.
	if c.Bool("docker-tls-verify") {
		node.CA = string(ca)
	}

	_, err := client.NodePost(&node)
	if err != nil {
		return err
	}

	fmt.Printf("Successfully added %s\n", node.Addr)
	return nil
}
<|endoftext|>


<commit_before>package dsalg

type direction *bool

var (
	left  direction = func() direction { b := false; return &b }()
	right direction = func() direction { b := true; return &b }()
)
<commit_msg>Minor amendments<commit_after>package dsalg

type direction *bool

var (
	left  = func() direction { b := false; return &b }()
	right = func() direction { b := true; return &b }()
)
<|endoftext|>


<commit_before>package interceptor

import (
	"time"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	"github.com/namely/mjolnir/logger"
	uuid "github.com/satori/go.uuid"
	"google.golang.org/grpc"
)

// Logger returns an interceptor that will set on the context a *logrus.Entry
// that will automatically be tagged with the request_id UUIDv4.
// It will log the start and end of the request including the duration of
// the call as well.
func Logger(l *logrus.Logger) grpc.UnaryServerInterceptor {
	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
		ctx = addLoggerToContext(l, ctx)
		entry := logger.FromContext(ctx)
		name := info.FullMethod

		entry.WithField("endpoint", name).Info("processing rpc")

		start := time.Now()
		out, err := handler(ctx, req)
		if err != nil {
			entry.WithError(err).WithField(
				"duration", time.Since(start).String()
			).Error("rpc endpoint failed")
			return nil, err
		}

		entry.WithFields(logrus.Fields{
			"endpoint": name,
			"duration": time.Since(start).String(),
		}).Info("finished rpc")

		return out, err
	}
}

func addLoggerToContext(l *logrus.Logger, ctx context.Context) context.Context {
	entry := l.WithField("request_id", uuid.NewV4().String())
	return logger.SetEntry(ctx, entry)
}
<commit_msg>Add comma after newline (#6)<commit_after>package interceptor

import (
	"time"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	"github.com/namely/mjolnir/logger"
	uuid "github.com/satori/go.uuid"
	"google.golang.org/grpc"
)

// Logger returns an interceptor that will set on the context a *logrus.Entry
// that will automatically be tagged with the request_id UUIDv4.
// It will log the start and end of the request including the duration of
// the call as well.
func Logger(l *logrus.Logger) grpc.UnaryServerInterceptor {
	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
		ctx = addLoggerToContext(l, ctx)
		entry := logger.FromContext(ctx)
		name := info.FullMethod

		entry.WithField("endpoint", name).Info("processing rpc")

		start := time.Now()
		out, err := handler(ctx, req)
		if err != nil {
			entry.WithError(err).WithField(
				"duration", time.Since(start).String(),
			).Error("rpc endpoint failed")
			return nil, err
		}

		entry.WithFields(logrus.Fields{
			"endpoint": name,
			"duration": time.Since(start).String(),
		}).Info("finished rpc")

		return out, err
	}
}

func addLoggerToContext(l *logrus.Logger, ctx context.Context) context.Context {
	entry := l.WithField("request_id", uuid.NewV4().String())
	return logger.SetEntry(ctx, entry)
}
<|endoftext|>


<commit_before>package v7pushaction_test

import (
	"errors"
	"time"

	"code.cloudfoundry.org/cli/actor/actionerror"
	"code.cloudfoundry.org/cli/actor/v7action"

	. "code.cloudfoundry.org/cli/actor/v7pushaction"
	"code.cloudfoundry.org/cli/actor/v7pushaction/v7pushactionfakes"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	log "github.com/sirupsen/logrus"
)

func PrepareSpaceStreamsDrainedAndClosed(
	pushPlansStream <-chan []PushPlan,
	eventStream <-chan Event,
	warningsStream <-chan Warnings,
	errorStream <-chan error,
) bool {
	var configStreamClosed, eventStreamClosed, warningsStreamClosed, errorStreamClosed bool
	for {
		select {
		case _, ok := <-pushPlansStream:
			if !ok {
				configStreamClosed = true
			}
		case _, ok := <-eventStream:
			if !ok {
				eventStreamClosed = true
			}
		case _, ok := <-warningsStream:
			if !ok {
				warningsStreamClosed = true
			}
		case _, ok := <-errorStream:
			if !ok {
				errorStreamClosed = true
			}
		}
		if configStreamClosed && eventStreamClosed && warningsStreamClosed && errorStreamClosed {
			break
		}
	}
	return true
}

func getPrepareNextEvent(c <-chan []PushPlan, e <-chan Event, w <-chan Warnings) func() Event {
	timeOut := time.Tick(500 * time.Millisecond)

	return func() Event {
		for {
			select {
			case <-c:
			case event, ok := <-e:
				if ok {
					log.WithField("event", event).Debug("getNextEvent")
					return event
				}
				return ""
			case <-w:
			case <-timeOut:
				return ""
			}
		}
	}
}

var _ = Describe("PrepareSpace", func() {
	var (
		actor       *Actor
		fakeV7Actor *v7pushactionfakes.FakeV7Actor

		pushPlans          []PushPlan
		fakeManifestParser *v7pushactionfakes.FakeManifestParser

		spaceGUID string

		pushPlansStream <-chan []PushPlan
		eventStream     <-chan Event
		warningsStream  <-chan Warnings
		errorStream     <-chan error
	)

	BeforeEach(func() {
		fakeV7Actor = new(v7pushactionfakes.FakeV7Actor) // TODO why do we new this up?
		actor, _, fakeV7Actor, _ = getTestPushActor()

		spaceGUID = "space"

		fakeManifestParser = new(v7pushactionfakes.FakeManifestParser)
	})

	AfterEach(func() {
		Eventually(PrepareSpaceStreamsDrainedAndClosed(pushPlansStream, eventStream, warningsStream, errorStream)).Should(BeTrue())
	})

	JustBeforeEach(func() {
		pushPlansStream, eventStream, warningsStream, errorStream = actor.PrepareSpace(pushPlans, fakeManifestParser)
	})

	When("there is a single push state and no manifest", func() {
		var appName = "app-name"
		BeforeEach(func() {
			fakeManifestParser.FullRawManifestReturns(nil)
		})
		When("Creating the app succeeds", func() {
			BeforeEach(func() {
				pushPlans = []PushPlan{{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName}}}
				fakeV7Actor.CreateApplicationInSpaceReturns(
					v7action.Application{Name: appName},
					v7action.Warnings{"create-app-warning"},
					nil,
				)
			})
			It("creates the app using the API", func() {
				Consistently(fakeV7Actor.SetSpaceManifestCallCount).Should(Equal(0))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(CreatingApplication))
				Eventually(fakeV7Actor.CreateApplicationInSpaceCallCount).Should(Equal(1))
				actualApp, actualSpaceGUID := fakeV7Actor.CreateApplicationInSpaceArgsForCall(0)
				Expect(actualApp).To(Equal(v7action.Application{Name: appName}))
				Expect(actualSpaceGUID).To(Equal(spaceGUID))
				Eventually(warningsStream).Should(Receive(Equal(Warnings{"create-app-warning"})))
				Eventually(errorStream).Should(Receive(Succeed()))
				Eventually(pushPlansStream).Should(Receive(ConsistOf(PushPlan{
					SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName},
				})))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(CreatedApplication))
			})
		})

		When("the app already exists", func() {
			BeforeEach(func() {
				pushPlans = []PushPlan{{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName}}}
				fakeV7Actor.CreateApplicationInSpaceReturns(
					v7action.Application{},
					v7action.Warnings{"create-app-warning"},
					actionerror.ApplicationAlreadyExistsError{},
				)
			})
			It("Sends already exists events", func() {
				Consistently(fakeV7Actor.SetSpaceManifestCallCount).Should(Equal(0))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(SkippingApplicationCreation))
				Eventually(fakeV7Actor.CreateApplicationInSpaceCallCount).Should(Equal(1))
				actualApp, actualSpaceGUID := fakeV7Actor.CreateApplicationInSpaceArgsForCall(0)
				Expect(actualApp).To(Equal(v7action.Application{Name: appName}))
				Expect(actualSpaceGUID).To(Equal(spaceGUID))
				Eventually(warningsStream).Should(Receive(Equal(Warnings{"create-app-warning"})))
				Eventually(errorStream).Should(Receive(Succeed()))
				Eventually(pushPlansStream).Should(Receive(ConsistOf(PushPlan{
					SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName},
				})))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(ApplicationAlreadyExists))
			})
		})

		When("creating the app fails", func() {
			BeforeEach(func() {
				pushPlans = []PushPlan{{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName}}}
				fakeV7Actor.CreateApplicationInSpaceReturns(
					v7action.Application{},
					v7action.Warnings{"create-app-warning"},
					errors.New("some-create-error"),
				)
			})

			It("Returns the error", func() {
				Consistently(fakeV7Actor.SetSpaceManifestCallCount).Should(Equal(0))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(CreatingApplication))
				Eventually(fakeV7Actor.CreateApplicationInSpaceCallCount).Should(Equal(1))
				actualApp, actualSpaceGuid := fakeV7Actor.CreateApplicationInSpaceArgsForCall(0)
				Expect(actualApp.Name).To(Equal(appName))
				Expect(actualSpaceGuid).To(Equal(spaceGUID))
				Eventually(warningsStream).Should(Receive(Equal(Warnings{"create-app-warning"})))
				Eventually(errorStream).Should(Receive(Equal(errors.New("some-create-error"))))
				Consistently(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).ShouldNot(Equal(ApplicationAlreadyExists))
			})
		})
	})

	When("There is a a manifest", func() {
		var (
			manifest = []byte("app manifest")
			appName1 = "app-name1"
			appName2 = "app-name2"
		)
		When("applying the manifest fails", func() {
			BeforeEach(func() {
				pushPlans = []PushPlan{{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName1}}}
				fakeManifestParser.FullRawManifestReturns(manifest)
				fakeManifestParser.RawAppManifestReturns(manifest, nil)
				fakeV7Actor.SetSpaceManifestReturns(v7action.Warnings{"apply-manifest-warnings"}, errors.New("some-error"))
			})

			It("returns the error and exits", func() {
				Consistently(fakeV7Actor.CreateApplicationInSpaceCallCount).Should(Equal(0))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(ApplyManifest))
				Eventually(fakeV7Actor.SetSpaceManifestCallCount).Should(Equal(1))
				actualSpaceGuid, actualManifest := fakeV7Actor.SetSpaceManifestArgsForCall(0)
				Expect(actualSpaceGuid).To(Equal(spaceGUID))
				Expect(actualManifest).To(Equal(manifest))
				Eventually(warningsStream).Should(Receive(Equal(Warnings{"apply-manifest-warnings"})))
				Eventually(errorStream).Should(Receive(Equal(errors.New("some-error"))))
				Consistently(pushPlansStream).ShouldNot(Receive(ConsistOf(PushPlan{
					SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName1},
				})))
				Consistently(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).ShouldNot(Equal(ApplyManifestComplete))
			})
		})

		When("There is a single pushPlan", func() {

			BeforeEach(func() {
				pushPlans = []PushPlan{{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName1}}}
				fakeManifestParser.FullRawManifestReturns(manifest)
				fakeManifestParser.RawAppManifestReturns(manifest, nil)
				fakeV7Actor.SetSpaceManifestReturns(v7action.Warnings{"apply-manifest-warnings"}, nil)
			})

			It("applies the app specific manifest", func() {
				Consistently(fakeV7Actor.CreateApplicationInSpaceCallCount).Should(Equal(0))
				Consistently(fakeManifestParser.FullRawManifestCallCount).Should(Equal(1))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(ApplyManifest))
				Eventually(fakeManifestParser.RawAppManifestCallCount).Should(Equal(1))
				actualAppName := fakeManifestParser.RawAppManifestArgsForCall(0)
				Expect(actualAppName).To(Equal(appName1))
				Eventually(fakeV7Actor.SetSpaceManifestCallCount).Should(Equal(1))
				actualSpaceGUID, actualManifest := fakeV7Actor.SetSpaceManifestArgsForCall(0)
				Expect(actualManifest).To(Equal(manifest))
				Expect(actualSpaceGUID).To(Equal(spaceGUID))
				Eventually(warningsStream).Should(Receive(Equal(Warnings{"apply-manifest-warnings"})))
				Eventually(errorStream).Should(Receive(Succeed()))
				Eventually(pushPlansStream).Should(Receive(ConsistOf(PushPlan{
					SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName1},
				})))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(ApplyManifestComplete))
			})
		})

		When("There are multiple push states", func() {
			BeforeEach(func() {
				pushPlans = []PushPlan{
					{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName1}},
					{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName2}},
				}
				fakeManifestParser.FullRawManifestReturns(manifest)
				fakeV7Actor.SetSpaceManifestReturns(v7action.Warnings{"apply-manifest-warnings"}, nil)
			})

			It("Applies the entire manifest", func() {
				Consistently(fakeV7Actor.CreateApplicationInSpaceCallCount).Should(Equal(0))
				Consistently(fakeManifestParser.RawAppManifestCallCount).Should(Equal(0))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(ApplyManifest))
				Eventually(fakeManifestParser.FullRawManifestCallCount).Should(Equal(2))
				Eventually(fakeV7Actor.SetSpaceManifestCallCount).Should(Equal(1))
				actualSpaceGUID, actualManifest := fakeV7Actor.SetSpaceManifestArgsForCall(0)
				Expect(actualManifest).To(Equal(manifest))
				Expect(actualSpaceGUID).To(Equal(spaceGUID))
				Eventually(warningsStream).Should(Receive(Equal(Warnings{"apply-manifest-warnings"})))
				Eventually(errorStream).Should(Receive(Succeed()))
				Eventually(pushPlansStream).Should(Receive(ConsistOf(
					PushPlan{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName1}},
					PushPlan{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName2}},
				)))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(ApplyManifestComplete))
			})
		})
	})

})
<commit_msg>remove unnecessary fake creation<commit_after>package v7pushaction_test

import (
	"errors"
	"time"

	"code.cloudfoundry.org/cli/actor/actionerror"
	"code.cloudfoundry.org/cli/actor/v7action"

	. "code.cloudfoundry.org/cli/actor/v7pushaction"
	"code.cloudfoundry.org/cli/actor/v7pushaction/v7pushactionfakes"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	log "github.com/sirupsen/logrus"
)

func PrepareSpaceStreamsDrainedAndClosed(
	pushPlansStream <-chan []PushPlan,
	eventStream <-chan Event,
	warningsStream <-chan Warnings,
	errorStream <-chan error,
) bool {
	var configStreamClosed, eventStreamClosed, warningsStreamClosed, errorStreamClosed bool
	for {
		select {
		case _, ok := <-pushPlansStream:
			if !ok {
				configStreamClosed = true
			}
		case _, ok := <-eventStream:
			if !ok {
				eventStreamClosed = true
			}
		case _, ok := <-warningsStream:
			if !ok {
				warningsStreamClosed = true
			}
		case _, ok := <-errorStream:
			if !ok {
				errorStreamClosed = true
			}
		}
		if configStreamClosed && eventStreamClosed && warningsStreamClosed && errorStreamClosed {
			break
		}
	}
	return true
}

func getPrepareNextEvent(c <-chan []PushPlan, e <-chan Event, w <-chan Warnings) func() Event {
	timeOut := time.Tick(500 * time.Millisecond)

	return func() Event {
		for {
			select {
			case <-c:
			case event, ok := <-e:
				if ok {
					log.WithField("event", event).Debug("getNextEvent")
					return event
				}
				return ""
			case <-w:
			case <-timeOut:
				return ""
			}
		}
	}
}

var _ = Describe("PrepareSpace", func() {
	var (
		actor       *Actor
		fakeV7Actor *v7pushactionfakes.FakeV7Actor

		pushPlans          []PushPlan
		fakeManifestParser *v7pushactionfakes.FakeManifestParser

		spaceGUID string

		pushPlansStream <-chan []PushPlan
		eventStream     <-chan Event
		warningsStream  <-chan Warnings
		errorStream     <-chan error
	)

	BeforeEach(func() {
		actor, _, fakeV7Actor, _ = getTestPushActor()

		spaceGUID = "space"

		fakeManifestParser = new(v7pushactionfakes.FakeManifestParser)
	})

	AfterEach(func() {
		Eventually(PrepareSpaceStreamsDrainedAndClosed(pushPlansStream, eventStream, warningsStream, errorStream)).Should(BeTrue())
	})

	JustBeforeEach(func() {
		pushPlansStream, eventStream, warningsStream, errorStream = actor.PrepareSpace(pushPlans, fakeManifestParser)
	})

	When("there is a single push state and no manifest", func() {
		var appName = "app-name"
		BeforeEach(func() {
			fakeManifestParser.FullRawManifestReturns(nil)
		})
		When("Creating the app succeeds", func() {
			BeforeEach(func() {
				pushPlans = []PushPlan{{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName}}}
				fakeV7Actor.CreateApplicationInSpaceReturns(
					v7action.Application{Name: appName},
					v7action.Warnings{"create-app-warning"},
					nil,
				)
			})
			It("creates the app using the API", func() {
				Consistently(fakeV7Actor.SetSpaceManifestCallCount).Should(Equal(0))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(CreatingApplication))
				Eventually(fakeV7Actor.CreateApplicationInSpaceCallCount).Should(Equal(1))
				actualApp, actualSpaceGUID := fakeV7Actor.CreateApplicationInSpaceArgsForCall(0)
				Expect(actualApp).To(Equal(v7action.Application{Name: appName}))
				Expect(actualSpaceGUID).To(Equal(spaceGUID))
				Eventually(warningsStream).Should(Receive(Equal(Warnings{"create-app-warning"})))
				Eventually(errorStream).Should(Receive(Succeed()))
				Eventually(pushPlansStream).Should(Receive(ConsistOf(PushPlan{
					SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName},
				})))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(CreatedApplication))
			})
		})

		When("the app already exists", func() {
			BeforeEach(func() {
				pushPlans = []PushPlan{{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName}}}
				fakeV7Actor.CreateApplicationInSpaceReturns(
					v7action.Application{},
					v7action.Warnings{"create-app-warning"},
					actionerror.ApplicationAlreadyExistsError{},
				)
			})
			It("Sends already exists events", func() {
				Consistently(fakeV7Actor.SetSpaceManifestCallCount).Should(Equal(0))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(SkippingApplicationCreation))
				Eventually(fakeV7Actor.CreateApplicationInSpaceCallCount).Should(Equal(1))
				actualApp, actualSpaceGUID := fakeV7Actor.CreateApplicationInSpaceArgsForCall(0)
				Expect(actualApp).To(Equal(v7action.Application{Name: appName}))
				Expect(actualSpaceGUID).To(Equal(spaceGUID))
				Eventually(warningsStream).Should(Receive(Equal(Warnings{"create-app-warning"})))
				Eventually(errorStream).Should(Receive(Succeed()))
				Eventually(pushPlansStream).Should(Receive(ConsistOf(PushPlan{
					SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName},
				})))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(ApplicationAlreadyExists))
			})
		})

		When("creating the app fails", func() {
			BeforeEach(func() {
				pushPlans = []PushPlan{{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName}}}
				fakeV7Actor.CreateApplicationInSpaceReturns(
					v7action.Application{},
					v7action.Warnings{"create-app-warning"},
					errors.New("some-create-error"),
				)
			})

			It("Returns the error", func() {
				Consistently(fakeV7Actor.SetSpaceManifestCallCount).Should(Equal(0))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(CreatingApplication))
				Eventually(fakeV7Actor.CreateApplicationInSpaceCallCount).Should(Equal(1))
				actualApp, actualSpaceGuid := fakeV7Actor.CreateApplicationInSpaceArgsForCall(0)
				Expect(actualApp.Name).To(Equal(appName))
				Expect(actualSpaceGuid).To(Equal(spaceGUID))
				Eventually(warningsStream).Should(Receive(Equal(Warnings{"create-app-warning"})))
				Eventually(errorStream).Should(Receive(Equal(errors.New("some-create-error"))))
				Consistently(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).ShouldNot(Equal(ApplicationAlreadyExists))
			})
		})
	})

	When("There is a a manifest", func() {
		var (
			manifest = []byte("app manifest")
			appName1 = "app-name1"
			appName2 = "app-name2"
		)
		When("applying the manifest fails", func() {
			BeforeEach(func() {
				pushPlans = []PushPlan{{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName1}}}
				fakeManifestParser.FullRawManifestReturns(manifest)
				fakeManifestParser.RawAppManifestReturns(manifest, nil)
				fakeV7Actor.SetSpaceManifestReturns(v7action.Warnings{"apply-manifest-warnings"}, errors.New("some-error"))
			})

			It("returns the error and exits", func() {
				Consistently(fakeV7Actor.CreateApplicationInSpaceCallCount).Should(Equal(0))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(ApplyManifest))
				Eventually(fakeV7Actor.SetSpaceManifestCallCount).Should(Equal(1))
				actualSpaceGuid, actualManifest := fakeV7Actor.SetSpaceManifestArgsForCall(0)
				Expect(actualSpaceGuid).To(Equal(spaceGUID))
				Expect(actualManifest).To(Equal(manifest))
				Eventually(warningsStream).Should(Receive(Equal(Warnings{"apply-manifest-warnings"})))
				Eventually(errorStream).Should(Receive(Equal(errors.New("some-error"))))
				Consistently(pushPlansStream).ShouldNot(Receive(ConsistOf(PushPlan{
					SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName1},
				})))
				Consistently(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).ShouldNot(Equal(ApplyManifestComplete))
			})
		})

		When("There is a single pushPlan", func() {

			BeforeEach(func() {
				pushPlans = []PushPlan{{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName1}}}
				fakeManifestParser.FullRawManifestReturns(manifest)
				fakeManifestParser.RawAppManifestReturns(manifest, nil)
				fakeV7Actor.SetSpaceManifestReturns(v7action.Warnings{"apply-manifest-warnings"}, nil)
			})

			It("applies the app specific manifest", func() {
				Consistently(fakeV7Actor.CreateApplicationInSpaceCallCount).Should(Equal(0))
				Consistently(fakeManifestParser.FullRawManifestCallCount).Should(Equal(1))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(ApplyManifest))
				Eventually(fakeManifestParser.RawAppManifestCallCount).Should(Equal(1))
				actualAppName := fakeManifestParser.RawAppManifestArgsForCall(0)
				Expect(actualAppName).To(Equal(appName1))
				Eventually(fakeV7Actor.SetSpaceManifestCallCount).Should(Equal(1))
				actualSpaceGUID, actualManifest := fakeV7Actor.SetSpaceManifestArgsForCall(0)
				Expect(actualManifest).To(Equal(manifest))
				Expect(actualSpaceGUID).To(Equal(spaceGUID))
				Eventually(warningsStream).Should(Receive(Equal(Warnings{"apply-manifest-warnings"})))
				Eventually(errorStream).Should(Receive(Succeed()))
				Eventually(pushPlansStream).Should(Receive(ConsistOf(PushPlan{
					SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName1},
				})))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(ApplyManifestComplete))
			})
		})

		When("There are multiple push states", func() {
			BeforeEach(func() {
				pushPlans = []PushPlan{
					{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName1}},
					{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName2}},
				}
				fakeManifestParser.FullRawManifestReturns(manifest)
				fakeV7Actor.SetSpaceManifestReturns(v7action.Warnings{"apply-manifest-warnings"}, nil)
			})

			It("Applies the entire manifest", func() {
				Consistently(fakeV7Actor.CreateApplicationInSpaceCallCount).Should(Equal(0))
				Consistently(fakeManifestParser.RawAppManifestCallCount).Should(Equal(0))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(ApplyManifest))
				Eventually(fakeManifestParser.FullRawManifestCallCount).Should(Equal(2))
				Eventually(fakeV7Actor.SetSpaceManifestCallCount).Should(Equal(1))
				actualSpaceGUID, actualManifest := fakeV7Actor.SetSpaceManifestArgsForCall(0)
				Expect(actualManifest).To(Equal(manifest))
				Expect(actualSpaceGUID).To(Equal(spaceGUID))
				Eventually(warningsStream).Should(Receive(Equal(Warnings{"apply-manifest-warnings"})))
				Eventually(errorStream).Should(Receive(Succeed()))
				Eventually(pushPlansStream).Should(Receive(ConsistOf(
					PushPlan{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName1}},
					PushPlan{SpaceGUID: spaceGUID, Application: v7action.Application{Name: appName2}},
				)))
				Eventually(getPrepareNextEvent(pushPlansStream, eventStream, warningsStream)).Should(Equal(ApplyManifestComplete))
			})
		})
	})

})
<|endoftext|>


<commit_before>package registry

import (
	"code.google.com/p/go-uuid/uuid"

	"github.com/wantedly/risu/schema"
)

type Registry interface {
	Set(build schema.Build) error
	Get(id uuid.UUID) (schema.Build, error)
}

func NewRegistry(backend string, endpoint string) Registry {
	switch backend {
	case "etcd":
		return NewEtcdRegistry(endpoint)
	case "localfs":
		return NewLocalFsRegistry(endpoint)
	default:
		return NewLocalFsRegistry(endpoint)
	}
}
<commit_msg>Registry can list builds<commit_after>package registry

import (
	"code.google.com/p/go-uuid/uuid"

	"github.com/wantedly/risu/schema"
)

type Registry interface {
	Set(build schema.Build) error
	Get(id uuid.UUID) (schema.Build, error)
	List() ([]schema.Build, error)
}

func NewRegistry(backend string, endpoint string) Registry {
	switch backend {
	case "etcd":
		return NewEtcdRegistry(endpoint)
	case "localfs":
		return NewLocalFsRegistry(endpoint)
	default:
		return NewLocalFsRegistry(endpoint)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"github.com/codegangsta/cli"
	"github.com/control-center/serviced/isvcs"
	"github.com/control-center/serviced/servicedversion"

	"encoding/json"
	"fmt"
	"os"
)

// Initializer for serviced version
func (c *ServicedCli) initVersion() {
	c.app.Commands = append(c.app.Commands, cli.Command{
		Name:        "version",
		Usage:       "shows version information",
		Description: "",
		Action:      c.cmdVersion,
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "verbose, v",
				Usage: "Show JSON format",
			},
		},
	})
}

// serviced version
func (c *ServicedCli) cmdVersion(ctx *cli.Context) {

	var versionInfo = map[string]string{
		"Version":      servicedversion.Version,
		"GoVersion":    servicedversion.GoVersion,
		"Gitcommit":    servicedversion.Gitcommit,
		"Gitbranch":    servicedversion.Gitbranch,
		"Date":         servicedversion.Date,
		"Release":      servicedversion.Release,
		"IsvcsImage":   fmt.Sprintf("%s:%s", isvcs.IMAGE_REPO, isvcs.IMAGE_TAG),
		"IsvcsZKImage": fmt.Sprintf("%s:%s", isvcs.ZK_IMAGE_REPO, isvcs.ZK_IMAGE_TAG),
	}

	if ctx.Bool("verbose") {
		if jsonVersion, err := json.MarshalIndent(versionInfo, " ", "  "); err != nil {
			fmt.Fprintf(os.Stderr, "failed to marshal version info: %s", err)
			os.Exit(1)
		} else {
			fmt.Println(string(jsonVersion))
		}
	} else {

		fmt.Println(versionInfo["IsvcsImage"])
		fmt.Println(versionInfo["IsvcsZKImage"])

		fmt.Printf("Version:    %s\n", versionInfo["Version"])
		fmt.Printf("GoVersion:  %s\n", versionInfo["GoVersion"])
		fmt.Printf("Gitcommit:  %s\n", versionInfo["Gitcommit"])
		fmt.Printf("Gitbranch:  %s\n", versionInfo["Gitbranch"])
		fmt.Printf("Date:       %s\n", versionInfo["Date"])
		fmt.Printf("Buildtag:   %s\n", versionInfo["Buildtag"])
		fmt.Printf("Release:    %s\n", versionInfo["Release"])
		images := []string{
			versionInfo["IsvcsImage"], versionInfo["IsvcsZKImage"],
		}
		fmt.Printf("IsvcsImages: %v\n", images)
	}
}
<commit_msg>Remove debugging print statements.<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"github.com/codegangsta/cli"
	"github.com/control-center/serviced/isvcs"
	"github.com/control-center/serviced/servicedversion"

	"encoding/json"
	"fmt"
	"os"
)

// Initializer for serviced version
func (c *ServicedCli) initVersion() {
	c.app.Commands = append(c.app.Commands, cli.Command{
		Name:        "version",
		Usage:       "shows version information",
		Description: "",
		Action:      c.cmdVersion,
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "verbose, v",
				Usage: "Show JSON format",
			},
		},
	})
}

// serviced version
func (c *ServicedCli) cmdVersion(ctx *cli.Context) {

	var versionInfo = map[string]string{
		"Version":      servicedversion.Version,
		"GoVersion":    servicedversion.GoVersion,
		"Gitcommit":    servicedversion.Gitcommit,
		"Gitbranch":    servicedversion.Gitbranch,
		"Date":         servicedversion.Date,
		"Release":      servicedversion.Release,
		"IsvcsImage":   fmt.Sprintf("%s:%s", isvcs.IMAGE_REPO, isvcs.IMAGE_TAG),
		"IsvcsZKImage": fmt.Sprintf("%s:%s", isvcs.ZK_IMAGE_REPO, isvcs.ZK_IMAGE_TAG),
	}

	if ctx.Bool("verbose") {
		if jsonVersion, err := json.MarshalIndent(versionInfo, " ", "  "); err != nil {
			fmt.Fprintf(os.Stderr, "failed to marshal version info: %s", err)
			os.Exit(1)
		} else {
			fmt.Println(string(jsonVersion))
		}
	} else {
		fmt.Printf("Version:    %s\n", versionInfo["Version"])
		fmt.Printf("GoVersion:  %s\n", versionInfo["GoVersion"])
		fmt.Printf("Gitcommit:  %s\n", versionInfo["Gitcommit"])
		fmt.Printf("Gitbranch:  %s\n", versionInfo["Gitbranch"])
		fmt.Printf("Date:       %s\n", versionInfo["Date"])
		fmt.Printf("Buildtag:   %s\n", versionInfo["Buildtag"])
		fmt.Printf("Release:    %s\n", versionInfo["Release"])
		images := []string{
			versionInfo["IsvcsImage"], versionInfo["IsvcsZKImage"],
		}
		fmt.Printf("IsvcsImages: %v\n", images)
	}
}
<|endoftext|>


<commit_before>package scheduler

import (
	"runtime"
	"sort"
	"sync"
	"sync/atomic"

	"github.com/henrylee2cn/pholcus/app/aid/history"
	"github.com/henrylee2cn/pholcus/app/downloader/request"
	"github.com/henrylee2cn/pholcus/common/util"
	"github.com/henrylee2cn/pholcus/logs"
	"github.com/henrylee2cn/pholcus/runtime/cache"
	"github.com/henrylee2cn/pholcus/runtime/status"
)

// 一个Spider实例的请求矩阵
type Matrix struct {
	spiderName      string                      // 所属Spider
	resCount        int32                       // 资源使用情况计数
	maxPage         int64                       // 最大采集页数，以负数形式表示
	reqs            map[int][]*request.Request  // [优先级]队列，优先级默认为0
	priorities      []int                       // 优先级顺序，从低到高
	history         history.Historier           // 历史记录
	tempHistory     map[string]bool             // 临时记录 [hash(url+method)]true
	failures        map[string]*request.Request // 历史及本次失败请求
	tempHistoryLock sync.RWMutex
	failureLock     sync.Mutex
	sync.Mutex
}

func newMatrix(spiderName string, maxPage int64) *Matrix {
	matrix := &Matrix{
		spiderName:  spiderName,
		maxPage:     maxPage,
		reqs:        make(map[int][]*request.Request),
		priorities:  []int{},
		history:     history.New(spiderName),
		tempHistory: make(map[string]bool),
		failures:    make(map[string]*request.Request),
	}
	if cache.Task.Mode != status.SERVER {
		matrix.history.ReadSuccess(cache.Task.OutType, cache.Task.SuccessInherit)
		matrix.history.ReadFailure(cache.Task.OutType, cache.Task.FailureInherit)
		matrix.setFailures(matrix.history.PullFailure())
	}
	return matrix
}

// 添加请求到队列，并发安全
func (self *Matrix) Push(req *request.Request) {
	if sdl.checkStatus(status.STOP) {
		return
	}

	// 禁止并发，降低请求积存量
	self.Lock()
	defer self.Unlock()

	// 达到请求上限，停止该规则运行
	if self.maxPage >= 0 {
		return
	}

	// 暂停状态时等待，降低请求积存量
	waited := false
	for sdl.checkStatus(status.PAUSE) {
		waited = true
		runtime.Gosched()
	}
	if waited && sdl.checkStatus(status.STOP) {
		return
	}

	// 资源使用过多时等待，降低请求积存量
	waited = false
	for self.resCount > sdl.avgRes() {
		waited = true
		runtime.Gosched()
	}
	if waited && sdl.checkStatus(status.STOP) {
		return
	}

	// 不可重复下载的req
	if !req.IsReloadable() {
		hash := makeUnique(req)
		// 已存在成功记录时退出
		if self.hasHistory(hash) {
			return
		}
		// 添加到临时记录
		self.insertTempHistory(hash)
	}

	var priority = req.GetPriority()

	// 初始化该蜘蛛下该优先级队列
	if _, found := self.reqs[priority]; !found {
		self.priorities = append(self.priorities, priority)
		sort.Ints(self.priorities) // 从小到大排序
		self.reqs[priority] = []*request.Request{}
	}

	// 添加请求到队列
	self.reqs[priority] = append(self.reqs[priority], req)

	// 大致限制加入队列的请求量，并发情况下应该会比maxPage多
	atomic.AddInt64(&self.maxPage, 1)
}

// 从队列取出请求，不存在时返回nil，并发安全
func (self *Matrix) Pull() (req *request.Request) {
	if !sdl.checkStatus(status.RUN) {
		return
	}
	self.Lock()
	defer self.Unlock()
	// 按优先级从高到低取出请求
	for i := len(self.reqs) - 1; i >= 0; i-- {
		idx := self.priorities[i]
		if len(self.reqs[idx]) > 0 {
			req = self.reqs[idx][0]
			self.reqs[idx] = self.reqs[idx][1:]
			if sdl.useProxy {
				req.SetProxy(sdl.proxy.GetOne(req.GetUrl()))
			} else {
				req.SetProxy("")
			}
			return
		}
	}
	return
}

func (self *Matrix) Use() {
	defer func() {
		recover()
	}()
	sdl.count <- true
	atomic.AddInt32(&self.resCount, 1)
}

func (self *Matrix) Free() {
	<-sdl.count
	atomic.AddInt32(&self.resCount, -1)
}

// 返回是否作为新的失败请求被添加至队列尾部
func (self *Matrix) DoHistory(req *request.Request, ok bool) bool {
	hash := makeUnique(req)

	if !req.IsReloadable() {
		self.tempHistoryLock.Lock()
		delete(self.tempHistory, hash)
		self.tempHistoryLock.Unlock()

		if ok {
			self.history.UpsertSuccess(hash)
			return false
		}
	}

	if ok {
		return false
	}

	self.failureLock.Lock()
	defer self.failureLock.Unlock()
	if _, ok := self.failures[hash]; !ok {
		// 首次失败时，在任务队列末尾重新执行一次
		self.failures[hash] = req
		logs.Log.Informational(" *     + 失败请求: [%v]\n", req.GetUrl())
		return true
	}
	// 失败两次后，加入历史失败记录
	self.history.UpsertFailure(req)
	return false
}

func (self *Matrix) CanStop() bool {
	if sdl.checkStatus(status.STOP) {
		return true
	}
	if self.maxPage >= 0 {
		return true
	}
	if self.resCount != 0 {
		return false
	}
	if self.Len() > 0 {
		return false
	}

	self.failureLock.Lock()
	defer self.failureLock.Unlock()
	if len(self.failures) > 0 {
		// 重新下载历史记录中失败的请求
		var goon bool
		for hash, req := range self.failures {
			if req == nil {
				continue
			}
			self.failures[hash] = nil
			goon = true
			logs.Log.Informational(" *     - 失败请求: [%v]\n", req.GetUrl())
			self.Push(req)
		}
		if goon {
			return false
		}
	}
	return true
}

func (self *Matrix) TryFlushHistory() {
	if cache.Task.SuccessInherit {
		self.history.FlushSuccess(cache.Task.OutType)
	}
	if cache.Task.FailureInherit {
		self.history.FlushFailure(cache.Task.OutType)
	}
}

// 等待处理中的请求完成
func (self *Matrix) Wait() {
	for self.resCount != 0 {
		runtime.Gosched()
	}
}

func (self *Matrix) Len() int {
	self.Lock()
	defer self.Unlock()
	var l int
	for _, reqs := range self.reqs {
		l += len(reqs)
	}
	return l
}

func (self *Matrix) hasHistory(hash string) bool {
	if self.history.HasSuccess(hash) {
		return true
	}
	self.tempHistoryLock.RLock()
	has := self.tempHistory[hash]
	self.tempHistoryLock.RUnlock()
	return has
}

func (self *Matrix) insertTempHistory(hash string) {
	self.tempHistoryLock.Lock()
	self.tempHistory[hash] = true
	self.tempHistoryLock.Unlock()
}

func (self *Matrix) setFailures(reqs map[*request.Request]bool) {
	self.failureLock.Lock()
	defer self.failureLock.Unlock()
	for req := range reqs {
		self.failures[makeUnique(req)] = req
		logs.Log.Informational(" *     + 失败请求: [%v]\n", req.GetUrl())
	}
}

func makeUnique(req *request.Request) string {
	return util.MakeUnique(req.GetUrl() + req.GetMethod())
}
<commit_msg>修复32位系统下panic的bug<commit_after>package scheduler

import (
	"runtime"
	"sort"
	"sync"
	"sync/atomic"

	"github.com/henrylee2cn/pholcus/app/aid/history"
	"github.com/henrylee2cn/pholcus/app/downloader/request"
	"github.com/henrylee2cn/pholcus/common/util"
	"github.com/henrylee2cn/pholcus/logs"
	"github.com/henrylee2cn/pholcus/runtime/cache"
	"github.com/henrylee2cn/pholcus/runtime/status"
)

// 一个Spider实例的请求矩阵
type Matrix struct {
	maxPage         int64                       // 最大采集页数，以负数形式表示
	resCount        int32                       // 资源使用情况计数
	spiderName      string                      // 所属Spider
	reqs            map[int][]*request.Request  // [优先级]队列，优先级默认为0
	priorities      []int                       // 优先级顺序，从低到高
	history         history.Historier           // 历史记录
	tempHistory     map[string]bool             // 临时记录 [hash(url+method)]true
	failures        map[string]*request.Request // 历史及本次失败请求
	tempHistoryLock sync.RWMutex
	failureLock     sync.Mutex
	sync.Mutex
}

func newMatrix(spiderName string, maxPage int64) *Matrix {
	matrix := &Matrix{
		spiderName:  spiderName,
		maxPage:     maxPage,
		reqs:        make(map[int][]*request.Request),
		priorities:  []int{},
		history:     history.New(spiderName),
		tempHistory: make(map[string]bool),
		failures:    make(map[string]*request.Request),
	}
	if cache.Task.Mode != status.SERVER {
		matrix.history.ReadSuccess(cache.Task.OutType, cache.Task.SuccessInherit)
		matrix.history.ReadFailure(cache.Task.OutType, cache.Task.FailureInherit)
		matrix.setFailures(matrix.history.PullFailure())
	}
	return matrix
}

// 添加请求到队列，并发安全
func (self *Matrix) Push(req *request.Request) {
	if sdl.checkStatus(status.STOP) {
		return
	}

	// 禁止并发，降低请求积存量
	self.Lock()
	defer self.Unlock()

	// 达到请求上限，停止该规则运行
	if self.maxPage >= 0 {
		return
	}

	// 暂停状态时等待，降低请求积存量
	waited := false
	for sdl.checkStatus(status.PAUSE) {
		waited = true
		runtime.Gosched()
	}
	if waited && sdl.checkStatus(status.STOP) {
		return
	}

	// 资源使用过多时等待，降低请求积存量
	waited = false
	for self.resCount > sdl.avgRes() {
		waited = true
		runtime.Gosched()
	}
	if waited && sdl.checkStatus(status.STOP) {
		return
	}

	// 不可重复下载的req
	if !req.IsReloadable() {
		hash := makeUnique(req)
		// 已存在成功记录时退出
		if self.hasHistory(hash) {
			return
		}
		// 添加到临时记录
		self.insertTempHistory(hash)
	}

	var priority = req.GetPriority()

	// 初始化该蜘蛛下该优先级队列
	if _, found := self.reqs[priority]; !found {
		self.priorities = append(self.priorities, priority)
		sort.Ints(self.priorities) // 从小到大排序
		self.reqs[priority] = []*request.Request{}
	}

	// 添加请求到队列
	self.reqs[priority] = append(self.reqs[priority], req)

	// 大致限制加入队列的请求量，并发情况下应该会比maxPage多
	atomic.AddInt64(&self.maxPage, 1)
}

// 从队列取出请求，不存在时返回nil，并发安全
func (self *Matrix) Pull() (req *request.Request) {
	if !sdl.checkStatus(status.RUN) {
		return
	}
	self.Lock()
	defer self.Unlock()
	// 按优先级从高到低取出请求
	for i := len(self.reqs) - 1; i >= 0; i-- {
		idx := self.priorities[i]
		if len(self.reqs[idx]) > 0 {
			req = self.reqs[idx][0]
			self.reqs[idx] = self.reqs[idx][1:]
			if sdl.useProxy {
				req.SetProxy(sdl.proxy.GetOne(req.GetUrl()))
			} else {
				req.SetProxy("")
			}
			return
		}
	}
	return
}

func (self *Matrix) Use() {
	defer func() {
		recover()
	}()
	sdl.count <- true
	atomic.AddInt32(&self.resCount, 1)
}

func (self *Matrix) Free() {
	<-sdl.count
	atomic.AddInt32(&self.resCount, -1)
}

// 返回是否作为新的失败请求被添加至队列尾部
func (self *Matrix) DoHistory(req *request.Request, ok bool) bool {
	hash := makeUnique(req)

	if !req.IsReloadable() {
		self.tempHistoryLock.Lock()
		delete(self.tempHistory, hash)
		self.tempHistoryLock.Unlock()

		if ok {
			self.history.UpsertSuccess(hash)
			return false
		}
	}

	if ok {
		return false
	}

	self.failureLock.Lock()
	defer self.failureLock.Unlock()
	if _, ok := self.failures[hash]; !ok {
		// 首次失败时，在任务队列末尾重新执行一次
		self.failures[hash] = req
		logs.Log.Informational(" *     + 失败请求: [%v]\n", req.GetUrl())
		return true
	}
	// 失败两次后，加入历史失败记录
	self.history.UpsertFailure(req)
	return false
}

func (self *Matrix) CanStop() bool {
	if sdl.checkStatus(status.STOP) {
		return true
	}
	if self.maxPage >= 0 {
		return true
	}
	if self.resCount != 0 {
		return false
	}
	if self.Len() > 0 {
		return false
	}

	self.failureLock.Lock()
	defer self.failureLock.Unlock()
	if len(self.failures) > 0 {
		// 重新下载历史记录中失败的请求
		var goon bool
		for hash, req := range self.failures {
			if req == nil {
				continue
			}
			self.failures[hash] = nil
			goon = true
			logs.Log.Informational(" *     - 失败请求: [%v]\n", req.GetUrl())
			self.Push(req)
		}
		if goon {
			return false
		}
	}
	return true
}

func (self *Matrix) TryFlushHistory() {
	if cache.Task.SuccessInherit {
		self.history.FlushSuccess(cache.Task.OutType)
	}
	if cache.Task.FailureInherit {
		self.history.FlushFailure(cache.Task.OutType)
	}
}

// 等待处理中的请求完成
func (self *Matrix) Wait() {
	for self.resCount != 0 {
		runtime.Gosched()
	}
}

func (self *Matrix) Len() int {
	self.Lock()
	defer self.Unlock()
	var l int
	for _, reqs := range self.reqs {
		l += len(reqs)
	}
	return l
}

func (self *Matrix) hasHistory(hash string) bool {
	if self.history.HasSuccess(hash) {
		return true
	}
	self.tempHistoryLock.RLock()
	has := self.tempHistory[hash]
	self.tempHistoryLock.RUnlock()
	return has
}

func (self *Matrix) insertTempHistory(hash string) {
	self.tempHistoryLock.Lock()
	self.tempHistory[hash] = true
	self.tempHistoryLock.Unlock()
}

func (self *Matrix) setFailures(reqs map[*request.Request]bool) {
	self.failureLock.Lock()
	defer self.failureLock.Unlock()
	for req := range reqs {
		self.failures[makeUnique(req)] = req
		logs.Log.Informational(" *     + 失败请求: [%v]\n", req.GetUrl())
	}
}

func makeUnique(req *request.Request) string {
	return util.MakeUnique(req.GetUrl() + req.GetMethod())
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"owl/common/types"
	"path/filepath"

	"github.com/wuyingsong/tcp"
)

type handle struct {
}

type callback struct {
}

func (cb *callback) OnConnected(conn *tcp.TCPConn) {
	lg.Info("callback:%s connected", conn.GetRemoteAddr().String())
}

//链接断开回调
func (cb *callback) OnDisconnected(conn *tcp.TCPConn) {
	lg.Info("callback:%s disconnect ", conn.GetRemoteAddr().String())
}

//错误回调
func (cb *callback) OnError(err error) {
	lg.Error("callback: %s", err)
}

func (cb *callback) OnMessage(conn *tcp.TCPConn, p tcp.Packet) {
	defer func() {
		if r := recover(); r != nil {
			lg.Error("Recovered in OnMessage", r)
		}
	}()
	pkt := p.(*tcp.DefaultPacket)
	switch pkt.Type {
	case types.MsgCFCSendPluginsList:
		resp := types.GetPluginResp{}
		if err := resp.Decode(pkt.Body); err != nil {
			lg.Error("decode plugin response error %s", err)
			return
		}
		lg.Debug("recive message, type:%s, body:%s", types.MsgTextMap[pkt.Type], string(pkt.Body))
		removeNoUsePlugin(resp.Plugins)
		mergePlugin(resp.Plugins)
	case types.MsgCFCSendReconnect:
		conn.Close()
	case types.MsgCFCSendPlugin:
		sp := types.SyncPluginResponse{}
		if err := sp.Decode(pkt.Body); err != nil {
			lg.Error("%s", err)
			return
		}
		lg.Debug("recive message, %s %s", types.MsgTextMap[pkt.Type], sp.Path)
		filename := filepath.Join(GlobalConfig.PluginDir, sp.Path)
	retry:
		fd, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
		if err != nil {
			if os.IsNotExist(err) {
				dir := filepath.Dir(filename)
				lg.Warn("plugin dir(%s) is not exists, create", dir)
				if err = os.MkdirAll(filepath.Dir(dir), 0755); err != nil {
					lg.Warn("mkdir %s failed, error:%s", dir, err.Error())
					return
				}
				goto retry
			}
			lg.Error("%s", err)
			return
		}
		defer fd.Close()
		writeLen, err := fd.Write(sp.Body)
		if err != nil {
			lg.Error("create plugin error(%s), %s", err, sp.Path)
			return
		}
		lg.Debug("create plugin(%s) successfully, write %d bytes.", sp.Path, writeLen)
	default:
		lg.Error("unsupport packet type %v", pkt.Type)
		conn.Close()
	}

}
<commit_msg>bugfix: 创建插件目录传参错误导致循环创建<commit_after>package main

import (
	"os"
	"owl/common/types"
	"path/filepath"

	"github.com/wuyingsong/tcp"
)

type handle struct {
}

type callback struct {
}

func (cb *callback) OnConnected(conn *tcp.TCPConn) {
	lg.Info("callback:%s connected", conn.GetRemoteAddr().String())
}

//链接断开回调
func (cb *callback) OnDisconnected(conn *tcp.TCPConn) {
	lg.Info("callback:%s disconnect ", conn.GetRemoteAddr().String())
}

//错误回调
func (cb *callback) OnError(err error) {
	lg.Error("callback: %s", err)
}

func (cb *callback) OnMessage(conn *tcp.TCPConn, p tcp.Packet) {
	defer func() {
		if r := recover(); r != nil {
			lg.Error("Recovered in OnMessage", r)
		}
	}()
	pkt := p.(*tcp.DefaultPacket)
	switch pkt.Type {
	case types.MsgCFCSendPluginsList:
		resp := types.GetPluginResp{}
		if err := resp.Decode(pkt.Body); err != nil {
			lg.Error("decode plugin response error %s", err)
			return
		}
		lg.Debug("recive message, type:%s, body:%s", types.MsgTextMap[pkt.Type], string(pkt.Body))
		removeNoUsePlugin(resp.Plugins)
		mergePlugin(resp.Plugins)
	case types.MsgCFCSendReconnect:
		conn.Close()
	case types.MsgCFCSendPlugin:
		sp := types.SyncPluginResponse{}
		if err := sp.Decode(pkt.Body); err != nil {
			lg.Error("%s", err)
			return
		}
		lg.Debug("recive message, %s %s", types.MsgTextMap[pkt.Type], sp.Path)
		filename := filepath.Join(GlobalConfig.PluginDir, sp.Path)
	retry:
		fd, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
		if err != nil {
			if os.IsNotExist(err) {
				dir := filepath.Dir(filename)
				lg.Warn("create plugin failed, dir(%s) is not exists, create", dir)
				if err = os.MkdirAll(dir, 0755); err != nil {
					lg.Warn("mkdir %s failed, error:%s", dir, err.Error())
					return
				}
				goto retry
			}
			lg.Error("%s", err)
			return
		}
		defer fd.Close()
		writeLen, err := fd.Write(sp.Body)
		if err != nil {
			lg.Error("create plugin error(%s), %s", err, sp.Path)
			return
		}
		lg.Debug("create plugin(%s) successfully, write %d bytes.", sp.Path, writeLen)
	default:
		lg.Error("unsupport packet type %v", pkt.Type)
		conn.Close()
	}

}
<|endoftext|>


<commit_before>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

// NodeType identifies the type of a parse tree node.
type NodeType int

const (
	NodeSection NodeType = iota
	NodeParagraph
	NodeBlankLine
	NodeAdornment
)

var nodeTypes = [...]string{
	"NodeSection",
	"NodeParagraph",
	"NodeBlankLine",
	"NodeAdornment",
}

func (n NodeType) Type() NodeType {
	return n
}

func (n NodeType) String() string {
	return nodeTypes[n]
}

func (n NodeType) MarshalText() ([]byte, error) {
	return []byte(n.String()), nil
}

type Node interface {
	LineNumber() Line
	NodeType() NodeType
	Position() StartPosition
}

type NodeList []Node

func newList() *NodeList {
	return new(NodeList)
}

func (l *NodeList) append(n Node) {
	*l = append(*l, n)

}

type SectionNode struct {
	Id            int      `json:"id"`
	Type          NodeType `json:"type"`
	Text          string   `json:"text"`
	Level         int      `json:"level"`
	Length        int      `json:"length"`
	StartPosition `json:"startPosition"`
	Line          `json:"line"`
	OverLine      *AdornmentNode `json:"overLine"`
	UnderLine     *AdornmentNode `json:"underLine"`
	NodeList      NodeList       `json:"nodeList"`
}

func (s *SectionNode) NodeType() NodeType {
	return s.Type
}

func newSection(i item, id *int, level int, overAdorn item, underAdorn item) *SectionNode {
	*id++
	n := &SectionNode{
		Id:            *id,
		Type:          NodeSection,
		Text:          i.Text.(string),
		Level:         level,
		StartPosition: i.StartPosition,
		Length:        i.Length,
		Line:          i.Line,
	}

	if overAdorn.Text != nil {
		*id++
		Rune := rune(overAdorn.Text.(string)[0])
		n.OverLine = &AdornmentNode{
			Id:            *id,
			Type:          NodeAdornment,
			Rune:          Rune,
			StartPosition: overAdorn.StartPosition,
			Line:          overAdorn.Line,
			Length:        overAdorn.Length,
		}
	}

	*id++
	Rune := rune(underAdorn.Text.(string)[0])
	n.UnderLine = &AdornmentNode{
		Id:            *id,
		Rune:          Rune,
		Type:          NodeAdornment,
		StartPosition: underAdorn.StartPosition,
		Line:          underAdorn.Line,
		Length:        underAdorn.Length,
	}

	return n
}

type AdornmentNode struct {
	Id            int      `json:"id"`
	Type          NodeType `json:"type"`
	Rune          rune     `json:"rune"`
	Length        int      `json:"length"`
	Line          `json:"line"`
	StartPosition `json:"startPosition"`
}

func (a AdornmentNode) NodeType() NodeType {
	return a.Type
}

type BlankLineNode struct {
	Id            int      `json:"id"`
	Type          NodeType `json:"nodetype"`
	Line          `json:"line"`
	StartPosition `json:"startPosition"`
}

func (b BlankLineNode) NodeType() NodeType {
	return b.Type
}

func newBlankLine(i item, id *int) *BlankLineNode {
	*id++
	return &BlankLineNode{
		Id:            *id,
		Type:          NodeBlankLine,
		Line:          i.Line,
		StartPosition: i.StartPosition,
	}
}

type ParagraphNode struct {
	Id            int      `json:"id"`
	Type          NodeType `json:"type"`
	Text          string   `json:"text"`
	Length        int      `json:"length"`
	Line          `json:"line"`
	StartPosition `json:"startPosition"`
}

func newParagraph(i item, id *int) *ParagraphNode {
	*id++
	return &ParagraphNode{
		Id:            *id,
		Type:          NodeParagraph,
		Text:          i.Text.(string),
		Length:        i.Length,
		Line:          i.Line,
		StartPosition: i.StartPosition,
	}
}

func (p ParagraphNode) NodeType() NodeType {
	return p.Type
}
<commit_msg>node.go: Add missing fields to BlankLineNode<commit_after>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

// NodeType identifies the type of a parse tree node.
type NodeType int

const (
	NodeSection NodeType = iota
	NodeParagraph
	NodeBlankLine
	NodeAdornment
)

var nodeTypes = [...]string{
	"NodeSection",
	"NodeParagraph",
	"NodeBlankLine",
	"NodeAdornment",
}

func (n NodeType) Type() NodeType {
	return n
}

func (n NodeType) String() string {
	return nodeTypes[n]
}

func (n NodeType) MarshalText() ([]byte, error) {
	return []byte(n.String()), nil
}

type Node interface {
	LineNumber() Line
	NodeType() NodeType
	Position() StartPosition
}

type NodeList []Node

func newList() *NodeList {
	return new(NodeList)
}

func (l *NodeList) append(n Node) {
	*l = append(*l, n)

}

type SectionNode struct {
	Id            int      `json:"id"`
	Type          NodeType `json:"type"`
	Text          string   `json:"text"`
	Level         int      `json:"level"`
	Length        int      `json:"length"`
	StartPosition `json:"startPosition"`
	Line          `json:"line"`
	OverLine      *AdornmentNode `json:"overLine"`
	UnderLine     *AdornmentNode `json:"underLine"`
	NodeList      NodeList       `json:"nodeList"`
}

func (s *SectionNode) NodeType() NodeType {
	return s.Type
}

func newSection(i item, id *int, level int, overAdorn item, underAdorn item) *SectionNode {
	*id++
	n := &SectionNode{
		Id:            *id,
		Type:          NodeSection,
		Text:          i.Text.(string),
		Level:         level,
		StartPosition: i.StartPosition,
		Length:        i.Length,
		Line:          i.Line,
	}

	if overAdorn.Text != nil {
		*id++
		Rune := rune(overAdorn.Text.(string)[0])
		n.OverLine = &AdornmentNode{
			Id:            *id,
			Type:          NodeAdornment,
			Rune:          Rune,
			StartPosition: overAdorn.StartPosition,
			Line:          overAdorn.Line,
			Length:        overAdorn.Length,
		}
	}

	*id++
	Rune := rune(underAdorn.Text.(string)[0])
	n.UnderLine = &AdornmentNode{
		Id:            *id,
		Rune:          Rune,
		Type:          NodeAdornment,
		StartPosition: underAdorn.StartPosition,
		Line:          underAdorn.Line,
		Length:        underAdorn.Length,
	}

	return n
}

type AdornmentNode struct {
	Id            int      `json:"id"`
	Type          NodeType `json:"type"`
	Rune          rune     `json:"rune"`
	Length        int      `json:"length"`
	Line          `json:"line"`
	StartPosition `json:"startPosition"`
}

func (a AdornmentNode) NodeType() NodeType {
	return a.Type
}

type BlankLineNode struct {
	Id            int      `json:"id"`
	Type          NodeType `json:"nodetype"`
	Text	      string `json:"text"`
	Length        int      `json:"length"`
	Line          `json:"line"`
	StartPosition `json:"startPosition"`
}

func (b BlankLineNode) NodeType() NodeType {
	return b.Type
}

func newBlankLine(i item, id *int) *BlankLineNode {
	*id++
	return &BlankLineNode{
		Id:            *id,
		Type:          NodeBlankLine,
		Text:	       i.Text.(string),
		Length:	       i.Length,
		Line:          i.Line,
		StartPosition: i.StartPosition,
	}
}

type ParagraphNode struct {
	Id            int      `json:"id"`
	Type          NodeType `json:"type"`
	Text          string   `json:"text"`
	Length        int      `json:"length"`
	Line          `json:"line"`
	StartPosition `json:"startPosition"`
}

func newParagraph(i item, id *int) *ParagraphNode {
	*id++
	return &ParagraphNode{
		Id:            *id,
		Type:          NodeParagraph,
		Text:          i.Text.(string),
		Length:        i.Length,
		Line:          i.Line,
		StartPosition: i.StartPosition,
	}
}

func (p ParagraphNode) NodeType() NodeType {
	return p.Type
}
<|endoftext|>


<commit_before>package narcissus

import (
	"testing"

	"honnef.co/go/augeas"
)

type foo struct {
	augeasPath string
	A          string `path:"a"`
}

type bar struct{}

func TestParseNotAPtr(t *testing.T) {
	aug, err := augeas.New("", "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)
	err = n.Parse(foo{
		augeasPath: "/files/some/path",
	})

	if err == nil {
		t.Error("Expected an error, got nothing")
	}

	if err.Error() != "not a ptr" {
		t.Errorf("Expected error not a ptr, got %s", err.Error())
	}
}

func TestParseNotAStruct(t *testing.T) {
	aug, err := augeas.New("", "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)
	f := "foo"
	err = n.Parse(&f)

	if err == nil {
		t.Error("Expected an error, got nothing")
	}

	if err.Error() != "not a struct" {
		t.Errorf("Expected error not a struct, got %s", err.Error())
	}
}

func TestParseFieldNotFound(t *testing.T) {
	aug, err := augeas.New("", "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)
	err = n.Parse(&foo{
		augeasPath: "/files/some/path",
	})

	if err == nil {
		t.Error("Expected an error, got nothing")
	}
}

func TestNoAugeasPathValue(t *testing.T) {
	aug, err := augeas.New("", "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)
	err = n.Parse(&foo{})

	if err == nil {
		t.Error("Expected an error, got nothing")
	}

	if err.Error() != "no augeasPath value and no default" {
		t.Errorf("Expected error no augeasPath value and no default, got %s", err.Error())
	}
}

func TestNoAugeasPathField(t *testing.T) {
	aug, err := augeas.New("", "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)
	err = n.Parse(&bar{})

	if err == nil {
		t.Error("Expected an error, got nothing")
	}

	if err.Error() != "no augeasPath field" {
		t.Errorf("Expected error no augeasPath field, got %s", err.Error())
	}
}

type simpleValues struct {
	augeasPath string
	Str        string   `path:"str"`
	Int        int      `path:"int"`
	Bool       bool     `path:"bool"`
	SlStr      []string `path:"slstr"`
}

func TestGetStringField(t *testing.T) {
	aug, err := augeas.New("", "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)
	n.Augeas.Set("/test/str", "foo")
	n.Augeas.Set("/test/int", "42")
	n.Augeas.Set("/test/bool", "true")
	n.Augeas.Set("/test/slstr[1]", "a")
	n.Augeas.Set("/test/slstr[2]", "b")
	s := &simpleValues{
		augeasPath: "/test",
	}
	err = n.Parse(s)

	if err != nil {
		t.Errorf("Expected no error, got %v", err)
	}

	if s.Str != "foo" {
		t.Errorf("Expected foo, got %s", s.Str)
	}

	if s.Int != 42 {
		t.Errorf("Expected 42, got %v", s.Int)
	}

	if s.Bool != true {
		t.Errorf("Expected true, got %v", s.Bool)
	}

	if len(s.SlStr) != 2 {
		t.Errorf("Expected 2 elements, got %v", len(s.SlStr))
	}

	if s.SlStr[1] != "b" {
		t.Errorf("Expected element to be b, got %s", s.SlStr[1])
	}

}
<commit_msg>Skip a test<commit_after>package narcissus

import (
	"testing"

	"honnef.co/go/augeas"
)

type foo struct {
	augeasPath string
	A          string `path:"a"`
}

type bar struct{}

func TestParseNotAPtr(t *testing.T) {
	aug, err := augeas.New("", "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)
	err = n.Parse(foo{
		augeasPath: "/files/some/path",
	})

	if err == nil {
		t.Error("Expected an error, got nothing")
	}

	if err.Error() != "not a ptr" {
		t.Errorf("Expected error not a ptr, got %s", err.Error())
	}
}

func TestParseNotAStruct(t *testing.T) {
	aug, err := augeas.New("", "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)
	f := "foo"
	err = n.Parse(&f)

	if err == nil {
		t.Error("Expected an error, got nothing")
	}

	if err.Error() != "not a struct" {
		t.Errorf("Expected error not a struct, got %s", err.Error())
	}
}

func TestParseFieldNotFound(t *testing.T) {
	aug, err := augeas.New("", "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)
	err = n.Parse(&foo{
		augeasPath: "/files/some/path",
	})

	t.Skip("Fix this")

	if err == nil {
		t.Error("Expected an error, got nothing")
	}
}

func TestNoAugeasPathValue(t *testing.T) {
	aug, err := augeas.New("", "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)
	err = n.Parse(&foo{})

	if err == nil {
		t.Error("Expected an error, got nothing")
	}

	if err.Error() != "no augeasPath value and no default" {
		t.Errorf("Expected error no augeasPath value and no default, got %s", err.Error())
	}
}

func TestNoAugeasPathField(t *testing.T) {
	aug, err := augeas.New("", "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)
	err = n.Parse(&bar{})

	if err == nil {
		t.Error("Expected an error, got nothing")
	}

	if err.Error() != "no augeasPath field" {
		t.Errorf("Expected error no augeasPath field, got %s", err.Error())
	}
}

type simpleValues struct {
	augeasPath string
	Str        string   `path:"str"`
	Int        int      `path:"int"`
	Bool       bool     `path:"bool"`
	SlStr      []string `path:"slstr"`
}

func TestGetStringField(t *testing.T) {
	aug, err := augeas.New("", "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)
	n.Augeas.Set("/test/str", "foo")
	n.Augeas.Set("/test/int", "42")
	n.Augeas.Set("/test/bool", "true")
	n.Augeas.Set("/test/slstr[1]", "a")
	n.Augeas.Set("/test/slstr[2]", "b")
	s := &simpleValues{
		augeasPath: "/test",
	}
	err = n.Parse(s)

	if err != nil {
		t.Errorf("Expected no error, got %v", err)
	}

	if s.Str != "foo" {
		t.Errorf("Expected foo, got %s", s.Str)
	}

	if s.Int != 42 {
		t.Errorf("Expected 42, got %v", s.Int)
	}

	if s.Bool != true {
		t.Errorf("Expected true, got %v", s.Bool)
	}

	if len(s.SlStr) != 2 {
		t.Errorf("Expected 2 elements, got %v", len(s.SlStr))
	}

	if s.SlStr[1] != "b" {
		t.Errorf("Expected element to be b, got %s", s.SlStr[1])
	}

}
<|endoftext|>


<commit_before>package parser

import (
	"fmt"
	"plaid/lexer"
	"strconv"
	"strings"
)

// Node is the ancestor of all AST nodes
type Node interface {
	Start() lexer.Loc
	String() string
	isNode()
}

// Program describes all top-level statements within a script
type Program struct {
	stmts []Stmt
}

// Start returns a location that this node can be considered to start at
func (p Program) Start() lexer.Loc {
	if len(p.stmts) > 0 {
		return p.stmts[0].Start()
	}

	return lexer.Loc{Line: 1, Col: 1}
}

func (p Program) String() string {
	out := ""
	for i, stmt := range p.stmts {
		if i > 0 {
			out += "\n"
		}

		out += stmt.String()
	}
	return out
}

func (p Program) isNode() {}

// Stmt describes all constructs that return no value
type Stmt interface {
	Start() lexer.Loc
	String() string
	isNode()
	isStmt()
}

// StmtBlock describes any series of statements bounded by curly braces
type StmtBlock struct {
	left  lexer.Token
	stmts []Stmt
	right lexer.Token
}

// Start returns a location that this node can be considered to start at
func (sb StmtBlock) Start() lexer.Loc { return sb.left.Loc }

func (sb StmtBlock) String() string {
	out := "{"
	for _, stmt := range sb.stmts {
		out += "\n" + indentBlock("  ", stmt.String())
	}
	return out + "}"
}

func (sb StmtBlock) isNode() {}

// DeclarationStmt describes the declaration and assignment of a variable
type DeclarationStmt struct {
	tok  lexer.Token
	name IdentExpr
	expr Expr
}

// Start returns a location that this node can be considered to start at
func (ds DeclarationStmt) Start() lexer.Loc { return ds.tok.Loc }
func (ds DeclarationStmt) String() string   { return fmt.Sprintf("(let %s %s)", ds.name, ds.expr) }
func (ds DeclarationStmt) isNode()          {}
func (ds DeclarationStmt) isStmt()          {}

// ReturnStmt describes a return keyword and an optional returned expression.
type ReturnStmt struct {
	tok  lexer.Token
	expr Expr
}

// Start returns a location that this node can be considered to start at
func (rs ReturnStmt) Start() lexer.Loc { return rs.tok.Loc }

func (rs ReturnStmt) String() string {
	if rs.expr != nil {
		return fmt.Sprintf("(return %s)", rs.expr)
	}

	return "(return)"
}

func (rs ReturnStmt) isNode() {}
func (rs ReturnStmt) isStmt() {}

// ExprStmt describes certain expressions that can be used in the place of statements
type ExprStmt struct {
	expr Expr
}

// Start returns a location that this node can be considered to start at
func (es ExprStmt) Start() lexer.Loc { return es.expr.Start() }
func (es ExprStmt) String() string   { return es.expr.String() }
func (es ExprStmt) isNode()          {}
func (es ExprStmt) isStmt()          {}

// TypeSig describes a syntax type annotation
type TypeSig interface {
	Start() lexer.Loc
	String() string
	isNode()
	isType()
}

// TypeTuple describes a set of 0 or more types wrapped in parentheses
type TypeTuple struct {
	tok   lexer.Token
	elems []TypeSig
}

// Start returns a location that this node can be considered to start at
func (tt TypeTuple) Start() lexer.Loc { return tt.tok.Loc }

func (tt TypeTuple) String() string {
	out := "("
	for i, elem := range tt.elems {
		if i > 0 {
			out += " "
		}
		out += elem.String()
	}
	out += ")"
	return out
}

func (tt TypeTuple) isNode() {}
func (tt TypeTuple) isType() {}

// TypeFunction describes a function type annotation
type TypeFunction struct {
	params TypeTuple
	ret    TypeSig
}

// Start returns a location that this node can be considered to start at
func (tf TypeFunction) Start() lexer.Loc { return tf.params.Start() }

func (tf TypeFunction) String() string {
	out := tf.params.String()
	out += " => "
	out += tf.ret.String()
	return out
}

func (tf TypeFunction) isNode() {}
func (tf TypeFunction) isType() {}

// TypeIdent describes a named reference to a type
type TypeIdent struct {
	tok  lexer.Token
	name string
}

// Start returns a location that this node can be considered to start at
func (ti TypeIdent) Start() lexer.Loc { return ti.tok.Loc }
func (ti TypeIdent) String() string   { return ti.name }
func (ti TypeIdent) isNode()          {}
func (ti TypeIdent) isType()          {}

// TypeList describes a list type
type TypeList struct {
	tok   lexer.Token
	child TypeSig
}

// Start returns a location that this node can be considered to start at
func (tl TypeList) Start() lexer.Loc { return tl.tok.Loc }
func (tl TypeList) String() string   { return fmt.Sprintf("[%s]", tl.child) }
func (tl TypeList) isNode()          {}
func (tl TypeList) isType()          {}

// TypeOptional describes a list type
type TypeOptional struct {
	tok   lexer.Token
	child TypeSig
}

// Start returns a location that this node can be considered to start at
func (to TypeOptional) Start() lexer.Loc { return to.child.Start() }
func (to TypeOptional) String() string   { return fmt.Sprintf("%s?", to.child) }
func (to TypeOptional) isNode()          {}
func (to TypeOptional) isType()          {}

// Expr describes all constructs that resolve to a value
type Expr interface {
	Start() lexer.Loc
	String() string
	isNode()
	isExpr()
}

// FunctionExpr describes a function's entire type signature and body
type FunctionExpr struct {
	tok    lexer.Token
	params []FunctionParam
	ret    TypeSig
	block  StmtBlock
}

// Start returns a location that this node can be considered to start at
func (fe FunctionExpr) Start() lexer.Loc { return fe.tok.Loc }

func (fe FunctionExpr) String() string {
	out := "(fn ("
	for i, param := range fe.params {
		if i > 0 {
			out += " "
		}
		out += param.String()
	}
	out += ")"
	if fe.ret != nil {
		out += fmt.Sprintf(":%s", fe.ret)
	}
	out += fmt.Sprintf(" %s)", fe.block)
	return out
}

func (fe FunctionExpr) isExpr() {}
func (fe FunctionExpr) isNode() {}

// FunctionParam describes a single function argument's name and type signature
type FunctionParam struct {
	name IdentExpr
	sig  TypeSig
}

// Start returns a location that this node can be considered to start at
func (fp FunctionParam) Start() lexer.Loc { return fp.name.Start() }

func (fp FunctionParam) String() string {
	if fp.sig != nil {
		return fmt.Sprintf("%s:%s", fp.name, fp.sig)
	}

	return fp.name.String()
}

func (fp FunctionParam) isNode() {}

// DispatchExpr describes a function call including the callee and any arguments
type DispatchExpr struct {
	callee Expr
	args   []Expr
}

// Start returns a location that this node can be considered to start at
func (de DispatchExpr) Start() lexer.Loc { return de.callee.Start() }

func (de DispatchExpr) String() string {
	out := "("
	out += de.callee.String()
	out += " ("
	for i, arg := range de.args {
		if i > 0 {
			out += " "
		}
		out += arg.String()
	}
	out += "))"
	return out
}

func (de DispatchExpr) isNode() {}
func (de DispatchExpr) isExpr() {}

// AssignExpr describes the binding of a value to an assignable expression
type AssignExpr struct {
	tok   lexer.Token
	left  Expr
	right Expr
}

// Start returns a location that this node can be considered to start at
func (ae AssignExpr) Start() lexer.Loc { return ae.left.Start() }
func (ae AssignExpr) String() string   { return fmt.Sprintf("(= %s %s)", ae.left, ae.right) }
func (ae AssignExpr) isNode()          {}
func (ae AssignExpr) isExpr()          {}

// BinaryExpr describes any two expressions associated by an operator
type BinaryExpr struct {
	oper  string
	tok   lexer.Token
	left  Expr
	right Expr
}

// Start returns a location that this node can be considered to start at
func (be BinaryExpr) Start() lexer.Loc { return be.left.Start() }
func (be BinaryExpr) String() string   { return fmt.Sprintf("(%s %s %s)", be.oper, be.left, be.right) }
func (be BinaryExpr) isNode()          {}
func (be BinaryExpr) isExpr()          {}

// UnaryExpr describes any single expression associated to an operator
type UnaryExpr struct {
	oper string
	tok  lexer.Token
	expr Expr
}

// Start returns a location that this node can be considered to start at
func (ue UnaryExpr) Start() lexer.Loc { return lexer.SmallerLoc(ue.tok.Loc, ue.expr.Start()) }
func (ue UnaryExpr) String() string   { return fmt.Sprintf("(%s %s)", ue.oper, ue.expr) }
func (ue UnaryExpr) isNode()          {}
func (ue UnaryExpr) isExpr()          {}

// IdentExpr describes an identifier
type IdentExpr struct {
	tok  lexer.Token
	name string
}

// Start returns a location that this node can be considered to start at
func (ie IdentExpr) Start() lexer.Loc { return ie.tok.Loc }
func (ie IdentExpr) String() string   { return ie.name }
func (ie IdentExpr) isNode()          {}
func (ie IdentExpr) isExpr()          {}

// StringExpr describes a string literal
type StringExpr struct {
	tok lexer.Token
	val string
}

// Start returns a location that this node can be considered to start at
func (se StringExpr) Start() lexer.Loc { return se.tok.Loc }
func (se StringExpr) String() string   { return fmt.Sprintf("\"%s\"", se.val) }
func (se StringExpr) isNode()          {}
func (se StringExpr) isExpr()          {}

// NumberExpr describes a string literal
type NumberExpr struct {
	tok lexer.Token
	val int
}

// Start returns a location that this node can be considered to start at
func (ne NumberExpr) Start() lexer.Loc { return ne.tok.Loc }
func (ne NumberExpr) String() string   { return strconv.Itoa(ne.val) }
func (ne NumberExpr) isNode()          {}
func (ne NumberExpr) isExpr()          {}

func indentBlock(indent string, source string) string {
	lines := strings.Split(source, "\n")
	for i, line := range lines {
		lines[i] = indent + line
	}

	return strings.Join(lines, "\n")
}
<commit_msg>export Stmts property<commit_after>package parser

import (
	"fmt"
	"plaid/lexer"
	"strconv"
	"strings"
)

// Node is the ancestor of all AST nodes
type Node interface {
	Start() lexer.Loc
	String() string
	isNode()
}

// Program describes all top-level statements within a script
type Program struct {
	Stmts []Stmt
}

// Start returns a location that this node can be considered to start at
func (p Program) Start() lexer.Loc {
	if len(p.Stmts) > 0 {
		return p.Stmts[0].Start()
	}

	return lexer.Loc{Line: 1, Col: 1}
}

func (p Program) String() string {
	out := ""
	for i, stmt := range p.Stmts {
		if i > 0 {
			out += "\n"
		}

		out += stmt.String()
	}
	return out
}

func (p Program) isNode() {}

// Stmt describes all constructs that return no value
type Stmt interface {
	Start() lexer.Loc
	String() string
	isNode()
	isStmt()
}

// StmtBlock describes any series of statements bounded by curly braces
type StmtBlock struct {
	left  lexer.Token
	stmts []Stmt
	right lexer.Token
}

// Start returns a location that this node can be considered to start at
func (sb StmtBlock) Start() lexer.Loc { return sb.left.Loc }

func (sb StmtBlock) String() string {
	out := "{"
	for _, stmt := range sb.stmts {
		out += "\n" + indentBlock("  ", stmt.String())
	}
	return out + "}"
}

func (sb StmtBlock) isNode() {}

// DeclarationStmt describes the declaration and assignment of a variable
type DeclarationStmt struct {
	tok  lexer.Token
	name IdentExpr
	expr Expr
}

// Start returns a location that this node can be considered to start at
func (ds DeclarationStmt) Start() lexer.Loc { return ds.tok.Loc }
func (ds DeclarationStmt) String() string   { return fmt.Sprintf("(let %s %s)", ds.name, ds.expr) }
func (ds DeclarationStmt) isNode()          {}
func (ds DeclarationStmt) isStmt()          {}

// ReturnStmt describes a return keyword and an optional returned expression.
type ReturnStmt struct {
	tok  lexer.Token
	expr Expr
}

// Start returns a location that this node can be considered to start at
func (rs ReturnStmt) Start() lexer.Loc { return rs.tok.Loc }

func (rs ReturnStmt) String() string {
	if rs.expr != nil {
		return fmt.Sprintf("(return %s)", rs.expr)
	}

	return "(return)"
}

func (rs ReturnStmt) isNode() {}
func (rs ReturnStmt) isStmt() {}

// ExprStmt describes certain expressions that can be used in the place of statements
type ExprStmt struct {
	expr Expr
}

// Start returns a location that this node can be considered to start at
func (es ExprStmt) Start() lexer.Loc { return es.expr.Start() }
func (es ExprStmt) String() string   { return es.expr.String() }
func (es ExprStmt) isNode()          {}
func (es ExprStmt) isStmt()          {}

// TypeSig describes a syntax type annotation
type TypeSig interface {
	Start() lexer.Loc
	String() string
	isNode()
	isType()
}

// TypeTuple describes a set of 0 or more types wrapped in parentheses
type TypeTuple struct {
	tok   lexer.Token
	elems []TypeSig
}

// Start returns a location that this node can be considered to start at
func (tt TypeTuple) Start() lexer.Loc { return tt.tok.Loc }

func (tt TypeTuple) String() string {
	out := "("
	for i, elem := range tt.elems {
		if i > 0 {
			out += " "
		}
		out += elem.String()
	}
	out += ")"
	return out
}

func (tt TypeTuple) isNode() {}
func (tt TypeTuple) isType() {}

// TypeFunction describes a function type annotation
type TypeFunction struct {
	params TypeTuple
	ret    TypeSig
}

// Start returns a location that this node can be considered to start at
func (tf TypeFunction) Start() lexer.Loc { return tf.params.Start() }

func (tf TypeFunction) String() string {
	out := tf.params.String()
	out += " => "
	out += tf.ret.String()
	return out
}

func (tf TypeFunction) isNode() {}
func (tf TypeFunction) isType() {}

// TypeIdent describes a named reference to a type
type TypeIdent struct {
	tok  lexer.Token
	name string
}

// Start returns a location that this node can be considered to start at
func (ti TypeIdent) Start() lexer.Loc { return ti.tok.Loc }
func (ti TypeIdent) String() string   { return ti.name }
func (ti TypeIdent) isNode()          {}
func (ti TypeIdent) isType()          {}

// TypeList describes a list type
type TypeList struct {
	tok   lexer.Token
	child TypeSig
}

// Start returns a location that this node can be considered to start at
func (tl TypeList) Start() lexer.Loc { return tl.tok.Loc }
func (tl TypeList) String() string   { return fmt.Sprintf("[%s]", tl.child) }
func (tl TypeList) isNode()          {}
func (tl TypeList) isType()          {}

// TypeOptional describes a list type
type TypeOptional struct {
	tok   lexer.Token
	child TypeSig
}

// Start returns a location that this node can be considered to start at
func (to TypeOptional) Start() lexer.Loc { return to.child.Start() }
func (to TypeOptional) String() string   { return fmt.Sprintf("%s?", to.child) }
func (to TypeOptional) isNode()          {}
func (to TypeOptional) isType()          {}

// Expr describes all constructs that resolve to a value
type Expr interface {
	Start() lexer.Loc
	String() string
	isNode()
	isExpr()
}

// FunctionExpr describes a function's entire type signature and body
type FunctionExpr struct {
	tok    lexer.Token
	params []FunctionParam
	ret    TypeSig
	block  StmtBlock
}

// Start returns a location that this node can be considered to start at
func (fe FunctionExpr) Start() lexer.Loc { return fe.tok.Loc }

func (fe FunctionExpr) String() string {
	out := "(fn ("
	for i, param := range fe.params {
		if i > 0 {
			out += " "
		}
		out += param.String()
	}
	out += ")"
	if fe.ret != nil {
		out += fmt.Sprintf(":%s", fe.ret)
	}
	out += fmt.Sprintf(" %s)", fe.block)
	return out
}

func (fe FunctionExpr) isExpr() {}
func (fe FunctionExpr) isNode() {}

// FunctionParam describes a single function argument's name and type signature
type FunctionParam struct {
	name IdentExpr
	sig  TypeSig
}

// Start returns a location that this node can be considered to start at
func (fp FunctionParam) Start() lexer.Loc { return fp.name.Start() }

func (fp FunctionParam) String() string {
	if fp.sig != nil {
		return fmt.Sprintf("%s:%s", fp.name, fp.sig)
	}

	return fp.name.String()
}

func (fp FunctionParam) isNode() {}

// DispatchExpr describes a function call including the callee and any arguments
type DispatchExpr struct {
	callee Expr
	args   []Expr
}

// Start returns a location that this node can be considered to start at
func (de DispatchExpr) Start() lexer.Loc { return de.callee.Start() }

func (de DispatchExpr) String() string {
	out := "("
	out += de.callee.String()
	out += " ("
	for i, arg := range de.args {
		if i > 0 {
			out += " "
		}
		out += arg.String()
	}
	out += "))"
	return out
}

func (de DispatchExpr) isNode() {}
func (de DispatchExpr) isExpr() {}

// AssignExpr describes the binding of a value to an assignable expression
type AssignExpr struct {
	tok   lexer.Token
	left  Expr
	right Expr
}

// Start returns a location that this node can be considered to start at
func (ae AssignExpr) Start() lexer.Loc { return ae.left.Start() }
func (ae AssignExpr) String() string   { return fmt.Sprintf("(= %s %s)", ae.left, ae.right) }
func (ae AssignExpr) isNode()          {}
func (ae AssignExpr) isExpr()          {}

// BinaryExpr describes any two expressions associated by an operator
type BinaryExpr struct {
	oper  string
	tok   lexer.Token
	left  Expr
	right Expr
}

// Start returns a location that this node can be considered to start at
func (be BinaryExpr) Start() lexer.Loc { return be.left.Start() }
func (be BinaryExpr) String() string   { return fmt.Sprintf("(%s %s %s)", be.oper, be.left, be.right) }
func (be BinaryExpr) isNode()          {}
func (be BinaryExpr) isExpr()          {}

// UnaryExpr describes any single expression associated to an operator
type UnaryExpr struct {
	oper string
	tok  lexer.Token
	expr Expr
}

// Start returns a location that this node can be considered to start at
func (ue UnaryExpr) Start() lexer.Loc { return lexer.SmallerLoc(ue.tok.Loc, ue.expr.Start()) }
func (ue UnaryExpr) String() string   { return fmt.Sprintf("(%s %s)", ue.oper, ue.expr) }
func (ue UnaryExpr) isNode()          {}
func (ue UnaryExpr) isExpr()          {}

// IdentExpr describes an identifier
type IdentExpr struct {
	tok  lexer.Token
	name string
}

// Start returns a location that this node can be considered to start at
func (ie IdentExpr) Start() lexer.Loc { return ie.tok.Loc }
func (ie IdentExpr) String() string   { return ie.name }
func (ie IdentExpr) isNode()          {}
func (ie IdentExpr) isExpr()          {}

// StringExpr describes a string literal
type StringExpr struct {
	tok lexer.Token
	val string
}

// Start returns a location that this node can be considered to start at
func (se StringExpr) Start() lexer.Loc { return se.tok.Loc }
func (se StringExpr) String() string   { return fmt.Sprintf("\"%s\"", se.val) }
func (se StringExpr) isNode()          {}
func (se StringExpr) isExpr()          {}

// NumberExpr describes a string literal
type NumberExpr struct {
	tok lexer.Token
	val int
}

// Start returns a location that this node can be considered to start at
func (ne NumberExpr) Start() lexer.Loc { return ne.tok.Loc }
func (ne NumberExpr) String() string   { return strconv.Itoa(ne.val) }
func (ne NumberExpr) isNode()          {}
func (ne NumberExpr) isExpr()          {}

func indentBlock(indent string, source string) string {
	lines := strings.Split(source, "\n")
	for i, line := range lines {
		lines[i] = indent + line
	}

	return strings.Join(lines, "\n")
}
<|endoftext|>


<commit_before>package parser

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"cloud.google.com/go/bigquery"
	"github.com/m-lab/etl/web100"
)

type NDTParser struct {
	Parser
	tmpDir string
}

func (n *NDTParser) Parse(meta map[string]bigquery.Value, fn string, table string, rawSnapLog []byte) (interface{}, error) {
	// TODO(prod): do not write to a temporary file; operate on byte array directly.
	// Write rawSnapLog to /mnt/tmpfs.
	tmpFile := fmt.Sprintf("%s/%s", n.tmpDir, fn)
	err := ioutil.WriteFile(tmpFile, rawSnapLog, 0644)
	if err != nil {
		return nil, err
	}
	// TODO(dev): log possible remove errors.
	defer os.Remove(tmpFile)

	// TODO(dev): only do this once.
	// Parse the tcp-kis.txt web100 variable definition file.
	data, err := web100.Asset("tcp-kis.txt")
	if err != nil {
		// Asset missing from build.
		return nil, err
	}
	b := bytes.NewBuffer(data)
	legacyNames, err := web100.ParseWeb100Definitions(b)
	if err != nil {
		return nil, err
	}

	// Open the file we created above.
	w, err := web100.Open(tmpFile, legacyNames)
	if err != nil {
		return nil, err
	}
	defer w.Close()

	// Find the last web100 snapshot.
	for {
		err = w.Next()
		if err != nil {
			break
		}
	}
	// We expect EOF.
	if err != io.EOF {
		return nil, err
	}

	// Extract the values from the last snapshot.
	results, err := w.Values()
	if err != nil {
		return nil, err
	}
	return results, nil
}
<commit_msg>tweak parameter name<commit_after>package parser

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"cloud.google.com/go/bigquery"
	"github.com/m-lab/etl/web100"
)

type NDTParser struct {
	Parser
	tmpDir string
}

func (n *NDTParser) Parse(meta map[string]bigquery.Value, testName string, table string, rawSnapLog []byte) (interface{}, error) {
	// TODO(prod): do not write to a temporary file; operate on byte array directly.
	// Write rawSnapLog to /mnt/tmpfs.
	tmpFile := fmt.Sprintf("%s/%s", n.tmpDir, testName)
	err := ioutil.WriteFile(tmpFile, rawSnapLog, 0644)
	if err != nil {
		return nil, err
	}
	// TODO(dev): log possible remove errors.
	defer os.Remove(tmpFile)

	// TODO(dev): only do this once.
	// Parse the tcp-kis.txt web100 variable definition file.
	data, err := web100.Asset("tcp-kis.txt")
	if err != nil {
		// Asset missing from build.
		return nil, err
	}
	b := bytes.NewBuffer(data)
	legacyNames, err := web100.ParseWeb100Definitions(b)
	if err != nil {
		return nil, err
	}

	// Open the file we created above.
	w, err := web100.Open(tmpFile, legacyNames)
	if err != nil {
		return nil, err
	}
	defer w.Close()

	// Find the last web100 snapshot.
	for {
		err = w.Next()
		if err != nil {
			break
		}
	}
	// We expect EOF.
	if err != io.EOF {
		return nil, err
	}

	// Extract the values from the last snapshot.
	results, err := w.Values()
	if err != nil {
		return nil, err
	}
	return results, nil
}
<|endoftext|>