<commit_before>// Copyright 2018 The Go Cloud Development Kit Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub_test

import (
	"context"
	"fmt"
	"log"
	"sort"
	"sync"
	"time"

	"gocloud.dev/pubsub"
	"gocloud.dev/pubsub/mempubsub"
)

func Example_sendReceive() {
	// Open a topic and corresponding subscription.
	ctx := context.Background()
	t := mempubsub.NewTopic()
	defer t.Shutdown(ctx)
	s := mempubsub.NewSubscription(t, time.Second)
	defer s.Shutdown(ctx)

	// Send a message to the topic.
	if err := t.Send(ctx, &pubsub.Message{Body: []byte("Hello, world!")}); err != nil {
		log.Fatal(err)
	}

	// Receive a message from the subscription.
	m, err := s.Receive(ctx)
	if err != nil {
		log.Fatal(err)
	}

	// Print out the received message.
	fmt.Printf("%s\n", m.Body)

	// Acknowledge the message.
	m.Ack()

	// Output:
	// Hello, world!
}

func Example_sendReceiveMultipleMessages() {
	// Open a topic and corresponding subscription.
	ctx := context.Background()
	t := mempubsub.NewTopic()
	defer t.Shutdown(ctx)
	s := mempubsub.NewSubscription(t, time.Second)
	defer s.Shutdown(ctx)

	// Send messages to the topic.
	ms := []*pubsub.Message{
		{Body: []byte("a")},
		{Body: []byte("b")},
		{Body: []byte("c")},
	}
	for _, m := range ms {
		if err := t.Send(ctx, m); err != nil {
			log.Fatal(err)
		}
	}

	// Receive messages from the subscription.
	ms2 := []string{}
	for i := 0; i < len(ms); i++ {
		m2, err := s.Receive(ctx)
		if err != nil {
			log.Fatal(err)
		}
		ms2 = append(ms2, string(m2.Body))
		m2.Ack()
	}

	// The messages may be received in a different order than they were
	// sent.
	sort.Strings(ms2)

	// Print out and acknowledge the received messages.
	for _, m2 := range ms2 {
		fmt.Println(m2)
	}

	// Output:
	// a
	// b
	// c
}

func Example_receiveWithInvertedWorkerPool() {
	// Open a topic and corresponding subscription.
	ctx := context.Background()
	t := mempubsub.NewTopic()
	defer t.Shutdown(ctx)
	s := mempubsub.NewSubscription(t, time.Second)
	defer s.Shutdown(ctx)

	// Send a bunch of messages to the topic.
	const nMessages = 100
	for n := 0; n < nMessages; n++ {
		m := &pubsub.Message{
			Body: []byte(fmt.Sprintf("message %d", n)),
		}
		if err := t.Send(ctx, m); err != nil {
			log.Fatal(err)
		}
	}

	// In order to make our test exit, we keep track of how many messages were
	// processed with wg, and cancel the receiveCtx when we've processed them all.
	// A more realistic application would not need this WaitGroup.
	var wg sync.WaitGroup
	wg.Add(nMessages)
	receiveCtx, cancel := context.WithCancel(ctx)
	go func() {
		wg.Wait()
		cancel()
	}()

	// Process messages using an inverted worker pool, as described here:
	// https://www.youtube.com/watch?v=5zXAHh5tJqQ&t=26m58s
	// It uses a buffered channel, sem, as a semaphore to manage the maximum
	// number of workers.
	const poolSize = 10
	sem := make(chan struct{}, poolSize)
	for {
		// Read a message. Receive will block until a message is available.
		msg, err := s.Receive(receiveCtx)
		if err != nil {
			// An error from Receive is fatal; Receive will never succeed again
			// so the application should exit.
			// In this example, we expect to get a error here when we've read all the
			// messages and receiveCtx is canceled.
			break
		}

		// Write a token to the semaphore; if there are already poolSize workers
		// active, this will block until one of them completes.
		sem <- struct{}{}
		// Process the message. For many applications, this can be expensive, so
		// we do it in a goroutine, allowing this loop to continue and Receive more
		// messages.
		go func() {
			// Record that we've processed this message, and Ack it.
			msg.Ack()
			wg.Done()
			// Read a token from the semaphore before exiting this goroutine, freeing
			// up the slot for another goroutine.
			<-sem
		}()
	}

	// Wait for all workers to finish.
	for n := poolSize; n > 0; n-- {
		sem <- struct{}{}
	}
	fmt.Printf("Read %d messages", nMessages)

	// Output:
	// Read 100 messages
}

func Example_receiveWithTraditionalWorkerPool() {
	// Open a topic and corresponding subscription.
	ctx := context.Background()
	t := mempubsub.NewTopic()
	defer t.Shutdown(ctx)
	s := mempubsub.NewSubscription(t, time.Second)
	defer s.Shutdown(ctx)

	// Send a bunch of messages to the topic.
	const nMessages = 100
	for n := 0; n < nMessages; n++ {
		m := &pubsub.Message{
			Body: []byte(fmt.Sprintf("message %d", n)),
		}
		if err := t.Send(ctx, m); err != nil {
			log.Fatal(err)
		}
	}

	// In order to make our test exit, we keep track of how many messages were
	// processed with wg, and cancel the receiveCtx when we've processed them all.
	// A more realistic application would not need this WaitGroup.
	var wg sync.WaitGroup
	wg.Add(nMessages)
	receiveCtx, cancel := context.WithCancel(ctx)
	go func() {
		wg.Wait()
		cancel()
	}()

	// Process messages using a traditional worker pool. Consider using an
	// inverted pool instead (see the other example).
	const poolSize = 10
	var workerWg sync.WaitGroup
	for n := 0; n < poolSize; n++ {
		workerWg.Add(1)
		go func() {
			for {
				// Read a message. Receive will block until a message is available.
				// It's fine to call Receive from many goroutines.
				msg, err := s.Receive(receiveCtx)
				if err != nil {
					// An error from Receive is fatal; Receive will never succeed again
					// so the application should exit.
					// In this example, we expect to get a error here when we've read all
					// the messages and receiveCtx is canceled.
					workerWg.Done()
					return
				}

				// Process the message and Ack it.
				msg.Ack()
				wg.Done()
			}
		}()
	}

	// Wait for all workers to finish.
	workerWg.Wait()
	fmt.Printf("Read %d messages", nMessages)

	// Output:
	// Read 100 messages
}
<commit_msg>pubsub: fix inaccuracy in example comment (#1537)<commit_after>// Copyright 2018 The Go Cloud Development Kit Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub_test

import (
	"context"
	"fmt"
	"log"
	"sort"
	"sync"
	"time"

	"gocloud.dev/pubsub"
	"gocloud.dev/pubsub/mempubsub"
)

func Example_sendReceive() {
	// Open a topic and corresponding subscription.
	ctx := context.Background()
	t := mempubsub.NewTopic()
	defer t.Shutdown(ctx)
	s := mempubsub.NewSubscription(t, time.Second)
	defer s.Shutdown(ctx)

	// Send a message to the topic.
	if err := t.Send(ctx, &pubsub.Message{Body: []byte("Hello, world!")}); err != nil {
		log.Fatal(err)
	}

	// Receive a message from the subscription.
	m, err := s.Receive(ctx)
	if err != nil {
		log.Fatal(err)
	}

	// Print out the received message.
	fmt.Printf("%s\n", m.Body)

	// Acknowledge the message.
	m.Ack()

	// Output:
	// Hello, world!
}

func Example_sendReceiveMultipleMessages() {
	// Open a topic and corresponding subscription.
	ctx := context.Background()
	t := mempubsub.NewTopic()
	defer t.Shutdown(ctx)
	s := mempubsub.NewSubscription(t, time.Second)
	defer s.Shutdown(ctx)

	// Send messages to the topic.
	ms := []*pubsub.Message{
		{Body: []byte("a")},
		{Body: []byte("b")},
		{Body: []byte("c")},
	}
	for _, m := range ms {
		if err := t.Send(ctx, m); err != nil {
			log.Fatal(err)
		}
	}

	// Receive and acknowledge messages from the subscription.
	ms2 := []string{}
	for i := 0; i < len(ms); i++ {
		m2, err := s.Receive(ctx)
		if err != nil {
			log.Fatal(err)
		}
		ms2 = append(ms2, string(m2.Body))
		m2.Ack()
	}

	// The messages may be received in a different order than they were
	// sent.
	sort.Strings(ms2)

	// Print out the received messages.
	for _, m2 := range ms2 {
		fmt.Println(m2)
	}

	// Output:
	// a
	// b
	// c
}

func Example_receiveWithInvertedWorkerPool() {
	// Open a topic and corresponding subscription.
	ctx := context.Background()
	t := mempubsub.NewTopic()
	defer t.Shutdown(ctx)
	s := mempubsub.NewSubscription(t, time.Second)
	defer s.Shutdown(ctx)

	// Send a bunch of messages to the topic.
	const nMessages = 100
	for n := 0; n < nMessages; n++ {
		m := &pubsub.Message{
			Body: []byte(fmt.Sprintf("message %d", n)),
		}
		if err := t.Send(ctx, m); err != nil {
			log.Fatal(err)
		}
	}

	// In order to make our test exit, we keep track of how many messages were
	// processed with wg, and cancel the receiveCtx when we've processed them all.
	// A more realistic application would not need this WaitGroup.
	var wg sync.WaitGroup
	wg.Add(nMessages)
	receiveCtx, cancel := context.WithCancel(ctx)
	go func() {
		wg.Wait()
		cancel()
	}()

	// Process messages using an inverted worker pool, as described here:
	// https://www.youtube.com/watch?v=5zXAHh5tJqQ&t=26m58s
	// It uses a buffered channel, sem, as a semaphore to manage the maximum
	// number of workers.
	const poolSize = 10
	sem := make(chan struct{}, poolSize)
	for {
		// Read a message. Receive will block until a message is available.
		msg, err := s.Receive(receiveCtx)
		if err != nil {
			// An error from Receive is fatal; Receive will never succeed again
			// so the application should exit.
			// In this example, we expect to get a error here when we've read all the
			// messages and receiveCtx is canceled.
			break
		}

		// Write a token to the semaphore; if there are already poolSize workers
		// active, this will block until one of them completes.
		sem <- struct{}{}
		// Process the message. For many applications, this can be expensive, so
		// we do it in a goroutine, allowing this loop to continue and Receive more
		// messages.
		go func() {
			// Record that we've processed this message, and Ack it.
			msg.Ack()
			wg.Done()
			// Read a token from the semaphore before exiting this goroutine, freeing
			// up the slot for another goroutine.
			<-sem
		}()
	}

	// Wait for all workers to finish.
	for n := poolSize; n > 0; n-- {
		sem <- struct{}{}
	}
	fmt.Printf("Read %d messages", nMessages)

	// Output:
	// Read 100 messages
}

func Example_receiveWithTraditionalWorkerPool() {
	// Open a topic and corresponding subscription.
	ctx := context.Background()
	t := mempubsub.NewTopic()
	defer t.Shutdown(ctx)
	s := mempubsub.NewSubscription(t, time.Second)
	defer s.Shutdown(ctx)

	// Send a bunch of messages to the topic.
	const nMessages = 100
	for n := 0; n < nMessages; n++ {
		m := &pubsub.Message{
			Body: []byte(fmt.Sprintf("message %d", n)),
		}
		if err := t.Send(ctx, m); err != nil {
			log.Fatal(err)
		}
	}

	// In order to make our test exit, we keep track of how many messages were
	// processed with wg, and cancel the receiveCtx when we've processed them all.
	// A more realistic application would not need this WaitGroup.
	var wg sync.WaitGroup
	wg.Add(nMessages)
	receiveCtx, cancel := context.WithCancel(ctx)
	go func() {
		wg.Wait()
		cancel()
	}()

	// Process messages using a traditional worker pool. Consider using an
	// inverted pool instead (see the other example).
	const poolSize = 10
	var workerWg sync.WaitGroup
	for n := 0; n < poolSize; n++ {
		workerWg.Add(1)
		go func() {
			for {
				// Read a message. Receive will block until a message is available.
				// It's fine to call Receive from many goroutines.
				msg, err := s.Receive(receiveCtx)
				if err != nil {
					// An error from Receive is fatal; Receive will never succeed again
					// so the application should exit.
					// In this example, we expect to get a error here when we've read all
					// the messages and receiveCtx is canceled.
					workerWg.Done()
					return
				}

				// Process the message and Ack it.
				msg.Ack()
				wg.Done()
			}
		}()
	}

	// Wait for all workers to finish.
	workerWg.Wait()
	fmt.Printf("Read %d messages", nMessages)

	// Output:
	// Read 100 messages
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package repr_test

import (
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/fs"
	"github.com/jacobsa/comeback/repr"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"os"
	"testing"
	"time"
)

func TestRoundtripTest(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func makeLegalEntry() *fs.DirectoryEntry {
	entry := new(fs.DirectoryEntry)
	entry.Type = fs.TypeDirectory
	return entry
}

type RoundtripTest struct {
}

func init() { RegisterTestSuite(&RoundtripTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *RoundtripTest) NoEntries() {
	// Input
	in := []*fs.DirectoryEntry{}

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	ExpectThat(out, ElementsAre())
}

func (t *RoundtripTest) UnknownType() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[1].Type = 17

	// Marshal
	_, err := repr.Marshal(in)

	ExpectThat(err, Error(HasSubstr("EntryType")))
	ExpectThat(err, Error(HasSubstr("17")))
}

func (t *RoundtripTest) PreservesTypes() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Type = fs.TypeFile
	in[1].Type = fs.TypeDirectory
	in[2].Type = fs.TypeSymlink
	in[3].Type = fs.TypeBlockDevice
	in[4].Type = fs.TypeCharDevice
	in[5].Type = fs.TypeNamedPipe

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertEq(6, len(out))

	ExpectEq(in[0].Type, out[0].Type)
	ExpectEq(in[1].Type, out[1].Type)
	ExpectEq(in[2].Type, out[2].Type)
	ExpectEq(in[3].Type, out[3].Type)
	ExpectEq(in[4].Type, out[4].Type)
	ExpectEq(in[5].Type, out[5].Type)
}

func (t *RoundtripTest) PreservesNames() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Name = "taco"
	in[1].Name = "burrito"

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Name, out[0].Name)
	ExpectEq(in[1].Name, out[1].Name)
}

func (t *RoundtripTest) PreservesPermissions() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Permissions = 0644 | os.ModeSticky | os.ModeSetuid
	in[1].Permissions = 0755 | os.ModeSetgid

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Permissions, out[0].Permissions)
	ExpectEq(in[1].Permissions, out[1].Permissions)
}

func (t *RoundtripTest) PreservesUids() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Uid = 17
	in[1].Uid = 19

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Uid, out[0].Uid)
	ExpectEq(in[1].Uid, out[1].Uid)
}

func (t *RoundtripTest) PreservesUsernames() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	s := "taco"
	in[0].Username = &s

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	AssertNe(nil, out[0].Username)
	ExpectEq(*in[0].Username, *out[0].Username)
	ExpectEq(nil, out[1].Username)
}

func (t *RoundtripTest) PreservesGids() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Gid = 17
	in[1].Gid = 19

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Gid, out[0].Gid)
	ExpectEq(in[1].Gid, out[1].Gid)
}

func (t *RoundtripTest) PreservesGroupnames() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	s := "taco"
	in[0].Groupname = &s

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	AssertNe(nil, out[0].Groupname)
	ExpectEq(*in[0].Groupname, *out[0].Groupname)
	ExpectEq(nil, out[1].Groupname)
}

func (t *RoundtripTest) PreservesModTimes() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].MTime = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
	in[1].MTime = time.Date(1985, time.March, 18, 15, 33, 0, 0, time.UTC)

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectTrue(in[0].MTime.Equal(out[0].MTime), "%v", out[0].MTime)
	ExpectTrue(in[1].MTime.Equal(out[1].MTime), "%v", out[1].MTime)
}

func (t *RoundtripTest) PreservesScores() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	score00 := blob.ComputeScore([]byte("taco"))
	score01 := blob.ComputeScore([]byte("burrito"))
	score10 := blob.ComputeScore([]byte("enchilada"))

	in[0].Scores = []blob.Score{score00, score01}
	in[1].Scores = []blob.Score{score10}

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	AssertThat(out[0].Scores, ElementsAre(Any(), Any()))
	ExpectThat(out[0].Scores[0].Sha1Hash(), DeepEquals(score00.Sha1Hash()))
	ExpectThat(out[0].Scores[1].Sha1Hash(), DeepEquals(score01.Sha1Hash()))

	AssertThat(out[1].Scores, ElementsAre(Any()))
	ExpectThat(out[1].Scores[0].Sha1Hash(), DeepEquals(score10.Sha1Hash()))
}

func (t *RoundtripTest) PreservesSymlinkTargets() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Type = fs.TypeSymlink
	in[1].Type = fs.TypeSymlink

	in[0].Target = "taco"
	in[1].Target = "burrito"

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Target, out[0].Target)
	ExpectEq(in[1].Target, out[1].Target)
}

func (t *RoundtripTest) PreservesDeviceNumbers() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Type = fs.TypeCharDevice
	in[1].Type = fs.TypeBlockDevice

	in[0].Device = 17
	in[1].Device = 19

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Device, out[0].Device)
	ExpectEq(in[1].Device, out[1].Device)
}
<commit_msg>RoundtripTest.PreservesHardLinkTargets<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package repr_test

import (
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/fs"
	"github.com/jacobsa/comeback/repr"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"os"
	"testing"
	"time"
)

func TestRoundtripTest(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func makeLegalEntry() *fs.DirectoryEntry {
	entry := new(fs.DirectoryEntry)
	entry.Type = fs.TypeDirectory
	return entry
}

type RoundtripTest struct {
}

func init() { RegisterTestSuite(&RoundtripTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *RoundtripTest) NoEntries() {
	// Input
	in := []*fs.DirectoryEntry{}

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	ExpectThat(out, ElementsAre())
}

func (t *RoundtripTest) UnknownType() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[1].Type = 17

	// Marshal
	_, err := repr.Marshal(in)

	ExpectThat(err, Error(HasSubstr("EntryType")))
	ExpectThat(err, Error(HasSubstr("17")))
}

func (t *RoundtripTest) PreservesTypes() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Type = fs.TypeFile
	in[1].Type = fs.TypeDirectory
	in[2].Type = fs.TypeSymlink
	in[3].Type = fs.TypeBlockDevice
	in[4].Type = fs.TypeCharDevice
	in[5].Type = fs.TypeNamedPipe

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertEq(6, len(out))

	ExpectEq(in[0].Type, out[0].Type)
	ExpectEq(in[1].Type, out[1].Type)
	ExpectEq(in[2].Type, out[2].Type)
	ExpectEq(in[3].Type, out[3].Type)
	ExpectEq(in[4].Type, out[4].Type)
	ExpectEq(in[5].Type, out[5].Type)
}

func (t *RoundtripTest) PreservesNames() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Name = "taco"
	in[1].Name = "burrito"

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Name, out[0].Name)
	ExpectEq(in[1].Name, out[1].Name)
}

func (t *RoundtripTest) PreservesPermissions() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Permissions = 0644 | os.ModeSticky | os.ModeSetuid
	in[1].Permissions = 0755 | os.ModeSetgid

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Permissions, out[0].Permissions)
	ExpectEq(in[1].Permissions, out[1].Permissions)
}

func (t *RoundtripTest) PreservesUids() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Uid = 17
	in[1].Uid = 19

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Uid, out[0].Uid)
	ExpectEq(in[1].Uid, out[1].Uid)
}

func (t *RoundtripTest) PreservesUsernames() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	s := "taco"
	in[0].Username = &s

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	AssertNe(nil, out[0].Username)
	ExpectEq(*in[0].Username, *out[0].Username)
	ExpectEq(nil, out[1].Username)
}

func (t *RoundtripTest) PreservesGids() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Gid = 17
	in[1].Gid = 19

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Gid, out[0].Gid)
	ExpectEq(in[1].Gid, out[1].Gid)
}

func (t *RoundtripTest) PreservesGroupnames() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	s := "taco"
	in[0].Groupname = &s

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	AssertNe(nil, out[0].Groupname)
	ExpectEq(*in[0].Groupname, *out[0].Groupname)
	ExpectEq(nil, out[1].Groupname)
}

func (t *RoundtripTest) PreservesModTimes() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].MTime = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
	in[1].MTime = time.Date(1985, time.March, 18, 15, 33, 0, 0, time.UTC)

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectTrue(in[0].MTime.Equal(out[0].MTime), "%v", out[0].MTime)
	ExpectTrue(in[1].MTime.Equal(out[1].MTime), "%v", out[1].MTime)
}

func (t *RoundtripTest) PreservesScores() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	score00 := blob.ComputeScore([]byte("taco"))
	score01 := blob.ComputeScore([]byte("burrito"))
	score10 := blob.ComputeScore([]byte("enchilada"))

	in[0].Scores = []blob.Score{score00, score01}
	in[1].Scores = []blob.Score{score10}

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	AssertThat(out[0].Scores, ElementsAre(Any(), Any()))
	ExpectThat(out[0].Scores[0].Sha1Hash(), DeepEquals(score00.Sha1Hash()))
	ExpectThat(out[0].Scores[1].Sha1Hash(), DeepEquals(score01.Sha1Hash()))

	AssertThat(out[1].Scores, ElementsAre(Any()))
	ExpectThat(out[1].Scores[0].Sha1Hash(), DeepEquals(score10.Sha1Hash()))
}

func (t *RoundtripTest) PreservesHardLinkTargets() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	s := "taco"
	in[0].HardLinkTarget = &s

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	AssertNe(nil, out[0].HardLinkTarget)
	ExpectEq(*in[0].HardLinkTarget, *out[0].HardLinkTarget)
	ExpectEq(nil, out[1].HardLinkTarget)
}

func (t *RoundtripTest) PreservesSymlinkTargets() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Type = fs.TypeSymlink
	in[1].Type = fs.TypeSymlink

	in[0].Target = "taco"
	in[1].Target = "burrito"

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Target, out[0].Target)
	ExpectEq(in[1].Target, out[1].Target)
}

func (t *RoundtripTest) PreservesDeviceNumbers() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Type = fs.TypeCharDevice
	in[1].Type = fs.TypeBlockDevice

	in[0].Device = 17
	in[1].Device = 19

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Device, out[0].Device)
	ExpectEq(in[1].Device, out[1].Device)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	_ "github.com/mattn/go-sqlite3"
)

type SqliteItemRepository struct {
	dbPath string
}

func NewSqliteItemRepository(dbPath string) (this *SqliteItemRepository) {
	this = new(SqliteItemRepository)
	this.dbPath = dbPath
	return
}

func (this *SqliteItemRepository) Items() (items []*Item, err error) {
	db, err := sql.Open("sqlite3", this.dbPath)
	if err != nil {
		return
	}
	defer db.Close()

	rows, err := db.Query("select Z_PK, ZTITLE, ZBODY, ZRAW_BODY from ZITEM order by ZUPDATED_AT desc")

	if err != nil {
		return
	}

	defer rows.Close()

	for rows.Next() {
		var id int
		var title string
		var html sql.NullString
		var markdown sql.NullString
		if err := rows.Scan(&id, &title, &html, &markdown); err != nil {
			return items, err
		}

		items = append(items, NewItem(id, title, html.String, markdown.String))
	}

	return
}

func (this *SqliteItemRepository) ItemOfId(id int) (item *Item, err error) {
	db, err := sql.Open("sqlite3", this.dbPath)
	if err != nil {
		return
	}
	defer db.Close()

	stmt, err := db.Prepare("select ZTITLE, ZBODY, ZRAW_BODY from ZITEM where Z_PK = ?")

	if err != nil {
		return
	}

	defer stmt.Close()

	var title string
	var html sql.NullString
	var markdown sql.NullString

	err = stmt.QueryRow(id).Scan(&title, &html, &markdown)

	if err != nil {
		return
	}

	item = NewItem(id, title, html.String, markdown.String)

	return
}
<commit_msg>削除済みアイテムを非表示<commit_after>package main

import (
	"database/sql"
	_ "github.com/mattn/go-sqlite3"
)

type SqliteItemRepository struct {
	dbPath string
}

func NewSqliteItemRepository(dbPath string) (this *SqliteItemRepository) {
	this = new(SqliteItemRepository)
	this.dbPath = dbPath
	return
}

func (this *SqliteItemRepository) Items() (items []*Item, err error) {
	db, err := sql.Open("sqlite3", this.dbPath)
	if err != nil {
		return
	}
	defer db.Close()

	rows, err := db.Query("select Z_PK, ZTITLE, ZBODY, ZRAW_BODY from ZITEM where ZIN_TRASH is null order by ZUPDATED_AT desc")

	if err != nil {
		return
	}

	defer rows.Close()

	for rows.Next() {
		var id int
		var title string
		var html sql.NullString
		var markdown sql.NullString
		if err := rows.Scan(&id, &title, &html, &markdown); err != nil {
			return items, err
		}

		items = append(items, NewItem(id, title, html.String, markdown.String))
	}

	return
}

func (this *SqliteItemRepository) ItemOfId(id int) (item *Item, err error) {
	db, err := sql.Open("sqlite3", this.dbPath)
	if err != nil {
		return
	}
	defer db.Close()

	stmt, err := db.Prepare("select ZTITLE, ZBODY, ZRAW_BODY from ZITEM where Z_PK = ?")

	if err != nil {
		return
	}

	defer stmt.Close()

	var title string
	var html sql.NullString
	var markdown sql.NullString

	err = stmt.QueryRow(id).Scan(&title, &html, &markdown)

	if err != nil {
		return
	}

	item = NewItem(id, title, html.String, markdown.String)

	return
}
<|endoftext|>


<commit_before>package handlers

import (
	"errors"
	"io"
	"net/http"

	ctxu "github.com/docker/distribution/context"
	"github.com/docker/distribution/registry/api/errcode"
)

// closeResources closes all the provided resources after running the target
// handler.
func closeResources(handler http.Handler, closers ...io.Closer) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		for _, closer := range closers {
			defer closer.Close()
		}
		handler.ServeHTTP(w, r)
	})
}

// copyFullPayload copies the payload of a HTTP request to destWriter. If it
// receives less content than expected, and the client disconnected during the
// upload, it avoids sending a 400 error to keep the logs cleaner.
func copyFullPayload(responseWriter http.ResponseWriter, r *http.Request, destWriter io.Writer, context ctxu.Context, action string, errSlice *errcode.Errors) error {
	// Get a channel that tells us if the client disconnects
	var clientClosed <-chan bool
	if notifier, ok := responseWriter.(http.CloseNotifier); ok {
		clientClosed = notifier.CloseNotify()
	} else {
		panic("the ResponseWriter does not implement CloseNotifier")
	}

	// Read in the data, if any.
	copied, err := io.Copy(destWriter, r.Body)
	if clientClosed != nil && (err != nil || (r.ContentLength > 0 && copied < r.ContentLength)) {
		// Didn't recieve as much content as expected. Did the client
		// disconnect during the request? If so, avoid returning a 400
		// error to keep the logs cleaner.
		select {
		case <-clientClosed:
			// Set the response code to "499 Client Closed Request"
			// Even though the connection has already been closed,
			// this causes the logger to pick up a 499 error
			// instead of showing 0 for the HTTP status.
			responseWriter.WriteHeader(499)

			ctxu.GetLogger(context).Error("client disconnected during " + action)
			return errors.New("client disconnected")
		default:
		}
	}

	if err != nil {
		ctxu.GetLogger(context).Errorf("unknown error reading request payload: %v", err)
		*errSlice = append(*errSlice, errcode.ErrorCodeUnknown.WithDetail(err))
		return err
	}

	return nil
}
<commit_msg>Don't panic when a http.ResponseWriter does not implement CloseNotifier<commit_after>package handlers

import (
	"errors"
	"io"
	"net/http"

	ctxu "github.com/docker/distribution/context"
	"github.com/docker/distribution/registry/api/errcode"
)

// closeResources closes all the provided resources after running the target
// handler.
func closeResources(handler http.Handler, closers ...io.Closer) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		for _, closer := range closers {
			defer closer.Close()
		}
		handler.ServeHTTP(w, r)
	})
}

// copyFullPayload copies the payload of a HTTP request to destWriter. If it
// receives less content than expected, and the client disconnected during the
// upload, it avoids sending a 400 error to keep the logs cleaner.
func copyFullPayload(responseWriter http.ResponseWriter, r *http.Request, destWriter io.Writer, context ctxu.Context, action string, errSlice *errcode.Errors) error {
	// Get a channel that tells us if the client disconnects
	var clientClosed <-chan bool
	if notifier, ok := responseWriter.(http.CloseNotifier); ok {
		clientClosed = notifier.CloseNotify()
	} else {
		ctxu.GetLogger(context).Warn("the ResponseWriter does not implement CloseNotifier")
	}

	// Read in the data, if any.
	copied, err := io.Copy(destWriter, r.Body)
	if clientClosed != nil && (err != nil || (r.ContentLength > 0 && copied < r.ContentLength)) {
		// Didn't recieve as much content as expected. Did the client
		// disconnect during the request? If so, avoid returning a 400
		// error to keep the logs cleaner.
		select {
		case <-clientClosed:
			// Set the response code to "499 Client Closed Request"
			// Even though the connection has already been closed,
			// this causes the logger to pick up a 499 error
			// instead of showing 0 for the HTTP status.
			responseWriter.WriteHeader(499)

			ctxu.GetLogger(context).Error("client disconnected during " + action)
			return errors.New("client disconnected")
		default:
		}
	}

	if err != nil {
		ctxu.GetLogger(context).Errorf("unknown error reading request payload: %v", err)
		*errSlice = append(*errSlice, errcode.ErrorCodeUnknown.WithDetail(err))
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Package romulus contains all the logic for monitoring the kubernetes api
// and registering services with vulcan proxy
package romulus

var version = "v0.1.0"
<commit_msg>More documentation<commit_after>// Package romulus contains all the logic for monitoring the kubernetes api
// and registering services with vulcan proxy
//
// See https://docs.vulcand.io/index.html for vulcan proxy documentation
// See https://github.com/GoogleCloudPlatform/kubernetes/tree/master/docs for kubernetes documentation
package romulus

var version = "v0.1.0"
<|endoftext|>


<commit_before>package nndb

import (
	"fmt"
	"strings"
	"testing"
)

var nutrientInput = `~01001~^~203~^0.85^16^0.074^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01001~^~204~^81.11^580^0.065^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01001~^~205~^0.06^0^^~4~^~NC~^~~^~~^^^^^^^~~^11/1976^
~01001~^~207~^2.11^35^0.054^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01001~^~208~^717^0^^~4~^~NC~^~~^~~^^^^^^^~~^08/2010^
~01001~^~221~^0.0^0^^~7~^~~^~~^~~^^^^^^^~~^04/1985^
~01001~^~255~^15.87^522^0.061^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01001~^~262~^0^0^^~7~^~Z~^~~^~~^^^^^^^~~^02/2001^
~01001~^~263~^0^0^^~7~^~Z~^~~^~~^^^^^^^~~^02/2001^
~01001~^~268~^2999^0^^~4~^~NC~^~~^~~^^^^^^^~~^09/2015^
~01001~^~269~^0.06^0^^~4~^~NR~^~~^~~^^^^^^^~~^11/2002^
~01001~^~291~^0.0^0^^~4~^~~^~~^~~^^^^^^^~~^^
~01001~^~301~^24^17^0.789^~1~^~A~^~~^~~^7^19^30^4^22.021^26.496^~2, 3~^11/2002^
~01002~^~208~^555^0^^~4~^~NC~^~~^~~^^^^^^^~~^08/2010^
~01116~^~203~^3.47^0^^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01116~^~204~^3.25^0^^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01116~^~205~^4.66^0^^~4~^~NC~^~~^~~^^^^^^^~~^11/1976^
~01116~^~207~^0.72^0^^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01116~^~208~^61^0^^~4~^~NC~^~~^~~^^^^^^^~~^02/2009^
~01116~^~221~^0.0^0^^~7~^~~^~~^~~^^^^^^^~~^04/1985^
~01116~^~255~^87.90^0^^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01116~^~262~^0^0^^~7~^~Z~^~~^~~^^^^^^^~~^01/2003^
~01116~^~263~^0^0^^~7~^~Z~^~~^~~^^^^^^^~~^01/2003^
~01116~^~268~^257^0^^~4~^~NC~^~~^~~^^^^^^^~~^02/2009^
~01116~^~269~^4.66^0^^~4~^~NR~^~~^~~^^^^^^^~~^01/2003^
~01116~^~291~^0.0^0^^~4~^~~^~~^~~^^^^^^^~~^11/1976^`
var foodInput = "~01116~^~0100~^~Yogurt, plain, whole milk, 8 grams protein per 8 ounce~^~YOGURT,PLN,WHL MILK,8 GRAMS PROT PER 8 OZ~^~~^~~^~Y~^~~^0^~~^6.38^4.27^8.79^3.87"
var foodGroupInput = "~0100~^~Dairy and Egg Products~\n~0200~^~Spices and Herbs~\n"

func TestParseNutrientDescription(t *testing.T) {

	parser, error := NewParser(strings.NewReader(""), strings.NewReader(""), strings.NewReader(nutrientInput))
	if error != nil {
		t.Errorf(`newReaderParser returned an error %v`, error)
	}
	if parser == nil {
		t.Error(`newReaderParser returned nil parser`)
	}
	nutrients, error := parser.parseNutrients()
	if error != nil {
		t.Errorf(`Parse() returned an error %v`, error)
	}
	if nutrients == nil {
		t.Error(`parse returned nil`)
	}

	if len(nutrients) != 3 {
		t.Errorf(`incorrect number of results %v`, len(nutrients))
	}
	if nutrients[1001].Calories != 717 {
		t.Errorf(`incorrect amount of calories %v`, nutrients[1001].Calories)
	}
	if nutrients[1001].Fiber != 0 {
		t.Errorf(`incorrect amount of fiber %v`, nutrients[1001].Fiber)
	}
	if nutrients[1001].Fat != 81.11 {
		t.Errorf(`incorrect amount of fat %v`, nutrients[1001].Fat)
	}
	if nutrients[1001].Protein != 0.85 {
		t.Errorf(`incorrect amount of protein %v`, nutrients[1001].Protein)
	}
	if nutrients[1001].Sugar != 0.06 {
		t.Errorf(`incorrect amount of sugar %v`, nutrients[1001].Sugar)
	}
	if nutrients[1002].Calories != 555 {
		t.Errorf(`incorrect amount of calories %v`, nutrients[1002].Calories)
	}
	if nutrients[1002].Fat != 0 {
		t.Errorf(`incorrect amount of Fat %v`, nutrients[1002].Fat)
	}
}

func TestParse(t *testing.T) {
	parser, error := NewParser(strings.NewReader(foodInput), strings.NewReader(foodGroupInput), strings.NewReader(nutrientInput))
	if error != nil {
		t.Errorf(`newReaderParser returned an error %v`, error)
	}
	if parser == nil {
		t.Error(`newReaderParser returned nil parser`)
	}

	food, error := parser.Parse()

	if error != nil {
		t.Errorf(`Parse() returned an error %v`, error)
	}
	if food == nil {
		t.Error(`parse returned nil`)
	}
	if food[0].ID != 1116 {
		t.Error(`incorect value for foodGroups[0].ndbNo`)
	}
	if food[0].FoodGroup.ID != 100 {
		t.Error(`incorect food group`)
	}
	if food[0].Nutrients.Calories == 0 {
		t.Error(`Nutrients should be present`)
	}
	if food[0].Name != "Yogurt, plain, whole milk, 8 grams protein per 8 ounce" {
		t.Error(`Incorrect value for name`)
	}

	fmt.Println(food[0].String())
	fmt.Println(food[0].Fields())
}

func TestParseFoodGroup(t *testing.T) {
	parser, error := NewParser(strings.NewReader(""), strings.NewReader(foodGroupInput), strings.NewReader(""))

	if error != nil {
		t.Errorf(`newReaderParser returned an error %v`, error)
	}
	if parser == nil {
		t.Error(`newReaderParser returned nil parser`)
	}

	groups, error := parser.parseFoodGroups()

	if error != nil {
		t.Errorf(`Parse() returned an error %v`, error)
	}
	if groups == nil {
		t.Error(`parse returned nil`)
	}

	if groups[100].ID != 100 {
		t.Error(`incorect value for foodGroups[100].code`)
	}
	if groups[100].Name != "Dairy and Egg Products" {
		t.Error(`incorect value for foodGroups[100].code`)
	}

	if groups[200].ID != 200 {
		t.Error(`incorect value for foodGroups[200].code`)
	}
	if groups[200].Name != "Spices and Herbs" {
		t.Error(`incorect value for foodGroups[200].code`)
	}
}

func TestParseWrongFileFormat(t *testing.T) {
	input := "blahblah"
	parser, _ := NewParser(strings.NewReader(input), strings.NewReader(input), strings.NewReader(input))

	_, error := parser.parseFoodGroups()
	if error == nil {
		t.Error(`invalid data should cause error`)
	}

	_, error = parser.parseNutrients()
	if error == nil {
		t.Error(`invalid data should cause error`)
	}

	_, error = parser.Parse()
	if error == nil {
		t.Error(`invalid data should cause error`)
	}
}

func TestParseEmptyDataOK(t *testing.T) {
	input := ""
	parser, _ := NewParser(strings.NewReader(input), strings.NewReader(input), strings.NewReader(input))

	_, error := parser.parseFoodGroups()
	if error != nil {
		t.Error(`empty data should not cause error`)
	}

	_, error = parser.parseNutrients()
	if error != nil {
		t.Error(`empty data should not cause error`)
	}

	_, error = parser.Parse()
	if error != nil {
		t.Error(`empty data should not cause error`)
	}

}
<commit_msg>Removed extraneious output<commit_after>package nndb

import (
	"strings"
	"testing"
)

var nutrientInput = `~01001~^~203~^0.85^16^0.074^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01001~^~204~^81.11^580^0.065^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01001~^~205~^0.06^0^^~4~^~NC~^~~^~~^^^^^^^~~^11/1976^
~01001~^~207~^2.11^35^0.054^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01001~^~208~^717^0^^~4~^~NC~^~~^~~^^^^^^^~~^08/2010^
~01001~^~221~^0.0^0^^~7~^~~^~~^~~^^^^^^^~~^04/1985^
~01001~^~255~^15.87^522^0.061^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01001~^~262~^0^0^^~7~^~Z~^~~^~~^^^^^^^~~^02/2001^
~01001~^~263~^0^0^^~7~^~Z~^~~^~~^^^^^^^~~^02/2001^
~01001~^~268~^2999^0^^~4~^~NC~^~~^~~^^^^^^^~~^09/2015^
~01001~^~269~^0.06^0^^~4~^~NR~^~~^~~^^^^^^^~~^11/2002^
~01001~^~291~^0.0^0^^~4~^~~^~~^~~^^^^^^^~~^^
~01001~^~301~^24^17^0.789^~1~^~A~^~~^~~^7^19^30^4^22.021^26.496^~2, 3~^11/2002^
~01002~^~208~^555^0^^~4~^~NC~^~~^~~^^^^^^^~~^08/2010^
~01116~^~203~^3.47^0^^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01116~^~204~^3.25^0^^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01116~^~205~^4.66^0^^~4~^~NC~^~~^~~^^^^^^^~~^11/1976^
~01116~^~207~^0.72^0^^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01116~^~208~^61^0^^~4~^~NC~^~~^~~^^^^^^^~~^02/2009^
~01116~^~221~^0.0^0^^~7~^~~^~~^~~^^^^^^^~~^04/1985^
~01116~^~255~^87.90^0^^~1~^~~^~~^~~^^^^^^^~~^11/1976^
~01116~^~262~^0^0^^~7~^~Z~^~~^~~^^^^^^^~~^01/2003^
~01116~^~263~^0^0^^~7~^~Z~^~~^~~^^^^^^^~~^01/2003^
~01116~^~268~^257^0^^~4~^~NC~^~~^~~^^^^^^^~~^02/2009^
~01116~^~269~^4.66^0^^~4~^~NR~^~~^~~^^^^^^^~~^01/2003^
~01116~^~291~^0.0^0^^~4~^~~^~~^~~^^^^^^^~~^11/1976^`
var foodInput = "~01116~^~0100~^~Yogurt, plain, whole milk, 8 grams protein per 8 ounce~^~YOGURT,PLN,WHL MILK,8 GRAMS PROT PER 8 OZ~^~~^~~^~Y~^~~^0^~~^6.38^4.27^8.79^3.87"
var foodGroupInput = "~0100~^~Dairy and Egg Products~\n~0200~^~Spices and Herbs~\n"

func TestParseNutrientDescription(t *testing.T) {

	parser, error := NewParser(strings.NewReader(""), strings.NewReader(""), strings.NewReader(nutrientInput))
	if error != nil {
		t.Errorf(`newReaderParser returned an error %v`, error)
	}
	if parser == nil {
		t.Error(`newReaderParser returned nil parser`)
	}
	nutrients, error := parser.parseNutrients()
	if error != nil {
		t.Errorf(`Parse() returned an error %v`, error)
	}
	if nutrients == nil {
		t.Error(`parse returned nil`)
	}

	if len(nutrients) != 3 {
		t.Errorf(`incorrect number of results %v`, len(nutrients))
	}
	if nutrients[1001].Calories != 717 {
		t.Errorf(`incorrect amount of calories %v`, nutrients[1001].Calories)
	}
	if nutrients[1001].Fiber != 0 {
		t.Errorf(`incorrect amount of fiber %v`, nutrients[1001].Fiber)
	}
	if nutrients[1001].Fat != 81.11 {
		t.Errorf(`incorrect amount of fat %v`, nutrients[1001].Fat)
	}
	if nutrients[1001].Protein != 0.85 {
		t.Errorf(`incorrect amount of protein %v`, nutrients[1001].Protein)
	}
	if nutrients[1001].Sugar != 0.06 {
		t.Errorf(`incorrect amount of sugar %v`, nutrients[1001].Sugar)
	}
	if nutrients[1002].Calories != 555 {
		t.Errorf(`incorrect amount of calories %v`, nutrients[1002].Calories)
	}
	if nutrients[1002].Fat != 0 {
		t.Errorf(`incorrect amount of Fat %v`, nutrients[1002].Fat)
	}
}

func TestParse(t *testing.T) {
	parser, error := NewParser(strings.NewReader(foodInput), strings.NewReader(foodGroupInput), strings.NewReader(nutrientInput))
	if error != nil {
		t.Errorf(`newReaderParser returned an error %v`, error)
	}
	if parser == nil {
		t.Error(`newReaderParser returned nil parser`)
	}

	food, error := parser.Parse()

	if error != nil {
		t.Errorf(`Parse() returned an error %v`, error)
	}
	if food == nil {
		t.Error(`parse returned nil`)
	}
	if food[0].ID != 1116 {
		t.Error(`incorect value for foodGroups[0].ndbNo`)
	}
	if food[0].FoodGroup.ID != 100 {
		t.Error(`incorect food group`)
	}
	if food[0].Nutrients.Calories == 0 {
		t.Error(`Nutrients should be present`)
	}
	if food[0].Name != "Yogurt, plain, whole milk, 8 grams protein per 8 ounce" {
		t.Error(`Incorrect value for name`)
	}
}

func TestParseFoodGroup(t *testing.T) {
	parser, error := NewParser(strings.NewReader(""), strings.NewReader(foodGroupInput), strings.NewReader(""))

	if error != nil {
		t.Errorf(`newReaderParser returned an error %v`, error)
	}
	if parser == nil {
		t.Error(`newReaderParser returned nil parser`)
	}

	groups, error := parser.parseFoodGroups()

	if error != nil {
		t.Errorf(`Parse() returned an error %v`, error)
	}
	if groups == nil {
		t.Error(`parse returned nil`)
	}

	if groups[100].ID != 100 {
		t.Error(`incorect value for foodGroups[100].code`)
	}
	if groups[100].Name != "Dairy and Egg Products" {
		t.Error(`incorect value for foodGroups[100].code`)
	}

	if groups[200].ID != 200 {
		t.Error(`incorect value for foodGroups[200].code`)
	}
	if groups[200].Name != "Spices and Herbs" {
		t.Error(`incorect value for foodGroups[200].code`)
	}
}

func TestParseWrongFileFormat(t *testing.T) {
	input := "blahblah"
	parser, _ := NewParser(strings.NewReader(input), strings.NewReader(input), strings.NewReader(input))

	_, error := parser.parseFoodGroups()
	if error == nil {
		t.Error(`invalid data should cause error`)
	}

	_, error = parser.parseNutrients()
	if error == nil {
		t.Error(`invalid data should cause error`)
	}

	_, error = parser.Parse()
	if error == nil {
		t.Error(`invalid data should cause error`)
	}
}

func TestParseEmptyDataOK(t *testing.T) {
	input := ""
	parser, _ := NewParser(strings.NewReader(input), strings.NewReader(input), strings.NewReader(input))

	_, error := parser.parseFoodGroups()
	if error != nil {
		t.Error(`empty data should not cause error`)
	}

	_, error = parser.parseNutrients()
	if error != nil {
		t.Error(`empty data should not cause error`)
	}

	_, error = parser.Parse()
	if error != nil {
		t.Error(`empty data should not cause error`)
	}

}
<|endoftext|>


<commit_before>//
// Copyright (c) 2015 The heketi Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package glusterfs

import (
	"fmt"
	"strconv"
)

// Elements in the balanced list
type SimpleDevice struct {
	zone             int
	nodeId, deviceId string
}

// Pretty pring a SimpleDevice
func (s *SimpleDevice) String() string {
	return fmt.Sprintf("{Z:%v N:%v D:%v}",
		s.zone,
		s.nodeId,
		s.deviceId)
}

// Simple Devices so that we have no pointers and no race conditions
type SimpleDevices []SimpleDevice

// A node is a collection of devices
type SimpleNode []*SimpleDevice

// A zone is a collection of nodes
type SimpleZone []SimpleNode

// The allocation ring will contain a map composed of all
// the devices available in the cluster.  Call Rebalance()
// for it to create a balanced list.
type SimpleAllocatorRing struct {

	// Map [zone] to [node] to slice of SimpleDevices
	ring         map[int]map[string][]*SimpleDevice
	balancedList SimpleDevices
}

// Create a new simple ring
func NewSimpleAllocatorRing() *SimpleAllocatorRing {
	s := &SimpleAllocatorRing{}
	s.ring = make(map[int]map[string][]*SimpleDevice)

	return s
}

// Convert the ring map into a consumable list of lists.
// This allows the rebalancer to go through the lists and remove
// elements as it balances
func (s *SimpleAllocatorRing) createZoneLists() []SimpleZone {
	zones := make([]SimpleZone, 0)

	for _, n := range s.ring {

		zone := make([]SimpleNode, 0)
		for _, d := range n {
			zone = append(zone, d)
		}
		zones = append(zones, zone)
	}

	return zones
}

// Add a device to the ring map
func (s *SimpleAllocatorRing) Add(d *SimpleDevice) {

	if nodes, ok := s.ring[d.zone]; ok {
		if _, ok := nodes[d.nodeId]; ok {
			nodes[d.nodeId] = append(nodes[d.nodeId], d)
		} else {
			nodes[d.nodeId] = []*SimpleDevice{d}
		}
	} else {
		s.ring[d.zone] = make(map[string][]*SimpleDevice)
		s.ring[d.zone][d.nodeId] = []*SimpleDevice{d}
	}

	s.balancedList = nil
}

// Remove device from the ring map
func (s *SimpleAllocatorRing) Remove(d *SimpleDevice) {

	if nodes, ok := s.ring[d.zone]; ok {
		if devices, ok := nodes[d.nodeId]; ok {
			for index, device := range devices {
				if device.deviceId == d.deviceId {
					// Found device, now delete it from the ring map
					nodes[d.nodeId] = append(nodes[d.nodeId][:index], nodes[d.nodeId][index+1:]...)

					if len(nodes[d.nodeId]) == 0 {
						delete(nodes, d.nodeId)
					}
					if len(s.ring[d.zone]) == 0 {
						delete(s.ring, d.zone)
					}
				}
			}
		}
	}

	s.balancedList = nil
}

// Rebalance the ring and place the rebalanced list
// into balancedList.
// The idea is to setup an array/slice where each continguous SimpleDevice
// is from either a different zone, or node.
func (s *SimpleAllocatorRing) Rebalance() {

	if s.balancedList != nil {
		return
	}

	// Copy map data to slices
	zones := s.createZoneLists()

	// Create a list
	list := make(SimpleDevices, 0)

	// Populate the list
	var device *SimpleDevice
	for i := 0; len(zones) != 0; i++ {
		zone := i % len(zones)
		node := i % len(zones[zone])

		// pop device
		device, zones[zone][node] = zones[zone][node][len(zones[zone][node])-1], zones[zone][node][:len(zones[zone][node])-1]
		list = append(list, *device)

		// delete node
		if len(zones[zone][node]) == 0 {
			zones[zone] = append(zones[zone][:node], zones[zone][node+1:]...)

			// delete zone
			if len(zones[zone]) == 0 {
				zones = append(zones[:zone], zones[zone+1:]...)
			}
		}
	}

	s.balancedList = list
}

// Use a uuid to point at a position in the ring.  Return a list of devices
// from that point in the ring.
func (s *SimpleAllocatorRing) GetDeviceList(uuid string) SimpleDevices {

	if s.balancedList == nil {
		s.Rebalance()
	}
	if len(s.balancedList) == 0 {
		return SimpleDevices{}
	}

	// Instead of using 8 characters to convert to a int32, use 7 which avoids
	// negative numbers
	index64, err := strconv.ParseInt(uuid[:7], 16, 32)
	if err != nil {
		logger.Err(err)
		return s.balancedList
	}

	// Point to a position on the ring
	index := int(index64) % len(s.balancedList)

	// Return a list according to the position in the list
	return append(s.balancedList[index:], s.balancedList[:index]...)

}
<commit_msg>Create a new copied ring list to avoid race<commit_after>//
// Copyright (c) 2015 The heketi Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package glusterfs

import (
	"fmt"
	"strconv"
)

// Elements in the balanced list
type SimpleDevice struct {
	zone             int
	nodeId, deviceId string
}

// Pretty pring a SimpleDevice
func (s *SimpleDevice) String() string {
	return fmt.Sprintf("{Z:%v N:%v D:%v}",
		s.zone,
		s.nodeId,
		s.deviceId)
}

// Simple Devices so that we have no pointers and no race conditions
type SimpleDevices []SimpleDevice

// A node is a collection of devices
type SimpleNode []*SimpleDevice

// A zone is a collection of nodes
type SimpleZone []SimpleNode

// The allocation ring will contain a map composed of all
// the devices available in the cluster.  Call Rebalance()
// for it to create a balanced list.
type SimpleAllocatorRing struct {

	// Map [zone] to [node] to slice of SimpleDevices
	ring         map[int]map[string][]*SimpleDevice
	balancedList SimpleDevices
}

// Create a new simple ring
func NewSimpleAllocatorRing() *SimpleAllocatorRing {
	s := &SimpleAllocatorRing{}
	s.ring = make(map[int]map[string][]*SimpleDevice)

	return s
}

// Convert the ring map into a consumable list of lists.
// This allows the rebalancer to go through the lists and remove
// elements as it balances
func (s *SimpleAllocatorRing) createZoneLists() []SimpleZone {
	zones := make([]SimpleZone, 0)

	for _, n := range s.ring {

		zone := make([]SimpleNode, 0)
		for _, d := range n {
			zone = append(zone, d)
		}
		zones = append(zones, zone)
	}

	return zones
}

// Add a device to the ring map
func (s *SimpleAllocatorRing) Add(d *SimpleDevice) {

	if nodes, ok := s.ring[d.zone]; ok {
		if _, ok := nodes[d.nodeId]; ok {
			nodes[d.nodeId] = append(nodes[d.nodeId], d)
		} else {
			nodes[d.nodeId] = []*SimpleDevice{d}
		}
	} else {
		s.ring[d.zone] = make(map[string][]*SimpleDevice)
		s.ring[d.zone][d.nodeId] = []*SimpleDevice{d}
	}

	s.balancedList = nil
}

// Remove device from the ring map
func (s *SimpleAllocatorRing) Remove(d *SimpleDevice) {

	if nodes, ok := s.ring[d.zone]; ok {
		if devices, ok := nodes[d.nodeId]; ok {
			for index, device := range devices {
				if device.deviceId == d.deviceId {
					// Found device, now delete it from the ring map
					nodes[d.nodeId] = append(nodes[d.nodeId][:index], nodes[d.nodeId][index+1:]...)

					if len(nodes[d.nodeId]) == 0 {
						delete(nodes, d.nodeId)
					}
					if len(s.ring[d.zone]) == 0 {
						delete(s.ring, d.zone)
					}
				}
			}
		}
	}

	s.balancedList = nil
}

// Rebalance the ring and place the rebalanced list
// into balancedList.
// The idea is to setup an array/slice where each continguous SimpleDevice
// is from either a different zone, or node.
func (s *SimpleAllocatorRing) Rebalance() {

	if s.balancedList != nil {
		return
	}

	// Copy map data to slices
	zones := s.createZoneLists()

	// Create a list
	list := make(SimpleDevices, 0)

	// Populate the list
	var device *SimpleDevice
	for i := 0; len(zones) != 0; i++ {
		zone := i % len(zones)
		node := i % len(zones[zone])

		// pop device
		device, zones[zone][node] = zones[zone][node][len(zones[zone][node])-1], zones[zone][node][:len(zones[zone][node])-1]
		list = append(list, *device)

		// delete node
		if len(zones[zone][node]) == 0 {
			zones[zone] = append(zones[zone][:node], zones[zone][node+1:]...)

			// delete zone
			if len(zones[zone]) == 0 {
				zones = append(zones[:zone], zones[zone+1:]...)
			}
		}
	}

	s.balancedList = list
}

// Use a uuid to point at a position in the ring.  Return a list of devices
// from that point in the ring.
func (s *SimpleAllocatorRing) GetDeviceList(uuid string) SimpleDevices {

	if s.balancedList == nil {
		s.Rebalance()
	}
	if len(s.balancedList) == 0 {
		return SimpleDevices{}
	}

	// Create a new list to avoid race conditions
	devices := make(SimpleDevices, len(s.balancedList))
	copy(devices, s.balancedList)

	// Instead of using 8 characters to convert to a int32, use 7 which avoids
	// negative numbers
	index64, err := strconv.ParseInt(uuid[:7], 16, 32)
	if err != nil {
		logger.Err(err)
		return devices
	}

	// Point to a position on the ring
	index := int(index64) % len(s.balancedList)

	// Return a list according to the position in the list
	return append(devices[index:], devices[:index]...)

}
<|endoftext|>


<commit_before>package job_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	. "github.com/cloudfoundry/bosh-micro-cli/state/job"

	"code.google.com/p/gomock/gomock"
	mock_state_package "github.com/cloudfoundry/bosh-micro-cli/state/pkg/mocks"

	bmreljob "github.com/cloudfoundry/bosh-micro-cli/release/job"
	bmrelpkg "github.com/cloudfoundry/bosh-micro-cli/release/pkg"
	bmstatepkg "github.com/cloudfoundry/bosh-micro-cli/state/pkg"

	fakebmui "github.com/cloudfoundry/bosh-micro-cli/ui/fakes"
)

var _ = Describe("DependencyCompiler", func() {
	var mockCtrl *gomock.Controller

	BeforeEach(func() {
		mockCtrl = gomock.NewController(GinkgoT())
	})

	AfterEach(func() {
		mockCtrl.Finish()
	})

	var (
		mockPackageCompiler *mock_state_package.MockCompiler

		dependencyCompiler DependencyCompiler

		releaseJobs []bmreljob.Job
		fakeStage   *fakebmui.FakeStage

		releasePackage1 *bmrelpkg.Package
		releasePackage2 *bmrelpkg.Package

		releaseJob bmreljob.Job

		expectCompilePkg1 *gomock.Call
		expectCompilePkg2 *gomock.Call
	)

	BeforeEach(func() {
		mockPackageCompiler = mock_state_package.NewMockCompiler(mockCtrl)

		dependencyCompiler = NewDependencyCompiler(mockPackageCompiler)

		fakeStage = fakebmui.NewFakeStage()

		releasePackage1 = &bmrelpkg.Package{
			Name:          "fake-release-package-name-1",
			Fingerprint:   "fake-release-package-fingerprint-1",
			SHA1:          "fake-release-package-sha1-1",
			Dependencies:  []*bmrelpkg.Package{},
			ExtractedPath: "/extracted-release-path/extracted_packages/fake-release-package-name-1",
		}

		releasePackage2 = &bmrelpkg.Package{
			Name:          "fake-release-package-name-2",
			Fingerprint:   "fake-release-package-fingerprint-2",
			SHA1:          "fake-release-package-sha1-2",
			Dependencies:  []*bmrelpkg.Package{releasePackage1},
			ExtractedPath: "/extracted-release-path/extracted_packages/fake-release-package-name-2",
		}

		releaseJob = bmreljob.Job{
			Name:          "cpi",
			Fingerprint:   "fake-release-job-fingerprint",
			SHA1:          "fake-release-job-sha1",
			ExtractedPath: "/extracted-release-path/extracted_jobs/cpi",
			Templates: map[string]string{
				"cpi.erb":     "bin/cpi",
				"cpi.yml.erb": "config/cpi.yml",
			},
			PackageNames: []string{releasePackage2.Name},
			Packages:     []*bmrelpkg.Package{releasePackage2},
			Properties:   map[string]bmreljob.PropertyDefinition{},
		}
		releaseJobs = []bmreljob.Job{releaseJob}
	})

	JustBeforeEach(func() {
		compiledPackageRecord1 := bmstatepkg.CompiledPackageRecord{
			BlobID:   "fake-compiled-package-blobstore-id-1",
			BlobSHA1: "fake-compiled-package-sha1-1",
		}
		expectCompilePkg1 = mockPackageCompiler.EXPECT().Compile(releasePackage1).Return(compiledPackageRecord1, nil).AnyTimes()

		compiledPackageRecord2 := bmstatepkg.CompiledPackageRecord{
			BlobID:   "fake-compiled-package-blobstore-id-2",
			BlobSHA1: "fake-compiled-package-sha1-2",
		}
		expectCompilePkg2 = mockPackageCompiler.EXPECT().Compile(releasePackage2).Return(compiledPackageRecord2, nil).AnyTimes()
	})

	It("compiles all the job dependencies (packages) such that no package is compiled before its dependencies", func() {
		gomock.InOrder(
			expectCompilePkg1.Times(1),
			expectCompilePkg2.Times(1),
		)

		_, err := dependencyCompiler.Compile(releaseJobs, fakeStage)
		Expect(err).ToNot(HaveOccurred())
	})

	It("returns references to the compiled packages", func() {
		compiledPackageRefs, err := dependencyCompiler.Compile(releaseJobs, fakeStage)
		Expect(err).ToNot(HaveOccurred())

		Expect(compiledPackageRefs).To(Equal([]CompiledPackageRef{
			{
				Name:        "fake-release-package-name-1",
				Version:     "fake-release-package-fingerprint-1",
				BlobstoreID: "fake-compiled-package-blobstore-id-1",
				SHA1:        "fake-compiled-package-sha1-1",
			},
			{
				Name:        "fake-release-package-name-2",
				Version:     "fake-release-package-fingerprint-2",
				BlobstoreID: "fake-compiled-package-blobstore-id-2",
				SHA1:        "fake-compiled-package-sha1-2",
			},
		}))
	})

	It("logs compile stages", func() {
		_, err := dependencyCompiler.Compile(releaseJobs, fakeStage)
		Expect(err).ToNot(HaveOccurred())

		Expect(fakeStage.PerformCalls).To(Equal([]fakebmui.PerformCall{
			{Name: "Compiling package 'fake-release-package-name-1/fake-release-package-fingerprint-1'"},
			{Name: "Compiling package 'fake-release-package-name-2/fake-release-package-fingerprint-2'"},
		}))
	})

	Context("when multiple jobs depend on the same package", func() {
		JustBeforeEach(func() {
			releaseJob2 := bmreljob.Job{
				Name:         "fake-other-job",
				Fingerprint:  "fake-other-job-fingerprint",
				PackageNames: []string{releasePackage2.Name},
				Packages:     []*bmrelpkg.Package{releasePackage2},
			}
			releaseJobs = append(releaseJobs, releaseJob2)
		})

		It("only compiles each package once", func() {
			gomock.InOrder(
				expectCompilePkg1.Times(1),
				expectCompilePkg2.Times(1),
			)

			_, err := dependencyCompiler.Compile(releaseJobs, fakeStage)
			Expect(err).ToNot(HaveOccurred())
		})
	})

	Context("when multiple packages depend on the same package", func() {
		var (
			releasePackage3 *bmrelpkg.Package

			expectCompilePkg3 *gomock.Call
		)

		BeforeEach(func() {
			releasePackage3 = &bmrelpkg.Package{
				Name:          "fake-release-package-name-3",
				Fingerprint:   "fake-release-package-fingerprint-3",
				SHA1:          "fake-release-package-sha1-3",
				Dependencies:  []*bmrelpkg.Package{releasePackage1},
				ExtractedPath: "/extracted-release-path/extracted_packages/fake-release-package-name-3",
			}

			releaseJob.PackageNames = append(releaseJob.PackageNames, releasePackage3.Name)
			releaseJob.Packages = append(releaseJob.Packages, releasePackage3)
			releaseJobs = []bmreljob.Job{releaseJob}
		})

		JustBeforeEach(func() {
			compiledPackageRecord3 := bmstatepkg.CompiledPackageRecord{
				BlobID:   "fake-compiled-package-blobstore-id-3",
				BlobSHA1: "fake-compiled-package-sha1-3",
			}
			expectCompilePkg3 = mockPackageCompiler.EXPECT().Compile(releasePackage3).Return(compiledPackageRecord3, nil).AnyTimes()
		})

		It("only compiles each package once", func() {
			gomock.InOrder(
				expectCompilePkg1.Times(1),
				expectCompilePkg2.Times(1),
				expectCompilePkg3.Times(1),
			)

			_, err := dependencyCompiler.Compile(releaseJobs, fakeStage)
			Expect(err).ToNot(HaveOccurred())
		})
	})
})
<commit_msg>Fix flakey test from dependency ordering<commit_after>package job_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	. "github.com/cloudfoundry/bosh-micro-cli/state/job"

	"code.google.com/p/gomock/gomock"
	mock_state_package "github.com/cloudfoundry/bosh-micro-cli/state/pkg/mocks"

	bmreljob "github.com/cloudfoundry/bosh-micro-cli/release/job"
	bmrelpkg "github.com/cloudfoundry/bosh-micro-cli/release/pkg"
	bmstatepkg "github.com/cloudfoundry/bosh-micro-cli/state/pkg"

	fakebmui "github.com/cloudfoundry/bosh-micro-cli/ui/fakes"
)

var _ = Describe("DependencyCompiler", func() {
	var mockCtrl *gomock.Controller

	BeforeEach(func() {
		mockCtrl = gomock.NewController(GinkgoT())
	})

	AfterEach(func() {
		mockCtrl.Finish()
	})

	var (
		mockPackageCompiler *mock_state_package.MockCompiler

		dependencyCompiler DependencyCompiler

		releaseJobs []bmreljob.Job
		fakeStage   *fakebmui.FakeStage

		releasePackage1 *bmrelpkg.Package
		releasePackage2 *bmrelpkg.Package

		releaseJob bmreljob.Job

		expectCompilePkg1 *gomock.Call
		expectCompilePkg2 *gomock.Call
	)

	BeforeEach(func() {
		mockPackageCompiler = mock_state_package.NewMockCompiler(mockCtrl)

		dependencyCompiler = NewDependencyCompiler(mockPackageCompiler)

		fakeStage = fakebmui.NewFakeStage()

		releasePackage1 = &bmrelpkg.Package{
			Name:          "fake-release-package-name-1",
			Fingerprint:   "fake-release-package-fingerprint-1",
			SHA1:          "fake-release-package-sha1-1",
			Dependencies:  []*bmrelpkg.Package{},
			ExtractedPath: "/extracted-release-path/extracted_packages/fake-release-package-name-1",
		}

		releasePackage2 = &bmrelpkg.Package{
			Name:          "fake-release-package-name-2",
			Fingerprint:   "fake-release-package-fingerprint-2",
			SHA1:          "fake-release-package-sha1-2",
			Dependencies:  []*bmrelpkg.Package{releasePackage1},
			ExtractedPath: "/extracted-release-path/extracted_packages/fake-release-package-name-2",
		}

		releaseJob = bmreljob.Job{
			Name:          "cpi",
			Fingerprint:   "fake-release-job-fingerprint",
			SHA1:          "fake-release-job-sha1",
			ExtractedPath: "/extracted-release-path/extracted_jobs/cpi",
			Templates: map[string]string{
				"cpi.erb":     "bin/cpi",
				"cpi.yml.erb": "config/cpi.yml",
			},
			PackageNames: []string{releasePackage2.Name},
			Packages:     []*bmrelpkg.Package{releasePackage2},
			Properties:   map[string]bmreljob.PropertyDefinition{},
		}
		releaseJobs = []bmreljob.Job{releaseJob}
	})

	JustBeforeEach(func() {
		compiledPackageRecord1 := bmstatepkg.CompiledPackageRecord{
			BlobID:   "fake-compiled-package-blobstore-id-1",
			BlobSHA1: "fake-compiled-package-sha1-1",
		}
		expectCompilePkg1 = mockPackageCompiler.EXPECT().Compile(releasePackage1).Return(compiledPackageRecord1, nil).AnyTimes()

		compiledPackageRecord2 := bmstatepkg.CompiledPackageRecord{
			BlobID:   "fake-compiled-package-blobstore-id-2",
			BlobSHA1: "fake-compiled-package-sha1-2",
		}
		expectCompilePkg2 = mockPackageCompiler.EXPECT().Compile(releasePackage2).Return(compiledPackageRecord2, nil).AnyTimes()
	})

	It("compiles all the job dependencies (packages) such that no package is compiled before its dependencies", func() {
		gomock.InOrder(
			expectCompilePkg1.Times(1),
			expectCompilePkg2.Times(1),
		)

		_, err := dependencyCompiler.Compile(releaseJobs, fakeStage)
		Expect(err).ToNot(HaveOccurred())
	})

	It("returns references to the compiled packages", func() {
		compiledPackageRefs, err := dependencyCompiler.Compile(releaseJobs, fakeStage)
		Expect(err).ToNot(HaveOccurred())

		Expect(compiledPackageRefs).To(Equal([]CompiledPackageRef{
			{
				Name:        "fake-release-package-name-1",
				Version:     "fake-release-package-fingerprint-1",
				BlobstoreID: "fake-compiled-package-blobstore-id-1",
				SHA1:        "fake-compiled-package-sha1-1",
			},
			{
				Name:        "fake-release-package-name-2",
				Version:     "fake-release-package-fingerprint-2",
				BlobstoreID: "fake-compiled-package-blobstore-id-2",
				SHA1:        "fake-compiled-package-sha1-2",
			},
		}))
	})

	It("logs compile stages", func() {
		_, err := dependencyCompiler.Compile(releaseJobs, fakeStage)
		Expect(err).ToNot(HaveOccurred())

		Expect(fakeStage.PerformCalls).To(Equal([]fakebmui.PerformCall{
			{Name: "Compiling package 'fake-release-package-name-1/fake-release-package-fingerprint-1'"},
			{Name: "Compiling package 'fake-release-package-name-2/fake-release-package-fingerprint-2'"},
		}))
	})

	Context("when multiple jobs depend on the same package", func() {
		JustBeforeEach(func() {
			releaseJob2 := bmreljob.Job{
				Name:         "fake-other-job",
				Fingerprint:  "fake-other-job-fingerprint",
				PackageNames: []string{releasePackage2.Name},
				Packages:     []*bmrelpkg.Package{releasePackage2},
			}
			releaseJobs = append(releaseJobs, releaseJob2)
		})

		It("only compiles each package once", func() {
			gomock.InOrder(
				expectCompilePkg1.Times(1),
				expectCompilePkg2.Times(1),
			)

			_, err := dependencyCompiler.Compile(releaseJobs, fakeStage)
			Expect(err).ToNot(HaveOccurred())
		})
	})

	Context("when multiple packages depend on the same package", func() {
		var (
			releasePackage3 *bmrelpkg.Package

			expectCompilePkg3 *gomock.Call
		)

		BeforeEach(func() {
			releasePackage3 = &bmrelpkg.Package{
				Name:          "fake-release-package-name-3",
				Fingerprint:   "fake-release-package-fingerprint-3",
				SHA1:          "fake-release-package-sha1-3",
				Dependencies:  []*bmrelpkg.Package{releasePackage1},
				ExtractedPath: "/extracted-release-path/extracted_packages/fake-release-package-name-3",
			}

			releaseJob.PackageNames = append(releaseJob.PackageNames, releasePackage3.Name)
			releaseJob.Packages = append(releaseJob.Packages, releasePackage3)
			releaseJobs = []bmreljob.Job{releaseJob}
		})

		JustBeforeEach(func() {
			compiledPackageRecord3 := bmstatepkg.CompiledPackageRecord{
				BlobID:   "fake-compiled-package-blobstore-id-3",
				BlobSHA1: "fake-compiled-package-sha1-3",
			}
			expectCompilePkg3 = mockPackageCompiler.EXPECT().Compile(releasePackage3).Return(compiledPackageRecord3, nil).AnyTimes()
		})

		It("only compiles each package once", func() {
			expectCompilePkg1.Times(1)
			expectCompilePkg2.After(expectCompilePkg1)
			expectCompilePkg3.After(expectCompilePkg1)

			_, err := dependencyCompiler.Compile(releaseJobs, fakeStage)
			Expect(err).ToNot(HaveOccurred())
		})
	})
})
<|endoftext|>


<commit_before>package apps

import (
	"encoding/json"
	"io"
	"net/url"
	"regexp"
	"time"

	"github.com/Masterminds/semver"
	"github.com/cozy/cozy-stack/pkg/hooks"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/prefixer"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/registry"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/sirupsen/logrus"
)

var slugReg = regexp.MustCompile(`^[a-z0-9\-]+$`)

// Operation is the type of operation the installer is created for.
type Operation int

const (
	// Install operation for installing an application
	Install Operation = iota + 1
	// Update operation for updating an application
	Update
	// Delete operation for deleting an application
	Delete
)

// Installer is used to install or update applications.
type Installer struct {
	fetcher  Fetcher
	op       Operation
	fs       Copier
	db       prefixer.Prefixer
	endState State

	overridenParameters *json.RawMessage
	permissionsAcked    bool

	man  Manifest
	src  *url.URL
	slug string

	manc chan Manifest
	log  *logrus.Entry
}

// InstallerOptions provides the slug name of the application along with the
// source URL.
type InstallerOptions struct {
	Type             AppType
	Operation        Operation
	Manifest         Manifest
	Slug             string
	SourceURL        string
	Deactivated      bool
	PermissionsAcked bool
	Registries       []*url.URL

	// Used to override the "Parameters" field of konnectors during installation.
	// This modification is useful to allow the parameterization of a konnector
	// at its installation as we do not have yet a registry up and running.
	OverridenParameters *json.RawMessage
}

// Fetcher interface should be implemented by the underlying transport
// used to fetch the application data.
type Fetcher interface {
	// FetchManifest should returns an io.ReadCloser to read the
	// manifest data
	FetchManifest(src *url.URL) (io.ReadCloser, error)
	// Fetch should download the application and install it in the given
	// directory.
	Fetch(src *url.URL, fs Copier, man Manifest) error
}

// NewInstaller creates a new Installer
func NewInstaller(db prefixer.Prefixer, fs Copier, opts *InstallerOptions) (*Installer, error) {
	man, err := initManifest(db, opts)
	if err != nil {
		return nil, err
	}

	var src *url.URL
	switch opts.Operation {
	case Install:
		if opts.SourceURL == "" {
			return nil, ErrMissingSource
		}
		src, err = url.Parse(opts.SourceURL)
	case Update, Delete:
		var srcString string
		if opts.SourceURL == "" {
			srcString = man.Source()
		} else {
			srcString = opts.SourceURL
		}
		src, err = url.Parse(srcString)
	default:
		panic("Unknown installer operation")
	}
	if err != nil {
		return nil, err
	}

	var endState State
	if opts.Deactivated || man.State() == Installed {
		endState = Installed
	} else {
		endState = Ready
	}

	var installType string
	switch opts.Operation {
	case Install:
		installType = "install"
	case Update:
		installType = "update"
	case Delete:
		installType = "delete"
	}

	log := logger.WithDomain(db.DomainName()).WithFields(logrus.Fields{
		"nspace":        "apps",
		"slug":          man.Slug(),
		"version_start": man.Version(),
		"type":          installType,
	})

	var manFilename string
	switch man.AppType() {
	case Webapp:
		manFilename = WebappManifestName
	case Konnector:
		manFilename = KonnectorManifestName
	}

	var fetcher Fetcher
	switch src.Scheme {
	case "git", "git+ssh", "ssh+git":
		fetcher = newGitFetcher(manFilename, log)
	case "http", "https":
		fetcher = newHTTPFetcher(manFilename, log)
	case "registry":
		fetcher = newRegistryFetcher(opts.Registries, log)
	case "file":
		fetcher = newFileFetcher(manFilename, log)
	default:
		return nil, ErrNotSupportedSource
	}

	return &Installer{
		fetcher:  fetcher,
		op:       opts.Operation,
		db:       db,
		fs:       fs,
		endState: endState,

		overridenParameters: opts.OverridenParameters,
		permissionsAcked:    opts.PermissionsAcked,

		man:  man,
		src:  src,
		slug: man.Slug(),

		manc: make(chan Manifest, 2),
		log:  log,
	}, nil
}

func initManifest(db prefixer.Prefixer, opts *InstallerOptions) (man Manifest, err error) {
	if man = opts.Manifest; man != nil {
		return man, nil
	}

	slug := opts.Slug
	if slug == "" || !slugReg.MatchString(slug) {
		return nil, ErrInvalidSlugName
	}

	if opts.Operation == Install {
		_, err = GetBySlug(db, slug, opts.Type)
		if err == nil {
			return nil, ErrAlreadyExists
		}
		if err != ErrNotFound {
			return nil, err
		}
		switch opts.Type {
		case Webapp:
			man = &WebappManifest{DocSlug: slug}
		case Konnector:
			man = &KonnManifest{DocSlug: slug}
		}
	} else {
		man, err = GetBySlug(db, slug, opts.Type)
		if err != nil {
			return nil, err
		}
	}

	if man == nil {
		panic("Bad or missing installer type")
	}

	return man, nil
}

// Slug return the slug of the application being installed.
func (i *Installer) Slug() string {
	return i.slug
}

// Domain return the domain of instance associated with the installer.
func (i *Installer) Domain() string {
	return i.db.DomainName()
}

// Run will install, update or delete the application linked to the installer,
// depending on specified operation. It will report its progress or error (see
// Poll method) and should be run asynchronously.
func (i *Installer) Run() {
	if err := i.run(); err != nil {
		i.man.SetError(err)
		realtime.GetHub().Publish(i.db, realtime.EventUpdate, i.man.Clone(), nil)
	}
	i.notifyChannel()
}

// RunSync does the same work as Run but can be used synchronously.
func (i *Installer) RunSync() (Manifest, error) {
	i.manc = nil
	if err := i.run(); err != nil {
		return nil, err
	}
	return i.man.Clone().(Manifest), nil
}

func (i *Installer) run() (err error) {
	if i.man == nil {
		panic("Manifest is nil")
	}
	defer func() {
		if err != nil {
			i.log.Errorf("Could not commit installer process: %s", err)
		} else {
			i.log.Infof("Successful installer process: %s", i.man.Version())
		}
	}()
	i.log.Info("Start")
	switch i.op {
	case Install:
		return i.install()
	case Update:
		return i.update()
	case Delete:
		return i.delete()
	default:
		panic("Unknown operation")
	}
}

// install will perform the installation of an application. It returns the
// freshly fetched manifest from the source along with a possible error in case
// the installation went wrong.
//
// Note that the fetched manifest is returned even if an error occurred while
// upgrading.
func (i *Installer) install() error {
	args := []string{i.db.DomainName(), i.slug}
	return hooks.Execute("install-app", args, func() error {
		newManifest, err := i.ReadManifest(Installing)
		if err != nil {
			return err
		}
		i.man = newManifest
		i.sendRealtimeEvent()
		i.notifyChannel()
		if err := i.fetcher.Fetch(i.src, i.fs, i.man); err != nil {
			return err
		}
		i.man.SetState(i.endState)
		return i.man.Create(i.db)
	})
}

// update will perform the update of an already installed application. It
// returns the freshly fetched manifest from the source along with a possible
// error in case the update went wrong.
//
// Note that the fetched manifest is returned even if an error occurred while
// upgrading.
func (i *Installer) update() error {
	if err := i.checkState(i.man); err != nil {
		return err
	}

	oldManifest := i.man
	newManifest, err := i.ReadManifest(Upgrading)
	if err != nil {
		return err
	}

	// Fast path for registry:// and http:// sources: we do not need to go
	// further in the case where the fetched manifest has the same version has
	// the one in database.
	//
	// For git:// and file:// sources, it may be more complicated since we need
	// to actually fetch the data to extract the exact version of the manifest.
	makeUpdate := true
	availableVersion := ""
	switch i.src.Scheme {
	case "registry", "http", "https":
		makeUpdate = (newManifest.Version() != oldManifest.Version())
	}

	// Check the possible permissions changes before updating. If the
	// verifyPermissions flag is activated (for non manual updates for example),
	// we cancel out the update and mark the UpdateAvailable field of the
	// application instead of actually updating.
	if makeUpdate && !isPlatformApp(oldManifest) {
		oldPermissions := oldManifest.Permissions()
		newPermissions := newManifest.Permissions()
		samePermissions := newPermissions != nil && oldPermissions != nil &&
			newPermissions.HasSameRules(oldPermissions)
		if !samePermissions && !i.permissionsAcked {
			makeUpdate = false
			availableVersion = newManifest.Version()
		}
	}

	oldTermsVersion := oldManifest.Terms().Version
	newTermsVersion := newManifest.Terms().Version

	termsAdded := oldTermsVersion == "" && newTermsVersion != ""
	termsUpdated := oldTermsVersion != newTermsVersion

	if termsAdded || termsUpdated {
		makeUpdate = false
		availableVersion = newManifest.Version()
	}

	if makeUpdate {
		i.man = newManifest
		i.sendRealtimeEvent()
		i.notifyChannel()
		if err := i.fetcher.Fetch(i.src, i.fs, i.man); err != nil {
			return err
		}
		i.man.SetState(i.endState)
	} else {
		i.man.SetSource(i.src)
		if availableVersion != "" {
			i.man.SetAvailableVersion(availableVersion)
		}
		i.sendRealtimeEvent()
		i.notifyChannel()
	}

	return i.man.Update(i.db)
}

func (i *Installer) notifyChannel() {
	if i.manc != nil {
		i.manc <- i.man.Clone().(Manifest)
	}
}

func (i *Installer) delete() error {
	if err := i.checkState(i.man); err != nil {
		return err
	}
	args := []string{i.db.DomainName(), i.slug}
	return hooks.Execute("uninstall-app", args, func() error {
		return i.man.Delete(i.db)
	})
}

// checkState returns whether or not the manifest is in the right state to
// perform an update or deletion.
func (i *Installer) checkState(man Manifest) error {
	state := man.State()
	if state == Ready || state == Installed {
		return nil
	}
	if time.Since(man.LastUpdate()) > 15*time.Minute {
		return nil
	}
	return ErrBadState
}

// ReadManifest will fetch the manifest and read its JSON content into the
// passed manifest pointer.
//
// The State field of the manifest will be set to the specified state.
func (i *Installer) ReadManifest(state State) (Manifest, error) {
	r, err := i.fetcher.FetchManifest(i.src)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	newManifest, err := i.man.ReadManifest(io.LimitReader(r, ManifestMaxSize), i.slug, i.src.String())
	if err != nil {
		return nil, err
	}
	newManifest.SetState(state)

	shouldOverrideParameters := (i.overridenParameters != nil &&
		i.man.AppType() == Konnector &&
		i.src.Scheme != "registry")
	if shouldOverrideParameters {
		if m, ok := newManifest.(*KonnManifest); ok {
			m.Parameters = i.overridenParameters
		}
	}
	return newManifest, nil
}

func (i *Installer) sendRealtimeEvent() {
	realtime.GetHub().Publish(i.db, realtime.EventUpdate, i.man.Clone(), nil)
}

// Poll should be used to monitor the progress of the Installer.
func (i *Installer) Poll() (Manifest, bool, error) {
	man := <-i.manc
	done := false
	if s := man.State(); s == Ready || s == Installed || s == Errored {
		done = true
	}
	return man, done, man.Error()
}

// DoLazyUpdate tries to update an application before using it
func DoLazyUpdate(db prefixer.Prefixer, man Manifest, availableVersion string, copier Copier, registries []*url.URL) Manifest {
	src, err := url.Parse(man.Source())
	if err != nil || src.Scheme != "registry" {
		return man
	}
	var v *registry.Version
	channel, _ := getRegistryChannel(src)
	v, errv := registry.GetLatestVersion(man.Slug(), channel, registries)
	if errv != nil || v.Version == man.Version() {
		return man
	}
	if availableVersion != "" && v.Version == availableVersion {
		return man
	}
	if channel == "stable" && !isMoreRecent(man.Version(), v.Version) {
		return man
	}
	inst, err := NewInstaller(db, copier, &InstallerOptions{
		Operation:  Update,
		Manifest:   man,
		Registries: registries,
		SourceURL:  src.String(),
	})
	if err != nil {
		return man
	}
	newman, err := inst.RunSync()
	if err != nil {
		return man
	}
	return newman
}

// isMoreRecent returns true if b is greater than a
func isMoreRecent(a, b string) bool {
	vA, err := semver.NewVersion(a)
	if err != nil {
		return true
	}
	vB, err := semver.NewVersion(b)
	if err != nil {
		return false
	}
	return vB.GreaterThan(vA)
}

func isPlatformApp(man Manifest) bool {
	if man.AppType() != Webapp {
		return false
	}
	return utils.IsInArray(man.Slug(), []string{
		"onboarding",
		"settings",
		"collect",
		"home",
		"photos",
		"drive",
		"store",
		"banks",
		"contacts",
	})
}
<commit_msg>Fix konnectors install after new terms (#1677)<commit_after>package apps

import (
	"encoding/json"
	"io"
	"net/url"
	"regexp"
	"time"

	"github.com/Masterminds/semver"
	"github.com/cozy/cozy-stack/pkg/hooks"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/prefixer"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/registry"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/sirupsen/logrus"
)

var slugReg = regexp.MustCompile(`^[a-z0-9\-]+$`)

// Operation is the type of operation the installer is created for.
type Operation int

const (
	// Install operation for installing an application
	Install Operation = iota + 1
	// Update operation for updating an application
	Update
	// Delete operation for deleting an application
	Delete
)

// Installer is used to install or update applications.
type Installer struct {
	fetcher  Fetcher
	op       Operation
	fs       Copier
	db       prefixer.Prefixer
	endState State

	overridenParameters *json.RawMessage
	permissionsAcked    bool

	man  Manifest
	src  *url.URL
	slug string

	manc chan Manifest
	log  *logrus.Entry
}

// InstallerOptions provides the slug name of the application along with the
// source URL.
type InstallerOptions struct {
	Type             AppType
	Operation        Operation
	Manifest         Manifest
	Slug             string
	SourceURL        string
	Deactivated      bool
	PermissionsAcked bool
	Registries       []*url.URL

	// Used to override the "Parameters" field of konnectors during installation.
	// This modification is useful to allow the parameterization of a konnector
	// at its installation as we do not have yet a registry up and running.
	OverridenParameters *json.RawMessage
}

// Fetcher interface should be implemented by the underlying transport
// used to fetch the application data.
type Fetcher interface {
	// FetchManifest should returns an io.ReadCloser to read the
	// manifest data
	FetchManifest(src *url.URL) (io.ReadCloser, error)
	// Fetch should download the application and install it in the given
	// directory.
	Fetch(src *url.URL, fs Copier, man Manifest) error
}

// NewInstaller creates a new Installer
func NewInstaller(db prefixer.Prefixer, fs Copier, opts *InstallerOptions) (*Installer, error) {
	man, err := initManifest(db, opts)
	if err != nil {
		return nil, err
	}

	var src *url.URL
	switch opts.Operation {
	case Install:
		if opts.SourceURL == "" {
			return nil, ErrMissingSource
		}
		src, err = url.Parse(opts.SourceURL)
	case Update, Delete:
		var srcString string
		if opts.SourceURL == "" {
			srcString = man.Source()
		} else {
			srcString = opts.SourceURL
		}
		src, err = url.Parse(srcString)
	default:
		panic("Unknown installer operation")
	}
	if err != nil {
		return nil, err
	}

	var endState State
	if opts.Deactivated || man.State() == Installed {
		endState = Installed
	} else {
		endState = Ready
	}

	var installType string
	switch opts.Operation {
	case Install:
		installType = "install"
	case Update:
		installType = "update"
	case Delete:
		installType = "delete"
	}

	log := logger.WithDomain(db.DomainName()).WithFields(logrus.Fields{
		"nspace":        "apps",
		"slug":          man.Slug(),
		"version_start": man.Version(),
		"type":          installType,
	})

	var manFilename string
	switch man.AppType() {
	case Webapp:
		manFilename = WebappManifestName
	case Konnector:
		manFilename = KonnectorManifestName
	}

	var fetcher Fetcher
	switch src.Scheme {
	case "git", "git+ssh", "ssh+git":
		fetcher = newGitFetcher(manFilename, log)
	case "http", "https":
		fetcher = newHTTPFetcher(manFilename, log)
	case "registry":
		fetcher = newRegistryFetcher(opts.Registries, log)
	case "file":
		fetcher = newFileFetcher(manFilename, log)
	default:
		return nil, ErrNotSupportedSource
	}

	return &Installer{
		fetcher:  fetcher,
		op:       opts.Operation,
		db:       db,
		fs:       fs,
		endState: endState,

		overridenParameters: opts.OverridenParameters,
		permissionsAcked:    opts.PermissionsAcked,

		man:  man,
		src:  src,
		slug: man.Slug(),

		manc: make(chan Manifest, 2),
		log:  log,
	}, nil
}

func initManifest(db prefixer.Prefixer, opts *InstallerOptions) (man Manifest, err error) {
	if man = opts.Manifest; man != nil {
		return man, nil
	}

	slug := opts.Slug
	if slug == "" || !slugReg.MatchString(slug) {
		return nil, ErrInvalidSlugName
	}

	if opts.Operation == Install {
		_, err = GetBySlug(db, slug, opts.Type)
		if err == nil {
			return nil, ErrAlreadyExists
		}
		if err != ErrNotFound {
			return nil, err
		}
		switch opts.Type {
		case Webapp:
			man = &WebappManifest{DocSlug: slug}
		case Konnector:
			man = &KonnManifest{DocSlug: slug}
		}
	} else {
		man, err = GetBySlug(db, slug, opts.Type)
		if err != nil {
			return nil, err
		}
	}

	if man == nil {
		panic("Bad or missing installer type")
	}

	return man, nil
}

// Slug return the slug of the application being installed.
func (i *Installer) Slug() string {
	return i.slug
}

// Domain return the domain of instance associated with the installer.
func (i *Installer) Domain() string {
	return i.db.DomainName()
}

// Run will install, update or delete the application linked to the installer,
// depending on specified operation. It will report its progress or error (see
// Poll method) and should be run asynchronously.
func (i *Installer) Run() {
	if err := i.run(); err != nil {
		i.man.SetError(err)
		realtime.GetHub().Publish(i.db, realtime.EventUpdate, i.man.Clone(), nil)
	}
	i.notifyChannel()
}

// RunSync does the same work as Run but can be used synchronously.
func (i *Installer) RunSync() (Manifest, error) {
	i.manc = nil
	if err := i.run(); err != nil {
		return nil, err
	}
	return i.man.Clone().(Manifest), nil
}

func (i *Installer) run() (err error) {
	if i.man == nil {
		panic("Manifest is nil")
	}
	defer func() {
		if err != nil {
			i.log.Errorf("Could not commit installer process: %s", err)
		} else {
			i.log.Infof("Successful installer process: %s", i.man.Version())
		}
	}()
	i.log.Info("Start")
	switch i.op {
	case Install:
		return i.install()
	case Update:
		return i.update()
	case Delete:
		return i.delete()
	default:
		panic("Unknown operation")
	}
}

// install will perform the installation of an application. It returns the
// freshly fetched manifest from the source along with a possible error in case
// the installation went wrong.
//
// Note that the fetched manifest is returned even if an error occurred while
// upgrading.
func (i *Installer) install() error {
	args := []string{i.db.DomainName(), i.slug}
	return hooks.Execute("install-app", args, func() error {
		newManifest, err := i.ReadManifest(Installing)
		if err != nil {
			return err
		}
		i.man = newManifest
		i.sendRealtimeEvent()
		i.notifyChannel()
		if err := i.fetcher.Fetch(i.src, i.fs, i.man); err != nil {
			return err
		}
		i.man.SetState(i.endState)
		return i.man.Create(i.db)
	})
}

// update will perform the update of an already installed application. It
// returns the freshly fetched manifest from the source along with a possible
// error in case the update went wrong.
//
// Note that the fetched manifest is returned even if an error occurred while
// upgrading.
func (i *Installer) update() error {
	if err := i.checkState(i.man); err != nil {
		return err
	}

	oldManifest := i.man
	newManifest, err := i.ReadManifest(Upgrading)
	if err != nil {
		return err
	}

	// Fast path for registry:// and http:// sources: we do not need to go
	// further in the case where the fetched manifest has the same version has
	// the one in database.
	//
	// For git:// and file:// sources, it may be more complicated since we need
	// to actually fetch the data to extract the exact version of the manifest.
	makeUpdate := true
	availableVersion := ""
	switch i.src.Scheme {
	case "registry", "http", "https":
		makeUpdate = (newManifest.Version() != oldManifest.Version())
	}

	// Check the possible permissions changes before updating. If the
	// verifyPermissions flag is activated (for non manual updates for example),
	// we cancel out the update and mark the UpdateAvailable field of the
	// application instead of actually updating.
	if makeUpdate && !isPlatformApp(oldManifest) {
		oldPermissions := oldManifest.Permissions()
		newPermissions := newManifest.Permissions()
		samePermissions := newPermissions != nil && oldPermissions != nil &&
			newPermissions.HasSameRules(oldPermissions)
		if !samePermissions && !i.permissionsAcked {
			makeUpdate = false
			availableVersion = newManifest.Version()
		}
	}

	oldTermsVersion := oldManifest.Terms().Version
	newTermsVersion := newManifest.Terms().Version

	termsAdded := oldTermsVersion == "" && newTermsVersion != ""
	termsUpdated := oldTermsVersion != newTermsVersion

	if (termsAdded || termsUpdated) && !i.permissionsAcked {
		makeUpdate = false
		availableVersion = newManifest.Version()
	}

	if makeUpdate {
		i.man = newManifest
		i.sendRealtimeEvent()
		i.notifyChannel()
		if err := i.fetcher.Fetch(i.src, i.fs, i.man); err != nil {
			return err
		}
		i.man.SetState(i.endState)
	} else {
		i.man.SetSource(i.src)
		if availableVersion != "" {
			i.man.SetAvailableVersion(availableVersion)
		}
		i.sendRealtimeEvent()
		i.notifyChannel()
	}

	return i.man.Update(i.db)
}

func (i *Installer) notifyChannel() {
	if i.manc != nil {
		i.manc <- i.man.Clone().(Manifest)
	}
}

func (i *Installer) delete() error {
	if err := i.checkState(i.man); err != nil {
		return err
	}
	args := []string{i.db.DomainName(), i.slug}
	return hooks.Execute("uninstall-app", args, func() error {
		return i.man.Delete(i.db)
	})
}

// checkState returns whether or not the manifest is in the right state to
// perform an update or deletion.
func (i *Installer) checkState(man Manifest) error {
	state := man.State()
	if state == Ready || state == Installed {
		return nil
	}
	if time.Since(man.LastUpdate()) > 15*time.Minute {
		return nil
	}
	return ErrBadState
}

// ReadManifest will fetch the manifest and read its JSON content into the
// passed manifest pointer.
//
// The State field of the manifest will be set to the specified state.
func (i *Installer) ReadManifest(state State) (Manifest, error) {
	r, err := i.fetcher.FetchManifest(i.src)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	newManifest, err := i.man.ReadManifest(io.LimitReader(r, ManifestMaxSize), i.slug, i.src.String())
	if err != nil {
		return nil, err
	}
	newManifest.SetState(state)

	shouldOverrideParameters := (i.overridenParameters != nil &&
		i.man.AppType() == Konnector &&
		i.src.Scheme != "registry")
	if shouldOverrideParameters {
		if m, ok := newManifest.(*KonnManifest); ok {
			m.Parameters = i.overridenParameters
		}
	}
	return newManifest, nil
}

func (i *Installer) sendRealtimeEvent() {
	realtime.GetHub().Publish(i.db, realtime.EventUpdate, i.man.Clone(), nil)
}

// Poll should be used to monitor the progress of the Installer.
func (i *Installer) Poll() (Manifest, bool, error) {
	man := <-i.manc
	done := false
	if s := man.State(); s == Ready || s == Installed || s == Errored {
		done = true
	}
	return man, done, man.Error()
}

// DoLazyUpdate tries to update an application before using it
func DoLazyUpdate(db prefixer.Prefixer, man Manifest, availableVersion string, copier Copier, registries []*url.URL) Manifest {
	src, err := url.Parse(man.Source())
	if err != nil || src.Scheme != "registry" {
		return man
	}
	var v *registry.Version
	channel, _ := getRegistryChannel(src)
	v, errv := registry.GetLatestVersion(man.Slug(), channel, registries)
	if errv != nil || v.Version == man.Version() {
		return man
	}
	if availableVersion != "" && v.Version == availableVersion {
		return man
	}
	if channel == "stable" && !isMoreRecent(man.Version(), v.Version) {
		return man
	}
	inst, err := NewInstaller(db, copier, &InstallerOptions{
		Operation:  Update,
		Manifest:   man,
		Registries: registries,
		SourceURL:  src.String(),
	})
	if err != nil {
		return man
	}
	newman, err := inst.RunSync()
	if err != nil {
		return man
	}
	return newman
}

// isMoreRecent returns true if b is greater than a
func isMoreRecent(a, b string) bool {
	vA, err := semver.NewVersion(a)
	if err != nil {
		return true
	}
	vB, err := semver.NewVersion(b)
	if err != nil {
		return false
	}
	return vB.GreaterThan(vA)
}

func isPlatformApp(man Manifest) bool {
	if man.AppType() != Webapp {
		return false
	}
	return utils.IsInArray(man.Slug(), []string{
		"onboarding",
		"settings",
		"collect",
		"home",
		"photos",
		"drive",
		"store",
		"banks",
		"contacts",
	})
}
<|endoftext|>