<commit_before>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scorch

import (
	"fmt"
	"sync/atomic"

	"github.com/RoaringBitmap/roaring"
	"github.com/blevesearch/bleve/index/scorch/segment"
)

type segmentIntroduction struct {
	id        uint64
	data      segment.Segment
	obsoletes map[uint64]*roaring.Bitmap
	ids       []string
	internal  map[string][]byte

	applied   chan error
	persisted chan error
}

type epochWatcher struct {
	epoch    uint64
	notifyCh notificationChan
}

type snapshotReversion struct {
	snapshot  *IndexSnapshot
	applied   chan error
	persisted chan error
}

func (s *Scorch) mainLoop() {
	var epochWatchers []*epochWatcher
OUTER:
	for {
		select {
		case <-s.closeCh:
			break OUTER

		case epochWatcher := <-s.introducerNotifier:
			epochWatchers = append(epochWatchers, epochWatcher)

		case nextMerge := <-s.merges:
			s.introduceMerge(nextMerge)

		case next := <-s.introductions:
			err := s.introduceSegment(next)
			if err != nil {
				continue OUTER
			}

		case revertTo := <-s.revertToSnapshots:
			err := s.revertToSnapshot(revertTo)
			if err != nil {
				continue OUTER
			}
		}

		var epochCurr uint64
		s.rootLock.RLock()
		if s.root != nil {
			epochCurr = s.root.epoch
		}
		s.rootLock.RUnlock()
		var epochWatchersNext []*epochWatcher
		for _, w := range epochWatchers {
			if w.epoch < epochCurr {
				close(w.notifyCh)
			} else {
				epochWatchersNext = append(epochWatchersNext, w)
			}
		}
		epochWatchers = epochWatchersNext
	}

	s.asyncTasks.Done()
}

func (s *Scorch) introduceSegment(next *segmentIntroduction) error {
	// acquire lock
	s.rootLock.Lock()

	nsegs := len(s.root.segment)

	// prepare new index snapshot
	newSnapshot := &IndexSnapshot{
		parent:   s,
		segment:  make([]*SegmentSnapshot, nsegs, nsegs+1),
		offsets:  make([]uint64, nsegs, nsegs+1),
		internal: make(map[string][]byte, len(s.root.internal)),
		epoch:    s.nextSnapshotEpoch,
		refs:     1,
	}
	s.nextSnapshotEpoch++

	// iterate through current segments
	var running uint64
	for i := range s.root.segment {
		// see if optimistic work included this segment
		delta, ok := next.obsoletes[s.root.segment[i].id]
		if !ok {
			var err error
			delta, err = s.root.segment[i].segment.DocNumbers(next.ids)
			if err != nil {
				next.applied <- fmt.Errorf("error computing doc numbers: %v", err)
				close(next.applied)
				_ = newSnapshot.DecRef()
				return err
			}
		}
		newSnapshot.segment[i] = &SegmentSnapshot{
			id:         s.root.segment[i].id,
			segment:    s.root.segment[i].segment,
			cachedDocs: s.root.segment[i].cachedDocs,
		}
		s.root.segment[i].segment.AddRef()

		// apply new obsoletions
		if s.root.segment[i].deleted == nil {
			newSnapshot.segment[i].deleted = delta
		} else {
			newSnapshot.segment[i].deleted = roaring.Or(s.root.segment[i].deleted, delta)
		}

		newSnapshot.offsets[i] = running
		running += s.root.segment[i].Count()

	}
	// append new segment, if any, to end of the new index snapshot
	if next.data != nil {
		newSegmentSnapshot := &SegmentSnapshot{
			id:         next.id,
			segment:    next.data, // take ownership of next.data's ref-count
			cachedDocs: &cachedDocs{cache: nil},
		}
		newSnapshot.segment = append(newSnapshot.segment, newSegmentSnapshot)
		newSnapshot.offsets = append(newSnapshot.offsets, running)

		// increment numItemsIntroduced which tracks the number of items
		// queued for persistence.
		atomic.AddUint64(&s.stats.numItemsIntroduced, newSegmentSnapshot.Count())
	}
	// copy old values
	for key, oldVal := range s.root.internal {
		newSnapshot.internal[key] = oldVal
	}
	// set new values and apply deletes
	for key, newVal := range next.internal {
		if newVal != nil {
			newSnapshot.internal[key] = newVal
		} else {
			delete(newSnapshot.internal, key)
		}
	}
	if next.persisted != nil {
		s.rootPersisted = append(s.rootPersisted, next.persisted)
	}
	// swap in new index snapshot
	rootPrev := s.root
	s.root = newSnapshot
	// release lock
	s.rootLock.Unlock()

	if rootPrev != nil {
		_ = rootPrev.DecRef()
	}

	close(next.applied)

	return nil
}

func (s *Scorch) introduceMerge(nextMerge *segmentMerge) {
	// acquire lock
	s.rootLock.Lock()

	// prepare new index snapshot
	currSize := len(s.root.segment)
	newSize := currSize + 1 - len(nextMerge.old)
	newSnapshot := &IndexSnapshot{
		parent:   s,
		segment:  make([]*SegmentSnapshot, 0, newSize),
		offsets:  make([]uint64, 0, newSize),
		internal: s.root.internal,
		epoch:    s.nextSnapshotEpoch,
		refs:     1,
	}
	s.nextSnapshotEpoch++

	// iterate through current segments
	newSegmentDeleted := roaring.NewBitmap()
	var running uint64
	for i := range s.root.segment {
		segmentID := s.root.segment[i].id
		if segSnapAtMerge, ok := nextMerge.old[segmentID]; ok {
			// this segment is going away, see if anything else was deleted since we started the merge
			if s.root.segment[i].deleted != nil {
				// assume all these deletes are new
				deletedSince := s.root.segment[i].deleted
				// if we already knew about some of them, remove
				if segSnapAtMerge.deleted != nil {
					deletedSince = roaring.AndNot(s.root.segment[i].deleted, segSnapAtMerge.deleted)
				}
				deletedSinceItr := deletedSince.Iterator()
				for deletedSinceItr.HasNext() {
					oldDocNum := deletedSinceItr.Next()
					newDocNum := nextMerge.oldNewDocNums[segmentID][oldDocNum]
					newSegmentDeleted.Add(uint32(newDocNum))
				}
			}
		} else {
			// this segment is staying
			newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{
				id:         s.root.segment[i].id,
				segment:    s.root.segment[i].segment,
				deleted:    s.root.segment[i].deleted,
				cachedDocs: s.root.segment[i].cachedDocs,
			})
			s.root.segment[i].segment.AddRef()
			newSnapshot.offsets = append(newSnapshot.offsets, running)
			running += s.root.segment[i].Count()
		}
	}

	// put new segment at end
	newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{
		id:         nextMerge.id,
		segment:    nextMerge.new, // take ownership for nextMerge.new's ref-count
		deleted:    newSegmentDeleted,
		cachedDocs: &cachedDocs{cache: nil},
	})
	newSnapshot.offsets = append(newSnapshot.offsets, running)

	// swap in new segment
	rootPrev := s.root
	s.root = newSnapshot
	// release lock
	s.rootLock.Unlock()

	if rootPrev != nil {
		_ = rootPrev.DecRef()
	}

	// notify merger we incorporated this
	close(nextMerge.notify)
}

func (s *Scorch) revertToSnapshot(revertTo *snapshotReversion) error {
	if revertTo.snapshot == nil {
		err := fmt.Errorf("Cannot revert to a nil snapshot")
		revertTo.applied <- err
		return err
	}

	// acquire lock
	s.rootLock.Lock()

	// prepare a new index snapshot, based on next snapshot
	newSnapshot := &IndexSnapshot{
		parent:   s,
		segment:  make([]*SegmentSnapshot, len(revertTo.snapshot.segment)),
		offsets:  revertTo.snapshot.offsets,
		internal: revertTo.snapshot.internal,
		epoch:    s.nextSnapshotEpoch,
		refs:     1,
	}
	s.nextSnapshotEpoch++

	// iterate through segments
	for i, segmentSnapshot := range revertTo.snapshot.segment {
		newSnapshot.segment[i] = &SegmentSnapshot{
			id:         segmentSnapshot.id,
			segment:    segmentSnapshot.segment,
			deleted:    segmentSnapshot.deleted,
			cachedDocs: segmentSnapshot.cachedDocs,
		}
		newSnapshot.segment[i].segment.AddRef()

		// remove segment from ineligibleForRemoval map
		filename := zapFileName(segmentSnapshot.id)
		delete(s.ineligibleForRemoval, filename)
	}

	if revertTo.persisted != nil {
		s.rootPersisted = append(s.rootPersisted, revertTo.persisted)
	}

	// swap in new snapshot
	rootPrev := s.root
	s.root = newSnapshot
	// release lock
	s.rootLock.Unlock()

	if rootPrev != nil {
		_ = rootPrev.DecRef()
	}

	close(revertTo.applied)

	return nil
}
<commit_msg>scorch unlocks in introduceSegment's DocNumbers() error codepath<commit_after>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scorch

import (
	"fmt"
	"sync/atomic"

	"github.com/RoaringBitmap/roaring"
	"github.com/blevesearch/bleve/index/scorch/segment"
)

type segmentIntroduction struct {
	id        uint64
	data      segment.Segment
	obsoletes map[uint64]*roaring.Bitmap
	ids       []string
	internal  map[string][]byte

	applied   chan error
	persisted chan error
}

type epochWatcher struct {
	epoch    uint64
	notifyCh notificationChan
}

type snapshotReversion struct {
	snapshot  *IndexSnapshot
	applied   chan error
	persisted chan error
}

func (s *Scorch) mainLoop() {
	var epochWatchers []*epochWatcher
OUTER:
	for {
		select {
		case <-s.closeCh:
			break OUTER

		case epochWatcher := <-s.introducerNotifier:
			epochWatchers = append(epochWatchers, epochWatcher)

		case nextMerge := <-s.merges:
			s.introduceMerge(nextMerge)

		case next := <-s.introductions:
			err := s.introduceSegment(next)
			if err != nil {
				continue OUTER
			}

		case revertTo := <-s.revertToSnapshots:
			err := s.revertToSnapshot(revertTo)
			if err != nil {
				continue OUTER
			}
		}

		var epochCurr uint64
		s.rootLock.RLock()
		if s.root != nil {
			epochCurr = s.root.epoch
		}
		s.rootLock.RUnlock()
		var epochWatchersNext []*epochWatcher
		for _, w := range epochWatchers {
			if w.epoch < epochCurr {
				close(w.notifyCh)
			} else {
				epochWatchersNext = append(epochWatchersNext, w)
			}
		}
		epochWatchers = epochWatchersNext
	}

	s.asyncTasks.Done()
}

func (s *Scorch) introduceSegment(next *segmentIntroduction) error {
	// acquire lock
	s.rootLock.Lock()

	nsegs := len(s.root.segment)

	// prepare new index snapshot
	newSnapshot := &IndexSnapshot{
		parent:   s,
		segment:  make([]*SegmentSnapshot, nsegs, nsegs+1),
		offsets:  make([]uint64, nsegs, nsegs+1),
		internal: make(map[string][]byte, len(s.root.internal)),
		epoch:    s.nextSnapshotEpoch,
		refs:     1,
	}
	s.nextSnapshotEpoch++

	// iterate through current segments
	var running uint64
	for i := range s.root.segment {
		// see if optimistic work included this segment
		delta, ok := next.obsoletes[s.root.segment[i].id]
		if !ok {
			var err error
			delta, err = s.root.segment[i].segment.DocNumbers(next.ids)
			if err != nil {
				s.rootLock.Unlock()
				next.applied <- fmt.Errorf("error computing doc numbers: %v", err)
				close(next.applied)
				_ = newSnapshot.DecRef()
				return err
			}
		}
		newSnapshot.segment[i] = &SegmentSnapshot{
			id:         s.root.segment[i].id,
			segment:    s.root.segment[i].segment,
			cachedDocs: s.root.segment[i].cachedDocs,
		}
		s.root.segment[i].segment.AddRef()

		// apply new obsoletions
		if s.root.segment[i].deleted == nil {
			newSnapshot.segment[i].deleted = delta
		} else {
			newSnapshot.segment[i].deleted = roaring.Or(s.root.segment[i].deleted, delta)
		}

		newSnapshot.offsets[i] = running
		running += s.root.segment[i].Count()

	}
	// append new segment, if any, to end of the new index snapshot
	if next.data != nil {
		newSegmentSnapshot := &SegmentSnapshot{
			id:         next.id,
			segment:    next.data, // take ownership of next.data's ref-count
			cachedDocs: &cachedDocs{cache: nil},
		}
		newSnapshot.segment = append(newSnapshot.segment, newSegmentSnapshot)
		newSnapshot.offsets = append(newSnapshot.offsets, running)

		// increment numItemsIntroduced which tracks the number of items
		// queued for persistence.
		atomic.AddUint64(&s.stats.numItemsIntroduced, newSegmentSnapshot.Count())
	}
	// copy old values
	for key, oldVal := range s.root.internal {
		newSnapshot.internal[key] = oldVal
	}
	// set new values and apply deletes
	for key, newVal := range next.internal {
		if newVal != nil {
			newSnapshot.internal[key] = newVal
		} else {
			delete(newSnapshot.internal, key)
		}
	}
	if next.persisted != nil {
		s.rootPersisted = append(s.rootPersisted, next.persisted)
	}
	// swap in new index snapshot
	rootPrev := s.root
	s.root = newSnapshot
	// release lock
	s.rootLock.Unlock()

	if rootPrev != nil {
		_ = rootPrev.DecRef()
	}

	close(next.applied)

	return nil
}

func (s *Scorch) introduceMerge(nextMerge *segmentMerge) {
	// acquire lock
	s.rootLock.Lock()

	// prepare new index snapshot
	currSize := len(s.root.segment)
	newSize := currSize + 1 - len(nextMerge.old)
	newSnapshot := &IndexSnapshot{
		parent:   s,
		segment:  make([]*SegmentSnapshot, 0, newSize),
		offsets:  make([]uint64, 0, newSize),
		internal: s.root.internal,
		epoch:    s.nextSnapshotEpoch,
		refs:     1,
	}
	s.nextSnapshotEpoch++

	// iterate through current segments
	newSegmentDeleted := roaring.NewBitmap()
	var running uint64
	for i := range s.root.segment {
		segmentID := s.root.segment[i].id
		if segSnapAtMerge, ok := nextMerge.old[segmentID]; ok {
			// this segment is going away, see if anything else was deleted since we started the merge
			if s.root.segment[i].deleted != nil {
				// assume all these deletes are new
				deletedSince := s.root.segment[i].deleted
				// if we already knew about some of them, remove
				if segSnapAtMerge.deleted != nil {
					deletedSince = roaring.AndNot(s.root.segment[i].deleted, segSnapAtMerge.deleted)
				}
				deletedSinceItr := deletedSince.Iterator()
				for deletedSinceItr.HasNext() {
					oldDocNum := deletedSinceItr.Next()
					newDocNum := nextMerge.oldNewDocNums[segmentID][oldDocNum]
					newSegmentDeleted.Add(uint32(newDocNum))
				}
			}
		} else {
			// this segment is staying
			newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{
				id:         s.root.segment[i].id,
				segment:    s.root.segment[i].segment,
				deleted:    s.root.segment[i].deleted,
				cachedDocs: s.root.segment[i].cachedDocs,
			})
			s.root.segment[i].segment.AddRef()
			newSnapshot.offsets = append(newSnapshot.offsets, running)
			running += s.root.segment[i].Count()
		}
	}

	// put new segment at end
	newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{
		id:         nextMerge.id,
		segment:    nextMerge.new, // take ownership for nextMerge.new's ref-count
		deleted:    newSegmentDeleted,
		cachedDocs: &cachedDocs{cache: nil},
	})
	newSnapshot.offsets = append(newSnapshot.offsets, running)

	// swap in new segment
	rootPrev := s.root
	s.root = newSnapshot
	// release lock
	s.rootLock.Unlock()

	if rootPrev != nil {
		_ = rootPrev.DecRef()
	}

	// notify merger we incorporated this
	close(nextMerge.notify)
}

func (s *Scorch) revertToSnapshot(revertTo *snapshotReversion) error {
	if revertTo.snapshot == nil {
		err := fmt.Errorf("Cannot revert to a nil snapshot")
		revertTo.applied <- err
		return err
	}

	// acquire lock
	s.rootLock.Lock()

	// prepare a new index snapshot, based on next snapshot
	newSnapshot := &IndexSnapshot{
		parent:   s,
		segment:  make([]*SegmentSnapshot, len(revertTo.snapshot.segment)),
		offsets:  revertTo.snapshot.offsets,
		internal: revertTo.snapshot.internal,
		epoch:    s.nextSnapshotEpoch,
		refs:     1,
	}
	s.nextSnapshotEpoch++

	// iterate through segments
	for i, segmentSnapshot := range revertTo.snapshot.segment {
		newSnapshot.segment[i] = &SegmentSnapshot{
			id:         segmentSnapshot.id,
			segment:    segmentSnapshot.segment,
			deleted:    segmentSnapshot.deleted,
			cachedDocs: segmentSnapshot.cachedDocs,
		}
		newSnapshot.segment[i].segment.AddRef()

		// remove segment from ineligibleForRemoval map
		filename := zapFileName(segmentSnapshot.id)
		delete(s.ineligibleForRemoval, filename)
	}

	if revertTo.persisted != nil {
		s.rootPersisted = append(s.rootPersisted, revertTo.persisted)
	}

	// swap in new snapshot
	rootPrev := s.root
	s.root = newSnapshot
	// release lock
	s.rootLock.Unlock()

	if rootPrev != nil {
		_ = rootPrev.DecRef()
	}

	close(revertTo.applied)

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2009 Samuel Tesla <samuel.tesla@gmail.com>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package main

import (
	"fmt";
	"os";
	"specify";
	"strings";
)

func HavePassing(expected interface{}) reporterMatcher {
	return reporterMatcher{expected, func(r TestingReporter) interface{} { return r.PassingCount() }}
}

func HavePending(expected interface{}) reporterMatcher {
	return reporterMatcher{expected, func(r TestingReporter) interface{} { return r.PendingCount() }}
}

func HaveFailing(expected interface{}) reporterMatcher {
	return reporterMatcher{expected, func(r TestingReporter) interface{} { return r.FailingCount() }}
}

func HaveErrors(expected interface{}) reporterMatcher {
	return reporterMatcher{expected, func(r TestingReporter) interface{} { return r.ErrorCount() }}
}

type reporterMatcher struct {
	expected	interface{};
	actualFunc	func(TestingReporter) interface{};
}

func toTestingReporter(value interface{}) (reporter TestingReporter, err os.Error) {
	var ok bool;
	if reporter, ok = value.(TestingReporter); !ok {
		err = os.NewError("Not a TestingReporter")
	}
	return;
}

func (self reporterMatcher) Should(actual interface{}) (result os.Error) {
	if reporter, error := toTestingReporter(actual); error != nil {
		result = error
	} else {
		result = specify.Be(self.expected).Should(self.actualFunc(reporter))
	}
	return;
}
func (self reporterMatcher) ShouldNot(actual interface{}) (result os.Error) {
	if reporter, error := toTestingReporter(actual); error != nil {
		result = error
	} else {
		result = specify.Be(self.expected).ShouldNot(self.actualFunc(reporter))
	}
	return;
}

func HaveFailureIncluding(s string) haveFailureIncluding {
	return (haveFailureIncluding)(s)
}

type haveFailureIncluding string

func (s haveFailureIncluding) Match(r TestingReporter) bool {
	for report := range r.EachFailure() {
		if strings.Count(report.Title(), (string)(s)) > 0 {
			return true
		}
	}
	return false;
}
func (s haveFailureIncluding) Should(val interface{}) os.Error {
	if reporter, error := toTestingReporter(val); error != nil {
		return error
	} else {
		if !s.Match(reporter) {
			return os.NewError(fmt.Sprintf("expected failing example including `%v`", s))
		}
	}
	return nil;
}
func (haveFailureIncluding) ShouldNot(val interface{}) os.Error {
	return os.NewError("matcher not implemented")
}

func HavePendingIncluding(s string) havePendingIncluding {
	return (havePendingIncluding)(s)
}

type havePendingIncluding string

func (s havePendingIncluding) Match(r TestingReporter) bool {
	for report := range r.EachPending() {
		if strings.Count(report.Title(), (string)(s)) > 0 {
			return true
		}
	}
	return false;
}
func (s havePendingIncluding) Should(val interface{}) os.Error {
	if reporter, error := toTestingReporter(val); error != nil {
		return error
	} else {
		if !s.Match(reporter) {
			return os.NewError(fmt.Sprintf("expected pending example including `%v`", s))
		}
	}
	return nil;
}
func (havePendingIncluding) ShouldNot(val interface{}) os.Error {
	return os.NewError("matcher not implemented")
}

func HaveErrorIncluding(s string) haveErrorIncluding {
	return (haveErrorIncluding)(s)
}

type haveErrorIncluding string

func (s haveErrorIncluding) Match(r TestingReporter) bool {
	for report := range r.EachError() {
		if strings.Count(report.Title(), (string)(s)) > 0 {
			return true
		}
	}
	return false;
}
func (s haveErrorIncluding) Should(val interface{}) os.Error {
	if reporter, error := toTestingReporter(val); error != nil {
		return error
	} else {
		if !s.Match(reporter) {
			return os.NewError(fmt.Sprintf("expected error example including `%v`", s))
		}
	}
	return nil;
}
func (haveErrorIncluding) ShouldNot(val interface{}) os.Error {
	return os.NewError("matcher not implemented")
}

func HaveFailureAt(loc string) haveFailureAt	{ return (haveFailureAt)(loc) }

type haveFailureAt string

func (loc haveFailureAt) Match(r TestingReporter) bool {
	for report := range r.EachFailure() {
		if strings.HasSuffix(report.Location().String(), (string)(loc)) {
			return true
		}
	}
	return false;
}
func (loc haveFailureAt) Should(val interface{}) os.Error {
	if reporter, error := toTestingReporter(val); error != nil {
		return error
	} else {
		if !loc.Match(reporter) {
			return os.NewError(fmt.Sprintf("expected failure at `%v`", loc))
		}
	}
	return nil;
}
func (haveFailureAt) ShouldNot(val interface{}) os.Error {
	return os.NewError("matcher not implemented")
}

func HavePendingAt(loc string) havePendingAt	{ return (havePendingAt)(loc) }

type havePendingAt string

func (loc havePendingAt) Match(r TestingReporter) bool {
	for report := range r.EachPending() {
		if strings.HasSuffix(report.Location().String(), (string)(loc)) {
			return true
		}
	}
	return false;
}
func (loc havePendingAt) Should(val interface{}) os.Error {
	if reporter, error := toTestingReporter(val); error != nil {
		return error
	} else {
		if !loc.Match(reporter) {
			return os.NewError(fmt.Sprintf("expected failure at `%v`", loc))
		}
	}
	return nil;
}
func (havePendingAt) ShouldNot(val interface{}) os.Error {
	return os.NewError("matcher not implemented")
}

func HaveErrorAt(loc string) haveErrorAt	{ return (haveErrorAt)(loc) }

type haveErrorAt string

func (loc haveErrorAt) Match(r TestingReporter) bool {
	for report := range r.EachError() {
		if strings.HasSuffix(report.Location().String(), (string)(loc)) {
			return true
		}
	}
	return false;
}
func (loc haveErrorAt) Should(val interface{}) os.Error {
	if reporter, error := toTestingReporter(val); error != nil {
		return error
	} else {
		if !loc.Match(reporter) {
			return os.NewError(fmt.Sprintf("expected failure at `%v`", loc))
		}
	}
	return nil;
}
func (haveErrorAt) ShouldNot(val interface{}) os.Error {
	return os.NewError("matcher not implemented")
}
<commit_msg>DRY up matchers.<commit_after>/*
Copyright (c) 2009 Samuel Tesla <samuel.tesla@gmail.com>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package main

import (
	"fmt";
	"os";
	"specify";
	"strings";

	t "../src/testspecify";
)

func HavePassing(expected interface{}) reporterMatcher {
	return reporterMatcher{expected, func(r TestingReporter) interface{} { return r.PassingCount() }}
}

func HavePending(expected interface{}) reporterMatcher {
	return reporterMatcher{expected, func(r TestingReporter) interface{} { return r.PendingCount() }}
}

func HaveFailing(expected interface{}) reporterMatcher {
	return reporterMatcher{expected, func(r TestingReporter) interface{} { return r.FailingCount() }}
}

func HaveErrors(expected interface{}) reporterMatcher {
	return reporterMatcher{expected, func(r TestingReporter) interface{} { return r.ErrorCount() }}
}

func HaveFailureIncluding(s string) eachMatcher {
	return eachMatcher{s, "failing example", eachFailure, matchTitle}
}

func HavePendingIncluding(s string) eachMatcher {
	return eachMatcher{s, "pending example", eachPending, matchTitle}
}

func HaveErrorIncluding(s string) eachMatcher {
	return eachMatcher{s, "error", eachError, matchTitle}
}

func HaveFailureAt(loc string) eachMatcher {
	return eachMatcher{loc, "failure", eachFailure, matchLocation}
}

func HavePendingAt(loc string) eachMatcher {
	return eachMatcher{loc, "pending example", eachPending, matchLocation}
}

func HaveErrorAt(loc string) eachMatcher {
	return eachMatcher{loc, "error", eachError, matchLocation}
}

func toTestingReporter(value interface{}) (reporter TestingReporter, err os.Error) {
	var ok bool;
	if reporter, ok = value.(TestingReporter); !ok {
		err = os.NewError("Not a TestingReporter")
	}
	return;
}

type reporterMatcher struct {
	expected	interface{};
	actualFunc	func(TestingReporter) interface{};
}

func (self reporterMatcher) Should(actual interface{}) (result os.Error) {
	if reporter, error := toTestingReporter(actual); error != nil {
		result = error
	} else {
		result = specify.Be(self.expected).Should(self.actualFunc(reporter))
	}
	return;
}
func (self reporterMatcher) ShouldNot(actual interface{}) (result os.Error) {
	if reporter, error := toTestingReporter(actual); error != nil {
		result = error
	} else {
		result = specify.Be(self.expected).ShouldNot(self.actualFunc(reporter))
	}
	return;
}

type eachMatcher struct {
	s, message	string;
	each		func(TestingReporter) <-chan t.Report;
	f		func(t.Report, string) bool;
}

func (self eachMatcher) match(r TestingReporter) bool {
	for report := range self.each(r) {
		if self.f(report, self.s) {
			return true
		}
	}
	return false;
}
func (self eachMatcher) Should(val interface{}) os.Error {
	if reporter, error := toTestingReporter(val); error != nil {
		return error
	} else {
		if !self.match(reporter) {
			return os.NewError(fmt.Sprintf("expected %v including `%v`", self.message, self.s))
		}
	}
	return nil;
}
func (eachMatcher) ShouldNot(val interface{}) os.Error {
	return os.NewError("matcher not implemented")
}

func matchTitle(r t.Report, s string) bool	{ return strings.Count(r.Title(), s) > 0 }

func matchLocation(r t.Report, s string) bool {
	return strings.HasSuffix(r.Location().String(), s)
}

func eachFailure(r TestingReporter) <-chan t.Report {
	return r.EachFailure()
}

func eachPending(r TestingReporter) <-chan t.Report {
	return r.EachPending()
}

func eachError(r TestingReporter) <-chan t.Report {
	return r.EachError()
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/flags"
	"github.com/GoogleContainerTools/skaffold/integration/skaffold"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/walk"
	"github.com/GoogleContainerTools/skaffold/testutil"
)

func TestBuildDeploy(t *testing.T) {
	// TODO: This test shall pass once render v2 is completed.
	t.SkipNow()

	MarkIntegrationTest(t, NeedsGcp)

	ns, client := SetupNamespace(t)

	outputBytes := skaffold.Build("--quiet").InDir("examples/microservices").InNs(ns.Name).RunOrFailOutput(t)
	// Parse the Build Output
	buildArtifacts, err := flags.ParseBuildOutput(outputBytes)
	failNowIfError(t, err)
	if len(buildArtifacts.Builds) != 3 {
		t.Fatalf("expected 3 artifacts to be built, but found %d", len(buildArtifacts.Builds))
	}

	var webTag, appTag string
	for _, a := range buildArtifacts.Builds {
		if a.ImageName == "leeroy-web" {
			webTag = a.Tag
		}
		if a.ImageName == "leeroy-app" {
			appTag = a.Tag
		}
	}
	if webTag == "" {
		t.Fatalf("expected to find a tag for leeroy-web but found none %s", webTag)
	}
	if appTag == "" {
		t.Fatalf("expected to find a tag for leeroy-app but found none %s", appTag)
	}

	tmpDir := testutil.NewTempDir(t)
	buildOutputFile := tmpDir.Path("build.out")
	tmpDir.Write("build.out", string(outputBytes))

	// Run Deploy using the build output
	// See https://github.com/GoogleContainerTools/skaffold/issues/2372 on why status-check=false
	skaffold.Deploy("--build-artifacts", buildOutputFile, "--status-check=false").InDir("examples/microservices").InNs(ns.Name).RunOrFail(t)

	depApp := client.GetDeployment("leeroy-app")
	testutil.CheckDeepEqual(t, appTag, depApp.Spec.Template.Spec.Containers[0].Image)

	depWeb := client.GetDeployment("leeroy-web")
	testutil.CheckDeepEqual(t, webTag, depWeb.Spec.Template.Spec.Containers[0].Image)
}

func TestDeploy(t *testing.T) {
	// TODO: This test shall pass once render v2 is completed.
	t.SkipNow()

	MarkIntegrationTest(t, CanRunWithoutGcp)

	ns, client := SetupNamespace(t)

	// `--default-repo=` is used to cancel the default repo that is set by default.
	skaffold.Deploy("--images", "index.docker.io/library/busybox:1", "--default-repo=").InDir("examples/kustomize").InNs(ns.Name).RunOrFail(t)

	dep := client.GetDeployment("kustomize-test")
	testutil.CheckDeepEqual(t, "index.docker.io/library/busybox:1", dep.Spec.Template.Spec.Containers[0].Image)
}

func TestDeployWithBuildArtifacts(t *testing.T) {
	MarkIntegrationTest(t, CanRunWithoutGcp)

	ns, client := SetupNamespace(t)

	// first build the artifacts and output to file
	skaffold.Build("--file-output=images.json", "--default-repo=").InDir("examples/getting-started").RunOrFail(t)

	// `--default-repo=` is used to cancel the default repo that is set by default.
	skaffold.Deploy("--build-artifacts=images.json", "--default-repo=", "--load-images=true").InDir("examples/getting-started").InNs(ns.Name).RunOrFail(t)

	pod := client.GetPod("getting-started")
	testutil.CheckContains(t, "skaffold-example", pod.Spec.Containers[0].Image)
}

func TestDeployWithImages(t *testing.T) {
	MarkIntegrationTest(t, CanRunWithoutGcp)

	ns, client := SetupNamespace(t)

	// first build the artifacts and output to file
	skaffold.Build("--file-output=artifacts.json", "--default-repo=").InDir("examples/getting-started").RunOrFail(t)

	var artifacts flags.BuildOutput
	if ba, err := ioutil.ReadFile("examples/getting-started/artifacts.json"); err != nil {
		t.Fatal("could not read artifacts.json", err)
	} else if err := json.Unmarshal(ba, &artifacts); err != nil {
		t.Fatal("could not decode artifacts.json", err)
	}

	var images []string
	for _, a := range artifacts.Builds {
		images = append(images, a.ImageName+"="+a.Tag)
	}

	// `--default-repo=` is used to cancel the default repo that is set by default.
	skaffold.Deploy("--images="+strings.Join(images, ","), "--default-repo=", "--load-images=true").InDir("examples/getting-started").InNs(ns.Name).RunOrFail(t)

	pod := client.GetPod("getting-started")
	testutil.CheckContains(t, "skaffold-example", pod.Spec.Containers[0].Image)
}

func TestDeployTail(t *testing.T) {
	MarkIntegrationTest(t, CanRunWithoutGcp)

	ns, _ := SetupNamespace(t)

	// `--default-repo=` is used to cancel the default repo that is set by default.
	out := skaffold.Deploy("--tail", "--images", "busybox:latest", "--default-repo=").InDir("testdata/deploy-hello-tail").InNs(ns.Name).RunLive(t)

	WaitForLogs(t, out, "Hello world!")
}

func TestDeployTailDefaultNamespace(t *testing.T) {
	MarkIntegrationTest(t, CanRunWithoutGcp)
	t.Skipf("fix https://github.com/GoogleContainerTools/skaffold/issues/7054")
	// `--default-repo=` is used to cancel the default repo that is set by default.
	out := skaffold.Deploy("--tail", "--images", "busybox:latest", "--default-repo=").InDir("testdata/deploy-hello-tail").RunLive(t)

	defer skaffold.Delete().InDir("testdata/deploy-hello-tail").RunBackground(t)
	WaitForLogs(t, out, "Hello world!")
}

func TestDeployWithInCorrectConfig(t *testing.T) {
	MarkIntegrationTest(t, CanRunWithoutGcp)

	ns, _ := SetupNamespace(t)

	// We're not providing a tag for the getting-started image
	output, err := skaffold.Deploy().InDir("examples/getting-started").InNs(ns.Name).RunWithCombinedOutput(t)
	if err == nil {
		t.Errorf("expected to see an error since not every image tag is provided: %s", output)
	} else if !strings.Contains(string(output), "no tag provided for image [skaffold-example]") {
		t.Errorf("failed without saying the reason: %s", output)
	}
}

// Verify that we can deploy without artifact details (https://github.com/GoogleContainerTools/skaffold/issues/4616)
func TestDeployWithoutWorkspaces(t *testing.T) {
	MarkIntegrationTest(t, NeedsGcp)

	ns, _ := SetupNamespace(t)

	outputBytes := skaffold.Build("--quiet").InDir("examples/nodejs").InNs(ns.Name).RunOrFailOutput(t)
	// Parse the Build Output
	buildArtifacts, err := flags.ParseBuildOutput(outputBytes)
	failNowIfError(t, err)
	if len(buildArtifacts.Builds) != 1 {
		t.Fatalf("expected 1 artifact to be built, but found %d", len(buildArtifacts.Builds))
	}

	tmpDir := testutil.NewTempDir(t)
	buildOutputFile := tmpDir.Path("build.out")
	tmpDir.Write("build.out", string(outputBytes))
	copyFiles(tmpDir.Root(), "examples/nodejs/skaffold.yaml")
	copyFiles(tmpDir.Root(), "examples/nodejs/k8s")

	// Run Deploy using the build output
	// See https://github.com/GoogleContainerTools/skaffold/issues/2372 on why status-check=false
	skaffold.Deploy("--build-artifacts", buildOutputFile, "--status-check=false").InDir(tmpDir.Root()).InNs(ns.Name).RunOrFail(t)
}

// Copies a file or directory tree.  There are 2x3 cases:
//   1. If _src_ is a file,
//      1. and _dst_ exists and is a file then _src_ is copied into _dst_
//      2. and _dst_ exists and is a directory, then _src_ is copied as _dst/$(basename src)_
//      3. and _dst_ does not exist, then _src_ is copied as _dst_.
//   2. If _src_ is a directory,
//      1. and _dst_ exists and is a file, then return an error
//      2. and _dst_ exists and is a directory, then src is copied as _dst/$(basename src)_
//      3. and _dst_ does not exist, then src is copied as _dst/src[1:]_.
func copyFiles(dst, src string) error {
	if util.IsFile(src) {
		switch {
		case util.IsFile(dst): // copy _src_ to _dst_
		case util.IsDir(dst): // copy _src_ to _dst/src[-1]
			dst = filepath.Join(dst, filepath.Base(src))
		default: // copy _src_ to _dst_
			if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil {
				return err
			}
		}
		in, err := os.Open(src)
		if err != nil {
			return err
		}
		out, err := os.Create(dst)
		if err != nil {
			return err
		}
		_, err = io.Copy(out, in)
		return err
	} else if !util.IsDir(src) {
		return errors.New("src does not exist")
	}
	// so src is a directory
	if util.IsFile(dst) {
		return errors.New("cannot copy directory into file")
	}
	srcPrefix := src
	if util.IsDir(dst) { // src is copied to _dst/$(basename src)
		srcPrefix = filepath.Dir(src)
	} else if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil {
		return err
	}
	return walk.From(src).Unsorted().WhenIsFile().Do(func(path string, _ walk.Dirent) error {
		rel, err := filepath.Rel(srcPrefix, path)
		if err != nil {
			return err
		}
		in, err := os.Open(path)
		if err != nil {
			return err
		}
		defer in.Close()

		destFile := filepath.Join(dst, rel)
		if err := os.MkdirAll(filepath.Dir(destFile), os.ModePerm); err != nil {
			return err
		}

		out, err := os.Create(destFile)
		if err != nil {
			return err
		}

		_, err = io.Copy(out, in)
		return err
	})
}
<commit_msg>chore: reenable TestDeployTailDefaultNamespace (#7363)<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/flags"
	"github.com/GoogleContainerTools/skaffold/integration/skaffold"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/walk"
	"github.com/GoogleContainerTools/skaffold/testutil"
)

func TestBuildDeploy(t *testing.T) {
	// TODO: This test shall pass once render v2 is completed.
	t.SkipNow()

	MarkIntegrationTest(t, NeedsGcp)

	ns, client := SetupNamespace(t)

	outputBytes := skaffold.Build("--quiet").InDir("examples/microservices").InNs(ns.Name).RunOrFailOutput(t)
	// Parse the Build Output
	buildArtifacts, err := flags.ParseBuildOutput(outputBytes)
	failNowIfError(t, err)
	if len(buildArtifacts.Builds) != 3 {
		t.Fatalf("expected 3 artifacts to be built, but found %d", len(buildArtifacts.Builds))
	}

	var webTag, appTag string
	for _, a := range buildArtifacts.Builds {
		if a.ImageName == "leeroy-web" {
			webTag = a.Tag
		}
		if a.ImageName == "leeroy-app" {
			appTag = a.Tag
		}
	}
	if webTag == "" {
		t.Fatalf("expected to find a tag for leeroy-web but found none %s", webTag)
	}
	if appTag == "" {
		t.Fatalf("expected to find a tag for leeroy-app but found none %s", appTag)
	}

	tmpDir := testutil.NewTempDir(t)
	buildOutputFile := tmpDir.Path("build.out")
	tmpDir.Write("build.out", string(outputBytes))

	// Run Deploy using the build output
	// See https://github.com/GoogleContainerTools/skaffold/issues/2372 on why status-check=false
	skaffold.Deploy("--build-artifacts", buildOutputFile, "--status-check=false").InDir("examples/microservices").InNs(ns.Name).RunOrFail(t)

	depApp := client.GetDeployment("leeroy-app")
	testutil.CheckDeepEqual(t, appTag, depApp.Spec.Template.Spec.Containers[0].Image)

	depWeb := client.GetDeployment("leeroy-web")
	testutil.CheckDeepEqual(t, webTag, depWeb.Spec.Template.Spec.Containers[0].Image)
}

func TestDeploy(t *testing.T) {
	// TODO: This test shall pass once render v2 is completed.
	t.SkipNow()

	MarkIntegrationTest(t, CanRunWithoutGcp)

	ns, client := SetupNamespace(t)

	// `--default-repo=` is used to cancel the default repo that is set by default.
	skaffold.Deploy("--images", "index.docker.io/library/busybox:1", "--default-repo=").InDir("examples/kustomize").InNs(ns.Name).RunOrFail(t)

	dep := client.GetDeployment("kustomize-test")
	testutil.CheckDeepEqual(t, "index.docker.io/library/busybox:1", dep.Spec.Template.Spec.Containers[0].Image)
}

func TestDeployWithBuildArtifacts(t *testing.T) {
	MarkIntegrationTest(t, CanRunWithoutGcp)

	ns, client := SetupNamespace(t)

	// first build the artifacts and output to file
	skaffold.Build("--file-output=images.json", "--default-repo=").InDir("examples/getting-started").RunOrFail(t)

	// `--default-repo=` is used to cancel the default repo that is set by default.
	skaffold.Deploy("--build-artifacts=images.json", "--default-repo=", "--load-images=true").InDir("examples/getting-started").InNs(ns.Name).RunOrFail(t)

	pod := client.GetPod("getting-started")
	testutil.CheckContains(t, "skaffold-example", pod.Spec.Containers[0].Image)
}

func TestDeployWithImages(t *testing.T) {
	MarkIntegrationTest(t, CanRunWithoutGcp)

	ns, client := SetupNamespace(t)

	// first build the artifacts and output to file
	skaffold.Build("--file-output=artifacts.json", "--default-repo=").InDir("examples/getting-started").RunOrFail(t)

	var artifacts flags.BuildOutput
	if ba, err := ioutil.ReadFile("examples/getting-started/artifacts.json"); err != nil {
		t.Fatal("could not read artifacts.json", err)
	} else if err := json.Unmarshal(ba, &artifacts); err != nil {
		t.Fatal("could not decode artifacts.json", err)
	}

	var images []string
	for _, a := range artifacts.Builds {
		images = append(images, a.ImageName+"="+a.Tag)
	}

	// `--default-repo=` is used to cancel the default repo that is set by default.
	skaffold.Deploy("--images="+strings.Join(images, ","), "--default-repo=", "--load-images=true").InDir("examples/getting-started").InNs(ns.Name).RunOrFail(t)

	pod := client.GetPod("getting-started")
	testutil.CheckContains(t, "skaffold-example", pod.Spec.Containers[0].Image)
}

func TestDeployTail(t *testing.T) {
	MarkIntegrationTest(t, CanRunWithoutGcp)

	ns, _ := SetupNamespace(t)

	// `--default-repo=` is used to cancel the default repo that is set by default.
	out := skaffold.Deploy("--tail", "--images", "busybox:latest", "--default-repo=").InDir("testdata/deploy-hello-tail").InNs(ns.Name).RunLive(t)

	WaitForLogs(t, out, "Hello world!")
}

func TestDeployTailDefaultNamespace(t *testing.T) {
	MarkIntegrationTest(t, CanRunWithoutGcp)

	// `--default-repo=` is used to cancel the default repo that is set by default.
	out := skaffold.Deploy("--tail", "--images", "busybox:latest", "--default-repo=").InDir("testdata/deploy-hello-tail").RunLive(t)

	defer skaffold.Delete().InDir("testdata/deploy-hello-tail").RunBackground(t)
	WaitForLogs(t, out, "Hello world!")
}

func TestDeployWithInCorrectConfig(t *testing.T) {
	MarkIntegrationTest(t, CanRunWithoutGcp)

	ns, _ := SetupNamespace(t)

	// We're not providing a tag for the getting-started image
	output, err := skaffold.Deploy().InDir("examples/getting-started").InNs(ns.Name).RunWithCombinedOutput(t)
	if err == nil {
		t.Errorf("expected to see an error since not every image tag is provided: %s", output)
	} else if !strings.Contains(string(output), "no tag provided for image [skaffold-example]") {
		t.Errorf("failed without saying the reason: %s", output)
	}
}

// Verify that we can deploy without artifact details (https://github.com/GoogleContainerTools/skaffold/issues/4616)
func TestDeployWithoutWorkspaces(t *testing.T) {
	MarkIntegrationTest(t, NeedsGcp)

	ns, _ := SetupNamespace(t)

	outputBytes := skaffold.Build("--quiet").InDir("examples/nodejs").InNs(ns.Name).RunOrFailOutput(t)
	// Parse the Build Output
	buildArtifacts, err := flags.ParseBuildOutput(outputBytes)
	failNowIfError(t, err)
	if len(buildArtifacts.Builds) != 1 {
		t.Fatalf("expected 1 artifact to be built, but found %d", len(buildArtifacts.Builds))
	}

	tmpDir := testutil.NewTempDir(t)
	buildOutputFile := tmpDir.Path("build.out")
	tmpDir.Write("build.out", string(outputBytes))
	copyFiles(tmpDir.Root(), "examples/nodejs/skaffold.yaml")
	copyFiles(tmpDir.Root(), "examples/nodejs/k8s")

	// Run Deploy using the build output
	// See https://github.com/GoogleContainerTools/skaffold/issues/2372 on why status-check=false
	skaffold.Deploy("--build-artifacts", buildOutputFile, "--status-check=false").InDir(tmpDir.Root()).InNs(ns.Name).RunOrFail(t)
}

// Copies a file or directory tree.  There are 2x3 cases:
//   1. If _src_ is a file,
//      1. and _dst_ exists and is a file then _src_ is copied into _dst_
//      2. and _dst_ exists and is a directory, then _src_ is copied as _dst/$(basename src)_
//      3. and _dst_ does not exist, then _src_ is copied as _dst_.
//   2. If _src_ is a directory,
//      1. and _dst_ exists and is a file, then return an error
//      2. and _dst_ exists and is a directory, then src is copied as _dst/$(basename src)_
//      3. and _dst_ does not exist, then src is copied as _dst/src[1:]_.
func copyFiles(dst, src string) error {
	if util.IsFile(src) {
		switch {
		case util.IsFile(dst): // copy _src_ to _dst_
		case util.IsDir(dst): // copy _src_ to _dst/src[-1]
			dst = filepath.Join(dst, filepath.Base(src))
		default: // copy _src_ to _dst_
			if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil {
				return err
			}
		}
		in, err := os.Open(src)
		if err != nil {
			return err
		}
		out, err := os.Create(dst)
		if err != nil {
			return err
		}
		_, err = io.Copy(out, in)
		return err
	} else if !util.IsDir(src) {
		return errors.New("src does not exist")
	}
	// so src is a directory
	if util.IsFile(dst) {
		return errors.New("cannot copy directory into file")
	}
	srcPrefix := src
	if util.IsDir(dst) { // src is copied to _dst/$(basename src)
		srcPrefix = filepath.Dir(src)
	} else if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil {
		return err
	}
	return walk.From(src).Unsorted().WhenIsFile().Do(func(path string, _ walk.Dirent) error {
		rel, err := filepath.Rel(srcPrefix, path)
		if err != nil {
			return err
		}
		in, err := os.Open(path)
		if err != nil {
			return err
		}
		defer in.Close()

		destFile := filepath.Join(dst, rel)
		if err := os.MkdirAll(filepath.Dir(destFile), os.ModePerm); err != nil {
			return err
		}

		out, err := os.Create(destFile)
		if err != nil {
			return err
		}

		_, err = io.Copy(out, in)
		return err
	})
}
<|endoftext|>


<commit_before>package helpers

import (
	"archive/zip"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"strings"

	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
	yaml "gopkg.in/yaml.v2"
)

func WithManifest(manifest map[string]interface{}, f func(manifestDir string)) {
	dir, err := ioutil.TempDir("", "simple-app")
	Expect(err).ToNot(HaveOccurred())
	defer os.RemoveAll(dir)

	WriteManifest(filepath.Join(dir, "manifest.yml"), manifest)
	f(dir)
}

// WithHelloWorldApp creates a simple application to use with your CLI command
// (typically CF Push). When pushing, be aware of specifying '-b
// staticfile_buildpack" so that your app will correctly start up with the
// proper buildpack.
func WithHelloWorldApp(f func(dir string)) {
	dir, err := ioutil.TempDir("", "simple-app")
	Expect(err).ToNot(HaveOccurred())
	defer os.RemoveAll(dir)

	tempfile := filepath.Join(dir, "index.html")
	err = ioutil.WriteFile(tempfile, []byte(fmt.Sprintf("hello world %d", rand.Int())), 0666)
	Expect(err).ToNot(HaveOccurred())

	err = ioutil.WriteFile(filepath.Join(dir, "Staticfile"), nil, 0666)
	Expect(err).ToNot(HaveOccurred())

	f(dir)
}

// WithNoResourceMatchedApp creates a simple application to use with your CLI
// command (typically CF Push). When pushing, be aware of specifying '-b
// staticfile_buildpack" so that your app will correctly start up with the
// proper buildpack.
func WithNoResourceMatchedApp(f func(dir string)) {
	dir, err := ioutil.TempDir("", "simple-app")
	Expect(err).ToNot(HaveOccurred())
	defer os.RemoveAll(dir)

	tempfile := filepath.Join(dir, "index.html")

	err = ioutil.WriteFile(tempfile, []byte(fmt.Sprintf("hello world %s", strings.Repeat("a", 65*1024))), 0666)
	Expect(err).ToNot(HaveOccurred())

	f(dir)
}

func WithMultiBuildpackApp(f func(dir string)) {
	f("../assets/go_calls_ruby")
}

// WithProcfileApp creates an application to use with your CLI command
// that contains Procfile defining web and worker processes.
func WithProcfileApp(f func(dir string)) {
	dir, err := ioutil.TempDir("", "simple-ruby-app")
	Expect(err).ToNot(HaveOccurred())
	defer os.RemoveAll(dir)

	err = ioutil.WriteFile(filepath.Join(dir, "Procfile"), []byte(`---
web: ruby -run -e httpd . -p $PORT
console: bundle exec irb`,
	), 0666)
	Expect(err).ToNot(HaveOccurred())

	err = ioutil.WriteFile(filepath.Join(dir, "Gemfile"), nil, 0666)
	Expect(err).ToNot(HaveOccurred())

	err = ioutil.WriteFile(filepath.Join(dir, "Gemfile.lock"), []byte(`
GEM
  specs:

PLATFORMS
  ruby

DEPENDENCIES

BUNDLED WITH
   1.15.0
	`), 0666)
	Expect(err).ToNot(HaveOccurred())

	f(dir)
}

func WithCrashingApp(f func(dir string)) {
	dir, err := ioutil.TempDir("", "crashing-ruby-app")
	Expect(err).ToNot(HaveOccurred())
	defer os.RemoveAll(dir)

	err = ioutil.WriteFile(filepath.Join(dir, "Procfile"), []byte(`---
web: bogus bogus`,
	), 0666)
	Expect(err).ToNot(HaveOccurred())

	err = ioutil.WriteFile(filepath.Join(dir, "Gemfile"), nil, 0666)
	Expect(err).ToNot(HaveOccurred())

	err = ioutil.WriteFile(filepath.Join(dir, "Gemfile.lock"), []byte(`
GEM
  specs:

PLATFORMS
  ruby

DEPENDENCIES

BUNDLED WITH
   1.15.0
	`), 0666)
	Expect(err).ToNot(HaveOccurred())

	f(dir)
}

// WithBananaPantsApp creates a simple application to use with your CLI command
// (typically CF Push). When pushing, be aware of specifying '-b
// staticfile_buildpack" so that your app will correctly start up with the
// proper buildpack.
func WithBananaPantsApp(f func(dir string)) {
	dir, err := ioutil.TempDir("", "simple-app")
	Expect(err).ToNot(HaveOccurred())
	defer os.RemoveAll(dir)

	tempfile := filepath.Join(dir, "index.html")
	err = ioutil.WriteFile(tempfile, []byte("Banana Pants"), 0666)
	Expect(err).ToNot(HaveOccurred())

	err = ioutil.WriteFile(filepath.Join(dir, "Staticfile"), nil, 0666)
	Expect(err).ToNot(HaveOccurred())

	f(dir)
}

// AppGUID returns the GUID for an app in the currently targeted space.
func AppGUID(appName string) string {
	session := CF("app", appName, "--guid")
	Eventually(session).Should(Exit(0))
	return strings.TrimSpace(string(session.Out.Contents()))
}

func WriteManifest(path string, manifest map[string]interface{}) {
	body, err := yaml.Marshal(manifest)
	Expect(err).ToNot(HaveOccurred())
	err = ioutil.WriteFile(path, body, 0666)
	Expect(err).ToNot(HaveOccurred())
}

func ConfirmStagingLogs(session *Session) {
	Eventually(session).Should(Say("(?i)Creating container|Successfully created container|Staging\\.\\.\\.|Staging process started \\.\\.\\.|Staging Complete"))
}

// Zipit zips the source into a .zip file in the target dir
func Zipit(source, target, prefix string) error {
	// Thanks to Svett Ralchev
	// http://blog.ralch.com/tutorial/golang-working-with-zip/

	zipfile, err := os.Create(target)
	if err != nil {
		return err
	}
	defer zipfile.Close()

	if prefix != "" {
		_, err = io.WriteString(zipfile, prefix)
		if err != nil {
			return err
		}
	}

	archive := zip.NewWriter(zipfile)
	defer archive.Close()

	err = filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if path == source {
			return nil
		}

		header, err := zip.FileInfoHeader(info)
		if err != nil {
			return err
		}
		header.Name, err = filepath.Rel(source, path)
		if err != nil {
			return err
		}

		header.Name = filepath.ToSlash(header.Name)

		if info.IsDir() {
			header.Name += "/"
			header.SetMode(0755)
		} else {
			header.Method = zip.Deflate
			header.SetMode(0744)
		}

		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()

		_, err = io.Copy(writer, file)
		return err
	})

	return err
}
<commit_msg>update the staging logs check as they have changed<commit_after>package helpers

import (
	"archive/zip"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"strings"

	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
	yaml "gopkg.in/yaml.v2"
)

func WithManifest(manifest map[string]interface{}, f func(manifestDir string)) {
	dir, err := ioutil.TempDir("", "simple-app")
	Expect(err).ToNot(HaveOccurred())
	defer os.RemoveAll(dir)

	WriteManifest(filepath.Join(dir, "manifest.yml"), manifest)
	f(dir)
}

// WithHelloWorldApp creates a simple application to use with your CLI command
// (typically CF Push). When pushing, be aware of specifying '-b
// staticfile_buildpack" so that your app will correctly start up with the
// proper buildpack.
func WithHelloWorldApp(f func(dir string)) {
	dir, err := ioutil.TempDir("", "simple-app")
	Expect(err).ToNot(HaveOccurred())
	defer os.RemoveAll(dir)

	tempfile := filepath.Join(dir, "index.html")
	err = ioutil.WriteFile(tempfile, []byte(fmt.Sprintf("hello world %d", rand.Int())), 0666)
	Expect(err).ToNot(HaveOccurred())

	err = ioutil.WriteFile(filepath.Join(dir, "Staticfile"), nil, 0666)
	Expect(err).ToNot(HaveOccurred())

	f(dir)
}

// WithNoResourceMatchedApp creates a simple application to use with your CLI
// command (typically CF Push). When pushing, be aware of specifying '-b
// staticfile_buildpack" so that your app will correctly start up with the
// proper buildpack.
func WithNoResourceMatchedApp(f func(dir string)) {
	dir, err := ioutil.TempDir("", "simple-app")
	Expect(err).ToNot(HaveOccurred())
	defer os.RemoveAll(dir)

	tempfile := filepath.Join(dir, "index.html")

	err = ioutil.WriteFile(tempfile, []byte(fmt.Sprintf("hello world %s", strings.Repeat("a", 65*1024))), 0666)
	Expect(err).ToNot(HaveOccurred())

	f(dir)
}

func WithMultiBuildpackApp(f func(dir string)) {
	f("../assets/go_calls_ruby")
}

// WithProcfileApp creates an application to use with your CLI command
// that contains Procfile defining web and worker processes.
func WithProcfileApp(f func(dir string)) {
	dir, err := ioutil.TempDir("", "simple-ruby-app")
	Expect(err).ToNot(HaveOccurred())
	defer os.RemoveAll(dir)

	err = ioutil.WriteFile(filepath.Join(dir, "Procfile"), []byte(`---
web: ruby -run -e httpd . -p $PORT
console: bundle exec irb`,
	), 0666)
	Expect(err).ToNot(HaveOccurred())

	err = ioutil.WriteFile(filepath.Join(dir, "Gemfile"), nil, 0666)
	Expect(err).ToNot(HaveOccurred())

	err = ioutil.WriteFile(filepath.Join(dir, "Gemfile.lock"), []byte(`
GEM
  specs:

PLATFORMS
  ruby

DEPENDENCIES

BUNDLED WITH
   1.15.0
	`), 0666)
	Expect(err).ToNot(HaveOccurred())

	f(dir)
}

func WithCrashingApp(f func(dir string)) {
	dir, err := ioutil.TempDir("", "crashing-ruby-app")
	Expect(err).ToNot(HaveOccurred())
	defer os.RemoveAll(dir)

	err = ioutil.WriteFile(filepath.Join(dir, "Procfile"), []byte(`---
web: bogus bogus`,
	), 0666)
	Expect(err).ToNot(HaveOccurred())

	err = ioutil.WriteFile(filepath.Join(dir, "Gemfile"), nil, 0666)
	Expect(err).ToNot(HaveOccurred())

	err = ioutil.WriteFile(filepath.Join(dir, "Gemfile.lock"), []byte(`
GEM
  specs:

PLATFORMS
  ruby

DEPENDENCIES

BUNDLED WITH
   1.15.0
	`), 0666)
	Expect(err).ToNot(HaveOccurred())

	f(dir)
}

// WithBananaPantsApp creates a simple application to use with your CLI command
// (typically CF Push). When pushing, be aware of specifying '-b
// staticfile_buildpack" so that your app will correctly start up with the
// proper buildpack.
func WithBananaPantsApp(f func(dir string)) {
	dir, err := ioutil.TempDir("", "simple-app")
	Expect(err).ToNot(HaveOccurred())
	defer os.RemoveAll(dir)

	tempfile := filepath.Join(dir, "index.html")
	err = ioutil.WriteFile(tempfile, []byte("Banana Pants"), 0666)
	Expect(err).ToNot(HaveOccurred())

	err = ioutil.WriteFile(filepath.Join(dir, "Staticfile"), nil, 0666)
	Expect(err).ToNot(HaveOccurred())

	f(dir)
}

// AppGUID returns the GUID for an app in the currently targeted space.
func AppGUID(appName string) string {
	session := CF("app", appName, "--guid")
	Eventually(session).Should(Exit(0))
	return strings.TrimSpace(string(session.Out.Contents()))
}

func WriteManifest(path string, manifest map[string]interface{}) {
	body, err := yaml.Marshal(manifest)
	Expect(err).ToNot(HaveOccurred())
	err = ioutil.WriteFile(path, body, 0666)
	Expect(err).ToNot(HaveOccurred())
}

func ConfirmStagingLogs(session *Session) {
	Eventually(session).Should(Say(`(?i)Creating container|Successfully created container|Staging\.\.\.|Staging process started \.\.\.|Staging Complete|Exit status 0|Uploading droplet\.\.\.|Uploading complete`))
}

// Zipit zips the source into a .zip file in the target dir
func Zipit(source, target, prefix string) error {
	// Thanks to Svett Ralchev
	// http://blog.ralch.com/tutorial/golang-working-with-zip/

	zipfile, err := os.Create(target)
	if err != nil {
		return err
	}
	defer zipfile.Close()

	if prefix != "" {
		_, err = io.WriteString(zipfile, prefix)
		if err != nil {
			return err
		}
	}

	archive := zip.NewWriter(zipfile)
	defer archive.Close()

	err = filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if path == source {
			return nil
		}

		header, err := zip.FileInfoHeader(info)
		if err != nil {
			return err
		}
		header.Name, err = filepath.Rel(source, path)
		if err != nil {
			return err
		}

		header.Name = filepath.ToSlash(header.Name)

		if info.IsDir() {
			header.Name += "/"
			header.SetMode(0755)
		} else {
			header.Method = zip.Deflate
			header.SetMode(0744)
		}

		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()

		_, err = io.Copy(writer, file)
		return err
	})

	return err
}
<|endoftext|>


<commit_before>package network

import (
	"io"
	"io/ioutil"
	"net/http"
)

func NewClient(config Config) Client {
	return Client{
		config: config,
	}
}

type Client struct {
	config Config
}

type Config struct {
	Host          string
	SkipVerifySSL bool
	TraceWriter   io.Writer
}

type Request struct {
	Method                string
	Path                  string
	Authorization         authorization
	IfMatch               string
	Body                  requestBody
	AcceptableStatusCodes []int
	DoNotFollowRedirects  bool
}

type Response struct {
	Code    int
	Body    []byte
	Headers http.Header
}

func (c Client) MakeRequest(req Request) (Response, error) {
	if req.AcceptableStatusCodes == nil {
		panic("acceptable status codes for this request were not set")
	}

	var bodyReader io.Reader
	var contentType string
	if req.Body != nil {
		var err error
		bodyReader, contentType, err = req.Body.Encode()
		if err != nil {
			return Response{}, newRequestBodyMarshalError(err)
		}
	}

	requestURL := c.config.Host + req.Path
	request, err := http.NewRequest(req.Method, requestURL, bodyReader)
	if err != nil {
		return Response{}, newRequestConfigurationError(err)
	}

	if req.Authorization != nil {
		request.Header.Set("Authorization", req.Authorization.Authorization())
	}

	request.Header.Set("Accept", "application/json")

	if contentType != "" {
		request.Header.Set("Content-Type", contentType)
	}
	if req.IfMatch != "" {
		request.Header.Set("If-Match", req.IfMatch)
	}

	c.printRequest(request)

	var resp *http.Response
	transport := buildTransport(c.config.SkipVerifySSL)
	if req.DoNotFollowRedirects {
		resp, err = transport.RoundTrip(request)
	} else {
		client := &http.Client{Transport: transport}
		resp, err = client.Do(request)
	}
	if err != nil {
		return Response{}, newRequestHTTPError(err)
	}
	defer resp.Body.Close()

	responseBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return Response{}, newResponseReadError(err)
	}

	parsedResponse := Response{
		Code:    resp.StatusCode,
		Body:    responseBody,
		Headers: resp.Header,
	}
	c.printResponse(parsedResponse)

	if resp.StatusCode == http.StatusNotFound {
		return Response{}, newNotFoundError(responseBody)
	}

	if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
		return Response{}, newUnauthorizedError(responseBody)
	}

	for _, acceptableCode := range req.AcceptableStatusCodes {
		if resp.StatusCode == acceptableCode {
			return parsedResponse, nil
		}
	}

	return Response{}, newUnexpectedStatusError(resp.StatusCode, responseBody)
}
<commit_msg>Pulls out some of the conditional logic for network.Client.MakeRequest<commit_after>package network

import (
	"io"
	"io/ioutil"
	"net/http"
)

func NewClient(config Config) Client {
	return Client{
		config: config,
	}
}

type Client struct {
	config Config
}

type Config struct {
	Host          string
	SkipVerifySSL bool
	TraceWriter   io.Writer
}

type Request struct {
	Method                string
	Path                  string
	Authorization         authorization
	IfMatch               string
	Body                  requestBody
	AcceptableStatusCodes []int
	DoNotFollowRedirects  bool
}

type Response struct {
	Code    int
	Body    []byte
	Headers http.Header
}

func (c Client) MakeRequest(req Request) (Response, error) {
	if req.AcceptableStatusCodes == nil {
		panic("acceptable status codes for this request were not set")
	}

	request, err := c.buildRequest(req)
	if err != nil {
		return Response{}, err
	}

	var resp *http.Response
	transport := buildTransport(c.config.SkipVerifySSL)
	if req.DoNotFollowRedirects {
		resp, err = transport.RoundTrip(request)
	} else {
		client := &http.Client{Transport: transport}
		resp, err = client.Do(request)
	}
	if err != nil {
		return Response{}, newRequestHTTPError(err)
	}
	defer resp.Body.Close()

	responseBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return Response{}, newResponseReadError(err)
	}

	return c.handleResponse(req, Response{
		Code:    resp.StatusCode,
		Body:    responseBody,
		Headers: resp.Header,
	})
}

func (c Client) buildRequest(req Request) (*http.Request, error) {
	var bodyReader io.Reader
	var contentType string
	if req.Body != nil {
		var err error
		bodyReader, contentType, err = req.Body.Encode()
		if err != nil {
			return &http.Request{}, newRequestBodyMarshalError(err)
		}
	}

	requestURL := c.config.Host + req.Path
	request, err := http.NewRequest(req.Method, requestURL, bodyReader)
	if err != nil {
		return &http.Request{}, newRequestConfigurationError(err)
	}

	if req.Authorization != nil {
		request.Header.Set("Authorization", req.Authorization.Authorization())
	}

	request.Header.Set("Accept", "application/json")

	if contentType != "" {
		request.Header.Set("Content-Type", contentType)
	}
	if req.IfMatch != "" {
		request.Header.Set("If-Match", req.IfMatch)
	}

	c.printRequest(request)

	return request, nil
}

func (c Client) handleResponse(request Request, response Response) (Response, error) {
	c.printResponse(response)

	if response.Code == http.StatusNotFound {
		return Response{}, newNotFoundError(response.Body)
	}

	if response.Code == http.StatusUnauthorized || response.Code == http.StatusForbidden {
		return Response{}, newUnauthorizedError(response.Body)
	}

	for _, acceptableCode := range request.AcceptableStatusCodes {
		if response.Code == acceptableCode {
			return response, nil
		}
	}

	return Response{}, newUnexpectedStatusError(response.Code, response.Body)
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package publish defines the exposure keys publishing API.
package publish

import (
	"time"

	"github.com/google/exposure-notifications-server/internal/authorizedapp"
	"github.com/google/exposure-notifications-server/internal/database"
	"github.com/google/exposure-notifications-server/internal/setup"
)

// Compile-time check to assert this config matches requirements.
var _ setup.AuthorizedAppConfigProvider = (*Config)(nil)
var _ setup.DBConfigProvider = (*Config)(nil)

// Config represents the configuration and associated environment variables for
// the publish components.
type Config struct {
	Port               string        `envconfig:"PORT" default:"8080"`
	MinRequestDuration time.Duration `envconfig:"TARGET_REQUEST_DURATION" default:"5s"`
	MaxKeysOnPublish   int           `envconfig:"MAX_KEYS_ON_PUBLISH" default:"14"`
	MaxIntervalAge     time.Duration `envconfig:"MAX_INTERVAL_AGE_ON_PUBLISH" default:"360h"`
	TruncateWindow     time.Duration `envconfig:"TRUNCATE_WINDOW" default:"1h"`

	// Bypass flags for local development and testing.
	BypassSafetyNet   bool `envconfig:"BYPASS_SAFETYNET"`
	BypassDeviceCheck bool `envconfig:"BYPASS_DEVICECHECK"`

	AuthorizedApp *authorizedapp.Config
	Database      *database.Config
}

// AuthorizedApp returns the configuration for authorizedapp.
func (c *Config) AuthorizedAppConfig() *authorizedapp.Config {
	return c.AuthorizedApp
}

// DB returns the configuration for the databse.
func (c *Config) DB() *database.Config {
	return c.Database
}
<commit_msg>default max number of keys should be 15, per e2e testing w/ client team (#339)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package publish defines the exposure keys publishing API.
package publish

import (
	"time"

	"github.com/google/exposure-notifications-server/internal/authorizedapp"
	"github.com/google/exposure-notifications-server/internal/database"
	"github.com/google/exposure-notifications-server/internal/setup"
)

// Compile-time check to assert this config matches requirements.
var _ setup.AuthorizedAppConfigProvider = (*Config)(nil)
var _ setup.DBConfigProvider = (*Config)(nil)

// Config represents the configuration and associated environment variables for
// the publish components.
type Config struct {
	Port               string        `envconfig:"PORT" default:"8080"`
	MinRequestDuration time.Duration `envconfig:"TARGET_REQUEST_DURATION" default:"5s"`
	MaxKeysOnPublish   int           `envconfig:"MAX_KEYS_ON_PUBLISH" default:"15"`
	MaxIntervalAge     time.Duration `envconfig:"MAX_INTERVAL_AGE_ON_PUBLISH" default:"360h"`
	TruncateWindow     time.Duration `envconfig:"TRUNCATE_WINDOW" default:"1h"`

	// Bypass flags for local development and testing.
	BypassSafetyNet   bool `envconfig:"BYPASS_SAFETYNET"`
	BypassDeviceCheck bool `envconfig:"BYPASS_DEVICECHECK"`

	AuthorizedApp *authorizedapp.Config
	Database      *database.Config
}

// AuthorizedApp returns the configuration for authorizedapp.
func (c *Config) AuthorizedAppConfig() *authorizedapp.Config {
	return c.AuthorizedApp
}

// DB returns the configuration for the databse.
func (c *Config) DB() *database.Config {
	return c.Database
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package publish defines the exposure keys publishing API.
package publish

import (
	"time"

	"github.com/google/exposure-notifications-server/internal/authorizedapp"
	"github.com/google/exposure-notifications-server/internal/database"
	"github.com/google/exposure-notifications-server/internal/observability"
	"github.com/google/exposure-notifications-server/internal/publish/model"
	"github.com/google/exposure-notifications-server/internal/setup"
	"github.com/google/exposure-notifications-server/internal/verification"
	"github.com/google/exposure-notifications-server/pkg/secrets"
)

// Compile-time check to assert this config matches requirements.
var _ setup.AuthorizedAppConfigProvider = (*Config)(nil)
var _ setup.DatabaseConfigProvider = (*Config)(nil)
var _ setup.SecretManagerConfigProvider = (*Config)(nil)
var _ setup.ObservabilityExporterConfigProvider = (*Config)(nil)
var _ model.TransformerConfig = (*Config)(nil)

// Config represents the configuration and associated environment variables for
// the publish components.
type Config struct {
	AuthorizedApp         authorizedapp.Config
	Database              database.Config
	SecretManager         secrets.Config
	Verification          verification.Config
	ObservabilityExporter observability.Config

	Port             string `env:"PORT, default=8080"`
	MaxKeysOnPublish uint   `env:"MAX_KEYS_ON_PUBLISH, default=20"`
	// Provides compatibility w/ 1.5 release.
	MaxSameStartIntervalKeys     uint          `env:"MAX_SAME_START_INTERVAL_KEYS, default=3"`
	MaxIntervalAge               time.Duration `env:"MAX_INTERVAL_AGE_ON_PUBLISH, default=360h"`
	MaxMagnitudeSymptomOnsetDays uint          `env:"MAX_SYMPTOM_ONSET_DAYS, default=21"`
	CreatedAtTruncateWindow      time.Duration `env:"TRUNCATE_WINDOW, default=1h"`

	// Flags for local development and testing. This will cause still valid keys
	// to not be embargoed.
	// Normally "still valid" keys can be accepted, but are embargoed.
	ReleaseSameDayKeys bool `env:"DEBUG_RELEASE_SAME_DAY_KEYS"`
}

func (c *Config) MaxExposureKeys() uint {
	return c.MaxKeysOnPublish
}

func (c *Config) MaxSameDayKeys() uint {
	return c.MaxSameStartIntervalKeys
}

func (c *Config) MaxIntervalStartAge() time.Duration {
	return c.MaxIntervalAge
}

func (c *Config) TruncateWindow() time.Duration {
	return c.CreatedAtTruncateWindow
}

func (c *Config) MaxSymptomOnsetDays() uint {
	return c.MaxMagnitudeSymptomOnsetDays
}

func (c *Config) DebugReleaseSameDayKeys() bool {
	return c.ReleaseSameDayKeys
}

func (c *Config) AuthorizedAppConfig() *authorizedapp.Config {
	return &c.AuthorizedApp
}

func (c *Config) DatabaseConfig() *database.Config {
	return &c.Database
}

func (c *Config) SecretManagerConfig() *secrets.Config {
	return &c.SecretManager
}

func (c *Config) ObservabilityExporterConfig() *observability.Config {
	return &c.ObservabilityExporter
}
<commit_msg>up the default key batch size. Supports early key release and post diagnosis upload for 2 weeks. (#732)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package publish defines the exposure keys publishing API.
package publish

import (
	"time"

	"github.com/google/exposure-notifications-server/internal/authorizedapp"
	"github.com/google/exposure-notifications-server/internal/database"
	"github.com/google/exposure-notifications-server/internal/observability"
	"github.com/google/exposure-notifications-server/internal/publish/model"
	"github.com/google/exposure-notifications-server/internal/setup"
	"github.com/google/exposure-notifications-server/internal/verification"
	"github.com/google/exposure-notifications-server/pkg/secrets"
)

// Compile-time check to assert this config matches requirements.
var _ setup.AuthorizedAppConfigProvider = (*Config)(nil)
var _ setup.DatabaseConfigProvider = (*Config)(nil)
var _ setup.SecretManagerConfigProvider = (*Config)(nil)
var _ setup.ObservabilityExporterConfigProvider = (*Config)(nil)
var _ model.TransformerConfig = (*Config)(nil)

// Config represents the configuration and associated environment variables for
// the publish components.
type Config struct {
	AuthorizedApp         authorizedapp.Config
	Database              database.Config
	SecretManager         secrets.Config
	Verification          verification.Config
	ObservabilityExporter observability.Config

	Port             string `env:"PORT, default=8080"`
	MaxKeysOnPublish uint   `env:"MAX_KEYS_ON_PUBLISH, default=30"`
	// Provides compatibility w/ 1.5 release.
	MaxSameStartIntervalKeys     uint          `env:"MAX_SAME_START_INTERVAL_KEYS, default=3"`
	MaxIntervalAge               time.Duration `env:"MAX_INTERVAL_AGE_ON_PUBLISH, default=360h"`
	MaxMagnitudeSymptomOnsetDays uint          `env:"MAX_SYMPTOM_ONSET_DAYS, default=21"`
	CreatedAtTruncateWindow      time.Duration `env:"TRUNCATE_WINDOW, default=1h"`

	// Flags for local development and testing. This will cause still valid keys
	// to not be embargoed.
	// Normally "still valid" keys can be accepted, but are embargoed.
	ReleaseSameDayKeys bool `env:"DEBUG_RELEASE_SAME_DAY_KEYS"`
}

func (c *Config) MaxExposureKeys() uint {
	return c.MaxKeysOnPublish
}

func (c *Config) MaxSameDayKeys() uint {
	return c.MaxSameStartIntervalKeys
}

func (c *Config) MaxIntervalStartAge() time.Duration {
	return c.MaxIntervalAge
}

func (c *Config) TruncateWindow() time.Duration {
	return c.CreatedAtTruncateWindow
}

func (c *Config) MaxSymptomOnsetDays() uint {
	return c.MaxMagnitudeSymptomOnsetDays
}

func (c *Config) DebugReleaseSameDayKeys() bool {
	return c.ReleaseSameDayKeys
}

func (c *Config) AuthorizedAppConfig() *authorizedapp.Config {
	return &c.AuthorizedApp
}

func (c *Config) DatabaseConfig() *database.Config {
	return &c.Database
}

func (c *Config) SecretManagerConfig() *secrets.Config {
	return &c.SecretManager
}

func (c *Config) ObservabilityExporterConfig() *observability.Config {
	return &c.ObservabilityExporter
}
<|endoftext|>


<commit_before>package interpreter

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"
)

type CommandNotFound struct {
	Name string
	Err  error
}

func (this CommandNotFound) Stringer() string {
	return fmt.Sprintf("'%s' is not recognized as an internal or external command,\noperable program or batch file", this.Name)
}

func (this CommandNotFound) Error() string {
	return this.Stringer()
}

type NextT int

const (
	THROUGH  NextT = 0
	CONTINUE NextT = 1
	SHUTDOWN NextT = 2
)

func (this NextT) String() string {
	switch this {
	case THROUGH:
		return "THROUGH"
	case CONTINUE:
		return "CONTINUE"
	case SHUTDOWN:
		return "SHUTDOWN"
	default:
		return "UNKNOWN"
	}
}

type Interpreter struct {
	exec.Cmd
	Stdio        [3]*os.File
	HookCount    int
	Closer       []io.Closer
	Tag          interface{}
	PipeSeq      [2]uint
	IsBackGround bool
}

func (this *Interpreter) closeAtEnd() {
	if this.Closer != nil {
		for _, c := range this.Closer {
			c.Close()
		}
		this.Closer = nil
	}
}

func New() *Interpreter {
	this := Interpreter{
		Stdio: [3]*os.File{os.Stdin, os.Stdout, os.Stderr},
	}
	this.Stdin = os.Stdin
	this.Stdout = os.Stdout
	this.Stderr = os.Stderr
	this.PipeSeq[0] = pipeSeq
	this.PipeSeq[1] = 0
	this.Tag = nil
	return &this
}

func (this *Interpreter) SetStdin(f *os.File) {
	this.Stdio[0] = f
	this.Stdin = f
}
func (this *Interpreter) SetStdout(f *os.File) {
	this.Stdio[1] = f
	this.Stdout = f
}
func (this *Interpreter) SetStderr(f *os.File) {
	this.Stdio[2] = f
	this.Stderr = f
}

func (this *Interpreter) Clone() *Interpreter {
	rv := new(Interpreter)
	rv.Stdio[0] = this.Stdio[0]
	rv.Stdio[1] = this.Stdio[1]
	rv.Stdio[2] = this.Stdio[2]
	rv.Stdin = this.Stdin
	rv.Stdout = this.Stdout
	rv.Stderr = this.Stderr
	rv.HookCount = this.HookCount
	rv.Tag = this.Tag
	rv.PipeSeq = rv.PipeSeq
	rv.Closer = nil
	return rv
}

type ArgsHookT func(it *Interpreter, args []string) []string

var argsHook = func(it *Interpreter, args []string) []string {
	return args
}

func SetArgsHook(argsHook_ ArgsHookT) (rv ArgsHookT) {
	rv, argsHook = argsHook, argsHook_
	return
}

type HookT func(*Interpreter) (NextT, error)

var hook = func(*Interpreter) (NextT, error) {
	return THROUGH, nil
}

func SetHook(hook_ HookT) (rv HookT) {
	rv, hook = hook, hook_
	return
}

var OnCommandNotFound = func(this *Interpreter, err error) error {
	err = &CommandNotFound{this.Args[0], err}
	return err
}

var errorStatusPattern = regexp.MustCompile("^exit status ([0-9]+)")
var ErrorLevel string

func nvl(a *os.File, b *os.File) *os.File {
	if a != nil {
		return a
	} else {
		return b
	}
}

func (this *Interpreter) Spawnvp() (NextT, error) {
	var whatToDo NextT = CONTINUE
	var err error = nil

	if len(this.Args) > 0 {
		whatToDo, err = hook(this)
		if whatToDo == THROUGH {
			this.Path, err = exec.LookPath(this.Args[0])
			if err == nil {
				err = this.Run()
			} else {
				err = OnCommandNotFound(this, err)
			}
			whatToDo = CONTINUE
		}
	}
	return whatToDo, err
}

type result_t struct {
	NextValue NextT
	Error     error
}

var pipeSeq uint = 0

func (this *Interpreter) Interpret(text string) (next NextT, err error) {
	next = CONTINUE
	err = nil

	statements, statementsErr := Parse(text)
	if statementsErr != nil {
		return CONTINUE, statementsErr
	}
	if argsHook != nil {
		for _, pipeline := range statements {
			for _, state := range pipeline {
				state.Argv = argsHook(this, state.Argv)
			}
		}
	}
	for _, pipeline := range statements {
		var pipeIn *os.File = nil
		pipeSeq++
		isBackGround := false

		for i, state := range pipeline {
			if state.Term == "&" {
				isBackGround = true
			}

			cmd := new(Interpreter)
			cmd.PipeSeq[0] = pipeSeq
			cmd.PipeSeq[1] = uint(1 + i)
			cmd.IsBackGround = isBackGround
			cmd.Tag = this.Tag
			cmd.HookCount = this.HookCount
			cmd.SetStdin(nvl(this.Stdio[0], os.Stdin))
			cmd.SetStdout(nvl(this.Stdio[1], os.Stdout))
			cmd.SetStderr(nvl(this.Stdio[2], os.Stderr))

			var err error = nil

			if pipeIn != nil {
				cmd.SetStdin(pipeIn)
				cmd.Closer = append(cmd.Closer, pipeIn)
				pipeIn = nil
			}

			if state.Term[0] == '|' {
				var pipeOut *os.File
				pipeIn, pipeOut, err = os.Pipe()
				cmd.SetStdout(pipeOut)
				if state.Term == "|&" {
					cmd.SetStderr(pipeOut)
				}
				cmd.Closer = append(cmd.Closer, pipeOut)
			}

			for _, red := range state.Redirect {
				err = red.OpenOn(cmd)
				if err != nil {
					return CONTINUE, err
				}
			}

			cmd.Args = state.Argv
			if i == len(pipeline)-1 && state.Term != "&" {
				next, err = cmd.Spawnvp()
				cmd.closeAtEnd()
				if err != nil {
					m := errorStatusPattern.FindStringSubmatch(err.Error())
					if m != nil {
						ErrorLevel = m[1]
						err = nil
					} else {
						ErrorLevel = "-1"
					}
				} else {
					ErrorLevel = "0"
				}
			} else {
				go func(cmd1 *Interpreter) {
					cmd1.Spawnvp()
					cmd1.closeAtEnd()
				}(cmd)
			}
		}
	}
	return
}
<commit_msg>Fixed: CommandNotFound Error was not printed when hook is not used.<commit_after>package interpreter

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"
)

type CommandNotFound struct {
	Name string
	Err  error
}

func (this CommandNotFound) Stringer() string {
	return fmt.Sprintf("'%s' is not recognized as an internal or external command,\noperable program or batch file", this.Name)
}

func (this CommandNotFound) Error() string {
	return this.Stringer()
}

type NextT int

const (
	THROUGH  NextT = 0
	CONTINUE NextT = 1
	SHUTDOWN NextT = 2
)

func (this NextT) String() string {
	switch this {
	case THROUGH:
		return "THROUGH"
	case CONTINUE:
		return "CONTINUE"
	case SHUTDOWN:
		return "SHUTDOWN"
	default:
		return "UNKNOWN"
	}
}

type Interpreter struct {
	exec.Cmd
	Stdio        [3]*os.File
	HookCount    int
	Closer       []io.Closer
	Tag          interface{}
	PipeSeq      [2]uint
	IsBackGround bool
}

func (this *Interpreter) closeAtEnd() {
	if this.Closer != nil {
		for _, c := range this.Closer {
			c.Close()
		}
		this.Closer = nil
	}
}

func New() *Interpreter {
	this := Interpreter{
		Stdio: [3]*os.File{os.Stdin, os.Stdout, os.Stderr},
	}
	this.Stdin = os.Stdin
	this.Stdout = os.Stdout
	this.Stderr = os.Stderr
	this.PipeSeq[0] = pipeSeq
	this.PipeSeq[1] = 0
	this.Tag = nil
	return &this
}

func (this *Interpreter) SetStdin(f *os.File) {
	this.Stdio[0] = f
	this.Stdin = f
}
func (this *Interpreter) SetStdout(f *os.File) {
	this.Stdio[1] = f
	this.Stdout = f
}
func (this *Interpreter) SetStderr(f *os.File) {
	this.Stdio[2] = f
	this.Stderr = f
}

func (this *Interpreter) Clone() *Interpreter {
	rv := new(Interpreter)
	rv.Stdio[0] = this.Stdio[0]
	rv.Stdio[1] = this.Stdio[1]
	rv.Stdio[2] = this.Stdio[2]
	rv.Stdin = this.Stdin
	rv.Stdout = this.Stdout
	rv.Stderr = this.Stderr
	rv.HookCount = this.HookCount
	rv.Tag = this.Tag
	rv.PipeSeq = rv.PipeSeq
	rv.Closer = nil
	return rv
}

type ArgsHookT func(it *Interpreter, args []string) []string

var argsHook = func(it *Interpreter, args []string) []string {
	return args
}

func SetArgsHook(argsHook_ ArgsHookT) (rv ArgsHookT) {
	rv, argsHook = argsHook, argsHook_
	return
}

type HookT func(*Interpreter) (NextT, error)

var hook = func(*Interpreter) (NextT, error) {
	return THROUGH, nil
}

func SetHook(hook_ HookT) (rv HookT) {
	rv, hook = hook, hook_
	return
}

var OnCommandNotFound = func(this *Interpreter, err error) error {
	err = &CommandNotFound{this.Args[0], err}
	fmt.Fprintln(this.Stderr, err.Error())
	return err
}

var errorStatusPattern = regexp.MustCompile("^exit status ([0-9]+)")
var ErrorLevel string

func nvl(a *os.File, b *os.File) *os.File {
	if a != nil {
		return a
	} else {
		return b
	}
}

func (this *Interpreter) Spawnvp() (NextT, error) {
	var whatToDo NextT = CONTINUE
	var err error = nil

	if len(this.Args) > 0 {
		whatToDo, err = hook(this)
		if whatToDo == THROUGH {
			this.Path, err = exec.LookPath(this.Args[0])
			if err == nil {
				err = this.Run()
			} else {
				err = OnCommandNotFound(this, err)
			}
			whatToDo = CONTINUE
		}
	}
	return whatToDo, err
}

type result_t struct {
	NextValue NextT
	Error     error
}

var pipeSeq uint = 0

func (this *Interpreter) Interpret(text string) (next NextT, err error) {
	next = CONTINUE
	err = nil

	statements, statementsErr := Parse(text)
	if statementsErr != nil {
		return CONTINUE, statementsErr
	}
	if argsHook != nil {
		for _, pipeline := range statements {
			for _, state := range pipeline {
				state.Argv = argsHook(this, state.Argv)
			}
		}
	}
	for _, pipeline := range statements {
		var pipeIn *os.File = nil
		pipeSeq++
		isBackGround := false

		for i, state := range pipeline {
			if state.Term == "&" {
				isBackGround = true
			}

			cmd := new(Interpreter)
			cmd.PipeSeq[0] = pipeSeq
			cmd.PipeSeq[1] = uint(1 + i)
			cmd.IsBackGround = isBackGround
			cmd.Tag = this.Tag
			cmd.HookCount = this.HookCount
			cmd.SetStdin(nvl(this.Stdio[0], os.Stdin))
			cmd.SetStdout(nvl(this.Stdio[1], os.Stdout))
			cmd.SetStderr(nvl(this.Stdio[2], os.Stderr))

			var err error = nil

			if pipeIn != nil {
				cmd.SetStdin(pipeIn)
				cmd.Closer = append(cmd.Closer, pipeIn)
				pipeIn = nil
			}

			if state.Term[0] == '|' {
				var pipeOut *os.File
				pipeIn, pipeOut, err = os.Pipe()
				cmd.SetStdout(pipeOut)
				if state.Term == "|&" {
					cmd.SetStderr(pipeOut)
				}
				cmd.Closer = append(cmd.Closer, pipeOut)
			}

			for _, red := range state.Redirect {
				err = red.OpenOn(cmd)
				if err != nil {
					return CONTINUE, err
				}
			}

			cmd.Args = state.Argv
			if i == len(pipeline)-1 && state.Term != "&" {
				next, err = cmd.Spawnvp()
				cmd.closeAtEnd()
				if err != nil {
					m := errorStatusPattern.FindStringSubmatch(err.Error())
					if m != nil {
						ErrorLevel = m[1]
						err = nil
					} else {
						ErrorLevel = "-1"
					}
				} else {
					ErrorLevel = "0"
				}
			} else {
				go func(cmd1 *Interpreter) {
					cmd1.Spawnvp()
					cmd1.closeAtEnd()
				}(cmd)
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>package di_test

import (
	"net/http"

	"github.com/clavoie/di"
)

type HttpDep interface {
	// optional: only dependencies that perform cleanup need to implement
	// IHttpClosable
	di.IHttpClosable
}

type HttpImpl struct{}

func (hi *HttpImpl) Di_HttpClose() {
	// called after the request has ended
}

func NewHttpDep() HttpDep { return new(HttpImpl) }

func ExampleIHttpResolver_httpHandler() {
	defs := di.NewDefs()

	// one instance of HttpDep created per http request
	err := defs.Add(NewHttpDep, di.PerHttpRequest)

	if err != nil {
		panic(err)
	}

	resolver, err := di.NewResolver(defs)
	if err != nil {
		panic(err)
	}

	logger := func(error) {}
	errHandler := func(err error, w http.ResponseWriter, r *http.Request) {
		logger(err)
		w.WriteHeader(http.StatusInternalServerError)
	}
	depHandler := func(dep HttpDep) {}
	normalHandler := func(http.ResponseWriter, *http.Request) {}
	urlDefs := []struct {
		url     string
		handler interface{}
	}{
		{"/", depHandler},
		{"/thing", normalHandler},
	}

	for _, urlDef := range urlDefs {
		handleFunc, err := resolver.HttpHandler(urlDef.handler, errHandler)
		if err != nil {
			panic(err)
		}

		http.HandleFunc(urlDef.url, handleFunc)
	}

	// listen and serve
}
<commit_msg>a better http example<commit_after>package di_test

import (
	"net/http"

	"github.com/clavoie/di"
)

type HttpDep interface {
	// optional: only dependencies that perform cleanup need to implement
	// IHttpClosable
	di.IHttpClosable
}

type HttpImpl struct{}

func (hi *HttpImpl) Di_HttpClose() {
	// called after the request has ended
}

func NewHttpDep() HttpDep { return new(HttpImpl) }

func NewHttpResolver() di.IHttpResolver {
	defs := di.NewDefs()

	// one instance of HttpDep created per http request
	err := defs.Add(NewHttpDep, di.PerHttpRequest)

	if err != nil {
		panic(err)
	}

	resolver, err := di.NewResolver(defs)
	if err != nil {
		panic(err)
	}

	return resolver
}

var resolver = NewHttpResolver()

func DepHandler(dep HttpDep)                         {}
func HttpHandler(http.ResponseWriter, *http.Request) {}
func WriteToLog(err error)                           { /* etc */ }

func ErrHandler(err error, w http.ResponseWriter, r *http.Request) {
	WriteToLog(err)
	w.WriteHeader(http.StatusInternalServerError)
}

var urlDefs = []struct {
	url     string
	handler interface{}
}{
	{"/", DepHandler},
	{"/thing", HttpHandler},
}

func ExampleIHttpResolver_httpHandler() {
	for _, urlDef := range urlDefs {
		handleFunc, err := resolver.HttpHandler(urlDef.handler, ErrHandler)
		if err != nil {
			panic(err)
		}

		http.HandleFunc(urlDef.url, handleFunc)
	}

	// listen and serve
}
<|endoftext|>


<commit_before>package docker_app_runner

//go:generate counterfeiter -o fake_app_runner/fake_app_runner.go . AppRunner

import (
	"fmt"

	"errors"
	"github.com/cloudfoundry-incubator/receptor"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/pivotal-cf-experimental/lattice-cli/app_runner/docker_repository_name_formatter"
)

type AppRunner interface {
	StartDockerApp(params StartDockerAppParams) error
	ScaleApp(name string, instances int) error
	RemoveApp(name string) error
	AppExists(name string) (bool, error)
	NumOfRunningAppInstances(name string) (int, error)
}

type PortConfig struct {
	Monitored uint32
	Exposed   []uint32
}

func (portConfig PortConfig) IsEmpty() bool {
	return portConfig.Monitored == 0 && len(portConfig.Exposed) == 0
}

type StartDockerAppParams struct {
	Name                 string
	StartCommand         string
	DockerImagePath      string
	AppArgs              []string
	EnvironmentVariables map[string]string
	Privileged           bool
	Monitor              bool
	Instances            int
	MemoryMB             int
	DiskMB               int
	Ports                PortConfig
	WorkingDir           string
}

const (
	healthcheckDownloadUrl string = "http://file_server.service.dc1.consul:8080/v1/static/healthcheck.tgz"
	lrpDomain              string = "lattice"
)

type appRunner struct {
	receptorClient receptor.Client
	systemDomain   string
}

func New(receptorClient receptor.Client, systemDomain string) AppRunner {
	return &appRunner{receptorClient, systemDomain}
}

func (appRunner *appRunner) StartDockerApp(params StartDockerAppParams) error {
	exposedContainsMonitored := false
	for _, port := range params.Ports.Exposed {
		if port == params.Ports.Monitored {
			exposedContainsMonitored = true
			break
		}
	}

	if params.Monitor && !exposedContainsMonitored {
		return errors.New("Monitored Port must be in the Exposed Ports.")
	}

	if exists, err := appRunner.desiredLRPExists(params.Name); err != nil {
		return err
	} else if exists {
		return newExistingAppError(params.Name)
	}

	if err := appRunner.receptorClient.UpsertDomain(lrpDomain, 0); err != nil {
		return err
	}

	return appRunner.desireLrp(params)
}

func (appRunner *appRunner) ScaleApp(name string, instances int) error {
	if exists, err := appRunner.desiredLRPExists(name); err != nil {
		return err
	} else if !exists {
		return newAppNotStartedError(name)
	}

	return appRunner.updateLrp(name, instances)
}

func (appRunner *appRunner) RemoveApp(name string) error {
	if lrpExists, err := appRunner.desiredLRPExists(name); err != nil {
		return err
	} else if !lrpExists {
		return newAppNotStartedError(name)
	}

	return appRunner.receptorClient.DeleteDesiredLRP(name)
}

func (appRunner *appRunner) AppExists(name string) (bool, error) {
	actualLRPs, err := appRunner.receptorClient.ActualLRPs()
	if err != nil {
		return false, err
	}

	for _, actualLRP := range actualLRPs {
		if actualLRP.ProcessGuid == name {
			return true, nil
		}
	}

	return false, nil
}

func (appRunner *appRunner) NumOfRunningAppInstances(name string) (count int, err error) {
	runningInstances := 0
	instances, err := appRunner.receptorClient.ActualLRPsByProcessGuid(name)
	if err != nil {
		return 0, err
	}

	for _, instance := range instances {
		if instance.State == receptor.ActualLRPStateRunning {
			runningInstances += 1
		}
	}

	return runningInstances, nil
}

func (appRunner *appRunner) desiredLRPExists(name string) (exists bool, err error) {
	desiredLRPs, err := appRunner.receptorClient.DesiredLRPs()
	if err != nil {
		return false, err
	}

	for _, desiredLRP := range desiredLRPs {
		if desiredLRP.ProcessGuid == name {
			return true, nil
		}
	}

	return false, nil
}

func (appRunner *appRunner) desireLrp(params StartDockerAppParams) error {
	dockerImageUrl, err := docker_repository_name_formatter.FormatForReceptor(params.DockerImagePath)
	if err != nil {
		return err
	}

	req := receptor.DesiredLRPCreateRequest{
		ProcessGuid:          params.Name,
		Domain:               lrpDomain,
		RootFSPath:           dockerImageUrl,
		Instances:            params.Instances,
		Stack:                "lucid64",
		Routes:               []string{fmt.Sprintf("%s.%s", params.Name, appRunner.systemDomain)},
		MemoryMB:             params.MemoryMB,
		DiskMB:               params.DiskMB,
		Ports:                params.Ports.Exposed,
		LogGuid:              params.Name,
		LogSource:            "APP",
		EnvironmentVariables: buildEnvironmentVariables(params.EnvironmentVariables, params.Ports.Monitored),
		Setup: &models.DownloadAction{
			From: healthcheckDownloadUrl,
			To:   "/tmp",
		},
		Action: &models.RunAction{
			Path:       params.StartCommand,
			Args:       params.AppArgs,
			Privileged: params.Privileged,
			Dir:        params.WorkingDir,
		},
	}

	if params.Monitor {
		req.Monitor = &models.RunAction{
			Path:      "/tmp/healthcheck",
			Args:      []string{"-port", fmt.Sprintf("%d", params.Ports.Monitored)},
			LogSource: "HEALTH",
		}
	}
	err = appRunner.receptorClient.CreateDesiredLRP(req)

	return err
}

func buildEnvironmentVariables(environmentVariables map[string]string, port uint32) []receptor.EnvironmentVariable {
	appEnvVars := make([]receptor.EnvironmentVariable, 0, len(environmentVariables)+1)
	for name, value := range environmentVariables {
		appEnvVars = append(appEnvVars, receptor.EnvironmentVariable{Name: name, Value: value})
	}
	return append(appEnvVars, receptor.EnvironmentVariable{Name: "PORT", Value: fmt.Sprintf("%d", port)})
}

func (appRunner *appRunner) updateLrp(name string, instances int) error {
	err := appRunner.receptorClient.UpdateDesiredLRP(
		name,
		receptor.DesiredLRPUpdateRequest{
			Instances: &instances,
		},
	)

	return err
}
<commit_msg>Clean up building env var slice<commit_after>package docker_app_runner

//go:generate counterfeiter -o fake_app_runner/fake_app_runner.go . AppRunner

import (
	"fmt"

	"errors"
	"github.com/cloudfoundry-incubator/receptor"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/pivotal-cf-experimental/lattice-cli/app_runner/docker_repository_name_formatter"
)

type AppRunner interface {
	StartDockerApp(params StartDockerAppParams) error
	ScaleApp(name string, instances int) error
	RemoveApp(name string) error
	AppExists(name string) (bool, error)
	NumOfRunningAppInstances(name string) (int, error)
}

type PortConfig struct {
	Monitored uint32
	Exposed   []uint32
}

func (portConfig PortConfig) IsEmpty() bool {
	return portConfig.Monitored == 0 && len(portConfig.Exposed) == 0
}

type StartDockerAppParams struct {
	Name                 string
	StartCommand         string
	DockerImagePath      string
	AppArgs              []string
	EnvironmentVariables map[string]string
	Privileged           bool
	Monitor              bool
	Instances            int
	MemoryMB             int
	DiskMB               int
	Ports                PortConfig
	WorkingDir           string
}

const (
	healthcheckDownloadUrl string = "http://file_server.service.dc1.consul:8080/v1/static/healthcheck.tgz"
	lrpDomain              string = "lattice"
)

type appRunner struct {
	receptorClient receptor.Client
	systemDomain   string
}

func New(receptorClient receptor.Client, systemDomain string) AppRunner {
	return &appRunner{receptorClient, systemDomain}
}

func (appRunner *appRunner) StartDockerApp(params StartDockerAppParams) error {
	exposedContainsMonitored := false
	for _, port := range params.Ports.Exposed {
		if port == params.Ports.Monitored {
			exposedContainsMonitored = true
			break
		}
	}

	if params.Monitor && !exposedContainsMonitored {
		return errors.New("Monitored Port must be in the Exposed Ports.")
	}

	if exists, err := appRunner.desiredLRPExists(params.Name); err != nil {
		return err
	} else if exists {
		return newExistingAppError(params.Name)
	}

	if err := appRunner.receptorClient.UpsertDomain(lrpDomain, 0); err != nil {
		return err
	}

	return appRunner.desireLrp(params)
}

func (appRunner *appRunner) ScaleApp(name string, instances int) error {
	if exists, err := appRunner.desiredLRPExists(name); err != nil {
		return err
	} else if !exists {
		return newAppNotStartedError(name)
	}

	return appRunner.updateLrp(name, instances)
}

func (appRunner *appRunner) RemoveApp(name string) error {
	if lrpExists, err := appRunner.desiredLRPExists(name); err != nil {
		return err
	} else if !lrpExists {
		return newAppNotStartedError(name)
	}

	return appRunner.receptorClient.DeleteDesiredLRP(name)
}

func (appRunner *appRunner) AppExists(name string) (bool, error) {
	actualLRPs, err := appRunner.receptorClient.ActualLRPs()
	if err != nil {
		return false, err
	}

	for _, actualLRP := range actualLRPs {
		if actualLRP.ProcessGuid == name {
			return true, nil
		}
	}

	return false, nil
}

func (appRunner *appRunner) NumOfRunningAppInstances(name string) (count int, err error) {
	runningInstances := 0
	instances, err := appRunner.receptorClient.ActualLRPsByProcessGuid(name)
	if err != nil {
		return 0, err
	}

	for _, instance := range instances {
		if instance.State == receptor.ActualLRPStateRunning {
			runningInstances += 1
		}
	}

	return runningInstances, nil
}

func (appRunner *appRunner) desiredLRPExists(name string) (exists bool, err error) {
	desiredLRPs, err := appRunner.receptorClient.DesiredLRPs()
	if err != nil {
		return false, err
	}

	for _, desiredLRP := range desiredLRPs {
		if desiredLRP.ProcessGuid == name {
			return true, nil
		}
	}

	return false, nil
}

func (appRunner *appRunner) desireLrp(params StartDockerAppParams) error {
	dockerImageUrl, err := docker_repository_name_formatter.FormatForReceptor(params.DockerImagePath)
	if err != nil {
		return err
	}

	envVars := buildEnvironmentVariables(params.EnvironmentVariables)
	envVars = append(envVars, receptor.EnvironmentVariable{Name: "PORT", Value: fmt.Sprintf("%d", params.Ports.Monitored)})

	req := receptor.DesiredLRPCreateRequest{
		ProcessGuid:          params.Name,
		Domain:               lrpDomain,
		RootFSPath:           dockerImageUrl,
		Instances:            params.Instances,
		Stack:                "lucid64",
		Routes:               []string{fmt.Sprintf("%s.%s", params.Name, appRunner.systemDomain)},
		MemoryMB:             params.MemoryMB,
		DiskMB:               params.DiskMB,
		Ports:                params.Ports.Exposed,
		LogGuid:              params.Name,
		LogSource:            "APP",
		EnvironmentVariables: envVars,
		Setup: &models.DownloadAction{
			From: healthcheckDownloadUrl,
			To:   "/tmp",
		},
		Action: &models.RunAction{
			Path:       params.StartCommand,
			Args:       params.AppArgs,
			Privileged: params.Privileged,
			Dir:        params.WorkingDir,
		},
	}

	if params.Monitor {
		req.Monitor = &models.RunAction{
			Path:      "/tmp/healthcheck",
			Args:      []string{"-port", fmt.Sprintf("%d", params.Ports.Monitored)},
			LogSource: "HEALTH",
		}
	}
	err = appRunner.receptorClient.CreateDesiredLRP(req)

	return err
}

func buildEnvironmentVariables(environmentVariables map[string]string) []receptor.EnvironmentVariable {
	appEnvVars := make([]receptor.EnvironmentVariable, 0, len(environmentVariables)+1)
	for name, value := range environmentVariables {
		appEnvVars = append(appEnvVars, receptor.EnvironmentVariable{Name: name, Value: value})
	}
	return appEnvVars
}

func (appRunner *appRunner) updateLrp(name string, instances int) error {
	err := appRunner.receptorClient.UpdateDesiredLRP(
		name,
		receptor.DesiredLRPUpdateRequest{
			Instances: &instances,
		},
	)

	return err
}
<|endoftext|>