<commit_before>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package go_kafka_client

import (
	"fmt"
	"math"
	"sync"
	"time"
)

type consumerFetcherManager struct {
	config                         *ConsumerConfig
	numStreams                     int
	closeFinished                  chan bool
	updateLock                     sync.RWMutex
	partitionMap                   map[TopicAndPartition]*partitionTopicInfo
	fetcherRoutineMap              map[int]*consumerFetcherRoutine
	shuttingDown                   bool
	updateInProgress               bool
	updatedCond                    *sync.Cond
	disconnectChannelsForPartition chan TopicAndPartition

	metrics *consumerMetrics
	client  LowLevelClient
}

func (m *consumerFetcherManager) String() string {
	return fmt.Sprintf("%s-manager", m.config.Consumerid)
}

func newConsumerFetcherManager(config *ConsumerConfig, disconnectChannelsForPartition chan TopicAndPartition, metrics *consumerMetrics) *consumerFetcherManager {
	manager := &consumerFetcherManager{
		config:                         config,
		closeFinished:                  make(chan bool),
		partitionMap:                   make(map[TopicAndPartition]*partitionTopicInfo),
		fetcherRoutineMap:              make(map[int]*consumerFetcherRoutine),
		disconnectChannelsForPartition: disconnectChannelsForPartition,
		client:  config.LowLevelClient,
		metrics: metrics,
	}
	manager.updatedCond = sync.NewCond(manager.updateLock.RLocker())

	return manager
}

func (m *consumerFetcherManager) startConnections(topicInfos []*partitionTopicInfo, numStreams int) {
	Debug(m, "Fetcher Manager started")
	Debugf(m, "TopicInfos = %s", topicInfos)
	m.numStreams = numStreams

	partitionAndOffsets := make(map[TopicAndPartition]int64)

	m.updateInProgress = true
	inWriteLock(&m.updateLock, func() {
		Debug(m, "Updating fetcher configuration")
		newPartitionMap := make(map[TopicAndPartition]*partitionTopicInfo)
		for _, providedInfo := range topicInfos {
			topicAndPartition := TopicAndPartition{providedInfo.Topic, providedInfo.Partition}
			if currentInfo, alreadyFetching := m.partitionMap[topicAndPartition]; alreadyFetching {
				newPartitionMap[topicAndPartition] = currentInfo
				continue
			} else {
				newPartitionMap[topicAndPartition] = providedInfo
				partitionAndOffsets[topicAndPartition] = providedInfo.FetchedOffset
			}
		}

		Debugf(m, "Got new list of partitions to process %v", newPartitionMap)
		Debugf(m, "All partitions map: %v", m.partitionMap)
		//receive obsolete partitions map
		for k := range newPartitionMap {
			delete(m.partitionMap, k)
		}

		//removing obsolete partitions and tearing down associated jobs
		topicPartitionsToRemove := make([]TopicAndPartition, 0)
		for tp := range m.partitionMap {
			topicPartitionsToRemove = append(topicPartitionsToRemove, tp)
			m.disconnectChannelsForPartition <- tp
			delete(m.partitionMap, tp)
		}
		Debugf(m, "There are obsolete partitions %v", topicPartitionsToRemove)

		//removing unnecessary partition-fetchRoutine bindings
		for _, fetcher := range m.fetcherRoutineMap {
			Debugf(m, "Fetcher %s parition map before obsolete partitions removal", fetcher, fetcher.partitionMap)
			fetcher.removePartitions(topicPartitionsToRemove)
			Debugf(m, "Fetcher %s parition map after obsolete partitions removal", fetcher, fetcher.partitionMap)
		}
		m.shutdownIdleFetchers()

		//updating partitions map with requested partitions
		for k, v := range newPartitionMap {
			m.partitionMap[k] = v
		}
		m.addFetcherForPartitions(partitionAndOffsets)

		m.updateInProgress = false
		Debugf(m, "Applied new partition map %v", m.partitionMap)
	})

	Debug(m, "Notifying all waiters about completed update")
	m.updatedCond.Broadcast()
}

func (m *consumerFetcherManager) addFetcherForPartitions(partitionAndOffsets map[TopicAndPartition]int64) {
	Infof(m, "Adding fetcher for partitions %v", partitionAndOffsets)
	partitionsPerFetcher := make(map[int]map[TopicAndPartition]int64)
	for topicAndPartition, offset := range partitionAndOffsets {
		fetcherId := m.getFetcherId(topicAndPartition.Topic, topicAndPartition.Partition)
		if partitionsPerFetcher[fetcherId] == nil {
			partitionsPerFetcher[fetcherId] = make(map[TopicAndPartition]int64)
		}
		partitionsPerFetcher[fetcherId][topicAndPartition] = offset
	}

	Debugf(m, "partitionsPerFetcher: %v", partitionsPerFetcher)
	for fetcherId, partitionOffsets := range partitionsPerFetcher {
		if m.fetcherRoutineMap[fetcherId] == nil {
			Debugf(m, "Starting new fetcher")
			fetcherRoutine := newConsumerFetcher(m,
				fmt.Sprintf("ConsumerFetcherRoutine-%s-%d", m.config.Consumerid, fetcherId),
				m.partitionMap)
			m.fetcherRoutineMap[fetcherId] = fetcherRoutine
			go fetcherRoutine.start()
		}

		partitionToOffsetMap := make(map[TopicAndPartition]int64)
		for tp, offset := range partitionOffsets {
			partitionToOffsetMap[tp] = offset
		}
		m.fetcherRoutineMap[fetcherId].addPartitions(partitionToOffsetMap)
	}
}

func (m *consumerFetcherManager) getFetcherId(topic string, partitionId int32) int {
	return int(math.Abs(float64(31*hash(topic)+partitionId))) % int(m.numStreams)
}

func (m *consumerFetcherManager) shutdownIdleFetchers() {
	Debug(m, "Shutting down idle fetchers")
	for key, fetcher := range m.fetcherRoutineMap {
		if len(fetcher.partitionMap) <= 0 {
			Debugf(m, "There is idle fetcher: %s", fetcher)
			<-fetcher.close()
			delete(m.fetcherRoutineMap, key)
		}
	}
	Debug(m, "Closed idle fetchers")
}

func (m *consumerFetcherManager) closeAllFetchers() {
	Info(m, "Closing fetchers")
	inWriteLock(&m.updateLock, func() {
		Debug(m, "Stopping all message buffers and removing paritions")
		for k, v := range m.partitionMap {
			v.Buffer.stop()
			delete(m.partitionMap, k)
		}

		Debugf(m, "Trying to close %d fetchers", len(m.fetcherRoutineMap))
		for key, fetcher := range m.fetcherRoutineMap {
			Debugf(m, "Closing %s", fetcher)
			<-fetcher.close()
			Debugf(m, "Closed %s", fetcher)
			delete(m.fetcherRoutineMap, key)
		}
	})
}

func (m *consumerFetcherManager) close() <-chan bool {
	Info(m, "Closing manager")
	go func() {
		Info(m, "Setting shutdown flag")
		m.shuttingDown = true
		m.closeAllFetchers()
		m.updatedCond.Broadcast()
		m.partitionMap = nil
		m.closeFinished <- true
		Info(m, "Successfully closed all fetcher manager routines")
	}()

	return m.closeFinished
}

type consumerFetcherRoutine struct {
	manager         *consumerFetcherManager
	name            string
	allPartitionMap map[TopicAndPartition]*partitionTopicInfo
	partitionMap    map[TopicAndPartition]int64
	lock            sync.Mutex
	closeFinished   chan bool
	fetchStopper    chan bool
	askNext         chan TopicAndPartition
}

func (f *consumerFetcherRoutine) String() string {
	return f.name
}

func newConsumerFetcher(m *consumerFetcherManager, name string, allPartitionMap map[TopicAndPartition]*partitionTopicInfo) *consumerFetcherRoutine {
	return &consumerFetcherRoutine{
		manager:         m,
		name:            name,
		allPartitionMap: allPartitionMap,
		partitionMap:    make(map[TopicAndPartition]int64),
		closeFinished:   make(chan bool),
		fetchStopper:    make(chan bool),
		askNext:         make(chan TopicAndPartition),
	}
}

func (f *consumerFetcherRoutine) start() {
	Info(f, "Fetcher started")
	for {
		Trace(f, "Waiting for asknext or die")
		ts := time.Now()
		select {
		case nextTopicPartition := <-f.askNext:
			{
				f.manager.metrics.FetchersIdleTimer().Update(time.Since(ts))
				Debugf(f, "Received asknext for %s", &nextTopicPartition)
				inLock(&f.lock, func() {
					if !f.manager.shuttingDown {
						Trace(f, "Next asked")
						offset := InvalidOffset
						Debugf(f, "Partition map: %v", f.partitionMap)
						if existingOffset, exists := f.partitionMap[nextTopicPartition]; exists {
							offset = existingOffset
						}

						if f.allPartitionMap[nextTopicPartition] == nil {
							Warnf(f, "Message buffer for partition %s has been terminated. Aborting processing task...", nextTopicPartition)
							return
						}

						var messages []*Message
						var err error
						f.manager.metrics.FetchDurationTimer().Time(func() {
							messages, err = f.manager.client.Fetch(nextTopicPartition.Topic, nextTopicPartition.Partition, offset)
						})

						if err != nil {
							if f.manager.client.IsOffsetOutOfRange(err) {
								Warnf(f, "Current offset %d for topic %s and partition %s is out of range.", offset, nextTopicPartition.Topic, nextTopicPartition.Partition)
								f.handleOffsetOutOfRange(&nextTopicPartition)
							} else {
								Warnf(f, "Got a fetch error: %s", err)
								//TODO new backoff type?
								time.Sleep(1 * time.Second)
							}
						}

						go f.processPartitionData(nextTopicPartition, messages)

						if len(messages) == 0 {
							go f.requeue(nextTopicPartition)
						}
					}
				})
				time.Sleep(f.manager.config.FetchRequestBackoff)
			}
		case <-f.fetchStopper:
			{
				Info(f, "Stopped fetcher")
				return
			}
		}
	}
}

func (f *consumerFetcherRoutine) requeue(topicPartition TopicAndPartition) {
	Debug(f, "Asknext received no messages, requeue request")
	time.Sleep(f.manager.config.RequeueAskNextBackoff)
	f.askNext <- topicPartition
	Debugf(f, "Requeued request %s", topicPartition)
}

func (f *consumerFetcherRoutine) addPartitions(partitionAndOffsets map[TopicAndPartition]int64) {
	Debugf(f, "Adding partitions: %v", partitionAndOffsets)
	newPartitions := make(map[TopicAndPartition]chan TopicAndPartition)
	inLock(&f.lock, func() {
		for topicAndPartition, offset := range partitionAndOffsets {
			if _, contains := f.partitionMap[topicAndPartition]; !contains {
				validOffset := offset + 1
				if isOffsetInvalid(offset) {
					f.handleOffsetOutOfRange(&topicAndPartition)
				} else {
					f.partitionMap[topicAndPartition] = validOffset
				}
				f.manager.partitionMap[topicAndPartition].Buffer.start(f.askNext)
				newPartitions[topicAndPartition] = f.askNext
				Debugf(f, "Owner of %s", topicAndPartition)
			}
		}
	})

	for topicAndPartition, askNext := range newPartitions {
		Debugf(f, "Sending ask next to %s for %s", f, topicAndPartition)
	Loop:
		for {
			select {
			case askNext <- topicAndPartition:
				break Loop
			case <-time.After(1 * time.Second):
				{
					if f.manager.shuttingDown {
						return
					}
				}
			}
		}
		Debugf(f, "Sent ask next to %s for %s", f, topicAndPartition)
	}
}

func (f *consumerFetcherRoutine) processPartitionData(topicAndPartition TopicAndPartition, messages []*Message) {
	Trace(f, "Trying to acquire lock for partition processing")
	inReadLock(&f.manager.updateLock, func() {
		for f.manager.updateInProgress {
			f.manager.updatedCond.Wait()
		}
		Tracef(f, "Processing partition data for %s", topicAndPartition)
		if len(messages) > 0 {
			f.partitionMap[topicAndPartition] = messages[len(messages)-1].Offset + 1
			f.allPartitionMap[topicAndPartition].Buffer.addBatch(messages)
			Debugf(f, "Sent partition data to %s", topicAndPartition)
		} else {
			Trace(f, "Got empty message. Ignoring...")
		}
	})
}

func (f *consumerFetcherRoutine) handleOffsetOutOfRange(topicAndPartition *TopicAndPartition) {
	newOffset, err := f.manager.client.GetAvailableOffset(topicAndPartition.Topic, topicAndPartition.Partition, f.manager.config.AutoOffsetReset)
	if err != nil {
		Errorf(f, "Cannot get available offset for %s. Reason: %s", topicAndPartition, err)
		return
	}

    // Do not use a lock here just because it's faster and it will be checked afterwards if we should still fetch that TopicPartition
    // This just guarantees we dont get a nil pointer dereference here
    if topicInfo, exists := f.allPartitionMap[*topicAndPartition]; exists {
        topicInfo.FetchedOffset = newOffset
        f.partitionMap[*topicAndPartition] = newOffset
    }
}

func (f *consumerFetcherRoutine) removeAllPartitions() {
	partitions := make([]TopicAndPartition, 0)
	for topicPartition, _ := range f.partitionMap {
		partitions = append(partitions, topicPartition)
	}
	f.removePartitions(partitions)
}

func (f *consumerFetcherRoutine) removePartitions(partitions []TopicAndPartition) {
	Debug(f, "Remove partitions")
	inLock(&f.lock, func() {
		for _, topicAndPartition := range partitions {
			delete(f.partitionMap, topicAndPartition)
		}
	})
}

func (f *consumerFetcherRoutine) close() <-chan bool {
	Info(f, "Closing fetcher")
	go func() {
		f.fetchStopper <- true
		f.removeAllPartitions()
		Debug(f, "Sending close finished")
		f.closeFinished <- true
		Debug(f, "Sent close finished")
	}()
	return f.closeFinished
}
<commit_msg>removed fetch backoff<commit_after>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package go_kafka_client

import (
	"fmt"
	"math"
	"sync"
	"time"
)

type consumerFetcherManager struct {
	config                         *ConsumerConfig
	numStreams                     int
	closeFinished                  chan bool
	updateLock                     sync.RWMutex
	partitionMap                   map[TopicAndPartition]*partitionTopicInfo
	fetcherRoutineMap              map[int]*consumerFetcherRoutine
	shuttingDown                   bool
	updateInProgress               bool
	updatedCond                    *sync.Cond
	disconnectChannelsForPartition chan TopicAndPartition

	metrics *consumerMetrics
	client  LowLevelClient
}

func (m *consumerFetcherManager) String() string {
	return fmt.Sprintf("%s-manager", m.config.Consumerid)
}

func newConsumerFetcherManager(config *ConsumerConfig, disconnectChannelsForPartition chan TopicAndPartition, metrics *consumerMetrics) *consumerFetcherManager {
	manager := &consumerFetcherManager{
		config:                         config,
		closeFinished:                  make(chan bool),
		partitionMap:                   make(map[TopicAndPartition]*partitionTopicInfo),
		fetcherRoutineMap:              make(map[int]*consumerFetcherRoutine),
		disconnectChannelsForPartition: disconnectChannelsForPartition,
		client:  config.LowLevelClient,
		metrics: metrics,
	}
	manager.updatedCond = sync.NewCond(manager.updateLock.RLocker())

	return manager
}

func (m *consumerFetcherManager) startConnections(topicInfos []*partitionTopicInfo, numStreams int) {
	Debug(m, "Fetcher Manager started")
	Debugf(m, "TopicInfos = %s", topicInfos)
	m.numStreams = numStreams

	partitionAndOffsets := make(map[TopicAndPartition]int64)

	m.updateInProgress = true
	inWriteLock(&m.updateLock, func() {
		Debug(m, "Updating fetcher configuration")
		newPartitionMap := make(map[TopicAndPartition]*partitionTopicInfo)
		for _, providedInfo := range topicInfos {
			topicAndPartition := TopicAndPartition{providedInfo.Topic, providedInfo.Partition}
			if currentInfo, alreadyFetching := m.partitionMap[topicAndPartition]; alreadyFetching {
				newPartitionMap[topicAndPartition] = currentInfo
				continue
			} else {
				newPartitionMap[topicAndPartition] = providedInfo
				partitionAndOffsets[topicAndPartition] = providedInfo.FetchedOffset
			}
		}

		Debugf(m, "Got new list of partitions to process %v", newPartitionMap)
		Debugf(m, "All partitions map: %v", m.partitionMap)
		//receive obsolete partitions map
		for k := range newPartitionMap {
			delete(m.partitionMap, k)
		}

		//removing obsolete partitions and tearing down associated jobs
		topicPartitionsToRemove := make([]TopicAndPartition, 0)
		for tp := range m.partitionMap {
			topicPartitionsToRemove = append(topicPartitionsToRemove, tp)
			m.disconnectChannelsForPartition <- tp
			delete(m.partitionMap, tp)
		}
		Debugf(m, "There are obsolete partitions %v", topicPartitionsToRemove)

		//removing unnecessary partition-fetchRoutine bindings
		for _, fetcher := range m.fetcherRoutineMap {
			Debugf(m, "Fetcher %s parition map before obsolete partitions removal", fetcher, fetcher.partitionMap)
			fetcher.removePartitions(topicPartitionsToRemove)
			Debugf(m, "Fetcher %s parition map after obsolete partitions removal", fetcher, fetcher.partitionMap)
		}
		m.shutdownIdleFetchers()

		//updating partitions map with requested partitions
		for k, v := range newPartitionMap {
			m.partitionMap[k] = v
		}
		m.addFetcherForPartitions(partitionAndOffsets)

		m.updateInProgress = false
		Debugf(m, "Applied new partition map %v", m.partitionMap)
	})

	Debug(m, "Notifying all waiters about completed update")
	m.updatedCond.Broadcast()
}

func (m *consumerFetcherManager) addFetcherForPartitions(partitionAndOffsets map[TopicAndPartition]int64) {
	Infof(m, "Adding fetcher for partitions %v", partitionAndOffsets)
	partitionsPerFetcher := make(map[int]map[TopicAndPartition]int64)
	for topicAndPartition, offset := range partitionAndOffsets {
		fetcherId := m.getFetcherId(topicAndPartition.Topic, topicAndPartition.Partition)
		if partitionsPerFetcher[fetcherId] == nil {
			partitionsPerFetcher[fetcherId] = make(map[TopicAndPartition]int64)
		}
		partitionsPerFetcher[fetcherId][topicAndPartition] = offset
	}

	Debugf(m, "partitionsPerFetcher: %v", partitionsPerFetcher)
	for fetcherId, partitionOffsets := range partitionsPerFetcher {
		if m.fetcherRoutineMap[fetcherId] == nil {
			Debugf(m, "Starting new fetcher")
			fetcherRoutine := newConsumerFetcher(m,
				fmt.Sprintf("ConsumerFetcherRoutine-%s-%d", m.config.Consumerid, fetcherId),
				m.partitionMap)
			m.fetcherRoutineMap[fetcherId] = fetcherRoutine
			go fetcherRoutine.start()
		}

		partitionToOffsetMap := make(map[TopicAndPartition]int64)
		for tp, offset := range partitionOffsets {
			partitionToOffsetMap[tp] = offset
		}
		m.fetcherRoutineMap[fetcherId].addPartitions(partitionToOffsetMap)
	}
}

func (m *consumerFetcherManager) getFetcherId(topic string, partitionId int32) int {
	return int(math.Abs(float64(31*hash(topic)+partitionId))) % int(m.numStreams)
}

func (m *consumerFetcherManager) shutdownIdleFetchers() {
	Debug(m, "Shutting down idle fetchers")
	for key, fetcher := range m.fetcherRoutineMap {
		if len(fetcher.partitionMap) <= 0 {
			Debugf(m, "There is idle fetcher: %s", fetcher)
			<-fetcher.close()
			delete(m.fetcherRoutineMap, key)
		}
	}
	Debug(m, "Closed idle fetchers")
}

func (m *consumerFetcherManager) closeAllFetchers() {
	Info(m, "Closing fetchers")
	inWriteLock(&m.updateLock, func() {
		Debug(m, "Stopping all message buffers and removing paritions")
		for k, v := range m.partitionMap {
			v.Buffer.stop()
			delete(m.partitionMap, k)
		}

		Debugf(m, "Trying to close %d fetchers", len(m.fetcherRoutineMap))
		for key, fetcher := range m.fetcherRoutineMap {
			Debugf(m, "Closing %s", fetcher)
			<-fetcher.close()
			Debugf(m, "Closed %s", fetcher)
			delete(m.fetcherRoutineMap, key)
		}
	})
}

func (m *consumerFetcherManager) close() <-chan bool {
	Info(m, "Closing manager")
	go func() {
		Info(m, "Setting shutdown flag")
		m.shuttingDown = true
		m.closeAllFetchers()
		m.updatedCond.Broadcast()
		m.partitionMap = nil
		m.closeFinished <- true
		Info(m, "Successfully closed all fetcher manager routines")
	}()

	return m.closeFinished
}

type consumerFetcherRoutine struct {
	manager         *consumerFetcherManager
	name            string
	allPartitionMap map[TopicAndPartition]*partitionTopicInfo
	partitionMap    map[TopicAndPartition]int64
	lock            sync.Mutex
	closeFinished   chan bool
	fetchStopper    chan bool
	askNext         chan TopicAndPartition
}

func (f *consumerFetcherRoutine) String() string {
	return f.name
}

func newConsumerFetcher(m *consumerFetcherManager, name string, allPartitionMap map[TopicAndPartition]*partitionTopicInfo) *consumerFetcherRoutine {
	return &consumerFetcherRoutine{
		manager:         m,
		name:            name,
		allPartitionMap: allPartitionMap,
		partitionMap:    make(map[TopicAndPartition]int64),
		closeFinished:   make(chan bool),
		fetchStopper:    make(chan bool),
		askNext:         make(chan TopicAndPartition),
	}
}

func (f *consumerFetcherRoutine) start() {
	Info(f, "Fetcher started")
	for {
		Trace(f, "Waiting for asknext or die")
		ts := time.Now()
		select {
		case nextTopicPartition := <-f.askNext:
			{
				f.manager.metrics.FetchersIdleTimer().Update(time.Since(ts))
				Debugf(f, "Received asknext for %s", &nextTopicPartition)
				inLock(&f.lock, func() {
					if !f.manager.shuttingDown {
						Trace(f, "Next asked")
						offset := InvalidOffset
						Debugf(f, "Partition map: %v", f.partitionMap)
						if existingOffset, exists := f.partitionMap[nextTopicPartition]; exists {
							offset = existingOffset
						}

						if f.allPartitionMap[nextTopicPartition] == nil {
							Warnf(f, "Message buffer for partition %s has been terminated. Aborting processing task...", nextTopicPartition)
							return
						}

						var messages []*Message
						var err error
						f.manager.metrics.FetchDurationTimer().Time(func() {
							messages, err = f.manager.client.Fetch(nextTopicPartition.Topic, nextTopicPartition.Partition, offset)
						})

						if err != nil {
							if f.manager.client.IsOffsetOutOfRange(err) {
								Warnf(f, "Current offset %d for topic %s and partition %s is out of range.", offset, nextTopicPartition.Topic, nextTopicPartition.Partition)
								f.handleOffsetOutOfRange(&nextTopicPartition)
							} else {
								Warnf(f, "Got a fetch error: %s", err)
								//TODO new backoff type?
								time.Sleep(1 * time.Second)
							}
						}

						go f.processPartitionData(nextTopicPartition, messages)

						if len(messages) == 0 {
							go f.requeue(nextTopicPartition)
						}
					}
				})
			}
		case <-f.fetchStopper:
			{
				Info(f, "Stopped fetcher")
				return
			}
		}
	}
}

func (f *consumerFetcherRoutine) requeue(topicPartition TopicAndPartition) {
	Debug(f, "Asknext received no messages, requeue request")
	time.Sleep(f.manager.config.RequeueAskNextBackoff)
	f.askNext <- topicPartition
	Debugf(f, "Requeued request %s", topicPartition)
}

func (f *consumerFetcherRoutine) addPartitions(partitionAndOffsets map[TopicAndPartition]int64) {
	Debugf(f, "Adding partitions: %v", partitionAndOffsets)
	newPartitions := make(map[TopicAndPartition]chan TopicAndPartition)
	inLock(&f.lock, func() {
		for topicAndPartition, offset := range partitionAndOffsets {
			if _, contains := f.partitionMap[topicAndPartition]; !contains {
				validOffset := offset + 1
				if isOffsetInvalid(offset) {
					f.handleOffsetOutOfRange(&topicAndPartition)
				} else {
					f.partitionMap[topicAndPartition] = validOffset
				}
				f.manager.partitionMap[topicAndPartition].Buffer.start(f.askNext)
				newPartitions[topicAndPartition] = f.askNext
				Debugf(f, "Owner of %s", topicAndPartition)
			}
		}
	})

	for topicAndPartition, askNext := range newPartitions {
		Debugf(f, "Sending ask next to %s for %s", f, topicAndPartition)
	Loop:
		for {
			select {
			case askNext <- topicAndPartition:
				break Loop
			case <-time.After(1 * time.Second):
				{
					if f.manager.shuttingDown {
						return
					}
				}
			}
		}
		Debugf(f, "Sent ask next to %s for %s", f, topicAndPartition)
	}
}

func (f *consumerFetcherRoutine) processPartitionData(topicAndPartition TopicAndPartition, messages []*Message) {
	Trace(f, "Trying to acquire lock for partition processing")
	inReadLock(&f.manager.updateLock, func() {
		for f.manager.updateInProgress {
			f.manager.updatedCond.Wait()
		}
		Tracef(f, "Processing partition data for %s", topicAndPartition)
		if len(messages) > 0 {
			f.partitionMap[topicAndPartition] = messages[len(messages)-1].Offset + 1
			f.allPartitionMap[topicAndPartition].Buffer.addBatch(messages)
			Debugf(f, "Sent partition data to %s", topicAndPartition)
		} else {
			Trace(f, "Got empty message. Ignoring...")
		}
	})
}

func (f *consumerFetcherRoutine) handleOffsetOutOfRange(topicAndPartition *TopicAndPartition) {
	newOffset, err := f.manager.client.GetAvailableOffset(topicAndPartition.Topic, topicAndPartition.Partition, f.manager.config.AutoOffsetReset)
	if err != nil {
		Errorf(f, "Cannot get available offset for %s. Reason: %s", topicAndPartition, err)
		return
	}

    // Do not use a lock here just because it's faster and it will be checked afterwards if we should still fetch that TopicPartition
    // This just guarantees we dont get a nil pointer dereference here
    if topicInfo, exists := f.allPartitionMap[*topicAndPartition]; exists {
        topicInfo.FetchedOffset = newOffset
        f.partitionMap[*topicAndPartition] = newOffset
    }
}

func (f *consumerFetcherRoutine) removeAllPartitions() {
	partitions := make([]TopicAndPartition, 0)
	for topicPartition, _ := range f.partitionMap {
		partitions = append(partitions, topicPartition)
	}
	f.removePartitions(partitions)
}

func (f *consumerFetcherRoutine) removePartitions(partitions []TopicAndPartition) {
	Debug(f, "Remove partitions")
	inLock(&f.lock, func() {
		for _, topicAndPartition := range partitions {
			delete(f.partitionMap, topicAndPartition)
		}
	})
}

func (f *consumerFetcherRoutine) close() <-chan bool {
	Info(f, "Closing fetcher")
	go func() {
		f.fetchStopper <- true
		f.removeAllPartitions()
		Debug(f, "Sending close finished")
		f.closeFinished <- true
		Debug(f, "Sent close finished")
	}()
	return f.closeFinished
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	"socialapi/workers/common/mux"
	"socialapi/workers/realtime/gatekeeper"

	"github.com/koding/runner"
)

var (
	Name = "IntegrationWebhook"
)

func main() {
	r := runner.New(Name)
	if err := r.Init(); err != nil {
		fmt.Println(err)
		return
	}

	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)

	iConfig := appConfig.Integration

	mc := mux.NewConfig(Name, iConfig.Host, iConfig.Port)
	m := mux.New(mc, r.Log)
	m.Metrics = r.Metrics

	h, err := api.NewHandler(r.Log)
	if err != nil {
		r.Log.Fatal("Could not initialize webhook worker")
	}
	h.AddHandlers(m)

	go r.Listen()

	m.Listen()
	defer m.Close()

	r.Wait()
}
<commit_msg>webhook: print handler errors in initialization<commit_after>package main

import (
	"fmt"
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	"socialapi/workers/common/mux"
	"socialapi/workers/realtime/gatekeeper"

	"github.com/koding/runner"
)

var (
	Name = "IntegrationWebhook"
)

func main() {
	r := runner.New(Name)
	if err := r.Init(); err != nil {
		fmt.Println(err)
		return
	}

	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)

	iConfig := appConfig.Integration

	mc := mux.NewConfig(Name, iConfig.Host, iConfig.Port)
	m := mux.New(mc, r.Log)
	m.Metrics = r.Metrics

	h, err := api.NewHandler(r.Log)
	if err != nil {
		r.Log.Fatal("Could not initialize webhook worker: %s", err)
	}
	h.AddHandlers(m)

	go r.Listen()

	m.Listen()
	defer m.Close()

	r.Wait()
}
<|endoftext|>


<commit_before>package analytics

import . "github.com/visionmedia/go-debug"
import "github.com/jehiah/go-strftime"
import "github.com/xtgo/uuid"
import . "encoding/json"
import "io/ioutil"
import "net/http"
import "errors"
import "bytes"
import "sync"
import "time"

// Library version.
const Version = "1.1.0"

// Default API end-point.
const api = "https://api.segment.io"

// Message type.
type Message map[string]interface{}

// Debug.
var debug = Debug("analytics")

// Segment.io client
type Client struct {
	FlushAt    int
	FlushAfter time.Duration
	Endpoint   string
	Key        string
	buffer     []Message
	wg         sync.WaitGroup
	sync.Mutex
}

// Batch message.
type batch struct {
	Messages  []Message `json:"batch"`
	MessageId string    `json:"messageId"`
}

// New creates a new Segment.io client
// with the given write key.
func New(key string) *Client {
	c := &Client{
		FlushAt:    20,
		FlushAfter: 5 * time.Second,
		Key:        key,
		Endpoint:   api,
		buffer:     make([]Message, 0),
	}

	go c.start()

	return c
}

// Stop the client, flush messages and wait for requests to complete.
func (c *Client) Stop() {
	c.flush()
	c.wg.Wait()
}

// start flusher.
func (c *Client) start() {
	go func() {
		for {
			time.Sleep(c.FlushAfter)
			debug("interval %v reached", c.FlushAfter)
			c.flush()
		}
	}()
}

// Alias buffers an "alias" message.
func (c *Client) Alias(msg Message) error {
	if msg["userId"] == nil {
		return errors.New("You must pass a 'userId'.")
	}

	if msg["previousId"] == nil {
		return errors.New("You must pass a 'previousId'.")
	}

	c.queue(message(msg, "alias"))

	return nil
}

// Page buffers an "page" message.
func (c *Client) Page(msg Message) error {
	if msg["userId"] == nil && msg["anonymousId"] == nil {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	c.queue(message(msg, "page"))

	return nil
}

// Screen buffers an "screen" message.
func (c *Client) Screen(msg Message) error {
	if msg["userId"] == nil && msg["anonymousId"] == nil {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	c.queue(message(msg, "screen"))

	return nil
}

// Group buffers an "group" message.
func (c *Client) Group(msg Message) error {
	if msg["groupId"] == nil {
		return errors.New("You must pass a 'groupId'.")
	}

	if msg["userId"] == nil && msg["anonymousId"] == nil {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	c.queue(message(msg, "group"))

	return nil
}

// Identify buffers an "identify" message.
func (c *Client) Identify(msg Message) error {
	if msg["userId"] == nil && msg["anonymousId"] == nil {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	c.queue(message(msg, "identify"))

	return nil
}

// Track buffers an "track" message.
func (c *Client) Track(msg Message) error {
	if msg["event"] == nil {
		return errors.New("You must pass 'event'.")
	}

	if msg["userId"] == nil && msg["anonymousId"] == nil {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	c.queue(message(msg, "track"))

	return nil
}

// Return a new initialized message map
// with `msg` values and context merged.
func message(msg Message, call string) Message {
	m := newMessage(call)

	if ctx, ok := msg["context"].(map[string]interface{}); ok {
		merge(m["context"].(map[string]interface{}), ctx)
		delete(msg, "context")
	}

	merge(m, msg)

	return m
}

// Return new initialzed message map.
func newMessage(call string) Message {
	return Message{
		"type":      call,
		"timestamp": timestamp(),
		"messageId": uid(),
		"context": map[string]interface{}{
			"library": map[string]interface{}{
				"name":    "analytics-go",
				"version": Version,
			},
		},
	}
}

// Merge two maps.
func merge(dst Message, src Message) {
	for k, v := range src {
		dst[k] = v
	}
}

// Return uuid.
func uid() string {
	return uuid.NewRandom().String()
}

// Return formatted timestamp.
func timestamp() string {
	return strftime.Format("%Y-%m-%dT%H:%M:%S%z", time.Now())
}

// Buffer the given message and flush
// when the buffer exceeds .FlushAt.
func (c *Client) queue(msg Message) {
	c.Lock()
	defer c.Unlock()

	c.wg.Add(1)
	c.buffer = append(c.buffer, msg)

	debug("buffer (%d/%d) %v", len(c.buffer), c.FlushAt, msg)

	if len(c.buffer) >= c.FlushAt {
		go c.flush()
	}
}

// Return a batch message primed
// with context properties.
func batchMessage(msgs []Message) *batch {
	return &batch{
		MessageId: uid(),
		Messages:  msgs,
	}
}

// Flush the buffered messages.
func (c *Client) flush() error {
	c.Lock()

	if len(c.buffer) == 0 {
		debug("no messages to flush")
		c.Unlock()
		return nil
	}

	debug("flushing %d messages", len(c.buffer))
	defer c.wg.Add(-len(c.buffer))
	json, err := Marshal(batchMessage(c.buffer))

	if err != nil {
		debug("error: %v", err)
		c.Unlock()
		return err
	}

	c.buffer = nil
	c.Unlock()

	client := &http.Client{}
	url := c.Endpoint + "/v1/import"
	debug("POST %s with %d bytes", url, len(json))
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(json))

	if err != nil {
		debug("error: %v", err)
		return err
	}

	req.Header.Add("User-Agent", "analytics-go (version: "+Version+")")
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Content-Length", string(len(json)))
	req.SetBasicAuth(c.Key, "")

	res, err := client.Do(req)
	if err != nil {
		debug("error: %v", err)
		return err
	}
	defer res.Body.Close()

	debug("%d response", res.StatusCode)

	if res.StatusCode >= 400 {
		body, _ := ioutil.ReadAll(res.Body)
		debug("error: %s", string(body))
		debug("error: %s", string(json))
	}

	return err
}
<commit_msg>add public .Flush() method<commit_after>package analytics

import . "github.com/visionmedia/go-debug"
import "github.com/jehiah/go-strftime"
import "github.com/xtgo/uuid"
import . "encoding/json"
import "io/ioutil"
import "net/http"
import "errors"
import "bytes"
import "sync"
import "time"

// Library version.
const Version = "1.1.0"

// Default API end-point.
const api = "https://api.segment.io"

// Message type.
type Message map[string]interface{}

// Debug.
var debug = Debug("analytics")

// Segment.io client
type Client struct {
	FlushAt    int
	FlushAfter time.Duration
	Endpoint   string
	Key        string
	buffer     []Message
	wg         sync.WaitGroup
	sync.Mutex
}

// Batch message.
type batch struct {
	Messages  []Message `json:"batch"`
	MessageId string    `json:"messageId"`
}

// New creates a new Segment.io client
// with the given write key.
func New(key string) *Client {
	c := &Client{
		FlushAt:    20,
		FlushAfter: 5 * time.Second,
		Key:        key,
		Endpoint:   api,
		buffer:     make([]Message, 0),
	}

	go c.start()

	return c
}

// Stop the client, flush messages and wait for requests to complete.
func (c *Client) Stop() {
	c.Flush()
	c.wg.Wait()
}

// start flusher.
func (c *Client) start() {
	go func() {
		for {
			time.Sleep(c.FlushAfter)
			debug("interval %v reached", c.FlushAfter)
			c.Flush()
		}
	}()
}

// Alias buffers an "alias" message.
func (c *Client) Alias(msg Message) error {
	if msg["userId"] == nil {
		return errors.New("You must pass a 'userId'.")
	}

	if msg["previousId"] == nil {
		return errors.New("You must pass a 'previousId'.")
	}

	c.queue(message(msg, "alias"))

	return nil
}

// Page buffers an "page" message.
func (c *Client) Page(msg Message) error {
	if msg["userId"] == nil && msg["anonymousId"] == nil {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	c.queue(message(msg, "page"))

	return nil
}

// Screen buffers an "screen" message.
func (c *Client) Screen(msg Message) error {
	if msg["userId"] == nil && msg["anonymousId"] == nil {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	c.queue(message(msg, "screen"))

	return nil
}

// Group buffers an "group" message.
func (c *Client) Group(msg Message) error {
	if msg["groupId"] == nil {
		return errors.New("You must pass a 'groupId'.")
	}

	if msg["userId"] == nil && msg["anonymousId"] == nil {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	c.queue(message(msg, "group"))

	return nil
}

// Identify buffers an "identify" message.
func (c *Client) Identify(msg Message) error {
	if msg["userId"] == nil && msg["anonymousId"] == nil {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	c.queue(message(msg, "identify"))

	return nil
}

// Track buffers an "track" message.
func (c *Client) Track(msg Message) error {
	if msg["event"] == nil {
		return errors.New("You must pass 'event'.")
	}

	if msg["userId"] == nil && msg["anonymousId"] == nil {
		return errors.New("You must pass either an 'anonymousId' or 'userId'.")
	}

	c.queue(message(msg, "track"))

	return nil
}

// Return a new initialized message map
// with `msg` values and context merged.
func message(msg Message, call string) Message {
	m := newMessage(call)

	if ctx, ok := msg["context"].(map[string]interface{}); ok {
		merge(m["context"].(map[string]interface{}), ctx)
		delete(msg, "context")
	}

	merge(m, msg)

	return m
}

// Return new initialzed message map.
func newMessage(call string) Message {
	return Message{
		"type":      call,
		"timestamp": timestamp(),
		"messageId": uid(),
		"context": map[string]interface{}{
			"library": map[string]interface{}{
				"name":    "analytics-go",
				"version": Version,
			},
		},
	}
}

// Merge two maps.
func merge(dst Message, src Message) {
	for k, v := range src {
		dst[k] = v
	}
}

// Return uuid.
func uid() string {
	return uuid.NewRandom().String()
}

// Return formatted timestamp.
func timestamp() string {
	return strftime.Format("%Y-%m-%dT%H:%M:%S%z", time.Now())
}

// Buffer the given message and flush
// when the buffer exceeds .FlushAt.
func (c *Client) queue(msg Message) {
	c.Lock()
	defer c.Unlock()

	c.wg.Add(1)
	c.buffer = append(c.buffer, msg)

	debug("buffer (%d/%d) %v", len(c.buffer), c.FlushAt, msg)

	if len(c.buffer) >= c.FlushAt {
		go c.Flush()
	}
}

// Return a batch message primed
// with context properties.
func batchMessage(msgs []Message) *batch {
	return &batch{
		MessageId: uid(),
		Messages:  msgs,
	}
}

// Flush the buffered messages, returning an error if the request fails.
func (c *Client) Flush() error {
	c.Lock()

	if len(c.buffer) == 0 {
		debug("no messages to flush")
		c.Unlock()
		return nil
	}

	debug("flushing %d messages", len(c.buffer))
	defer c.wg.Add(-len(c.buffer))
	json, err := Marshal(batchMessage(c.buffer))

	if err != nil {
		debug("error: %v", err)
		c.Unlock()
		return err
	}

	c.buffer = nil
	c.Unlock()

	client := &http.Client{}
	url := c.Endpoint + "/v1/import"
	debug("POST %s with %d bytes", url, len(json))
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(json))

	if err != nil {
		debug("error: %v", err)
		return err
	}

	req.Header.Add("User-Agent", "analytics-go (version: "+Version+")")
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Content-Length", string(len(json)))
	req.SetBasicAuth(c.Key, "")

	res, err := client.Do(req)
	if err != nil {
		debug("error: %v", err)
		return err
	}
	defer res.Body.Close()

	debug("%d response", res.StatusCode)

	if res.StatusCode >= 400 {
		body, _ := ioutil.ReadAll(res.Body)
		debug("error: %s", string(body))
		debug("error: %s", string(json))
	}

	return err
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"fmt"
	"math"
	"net/http"
	"path"
	"strings"

	"golang.org/x/discovery/internal"
	"golang.org/x/discovery/internal/derrors"
	"golang.org/x/discovery/internal/log"
	"golang.org/x/discovery/internal/postgres"
	"golang.org/x/xerrors"
)

const defaultSearchLimit = 10

// SearchPage contains all of the data that the search template needs to
// populate.
type SearchPage struct {
	basePage
	Pagination pagination
	Results    []*SearchResult
}

// SearchResult contains data needed to display a single search result.
type SearchResult struct {
	Name          string
	PackagePath   string
	ModulePath    string
	Synopsis      string
	Version       string
	Licenses      []string
	CommitTime    string
	NumImportedBy uint64
	Approximate   bool
}

// fetchSearchPage fetches data matching the search query from the database and
// returns a SearchPage.
func fetchSearchPage(ctx context.Context, ds DataSource, query, method string, pageParams paginationParams) (*SearchPage, error) {
	var (
		dbresults []*postgres.SearchResult
		err       error
	)
	switch method {
	case "slow":
		dbresults, err = ds.Search(ctx, query, pageParams.limit, pageParams.offset())
	case "deep":
		dbresults, err = ds.DeepSearch(ctx, query, pageParams.limit, pageParams.offset())
	case "partial-fast":
		dbresults, err = ds.PartialFastSearch(ctx, query, pageParams.limit, pageParams.offset())
	case "popular":
		dbresults, err = ds.PopularSearch(ctx, query, pageParams.limit, pageParams.offset())
	default:
		dbresults, err = ds.FastSearch(ctx, query, pageParams.limit, pageParams.offset())
	}
	if err != nil {
		return nil, err
	}

	var results []*SearchResult
	for _, r := range dbresults {
		fmtVersion := formattedVersion(r.Version, r.ModulePath)
		results = append(results, &SearchResult{
			Name:          r.Name,
			PackagePath:   r.PackagePath,
			ModulePath:    r.ModulePath,
			Synopsis:      r.Synopsis,
			Version:       fmtVersion,
			Licenses:      r.Licenses,
			CommitTime:    elapsedTime(r.CommitTime),
			NumImportedBy: r.NumImportedBy,
		})
	}

	var (
		numResults  int
		approximate bool
	)
	if len(dbresults) > 0 {
		numResults = int(dbresults[0].NumResults)
		if dbresults[0].Approximate {
			// 128 buckets corresponds to a standard error of 10%.
			// http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf
			numResults = approximateNumber(numResults, 0.1)
			approximate = true
		}
	}

	pgs := newPagination(pageParams, len(results), numResults)
	pgs.Approximate = approximate
	return &SearchPage{
		Results:    results,
		Pagination: pgs,
	}, nil
}

// approximateNumber returns an approximation of the estimate, calibrated by
// the statistical estimate of standard error.
// i.e., a number that isn't misleading when we say '1-10 of approximately N
// results', but that is still close to our estimate.
func approximateNumber(estimate int, sigma float64) int {
	expectedErr := sigma * float64(estimate)
	// Compute the unit by rounding the error the logarithmically closest power
	// of 10, so that 300->100, but 400->1000.
	unit := math.Pow(10, math.Round(math.Log10(expectedErr)))
	// Now round the estimate to the nearest unit.
	return int(unit * math.Round(float64(estimate)/unit))
}

// handleSearch applies database data to the search template. Handles endpoint
// /search?q=<query>. If <query> is an exact match for a package path, the user
// will be redirected to the details page.
func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	query := searchQuery(r)
	if query == "" {
		http.Redirect(w, r, "/", http.StatusFound)
		return
	}

	if strings.Contains(query, "/") {
		pkg, err := s.ds.GetPackage(ctx, path.Clean(query), internal.UnknownModulePath, internal.LatestVersion)
		if err == nil {
			http.Redirect(w, r, fmt.Sprintf("/%s", pkg.Path), http.StatusFound)
			return
		} else if !xerrors.Is(err, derrors.NotFound) {
			log.Errorf("error getting package for %s: %v", path.Clean(query), err)
		}
	}

	searchMethod := r.FormValue("method")
	page, err := fetchSearchPage(ctx, s.ds, query, searchMethod, newPaginationParams(r, defaultSearchLimit))
	if err != nil {
		log.Errorf("fetchSearchDetails(ctx, db, %q): %v", query, err)
		s.serveErrorPage(w, r, http.StatusInternalServerError, nil)
		return
	}
	page.basePage = newBasePage(r, query)
	s.servePage(w, "search.tmpl", page)
}

// searchQuery extracts a search query from the request.
func searchQuery(r *http.Request) string {
	return strings.TrimSpace(r.FormValue("q"))
}
<commit_msg>internal/frontend: link search results to overview tab<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"fmt"
	"math"
	"net/http"
	"path"
	"strings"

	"golang.org/x/discovery/internal"
	"golang.org/x/discovery/internal/derrors"
	"golang.org/x/discovery/internal/log"
	"golang.org/x/discovery/internal/postgres"
	"golang.org/x/xerrors"
)

const defaultSearchLimit = 10

// SearchPage contains all of the data that the search template needs to
// populate.
type SearchPage struct {
	basePage
	Pagination pagination
	Results    []*SearchResult
}

// SearchResult contains data needed to display a single search result.
type SearchResult struct {
	Name          string
	PackagePath   string
	ModulePath    string
	Synopsis      string
	Version       string
	Licenses      []string
	CommitTime    string
	NumImportedBy uint64
	Approximate   bool
}

// fetchSearchPage fetches data matching the search query from the database and
// returns a SearchPage.
func fetchSearchPage(ctx context.Context, ds DataSource, query, method string, pageParams paginationParams) (*SearchPage, error) {
	var (
		dbresults []*postgres.SearchResult
		err       error
	)
	switch method {
	case "slow":
		dbresults, err = ds.Search(ctx, query, pageParams.limit, pageParams.offset())
	case "deep":
		dbresults, err = ds.DeepSearch(ctx, query, pageParams.limit, pageParams.offset())
	case "partial-fast":
		dbresults, err = ds.PartialFastSearch(ctx, query, pageParams.limit, pageParams.offset())
	case "popular":
		dbresults, err = ds.PopularSearch(ctx, query, pageParams.limit, pageParams.offset())
	default:
		dbresults, err = ds.FastSearch(ctx, query, pageParams.limit, pageParams.offset())
	}
	if err != nil {
		return nil, err
	}

	var results []*SearchResult
	for _, r := range dbresults {
		fmtVersion := formattedVersion(r.Version, r.ModulePath)
		results = append(results, &SearchResult{
			Name:          r.Name,
			PackagePath:   r.PackagePath,
			ModulePath:    r.ModulePath,
			Synopsis:      r.Synopsis,
			Version:       fmtVersion,
			Licenses:      r.Licenses,
			CommitTime:    elapsedTime(r.CommitTime),
			NumImportedBy: r.NumImportedBy,
		})
	}

	var (
		numResults  int
		approximate bool
	)
	if len(dbresults) > 0 {
		numResults = int(dbresults[0].NumResults)
		if dbresults[0].Approximate {
			// 128 buckets corresponds to a standard error of 10%.
			// http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf
			numResults = approximateNumber(numResults, 0.1)
			approximate = true
		}
	}

	pgs := newPagination(pageParams, len(results), numResults)
	pgs.Approximate = approximate
	return &SearchPage{
		Results:    results,
		Pagination: pgs,
	}, nil
}

// approximateNumber returns an approximation of the estimate, calibrated by
// the statistical estimate of standard error.
// i.e., a number that isn't misleading when we say '1-10 of approximately N
// results', but that is still close to our estimate.
func approximateNumber(estimate int, sigma float64) int {
	expectedErr := sigma * float64(estimate)
	// Compute the unit by rounding the error the logarithmically closest power
	// of 10, so that 300->100, but 400->1000.
	unit := math.Pow(10, math.Round(math.Log10(expectedErr)))
	// Now round the estimate to the nearest unit.
	return int(unit * math.Round(float64(estimate)/unit))
}

// handleSearch applies database data to the search template. Handles endpoint
// /search?q=<query>. If <query> is an exact match for a package path, the user
// will be redirected to the details page.
func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	query := searchQuery(r)
	if query == "" {
		http.Redirect(w, r, "/", http.StatusFound)
		return
	}

	if strings.Contains(query, "/") {
		pkg, err := s.ds.GetPackage(ctx, path.Clean(query), internal.UnknownModulePath, internal.LatestVersion)
		if err == nil {
			http.Redirect(w, r, fmt.Sprintf("/%s?tab=overview", pkg.Path), http.StatusFound)
			return
		} else if !xerrors.Is(err, derrors.NotFound) {
			log.Errorf("error getting package for %s: %v", path.Clean(query), err)
		}
	}

	searchMethod := r.FormValue("method")
	page, err := fetchSearchPage(ctx, s.ds, query, searchMethod, newPaginationParams(r, defaultSearchLimit))
	if err != nil {
		log.Errorf("fetchSearchDetails(ctx, db, %q): %v", query, err)
		s.serveErrorPage(w, r, http.StatusInternalServerError, nil)
		return
	}
	page.basePage = newBasePage(r, query)
	s.servePage(w, "search.tmpl", page)
}

// searchQuery extracts a search query from the request.
func searchQuery(r *http.Request) string {
	return strings.TrimSpace(r.FormValue("q"))
}
<|endoftext|>


<commit_before>package integration

import (
	"fmt"
	"io/ioutil"
	"log"

	yaml "gopkg.in/yaml.v2"

	"os"
	"os/exec"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Happy Path Installation Tests", func() {
	kisPath := CopyKismaticToTemp()

	BeforeSuite(func() {
		fmt.Println("Unpacking kismatic to", kisPath)
		c := exec.Command("tar", "-zxf", "../out/kismatic.tar.gz", "-C", kisPath)
		tarOut, tarErr := c.CombinedOutput()
		if tarErr != nil {
			log.Fatal("Error unpacking installer", string(tarOut), tarErr)
		}
		os.Chdir(kisPath)
	})

	AfterSuite(func() {
		if !leaveIt() {
			os.RemoveAll(kisPath)
		}
	})

	Describe("Calling installer with no input", func() {
		It("should output help text", func() {
			c := exec.Command("./kismatic")
			helpbytes, helperr := c.Output()
			Expect(helperr).To(BeNil())
			helpText := string(helpbytes)
			Expect(helpText).To(ContainSubstring("Usage"))
		})
	})

	Describe("Calling installer with 'install plan'", func() {
		Context("and just hitting enter", func() {
			It("should result in the output of a well formed default plan file", func() {
				By("Outputing a file")
				c := exec.Command("./kismatic", "install", "plan")
				helpbytes, helperr := c.Output()
				Expect(helperr).To(BeNil())
				helpText := string(helpbytes)
				Expect(helpText).To(ContainSubstring("Generating installation plan file with 3 etcd nodes, 2 master nodes and 3 worker nodes"))
				Expect(FileExists("kismatic-cluster.yaml")).To(Equal(true))

				By("Outputing a file with valid YAML")
				yamlBytes, err := ioutil.ReadFile("kismatic-cluster.yaml")
				if err != nil {
					Fail("Could not read cluster file")
				}
				yamlBlob := string(yamlBytes)

				planFromYaml := ClusterPlan{}

				unmarshallErr := yaml.Unmarshal([]byte(yamlBlob), &planFromYaml)
				if unmarshallErr != nil {
					Fail("Could not unmarshall cluster yaml: %v")
				}
			})
		})
	})

	Describe("Calling installer with a plan targetting AWS", func() {
		Context("Using a 1/1/1 Ubtunu 16.04 layout", func() {
			It("should result in a working cluster", func() {
				InstallKismatic(AMIUbuntu1604USEAST, "ubuntu")
			})
		})
		Context("Using a 1/1/1 CentOS 7 layout", func() {
			It("should result in a working cluster", func() {
				InstallKismatic(AMICentos7UsEast, "centos")
			})
		})
		Context("Using a Minikube CentOS 7 layout", func() {
			It("should result in a working cluster", func() {
				InstallKismaticMini(AMICentos7UsEast, "centos")
			})
		})
		Context("Using a 3/2/3 CentOS 7 layout", func() {
			It("should result in a working cluster", func() {
				InstallBigKismatic(
					NodeCount{
						Etcd:   3,
						Master: 2,
						Worker: 3,
					},
					AMICentos7UsEast, "centos")
			})
		})
	})
})
<commit_msg>Added commented-out test that proves a bad plan file won't cause kismatic to pause<commit_after>package integration

import (
	"bufio"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"time"

	yaml "gopkg.in/yaml.v2"

	"os"
	"os/exec"

	homedir "github.com/mitchellh/go-homedir"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Happy Path Installation Tests", func() {
	kisPath := CopyKismaticToTemp()

	BeforeSuite(func() {
		fmt.Println("Unpacking kismatic to", kisPath)
		c := exec.Command("tar", "-zxf", "../out/kismatic.tar.gz", "-C", kisPath)
		tarOut, tarErr := c.CombinedOutput()
		if tarErr != nil {
			log.Fatal("Error unpacking installer", string(tarOut), tarErr)
		}
		os.Chdir(kisPath)
	})

	AfterSuite(func() {
		if !leaveIt() {
			os.RemoveAll(kisPath)
		}
	})

	Describe("Calling installer with no input", func() {
		It("should output help text", func() {
			c := exec.Command("./kismatic")
			helpbytes, helperr := c.Output()
			Expect(helperr).To(BeNil())
			helpText := string(helpbytes)
			Expect(helpText).To(ContainSubstring("Usage"))
		})
	})

	Describe("Calling installer with 'install plan'", func() {
		Context("and just hitting enter", func() {
			It("should result in the output of a well formed default plan file", func() {
				By("Outputing a file")
				c := exec.Command("./kismatic", "install", "plan")
				helpbytes, helperr := c.Output()
				Expect(helperr).To(BeNil())
				helpText := string(helpbytes)
				Expect(helpText).To(ContainSubstring("Generating installation plan file with 3 etcd nodes, 2 master nodes and 3 worker nodes"))
				Expect(FileExists("kismatic-cluster.yaml")).To(Equal(true))

				By("Outputing a file with valid YAML")
				yamlBytes, err := ioutil.ReadFile("kismatic-cluster.yaml")
				if err != nil {
					Fail("Could not read cluster file")
				}
				yamlBlob := string(yamlBytes)

				planFromYaml := ClusterPlan{}

				unmarshallErr := yaml.Unmarshal([]byte(yamlBlob), &planFromYaml)
				if unmarshallErr != nil {
					Fail("Could not unmarshall cluster yaml: %v")
				}
			})
		})
	})

	Describe("Calling installer with a plan targeting bad infrastructure", func() {
		//TODO: Fix timeout that causes us to comment this out.
		//
		// Context("Using a 1/1/1 Ubtunu 16.04 layout pointing to bad ip addresses", func() {
		// 	It("should bomb validate and apply", func() {
		// 		if !completesInTime(InstallKismaticWithABadNode, 30*time.Second) {
		// 			Fail("It shouldn't take 30 seconds for Kismatic to fail with bad nodes.")
		// 		}
		// 	})
		// })
	})

	Describe("Calling installer with a plan targetting AWS", func() {
		Context("Using a 1/1/1 Ubtunu 16.04 layout", func() {
			It("should result in a working cluster", func() {
				InstallKismatic(AMIUbuntu1604USEAST, "ubuntu")
			})
		})
		Context("Using a 1/1/1 CentOS 7 layout", func() {
			It("should result in a working cluster", func() {
				InstallKismatic(AMICentos7UsEast, "centos")
			})
		})
		Context("Using a Minikube CentOS 7 layout", func() {
			It("should result in a working cluster", func() {
				InstallKismaticMini(AMICentos7UsEast, "centos")
			})
		})
		Context("Using a 3/2/3 CentOS 7 layout", func() {
			It("should result in a working cluster", func() {
				InstallBigKismatic(
					NodeCount{
						Etcd:   3,
						Master: 2,
						Worker: 3,
					},
					AMICentos7UsEast, "centos")
			})
		})
	})
})

func InstallKismaticWithABadNode() {
	By("Building a template")
	template, err := template.New("planAWSOverlay").Parse(planAWSOverlay)
	FailIfError(err, "Couldn't parse template")

	By("Faking infrastructure")
	fakeNode := AWSNodeDeets{
		Instanceid: "FakeId",
		Publicip:   "10.0.0.0",
		Hostname:   "FakeHostname",
	}

	By("Building a plan to set up an overlay network cluster on this hardware")
	nodes := PlanAWS{
		Etcd:                []AWSNodeDeets{fakeNode},
		Master:              []AWSNodeDeets{fakeNode},
		Worker:              []AWSNodeDeets{fakeNode},
		MasterNodeFQDN:      "yep.nope",
		MasterNodeShortName: "yep",
		User:                "Billy Rubin",
	}
	var hdErr error
	nodes.HomeDirectory, hdErr = homedir.Dir()
	FailIfError(hdErr, "Error getting home directory")

	f, fileErr := os.Create("kismatic-testing.yaml")
	FailIfError(fileErr, "Error waiting for nodes")
	defer f.Close()
	w := bufio.NewWriter(f)
	execErr := template.Execute(w, &nodes)
	FailIfError(execErr, "Error filling in plan template")
	w.Flush()

	f.Close()

	// By("Validing our plan")
	// ver := exec.Command("./kismatic", "install", "validate", "-f", f.Name())
	// ver.Stdout = os.Stdout
	// ver.Stderr = os.Stderr
	// verErr := ver.Run()

	// if verErr == nil {
	// 	// This should really be a failure but at the moment validation does not run tests against target nodes
	// 	// Fail("Validation succeeeded even though it shouldn't have")
	// }

	By("Well, try it anyway")
	app := exec.Command("./kismatic", "install", "apply", "-f", f.Name())
	app.Stdout = os.Stdout
	app.Stderr = os.Stderr
	appErr := app.Run()

	if appErr == nil {
		Fail("Application succeeeded even though it shouldn't have")
	}
}

func completesInTime(dothis func(), howLong time.Duration) bool {
	c1 := make(chan string, 1)
	go func() {
		dothis()
		c1 <- "completed"
	}()

	select {
	case <-c1:
		return true
	case <-time.After(howLong):
		return false
	}
}
<|endoftext|>


<commit_before>package tests

import (
	"context"
	"strings"
	"sync"
	"time"

	"github.com/fnproject/fn/api/server"

	"fmt"
	"io"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"

	"github.com/funcy/functions_go/client"
	httptransport "github.com/go-openapi/runtime/client"
	"github.com/go-openapi/strfmt"
	"github.com/spf13/viper"
)

const lBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

func Host() string {
	apiURL := os.Getenv("API_URL")
	if apiURL == "" {
		apiURL = "http://localhost:8080"
	}

	u, err := url.Parse(apiURL)
	if err != nil {
		log.Fatalln("Couldn't parse API URL:", err)
	}
	return u.Host
}

func APIClient() *client.Functions {
	transport := httptransport.New(Host(), "/v1", []string{"http"})
	if os.Getenv("FN_TOKEN") != "" {
		transport.DefaultAuthentication = httptransport.BearerToken(os.Getenv("FN_TOKEN"))
	}

	// create the API client, with the transport
	return client.New(transport, strfmt.Default)
}

var (
	getServer sync.Once
	cancel2   context.CancelFunc
	s         *server.Server
)

func getServerWithCancel() (*server.Server, context.CancelFunc) {
	getServer.Do(func() {
		ctx, cancel := context.WithCancel(context.Background())

		viper.Set(server.EnvPort, "8080")
		viper.Set(server.EnvAPIURL, "http://localhost:8080")
		viper.Set(server.EnvLogLevel, "fatal")
		timeString := time.Now().Format("2006_01_02_15_04_05")
		db_url := os.Getenv("DB_URL")
		tmpDir := os.TempDir()
		tmpMq := fmt.Sprintf("%s/fn_integration_test_%s_worker_mq.db", tmpDir, timeString)
		tmpDb := fmt.Sprintf("%s/fn_integration_test_%s_fn.db", tmpDir, timeString)
		viper.Set(server.EnvMQURL, fmt.Sprintf("bolt://%s", tmpMq))
		if db_url == "" {
			db_url = fmt.Sprintf("sqlite3://%s", tmpDb)
		}
		viper.Set(server.EnvDBURL, db_url)

		s = server.NewFromEnv(ctx)

		go s.Start(ctx)
		started := false
		time.AfterFunc(time.Second*10, func() {
			if !started {
				panic("Failed to start server.")
			}
		})
		log.Println(server.EnvAPIURL)
		_, err := http.Get(viper.GetString(server.EnvAPIURL) + "/version")
		for err != nil {
			_, err = http.Get(viper.GetString(server.EnvAPIURL) + "/version")
		}
		started = true
		cancel2 = context.CancelFunc(func() {
			cancel()
			os.Remove(tmpMq)
			os.Remove(tmpDb)
		})
	})
	return s, cancel2
}

type SuiteSetup struct {
	Context      context.Context
	Client       *client.Functions
	AppName      string
	RoutePath    string
	Image        string
	RouteType    string
	Format       string
	Memory       uint64
	RouteConfig  map[string]string
	RouteHeaders map[string][]string
	Cancel       context.CancelFunc
}

func RandStringBytes(n int) string {
	b := make([]byte, n)
	for i := range b {
		b[i] = lBytes[rand.Intn(len(lBytes))]
	}
	return strings.ToLower(string(b))
}

func SetupDefaultSuite() *SuiteSetup {
	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
	ss := &SuiteSetup{
		Context:      ctx,
		Client:       APIClient(),
		AppName:      RandStringBytes(10),
		RoutePath:    "/" + RandStringBytes(10),
		Image:        "funcy/hello",
		Format:       "default",
		RouteType:    "async",
		RouteConfig:  map[string]string{},
		RouteHeaders: map[string][]string{},
		Cancel:       cancel,
		Memory:       uint64(256),
	}

	_, ok := ss.Client.Version.GetVersion(nil)
	if ok != nil {
		if Host() != "localhost:8080" {
			_, ok := http.Get(fmt.Sprintf("http://%s/version", Host()))
			if ok != nil {
				panic("Cannot reach remote api for functions")
			}
		} else {
			_, ok := http.Get(fmt.Sprintf("http://%s/version", Host()))
			if ok != nil {
				log.Println("Making functions server")
				_, cancel := getServerWithCancel()
				ss.Cancel = cancel
			}
		}
	}

	return ss
}

func EnvAsHeader(req *http.Request, selectedEnv []string) {
	detectedEnv := os.Environ()
	if len(selectedEnv) > 0 {
		detectedEnv = selectedEnv
	}

	for _, e := range detectedEnv {
		kv := strings.Split(e, "=")
		name := kv[0]
		req.Header.Set(name, os.Getenv(name))
	}
}

func CallFN(u string, content io.Reader, output io.Writer, method string, env []string) error {
	if method == "" {
		if content == nil {
			method = "GET"
		} else {
			method = "POST"
		}
	}

	req, err := http.NewRequest(method, u, content)
	if err != nil {
		return fmt.Errorf("error running route: %s", err)
	}

	req.Header.Set("Content-Type", "application/json")

	if len(env) > 0 {
		EnvAsHeader(req, env)
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return fmt.Errorf("error running route: %s", err)
	}

	io.Copy(output, resp.Body)

	return nil
}
<commit_msg>Updating API tests<commit_after>package tests

import (
	"context"
	"strings"
	"sync"
	"time"

	"github.com/fnproject/fn/api/server"

	"fmt"
	"io"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"

	"github.com/funcy/functions_go/client"
	httptransport "github.com/go-openapi/runtime/client"
	"github.com/go-openapi/strfmt"
	"github.com/spf13/viper"
)

const lBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

func Host() string {
	apiURL := os.Getenv("API_URL")
	if apiURL == "" {
		apiURL = "http://localhost:8080"
	}

	u, err := url.Parse(apiURL)
	if err != nil {
		log.Fatalln("Couldn't parse API URL:", err)
	}
	return u.Host
}

func APIClient() *client.Functions {
	transport := httptransport.New(Host(), "/v1", []string{"http"})
	if os.Getenv("FN_TOKEN") != "" {
		transport.DefaultAuthentication = httptransport.BearerToken(os.Getenv("FN_TOKEN"))
	}

	// create the API client, with the transport
	return client.New(transport, strfmt.Default)
}

var (
	getServer sync.Once
	cancel2   context.CancelFunc
	s         *server.Server
)

func getServerWithCancel() (*server.Server, context.CancelFunc) {
	getServer.Do(func() {
		ctx, cancel := context.WithCancel(context.Background())

		viper.Set(server.EnvPort, "8080")
		viper.Set(server.EnvAPIURL, "http://localhost:8080")
		viper.Set(server.EnvLogLevel, "fatal")
		timeString := time.Now().Format("2006_01_02_15_04_05")
		db_url := os.Getenv("DB_URL")
		tmpDir := os.TempDir()
		tmpMq := fmt.Sprintf("%s/fn_integration_test_%s_worker_mq.db", tmpDir, timeString)
		tmpDb := fmt.Sprintf("%s/fn_integration_test_%s_fn.db", tmpDir, timeString)
		viper.Set(server.EnvMQURL, fmt.Sprintf("bolt://%s", tmpMq))
		if db_url == "" {
			db_url = fmt.Sprintf("sqlite3://%s", tmpDb)
		}
		viper.Set(server.EnvDBURL, db_url)

		s = server.NewFromEnv(ctx)

		go s.Start(ctx)
		started := false
		time.AfterFunc(time.Second*10, func() {
			if !started {
				panic("Failed to start server.")
			}
		})
		log.Println(server.EnvAPIURL)
		_, err := http.Get(viper.GetString(server.EnvAPIURL) + "/version")
		for err != nil {
			_, err = http.Get(viper.GetString(server.EnvAPIURL) + "/version")
		}
		started = true
		cancel2 = context.CancelFunc(func() {
			cancel()
			os.Remove(tmpMq)
			os.Remove(tmpDb)
		})
	})
	return s, cancel2
}

type SuiteSetup struct {
	Context      context.Context
	Client       *client.Functions
	AppName      string
	RoutePath    string
	Image        string
	RouteType    string
	Format       string
	Memory       uint64
	RouteConfig  map[string]string
	RouteHeaders map[string][]string
	Cancel       context.CancelFunc
}

func RandStringBytes(n int) string {
	b := make([]byte, n)
	for i := range b {
		b[i] = lBytes[rand.Intn(len(lBytes))]
	}
	return strings.ToLower(string(b))
}

func SetupDefaultSuite() *SuiteSetup {
	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
	ss := &SuiteSetup{
		Context:      ctx,
		Client:       APIClient(),
		AppName:      RandStringBytes(10),
		RoutePath:    "/" + RandStringBytes(10),
		Image:        "funcy/hello",
		Format:       "default",
		RouteType:    "async",
		RouteConfig:  map[string]string{},
		RouteHeaders: map[string][]string{},
		Cancel:       cancel,
		Memory:       uint64(256),
	}

	if Host() != "localhost:8080" {
		_, ok := http.Get(fmt.Sprintf("http://%s/version", Host()))
		if ok != nil {
			panic("Cannot reach remote api for functions")
		}
	} else {
		_, ok := http.Get(fmt.Sprintf("http://%s/version", Host()))
		if ok != nil {
			log.Println("Making functions server")
			_, cancel := getServerWithCancel()
			ss.Cancel = cancel
		}
	}

	return ss
}

func EnvAsHeader(req *http.Request, selectedEnv []string) {
	detectedEnv := os.Environ()
	if len(selectedEnv) > 0 {
		detectedEnv = selectedEnv
	}

	for _, e := range detectedEnv {
		kv := strings.Split(e, "=")
		name := kv[0]
		req.Header.Set(name, os.Getenv(name))
	}
}

func CallFN(u string, content io.Reader, output io.Writer, method string, env []string) error {
	if method == "" {
		if content == nil {
			method = "GET"
		} else {
			method = "POST"
		}
	}

	req, err := http.NewRequest(method, u, content)
	if err != nil {
		return fmt.Errorf("error running route: %s", err)
	}

	req.Header.Set("Content-Type", "application/json")

	if len(env) > 0 {
		EnvAsHeader(req, env)
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return fmt.Errorf("error running route: %s", err)
	}

	io.Copy(output, resp.Body)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pathtools

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/google/blueprint/deptools"
)

var GlobMultipleRecursiveErr = errors.New("pattern contains multiple '**'")
var GlobLastRecursiveErr = errors.New("pattern has '**' as last path element")
var GlobInvalidRecursiveErr = errors.New("pattern contains other characters between '**' and path separator")

// Glob returns the list of files and directories that match the given pattern
// but do not match the given exclude patterns, along with the list of
// directories and other dependencies that were searched to construct the file
// list.  The supported glob and exclude patterns are equivalent to
// filepath.Glob, with an extension that recursive glob (** matching zero or
// more complete path entries) is supported. Any directories in the matches
// list will have a '/' suffix.
//
// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
// should be used instead, as they will automatically set up dependencies
// to rerun the primary builder when the list of matching files changes.
func Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (matches, deps []string, err error) {
	return startGlob(OsFs, pattern, excludes, follow)
}

func startGlob(fs FileSystem, pattern string, excludes []string,
	follow ShouldFollowSymlinks) (matches, deps []string, err error) {

	if filepath.Base(pattern) == "**" {
		return nil, nil, GlobLastRecursiveErr
	} else {
		matches, deps, err = glob(fs, pattern, false, follow)
	}

	if err != nil {
		return nil, nil, err
	}

	matches, err = filterExcludes(matches, excludes)
	if err != nil {
		return nil, nil, err
	}

	// If the pattern has wildcards, we added dependencies on the
	// containing directories to know about changes.
	//
	// If the pattern didn't have wildcards, and didn't find matches, the
	// most specific found directories were added.
	//
	// But if it didn't have wildcards, and did find a match, no
	// dependencies were added, so add the match itself to detect when it
	// is removed.
	if !isWild(pattern) {
		deps = append(deps, matches...)
	}

	for i, match := range matches {
		var info os.FileInfo
		if follow == DontFollowSymlinks {
			info, err = fs.Lstat(match)
		} else {
			info, err = fs.Stat(match)
		}
		if err != nil {
			return nil, nil, err
		}

		if info.IsDir() {
			matches[i] = match + "/"
		}
	}

	return matches, deps, nil
}

// glob is a recursive helper function to handle globbing each level of the pattern individually,
// allowing searched directories to be tracked.  Also handles the recursive glob pattern, **.
func glob(fs FileSystem, pattern string, hasRecursive bool,
	follow ShouldFollowSymlinks) (matches, dirs []string, err error) {

	if !isWild(pattern) {
		// If there are no wilds in the pattern, check whether the file exists or not.
		// Uses filepath.Glob instead of manually statting to get consistent results.
		pattern = filepath.Clean(pattern)
		matches, err = fs.glob(pattern)
		if err != nil {
			return matches, dirs, err
		}

		if len(matches) == 0 {
			// Some part of the non-wild pattern didn't exist.  Add the last existing directory
			// as a dependency.
			var matchDirs []string
			for len(matchDirs) == 0 {
				pattern = filepath.Dir(pattern)
				matchDirs, err = fs.glob(pattern)
				if err != nil {
					return matches, dirs, err
				}
			}
			dirs = append(dirs, matchDirs...)
		}
		return matches, dirs, err
	}

	dir, file := saneSplit(pattern)

	if file == "**" {
		if hasRecursive {
			return matches, dirs, GlobMultipleRecursiveErr
		}
		hasRecursive = true
	} else if strings.Contains(file, "**") {
		return matches, dirs, GlobInvalidRecursiveErr
	}

	dirMatches, dirs, err := glob(fs, dir, hasRecursive, follow)
	if err != nil {
		return nil, nil, err
	}

	for _, m := range dirMatches {
		isDir, err := fs.IsDir(m)
		if os.IsNotExist(err) {
			if isSymlink, _ := fs.IsSymlink(m); isSymlink {
				return nil, nil, fmt.Errorf("dangling symlink: %s", m)
			}
		}
		if err != nil {
			return nil, nil, fmt.Errorf("unexpected error after glob: %s", err)
		}

		if isDir {
			if file == "**" {
				recurseDirs, err := fs.ListDirsRecursive(m, follow)
				if err != nil {
					return nil, nil, err
				}
				matches = append(matches, recurseDirs...)
			} else {
				dirs = append(dirs, m)
				newMatches, err := fs.glob(filepath.Join(MatchEscape(m), file))
				if err != nil {
					return nil, nil, err
				}
				if file[0] != '.' {
					newMatches = filterDotFiles(newMatches)
				}
				matches = append(matches, newMatches...)
			}
		}
	}

	return matches, dirs, nil
}

// Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations
// Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is
// not "/".  Returns ".", "" if path is "."
func saneSplit(path string) (dir, file string) {
	if path == "." {
		return ".", ""
	}
	dir, file = filepath.Split(path)
	switch dir {
	case "":
		dir = "."
	case "/":
		// Nothing
	default:
		dir = dir[:len(dir)-1]
	}
	return dir, file
}

func isWild(pattern string) bool {
	return strings.ContainsAny(pattern, "*?[")
}

// Filters the strings in matches based on the glob patterns in excludes.  Hierarchical (a/*) and
// recursive (**) glob patterns are supported.
func filterExcludes(matches []string, excludes []string) ([]string, error) {
	if len(excludes) == 0 {
		return matches, nil
	}

	var ret []string
matchLoop:
	for _, m := range matches {
		for _, e := range excludes {
			exclude, err := Match(e, m)
			if err != nil {
				return nil, err
			}
			if exclude {
				continue matchLoop
			}
		}
		ret = append(ret, m)
	}

	return ret, nil
}

// filterDotFiles filters out files that start with '.'
func filterDotFiles(matches []string) []string {
	ret := make([]string, 0, len(matches))

	for _, match := range matches {
		_, name := filepath.Split(match)
		if name[0] == '.' {
			continue
		}
		ret = append(ret, match)
	}

	return ret
}

// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting
// recursive globs (**).
func Match(pattern, name string) (bool, error) {
	if filepath.Base(pattern) == "**" {
		return false, GlobLastRecursiveErr
	}

	patternDir := pattern[len(pattern)-1] == '/'
	nameDir := name[len(name)-1] == '/'

	if patternDir != nameDir {
		return false, nil
	}

	if nameDir {
		name = name[:len(name)-1]
		pattern = pattern[:len(pattern)-1]
	}

	for {
		var patternFile, nameFile string
		pattern, patternFile = filepath.Dir(pattern), filepath.Base(pattern)

		if patternFile == "**" {
			if strings.Contains(pattern, "**") {
				return false, GlobMultipleRecursiveErr
			}
			// Test if the any prefix of name matches the part of the pattern before **
			for {
				if name == "." || name == "/" {
					return name == pattern, nil
				}
				if match, err := filepath.Match(pattern, name); err != nil {
					return false, err
				} else if match {
					return true, nil
				}
				name = filepath.Dir(name)
			}
		} else if strings.Contains(patternFile, "**") {
			return false, GlobInvalidRecursiveErr
		}

		name, nameFile = filepath.Dir(name), filepath.Base(name)

		if nameFile == "." && patternFile == "." {
			return true, nil
		} else if nameFile == "/" && patternFile == "/" {
			return true, nil
		} else if nameFile == "." || patternFile == "." || nameFile == "/" || patternFile == "/" {
			return false, nil
		}

		match, err := filepath.Match(patternFile, nameFile)
		if err != nil || !match {
			return match, err
		}
	}
}

func GlobPatternList(patterns []string, prefix string) (globedList []string, depDirs []string, err error) {
	var (
		matches []string
		deps    []string
	)

	globedList = make([]string, 0)
	depDirs = make([]string, 0)

	for _, pattern := range patterns {
		if isWild(pattern) {
			matches, deps, err = Glob(filepath.Join(prefix, pattern), nil, FollowSymlinks)
			if err != nil {
				return nil, nil, err
			}
			globedList = append(globedList, matches...)
			depDirs = append(depDirs, deps...)
		} else {
			globedList = append(globedList, filepath.Join(prefix, pattern))
		}
	}
	return globedList, depDirs, nil
}

// IsGlob returns true if the pattern contains any glob characters (*, ?, or [).
func IsGlob(pattern string) bool {
	return strings.IndexAny(pattern, "*?[") >= 0
}

// HasGlob returns true if any string in the list contains any glob characters (*, ?, or [).
func HasGlob(in []string) bool {
	for _, s := range in {
		if IsGlob(s) {
			return true
		}
	}

	return false
}

// GlobWithDepFile finds all files and directories that match glob.  Directories
// will have a trailing '/'.  It compares the list of matches against the
// contents of fileListFile, and rewrites fileListFile if it has changed.  It
// also writes all of the the directories it traversed as dependencies on
// fileListFile to depFile.
//
// The format of glob is either path/*.ext for a single directory glob, or
// path/**/*.ext for a recursive glob.
//
// Returns a list of file paths, and an error.
//
// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
// should be used instead, as they will automatically set up dependencies
// to rerun the primary builder when the list of matching files changes.
func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) {
	files, deps, err := Glob(glob, excludes, FollowSymlinks)
	if err != nil {
		return nil, err
	}

	fileList := strings.Join(files, "\n") + "\n"

	WriteFileIfChanged(fileListFile, []byte(fileList), 0666)
	deptools.WriteDepFile(depFile, fileListFile, deps)

	return
}

// WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if
// the files does not already exist with identical contents.  This can be used
// along with ninja restat rules to skip rebuilding downstream rules if no
// changes were made by a rule.
func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error {
	var isChanged bool

	dir := filepath.Dir(filename)
	err := os.MkdirAll(dir, 0777)
	if err != nil {
		return err
	}

	info, err := os.Stat(filename)
	if err != nil {
		if os.IsNotExist(err) {
			// The file does not exist yet.
			isChanged = true
		} else {
			return err
		}
	} else {
		if info.Size() != int64(len(data)) {
			isChanged = true
		} else {
			oldData, err := ioutil.ReadFile(filename)
			if err != nil {
				return err
			}

			if len(oldData) != len(data) {
				isChanged = true
			} else {
				for i := range data {
					if oldData[i] != data[i] {
						isChanged = true
						break
					}
				}
			}
		}
	}

	if isChanged {
		err = ioutil.WriteFile(filename, data, perm)
		if err != nil {
			return err
		}
	}

	return nil
}

var matchEscaper = strings.NewReplacer(
	`*`, `\*`,
	`?`, `\?`,
	`[`, `\[`,
	`]`, `\]`,
)

// MatchEscape returns its inputs with characters that would be interpreted by
func MatchEscape(s string) string {
	return matchEscaper.Replace(s)
}
<commit_msg>Remove unused GlobPatternList function<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pathtools

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/google/blueprint/deptools"
)

var GlobMultipleRecursiveErr = errors.New("pattern contains multiple '**'")
var GlobLastRecursiveErr = errors.New("pattern has '**' as last path element")
var GlobInvalidRecursiveErr = errors.New("pattern contains other characters between '**' and path separator")

// Glob returns the list of files and directories that match the given pattern
// but do not match the given exclude patterns, along with the list of
// directories and other dependencies that were searched to construct the file
// list.  The supported glob and exclude patterns are equivalent to
// filepath.Glob, with an extension that recursive glob (** matching zero or
// more complete path entries) is supported. Any directories in the matches
// list will have a '/' suffix.
//
// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
// should be used instead, as they will automatically set up dependencies
// to rerun the primary builder when the list of matching files changes.
func Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (matches, deps []string, err error) {
	return startGlob(OsFs, pattern, excludes, follow)
}

func startGlob(fs FileSystem, pattern string, excludes []string,
	follow ShouldFollowSymlinks) (matches, deps []string, err error) {

	if filepath.Base(pattern) == "**" {
		return nil, nil, GlobLastRecursiveErr
	} else {
		matches, deps, err = glob(fs, pattern, false, follow)
	}

	if err != nil {
		return nil, nil, err
	}

	matches, err = filterExcludes(matches, excludes)
	if err != nil {
		return nil, nil, err
	}

	// If the pattern has wildcards, we added dependencies on the
	// containing directories to know about changes.
	//
	// If the pattern didn't have wildcards, and didn't find matches, the
	// most specific found directories were added.
	//
	// But if it didn't have wildcards, and did find a match, no
	// dependencies were added, so add the match itself to detect when it
	// is removed.
	if !isWild(pattern) {
		deps = append(deps, matches...)
	}

	for i, match := range matches {
		var info os.FileInfo
		if follow == DontFollowSymlinks {
			info, err = fs.Lstat(match)
		} else {
			info, err = fs.Stat(match)
		}
		if err != nil {
			return nil, nil, err
		}

		if info.IsDir() {
			matches[i] = match + "/"
		}
	}

	return matches, deps, nil
}

// glob is a recursive helper function to handle globbing each level of the pattern individually,
// allowing searched directories to be tracked.  Also handles the recursive glob pattern, **.
func glob(fs FileSystem, pattern string, hasRecursive bool,
	follow ShouldFollowSymlinks) (matches, dirs []string, err error) {

	if !isWild(pattern) {
		// If there are no wilds in the pattern, check whether the file exists or not.
		// Uses filepath.Glob instead of manually statting to get consistent results.
		pattern = filepath.Clean(pattern)
		matches, err = fs.glob(pattern)
		if err != nil {
			return matches, dirs, err
		}

		if len(matches) == 0 {
			// Some part of the non-wild pattern didn't exist.  Add the last existing directory
			// as a dependency.
			var matchDirs []string
			for len(matchDirs) == 0 {
				pattern = filepath.Dir(pattern)
				matchDirs, err = fs.glob(pattern)
				if err != nil {
					return matches, dirs, err
				}
			}
			dirs = append(dirs, matchDirs...)
		}
		return matches, dirs, err
	}

	dir, file := saneSplit(pattern)

	if file == "**" {
		if hasRecursive {
			return matches, dirs, GlobMultipleRecursiveErr
		}
		hasRecursive = true
	} else if strings.Contains(file, "**") {
		return matches, dirs, GlobInvalidRecursiveErr
	}

	dirMatches, dirs, err := glob(fs, dir, hasRecursive, follow)
	if err != nil {
		return nil, nil, err
	}

	for _, m := range dirMatches {
		isDir, err := fs.IsDir(m)
		if os.IsNotExist(err) {
			if isSymlink, _ := fs.IsSymlink(m); isSymlink {
				return nil, nil, fmt.Errorf("dangling symlink: %s", m)
			}
		}
		if err != nil {
			return nil, nil, fmt.Errorf("unexpected error after glob: %s", err)
		}

		if isDir {
			if file == "**" {
				recurseDirs, err := fs.ListDirsRecursive(m, follow)
				if err != nil {
					return nil, nil, err
				}
				matches = append(matches, recurseDirs...)
			} else {
				dirs = append(dirs, m)
				newMatches, err := fs.glob(filepath.Join(MatchEscape(m), file))
				if err != nil {
					return nil, nil, err
				}
				if file[0] != '.' {
					newMatches = filterDotFiles(newMatches)
				}
				matches = append(matches, newMatches...)
			}
		}
	}

	return matches, dirs, nil
}

// Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations
// Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is
// not "/".  Returns ".", "" if path is "."
func saneSplit(path string) (dir, file string) {
	if path == "." {
		return ".", ""
	}
	dir, file = filepath.Split(path)
	switch dir {
	case "":
		dir = "."
	case "/":
		// Nothing
	default:
		dir = dir[:len(dir)-1]
	}
	return dir, file
}

func isWild(pattern string) bool {
	return strings.ContainsAny(pattern, "*?[")
}

// Filters the strings in matches based on the glob patterns in excludes.  Hierarchical (a/*) and
// recursive (**) glob patterns are supported.
func filterExcludes(matches []string, excludes []string) ([]string, error) {
	if len(excludes) == 0 {
		return matches, nil
	}

	var ret []string
matchLoop:
	for _, m := range matches {
		for _, e := range excludes {
			exclude, err := Match(e, m)
			if err != nil {
				return nil, err
			}
			if exclude {
				continue matchLoop
			}
		}
		ret = append(ret, m)
	}

	return ret, nil
}

// filterDotFiles filters out files that start with '.'
func filterDotFiles(matches []string) []string {
	ret := make([]string, 0, len(matches))

	for _, match := range matches {
		_, name := filepath.Split(match)
		if name[0] == '.' {
			continue
		}
		ret = append(ret, match)
	}

	return ret
}

// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting
// recursive globs (**).
func Match(pattern, name string) (bool, error) {
	if filepath.Base(pattern) == "**" {
		return false, GlobLastRecursiveErr
	}

	patternDir := pattern[len(pattern)-1] == '/'
	nameDir := name[len(name)-1] == '/'

	if patternDir != nameDir {
		return false, nil
	}

	if nameDir {
		name = name[:len(name)-1]
		pattern = pattern[:len(pattern)-1]
	}

	for {
		var patternFile, nameFile string
		pattern, patternFile = filepath.Dir(pattern), filepath.Base(pattern)

		if patternFile == "**" {
			if strings.Contains(pattern, "**") {
				return false, GlobMultipleRecursiveErr
			}
			// Test if the any prefix of name matches the part of the pattern before **
			for {
				if name == "." || name == "/" {
					return name == pattern, nil
				}
				if match, err := filepath.Match(pattern, name); err != nil {
					return false, err
				} else if match {
					return true, nil
				}
				name = filepath.Dir(name)
			}
		} else if strings.Contains(patternFile, "**") {
			return false, GlobInvalidRecursiveErr
		}

		name, nameFile = filepath.Dir(name), filepath.Base(name)

		if nameFile == "." && patternFile == "." {
			return true, nil
		} else if nameFile == "/" && patternFile == "/" {
			return true, nil
		} else if nameFile == "." || patternFile == "." || nameFile == "/" || patternFile == "/" {
			return false, nil
		}

		match, err := filepath.Match(patternFile, nameFile)
		if err != nil || !match {
			return match, err
		}
	}
}

// IsGlob returns true if the pattern contains any glob characters (*, ?, or [).
func IsGlob(pattern string) bool {
	return strings.IndexAny(pattern, "*?[") >= 0
}

// HasGlob returns true if any string in the list contains any glob characters (*, ?, or [).
func HasGlob(in []string) bool {
	for _, s := range in {
		if IsGlob(s) {
			return true
		}
	}

	return false
}

// GlobWithDepFile finds all files and directories that match glob.  Directories
// will have a trailing '/'.  It compares the list of matches against the
// contents of fileListFile, and rewrites fileListFile if it has changed.  It
// also writes all of the the directories it traversed as dependencies on
// fileListFile to depFile.
//
// The format of glob is either path/*.ext for a single directory glob, or
// path/**/*.ext for a recursive glob.
//
// Returns a list of file paths, and an error.
//
// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
// should be used instead, as they will automatically set up dependencies
// to rerun the primary builder when the list of matching files changes.
func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) {
	files, deps, err := Glob(glob, excludes, FollowSymlinks)
	if err != nil {
		return nil, err
	}

	fileList := strings.Join(files, "\n") + "\n"

	WriteFileIfChanged(fileListFile, []byte(fileList), 0666)
	deptools.WriteDepFile(depFile, fileListFile, deps)

	return
}

// WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if
// the files does not already exist with identical contents.  This can be used
// along with ninja restat rules to skip rebuilding downstream rules if no
// changes were made by a rule.
func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error {
	var isChanged bool

	dir := filepath.Dir(filename)
	err := os.MkdirAll(dir, 0777)
	if err != nil {
		return err
	}

	info, err := os.Stat(filename)
	if err != nil {
		if os.IsNotExist(err) {
			// The file does not exist yet.
			isChanged = true
		} else {
			return err
		}
	} else {
		if info.Size() != int64(len(data)) {
			isChanged = true
		} else {
			oldData, err := ioutil.ReadFile(filename)
			if err != nil {
				return err
			}

			if len(oldData) != len(data) {
				isChanged = true
			} else {
				for i := range data {
					if oldData[i] != data[i] {
						isChanged = true
						break
					}
				}
			}
		}
	}

	if isChanged {
		err = ioutil.WriteFile(filename, data, perm)
		if err != nil {
			return err
		}
	}

	return nil
}

var matchEscaper = strings.NewReplacer(
	`*`, `\*`,
	`?`, `\?`,
	`[`, `\[`,
	`]`, `\]`,
)

// MatchEscape returns its inputs with characters that would be interpreted by
func MatchEscape(s string) string {
	return matchEscaper.Replace(s)
}
<|endoftext|>