<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"

	log "github.com/couchbaselabs/clog"
)

// A janitor maintains PIndexes and Feeds, creating, deleting &
// hooking them up as necessary to try to match to latest plans.

func (mgr *Manager) JanitorNOOP(msg string) {
	SyncWorkReq(mgr.janitorCh, WORK_NOOP, msg)
}

func (mgr *Manager) JanitorKick(msg string) {
	SyncWorkReq(mgr.janitorCh, WORK_KICK, msg)
}

func (mgr *Manager) JanitorLoop() {
	for m := range mgr.janitorCh {
		if m.op == WORK_KICK {
			err := mgr.JanitorOnce(m.msg)
			if err != nil {
				log.Printf("error: JanitorOnce, err: %v", err)
				// Keep looping as perhaps it's a transient issue.
				// TODO: Need a better way to track warnings from errors,
				// and which need a future, rescheduled janitor kick.
			}
		} else if m.op == WORK_NOOP {
			// NOOP.
		} else {
			log.Printf("error: unknown janitor op: %s, m: %#v", m.op, m)
		}
		if m.resCh != nil {
			close(m.resCh)
		}
	}
}

func (mgr *Manager) JanitorOnce(reason string) error {
	log.Printf("janitor awakes, reason: %s", reason)

	if mgr.cfg == nil { // Can occur during testing.
		return fmt.Errorf("janitor skipped due to nil cfg")
	}

	planPIndexes, _, err := CfgGetPlanPIndexes(mgr.cfg)
	if err != nil {
		return fmt.Errorf("janitor skipped on CfgGetPlanPIndexes err: %v", err)
	}
	if planPIndexes == nil {
		// Might happen if janitor wins a race.
		// TODO: Should we reschedule another future janitor kick?
		return fmt.Errorf("janitor skipped on nil planPIndexes")
	}

	currFeeds, currPIndexes := mgr.CurrentMaps()

	addPlanPIndexes, removePIndexes :=
		CalcPIndexesDelta(mgr.uuid, currPIndexes, planPIndexes)

	log.Printf("janitor pindexes to add:")
	for _, ppi := range addPlanPIndexes {
		log.Printf("  %+v", ppi)
	}
	log.Printf("janitor pindexes to remove:")
	for _, pi := range removePIndexes {
		log.Printf("  %+v", pi)
	}

	// First, teardown pindexes that need to be removed.
	for _, removePIndex := range removePIndexes {
		log.Printf("janitor removing pindex: %s", removePIndex.Name)
		err = mgr.stopPIndex(removePIndex)
		if err != nil {
			return fmt.Errorf("error: janitor removing pindex: %s, err: %v",
				removePIndex.Name, err)
		}
	}
	// Then, (re-)create pindexes that we're missing.
	for _, addPlanPIndex := range addPlanPIndexes {
		log.Printf("janitor adding pindex: %s", addPlanPIndex.Name)
		err = mgr.startPIndex(addPlanPIndex)
		if err != nil {
			return fmt.Errorf("error: janitor adding pindex: %s, err: %v",
				addPlanPIndex.Name, err)
		}
	}

	currFeeds, currPIndexes = mgr.CurrentMaps()

	addFeeds, removeFeeds := CalcFeedsDelta(currFeeds, currPIndexes)

	log.Printf("janitor feeds to add:")
	for _, targetPIndexes := range addFeeds {
		if len(targetPIndexes) > 0 {
			log.Printf("  %s", FeedName(targetPIndexes[0]))
		}
	}
	log.Printf("janitor feeds to remove:")
	for _, removeFeed := range removeFeeds {
		log.Printf("  %s", removeFeed.Name())
	}

	// First, teardown feeds that need to be removed.
	for _, removeFeed := range removeFeeds {
		err = mgr.stopFeed(removeFeed)
		if err != nil {
			return fmt.Errorf("error: janitor removing feed name: %s, err: %v",
				removeFeed.Name, err)
		}
	}
	// Then, (re-)create feeds that we're missing.
	for addFeedName, targetPIndexes := range addFeeds {
		err = mgr.startFeed(targetPIndexes)
		if err != nil {
			return fmt.Errorf("error: janitor adding feed, addFeedName: %s, err: %v",
				addFeedName, err)
		}
	}

	return nil
}

// --------------------------------------------------------

// Functionally determine the delta of which pindexes need creation
// and which should be shut down on our local node (mgrUUID).
func CalcPIndexesDelta(mgrUUID string,
	currPIndexes map[string]*PIndex,
	wantedPlanPIndexes *PlanPIndexes) (
	addPlanPIndexes []*PlanPIndex,
	removePIndexes []*PIndex) {
	// Allocate our return arrays.
	addPlanPIndexes = make([]*PlanPIndex, 0)
	removePIndexes = make([]*PIndex, 0)

	// Just for fast transient lookups.
	mapWantedPlanPIndex := make(map[string]*PlanPIndex)
	mapRemovePIndex := make(map[string]*PIndex)

	// For each wanted plan pindex, if a pindex does not exist or is
	// different, then schedule to add.
	for _, wantedPlanPIndex := range wantedPlanPIndexes.PlanPIndexes {
	nodeUUIDs:
		for nodeUUID, nodeState := range wantedPlanPIndex.NodeUUIDs {
			if nodeUUID != mgrUUID || nodeState != "active" {
				continue nodeUUIDs
			}

			mapWantedPlanPIndex[wantedPlanPIndex.Name] = wantedPlanPIndex

			currPIndex, exists := currPIndexes[wantedPlanPIndex.Name]
			if !exists {
				addPlanPIndexes = append(addPlanPIndexes, wantedPlanPIndex)
			} else if PIndexMatchesPlan(currPIndex, wantedPlanPIndex) == false {
				addPlanPIndexes = append(addPlanPIndexes, wantedPlanPIndex)
				removePIndexes = append(removePIndexes, currPIndex)
				mapRemovePIndex[currPIndex.Name] = currPIndex
			}

			break nodeUUIDs
		}
	}

	// For each existing pindex, if not part of wanted plan pindex,
	// then schedule for removal.
	for _, currPIndex := range currPIndexes {
		if _, exists := mapWantedPlanPIndex[currPIndex.Name]; !exists {
			if _, exists = mapRemovePIndex[currPIndex.Name]; !exists {
				removePIndexes = append(removePIndexes, currPIndex)
				mapRemovePIndex[currPIndex.Name] = currPIndex
			}
		}
	}

	return addPlanPIndexes, removePIndexes
}

func PIndexMatchesPlan(pindex *PIndex, planPIndex *PlanPIndex) bool {
	same := pindex.Name == planPIndex.Name &&
		pindex.IndexName == planPIndex.IndexName &&
		pindex.IndexUUID == planPIndex.IndexUUID &&
		pindex.IndexMapping == planPIndex.IndexMapping &&
		pindex.SourceType == planPIndex.SourceType &&
		pindex.SourceName == planPIndex.SourceName &&
		pindex.SourceUUID == planPIndex.SourceUUID &&
		pindex.SourcePartitions == planPIndex.SourcePartitions
	if !same {
		log.Printf("PIndexMatchesPlan false, pindex: %#v, planPIndex: %#v",
			pindex, planPIndex)
	}
	return same
}

// --------------------------------------------------------

// Functionally determine the delta of which feeds need creation and
// which should be shut down.
func CalcFeedsDelta(currFeeds map[string]Feed, pindexes map[string]*PIndex) (
	addFeeds [][]*PIndex, removeFeeds []Feed) {
	// Allocate result holders.
	addFeeds = make([][]*PIndex, 0)
	removeFeeds = make([]Feed, 0)

	// Group the pindexes by their feed names.
	groupedPIndexes := make(map[string][]*PIndex)
	for _, pindex := range pindexes {
		feedName := FeedName(pindex)
		arr, exists := groupedPIndexes[feedName]
		if !exists {
			arr = make([]*PIndex, 0)
		}
		groupedPIndexes[feedName] = append(arr, pindex)
	}

	for feedName, feedPIndexes := range groupedPIndexes {
		if _, exists := currFeeds[feedName]; !exists {
			addFeeds = append(addFeeds, feedPIndexes)
		}
	}

	for currFeedName, currFeed := range currFeeds {
		if _, exists := groupedPIndexes[currFeedName]; !exists {
			removeFeeds = append(removeFeeds, currFeed)
		}
	}

	return addFeeds, removeFeeds
}

func FeedName(pindex *PIndex) string {
	// TODO: Different feed types might have different name formulas
	// depending on whether they have a "single cluster" abstraction
	// or work on a "node by node" basis (in where stream destinations
	// need to be part of the name encoding).
	//
	// NOTE: We're depending on the IndexName/IndexUUID to
	// functionally "cover" the SourceType/SourceName/SourceUUID, so
	// we don't need to encode the source parts into the feed name.
	return pindex.IndexName + "_" + pindex.IndexUUID
}

// --------------------------------------------------------

func (mgr *Manager) startPIndex(planPIndex *PlanPIndex) error {
	pindex, err := NewPIndex(planPIndex.Name, NewUUID(),
		planPIndex.IndexType,
		planPIndex.IndexName,
		planPIndex.IndexUUID,
		planPIndex.IndexMapping,
		planPIndex.SourceType,
		planPIndex.SourceName,
		planPIndex.SourceUUID,
		planPIndex.SourcePartitions,
		mgr.PIndexPath(planPIndex.Name))
	if err != nil {
		return fmt.Errorf("error: NewPIndex, name: %s, err: %v",
			planPIndex.Name, err)
	}

	if err = mgr.registerPIndex(pindex); err != nil {
		close(pindex.Stream)
		return err
	}

	return nil
}

func (mgr *Manager) stopPIndex(pindex *PIndex) error {
	feeds, _ := mgr.CurrentMaps()
	for _, feed := range feeds {
		for _, stream := range feed.Streams() {
			if stream == pindex.Stream {
				if err := mgr.stopFeed(feed); err != nil {
					panic(fmt.Sprintf("error: could not stop feed, err: %v", err))
				}
			}
		}
	}

	pindexUnreg := mgr.unregisterPIndex(pindex.Name)
	if pindexUnreg != nil && pindexUnreg != pindex {
		panic("unregistered pindex isn't the one we're stopping")
	}

	close(pindex.Stream)

	return nil
}

// --------------------------------------------------------

func (mgr *Manager) startFeed(pindexes []*PIndex) error {
	if len(pindexes) <= 0 {
		return nil
	}

	pindexFirst := pindexes[0]
	feedName := FeedName(pindexFirst)

	streams := make(map[string]Stream)
	for _, pindex := range pindexes {
		if f := FeedName(pindex); f != feedName {
			panic(fmt.Sprintf("error: unexpected feedName: %s != %s", f, feedName))
		}

		streams[pindex.SourcePartitions] = pindex.Stream
	}

	return mgr.startFeedByType(feedName,
		pindexFirst.IndexName, pindexFirst.IndexUUID,
		pindexFirst.SourceType, pindexFirst.SourceName, pindexFirst.SourceUUID,
		streams)
}

func (mgr *Manager) stopFeed(feed Feed) error {
	feedUnreg := mgr.unregisterFeed(feed.Name())
	if feedUnreg != nil && feedUnreg != feed {
		panic("error: unregistered feed isn't the one we're closing")
	}

	// NOTE: We're depending on feed to synchronously close, so we
	// know it'll no longer be sending to any of its streams anymore.
	//
	return feed.Close()
}

// --------------------------------------------------------

func (mgr *Manager) startFeedByType(feedName, indexName, indexUUID,
	sourceType, sourceName, sourceUUID string,
	streams map[string]Stream) error {
	if sourceType == "couchbase" {
		// TODO: Should default to DCP feed or projector feed one day.
		return mgr.startTAPFeed(feedName, indexName, indexUUID,
			sourceName, sourceUUID, streams)
	}

	if sourceType == "nil" {
		return mgr.registerFeed(NewNILFeed(feedName, streams))
	}

	return fmt.Errorf("error: startFeed() got unknown source type: %s", sourceType)
}

func (mgr *Manager) startTAPFeed(feedName, indexName, indexUUID,
	bucketName, bucketUUID string, streams map[string]Stream) error {
	feed, err := NewTAPFeed(feedName, mgr.server, "default",
		bucketName, bucketUUID, streams)
	if err != nil {
		return fmt.Errorf("error: could not prepare TAP stream to server: %s,"+
			" bucketName: %s, indexName: %s, err: %v",
			mgr.server, bucketName, indexName, err)
		// TODO: need a way to collect these errors so REST api
		// can show them to user ("hey, perhaps you deleted a bucket
		// and should delete these related full-text indexes?
		// or the couchbase cluster is just down.");
		// perhaps as specialized clog writer?
		// TODO: cleanup on error?
	}

	if err = feed.Start(); err != nil {
		// TODO: need way to track dead cows (non-beef)
		// TODO: cleanup?
		return fmt.Errorf("error: could not start feed, server: %s, err: %v",
			mgr.server, err)
	}

	if err = mgr.registerFeed(feed); err != nil {
		// TODO: cleanup?
		return err
	}

	return nil
}
<commit_msg>panic on SourcePartition collision during feed creation<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"

	log "github.com/couchbaselabs/clog"
)

// A janitor maintains PIndexes and Feeds, creating, deleting &
// hooking them up as necessary to try to match to latest plans.

func (mgr *Manager) JanitorNOOP(msg string) {
	SyncWorkReq(mgr.janitorCh, WORK_NOOP, msg)
}

func (mgr *Manager) JanitorKick(msg string) {
	SyncWorkReq(mgr.janitorCh, WORK_KICK, msg)
}

func (mgr *Manager) JanitorLoop() {
	for m := range mgr.janitorCh {
		if m.op == WORK_KICK {
			err := mgr.JanitorOnce(m.msg)
			if err != nil {
				log.Printf("error: JanitorOnce, err: %v", err)
				// Keep looping as perhaps it's a transient issue.
				// TODO: Need a better way to track warnings from errors,
				// and which need a future, rescheduled janitor kick.
			}
		} else if m.op == WORK_NOOP {
			// NOOP.
		} else {
			log.Printf("error: unknown janitor op: %s, m: %#v", m.op, m)
		}
		if m.resCh != nil {
			close(m.resCh)
		}
	}
}

func (mgr *Manager) JanitorOnce(reason string) error {
	log.Printf("janitor awakes, reason: %s", reason)

	if mgr.cfg == nil { // Can occur during testing.
		return fmt.Errorf("janitor skipped due to nil cfg")
	}

	planPIndexes, _, err := CfgGetPlanPIndexes(mgr.cfg)
	if err != nil {
		return fmt.Errorf("janitor skipped on CfgGetPlanPIndexes err: %v", err)
	}
	if planPIndexes == nil {
		// Might happen if janitor wins a race.
		// TODO: Should we reschedule another future janitor kick?
		return fmt.Errorf("janitor skipped on nil planPIndexes")
	}

	currFeeds, currPIndexes := mgr.CurrentMaps()

	addPlanPIndexes, removePIndexes :=
		CalcPIndexesDelta(mgr.uuid, currPIndexes, planPIndexes)

	log.Printf("janitor pindexes to add:")
	for _, ppi := range addPlanPIndexes {
		log.Printf("  %+v", ppi)
	}
	log.Printf("janitor pindexes to remove:")
	for _, pi := range removePIndexes {
		log.Printf("  %+v", pi)
	}

	// First, teardown pindexes that need to be removed.
	for _, removePIndex := range removePIndexes {
		log.Printf("janitor removing pindex: %s", removePIndex.Name)
		err = mgr.stopPIndex(removePIndex)
		if err != nil {
			return fmt.Errorf("error: janitor removing pindex: %s, err: %v",
				removePIndex.Name, err)
		}
	}
	// Then, (re-)create pindexes that we're missing.
	for _, addPlanPIndex := range addPlanPIndexes {
		log.Printf("janitor adding pindex: %s", addPlanPIndex.Name)
		err = mgr.startPIndex(addPlanPIndex)
		if err != nil {
			return fmt.Errorf("error: janitor adding pindex: %s, err: %v",
				addPlanPIndex.Name, err)
		}
	}

	currFeeds, currPIndexes = mgr.CurrentMaps()

	addFeeds, removeFeeds := CalcFeedsDelta(currFeeds, currPIndexes)

	log.Printf("janitor feeds to add:")
	for _, targetPIndexes := range addFeeds {
		if len(targetPIndexes) > 0 {
			log.Printf("  %s", FeedName(targetPIndexes[0]))
		}
	}
	log.Printf("janitor feeds to remove:")
	for _, removeFeed := range removeFeeds {
		log.Printf("  %s", removeFeed.Name())
	}

	// First, teardown feeds that need to be removed.
	for _, removeFeed := range removeFeeds {
		err = mgr.stopFeed(removeFeed)
		if err != nil {
			return fmt.Errorf("error: janitor removing feed name: %s, err: %v",
				removeFeed.Name, err)
		}
	}
	// Then, (re-)create feeds that we're missing.
	for addFeedName, targetPIndexes := range addFeeds {
		err = mgr.startFeed(targetPIndexes)
		if err != nil {
			return fmt.Errorf("error: janitor adding feed, addFeedName: %s, err: %v",
				addFeedName, err)
		}
	}

	return nil
}

// --------------------------------------------------------

// Functionally determine the delta of which pindexes need creation
// and which should be shut down on our local node (mgrUUID).
func CalcPIndexesDelta(mgrUUID string,
	currPIndexes map[string]*PIndex,
	wantedPlanPIndexes *PlanPIndexes) (
	addPlanPIndexes []*PlanPIndex,
	removePIndexes []*PIndex) {
	// Allocate our return arrays.
	addPlanPIndexes = make([]*PlanPIndex, 0)
	removePIndexes = make([]*PIndex, 0)

	// Just for fast transient lookups.
	mapWantedPlanPIndex := make(map[string]*PlanPIndex)
	mapRemovePIndex := make(map[string]*PIndex)

	// For each wanted plan pindex, if a pindex does not exist or is
	// different, then schedule to add.
	for _, wantedPlanPIndex := range wantedPlanPIndexes.PlanPIndexes {
	nodeUUIDs:
		for nodeUUID, nodeState := range wantedPlanPIndex.NodeUUIDs {
			if nodeUUID != mgrUUID || nodeState != "active" {
				continue nodeUUIDs
			}

			mapWantedPlanPIndex[wantedPlanPIndex.Name] = wantedPlanPIndex

			currPIndex, exists := currPIndexes[wantedPlanPIndex.Name]
			if !exists {
				addPlanPIndexes = append(addPlanPIndexes, wantedPlanPIndex)
			} else if PIndexMatchesPlan(currPIndex, wantedPlanPIndex) == false {
				addPlanPIndexes = append(addPlanPIndexes, wantedPlanPIndex)
				removePIndexes = append(removePIndexes, currPIndex)
				mapRemovePIndex[currPIndex.Name] = currPIndex
			}

			break nodeUUIDs
		}
	}

	// For each existing pindex, if not part of wanted plan pindex,
	// then schedule for removal.
	for _, currPIndex := range currPIndexes {
		if _, exists := mapWantedPlanPIndex[currPIndex.Name]; !exists {
			if _, exists = mapRemovePIndex[currPIndex.Name]; !exists {
				removePIndexes = append(removePIndexes, currPIndex)
				mapRemovePIndex[currPIndex.Name] = currPIndex
			}
		}
	}

	return addPlanPIndexes, removePIndexes
}

func PIndexMatchesPlan(pindex *PIndex, planPIndex *PlanPIndex) bool {
	same := pindex.Name == planPIndex.Name &&
		pindex.IndexName == planPIndex.IndexName &&
		pindex.IndexUUID == planPIndex.IndexUUID &&
		pindex.IndexMapping == planPIndex.IndexMapping &&
		pindex.SourceType == planPIndex.SourceType &&
		pindex.SourceName == planPIndex.SourceName &&
		pindex.SourceUUID == planPIndex.SourceUUID &&
		pindex.SourcePartitions == planPIndex.SourcePartitions
	if !same {
		log.Printf("PIndexMatchesPlan false, pindex: %#v, planPIndex: %#v",
			pindex, planPIndex)
	}
	return same
}

// --------------------------------------------------------

// Functionally determine the delta of which feeds need creation and
// which should be shut down.
func CalcFeedsDelta(currFeeds map[string]Feed, pindexes map[string]*PIndex) (
	addFeeds [][]*PIndex, removeFeeds []Feed) {
	// Allocate result holders.
	addFeeds = make([][]*PIndex, 0)
	removeFeeds = make([]Feed, 0)

	// Group the pindexes by their feed names.
	groupedPIndexes := make(map[string][]*PIndex)
	for _, pindex := range pindexes {
		feedName := FeedName(pindex)
		arr, exists := groupedPIndexes[feedName]
		if !exists {
			arr = make([]*PIndex, 0)
		}
		groupedPIndexes[feedName] = append(arr, pindex)
	}

	for feedName, feedPIndexes := range groupedPIndexes {
		if _, exists := currFeeds[feedName]; !exists {
			addFeeds = append(addFeeds, feedPIndexes)
		}
	}

	for currFeedName, currFeed := range currFeeds {
		if _, exists := groupedPIndexes[currFeedName]; !exists {
			removeFeeds = append(removeFeeds, currFeed)
		}
	}

	return addFeeds, removeFeeds
}

func FeedName(pindex *PIndex) string {
	// TODO: Different feed types might have different name formulas
	// depending on whether they have a "single cluster" abstraction
	// or work on a "node by node" basis (in where stream destinations
	// need to be part of the name encoding).
	//
	// NOTE: We're depending on the IndexName/IndexUUID to
	// functionally "cover" the SourceType/SourceName/SourceUUID, so
	// we don't need to encode the source parts into the feed name.
	return pindex.IndexName + "_" + pindex.IndexUUID
}

// --------------------------------------------------------

func (mgr *Manager) startPIndex(planPIndex *PlanPIndex) error {
	pindex, err := NewPIndex(planPIndex.Name, NewUUID(),
		planPIndex.IndexType,
		planPIndex.IndexName,
		planPIndex.IndexUUID,
		planPIndex.IndexMapping,
		planPIndex.SourceType,
		planPIndex.SourceName,
		planPIndex.SourceUUID,
		planPIndex.SourcePartitions,
		mgr.PIndexPath(planPIndex.Name))
	if err != nil {
		return fmt.Errorf("error: NewPIndex, name: %s, err: %v",
			planPIndex.Name, err)
	}

	if err = mgr.registerPIndex(pindex); err != nil {
		close(pindex.Stream)
		return err
	}

	return nil
}

func (mgr *Manager) stopPIndex(pindex *PIndex) error {
	feeds, _ := mgr.CurrentMaps()
	for _, feed := range feeds {
		for _, stream := range feed.Streams() {
			if stream == pindex.Stream {
				if err := mgr.stopFeed(feed); err != nil {
					panic(fmt.Sprintf("error: could not stop feed, err: %v", err))
				}
			}
		}
	}

	pindexUnreg := mgr.unregisterPIndex(pindex.Name)
	if pindexUnreg != nil && pindexUnreg != pindex {
		panic("unregistered pindex isn't the one we're stopping")
	}

	close(pindex.Stream)

	return nil
}

// --------------------------------------------------------

func (mgr *Manager) startFeed(pindexes []*PIndex) error {
	if len(pindexes) <= 0 {
		return nil
	}

	pindexFirst := pindexes[0]
	feedName := FeedName(pindexFirst)

	streams := make(map[string]Stream)
	for _, pindex := range pindexes {
		if f := FeedName(pindex); f != feedName {
			panic(fmt.Sprintf("error: unexpected feedName: %s != %s", f, feedName))
		}
		if _, exists := streams[pindex.SourcePartitions]; exists {
			panic(fmt.Sprintf("error: SourcePartitions collision: %s",
				pindex.SourcePartitions))
		}

		streams[pindex.SourcePartitions] = pindex.Stream
	}

	return mgr.startFeedByType(feedName,
		pindexFirst.IndexName, pindexFirst.IndexUUID,
		pindexFirst.SourceType, pindexFirst.SourceName, pindexFirst.SourceUUID,
		streams)
}

func (mgr *Manager) stopFeed(feed Feed) error {
	feedUnreg := mgr.unregisterFeed(feed.Name())
	if feedUnreg != nil && feedUnreg != feed {
		panic("error: unregistered feed isn't the one we're closing")
	}

	// NOTE: We're depending on feed to synchronously close, so we
	// know it'll no longer be sending to any of its streams anymore.
	//
	return feed.Close()
}

// --------------------------------------------------------

func (mgr *Manager) startFeedByType(feedName, indexName, indexUUID,
	sourceType, sourceName, sourceUUID string,
	streams map[string]Stream) error {
	if sourceType == "couchbase" {
		// TODO: Should default to DCP feed or projector feed one day.
		return mgr.startTAPFeed(feedName, indexName, indexUUID,
			sourceName, sourceUUID, streams)
	}

	if sourceType == "nil" {
		return mgr.registerFeed(NewNILFeed(feedName, streams))
	}

	return fmt.Errorf("error: startFeed() got unknown source type: %s", sourceType)
}

func (mgr *Manager) startTAPFeed(feedName, indexName, indexUUID,
	bucketName, bucketUUID string, streams map[string]Stream) error {
	feed, err := NewTAPFeed(feedName, mgr.server, "default",
		bucketName, bucketUUID, streams)
	if err != nil {
		return fmt.Errorf("error: could not prepare TAP stream to server: %s,"+
			" bucketName: %s, indexName: %s, err: %v",
			mgr.server, bucketName, indexName, err)
		// TODO: need a way to collect these errors so REST api
		// can show them to user ("hey, perhaps you deleted a bucket
		// and should delete these related full-text indexes?
		// or the couchbase cluster is just down.");
		// perhaps as specialized clog writer?
		// TODO: cleanup on error?
	}

	if err = feed.Start(); err != nil {
		// TODO: need way to track dead cows (non-beef)
		// TODO: cleanup?
		return fmt.Errorf("error: could not start feed, server: %s, err: %v",
			mgr.server, err)
	}

	if err = mgr.registerFeed(feed); err != nil {
		// TODO: cleanup?
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package mangadownloader

import (
	"code.google.com/p/go.net/html"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
)

var (
	regexpImageContentType *regexp.Regexp
)

func init() {
	regexpImageContentType, _ = regexp.Compile("^image/(.+)$")
}

type MangaDownloader struct {
	Services []Service
}

func CreateDefaultMangeDownloader() *MangaDownloader {
	md := &MangaDownloader{}

	md.Services = append(md.Services, &MangaReaderService{
		Md: md,
	})

	return md
}

func (md *MangaDownloader) Identify(u *url.URL) (interface{}, error) {
	for _, service := range md.Services {
		if service.Supports(u) {
			return service.Identify(u)
		}
	}

	return nil, errors.New("Unsupported url")
}

func (md *MangaDownloader) DownloadManga(manga *Manga, out string) error {
	name, err := manga.Name()
	if err != nil {
		return err
	}
	out = filepath.Join(out, name)
	chapters, err := manga.Chapters()
	if err != nil {
		return err
	}
	for _, chapter := range chapters {
		err := md.DownloadChapter(chapter, out)
		if err != nil {
			return err
		}
	}

	return nil
}

func (md *MangaDownloader) DownloadChapter(chapter *Chapter, out string) error {
	name, err := chapter.Name()
	if err != nil {
		return err
	}
	out = filepath.Join(out, name)
	pages, err := chapter.Pages()
	if err != nil {
		return err
	}
	for _, page := range pages {
		err := md.DownloadPage(page, out)
		if err != nil {
			return err
		}
	}

	return nil
}

func (md *MangaDownloader) DownloadPage(page *Page, out string) error {
	index, err := page.Index()
	if err != nil {
		return err
	}
	out = filepath.Join(out, fmt.Sprintf("%04d", index))

	image, err := page.Image()
	if err != nil {
		return err
	}
	var response *http.Response
	for {
		response, err = md.HttpGet(image.Url)
		if err == nil {
			break
		}
		fmt.Println(err)
	}
	defer response.Body.Close()
	var extension string
	if len(extension) == 0 {
		contentType := response.Header.Get("content-type")
		if len(contentType) > 0 {
			matches := regexpImageContentType.FindStringSubmatch(contentType)
			if matches != nil && len(matches) == 2 {
				extension = matches[1]
			}
		}
	}
	if len(extension) > 0 {
		if extension == "jpeg" {
			extension = "jpg"
		}
		out += "." + extension
	}
	err = os.MkdirAll(filepath.Dir(out), 0755)
	if err != nil {
		return err
	}
	file, err := os.OpenFile(out, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = io.Copy(file, response.Body)
	if err != nil {
		return err
	}

	return nil
}

func (md *MangaDownloader) HttpGet(u *url.URL) (*http.Response, error) {
	return http.Get(u.String())
}

func (md *MangaDownloader) HttpGetHtml(u *url.URL) (*html.Node, error) {
	response, err := md.HttpGet(u)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	node, err := html.Parse(response.Body)
	return node, err
}
<commit_msg>Basic http retry<commit_after>package mangadownloader

import (
	"code.google.com/p/go.net/html"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
)

var (
	regexpImageContentType *regexp.Regexp
)

func init() {
	regexpImageContentType, _ = regexp.Compile("^image/(.+)$")
}

type MangaDownloader struct {
	Services []Service
}

func CreateDefaultMangeDownloader() *MangaDownloader {
	md := &MangaDownloader{}

	md.Services = append(md.Services, &MangaReaderService{
		Md: md,
	})

	return md
}

func (md *MangaDownloader) Identify(u *url.URL) (interface{}, error) {
	for _, service := range md.Services {
		if service.Supports(u) {
			return service.Identify(u)
		}
	}

	return nil, errors.New("Unsupported url")
}

func (md *MangaDownloader) DownloadManga(manga *Manga, out string) error {
	name, err := manga.Name()
	if err != nil {
		return err
	}
	out = filepath.Join(out, name)
	chapters, err := manga.Chapters()
	if err != nil {
		return err
	}
	for _, chapter := range chapters {
		err := md.DownloadChapter(chapter, out)
		if err != nil {
			return err
		}
	}

	return nil
}

func (md *MangaDownloader) DownloadChapter(chapter *Chapter, out string) error {
	name, err := chapter.Name()
	if err != nil {
		return err
	}
	out = filepath.Join(out, name)
	pages, err := chapter.Pages()
	if err != nil {
		return err
	}
	for _, page := range pages {
		err := md.DownloadPage(page, out)
		if err != nil {
			return err
		}
	}

	return nil
}

func (md *MangaDownloader) DownloadPage(page *Page, out string) error {
	index, err := page.Index()
	if err != nil {
		return err
	}
	out = filepath.Join(out, fmt.Sprintf("%04d", index))

	image, err := page.Image()
	if err != nil {
		return err
	}
	response, err := md.HttpGet(image.Url)
	defer response.Body.Close()
	var extension string
	if len(extension) == 0 {
		contentType := response.Header.Get("content-type")
		if len(contentType) > 0 {
			matches := regexpImageContentType.FindStringSubmatch(contentType)
			if matches != nil && len(matches) == 2 {
				extension = matches[1]
			}
		}
	}
	if len(extension) > 0 {
		if extension == "jpeg" {
			extension = "jpg"
		}
		out += "." + extension
	}
	err = os.MkdirAll(filepath.Dir(out), 0755)
	if err != nil {
		return err
	}
	file, err := os.OpenFile(out, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = io.Copy(file, response.Body)
	if err != nil {
		return err
	}

	return nil
}

func (md *MangaDownloader) HttpGet(u *url.URL) (response *http.Response, err error) {
	for i := 0; i < 5; i++ {
		response, err = http.Get(u.String())
		if err == nil {
			break
		}
		fmt.Println(err)
	}
	return
}

func (md *MangaDownloader) HttpGetHtml(u *url.URL) (*html.Node, error) {
	response, err := md.HttpGet(u)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	node, err := html.Parse(response.Body)
	return node, err
}
<|endoftext|>


<commit_before>package orm

import (
	"fmt"

	c "github.com/d0ngw/go/common"
)

// DBConfigurer DB配置器
type DBConfigurer interface {
	c.Configurer
	DBConfig() *DBConfig
}

// DBShardConfigurer db shard configurer
type DBShardConfigurer interface {
	c.Configurer
	DBShardConfig() *DBShardConfig
}

// EntityShardConfigurer entity shard configurer
type EntityShardConfigurer interface {
	c.Configurer
	EntityShardConfig() *EntityShardConfig
}

//DBConfig 数据库配置
type DBConfig struct {
	User          string `yaml:"user"`
	Pass          string `yaml:"pass"`
	URL           string `yaml:"url"`
	Schema        string `yaml:"schema"`
	MaxConn       int    `yaml:"maxConn"`
	MaxIdle       int    `yaml:"maxIdle"`
	MaxTimeSecond int    `yaml:"maxTimeSecond"`
	Charset       string `yaml:"charset"`
}

// Parse implements DBConfigurer
func (p *DBConfig) Parse() error {
	if p.URL == "" {
		return fmt.Errorf("need url")
	}
	if p.Schema == "" {
		return fmt.Errorf("need schema")
	}
	return nil
}

// DBConfig implements DBConfigurer
func (p *DBConfig) DBConfig() *DBConfig {
	return p
}

// DBShardConfig db shard config
type DBShardConfig struct {
	Shards  map[string]*DBConfig `yaml:"shards"`
	Default string               `yaml:"default"`
}

// Parse implements Configurer.Parse
func (p *DBShardConfig) Parse() error {
	c.Infof("db shards count:%d", len(p.Shards))

	for k, v := range p.Shards {
		if v == nil {
			return fmt.Errorf("no db config for %s", k)
		}
		if err := v.Parse(); err != nil {
			return err
		}
	}

	if p.Default != "" {
		if p.Shards[p.Default] == nil {
			return fmt.Errorf("can't find default shard %s", p.Default)
		}
	}
	return nil
}

// DBShardConfig implements DBShardConfigurer
func (p *DBShardConfig) DBShardConfig() *DBShardConfig {
	return p
}

// EntityShardRuleConfig 实体的shard规则
type EntityShardRuleConfig struct {
	Name       string   `yaml:"name"`        //名称
	DBShard    *OneRule `yaml:"db_shard"`    //数据库实例的配置
	TableShard *OneRule `yaml:"table_shard"` //数据库表的配置
	Default    bool     `yaml:"default"`     //是否是默认规则
	meta       Meta
}

// Parse implements Configurer.Parse
func (p *EntityShardRuleConfig) Parse() error {
	if p.Name == "" {
		return fmt.Errorf("invalid name")
	}
	if p.DBShard != nil {
		if err := p.DBShard.Parse(); err != nil {
			return fmt.Errorf("parse db_shard fail,name:%s,err:%v", p.Name, err)
		}
	}
	if p.TableShard != nil {
		if err := p.TableShard.Parse(); err != nil {
			return fmt.Errorf("parse table_shard fail,name:%s,err:%v", p.Name, err)
		}
	}
	return nil
}

type entityRule struct {
	meta        Meta
	defaultRule *EntityShardRuleConfig
	rules       map[string]*EntityShardRuleConfig
}

// EntityShardConfig entity shad config
type EntityShardConfig struct {
	// pkgPath -> entity name -> rules
	Entities map[string]map[string][]*EntityShardRuleConfig `yaml:"entities"`
	entities map[string]map[string]*entityRule
}

// Parse implements Configurer.Parse
func (p *EntityShardConfig) Parse() error {
	if len(p.Entities) == 0 {
		c.Infof("no entities")
		return nil
	}

	entities := map[string]map[string]*entityRule{}

	for pkgPath, pkgEntities := range p.Entities {
		pkg := map[string]*entityRule{}
		entities[pkgPath] = pkg
		for entityName, rules := range pkgEntities {
			meta := findMetaWithPkgAndName(pkgPath, entityName)
			if meta == nil {
				return fmt.Errorf("can't find meta for %s.%s", pkgPath, entityName)
			}

			entity := &entityRule{rules: map[string]*EntityShardRuleConfig{}, meta: meta}
			pkg[entityName] = entity

			for _, rule := range rules {
				if err := rule.Parse(); err != nil {
					return fmt.Errorf("parse %s/%s %s fail,err:%v", pkgPath, entityName, rule.Name, err)
				}
				rule.meta = meta
				if rule.Default {
					if entity.defaultRule == nil {
						entity.defaultRule = rule
					} else {
						return fmt.Errorf("duplicate default rule for %s/%s", pkgPath, entityName)
					}
				}
				if entity.rules[rule.Name] != nil {
					return fmt.Errorf("duplicate default rule for %s/%s", pkgPath, entityName)
				}
				entity.rules[rule.Name] = rule
			}
		}
	}
	p.entities = entities
	return nil
}

// EntityShardConfig implements EntityShardConfigurer
func (p *EntityShardConfig) EntityShardConfig() *EntityShardConfig {
	return p
}
<commit_msg>allow empty DBShardConfig<commit_after>package orm

import (
	"fmt"

	c "github.com/d0ngw/go/common"
)

// DBConfigurer DB配置器
type DBConfigurer interface {
	c.Configurer
	DBConfig() *DBConfig
}

// DBShardConfigurer db shard configurer
type DBShardConfigurer interface {
	c.Configurer
	DBShardConfig() *DBShardConfig
}

// EntityShardConfigurer entity shard configurer
type EntityShardConfigurer interface {
	c.Configurer
	EntityShardConfig() *EntityShardConfig
}

//DBConfig 数据库配置
type DBConfig struct {
	User          string `yaml:"user"`
	Pass          string `yaml:"pass"`
	URL           string `yaml:"url"`
	Schema        string `yaml:"schema"`
	MaxConn       int    `yaml:"maxConn"`
	MaxIdle       int    `yaml:"maxIdle"`
	MaxTimeSecond int    `yaml:"maxTimeSecond"`
	Charset       string `yaml:"charset"`
}

// Parse implements DBConfigurer
func (p *DBConfig) Parse() error {
	if p.URL == "" {
		return fmt.Errorf("need url")
	}
	if p.Schema == "" {
		return fmt.Errorf("need schema")
	}
	return nil
}

// DBConfig implements DBConfigurer
func (p *DBConfig) DBConfig() *DBConfig {
	return p
}

// DBShardConfig db shard config
type DBShardConfig struct {
	Shards  map[string]*DBConfig `yaml:"shards"`
	Default string               `yaml:"default"`
}

// Parse implements Configurer.Parse
func (p *DBShardConfig) Parse() error {
	if p == nil {
		c.Warnf("no db config")
		return nil
	}

	c.Infof("db shards count:%d", len(p.Shards))
	for k, v := range p.Shards {
		if v == nil {
			return fmt.Errorf("no db config for %s", k)
		}
		if err := v.Parse(); err != nil {
			return err
		}
	}

	if p.Default != "" {
		if p.Shards[p.Default] == nil {
			return fmt.Errorf("can't find default shard %s", p.Default)
		}
	}
	return nil
}

// DBShardConfig implements DBShardConfigurer
func (p *DBShardConfig) DBShardConfig() *DBShardConfig {
	return p
}

// EntityShardRuleConfig 实体的shard规则
type EntityShardRuleConfig struct {
	Name       string   `yaml:"name"`        //名称
	DBShard    *OneRule `yaml:"db_shard"`    //数据库实例的配置
	TableShard *OneRule `yaml:"table_shard"` //数据库表的配置
	Default    bool     `yaml:"default"`     //是否是默认规则
	meta       Meta
}

// Parse implements Configurer.Parse
func (p *EntityShardRuleConfig) Parse() error {
	if p.Name == "" {
		return fmt.Errorf("invalid name")
	}
	if p.DBShard != nil {
		if err := p.DBShard.Parse(); err != nil {
			return fmt.Errorf("parse db_shard fail,name:%s,err:%v", p.Name, err)
		}
	}
	if p.TableShard != nil {
		if err := p.TableShard.Parse(); err != nil {
			return fmt.Errorf("parse table_shard fail,name:%s,err:%v", p.Name, err)
		}
	}
	return nil
}

type entityRule struct {
	meta        Meta
	defaultRule *EntityShardRuleConfig
	rules       map[string]*EntityShardRuleConfig
}

// EntityShardConfig entity shad config
type EntityShardConfig struct {
	// pkgPath -> entity name -> rules
	Entities map[string]map[string][]*EntityShardRuleConfig `yaml:"entities"`
	entities map[string]map[string]*entityRule
}

// Parse implements Configurer.Parse
func (p *EntityShardConfig) Parse() error {
	if len(p.Entities) == 0 {
		c.Infof("no entities")
		return nil
	}

	entities := map[string]map[string]*entityRule{}

	for pkgPath, pkgEntities := range p.Entities {
		pkg := map[string]*entityRule{}
		entities[pkgPath] = pkg
		for entityName, rules := range pkgEntities {
			meta := findMetaWithPkgAndName(pkgPath, entityName)
			if meta == nil {
				return fmt.Errorf("can't find meta for %s.%s", pkgPath, entityName)
			}

			entity := &entityRule{rules: map[string]*EntityShardRuleConfig{}, meta: meta}
			pkg[entityName] = entity

			for _, rule := range rules {
				if err := rule.Parse(); err != nil {
					return fmt.Errorf("parse %s/%s %s fail,err:%v", pkgPath, entityName, rule.Name, err)
				}
				rule.meta = meta
				if rule.Default {
					if entity.defaultRule == nil {
						entity.defaultRule = rule
					} else {
						return fmt.Errorf("duplicate default rule for %s/%s", pkgPath, entityName)
					}
				}
				if entity.rules[rule.Name] != nil {
					return fmt.Errorf("duplicate default rule for %s/%s", pkgPath, entityName)
				}
				entity.rules[rule.Name] = rule
			}
		}
	}
	p.entities = entities
	return nil
}

// EntityShardConfig implements EntityShardConfigurer
func (p *EntityShardConfig) EntityShardConfig() *EntityShardConfig {
	return p
}
<|endoftext|>


<commit_before>package orm

import (
	"fmt"
	"reflect"
	"sync"
)

var _tables = newTables()

type tableInProgress struct {
	table *Table
	wg    sync.WaitGroup
}

// GetTable returns a Table for a struct type.
func GetTable(typ reflect.Type) *Table {
	return _tables.Get(typ)
}

// RegisterTable registers a struct as SQL table.
// It is usually used to register intermediate table
// in many to many relationship.
func RegisterTable(strct interface{}) {
	_tables.Register(strct)
}

type tables struct {
	mu         sync.RWMutex
	inProgress map[reflect.Type]*tableInProgress
	tables     map[reflect.Type]*Table
}

func newTables() *tables {
	return &tables{
		inProgress: make(map[reflect.Type]*tableInProgress),
		tables:     make(map[reflect.Type]*Table),
	}
}

func (t *tables) Register(strct interface{}) {
	typ := reflect.TypeOf(strct)
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}
	_ = t.Get(typ)
}

func (t *tables) get(typ reflect.Type, allowInProgress bool) *Table {
	if typ.Kind() != reflect.Struct {
		panic(fmt.Errorf("got %s, wanted %s", typ.Kind(), reflect.Struct))
	}

	t.mu.RLock()
	table, ok := t.tables[typ]
	t.mu.RUnlock()
	if ok {
		return table
	}

	t.mu.Lock()

	table, ok = t.tables[typ]
	if ok {
		t.mu.Unlock()
		return table
	}

	inProgress := t.inProgress[typ]
	if inProgress != nil {
		t.mu.Unlock()
		if !allowInProgress {
			inProgress.wg.Wait()
		}
		return inProgress.table
	}

	table = newTable(typ)
	inProgress = &tableInProgress{
		table: table,
	}
	inProgress.wg.Add(1)
	t.inProgress[typ] = inProgress

	t.mu.Unlock()
	table.init()
	inProgress.wg.Done()
	t.mu.Lock()

	delete(t.inProgress, typ)
	t.tables[typ] = table

	t.mu.Unlock()
	return table
}

func (t *tables) Get(typ reflect.Type) *Table {
	return t.get(typ, false)
}

func (t *tables) getByName(name string) *Table {
	t.mu.RLock()
	defer t.mu.RUnlock()

	for _, t := range t.tables {
		if string(t.Name) == name {
			return t
		}
	}

	return nil
}
<commit_msg>orm: fix getByName<commit_after>package orm

import (
	"fmt"
	"reflect"
	"sync"
)

var _tables = newTables()

type tableInProgress struct {
	table *Table
	wg    sync.WaitGroup
}

// GetTable returns a Table for a struct type.
func GetTable(typ reflect.Type) *Table {
	return _tables.Get(typ)
}

// RegisterTable registers a struct as SQL table.
// It is usually used to register intermediate table
// in many to many relationship.
func RegisterTable(strct interface{}) {
	_tables.Register(strct)
}

type tables struct {
	mu         sync.RWMutex
	inProgress map[reflect.Type]*tableInProgress
	tables     map[reflect.Type]*Table
}

func newTables() *tables {
	return &tables{
		inProgress: make(map[reflect.Type]*tableInProgress),
		tables:     make(map[reflect.Type]*Table),
	}
}

func (t *tables) Register(strct interface{}) {
	typ := reflect.TypeOf(strct)
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}
	_ = t.Get(typ)
}

func (t *tables) get(typ reflect.Type, allowInProgress bool) *Table {
	if typ.Kind() != reflect.Struct {
		panic(fmt.Errorf("got %s, wanted %s", typ.Kind(), reflect.Struct))
	}

	t.mu.RLock()
	table, ok := t.tables[typ]
	t.mu.RUnlock()
	if ok {
		return table
	}

	t.mu.Lock()

	table, ok = t.tables[typ]
	if ok {
		t.mu.Unlock()
		return table
	}

	inProgress := t.inProgress[typ]
	if inProgress != nil {
		t.mu.Unlock()
		if !allowInProgress {
			inProgress.wg.Wait()
		}
		return inProgress.table
	}

	table = newTable(typ)
	inProgress = &tableInProgress{
		table: table,
	}
	inProgress.wg.Add(1)
	t.inProgress[typ] = inProgress

	t.mu.Unlock()
	table.init()
	inProgress.wg.Done()
	t.mu.Lock()

	delete(t.inProgress, typ)
	t.tables[typ] = table

	t.mu.Unlock()
	return table
}

func (t *tables) Get(typ reflect.Type) *Table {
	return t.get(typ, false)
}

func (t *tables) getByName(name string) *Table {
	t.mu.RLock()
	defer t.mu.RUnlock()

	for _, t := range t.tables {
		if t.ModelName == name {
			return t
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package migrations

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"io"

	"github.com/BurntSushi/migration"
)

func AddBuildEvents(tx migration.LimitedTx) error {
	_, err := tx.Exec(`
    CREATE TABLE build_events (
      id serial PRIMARY KEY,
      build_id integer REFERENCES builds (id),
			type varchar(32) NOT NULL,
      payload text NOT NULL
    )
  `)
	if err != nil {
		return err
	}

	cursor := 0

	for {
		var id int
		var buildLog sql.NullString

		err := tx.QueryRow(`
			SELECT id, log
			FROM builds
			WHERE id > $1
			LIMIT 1
		`, cursor).Scan(&id, &buildLog)
		if err != nil {
			if err == sql.ErrNoRows {
				break
			}

			return err
		}

		cursor = id

		if !buildLog.Valid {
			continue
		}

		decoder := json.NewDecoder(bytes.NewBufferString(buildLog.String))

		var version version
		err = decoder.Decode(&version)
		if err != nil {
			_, err = tx.Exec(`
					INSERT INTO build_events (build_id, type, payload)
					VALUES ($1, $2, $3)
				`, id, "version", "0.0")
			if err != nil {
				return err
			}

			_, err = tx.Exec(`
						INSERT INTO build_events (build_id, type, payload)
						VALUES ($1, $2, $3)
					`, id, "log", buildLog.String)
			if err != nil {
				return err
			}

			continue
		}

		versionEnc, err := json.Marshal(version.Version)
		if err != nil {
			return err
		}

		_, err = tx.Exec(`
				INSERT INTO build_events (build_id, type, payload)
				VALUES ($1, $2, $3)
			`, id, "version", versionEnc)
		if err != nil {
			return err
		}

		for {
			var event eventEnvelope
			err := decoder.Decode(&event)
			if err != nil {
				if err == io.EOF {
					break
				}

				return err
			}

			_, err = tx.Exec(`
					INSERT INTO build_events (build_id, type, payload)
					VALUES ($1, $2, $3)
				`, id, event.Type, []byte(*event.EventPayload))
			if err != nil {
				return err
			}
		}
	}

	_, err = tx.Exec(`
		ALTER TABLE builds
		DROP COLUMN log
	`)
	if err != nil {
		return err
	}

	return nil
}

type eventEnvelope struct {
	Type         string           `json:"type"`
	EventPayload *json.RawMessage `json:"event"`
}

type version struct {
	Version string `json:"version"`
}
<commit_msg>make events migration more durable<commit_after>package migrations

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"fmt"
	"io"

	"github.com/BurntSushi/migration"
)

func AddBuildEvents(tx migration.LimitedTx) error {
	_, err := tx.Exec(`
    CREATE TABLE build_events (
      id serial PRIMARY KEY,
      build_id integer REFERENCES builds (id),
			type varchar(32) NOT NULL,
      payload text NOT NULL
    )
  `)
	if err != nil {
		return err
	}

	cursor := 0

	for {
		var id int
		var buildLog sql.NullString

		err := tx.QueryRow(`
			SELECT id, log
			FROM builds
			WHERE id > $1
			LIMIT 1
		`, cursor).Scan(&id, &buildLog)
		if err != nil {
			if err == sql.ErrNoRows {
				break
			}

			return err
		}

		cursor = id

		if !buildLog.Valid {
			continue
		}

		logBuf := bytes.NewBufferString(buildLog.String)
		decoder := json.NewDecoder(logBuf)

		for {
			var entry logEntry

			err := decoder.Decode(&entry)
			if err != nil {
				if err != io.EOF {
					// non-JSON log; assume v0.0

					_, err = tx.Exec(`
						INSERT INTO build_events (build_id, type, payload)
						VALUES ($1, $2, $3)
					`, id, "version", "0.0")
					if err != nil {
						return err
					}

					_, err = tx.Exec(`
							INSERT INTO build_events (build_id, type, payload)
							VALUES ($1, $2, $3)
						`, id, "log", buildLog.String)
					if err != nil {
						return err
					}
				}

				break
			}

			if entry.Type != "" && entry.EventPayload != nil {
				_, err = tx.Exec(`
						INSERT INTO build_events (build_id, type, payload)
						VALUES ($1, $2, $3)
					`, id, entry.Type, []byte(*entry.EventPayload))
				if err != nil {
					return err
				}

				continue
			}

			if entry.Version != "" {
				versionEnc, err := json.Marshal(entry.Version)
				if err != nil {
					return err
				}

				_, err = tx.Exec(`
					INSERT INTO build_events (build_id, type, payload)
					VALUES ($1, $2, $3)
				`, id, "version", versionEnc)
				if err != nil {
					return err
				}

				continue
			}

			return fmt.Errorf("malformed event stream; got stuck at %s", logBuf.String())
		}
	}

	_, err = tx.Exec(`
		ALTER TABLE builds
		DROP COLUMN log
	`)
	if err != nil {
		return err
	}

	return nil
}

type logEntry struct {
	// either an event...
	Type         string           `json:"type"`
	EventPayload *json.RawMessage `json:"event"`

	// ...or a version
	Version string `json:"version"`
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/datasektionen/taitan/pages"
	"golang.org/x/exp/inotify"
)

var (
	debug     bool   // Show debug level messages.
	info      bool   // Show info level messages.
	responses Atomic // Our parsed responses.
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] ROOT\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(1)
}

func init() {
	flag.BoolVar(&debug, "vv", false, "Print debug messages.")
	flag.BoolVar(&info, "v", false, "Print info messages.")
	flag.Usage = usage
	flag.Parse()
}

func getEnv(env string) string {
	e := os.Getenv(env)
	if e == "" {
		log.Fatalf("$%s environmental variable is not set.\n", env)
	}
	return e
}

func getRoot() string {
	content := getEnv("CONTENT_URL")
	u, err := url.Parse(content)
	if err != nil {
		log.Fatalln("getContent: ", err)
	}

	// https://<token>@github.com/username/repo.git
	u.User = url.User(getEnv("TOKEN"))

	base := filepath.Base(u.Path)
	return strings.TrimSuffix(base, filepath.Ext(base))
}

func getContent() {
	content := getEnv("CONTENT_URL")
	u, err := url.Parse(content)
	if err != nil {
		log.Fatalln("getContent: ", err)
	}

	// https://<token>@github.com/username/repo.git
	u.User = url.User(getEnv("TOKEN"))

	root := getRoot()
	if _, err = os.Stat(root); os.IsNotExist(err) {
		log.Debugln("No root directory - cloning content url!")
		cmd := exec.Command("git", "clone", u.String())
		err = cmd.Start()
		if err != nil {
			log.Fatal(err)
		}
		log.Debugln("Waiting for git clone to finish...")
		err = cmd.Wait()
		if err != nil {
			log.Warnln("Cloned with error: %v\n", err)
		}
	} else {
		log.Debugln("Found root directory - pulling updates!")
		cmd := exec.Command("git", fmt.Sprintf("--git-dir=%s/.git", root), "pull")
		err = cmd.Start()
		if err != nil {
			log.Fatal(err)
		}
		log.Debugln("Waiting for git pull to finish...")
		err = cmd.Wait()
		if err != nil {
			log.Warnf("Pulled with error: %v\n", err)
		}
	}
}

// setVerbosity sets the amount of messages printed.
func setVerbosity() {
	switch {
	case debug:
		log.SetLevel(log.DebugLevel)
	case info:
		log.SetLevel(log.InfoLevel)
	default:
		log.SetLevel(log.WarnLevel)
	}
}

// Atomic responses.
type Atomic struct {
	sync.Mutex
	Resps map[string]*pages.Resp
}

func validRoot(root string) {
	fi, err := os.Stat(root)
	if err != nil {
		if os.IsNotExist(err) {
			log.Fatalf("Directory doesn't exist: %q", root)
		}
		log.Fatalln(err)
	}
	if !fi.IsDir() {
		log.Fatalf("Supplied path is not a directory: %q", root)
	}
}

func main() {
	setVerbosity()

	// Get port or die.
	port := getEnv("PORT")

	// Get content or die.
	getContent()
	go func() {
		for {
			time.Sleep(time.Second * 20)
			getContent()
		}
	}()

	root := getRoot()
	log.WithField("Root", root).Info("Our root directory")

	// We'll parse and store the responses ahead of time.
	resps, err := pages.Load(root)
	if err != nil {
		log.Fatalf("pages.Load: unexpected error: %s", err)
	}
	log.WithField("Resps", resps).Debug("The parsed responses")
	responses = Atomic{Resps: resps}

	// Watch the directory for any changes. If the directory has any changes we'll
	// update our responses.
	go watch(root)

	log.Info("Starting server.")
	log.Info("Listening on port: ", port)

	// Our request handler.
	http.HandleFunc("/", handler)

	// Listen on port and serve with our handler.
	err = http.ListenAndServe(":"+port, nil)
	if err != nil {
		panic(err)
	}
}

func watch(root string) {
	wd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}
	path := filepath.Join(wd, root)
	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}
	err = watcher.AddWatch(path,
		inotify.IN_CLOSE_WRITE|
			inotify.IN_CREATE|
			inotify.IN_DELETE|
			inotify.IN_MODIFY|
			inotify.IN_MOVED_FROM|
			inotify.IN_MOVED_TO|
			inotify.IN_MOVE)
	if err != nil {
		log.Fatal(err)
	}
	last := time.Now()
	for {
		select {
		case ev := <-watcher.Event:
			if time.Now().Sub(last) < 10*time.Second {
				continue
			}
			log.Println("event:", ev)
			last = time.Now()
			responses.Lock()
			responses.Resps, err = pages.Load(root)
			if err != nil {
				log.Error(err)
			}
			responses.Unlock()
		case err := <-watcher.Error:
			log.Warn("error:", err)
		}
	}
}

// handler parses and serves responses to our file queries.
func handler(res http.ResponseWriter, req *http.Request) {
	if req.Header.Get("X-Github-Event") != "" {
		log.Debugln("GITHUB EVENT - BO YAH - run get content here :D")
	}
	// Requested URL. We extract the path.
	query := req.URL.Path
	log.WithField("query", query).Info("Recieved query")

	clean := filepath.Clean(query)
	log.WithField("clean", clean).Info("Sanitized path")

	responses.Lock()
	r, ok := responses.Resps[clean]
	responses.Unlock()
	if !ok {
		log.WithField("page", clean).Warn("Page doesn't exist")
		res.WriteHeader(http.StatusNotFound)
		return
	}
	log.Info("Marshaling the response.")
	buf, err := json.Marshal(r)
	if err != nil {
		log.Warnf("handler: unexpected error: %#v\n", err)
		res.WriteHeader(http.StatusInternalServerError)
		return
	}
	log.Info("Serve the response.")
	log.Debug("Response: %#v\n", string(buf))
	res.Header().Set("Content-Type", "application/json")
	res.Write(buf)
}
<commit_msg>change debug level<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/datasektionen/taitan/pages"
	"golang.org/x/exp/inotify"
)

var (
	debug     bool   // Show debug level messages.
	info      bool   // Show info level messages.
	responses Atomic // Our parsed responses.
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] ROOT\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(1)
}

func init() {
	flag.BoolVar(&debug, "vv", false, "Print debug messages.")
	flag.BoolVar(&info, "v", false, "Print info messages.")
	flag.Usage = usage
	flag.Parse()
}

func getEnv(env string) string {
	e := os.Getenv(env)
	if e == "" {
		log.Fatalf("$%s environmental variable is not set.\n", env)
	}
	return e
}

func getRoot() string {
	content := getEnv("CONTENT_URL")
	u, err := url.Parse(content)
	if err != nil {
		log.Fatalln("getContent: ", err)
	}

	// https://<token>@github.com/username/repo.git
	u.User = url.User(getEnv("TOKEN"))

	base := filepath.Base(u.Path)
	return strings.TrimSuffix(base, filepath.Ext(base))
}

func getContent() {
	content := getEnv("CONTENT_URL")
	u, err := url.Parse(content)
	if err != nil {
		log.Fatalln("getContent: ", err)
	}

	// https://<token>@github.com/username/repo.git
	u.User = url.User(getEnv("TOKEN"))

	root := getRoot()
	if _, err = os.Stat(root); os.IsNotExist(err) {
		log.Debugln("No root directory - cloning content url!")
		cmd := exec.Command("git", "clone", u.String())
		err = cmd.Start()
		if err != nil {
			log.Fatal(err)
		}
		log.Debugln("Waiting for git clone to finish...")
		err = cmd.Wait()
		if err != nil {
			log.Warnln("Cloned with error: %v\n", err)
		}
	} else {
		log.Debugln("Found root directory - pulling updates!")
		cmd := exec.Command("git", fmt.Sprintf("--git-dir=%s/.git", root), "pull")
		err = cmd.Start()
		if err != nil {
			log.Fatal(err)
		}
		log.Debugln("Waiting for git pull to finish...")
		err = cmd.Wait()
		if err != nil {
			log.Warnf("Pulled with error: %v\n", err)
		}
	}
}

// setVerbosity sets the amount of messages printed.
func setVerbosity() {
	switch {
	case debug:
		log.SetLevel(log.DebugLevel)
	case info:
		log.SetLevel(log.InfoLevel)
	default:
		log.SetLevel(log.WarnLevel)
	}
}

// Atomic responses.
type Atomic struct {
	sync.Mutex
	Resps map[string]*pages.Resp
}

func validRoot(root string) {
	fi, err := os.Stat(root)
	if err != nil {
		if os.IsNotExist(err) {
			log.Fatalf("Directory doesn't exist: %q", root)
		}
		log.Fatalln(err)
	}
	if !fi.IsDir() {
		log.Fatalf("Supplied path is not a directory: %q", root)
	}
}

func main() {
	setVerbosity()

	// Get port or die.
	port := getEnv("PORT")

	// Get content or die.
	getContent()
	go func() {
		for {
			time.Sleep(time.Second * 20)
			getContent()
		}
	}()

	root := getRoot()
	log.WithField("Root", root).Info("Our root directory")

	// We'll parse and store the responses ahead of time.
	resps, err := pages.Load(root)
	if err != nil {
		log.Fatalf("pages.Load: unexpected error: %s", err)
	}
	log.WithField("Resps", resps).Debug("The parsed responses")
	responses = Atomic{Resps: resps}

	// Watch the directory for any changes. If the directory has any changes we'll
	// update our responses.
	go watch(root)

	log.Info("Starting server.")
	log.Info("Listening on port: ", port)

	// Our request handler.
	http.HandleFunc("/", handler)

	// Listen on port and serve with our handler.
	err = http.ListenAndServe(":"+port, nil)
	if err != nil {
		panic(err)
	}
}

func watch(root string) {
	wd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}
	path := filepath.Join(wd, root)
	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}
	err = watcher.AddWatch(path,
		inotify.IN_CLOSE_WRITE|
			inotify.IN_CREATE|
			inotify.IN_DELETE|
			inotify.IN_MODIFY|
			inotify.IN_MOVED_FROM|
			inotify.IN_MOVED_TO|
			inotify.IN_MOVE)
	if err != nil {
		log.Fatal(err)
	}
	last := time.Now()
	for {
		select {
		case ev := <-watcher.Event:
			if time.Now().Sub(last) < 10*time.Second {
				continue
			}
			log.Println("event:", ev)
			last = time.Now()
			responses.Lock()
			responses.Resps, err = pages.Load(root)
			if err != nil {
				log.Error(err)
			}
			responses.Unlock()
		case err := <-watcher.Error:
			log.Warn("error:", err)
		}
	}
}

// handler parses and serves responses to our file queries.
func handler(res http.ResponseWriter, req *http.Request) {
	if req.Header.Get("X-Github-Event") != "" {
		log.Infoln("GITHUB EVENT - BO YAH - run get content here :D")
	}
	// Requested URL. We extract the path.
	query := req.URL.Path
	log.WithField("query", query).Info("Recieved query")

	clean := filepath.Clean(query)
	log.WithField("clean", clean).Info("Sanitized path")

	responses.Lock()
	r, ok := responses.Resps[clean]
	responses.Unlock()
	if !ok {
		log.WithField("page", clean).Warn("Page doesn't exist")
		res.WriteHeader(http.StatusNotFound)
		return
	}
	log.Info("Marshaling the response.")
	buf, err := json.Marshal(r)
	if err != nil {
		log.Warnf("handler: unexpected error: %#v\n", err)
		res.WriteHeader(http.StatusInternalServerError)
		return
	}
	log.Info("Serve the response.")
	log.Debug("Response: %#v\n", string(buf))
	res.Header().Set("Content-Type", "application/json")
	res.Write(buf)
}
<|endoftext|>


<commit_before>package tarpan

import (
	"errors"
	"log"
	"strconv"
	//"strings"
	"sync"
	"time"

	"github.com/soniah/gosnmp"
)

// Tarpan constants
const (
	ConcurrentProcesses = 5
)

// Exit code
const (
	ExitCodeOK = iota
	ExitCodeConfError
	ExitCodeParseFlagError
)

// SNMP constants
const (
	COMMUNITY = "public"
	PORT      = 161
	TIMEOUT   = 2
	RETRY     = 3
	//MaximumPDUBytes = 512
)

// JSON structure definition
type DataSet struct {
	Default Defaults `json:"defaults"`
	Targets []Target `json:"targets"`
}

type Defaults struct {
	Port      uint16 `json:"port"`
	Version   string `json:"version"`
	Community string `json:"community"`
}

type Target struct {
	Name      string `json:"name"`
	Address   string `json:"address"`
	Port      uint16 `json:"port"`
	Version   string `json:"version"`
	Community string `json:"community"`
	OIDs      []OID  `json:"oids"`
}

type OID struct {
	OID         string `json:"oid"`
	Description string `json:"description"`
}

// Tarpan result structures
type TarpanResult struct {
	Name      string    `json:"name"`
	Address   string    `json:"address"`
	Port      uint16    `json:"port"`
	Version   string    `json:"version"`
	Community string    `json:"community"`
	VarBinds  []VarBind `json:"varbinds"`
}

type VarBind struct {
	Description string         `json:"description"`
	OID         string         `json:"oid"`
	Type        gosnmp.Asn1BER `json:"type"`
	Value       interface{}    `json:"value"`
	Time        int64          `json:"time"`
}

type Tarpan interface {
	makeRequestBody()
	SetManager()
	SetParams()
	Get(params map[string]string, oids []string) ([]gosnmp.SnmpPDU, error)
	Run()
}

type TarpanManager struct {
	snmp *gosnmp.GoSNMP
}

type RequestParams struct {
	target    string
	community string
	version   string
	port      string
	timeout   uint8
	retry     uint8
}

type SnmpResult struct {
	index   int
	target  Target
	results []gosnmp.SnmpPDU
	time    int64
}

type Channels struct {
	semaphoe chan int
	results  chan SnmpResult
}

func (t *TarpanManager) Get(params *RequestParams,
	oids []string) ([]gosnmp.SnmpPDU, error) {
	t.SetParams(params)
	connection_err := t.snmp.Connect()
	defer t.snmp.Conn.Close()
	if connection_err != nil {
		log.Fatalf("Connection error: %")
		return nil, errors.New(connection_err.Error())
	}
	snmpPacket, request_err := t.snmp.Get(oids)
	if request_err != nil {
		return nil, errors.New(request_err.Error())
	}
	/*
		now = time.Now()
		&SnmpResult{
			index:   "",
			target:  "",
			results: snmpPacket.Variables,
			time:    now.Unix(),
		}
	*/

	return snmpPacket.Variables, nil
}

func (t *TarpanManager) SetManager(manager *gosnmp.GoSNMP) {
	t.snmp = manager
}

func (t *TarpanManager) SetParams(p *RequestParams) {
	if p.target != "" {
		t.snmp.Target = p.target
	}
	if p.port != "" {
		port, _ := strconv.ParseUint(p.port, 10, 16)
		t.snmp.Port = uint16(port)
	} else {
		t.snmp.Port = uint16(PORT)
	}
	if p.community != "" {
		t.snmp.Community = p.community
	} else {
		t.snmp.Community = COMMUNITY
	}
	if p.version != "" {
		if p.version == "2c" {
			t.snmp.Version = gosnmp.Version2c
		}
	} else {
		t.snmp.Version = gosnmp.Version2c
	}
	if p.timeout != 0 {
		t.snmp.Timeout = time.Duration(p.timeout) * time.Second
	} else {
		t.snmp.Timeout = time.Duration(TIMEOUT) * time.Second
	}
	if p.retry != 0 {
		t.snmp.Retries = int(p.retry)
	} else {
		t.snmp.Retries = RETRY
	}
}

func getRequestParams(ds *DataSet, idx int) (*RequestParams, error) {
	var address string
	err := validateIP(ds.Targets[idx].Address)
	if err == nil {
		address = ds.Targets[idx].Address
	} else {
		return &RequestParams{}, err
	}
	community := ds.Targets[idx].Community
	if community == "" {
		community = ds.Default.Community
	}
	version := ds.Targets[idx].Version
	if version == "" {
		version = ds.Default.Version
	}
	port := ds.Targets[idx].Port
	if port == 0 {
		port = ds.Default.Port
	}

	requestParams := &RequestParams{
		target:    address,
		community: community,
		version:   version,
		port:      strconv.Itoa(int(port)),
	}

	return requestParams, nil
}

func makeChannel(result_buffer_size int) *Channels {
	channel := &Channels{
		semaphoe: make(chan int, ConcurrentProcesses),
		results:  make(chan SnmpResult, result_buffer_size),
	}

	return channel
}

func getTargetOIDDescription(oid string, oids []OID) (string, error) {
	for _, o := range oids {
		if o.OID == oid {
			return o.Description, nil
		}
	}

	return "", errors.New("oid not found")
}

func makeTarpanResult(result SnmpResult) *TarpanResult {
	var varbinds []VarBind
	for _, val := range result.results {
		//n := strings.TrimLeft(val.Name, ".")
		desc, _ := getTargetOIDDescription(val.Name, result.target.OIDs)
		v := VarBind{
			Description: desc,
			OID:         val.Name,
			Type:        val.Type,
			Value:       val.Value,
			Time:        result.time,
		}
		varbinds = append(varbinds, v)
	}
	tarpanResult := &TarpanResult{
		Name:      result.target.Name,
		Address:   result.target.Address,
		Port:      result.target.Port,
		Version:   result.target.Version,
		Community: result.target.Community,
		VarBinds:  varbinds,
	}

	return tarpanResult
}

func removeNilResult(tarpanResults []*TarpanResult) []*TarpanResult {
	var responseResult []*TarpanResult
	for _, v := range tarpanResults {
		if v != nil {
			responseResult = append(responseResult, v)
		}
	}

	return responseResult
}

func makeTarpanResults(c *Channels) []*TarpanResult {
	result_length := len(c.results)
	tarpanResults := make([]*TarpanResult, result_length)
	var result SnmpResult
	for i := 0; i < result_length; i++ {
		result = <-c.results
		tarpanResults[result.index] = makeTarpanResult(result)
	}

	return removeNilResult(tarpanResults)
}

func makeManagers(numberOfTargets int) []*TarpanManager {
	var managers []*TarpanManager
	for i := 0; i < numberOfTargets; i++ {
		managers = append(managers, &TarpanManager{
			snmp: &gosnmp.GoSNMP{},
		})
	}

	return managers
}

func Collect(dataset *DataSet) []*TarpanResult {
	var waitGroup sync.WaitGroup
	var oids []string

	managers := makeManagers(len(dataset.Targets))
	channels := makeChannel(len(dataset.Targets))
	for t_idx := range dataset.Targets {

		// TODO: split oid slice depending on PDU size
		oids = []string{}
		for o_idx := range dataset.Targets[t_idx].OIDs {
			oids = append(oids, dataset.Targets[t_idx].OIDs[o_idx].OID)
		}

		waitGroup.Add(1)
		go func(m *TarpanManager, ds *DataSet, idx int, o []string, c *Channels) {
			defer func() {
				waitGroup.Done()
				<-c.semaphoe
			}()
			c.semaphoe <- 0
			param, param_err := getRequestParams(ds, idx)
			if param_err != nil {
				log.Print(param_err)
				return
			}
			results, err := m.Get(param, o)
			t := time.Now()
			if err != nil {
				log.Print(err)
				return
			}
			c.results <- SnmpResult{
				index:   idx,
				target:  ds.Targets[idx],
				results: results,
				time:    t.Unix(),
			}
		}(managers[t_idx], dataset, t_idx, oids, channels)
	}
	waitGroup.Wait()
	tarpanResults := makeTarpanResults(channels)

	return tarpanResults
}

func Run(target string, output string, debug bool) (int, error) {
	var err error

	// Load targets
	dataset, loadErr := loadConfig(target)
	if loadErr != nil {
		return ExitCodeConfError, loadErr
	}

	// collect data
	tr := Collect(dataset)

	// output
	Write(output, tr)

	return ExitCodeOK, err
}
<commit_msg>mainly modifing Get method to return TarpanResult<commit_after>package tarpan

import (
	"errors"
	"log"
	"strconv"
	//"strings"
	"sync"
	"time"

	"github.com/soniah/gosnmp"
)

// Tarpan constants
const (
	ConcurrentProcesses = 5
)

// Exit code
const (
	ExitCodeOK = iota
	ExitCodeConfError
	ExitCodeParseFlagError
)

// SNMP constants
const (
	COMMUNITY = "public"
	PORT      = 161
	TIMEOUT   = 2
	RETRY     = 3
	//MaximumPDUBytes = 512
)

// JSON structure definition
type DataSet struct {
	Default Defaults `json:"defaults"`
	Targets []Target `json:"targets"`
}

type Defaults struct {
	Port      uint16 `json:"port"`
	Version   string `json:"version"`
	Community string `json:"community"`
}

type Target struct {
	Name      string `json:"name"`
	Address   string `json:"address"`
	Port      uint16 `json:"port"`
	Version   string `json:"version"`
	Community string `json:"community"`
	OIDs      []OID  `json:"oids"`
}

type OID struct {
	OID         string `json:"oid"`
	Description string `json:"description"`
}

// Tarpan result structures
type TarpanResult struct {
	Name      string    `json:"name"`
	Address   string    `json:"address"`
	Port      uint16    `json:"port"`
	Version   string    `json:"version"`
	Community string    `json:"community"`
	VarBinds  []VarBind `json:"varbinds"`
}

type TarpanResults []*TarpanResult

type VarBind struct {
	Description string         `json:"description"`
	OID         string         `json:"oid"`
	Type        gosnmp.Asn1BER `json:"type"`
	Value       interface{}    `json:"value"`
	Time        int64          `json:"time"`
}

type Tarpan interface {
	makeRequestBody()
	SetManager()
	SetParams()
	Get(params map[string]string, oids []string) ([]gosnmp.SnmpPDU, error)
	Run()
}

type TarpanManager struct {
	target *Target
	snmp   *gosnmp.GoSNMP
}

type TarpanManagers []*TarpanManager

type RequestParams struct {
	managerIndex int
	address      string
	community    string
	version      string
	port         string
	timeout      uint8
	retry        uint8
}

type Channels struct {
	semaphoe chan int
	results  chan TarpanResult
}

func (m *TarpanManager) setTarget(ds *DataSet, target_index int) error {
	params, param_err := getRequestParams(ds, target_index)
	if param_err != nil {
		log.Print(param_err)
		errors.New(param_err.Error())
	}
	m.SetParams(params)
	m.target = &ds.Targets[target_index]

	return nil
}

func (m *TarpanManager) Get(oids []string) (TarpanResult, error) {
	connection_err := m.snmp.Connect()
	defer m.snmp.Conn.Close()
	if connection_err != nil {
		log.Fatalf("Connection error: %")
		return TarpanResult{}, errors.New(connection_err.Error())
	}
	snmpPacket, request_err := m.snmp.Get(oids)
	if request_err != nil {
		return TarpanResult{}, errors.New(request_err.Error())
	}
	tarpanResult := m.makeTarpanResult(snmpPacket)

	return tarpanResult, nil
}

func (t *TarpanManager) SetManager(manager *gosnmp.GoSNMP) {
	t.snmp = manager
}

func (t *TarpanManager) SetParams(p *RequestParams) {
	if p.address != "" {
		t.snmp.Target = p.address
	}
	if p.port != "" {
		port, _ := strconv.ParseUint(p.port, 10, 16)
		t.snmp.Port = uint16(port)
	} else {
		t.snmp.Port = uint16(PORT)
	}
	if p.community != "" {
		t.snmp.Community = p.community
	} else {
		t.snmp.Community = COMMUNITY
	}
	if p.version != "" {
		if p.version == "2c" {
			t.snmp.Version = gosnmp.Version2c
		}
	} else {
		t.snmp.Version = gosnmp.Version2c
	}
	if p.timeout != 0 {
		t.snmp.Timeout = time.Duration(p.timeout) * time.Second
	} else {
		t.snmp.Timeout = time.Duration(TIMEOUT) * time.Second
	}
	if p.retry != 0 {
		t.snmp.Retries = int(p.retry)
	} else {
		t.snmp.Retries = RETRY
	}
}

func getRequestParams(ds *DataSet, idx int) (*RequestParams, error) {
	var address string
	err := validateIP(ds.Targets[idx].Address)
	if err == nil {
		address = ds.Targets[idx].Address
	} else {
		return &RequestParams{}, err
	}
	community := ds.Targets[idx].Community
	if community == "" {
		community = ds.Default.Community
	}
	version := ds.Targets[idx].Version
	if version == "" {
		version = ds.Default.Version
	}
	port := ds.Targets[idx].Port
	if port == 0 {
		port = ds.Default.Port
	}

	requestParams := &RequestParams{
		managerIndex: idx,
		address:      address,
		community:    community,
		version:      version,
		port:         strconv.Itoa(int(port)),
	}

	return requestParams, nil
}

func makeChannel(result_buffer_size int) *Channels {
	channel := &Channels{
		semaphoe: make(chan int, ConcurrentProcesses),
		results:  make(chan TarpanResult, result_buffer_size),
	}

	return channel
}

//func getTargetOIDDescription(oid string, oids []OID) (string, error) {
func (m *TarpanManager) getTargetOIDDescription(oid string) (string, error) {
	for _, o := range m.target.OIDs {
		if o.OID == oid {
			return o.Description, nil
		}
	}

	return "", errors.New("oid not found")
}

func removeNilResult(tarpanResults []*TarpanResult) []*TarpanResult {
	var responseResult []*TarpanResult
	for _, v := range tarpanResults {
		if v != nil {
			responseResult = append(responseResult, v)
		}
	}

	return responseResult
}

func (m *TarpanManager) makeTarpanResult(sp *gosnmp.SnmpPacket) TarpanResult {
	now := time.Now()
	var varbinds []VarBind
	for _, val := range sp.Variables {
		//n := strings.TrimLeft(val.Name, ".")
		desc, _ := m.getTargetOIDDescription(val.Name)
		//asn1ber_name, _ := getAsn1BERName(val.Type)
		v := VarBind{
			Description: desc,
			OID:         val.Name,
			Type:        val.Type,
			Value:       val.Value,
			Time:        now.Unix(),
		}
		varbinds = append(varbinds, v)
	}
	var version string
	switch m.snmp.Version {
	case gosnmp.Version1:
		version = "1"
	case gosnmp.Version2c:
		version = "2c"
	case gosnmp.Version3:
		version = "3"
	default:
		version = ""
	}
	tarpanResult := TarpanResult{
		Name:      m.target.Name,
		Address:   m.target.Address,
		Port:      m.snmp.Port,
		Version:   version,
		Community: m.snmp.Community,
		VarBinds:  varbinds,
	}

	return tarpanResult
}

func makeTarpanResults(c *Channels) []*TarpanResult {
	result_length := len(c.results)
	tarpanResults := make(TarpanResults, result_length)
	for i := 0; i < result_length; i++ {
		tarpanResult := <-c.results
		tarpanResults[i] = &tarpanResult
	}

	return removeNilResult(tarpanResults)
}

func makeManagers(ds *DataSet) []*TarpanManager {
	var manager *TarpanManager
	var managers TarpanManagers
	var oids []string

	// Set Target
	for t_idx := range ds.Targets {
		// TODO: split oid slice depending on PDU size
		oids = []string{}
		for o_idx := range ds.Targets[t_idx].OIDs {
			oids = append(oids, ds.Targets[t_idx].OIDs[o_idx].OID)
		}
		manager = &TarpanManager{
			snmp: &gosnmp.GoSNMP{},
		}
		err := manager.setTarget(ds, t_idx)
		if err != nil {
			log.Println("set target error")
		}
		managers = append(managers, manager)
	}

	return managers
}

func Collect(dataset *DataSet) []*TarpanResult {
	var waitGroup sync.WaitGroup
	var oids []string
	var tarpanResults []*TarpanResult

	managers := makeManagers(dataset)
	channels := makeChannel(len(dataset.Targets))
	for t_idx := range dataset.Targets {

		// TODO: split oid slice depending on PDU size
		oids = []string{}
		for o_idx := range dataset.Targets[t_idx].OIDs {
			oids = append(oids, dataset.Targets[t_idx].OIDs[o_idx].OID)
		}

		waitGroup.Add(1)
		go func(m *TarpanManager, o []string, c *Channels) {
			c.semaphoe <- 0
			defer func() {
				waitGroup.Done()
				<-c.semaphoe
			}()
			result, err := m.Get(o)
			if err != nil {
				log.Print(err)
				return
			}
			c.results <- result
		}(managers[t_idx], oids, channels)
	}
	waitGroup.Wait()
	tarpanResults = makeTarpanResults(channels)

	return tarpanResults
}

func Run(target string, output string, debug bool) (int, error) {
	var err error

	// Load targets
	dataset, loadErr := loadConfig(target)
	if loadErr != nil {
		return ExitCodeConfError, loadErr
	}

	// collect data
	tr := Collect(dataset)

	// output
	Write(output, tr)

	return ExitCodeOK, err
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	httptransport "github.com/go-openapi/runtime/client"
	strfmt "github.com/go-openapi/strfmt"
	client_models "github.com/iron-io/titan/runner/client/models"
	titan_client "github.com/iron-io/titan/runner/client/titan"
	"github.com/iron-io/titan/runner/client/titan/jobs"
)

type Tasker struct {
	api   *titan_client.Titan
	dummy os.File
	log   log.FieldLogger
}

// Titan tasker.
func NewTasker(config *Config, log log.FieldLogger) *Tasker {
	api := titan_client.New(httptransport.New("localhost:8080", "/v1", []string{"http"}), strfmt.Default)
	f, _ := ioutil.TempFile("", "crap-")
	return &Tasker{api, *f, log}
}

func (t *Tasker) Job() *client_models.Job {
	l := t.log.WithField("action", "DequeueJob")
	var job *client_models.Job
	param := jobs.NewGetJobsParams()
	for {
		jobs, err := t.api.Jobs.GetJobs(param)
		if err != nil {
			l.WithError(err).Errorln("dequeue job from api")
		} else if len(jobs.Payload.Jobs) > 0 && jobs.Payload.Jobs[0] != nil {
			job = jobs.Payload.Jobs[0]
			break
		}
		time.Sleep(1 * time.Second)
	}
	return job
}

func (t *Tasker) Start(job *client_models.Job) error {
	l := t.log.WithFields(log.Fields{
		"action": "Start",
		"job_id": job.ID,
	})
	params := jobs.NewPostGroupsGroupNameJobsIDStartParams().WithGroupName(job.GroupName).WithID(job.ID)
	body := &client_models.Start{
		StartedAt: strfmt.DateTime(time.Now()),
	}
	params.WithBody(body)

	j, err := t.api.Jobs.PostGroupsGroupNameJobsIDStart(params)
	if err != nil {
		l.WithError(err).Errorln("failed")
		return err
	}
	l.Infoln("Got back", j)
	return nil
}

// TODO: this should be on server side
func (t *Tasker) RetryTask(job *client_models.Job) error {
	panic("Not implemented Retry")
}

func (t *Tasker) IsCancelled(job *client_models.Job) bool {
	wrapper, err := t.api.Jobs.GetGroupsGroupNameJobsID(jobs.NewGetGroupsGroupNameJobsIDParams().WithGroupName(job.GroupName).WithID(job.ID))
	if err != nil {
		log.Errorln("JobIdGet from Cancel", "err", err)
		return false
	}

	return wrapper.Payload.Job.Status == "cancelled"
}

func (t *Tasker) Succeeded(job *client_models.Job, r *os.File) error {
	param := jobs.NewPostGroupsGroupNameJobsIDSuccessParams().WithGroupName(job.GroupName).WithID(job.ID)
	_, err := t.api.Jobs.PostGroupsGroupNameJobsIDSuccess(param)
	if err != nil {
		log.Errorln("JobIdSuccessPost", "jobId", job.ID, "err", err)
	}
	return t.Log(job, r)
}

func (t *Tasker) Failed(job *client_models.Job, reason string, r *os.File) error {
	param := jobs.NewPostGroupsGroupNameJobsIDErrorParams().WithGroupName(job.GroupName).WithID(job.ID).WithReason(reason)

	_, err := t.api.Jobs.PostGroupsGroupNameJobsIDError(param)
	if err != nil {
		log.Errorln("JobIdFailPost", "jobId", job.ID, "err", err)
	}
	return t.Log(job, r)
}

func (t *Tasker) Log(job *client_models.Job, logFile *os.File) error {
	param := jobs.NewPostGroupsGroupNameJobsIDLogParams().WithGroupName(job.GroupName).WithID(job.ID)
	if logFile != nil {
		param.WithLog(*logFile)
	} else {
		param.WithLog(t.dummy)
	}
	r, err := t.api.Jobs.PostGroupsGroupNameJobsIDLog(param)
	log.Println(r, err)
	if err != nil {
		log.WithError(err).Errorln("Uploading log")
	}

	return err
}
<commit_msg>Use UTC time for job time fields. Fixes #122.<commit_after>package main

import (
	"io/ioutil"
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	httptransport "github.com/go-openapi/runtime/client"
	strfmt "github.com/go-openapi/strfmt"
	client_models "github.com/iron-io/titan/runner/client/models"
	titan_client "github.com/iron-io/titan/runner/client/titan"
	"github.com/iron-io/titan/runner/client/titan/jobs"
)

type Tasker struct {
	api   *titan_client.Titan
	dummy os.File
	log   log.FieldLogger
}

// Titan tasker.
func NewTasker(config *Config, log log.FieldLogger) *Tasker {
	api := titan_client.New(httptransport.New("localhost:8080", "/v1", []string{"http"}), strfmt.Default)
	f, _ := ioutil.TempFile("", "crap-")
	return &Tasker{api, *f, log}
}

func (t *Tasker) Job() *client_models.Job {
	l := t.log.WithField("action", "DequeueJob")
	var job *client_models.Job
	param := jobs.NewGetJobsParams()
	for {
		jobs, err := t.api.Jobs.GetJobs(param)
		if err != nil {
			l.WithError(err).Errorln("dequeue job from api")
		} else if len(jobs.Payload.Jobs) > 0 && jobs.Payload.Jobs[0] != nil {
			job = jobs.Payload.Jobs[0]
			break
		}
		time.Sleep(1 * time.Second)
	}
	return job
}

func (t *Tasker) Start(job *client_models.Job) error {
	l := t.log.WithFields(log.Fields{
		"action": "Start",
		"job_id": job.ID,
	})
	params := jobs.NewPostGroupsGroupNameJobsIDStartParams().WithGroupName(job.GroupName).WithID(job.ID)
	body := &client_models.Start{
		StartedAt: strfmt.DateTime(time.Now().UTC()),
	}
	params.WithBody(body)

	j, err := t.api.Jobs.PostGroupsGroupNameJobsIDStart(params)
	if err != nil {
		l.WithError(err).Errorln("failed")
		return err
	}
	l.Infoln("Got back", j)
	return nil
}

// TODO: this should be on server side
func (t *Tasker) RetryTask(job *client_models.Job) error {
	panic("Not implemented Retry")
}

func (t *Tasker) IsCancelled(job *client_models.Job) bool {
	wrapper, err := t.api.Jobs.GetGroupsGroupNameJobsID(jobs.NewGetGroupsGroupNameJobsIDParams().WithGroupName(job.GroupName).WithID(job.ID))
	if err != nil {
		log.Errorln("JobIdGet from Cancel", "err", err)
		return false
	}

	return wrapper.Payload.Job.Status == "cancelled"
}

func (t *Tasker) Succeeded(job *client_models.Job, r *os.File) error {
	param := jobs.NewPostGroupsGroupNameJobsIDSuccessParams().WithGroupName(job.GroupName).WithID(job.ID)
	_, err := t.api.Jobs.PostGroupsGroupNameJobsIDSuccess(param)
	if err != nil {
		log.Errorln("JobIdSuccessPost", "jobId", job.ID, "err", err)
	}
	return t.Log(job, r)
}

func (t *Tasker) Failed(job *client_models.Job, reason string, r *os.File) error {
	param := jobs.NewPostGroupsGroupNameJobsIDErrorParams().WithGroupName(job.GroupName).WithID(job.ID).WithReason(reason)

	_, err := t.api.Jobs.PostGroupsGroupNameJobsIDError(param)
	if err != nil {
		log.Errorln("JobIdFailPost", "jobId", job.ID, "err", err)
	}
	return t.Log(job, r)
}

func (t *Tasker) Log(job *client_models.Job, logFile *os.File) error {
	param := jobs.NewPostGroupsGroupNameJobsIDLogParams().WithGroupName(job.GroupName).WithID(job.ID)
	if logFile != nil {
		param.WithLog(*logFile)
	} else {
		param.WithLog(t.dummy)
	}
	r, err := t.api.Jobs.PostGroupsGroupNameJobsIDLog(param)
	log.Println(r, err)
	if err != nil {
		log.WithError(err).Errorln("Uploading log")
	}

	return err
}
<|endoftext|>


<commit_before>package client

import (
	"fmt"
	"net/url"
)

// Search makes a GET request to return the Ponzu Search API response for the
// enpoint: `/api/search?type=<Type>&q=<Query String>`
func (c *Client) Search(contentType string, q string, opts *QueryOptions) (*APIResponse, error) {
	opts = setDefaultOpts(opts)

	endpoint := fmt.Sprintf(
		"%s/api/search?type=%s&q=%s&count=%d&offest=%d",
		c.Conf.Host, contentType, url.QueryEscape(q), opts.Count, opts.Offset,
	)

	if c.CacheEnabled() {
		ok, resp := c.Cache.Check(endpoint)
		if ok {
			return resp, nil
		}
	}

	w, err := c.Get(endpoint)
	if err != nil {
		return nil, err
	}

	resp := &APIResponse{
		Response: w,
	}

	err = resp.process()
	if err != nil {
		return resp, err
	}

	if c.CacheEnabled() {
		err := c.Cache.Add(endpoint, resp)
		return resp, err
	}

	return resp, err
}
<commit_msg>Fixed typo in endpoint variable (#12)<commit_after>package client

import (
	"fmt"
	"net/url"
)

// Search makes a GET request to return the Ponzu Search API response for the
// enpoint: `/api/search?type=<Type>&q=<Query String>`
func (c *Client) Search(contentType string, q string, opts *QueryOptions) (*APIResponse, error) {
	opts = setDefaultOpts(opts)

	endpoint := fmt.Sprintf(
		"%s/api/search?type=%s&q=%s&count=%d&offset=%d",
		c.Conf.Host, contentType, url.QueryEscape(q), opts.Count, opts.Offset,
	)

	if c.CacheEnabled() {
		ok, resp := c.Cache.Check(endpoint)
		if ok {
			return resp, nil
		}
	}

	w, err := c.Get(endpoint)
	if err != nil {
		return nil, err
	}

	resp := &APIResponse{
		Response: w,
	}

	err = resp.process()
	if err != nil {
		return resp, err
	}

	if c.CacheEnabled() {
		err := c.Cache.Add(endpoint, resp)
		return resp, err
	}

	return resp, err
}
<|endoftext|>


<commit_before>package acceptance_test

import (
	"net"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
)

var _ = Describe("plain HTTP requests", func() {
	const (
		CONNECTION_TIMEOUT = 11 * time.Second
	)

	Describe("to the API", func() {
		It("has the connection refused", func() {
			uri := config.ApiEndpoint + ":80"
			_, err := net.DialTimeout("tcp", uri, CONNECTION_TIMEOUT)
			Expect(err).To(HaveOccurred(), "should not connect")
			Expect(err.(net.Error).Timeout()).To(BeTrue(), "should timeout")
		})
	})

	Describe("to an app", func() {
		It("has the connection refused", func() {
			appName := generator.PrefixedRandomName("CATS-APP-")
			Expect(cf.Cf(
				"push", appName,
				"-b", config.StaticFileBuildpackName,
				"-p", "../../example-apps/static-app",
				"-d", config.AppsDomain,
			).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

			uri := appName + "." + config.AppsDomain + ":80"
			_, err := net.DialTimeout("tcp", uri, CONNECTION_TIMEOUT)
			Expect(err).To(HaveOccurred(), "should not connect")
			Expect(err.(net.Error).Timeout()).To(BeTrue(), "should timeout")
		})
	})
})
<commit_msg>Add acceptance tests for http redirect behaviour.<commit_after>package acceptance_test

import (
	"fmt"
	"net"
	"net/http"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("plain HTTP requests", func() {
	const (
		CONNECTION_TIMEOUT = 11 * time.Second
	)

	Describe("to the API", func() {
		It("has the connection refused", func() {
			uri := config.ApiEndpoint + ":80"
			_, err := net.DialTimeout("tcp", uri, CONNECTION_TIMEOUT)
			Expect(err).To(HaveOccurred(), "should not connect")
			Expect(err.(net.Error).Timeout()).To(BeTrue(), "should timeout")
		})
	})

	Describe("to an app", func() {
		It("is redirected to the https endpoint", func() {
			req, err := http.NewRequest("GET", fmt.Sprintf("http://foo.%s/", config.AppsDomain), nil)
			Expect(err).NotTo(HaveOccurred())
			resp, err := http.DefaultTransport.RoundTrip(req)
			Expect(err).NotTo(HaveOccurred())

			Expect(resp.StatusCode).To(Equal(301))
			Expect(resp.Header.Get("Location")).To(Equal(fmt.Sprintf("https://foo.%s/", config.AppsDomain)))
		})

		It("has any path and query components removed when redirecting", func() {
			req, err := http.NewRequest("GET", fmt.Sprintf("http://foo.%s/bar?baz=qux", config.AppsDomain), nil)
			Expect(err).NotTo(HaveOccurred())
			resp, err := http.DefaultTransport.RoundTrip(req)
			Expect(err).NotTo(HaveOccurred())

			Expect(resp.StatusCode).To(Equal(301))
			Expect(resp.Header.Get("Location")).To(Equal(fmt.Sprintf("https://foo.%s/", config.AppsDomain)))
		})
	})
})
<|endoftext|>


<commit_before>package dbsearch

import (
	"fmt"
	"github.com/iostrovok/go-iutils/iutils"
	"regexp"
	"strconv"
	"strings"
)

// construct a regexp to extract values:
var (
	unquotedRe  = regexp.MustCompile(`([^",\\{}\s]|NULL)+,`)
	_arrayValue = fmt.Sprintf("\"(%s)+\",", `[^"\\]|\\"|\\\\`)
	quotedRe    = regexp.MustCompile(_arrayValue)

	intArrayBrace = regexp.MustCompile(`[^-0-9\.\,]+`)
	intArraySplit = regexp.MustCompile(`,`)
	intArrayTail  = regexp.MustCompile(`\.[0-9]*`)

	noNumberDots      = regexp.MustCompile(`[^-0-9\.,]+`)
	noNumberDotsSplit = regexp.MustCompile(`(,|\s+)+`)

	parseBoolArrayRe     = regexp.MustCompile(`[^FTft]+`)
	parseBoolArrayReTail = regexp.MustCompile(`^[^FTft]+|[^FTft]+$`)
)

func parseBoolArrayForBool(s interface{}) []bool {
	line := parseBoolArrayReTail.ReplaceAllString(_AnyToString(s), "")
	r := parseBoolArrayRe.Split(line, -1)
	out := make([]bool, len(r))
	for i, v := range r {
		if v == "t" || v == "T" {
			out[i] = true
		} else {
			out[i] = false
		}
	}
	return out
}

func parseBoolArrayForString(s interface{}) []bool {
	r := parseArray(_AnyToString(s))
	out := make([]bool, len(r))
	for i, v := range r {
		v := strings.TrimSpace(v)
		if v == "" {
			out[i] = false
		} else {
			out[i] = true
		}
	}
	return out
}

func parseBoolArrayForReal(s interface{}) []bool {
	r := parseFloat64Array(s)
	out := make([]bool, len(r))
	for i, v := range r {
		if v == 0.0 {
			out[i] = false
		} else {
			out[i] = true
		}
	}
	return out
}

func parseBoolArrayForNumber(s interface{}) []bool {
	r := parseInt64Array(s)
	out := make([]bool, len(r))
	for i, v := range r {
		if v == 0 {
			out[i] = false
		} else {
			out[i] = true
		}
	}
	return out
}

func parseUint64Array(s interface{}) []uint64 {
	r := parseInt64Array(s)
	out := make([]uint64, len(r))
	for i, v := range r {
		if v < 0 {
			out[i] = 0
		} else {
			out[i] = uint64(v)
		}
	}
	return out
}

func parseUint8Array(s interface{}) []uint8 {
	r := parseIntArray(s)
	out := make([]uint8, len(r))
	for i, v := range r {
		if v < 0 {
			out[i] = 0
		} else {
			out[i] = uint8(v)
		}
	}
	return out
}

func parseUintArray(s interface{}) []uint {
	r := parseIntArray(s)
	out := make([]uint, len(r))
	for i, v := range r {
		if v < 0 {
			out[i] = 0
		} else {
			out[i] = uint(v)
		}
	}
	return out
}

func parseInt64Array(s interface{}) []int64 {
	str := strings.TrimSpace(_AnyToString(s))
	str = intArrayBrace.ReplaceAllString(str, "")
	str = intArrayTail.ReplaceAllString(str, "")
	k := intArraySplit.Split(str, -1)

	out := make([]int64, len(k))

	for i, v := range k {
		v := intArrayTail.ReplaceAllString(v, "")

		if v == "" {
			out[i] = 0
			continue
		}

		j, err := strconv.Atoi(v)
		if err != nil {
			//log.Println(err)
			out[i] = 0
			continue
		}
		out[i] = int64(j)
	}
	return out
}

func parseIntArray(s interface{}) []int {
	k := parseInt64Array(s)
	out := make([]int, len(k))
	for i, v := range k {
		out[i] = int(v)
	}
	return out
}

func parseFloat64Array(s interface{}) []float64 {
	out := []float64{}

	str := strings.TrimSpace(iutils.AnyToString(s))
	str = noNumberDots.ReplaceAllString(str, "")
	list := noNumberDotsSplit.Split(str, -1)

	for _, v := range list {
		out = append(out, iutils.AnyToFloat64(v))
	}

	return out
}

func parseFloat32Array(s interface{}) []float32 {
	out := []float32{}

	str := strings.TrimSpace(iutils.AnyToString(s))
	str = noNumberDots.ReplaceAllString(str, "")
	list := noNumberDotsSplit.Split(str, -1)

	for _, v := range list {
		out = append(out, float32(iutils.AnyToFloat64(v)))
	}

	return out
}
func parseArray(line string) []string {

	out := []string{}
	if line == "{}" {
		return out
	}

	if len(line)-1 != strings.LastIndex(line, "}") || strings.Index(line, "{") != 0 {
		return out
	}

	/* Removes lead & last {} and adds "," to end of string */
	line = strings.TrimPrefix(line, "{")
	line = strings.TrimSuffix(line, "}") + ","

	for len(line) > 0 {
		s := ""
		if strings.Index(line, `""`) == 0 {
			/* Empty line */
			s = ""
			line = line[3:]
		} else if strings.Index(line, `"`) != 0 {
			s = unquotedRe.FindString(line)
			line = line[strings.Index(line, ",")+1:]
			s = strings.TrimSuffix(s, ",")

			/* counvert NULL to empty string6 however we need nil string */
			if s == "NULL" {
				s = ""
			}
		} else {
			s = quotedRe.FindString(line)
			line = strings.TrimPrefix(line, s)
			s = strings.TrimPrefix(s, "\"")
			s = strings.TrimSuffix(s, "\",")
			s = strings.Join(strings.Split(s, "\\\\"), "\\")
			s = strings.Join(strings.Split(s, "\\\""), "\"")
		}
		out = append(out, s)
	}

	return out
}
<commit_msg>Removes outside dependencies.<commit_after>package dbsearch

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

// construct a regexp to extract values:
var (
	unquotedRe  = regexp.MustCompile(`([^",\\{}\s]|NULL)+,`)
	_arrayValue = fmt.Sprintf("\"(%s)+\",", `[^"\\]|\\"|\\\\`)
	quotedRe    = regexp.MustCompile(_arrayValue)

	intArrayBrace = regexp.MustCompile(`[^-0-9\.\,]+`)
	intArraySplit = regexp.MustCompile(`,`)
	intArrayTail  = regexp.MustCompile(`\.[0-9]*`)

	noNumberDots      = regexp.MustCompile(`[^-0-9\.,]+`)
	noNumberDotsSplit = regexp.MustCompile(`(,|\s+)+`)

	parseBoolArrayRe     = regexp.MustCompile(`[^FTft]+`)
	parseBoolArrayReTail = regexp.MustCompile(`^[^FTft]+|[^FTft]+$`)
)

func parseBoolArrayForBool(s interface{}) []bool {
	line := parseBoolArrayReTail.ReplaceAllString(_AnyToString(s), "")
	r := parseBoolArrayRe.Split(line, -1)
	out := make([]bool, len(r))
	for i, v := range r {
		if v == "t" || v == "T" {
			out[i] = true
		} else {
			out[i] = false
		}
	}
	return out
}

func parseBoolArrayForString(s interface{}) []bool {
	r := parseArray(_AnyToString(s))
	out := make([]bool, len(r))
	for i, v := range r {
		v := strings.TrimSpace(v)
		if v == "" {
			out[i] = false
		} else {
			out[i] = true
		}
	}
	return out
}

func parseBoolArrayForReal(s interface{}) []bool {
	r := parseFloat64Array(s)
	out := make([]bool, len(r))
	for i, v := range r {
		if v == 0.0 {
			out[i] = false
		} else {
			out[i] = true
		}
	}
	return out
}

func parseBoolArrayForNumber(s interface{}) []bool {
	r := parseInt64Array(s)
	out := make([]bool, len(r))
	for i, v := range r {
		if v == 0 {
			out[i] = false
		} else {
			out[i] = true
		}
	}
	return out
}

func parseUint64Array(s interface{}) []uint64 {
	r := parseInt64Array(s)
	out := make([]uint64, len(r))
	for i, v := range r {
		if v < 0 {
			out[i] = 0
		} else {
			out[i] = uint64(v)
		}
	}
	return out
}

func parseUint8Array(s interface{}) []uint8 {
	r := parseIntArray(s)
	out := make([]uint8, len(r))
	for i, v := range r {
		if v < 0 {
			out[i] = 0
		} else {
			out[i] = uint8(v)
		}
	}
	return out
}

func parseUintArray(s interface{}) []uint {
	r := parseIntArray(s)
	out := make([]uint, len(r))
	for i, v := range r {
		if v < 0 {
			out[i] = 0
		} else {
			out[i] = uint(v)
		}
	}
	return out
}

func parseInt64Array(s interface{}) []int64 {
	str := strings.TrimSpace(_AnyToString(s))
	str = intArrayBrace.ReplaceAllString(str, "")
	str = intArrayTail.ReplaceAllString(str, "")
	k := intArraySplit.Split(str, -1)

	out := make([]int64, len(k))

	for i, v := range k {
		v := intArrayTail.ReplaceAllString(v, "")

		if v == "" {
			out[i] = 0
			continue
		}

		j, err := strconv.Atoi(v)
		if err != nil {
			//log.Println(err)
			out[i] = 0
			continue
		}
		out[i] = int64(j)
	}
	return out
}

func parseIntArray(s interface{}) []int {
	k := parseInt64Array(s)
	out := make([]int, len(k))
	for i, v := range k {
		out[i] = int(v)
	}
	return out
}

func parseFloat64Array(s interface{}) []float64 {
	out := []float64{}

	str := strings.TrimSpace(_AnyToString(s))
	str = noNumberDots.ReplaceAllString(str, "")
	list := noNumberDotsSplit.Split(str, -1)

	for _, v := range list {
		out = append(out, _AnyToFloat64(v))
	}

	return out
}

func parseFloat32Array(s interface{}) []float32 {
	out := []float32{}

	str := strings.TrimSpace(_AnyToString(s))
	str = noNumberDots.ReplaceAllString(str, "")
	list := noNumberDotsSplit.Split(str, -1)

	for _, v := range list {
		out = append(out, float32(_AnyToFloat64(v)))
	}

	return out
}

func parseArray(line string) []string {

	out := []string{}
	if line == "{}" {
		return out
	}

	if len(line)-1 != strings.LastIndex(line, "}") || strings.Index(line, "{") != 0 {
		return out
	}

	/* Removes lead & last {} and adds "," to end of string */
	line = strings.TrimPrefix(line, "{")
	line = strings.TrimSuffix(line, "}") + ","

	for len(line) > 0 {
		s := ""
		if strings.Index(line, `""`) == 0 {
			/* Empty line */
			s = ""
			line = line[3:]
		} else if strings.Index(line, `"`) != 0 {
			s = unquotedRe.FindString(line)
			line = line[strings.Index(line, ",")+1:]
			s = strings.TrimSuffix(s, ",")

			/* counvert NULL to empty string6 however we need nil string */
			if s == "NULL" {
				s = ""
			}
		} else {
			s = quotedRe.FindString(line)
			line = strings.TrimPrefix(line, s)
			s = strings.TrimPrefix(s, "\"")
			s = strings.TrimSuffix(s, "\",")
			s = strings.Join(strings.Split(s, "\\\\"), "\\")
			s = strings.Join(strings.Split(s, "\\\""), "\"")
		}
		out = append(out, s)
	}

	return out
}

func _AnyToFloat64(s interface{}) float64 {
	if s == nil {
		return 0
	}

	switch s.(type) {
	case string:
		st := noNumberDots.ReplaceAllString(s.(string), "")
		if st == "" {
			return 0.0
		}
		i, err := strconv.ParseFloat(st, 64)
		if err != nil {
			return 0.0
		}
		return float64(i)

	case []uint8:
		return _AnyToFloat64(string(s.([]uint8)))
	case int:
		return float64(s.(int))
	case int32:
		return float64(s.(int32))
	case int64:
		return float64(s.(int64))
	case *int32:
		return float64(*s.(*int32))
	case *int64:
		return float64(*s.(*int64))
	case *int:
		return float64(*s.(*int))
	case float32:
		return float64(s.(float32))
	case *float32:
		return float64(*s.(*float32))
	case *float64:
		return *s.(*float64)
	case float64:
		return s.(float64)

	default:
	}
	return 0.0
}
<|endoftext|>