<commit_before>package bbs_test

import (
	"github.com/cloudfoundry/storeadapter"
	"github.com/onsi/ginkgo/config"
	"os"
	"os/signal"
	"testing"

	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var etcdRunner *etcdstorerunner.ETCDClusterRunner
var store storeadapter.StoreAdapter

func TestBBS(t *testing.T) {
	registerSignalHandler()
	RegisterFailHandler(Fail)

	etcdRunner = etcdstorerunner.NewETCDClusterRunner(5001+config.GinkgoConfig.ParallelNode, 1)
	etcdRunner.Start()

	store = etcdRunner.Adapter()

	RunSpecs(t, "BBS Suite")

	etcdRunner.Stop()
}

var _ = BeforeEach(func() {
	etcdRunner.Stop()
	etcdRunner.Start()
})

func registerSignalHandler() {
	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt, os.Kill)

		select {
		case <-c:
			etcdRunner.Stop()
			os.Exit(0)
		}
	}()
}
<commit_msg>verify the fake works<commit_after>package bbs_test

import (
	"github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/fakebbs"
	"github.com/cloudfoundry/storeadapter"
	"github.com/onsi/ginkgo/config"
	"os"
	"os/signal"
	"testing"

	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var etcdRunner *etcdstorerunner.ETCDClusterRunner
var store storeadapter.StoreAdapter

func TestBBS(t *testing.T) {
	registerSignalHandler()
	RegisterFailHandler(Fail)

	etcdRunner = etcdstorerunner.NewETCDClusterRunner(5001+config.GinkgoConfig.ParallelNode, 1)
	etcdRunner.Start()

	store = etcdRunner.Adapter()

	RunSpecs(t, "BBS Suite")

	etcdRunner.Stop()
}

var _ = BeforeEach(func() {
	etcdRunner.Stop()
	etcdRunner.Start()
})

var _ = It("should have a valid fake", func() {
	var fakeExecutorBBS bbs.ExecutorBBS
	fakeExecutorBBS = fakebbs.NewFakeExecutorBBS()
	Ω(fakeExecutorBBS).ShouldNot(BeNil())

	var fakeStagerBBS bbs.StagerBBS
	fakeStagerBBS = fakebbs.NewFakeStagerBBS()
	Ω(fakeStagerBBS).ShouldNot(BeNil())
})

func registerSignalHandler() {
	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt, os.Kill)

		select {
		case <-c:
			etcdRunner.Stop()
			os.Exit(0)
		}
	}()
}
<|endoftext|>


<commit_before>// Description: ginweb is light weight encapsulation of gin framework
// Author: ZHU HAIHUA
// Since: 2016-02-28 14:57
package ginweb

import (
	"github.com/gin-gonic/gin"
	. "github.com/kimiazhu/ginweb/midware"
	"github.com/kimiazhu/ginweb/server"
	"github.com/kimiazhu/ginweb/conf"
	"github.com/kimiazhu/log4go"
)

const VERSION = "0.0.1"

// an component is different from midware,
// it will be initialize just before the
// application running, You can use it in
// the entire life cycle of application
type component struct {
	name string
	config interface{}
	initialize func(config interface{}) (error)
}

var components []component

func New() *gin.Engine {
	gin.SetMode(conf.Conf.SERVER.RUNMODE)
	g := gin.New()
	g.Use(Recovery())
	g.Use(AccessLog())
	return g
}

func RegisterComponent(name string, config interface{}, initialize func(config interface{}) (error))  {
	components = append(components, component{name, config, initialize})
}

func Run(port string, engin *gin.Engine) {
	initialize()
	server.Start(":"+port, engin)
}

// initialize used to init all components before the app start
func initialize() {
	for _, c := range components {
		e := c.initialize(c.config)
		if e != nil {
			log4go.Error("initialize component [%s] error! %v", c.name, e)
		} else {
			log4go.Debug("initialize component [%s] success", c.name)
		}
	}
}<commit_msg>init just once the components<commit_after>// Description: ginweb is light weight encapsulation of gin framework
// Author: ZHU HAIHUA
// Since: 2016-02-28 14:57
package ginweb

import (
	"github.com/gin-gonic/gin"
	. "github.com/kimiazhu/ginweb/midware"
	"github.com/kimiazhu/ginweb/server"
	"github.com/kimiazhu/ginweb/conf"
	"github.com/kimiazhu/log4go"
	"sync"
)

const VERSION = "0.0.1"

// an component is different from midware,
// it will be initialize just before the
// application running, You can use it in
// the entire life cycle of application
type component struct {
	name string
	config interface{}
	initialize func(config interface{}) (error)
}

var initCompOnce sync.Once
var components []component

func New() *gin.Engine {
	gin.SetMode(conf.Conf.SERVER.RUNMODE)
	g := gin.New()
	g.Use(Recovery())
	g.Use(AccessLog())
	return g
}

func RegisterComponent(name string, config interface{}, initialize func(config interface{}) (error))  {
	components = append(components, component{name, config, initialize})
}

func Run(port string, engin *gin.Engine) {
	initCompOnce.Do(initialize)
	server.Start(":"+port, engin)
}

// initialize used to init all components before the app start
func initialize() {
	for _, c := range components {
		e := c.initialize(c.config)
		if e != nil {
			log4go.Error("initialize component [%s] error! %v", c.name, e)
		} else {
			log4go.Debug("initialize component [%s] success", c.name)
		}
	}
}<|endoftext|>


<commit_before>// Package manager provides interface between handler and lower level implementation
// such as geoloader.
package manager

// The implementation is currently rather naive.  Eviction is done based only on whether
// there is a pending request, and there are already the max number of datasets loaded.
// A later implementation will use LRU and dead time to make this determination.
//
// Behavior:
//   If a legacy dataset is requests, return the CurrentAnnotator instead.
//   If the requested dataset is loaded, return it.
//   If the requested dataset is loading, return ErrPendingAnnotatorLoad
//   If the dataset is not loaded or pending, check:
//      A: If there are already MaxPending loads in process:
//        Do nothing and reply with ErrPendingAnnotatorLoad (even though this isn't true)
//      B: If there is room to load it?
//       YES: start loading it, and return ErrPendingAnnotatorLoad
//        NO: kick out an existing dataset and return ErrPendingAnnotatorLoad.
//
// Please modify with extreme caution.  The lock MUST be held when ACCESSING any field
// of AnnotatorMap.

// Note that the system may evict up to the number of pending loads, so at any given time,
// there may only be MaxDatasetInMemory = MaxPending actually loaded.

// Also note that anyone holding an annotator will prevent it from being collected by the
// GC, so simply evicting it is not a guarantee that the memory will be reclaimed.

import (
	"errors"
	"log"
	"sync"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/geoloader"
	"github.com/m-lab/annotation-service/metrics"
)

var (
	// These are vars instead of consts to facilitate testing.
	MaxDatasetInMemory = 12 // Limit on number of loaded datasets
	MaxPending         = 2  // Limit on number of concurrently loading datasets.

	// ErrNilDataset is returned when CurrentAnnotator is nil.
	ErrNilDataset = errors.New("CurrentAnnotator is nil")

	// ErrPendingAnnotatorLoad is returned when a new annotator is requested, but not yet loaded.
	ErrPendingAnnotatorLoad = errors.New("annotator is loading")

	// ErrAnnotatorLoadFailed is returned when a requested annotator has failed to load.
	ErrAnnotatorLoadFailed = errors.New("unable to load annoator")

	// These are UNEXPECTED errors!!

	// ErrNoAppropriateDataset is returned when directory is empty.
	ErrNoAppropriateDataset = errors.New("No Appropriate Dataset")
	// ErrGoroutineNotOwner indicates multithreaded code problem with reservation.
	ErrGoroutineNotOwner = errors.New("Goroutine not owner")
	// ErrMapEntryAlreadySet indicates multithreaded code problem setting map entry.
	ErrMapEntryAlreadySet = errors.New("Map entry already set")

	// TODO remove these and keep current in the map.
	// A mutex to make sure that we are not reading from the CurrentAnnotator
	// pointer while trying to update it
	currentDataMutex = &sync.RWMutex{}

	// CurrentAnnotator points to a GeoDataset struct containing the absolute
	// latest data for the annotator to search and reply with
	CurrentAnnotator api.Annotator

	// ArchivedLoader points to a AnnotatorMap struct containing the archived
	// Geolite2 and legacy dataset in memory.
	archivedAnnotator = NewAnnotatorMap(geoloader.ArchivedLoader)
)

// AnnotatorMap manages all loading and fetching of Annotators.
// TODO - should we call this AnnotatorCache?
// TODO - should this be a generic cache of interface{}?
//
// Synchronization:
//  All accesses must hold the mutex.  If an element is not found, the
//  goroutine may attempt to take responsibility for loading it by obtaining
//  the write lock, and writing an entry with a nil pointer.
// TODO - still need a strategy for dealing with persistent errors.
type AnnotatorMap struct {
	// Keys are filename of the datasets.
	annotators map[string]api.Annotator
	// Lock to be held when reading or writing the map.
	mutex      sync.RWMutex
	numPending int
	loader     func(string) (api.Annotator, error)
}

// NewAnnotatorMap creates a new map that will use the provided loader for loading new Annotators.
func NewAnnotatorMap(loader func(string) (api.Annotator, error)) *AnnotatorMap {
	return &AnnotatorMap{annotators: make(map[string]api.Annotator), loader: loader}
}

// NOTE: Should only be called by checkAndLoadAnnotator.
// The calling goroutine should "own" the responsibility for
// setting the annotator.
func (am *AnnotatorMap) setAnnotatorIfNil(key string, ann api.Annotator) error {
	am.mutex.Lock()
	defer am.mutex.Unlock()

	old, ok := am.annotators[key]
	if !ok {
		log.Println("This should never happen", ErrGoroutineNotOwner)
		metrics.ErrorTotal.WithLabelValues("WrongOwner").Inc()
		return ErrGoroutineNotOwner
	}
	if old != nil {
		log.Println("This should never happen", ErrMapEntryAlreadySet)
		metrics.ErrorTotal.WithLabelValues("MapEntryAlreadySet").Inc()
		return ErrMapEntryAlreadySet
	}

	am.annotators[key] = ann
	metrics.LoadCount.Inc()
	metrics.PendingLoads.Dec()
	metrics.DatasetCount.Inc()
	am.numPending--
	log.Println("Loaded", key)
	return nil
}

// This creates a reservation for loading a dataset, IFF map entry is empty (not nil or populated)
//   If the dataset is not loaded or pending, check:
//      A: If there are already MaxPending loads in process:
//        Do nothing and reply false
//      B: If there is room to load it?
//       YES: make the reservation (by setting entry to nil) and return true.
//        NO: kick out an existing dataset and return false.
func (am *AnnotatorMap) maybeSetNil(key string) bool {
	am.mutex.Lock()
	defer am.mutex.Unlock()
	_, ok := am.annotators[key]
	if ok {
		// Another goroutine is already responsible for loading.
		return false
	}

	if am.numPending >= MaxPending {
		return false
	}
	// Check the number of datasets in memory. Given the memory
	// limit, some dataset may be removed from memory if needed.
	if len(am.annotators) >= MaxDatasetInMemory {
		for fileKey := range am.annotators {
			ann, ok := am.annotators[fileKey]
			if ok {
				log.Println("removing dataset " + fileKey)
				ann.Close()
				delete(am.annotators, fileKey)
				metrics.EvictionCount.Inc()
				metrics.DatasetCount.Dec()
				break
			}
		}
		return false
	}

	// Place marker so that other requesters know it is loading.
	am.annotators[key] = nil
	metrics.PendingLoads.Inc()
	am.numPending++
	return true
}

// This synchronously attempts to set map entry to nil, and
// if successful, proceeds to asynchronously load the new dataset.
func (am *AnnotatorMap) checkAndLoadAnnotator(key string) {
	reserved := am.maybeSetNil(key)
	if reserved {
		// This goroutine now has exclusive ownership of the
		// map entry, and the responsibility for loading the annotator.
		go func(key string) {
			newAnn, err := am.loader(key)
			if err != nil {
				metrics.ErrorTotal.WithLabelValues(err.Error()).Inc()
				log.Println(err)
				return
			}
			// Set the new annotator value.  Entry should be nil.
			err = am.setAnnotatorIfNil(key, newAnn)
			if err != nil {
				metrics.ErrorTotal.WithLabelValues(err.Error()).Inc()
				log.Println(err)
			}
		}(key)
	}
}

// GetAnnotator returns the annoator in memory based on filename.
func (am *AnnotatorMap) GetAnnotator(key string) (api.Annotator, error) {
	am.mutex.RLock()
	ann, _ := am.annotators[key]
	am.mutex.RUnlock()

	if ann == nil {
		metrics.RejectionCount.WithLabelValues("Dataset not loaded").Inc()
		return nil, ErrPendingAnnotatorLoad
	}
	return ann, nil
}

// LoadAllDatasets load all available datasets into memory
// Must be called after geoloader.UpdateArchivedFilenames()
func (am *AnnotatorMap) LoadAllDatasets() error {
	df := geoloader.getDatasetFilenames()
	for _, filename := range df {
		ann, err := am.loader(filename)
		if err != nil {
			continue
		}
		am.mutex.Lock()
		am.annotators[filename] = ann
		am.mutex.Unlock()
	}
	return nil
}

func (am *AnnotatorMap) NumDatasetInMemory() int {
	return len(am.annotators)
}

// GetAnnotator returns the correct annotator to use for a given timestamp.
func GetAnnotator(request *api.RequestData) (api.Annotator, error) {
	// key := strconv.FormatInt(date.Unix(), encodingBase)
	date := request.Timestamp
	if date.After(geoloader.LatestDatasetDate()) {
		currentDataMutex.RLock()
		ann := CurrentAnnotator
		currentDataMutex.RUnlock()
		return ann, nil
	}

	filename := geoloader.BestAnnotatorFilename(request)

	if filename == "" {
		metrics.ErrorTotal.WithLabelValues("No Appropriate Dataset").Inc()
		return nil, errors.New("No Appropriate Dataset")
	}

	// Since all datasets have been loaded into memory during initialization,
	// We can fetch any annotator by filename.
	return archivedAnnotator.GetAnnotator(filename)
}

// InitDataset will update the filename list of archived dataset in memory
// and load ALL legacy and Geolite2 dataset in memory.
func InitDataset() {
	geoloader.UpdateArchivedFilenames()

	ann := geoloader.GetLatestData()
	currentDataMutex.Lock()
	CurrentAnnotator = ann
	currentDataMutex.Unlock()

	archivedAnnotator.LoadAllDatasets()
}
<commit_msg>Update manager.go<commit_after>// Package manager provides interface between handler and lower level implementation
// such as geoloader.
package manager

// The implementation is currently rather naive.  Eviction is done based only on whether
// there is a pending request, and there are already the max number of datasets loaded.
// A later implementation will use LRU and dead time to make this determination.
//
// Behavior:
//   If a legacy dataset is requests, return the CurrentAnnotator instead.
//   If the requested dataset is loaded, return it.
//   If the requested dataset is loading, return ErrPendingAnnotatorLoad
//   If the dataset is not loaded or pending, check:
//      A: If there are already MaxPending loads in process:
//        Do nothing and reply with ErrPendingAnnotatorLoad (even though this isn't true)
//      B: If there is room to load it?
//       YES: start loading it, and return ErrPendingAnnotatorLoad
//        NO: kick out an existing dataset and return ErrPendingAnnotatorLoad.
//
// Please modify with extreme caution.  The lock MUST be held when ACCESSING any field
// of AnnotatorMap.

// Note that the system may evict up to the number of pending loads, so at any given time,
// there may only be MaxDatasetInMemory = MaxPending actually loaded.

// Also note that anyone holding an annotator will prevent it from being collected by the
// GC, so simply evicting it is not a guarantee that the memory will be reclaimed.

import (
	"errors"
	"log"
	"sync"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/geoloader"
	"github.com/m-lab/annotation-service/metrics"
)

var (
	// These are vars instead of consts to facilitate testing.
	MaxDatasetInMemory = 12 // Limit on number of loaded datasets
	MaxPending         = 2  // Limit on number of concurrently loading datasets.

	// ErrNilDataset is returned when CurrentAnnotator is nil.
	ErrNilDataset = errors.New("CurrentAnnotator is nil")

	// ErrPendingAnnotatorLoad is returned when a new annotator is requested, but not yet loaded.
	ErrPendingAnnotatorLoad = errors.New("annotator is loading")

	// ErrAnnotatorLoadFailed is returned when a requested annotator has failed to load.
	ErrAnnotatorLoadFailed = errors.New("unable to load annoator")

	// These are UNEXPECTED errors!!

	// ErrNoAppropriateDataset is returned when directory is empty.
	ErrNoAppropriateDataset = errors.New("No Appropriate Dataset")
	// ErrGoroutineNotOwner indicates multithreaded code problem with reservation.
	ErrGoroutineNotOwner = errors.New("Goroutine not owner")
	// ErrMapEntryAlreadySet indicates multithreaded code problem setting map entry.
	ErrMapEntryAlreadySet = errors.New("Map entry already set")

	// TODO remove these and keep current in the map.
	// A mutex to make sure that we are not reading from the CurrentAnnotator
	// pointer while trying to update it
	currentDataMutex = &sync.RWMutex{}

	// CurrentAnnotator points to a GeoDataset struct containing the absolute
	// latest data for the annotator to search and reply with
	CurrentAnnotator api.Annotator

	// ArchivedLoader points to a AnnotatorMap struct containing the archived
	// Geolite2 and legacy dataset in memory.
	archivedAnnotator = NewAnnotatorMap(geoloader.ArchivedLoader)
)

// AnnotatorMap manages all loading and fetching of Annotators.
// TODO - should we call this AnnotatorCache?
// TODO - should this be a generic cache of interface{}?
//
// Synchronization:
//  All accesses must hold the mutex.  If an element is not found, the
//  goroutine may attempt to take responsibility for loading it by obtaining
//  the write lock, and writing an entry with a nil pointer.
// TODO - still need a strategy for dealing with persistent errors.
type AnnotatorMap struct {
	// Keys are filename of the datasets.
	annotators map[string]api.Annotator
	// Lock to be held when reading or writing the map.
	mutex      sync.RWMutex
	numPending int
	loader     func(string) (api.Annotator, error)
}

// NewAnnotatorMap creates a new map that will use the provided loader for loading new Annotators.
func NewAnnotatorMap(loader func(string) (api.Annotator, error)) *AnnotatorMap {
	return &AnnotatorMap{annotators: make(map[string]api.Annotator), loader: loader}
}

// NOTE: Should only be called by checkAndLoadAnnotator.
// The calling goroutine should "own" the responsibility for
// setting the annotator.
func (am *AnnotatorMap) setAnnotatorIfNil(key string, ann api.Annotator) error {
	am.mutex.Lock()
	defer am.mutex.Unlock()

	old, ok := am.annotators[key]
	if !ok {
		log.Println("This should never happen", ErrGoroutineNotOwner)
		metrics.ErrorTotal.WithLabelValues("WrongOwner").Inc()
		return ErrGoroutineNotOwner
	}
	if old != nil {
		log.Println("This should never happen", ErrMapEntryAlreadySet)
		metrics.ErrorTotal.WithLabelValues("MapEntryAlreadySet").Inc()
		return ErrMapEntryAlreadySet
	}

	am.annotators[key] = ann
	metrics.LoadCount.Inc()
	metrics.PendingLoads.Dec()
	metrics.DatasetCount.Inc()
	am.numPending--
	log.Println("Loaded", key)
	return nil
}

// This creates a reservation for loading a dataset, IFF map entry is empty (not nil or populated)
//   If the dataset is not loaded or pending, check:
//      A: If there are already MaxPending loads in process:
//        Do nothing and reply false
//      B: If there is room to load it?
//       YES: make the reservation (by setting entry to nil) and return true.
//        NO: kick out an existing dataset and return false.
func (am *AnnotatorMap) maybeSetNil(key string) bool {
	am.mutex.Lock()
	defer am.mutex.Unlock()
	_, ok := am.annotators[key]
	if ok {
		// Another goroutine is already responsible for loading.
		return false
	}

	if am.numPending >= MaxPending {
		return false
	}
	// Check the number of datasets in memory. Given the memory
	// limit, some dataset may be removed from memory if needed.
	if len(am.annotators) >= MaxDatasetInMemory {
		for fileKey := range am.annotators {
			ann, ok := am.annotators[fileKey]
			if ok {
				log.Println("removing dataset " + fileKey)
				ann.Close()
				delete(am.annotators, fileKey)
				metrics.EvictionCount.Inc()
				metrics.DatasetCount.Dec()
				break
			}
		}
		return false
	}

	// Place marker so that other requesters know it is loading.
	am.annotators[key] = nil
	metrics.PendingLoads.Inc()
	am.numPending++
	return true
}

// This synchronously attempts to set map entry to nil, and
// if successful, proceeds to asynchronously load the new dataset.
func (am *AnnotatorMap) checkAndLoadAnnotator(key string) {
	reserved := am.maybeSetNil(key)
	if reserved {
		// This goroutine now has exclusive ownership of the
		// map entry, and the responsibility for loading the annotator.
		go func(key string) {
			newAnn, err := am.loader(key)
			if err != nil {
				metrics.ErrorTotal.WithLabelValues(err.Error()).Inc()
				log.Println(err)
				return
			}
			// Set the new annotator value.  Entry should be nil.
			err = am.setAnnotatorIfNil(key, newAnn)
			if err != nil {
				metrics.ErrorTotal.WithLabelValues(err.Error()).Inc()
				log.Println(err)
			}
		}(key)
	}
}

// GetAnnotator returns the annoator in memory based on filename.
func (am *AnnotatorMap) GetAnnotator(key string) (api.Annotator, error) {
	am.mutex.RLock()
	ann, _ := am.annotators[key]
	am.mutex.RUnlock()

	if ann == nil {
		metrics.RejectionCount.WithLabelValues("Dataset not loaded").Inc()
		return nil, ErrPendingAnnotatorLoad
	}
	return ann, nil
}

// LoadAllDatasets load all available datasets into memory
// Must be called after geoloader.UpdateArchivedFilenames()
func (am *AnnotatorMap) LoadAllDatasets() error {
	df := geoloader.GetDatasetFilenames()
	for _, filename := range df {
		ann, err := am.loader(filename)
		if err != nil {
			continue
		}
		am.mutex.Lock()
		am.annotators[filename] = ann
		am.mutex.Unlock()
	}
	return nil
}

func (am *AnnotatorMap) NumDatasetInMemory() int {
	return len(am.annotators)
}

// GetAnnotator returns the correct annotator to use for a given timestamp.
func GetAnnotator(request *api.RequestData) (api.Annotator, error) {
	// key := strconv.FormatInt(date.Unix(), encodingBase)
	date := request.Timestamp
	if date.After(geoloader.LatestDatasetDate()) {
		currentDataMutex.RLock()
		ann := CurrentAnnotator
		currentDataMutex.RUnlock()
		return ann, nil
	}

	filename := geoloader.BestAnnotatorFilename(request)

	if filename == "" {
		metrics.ErrorTotal.WithLabelValues("No Appropriate Dataset").Inc()
		return nil, errors.New("No Appropriate Dataset")
	}

	// Since all datasets have been loaded into memory during initialization,
	// We can fetch any annotator by filename.
	return archivedAnnotator.GetAnnotator(filename)
}

// InitDataset will update the filename list of archived dataset in memory
// and load ALL legacy and Geolite2 dataset in memory.
func InitDataset() {
	geoloader.UpdateArchivedFilenames()

	ann := geoloader.GetLatestData()
	currentDataMutex.Lock()
	CurrentAnnotator = ann
	currentDataMutex.Unlock()

	archivedAnnotator.LoadAllDatasets()
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"

	log "github.com/couchbaselabs/clog"
)

const PLANNER_VERSION = "0.0.0" // Follow semver rules.

type Plan struct {
}

type LogicalIndex struct {
}

type LogicalIndexes []*LogicalIndex

type Indexer struct {
}

type Indexers []*Indexer

// A planner assigns partitions to cbft's and to PIndexes on each cbft.
func (mgr *Manager) PlannerLoop() {
	for _ = range mgr.plannerCh {
		if !mgr.CheckPlannerVersion() {
			continue
		}
		plan, err := mgr.CalcPlan(nil, nil)
		if err != nil {
			log.Printf("error: CalcPlan, err: %v", err)
		}
		if plan != nil {
			// TODO: save the plan.
		}
	}
}

func (mgr *Manager) CalcPlan(logicalIndexes LogicalIndexes,
	indexers Indexers) (*Plan, error) {
	// TODO: implement the grand plans for the planner.
	// First gen planner should keep it simple, such as...
	// - a single Feed for every datasource node.
	// - a Feed might "fan out" to multiple Streams/PIndexes.
	// - have a single PIndex for all datasource partitions
	//   (vbuckets) to start.
	return nil, fmt.Errorf("TODO")
}

func (mgr *Manager) CheckPlannerVersion() bool {
	for mgr.cfg != nil {
		version, cas, err := mgr.cfg.Get("plannerVersion", 0)
		if version == nil || version == "" || err != nil {
			version = PLANNER_VERSION
		}
		if !VersionGTE(PLANNER_VERSION, version.(string)) {
			log.Printf("planning skipped for obsoleted version: %v < %v",
				PLANNER_VERSION, version)
			return false
		}
		if PLANNER_VERSION == version {
			return true
		}
		// We have a higher PLANNER_VERSION than the read
		// version, so save PLANNER_VERSION and retry.
		mgr.cfg.Set("plannerVersion", PLANNER_VERSION, cas)
	}

	return false // Never reached.
}
<commit_msg>move planner init logging to higher level<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"

	log "github.com/couchbaselabs/clog"
)

const PLANNER_VERSION = "0.0.0" // Follow semver rules.

type Plan struct {
}

type LogicalIndex struct {
}

type LogicalIndexes []*LogicalIndex

type Indexer struct {
}

type Indexers []*Indexer

// A planner assigns partitions to cbft's and to PIndexes on each cbft.
func (mgr *Manager) PlannerLoop() {
	for _ = range mgr.plannerCh {
		if !mgr.CheckPlannerVersion() {
			log.Printf("planning skipped for obsoleted version: %v",
				PLANNER_VERSION)
			continue
		}
		plan, err := mgr.CalcPlan(nil, nil)
		if err != nil {
			log.Printf("error: CalcPlan, err: %v", err)
		}
		if plan != nil {
			// TODO: save the plan.
		}
	}
}

func (mgr *Manager) CalcPlan(logicalIndexes LogicalIndexes,
	indexers Indexers) (*Plan, error) {
	// TODO: implement the grand plans for the planner.
	// First gen planner should keep it simple, such as...
	// - a single Feed for every datasource node.
	// - a Feed might "fan out" to multiple Streams/PIndexes.
	// - have a single PIndex for all datasource partitions
	//   (vbuckets) to start.
	return nil, fmt.Errorf("TODO")
}

func (mgr *Manager) CheckPlannerVersion() bool {
	for mgr.cfg != nil {
		version, cas, err := mgr.cfg.Get("plannerVersion", 0)
		if version == nil || version == "" || err != nil {
			version = PLANNER_VERSION
		}
		if !VersionGTE(PLANNER_VERSION, version.(string)) {
			return false
		}
		if PLANNER_VERSION == version {
			return true
		}
		// We have a higher PLANNER_VERSION than the read
		// version, so save PLANNER_VERSION and retry.
		mgr.cfg.Set("plannerVersion", PLANNER_VERSION, cas)
	}

	return false // Never reached.
}
<|endoftext|>


<commit_before>package mangadownloader

import (
	"code.google.com/p/go.net/html"
	"errors"
	"fmt"
	"github.com/pierrre/archivefile/zip"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
)

var (
	regexpImageContentType, _ = regexp.Compile("^image/(.+)$")
	filenameCleanReplacer     *strings.Replacer
)

func init() {
	filenameCleanReplacements := make([]string, len(filenameReservedCharacters)*2)
	for _, char := range filenameReservedCharacters {
		filenameCleanReplacements = append(filenameCleanReplacements, string(char))
		filenameCleanReplacements = append(filenameCleanReplacements, " ")
	}
	filenameCleanReplacer = strings.NewReplacer(filenameCleanReplacements...)
}

type MangaDownloader struct {
	Services  map[string]Service
	HttpRetry int
}

func NewMangaDownloader() *MangaDownloader {
	md := new(MangaDownloader)
	md.Services = make(map[string]Service)

	return md
}

func CreateDefaultMangeDownloader() *MangaDownloader {
	md := NewMangaDownloader()

	md.Services["mangafox"] = &MangaFoxService{
		Md: md,
	}

	md.Services["mangahere"] = &MangaHereService{
		Md: md,
	}

	md.Services["mangareader"] = &MangaReaderService{
		Md: md,
	}

	md.Services["mangawall"] = &MangaWallService{
		Md: md,
	}

	md.Services["tenmanga"] = &TenMangaService{
		Md: md,
	}

	return md
}

func (md *MangaDownloader) Identify(u *url.URL) (interface{}, error) {
	for _, service := range md.Services {
		if service.Supports(u) {
			return service.Identify(u)
		}
	}

	return nil, errors.New("Unsupported url")
}

func (md *MangaDownloader) DownloadManga(manga *Manga, out string, options *Options) error {
	name, err := manga.Name()
	if err != nil {
		return err
	}

	out = filepath.Join(out, cleanFilename(name))

	chapters, err := manga.Chapters()
	if err != nil {
		return err
	}

	err = md.downloadChapters(chapters, out, options)
	if err != nil {
		return err
	}

	return nil
}

func (md *MangaDownloader) downloadChapters(chapters []*Chapter, out string, options *Options) error {
	work := make(chan *Chapter)
	go func() {
		for _, chapter := range chapters {
			work <- chapter
		}
		close(work)
	}()

	parallelChapter := options.ParallelChapter
	if parallelChapter < 1 {
		parallelChapter = 1
	}
	wg := new(sync.WaitGroup)
	wg.Add(parallelChapter)
	result := make(chan error)
	for i := 0; i < parallelChapter; i++ {
		go func() {
			for chapter := range work {
				result <- md.DownloadChapter(chapter, out, options)
			}
			wg.Done()
		}()
	}
	go func() {
		wg.Wait()
		close(result)
	}()

	errs := make(MultiError, 0)
	for err := range result {
		if err != nil {
			errs = append(errs, err)
		}
	}
	if len(errs) > 0 {
		return errs
	}

	return nil
}

func (md *MangaDownloader) DownloadChapter(chapter *Chapter, out string, options *Options) error {
	name, err := chapter.Name()
	if err != nil {
		return err
	}

	out = filepath.Join(out, cleanFilename(name))
	var outFinal string
	if options.Cbz {
		outFinal = getCbzPath(out)
	} else {
		outFinal = out
	}
	if fileExists(outFinal) {
		return nil
	}

	outTmp := out + ".tmp"
	if fileExists(outTmp) {
		err = os.RemoveAll(outTmp)
		if err != nil {
			return err
		}
	}

	pages, err := chapter.Pages()
	if err != nil {
		return err
	}

	err = md.downloadPages(pages, outTmp, options)
	if err != nil {
		return err
	}

	err = os.Rename(outTmp, out)
	if err != nil {
		return err
	}

	if options.Cbz {
		outCbz := getCbzPath(out)
		outCbzTmp := outCbz + ".tmp"
		err = zip.ArchiveFile(out+string(filepath.Separator), outCbzTmp, nil)
		if err != nil {
			return err
		}
		err = os.RemoveAll(out)
		if err != nil {
			return err
		}
		err = os.Rename(outCbzTmp, outCbz)
		if err != nil {
			return err
		}
	}

	return nil
}

func (md *MangaDownloader) downloadPages(pages []*Page, out string, options *Options) error {
	type pageWork struct {
		page  *Page
		index int
	}

	work := make(chan *pageWork)
	go func() {
		for index, page := range pages {
			work <- &pageWork{
				page:  page,
				index: index,
			}
		}
		close(work)
	}()

	parallelPage := options.ParallelPage
	if parallelPage < 1 {
		parallelPage = 1
	}
	wg := new(sync.WaitGroup)
	wg.Add(parallelPage)
	result := make(chan error)
	for i := 0; i < parallelPage; i++ {
		go func() {
			for chapterPageWork := range work {
				result <- md.downloadPageWithIndex(chapterPageWork.page, out, chapterPageWork.index, options)
			}
			wg.Done()
		}()
	}
	go func() {
		wg.Wait()
		close(result)
	}()

	errs := make(MultiError, 0)
	for err := range result {
		if err != nil {
			errs = append(errs, err)
		}
	}
	if len(errs) > 0 {
		return errs
	}

	return nil
}

func (md *MangaDownloader) downloadPageWithIndex(page *Page, out string, index int, options *Options) error {
	filenameFormat := "%0" + strconv.Itoa(options.PageDigitCount) + "d"
	filename := fmt.Sprintf(filenameFormat, index+1)
	return md.DownloadPage(page, out, filename, options)
}

func (md *MangaDownloader) DownloadPage(page *Page, out string, filename string, options *Options) error {
	out = filepath.Join(out, filename)

	imageUrl, err := page.ImageUrl()
	if err != nil {
		return err
	}

	response, err := md.HttpGet(imageUrl)
	if err != nil {
		return err
	}
	defer response.Body.Close()
	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return err
	}

	var extension string
	if len(extension) == 0 {
		contentType := response.Header.Get("content-type")
		if len(contentType) > 0 {
			matches := regexpImageContentType.FindStringSubmatch(contentType)
			if matches != nil && len(matches) == 2 {
				extension = matches[1]
			}
		}
	}
	if len(extension) > 0 {
		if extension == "jpeg" {
			extension = "jpg"
		}
		out += "." + extension
	}

	err = os.MkdirAll(filepath.Dir(out), 0755)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(out, data, 0644)
	if err != nil {
		return err
	}

	return nil
}

func (md *MangaDownloader) HttpGet(u *url.URL) (response *http.Response, err error) {
	httpRetry := md.HttpRetry
	if httpRetry < 1 {
		httpRetry = 1
	}

	errs := make(MultiError, 0)
	for i := 0; i < httpRetry; i++ {
		response, err := http.Get(u.String())
		if err == nil {
			return response, nil
		}
		errs = append(errs, err)
	}
	return nil, errs
}

func (md *MangaDownloader) HttpGetHtml(u *url.URL) (*html.Node, error) {
	response, err := md.HttpGet(u)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	node, err := html.Parse(response.Body)
	return node, err
}

type Options struct {
	Cbz             bool
	PageDigitCount  int
	ParallelChapter int
	ParallelPage    int
}

func cleanFilename(name string) string {
	return filenameCleanReplacer.Replace(name)
}

func getCbzPath(filePath string) string {
	return filePath + ".cbz"
}
<commit_msg>Improve download cbz / tmp<commit_after>package mangadownloader

import (
	"code.google.com/p/go.net/html"
	"errors"
	"fmt"
	"github.com/pierrre/archivefile/zip"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
)

var (
	regexpImageContentType, _ = regexp.Compile("^image/(.+)$")
	filenameCleanReplacer     *strings.Replacer
)

func init() {
	filenameCleanReplacements := make([]string, len(filenameReservedCharacters)*2)
	for _, char := range filenameReservedCharacters {
		filenameCleanReplacements = append(filenameCleanReplacements, string(char))
		filenameCleanReplacements = append(filenameCleanReplacements, " ")
	}
	filenameCleanReplacer = strings.NewReplacer(filenameCleanReplacements...)
}

type MangaDownloader struct {
	Services  map[string]Service
	HttpRetry int
}

func NewMangaDownloader() *MangaDownloader {
	md := new(MangaDownloader)
	md.Services = make(map[string]Service)

	return md
}

func CreateDefaultMangeDownloader() *MangaDownloader {
	md := NewMangaDownloader()

	md.Services["mangafox"] = &MangaFoxService{
		Md: md,
	}

	md.Services["mangahere"] = &MangaHereService{
		Md: md,
	}

	md.Services["mangareader"] = &MangaReaderService{
		Md: md,
	}

	md.Services["mangawall"] = &MangaWallService{
		Md: md,
	}

	md.Services["tenmanga"] = &TenMangaService{
		Md: md,
	}

	return md
}

func (md *MangaDownloader) Identify(u *url.URL) (interface{}, error) {
	for _, service := range md.Services {
		if service.Supports(u) {
			return service.Identify(u)
		}
	}

	return nil, errors.New("Unsupported url")
}

func (md *MangaDownloader) DownloadManga(manga *Manga, out string, options *Options) error {
	name, err := manga.Name()
	if err != nil {
		return err
	}

	out = filepath.Join(out, cleanFilename(name))

	chapters, err := manga.Chapters()
	if err != nil {
		return err
	}

	err = md.downloadChapters(chapters, out, options)
	if err != nil {
		return err
	}

	return nil
}

func (md *MangaDownloader) downloadChapters(chapters []*Chapter, out string, options *Options) error {
	work := make(chan *Chapter)
	go func() {
		for _, chapter := range chapters {
			work <- chapter
		}
		close(work)
	}()

	parallelChapter := options.ParallelChapter
	if parallelChapter < 1 {
		parallelChapter = 1
	}
	wg := new(sync.WaitGroup)
	wg.Add(parallelChapter)
	result := make(chan error)
	for i := 0; i < parallelChapter; i++ {
		go func() {
			for chapter := range work {
				result <- md.DownloadChapter(chapter, out, options)
			}
			wg.Done()
		}()
	}
	go func() {
		wg.Wait()
		close(result)
	}()

	errs := make(MultiError, 0)
	for err := range result {
		if err != nil {
			errs = append(errs, err)
		}
	}
	if len(errs) > 0 {
		return errs
	}

	return nil
}

func (md *MangaDownloader) DownloadChapter(chapter *Chapter, out string, options *Options) error {
	name, err := chapter.Name()
	if err != nil {
		return err
	}
	out = filepath.Join(out, cleanFilename(name))

	if options.Cbz {
		return md.downloadChapterCbz(chapter, out, options)
	} else {
		return md.downloadChapter(chapter, out, options)
	}
}

func (md *MangaDownloader) downloadChapter(chapter *Chapter, out string, options *Options) error {
	if fileExists(out) {
		return nil
	}

	outTmp := out + ".tmp"
	if fileExists(outTmp) {
		err := os.RemoveAll(outTmp)
		if err != nil {
			return err
		}
	}

	pages, err := chapter.Pages()
	if err != nil {
		return err
	}

	err = md.downloadPages(pages, outTmp, options)
	if err != nil {
		return err
	}

	err = os.Rename(outTmp, out)
	if err != nil {
		return err
	}

	return nil
}

func (md *MangaDownloader) downloadChapterCbz(chapter *Chapter, out string, options *Options) error {
	outCbz := out + ".cbz"
	if fileExists(outCbz) {
		return nil
	}

	outCbzTmp := outCbz + ".tmp"
	if fileExists(outCbzTmp) {
		err := os.RemoveAll(outCbzTmp)
		if err != nil {
			return err
		}
	}

	err := md.downloadChapter(chapter, out, options)
	if err != nil {
		return err
	}

	err = zip.ArchiveFile(out+string(filepath.Separator), outCbzTmp, nil)
	if err != nil {
		return err
	}

	err = os.Rename(outCbzTmp, outCbz)
	if err != nil {
		return err
	}

	err = os.RemoveAll(out)
	if err != nil {
		return err
	}

	return nil
}

func (md *MangaDownloader) downloadPages(pages []*Page, out string, options *Options) error {
	type pageWork struct {
		page  *Page
		index int
	}

	work := make(chan *pageWork)
	go func() {
		for index, page := range pages {
			work <- &pageWork{
				page:  page,
				index: index,
			}
		}
		close(work)
	}()

	parallelPage := options.ParallelPage
	if parallelPage < 1 {
		parallelPage = 1
	}
	wg := new(sync.WaitGroup)
	wg.Add(parallelPage)
	result := make(chan error)
	for i := 0; i < parallelPage; i++ {
		go func() {
			for chapterPageWork := range work {
				result <- md.downloadPageWithIndex(chapterPageWork.page, out, chapterPageWork.index, options)
			}
			wg.Done()
		}()
	}
	go func() {
		wg.Wait()
		close(result)
	}()

	errs := make(MultiError, 0)
	for err := range result {
		if err != nil {
			errs = append(errs, err)
		}
	}
	if len(errs) > 0 {
		return errs
	}

	return nil
}

func (md *MangaDownloader) downloadPageWithIndex(page *Page, out string, index int, options *Options) error {
	filenameFormat := "%0" + strconv.Itoa(options.PageDigitCount) + "d"
	filename := fmt.Sprintf(filenameFormat, index+1)
	return md.DownloadPage(page, out, filename, options)
}

func (md *MangaDownloader) DownloadPage(page *Page, out string, filename string, options *Options) error {
	out = filepath.Join(out, filename)

	imageUrl, err := page.ImageUrl()
	if err != nil {
		return err
	}

	response, err := md.HttpGet(imageUrl)
	if err != nil {
		return err
	}
	defer response.Body.Close()
	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return err
	}

	var extension string
	if len(extension) == 0 {
		contentType := response.Header.Get("content-type")
		if len(contentType) > 0 {
			matches := regexpImageContentType.FindStringSubmatch(contentType)
			if matches != nil && len(matches) == 2 {
				extension = matches[1]
			}
		}
	}
	if len(extension) > 0 {
		if extension == "jpeg" {
			extension = "jpg"
		}
		out += "." + extension
	}

	err = os.MkdirAll(filepath.Dir(out), 0755)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(out, data, 0644)
	if err != nil {
		return err
	}

	return nil
}

func (md *MangaDownloader) HttpGet(u *url.URL) (response *http.Response, err error) {
	httpRetry := md.HttpRetry
	if httpRetry < 1 {
		httpRetry = 1
	}

	errs := make(MultiError, 0)
	for i := 0; i < httpRetry; i++ {
		response, err := http.Get(u.String())
		if err == nil {
			return response, nil
		}
		errs = append(errs, err)
	}
	return nil, errs
}

func (md *MangaDownloader) HttpGetHtml(u *url.URL) (*html.Node, error) {
	response, err := md.HttpGet(u)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	node, err := html.Parse(response.Body)
	return node, err
}

type Options struct {
	Cbz             bool
	PageDigitCount  int
	ParallelChapter int
	ParallelPage    int
}

func cleanFilename(name string) string {
	return filenameCleanReplacer.Replace(name)
}
<|endoftext|>


<commit_before>package xlog

import "io"

// globalInstance stores the global logger.
var globalInstance *Logger

// Close releases any resources held by the global logger. The logger should
// not be used again after calling this method without re-configuring it, as
// this method sets the global instance to nil.
func Close() {
	instance().Close()
	globalInstance = nil
}

// SetName sets the name of the global logger.
func SetName(name string) {
	instance().SetName(name)
}

// Append adds a file to the global logger.
func Append(file string, level Level) {
	instance().Append(file, level)
}

// MultiAppend adds one or more files to the global logger.
func MultiAppend(files []string, level Level) {
	instance().MultiAppend(files, level)
}

// AppendWriter adds a writer to the global logger.
func AppendWriter(writer io.Writer, level Level) {
	instance().AppendWriter(writer, level)
}

// MultiAppendWriter adds one or more io.Writer instances to the global logger.
func MultiAppendWriter(writers []io.Writer, level Level) {
	instance().MultiAppendWriter(writers, level)
}

// Writable returns true when global logging is enabled, and the global logger
// hasn't been closed.
func Writable() bool {
	return instance().Writable()
}

// Log writes the message to each logger appended to the global logger at given level
// or higher.
func Log(level Level, v ...interface{}) {
	instance().Log(level, v...)
}

// Logf writes the message to each logger appended to the global logger given
// level or higher.
func Logf(level Level, format string, v ...interface{}) {
	instance().Logf(level, format, v...)
}

// Debug writes to the global logger at DebugLevel.
// Arguments are handled in the manner of fmt.Print.
func Debug(v ...interface{}) {
	instance().Debug(v...)
}

// Debugf writes to the global logger at DebugLevel.
// Arguments are handled in the manner of fmt.Printf.
func Debugf(format string, v ...interface{}) {
	instance().Debugf(format, v...)
}

// Info writes to the global logger at InfoLevel.
// Arguments are handled in the manner of fmt.Print.
func Info(v ...interface{}) {
	instance().Info(v...)
}

// Infof writes to the global logger at InfoLevel.
// Arguments are handled in the manner of fmt.Printf.
func Infof(format string, v ...interface{}) {
	instance().Infof(format, v...)
}

// Notice writes to the global logger at NoticeLevel.
// Arguments are handled in the manner of fmt.Print.
func Notice(v ...interface{}) {
	instance().Notice(v...)
}

// Noticef writes to the global logger at NoticeLevel.
// Arguments are handled in the manner of fmt.Printf.
func Noticef(format string, v ...interface{}) {
	instance().Noticef(format, v...)
}

// Warning writes to the global logger at WarningLevel.
// Arguments are handled in the manner of fmt.Print.
func Warning(v ...interface{}) {
	instance().Warning(v...)
}

// Warningf writes to the global logger at WarningLevel.
// Arguments are handled in the manner of fmt.Printf.
func Warningf(format string, v ...interface{}) {
	instance().Warningf(format, v...)
}

// Error writes to the global logger at ErrorLevel.
// Arguments are handled in the manner of fmt.Print.
func Error(v ...interface{}) {
	instance().Error(v...)
}

// Errorf writes to the global logger at ErrorLevel.
// Arguments are handled in the manner of fmt.Printf.
func Errorf(format string, v ...interface{}) {
	instance().Errorf(format, v...)
}

// Critical writes to the global logger at CriticalLevel.
// Arguments are handled in the manner of fmt.Print.
func Critical(v ...interface{}) {
	instance().Critical(v...)
}

// Criticalf writes to the global logger at CriticalLevel.
// Arguments are handled in the manner of fmt.Printf.
func Criticalf(format string, v ...interface{}) {
	instance().Criticalf(format, v...)
}

// Alert writes to the global logger at AlertLevel.
// Arguments are handled in the manner of fmt.Print.
func Alert(v ...interface{}) {
	instance().Alert(v...)
}

// Alertf writes to the global logger at AlertLevel.
// Arguments are handled in the manner of fmt.Printf.
func Alertf(format string, v ...interface{}) {
	instance().Alertf(format, v...)
}

// Emergency writes to the global logger at EmergencyLevel.
// Arguments are handled in the manner of fmt.Print.
func Emergency(v ...interface{}) {
	instance().Emergency(v...)
}

// Emergencyf writes to the global logger at EmergencyLevel.
// Arguments are handled in the manner of fmt.Printf.
func Emergencyf(format string, v ...interface{}) {
	instance().Emergencyf(format, v...)
}

// instance calls panic() when the global logger has not been configured.
func instance() *Logger {
	if globalInstance == nil {
		globalInstance = NewLogger("xlog")
	}
	return globalInstance
}
<commit_msg>Changed the name of a function<commit_after>package xlog

import "io"

// globalInstance stores the global logger.
var globalInstance *Logger

// Close releases any resources held by the global logger. The logger should
// not be used again after calling this method without re-configuring it, as
// this method sets the global instance to nil.
func Close() {
	instance().Close()
	globalInstance = nil
}

// SetName sets the name of the global logger.
func SetName(name string) {
	instance().SetName(name)
}

// Append adds a file to the global logger.
func Append(file string, level Level) {
	instance().Append(file, level)
}

// MultiAppend adds one or more files to the global logger.
func MultiAppend(files []string, level Level) {
	instance().MultiAppend(files, level)
}

// AppendWriter adds a writer to the global logger.
func AppendWriter(writer io.Writer, level Level) {
	instance().AppendWriter(writer, level)
}

// MultiAppendWriters adds one or more io.Writer instances to the global logger.
func MultiAppendWriters(writers []io.Writer, level Level) {
	instance().MultiAppendWriters(writers, level)
}

// Writable returns true when global logging is enabled, and the global logger
// hasn't been closed.
func Writable() bool {
	return instance().Writable()
}

// Log writes the message to each logger appended to the global logger at given level
// or higher.
func Log(level Level, v ...interface{}) {
	instance().Log(level, v...)
}

// Logf writes the message to each logger appended to the global logger given
// level or higher.
func Logf(level Level, format string, v ...interface{}) {
	instance().Logf(level, format, v...)
}

// Debug writes to the global logger at DebugLevel.
// Arguments are handled in the manner of fmt.Print.
func Debug(v ...interface{}) {
	instance().Debug(v...)
}

// Debugf writes to the global logger at DebugLevel.
// Arguments are handled in the manner of fmt.Printf.
func Debugf(format string, v ...interface{}) {
	instance().Debugf(format, v...)
}

// Info writes to the global logger at InfoLevel.
// Arguments are handled in the manner of fmt.Print.
func Info(v ...interface{}) {
	instance().Info(v...)
}

// Infof writes to the global logger at InfoLevel.
// Arguments are handled in the manner of fmt.Printf.
func Infof(format string, v ...interface{}) {
	instance().Infof(format, v...)
}

// Notice writes to the global logger at NoticeLevel.
// Arguments are handled in the manner of fmt.Print.
func Notice(v ...interface{}) {
	instance().Notice(v...)
}

// Noticef writes to the global logger at NoticeLevel.
// Arguments are handled in the manner of fmt.Printf.
func Noticef(format string, v ...interface{}) {
	instance().Noticef(format, v...)
}

// Warning writes to the global logger at WarningLevel.
// Arguments are handled in the manner of fmt.Print.
func Warning(v ...interface{}) {
	instance().Warning(v...)
}

// Warningf writes to the global logger at WarningLevel.
// Arguments are handled in the manner of fmt.Printf.
func Warningf(format string, v ...interface{}) {
	instance().Warningf(format, v...)
}

// Error writes to the global logger at ErrorLevel.
// Arguments are handled in the manner of fmt.Print.
func Error(v ...interface{}) {
	instance().Error(v...)
}

// Errorf writes to the global logger at ErrorLevel.
// Arguments are handled in the manner of fmt.Printf.
func Errorf(format string, v ...interface{}) {
	instance().Errorf(format, v...)
}

// Critical writes to the global logger at CriticalLevel.
// Arguments are handled in the manner of fmt.Print.
func Critical(v ...interface{}) {
	instance().Critical(v...)
}

// Criticalf writes to the global logger at CriticalLevel.
// Arguments are handled in the manner of fmt.Printf.
func Criticalf(format string, v ...interface{}) {
	instance().Criticalf(format, v...)
}

// Alert writes to the global logger at AlertLevel.
// Arguments are handled in the manner of fmt.Print.
func Alert(v ...interface{}) {
	instance().Alert(v...)
}

// Alertf writes to the global logger at AlertLevel.
// Arguments are handled in the manner of fmt.Printf.
func Alertf(format string, v ...interface{}) {
	instance().Alertf(format, v...)
}

// Emergency writes to the global logger at EmergencyLevel.
// Arguments are handled in the manner of fmt.Print.
func Emergency(v ...interface{}) {
	instance().Emergency(v...)
}

// Emergencyf writes to the global logger at EmergencyLevel.
// Arguments are handled in the manner of fmt.Printf.
func Emergencyf(format string, v ...interface{}) {
	instance().Emergencyf(format, v...)
}

// instance calls panic() when the global logger has not been configured.
func instance() *Logger {
	if globalInstance == nil {
		globalInstance = NewLogger("xlog")
	}
	return globalInstance
}
<|endoftext|>


<commit_before>package godoc

import (
	"bytes"
	"go/build"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"testing"
	"text/template"

	"code.google.com/p/go.tools/godoc/vfs"
	"code.google.com/p/go.tools/godoc/vfs/mapfs"
)

// setupGoroot creates temporary directory to act as GOROOT when running tests
// that depend upon the build package.  It updates build.Default to point to the
// new GOROOT.
// It returns a function that can be called to reset build.Default and remove
// the temporary directory.
func setupGoroot(t *testing.T) (cleanup func()) {
	var stdLib = map[string]string{
		"src/pkg/fmt/fmt.go": `// Package fmt implements formatted I/O.
package fmt

type Stringer interface {
	String() string
}
`,
	}
	goroot, err := ioutil.TempDir("", "cmdline_test")
	if err != nil {
		t.Fatal(err)
	}
	origContext := build.Default
	build.Default = build.Context{
		GOROOT:   goroot,
		Compiler: "gc",
	}
	for relname, contents := range stdLib {
		name := filepath.Join(goroot, relname)
		if err := os.MkdirAll(filepath.Dir(name), 0770); err != nil {
			t.Fatal(err)
		}
		if err := ioutil.WriteFile(name, []byte(contents), 0770); err != nil {
			t.Fatal(err)
		}
	}

	return func() {
		if err := os.RemoveAll(goroot); err != nil {
			t.Log(err)
		}
		build.Default = origContext
	}
}

func TestPaths(t *testing.T) {
	cleanup := setupGoroot(t)
	defer cleanup()

	pres := &Presentation{
		pkgHandler: handlerServer{
			fsRoot: "/fsroot",
		},
	}
	fs := make(vfs.NameSpace)

	for _, tc := range []struct {
		desc   string
		path   string
		expAbs string
		expRel string
	}{
		{
			"Absolute path",
			"/foo/fmt",
			"/target",
			"/target",
		},
		{
			"Local import",
			"../foo/fmt",
			"/target",
			"/target",
		},
		{
			"Import",
			"fmt",
			"/target",
			"fmt",
		},
		{
			"Default",
			"unknownpkg",
			"/fsroot/unknownpkg",
			"unknownpkg",
		},
	} {
		abs, rel := paths(fs, pres, tc.path)
		if abs != tc.expAbs || rel != tc.expRel {
			t.Errorf("%s: paths(%q) = %s,%s; want %s,%s", tc.desc, tc.path, abs, rel, tc.expAbs, tc.expRel)
		}
	}
}

func TestMakeRx(t *testing.T) {
	for _, tc := range []struct {
		desc  string
		names []string
		exp   string
	}{
		{
			desc:  "empty string",
			names: []string{""},
			exp:   `^$`,
		},
		{
			desc:  "simple text",
			names: []string{"a"},
			exp:   `^a$`,
		},
		{
			desc:  "two words",
			names: []string{"foo", "bar"},
			exp:   `^foo$|^bar$`,
		},
		{
			desc:  "word & non-trivial",
			names: []string{"foo", `ab?c`},
			exp:   `^foo$|ab?c`,
		},
		{
			desc:  "bad regexp",
			names: []string{`(."`},
			exp:   `(."`,
		},
	} {
		expRE, expErr := regexp.Compile(tc.exp)
		if re, err := makeRx(tc.names); !reflect.DeepEqual(err, expErr) && !reflect.DeepEqual(re, expRE) {
			t.Errorf("%s: makeRx(%v) = %q,%q; want %q,%q", tc.desc, tc.names, re, err, expRE, expErr)
		}
	}
}

func TestCommandLine(t *testing.T) {
	cleanup := setupGoroot(t)
	defer cleanup()
	mfs := mapfs.New(map[string]string{
		"src/pkg/bar/bar.go": `// Package bar is an example.
package bar
`,
		"src/cmd/go/doc.go": `// The go command
package main
`,
		"src/cmd/gofmt/doc.go": `// The gofmt command
package main
`,
	})
	fs := make(vfs.NameSpace)
	fs.Bind("/", mfs, "/", vfs.BindReplace)
	c := NewCorpus(fs)
	p := &Presentation{Corpus: c}
	p.cmdHandler = handlerServer{p, c, "/cmd/", "/src/cmd"}
	p.pkgHandler = handlerServer{p, c, "/pkg/", "/src/pkg"}
	p.initFuncMap()
	p.PackageText = template.Must(template.New("PackageText").Funcs(p.FuncMap()).Parse(`{{with .PAst}}{{node $ .}}{{end}}{{with .PDoc}}{{if $.IsMain}}COMMAND {{.Doc}}{{else}}PACKAGE {{.Doc}}{{end}}{{end}}`))

	for _, tc := range []struct {
		desc string
		args []string
		exp  string
		err  bool
	}{
		{
			desc: "standard package",
			args: []string{"fmt"},
			exp:  "PACKAGE Package fmt implements formatted I/O.\n",
		},
		{
			desc: "package",
			args: []string{"bar"},
			exp:  "PACKAGE Package bar is an example.\n",
		},
		{
			desc: "source mode",
			args: []string{"src/bar"},
			exp:  "// Package bar is an example.\npackage bar\n",
		},
		{
			desc: "command",
			args: []string{"go"},
			exp:  "COMMAND The go command\n",
		},
		{
			desc: "forced command",
			args: []string{"cmd/gofmt"},
			exp:  "COMMAND The gofmt command\n",
		},
		{
			desc: "bad arg",
			args: []string{"doesnotexist"},
			err:  true,
		},
	} {
		w := new(bytes.Buffer)
		err := CommandLine(w, fs, p, tc.args)
		if got, want := w.String(), tc.exp; got != want || tc.err == (err == nil) {
			t.Errorf("%s: CommandLine(%v) = %q,%v; want %q,%v",
				tc.desc, tc.args, got, err, want, tc.err)
		}
	}
}
<commit_msg>godoc: fix cmdline_test under Windows.  Make sure the absolute path testcase starts with a volume name for windows platforms.<commit_after>package godoc

import (
	"bytes"
	"go/build"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"runtime"
	"testing"
	"text/template"

	"code.google.com/p/go.tools/godoc/vfs"
	"code.google.com/p/go.tools/godoc/vfs/mapfs"
)

// setupGoroot creates temporary directory to act as GOROOT when running tests
// that depend upon the build package.  It updates build.Default to point to the
// new GOROOT.
// It returns a function that can be called to reset build.Default and remove
// the temporary directory.
func setupGoroot(t *testing.T) (cleanup func()) {
	var stdLib = map[string]string{
		"src/pkg/fmt/fmt.go": `// Package fmt implements formatted I/O.
package fmt

type Stringer interface {
	String() string
}
`,
	}
	goroot, err := ioutil.TempDir("", "cmdline_test")
	if err != nil {
		t.Fatal(err)
	}
	origContext := build.Default
	build.Default = build.Context{
		GOROOT:   goroot,
		Compiler: "gc",
	}
	for relname, contents := range stdLib {
		name := filepath.Join(goroot, relname)
		if err := os.MkdirAll(filepath.Dir(name), 0770); err != nil {
			t.Fatal(err)
		}
		if err := ioutil.WriteFile(name, []byte(contents), 0770); err != nil {
			t.Fatal(err)
		}
	}

	return func() {
		if err := os.RemoveAll(goroot); err != nil {
			t.Log(err)
		}
		build.Default = origContext
	}
}

func TestPaths(t *testing.T) {
	cleanup := setupGoroot(t)
	defer cleanup()

	pres := &Presentation{
		pkgHandler: handlerServer{
			fsRoot: "/fsroot",
		},
	}
	fs := make(vfs.NameSpace)

	absPath := "/foo/fmt"
	if runtime.GOOS == "windows" {
		absPath = `c:\foo\fmt`
	}

	for _, tc := range []struct {
		desc   string
		path   string
		expAbs string
		expRel string
	}{
		{
			"Absolute path",
			absPath,
			"/target",
			"/target",
		},
		{
			"Local import",
			"../foo/fmt",
			"/target",
			"/target",
		},
		{
			"Import",
			"fmt",
			"/target",
			"fmt",
		},
		{
			"Default",
			"unknownpkg",
			"/fsroot/unknownpkg",
			"unknownpkg",
		},
	} {
		abs, rel := paths(fs, pres, tc.path)
		if abs != tc.expAbs || rel != tc.expRel {
			t.Errorf("%s: paths(%q) = %s,%s; want %s,%s", tc.desc, tc.path, abs, rel, tc.expAbs, tc.expRel)
		}
	}
}

func TestMakeRx(t *testing.T) {
	for _, tc := range []struct {
		desc  string
		names []string
		exp   string
	}{
		{
			desc:  "empty string",
			names: []string{""},
			exp:   `^$`,
		},
		{
			desc:  "simple text",
			names: []string{"a"},
			exp:   `^a$`,
		},
		{
			desc:  "two words",
			names: []string{"foo", "bar"},
			exp:   `^foo$|^bar$`,
		},
		{
			desc:  "word & non-trivial",
			names: []string{"foo", `ab?c`},
			exp:   `^foo$|ab?c`,
		},
		{
			desc:  "bad regexp",
			names: []string{`(."`},
			exp:   `(."`,
		},
	} {
		expRE, expErr := regexp.Compile(tc.exp)
		if re, err := makeRx(tc.names); !reflect.DeepEqual(err, expErr) && !reflect.DeepEqual(re, expRE) {
			t.Errorf("%s: makeRx(%v) = %q,%q; want %q,%q", tc.desc, tc.names, re, err, expRE, expErr)
		}
	}
}

func TestCommandLine(t *testing.T) {
	cleanup := setupGoroot(t)
	defer cleanup()
	mfs := mapfs.New(map[string]string{
		"src/pkg/bar/bar.go": `// Package bar is an example.
package bar
`,
		"src/cmd/go/doc.go": `// The go command
package main
`,
		"src/cmd/gofmt/doc.go": `// The gofmt command
package main
`,
	})
	fs := make(vfs.NameSpace)
	fs.Bind("/", mfs, "/", vfs.BindReplace)
	c := NewCorpus(fs)
	p := &Presentation{Corpus: c}
	p.cmdHandler = handlerServer{p, c, "/cmd/", "/src/cmd"}
	p.pkgHandler = handlerServer{p, c, "/pkg/", "/src/pkg"}
	p.initFuncMap()
	p.PackageText = template.Must(template.New("PackageText").Funcs(p.FuncMap()).Parse(`{{with .PAst}}{{node $ .}}{{end}}{{with .PDoc}}{{if $.IsMain}}COMMAND {{.Doc}}{{else}}PACKAGE {{.Doc}}{{end}}{{end}}`))

	for _, tc := range []struct {
		desc string
		args []string
		exp  string
		err  bool
	}{
		{
			desc: "standard package",
			args: []string{"fmt"},
			exp:  "PACKAGE Package fmt implements formatted I/O.\n",
		},
		{
			desc: "package",
			args: []string{"bar"},
			exp:  "PACKAGE Package bar is an example.\n",
		},
		{
			desc: "source mode",
			args: []string{"src/bar"},
			exp:  "// Package bar is an example.\npackage bar\n",
		},
		{
			desc: "command",
			args: []string{"go"},
			exp:  "COMMAND The go command\n",
		},
		{
			desc: "forced command",
			args: []string{"cmd/gofmt"},
			exp:  "COMMAND The gofmt command\n",
		},
		{
			desc: "bad arg",
			args: []string{"doesnotexist"},
			err:  true,
		},
	} {
		w := new(bytes.Buffer)
		err := CommandLine(w, fs, p, tc.args)
		if got, want := w.String(), tc.exp; got != want || tc.err == (err == nil) {
			t.Errorf("%s: CommandLine(%v) = %q,%v; want %q,%v",
				tc.desc, tc.args, got, err, want, tc.err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package table

import (
	"bytes"
	"github.com/osrg/gobgp/packet"
	"hash/fnv"
)

func UpdatePathAttrs2ByteAs(msg *bgp.BGPUpdate) error {
	var asAttr *bgp.PathAttributeAsPath
	idx := 0
	for i, attr := range msg.PathAttributes {
		switch attr.(type) {
		case *bgp.PathAttributeAsPath:
			asAttr = attr.(*bgp.PathAttributeAsPath)
			idx = i
		}
	}

	if asAttr == nil {
		return nil
	}

	msg.PathAttributes = cloneAttrSlice(msg.PathAttributes)
	asAttr = msg.PathAttributes[idx].(*bgp.PathAttributeAsPath)
	as4pathParam := make([]*bgp.As4PathParam, 0)
	newASparams := make([]bgp.AsPathParamInterface, len(asAttr.Value))
	for i, param := range asAttr.Value {
		asParam := param.(*bgp.As4PathParam)

		newAs := make([]uint32, 0)
		oldAs := make([]uint16, len(asParam.AS))
		for j := 0; j < len(asParam.AS); j++ {
			if asParam.AS[j] > (1<<16)-1 {
				oldAs[j] = bgp.AS_TRANS
				newAs = append(newAs, asParam.AS[j])
			} else {
				oldAs[j] = uint16(asParam.AS[j])
			}
		}

		newASparams[i] = bgp.NewAsPathParam(asParam.Type, oldAs)
		if len(newAs) > 0 {
			as4pathParam = append(as4pathParam, bgp.NewAs4PathParam(asParam.Type, newAs))
		}
	}
	msg.PathAttributes[idx] = bgp.NewPathAttributeAsPath(newASparams)
	if len(as4pathParam) > 0 {
		msg.PathAttributes = append(msg.PathAttributes, bgp.NewPathAttributeAs4Path(as4pathParam))
	}
	return nil
}

func UpdatePathAttrs4ByteAs(msg *bgp.BGPUpdate) error {
	newPathAttrs := make([]bgp.PathAttributeInterface, 0)
	var asAttr *bgp.PathAttributeAsPath
	var as4Attr *bgp.PathAttributeAs4Path

	for _, attr := range msg.PathAttributes {
		switch attr.(type) {
		case *bgp.PathAttributeAsPath:
			asAttr = attr.(*bgp.PathAttributeAsPath)
			newPathAttrs = append(newPathAttrs, attr)
		case *bgp.PathAttributeAs4Path:
			as4Attr = attr.(*bgp.PathAttributeAs4Path)
		default:
			newPathAttrs = append(newPathAttrs, attr)
		}
	}

	if asAttr == nil {
		return nil
	}

	AS := make([]uint32, 0)
	if as4Attr != nil {
		for _, p := range as4Attr.Value {
			AS = append(AS, p.AS...)
		}
		msg.PathAttributes = newPathAttrs
	}

	transIdx := 0
	for i, param := range asAttr.Value {
		asParam, y := param.(*bgp.AsPathParam)
		if !y {
			continue
		}

		newAS := make([]uint32, len(asParam.AS))
		for j := 0; j < len(asParam.AS); j++ {
			if asParam.AS[j] == bgp.AS_TRANS {
				if transIdx == len(AS) {
					//return error
				}
				newAS[j] = AS[transIdx]
				transIdx++
			} else {
				newAS[j] = uint32(asParam.AS[j])
			}
		}
		asAttr.Value[i] = bgp.NewAs4PathParam(asParam.Type, newAS)
	}
	if len(AS) != transIdx {
		//return error
	}
	return nil
}

func cloneAttrSlice(attrs []bgp.PathAttributeInterface) []bgp.PathAttributeInterface {
	clonedAttrs := make([]bgp.PathAttributeInterface, 0)
	clonedAttrs = append(clonedAttrs, attrs...)
	return clonedAttrs
}

func createUpdateMsgFromPath(path *Path, msg *bgp.BGPMessage) *bgp.BGPMessage {
	rf := path.GetRouteFamily()

	if rf == bgp.RF_IPv4_UC {
		if path.IsWithdraw {
			draw := path.GetNlri().(*bgp.WithdrawnRoute)
			if msg != nil {
				u := msg.Body.(*bgp.BGPUpdate)
				u.WithdrawnRoutes = append(u.WithdrawnRoutes, *draw)
				return nil
			} else {
				return bgp.NewBGPUpdateMessage([]bgp.WithdrawnRoute{*draw}, []bgp.PathAttributeInterface{}, []bgp.NLRInfo{})
			}
		} else {
			nlri := path.GetNlri().(*bgp.NLRInfo)
			if msg != nil {
				u := msg.Body.(*bgp.BGPUpdate)
				u.NLRI = append(u.NLRI, *nlri)
			} else {
				pathAttrs := path.GetPathAttrs()
				return bgp.NewBGPUpdateMessage([]bgp.WithdrawnRoute{}, pathAttrs, []bgp.NLRInfo{*nlri})
			}
		}
	} else {
		if path.IsWithdraw {
			if msg != nil {
				idx, _ := path.getPathAttr(bgp.BGP_ATTR_TYPE_MP_UNREACH_NLRI)
				u := msg.Body.(*bgp.BGPUpdate)
				unreach := u.PathAttributes[idx].(*bgp.PathAttributeMpUnreachNLRI)
				unreach.Value = append(unreach.Value, path.GetNlri())
			} else {
				clonedAttrs := cloneAttrSlice(path.GetPathAttrs())
				idx, attr := path.getPathAttr(bgp.BGP_ATTR_TYPE_MP_REACH_NLRI)
				reach := attr.(*bgp.PathAttributeMpReachNLRI)
				clonedAttrs[idx] = bgp.NewPathAttributeMpUnreachNLRI(reach.Value)
				return bgp.NewBGPUpdateMessage([]bgp.WithdrawnRoute{}, clonedAttrs, []bgp.NLRInfo{})
			}
		} else {
			if msg != nil {
				idx, _ := path.getPathAttr(bgp.BGP_ATTR_TYPE_MP_REACH_NLRI)
				u := msg.Body.(*bgp.BGPUpdate)
				reachAttr := u.PathAttributes[idx].(*bgp.PathAttributeMpReachNLRI)
				u.PathAttributes[idx] = bgp.NewPathAttributeMpReachNLRI(reachAttr.Nexthop.String(),
					append(reachAttr.Value, path.GetNlri()))
			} else {
				// we don't need to clone here but we
				// might merge path to this message in
				// the future so let's clone anyway.
				clonedAttrs := cloneAttrSlice(path.GetPathAttrs())
				return bgp.NewBGPUpdateMessage([]bgp.WithdrawnRoute{}, clonedAttrs, []bgp.NLRInfo{})
			}
		}
	}
	return nil
}

type bucket struct {
	attrs []byte
	paths []*Path
}

func CreateUpdateMsgFromPaths(pathList []*Path) []*bgp.BGPMessage {
	var msgs []*bgp.BGPMessage

	pathByAttrs := make(map[uint32][]*bucket)

	for _, path := range pathList {
		y := func(p *Path) bool {
			if p.GetRouteFamily() != bgp.RF_IPv4_UC {
				return false
			}
			if p.IsWithdraw {
				return false
			}
			return true
		}(path)

		if y {
			key, attrs := func(p *Path) (uint32, []byte) {
				h := fnv.New32()
				total := bytes.NewBuffer(make([]byte, 0))
				for _, v := range p.GetPathAttrs() {
					b, _ := v.Serialize()
					total.Write(b)
				}
				h.Write(total.Bytes())
				return h.Sum32(), total.Bytes()
			}(path)

			if bl, y := pathByAttrs[key]; y {
				found := false
				for _, b := range bl {
					if bytes.Compare(b.attrs, attrs) == 0 {
						b.paths = append(b.paths, path)
						found = true
						break
					}
				}
				if found == false {
					nb := &bucket{
						attrs: attrs,
						paths: []*Path{path},
					}
					pathByAttrs[key] = append(pathByAttrs[key], nb)
				}
			} else {
				nb := &bucket{
					attrs: attrs,
					paths: []*Path{path},
				}
				pathByAttrs[key] = []*bucket{nb}
			}
		} else {
			msg := createUpdateMsgFromPath(path, nil)
			msgs = append(msgs, msg)
		}
	}

	for _, bList := range pathByAttrs {
		for _, b := range bList {
			var msg *bgp.BGPMessage
			for i, path := range b.paths {
				if i == 0 {
					msg = createUpdateMsgFromPath(path, nil)
					msgs = append(msgs, msg)
				} else {
					msgLen := func(u *bgp.BGPUpdate) int {
						attrsLen := 0
						for _, a := range u.PathAttributes {
							attrsLen += a.Len()
						}
						// Header + Update (WithdrawnRoutesLen +
						// TotalPathAttributeLen + attributes + maxlen of
						// NLRI). Note that we try to add one NLRI.
						return 19 + 2 + 2 + attrsLen + (len(u.NLRI)+1)*5
					}(msg.Body.(*bgp.BGPUpdate))

					if msgLen+32 > bgp.BGP_MAX_MESSAGE_LENGTH {
						// don't marge
						msg = createUpdateMsgFromPath(path, nil)
						msgs = append(msgs, msg)
					} else {
						createUpdateMsgFromPath(path, msg)
					}
				}
			}
		}
	}

	return msgs
}
<commit_msg>table: disable merging NLRIs if we don't have many NLRIs<commit_after>// Copyright (C) 2014 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package table

import (
	"bytes"
	"github.com/osrg/gobgp/packet"
	"hash/fnv"
)

func UpdatePathAttrs2ByteAs(msg *bgp.BGPUpdate) error {
	var asAttr *bgp.PathAttributeAsPath
	idx := 0
	for i, attr := range msg.PathAttributes {
		switch attr.(type) {
		case *bgp.PathAttributeAsPath:
			asAttr = attr.(*bgp.PathAttributeAsPath)
			idx = i
		}
	}

	if asAttr == nil {
		return nil
	}

	msg.PathAttributes = cloneAttrSlice(msg.PathAttributes)
	asAttr = msg.PathAttributes[idx].(*bgp.PathAttributeAsPath)
	as4pathParam := make([]*bgp.As4PathParam, 0)
	newASparams := make([]bgp.AsPathParamInterface, len(asAttr.Value))
	for i, param := range asAttr.Value {
		asParam := param.(*bgp.As4PathParam)

		newAs := make([]uint32, 0)
		oldAs := make([]uint16, len(asParam.AS))
		for j := 0; j < len(asParam.AS); j++ {
			if asParam.AS[j] > (1<<16)-1 {
				oldAs[j] = bgp.AS_TRANS
				newAs = append(newAs, asParam.AS[j])
			} else {
				oldAs[j] = uint16(asParam.AS[j])
			}
		}

		newASparams[i] = bgp.NewAsPathParam(asParam.Type, oldAs)
		if len(newAs) > 0 {
			as4pathParam = append(as4pathParam, bgp.NewAs4PathParam(asParam.Type, newAs))
		}
	}
	msg.PathAttributes[idx] = bgp.NewPathAttributeAsPath(newASparams)
	if len(as4pathParam) > 0 {
		msg.PathAttributes = append(msg.PathAttributes, bgp.NewPathAttributeAs4Path(as4pathParam))
	}
	return nil
}

func UpdatePathAttrs4ByteAs(msg *bgp.BGPUpdate) error {
	newPathAttrs := make([]bgp.PathAttributeInterface, 0)
	var asAttr *bgp.PathAttributeAsPath
	var as4Attr *bgp.PathAttributeAs4Path

	for _, attr := range msg.PathAttributes {
		switch attr.(type) {
		case *bgp.PathAttributeAsPath:
			asAttr = attr.(*bgp.PathAttributeAsPath)
			newPathAttrs = append(newPathAttrs, attr)
		case *bgp.PathAttributeAs4Path:
			as4Attr = attr.(*bgp.PathAttributeAs4Path)
		default:
			newPathAttrs = append(newPathAttrs, attr)
		}
	}

	if asAttr == nil {
		return nil
	}

	AS := make([]uint32, 0)
	if as4Attr != nil {
		for _, p := range as4Attr.Value {
			AS = append(AS, p.AS...)
		}
		msg.PathAttributes = newPathAttrs
	}

	transIdx := 0
	for i, param := range asAttr.Value {
		asParam, y := param.(*bgp.AsPathParam)
		if !y {
			continue
		}

		newAS := make([]uint32, len(asParam.AS))
		for j := 0; j < len(asParam.AS); j++ {
			if asParam.AS[j] == bgp.AS_TRANS {
				if transIdx == len(AS) {
					//return error
				}
				newAS[j] = AS[transIdx]
				transIdx++
			} else {
				newAS[j] = uint32(asParam.AS[j])
			}
		}
		asAttr.Value[i] = bgp.NewAs4PathParam(asParam.Type, newAS)
	}
	if len(AS) != transIdx {
		//return error
	}
	return nil
}

func cloneAttrSlice(attrs []bgp.PathAttributeInterface) []bgp.PathAttributeInterface {
	clonedAttrs := make([]bgp.PathAttributeInterface, 0)
	clonedAttrs = append(clonedAttrs, attrs...)
	return clonedAttrs
}

func createUpdateMsgFromPath(path *Path, msg *bgp.BGPMessage) *bgp.BGPMessage {
	rf := path.GetRouteFamily()

	if rf == bgp.RF_IPv4_UC {
		if path.IsWithdraw {
			draw := path.GetNlri().(*bgp.WithdrawnRoute)
			if msg != nil {
				u := msg.Body.(*bgp.BGPUpdate)
				u.WithdrawnRoutes = append(u.WithdrawnRoutes, *draw)
				return nil
			} else {
				return bgp.NewBGPUpdateMessage([]bgp.WithdrawnRoute{*draw}, []bgp.PathAttributeInterface{}, []bgp.NLRInfo{})
			}
		} else {
			nlri := path.GetNlri().(*bgp.NLRInfo)
			if msg != nil {
				u := msg.Body.(*bgp.BGPUpdate)
				u.NLRI = append(u.NLRI, *nlri)
			} else {
				pathAttrs := path.GetPathAttrs()
				return bgp.NewBGPUpdateMessage([]bgp.WithdrawnRoute{}, pathAttrs, []bgp.NLRInfo{*nlri})
			}
		}
	} else {
		if path.IsWithdraw {
			if msg != nil {
				idx, _ := path.getPathAttr(bgp.BGP_ATTR_TYPE_MP_UNREACH_NLRI)
				u := msg.Body.(*bgp.BGPUpdate)
				unreach := u.PathAttributes[idx].(*bgp.PathAttributeMpUnreachNLRI)
				unreach.Value = append(unreach.Value, path.GetNlri())
			} else {
				clonedAttrs := cloneAttrSlice(path.GetPathAttrs())
				idx, attr := path.getPathAttr(bgp.BGP_ATTR_TYPE_MP_REACH_NLRI)
				reach := attr.(*bgp.PathAttributeMpReachNLRI)
				clonedAttrs[idx] = bgp.NewPathAttributeMpUnreachNLRI(reach.Value)
				return bgp.NewBGPUpdateMessage([]bgp.WithdrawnRoute{}, clonedAttrs, []bgp.NLRInfo{})
			}
		} else {
			if msg != nil {
				idx, _ := path.getPathAttr(bgp.BGP_ATTR_TYPE_MP_REACH_NLRI)
				u := msg.Body.(*bgp.BGPUpdate)
				reachAttr := u.PathAttributes[idx].(*bgp.PathAttributeMpReachNLRI)
				u.PathAttributes[idx] = bgp.NewPathAttributeMpReachNLRI(reachAttr.Nexthop.String(),
					append(reachAttr.Value, path.GetNlri()))
			} else {
				// we don't need to clone here but we
				// might merge path to this message in
				// the future so let's clone anyway.
				clonedAttrs := cloneAttrSlice(path.GetPathAttrs())
				return bgp.NewBGPUpdateMessage([]bgp.WithdrawnRoute{}, clonedAttrs, []bgp.NLRInfo{})
			}
		}
	}
	return nil
}

type bucket struct {
	attrs []byte
	paths []*Path
}

func CreateUpdateMsgFromPaths(pathList []*Path) []*bgp.BGPMessage {
	var msgs []*bgp.BGPMessage

	pathByAttrs := make(map[uint32][]*bucket)
	pathLen := len(pathList)
	for _, path := range pathList {
		y := func(p *Path) bool {
			// the merging logic makes gobgpd slower so if
			// paths are not many, let's avoid mering.
			if pathLen < 1024 {
				return false
			}
			if p.GetRouteFamily() != bgp.RF_IPv4_UC {
				return false
			}
			if p.IsWithdraw {
				return false
			}
			return true
		}(path)

		if y {
			key, attrs := func(p *Path) (uint32, []byte) {
				h := fnv.New32()
				total := bytes.NewBuffer(make([]byte, 0))
				for _, v := range p.GetPathAttrs() {
					b, _ := v.Serialize()
					total.Write(b)
				}
				h.Write(total.Bytes())
				return h.Sum32(), total.Bytes()
			}(path)

			if bl, y := pathByAttrs[key]; y {
				found := false
				for _, b := range bl {
					if bytes.Compare(b.attrs, attrs) == 0 {
						b.paths = append(b.paths, path)
						found = true
						break
					}
				}
				if found == false {
					nb := &bucket{
						attrs: attrs,
						paths: []*Path{path},
					}
					pathByAttrs[key] = append(pathByAttrs[key], nb)
				}
			} else {
				nb := &bucket{
					attrs: attrs,
					paths: []*Path{path},
				}
				pathByAttrs[key] = []*bucket{nb}
			}
		} else {
			msg := createUpdateMsgFromPath(path, nil)
			msgs = append(msgs, msg)
		}
	}

	for _, bList := range pathByAttrs {
		for _, b := range bList {
			var msg *bgp.BGPMessage
			for i, path := range b.paths {
				if i == 0 {
					msg = createUpdateMsgFromPath(path, nil)
					msgs = append(msgs, msg)
				} else {
					msgLen := func(u *bgp.BGPUpdate) int {
						attrsLen := 0
						for _, a := range u.PathAttributes {
							attrsLen += a.Len()
						}
						// Header + Update (WithdrawnRoutesLen +
						// TotalPathAttributeLen + attributes + maxlen of
						// NLRI). Note that we try to add one NLRI.
						return 19 + 2 + 2 + attrsLen + (len(u.NLRI)+1)*5
					}(msg.Body.(*bgp.BGPUpdate))

					if msgLen+32 > bgp.BGP_MAX_MESSAGE_LENGTH {
						// don't marge
						msg = createUpdateMsgFromPath(path, nil)
						msgs = append(msgs, msg)
					} else {
						createUpdateMsgFromPath(path, msg)
					}
				}
			}
		}
	}

	return msgs
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
)

func main() {

	fullDate := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
	getData(fullDate)
	//parseFile("data-2015-09-12-0.gz")
}

func parseFile(fName string) {
	// https://groups.google.com/forum/#!topic/golang-nuts/GjIkryuCyAY
	fileOS, err := os.Open(fName)
	if err != nil {
		fmt.Fprintf(os.Stderr, "rollupParse: Can't open %s: error: %s\n", fName, err)
		os.Exit(1)
	}

	fileGzip, err := gzip.NewReader(fileOS)
	if err != nil {
		fmt.Printf("The file %v is not in gzip format.\n", fName)
		os.Exit(1)
	}

	fileRead := bufio.NewReader(fileGzip)
	i := 0
	for {
		line, err := fileRead.ReadString('\n')
		if err != nil {
			break
		}
		fmt.Printf("Line %v: %s", i, line)
		i++
	}
}

func getData(fullDate string) {

	urls := makeUrlArray(fullDate)

	for i, value := range urls {

		fmt.Println("fetching url", value)

		resp, archiveErr := http.Get(value)

		if resp != nil {
			defer resp.Body.Close()
		}

		if archiveErr != nil {
			handleError("Error getting github archive", archiveErr)
		}

		contents, readErr := ioutil.ReadAll(resp.Body)

		if readErr != nil {
			handleError("Error converting response", readErr)
		}

		fname := makeFileName(fullDate, i)

		fileErr := ioutil.WriteFile(fname, contents, 0644)

		if fileErr != nil {
			handleError("Error writing response to file", fileErr)
		}
	}
}

func makeUrlArray(fullDate string) [24]string {

	baseUrl := makeUrlBase(fullDate)
	urlEnd := ".json.gz"

	var urls [24]string

	for i := 0; i < 6; i++ {

		var buffer bytes.Buffer
		buffer.WriteString(baseUrl)
		buffer.WriteString("-")
		buffer.WriteString(strconv.Itoa(i))
		buffer.WriteString(urlEnd)
		url := buffer.String()

		urls[i] = url
	}

	return urls
}

func makeUrlBase(fullDate string) string {
	split := strings.Split(fullDate, "-")

	var buffer bytes.Buffer
	buffer.WriteString("http://data.githubarchive.org/")
	buffer.WriteString(split[0]) //year
	buffer.WriteString("-")
	buffer.WriteString(split[1]) //month
	buffer.WriteString("-")
	buffer.WriteString(split[2]) //day

	return buffer.String()
}

func makeFileName(fullDate string, i int) string {
	var buffer bytes.Buffer
	buffer.WriteString("data-")
	buffer.WriteString(fullDate)
	buffer.WriteString("-")
	buffer.WriteString(strconv.Itoa(i))
	buffer.WriteString(".gz")

	return buffer.String()

}

func handleError(message string, err error) {
	fmt.Println(message, err)
	os.Exit(1)
}
<commit_msg>download and parse files sequentially<commit_after>package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

func main() {

	fullDate := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
	getData(fullDate)
}

func parseFile(fName string) {
	// https://groups.google.com/forum/#!topic/golang-nuts/GjIkryuCyAY
	fileOS, err := os.Open(fName)
	if err != nil {
		fmt.Fprintf(os.Stderr, "rollupParse: Can't open %s: error: %s\n", fName, err)
		os.Exit(1)
	}

	fileGzip, err := gzip.NewReader(fileOS)
	if err != nil {
		fmt.Printf("The file %v is not in gzip format.\n", fName)
		os.Exit(1)
	}

	fileRead := bufio.NewReader(fileGzip)
	i := 0
	for {
		line, err := fileRead.ReadString('\n')
		if err != nil {
			break
		}
		fmt.Printf("Line %v: %s", i, line)
		i++
	}
}

func getData(fullDate string) {

	urls := makeUrlArray(fullDate)

	for i, value := range urls {

		fmt.Println("fetching url", value)

		resp, archiveErr := http.Get(value)

		if resp != nil {
			defer resp.Body.Close()
		}

		if archiveErr != nil {
			handleError("Error getting github archive", archiveErr)
		}

		contents, readErr := ioutil.ReadAll(resp.Body)

		if readErr != nil {
			handleError("Error converting response", readErr)
		}

		fname := makeFileName(fullDate, i)

		fileErr := ioutil.WriteFile(fname, contents, 0644)

		if fileErr != nil {
			handleError("Error writing response to file", fileErr)
		}

		parseFile(fname)

	}
}

func makeUrlArray(fullDate string) [24]string {

	baseUrl := makeUrlBase(fullDate)
	urlEnd := ".json.gz"

	var urls [24]string

	for i := 0; i < 6; i++ {

		var buffer bytes.Buffer
		buffer.WriteString(baseUrl)
		buffer.WriteString("-")
		buffer.WriteString(strconv.Itoa(i))
		buffer.WriteString(urlEnd)
		url := buffer.String()

		urls[i] = url
	}

	return urls
}

func makeUrlBase(fullDate string) string {
	split := strings.Split(fullDate, "-")

	var buffer bytes.Buffer
	buffer.WriteString("http://data.githubarchive.org/")
	buffer.WriteString(split[0]) //year
	buffer.WriteString("-")
	buffer.WriteString(split[1]) //month
	buffer.WriteString("-")
	buffer.WriteString(split[2]) //day

	return buffer.String()
}

func makeFileName(fullDate string, i int) string {
	var buffer bytes.Buffer
	buffer.WriteString("data-")
	buffer.WriteString(fullDate)
	buffer.WriteString("-")
	buffer.WriteString(strconv.Itoa(i))
	buffer.WriteString(".gz")

	return buffer.String()

}

func handleError(message string, err error) {
	fmt.Println(message, err)
	os.Exit(1)
}
<|endoftext|>


<commit_before>package main
// Author: Robert B Frangioso

import (
	"path/filepath"
	"os"
	"flag"
	"fmt"
	"io/ioutil"
	"sync"
	"os/exec"
	"bytes"
	"strconv"
	"strings"
)

type msg_type int

const (
	OUTPUT msg_type = 1 + iota
	ERROR
	CLOSE
)

type output_msg struct {
	mtype	msg_type
	buffer	bytes.Buffer
}

func find(root string, wg *sync.WaitGroup, flags []string, args []string, output chan output_msg) {

	defer wg.Done()
	var cmd_out, cmd_err bytes.Buffer
	var msg output_msg

	findstr := append(append(append([]string{}, flags... ), []string{root}... ), args... )
	cmd := exec.Command("find", findstr... )
	cmd.Stdout = &cmd_out
	cmd.Stderr = &cmd_err
	err := cmd.Run()

	if err == nil {
		msg.mtype = OUTPUT
		msg.buffer = cmd_out
		output <- msg
	} else {
		msg.mtype = ERROR
		msg.buffer = cmd_err
		output <- msg
	}

	return
}

func aggregator(wg *sync.WaitGroup, input chan output_msg) {

	defer wg.Done()
	var msg output_msg

	for true {
		msg = <- input
		switch msg.mtype {
		case CLOSE:
			return
		case OUTPUT:
			if msg.buffer.Len() > 0 {
				fmt.Printf("%s", msg.buffer.String())
			}
		case ERROR:
			if msg.buffer.Len() > 0 {
				fmt.Fprintf(os.Stderr, "%s", msg.buffer.String())
			}
		default:
		}
	}

}

func parseflags() []string {

	osx_find_flags := []string{"L", "H", "P", "E", "X", "d", "s", "x", "f"}
	set_flags := []string{}

	for f := range osx_find_flags {
		flag.Bool(osx_find_flags[f], false, "bool")
	}

	flag.Parse()
	for f := range osx_find_flags {
		flag_p := flag.Lookup(osx_find_flags[f])
		val, err := strconv.ParseBool(flag_p.Value.String())
		if err == nil && val == true {
			set_flags = append(set_flags, "-"+flag_p.Name)
		}
	}

	return set_flags
}

func parseargs(args []string) ([]string, []string) {

	var i int

	for i = range args {
		if strings.HasPrefix(args[i], "-")  { break }
		i++
	}

	rootdirs := append([]string{}, args[:i]... )
	options := append([]string{}, args[i:]... )
	return rootdirs, options
}

func gofind_usage() {
        fmt.Fprintf(os.Stderr, "Usage: gofind [find-flags] rootsearchdir[...] [find-options]\n(osx only find-flags atm)\n")
}

func main() {

	flag.Usage = gofind_usage
	var wg, wga sync.WaitGroup
	msg_channel := make(chan output_msg)

	set_flags := parseflags()
	argslice := flag.Args()
	basedirs := []string{}
	rootdirs, options := parseargs(argslice)

	for r := range rootdirs {
		dirs, direrr := ioutil.ReadDir(rootdirs[r])
		if(direrr != nil) {
			gofind_usage()
			//	fmt.Printf("Usage: gofind [find-flags] rootsearchdir[...] [find-options] \n")
			return
		}
		for dirindex := range dirs {
			if dirs[dirindex].IsDir() {
				basedirs = append(basedirs, filepath.Join(rootdirs[r], dirs[dirindex].Name()))
			}
		}
		shallowfind := append(append([]string{},[]string{"-maxdepth", "1"}... ), options... )
		wg.Add(1)
		go find(rootdirs[r], &wg, set_flags, shallowfind, msg_channel)
	}

	for  dir := range basedirs {
		wg.Add(1)
		go find(basedirs[dir], &wg, set_flags, options, msg_channel)
	}

	wga.Add(1)
	go aggregator(&wga, msg_channel)
	wg.Wait()

	msg_channel <- output_msg{CLOSE, bytes.Buffer{}}
	wga.Wait()
}

<commit_msg>add support for passing distro dependant find flags<commit_after>package main
// Author: Robert B Frangioso

import (
	"path/filepath"
	"os"
	"flag"
	"fmt"
	"io/ioutil"
	"sync"
	"os/exec"
	"bytes"
	"strconv"
	"strings"
	"runtime"
)

type msg_type int

const (
	OUTPUT msg_type = 1 + iota
	ERROR
	CLOSE
)

type output_msg struct {
	mtype	msg_type
	buffer	bytes.Buffer
}

func find(root string, wg *sync.WaitGroup, flags []string, args []string, output chan output_msg) {

	defer wg.Done()
	var cmd_out, cmd_err bytes.Buffer
	var msg output_msg

	findstr := append(append(append([]string{}, flags... ), []string{root}... ), args... )
	cmd := exec.Command("find", findstr... )
	cmd.Stdout = &cmd_out
	cmd.Stderr = &cmd_err
	err := cmd.Run()

	if err == nil {
		msg.mtype = OUTPUT
		msg.buffer = cmd_out
		output <- msg
	} else {
		msg.mtype = ERROR
		msg.buffer = cmd_err
		output <- msg
	}

	return
}

func aggregator(wg *sync.WaitGroup, input chan output_msg) {

	defer wg.Done()
	var msg output_msg

	for true {
		msg = <- input
		switch msg.mtype {
		case CLOSE:
			return
		case OUTPUT:
			if msg.buffer.Len() > 0 {
				fmt.Printf("%s", msg.buffer.String())
			}
		case ERROR:
			if msg.buffer.Len() > 0 {
				fmt.Fprintf(os.Stderr, "%s", msg.buffer.String())
			}
		default:
		}
	}

}

func getosfindflags() []string {
	var flags []string

	switch runtime.GOOS {
	case "freebsd":
	case "darwin":
		// osx find derived from freebsd and using same flags
		flags = []string{"L", "H", "P", "E", "X", "d", "s", "x", "f"}
	case "linux":
		flags = []string{"L", "H", "P", "D", "O"}
	case "windows":
		// assuming gnu find for windows
		flags = []string{"L", "H", "P", "D", "O"}
	case "netbsd":
		flags = []string{"L", "H", "P", "E", "X", "d", "s", "x", "f", "h"}
	case "openbsd":
		flags = []string{"d","H", "h", "L", "X", "x"}
	default:
		// assume freebsd variant
		flags = []string{"L", "H", "P", "E", "X", "d", "s", "x", "f"}
	}

	return flags
}

func parseflags() []string {

	os_find_flags := getosfindflags()
	set_flags := []string{}

	for f := range os_find_flags {
		flag.Bool(os_find_flags[f], false, "bool")
	}

	flag.Parse()
	for f := range os_find_flags {
		flag_p := flag.Lookup(os_find_flags[f])
		val, err := strconv.ParseBool(flag_p.Value.String())
		if err == nil && val == true {
			set_flags = append(set_flags, "-"+flag_p.Name)
		}
	}

	return set_flags
}

func parseargs(args []string) ([]string, []string) {

	var i int

	for i = range args {
		if strings.HasPrefix(args[i], "-")  { break }
		i++
	}

	rootdirs := append([]string{}, args[:i]... )
	options := append([]string{}, args[i:]... )
	return rootdirs, options
}

func gofind_usage() {
        fmt.Fprintf(os.Stderr, "Usage: gofind [find-flags] rootsearchdir[...] [find-options]\n(O & D find-flags for gnu find not supported atm)\n")
}

func main() {

	flag.Usage = gofind_usage
	var wg, wga sync.WaitGroup
	msg_channel := make(chan output_msg)

	set_flags := parseflags()
	argslice := flag.Args()
	basedirs := []string{}
	rootdirs, options := parseargs(argslice)

	for r := range rootdirs {
		dirs, direrr := ioutil.ReadDir(rootdirs[r])
		if(direrr != nil) {
			gofind_usage()
			return
		}
		for dirindex := range dirs {
			if dirs[dirindex].IsDir() {
				basedirs = append(basedirs, filepath.Join(rootdirs[r], dirs[dirindex].Name()))
			}
		}
		shallowfind := append(append([]string{},[]string{"-maxdepth", "1"}... ), options... )
		wg.Add(1)
		go find(rootdirs[r], &wg, set_flags, shallowfind, msg_channel)
	}

	for  dir := range basedirs {
		wg.Add(1)
		go find(basedirs[dir], &wg, set_flags, options, msg_channel)
	}

	wga.Add(1)
	go aggregator(&wga, msg_channel)
	wg.Wait()

	msg_channel <- output_msg{CLOSE, bytes.Buffer{}}
	wga.Wait()
}

<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"os/exec"
	"runtime"
	"strings"

	"github.com/codegangsta/cli"
)

func main() {

	app := cli.NewApp()
	app.Name = "google"
	app.Usage = "Quick Search on google"
	app.Action = func(c *cli.Context) {
		if len(c.Args()) == 0 {
			println("give a search query")
		} else {
			s := strings.Join(c.Args(), "+")
			println("let me google", s)
			cmd := new(exec.Cmd)
			switch runtime.GOOS {
			case "linux":
				cmd = exec.Command("xdg-open", "https://google.com/#q="+s)
			case "windows":
				cmd = exec.Command("start", "https://google.com/#q="+s)
			case "darwin":
				cmd = exec.Command("open", "https://google.com/#q="+s)
			}
			err := cmd.Start()
			if err != nil {
				log.Fatal(err)
			}
		}
	}
	app.Run(os.Args)
}
<commit_msg>Update<commit_after>package main

import (
	"log"
	"os"
	"os/exec"
	"runtime"
	"strings"

	"github.com/codegangsta/cli"
)

func main() {
	app := cli.NewApp()
	app.Name = "google"
	app.Usage = "Quick Search from terminal on google"
	app.Action = func(c *cli.Context) {
		if len(c.Args()) == 0 {
			println("give a search query, e.g. \"google hello world\" ")
		} else {
			s := strings.Join(c.Args(), "+")
			println("let me google", s)

			cmd := new(exec.Cmd)

			switch runtime.GOOS {

			case "linux":
				cmd = exec.Command("xdg-open", "https://google.com/#q="+s)
			case "windows":
				cmd = exec.Command("start", "https://google.com/#q="+s)
			case "darwin":
				cmd = exec.Command("open", "https://google.com/#q="+s)
			default:
				cmd = exec.Command("xdg-open", "https://google.com/#q="+s)
				//TODO: Add support for more Operating Systems
			}

			err := cmd.Start()
			if err != nil {
				log.Fatal(err)
			}
		}
	}
	app.Run(os.Args)
}
<|endoftext|>