<commit_before>/*
	rulehuntersrv - A server to find rules in data based on user specified goals
	Copyright (C) 2016 vLife Systems Ltd <http://vlifesystems.com>

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program; see the file COPYING.  If not, see
	<http://www.gnu.org/licenses/>.
*/

package main

import (
	"fmt"
	"github.com/kardianos/service"
	"github.com/vlifesystems/rulehuntersrv/config"
	"github.com/vlifesystems/rulehuntersrv/experiment"
	"github.com/vlifesystems/rulehuntersrv/logger"
	"github.com/vlifesystems/rulehuntersrv/progress"
	"github.com/vlifesystems/rulehuntersrv/quitter"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"
)

type program struct {
	config          *config.Config
	cmdFlags        cmdFlags
	progressMonitor *progress.ProgressMonitor
	logger          logger.Logger
	quitter         *quitter.Quitter
}

func newProgram(
	c *config.Config,
	p *progress.ProgressMonitor,
	l logger.Logger,
	q *quitter.Quitter,
) *program {
	return &program{
		config:          c,
		progressMonitor: p,
		quitter:         q,
		logger:          l,
	}
}

func (p *program) Start(s service.Service) error {
	go p.run()
	return nil
}

// Returns number of files processed and ifLoggedError
func (p *program) ProcessDir() (int, bool) {
	experimentFilenames, err := p.getExperimentFilenames()
	if err != nil {
		p.logger.Error(err.Error())
		return 0, true
	}
	for _, experimentFilename := range experimentFilenames {
		if err := p.progressMonitor.AddExperiment(experimentFilename); err != nil {
			p.logger.Error(err.Error())
			return 0, true
		}
	}

	numProcessed := 0
	ifLoggedError := false
	for _, experimentFilename := range experimentFilenames {
		err := experiment.Process(
			experimentFilename,
			p.config,
			p.logger,
			p.progressMonitor,
		)
		if err != nil {
			msg := fmt.Sprintf("Failed processing experiment: %s - %s",
				experimentFilename, err)
			p.logger.Error(msg)
			ifLoggedError = true
		}
	}
	return numProcessed, ifLoggedError
}

func (p *program) run() {
	sleepInSeconds := time.Duration(2)
	logWaitingForExperiments := true
	p.quitter.Add()
	defer p.quitter.Done()

	for !p.quitter.ShouldQuit() {
		if logWaitingForExperiments {
			logWaitingForExperiments = false
			p.logger.Info("Waiting for experiments to process")
		}

		if n, _ := p.ProcessDir(); n >= 1 {
			logWaitingForExperiments = true
		}

		// Sleeping prevents 'excessive' cpu use and disk access
		time.Sleep(sleepInSeconds * time.Second)
	}
}

func (p *program) getExperimentFilenames() ([]string, error) {
	experimentFilenames := make([]string, 0)
	files, err := ioutil.ReadDir(p.config.ExperimentsDir)
	if err != nil {
		return experimentFilenames, err
	}

	for _, file := range files {
		ext := filepath.Ext(file.Name())
		if !file.IsDir() && (ext == ".json" || ext == ".yaml") {
			experimentFilenames = append(experimentFilenames, file.Name())
		}
	}
	return experimentFilenames, nil
}

func (p *program) moveExperimentToProcessed(experimentFilename string) error {
	experimentFullFilename :=
		filepath.Join(p.config.ExperimentsDir, experimentFilename)
	experimentProcessedFullFilename :=
		filepath.Join(p.config.ExperimentsDir, "processed", experimentFilename)
	return os.Rename(experimentFullFilename, experimentProcessedFullFilename)
}

func (p *program) Stop(s service.Service) error {
	p.quitter.Quit()
	return nil
}
<commit_msg>Remove redundant moveExperimentToProcessed()<commit_after>/*
	rulehuntersrv - A server to find rules in data based on user specified goals
	Copyright (C) 2016 vLife Systems Ltd <http://vlifesystems.com>

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program; see the file COPYING.  If not, see
	<http://www.gnu.org/licenses/>.
*/

package main

import (
	"fmt"
	"github.com/kardianos/service"
	"github.com/vlifesystems/rulehuntersrv/config"
	"github.com/vlifesystems/rulehuntersrv/experiment"
	"github.com/vlifesystems/rulehuntersrv/logger"
	"github.com/vlifesystems/rulehuntersrv/progress"
	"github.com/vlifesystems/rulehuntersrv/quitter"
	"io/ioutil"
	"path/filepath"
	"time"
)

type program struct {
	config          *config.Config
	cmdFlags        cmdFlags
	progressMonitor *progress.ProgressMonitor
	logger          logger.Logger
	quitter         *quitter.Quitter
}

func newProgram(
	c *config.Config,
	p *progress.ProgressMonitor,
	l logger.Logger,
	q *quitter.Quitter,
) *program {
	return &program{
		config:          c,
		progressMonitor: p,
		quitter:         q,
		logger:          l,
	}
}

func (p *program) Start(s service.Service) error {
	go p.run()
	return nil
}

// Returns number of files processed and ifLoggedError
func (p *program) ProcessDir() (int, bool) {
	experimentFilenames, err := p.getExperimentFilenames()
	if err != nil {
		p.logger.Error(err.Error())
		return 0, true
	}
	for _, experimentFilename := range experimentFilenames {
		if err := p.progressMonitor.AddExperiment(experimentFilename); err != nil {
			p.logger.Error(err.Error())
			return 0, true
		}
	}

	numProcessed := 0
	ifLoggedError := false
	for _, experimentFilename := range experimentFilenames {
		err := experiment.Process(
			experimentFilename,
			p.config,
			p.logger,
			p.progressMonitor,
		)
		if err != nil {
			msg := fmt.Sprintf("Failed processing experiment: %s - %s",
				experimentFilename, err)
			p.logger.Error(msg)
			ifLoggedError = true
		}
	}
	return numProcessed, ifLoggedError
}

func (p *program) run() {
	sleepInSeconds := time.Duration(2)
	logWaitingForExperiments := true
	p.quitter.Add()
	defer p.quitter.Done()

	for !p.quitter.ShouldQuit() {
		if logWaitingForExperiments {
			logWaitingForExperiments = false
			p.logger.Info("Waiting for experiments to process")
		}

		if n, _ := p.ProcessDir(); n >= 1 {
			logWaitingForExperiments = true
		}

		// Sleeping prevents 'excessive' cpu use and disk access
		time.Sleep(sleepInSeconds * time.Second)
	}
}

func (p *program) getExperimentFilenames() ([]string, error) {
	experimentFilenames := make([]string, 0)
	files, err := ioutil.ReadDir(p.config.ExperimentsDir)
	if err != nil {
		return experimentFilenames, err
	}

	for _, file := range files {
		ext := filepath.Ext(file.Name())
		if !file.IsDir() && (ext == ".json" || ext == ".yaml") {
			experimentFilenames = append(experimentFilenames, file.Name())
		}
	}
	return experimentFilenames, nil
}

func (p *program) Stop(s service.Service) error {
	p.quitter.Quit()
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package db encapsulates tsuru connection with MongoDB.
//
// The function Open dials to MongoDB and returns a connection (represented by
// the Storage type). It manages an internal pool of connections, and
// reconnects in case of failures. That means that you should not store
// references to the connection, but always call Open.
package db

import (
	"fmt"
	"github.com/globocom/config"
	"labix.org/v2/mgo"
	"sync"
	"time"
)

var (
	conn        = make(map[string]*session) // pool of connections
	mut         sync.RWMutex                // for pool thread safety
	ticker      *time.Ticker                // for garbage collection
	maxIdleTime = 5 * time.Minute           // max idle time for connections
)

const period time.Duration = 7 * 24 * time.Hour

type session struct {
	s    *mgo.Session
	used time.Time
}

// Storage holds the connection with the database.
type Storage struct {
	session *mgo.Session
	dbname  string
}

func open(addr, dbname string) (*Storage, error) {
	sess, err := mgo.DialWithTimeout(addr, 1e9)
	if err != nil {
		return nil, err
	}
	copy := sess.Copy()
	go func() {
		time.Sleep(maxIdleTime)
		copy.Close()
	}()
	storage := &Storage{session: copy, dbname: dbname}
	mut.Lock()
	conn[addr] = &session{s: sess, used: time.Now()}
	mut.Unlock()
	return storage, nil
}

// Open dials to the MongoDB database, and return the connection (represented
// by the type Storage).
//
// addr is a MongoDB connection URI, and dbname is the name of the database.
//
// This function returns a pointer to a Storage, or a non-nil error in case of
// any failure.
func Open(addr, dbname string) (storage *Storage, err error) {
	defer func() {
		if r := recover(); r != nil {
			storage, err = open(addr, dbname)
		}
	}()
	mut.RLock()
	if session, ok := conn[addr]; ok {
		mut.RUnlock()
		if err = session.s.Ping(); err == nil {
			mut.Lock()
			session.used = time.Now()
			conn[addr] = session
			mut.Unlock()
			copy := session.s.Copy()
			go func() {
				time.Sleep(maxIdleTime)
				copy.Close()
			}()
			return &Storage{copy, dbname}, nil
		}
		return open(addr, dbname)
	}
	mut.RUnlock()
	return open(addr, dbname)
}

// Conn reads the tsuru config and calls Open to get a database connection.
//
// Most tsuru packages should probably use this function. Open is intended for
// use when supporting more than one database.
func Conn() (*Storage, error) {
	url, err := config.GetString("database:url")
	if err != nil {
		return nil, fmt.Errorf("configuration error: %s", err)
	}
	dbname, err := config.GetString("database:name")
	if err != nil {
		return nil, fmt.Errorf("configuration error: %s", err)
	}
	return Open(url, dbname)
}

// Close closes the storage, releasing the connection.
func (s *Storage) Close() {
	s.session.Close()
}

// Collection returns a collection by its name.
//
// If the collection does not exist, MongoDB will create it.
func (s *Storage) Collection(name string) *mgo.Collection {
	return s.session.DB(s.dbname).C(name)
}

// Apps returns the apps collection from MongoDB.
func (s *Storage) Apps() *mgo.Collection {
	nameIndex := mgo.Index{Key: []string{"name"}, Unique: true}
	c := s.Collection("apps")
	c.EnsureIndex(nameIndex)
	return c
}

// Services returns the services collection from MongoDB.
func (s *Storage) Services() *mgo.Collection {
	c := s.Collection("services")
	return c
}

// ServiceInstances returns the services_instances collection from MongoDB.
func (s *Storage) ServiceInstances() *mgo.Collection {
	return s.Collection("service_instances")
}

// Users returns the users collection from MongoDB.
func (s *Storage) Users() *mgo.Collection {
	emailIndex := mgo.Index{Key: []string{"email"}, Unique: true}
	c := s.Collection("users")
	c.EnsureIndex(emailIndex)
	return c
}

// Teams returns the teams collection from MongoDB.
func (s *Storage) Teams() *mgo.Collection {
	return s.Collection("teams")
}

func init() {
	ticker = time.NewTicker(time.Hour)
	go retire(ticker)
}

// retire retires old connections :-)
func retire(t *time.Ticker) {
	for _ = range t.C {
		now := time.Now()
		var old []string
		mut.RLock()
		for k, v := range conn {
			if now.Sub(v.used) >= period {
				old = append(old, k)
			}
		}
		mut.RUnlock()
		mut.Lock()
		for _, c := range old {
			conn[c].s.Close()
			delete(conn, c)
		}
		mut.Unlock()
	}
}
<commit_msg>db: copy the value of maxIdleTime to avoid data races<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package db encapsulates tsuru connection with MongoDB.
//
// The function Open dials to MongoDB and returns a connection (represented by
// the Storage type). It manages an internal pool of connections, and
// reconnects in case of failures. That means that you should not store
// references to the connection, but always call Open.
package db

import (
	"fmt"
	"github.com/globocom/config"
	"labix.org/v2/mgo"
	"sync"
	"time"
)

var (
	conn        = make(map[string]*session) // pool of connections
	mut         sync.RWMutex                // for pool thread safety
	ticker      *time.Ticker                // for garbage collection
	maxIdleTime = 5 * time.Minute           // max idle time for connections
)

const period time.Duration = 7 * 24 * time.Hour

type session struct {
	s    *mgo.Session
	used time.Time
}

// Storage holds the connection with the database.
type Storage struct {
	session *mgo.Session
	dbname  string
}

func open(addr, dbname string) (*Storage, error) {
	sess, err := mgo.DialWithTimeout(addr, 1e9)
	if err != nil {
		return nil, err
	}
	copy := sess.Copy()
	go func(t time.Duration) {
		time.Sleep(t)
		copy.Close()
	}(maxIdleTime)
	storage := &Storage{session: copy, dbname: dbname}
	mut.Lock()
	conn[addr] = &session{s: sess, used: time.Now()}
	mut.Unlock()
	return storage, nil
}

// Open dials to the MongoDB database, and return the connection (represented
// by the type Storage).
//
// addr is a MongoDB connection URI, and dbname is the name of the database.
//
// This function returns a pointer to a Storage, or a non-nil error in case of
// any failure.
func Open(addr, dbname string) (storage *Storage, err error) {
	defer func() {
		if r := recover(); r != nil {
			storage, err = open(addr, dbname)
		}
	}()
	mut.RLock()
	if session, ok := conn[addr]; ok {
		mut.RUnlock()
		if err = session.s.Ping(); err == nil {
			mut.Lock()
			session.used = time.Now()
			conn[addr] = session
			mut.Unlock()
			copy := session.s.Copy()
			go func() {
				time.Sleep(maxIdleTime)
				copy.Close()
			}()
			return &Storage{copy, dbname}, nil
		}
		return open(addr, dbname)
	}
	mut.RUnlock()
	return open(addr, dbname)
}

// Conn reads the tsuru config and calls Open to get a database connection.
//
// Most tsuru packages should probably use this function. Open is intended for
// use when supporting more than one database.
func Conn() (*Storage, error) {
	url, err := config.GetString("database:url")
	if err != nil {
		return nil, fmt.Errorf("configuration error: %s", err)
	}
	dbname, err := config.GetString("database:name")
	if err != nil {
		return nil, fmt.Errorf("configuration error: %s", err)
	}
	return Open(url, dbname)
}

// Close closes the storage, releasing the connection.
func (s *Storage) Close() {
	s.session.Close()
}

// Collection returns a collection by its name.
//
// If the collection does not exist, MongoDB will create it.
func (s *Storage) Collection(name string) *mgo.Collection {
	return s.session.DB(s.dbname).C(name)
}

// Apps returns the apps collection from MongoDB.
func (s *Storage) Apps() *mgo.Collection {
	nameIndex := mgo.Index{Key: []string{"name"}, Unique: true}
	c := s.Collection("apps")
	c.EnsureIndex(nameIndex)
	return c
}

// Services returns the services collection from MongoDB.
func (s *Storage) Services() *mgo.Collection {
	c := s.Collection("services")
	return c
}

// ServiceInstances returns the services_instances collection from MongoDB.
func (s *Storage) ServiceInstances() *mgo.Collection {
	return s.Collection("service_instances")
}

// Users returns the users collection from MongoDB.
func (s *Storage) Users() *mgo.Collection {
	emailIndex := mgo.Index{Key: []string{"email"}, Unique: true}
	c := s.Collection("users")
	c.EnsureIndex(emailIndex)
	return c
}

// Teams returns the teams collection from MongoDB.
func (s *Storage) Teams() *mgo.Collection {
	return s.Collection("teams")
}

func init() {
	ticker = time.NewTicker(time.Hour)
	go retire(ticker)
}

// retire retires old connections :-)
func retire(t *time.Ticker) {
	for _ = range t.C {
		now := time.Now()
		var old []string
		mut.RLock()
		for k, v := range conn {
			if now.Sub(v.used) >= period {
				old = append(old, k)
			}
		}
		mut.RUnlock()
		mut.Lock()
		for _, c := range old {
			conn[c].s.Close()
			delete(conn, c)
		}
		mut.Unlock()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"sort"
	"time"

	"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"

	"sigs.k8s.io/kustomize/api/internal/crawl/doc"

	"sigs.k8s.io/kustomize/api/internal/crawl/index"
)

const (
	githubAccessTokenVar = "GITHUB_ACCESS_TOKEN"
	retryCount           = 3
)

// iterateArr adds each item in arr into countMap.
func iterateArr(arr []string, countMap map[string]int) {
	for _, item := range arr {
		if _, ok := countMap[item]; !ok {
			countMap[item] = 1
		} else {
			countMap[item]++
		}
	}

}

// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
// in the map, and outputs the sorted keys as a slice.
func SortMapKeyByValue(m map[string]int) []string {
	keys := make([]string, 0, len(m))
	for key := range m {
		keys = append(keys, key)
	}
	// sort keys according to their values in the map m
	sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] })
	return keys
}

func GeneratorAndTransformerStats(ctx context.Context,
	generatorDocs []*doc.Document, transformerDocs []*doc.Document,
	idx *index.KustomizeIndex) {
	// allGenerators includes all the documents referred in the generators field
	allGenerators := doc.NewUniqueDocuments()

	// allTransformers includes all the documents referred in the transformers field
	allTransformers := doc.NewUniqueDocuments()

	// docUsingGeneratorCount counts the number of the kustomization files using generators
	docUsingGeneratorCount := 0

	// docUsingTransformerCount counts the number of the kustomization files using transformers
	docUsingTransformerCount := 0

	// collect all the documents referred in the generators and transformers fields
	for _, d := range generatorDocs {
		kdoc := doc.KustomizationDocument{
			Document: *d,
		}
		generators, err := kdoc.GetResources(false, false, true)
		if err != nil {
			log.Printf("failed to parse the generators field of the Document (%s): %v",
				d.Path(), err)
		}
		if len(generators) > 0 {
			docUsingGeneratorCount++
			allGenerators.AddDocuments(generators)
		}
	}

	for _, d := range transformerDocs {
		kdoc := doc.KustomizationDocument{
			Document: *d,
		}
		transformers, err := kdoc.GetResources(false, true, false)
		if err != nil {
			log.Printf("failed to parse the transformers field of the Document (%s): %v",
				d.Path(), err)
		}
		if len(transformers) > 0 {
			docUsingTransformerCount++
			allTransformers.AddDocuments(transformers)
		}
	}

	// fileGeneratorCount counts file-type generators
	// dirGeneratorCount counts dir-type generators
	fileGeneratorCount, dirGeneratorCount, generatorFiles, generatorDirs := DocumentTypeSummary(ctx, allGenerators.Documents())

	// fileTransformerCount counts file-type transformers
	// dirTransformerCount counts dir-type transformers
	fileTransformerCount, dirTransformerCount, transformerFiles, transformerDirs := DocumentTypeSummary(ctx, allTransformers.Documents())

	// check whether any of the generator files are not in the index
	nonExistGeneratorFileCount := ExistInIndex(idx, generatorFiles, "generator file ")
	// check whether any of the generator dirs are not in the index
	nonExistGeneratorDirCount := ExistInIndex(idx, generatorDirs, "generator dir ")

	// check whether any of the transformer files are not in the index
	nonExistTransformerFileCount := ExistInIndex(idx, transformerFiles, "transformer file ")
	// check whether any of the transformer dirs are not in the index
	nonExistTransformerDirCount := ExistInIndex(idx, transformerDirs, "transformer dir ")

	GitRepositorySummary(generatorFiles, "generator files")
	GitRepositorySummary(generatorDirs, "generator dirs")
	GitRepositorySummary(transformerFiles, "transformer files")
	GitRepositorySummary(transformerDirs, "transformer dirs")

	fmt.Printf(`%d kustomization files use generators: %d generators are files and %d generators are dirs.
%d kustomization files use tranformers: %d transformers are files and %d transformers are dirs.`,
		docUsingGeneratorCount, fileGeneratorCount, dirGeneratorCount,
		docUsingTransformerCount, fileTransformerCount, dirTransformerCount)
	fmt.Printf("\n")
	fmt.Printf("%d generator files do not exist in the index\n", nonExistGeneratorFileCount)
	fmt.Printf("%d generator dirs do not exist in the index\n", nonExistGeneratorDirCount)
	fmt.Printf("%d transformer files do not exist in the index\n", nonExistTransformerFileCount)
	fmt.Printf("%d transformer dirs do not exist in the index\n", nonExistTransformerDirCount)
}

// GitRepositorySummary counts the distribution of docs:
// 1) how many git repositories are these docs from?
// 2) how many docs are from each git repository?
func GitRepositorySummary(docs []*doc.Document, msgPrefix string) {
	m := make(map[string]int)
	for _, d := range docs {
		if _, ok := m[d.RepositoryURL]; ok {
			m[d.RepositoryURL]++
		} else {
			m[d.RepositoryURL] = 1
		}
	}
	sortedKeys := SortMapKeyByValue(m)
	for _, k := range sortedKeys {
		fmt.Printf("%d %s are from %s\n", m[k], msgPrefix, k)
	}
}

// ExistInIndex goes through each Document in docs, and check whether it is in the index or not.
// It returns the number of documents which does not exist in the index.
func ExistInIndex(idx *index.KustomizeIndex, docs []*doc.Document, msgPrefix string) int {
	nonExistCount := 0
	for _, d := range docs {
		exists, err := idx.Exists(d.ID())
		if err != nil {
			log.Println(err)
		}
		if !exists {
			log.Printf("%s (%s) does not exist in the index", msgPrefix, d.Path())
			nonExistCount++
		}
	}
	return nonExistCount
}

// DocumentTypeSummary goes through each doc in docs, and determines whether it is a file or dir.
func DocumentTypeSummary(ctx context.Context, docs []*doc.Document) (
	fileCount, dirCount int, files, dirs []*doc.Document) {
	githubToken := os.Getenv(githubAccessTokenVar)
	if githubToken == "" {
		log.Fatalf("Must set the variable '%s' to make github requests.\n",
			githubAccessTokenVar)
	}
	ghCrawler := github.NewCrawler(githubToken, retryCount, &http.Client{}, github.QueryWith())

	for _, d := range docs {
		oldFilePath := d.FilePath
		if err := ghCrawler.FetchDocument(ctx, d); err != nil {
			log.Printf("FetchDocument failed on %s: %v", d.Path(), err)
			continue
		}

		if d.FilePath == oldFilePath {
			fileCount++
			files = append(files, d)
		} else {
			dirCount++
			dirs = append(dirs, d)
		}
	}
	return fileCount, dirCount, files, dirs
}

// ExistInSlice checks where target exits in items.
func ExistInSlice(items []string, target string) bool {
	for _, item := range items {
		if item == target {
			return true
		}
	}
	return false
}

func main() {
	topKindsPtr := flag.Int(
		"kinds", -1,
		`the number of kubernetes object kinds to be listed according to their popularities.
By default, all the kinds will be listed.
If you only want to list the 10 most popular kinds, set the flag to 10.`)
	topIdentifiersPtr := flag.Int(
		"identifiers", -1,
		`the number of identifiers to be listed according to their popularities.
By default, all the identifiers will be listed.
If you only want to list the 10 most popular identifiers, set the flag to 10.`)
	topKustomizeFeaturesPtr := flag.Int(
		"kustomize-features", -1,
		`the number of kustomize features to be listed according to their popularities.
By default, all the features will be listed.
If you only want to list the 10 most popular features, set the flag to 10.`)
	indexNamePtr := flag.String(
		"index", "kustomize", "The name of the ElasticSearch index.")
	flag.Parse()

	ctx := context.Background()
	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
	if err != nil {
		log.Fatalf("Could not create an index: %v\n", err)
	}

	// count tracks the number of documents in the index
	count := 0

	// kustomizationFilecount tracks the number of kustomization files in the index
	kustomizationFilecount := 0

	kindsMap := make(map[string]int)
	identifiersMap := make(map[string]int)
	kustomizeIdentifiersMap := make(map[string]int)

	// ids tracks the unique IDs of the documents in the index
	ids := make(map[string]struct{})

	// generatorDocs includes all the docs using generators
	generatorDocs := make([]*doc.Document, 0)

	// transformersDocs includes all the docs using transformers
	transformersDocs := make([]*doc.Document, 0)

	// get all the documents in the index
	query := []byte(`{ "query":{ "match_all":{} } }`)
	it := idx.IterateQuery(query, 10000, 60*time.Second)
	for it.Next() {
		for _, hit := range it.Value().Hits.Hits {
			// check whether there is any duplicate IDs in the index
			if _, ok := ids[hit.ID]; !ok {
				ids[hit.ID] = struct{}{}
			} else {
				log.Printf("Found duplicate ID (%s)\n", hit.ID)
			}

			count++
			iterateArr(hit.Document.Kinds, kindsMap)
			iterateArr(hit.Document.Identifiers, identifiersMap)

			if doc.IsKustomizationFile(hit.Document.FilePath) {
				kustomizationFilecount++
				iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
				if ExistInSlice(hit.Document.Identifiers, "generators") {
					generatorDocs = append(generatorDocs, hit.Document.Copy())
				}
				if ExistInSlice(hit.Document.Identifiers, "transformers") {
					transformersDocs = append(transformersDocs, hit.Document.Copy())
				}
			}
		}
	}

	if err := it.Err(); err != nil {
		log.Fatalf("Error iterating: %v\n", err)
	}

	sortedKindsMapKeys := SortMapKeyByValue(kindsMap)
	sortedIdentifiersMapKeys := SortMapKeyByValue(identifiersMap)
	sortedKustomizeIdentifiersMapKeys := SortMapKeyByValue(kustomizeIdentifiersMap)

	fmt.Printf(`The count of unique document IDs in the kustomize index: %d
There are %d documents in the kustomize index.
%d kinds of kubernetes objects are customized:`, len(ids), count, len(kindsMap))
	fmt.Printf("\n")

	kindCount := 0
	for _, key := range sortedKindsMapKeys {
		if *topKindsPtr < 0 || (*topKindsPtr >= 0 && kindCount < *topKindsPtr) {
			fmt.Printf("\tkind `%s` is customimzed in %d documents\n", key, kindsMap[key])
			kindCount++
		}
	}

	fmt.Printf("%d kinds of identifiers are found:\n", len(identifiersMap))
	identifierCount := 0
	for _, key := range sortedIdentifiersMapKeys {
		if *topIdentifiersPtr < 0 || (*topIdentifiersPtr >= 0 && identifierCount < *topIdentifiersPtr) {
			fmt.Printf("\tidentifier `%s` appears in %d documents\n", key, identifiersMap[key])
			identifierCount++
		}
	}

	fmt.Printf(`There are %d kustomization files in the kustomize index.
%d kinds of kustomize features are found:`, kustomizationFilecount, len(kustomizeIdentifiersMap))
	fmt.Printf("\n")
	kustomizeFeatureCount := 0
	for _, key := range sortedKustomizeIdentifiersMapKeys {
		if *topKustomizeFeaturesPtr < 0 || (*topKustomizeFeaturesPtr >= 0 && kustomizeFeatureCount < *topKustomizeFeaturesPtr) {
			fmt.Printf("\tfeature `%s` is used in %d documents\n", key, kustomizeIdentifiersMap[key])
			kustomizeFeatureCount++
		}
	}

	GeneratorAndTransformerStats(ctx, generatorDocs, transformersDocs, idx)
}
<commit_msg>Refactor the stats code for generators and transformers<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"sort"
	"time"

	"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"

	"sigs.k8s.io/kustomize/api/internal/crawl/doc"

	"sigs.k8s.io/kustomize/api/internal/crawl/index"
)

const (
	githubAccessTokenVar = "GITHUB_ACCESS_TOKEN"
	retryCount           = 3
)

// iterateArr adds each item in arr into countMap.
func iterateArr(arr []string, countMap map[string]int) {
	for _, item := range arr {
		if _, ok := countMap[item]; !ok {
			countMap[item] = 1
		} else {
			countMap[item]++
		}
	}

}

// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
// in the map, and outputs the sorted keys as a slice.
func SortMapKeyByValue(m map[string]int) []string {
	keys := make([]string, 0, len(m))
	for key := range m {
		keys = append(keys, key)
	}
	// sort keys according to their values in the map m
	sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] })
	return keys
}

func GeneratorOrTransformerStats(ctx context.Context,
	docs []*doc.Document, isGenerator bool, idx *index.KustomizeIndex) {

	fieldName := "generators"
	if !isGenerator {
		fieldName = "transformers"
	}

	// allReferredDocs includes all the documents referred in the field
	allReferredDocs := doc.NewUniqueDocuments()

	// docUsingGeneratorCount counts the number of the kustomization files using generators or transformers
	docCount := 0

	// collect all the documents referred in the field
	for _, d := range docs {
		kdoc := doc.KustomizationDocument{
			Document: *d,
		}
		referredDocs, err := kdoc.GetResources(false, !isGenerator, isGenerator)
		if err != nil {
			log.Printf("failed to parse the %s field of the Document (%s): %v",
				fieldName, d.Path(), err)
		}
		if len(referredDocs) > 0 {
			docCount++
			allReferredDocs.AddDocuments(referredDocs)
		}
	}

	fileCount, dirCount, fileTypeDocs, dirTypeDocs := DocumentTypeSummary(ctx, allReferredDocs.Documents())

	// check whether any of the files are not in the index
	nonExistFileCount := ExistInIndex(idx, fileTypeDocs, fieldName + " file ")
	// check whether any of the dirs are not in the index
	nonExistDirCount := ExistInIndex(idx, dirTypeDocs, fieldName + " dir ")

	GitRepositorySummary(fileTypeDocs, fieldName + " files")
	GitRepositorySummary(dirTypeDocs, fieldName + " dirs")

	fmt.Printf("%d kustomization files use %s: %d %s are files and %d %s are dirs.\n",
		docCount, fieldName, fileCount, fieldName, dirCount, fieldName)
	fmt.Printf("%d %s files do not exist in the index\n", nonExistFileCount, fieldName)
	fmt.Printf("%d %s dirs do not exist in the index\n", nonExistDirCount, fieldName)
}

// GitRepositorySummary counts the distribution of docs:
// 1) how many git repositories are these docs from?
// 2) how many docs are from each git repository?
func GitRepositorySummary(docs []*doc.Document, msgPrefix string) {
	m := make(map[string]int)
	for _, d := range docs {
		if _, ok := m[d.RepositoryURL]; ok {
			m[d.RepositoryURL]++
		} else {
			m[d.RepositoryURL] = 1
		}
	}
	sortedKeys := SortMapKeyByValue(m)
	for _, k := range sortedKeys {
		fmt.Printf("%d %s are from %s\n", m[k], msgPrefix, k)
	}
}

// ExistInIndex goes through each Document in docs, and check whether it is in the index or not.
// It returns the number of documents which does not exist in the index.
func ExistInIndex(idx *index.KustomizeIndex, docs []*doc.Document, msgPrefix string) int {
	nonExistCount := 0
	for _, d := range docs {
		exists, err := idx.Exists(d.ID())
		if err != nil {
			log.Println(err)
		}
		if !exists {
			log.Printf("%s (%s) does not exist in the index", msgPrefix, d.Path())
			nonExistCount++
		}
	}
	return nonExistCount
}

// DocumentTypeSummary goes through each doc in docs, and determines whether it is a file or dir.
func DocumentTypeSummary(ctx context.Context, docs []*doc.Document) (
	fileCount, dirCount int, files, dirs []*doc.Document) {
	githubToken := os.Getenv(githubAccessTokenVar)
	if githubToken == "" {
		log.Fatalf("Must set the variable '%s' to make github requests.\n",
			githubAccessTokenVar)
	}
	ghCrawler := github.NewCrawler(githubToken, retryCount, &http.Client{}, github.QueryWith())

	for _, d := range docs {
		oldFilePath := d.FilePath
		if err := ghCrawler.FetchDocument(ctx, d); err != nil {
			log.Printf("FetchDocument failed on %s: %v", d.Path(), err)
			continue
		}

		if d.FilePath == oldFilePath {
			fileCount++
			files = append(files, d)
		} else {
			dirCount++
			dirs = append(dirs, d)
		}
	}
	return fileCount, dirCount, files, dirs
}

// ExistInSlice checks where target exits in items.
func ExistInSlice(items []string, target string) bool {
	for _, item := range items {
		if item == target {
			return true
		}
	}
	return false
}

func main() {
	topKindsPtr := flag.Int(
		"kinds", -1,
		`the number of kubernetes object kinds to be listed according to their popularities.
By default, all the kinds will be listed.
If you only want to list the 10 most popular kinds, set the flag to 10.`)
	topIdentifiersPtr := flag.Int(
		"identifiers", -1,
		`the number of identifiers to be listed according to their popularities.
By default, all the identifiers will be listed.
If you only want to list the 10 most popular identifiers, set the flag to 10.`)
	topKustomizeFeaturesPtr := flag.Int(
		"kustomize-features", -1,
		`the number of kustomize features to be listed according to their popularities.
By default, all the features will be listed.
If you only want to list the 10 most popular features, set the flag to 10.`)
	indexNamePtr := flag.String(
		"index", "kustomize", "The name of the ElasticSearch index.")
	flag.Parse()

	ctx := context.Background()
	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
	if err != nil {
		log.Fatalf("Could not create an index: %v\n", err)
	}

	// count tracks the number of documents in the index
	count := 0

	// kustomizationFilecount tracks the number of kustomization files in the index
	kustomizationFilecount := 0

	kindsMap := make(map[string]int)
	identifiersMap := make(map[string]int)
	kustomizeIdentifiersMap := make(map[string]int)

	// ids tracks the unique IDs of the documents in the index
	ids := make(map[string]struct{})

	// generatorDocs includes all the docs using generators
	generatorDocs := make([]*doc.Document, 0)

	// transformersDocs includes all the docs using transformers
	transformersDocs := make([]*doc.Document, 0)

	// get all the documents in the index
	query := []byte(`{ "query":{ "match_all":{} } }`)
	it := idx.IterateQuery(query, 10000, 60*time.Second)
	for it.Next() {
		for _, hit := range it.Value().Hits.Hits {
			// check whether there is any duplicate IDs in the index
			if _, ok := ids[hit.ID]; !ok {
				ids[hit.ID] = struct{}{}
			} else {
				log.Printf("Found duplicate ID (%s)\n", hit.ID)
			}

			count++
			iterateArr(hit.Document.Kinds, kindsMap)
			iterateArr(hit.Document.Identifiers, identifiersMap)

			if doc.IsKustomizationFile(hit.Document.FilePath) {
				kustomizationFilecount++
				iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
				if ExistInSlice(hit.Document.Identifiers, "generators") {
					generatorDocs = append(generatorDocs, hit.Document.Copy())
				}
				if ExistInSlice(hit.Document.Identifiers, "transformers") {
					transformersDocs = append(transformersDocs, hit.Document.Copy())
				}
			}
		}
	}

	if err := it.Err(); err != nil {
		log.Fatalf("Error iterating: %v\n", err)
	}

	sortedKindsMapKeys := SortMapKeyByValue(kindsMap)
	sortedIdentifiersMapKeys := SortMapKeyByValue(identifiersMap)
	sortedKustomizeIdentifiersMapKeys := SortMapKeyByValue(kustomizeIdentifiersMap)

	fmt.Printf(`The count of unique document IDs in the kustomize index: %d
There are %d documents in the kustomize index.
%d kinds of kubernetes objects are customized:`, len(ids), count, len(kindsMap))
	fmt.Printf("\n")

	kindCount := 0
	for _, key := range sortedKindsMapKeys {
		if *topKindsPtr < 0 || (*topKindsPtr >= 0 && kindCount < *topKindsPtr) {
			fmt.Printf("\tkind `%s` is customimzed in %d documents\n", key, kindsMap[key])
			kindCount++
		}
	}

	fmt.Printf("%d kinds of identifiers are found:\n", len(identifiersMap))
	identifierCount := 0
	for _, key := range sortedIdentifiersMapKeys {
		if *topIdentifiersPtr < 0 || (*topIdentifiersPtr >= 0 && identifierCount < *topIdentifiersPtr) {
			fmt.Printf("\tidentifier `%s` appears in %d documents\n", key, identifiersMap[key])
			identifierCount++
		}
	}

	fmt.Printf(`There are %d kustomization files in the kustomize index.
%d kinds of kustomize features are found:`, kustomizationFilecount, len(kustomizeIdentifiersMap))
	fmt.Printf("\n")
	kustomizeFeatureCount := 0
	for _, key := range sortedKustomizeIdentifiersMapKeys {
		if *topKustomizeFeaturesPtr < 0 || (*topKustomizeFeaturesPtr >= 0 && kustomizeFeatureCount < *topKustomizeFeaturesPtr) {
			fmt.Printf("\tfeature `%s` is used in %d documents\n", key, kustomizeIdentifiersMap[key])
			kustomizeFeatureCount++
		}
	}

	GeneratorOrTransformerStats(ctx, generatorDocs, true, idx)
	GeneratorOrTransformerStats(ctx, transformersDocs, false, idx)
}
<|endoftext|>


<commit_before>package base_image_test

import (
	"fmt"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/werf/werf/integration/pkg/utils"
	utilsDocker "github.com/werf/werf/integration/pkg/utils/docker"
)

var fromImageItFunc = func(appConfigName, fromImageConfigName string, extraAfterBuildChecks func(appConfigName, fromImageConfigName string)) {
	By(fmt.Sprintf("fromCacheVersion: %s", "0"))
	SuiteData.Stubs.SetEnv("FROM_CACHE_VERSION", "0")

	output := utils.SucceedCommandOutputString(
		SuiteData.TestDirPath,
		SuiteData.WerfBinPath,
		"build",
	)

	Ω(strings.Count(output, fmt.Sprintf("Building stage %s/from", appConfigName))).Should(Equal(2))

	extraAfterBuildChecks(appConfigName, fromImageConfigName)

	By(fmt.Sprintf("fromCacheVersion: %s", "1"))
	SuiteData.Stubs.SetEnv("FROM_CACHE_VERSION", "1")

	output = utils.SucceedCommandOutputString(
		SuiteData.TestDirPath,
		SuiteData.WerfBinPath,
		"build",
	)

	Ω(strings.Count(output, fmt.Sprintf("Building stage %s/from", appConfigName))).Should(Equal(2))

	extraAfterBuildChecks(appConfigName, fromImageConfigName)
}

var _ = XDescribe("fromImage", func() {
	BeforeEach(func() {
		SuiteData.TestDirPath = utils.FixturePath("from_image")
	})

	It("should be rebuilt", func() {
		fromImageItFunc("app", "fromImage", func(appConfigName, fromImageConfigName string) {
			appImageName := utils.SucceedCommandOutputString(
				SuiteData.TestDirPath,
				SuiteData.WerfBinPath,
				"stage", "image", appConfigName,
			)

			fromImageName := utils.SucceedCommandOutputString(
				SuiteData.TestDirPath,
				SuiteData.WerfBinPath,
				"stage", "image", fromImageConfigName,
			)

			Ω(utilsDocker.ImageParent(strings.TrimSpace(appImageName))).Should(Equal(utilsDocker.ImageID(strings.TrimSpace(fromImageName))))
		})
	})
})

var _ = XDescribe("fromArtifact", func() {
	BeforeEach(func() {
		SuiteData.TestDirPath = utils.FixturePath("from_artifact")
	})

	It("should be rebuilt", func() {
		fromImageItFunc("app", "fromArtifact", func(appConfigName, fromImageConfigName string) {})
	})
})
<commit_msg>tests<commit_after>package base_image_test

import (
	"fmt"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/werf/werf/integration/pkg/utils"
	utilsDocker "github.com/werf/werf/integration/pkg/utils/docker"
)

var fromImageItFunc = func(appConfigName, fromImageConfigName string, extraAfterBuildChecks func(appConfigName, fromImageConfigName string)) {
	By(fmt.Sprintf("fromCacheVersion: %s", "0"))
	SuiteData.Stubs.SetEnv("FROM_CACHE_VERSION", "0")

	output := utils.SucceedCommandOutputString(
		SuiteData.TestDirPath,
		SuiteData.WerfBinPath,
		"build",
	)

	Ω(strings.Count(output, fmt.Sprintf("Building stage %s/from", appConfigName))).Should(Equal(2))

	extraAfterBuildChecks(appConfigName, fromImageConfigName)

	By(fmt.Sprintf("fromCacheVersion: %s", "1"))
	SuiteData.Stubs.SetEnv("FROM_CACHE_VERSION", "1")

	output = utils.SucceedCommandOutputString(
		SuiteData.TestDirPath,
		SuiteData.WerfBinPath,
		"build",
	)

	Ω(strings.Count(output, fmt.Sprintf("Building stage %s/from", appConfigName))).Should(Equal(2))

	extraAfterBuildChecks(appConfigName, fromImageConfigName)
}

var _ = XDescribe("fromImage", func() {
	BeforeEach(func() {
		SuiteData.TestDirPath = utils.FixturePath("from_image")
	})

	It("should be rebuilt", func() {
		fromImageItFunc("app", "fromImage", func(appConfigName, fromImageConfigName string) {
			appImageName := utils.SucceedCommandOutputString(
				SuiteData.TestDirPath,
				SuiteData.WerfBinPath,
				"stage", "image", appConfigName,
			)

			fromImageName := utils.SucceedCommandOutputString(
				SuiteData.TestDirPath,
				SuiteData.WerfBinPath,
				"stage", "image", fromImageConfigName,
			)

			Ω(utilsDocker.ImageParent(strings.TrimSpace(appImageName))).Should(Equal(utilsDocker.ImageID(strings.TrimSpace(fromImageName))))
		})
	})
})
<|endoftext|>


<commit_before>package cli

import (
	"bytes"
	"fmt"
	"os"
	"runtime"
	"strings"

	cmds "github.com/ipfs/go-ipfs/commands"
	files "github.com/ipfs/go-ipfs/commands/files"
	u "github.com/ipfs/go-ipfs/util"
)

// Parse parses the input commandline string (cmd, flags, and args).
// returns the corresponding command Request object.
func Parse(input []string, stdin *os.File, root *cmds.Command) (cmds.Request, *cmds.Command, []string, error) {
	path, opts, stringVals, cmd, err := parseOpts(input, root)
	if err != nil {
		return nil, nil, path, err
	}

	optDefs, err := root.GetOptions(path)
	if err != nil {
		return nil, cmd, path, err
	}

	req, err := cmds.NewRequest(path, opts, nil, nil, cmd, optDefs)
	if err != nil {
		return nil, cmd, path, err
	}

	// if -r is provided, and it is associated with the package builtin
	// recursive path option, allow recursive file paths
	recursiveOpt := req.Option(cmds.RecShort)
	recursive := false
	if recursiveOpt != nil && recursiveOpt.Definition() == cmds.OptionRecursivePath {
		recursive, _, err = recursiveOpt.Bool()
		if err != nil {
			return req, nil, nil, u.ErrCast()
		}
	}

	stringArgs, fileArgs, err := parseArgs(stringVals, stdin, cmd.Arguments, recursive)
	if err != nil {
		return req, cmd, path, err
	}
	req.SetArguments(stringArgs)

	file := files.NewSliceFile("", fileArgs)
	req.SetFiles(file)

	err = cmd.CheckArguments(req)
	if err != nil {
		return req, cmd, path, err
	}

	return req, cmd, path, nil
}

// Parse a command line made up of sub-commands, short arguments, long arguments and positional arguments
func parseOpts(args []string, root *cmds.Command) (
	path []string,
	opts map[string]interface{},
	stringVals []string,
	cmd *cmds.Command,
	err error,
) {
	path = make([]string, 0, len(args))
	stringVals = make([]string, 0, len(args))
	optDefs := map[string]cmds.Option{}
	opts = map[string]interface{}{}
	cmd = root

	// parseFlag checks that a flag is valid and saves it into opts
	// Returns true if the optional second argument is used
	parseFlag := func(name string, arg *string, mustUse bool) (bool, error) {
		if _, ok := opts[name]; ok {
			return false, fmt.Errorf("Duplicate values for option '%s'", name)
		}

		optDef, found := optDefs[name]
		if !found {
			err = fmt.Errorf("Unrecognized option '%s'", name)
			return false, err
		}

		if optDef.Type() == cmds.Bool {
			if mustUse {
				return false, fmt.Errorf("Option '%s' takes no arguments, but was passed '%s'", name, *arg)
			}
			opts[name] = ""
			return false, nil
		} else {
			if arg == nil {
				return true, fmt.Errorf("Missing argument for option '%s'", name)
			}
			opts[name] = *arg
			return true, nil
		}
	}

	optDefs, err = root.GetOptions(path)
	if err != nil {
		return
	}

	consumed := false
	for i, arg := range args {
		switch {
		case consumed:
			// arg was already consumed by the preceding flag
			consumed = false
			continue

		case arg == "--":
			// treat all remaining arguments as positional arguments
			stringVals = append(stringVals, args[i+1:]...)
			return

		case strings.HasPrefix(arg, "--"):
			// arg is a long flag, with an optional argument specified
			// using `=' or in args[i+1]
			var slurped bool
			var next *string
			split := strings.SplitN(arg, "=", 2)
			if len(split) == 2 {
				slurped = false
				arg = split[0]
				next = &split[1]
			} else {
				slurped = true
				if i+1 < len(args) {
					next = &args[i+1]
				} else {
					next = nil
				}
			}
			consumed, err = parseFlag(arg[2:], next, len(split) == 2)
			if err != nil {
				return
			}
			if !slurped {
				consumed = false
			}

		case strings.HasPrefix(arg, "-") && arg != "-":
			// args is one or more flags in short form, followed by an optional argument
			// all flags except the last one have type bool
			for arg = arg[1:]; len(arg) != 0; arg = arg[1:] {
				var rest *string
				var slurped bool
				mustUse := false
				if len(arg) > 1 {
					slurped = false
					str := arg[1:]
					if len(str) > 0 && str[0] == '=' {
						str = str[1:]
						mustUse = true
					}
					rest = &str
				} else {
					slurped = true
					if i+1 < len(args) {
						rest = &args[i+1]
					} else {
						rest = nil
					}
				}
				var end bool
				end, err = parseFlag(arg[0:1], rest, mustUse)
				if err != nil {
					return
				}
				if end {
					consumed = slurped
					break
				}
			}

		default:
			// arg is a sub-command or a positional argument
			sub := cmd.Subcommand(arg)
			if sub != nil {
				cmd = sub
				path = append(path, arg)
				optDefs, err = root.GetOptions(path)
				if err != nil {
					return
				}
			} else {
				stringVals = append(stringVals, arg)
			}
		}
	}
	return
}

func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive bool) ([]string, []files.File, error) {
	// ignore stdin on Windows
	if runtime.GOOS == "windows" {
		stdin = nil
	}

	// check if stdin is coming from terminal or is being piped in
	if stdin != nil {
		if term, err := isTerminal(stdin); err != nil {
			return nil, nil, err
		} else if term {
			stdin = nil // set to nil so we ignore it
		}
	}

	// count required argument definitions
	numRequired := 0
	for _, argDef := range argDefs {
		if argDef.Required {
			numRequired++
		}
	}

	// count number of values provided by user.
	// if there is at least one ArgDef, we can safely trigger the inputs loop
	// below to parse stdin.
	numInputs := len(inputs)
	if argDef := getArgDef(0, argDefs); argDef != nil && stdin != nil {
		numInputs += 1
	}

	// if we have more arg values provided than argument definitions,
	// and the last arg definition is not variadic (or there are no definitions), return an error
	notVariadic := len(argDefs) == 0 || !argDefs[len(argDefs)-1].Variadic
	if notVariadic && len(inputs) > len(argDefs) {
		return nil, nil, fmt.Errorf("Expected %v arguments, got %v: %v", len(argDefs), len(inputs), inputs)
	}

	stringArgs := make([]string, 0, numInputs)
	fileArgs := make([]files.File, 0, numInputs)

	argDefIndex := 0 // the index of the current argument definition
	for i := 0; i < numInputs; i++ {
		argDef := getArgDef(argDefIndex, argDefs)

		// skip optional argument definitions if there aren't sufficient remaining inputs
		for numInputs-i <= numRequired && !argDef.Required {
			argDefIndex++
			argDef = getArgDef(argDefIndex, argDefs)
		}
		if argDef.Required {
			numRequired--
		}

		var err error
		if argDef.Type == cmds.ArgString {
			if stdin == nil || !argDef.SupportsStdin {
				// add string values
				stringArgs, inputs = appendString(stringArgs, inputs)

			} else {
				if len(inputs) > 0 {
					// don't use stdin if we have inputs
					stdin = nil
				} else {
					// if we have a stdin, read it in and use the data as a string value
					stringArgs, stdin, err = appendStdinAsString(stringArgs, stdin)
					if err != nil {
						return nil, nil, err
					}
				}
			}
		} else if argDef.Type == cmds.ArgFile {
			if stdin == nil || !argDef.SupportsStdin {
				// treat stringArg values as file paths
				fileArgs, inputs, err = appendFile(fileArgs, inputs, argDef, recursive)
				if err != nil {
					return nil, nil, err
				}

			} else {
				if len(inputs) > 0 {
					// don't use stdin if we have inputs
					stdin = nil
				} else {
					// if we have a stdin, create a file from it
					fileArgs, stdin = appendStdinAsFile(fileArgs, stdin)
				}
			}
		}

		argDefIndex++
	}

	// check to make sure we didn't miss any required arguments
	if len(argDefs) > argDefIndex {
		for _, argDef := range argDefs[argDefIndex:] {
			if argDef.Required {
				return nil, nil, fmt.Errorf("Argument '%s' is required", argDef.Name)
			}
		}
	}

	return stringArgs, fileArgs, nil
}

func getArgDef(i int, argDefs []cmds.Argument) *cmds.Argument {
	if i < len(argDefs) {
		// get the argument definition (usually just argDefs[i])
		return &argDefs[i]

	} else if len(argDefs) > 0 {
		// but if i > len(argDefs) we use the last argument definition)
		return &argDefs[len(argDefs)-1]
	}

	// only happens if there aren't any definitions
	return nil
}

func appendString(args, inputs []string) ([]string, []string) {
	return append(args, inputs[0]), inputs[1:]
}

func appendStdinAsString(args []string, stdin *os.File) ([]string, *os.File, error) {
	var buf bytes.Buffer

	_, err := buf.ReadFrom(stdin)
	if err != nil {
		return nil, nil, err
	}

	input := strings.TrimSpace(buf.String())
	return append(args, strings.Split(input, "\n")...), nil, nil
}

func appendFile(args []files.File, inputs []string, argDef *cmds.Argument, recursive bool) ([]files.File, []string, error) {
	path := inputs[0]

	file, err := os.Open(path)
	if err != nil {
		return nil, nil, err
	}

	stat, err := file.Stat()
	if err != nil {
		return nil, nil, err
	}

	if stat.IsDir() {
		if !argDef.Recursive {
			err = fmt.Errorf("Invalid path '%s', argument '%s' does not support directories",
				path, argDef.Name)
			return nil, nil, err
		}
		if !recursive {
			err = fmt.Errorf("'%s' is a directory, use the '-%s' flag to specify directories",
				path, cmds.RecShort)
			return nil, nil, err
		}
	}

	arg, err := files.NewSerialFile(path, file)
	if err != nil {
		return nil, nil, err
	}

	return append(args, arg), inputs[1:], nil
}

func appendStdinAsFile(args []files.File, stdin *os.File) ([]files.File, *os.File) {
	arg := files.NewReaderFile("", stdin, nil)
	return append(args, arg), nil
}

// isTerminal returns true if stdin is a Stdin pipe (e.g. `cat file | ipfs`),
// and false otherwise (e.g. nothing is being piped in, so stdin is
// coming from the terminal)
func isTerminal(stdin *os.File) (bool, error) {
	stat, err := stdin.Stat()
	if err != nil {
		return false, err
	}

	// if stdin is a CharDevice, return true
	return ((stat.Mode() & os.ModeCharDevice) != 0), nil
}
<commit_msg>parse: improve stdin fix<commit_after>package cli

import (
	"bytes"
	"fmt"
	"os"
	"runtime"
	"strings"

	cmds "github.com/ipfs/go-ipfs/commands"
	files "github.com/ipfs/go-ipfs/commands/files"
	u "github.com/ipfs/go-ipfs/util"
)

// Parse parses the input commandline string (cmd, flags, and args).
// returns the corresponding command Request object.
func Parse(input []string, stdin *os.File, root *cmds.Command) (cmds.Request, *cmds.Command, []string, error) {
	path, opts, stringVals, cmd, err := parseOpts(input, root)
	if err != nil {
		return nil, nil, path, err
	}

	optDefs, err := root.GetOptions(path)
	if err != nil {
		return nil, cmd, path, err
	}

	req, err := cmds.NewRequest(path, opts, nil, nil, cmd, optDefs)
	if err != nil {
		return nil, cmd, path, err
	}

	// if -r is provided, and it is associated with the package builtin
	// recursive path option, allow recursive file paths
	recursiveOpt := req.Option(cmds.RecShort)
	recursive := false
	if recursiveOpt != nil && recursiveOpt.Definition() == cmds.OptionRecursivePath {
		recursive, _, err = recursiveOpt.Bool()
		if err != nil {
			return req, nil, nil, u.ErrCast()
		}
	}

	stringArgs, fileArgs, err := parseArgs(stringVals, stdin, cmd.Arguments, recursive)
	if err != nil {
		return req, cmd, path, err
	}
	req.SetArguments(stringArgs)

	file := files.NewSliceFile("", fileArgs)
	req.SetFiles(file)

	err = cmd.CheckArguments(req)
	if err != nil {
		return req, cmd, path, err
	}

	return req, cmd, path, nil
}

// Parse a command line made up of sub-commands, short arguments, long arguments and positional arguments
func parseOpts(args []string, root *cmds.Command) (
	path []string,
	opts map[string]interface{},
	stringVals []string,
	cmd *cmds.Command,
	err error,
) {
	path = make([]string, 0, len(args))
	stringVals = make([]string, 0, len(args))
	optDefs := map[string]cmds.Option{}
	opts = map[string]interface{}{}
	cmd = root

	// parseFlag checks that a flag is valid and saves it into opts
	// Returns true if the optional second argument is used
	parseFlag := func(name string, arg *string, mustUse bool) (bool, error) {
		if _, ok := opts[name]; ok {
			return false, fmt.Errorf("Duplicate values for option '%s'", name)
		}

		optDef, found := optDefs[name]
		if !found {
			err = fmt.Errorf("Unrecognized option '%s'", name)
			return false, err
		}

		if optDef.Type() == cmds.Bool {
			if mustUse {
				return false, fmt.Errorf("Option '%s' takes no arguments, but was passed '%s'", name, *arg)
			}
			opts[name] = ""
			return false, nil
		} else {
			if arg == nil {
				return true, fmt.Errorf("Missing argument for option '%s'", name)
			}
			opts[name] = *arg
			return true, nil
		}
	}

	optDefs, err = root.GetOptions(path)
	if err != nil {
		return
	}

	consumed := false
	for i, arg := range args {
		switch {
		case consumed:
			// arg was already consumed by the preceding flag
			consumed = false
			continue

		case arg == "--":
			// treat all remaining arguments as positional arguments
			stringVals = append(stringVals, args[i+1:]...)
			return

		case strings.HasPrefix(arg, "--"):
			// arg is a long flag, with an optional argument specified
			// using `=' or in args[i+1]
			var slurped bool
			var next *string
			split := strings.SplitN(arg, "=", 2)
			if len(split) == 2 {
				slurped = false
				arg = split[0]
				next = &split[1]
			} else {
				slurped = true
				if i+1 < len(args) {
					next = &args[i+1]
				} else {
					next = nil
				}
			}
			consumed, err = parseFlag(arg[2:], next, len(split) == 2)
			if err != nil {
				return
			}
			if !slurped {
				consumed = false
			}

		case strings.HasPrefix(arg, "-") && arg != "-":
			// args is one or more flags in short form, followed by an optional argument
			// all flags except the last one have type bool
			for arg = arg[1:]; len(arg) != 0; arg = arg[1:] {
				var rest *string
				var slurped bool
				mustUse := false
				if len(arg) > 1 {
					slurped = false
					str := arg[1:]
					if len(str) > 0 && str[0] == '=' {
						str = str[1:]
						mustUse = true
					}
					rest = &str
				} else {
					slurped = true
					if i+1 < len(args) {
						rest = &args[i+1]
					} else {
						rest = nil
					}
				}
				var end bool
				end, err = parseFlag(arg[0:1], rest, mustUse)
				if err != nil {
					return
				}
				if end {
					consumed = slurped
					break
				}
			}

		default:
			// arg is a sub-command or a positional argument
			sub := cmd.Subcommand(arg)
			if sub != nil {
				cmd = sub
				path = append(path, arg)
				optDefs, err = root.GetOptions(path)
				if err != nil {
					return
				}
			} else {
				stringVals = append(stringVals, arg)
			}
		}
	}
	return
}

func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive bool) ([]string, []files.File, error) {
	// ignore stdin on Windows
	if runtime.GOOS == "windows" {
		stdin = nil
	}

	// check if stdin is coming from terminal or is being piped in
	if stdin != nil {
		if term, err := isTerminal(stdin); err != nil {
			return nil, nil, err
		} else if term {
			stdin = nil // set to nil so we ignore it
		}
	}

	// count required argument definitions
	numRequired := 0
	for _, argDef := range argDefs {
		if argDef.Required {
			numRequired++
		}
	}

	// count number of values provided by user.
	// if there is at least one ArgDef, we can safely trigger the inputs loop
	// below to parse stdin.
	numInputs := len(inputs)
	if len(argDefs) > 0 && stdin != nil {
		numInputs += 1
	}

	// if we have more arg values provided than argument definitions,
	// and the last arg definition is not variadic (or there are no definitions), return an error
	notVariadic := len(argDefs) == 0 || !argDefs[len(argDefs)-1].Variadic
	if notVariadic && len(inputs) > len(argDefs) {
		return nil, nil, fmt.Errorf("Expected %v arguments, got %v: %v", len(argDefs), len(inputs), inputs)
	}

	stringArgs := make([]string, 0, numInputs)
	fileArgs := make([]files.File, 0, numInputs)

	argDefIndex := 0 // the index of the current argument definition
	for i := 0; i < numInputs; i++ {
		argDef := getArgDef(argDefIndex, argDefs)

		// skip optional argument definitions if there aren't sufficient remaining inputs
		for numInputs-i <= numRequired && !argDef.Required {
			argDefIndex++
			argDef = getArgDef(argDefIndex, argDefs)
		}
		if argDef.Required {
			numRequired--
		}

		var err error
		if argDef.Type == cmds.ArgString {
			if stdin == nil || !argDef.SupportsStdin {
				// add string values
				stringArgs, inputs = appendString(stringArgs, inputs)

			} else {
				if len(inputs) > 0 {
					// don't use stdin if we have inputs
					stdin = nil
				} else {
					// if we have a stdin, read it in and use the data as a string value
					stringArgs, stdin, err = appendStdinAsString(stringArgs, stdin)
					if err != nil {
						return nil, nil, err
					}
				}
			}
		} else if argDef.Type == cmds.ArgFile {
			if stdin == nil || !argDef.SupportsStdin {
				// treat stringArg values as file paths
				fileArgs, inputs, err = appendFile(fileArgs, inputs, argDef, recursive)
				if err != nil {
					return nil, nil, err
				}

			} else {
				if len(inputs) > 0 {
					// don't use stdin if we have inputs
					stdin = nil
				} else {
					// if we have a stdin, create a file from it
					fileArgs, stdin = appendStdinAsFile(fileArgs, stdin)
				}
			}
		}

		argDefIndex++
	}

	// check to make sure we didn't miss any required arguments
	if len(argDefs) > argDefIndex {
		for _, argDef := range argDefs[argDefIndex:] {
			if argDef.Required {
				return nil, nil, fmt.Errorf("Argument '%s' is required", argDef.Name)
			}
		}
	}

	return stringArgs, fileArgs, nil
}

func getArgDef(i int, argDefs []cmds.Argument) *cmds.Argument {
	if i < len(argDefs) {
		// get the argument definition (usually just argDefs[i])
		return &argDefs[i]

	} else if len(argDefs) > 0 {
		// but if i > len(argDefs) we use the last argument definition)
		return &argDefs[len(argDefs)-1]
	}

	// only happens if there aren't any definitions
	return nil
}

func appendString(args, inputs []string) ([]string, []string) {
	return append(args, inputs[0]), inputs[1:]
}

func appendStdinAsString(args []string, stdin *os.File) ([]string, *os.File, error) {
	var buf bytes.Buffer

	_, err := buf.ReadFrom(stdin)
	if err != nil {
		return nil, nil, err
	}

	input := strings.TrimSpace(buf.String())
	return append(args, strings.Split(input, "\n")...), nil, nil
}

func appendFile(args []files.File, inputs []string, argDef *cmds.Argument, recursive bool) ([]files.File, []string, error) {
	path := inputs[0]

	file, err := os.Open(path)
	if err != nil {
		return nil, nil, err
	}

	stat, err := file.Stat()
	if err != nil {
		return nil, nil, err
	}

	if stat.IsDir() {
		if !argDef.Recursive {
			err = fmt.Errorf("Invalid path '%s', argument '%s' does not support directories",
				path, argDef.Name)
			return nil, nil, err
		}
		if !recursive {
			err = fmt.Errorf("'%s' is a directory, use the '-%s' flag to specify directories",
				path, cmds.RecShort)
			return nil, nil, err
		}
	}

	arg, err := files.NewSerialFile(path, file)
	if err != nil {
		return nil, nil, err
	}

	return append(args, arg), inputs[1:], nil
}

func appendStdinAsFile(args []files.File, stdin *os.File) ([]files.File, *os.File) {
	arg := files.NewReaderFile("", stdin, nil)
	return append(args, arg), nil
}

// isTerminal returns true if stdin is a Stdin pipe (e.g. `cat file | ipfs`),
// and false otherwise (e.g. nothing is being piped in, so stdin is
// coming from the terminal)
func isTerminal(stdin *os.File) (bool, error) {
	stat, err := stdin.Stat()
	if err != nil {
		return false, err
	}

	// if stdin is a CharDevice, return true
	return ((stat.Mode() & os.ModeCharDevice) != 0), nil
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"gopkg.in/alecthomas/kingpin.v2"
	"gopkg.in/yaml.v2"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"text/template"
)

const outputDir = "output"

// ConfigureBuildCommand setup "build" command
func ConfigureBuildCommand(app *kingpin.Application) {
	app.Command("build", "Build CV site from current directory").Action(runBuildCommand)
}

func runBuildCommand(*kingpin.ParseContext) error {
	userInfo := userInfo{}

	parseYAMLFiles(&userInfo)
	copyTemplate()
	render(&userInfo)

	return nil
}

func parseYAMLFiles(info *userInfo) {
	log.Println("build: parsing YAML files: start")

	var parsingMap = map[string]interface{}{
		aboutMeFileName:       &info.AboutMe,
		educationFileName:     &info.Educations,
		organizationsFileName: &info.Organizations,
		projectsFileName:      &info.Projects,
		skillsFileName:        &info.Skills,
	}

	for file, model := range parsingMap {
		b, err := ioutil.ReadFile(filepath.Join("./", file))
		if err != nil {
			log.Fatal(fmt.Errorf("build: read YAML file: %v", err))
		}

		err = yaml.Unmarshal(b, model)
		if err != nil {
			log.Fatal(fmt.Errorf("build: decoding YAML: %v", err))
		}
	}

	log.Println("build: parsing YAML files: finish")
}

func copyTemplate() {
	log.Println("build: copy template: start")

	cfg := GetConfig()

	inputDir := templatesDir
	if len(cfg.Template.Path) > 0 {
		inputDir = filepath.Join(inputDir, cfg.Template.Path)
	}

	stat, err := os.Stat(inputDir)
	if err != nil {
		log.Fatal(fmt.Errorf("build: read template dir: %v", err))
	}

	if !stat.IsDir() {
		log.Fatal("build: read template dir: isn't dir")
	}

	err = copyDir(inputDir, outputDir)
	if err != nil {
		log.Fatal(fmt.Errorf("build: copy template: %v", err))
	}

	log.Println("build: copy template: finish")
}

func render(info *userInfo) {
	log.Println("build: render template: start")

	cfg := GetConfig()

	if len(cfg.Template.Files) == 0 {
		log.Fatal("build: render template: template file name not specified")
	}

	for _, filename := range cfg.Template.Files {
		renderTemplate(filename, info)
	}

	log.Println("build: render template: finish")
}

func renderTemplate(filename string, info *userInfo) {
	cfg := GetConfig()

	log.Printf("build: render template: render %v", filepath.Join(outputDir, filename))

	f, err := os.Create(filepath.Join(outputDir, filename))
	if err != nil {
		log.Fatal(fmt.Errorf("build: create template file: %v", err))
	}
	defer f.Close()

	t, err := template.ParseFiles(filepath.Join(templatesDir, cfg.Template.Path, filename))
	if err != nil {
		log.Fatal(fmt.Errorf("build: parsing template: %v", err))
	}

	err = t.Execute(f, info)
	if err != nil {
		log.Fatal(fmt.Errorf("build: render template: %v", err))
	}
}

func copyDir(src, dst string) error {
	err := filepath.Walk(src, func(srcPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		localPath, err := filepath.Rel(src, srcPath)
		if err != nil {
			return err
		}

		dstPath := filepath.Join(dst, localPath)

		// Copy dir
		if info.IsDir() {
			err = os.MkdirAll(dstPath, info.Mode())
			return err
		}

		// Copy file
		srcFile, err := os.Open(srcPath)
		if err != nil {
			return err
		}
		defer srcFile.Close()

		dstFile, err := os.Create(dstPath)
		if err != nil {
			return err
		}
		defer dstFile.Close()

		err = dstFile.Chmod(info.Mode())
		if err != nil {
			return err
		}

		_, err = io.Copy(dstFile, srcFile)
		return err
	})

	return err
}
<commit_msg>Renaming<commit_after>package commands

import (
	"fmt"
	"gopkg.in/alecthomas/kingpin.v2"
	"gopkg.in/yaml.v2"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"text/template"
)

const outputDir = "output"

// ConfigureBuildCommand setup "build" command
func ConfigureBuildCommand(app *kingpin.Application) {
	app.Command("build", "Build CV site from current directory").Action(runBuildCommand)
}

func runBuildCommand(*kingpin.ParseContext) error {
	userInfo := userInfo{}

	parseYAMLFiles(&userInfo)
	copyTemplate()
	renderTemplate(&userInfo)

	return nil
}

func parseYAMLFiles(info *userInfo) {
	log.Println("build: parsing YAML files: start")

	var parsingMap = map[string]interface{}{
		aboutMeFileName:       &info.AboutMe,
		educationFileName:     &info.Educations,
		organizationsFileName: &info.Organizations,
		projectsFileName:      &info.Projects,
		skillsFileName:        &info.Skills,
	}

	for file, model := range parsingMap {
		b, err := ioutil.ReadFile(filepath.Join("./", file))
		if err != nil {
			log.Fatal(fmt.Errorf("build: read YAML file: %v", err))
		}

		err = yaml.Unmarshal(b, model)
		if err != nil {
			log.Fatal(fmt.Errorf("build: decoding YAML: %v", err))
		}
	}

	log.Println("build: parsing YAML files: finish")
}

func copyTemplate() {
	log.Println("build: copy template: start")

	cfg := GetConfig()

	inputDir := templatesDir
	if len(cfg.Template.Path) > 0 {
		inputDir = filepath.Join(inputDir, cfg.Template.Path)
	}

	stat, err := os.Stat(inputDir)
	if err != nil {
		log.Fatal(fmt.Errorf("build: read template dir: %v", err))
	}

	if !stat.IsDir() {
		log.Fatal("build: read template dir: isn't dir")
	}

	err = copyDir(inputDir, outputDir)
	if err != nil {
		log.Fatal(fmt.Errorf("build: copy template: %v", err))
	}

	log.Println("build: copy template: finish")
}

func renderTemplate(info *userInfo) {
	log.Println("build: render template: start")

	cfg := GetConfig()

	if len(cfg.Template.Files) == 0 {
		log.Fatal("build: render template: template file name not specified")
	}

	for _, filename := range cfg.Template.Files {
		renderTemplateFile(filename, info)
	}

	log.Println("build: render template: finish")
}

func renderTemplateFile(filename string, info *userInfo) {
	cfg := GetConfig()

	log.Printf("build: render template: render %v", filepath.Join(outputDir, filename))

	f, err := os.Create(filepath.Join(outputDir, filename))
	if err != nil {
		log.Fatal(fmt.Errorf("build: create template file: %v", err))
	}
	defer f.Close()

	t, err := template.ParseFiles(filepath.Join(templatesDir, cfg.Template.Path, filename))
	if err != nil {
		log.Fatal(fmt.Errorf("build: parsing template: %v", err))
	}

	err = t.Execute(f, info)
	if err != nil {
		log.Fatal(fmt.Errorf("build: render template: %v", err))
	}
}

func copyDir(src, dst string) error {
	err := filepath.Walk(src, func(srcPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		localPath, err := filepath.Rel(src, srcPath)
		if err != nil {
			return err
		}

		dstPath := filepath.Join(dst, localPath)

		// Copy dir
		if info.IsDir() {
			err = os.MkdirAll(dstPath, info.Mode())
			return err
		}

		// Copy file
		srcFile, err := os.Open(srcPath)
		if err != nil {
			return err
		}
		defer srcFile.Close()

		dstFile, err := os.Create(dstPath)
		if err != nil {
			return err
		}
		defer dstFile.Close()

		err = dstFile.Chmod(info.Mode())
		if err != nil {
			return err
		}

		_, err = io.Copy(dstFile, srcFile)
		return err
	})

	return err
}
<|endoftext|>


<commit_before>/*
 *    Copyright (C) 2014 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

// beehive's central module system.
package modules

import (
	"log"
)

// Interface which all modules need to implement
type ModuleInterface interface {
	// Name of the module
	Name() string
	// Namespace of the module
	Namespace() string
	// Description of the module
	Description() string

	// Activates the module
	Run(eventChannel chan Event)
	// Handles an action
	Action(action Action) []Placeholder
}

// An instance of a module is called a Bee
type Bee struct {
	Name        string
	Class       string
	Description string
	Options     []BeeOption
}

// An Event
type Event struct {
	Bee     string
	Name    string
	Options []Placeholder
}

// An Action
type Action struct {
	Bee     string
	Name    string
	Options []Placeholder
}

// A Filter
type Filter struct {
	Name    string
	Options []FilterOption
}

// A FilterOption used by filters
type FilterOption struct {
	Name            string
	Type            string
	Inverse         bool
	CaseInsensitive bool
	Trimmed         bool
	Value           interface{}
}

// A BeeOption is used to configure bees
type BeeOptions []BeeOption
type BeeOption struct {
	Name  string
	Type  string
	Value interface{}
}

// A Placeholder used by ins & outs of a module.
type Placeholder struct {
	Name  string
	Type  string
	Value interface{}
}

var (
	eventsIn                                     = make(chan Event)
	modules   map[string]*ModuleInterface        = make(map[string]*ModuleInterface)
	factories map[string]*ModuleFactoryInterface = make(map[string]*ModuleFactoryInterface)
	chains    []Chain
)

func (opts BeeOptions) GetValue(name string) interface{} {
	for _, opt := range opts {
		if opt.Name == name {
			return opt.Value
		}
	}

	return nil
}

// Handles incoming events and executes matching Chains.
func handleEvents() {
	for {
		event := <-eventsIn

		log.Println()
		log.Println("Event received:", event.Bee, "/", event.Name, "-", GetEventDescriptor(&event).Description)
		for _, v := range event.Options {
			log.Println("\tOptions:", v)
		}

		go func() {
			defer func() {
				if e := recover(); e != nil {
					log.Println("Fatal chain event:", e)
				}
			}()

			execChains(&event)
		}()
	}
}

// Modules need to call this method to register themselves
func RegisterModule(mod ModuleInterface) {
	log.Println("Worker bee ready:", mod.Name(), "-", mod.Description())

	modules[mod.Name()] = &mod
}

// Returns module with this name
func GetModule(identifier string) *ModuleInterface {
	mod, ok := modules[identifier]
	if ok {
		return mod
	}

	return nil
}

func startModule(mod *ModuleInterface, fatals int) {
	if fatals >= 3 {
		log.Println("Terminating evil bee", (*mod).Name(), "after", fatals, "failed tries!")
		return
	}

	defer func(mod *ModuleInterface) {
		if e := recover(); e != nil {
			log.Println("Fatal bee event:", e, fatals)
			startModule(mod, fatals+1)
		}
	}(mod)

	(*mod).Run(eventsIn)
}

// Starts all registered modules
func StartModules(bees []Bee) {
	for _, bee := range bees {
		mod := (*GetFactory(bee.Class)).New(bee.Name, bee.Description, bee.Options)
		RegisterModule(mod)
	}

	for _, m := range modules {
		go func(mod *ModuleInterface) {
			startModule(mod, 0)
		}(m)
	}
}

// Getter for chains
func Chains() []Chain {
	return chains
}

// Setter for chains
func SetChains(cs []Chain) {
	chains = cs
}

func init() {
	log.Println("Waking the bees...")
	go handleEvents()
}
<commit_msg>* Don't crash when encountering unknown bee classes.<commit_after>/*
 *    Copyright (C) 2014 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

// beehive's central module system.
package modules

import (
	"log"
)

// Interface which all modules need to implement
type ModuleInterface interface {
	// Name of the module
	Name() string
	// Namespace of the module
	Namespace() string
	// Description of the module
	Description() string

	// Activates the module
	Run(eventChannel chan Event)
	// Handles an action
	Action(action Action) []Placeholder
}

// An instance of a module is called a Bee
type Bee struct {
	Name        string
	Class       string
	Description string
	Options     []BeeOption
}

// An Event
type Event struct {
	Bee     string
	Name    string
	Options []Placeholder
}

// An Action
type Action struct {
	Bee     string
	Name    string
	Options []Placeholder
}

// A Filter
type Filter struct {
	Name    string
	Options []FilterOption
}

// A FilterOption used by filters
type FilterOption struct {
	Name            string
	Type            string
	Inverse         bool
	CaseInsensitive bool
	Trimmed         bool
	Value           interface{}
}

// A BeeOption is used to configure bees
type BeeOptions []BeeOption
type BeeOption struct {
	Name  string
	Type  string
	Value interface{}
}

// A Placeholder used by ins & outs of a module.
type Placeholder struct {
	Name  string
	Type  string
	Value interface{}
}

var (
	eventsIn                                     = make(chan Event)
	modules   map[string]*ModuleInterface        = make(map[string]*ModuleInterface)
	factories map[string]*ModuleFactoryInterface = make(map[string]*ModuleFactoryInterface)
	chains    []Chain
)

func (opts BeeOptions) GetValue(name string) interface{} {
	for _, opt := range opts {
		if opt.Name == name {
			return opt.Value
		}
	}

	return nil
}

// Handles incoming events and executes matching Chains.
func handleEvents() {
	for {
		event := <-eventsIn

		log.Println()
		log.Println("Event received:", event.Bee, "/", event.Name, "-", GetEventDescriptor(&event).Description)
		for _, v := range event.Options {
			log.Println("\tOptions:", v)
		}

		go func() {
			defer func() {
				if e := recover(); e != nil {
					log.Println("Fatal chain event:", e)
				}
			}()

			execChains(&event)
		}()
	}
}

// Modules need to call this method to register themselves
func RegisterModule(mod ModuleInterface) {
	log.Println("Worker bee ready:", mod.Name(), "-", mod.Description())

	modules[mod.Name()] = &mod
}

// Returns module with this name
func GetModule(identifier string) *ModuleInterface {
	mod, ok := modules[identifier]
	if ok {
		return mod
	}

	return nil
}

func startModule(mod *ModuleInterface, fatals int) {
	if fatals >= 3 {
		log.Println("Terminating evil bee", (*mod).Name(), "after", fatals, "failed tries!")
		return
	}

	defer func(mod *ModuleInterface) {
		if e := recover(); e != nil {
			log.Println("Fatal bee event:", e, fatals)
			startModule(mod, fatals+1)
		}
	}(mod)

	(*mod).Run(eventsIn)
}

// Starts all registered modules
func StartModules(bees []Bee) {
	for _, bee := range bees {
		factory := GetFactory(bee.Class)
		if factory == nil {
			panic("Unknown bee-class in config file: " + bee.Class)
		}
		mod := (*factory).New(bee.Name, bee.Description, bee.Options)
		RegisterModule(mod)
	}

	for _, m := range modules {
		go func(mod *ModuleInterface) {
			startModule(mod, 0)
		}(m)
	}
}

// Getter for chains
func Chains() []Chain {
	return chains
}

// Setter for chains
func SetChains(cs []Chain) {
	chains = cs
}

func init() {
	log.Println("Waking the bees...")
	go handleEvents()
}
<|endoftext|>


<commit_before>/*
 Copyright (c) 2014 Kouhei Maeda <mkouhei@palmtb.net>

 This software is release under the Expat License.
*/
package modules

import (
	"github.com/ajstarks/svgo"
	//"math/rand"
	"strconv"
)

func (canv Canvas) Network() *svg.SVG {
	canvas := svg.New(canv.W)
	canvas.Start(canv.Width, canv.Height)

	d := 5 // diameter
	linestyle := "fill:none;stroke:black"
	textstyle := "font-size:10px"
	circlestyle := "fill:gray"

	nd := 100
	xpoly := make([]int, nd)
	ypoly := make([]int, nd)
	xpoly[0] = canv.Width / 2
	ypoly[0] = canv.Height / 2
	delta := 5
	canvas.Circle(xpoly[0], ypoly[0], d, circlestyle)
	for i := 1; i < nd; i++ {
		if i%4 == 1 {
			xpoly[i] = xpoly[i-1] + delta*i
			ypoly[i] = ypoly[i-1] + delta
		} else if i%4 == 2 {
			xpoly[i] = xpoly[i-1] - delta
			ypoly[i] = ypoly[i-1] + delta*i
		} else if i%4 == 3 {
			xpoly[i] = xpoly[i-1] - delta*i
			ypoly[i] = ypoly[i-1] - delta
		} else if i%4 == 0 {
			xpoly[i] = xpoly[i-1] + delta
			ypoly[i] = ypoly[i-1] - delta*i
		}
		//xpoly[i] = rand.Intn(canv.Width)
		//ypoly[i] = rand.Intn(canv.Height)
		canvas.Circle(xpoly[i], ypoly[i], d, circlestyle)
		canvas.Text(xpoly[i]+d*2, ypoly[i]-d*2, "("+strconv.Itoa(xpoly[i])+","+strconv.Itoa(ypoly[i])+")", textstyle)
		if i > 0 {
			canvas.Line(xpoly[i-1], ypoly[i-1], xpoly[i], ypoly[i], linestyle)
		}
	}

	canvas.End()
	return canvas
}
<commit_msg>Remove comments.<commit_after>/*
 Copyright (c) 2014 Kouhei Maeda <mkouhei@palmtb.net>

 This software is release under the Expat License.
*/
package modules

import (
	"github.com/ajstarks/svgo"
	"strconv"
)

func (canv Canvas) Network() *svg.SVG {
	canvas := svg.New(canv.W)
	canvas.Start(canv.Width, canv.Height)

	d := 5 // diameter
	linestyle := "fill:none;stroke:black"
	textstyle := "font-size:10px"
	circlestyle := "fill:gray"

	nd := 100
	xpoly := make([]int, nd)
	ypoly := make([]int, nd)
	xpoly[0] = canv.Width / 2
	ypoly[0] = canv.Height / 2
	delta := 5
	canvas.Circle(xpoly[0], ypoly[0], d, circlestyle)
	for i := 1; i < nd; i++ {
		if i%4 == 1 {
			xpoly[i] = xpoly[i-1] + delta*i
			ypoly[i] = ypoly[i-1] + delta
		} else if i%4 == 2 {
			xpoly[i] = xpoly[i-1] - delta
			ypoly[i] = ypoly[i-1] + delta*i
		} else if i%4 == 3 {
			xpoly[i] = xpoly[i-1] - delta*i
			ypoly[i] = ypoly[i-1] - delta
		} else if i%4 == 0 {
			xpoly[i] = xpoly[i-1] + delta
			ypoly[i] = ypoly[i-1] - delta*i
		}
		canvas.Circle(xpoly[i], ypoly[i], d, circlestyle)
		canvas.Text(xpoly[i]+d*2, ypoly[i]-d*2, "("+strconv.Itoa(xpoly[i])+","+strconv.Itoa(ypoly[i])+")", textstyle)
		if i > 0 {
			canvas.Line(xpoly[i-1], ypoly[i-1], xpoly[i], ypoly[i], linestyle)
		}
	}

	canvas.End()
	return canvas
}
<|endoftext|>


<commit_before>package bitswap

import (
	"bytes"
	"testing"
	"time"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
	bstore "github.com/jbenet/go-ipfs/blockstore"
	exchange "github.com/jbenet/go-ipfs/exchange"
	notifications "github.com/jbenet/go-ipfs/exchange/bitswap/notifications"
	strategy "github.com/jbenet/go-ipfs/exchange/bitswap/strategy"
	tn "github.com/jbenet/go-ipfs/exchange/bitswap/testnet"
	peer "github.com/jbenet/go-ipfs/peer"
	testutil "github.com/jbenet/go-ipfs/util/testutil"
)

func TestGetBlockTimeout(t *testing.T) {

	net := tn.VirtualNetwork()
	rs := tn.VirtualRoutingServer()
	g := NewSessionGenerator(net, rs)

	self := g.Next()

	ctx, _ := context.WithTimeout(context.Background(), time.Nanosecond)
	block := testutil.NewBlockOrFail(t, "block")
	_, err := self.exchange.Block(ctx, block.Key())

	if err != context.DeadlineExceeded {
		t.Fatal("Expected DeadlineExceeded error")
	}
}

func TestProviderForKeyButNetworkCannotFind(t *testing.T) {

	net := tn.VirtualNetwork()
	rs := tn.VirtualRoutingServer()
	g := NewSessionGenerator(net, rs)

	block := testutil.NewBlockOrFail(t, "block")
	rs.Announce(&peer.Peer{}, block.Key()) // but not on network

	solo := g.Next()

	ctx, _ := context.WithTimeout(context.Background(), time.Nanosecond)
	_, err := solo.exchange.Block(ctx, block.Key())

	if err != context.DeadlineExceeded {
		t.Fatal("Expected DeadlineExceeded error")
	}
}

// TestGetBlockAfterRequesting...

func TestGetBlockFromPeerAfterPeerAnnounces(t *testing.T) {

	net := tn.VirtualNetwork()
	rs := tn.VirtualRoutingServer()
	block := testutil.NewBlockOrFail(t, "block")
	g := NewSessionGenerator(net, rs)

	hasBlock := g.Next()

	if err := hasBlock.blockstore.Put(block); err != nil {
		t.Fatal(err)
	}
	if err := hasBlock.exchange.HasBlock(context.Background(), block); err != nil {
		t.Fatal(err)
	}

	wantsBlock := g.Next()

	ctx, _ := context.WithTimeout(context.Background(), time.Second)
	received, err := wantsBlock.exchange.Block(ctx, block.Key())
	if err != nil {
		t.Log(err)
		t.Fatal("Expected to succeed")
	}

	if !bytes.Equal(block.Data, received.Data) {
		t.Fatal("Data doesn't match")
	}
}

func TestSendToWantingPeer(t *testing.T) {
	t.Log("I get a file from peer |w|. In this message, I receive |w|'s wants")
	t.Log("Peer |w| tells me it wants file |f|, but I don't have it")
	t.Log("Later, peer |o| sends |f| to me")
	t.Log("After receiving |f| from |o|, I send it to the wanting peer |w|")
}

func NewSessionGenerator(
	net tn.Network, rs tn.RoutingServer) SessionGenerator {
	return SessionGenerator{
		net: net,
		rs:  rs,
		seq: 0,
	}
}

type SessionGenerator struct {
	seq int
	net tn.Network
	rs  tn.RoutingServer
}

func (g *SessionGenerator) Next() testnetBitSwap {
	g.seq++
	return session(g.net, g.rs, []byte(string(g.seq)))
}

type testnetBitSwap struct {
	peer       *peer.Peer
	exchange   exchange.Interface
	blockstore bstore.Blockstore
}

// session creates a test bitswap session.
//
// NB: It's easy make mistakes by providing the same peer ID to two different
// sessions. To safeguard, use the SessionGenerator to generate sessions. It's
// just a much better idea.
func session(net tn.Network, rs tn.RoutingServer, id peer.ID) testnetBitSwap {
	p := &peer.Peer{ID: id}

	adapter := net.Adapter(p)
	htc := rs.Client(p)

	blockstore := bstore.NewBlockstore(ds.NewMapDatastore())
	bs := &bitswap{
		blockstore:    blockstore,
		notifications: notifications.New(),
		strategy:      strategy.New(),
		routing:       htc,
		sender:        adapter,
	}
	adapter.SetDelegate(bs)
	return testnetBitSwap{
		peer:       p,
		exchange:   bs,
		blockstore: blockstore,
	}
}
<commit_msg>test(bitswap) test with swarm of ~500 instances<commit_after>package bitswap

import (
	"bytes"
	"sync"
	"testing"
	"time"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
	"github.com/jbenet/go-ipfs/blocks"
	bstore "github.com/jbenet/go-ipfs/blockstore"
	exchange "github.com/jbenet/go-ipfs/exchange"
	notifications "github.com/jbenet/go-ipfs/exchange/bitswap/notifications"
	strategy "github.com/jbenet/go-ipfs/exchange/bitswap/strategy"
	tn "github.com/jbenet/go-ipfs/exchange/bitswap/testnet"
	peer "github.com/jbenet/go-ipfs/peer"
	testutil "github.com/jbenet/go-ipfs/util/testutil"
)

func TestGetBlockTimeout(t *testing.T) {

	net := tn.VirtualNetwork()
	rs := tn.VirtualRoutingServer()
	g := NewSessionGenerator(net, rs)

	self := g.Next()

	ctx, _ := context.WithTimeout(context.Background(), time.Nanosecond)
	block := testutil.NewBlockOrFail(t, "block")
	_, err := self.exchange.Block(ctx, block.Key())

	if err != context.DeadlineExceeded {
		t.Fatal("Expected DeadlineExceeded error")
	}
}

func TestProviderForKeyButNetworkCannotFind(t *testing.T) {

	net := tn.VirtualNetwork()
	rs := tn.VirtualRoutingServer()
	g := NewSessionGenerator(net, rs)

	block := testutil.NewBlockOrFail(t, "block")
	rs.Announce(&peer.Peer{}, block.Key()) // but not on network

	solo := g.Next()

	ctx, _ := context.WithTimeout(context.Background(), time.Nanosecond)
	_, err := solo.exchange.Block(ctx, block.Key())

	if err != context.DeadlineExceeded {
		t.Fatal("Expected DeadlineExceeded error")
	}
}

// TestGetBlockAfterRequesting...

func TestGetBlockFromPeerAfterPeerAnnounces(t *testing.T) {

	net := tn.VirtualNetwork()
	rs := tn.VirtualRoutingServer()
	block := testutil.NewBlockOrFail(t, "block")
	g := NewSessionGenerator(net, rs)

	hasBlock := g.Next()

	if err := hasBlock.blockstore.Put(block); err != nil {
		t.Fatal(err)
	}
	if err := hasBlock.exchange.HasBlock(context.Background(), block); err != nil {
		t.Fatal(err)
	}

	wantsBlock := g.Next()

	ctx, _ := context.WithTimeout(context.Background(), time.Second)
	received, err := wantsBlock.exchange.Block(ctx, block.Key())
	if err != nil {
		t.Log(err)
		t.Fatal("Expected to succeed")
	}

	if !bytes.Equal(block.Data, received.Data) {
		t.Fatal("Data doesn't match")
	}
}

func TestSwarm(t *testing.T) {
	net := tn.VirtualNetwork()
	rs := tn.VirtualRoutingServer()
	sg := NewSessionGenerator(net, rs)
	bg := NewBlockGenerator(t)

	t.Log("Create a ton of instances, and just a few blocks")

	numInstances := 500
	numBlocks := 2

	instances := sg.Instances(numInstances)
	blocks := bg.Blocks(numBlocks)

	t.Log("Give the blocks to the first instance")

	first := instances[0]
	for _, b := range blocks {
		first.blockstore.Put(*b)
		first.exchange.HasBlock(context.Background(), *b)
		rs.Announce(first.peer, b.Key())
	}

	t.Log("Distribute!")

	var wg sync.WaitGroup

	for _, inst := range instances {
		for _, b := range blocks {
			wg.Add(1)
			// NB: executing getOrFail concurrently puts tremendous pressure on
			// the goroutine scheduler
			getOrFail(inst, b, t, &wg)
		}
	}
	wg.Wait()

	t.Log("Verify!")

	for _, inst := range instances {
		for _, b := range blocks {
			if _, err := inst.blockstore.Get(b.Key()); err != nil {
				t.Fatal(err)
			}
		}
	}
}

func getOrFail(bitswap instance, b *blocks.Block, t *testing.T, wg *sync.WaitGroup) {
	if _, err := bitswap.blockstore.Get(b.Key()); err != nil {
		_, err := bitswap.exchange.Block(context.Background(), b.Key())
		if err != nil {
			t.Fatal(err)
		}
	}
	wg.Done()
}

func TestSendToWantingPeer(t *testing.T) {
	t.Log("I get a file from peer |w|. In this message, I receive |w|'s wants")
	t.Log("Peer |w| tells me it wants file |f|, but I don't have it")
	t.Log("Later, peer |o| sends |f| to me")
	t.Log("After receiving |f| from |o|, I send it to the wanting peer |w|")
}

func NewBlockGenerator(t *testing.T) BlockGenerator {
	return BlockGenerator{
		T: t,
	}
}

type BlockGenerator struct {
	*testing.T // b/c block generation can fail
	seq        int
}

func (bg *BlockGenerator) Next() blocks.Block {
	bg.seq++
	return testutil.NewBlockOrFail(bg.T, string(bg.seq))
}

func (bg *BlockGenerator) Blocks(n int) []*blocks.Block {
	blocks := make([]*blocks.Block, 0)
	for i := 0; i < n; i++ {
		b := bg.Next()
		blocks = append(blocks, &b)
	}
	return blocks
}

func NewSessionGenerator(
	net tn.Network, rs tn.RoutingServer) SessionGenerator {
	return SessionGenerator{
		net: net,
		rs:  rs,
		seq: 0,
	}
}

type SessionGenerator struct {
	seq int
	net tn.Network
	rs  tn.RoutingServer
}

func (g *SessionGenerator) Next() instance {
	g.seq++
	return session(g.net, g.rs, []byte(string(g.seq)))
}

func (g *SessionGenerator) Instances(n int) []instance {
	instances := make([]instance, 0)
	for j := 0; j < n; j++ {
		inst := g.Next()
		instances = append(instances, inst)
	}
	return instances
}

type instance struct {
	peer       *peer.Peer
	exchange   exchange.Interface
	blockstore bstore.Blockstore
}

// session creates a test bitswap session.
//
// NB: It's easy make mistakes by providing the same peer ID to two different
// sessions. To safeguard, use the SessionGenerator to generate sessions. It's
// just a much better idea.
func session(net tn.Network, rs tn.RoutingServer, id peer.ID) instance {
	p := &peer.Peer{ID: id}

	adapter := net.Adapter(p)
	htc := rs.Client(p)

	blockstore := bstore.NewBlockstore(ds.NewMapDatastore())
	bs := &bitswap{
		blockstore:    blockstore,
		notifications: notifications.New(),
		strategy:      strategy.New(),
		routing:       htc,
		sender:        adapter,
	}
	adapter.SetDelegate(bs)
	return instance{
		peer:       p,
		exchange:   bs,
		blockstore: blockstore,
	}
}
<|endoftext|>


<commit_before>package dcmnet

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"github.com/jeremyhuiskamp/dcm/stream"
	"io"
)

type PDUType uint8

//go:generate stringer -type PDUType
const (
	PDUAssociateRQ      PDUType = 0x01
	PDUAssociateAC      PDUType = 0x02
	PDUAssociateRJ      PDUType = 0x03
	PDUPresentationData PDUType = 0x04
	PDUReleaseRQ        PDUType = 0x05
	PDUReleaseRP        PDUType = 0x06
	PDUAbort            PDUType = 0x07
)

// Protocol Data Unit
type PDU struct {
	Type   PDUType
	Length uint32
	Data   stream.Stream
}

func (p PDU) String() string {
	return fmt.Sprintf("[%s, len=%d]", p.Type, p.Length)
}

// PDUDecoder parses a stream for PDUs
type PDUDecoder struct {
	data StreamDecoder
}

func NewPDUDecoder(data io.Reader) PDUDecoder {
	return PDUDecoder{StreamDecoder{data, nil}}
}

// Read the next PDU from the stream
func (d *PDUDecoder) NextPDU() (pdu *PDU, err error) {
	pdu = &PDU{}
	pdu.Data, err = d.data.NextChunk(6, func(header []byte) int64 {
		pdu.Type = PDUType(header[0])
		pdu.Length = binary.BigEndian.Uint32(header[2:6])
		return int64(pdu.Length)
	})

	// either error, or no more pdus
	if err != nil || pdu.Data == nil {
		return nil, err
	}

	return pdu, err
}

type PDUEncoder struct {
	out    io.Writer
	header [6]byte
}

func NewPDUEncoder(out io.Writer) PDUEncoder {
	return PDUEncoder{out: out}
}

func (w *PDUEncoder) NextPDU(pdu PDU) (err error) {
	w.header[0] = byte(pdu.Type)
	w.header[1] = 0
	// here, we could consider ignoring the passed-in length, but instead
	// writing the data to a buffer and calculating the length?
	// less efficient, but simpler for higher layers?
	// length should be capped at something reasonable based on assoc negotiation
	binary.BigEndian.PutUint32(w.header[2:6], pdu.Length)

	_, err = w.out.Write(w.header[:])
	if err != nil {
		return
	}

	// should we validate that the amount of data written matches what we said
	// the length was?
	_, err = pdu.Data.WriteTo(w.out)

	return err
}
<commit_msg>fix imports<commit_after>package dcmnet

import (
	"encoding/binary"
	"fmt"
	"io"

	"github.com/jeremyhuiskamp/dcm/stream"
)

type PDUType uint8

//go:generate stringer -type PDUType
const (
	PDUAssociateRQ      PDUType = 0x01
	PDUAssociateAC      PDUType = 0x02
	PDUAssociateRJ      PDUType = 0x03
	PDUPresentationData PDUType = 0x04
	PDUReleaseRQ        PDUType = 0x05
	PDUReleaseRP        PDUType = 0x06
	PDUAbort            PDUType = 0x07
)

// Protocol Data Unit
type PDU struct {
	Type   PDUType
	Length uint32
	Data   stream.Stream
}

func (p PDU) String() string {
	return fmt.Sprintf("[%s, len=%d]", p.Type, p.Length)
}

// PDUDecoder parses a stream for PDUs
type PDUDecoder struct {
	data StreamDecoder
}

func NewPDUDecoder(data io.Reader) PDUDecoder {
	return PDUDecoder{StreamDecoder{data, nil}}
}

// Read the next PDU from the stream
func (d *PDUDecoder) NextPDU() (pdu *PDU, err error) {
	pdu = &PDU{}
	pdu.Data, err = d.data.NextChunk(6, func(header []byte) int64 {
		pdu.Type = PDUType(header[0])
		pdu.Length = binary.BigEndian.Uint32(header[2:6])
		return int64(pdu.Length)
	})

	// either error, or no more pdus
	if err != nil || pdu.Data == nil {
		return nil, err
	}

	return pdu, err
}

type PDUEncoder struct {
	out    io.Writer
	header [6]byte
}

func NewPDUEncoder(out io.Writer) PDUEncoder {
	return PDUEncoder{out: out}
}

func (w *PDUEncoder) NextPDU(pdu PDU) (err error) {
	w.header[0] = byte(pdu.Type)
	w.header[1] = 0
	// here, we could consider ignoring the passed-in length, but instead
	// writing the data to a buffer and calculating the length?
	// less efficient, but simpler for higher layers?
	// length should be capped at something reasonable based on assoc negotiation
	binary.BigEndian.PutUint32(w.header[2:6], pdu.Length)

	_, err = w.out.Write(w.header[:])
	if err != nil {
		return
	}

	// should we validate that the amount of data written matches what we said
	// the length was?
	_, err = pdu.Data.WriteTo(w.out)

	return err
}
<|endoftext|>


<commit_before>package tracer

import (
	"context"
	"path/filepath"
	"runtime"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/log"
)

var DebugEnabled = int64(0)

func SetDebug(enabled bool) {
	if enabled {
		atomic.StoreInt64(&DebugEnabled, 1)
	} else {
		atomic.StoreInt64(&DebugEnabled, 0)
	}
}

type Segment struct {
	span    opentracing.Span
	records []opentracing.LogRecord
}

func (s *Segment) Finish() {
	s.span.FinishWithOptions(opentracing.FinishOptions{
		FinishTime: time.Now(),
		LogRecords: s.records,
	})
}

func (s *Segment) LogFields(fields ...log.Field) {
	s.span.LogFields(fields...)
}

func (s *Segment) SetBaggageItem(key, value string) {
	s.span.SetBaggageItem(key, value)
}

func (s *Segment) Info(msg string, fields ...log.Field) {
	if s.records == nil {
		s.records = make([]opentracing.LogRecord, 0, 8)
	}

	s.records = append(s.records, opentracing.LogRecord{
		Timestamp: time.Now(),
		Fields:    append(fields, log.String(MessageKey, msg), Caller(CallerKey, 2)),
	})
}

func (s *Segment) Debug(msg string, fields ...log.Field) {
	if DebugEnabled != 0 {
		return
	}

	if s.records == nil {
		s.records = make([]opentracing.LogRecord, 0, 8)
	}

	s.records = append(s.records, opentracing.LogRecord{
		Timestamp: time.Now(),
		Fields:    append(fields, log.String(MessageKey, msg), Caller(CallerKey, 2)),
	})
}

func Caller(key string, skip int) log.Field {
	_, file, line, _ := runtime.Caller(skip)
	return log.String(key, filepath.Base(filepath.Dir(file))+"/"+filepath.Base(file)+":"+strconv.Itoa(line))
}

func NewSegment(ctx context.Context, operationName string, fields ...log.Field) (*Segment, context.Context) {
	span, ctx := opentracing.StartSpanFromContext(ctx, operationName)
	span.LogFields(fields...)
	return &Segment{span: span}, ctx
}
<commit_msg>- Segment now handles nil gracefully<commit_after>package tracer

import (
	"context"
	"path/filepath"
	"runtime"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/log"
)

var DebugEnabled = int64(0)

var (
	NopSegment *Segment = nil
)

func SetDebug(enabled bool) {
	if enabled {
		atomic.StoreInt64(&DebugEnabled, 1)
	} else {
		atomic.StoreInt64(&DebugEnabled, 0)
	}
}

type Segment struct {
	span    opentracing.Span
	records []opentracing.LogRecord
}

func (s *Segment) Finish() {
	if s == nil {
		return
	}

	s.span.FinishWithOptions(opentracing.FinishOptions{
		FinishTime: time.Now(),
		LogRecords: s.records,
	})
}

func (s *Segment) LogFields(fields ...log.Field) {
	if s == nil {
		return
	}

	s.span.LogFields(fields...)
}

func (s *Segment) SetBaggageItem(key, value string) {
	if s == nil {
		return
	}

	s.span.SetBaggageItem(key, value)
}

func (s *Segment) Info(msg string, fields ...log.Field) {
	if s == nil {
		return
	}

	if s.records == nil {
		s.records = make([]opentracing.LogRecord, 0, 8)
	}

	s.records = append(s.records, opentracing.LogRecord{
		Timestamp: time.Now(),
		Fields:    append(fields, log.String(MessageKey, msg), Caller(CallerKey, 2)),
	})
}

func (s *Segment) Debug(msg string, fields ...log.Field) {
	if s == nil {
		return
	}

	if DebugEnabled != 0 {
		return
	}

	if s.records == nil {
		s.records = make([]opentracing.LogRecord, 0, 8)
	}

	s.records = append(s.records, opentracing.LogRecord{
		Timestamp: time.Now(),
		Fields:    append(fields, log.String(MessageKey, msg), Caller(CallerKey, 2)),
	})
}

func Caller(key string, skip int) log.Field {
	_, file, line, _ := runtime.Caller(skip)
	return log.String(key, filepath.Base(filepath.Dir(file))+"/"+filepath.Base(file)+":"+strconv.Itoa(line))
}

func NewSegment(ctx context.Context, operationName string, fields ...log.Field) (*Segment, context.Context) {
	span, ctx := opentracing.StartSpanFromContext(ctx, operationName)
	span.LogFields(fields...)
	return &Segment{span: span}, ctx
}
<|endoftext|>