<commit_before>package logx

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

var logger *Logger

func init() {
	chaos()
	if out == "file" { go poller() }
}

func chaos() {
	loadConfig()
	y, m, d := time.Now().Date()
	if logger == nil {
		logger = &Logger{
			look:        uint32(coreDead),
			fileName:    fileName,
			path:        filepath.Join(filePath, fileName),
			timestamp:   y*10000 + int(m)*100 + d*1,
			fileMaxSize: maxSize,
			bucket:      make(chan *bytes.Buffer, bucketLen),
			closeSignal: make(chan string),
			lock:        &sync.RWMutex{},
		}
	}
}

func (l *Logger) loadCurLogFile() error {
	l.link = filepath.Join(l.path, fileName+".log")
	actFileName, ok := isLinkFile(l.link)
	if !ok {
		return errors.New("is not link file")
	}
	l.fileName = filepath.Join(l.path, actFileName)
	f, err := openFile(l.fileName)
	if err != nil {
		return err
	}
	info, err := os.Stat(l.fileName)
	if err != nil { return err }
	sp := strings.Split(actFileName, ".")
	t, err := time.Parse("2006-01-02", sp[1])
	if err != nil {
		fmt.Errorf("loadCurrentLogFile |err=%v", err)
		return err
	}
	y, m, d := t.Date()
	l.timestamp = y*10000 + int(m)*100 + d*1
	l.file = f
	l.fileActualSize = int(info.Size())
	l.fileWriter = bufio.NewWriterSize(f, l.fileMaxSize)
	return nil
}

func (l *Logger) createFile() (err error) {
	if !pathIsExist(l.path) {
		if err = os.MkdirAll(l.path, os.ModePerm); err != nil { return }
	}
	now := time.Now()
	y, m, d := now.Date()
	l.timestamp = y*10000 + int(m)*100 + d*1
	l.fileName = filepath.Join(
		l.path,
		filepath.Base(os.Args[0])+"."+now.Format("2006-01-02.15.04.05.000")+".log")
	f, err := openFile(l.fileName)
	if err != nil { return err }
	l.file = f
	l.fileActualSize = 0
	l.fileWriter = bufio.NewWriterSize(f, l.fileMaxSize)
	l.link = filepath.Join(l.path, fileName+".log")
	return createLinkFile(l.fileName, l.link)
}

func (l *Logger) sync() {
	if l.lookRunning() { l.fileWriter.Flush() }
}

const fileMaxDelta = 100

func (l *Logger) rotate() bool {
	if !l.lookRunning() { return false }
	y, m, d := time.Now().Date()
	timestamp := y*10000 + int(m)*100 + d*1
	if l.fileActualSize <= l.fileMaxSize-fileMaxDelta && timestamp <= l.timestamp {
		return false
	}
	l.fileWriter.Flush()
	closeFile(l.file)
	if err := l.createFile(); err != nil { return false }
	return true
}

func (l *Logger) lookRunning() bool {
	if atomic.LoadUint32(&l.look) == uint32(coreRunning) { return true }
	return false
}

func (l *Logger) lookDead() bool {
	if atomic.LoadUint32(&l.look) == uint32(coreDead) { return true }
	return false
}

func (l *Logger) lookBlock() bool {
	if atomic.LoadUint32(&l.look) == uint32(coreBlock) { return true }
	return false
}

func (l *Logger) signalHandler() {
	var sigChan = make(chan os.Signal)
	signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT)
	for {
		select {
		case sig := <-sigChan:
			l.closeSignal <- "close"
			fmt.Println("LOGX receive os signal is ", sig)
			l.fileWriter.Flush()
			closeFile(l.file)
			atomic.SwapUint32(&l.look, uint32(coreDead))
			close(l.bucket)
			os.Exit(1)
		}
	}
}

func (l *Logger) release(buf *bytes.Buffer) { bufferPoolFree(buf) }

func caller() string {
	if pc, f, l, ok := runtime.Caller(2); ok {
		funcName := runtime.FuncForPC(pc).Name()
		return path.Base(f) + "|" + path.Base(funcName) + "|" + strconv.Itoa(l)
	}
	//pc := make([]uintptr, 3, 3)
	//cnt := runtime.Callers(6, pc)
	//
	//for i := 0; i < cnt; i++ {
	//	fu := runtime.FuncForPC(pc[i] - 1)
	//	name := fu.Name()
	//
	//	if !strings.Contains(name, "github.com/kafrax/logx") {
	//		f, l := fu.FileLine(pc[i] - 1)
	//		return path.Base(f) + "|" + path.Base(name) + "|" + strconv.Itoa(l)
	//	}
	//
	//	if pc, f, l, ok := runtime.Caller(8); ok {
	//		funcName := runtime.FuncForPC(pc).Name()
	//		return path.Base(f) + "|" + path.Base(funcName) + "|" + strconv.Itoa(l)
	//	}
	//}
	return ""
}

func print(buf *bytes.Buffer) {
	switch out {
	case "file": logger.bucket <- buf
	case "stdout": fmt.Print(buf.String())
	case "kafka": //todo send to kafka nsq etc.
	default: fmt.Println(buf.String())
	}
}

func Debugf(format string, msg ... interface{}) {
	if levelFlag > _DEBUG { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[DEBU][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Infof(format string, msg ... interface{}) {
	if levelFlag > _INFO { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[INFO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Warnf(format string, msg ... interface{}) {
	if levelFlag > _WARN { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[WARN][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Errorf(format string, msg ... interface{}) {
	if levelFlag > _ERR { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[ERRO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Fatalf(format string, msg ... interface{}) {
	if levelFlag > _DISASTER { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[FTAL][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Stackf(format string, msg ... interface{}) {
	s := fmt.Sprintf(format, msg...)
	s += "\n"
	buf := make([]byte, 1<<20)
	n := runtime.Stack(buf, true)
	s += string(buf[:n])
	s += "\n"
	fmt.Println("[STAC][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] " + s)
}

func Debug(msg ... interface{}) {
	if levelFlag > _DEBUG { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[DEBU][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Info(msg ... interface{}) {
	if levelFlag > _INFO { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[INFO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Warn(msg ... interface{}) {
	if levelFlag > _WARN { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[WARN][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Error(msg ... interface{}) {
	if levelFlag > _ERR { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[ERRO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Fatal(msg ... interface{}) {
	if levelFlag > _DISASTER { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[FTAL][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Stack(msg ... interface{}) {
	s := fmt.Sprintln(msg...)
	s += "\n"
	buf := make([]byte, 1<<20)
	n := runtime.Stack(buf, true)
	s += string(buf[:n])
	s += "\n"
	fmt.Println("[STAC][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] " + s)
}
<commit_msg>update<commit_after>package logx

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

var logger *Logger

func init() {
	chaos()
	if out == "file" { go poller() }
}

func chaos() {
	loadConfig()
	y, m, d := time.Now().Date()
	if logger == nil {
		logger = &Logger{
			look:        uint32(coreDead),
			fileName:    fileName,
			path:        filepath.Join(filePath, fileName),
			timestamp:   y*10000 + int(m)*100 + d*1,
			fileMaxSize: maxSize,
			bucket:      make(chan *bytes.Buffer, bucketLen),
			closeSignal: make(chan string),
			lock:        &sync.RWMutex{},
		}
	}
}

func (l *Logger) loadCurLogFile() error {
	l.link = filepath.Join(l.path, fileName+".log")
	actFileName, ok := isLinkFile(l.link)
	if !ok {
		return errors.New("is not link file")
	}
	l.fileName = filepath.Join(l.path, actFileName)
	f, err := openFile(l.fileName)
	if err != nil {
		return err
	}
	info, err := os.Stat(l.fileName)
	if err != nil { return err }
	sp := strings.Split(actFileName, ".")
	t, err := time.Parse("2006-01-02", sp[1])
	if err != nil {
		fmt.Errorf("loadCurrentLogFile |err=%v", err)
		return err
	}
	y, m, d := t.Date()
	l.timestamp = y*10000 + int(m)*100 + d*1
	l.file = f
	l.fileActualSize = int(info.Size())
	l.fileWriter = bufio.NewWriterSize(f, l.fileMaxSize)
	return nil
}

func (l *Logger) createFile() (err error) {
	if !pathIsExist(l.path) {
		if err = os.MkdirAll(l.path, os.ModePerm); err != nil { return }
	}
	now := time.Now()
	y, m, d := now.Date()
	l.timestamp = y*10000 + int(m)*100 + d*1
	l.fileName = filepath.Join(
		l.path,
		filepath.Base(os.Args[0])+"."+now.Format("2006-01-02.15.04.05.000")+".log")
	f, err := openFile(l.fileName)
	if err != nil { return err }
	l.file = f
	l.fileActualSize = 0
	l.fileWriter = bufio.NewWriterSize(f, l.fileMaxSize)
	l.link = filepath.Join(l.path, fileName+".log")
	return createLinkFile(l.fileName, l.link)
}

func (l *Logger) sync() {
	if l.lookRunning() { l.fileWriter.Flush() }
}

const fileMaxDelta = 100

func (l *Logger) rotate() bool {
	if !l.lookRunning() { return false }
	y, m, d := time.Now().Date()
	timestamp := y*10000 + int(m)*100 + d*1
	if l.fileActualSize <= l.fileMaxSize-fileMaxDelta && timestamp <= l.timestamp {
		return false
	}
	l.fileWriter.Flush()
	closeFile(l.file)
	if err := l.createFile(); err != nil { return false }
	return true
}

func (l *Logger) lookRunning() bool {
	if atomic.LoadUint32(&l.look) == uint32(coreRunning) { return true }
	return false
}

func (l *Logger) lookDead() bool {
	if atomic.LoadUint32(&l.look) == uint32(coreDead) { return true }
	return false
}

func (l *Logger) lookBlock() bool {
	if atomic.LoadUint32(&l.look) == uint32(coreBlock) { return true }
	return false
}

func (l *Logger) signalHandler() {
	var sigChan = make(chan os.Signal)
	signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL, syscall.SIGHUP, syscall.SIGQUIT)
	for {
		select {
		case sig := <-sigChan:
			l.closeSignal <- "close"
			fmt.Println("LOGX receive os signal is ", sig)
			l.fileWriter.Flush()
			closeFile(l.file)
			atomic.SwapUint32(&l.look, uint32(coreDead))
			close(l.bucket)
			os.Exit(1)
		}
	}
}

func (l *Logger) release(buf *bytes.Buffer) { bufferPoolFree(buf) }

func caller() string {
	if pc, f, l, ok := runtime.Caller(2); ok {
		funcName := runtime.FuncForPC(pc).Name()
		return path.Base(f) + "|" + path.Base(funcName) + "|" + strconv.Itoa(l)
	}
	//pc := make([]uintptr, 3, 3)
	//cnt := runtime.Callers(6, pc)
	//
	//for i := 0; i < cnt; i++ {
	//	fu := runtime.FuncForPC(pc[i] - 1)
	//	name := fu.Name()
	//
	//	if !strings.Contains(name, "github.com/kafrax/logx") {
	//		f, l := fu.FileLine(pc[i] - 1)
	//		return path.Base(f) + "|" + path.Base(name) + "|" + strconv.Itoa(l)
	//	}
	//
	//	if pc, f, l, ok := runtime.Caller(8); ok {
	//		funcName := runtime.FuncForPC(pc).Name()
	//		return path.Base(f) + "|" + path.Base(funcName) + "|" + strconv.Itoa(l)
	//	}
	//}
	return ""
}

func print(buf *bytes.Buffer) {
	switch out {
	case "file": logger.bucket <- buf
	case "stdout": fmt.Print(buf.String())
	case "kafka": //todo send to kafka nsq etc.
	default: fmt.Println(buf.String())
	}
}

func Debugf(format string, msg ... interface{}) {
	if levelFlag > _DEBUG { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[DEBU][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Infof(format string, msg ... interface{}) {
	if levelFlag > _INFO { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[INFO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Warnf(format string, msg ... interface{}) {
	if levelFlag > _WARN { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[WARN][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Errorf(format string, msg ... interface{}) {
	if levelFlag > _ERR { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[ERRO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Fatalf(format string, msg ... interface{}) {
	if levelFlag > _DISASTER { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[FTAL][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Stackf(format string, msg ... interface{}) {
	s := fmt.Sprintf(format, msg...)
	s += "\n"
	buf := make([]byte, 1<<20)
	n := runtime.Stack(buf, true)
	s += string(buf[:n])
	s += "\n"
	fmt.Println("[STAC][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] " + s)
}

func Debug(msg ... interface{}) {
	if levelFlag > _DEBUG { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[DEBU][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Info(msg ... interface{}) {
	if levelFlag > _INFO { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[INFO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Warn(msg ... interface{}) {
	if levelFlag > _WARN { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[WARN][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Error(msg ... interface{}) {
	if levelFlag > _ERR { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[ERRO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Fatal(msg ... interface{}) {
	if levelFlag > _DISASTER { return }
	buf := bufferPoolGet()
	buf.Write(s2b("[FTAL][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Stack(msg ... interface{}) {
	s := fmt.Sprintln(msg...)
	s += "\n"
	buf := make([]byte, 1<<20)
	n := runtime.Stack(buf, true)
	s += string(buf[:n])
	s += "\n"
	fmt.Println("[STAC][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] " + s)
}
<|endoftext|>


<commit_before>// Copyright 2014 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package orchestrator

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/route"
	"github.com/andreaskoch/allmark2/model"
	converter "github.com/andreaskoch/allmark2/services/converter/markdowntohtml"
	"github.com/andreaskoch/allmark2/web/view/viewmodel"
)

type ViewModelOrchestrator struct {
	*Orchestrator

	navigationOrchestrator *NavigationOrchestrator
	tagOrchestrator        *TagsOrchestrator
	fileOrchestrator       *FileOrchestrator
	locationOrchestrator   *LocationOrchestrator

	latestByRoute map[string][]*viewmodel.Model
}

func (orchestrator *ViewModelOrchestrator) blockingCacheWarmup() {
	orchestrator.GetLatest(route.New(), 5, 1)
}

func (orchestrator *ViewModelOrchestrator) GetFullViewModel(itemRoute route.Route) (viewModel viewmodel.Model, found bool) {

	// get the requested item
	item := orchestrator.getItem(itemRoute)
	if item == nil {
		return viewModel, false
	}

	// get the base view model
	viewModel = orchestrator.getViewModel(item)

	// navigation
	viewModel.ToplevelNavigation = orchestrator.navigationOrchestrator.GetToplevelNavigation()
	viewModel.BreadcrumbNavigation = orchestrator.navigationOrchestrator.GetBreadcrumbNavigation(itemRoute)
	viewModel.ItemNavigation = orchestrator.navigationOrchestrator.GetItemNavigation(itemRoute)

	// childs
	viewModel.Childs = orchestrator.getChildModels(itemRoute)

	// tags
	viewModel.Tags = orchestrator.tagOrchestrator.getItemTags(itemRoute)

	// Locations
	viewModel.Locations = orchestrator.locationOrchestrator.GetLocations(item.MetaData.Locations, func(i *model.Item) viewmodel.Model {
		return orchestrator.getViewModel(i)
	})

	// Geo Coordinates
	viewModel.GeoLocation = getGeoLocation(item)

	// Analytics Settings
	viewModel.Analytics = orchestrator.getAnalyticsSettings()

	// Hash / ETag
	viewModel.Hash = item.Hash

	// special viewmodel attributes
	isRepositoryItem := item.Type == model.TypeRepository
	if isRepositoryItem {

		// tag cloud
		repositoryIsNotEmpty := orchestrator.index().Size() >= 5 // don't bother to create a tag cloud if there aren't enough documents
		if repositoryIsNotEmpty {

			tagCloud := orchestrator.tagOrchestrator.GetTagCloud()
			viewModel.TagCloud = tagCloud

		}

	}

	return viewModel, true
}

func (orchestrator *ViewModelOrchestrator) GetViewModel(itemRoute route.Route) (viewModel viewmodel.Model, found bool) {

	// get the requested item
	item := orchestrator.getItem(itemRoute)
	if item == nil {
		return viewModel, false
	}

	return orchestrator.getViewModel(item), true

}

func (orchestrator *ViewModelOrchestrator) GetLatest(itemRoute route.Route, pageSize, page int) (latest []*viewmodel.Model, found bool) {

	cacheType := "latest"

	// load from cache
	if orchestrator.latestByRoute != nil {

		// re-prime the cache if it is stale
		if orchestrator.isCacheStale(cacheType) {
			go orchestrator.primeCache(cacheType)
		}

		// return the result
		if models, exists := orchestrator.latestByRoute[itemRoute.Value()]; exists {
			return pagedViewmodels(models, pageSize, page)
		}

		return []*viewmodel.Model{}, false

	}

	orchestrator.setCache(cacheType, func() {

		latestModelsByRoute := make(map[string][]*viewmodel.Model)

		for _, childRoute := range orchestrator.repository.Routes() {

			// get the latest items
			latestItems := orchestrator.getLatestItems(childRoute)

			// store the results
			latestModelsByRoute[childRoute.Value()] = orchestrator.getLastesViewModelsFromItemList(latestItems)

		}

		// save the result
		orchestrator.latestByRoute = latestModelsByRoute

	})

	// return a result
	if models, exists := orchestrator.latestByRoute[itemRoute.Value()]; exists {
		return pagedViewmodels(models, pageSize, page)
	}

	return []*viewmodel.Model{}, false
}

// Converts a list of model.Item elements into a view models for the latest-items controller
func (orchestrator *ViewModelOrchestrator) getLastesViewModelsFromItemList(items []*model.Item) []*viewmodel.Model {

	// create viewmodels
	models := make([]*viewmodel.Model, 0, len(items))
	for _, item := range items {

		viewModel := orchestrator.getViewModel(item)

		// prepare lazy loading
		viewModel.Content = converter.LazyLoad(viewModel.Content)

		models = append(models, &viewModel)
	}

	return models
}

func (orchestrator *ViewModelOrchestrator) getViewModel(item *model.Item) viewmodel.Model {

	itemRoute := item.Route()

	// get the root item
	root := orchestrator.rootItem()
	if root == nil {
		panic(fmt.Sprintf("Cannot get viewmodel for route %q because no root item was found.", itemRoute))
	}

	// convert content
	convertedContent, err := orchestrator.converter.Convert(orchestrator.getItemByAlias, orchestrator.relativePather(itemRoute), item)
	if err != nil {
		orchestrator.logger.Warn("Cannot convert content for item %q. Error: %s.", item.String(), err.Error())
		convertedContent = "<!-- Conversion Error -->"
	}

	// create a view model
	viewModel := viewmodel.Model{
		Base:    getBaseModel(root, item, orchestrator.itemPather()),
		Content: convertedContent,

		// files
		Files: orchestrator.fileOrchestrator.GetFiles(itemRoute),

		IsRepositoryItem: true,
	}

	return viewModel
}

func (orchestrator *ViewModelOrchestrator) getChildModels(itemRoute route.Route) []*viewmodel.Base {

	rootItem := orchestrator.rootItem()
	if rootItem == nil {
		orchestrator.logger.Fatal("No root item found")
	}

	pathProvider := orchestrator.relativePather(itemRoute)

	childModels := make([]*viewmodel.Base, 0)

	childItems := orchestrator.getChilds(itemRoute)
	for _, childItem := range childItems {
		baseModel := getBaseModel(rootItem, childItem, pathProvider)
		childModels = append(childModels, &baseModel)
	}

	// sort the models
	viewmodel.SortBaseModelBy(sortBaseModelsByDate).Sort(childModels)

	return childModels
}
<commit_msg>Cache getViewModel method<commit_after>// Copyright 2014 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package orchestrator

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/route"
	"github.com/andreaskoch/allmark2/model"
	converter "github.com/andreaskoch/allmark2/services/converter/markdowntohtml"
	"github.com/andreaskoch/allmark2/web/view/viewmodel"
)

type ViewModelOrchestrator struct {
	*Orchestrator

	navigationOrchestrator *NavigationOrchestrator
	tagOrchestrator        *TagsOrchestrator
	fileOrchestrator       *FileOrchestrator
	locationOrchestrator   *LocationOrchestrator

	latestByRoute     map[string][]*viewmodel.Model
	viewmodelsByRoute map[string]*viewmodel.Model
}

func (orchestrator *ViewModelOrchestrator) blockingCacheWarmup() {
	orchestrator.getViewModel(orchestrator.rootItem())
	orchestrator.GetLatest(route.New(), 5, 1)
}

func (orchestrator *ViewModelOrchestrator) GetFullViewModel(itemRoute route.Route) (viewModel viewmodel.Model, found bool) {

	// get the requested item
	item := orchestrator.getItem(itemRoute)
	if item == nil {
		return viewModel, false
	}

	// get the base view model
	viewModel = *orchestrator.getViewModel(item)

	// navigation
	viewModel.ToplevelNavigation = orchestrator.navigationOrchestrator.GetToplevelNavigation()
	viewModel.BreadcrumbNavigation = orchestrator.navigationOrchestrator.GetBreadcrumbNavigation(itemRoute)
	viewModel.ItemNavigation = orchestrator.navigationOrchestrator.GetItemNavigation(itemRoute)

	// childs
	viewModel.Childs = orchestrator.getChildModels(itemRoute)

	// tags
	viewModel.Tags = orchestrator.tagOrchestrator.getItemTags(itemRoute)

	// Locations
	viewModel.Locations = orchestrator.locationOrchestrator.GetLocations(item.MetaData.Locations, func(i *model.Item) viewmodel.Model {
		return *orchestrator.getViewModel(i)
	})

	// Geo Coordinates
	viewModel.GeoLocation = getGeoLocation(item)

	// Analytics Settings
	viewModel.Analytics = orchestrator.getAnalyticsSettings()

	// Hash / ETag
	viewModel.Hash = item.Hash

	// special viewmodel attributes
	isRepositoryItem := item.Type == model.TypeRepository
	if isRepositoryItem {

		// tag cloud
		repositoryIsNotEmpty := orchestrator.index().Size() >= 5 // don't bother to create a tag cloud if there aren't enough documents
		if repositoryIsNotEmpty {

			tagCloud := orchestrator.tagOrchestrator.GetTagCloud()
			viewModel.TagCloud = tagCloud

		}

	}

	return viewModel, true
}

func (orchestrator *ViewModelOrchestrator) GetViewModel(itemRoute route.Route) (viewModel viewmodel.Model, found bool) {

	// get the requested item
	item := orchestrator.getItem(itemRoute)
	if item == nil {
		return viewModel, false
	}

	return *orchestrator.getViewModel(item), true
}

func (orchestrator *ViewModelOrchestrator) GetLatest(itemRoute route.Route, pageSize, page int) (latest []*viewmodel.Model, found bool) {

	cacheType := "latest"

	// load from cache
	if orchestrator.latestByRoute != nil {

		// re-prime the cache if it is stale
		if orchestrator.isCacheStale(cacheType) {
			go orchestrator.primeCache(cacheType)
		}

		// return the result
		if models, exists := orchestrator.latestByRoute[itemRoute.Value()]; exists {
			return pagedViewmodels(models, pageSize, page)
		}

		return []*viewmodel.Model{}, false

	}

	orchestrator.setCache(cacheType, func() {

		latestModelsByRoute := make(map[string][]*viewmodel.Model)

		for _, childRoute := range orchestrator.repository.Routes() {

			// get the latest items
			latestItems := orchestrator.getLatestItems(childRoute)

			// store the results
			latestModelsByRoute[childRoute.Value()] = orchestrator.getLastesViewModelsFromItemList(latestItems)

		}

		// save the result
		orchestrator.latestByRoute = latestModelsByRoute

	})

	// return a result
	if models, exists := orchestrator.latestByRoute[itemRoute.Value()]; exists {
		return pagedViewmodels(models, pageSize, page)
	}

	return []*viewmodel.Model{}, false
}

// Converts a list of model.Item elements into a view models for the latest-items controller
func (orchestrator *ViewModelOrchestrator) getLastesViewModelsFromItemList(items []*model.Item) []*viewmodel.Model {

	// create viewmodels
	models := make([]*viewmodel.Model, 0, len(items))
	for _, item := range items {

		viewModel := orchestrator.getViewModel(item)

		// prepare lazy loading
		viewModel.Content = converter.LazyLoad(viewModel.Content)

		models = append(models, viewModel)
	}

	return models
}

func (orchestrator *ViewModelOrchestrator) getViewModel(item *model.Item) *viewmodel.Model {

	// get the root item
	root := orchestrator.rootItem()
	if root == nil {
		panic(fmt.Sprintf("Cannot get viewmodel for route %q because no root item was found.", item))
	}

	itemRoute := item.Route()
	cacheType := "viewmodels"

	// load from cache
	if orchestrator.viewmodelsByRoute != nil {

		// re-prime the cache if it is stale
		if orchestrator.isCacheStale(cacheType) {
			go orchestrator.primeCache(cacheType)
		}

		// return the result
		return orchestrator.viewmodelsByRoute[itemRoute.Value()]
	}

	orchestrator.setCache(cacheType, func() {

		viewmodelsByRoute := make(map[string]*viewmodel.Model)

		for _, child := range orchestrator.index().GetAllItems() {

			childRoute := child.Route()

			// convert content
			convertedContent, err := orchestrator.converter.Convert(orchestrator.getItemByAlias, orchestrator.relativePather(childRoute), child)
			if err != nil {
				orchestrator.logger.Warn("Cannot convert content for item %q. Error: %s.", child.String(), err.Error())
				convertedContent = "<!-- Conversion Error -->"
			}

			// create a view model
			viewModel := &viewmodel.Model{
				Base:    getBaseModel(root, child, orchestrator.itemPather()),
				Content: convertedContent,

				// files
				Files: orchestrator.fileOrchestrator.GetFiles(childRoute),

				IsRepositoryItem: true,
			}

			// store the view model
			viewmodelsByRoute[childRoute.Value()] = viewModel

		}

		orchestrator.viewmodelsByRoute = viewmodelsByRoute

	})

	return orchestrator.viewmodelsByRoute[itemRoute.Value()]
}

func (orchestrator *ViewModelOrchestrator) getChildModels(itemRoute route.Route) []*viewmodel.Base {

	rootItem := orchestrator.rootItem()
	if rootItem == nil {
		orchestrator.logger.Fatal("No root item found")
	}

	pathProvider := orchestrator.relativePather(itemRoute)

	childModels := make([]*viewmodel.Base, 0)

	childItems := orchestrator.getChilds(itemRoute)
	for _, childItem := range childItems {
		baseModel := getBaseModel(rootItem, childItem, pathProvider)
		childModels = append(childModels, &baseModel)
	}

	// sort the models
	viewmodel.SortBaseModelBy(sortBaseModelsByDate).Sort(childModels)

	return childModels
}
<|endoftext|>


<commit_before>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package prog

import (
	"fmt"

	"github.com/google/syzkaller/sys"
)

type validCtx struct {
	args map[*Arg]bool
	uses map[*Arg]*Arg
}

func (p *Prog) validate() error {
	ctx := &validCtx{make(map[*Arg]bool), make(map[*Arg]*Arg)}
	for _, c := range p.Calls {
		if err := c.validate(ctx); err != nil {
			return err
		}
	}
	for u, orig := range ctx.uses {
		if !ctx.args[u] {
			return fmt.Errorf("use of %+v referes to an out-of-tree arg\narg: %#v", *orig, u)
		}
	}
	return nil
}

func (c *Call) validate(ctx *validCtx) error {
	if c.Meta == nil {
		return fmt.Errorf("call does not have meta information")
	}
	if len(c.Args) != len(c.Meta.Args) {
		return fmt.Errorf("syscall %v: wrong number of arguments, want %v, got %v", c.Meta.Name, len(c.Meta.Args), len(c.Args))
	}
	var checkArg func(arg *Arg, typ sys.Type) error
	checkArg = func(arg *Arg, typ sys.Type) error {
		if arg == nil {
			return fmt.Errorf("syscall %v: nil arg", c.Meta.Name)
		}
		if ctx.args[arg] {
			return fmt.Errorf("syscall %v: arg is referenced several times in the tree", c.Meta.Name)
		}
		ctx.args[arg] = true
		for u := range arg.Uses {
			ctx.uses[u] = arg
		}
		if arg.Type == nil {
			return fmt.Errorf("syscall %v: no type", c.Meta.Name)
		}
		if arg.Type.Name() != typ.Name() {
			return fmt.Errorf("syscall %v: type name mismatch: %v vs %v", c.Meta.Name, arg.Type.Name(), typ.Name())
		}
		if arg.Type.Dir() == sys.DirOut {
			if arg.Val != 0 || arg.AddrPage != 0 || arg.AddrOffset != 0 {
				// We generate output len arguments, which makes sense
				// since it can be a length of a variable-length array
				// which is not known otherwise.
				if _, ok := arg.Type.(*sys.LenType); !ok {
					return fmt.Errorf("syscall %v: output arg '%v' has data", c.Meta.Name, typ.Name())
				}
			}
			for _, v := range arg.Data {
				if v != 0 {
					return fmt.Errorf("syscall %v: output arg '%v' has data", c.Meta.Name, typ.Name())
				}
			}
		}
		switch arg.Type.(type) {
		case *sys.ResourceType:
			switch arg.Kind {
			case ArgResult:
			case ArgReturn:
			case ArgConst:
				if arg.Type.Dir() == sys.DirOut && arg.Val != 0 {
					return fmt.Errorf("syscall %v: out resource arg '%v' has bad const value %v", c.Meta.Name, typ.Name(), arg.Val)
				}
			default:
				return fmt.Errorf("syscall %v: fd arg '%v' has bad kind %v", c.Meta.Name, typ.Name(), arg.Kind)
			}
		case *sys.StructType, *sys.ArrayType:
			switch arg.Kind {
			case ArgGroup:
			default:
				return fmt.Errorf("syscall %v: struct/array arg '%v' has bad kind %v", c.Meta.Name, typ.Name(), arg.Kind)
			}
		case *sys.UnionType:
			switch arg.Kind {
			case ArgUnion:
			default:
				return fmt.Errorf("syscall %v: union arg '%v' has bad kind %v", c.Meta.Name, typ.Name(), arg.Kind)
			}
		}
		switch arg.Kind {
		case ArgConst:
		case ArgResult:
			if arg.Res == nil {
				return fmt.Errorf("syscall %v: result arg '%v' has no reference", c.Meta.Name, typ.Name())
			}
			if !ctx.args[arg.Res] {
				return fmt.Errorf("syscall %v: result arg '%v' references out-of-tree result: %p%+v -> %p%+v",
					c.Meta.Name, typ.Name(), arg, arg, arg.Res, arg.Res)
			}
			if _, ok := arg.Res.Uses[arg]; !ok {
				return fmt.Errorf("syscall %v: result arg '%v' has broken link (%+v)", c.Meta.Name, typ.Name(), arg.Res.Uses)
			}
		case ArgPointer:
			switch typ1 := typ.(type) {
			case *sys.VmaType:
				if arg.Res != nil {
					return fmt.Errorf("syscall %v: vma arg '%v' has data", c.Meta.Name, typ.Name())
				}
				if arg.AddrPagesNum == 0 {
					return fmt.Errorf("syscall %v: vma arg '%v' has size 0", c.Meta.Name, typ.Name())
				}
			case *sys.PtrType:
				if arg.Type.Dir() != sys.DirIn {
					return fmt.Errorf("syscall %v: pointer arg '%v' has output direction", c.Meta.Name, typ.Name())
				}
				if arg.Res == nil && !typ.Optional() {
					return fmt.Errorf("syscall %v: non optional pointer arg '%v' is nil", c.Meta.Name, typ.Name())
				}
				if arg.Res != nil {
					if err := checkArg(arg.Res, typ1.Type); err != nil {
						return err
					}
				}
				if arg.AddrPagesNum != 0 {
					return fmt.Errorf("syscall %v: pointer arg '%v' has nonzero size", c.Meta.Name, typ.Name())
				}
			default:
				return fmt.Errorf("syscall %v: pointer arg '%v' has bad meta type %+v", c.Meta.Name, typ.Name(), typ)
			}
		case ArgPageSize:
		case ArgData:
		case ArgGroup:
			switch typ1 := typ.(type) {
			case *sys.StructType:
				if len(arg.Inner) != len(typ1.Fields) {
					return fmt.Errorf("syscall %v: struct arg '%v' has wrong number of fields: want %v, got %v", c.Meta.Name, typ.Name(), len(typ1.Fields), len(arg.Inner))
				}
				for i, arg1 := range arg.Inner {
					if err := checkArg(arg1, typ1.Fields[i]); err != nil {
						return err
					}
				}
			case *sys.ArrayType:
				for _, arg1 := range arg.Inner {
					if err := checkArg(arg1, typ1.Type); err != nil {
						return err
					}
				}
			default:
				return fmt.Errorf("syscall %v: group arg '%v' has bad underlying type %+v", c.Meta.Name, typ.Name(), typ)
			}
		case ArgUnion:
			typ1, ok := typ.(*sys.UnionType)
			if !ok {
				return fmt.Errorf("syscall %v: union arg '%v' has bad type", c.Meta.Name, typ.Name())
			}
			found := false
			for _, typ2 := range typ1.Options {
				if arg.OptionType.Name() == typ2.Name() {
					found = true
					break
				}
			}
			if !found {
				return fmt.Errorf("syscall %v: union arg '%v' has bad option", c.Meta.Name, typ.Name())
			}
			if err := checkArg(arg.Option, arg.OptionType); err != nil {
				return err
			}
		case ArgReturn:
		default:
			return fmt.Errorf("syscall %v: unknown arg '%v' kind", c.Meta.Name, typ.Name())
		}
		return nil
	}
	for i, arg := range c.Args {
		if arg.Kind == ArgReturn {
			return fmt.Errorf("syscall %v: arg '%v' has wrong return kind", c.Meta.Name, arg.Type.Name())
		}
		if err := checkArg(arg, c.Meta.Args[i]); err != nil {
			return err
		}
	}
	if c.Ret == nil {
		return fmt.Errorf("syscall %v: return value is absent", c.Meta.Name)
	}
	if c.Ret.Kind != ArgReturn {
		return fmt.Errorf("syscall %v: return value has wrong kind %v", c.Meta.Name, c.Ret.Kind)
	}
	if c.Meta.Ret != nil {
		if err := checkArg(c.Ret, c.Meta.Ret); err != nil {
			return err
		}
	} else if c.Ret.Type != nil {
		return fmt.Errorf("syscall %v: return value has spurious type: %+v", c.Meta.Name, c.Ret.Type)
	}
	return nil
}
<commit_msg>prog: better validate arg data<commit_after>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package prog

import (
	"fmt"

	"github.com/google/syzkaller/sys"
)

type validCtx struct {
	args map[*Arg]bool
	uses map[*Arg]*Arg
}

func (p *Prog) validate() error {
	ctx := &validCtx{make(map[*Arg]bool), make(map[*Arg]*Arg)}
	for _, c := range p.Calls {
		if err := c.validate(ctx); err != nil {
			return err
		}
	}
	for u, orig := range ctx.uses {
		if !ctx.args[u] {
			return fmt.Errorf("use of %+v referes to an out-of-tree arg\narg: %#v", *orig, u)
		}
	}
	return nil
}

func (c *Call) validate(ctx *validCtx) error {
	if c.Meta == nil {
		return fmt.Errorf("call does not have meta information")
	}
	if len(c.Args) != len(c.Meta.Args) {
		return fmt.Errorf("syscall %v: wrong number of arguments, want %v, got %v", c.Meta.Name, len(c.Meta.Args), len(c.Args))
	}
	var checkArg func(arg *Arg, typ sys.Type) error
	checkArg = func(arg *Arg, typ sys.Type) error {
		if arg == nil {
			return fmt.Errorf("syscall %v: nil arg", c.Meta.Name)
		}
		if ctx.args[arg] {
			return fmt.Errorf("syscall %v: arg is referenced several times in the tree", c.Meta.Name)
		}
		ctx.args[arg] = true
		for u := range arg.Uses {
			ctx.uses[u] = arg
		}
		if arg.Type == nil {
			return fmt.Errorf("syscall %v: no type", c.Meta.Name)
		}
		if arg.Type.Name() != typ.Name() {
			return fmt.Errorf("syscall %v: type name mismatch: %v vs %v", c.Meta.Name, arg.Type.Name(), typ.Name())
		}
		if arg.Type.Dir() == sys.DirOut {
			if arg.Val != 0 || arg.AddrPage != 0 || arg.AddrOffset != 0 {
				// We generate output len arguments, which makes sense
				// since it can be a length of a variable-length array
				// which is not known otherwise.
				if _, ok := arg.Type.(*sys.LenType); !ok {
					return fmt.Errorf("syscall %v: output arg '%v' has data", c.Meta.Name, typ.Name())
				}
			}
			for _, v := range arg.Data {
				if v != 0 {
					return fmt.Errorf("syscall %v: output arg '%v' has data", c.Meta.Name, typ.Name())
				}
			}
		}
		switch arg.Type.(type) {
		case *sys.ResourceType:
			switch arg.Kind {
			case ArgResult:
			case ArgReturn:
			case ArgConst:
				if arg.Type.Dir() == sys.DirOut && arg.Val != 0 {
					return fmt.Errorf("syscall %v: out resource arg '%v' has bad const value %v", c.Meta.Name, typ.Name(), arg.Val)
				}
			default:
				return fmt.Errorf("syscall %v: fd arg '%v' has bad kind %v", c.Meta.Name, typ.Name(), arg.Kind)
			}
		case *sys.StructType, *sys.ArrayType:
			switch arg.Kind {
			case ArgGroup:
			default:
				return fmt.Errorf("syscall %v: struct/array arg '%v' has bad kind %v", c.Meta.Name, typ.Name(), arg.Kind)
			}
		case *sys.UnionType:
			switch arg.Kind {
			case ArgUnion:
			default:
				return fmt.Errorf("syscall %v: union arg '%v' has bad kind %v", c.Meta.Name, typ.Name(), arg.Kind)
			}
		}
		switch arg.Kind {
		case ArgConst:
		case ArgResult:
			if arg.Res == nil {
				return fmt.Errorf("syscall %v: result arg '%v' has no reference", c.Meta.Name, typ.Name())
			}
			if !ctx.args[arg.Res] {
				return fmt.Errorf("syscall %v: result arg '%v' references out-of-tree result: %p%+v -> %p%+v",
					c.Meta.Name, typ.Name(), arg, arg, arg.Res, arg.Res)
			}
			if _, ok := arg.Res.Uses[arg]; !ok {
				return fmt.Errorf("syscall %v: result arg '%v' has broken link (%+v)", c.Meta.Name, typ.Name(), arg.Res.Uses)
			}
		case ArgPointer:
			switch typ1 := typ.(type) {
			case *sys.VmaType:
				if arg.Res != nil {
					return fmt.Errorf("syscall %v: vma arg '%v' has data", c.Meta.Name, typ.Name())
				}
				if arg.AddrPagesNum == 0 {
					return fmt.Errorf("syscall %v: vma arg '%v' has size 0", c.Meta.Name, typ.Name())
				}
			case *sys.PtrType:
				if arg.Type.Dir() != sys.DirIn {
					return fmt.Errorf("syscall %v: pointer arg '%v' has output direction", c.Meta.Name, typ.Name())
				}
				if arg.Res == nil && !typ.Optional() {
					return fmt.Errorf("syscall %v: non optional pointer arg '%v' is nil", c.Meta.Name, typ.Name())
				}
				if arg.Res != nil {
					if err := checkArg(arg.Res, typ1.Type); err != nil {
						return err
					}
				}
				if arg.AddrPagesNum != 0 {
					return fmt.Errorf("syscall %v: pointer arg '%v' has nonzero size", c.Meta.Name, typ.Name())
				}
			default:
				return fmt.Errorf("syscall %v: pointer arg '%v' has bad meta type %+v", c.Meta.Name, typ.Name(), typ)
			}
		case ArgPageSize:
		case ArgData:
			switch typ1 := typ.(type) {
			case *sys.ArrayType:
				if typ2, ok := typ1.Type.(*sys.IntType); !ok || typ2.Size() != 1 {
					return fmt.Errorf("syscall %v: data arg '%v' should be an array", c.Meta.Name, typ.Name())
				}
			}
		case ArgGroup:
			switch typ1 := typ.(type) {
			case *sys.StructType:
				if len(arg.Inner) != len(typ1.Fields) {
					return fmt.Errorf("syscall %v: struct arg '%v' has wrong number of fields: want %v, got %v", c.Meta.Name, typ.Name(), len(typ1.Fields), len(arg.Inner))
				}
				for i, arg1 := range arg.Inner {
					if err := checkArg(arg1, typ1.Fields[i]); err != nil {
						return err
					}
				}
			case *sys.ArrayType:
				for _, arg1 := range arg.Inner {
					if err := checkArg(arg1, typ1.Type); err != nil {
						return err
					}
				}
			default:
				return fmt.Errorf("syscall %v: group arg '%v' has bad underlying type %+v", c.Meta.Name, typ.Name(), typ)
			}
		case ArgUnion:
			typ1, ok := typ.(*sys.UnionType)
			if !ok {
				return fmt.Errorf("syscall %v: union arg '%v' has bad type", c.Meta.Name, typ.Name())
			}
			found := false
			for _, typ2 := range typ1.Options {
				if arg.OptionType.Name() == typ2.Name() {
					found = true
					break
				}
			}
			if !found {
				return fmt.Errorf("syscall %v: union arg '%v' has bad option", c.Meta.Name, typ.Name())
			}
			if err := checkArg(arg.Option, arg.OptionType); err != nil {
				return err
			}
		case ArgReturn:
		default:
			return fmt.Errorf("syscall %v: unknown arg '%v' kind", c.Meta.Name, typ.Name())
		}
		return nil
	}
	for i, arg := range c.Args {
		if arg.Kind == ArgReturn {
			return fmt.Errorf("syscall %v: arg '%v' has wrong return kind", c.Meta.Name, arg.Type.Name())
		}
		if err := checkArg(arg, c.Meta.Args[i]); err != nil {
			return err
		}
	}
	if c.Ret == nil {
		return fmt.Errorf("syscall %v: return value is absent", c.Meta.Name)
	}
	if c.Ret.Kind != ArgReturn {
		return fmt.Errorf("syscall %v: return value has wrong kind %v", c.Meta.Name, c.Ret.Kind)
	}
	if c.Meta.Ret != nil {
		if err := checkArg(c.Ret, c.Meta.Ret); err != nil {
			return err
		}
	} else if c.Ret.Type != nil {
		return fmt.Errorf("syscall %v: return value has spurious type: %+v", c.Meta.Name, c.Ret.Type)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"sync"

	"github.com/caelifer/dups/balancer"
)

// Node type
type Node struct {
	Path string // File path
	Hash string // Crypto signature in string form
}

// Constructor function from
func MakeNode(path string, fi os.FileInfo) (Node, error) {
	node := Node{Path: path}

	// Open file
	file, err := os.Open(path)
	if err != nil {
		return node, err
	}
	defer file.Close()

	var n int64 // bytes read
	hash := sha1.New()

	// Filesystem and memory optimal read
	n, err = io.Copy(hash, file)

	// Check for normal errors
	if err != nil {
		return node, err
	}

	// Paranoid sanity check
	if n != fi.Size() {
		return node, errors.New("Partial read")
	}

	// Add hash value
	node.Hash = fmt.Sprintf("%0x", hash.Sum(nil))

	// Collect garbage
	hash = nil

	return node, nil
}

// Dup type
type Dup struct {
	Hash  string   // Crypto signature
	Paths []string // Paths with matching signatures
}

func (d Dup) String() string {
	out := ""
	hash := d.Hash
	num := len(d.Paths)

	for _, p := range d.Paths {
		out += fmt.Sprintf("%s(%d):%q\n", hash, num, p)
	}
	return out
}

// Global pool manager
var WorkQueue = make(chan balancer.Request)

// Start balancer on the background
var _ = balancer.NewBalancer(WorkQueue)

func main() {
	// Use all available CPU cores
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Process command line params
	paths := os.Args[1:]

	if len(paths) == 0 {
		// Default is current directory
		paths = []string{"."}
	}
	// Start worker pool manager
	quit := make(chan bool)

	// Start map-reduce
	for dup := range Reduce(Map(paths)) {
		fmt.Println(dup)
	}
	// Stop pool manager
	// quit <- true
}

func Reduce(in <-chan Node) <-chan Dup {
	out := make(chan Dup)
	nodes := make(map[string][]string)

	// Synchronously populate thread-unsafe map
	for n := range in {
		if v, ok := nodes[n.Hash]; ok {
			// Found dups
			nodes[n.Hash] = append(v, n.Path)
			// log.Printf("DEBUG Found dups for %s - %+v\n", n.Hash, nodes[n.Hash])
		} else {
			// Add new node
			nodes[n.Hash] = []string{n.Path}
		}
	}

	// Asyncronously send output
	go func() {
		for hash, paths := range nodes {
			if len(paths) > 1 {
				// Construct Dup and send it out
				out <- Dup{
					Hash:  hash,
					Paths: paths,
				}
			}
		}
		// Don't foget to clean-up
		close(out)
	}()

	return out
}

func Map(paths []string) <-chan Node {
	out := make(chan Node)

	// Start filesystem tree walking
	go func() {
		var wg sync.WaitGroup

		// Process all command line paths
		for _, p := range paths {
			err := filepath.Walk(p, func(path string, info os.FileInfo, err error) error {
				// Handle passthrough error
				if err != nil {
					log.Println("WARN", err)
					return nil
				}

				// Only process simple files
				if IsFile(info) {
					// Add to wait group
					wg.Add(1)

					// Calculate hash using balancer
					WorkQueue <- func() {
						defer wg.Done() // Signal done

						n, err := MakeNode(path, info)
						if err != nil {
							log.Println("WARN", err)
							return
						}
						out <- n
					}
				}
				return nil
			})

			if err != nil {
				log.Fatal(err)
			}
		}

		// Wait until all results are in
		wg.Wait()

		// Close output channel when all nodes were processed
		close(out)
	}()

	return out
}

func IsFile(fi os.FileInfo) bool {
	return fi.Mode()&os.ModeType == 0
}
<commit_msg>Get rid of quit channel; the whole machine is going to be dismantle at exit anyway.<commit_after>package main

import (
	"crypto/sha1"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"sync"

	"github.com/caelifer/dups/balancer"
)

// Node type
type Node struct {
	Path string // File path
	Hash string // Crypto signature in string form
}

// Constructor function from
func MakeNode(path string, fi os.FileInfo) (Node, error) {
	node := Node{Path: path}

	// Open file
	file, err := os.Open(path)
	if err != nil {
		return node, err
	}
	defer file.Close()

	var n int64 // bytes read
	hash := sha1.New()

	// Filesystem and memory optimal read
	n, err = io.Copy(hash, file)

	// Check for normal errors
	if err != nil {
		return node, err
	}

	// Paranoid sanity check
	if n != fi.Size() {
		return node, errors.New("Partial read")
	}

	// Add hash value
	node.Hash = fmt.Sprintf("%0x", hash.Sum(nil))

	// Collect garbage
	hash = nil

	return node, nil
}

// Dup type
type Dup struct {
	Hash  string   // Crypto signature
	Paths []string // Paths with matching signatures
}

func (d Dup) String() string {
	out := ""
	hash := d.Hash
	num := len(d.Paths)

	for _, p := range d.Paths {
		out += fmt.Sprintf("%d:%s:%q\n", num, hash, p)
	}
	return out
}

// Global pool manager
var WorkQueue = make(chan balancer.Request)

// Start balancer on the background
var _ = balancer.NewBalancer(WorkQueue)

func main() {
	// Use all available CPU cores
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Process command line params
	paths := os.Args[1:]

	if len(paths) == 0 {
		// Default is current directory
		paths = []string{"."}
	}

	// Start map-reduce
	for dup := range Reduce(Map(paths)) {
		fmt.Println(dup)
	}
}

func Reduce(in <-chan Node) <-chan Dup {
	out := make(chan Dup)
	nodes := make(map[string][]string)

	// Synchronously populate thread-unsafe map
	for n := range in {
		if v, ok := nodes[n.Hash]; ok {
			// Found dups
			nodes[n.Hash] = append(v, n.Path)
			// log.Printf("DEBUG Found dups for %s - %+v\n", n.Hash, nodes[n.Hash])
		} else {
			// Add new node
			nodes[n.Hash] = []string{n.Path}
		}
	}

	// Asyncronously send output
	go func() {
		for hash, paths := range nodes {
			if len(paths) > 1 {
				// Construct Dup and send it out
				out <- Dup{
					Hash:  hash,
					Paths: paths,
				}
			}
		}
		// Don't foget to clean-up
		close(out)
	}()

	return out
}

func Map(paths []string) <-chan Node {
	out := make(chan Node)

	// Start filesystem tree walking
	go func() {
		var wg sync.WaitGroup

		// Process all command line paths
		for _, p := range paths {
			err := filepath.Walk(p, func(path string, info os.FileInfo, err error) error {
				// Handle passthrough error
				if err != nil {
					log.Println("WARN", err)
					return nil
				}

				// Only process simple files
				if IsFile(info) {
					// Add to wait group
					wg.Add(1)

					// Calculate hash using balancer
					WorkQueue <- func() {
						defer wg.Done() // Signal done

						n, err := MakeNode(path, info)
						if err != nil {
							log.Println("WARN", err)
							return
						}
						out <- n
					}
				}
				return nil
			})

			if err != nil {
				log.Fatal(err)
			}
		}

		// Wait until all results are in
		wg.Wait()

		// Close output channel when all nodes were processed
		close(out)
	}()

	return out
}

func IsFile(fi os.FileInfo) bool {
	return fi.Mode()&os.ModeType == 0
}
<|endoftext|>


<commit_before>package gorma

import (
	"fmt"
	"sort"
	"strings"
	"unicode"

	"bitbucket.org/pkg/inflect"

	"github.com/goadesign/goa/design"
	"github.com/goadesign/goa/dslengine"
	"github.com/goadesign/goa/goagen/codegen"
)

// NewRelationalModelDefinition returns an initialized
// RelationalModelDefinition.
func NewRelationalModelDefinition() *RelationalModelDefinition {
	baseAttr := &design.AttributeDefinition{}
	utd := &design.UserTypeDefinition{
		AttributeDefinition: baseAttr,
	}
	utd.Type = design.Object{}
	m := &RelationalModelDefinition{
		RelationalFields: make(map[string]*RelationalFieldDefinition),
		BuiltFrom:        make(map[string]*design.UserTypeDefinition),
		RenderTo:         make(map[string]*design.MediaTypeDefinition),
		BelongsTo:        make(map[string]*RelationalModelDefinition),
		HasMany:          make(map[string]*RelationalModelDefinition),
		HasOne:           make(map[string]*RelationalModelDefinition),
		ManyToMany:       make(map[string]*ManyToManyDefinition),
		UserTypeDefinition: &design.UserTypeDefinition{
			AttributeDefinition: baseAttr,
		},
	}
	return m
}

// Context returns the generic definition name used in error messages.
func (f *RelationalModelDefinition) Context() string {
	if f.ModelName != "" {
		return fmt.Sprintf("RelationalModel %#v", f.Name())
	}
	return "unnamed RelationalModel"
}

// DSL returns this object's DSL.
func (f *RelationalModelDefinition) DSL() func() {
	return f.DefinitionDSL
}

// TableName returns the TableName of the struct.
func (f RelationalModelDefinition) TableName() string {
	return inflect.Underscore(inflect.Pluralize(f.ModelName))
}

// Children returns a slice of this objects children.
func (f RelationalModelDefinition) Children() []dslengine.Definition {
	var stores []dslengine.Definition
	for _, s := range f.RelationalFields {
		stores = append(stores, s)
	}
	return stores
}

// PKAttributes constructs a pair of field + definition strings
// useful for method parameters.
func (f *RelationalModelDefinition) PKAttributes() string {
	var attr []string
	for _, pk := range f.PrimaryKeys {
		attr = append(attr, fmt.Sprintf("%s %s", codegen.Goify(pk.DatabaseFieldName, false), goDatatype(pk, true)))
	}
	return strings.Join(attr, ",")
}

// PKWhere returns an array of strings representing the where clause
// of a retrieval by primary key(s) -- x = ? and y = ?.
func (f *RelationalModelDefinition) PKWhere() string {
	var pkwhere []string
	for _, pk := range f.PrimaryKeys {
		def := fmt.Sprintf("%s = ?", pk.DatabaseFieldName)
		pkwhere = append(pkwhere, def)
	}
	return strings.Join(pkwhere, " and ")
}

// PKWhereFields returns the fields for a where clause for the primary
// keys of a model.
func (f *RelationalModelDefinition) PKWhereFields() string {
	var pkwhere []string
	for _, pk := range f.PrimaryKeys {
		def := fmt.Sprintf("%s", codegen.Goify(pk.DatabaseFieldName, false))
		pkwhere = append(pkwhere, def)
	}
	return strings.Join(pkwhere, ",")
}

// PKUpdateFields returns something?  This function doesn't look useful in
// current form.  Perhaps it isn't.
func (f *RelationalModelDefinition) PKUpdateFields(modelname string) string {
	var pkwhere []string
	for _, pk := range f.PrimaryKeys {
		def := fmt.Sprintf("%s.%s", modelname, codegen.Goify(pk.FieldName, true))
		pkwhere = append(pkwhere, def)
	}

	pkw := strings.Join(pkwhere, ",")
	return pkw
}

// StructDefinition returns the struct definition for the model.
func (f *RelationalModelDefinition) StructDefinition() string {
	header := fmt.Sprintf("type %s struct {\n", f.ModelName)
	var output string
	f.IterateFields(func(field *RelationalFieldDefinition) error {
		output = output + field.FieldDefinition()
		return nil
	})

	// Get a sortable slice of BelongsTo relationships
	var keys []string
	for k := range f.BelongsTo {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, k := range keys {
		output = output + f.BelongsTo[k].ModelName + "\t" + f.BelongsTo[k].ModelName + "\n"
	}
	footer := "}\n"
	return header + output + footer
}

// Attribute implements the Container interface of goa.
func (f *RelationalModelDefinition) Attribute() *design.AttributeDefinition {
	return f.AttributeDefinition
}

// Project does something interesting, and I don't remember if I use it
// anywhere.
//
// TODO find out
func (f *RelationalModelDefinition) Project(name, v string) *design.MediaTypeDefinition {
	p, _, _ := f.RenderTo[name].Project(v)
	return p
}

// LowerName returns the model name as a lowercase string.
func (f *RelationalModelDefinition) LowerName() string {
	return codegen.Goify(strings.ToLower(f.ModelName), false)
}

// Underscore returns the model name as a lowercase string in snake case.
func (f *RelationalModelDefinition) Underscore() string {
	runes := []rune(f.ModelName)
	length := len(runes)

	var out []rune
	for i := 0; i < length; i++ {
		if i > 0 && unicode.IsUpper(runes[i]) && ((i+1 < length && unicode.IsLower(runes[i+1])) || unicode.IsLower(runes[i-1])) {
			out = append(out, '_')
		}
		out = append(out, unicode.ToLower(runes[i]))
	}

	return string(out)
}

// IterateBuildSources runs an iterator function once per Model in the Store's model list.
func (f *RelationalModelDefinition) IterateBuildSources(it BuildSourceIterator) error {

	for _, bs := range f.BuildSources {
		if err := it(bs); err != nil {
			return err
		}
	}
	return nil
}

// IterateFields returns an iterator function useful for iterating through
// this model's field list.
func (f *RelationalModelDefinition) IterateFields(it FieldIterator) error {
	// Break out each type of fields

	var pkkeys []string
	pks := make(map[string]string)
	for n := range f.RelationalFields {
		if f.RelationalFields[n].PrimaryKey {
			pks[n] = n
			pkkeys = append(pkkeys, n)
		}
	}
	sort.Strings(pkkeys)

	var namekeys []string
	names := make(map[string]string)
	for n := range f.RelationalFields {
		if !f.RelationalFields[n].PrimaryKey && !f.RelationalFields[n].Timestamp {
			names[n] = n
			namekeys = append(namekeys, n)
		}
	}
	sort.Strings(namekeys)

	var datekeys []string
	dates := make(map[string]string)
	for n := range f.RelationalFields {
		if f.RelationalFields[n].Timestamp {
			dates[n] = n
			datekeys = append(datekeys, n)
		}
	}
	sort.Strings(datekeys)

	// Combine the sorted slices
	var fields []string
	fields = append(fields, pkkeys...)
	fields = append(fields, namekeys...)
	fields = append(fields, datekeys...)

	// Iterate them
	for _, n := range fields {
		if err := it(f.RelationalFields[n]); err != nil {
			return err
		}
	}
	return nil
}

// PopulateFromModeledType creates fields for the model
// based on the goa UserTypeDefinition it models, which is
// set using BuildsFrom().
// This happens before fields are processed, so it's
// ok to just assign without testing.
func (f *RelationalModelDefinition) PopulateFromModeledType() {
	if f.BuiltFrom == nil {
		return
	}
	for _, utd := range f.BuiltFrom {
		obj := utd.ToObject()
		obj.IterateAttributes(func(name string, att *design.AttributeDefinition) error {
			rf, ok := f.RelationalFields[codegen.Goify(name, true)]
			if ok {
				// We already have a mapping for this field.  What to do?
				if rf.Datatype != "" {
					return nil
				}
				// we may have seen the field but don't know its type
				// TODO(BJK) refactor this into separate func later
				switch att.Type.Kind() {
				case design.BooleanKind:
					rf.Datatype = Boolean
				case design.IntegerKind:
					rf.Datatype = Integer
				case design.NumberKind:
					rf.Datatype = Decimal
				case design.StringKind:
					rf.Datatype = String
				case design.DateTimeKind:
					rf.Datatype = Timestamp
				case design.MediaTypeKind:
					// Embedded MediaType
					// Skip for now?
					return nil

				default:
					dslengine.ReportError("Unsupported type: %#v %s", att.Type.Kind(), att.Type.Name())
				}
				if !utd.IsRequired(name) {
					rf.Nullable = true
				}
			}

			rf = &RelationalFieldDefinition{}
			rf.Parent = f
			rf.FieldName = codegen.Goify(name, true)

			if strings.HasSuffix(rf.FieldName, "Id") {
				rf.FieldName = strings.TrimSuffix(rf.FieldName, "Id")
				rf.FieldName = rf.FieldName + "ID"
			}
			switch att.Type.Kind() {
			case design.BooleanKind:
				rf.Datatype = Boolean
			case design.IntegerKind:
				rf.Datatype = Integer
			case design.NumberKind:
				rf.Datatype = Decimal
			case design.StringKind:
				rf.Datatype = String
			case design.DateTimeKind:
				rf.Datatype = Timestamp
			case design.MediaTypeKind:
				// Embedded MediaType
				// Skip for now?
				return nil

			default:
				dslengine.ReportError("Unsupported type: %#v %s", att.Type.Kind(), att.Type.Name())
			}
			if !utd.IsRequired(name) {
				rf.Nullable = true
			}
			// might need this later?
			rf.a = att
			f.RelationalFields[rf.FieldName] = rf

			addAttributeToModel(name, att, f)

			return nil
		})
	}
	return
}

func addAttributeToModel(name string, att *design.AttributeDefinition, m *RelationalModelDefinition) {
	var parent *design.AttributeDefinition
	parent = m.AttributeDefinition
	if parent != nil {
		if parent.Type == nil {
			parent.Type = design.Object{}
		}
		if _, ok := parent.Type.(design.Object); !ok {
			dslengine.ReportError("can't define child attributes on attribute of type %s", parent.Type.Name())
			return
		}

		parent.Type.(design.Object)[name] = att
	}

}

// copied from Goa
func parseAttributeArgs(baseAttr *design.AttributeDefinition, args ...interface{}) (design.DataType, string, func()) {
	var (
		dataType    design.DataType
		description string
		dsl         func()
		ok          bool
	)

	parseDataType := func(expected string, index int) {
		if name, ok := args[index].(string); ok {
			// Lookup type by name
			if dataType, ok = design.Design.Types[name]; !ok {
				if dataType = design.Design.MediaTypeWithIdentifier(name); dataType == nil {
					dslengine.InvalidArgError(expected, args[index])
				}
			}
			return
		}
		if dataType, ok = args[index].(design.DataType); !ok {
			dslengine.InvalidArgError(expected, args[index])
		}
	}
	parseDescription := func(expected string, index int) {
		if description, ok = args[index].(string); !ok {
			dslengine.InvalidArgError(expected, args[index])
		}
	}
	parseDSL := func(index int, success, failure func()) {
		if dsl, ok = args[index].(func()); ok {
			success()
		} else {
			failure()
		}
	}

	success := func() {}

	switch len(args) {
	case 0:
		if baseAttr != nil {
			dataType = baseAttr.Type
		} else {
			dataType = design.String
		}
	case 1:
		success = func() {
			if baseAttr != nil {
				dataType = baseAttr.Type
			}
		}
		parseDSL(0, success, func() { parseDataType("type, type name or func()", 0) })
	case 2:
		parseDataType("type or type name", 0)
		parseDSL(1, success, func() { parseDescription("string or func()", 1) })
	case 3:
		parseDataType("type or type name", 0)
		parseDescription("string", 1)
		parseDSL(2, success, func() { dslengine.InvalidArgError("func()", args[2]) })
	default:
		dslengine.ReportError("too many arguments in call to Attribute")
	}

	return dataType, description, dsl
}
<commit_msg>Change the Datatype for NumberKind to BigDecimal<commit_after>package gorma

import (
	"fmt"
	"sort"
	"strings"
	"unicode"

	"bitbucket.org/pkg/inflect"

	"github.com/goadesign/goa/design"
	"github.com/goadesign/goa/dslengine"
	"github.com/goadesign/goa/goagen/codegen"
)

// NewRelationalModelDefinition returns an initialized
// RelationalModelDefinition.
func NewRelationalModelDefinition() *RelationalModelDefinition {
	baseAttr := &design.AttributeDefinition{}
	utd := &design.UserTypeDefinition{
		AttributeDefinition: baseAttr,
	}
	utd.Type = design.Object{}
	m := &RelationalModelDefinition{
		RelationalFields: make(map[string]*RelationalFieldDefinition),
		BuiltFrom:        make(map[string]*design.UserTypeDefinition),
		RenderTo:         make(map[string]*design.MediaTypeDefinition),
		BelongsTo:        make(map[string]*RelationalModelDefinition),
		HasMany:          make(map[string]*RelationalModelDefinition),
		HasOne:           make(map[string]*RelationalModelDefinition),
		ManyToMany:       make(map[string]*ManyToManyDefinition),
		UserTypeDefinition: &design.UserTypeDefinition{
			AttributeDefinition: baseAttr,
		},
	}
	return m
}

// Context returns the generic definition name used in error messages.
func (f *RelationalModelDefinition) Context() string {
	if f.ModelName != "" {
		return fmt.Sprintf("RelationalModel %#v", f.Name())
	}
	return "unnamed RelationalModel"
}

// DSL returns this object's DSL.
func (f *RelationalModelDefinition) DSL() func() {
	return f.DefinitionDSL
}

// TableName returns the TableName of the struct.
func (f RelationalModelDefinition) TableName() string {
	return inflect.Underscore(inflect.Pluralize(f.ModelName))
}

// Children returns a slice of this objects children.
func (f RelationalModelDefinition) Children() []dslengine.Definition {
	var stores []dslengine.Definition
	for _, s := range f.RelationalFields {
		stores = append(stores, s)
	}
	return stores
}

// PKAttributes constructs a pair of field + definition strings
// useful for method parameters.
func (f *RelationalModelDefinition) PKAttributes() string {
	var attr []string
	for _, pk := range f.PrimaryKeys {
		attr = append(attr, fmt.Sprintf("%s %s", codegen.Goify(pk.DatabaseFieldName, false), goDatatype(pk, true)))
	}
	return strings.Join(attr, ",")
}

// PKWhere returns an array of strings representing the where clause
// of a retrieval by primary key(s) -- x = ? and y = ?.
func (f *RelationalModelDefinition) PKWhere() string {
	var pkwhere []string
	for _, pk := range f.PrimaryKeys {
		def := fmt.Sprintf("%s = ?", pk.DatabaseFieldName)
		pkwhere = append(pkwhere, def)
	}
	return strings.Join(pkwhere, " and ")
}

// PKWhereFields returns the fields for a where clause for the primary
// keys of a model.
func (f *RelationalModelDefinition) PKWhereFields() string {
	var pkwhere []string
	for _, pk := range f.PrimaryKeys {
		def := fmt.Sprintf("%s", codegen.Goify(pk.DatabaseFieldName, false))
		pkwhere = append(pkwhere, def)
	}
	return strings.Join(pkwhere, ",")
}

// PKUpdateFields returns something?  This function doesn't look useful in
// current form.  Perhaps it isn't.
func (f *RelationalModelDefinition) PKUpdateFields(modelname string) string {
	var pkwhere []string
	for _, pk := range f.PrimaryKeys {
		def := fmt.Sprintf("%s.%s", modelname, codegen.Goify(pk.FieldName, true))
		pkwhere = append(pkwhere, def)
	}

	pkw := strings.Join(pkwhere, ",")
	return pkw
}

// StructDefinition returns the struct definition for the model.
func (f *RelationalModelDefinition) StructDefinition() string {
	header := fmt.Sprintf("type %s struct {\n", f.ModelName)
	var output string
	f.IterateFields(func(field *RelationalFieldDefinition) error {
		output = output + field.FieldDefinition()
		return nil
	})

	// Get a sortable slice of BelongsTo relationships
	var keys []string
	for k := range f.BelongsTo {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, k := range keys {
		output = output + f.BelongsTo[k].ModelName + "\t" + f.BelongsTo[k].ModelName + "\n"
	}
	footer := "}\n"
	return header + output + footer
}

// Attribute implements the Container interface of goa.
func (f *RelationalModelDefinition) Attribute() *design.AttributeDefinition {
	return f.AttributeDefinition
}

// Project does something interesting, and I don't remember if I use it
// anywhere.
//
// TODO find out
func (f *RelationalModelDefinition) Project(name, v string) *design.MediaTypeDefinition {
	p, _, _ := f.RenderTo[name].Project(v)
	return p
}

// LowerName returns the model name as a lowercase string.
func (f *RelationalModelDefinition) LowerName() string {
	return codegen.Goify(strings.ToLower(f.ModelName), false)
}

// Underscore returns the model name as a lowercase string in snake case.
func (f *RelationalModelDefinition) Underscore() string {
	runes := []rune(f.ModelName)
	length := len(runes)

	var out []rune
	for i := 0; i < length; i++ {
		if i > 0 && unicode.IsUpper(runes[i]) && ((i+1 < length && unicode.IsLower(runes[i+1])) || unicode.IsLower(runes[i-1])) {
			out = append(out, '_')
		}
		out = append(out, unicode.ToLower(runes[i]))
	}

	return string(out)
}

// IterateBuildSources runs an iterator function once per Model in the Store's model list.
func (f *RelationalModelDefinition) IterateBuildSources(it BuildSourceIterator) error {

	for _, bs := range f.BuildSources {
		if err := it(bs); err != nil {
			return err
		}
	}
	return nil
}

// IterateFields returns an iterator function useful for iterating through
// this model's field list.
func (f *RelationalModelDefinition) IterateFields(it FieldIterator) error {
	// Break out each type of fields

	var pkkeys []string
	pks := make(map[string]string)
	for n := range f.RelationalFields {
		if f.RelationalFields[n].PrimaryKey {
			pks[n] = n
			pkkeys = append(pkkeys, n)
		}
	}
	sort.Strings(pkkeys)

	var namekeys []string
	names := make(map[string]string)
	for n := range f.RelationalFields {
		if !f.RelationalFields[n].PrimaryKey && !f.RelationalFields[n].Timestamp {
			names[n] = n
			namekeys = append(namekeys, n)
		}
	}
	sort.Strings(namekeys)

	var datekeys []string
	dates := make(map[string]string)
	for n := range f.RelationalFields {
		if f.RelationalFields[n].Timestamp {
			dates[n] = n
			datekeys = append(datekeys, n)
		}
	}
	sort.Strings(datekeys)

	// Combine the sorted slices
	var fields []string
	fields = append(fields, pkkeys...)
	fields = append(fields, namekeys...)
	fields = append(fields, datekeys...)

	// Iterate them
	for _, n := range fields {
		if err := it(f.RelationalFields[n]); err != nil {
			return err
		}
	}
	return nil
}

// PopulateFromModeledType creates fields for the model
// based on the goa UserTypeDefinition it models, which is
// set using BuildsFrom().
// This happens before fields are processed, so it's
// ok to just assign without testing.
func (f *RelationalModelDefinition) PopulateFromModeledType() {
	if f.BuiltFrom == nil {
		return
	}
	for _, utd := range f.BuiltFrom {
		obj := utd.ToObject()
		obj.IterateAttributes(func(name string, att *design.AttributeDefinition) error {
			rf, ok := f.RelationalFields[codegen.Goify(name, true)]
			if ok {
				// We already have a mapping for this field.  What to do?
				if rf.Datatype != "" {
					return nil
				}
				// we may have seen the field but don't know its type
				// TODO(BJK) refactor this into separate func later
				switch att.Type.Kind() {
				case design.BooleanKind:
					rf.Datatype = Boolean
				case design.IntegerKind:
					rf.Datatype = Integer
				case design.NumberKind:
					rf.Datatype = BigDecimal
				case design.StringKind:
					rf.Datatype = String
				case design.DateTimeKind:
					rf.Datatype = Timestamp
				case design.MediaTypeKind:
					// Embedded MediaType
					// Skip for now?
					return nil

				default:
					dslengine.ReportError("Unsupported type: %#v %s", att.Type.Kind(), att.Type.Name())
				}
				if !utd.IsRequired(name) {
					rf.Nullable = true
				}
			}

			rf = &RelationalFieldDefinition{}
			rf.Parent = f
			rf.FieldName = codegen.Goify(name, true)

			if strings.HasSuffix(rf.FieldName, "Id") {
				rf.FieldName = strings.TrimSuffix(rf.FieldName, "Id")
				rf.FieldName = rf.FieldName + "ID"
			}
			switch att.Type.Kind() {
			case design.BooleanKind:
				rf.Datatype = Boolean
			case design.IntegerKind:
				rf.Datatype = Integer
			case design.NumberKind:
				rf.Datatype = BigDecimal
			case design.StringKind:
				rf.Datatype = String
			case design.DateTimeKind:
				rf.Datatype = Timestamp
			case design.MediaTypeKind:
				// Embedded MediaType
				// Skip for now?
				return nil

			default:
				dslengine.ReportError("Unsupported type: %#v %s", att.Type.Kind(), att.Type.Name())
			}
			if !utd.IsRequired(name) {
				rf.Nullable = true
			}
			// might need this later?
			rf.a = att
			f.RelationalFields[rf.FieldName] = rf

			addAttributeToModel(name, att, f)

			return nil
		})
	}
	return
}

func addAttributeToModel(name string, att *design.AttributeDefinition, m *RelationalModelDefinition) {
	var parent *design.AttributeDefinition
	parent = m.AttributeDefinition
	if parent != nil {
		if parent.Type == nil {
			parent.Type = design.Object{}
		}
		if _, ok := parent.Type.(design.Object); !ok {
			dslengine.ReportError("can't define child attributes on attribute of type %s", parent.Type.Name())
			return
		}

		parent.Type.(design.Object)[name] = att
	}

}

// copied from Goa
func parseAttributeArgs(baseAttr *design.AttributeDefinition, args ...interface{}) (design.DataType, string, func()) {
	var (
		dataType    design.DataType
		description string
		dsl         func()
		ok          bool
	)

	parseDataType := func(expected string, index int) {
		if name, ok := args[index].(string); ok {
			// Lookup type by name
			if dataType, ok = design.Design.Types[name]; !ok {
				if dataType = design.Design.MediaTypeWithIdentifier(name); dataType == nil {
					dslengine.InvalidArgError(expected, args[index])
				}
			}
			return
		}
		if dataType, ok = args[index].(design.DataType); !ok {
			dslengine.InvalidArgError(expected, args[index])
		}
	}
	parseDescription := func(expected string, index int) {
		if description, ok = args[index].(string); !ok {
			dslengine.InvalidArgError(expected, args[index])
		}
	}
	parseDSL := func(index int, success, failure func()) {
		if dsl, ok = args[index].(func()); ok {
			success()
		} else {
			failure()
		}
	}

	success := func() {}

	switch len(args) {
	case 0:
		if baseAttr != nil {
			dataType = baseAttr.Type
		} else {
			dataType = design.String
		}
	case 1:
		success = func() {
			if baseAttr != nil {
				dataType = baseAttr.Type
			}
		}
		parseDSL(0, success, func() { parseDataType("type, type name or func()", 0) })
	case 2:
		parseDataType("type or type name", 0)
		parseDSL(1, success, func() { parseDescription("string or func()", 1) })
	case 3:
		parseDataType("type or type name", 0)
		parseDescription("string", 1)
		parseDSL(2, success, func() { dslengine.InvalidArgError("func()", args[2]) })
	default:
		dslengine.ReportError("too many arguments in call to Attribute")
	}

	return dataType, description, dsl
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"github.com/42wim/matterircd/config"
	"github.com/42wim/matterircd/mm-go-irckit"
	"github.com/sirupsen/logrus"
	"net"
	"os"
	"strings"
)

var (
	flagRestrict, flagDefaultTeam, flagDefaultServer, flagTLSBind, flagTLSDir *string
	flagInsecure, flagSkipTLSVerify                                           *bool
	version                                                                   = "0.19.2"
	githash                                                                   string
	logger                                                                    *logrus.Entry
	cfg                                                                       config.Config
)

func main() {
	ourlog := logrus.New()
	ourlog.Formatter = &logrus.TextFormatter{FullTimestamp: true}
	logger = ourlog.WithFields(logrus.Fields{"module": "matterircd"})
	config.Logger = logger

	// config related. instantiate a new config.Config to store flags
	cfg = config.Config{}
	flagConfig := flag.String("conf", "matterircd.toml", "config file")

	// bools for showing version/enabling debug
	flagVersion := flag.Bool("version", false, "show version")
	flag.BoolVar(&cfg.Debug, "debug", false, "enable debug logging")

	// bind related cfg
	flag.StringVar(&cfg.Bind, "bind", "127.0.0.1:6667", "interface:port to bind to.")

	// mattermost related cfg
	flag.StringVar(&cfg.Restrict, "restrict", "", "only allow connection to specified mattermost server/instances. Space delimited")
	flag.StringVar(&cfg.DefaultTeam, "mmteam", "", "specify default mattermost team")
	flag.StringVar(&cfg.DefaultServer, "mmserver", "", "specify default mattermost server/instance")
	flag.BoolVar(&cfg.Insecure, "mminsecure", false, "use http connection to mattermost")

	// TLS related cfg
	flag.BoolVar(&cfg.SkipTLSVerify, "mmskiptlsverify", false, "skip verification of mattermost certificate chain and hostname")
	flag.StringVar(&cfg.TLSBind, "tlsbind", "", "interface:port to bind to. (e.g 127.0.0.1:6697)")
	flag.StringVar(&cfg.TLSDir, "tlsdir", ".", "directory to look for key.pem and cert.pem.")
	flag.Parse()

	// migrate config settings to mattermost settings
	cfg = *config.Migrate(cfg)
	// Attempt to load values from the config file
	if _, err := os.Stat(*flagConfig); err == nil {
		cfg = *config.LoadConfig(*flagConfig, cfg)
	}

	if cfg.Debug {
		logger.Info("enabling debug")
		ourlog.Level = logrus.DebugLevel
		irckit.SetLogLevel("debug")
	}
	if *flagVersion {
		fmt.Printf("version: %s %s\n", version, githash)
		return
	}

	irckit.SetLogger(logger)

	logger.Infof("Running version %s %s", version, githash)
	if strings.Contains(version, "-dev") {
		logger.Infof("WARNING: THIS IS A DEVELOPMENT VERSION. Things may break.")
	}

	if cfg.TLSBind != "" {
		go func() {
			logger.Infof("Listening on %s (TLS)", cfg.TLSBind)
			socket := tlsbind()
			defer socket.Close()
			start(socket)
		}()
	}

	// backwards compatible
	if cfg.Bind == "127.0.0.1:6667" && cfg.BindInterface != "" && cfg.BindPort != 0 {
		cfg.Bind = fmt.Sprintf("%s:%d", cfg.BindInterface, cfg.BindPort)
	}

	if cfg.Bind != "" {
		go func() {
			socket, err := net.Listen("tcp", cfg.Bind)
			if err != nil {
				logger.Errorf("Can not listen on %s: %v", cfg.Bind, err)
			}
			logger.Infof("Listening on %s", cfg.Bind)
			defer socket.Close()
			start(socket)
		}()
	}
	select {}
}

func tlsbind() net.Listener {
	cert, err := tls.LoadX509KeyPair(cfg.TLSDir+"/cert.pem", cfg.TLSDir+"/key.pem")
	if err != nil {
		logger.Errorf("could not load TLS, incorrect directory? Error: %s", err)
		os.Exit(1)
	}
	config := tls.Config{Certificates: []tls.Certificate{cert}}
	listenerTLS, err := tls.Listen("tcp", cfg.TLSBind, &config)
	if err != nil {
		logger.Errorf("Can not listen on %s: %v\n", cfg.TLSBind, err)
		os.Exit(1)
	}
	logger.Info("TLS listening on ", cfg.TLSBind)
	return listenerTLS
}

func start(socket net.Listener) {
	for {
		conn, err := socket.Accept()
		if err != nil {
			logger.Errorf("Failed to accept connection: %v", err)
			return
		}

		go func() {
			irccfg := &irckit.MmCfg{Insecure: cfg.Insecure, SkipTLSVerify: cfg.SkipTLSVerify,
				SlackSettings: cfg.Slack, MattermostSettings: cfg.Mattermost, PasteBufferTimeout: cfg.PasteBufferTimeout}
			newsrv := irckit.ServerConfig{Name: "matterircd", Version: version}.Server()
			logger.Infof("New connection: %s", conn.RemoteAddr())
			err = newsrv.Connect(irckit.NewUserMM(conn, newsrv, irccfg))
			if err != nil {
				logger.Errorf("Failed to join: %v", err)
				return
			}
		}()
	}
}
<commit_msg>Bump version<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"github.com/42wim/matterircd/config"
	"github.com/42wim/matterircd/mm-go-irckit"
	"github.com/sirupsen/logrus"
	"net"
	"os"
	"strings"
)

var (
	flagRestrict, flagDefaultTeam, flagDefaultServer, flagTLSBind, flagTLSDir *string
	flagInsecure, flagSkipTLSVerify                                           *bool
	version                                                                   = "0.19.3-dev"
	githash                                                                   string
	logger                                                                    *logrus.Entry
	cfg                                                                       config.Config
)

func main() {
	ourlog := logrus.New()
	ourlog.Formatter = &logrus.TextFormatter{FullTimestamp: true}
	logger = ourlog.WithFields(logrus.Fields{"module": "matterircd"})
	config.Logger = logger

	// config related. instantiate a new config.Config to store flags
	cfg = config.Config{}
	flagConfig := flag.String("conf", "matterircd.toml", "config file")

	// bools for showing version/enabling debug
	flagVersion := flag.Bool("version", false, "show version")
	flag.BoolVar(&cfg.Debug, "debug", false, "enable debug logging")

	// bind related cfg
	flag.StringVar(&cfg.Bind, "bind", "127.0.0.1:6667", "interface:port to bind to.")

	// mattermost related cfg
	flag.StringVar(&cfg.Restrict, "restrict", "", "only allow connection to specified mattermost server/instances. Space delimited")
	flag.StringVar(&cfg.DefaultTeam, "mmteam", "", "specify default mattermost team")
	flag.StringVar(&cfg.DefaultServer, "mmserver", "", "specify default mattermost server/instance")
	flag.BoolVar(&cfg.Insecure, "mminsecure", false, "use http connection to mattermost")

	// TLS related cfg
	flag.BoolVar(&cfg.SkipTLSVerify, "mmskiptlsverify", false, "skip verification of mattermost certificate chain and hostname")
	flag.StringVar(&cfg.TLSBind, "tlsbind", "", "interface:port to bind to. (e.g 127.0.0.1:6697)")
	flag.StringVar(&cfg.TLSDir, "tlsdir", ".", "directory to look for key.pem and cert.pem.")
	flag.Parse()

	// migrate config settings to mattermost settings
	cfg = *config.Migrate(cfg)
	// Attempt to load values from the config file
	if _, err := os.Stat(*flagConfig); err == nil {
		cfg = *config.LoadConfig(*flagConfig, cfg)
	}

	if cfg.Debug {
		logger.Info("enabling debug")
		ourlog.Level = logrus.DebugLevel
		irckit.SetLogLevel("debug")
	}
	if *flagVersion {
		fmt.Printf("version: %s %s\n", version, githash)
		return
	}

	irckit.SetLogger(logger)

	logger.Infof("Running version %s %s", version, githash)
	if strings.Contains(version, "-dev") {
		logger.Infof("WARNING: THIS IS A DEVELOPMENT VERSION. Things may break.")
	}

	if cfg.TLSBind != "" {
		go func() {
			logger.Infof("Listening on %s (TLS)", cfg.TLSBind)
			socket := tlsbind()
			defer socket.Close()
			start(socket)
		}()
	}

	// backwards compatible
	if cfg.Bind == "127.0.0.1:6667" && cfg.BindInterface != "" && cfg.BindPort != 0 {
		cfg.Bind = fmt.Sprintf("%s:%d", cfg.BindInterface, cfg.BindPort)
	}

	if cfg.Bind != "" {
		go func() {
			socket, err := net.Listen("tcp", cfg.Bind)
			if err != nil {
				logger.Errorf("Can not listen on %s: %v", cfg.Bind, err)
			}
			logger.Infof("Listening on %s", cfg.Bind)
			defer socket.Close()
			start(socket)
		}()
	}
	select {}
}

func tlsbind() net.Listener {
	cert, err := tls.LoadX509KeyPair(cfg.TLSDir+"/cert.pem", cfg.TLSDir+"/key.pem")
	if err != nil {
		logger.Errorf("could not load TLS, incorrect directory? Error: %s", err)
		os.Exit(1)
	}
	config := tls.Config{Certificates: []tls.Certificate{cert}}
	listenerTLS, err := tls.Listen("tcp", cfg.TLSBind, &config)
	if err != nil {
		logger.Errorf("Can not listen on %s: %v\n", cfg.TLSBind, err)
		os.Exit(1)
	}
	logger.Info("TLS listening on ", cfg.TLSBind)
	return listenerTLS
}

func start(socket net.Listener) {
	for {
		conn, err := socket.Accept()
		if err != nil {
			logger.Errorf("Failed to accept connection: %v", err)
			return
		}

		go func() {
			irccfg := &irckit.MmCfg{Insecure: cfg.Insecure, SkipTLSVerify: cfg.SkipTLSVerify,
				SlackSettings: cfg.Slack, MattermostSettings: cfg.Mattermost, PasteBufferTimeout: cfg.PasteBufferTimeout}
			newsrv := irckit.ServerConfig{Name: "matterircd", Version: version}.Server()
			logger.Infof("New connection: %s", conn.RemoteAddr())
			err = newsrv.Connect(irckit.NewUserMM(conn, newsrv, irccfg))
			if err != nil {
				logger.Errorf("Failed to join: %v", err)
				return
			}
		}()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os/user"
	"path/filepath"
	"regexp"
	"strings"
)

func main() {
	curr := readCurrent()
	oldFirmwareNumber, oldRollupNumber := curr[0], curr[1]

	baseURL := "http://wiki.snom.com/"
	url := baseURL + "Category:Firmware:snom300"
	html := string(establishConnection(url))
	firmware, rollup := getFirmwareAndRollup(html)
	firmwareSiteLink, firmwareNumber := baseURL+firmware[0], firmware[1]
	fmt.Printf("Firmware number: %s ", firmwareNumber)
	if firmwareNumber != oldFirmwareNumber {
		fmt.Printf("(new)")
		link := getFirmwareLink(string(establishConnection(firmwareSiteLink)))
		ioutil.WriteFile(firmwareNumber+".bin", establishConnection(link), 0644)
	} else {
		fmt.Printf("(old)")
	}
	fmt.Println()

	rollupSiteLink, rollupNumber := baseURL+rollup[0], rollup[1]
	fmt.Printf("Rollup number: %s ", rollupNumber)
	if rollupNumber != oldRollupNumber {
		fmt.Printf("(new)")
		link := getRollupLink(string(establishConnection(rollupSiteLink)))
		ioutil.WriteFile(rollupNumber+".bin", establishConnection(link), 0644)
	} else {
		fmt.Printf("(old)")
	}
	fmt.Println()

	writeCurrent(firmwareNumber, rollupNumber)
}

func getFirmwareAndRollup(html string) ([]string, []string) {
	matches := regexp.MustCompile("Firmware/([^\"]+)").FindAllStringSubmatch(html, 3)

	if len(matches) >= 3 {
		return matches[1], matches[2]
	}

	return []string{""}, []string{""}
}

func getFirmwareLink(html string) string {
	matches := regexp.MustCompile("http://downloads\\.snom\\.com/fw/snom300-[0-9.]+-SIP-f.bin").FindAllStringSubmatch(html, 10)
	if len(matches) == 0 {
		return ""
	}

	return matches[0][0]
}

func getRollupLink(html string) string {
	matches := regexp.MustCompile("http://downloads\\.snom\\.com/fw/mru-preview/snom300-[0-9.]+-SIP-f.bin").FindAllStringSubmatch(html, 10)
	if len(matches) == 0 {
		return ""
	}

	return matches[0][0]
}

func writeCurrent(firmwareNumber, rollupNumber string) {
	usr, _ := user.Current()
	homedir := usr.HomeDir

	data := firmwareNumber + "\n"
	data += rollupNumber

	ioutil.WriteFile(filepath.Join(homedir, ".snom"), []byte(data), 0644)
}

func readCurrent() []string {
	usr, _ := user.Current()
	homedir := usr.HomeDir

	data, _ := ioutil.ReadFile(filepath.Join(homedir, ".snom"))

	numbers := strings.Split(string(data), "\n")
	if len(numbers) < 2 {
		return []string{"", ""}
	}

	return numbers
}
<commit_msg>Add 'force' option Forces the download<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os/user"
	"path/filepath"
	"regexp"
	"strings"
)

var force = flag.Bool("force", false, "Force the download.")

func main() {
	flag.Parse()
	curr := readCurrent()
	oldFirmwareNumber, oldRollupNumber := curr[0], curr[1]

	baseURL := "http://wiki.snom.com/"
	url := baseURL + "Category:Firmware:snom300"
	html := string(establishConnection(url))
	firmware, rollup := getFirmwareAndRollup(html)
	firmwareSiteLink, firmwareNumber := baseURL+firmware[0], firmware[1]
	fmt.Printf("Firmware number: %s ", firmwareNumber)
	if firmwareNumber != oldFirmwareNumber || *force {
		fmt.Printf("(new)")
		link := getFirmwareLink(string(establishConnection(firmwareSiteLink)))
		ioutil.WriteFile(firmwareNumber+".bin", establishConnection(link), 0644)
	} else {
		fmt.Printf("(old)")
	}
	fmt.Println()

	rollupSiteLink, rollupNumber := baseURL+rollup[0], rollup[1]
	fmt.Printf("Rollup number: %s ", rollupNumber)
	if rollupNumber != oldRollupNumber || *force {
		fmt.Printf("(new)")
		link := getRollupLink(string(establishConnection(rollupSiteLink)))
		ioutil.WriteFile(rollupNumber+".bin", establishConnection(link), 0644)
	} else {
		fmt.Printf("(old)")
	}
	fmt.Println()

	writeCurrent(firmwareNumber, rollupNumber)
}

func getFirmwareAndRollup(html string) ([]string, []string) {
	matches := regexp.MustCompile("Firmware/([^\"]+)").FindAllStringSubmatch(html, 3)

	if len(matches) >= 3 {
		return matches[1], matches[2]
	}

	return []string{""}, []string{""}
}

func getFirmwareLink(html string) string {
	matches := regexp.MustCompile("http://downloads\\.snom\\.com/fw/snom300-[0-9.]+-SIP-f.bin").FindAllStringSubmatch(html, 10)
	if len(matches) == 0 {
		return ""
	}

	return matches[0][0]
}

func getRollupLink(html string) string {
	matches := regexp.MustCompile("http://downloads\\.snom\\.com/fw/mru-preview/snom300-[0-9.]+-SIP-f.bin").FindAllStringSubmatch(html, 10)
	if len(matches) == 0 {
		return ""
	}

	return matches[0][0]
}

func writeCurrent(firmwareNumber, rollupNumber string) {
	usr, _ := user.Current()
	homedir := usr.HomeDir

	data := firmwareNumber + "\n"
	data += rollupNumber

	ioutil.WriteFile(filepath.Join(homedir, ".snom"), []byte(data), 0644)
}

func readCurrent() []string {
	usr, _ := user.Current()
	homedir := usr.HomeDir

	data, _ := ioutil.ReadFile(filepath.Join(homedir, ".snom"))

	numbers := strings.Split(string(data), "\n")
	if len(numbers) < 2 {
		return []string{"", ""}
	}

	return numbers
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"

	"github.com/caarlos0/env"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"

	"github.com/nuveo/prest/config"
	"github.com/nuveo/prest/controllers"
)

func main() {
	cfg := config.Prest{}
	env.Parse(&cfg)

	n := negroni.Classic()
	n.Use(negroni.HandlerFunc(handlerSet))
	r := mux.NewRouter()
	r.HandleFunc("/databases", controllers.GetDatabases).Methods("GET")
	r.HandleFunc("/schemas", controllers.GetSchemas).Methods("GET")
	r.HandleFunc("/tables", controllers.GetTables).Methods("GET")
	r.HandleFunc("/{database}/{schema}", controllers.GetTablesByDatabaseAndSchema).Methods("GET")

	n.UseHandler(r)
	n.Run(fmt.Sprintf(":%v", cfg.HTTPPort))
}

func handlerSet(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	w.Header().Set("Content-Type", "application/json")
	next(w, r)
}
<commit_msg>add select in tables route<commit_after>package main

import (
	"fmt"
	"net/http"

	"github.com/caarlos0/env"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"

	"github.com/nuveo/prest/config"
	"github.com/nuveo/prest/controllers"
)

func main() {
	cfg := config.Prest{}
	env.Parse(&cfg)

	n := negroni.Classic()
	n.Use(negroni.HandlerFunc(handlerSet))
	r := mux.NewRouter()
	r.HandleFunc("/databases", controllers.GetDatabases).Methods("GET")
	r.HandleFunc("/schemas", controllers.GetSchemas).Methods("GET")
	r.HandleFunc("/tables", controllers.GetTables).Methods("GET")
	r.HandleFunc("/{database}/{schema}", controllers.GetTablesByDatabaseAndSchema).Methods("GET")
	r.HandleFunc("/{database}/{schema}/{table}", controllers.SelectFromTables).Methods("GET")

	n.UseHandler(r)
	n.Run(fmt.Sprintf(":%v", cfg.HTTPPort))
}

func handlerSet(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	w.Header().Set("Content-Type", "application/json")
	next(w, r)
}
<|endoftext|>


<commit_before>package magic

import (
	"archive/zip"
	"encoding/xml"
	"io"
	"io/ioutil"
	"strings"
)

type DetectionFunc func(io.Reader) string

var zipDetectionMap = map[string]DetectionFunc{
	"[Content_Types].xml":  detectOfficeX,
	"META-INF/MANIFEST.MF": detectJAR,
}

func DetectZipExtension(reader io.ReaderAt, length int64) (string, error) {
	r, err := zip.NewReader(reader, length)
	if err != nil {
		return "", err
	}
	for _, f := range r.File {
		df, ok := zipDetectionMap[f.Name]
		if !ok {
			continue
		}

		fp, err := f.Open()
		if err != nil {
			return "", err
		}
		defer fp.Close()
		if ext := df(fp); ext != "" {
			return ext, nil
		}
	}
	return "zip", nil
}

func DetectZipMIME(reader io.ReaderAt, length int64) (string, error) {
	ext, err := DetectZipExtension(reader, length)
	if err != nil {
		return "", err
	}
	return MIMEMap[ext], nil
}

func detectJAR(r io.Reader) string {
	return "jar"
}

func detectOfficeX(r io.Reader) string {
	b, err := ioutil.ReadAll(r)
	if err != nil {
		return ""
	}

	// Parse xml
	var o officeX
	if err := xml.Unmarshal(b, &o); err != nil {
		return ""
	}
	if len(o.Overrides) == 0 {
		return ""
	}

	for _, override := range o.Overrides {
		pnFactors := strings.Split(override.PartName, "/")
		if len(pnFactors) < 2 {
			return ""
		}
		switch pnFactors[1] {
		case "word":
			return "docx"
		case "xl":
			return "xlsx"
		case "ppt":
			return "pptx"
		}
	}
	return ""

}

type officeX struct {
	Overrides []*override `xml:"Override"`
}

type override struct {
	PartName string `xml:"PartName,attr"`
}
<commit_msg>Add thmx detection<commit_after>package magic

import (
	"archive/zip"
	"encoding/xml"
	"io"
	"io/ioutil"
	"strings"
)

type DetectionFunc func(io.Reader) string

var zipDetectionMap = map[string]DetectionFunc{
	"[Content_Types].xml":  detectOfficeX,
	"META-INF/MANIFEST.MF": detectJAR,
}

func DetectZipExtension(reader io.ReaderAt, length int64) (string, error) {
	r, err := zip.NewReader(reader, length)
	if err != nil {
		return "", err
	}
	for _, f := range r.File {
		df, ok := zipDetectionMap[f.Name]
		if !ok {
			continue
		}

		fp, err := f.Open()
		if err != nil {
			return "", err
		}
		defer fp.Close()
		if ext := df(fp); ext != "" {
			return ext, nil
		}
	}
	return "zip", nil
}

func DetectZipMIME(reader io.ReaderAt, length int64) (string, error) {
	ext, err := DetectZipExtension(reader, length)
	if err != nil {
		return "", err
	}
	return MIMEMap[ext], nil
}

func detectJAR(r io.Reader) string {
	return "jar"
}

func detectOfficeX(r io.Reader) string {
	b, err := ioutil.ReadAll(r)
	if err != nil {
		return ""
	}

	// Parse xml
	var o officeX
	if err := xml.Unmarshal(b, &o); err != nil {
		return ""
	}
	if len(o.Overrides) == 0 {
		return ""
	}

	for _, override := range o.Overrides {
		pnFactors := strings.Split(override.PartName, "/")
		if len(pnFactors) < 2 {
			return ""
		}
		switch pnFactors[1] {
		case "word":
			return "docx"
		case "xl":
			return "xlsx"
		case "ppt":
			return "pptx"
		case "theme":
			return "thmx"
		}
	}
	return ""

}

type officeX struct {
	Overrides []*override `xml:"Override"`
}

type override struct {
	PartName string `xml:"PartName,attr"`
}
<|endoftext|>


<commit_before>package main

import (
    "bufio"
    "flag"
    "fmt"
    "log"
    "milo/utils"
    "os"
    "path/filepath"
    "regexp"
    "runtime"
    "strings"
)

var root string
var extensions []string
var pattern *regexp.Regexp

// getNames creates a filepath.WalkFunc suitable for passing to
// filepath.Walk which passes the filenames found into a channel.
func getNames(c chan string) filepath.WalkFunc {
    return func(path string, info os.FileInfo, err error) error {
        if !info.Mode().IsRegular() {
            return nil
        }
        if len(extensions) > 0 {
            for _, ext := range extensions {
                if filepath.Ext(path) == ext {
                    c <- path
                    return nil
                }
            }
            return nil
        }
        c <- path
        return nil
    }
}

// init parses the command-line arguments into the values
// used to execute. There should be a pattern at the very
// least. Optionally, a path (defaulting to "."), and
// file extensions to search may be provided.
func init() {
    flag.Parse()
    args := flag.Args()
    if len(args) == 0 {
        log.Fatalf("No arguments passed.")
    }
    args = getExts(args)
    args = getRoot(args)
    if len(args) != 1 {
        log.Fatalf("Unable to find pattern.\n")
    }
    p, err := regexp.Compile(args[0])
    if err != nil {
        log.Fatal(err)
    }
    pattern = p
    runtime.GOMAXPROCS(runtime.NumCPU())
}

// getExts sets the extensions global variable,
// removes any extension arguments from args,
// and returns args for further processing.
func getExts(args []string) []string {
    var unused []string
    for _, val := range args {
        if strings.HasPrefix(val, "--") {
            if len(val) < 3 {
                log.Fatalf("Invalid extension: '%s'\n", val)
            }
            extensions = append(extensions, "." + val[2:])
        } else {
            unused = append(unused, val)
        }
    }
    return unused
}

// getRoot finds a valid directory in the command-line
// args, sets it to the global "root" variable, and
// returns the remaining arguments.
func getRoot(args []string) []string {
    var unused []string
    for _, val := range args {
        if utils.IsDir(val) {
            if root != "" {
                log.Fatalf("Too many directory arguments\n")
            } else {
                root = val
            }
        } else {
            unused = append(unused, val)
        }
    }
    if root == "" {
        root = "."
    }
    return unused
}

func main() {
    filenames := make(chan string, 3333)
    results := make(chan bool, 3333)
    f := getNames(filenames)
    go func() {
        filepath.Walk(root, f)
        close(filenames)
    }()
    count := 0
    for filename := range filenames {
        count += 1
        go checkFile(filename, results)
    }
    for count > 0 {
        <- results
        count--
    }
}

// checkFile takes a filename and reads the file to determine
// whether the file contains the regex in the global pattern.
func checkFile(filename string, done chan bool) {
    file, err := os.Open(filename)
    if err != nil {
        log.Println(err)
        done <- true
        return
    }
    scanner := bufio.NewScanner(file)
    line := 0
    for scanner.Scan() {
        line += 1
        found := pattern.FindIndex(scanner.Bytes())
        if found != nil {
            fmt.Printf("%s: line %d: %s\n", filename, line, scanner.Text())
        }
    }
    done <- true
}
<commit_msg>close files<commit_after>package main

import (
    "bufio"
    "flag"
    "fmt"
    "log"
    "milo/utils"
    "os"
    "path/filepath"
    "regexp"
    "runtime"
    "strings"
)

var root string
var extensions []string
var pattern *regexp.Regexp

// getNames creates a filepath.WalkFunc suitable for passing to
// filepath.Walk which passes the filenames found into a channel.
func getNames(c chan string) filepath.WalkFunc {
    return func(path string, info os.FileInfo, err error) error {
        if !info.Mode().IsRegular() {
            return nil
        }
        if len(extensions) > 0 {
            for _, ext := range extensions {
                if filepath.Ext(path) == ext {
                    c <- path
                    return nil
                }
            }
            return nil
        }
        c <- path
        return nil
    }
}

// init parses the command-line arguments into the values
// used to execute. There should be a pattern at the very
// least. Optionally, a path (defaulting to "."), and
// file extensions to search may be provided.
func init() {
    flag.Parse()
    args := flag.Args()
    if len(args) == 0 {
        log.Fatalf("No arguments passed.")
    }
    args = getExts(args)
    args = getRoot(args)
    if len(args) != 1 {
        log.Fatalf("Unable to find pattern.\n")
    }
    p, err := regexp.Compile(args[0])
    if err != nil {
        log.Fatal(err)
    }
    pattern = p
    runtime.GOMAXPROCS(runtime.NumCPU())
}

// getExts sets the extensions global variable,
// removes any extension arguments from args,
// and returns args for further processing.
func getExts(args []string) []string {
    var unused []string
    for _, val := range args {
        if strings.HasPrefix(val, "--") {
            if len(val) < 3 {
                log.Fatalf("Invalid extension: '%s'\n", val)
            }
            extensions = append(extensions, "." + val[2:])
        } else {
            unused = append(unused, val)
        }
    }
    return unused
}

// getRoot finds a valid directory in the command-line
// args, sets it to the global "root" variable, and
// returns the remaining arguments.
func getRoot(args []string) []string {
    var unused []string
    for _, val := range args {
        if utils.IsDir(val) {
            if root != "" {
                log.Fatalf("Too many directory arguments\n")
            } else {
                root = val
            }
        } else {
            unused = append(unused, val)
        }
    }
    if root == "" {
        root = "."
    }
    return unused
}

func main() {
    filenames := make(chan string, 3333)
    results := make(chan bool, 3333)
    f := getNames(filenames)
    go func() {
        filepath.Walk(root, f)
        close(filenames)
    }()
    count := 0
    for filename := range filenames {
        count += 1
        go checkFile(filename, results)
    }
    for count > 0 {
        <- results
        count--
    }
}

// checkFile takes a filename and reads the file to determine
// whether the file contains the regex in the global pattern.
func checkFile(filename string, done chan bool) {
    file, err := os.Open(filename)
    if err != nil {
        log.Println(err)
        done <- true
        return
    }
    defer file.Close()
    scanner := bufio.NewScanner(file)
    line := 0
    for scanner.Scan() {
        line += 1
        found := pattern.FindIndex(scanner.Bytes())
        if found != nil {
            fmt.Printf("%s:%d: %s\n", filename, line, scanner.Text())
        }
    }
    done <- true
}
<|endoftext|>