<commit_before>package filesystem

import (
	"bufio"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"

	"github.com/discoviking/blog/article"
	"github.com/russross/blackfriday"
)

var (
	languages = map[string]article.Language{
		"eng": article.English,
		"jap": article.Japanese,
	}
)

// Filesystem is a datastore which just reads articles from files.
type Filesystem struct {
	articlePath string
}

func New(path string) *Filesystem {
	return &Filesystem{path}
}

func (f *Filesystem) getArticle(id string, includeBody bool) (*article.Article, error) {
	glob := filepath.Join(f.articlePath, id+"*")
	files, err := filepath.Glob(glob)
	if err != nil {
		return nil, err
	}

	last_create_time := time.Unix(0, 0)
	last_mod_time := time.Unix(0, 0)

	titles := make(map[article.Language]string, len(files))
	bodies := make(map[article.Language]string, len(files))

	for _, name := range files {
		// Check the file exists, and also gets us its length.
		info, err := os.Stat(name)
		if err != nil {
			return nil, err
		}

		stat, ok := info.Sys().(*syscall.Stat_t)
		if !ok {
			return nil, err
		}

		create_time := time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec))
		if create_time.After(last_create_time) {
			last_create_time = create_time
		}
		mod_time := info.ModTime()
		if mod_time.After(last_mod_time) {
			last_mod_time = mod_time
		}

		// Work out the language from the file extension.
		ext := filepath.Ext(name)
		lang, ok := languages[ext]
		if !ok {
			lang = article.English
		}

		// Open the file.
		file, err := os.Open(name)
		if err != nil {
			return nil, err
		}
		defer file.Close()

		// The first line contains the title.
		b := bufio.NewReader(file)
		title, err := b.ReadString('\n')
		if err != nil {
			return nil, err
		}
		titles[lang] = strings.TrimSpace(title)

		// The rest is the body.
		if includeBody {
			buf := make([]byte, info.Size())
			n, err := b.Read(buf)
			if err != nil {
				return nil, err
			}

			bodies[lang] = string(blackfriday.MarkdownCommon(buf[:n]))
		}
	}

	return &article.Article{
		Id:         id,
		Title:      titles,
		Body:       bodies,
		DatePosted: last_create_time,
		DateEdited: last_mod_time,
	}, nil
}

func (f *Filesystem) Article(name string) (*article.Article, error) {
	return f.getArticle(name, true)
}

func (f *Filesystem) List() ([]*article.Article, error) {
	dir, err := os.Open(f.articlePath)
	if err != nil {
		return nil, err
	}
	defer dir.Close()

	files, err := dir.Readdirnames(0)
	if err != nil {
		return nil, err
	}

	names := make(map[string]struct{}, len(files))
	for _, f := range files {
		ext := filepath.Ext(f)
		name := f[:len(f)-len(ext)]
		names[name] = struct{}{}
	}

	articles := make(article.ArticleList, 0, len(names))

	for name, _ := range names {
		a, err := f.getArticle(name, false)
		if err != nil {
			continue
		}
		articles = append(articles, a)
	}

	// Sort by creation time.
	sort.Sort(articles)

	return articles, nil
}

func (f *Filesystem) Store(a *article.Article) error {
	return nil
}
<commit_msg>Fix filesystem language lookup<commit_after>package filesystem

import (
	"bufio"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"

	"github.com/discoviking/blog/article"
	"github.com/russross/blackfriday"
)

var (
	languages = map[string]article.Language{
		".eng": article.English,
		".jap": article.Japanese,
	}
)

// Filesystem is a datastore which just reads articles from files.
type Filesystem struct {
	articlePath string
}

func New(path string) *Filesystem {
	return &Filesystem{path}
}

func (f *Filesystem) getArticle(id string, includeBody bool) (*article.Article, error) {
	glob := filepath.Join(f.articlePath, id+"*")
	files, err := filepath.Glob(glob)
	if err != nil {
		return nil, err
	}

	last_create_time := time.Unix(0, 0)
	last_mod_time := time.Unix(0, 0)

	titles := make(map[article.Language]string, len(files))
	bodies := make(map[article.Language]string, len(files))

	log.Print("Loading files %v", files)

	for _, name := range files {
		// Check the file exists, and also gets us its length.
		info, err := os.Stat(name)
		if err != nil {
			return nil, err
		}

		stat, ok := info.Sys().(*syscall.Stat_t)
		if !ok {
			return nil, err
		}

		create_time := time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec))
		if create_time.After(last_create_time) {
			last_create_time = create_time
		}
		mod_time := info.ModTime()
		if mod_time.After(last_mod_time) {
			last_mod_time = mod_time
		}

		// Work out the language from the file extension.
		ext := filepath.Ext(name)
		log.Print("File extension:", ext)
		lang, ok := languages[ext]
		if !ok {
			log.Print("Defaulting language to ENG")
			lang = article.English
		} else {
			log.Print("Found language:", lang)
		}

		// Open the file.
		file, err := os.Open(name)
		if err != nil {
			return nil, err
		}
		defer file.Close()

		// The first line contains the title.
		b := bufio.NewReader(file)
		title, err := b.ReadString('\n')
		if err != nil {
			return nil, err
		}
		titles[lang] = strings.TrimSpace(title)

		// The rest is the body.
		if includeBody {
			buf := make([]byte, info.Size())
			n, err := b.Read(buf)
			if err != nil {
				return nil, err
			}

			bodies[lang] = string(blackfriday.MarkdownCommon(buf[:n]))
		}
	}

	return &article.Article{
		Id:         id,
		Title:      titles,
		Body:       bodies,
		DatePosted: last_create_time,
		DateEdited: last_mod_time,
	}, nil
}

func (f *Filesystem) Article(name string) (*article.Article, error) {
	return f.getArticle(name, true)
}

func (f *Filesystem) List() ([]*article.Article, error) {
	dir, err := os.Open(f.articlePath)
	if err != nil {
		return nil, err
	}
	defer dir.Close()

	files, err := dir.Readdirnames(0)
	if err != nil {
		return nil, err
	}

	names := make(map[string]struct{}, len(files))
	for _, f := range files {
		ext := filepath.Ext(f)
		name := f[:len(f)-len(ext)]
		names[name] = struct{}{}
	}

	articles := make(article.ArticleList, 0, len(names))

	for name, _ := range names {
		a, err := f.getArticle(name, false)
		if err != nil {
			continue
		}
		articles = append(articles, a)
	}

	// Sort by creation time.
	sort.Sort(articles)

	return articles, nil
}

func (f *Filesystem) Store(a *article.Article) error {
	return nil
}
<|endoftext|>


<commit_before>package finder

import (
	"fmt"
	"log"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/caelifer/dups/fstree"
	"github.com/caelifer/dups/mapreduce"
	"github.com/caelifer/dups/node"
	"github.com/caelifer/scheduler"
)

// Finder
type Finder struct {
	// Work Queue
	sched scheduler.Scheduler

	// Stats
	totalDirs        uint64
	totalFiles       uint64
	totalCopies      uint64
	totalWastedSpace uint64
	totalTime        time.Duration
}

func New(nworkers, njobs int) *Finder {
	return &Finder{sched: scheduler.New(nworkers, njobs)}
}

func (f *Finder) SetTimeSpent(d time.Duration) {
	f.totalTime = d
}

func (f Finder) Stats() string {
	// Stats report
	return fmt.Sprintf("Examined %d files in %d directories [%s], found %d dups, total wasted space %.2fGiB",
		f.totalFiles, f.totalDirs, f.totalTime, f.totalCopies, float64(f.totalWastedSpace)/(1024*1024*1024))
}

func (f *Finder) AllDups(paths []string) <-chan mapreduce.Value {
	// Build a processing pipeline
	return mapreduce.Pipeline(
		[]mapreduce.MapReducePair{
			{
				f.makeNodeMap(paths),
				mapreduce.FilterDuplicates,
			}, {
				f.makeFileSizeMap(),
				mapreduce.FilterUniques,
			}, {
				f.makeFileHashMap(true),
				mapreduce.FilterUniques,
			}, {
				f.makeFileHashMap(false),
				mapreduce.FilterUniques,
			}, {
				f.mapDups(),
				f.reduceDups(),
			},
		}...,
	)
}

// makeNodeMap
func (f *Finder) makeNodeMap(paths []string) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue, _ <-chan mapreduce.Value) {
		// Process all command line paths
		for _, path_ := range paths {
			// err := filepath.Walk(path_, func(path string, info os.FileInfo, err error) error {
			err := fstree.Walk(f.sched, path_, func(path string, info os.FileInfo, err error) error {
				// Handle passthrough error
				if err != nil {
					log.Println("WARN", err)
					return nil
				}

				// Only process simple files
				if info.IsDir() {
					// Increase seen directory counter
					atomic.AddUint64(&f.totalDirs, 1)
				}

				// Only process simple files
				if isRegularFile(info) {
					size := info.Size()

					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(path),
						&node.Node{Path: path, Size: size},
					)

					// Increase seen files counter
					atomic.AddUint64(&f.totalFiles, 1)
				}
				return nil
			})

			if err != nil {
				log.Fatal(err)
			}
		}
	}
}

// Very simple function to map nodes by size
func (*Finder) makeFileSizeMap() mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue, in <-chan mapreduce.Value) {
		for x := range in {
			node := x.Value().(*node.Node) // Assert type

			out <- mapreduce.NewKVType(
				mapreduce.KeyTypeFromInt64(node.Size),
				node,
			)
		}
	}
}

func (f *Finder) makeFileHashMap(fast bool) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue, in <-chan mapreduce.Value) {
		wg := new(sync.WaitGroup) // Heap
		for x := range in {
			// Add to wait group
			wg.Add(1)

			node_ := x.Value().(*node.Node) // Assert type

			// Calculate hash using balancer
			go func(n *node.Node) {
				f.sched.Schedule(func() {
					defer wg.Done() // Signal done

					// Calculate hash using fast calculation if required
					err := n.CalculateHash(fast) // Fast hash calculation - SHA1 of first 1024 bytes
					if err != nil {
						// Skip files for which we failed to calculate SHA1 hash
						// log.Printf("WARN Unable calculate SHA1 hash for %q\n", node.Path)
						return
					}

					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(n.Hash),
						n,
					)
				})
			}(node_)
		}
		// Wait for all results be submitted
		wg.Wait()
	}
}

// fanal map
func (f *Finder) mapDups() mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue, in <-chan mapreduce.Value) {
		for x := range in {
			// Update stats
			atomic.AddUint64(&f.totalCopies, 1)
			n := x.Value().(*node.Node) // Type assert
			out <- mapreduce.NewKVType(
				mapreduce.KeyType(n.Hash),
				&Dup{Node: n},
			)
		}
	}
}

// final reduce
func (f *Finder) reduceDups() mapreduce.ReduceFn {
	return func(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
		byHash := make(map[string][]Dup)

		for x := range in {
			d := x.Value().(Dup) // Type assert

			// Aggregate
			if v, ok := byHash[d.Hash]; ok {
				// Found node with the same content
				byHash[d.Hash] = append(v, d)
			} else {
				byHash[d.Hash] = []Dup{d}
			}
		}

		// Reduce
		for _, dups := range byHash {
			count := len(dups)
			// Update free size stats
			atomic.AddUint64(&f.totalWastedSpace, uint64(dups[0].Size*int64(count-1)))

			for _, d := range dups {
				// Update dups number stats
				d.Count = count
				out <- d
			}
		}

		byHash = nil
	}
}

func isRegularFile(fi os.FileInfo) bool {
	return fi.Mode()&os.ModeType == 0
}
<commit_msg>Use initializer instead of local variable<commit_after>package finder

import (
	"fmt"
	"log"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/caelifer/dups/fstree"
	"github.com/caelifer/dups/mapreduce"
	"github.com/caelifer/dups/node"
	"github.com/caelifer/scheduler"
)

// Finder
type Finder struct {
	// Work Queue
	sched scheduler.Scheduler

	// Stats
	totalDirs        uint64
	totalFiles       uint64
	totalCopies      uint64
	totalWastedSpace uint64
	totalTime        time.Duration
}

func New(nworkers, njobs int) *Finder {
	return &Finder{sched: scheduler.New(nworkers, njobs)}
}

func (f *Finder) SetTimeSpent(d time.Duration) {
	f.totalTime = d
}

func (f Finder) Stats() string {
	// Stats report
	return fmt.Sprintf("Examined %d files in %d directories [%s], found %d dups, total wasted space %.2fGiB",
		f.totalFiles, f.totalDirs, f.totalTime, f.totalCopies, float64(f.totalWastedSpace)/(1024*1024*1024))
}

func (f *Finder) AllDups(paths []string) <-chan mapreduce.Value {
	// Build a processing pipeline
	return mapreduce.Pipeline(
		[]mapreduce.MapReducePair{
			{
				f.makeNodeMap(paths),
				mapreduce.FilterDuplicates,
			}, {
				f.makeFileSizeMap(),
				mapreduce.FilterUniques,
			}, {
				f.makeFileHashMap(true),
				mapreduce.FilterUniques,
			}, {
				f.makeFileHashMap(false),
				mapreduce.FilterUniques,
			}, {
				f.mapDups(),
				f.reduceDups(),
			},
		}...,
	)
}

// makeNodeMap
func (f *Finder) makeNodeMap(paths []string) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue, _ <-chan mapreduce.Value) {
		// Process all command line paths
		for _, path_ := range paths {
			// err := filepath.Walk(path_, func(path string, info os.FileInfo, err error) error {
			err := fstree.Walk(f.sched, path_, func(path string, info os.FileInfo, err error) error {
				// Handle passthrough error
				if err != nil {
					log.Println("WARN", err)
					return nil
				}

				// Only process simple files
				if info.IsDir() {
					// Increase seen directory counter
					atomic.AddUint64(&f.totalDirs, 1)
				}

				// Only process simple files
				if isRegularFile(info) {
					size := info.Size()

					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(path),
						&node.Node{Path: path, Size: size},
					)

					// Increase seen files counter
					atomic.AddUint64(&f.totalFiles, 1)
				}
				return nil
			})

			if err != nil {
				log.Fatal(err)
			}
		}
	}
}

// Very simple function to map nodes by size
func (*Finder) makeFileSizeMap() mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue, in <-chan mapreduce.Value) {
		for x := range in {
			node := x.Value().(*node.Node) // Assert type

			out <- mapreduce.NewKVType(
				mapreduce.KeyTypeFromInt64(node.Size),
				node,
			)
		}
	}
}

func (f *Finder) makeFileHashMap(fast bool) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue, in <-chan mapreduce.Value) {
		wg := new(sync.WaitGroup) // Heap
		for x := range in {
			// Add to wait group
			wg.Add(1)
			// Calculate hash using balancer
			go func(n *node.Node) {
				f.sched.Schedule(func() {
					defer wg.Done() // Signal done
					// Calculate hash using fast calculation if required
					err := n.CalculateHash(fast) // Fast hash calculation - SHA1 of first 1024 bytes
					if err != nil {
						// Skip files for which we failed to calculate SHA1 hash
						// log.Printf("WARN Unable calculate SHA1 hash for %q\n", node.Path)
						return
					}
					// Report result
					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(n.Hash),
						n,
					)
				})
			}(x.Value().(*node.Node))
		}
		// Wait for all results be submitted
		wg.Wait()
	}
}

// fanal map
func (f *Finder) mapDups() mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue, in <-chan mapreduce.Value) {
		for x := range in {
			// Update stats
			atomic.AddUint64(&f.totalCopies, 1)
			n := x.Value().(*node.Node) // Type assert
			out <- mapreduce.NewKVType(
				mapreduce.KeyType(n.Hash),
				&Dup{Node: n},
			)
		}
	}
}

// final reduce
func (f *Finder) reduceDups() mapreduce.ReduceFn {
	return func(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
		byHash := make(map[string][]Dup)

		for x := range in {
			d := x.Value().(Dup) // Type assert

			// Aggregate
			if v, ok := byHash[d.Hash]; ok {
				// Found node with the same content
				byHash[d.Hash] = append(v, d)
			} else {
				byHash[d.Hash] = []Dup{d}
			}
		}

		// Reduce
		for _, dups := range byHash {
			count := len(dups)
			// Update free size stats
			atomic.AddUint64(&f.totalWastedSpace, uint64(dups[0].Size*int64(count-1)))

			for _, d := range dups {
				// Update dups number stats
				d.Count = count
				out <- d
			}
		}

		byHash = nil
	}
}

func isRegularFile(fi os.FileInfo) bool {
	return fi.Mode()&os.ModeType == 0
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package boomer

import (
	"fmt"
	"sort"
	"strings"
	"time"
)

const (
	barChar = "∎"
)

type report struct {
	avgTotal float64
	fastest  float64
	slowest  float64
	average  float64
	rps      float64

	results chan *result
	total   time.Duration
	errors  map[string]int

	statusCodeDist map[int]int
	lats           []float64
	sizeTotal      int64

	output string
}

func printReport(size int, results chan *result, output string, total time.Duration) {
	r := &report{
		output:         output,
		results:        results,
		total:          total,
		statusCodeDist: make(map[int]int),
		errors:         make(map[string]int),
	}
	r.finalize()
}

func (r *report) finalize() {
	for {
		select {
		case res := <-r.results:
			if res.err != nil {
				r.errors[res.err.Error()]++
			} else {
				r.lats = append(r.lats, res.duration.Seconds())
				r.avgTotal += res.duration.Seconds()
				r.statusCodeDist[res.statusCode]++
				if res.contentLength > 0 {
					r.sizeTotal += res.contentLength
				}
			}
		default:
			r.rps = float64(len(r.lats)) / r.total.Seconds()
			r.average = r.avgTotal / float64(len(r.lats))
			r.print()
			return
		}
	}
}

func (r *report) print() {
	sort.Float64s(r.lats)

	if r.output == "csv" {
		r.printCSV()
		return
	}

	if len(r.lats) > 0 {
		r.fastest = r.lats[0]
		r.slowest = r.lats[len(r.lats)-1]
		fmt.Printf("\nSummary:\n")
		fmt.Printf("  Total:\t%4.4f secs.\n", r.total.Seconds())
		fmt.Printf("  Slowest:\t%4.4f secs.\n", r.slowest)
		fmt.Printf("  Fastest:\t%4.4f secs.\n", r.fastest)
		fmt.Printf("  Average:\t%4.4f secs.\n", r.average)
		fmt.Printf("  Requests/sec:\t%4.4f\n", r.rps)
		if r.sizeTotal > 0 {
			fmt.Printf("  Total Data Received:\t%d bytes.\n", r.sizeTotal)
			fmt.Printf("  Response Size per Request:\t%d bytes.\n", r.sizeTotal/int64(len(r.lats)))
		}
		r.printStatusCodes()
		r.printHistogram()
		r.printLatencies()
	}

	if len(r.errors) > 0 {
		r.printErrors()
	}
}

func (r *report) printCSV() {
	for i, val := range r.lats {
		fmt.Printf("%v,%4.4f\n", i+1, val)
	}
}

// Prints percentile latencies.
func (r *report) printLatencies() {
	pctls := []int{10, 25, 50, 75, 90, 95, 99}
	data := make([]float64, len(pctls))
	j := 0
	for i := 0; i < len(r.lats) && j < len(pctls); i++ {
		current := i * 100 / len(r.lats)
		if current >= pctls[j] {
			data[j] = r.lats[i]
			j++
		}
	}
	fmt.Printf("\nLatency distribution:\n")
	for i := 0; i < len(pctls); i++ {
		if data[i] > 0 {
			fmt.Printf("  %v%% in %4.4f secs.\n", pctls[i], data[i])
		}
	}
}

func (r *report) printHistogram() {
	bc := 10
	buckets := make([]float64, bc+1)
	counts := make([]int, bc+1)
	bs := (r.slowest - r.fastest) / float64(bc)
	for i := 0; i < bc; i++ {
		buckets[i] = r.fastest + bs*float64(i)
	}
	buckets[bc] = r.slowest
	var bi int
	var max int
	for i := 0; i < len(r.lats); {
		if r.lats[i] <= buckets[bi] {
			i++
			counts[bi]++
			if max < counts[bi] {
				max = counts[bi]
			}
		} else if bi < len(buckets)-1 {
			bi++
		}
	}
	fmt.Printf("\nResponse time histogram:\n")
	for i := 0; i < len(buckets); i++ {
		// Normalize bar lengths.
		var barLen int
		if max > 0 {
			barLen = counts[i] * 40 / max
		}
		fmt.Printf("  %4.3f [%v]\t|%v\n", buckets[i], counts[i], strings.Repeat(barChar, barLen))
	}
}

// Prints status code distribution.
func (r *report) printStatusCodes() {
	fmt.Printf("\nStatus code distribution:\n")
	for code, num := range r.statusCodeDist {
		fmt.Printf("  [%d]\t%d responses\n", code, num)
	}
}

func (r *report) printErrors() {
	fmt.Printf("\nError distribution:\n")
	for error, num := range r.errors {
		fmt.Printf("  [%d]\t%s\n", num, error)
	}
}
<commit_msg>Improving readability.<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package boomer

import (
	"fmt"
	"sort"
	"strings"
	"time"
)

const (
	barChar = "∎"
)

type report struct {
	avgTotal float64
	fastest  float64
	slowest  float64
	average  float64
	rps      float64

	results chan *result
	total   time.Duration

	errorDist      map[string]int
	statusCodeDist map[int]int
	lats           []float64
	sizeTotal      int64

	output string
}

func printReport(size int, results chan *result, output string, total time.Duration) {
	r := &report{
		output:         output,
		results:        results,
		total:          total,
		statusCodeDist: make(map[int]int),
		errorDist:      make(map[string]int),
	}
	r.finalize()
}

func (r *report) finalize() {
	for {
		select {
		case res := <-r.results:
			if res.err != nil {
				r.errorDist[res.err.Error()]++
			} else {
				r.lats = append(r.lats, res.duration.Seconds())
				r.avgTotal += res.duration.Seconds()
				r.statusCodeDist[res.statusCode]++
				if res.contentLength > 0 {
					r.sizeTotal += res.contentLength
				}
			}
		default:
			r.rps = float64(len(r.lats)) / r.total.Seconds()
			r.average = r.avgTotal / float64(len(r.lats))
			r.print()
			return
		}
	}
}

func (r *report) print() {
	sort.Float64s(r.lats)

	if r.output == "csv" {
		r.printCSV()
		return
	}

	if len(r.lats) > 0 {
		r.fastest = r.lats[0]
		r.slowest = r.lats[len(r.lats)-1]
		fmt.Printf("\nSummary:\n")
		fmt.Printf("  Total:\t%4.4f secs.\n", r.total.Seconds())
		fmt.Printf("  Slowest:\t%4.4f secs.\n", r.slowest)
		fmt.Printf("  Fastest:\t%4.4f secs.\n", r.fastest)
		fmt.Printf("  Average:\t%4.4f secs.\n", r.average)
		fmt.Printf("  Requests/sec:\t%4.4f\n", r.rps)
		if r.sizeTotal > 0 {
			fmt.Printf("  Total Data Received:\t%d bytes.\n", r.sizeTotal)
			fmt.Printf("  Response Size per Request:\t%d bytes.\n", r.sizeTotal/int64(len(r.lats)))
		}
		r.printStatusCodes()
		r.printHistogram()
		r.printLatencies()
	}

	if len(r.errorDist) > 0 {
		r.printErrors()
	}
}

func (r *report) printCSV() {
	for i, val := range r.lats {
		fmt.Printf("%v,%4.4f\n", i+1, val)
	}
}

// Prints percentile latencies.
func (r *report) printLatencies() {
	pctls := []int{10, 25, 50, 75, 90, 95, 99}
	data := make([]float64, len(pctls))
	j := 0
	for i := 0; i < len(r.lats) && j < len(pctls); i++ {
		current := i * 100 / len(r.lats)
		if current >= pctls[j] {
			data[j] = r.lats[i]
			j++
		}
	}
	fmt.Printf("\nLatency distribution:\n")
	for i := 0; i < len(pctls); i++ {
		if data[i] > 0 {
			fmt.Printf("  %v%% in %4.4f secs.\n", pctls[i], data[i])
		}
	}
}

func (r *report) printHistogram() {
	bc := 10
	buckets := make([]float64, bc+1)
	counts := make([]int, bc+1)
	bs := (r.slowest - r.fastest) / float64(bc)
	for i := 0; i < bc; i++ {
		buckets[i] = r.fastest + bs*float64(i)
	}
	buckets[bc] = r.slowest
	var bi int
	var max int
	for i := 0; i < len(r.lats); {
		if r.lats[i] <= buckets[bi] {
			i++
			counts[bi]++
			if max < counts[bi] {
				max = counts[bi]
			}
		} else if bi < len(buckets)-1 {
			bi++
		}
	}
	fmt.Printf("\nResponse time histogram:\n")
	for i := 0; i < len(buckets); i++ {
		// Normalize bar lengths.
		var barLen int
		if max > 0 {
			barLen = counts[i] * 40 / max
		}
		fmt.Printf("  %4.3f [%v]\t|%v\n", buckets[i], counts[i], strings.Repeat(barChar, barLen))
	}
}

// Prints status code distribution.
func (r *report) printStatusCodes() {
	fmt.Printf("\nStatus code distribution:\n")
	for code, num := range r.statusCodeDist {
		fmt.Printf("  [%d]\t%d responses\n", code, num)
	}
}

func (r *report) printErrors() {
	fmt.Printf("\nError distribution:\n")
	for err, num := range r.errorDist {
		fmt.Printf("  [%d]\t%s\n", num, err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"github.com/99designs/aws-vault/keyring"
	"github.com/aws/aws-sdk-go/aws/awserr"
)

type ExecCommandInput struct {
	Profile  string
	Command  string
	Args     []string
	Keyring  keyring.Keyring
	Duration time.Duration
	WriteEnv bool
	Signals  chan os.Signal
}

func ExecCommand(ui Ui, input ExecCommandInput) {
	creds, err := NewVaultCredentials(input.Keyring, input.Profile, VaultOptions{
		SessionDuration: input.Duration,
		WriteEnv:        input.WriteEnv,
	})
	if err != nil {
		ui.Error.Fatal(err)
	}

	val, err := creds.Get()
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoCredentialProviders" {
			ui.Error.Fatalf("No credentials found for profile %q", input.Profile)
		} else {
			ui.Error.Fatal(err)
		}
	}

	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		ui.Error.Fatal(err)
	}

	go func() {
		log.Printf("Metadata server listening on %s", l.Addr().String())
		ui.Error.Fatal(http.Serve(l, proxyHandler(NewMetadataHandler(creds))))
	}()

	cfg, err := profileConfig(input.Profile)
	if err != nil {
		ui.Error.Fatal(cfg)
	}

	env := os.Environ()
	env = overwriteEnv(env, "http_proxy", l.Addr().String())
	env = overwriteEnv(env, "no_proxy", "amazonaws.com")
	env = overwriteEnv(env, "AWS_CONFIG_FILE", cfg.Name())
	env = overwriteEnv(env, "AWS_DEFAULT_PROFILE", input.Profile)

	if input.WriteEnv {
		env = overwriteEnv(env, "AWS_ACCESS_KEY_ID", val.AccessKeyID)
		env = overwriteEnv(env, "AWS_SECRET_ACCESS_KEY", val.SecretAccessKey)

		if val.SessionToken != "" {
			env = overwriteEnv(env, "AWS_SESSION_TOKEN", val.SessionToken)
			env = overwriteEnv(env, "AWS_SECURITY_TOKEN", val.SessionToken)
		}
	}

	cmd := exec.Command(input.Command, input.Args...)
	cmd.Env = env
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	go func() {
		sig := <-input.Signals
		if cmd.Process != nil {
			cmd.Process.Signal(sig)
		}
	}()

	var waitStatus syscall.WaitStatus
	if err := cmd.Run(); err != nil {
		if err != nil {
			ui.Error.Println(err)
		}
		if exitError, ok := err.(*exec.ExitError); ok {
			waitStatus = exitError.Sys().(syscall.WaitStatus)
			os.Exit(waitStatus.ExitStatus())
		}
	}

}

// write out a config excluding role switching keys
func profileConfig(profile string) (*os.File, error) {
	conf, err := parseProfiles()
	if err != nil {
		return nil, err
	}

	tmpConfig, err := ioutil.TempFile(os.TempDir(), "aws-vault")
	if err != nil {
		return nil, err
	}

	// allow some time for keychain prompt
	newConfig := map[string]string{
		"metadata_service_timeout":      "15",
		"metadata_service_num_attempts": "2",
	}

	for k, v := range conf[profile] {
		if k != "source_profile" && k != "role_arn" {
			newConfig[k] = v
		}
	}

	return tmpConfig, writeProfiles(tmpConfig, profiles{profile: newConfig})
}

func overwriteEnv(env []string, key, val string) []string {
	var found bool

	for idx, e := range env {
		if strings.HasPrefix(key+"=", e) {
			env[idx] = key + "=" + val
			found = true
		} else {
			env[idx] = e
		}
	}

	if !found {
		env = append(env, key+"="+val)
	}

	return env
}

func connectProxy(w http.ResponseWriter, r *http.Request) error {
	d, err := net.Dial("tcp", r.RequestURI)
	if err != nil {
		return err
	}
	hj, ok := w.(http.Hijacker)
	if !ok {
		return err
	}
	conn, bufrw, err := hj.Hijack()
	if err != nil {
		return err
	}
	defer conn.Close()
	defer d.Close()

	bufrw.WriteString("HTTP/1.0 200 Connection established\r\n\r\n")
	bufrw.Flush()

	go io.Copy(d, conn)
	io.Copy(conn, d)

	return nil
}

func proxyHandler(upstream http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%s %s", r.Method, r.RequestURI)
		if r.Method == "GET" && strings.HasPrefix(r.URL.Path, "/latest/meta-data") {
			upstream.ServeHTTP(w, r)
		} else if r.Method == "CONNECT" {
			if err := connectProxy(w, r); err != nil {
				http.Error(w, "Error connecting to proxy: "+err.Error(), http.StatusGatewayTimeout)
			}
		} else {
			proxy := &httputil.ReverseProxy{Director: func(req *http.Request) {
				req.Method = "GET"
				req.URL.Scheme = "http"
				req.URL.Host = r.Host
			}}
			proxy.ServeHTTP(w, r)
		}
	})
}
<commit_msg>Added protocol to http_proxy, fixes #33<commit_after>package main

import (
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"github.com/99designs/aws-vault/keyring"
	"github.com/aws/aws-sdk-go/aws/awserr"
)

type ExecCommandInput struct {
	Profile  string
	Command  string
	Args     []string
	Keyring  keyring.Keyring
	Duration time.Duration
	WriteEnv bool
	Signals  chan os.Signal
}

func ExecCommand(ui Ui, input ExecCommandInput) {
	creds, err := NewVaultCredentials(input.Keyring, input.Profile, VaultOptions{
		SessionDuration: input.Duration,
		WriteEnv:        input.WriteEnv,
	})
	if err != nil {
		ui.Error.Fatal(err)
	}

	val, err := creds.Get()
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoCredentialProviders" {
			ui.Error.Fatalf("No credentials found for profile %q", input.Profile)
		} else {
			ui.Error.Fatal(err)
		}
	}

	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		ui.Error.Fatal(err)
	}

	go func() {
		log.Printf("Metadata server listening on %s", l.Addr().String())
		ui.Error.Fatal(http.Serve(l, proxyHandler(NewMetadataHandler(creds))))
	}()

	cfg, err := profileConfig(input.Profile)
	if err != nil {
		ui.Error.Fatal(cfg)
	}

	env := os.Environ()
	env = overwriteEnv(env, "http_proxy", "http://"+l.Addr().String())
	env = overwriteEnv(env, "no_proxy", "amazonaws.com")
	env = overwriteEnv(env, "AWS_CONFIG_FILE", cfg.Name())
	env = overwriteEnv(env, "AWS_DEFAULT_PROFILE", input.Profile)

	if input.WriteEnv {
		env = overwriteEnv(env, "AWS_ACCESS_KEY_ID", val.AccessKeyID)
		env = overwriteEnv(env, "AWS_SECRET_ACCESS_KEY", val.SecretAccessKey)

		if val.SessionToken != "" {
			env = overwriteEnv(env, "AWS_SESSION_TOKEN", val.SessionToken)
			env = overwriteEnv(env, "AWS_SECURITY_TOKEN", val.SessionToken)
		}
	}

	cmd := exec.Command(input.Command, input.Args...)
	cmd.Env = env
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	go func() {
		sig := <-input.Signals
		if cmd.Process != nil {
			cmd.Process.Signal(sig)
		}
	}()

	var waitStatus syscall.WaitStatus
	if err := cmd.Run(); err != nil {
		if err != nil {
			ui.Error.Println(err)
		}
		if exitError, ok := err.(*exec.ExitError); ok {
			waitStatus = exitError.Sys().(syscall.WaitStatus)
			os.Exit(waitStatus.ExitStatus())
		}
	}

}

// write out a config excluding role switching keys
func profileConfig(profile string) (*os.File, error) {
	conf, err := parseProfiles()
	if err != nil {
		return nil, err
	}

	tmpConfig, err := ioutil.TempFile(os.TempDir(), "aws-vault")
	if err != nil {
		return nil, err
	}

	// allow some time for keychain prompt
	newConfig := map[string]string{
		"metadata_service_timeout":      "15",
		"metadata_service_num_attempts": "2",
	}

	for k, v := range conf[profile] {
		if k != "source_profile" && k != "role_arn" {
			newConfig[k] = v
		}
	}

	return tmpConfig, writeProfiles(tmpConfig, profiles{profile: newConfig})
}

func overwriteEnv(env []string, key, val string) []string {
	var found bool

	for idx, e := range env {
		if strings.HasPrefix(key+"=", e) {
			env[idx] = key + "=" + val
			found = true
		} else {
			env[idx] = e
		}
	}

	if !found {
		env = append(env, key+"="+val)
	}

	return env
}

func connectProxy(w http.ResponseWriter, r *http.Request) error {
	d, err := net.Dial("tcp", r.RequestURI)
	if err != nil {
		return err
	}
	hj, ok := w.(http.Hijacker)
	if !ok {
		return err
	}
	conn, bufrw, err := hj.Hijack()
	if err != nil {
		return err
	}
	defer conn.Close()
	defer d.Close()

	bufrw.WriteString("HTTP/1.0 200 Connection established\r\n\r\n")
	bufrw.Flush()

	go io.Copy(d, conn)
	io.Copy(conn, d)

	return nil
}

func proxyHandler(upstream http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%s %s", r.Method, r.RequestURI)
		if r.Method == "GET" && strings.HasPrefix(r.URL.Path, "/latest/meta-data") {
			upstream.ServeHTTP(w, r)
		} else if r.Method == "CONNECT" {
			if err := connectProxy(w, r); err != nil {
				http.Error(w, "Error connecting to proxy: "+err.Error(), http.StatusGatewayTimeout)
			}
		} else {
			proxy := &httputil.ReverseProxy{Director: func(req *http.Request) {
				req.Method = "GET"
				req.URL.Scheme = "http"
				req.URL.Host = r.Host
			}}
			proxy.ServeHTTP(w, r)
		}
	})
}
<|endoftext|>


<commit_before>package amqpfeed

import (
	"bytes"
	"encoding/json"
	"github.com/straumur/straumur"
	"github.com/streadway/amqp"
)

type Consumer struct {
	conn    *amqp.Connection
	channel *amqp.Channel
	tag     string
	done    chan error
	events  chan straumur.Event
}

func NewConsumer(amqpURI, exchange, exchangeType, queueName, key, ctag string, durable, auto_delete bool) (*Consumer, error) {

	c := &Consumer{
		conn:    nil,
		channel: nil,
		tag:     ctag,
		done:    make(chan error),
		events:  make(chan straumur.Event),
	}

	var err error

	c.conn, err = amqp.Dial(amqpURI)
	if err != nil {
		return nil, err
	}

	c.channel, err = c.conn.Channel()
	if err != nil {
		return nil, err
	}

	if err = c.channel.ExchangeDeclare(
		exchange,     // name of the exchange
		exchangeType, // type
		durable,      // durable
		auto_delete,  // delete when complete
		false,        // internal
		false,        // noWait
		nil,          // arguments
	); err != nil {
		return nil, err
	}

	queue, err := c.channel.QueueDeclare(
		queueName,   // name of the queue
		durable,     // durable
		auto_delete, // delete when usused
		false,       // exclusive
		false,       // noWait
		nil,         // arguments
	)
	if err != nil {
		return nil, err
	}

	if err = c.channel.QueueBind(
		queue.Name, // name of the queue
		key,        // bindingKey
		exchange,   // sourceExchange
		false,      // noWait
		nil,        // arguments
	); err != nil {
		return nil, err
	}

	deliveries, err := c.channel.Consume(
		queue.Name, // name
		c.tag,      // consumerTag,
		false,      // noAck
		false,      // exclusive
		false,      // noLocal
		false,      // noWait
		nil,        // arguments
	)

	if err != nil {
		return nil, err
	}

	go c.handle(deliveries, c.done)

	return c, nil
}

func (c *Consumer) Close() error {

	// will close() the deliveries channel
	if err := c.channel.Cancel(c.tag, true); err != nil {
		return err
	}

	if err := c.conn.Close(); err != nil {
		return err
	}

	return <-c.done
}

func (c *Consumer) Updates() <-chan straumur.Event {
	return c.events
}

func (c *Consumer) handle(deliveries <-chan amqp.Delivery, done chan error) {
	for d := range deliveries {
		decoder := json.NewDecoder(bytes.NewReader(d.Body))
		var e straumur.Event
		err := decoder.Decode(&e)
		if err != nil {
			done <- err
		}
		c.events <- e
	}
	done <- nil
}
<commit_msg>Added TODO<commit_after>package amqpfeed

import (
	"bytes"
	"encoding/json"
	"github.com/straumur/straumur"
	"github.com/streadway/amqp"
)

type Consumer struct {
	conn    *amqp.Connection
	channel *amqp.Channel
	tag     string
	done    chan error
	events  chan straumur.Event
}

//TODO: Document different usage of queues.
func NewConsumer(amqpURI, exchange, exchangeType, queueName, key, ctag string, durable, auto_delete bool) (*Consumer, error) {

	c := &Consumer{
		conn:    nil,
		channel: nil,
		tag:     ctag,
		done:    make(chan error),
		events:  make(chan straumur.Event),
	}

	var err error

	c.conn, err = amqp.Dial(amqpURI)
	if err != nil {
		return nil, err
	}

	c.channel, err = c.conn.Channel()
	if err != nil {
		return nil, err
	}

	if err = c.channel.ExchangeDeclare(
		exchange,     // name of the exchange
		exchangeType, // type
		durable,      // durable
		auto_delete,  // delete when complete
		false,        // internal
		false,        // noWait
		nil,          // arguments
	); err != nil {
		return nil, err
	}

	queue, err := c.channel.QueueDeclare(
		queueName,   // name of the queue
		durable,     // durable
		auto_delete, // delete when usused
		false,       // exclusive
		false,       // noWait
		nil,         // arguments
	)
	if err != nil {
		return nil, err
	}

	if err = c.channel.QueueBind(
		queue.Name, // name of the queue
		key,        // bindingKey
		exchange,   // sourceExchange
		false,      // noWait
		nil,        // arguments
	); err != nil {
		return nil, err
	}

	deliveries, err := c.channel.Consume(
		queue.Name, // name
		c.tag,      // consumerTag,
		false,      // noAck
		false,      // exclusive
		false,      // noLocal
		false,      // noWait
		nil,        // arguments
	)

	if err != nil {
		return nil, err
	}

	go c.handle(deliveries, c.done)

	return c, nil
}

func (c *Consumer) Close() error {

	// will close() the deliveries channel
	if err := c.channel.Cancel(c.tag, true); err != nil {
		return err
	}

	if err := c.conn.Close(); err != nil {
		return err
	}

	return <-c.done
}

func (c *Consumer) Updates() <-chan straumur.Event {
	return c.events
}

func (c *Consumer) handle(deliveries <-chan amqp.Delivery, done chan error) {
	for d := range deliveries {
		decoder := json.NewDecoder(bytes.NewReader(d.Body))
		var e straumur.Event
		err := decoder.Decode(&e)
		if err != nil {
			done <- err
		}
		c.events <- e
	}
	done <- nil
}
<|endoftext|>


<commit_before>package gannoy

import (
	"bytes"
	"encoding/binary"
	"io"
	"os"
	"syscall"
)

type File struct {
	tree       int
	dim        int
	K          int
	file       *os.File
	filename   string
	appendFile *os.File
	createChan chan createArgs
	locker     Locker
}

func newFile(filename string, tree, dim, K int) *File {
	_, err := os.Stat(filename)
	if err != nil {
		f, _ := os.Create(filename)
		f.Close()
	}

	file, _ := os.OpenFile(filename, os.O_RDWR, 0)
	appendFile, _ := os.OpenFile(filename, os.O_RDWR|os.O_APPEND, 0)

	f := &File{
		tree:       tree,
		dim:        dim,
		K:          K,
		file:       file,
		filename:   filename,
		appendFile: appendFile,
		createChan: make(chan createArgs, 1),
		locker:     newLocker(),
	}
	go f.creator()
	return f
}

func (f *File) Create(n Node) (int, error) {
	args := createArgs{node: n, result: make(chan createResult)}
	f.createChan <- args
	result := <-args.result
	return result.id, result.err
}

func (f *File) create(n Node) (int, error) {
	buf := &bytes.Buffer{}
	f.nodeToBuf(buf, n)
	id := f.nodeCount()
	_, err := f.appendFile.Write(buf.Bytes())
	return id, err
}

func (f *File) Find(id int) (Node, error) {
	node := Node{}
	node.id = id
	node.storage = f
	offset := f.offset(id)
	err := f.locker.ReadLock(f.file.Fd(), offset, f.nodeSize())
	if err != nil {
		return node, err
	}
	defer f.locker.UnLock(f.file.Fd(), offset, f.nodeSize())

	b := make([]byte, f.nodeSize())
	_, err = syscall.Pread(int(f.file.Fd()), b, offset)
	if err != nil {
		return node, err
	}

	buf := bytes.NewReader(b)

	var free bool
	binary.Read(buf, binary.BigEndian, &free)
	node.free = free

	var nDescendants int32
	binary.Read(buf, binary.BigEndian, &nDescendants)
	node.nDescendants = int(nDescendants)

	var key int32
	binary.Read(buf, binary.BigEndian, &key)
	node.key = int(key)

	parents := make([]int32, f.tree)
	binary.Read(buf, binary.BigEndian, &parents)
	nodeParents := make([]int, f.tree)
	for i, parent := range parents {
		nodeParents[i] = int(parent)
	}
	node.parents = nodeParents

	if node.nDescendants == 1 {
		// leaf node
		buf.Seek(int64(4*2), io.SeekCurrent) // skip children
		node.children = []int{0, 0}

		vec := make([]float64, f.dim)
		binary.Read(buf, binary.BigEndian, &vec)
		node.v = vec
	} else if node.nDescendants <= f.K {
		// bucket node
		children := make([]int32, nDescendants)
		binary.Read(buf, binary.BigEndian, &children)
		nodeChildren := make([]int, nDescendants)
		for i, child := range children {
			nodeChildren[i] = int(child)
		}
		node.children = nodeChildren
	} else {
		// other node
		children := make([]int32, 2)
		binary.Read(buf, binary.BigEndian, &children)
		nodeChildren := make([]int, 2)
		for i, child := range children {
			nodeChildren[i] = int(child)
		}
		node.children = nodeChildren

		vec := make([]float64, f.dim)
		binary.Read(buf, binary.BigEndian, &vec)
		node.v = vec
	}
	return node, nil
}

func (f *File) Update(n Node) error {
	buf := &bytes.Buffer{}
	f.nodeToBuf(buf, n)
	offset := f.offset(n.id)
	file, _ := os.OpenFile(f.filename, os.O_RDWR, 0)
	defer file.Close()

	err := f.locker.WriteLock(file.Fd(), offset, f.nodeSize())
	if err != nil {
		return err
	}
	defer f.locker.UnLock(file.Fd(), offset, f.nodeSize())

	_, err = syscall.Pwrite(int(file.Fd()), buf.Bytes(), offset)
	return err
}

func (f *File) UpdateParent(id, rootIndex, parent int) error {
	offset := f.offset(id) +
		int64(1+ // free
			4+ // nDescendants
			4+ // key
			4*rootIndex) // parents
	buf := &bytes.Buffer{}
	binary.Write(buf, binary.BigEndian, int32(parent))

	file, _ := os.OpenFile(f.filename, os.O_RDWR, 0)
	defer file.Close()

	err := f.locker.WriteLock(file.Fd(), offset, 4)
	if err != nil {
		return err
	}
	defer f.locker.UnLock(file.Fd(), offset, 4)

	_, err = syscall.Pwrite(int(file.Fd()), buf.Bytes(), offset)
	return err
}

func (f *File) Delete(n Node) error {
	n.free = true
	return f.Update(n)
}

func (f *File) Iterate(c chan Node) {
	count := f.nodeCount()
	// TODO: Use goroutine
	for i := 0; i < count; i++ {
		n, err := f.Find(i)
		if err != nil {
			close(c)
		}
		c <- n
	}
	close(c)
}

func (f File) offset(id int) int64 {
	return (int64(id) * f.nodeSize())
}

func (f File) nodeCount() int {
	stat, _ := f.file.Stat()
	size := stat.Size()
	return int(size / f.nodeSize())
}

func (f File) nodeSize() int64 {
	return int64(1 + // free
		4 + // nDescendants
		4 + // key
		4*f.tree + // parents
		4*2 + // children
		8*f.dim) // v
}

func (f File) nodeToBuf(buf *bytes.Buffer, node Node) {
	// 1bytes free
	binary.Write(buf, binary.BigEndian, node.free)

	// 4bytes nDescendants
	binary.Write(buf, binary.BigEndian, int32(node.nDescendants))

	// 4bytes key
	binary.Write(buf, binary.BigEndian, int32(node.key))

	// 4bytes parents
	parents := make([]int32, len(node.parents))
	for i, parent := range node.parents {
		parents[i] = int32(parent)
	}
	binary.Write(buf, binary.BigEndian, parents)

	if node.isBucket() {
		// 4bytes children in K
		children := make([]int32, f.K)
		for i, child := range node.children {
			children[i] = int32(child)
		}
		binary.Write(buf, binary.BigEndian, children)

		// padding by zero
		remainingSize := ((2*4 + 8*f.dim) - (4 * f.K))
		binary.Write(buf, binary.BigEndian, make([]int32, remainingSize/4))
	} else {
		// 4bytes children in K
		children := make([]int32, 2)
		for i, child := range node.children {
			children[i] = int32(child)
		}
		binary.Write(buf, binary.BigEndian, children)

		// 8bytes v in f
		vec := make([]float64, f.dim)
		for i, v := range node.v {
			vec[i] = float64(v)
		}
		binary.Write(buf, binary.BigEndian, vec)
	}
}

type createArgs struct {
	node   Node
	result chan createResult
}

type createResult struct {
	id  int
	err error
}

func (f *File) creator() {
	for args := range f.createChan {
		id, err := f.create(args.node)
		args.result <- createResult{
			id:  id,
			err: err,
		}
	}
}

func (f File) size() int64 {
	info, _ := f.file.Stat()
	return info.Size()
}
<commit_msg>nodeSize is calculated only once.<commit_after>package gannoy

import (
	"bytes"
	"encoding/binary"
	"io"
	"os"
	"syscall"
)

type File struct {
	tree       int
	dim        int
	K          int
	file       *os.File
	filename   string
	appendFile *os.File
	createChan chan createArgs
	locker     Locker
	nodeSize   int64
}

func newFile(filename string, tree, dim, K int) *File {
	_, err := os.Stat(filename)
	if err != nil {
		f, _ := os.Create(filename)
		f.Close()
	}

	file, _ := os.OpenFile(filename, os.O_RDWR, 0)
	appendFile, _ := os.OpenFile(filename, os.O_RDWR|os.O_APPEND, 0)

	f := &File{
		tree:       tree,
		dim:        dim,
		K:          K,
		file:       file,
		filename:   filename,
		appendFile: appendFile,
		createChan: make(chan createArgs, 1),
		locker:     newLocker(),
		nodeSize: int64(1 + // free
			4 + // nDescendants
			4 + // key
			4*tree + // parents
			4*2 + // children
			8*dim), // v
	}
	go f.creator()
	return f
}

func (f *File) Create(n Node) (int, error) {
	args := createArgs{node: n, result: make(chan createResult)}
	f.createChan <- args
	result := <-args.result
	return result.id, result.err
}

func (f *File) create(n Node) (int, error) {
	buf := &bytes.Buffer{}
	f.nodeToBuf(buf, n)
	id := f.nodeCount()
	_, err := f.appendFile.Write(buf.Bytes())
	return id, err
}

func (f *File) Find(id int) (Node, error) {
	node := Node{}
	node.id = id
	node.storage = f
	offset := f.offset(id)
	err := f.locker.ReadLock(f.file.Fd(), offset, f.nodeSize)
	if err != nil {
		return node, err
	}
	defer f.locker.UnLock(f.file.Fd(), offset, f.nodeSize)

	b := make([]byte, f.nodeSize)
	_, err = syscall.Pread(int(f.file.Fd()), b, offset)
	if err != nil {
		return node, err
	}

	buf := bytes.NewReader(b)

	var free bool
	binary.Read(buf, binary.BigEndian, &free)
	node.free = free

	var nDescendants int32
	binary.Read(buf, binary.BigEndian, &nDescendants)
	node.nDescendants = int(nDescendants)

	var key int32
	binary.Read(buf, binary.BigEndian, &key)
	node.key = int(key)

	parents := make([]int32, f.tree)
	binary.Read(buf, binary.BigEndian, &parents)
	nodeParents := make([]int, f.tree)
	for i, parent := range parents {
		nodeParents[i] = int(parent)
	}
	node.parents = nodeParents

	if node.nDescendants == 1 {
		// leaf node
		buf.Seek(int64(4*2), io.SeekCurrent) // skip children
		node.children = []int{0, 0}

		vec := make([]float64, f.dim)
		binary.Read(buf, binary.BigEndian, &vec)
		node.v = vec
	} else if node.nDescendants <= f.K {
		// bucket node
		children := make([]int32, nDescendants)
		binary.Read(buf, binary.BigEndian, &children)
		nodeChildren := make([]int, nDescendants)
		for i, child := range children {
			nodeChildren[i] = int(child)
		}
		node.children = nodeChildren
	} else {
		// other node
		children := make([]int32, 2)
		binary.Read(buf, binary.BigEndian, &children)
		nodeChildren := make([]int, 2)
		for i, child := range children {
			nodeChildren[i] = int(child)
		}
		node.children = nodeChildren

		vec := make([]float64, f.dim)
		binary.Read(buf, binary.BigEndian, &vec)
		node.v = vec
	}
	return node, nil
}

func (f *File) Update(n Node) error {
	buf := &bytes.Buffer{}
	f.nodeToBuf(buf, n)
	offset := f.offset(n.id)
	file, _ := os.OpenFile(f.filename, os.O_RDWR, 0)
	defer file.Close()

	err := f.locker.WriteLock(file.Fd(), offset, f.nodeSize)
	if err != nil {
		return err
	}
	defer f.locker.UnLock(file.Fd(), offset, f.nodeSize)

	_, err = syscall.Pwrite(int(file.Fd()), buf.Bytes(), offset)
	return err
}

func (f *File) UpdateParent(id, rootIndex, parent int) error {
	offset := f.offset(id) +
		int64(1+ // free
			4+ // nDescendants
			4+ // key
			4*rootIndex) // parents
	buf := &bytes.Buffer{}
	binary.Write(buf, binary.BigEndian, int32(parent))

	file, _ := os.OpenFile(f.filename, os.O_RDWR, 0)
	defer file.Close()

	err := f.locker.WriteLock(file.Fd(), offset, 4)
	if err != nil {
		return err
	}
	defer f.locker.UnLock(file.Fd(), offset, 4)

	_, err = syscall.Pwrite(int(file.Fd()), buf.Bytes(), offset)
	return err
}

func (f *File) Delete(n Node) error {
	n.free = true
	return f.Update(n)
}

func (f *File) Iterate(c chan Node) {
	count := f.nodeCount()
	// TODO: Use goroutine
	for i := 0; i < count; i++ {
		n, err := f.Find(i)
		if err != nil {
			close(c)
		}
		c <- n
	}
	close(c)
}

func (f File) offset(id int) int64 {
	return (int64(id) * f.nodeSize)
}

func (f File) nodeCount() int {
	stat, _ := f.file.Stat()
	size := stat.Size()
	return int(size / f.nodeSize)
}

func (f File) nodeToBuf(buf *bytes.Buffer, node Node) {
	// 1bytes free
	binary.Write(buf, binary.BigEndian, node.free)

	// 4bytes nDescendants
	binary.Write(buf, binary.BigEndian, int32(node.nDescendants))

	// 4bytes key
	binary.Write(buf, binary.BigEndian, int32(node.key))

	// 4bytes parents
	parents := make([]int32, len(node.parents))
	for i, parent := range node.parents {
		parents[i] = int32(parent)
	}
	binary.Write(buf, binary.BigEndian, parents)

	if node.isBucket() {
		// 4bytes children in K
		children := make([]int32, f.K)
		for i, child := range node.children {
			children[i] = int32(child)
		}
		binary.Write(buf, binary.BigEndian, children)

		// padding by zero
		remainingSize := ((2*4 + 8*f.dim) - (4 * f.K))
		binary.Write(buf, binary.BigEndian, make([]int32, remainingSize/4))
	} else {
		// 4bytes children in K
		children := make([]int32, 2)
		for i, child := range node.children {
			children[i] = int32(child)
		}
		binary.Write(buf, binary.BigEndian, children)

		// 8bytes v in f
		vec := make([]float64, f.dim)
		for i, v := range node.v {
			vec[i] = float64(v)
		}
		binary.Write(buf, binary.BigEndian, vec)
	}
}

type createArgs struct {
	node   Node
	result chan createResult
}

type createResult struct {
	id  int
	err error
}

func (f *File) creator() {
	for args := range f.createChan {
		id, err := f.create(args.node)
		args.result <- createResult{
			id:  id,
			err: err,
		}
	}
}

func (f File) size() int64 {
	info, _ := f.file.Stat()
	return info.Size()
}
<|endoftext|>


<commit_before>package scipipe

import (
	"bytes"
	"io/ioutil"
	"os"
	"os/exec"
	"sync"
)

// ======= FileTarget ========

// FileTarget contains information and helper methods for a physical file on a
// normal disk.
type FileTarget struct {
	path     string
	buffer   *bytes.Buffer
	doStream bool
	lock     *sync.Mutex
}

// Create new FileTarget "object"
func NewFileTarget(path string) *FileTarget {
	ft := new(FileTarget)
	ft.path = path
	ft.lock = new(sync.Mutex)
	//Don't init buffer if not needed?
	//buf := make([]byte, 0, 128)
	//ft.buffer = bytes.NewBuffer(buf)
	return ft
}

// Get the (final) path of the physical file
func (ft *FileTarget) GetPath() string {
	return ft.path
}

// Get the temporary path of the physical file
func (ft *FileTarget) GetTempPath() string {
	return ft.path + ".tmp"
}

// Get the path to use when a FIFO file is used instead of a normal file
func (ft *FileTarget) GetFifoPath() string {
	return ft.path + ".fifo"
}

// Open the file and return a file handle (*os.File)
func (ft *FileTarget) Open() *os.File {
	f, err := os.Open(ft.GetPath())
	Check(err)
	return f
}

// Open the temp file and return a file handle (*os.File)
func (ft *FileTarget) OpenTemp() *os.File {
	f, err := os.Open(ft.GetTempPath())
	Check(err)
	return f
}

// Open the file for writing return a file handle (*os.File)
func (ft *FileTarget) OpenWriteTemp() *os.File {
	f, err := os.Create(ft.GetTempPath())
	Check(err)
	return f
}

// Read the whole content of the file and return as a byte array ([]byte)
func (ft *FileTarget) Read() []byte {
	dat, err := ioutil.ReadFile(ft.GetPath())
	Check(err)
	return dat
}

// Write a byte array ([]byte) to the file (first to its temp path, and then atomize)
func (ft *FileTarget) WriteTempFile(dat []byte) {
	err := ioutil.WriteFile(ft.GetTempPath(), dat, 0644)
	Check(err)
}

// Change from the temporary file name to the final file name
func (ft *FileTarget) Atomize() {
	Debug.Println("FileTarget: Atomizing", ft.GetTempPath(), "->", ft.GetPath())
	ft.lock.Lock()
	err := os.Rename(ft.GetTempPath(), ft.path)
	Check(err)
	ft.lock.Unlock()
	Debug.Println("FileTarget: Done atomizing", ft.GetTempPath(), "->", ft.GetPath())
}

// Create FIFO file for the FileTarget
func (ft *FileTarget) CreateFifo() {
	ft.lock.Lock()
	cmd := "mkfifo " + ft.GetFifoPath()
	Debug.Println("Now creating FIFO with command:", cmd)

	if _, err := os.Stat(ft.GetFifoPath()); err == nil {
		Warning.Println("FIFO already exists, so not creating a new one:", ft.GetFifoPath())
	} else {
		_, err := exec.Command("bash", "-c", cmd).Output()
		Check(err)
	}

	ft.lock.Unlock()
}

// Remove the FIFO file, if it exists
func (ft *FileTarget) RemoveFifo() {
	// FIXME: Shouldn't we check first whether the fifo exists?
	ft.lock.Lock()
	output, err := exec.Command("bash", "-c", "rm "+ft.GetFifoPath()).Output()
	Check(err)
	Debug.Println("Removed FIFO output: ", output)
	ft.lock.Unlock()
}

// Check if the file exists (at its final file name)
func (ft *FileTarget) Exists() bool {
	exists := false
	ft.lock.Lock()
	if _, err := os.Stat(ft.GetPath()); err == nil {
		exists = true
	}
	ft.lock.Unlock()
	return exists
}

// ======= FileQueue =======

// FileQueue is initialized by a set of strings with file paths, and from that
// will return instantiated FileTargets on its Out-port, when run.
type FileQueue struct {
	Process
	Out       chan *FileTarget
	FilePaths []string
}

func FQ(fps ...string) (fq *FileQueue) {
	return NewFileQueue(fps...)
}

// Initialize a new FileQueue component from a list of file paths
func NewFileQueue(filePaths ...string) (fq *FileQueue) {
	fq = &FileQueue{
		Out:       make(chan *FileTarget, BUFSIZE),
		FilePaths: filePaths,
	}
	return
}

// Execute the FileQueue, returning instantiated FileTargets
func (proc *FileQueue) Run() {
	defer close(proc.Out)
	for _, fp := range proc.FilePaths {
		proc.Out <- NewFileTarget(fp)
	}
}

// ======= Sink =======

// Sink is a simple component that just receives FileTargets on its In-port
// without doing anything with them
type Sink struct {
	Process
	In chan *FileTarget
}

// Instantiate a Sink component
func NewSink() (s *Sink) {
	return &Sink{
		In: make(chan *FileTarget, BUFSIZE),
	}
}

// Execute the Sink component
func (proc *Sink) Run() {
	for ft := range proc.In {
		Debug.Println("Received file in sink: ", ft.GetPath())
	}
}
<commit_msg>Allow taking multiple inputs, with an InPorts map in Sink<commit_after>package scipipe

import (
	"bytes"
	"io/ioutil"
	"os"
	"os/exec"
	"sync"
)

// ======= FileTarget ========

// FileTarget contains information and helper methods for a physical file on a
// normal disk.
type FileTarget struct {
	path     string
	buffer   *bytes.Buffer
	doStream bool
	lock     *sync.Mutex
}

// Create new FileTarget "object"
func NewFileTarget(path string) *FileTarget {
	ft := new(FileTarget)
	ft.path = path
	ft.lock = new(sync.Mutex)
	//Don't init buffer if not needed?
	//buf := make([]byte, 0, 128)
	//ft.buffer = bytes.NewBuffer(buf)
	return ft
}

// Get the (final) path of the physical file
func (ft *FileTarget) GetPath() string {
	return ft.path
}

// Get the temporary path of the physical file
func (ft *FileTarget) GetTempPath() string {
	return ft.path + ".tmp"
}

// Get the path to use when a FIFO file is used instead of a normal file
func (ft *FileTarget) GetFifoPath() string {
	return ft.path + ".fifo"
}

// Open the file and return a file handle (*os.File)
func (ft *FileTarget) Open() *os.File {
	f, err := os.Open(ft.GetPath())
	Check(err)
	return f
}

// Open the temp file and return a file handle (*os.File)
func (ft *FileTarget) OpenTemp() *os.File {
	f, err := os.Open(ft.GetTempPath())
	Check(err)
	return f
}

// Open the file for writing return a file handle (*os.File)
func (ft *FileTarget) OpenWriteTemp() *os.File {
	f, err := os.Create(ft.GetTempPath())
	Check(err)
	return f
}

// Read the whole content of the file and return as a byte array ([]byte)
func (ft *FileTarget) Read() []byte {
	dat, err := ioutil.ReadFile(ft.GetPath())
	Check(err)
	return dat
}

// Write a byte array ([]byte) to the file (first to its temp path, and then atomize)
func (ft *FileTarget) WriteTempFile(dat []byte) {
	err := ioutil.WriteFile(ft.GetTempPath(), dat, 0644)
	Check(err)
}

// Change from the temporary file name to the final file name
func (ft *FileTarget) Atomize() {
	Debug.Println("FileTarget: Atomizing", ft.GetTempPath(), "->", ft.GetPath())
	ft.lock.Lock()
	err := os.Rename(ft.GetTempPath(), ft.path)
	Check(err)
	ft.lock.Unlock()
	Debug.Println("FileTarget: Done atomizing", ft.GetTempPath(), "->", ft.GetPath())
}

// Create FIFO file for the FileTarget
func (ft *FileTarget) CreateFifo() {
	ft.lock.Lock()
	cmd := "mkfifo " + ft.GetFifoPath()
	Debug.Println("Now creating FIFO with command:", cmd)

	if _, err := os.Stat(ft.GetFifoPath()); err == nil {
		Warning.Println("FIFO already exists, so not creating a new one:", ft.GetFifoPath())
	} else {
		_, err := exec.Command("bash", "-c", cmd).Output()
		Check(err)
	}

	ft.lock.Unlock()
}

// Remove the FIFO file, if it exists
func (ft *FileTarget) RemoveFifo() {
	// FIXME: Shouldn't we check first whether the fifo exists?
	ft.lock.Lock()
	output, err := exec.Command("bash", "-c", "rm "+ft.GetFifoPath()).Output()
	Check(err)
	Debug.Println("Removed FIFO output: ", output)
	ft.lock.Unlock()
}

// Check if the file exists (at its final file name)
func (ft *FileTarget) Exists() bool {
	exists := false
	ft.lock.Lock()
	if _, err := os.Stat(ft.GetPath()); err == nil {
		exists = true
	}
	ft.lock.Unlock()
	return exists
}

// ======= FileQueue =======

// FileQueue is initialized by a set of strings with file paths, and from that
// will return instantiated FileTargets on its Out-port, when run.
type FileQueue struct {
	Process
	Out       chan *FileTarget
	FilePaths []string
}

func FQ(fps ...string) (fq *FileQueue) {
	return NewFileQueue(fps...)
}

// Initialize a new FileQueue component from a list of file paths
func NewFileQueue(filePaths ...string) (fq *FileQueue) {
	fq = &FileQueue{
		Out:       make(chan *FileTarget, BUFSIZE),
		FilePaths: filePaths,
	}
	return
}

// Execute the FileQueue, returning instantiated FileTargets
func (proc *FileQueue) Run() {
	defer close(proc.Out)
	for _, fp := range proc.FilePaths {
		proc.Out <- NewFileTarget(fp)
	}
}

// ======= Sink =======

// Sink is a simple component that just receives FileTargets on its In-port
// without doing anything with them
type Sink struct {
	Process
	In      chan *FileTarget
	InPorts map[string]chan *FileTarget
}

// Instantiate a Sink component
func NewSink() (s *Sink) {
	return &Sink{
		In:      nil,
		InPorts: make(map[string]chan *FileTarget),
	}
}

// Execute the Sink component
func (proc *Sink) Run() {
	if proc.In != nil {
		for ft := range proc.In {
			Debug.Println("Received file in sink: ", ft.GetPath())
		}
	}

	ok := true
	var ft *FileTarget
	for len(proc.InPorts) > 0 {
		for key, ch := range proc.InPorts {
			ft, ok = <-ch
			if !ok {
				delete(proc.InPorts, key)
				continue
			}
			Debug.Println("Received file in sink: ", ft.GetPath())
		}
	}
}
<|endoftext|>


<commit_before>package scipipe

import (
	"io/ioutil"
	"os"
	"time"
)

type FileTarget struct {
	path string
}

func NewFileTarget(path string) *FileTarget {
	ft := new(FileTarget)
	ft.path = path
	return ft
}

func (ft *FileTarget) GetPath() string {
	return ft.path
}

func (ft *FileTarget) GetTempPath() string {
	return ft.path + ".tmp"
}

func (ft *FileTarget) Atomize() {
	time.Sleep(1 * time.Second) // TODO: Remove in production. Just for demo purposes!
	err := os.Rename(ft.GetTempPath(), ft.path)
	Check(err)
}

func (ft *FileTarget) Exists() bool {
	if _, err := os.Stat(ft.GetPath()); err == nil {
		return true
	}
	return false
}
<commit_msg>More methods in FileTarget<commit_after>package scipipe

import (
	"io/ioutil"
	"os"
	"time"
)

type FileTarget struct {
	path string
}

func NewFileTarget(path string) *FileTarget {
	ft := new(FileTarget)
	ft.path = path
	return ft
}

func (ft *FileTarget) GetPath() string {
	return ft.path
}

func (ft *FileTarget) GetTempPath() string {
	return ft.path + ".tmp"
}

func (ft *FileTarget) Open() *os.File {
	f, err := os.Open(ft.GetPath())
	Check(err)
	return f
}

func (ft *FileTarget) Read() []byte {
	dat, err := ioutil.ReadFile(ft.GetPath())
	Check(err)
	return dat
}

func (ft *FileTarget) Write(dat []byte) {
	err := ioutil.WriteFile(ft.GetTempPath(), dat, 0644)
	ft.Atomize()
	Check(err)
}

func (ft *FileTarget) Atomize() {
	time.Sleep(1 * time.Millisecond) // TODO: Remove in production. Just for demo purposes!
	err := os.Rename(ft.GetTempPath(), ft.path)
	Check(err)
}

func (ft *FileTarget) Exists() bool {
	if _, err := os.Stat(ft.GetPath()); err == nil {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package models

import (
	"encoding/json"
	"socialapi/request"
)

func BuildChannelMessageContainer(id int64, query *request.Query) (*ChannelMessageContainer, error) {
	cm := NewChannelMessage()
	cm.Id = id
	if err := cm.ById(cm.Id); err != nil {
		return nil, err
	}

	cmc := NewChannelMessageContainer()
	cmc.PopulateWith(cm).SetGenerics(query)
	if cmc.Err != nil {
		return nil, cmc.Err
	}

	return cmc, nil
}

func CacheForChannelMessage(id int64) (string, error) {
	query := request.NewQuery().SetDefaults()
	// no need to add IsInteracted data into cache
	query.AddIsInteracted = false
	cmc, err := BuildChannelMessageContainer(id, query)
	d, err := json.Marshal(cmc)
	if err != nil {
		return "", err
	}

	return string(d), err
}

func BuildChannelContainer(id int64, query *request.Query) (*ChannelContainer, error) {
	c := NewChannel()
	c.Id = id
	if err := c.ById(c.Id); err != nil {
		return nil, err
	}

	cc := NewChannelContainer()
	cc.PopulateWith(*c, query.AccountId)

	return cc, cc.Err
}

func CacheForChannel(id int64) (string, error) {
	query := request.NewQuery().SetDefaults()
	cc, err := BuildChannelContainer(id, query)
	d, err := json.Marshal(cc)
	if err != nil {
		return "", err
	}

	return string(d), nil
}
<commit_msg>Social: create cache package<commit_after>package models

import (
	"encoding/json"
	mongomodels "koding/db/models"
	"koding/db/mongodb/modelhelper"

	"socialapi/request"

	"github.com/koding/cache"
)

var Cache *StaticCache

var cacheSize = 10000

// later, this can be lazily init, after database connection established
func init() {
	Cache = &StaticCache{
		Account: &AccountCache{
			oldId: cache.NewLRU(cacheSize),
			nick:  cache.NewLRU(cacheSize),
		},
		Session: &SessionCache{
			session: cache.NewLRU(cacheSize),
		},
	}
}

type StaticCache struct {
	Account *AccountCache
	Session *SessionCache
}

//////////////// Account Cache ////////////////////
type AccountCache struct {
	oldId cache.Cache
	nick  cache.Cache
}

func (a *AccountCache) ByNick(nick string) (*Account, error) {
	data, err := a.nick.Get(nick)
	if err != nil && err != cache.ErrNotFound {
		return nil, err
	}

	if err == nil {
		acc, ok := data.(*Account)
		if ok {
			return acc, nil
		}
	}

	account := NewAccount()
	if err := account.ByNick(nick); err != nil {
		return nil, err
	}

	if err := a.SetToCache(account); err != nil {
		return nil, err
	}

	return account, nil
}

func (a *AccountCache) SetToCache(acc *Account) error {
	if err := a.nick.Set(acc.Nick, acc); err != nil {
		return err
	}

	if err := a.oldId.Set(acc.OldId, acc); err != nil {
		return err
	}
	return nil
}

//////////////// Session Cache ////////////////////
type SessionCache struct {
	session cache.Cache
}

func (s *SessionCache) ById(id string) (*mongomodels.Session, error) {
	data, err := s.session.Get(id)
	if err != nil && err != cache.ErrNotFound {
		return nil, err
	}

	if err == nil {
		ses, ok := data.(*mongomodels.Session)
		if ok {
			return ses, nil
		}
	}

	session, err := modelhelper.GetSession(id)
	if err != nil {
		return nil, err
	}

	if err := s.SetToCache(session); err != nil {
		return nil, err
	}

	return session, nil
}

func (s *SessionCache) SetToCache(ses *mongomodels.Session) error {
	if err := s.session.Set(ses.ClientId, ses); err != nil {
		return err
	}

	return nil
}

func BuildChannelMessageContainer(id int64, query *request.Query) (*ChannelMessageContainer, error) {
	cm := NewChannelMessage()
	cm.Id = id
	if err := cm.ById(cm.Id); err != nil {
		return nil, err
	}

	cmc := NewChannelMessageContainer()
	cmc.PopulateWith(cm).SetGenerics(query)
	if cmc.Err != nil {
		return nil, cmc.Err
	}

	return cmc, nil
}

func CacheForChannelMessage(id int64) (string, error) {
	query := request.NewQuery().SetDefaults()
	// no need to add IsInteracted data into cache
	query.AddIsInteracted = false
	cmc, err := BuildChannelMessageContainer(id, query)
	d, err := json.Marshal(cmc)
	if err != nil {
		return "", err
	}

	return string(d), err
}

func BuildChannelContainer(id int64, query *request.Query) (*ChannelContainer, error) {
	c := NewChannel()
	c.Id = id
	if err := c.ById(c.Id); err != nil {
		return nil, err
	}

	cc := NewChannelContainer()
	cc.PopulateWith(*c, query.AccountId)

	return cc, cc.Err
}

func CacheForChannel(id int64) (string, error) {
	query := request.NewQuery().SetDefaults()
	cc, err := BuildChannelContainer(id, query)
	d, err := json.Marshal(cc)
	if err != nil {
		return "", err
	}

	return string(d), nil
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"context"
	"database/sql"
	"fmt"
	"time"

	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/logutil"
	"vitess.io/vitess/go/vt/topo/topoproto"
	"vitess.io/vitess/go/vt/vtadmin/cluster/discovery"
	"vitess.io/vitess/go/vt/vtadmin/errors"
	"vitess.io/vitess/go/vt/vtadmin/vtadminproto"
	"vitess.io/vitess/go/vt/vtadmin/vtctldclient"
	"vitess.io/vitess/go/vt/vtadmin/vtsql"

	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin"
)

// Cluster is the self-contained unit of services required for vtadmin to talk
// to a vitess cluster. This consists of a discovery service, a database
// connection, and a vtctl client.
type Cluster struct {
	ID        string
	Name      string
	Discovery discovery.Discovery

	DB     vtsql.DB
	Vtctld vtctldclient.Proxy

	// These fields are kept to power debug endpoints.
	// (TODO|@amason): Figure out if these are needed or if there's a way to
	// push down to the credentials / vtsql.
	// vtgateCredentialsPath string
}

// New creates a new Cluster from a Config.
func New(cfg Config) (*Cluster, error) {
	cluster := &Cluster{
		ID:   cfg.ID,
		Name: cfg.Name,
	}

	discoargs := buildPFlagSlice(cfg.DiscoveryFlagsByImpl[cfg.DiscoveryImpl])

	disco, err := discovery.New(cfg.DiscoveryImpl, cluster.ToProto(), discoargs)
	if err != nil {
		return nil, fmt.Errorf("error while creating discovery impl (%s): %w", cfg.DiscoveryImpl, err)
	}

	cluster.Discovery = disco

	protocluster := cluster.ToProto()

	vtsqlargs := buildPFlagSlice(cfg.VtSQLFlags)

	vtsqlCfg, err := vtsql.Parse(protocluster, disco, vtsqlargs)
	if err != nil {
		return nil, fmt.Errorf("error while creating vtsql connection config: %w", err)
	}

	vtctldargs := buildPFlagSlice(cfg.VtctldFlags)

	vtctldCfg, err := vtctldclient.Parse(protocluster, disco, vtctldargs)
	if err != nil {
		return nil, fmt.Errorf("error while creating vtctldclient proxy config: %w", err)
	}

	cluster.DB = vtsql.New(vtsqlCfg)
	cluster.Vtctld = vtctldclient.New(vtctldCfg)

	return cluster, nil
}

// ToProto returns a value-copy protobuf equivalent of the cluster.
func (c Cluster) ToProto() *vtadminpb.Cluster {
	return &vtadminpb.Cluster{
		Id:   c.ID,
		Name: c.Name,
	}
}

func buildPFlagSlice(flags map[string]string) []string {
	args := make([]string, 0, len(flags))
	for k, v := range flags {
		// The k=v syntax is needed to account for negating boolean flags.
		args = append(args, "--"+k+"="+v)
	}

	return args
}

// ParseTablets converts a set of *sql.Rows into a slice of Tablets, for the
// given cluster.
func (c Cluster) ParseTablets(rows *sql.Rows) ([]*vtadminpb.Tablet, error) {
	var tablets []*vtadminpb.Tablet

	for rows.Next() {
		if err := rows.Err(); err != nil {
			return nil, err
		}

		tablet, err := c.parseTablet(rows)
		if err != nil {
			return nil, err
		}

		tablets = append(tablets, tablet)
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	return tablets, nil
}

// Fields are:
// Cell | Keyspace | Shard | TabletType (string) | ServingState (string) | Alias | Hostname | MasterTermStartTime.
func (c Cluster) parseTablet(rows *sql.Rows) (*vtadminpb.Tablet, error) {
	var (
		cell            string
		tabletTypeStr   string
		servingStateStr string
		aliasStr        string
		mtstStr         string
		topotablet      topodatapb.Tablet

		err error
	)

	if err := rows.Scan(
		&cell,
		&topotablet.Keyspace,
		&topotablet.Shard,
		&tabletTypeStr,
		&servingStateStr,
		&aliasStr,
		&topotablet.Hostname,
		&mtstStr,
	); err != nil {
		return nil, err
	}

	tablet := &vtadminpb.Tablet{
		Cluster: &vtadminpb.Cluster{
			Id:   c.ID,
			Name: c.Name,
		},
		Tablet: &topotablet,
	}

	topotablet.Type, err = topoproto.ParseTabletType(tabletTypeStr)
	if err != nil {
		return nil, err
	}

	tablet.State = vtadminproto.ParseTabletServingState(servingStateStr)

	topotablet.Alias, err = topoproto.ParseTabletAlias(aliasStr)
	if err != nil {
		return nil, err
	}

	if topotablet.Alias.Cell != cell {
		// (TODO:@amason) ???
		log.Warningf("tablet cell %s does not match alias %s. ignoring for now", cell, topoproto.TabletAliasString(topotablet.Alias))
	}

	if mtstStr != "" {
		timeTime, err := time.Parse(time.RFC3339, mtstStr)
		if err != nil {
			return nil, err
		}

		topotablet.MasterTermStartTime = logutil.TimeToProto(timeTime)
	}

	return tablet, nil
}

func (c Cluster) GetTablets(ctx context.Context) ([]*vtadminpb.Tablet, error) {
	if err := c.DB.Dial(ctx, ""); err != nil {
		return nil, err
	}

	rows, err := c.DB.ShowTablets(ctx)
	if err != nil {
		return nil, err
	}

	return c.ParseTablets(rows)
}

// FindTablet returns the first tablet in a given cluster that satisfies the filter function.
func (c Cluster) FindTablet(ctx context.Context, filter func(*vtadminpb.Tablet) bool) (*vtadminpb.Tablet, error) {
	tablets, err := c.FindTablets(ctx, filter, 1)
	if err != nil {
		return nil, err
	}

	if len(tablets) != 1 {
		return nil, errors.ErrNoTablet
	}

	return tablets[0], nil
}

// FindTablets returns the first N tablets in the given cluster that satisfy
// the filter function. If N = -1, then all matching tablets are returned.
// Ordering is not guaranteed, and callers should write their filter functions accordingly.
func (c Cluster) FindTablets(ctx context.Context, filter func(*vtadminpb.Tablet) bool, n int) ([]*vtadminpb.Tablet, error) {
	tablets, err := c.GetTablets(ctx)
	if err != nil {
		return nil, err
	}

	if n == -1 {
		n = len(tablets)
	}

	results := make([]*vtadminpb.Tablet, 0, n)
	for _, t := range tablets {
		if len(results) >= n {
			break
		}

		if filter(t) {
			results = append(results, t)
		}
	}

	return results, nil
}
<commit_msg>:goat: Nits<commit_after>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"context"
	"database/sql"
	"fmt"
	"time"

	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/logutil"
	"vitess.io/vitess/go/vt/topo/topoproto"
	"vitess.io/vitess/go/vt/vtadmin/cluster/discovery"
	"vitess.io/vitess/go/vt/vtadmin/errors"
	"vitess.io/vitess/go/vt/vtadmin/vtadminproto"
	"vitess.io/vitess/go/vt/vtadmin/vtctldclient"
	"vitess.io/vitess/go/vt/vtadmin/vtsql"

	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin"
)

// Cluster is the self-contained unit of services required for vtadmin to talk
// to a vitess cluster. This consists of a discovery service, a database
// connection, and a vtctl client.
type Cluster struct {
	ID        string
	Name      string
	Discovery discovery.Discovery

	DB     vtsql.DB
	Vtctld vtctldclient.Proxy

	// These fields are kept to power debug endpoints.
	// (TODO|@amason): Figure out if these are needed or if there's a way to
	// push down to the credentials / vtsql.
	// vtgateCredentialsPath string
}

// New creates a new Cluster from a Config.
func New(cfg Config) (*Cluster, error) {
	cluster := &Cluster{
		ID:   cfg.ID,
		Name: cfg.Name,
	}

	discoargs := buildPFlagSlice(cfg.DiscoveryFlagsByImpl[cfg.DiscoveryImpl])

	disco, err := discovery.New(cfg.DiscoveryImpl, cluster.ToProto(), discoargs)
	if err != nil {
		return nil, fmt.Errorf("error while creating discovery impl (%s): %w", cfg.DiscoveryImpl, err)
	}

	cluster.Discovery = disco

	protocluster := cluster.ToProto()

	vtsqlargs := buildPFlagSlice(cfg.VtSQLFlags)

	vtsqlCfg, err := vtsql.Parse(protocluster, disco, vtsqlargs)
	if err != nil {
		return nil, fmt.Errorf("error while creating vtsql connection config: %w", err)
	}

	vtctldargs := buildPFlagSlice(cfg.VtctldFlags)

	vtctldCfg, err := vtctldclient.Parse(protocluster, disco, vtctldargs)
	if err != nil {
		return nil, fmt.Errorf("error while creating vtctldclient proxy config: %w", err)
	}

	cluster.DB = vtsql.New(vtsqlCfg)
	cluster.Vtctld = vtctldclient.New(vtctldCfg)

	return cluster, nil
}

// ToProto returns a value-copy protobuf equivalent of the cluster.
func (c Cluster) ToProto() *vtadminpb.Cluster {
	return &vtadminpb.Cluster{
		Id:   c.ID,
		Name: c.Name,
	}
}

func buildPFlagSlice(flags map[string]string) []string {
	args := make([]string, 0, len(flags))
	for k, v := range flags {
		// The k=v syntax is needed to account for negating boolean flags.
		args = append(args, "--"+k+"="+v)
	}

	return args
}

// ParseTablets converts a set of *sql.Rows into a slice of Tablets, for the
// given cluster.
func (c *Cluster) ParseTablets(rows *sql.Rows) ([]*vtadminpb.Tablet, error) {
	var tablets []*vtadminpb.Tablet

	for rows.Next() {
		if err := rows.Err(); err != nil {
			return nil, err
		}

		tablet, err := c.parseTablet(rows)
		if err != nil {
			return nil, err
		}

		tablets = append(tablets, tablet)
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	return tablets, nil
}

// Fields are:
// Cell | Keyspace | Shard | TabletType (string) | ServingState (string) | Alias | Hostname | MasterTermStartTime.
func (c *Cluster) parseTablet(rows *sql.Rows) (*vtadminpb.Tablet, error) {
	var (
		cell            string
		tabletTypeStr   string
		servingStateStr string
		aliasStr        string
		mtstStr         string
		topotablet      topodatapb.Tablet

		err error
	)

	if err := rows.Scan(
		&cell,
		&topotablet.Keyspace,
		&topotablet.Shard,
		&tabletTypeStr,
		&servingStateStr,
		&aliasStr,
		&topotablet.Hostname,
		&mtstStr,
	); err != nil {
		return nil, err
	}

	tablet := &vtadminpb.Tablet{
		Cluster: &vtadminpb.Cluster{
			Id:   c.ID,
			Name: c.Name,
		},
		Tablet: &topotablet,
	}

	topotablet.Type, err = topoproto.ParseTabletType(tabletTypeStr)
	if err != nil {
		return nil, err
	}

	tablet.State = vtadminproto.ParseTabletServingState(servingStateStr)

	topotablet.Alias, err = topoproto.ParseTabletAlias(aliasStr)
	if err != nil {
		return nil, err
	}

	if topotablet.Alias.Cell != cell {
		// (TODO:@amason) ???
		log.Warningf("tablet cell %s does not match alias %s. ignoring for now", cell, topoproto.TabletAliasString(topotablet.Alias))
	}

	if mtstStr != "" {
		timeTime, err := time.Parse(time.RFC3339, mtstStr)
		if err != nil {
			return nil, err
		}

		topotablet.MasterTermStartTime = logutil.TimeToProto(timeTime)
	}

	return tablet, nil
}

// GetTablets returns all tablets in the cluster.
func (c *Cluster) GetTablets(ctx context.Context) ([]*vtadminpb.Tablet, error) {
	if err := c.DB.Dial(ctx, ""); err != nil {
		return nil, err
	}

	rows, err := c.DB.ShowTablets(ctx)
	if err != nil {
		return nil, err
	}

	return c.ParseTablets(rows)
}

// FindTablet returns the first tablet in a given cluster that satisfies the filter function.
func (c *Cluster) FindTablet(ctx context.Context, filter func(*vtadminpb.Tablet) bool) (*vtadminpb.Tablet, error) {
	tablets, err := c.FindTablets(ctx, filter, 1)
	if err != nil {
		return nil, err
	}

	if len(tablets) != 1 {
		return nil, errors.ErrNoTablet
	}

	return tablets[0], nil
}

// FindTablets returns the first N tablets in the given cluster that satisfy
// the filter function. If N = -1, then all matching tablets are returned.
// Ordering is not guaranteed, and callers should write their filter functions accordingly.
func (c *Cluster) FindTablets(ctx context.Context, filter func(*vtadminpb.Tablet) bool, n int) ([]*vtadminpb.Tablet, error) {
	tablets, err := c.GetTablets(ctx)
	if err != nil {
		return nil, err
	}

	if n == -1 {
		n = len(tablets)
	}

	results := make([]*vtadminpb.Tablet, 0, n)
	for _, t := range tablets {
		if len(results) >= n {
			break
		}

		if filter(t) {
			results = append(results, t)
		}
	}

	return results, nil
}
<|endoftext|>


<commit_before>package goblet

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/url"
	"reflect"
	"strings"

	"github.com/extrame/unmarshall"
	"github.com/sirupsen/logrus"
)

type FormFillFn func(content string) (interface{}, error)
type MultiFormFillFn func(ctx *Context, id string) (interface{}, error)

func (s *Server) AddFillForTypeInForm(typ string, fn FormFillFn) {
	s.filler[typ] = fn
}

func (s *Server) AddFillForTypeInMultiForm(typ string, fn MultiFormFillFn) {
	s.multiFiller[typ] = fn
}

// types that impliment RequestDecoder can unmarshal
// the request body into an apropriate type/struct
type RequestDecoder interface {
	Unmarshal(cx *Context, v interface{}, autofill bool) error
}

// a JSON decoder for request body (just a wrapper to json.Unmarshal)
type JsonRequestDecoder struct{}

func (d *JsonRequestDecoder) Unmarshal(cx *Context, v interface{}, autofill bool) (err error) {
	// read body
	if cx.fill_bts == nil {
		cx.fill_bts, err = ioutil.ReadAll(cx.request.Body)
	}
	if err != nil {
		return err
	}
	return json.Unmarshal(cx.fill_bts, v)
}

// an XML decoder for request body
type XmlRequestDecoder struct{}

func (d *XmlRequestDecoder) Unmarshal(cx *Context, v interface{}, autofill bool) (err error) {
	// read body
	if cx.fill_bts == nil {
		cx.fill_bts, err = ioutil.ReadAll(cx.request.Body)
	}
	if err != nil {
		return err
	}
	if err = xml.Unmarshal(cx.fill_bts, v); err != nil {
		logrus.Errorf("[Fill Error]Request:%s,Err:%s\n", string(cx.fill_bts), err.Error())
	}
	return err
}

// a form-enc decoder for request body
type FormRequestDecoder struct{}

type FileGetter func(string) (multipart.File, *multipart.FileHeader, error)

func (d *FormRequestDecoder) Unmarshal(cx *Context, v interface{}, autofill bool) error {
	if cx.request.Form == nil {
		cx.request.ParseForm()
	}

	var maxlength = 0
	for k, _ := range cx.request.Form {
		if len(k) > maxlength {
			maxlength = len(k)
		}
	}

	var unmarshaller = unmarshall.Unmarshaller{
		Values: func() map[string][]string {
			return cx.request.Form
		},
		ValuesGetter: func(prefix string) url.Values {
			values := (*map[string][]string)(&cx.request.Form)
			var sub = make(url.Values)
			if values != nil {
				for k, v := range *values {
					if strings.HasPrefix(k, prefix+"[") {
						sub[k] = v
					}
				}
			}
			return sub
		},
		ValueGetter: func(tag string) []string {
			values := (*map[string][]string)(&cx.request.Form)
			if values != nil {
				if results, ok := (*values)[tag]; !ok {
					//get the value of [Tag] from [tag](lower case), it maybe a problem TODO
					return (*values)[strings.ToLower(tag)]
				} else {
					return results
				}
			}
			return []string{}
		},
		MaxLength:    maxlength,
		TagConcatter: concatPrefix,
		AutoFill:     autofill,
	}

	for typ, fn := range cx.Server.filler {
		if unmarshaller.FillForSpecifiledType == nil {
			unmarshaller.FillForSpecifiledType = make(map[string]func(id string) (reflect.Value, error))
		}
		unmarshaller.FillForSpecifiledType[typ] = func(content string) (reflect.Value, error) {
			obj, err := fn(content)
			return reflect.ValueOf(obj), err
		}
	}

	return unmarshaller.Unmarshall(v)
}

// a form-enc decoder for request body
type MultiFormRequestDecoder struct{}

func (d *MultiFormRequestDecoder) Unmarshal(cx *Context, v interface{}, autofill bool) error {
	cx.request.ParseMultipartForm(32 << 20)
	values := (map[string][]string)(cx.request.Form)
	if cx.request.MultipartForm == nil {
		return errors.New("MultipartForm is empty")
	}
	var maxlength = 0

	for k, v := range cx.request.MultipartForm.Value {
		values[k] = v
		if len(k) > maxlength {
			maxlength = len(k)
		}
	}

	for k, _ := range cx.request.MultipartForm.File {
		if len(k) > maxlength {
			maxlength = len(k)
		}
	}

	var unmarshaller = unmarshall.Unmarshaller{
		Values: func() map[string][]string {
			return values
		},
		MaxLength: maxlength,
		ValueGetter: func(tag string) []string {
			return values[tag]
		},
		ValuesGetter: func(prefix string) url.Values {
			var sub = make(url.Values)
			if values != nil {
				for k, v := range values {
					if strings.HasPrefix(k, prefix+"[") {
						sub[k] = v
					}
				}
			}
			return sub
		},
		TagConcatter: concatPrefix,
		FillForSpecifiledType: map[string]func(id string) (reflect.Value, error){
			"github.com/extrame/goblet.File": func(id string) (reflect.Value, error) {
				var file File
				var err error
				var f multipart.File
				var h *multipart.FileHeader
				if f, h, err = cx.request.FormFile(id); err == nil {
					file.Name = h.Filename
					file.Header = h.Header
					file.rc = f
					return reflect.ValueOf(file), err
				} else {
					return reflect.ValueOf(nil), err
				}
			},
		},
		AutoFill: autofill,
	}

	for typ, fn := range cx.Server.multiFiller {
		unmarshaller.FillForSpecifiledType[typ] = func(id string) (reflect.Value, error) {
			obj, err := fn(cx, id)
			return reflect.ValueOf(obj), err
		}
	}

	return unmarshaller.Unmarshall(v)
}

// map of Content-Type -> RequestDecoders
var decoders map[string]RequestDecoder = map[string]RequestDecoder{
	"application/json":                  new(JsonRequestDecoder),
	"application/xml":                   new(XmlRequestDecoder),
	"text/xml":                          new(XmlRequestDecoder),
	"application/x-www-form-urlencoded": new(FormRequestDecoder),
	"text/plain":                        new(FormRequestDecoder),
	"multipart/form-data":               new(MultiFormRequestDecoder),
}

// goweb.Context Helper function to fill a variable with the contents
// of the request body. The body will be decoded based
// on the content-type and an apropriate RequestDecoder
// automatically selected
// If you want to use md5 function for the specified field, please add
// md5 tag for it. AND the md5 tag must be the last one of the tags, so
// if you have no other tag, please add ',' before md5
func (cx *Context) Fill(v interface{}, fills ...bool) error {
	// get content type
	ct := cx.request.Header.Get("Content-Type")
	// default to urlencoded
	if ct == "" {
		if cx.Server.Basic.DefaultType != "" {
			ct = cx.Server.Basic.DefaultType
		} else {
			ct = "application/x-www-form-urlencoded"
		}
	} else if strings.HasPrefix(ct, "text/plain") && cx.Server.Basic.DefaultType != "" {
		ct = cx.Server.Basic.DefaultType
	}
	autofill := true
	if len(fills) > 0 {
		autofill = fills[0]
	}
	return cx.FillAs(v, autofill, ct)
}

func (cx *Context) FillAs(v interface{}, autofill bool, ct string) error {
	// ignore charset (after ';')
	ct = strings.Split(ct, ";")[0]
	// get request decoder
	decoder, ok := decoders[ct]
	if ok != true {
		return fmt.Errorf("Cannot decode request for %s data", ct)
	}
	// decode
	err := decoder.Unmarshal(cx, v, autofill)
	if err != nil {
		fmt.Println(err)
		return err
	}
	// all clear
	return nil
}

func concatPrefix(prefix, tag string) string {
	return prefix + "[" + tag + "]"
}
<commit_msg>add error handle for multipart form filling<commit_after>package goblet

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/url"
	"reflect"
	"strings"

	"github.com/extrame/unmarshall"
	"github.com/sirupsen/logrus"
)

type FormFillFn func(content string) (interface{}, error)
type MultiFormFillFn func(ctx *Context, id string) (interface{}, error)

func (s *Server) AddFillForTypeInForm(typ string, fn FormFillFn) {
	s.filler[typ] = fn
}

func (s *Server) AddFillForTypeInMultiForm(typ string, fn MultiFormFillFn) {
	s.multiFiller[typ] = fn
}

// types that impliment RequestDecoder can unmarshal
// the request body into an apropriate type/struct
type RequestDecoder interface {
	Unmarshal(cx *Context, v interface{}, autofill bool) error
}

// a JSON decoder for request body (just a wrapper to json.Unmarshal)
type JsonRequestDecoder struct{}

func (d *JsonRequestDecoder) Unmarshal(cx *Context, v interface{}, autofill bool) (err error) {
	// read body
	if cx.fill_bts == nil {
		cx.fill_bts, err = ioutil.ReadAll(cx.request.Body)
	}
	if err != nil {
		return err
	}
	return json.Unmarshal(cx.fill_bts, v)
}

// an XML decoder for request body
type XmlRequestDecoder struct{}

func (d *XmlRequestDecoder) Unmarshal(cx *Context, v interface{}, autofill bool) (err error) {
	// read body
	if cx.fill_bts == nil {
		cx.fill_bts, err = ioutil.ReadAll(cx.request.Body)
	}
	if err != nil {
		return err
	}
	if err = xml.Unmarshal(cx.fill_bts, v); err != nil {
		logrus.Errorf("[Fill Error]Request:%s,Err:%s\n", string(cx.fill_bts), err.Error())
	}
	return err
}

// a form-enc decoder for request body
type FormRequestDecoder struct{}

type FileGetter func(string) (multipart.File, *multipart.FileHeader, error)

func (d *FormRequestDecoder) Unmarshal(cx *Context, v interface{}, autofill bool) error {
	if cx.request.Form == nil {
		cx.request.ParseForm()
	}

	var maxlength = 0
	for k, _ := range cx.request.Form {
		if len(k) > maxlength {
			maxlength = len(k)
		}
	}

	var unmarshaller = unmarshall.Unmarshaller{
		Values: func() map[string][]string {
			return cx.request.Form
		},
		ValuesGetter: func(prefix string) url.Values {
			values := (*map[string][]string)(&cx.request.Form)
			var sub = make(url.Values)
			if values != nil {
				for k, v := range *values {
					if strings.HasPrefix(k, prefix+"[") {
						sub[k] = v
					}
				}
			}
			return sub
		},
		ValueGetter: func(tag string) []string {
			values := (*map[string][]string)(&cx.request.Form)
			if values != nil {
				if results, ok := (*values)[tag]; !ok {
					//get the value of [Tag] from [tag](lower case), it maybe a problem TODO
					return (*values)[strings.ToLower(tag)]
				} else {
					return results
				}
			}
			return []string{}
		},
		MaxLength:    maxlength,
		TagConcatter: concatPrefix,
		AutoFill:     autofill,
	}

	for typ, fn := range cx.Server.filler {
		if unmarshaller.FillForSpecifiledType == nil {
			unmarshaller.FillForSpecifiledType = make(map[string]func(id string) (reflect.Value, error))
		}
		unmarshaller.FillForSpecifiledType[typ] = func(content string) (reflect.Value, error) {
			obj, err := fn(content)
			return reflect.ValueOf(obj), err
		}
	}

	return unmarshaller.Unmarshall(v)
}

// a form-enc decoder for request body
type MultiFormRequestDecoder struct{}

func (d *MultiFormRequestDecoder) Unmarshal(cx *Context, v interface{}, autofill bool) error {
	err := cx.request.ParseMultipartForm(32 << 20)
	if err != nil {
		return err
	}
	values := (map[string][]string)(cx.request.Form)
	if cx.request.MultipartForm == nil {
		return errors.New("MultipartForm is empty")
	}
	var maxlength = 0

	for k, v := range cx.request.MultipartForm.Value {
		values[k] = v
		if len(k) > maxlength {
			maxlength = len(k)
		}
	}

	for k, _ := range cx.request.MultipartForm.File {
		if len(k) > maxlength {
			maxlength = len(k)
		}
	}

	var unmarshaller = unmarshall.Unmarshaller{
		Values: func() map[string][]string {
			return values
		},
		MaxLength: maxlength,
		ValueGetter: func(tag string) []string {
			return values[tag]
		},
		ValuesGetter: func(prefix string) url.Values {
			var sub = make(url.Values)
			if values != nil {
				for k, v := range values {
					if strings.HasPrefix(k, prefix+"[") {
						sub[k] = v
					}
				}
			}
			return sub
		},
		TagConcatter: concatPrefix,
		FillForSpecifiledType: map[string]func(id string) (reflect.Value, error){
			"github.com/extrame/goblet.File": func(id string) (reflect.Value, error) {
				var file File
				var err error
				var f multipart.File
				var h *multipart.FileHeader
				if f, h, err = cx.request.FormFile(id); err == nil {
					file.Name = h.Filename
					file.Header = h.Header
					file.rc = f
					return reflect.ValueOf(file), err
				} else {
					return reflect.ValueOf(nil), err
				}
			},
		},
		AutoFill: autofill,
	}

	for typ, fn := range cx.Server.multiFiller {
		unmarshaller.FillForSpecifiledType[typ] = func(id string) (reflect.Value, error) {
			obj, err := fn(cx, id)
			return reflect.ValueOf(obj), err
		}
	}

	return unmarshaller.Unmarshall(v)
}

// map of Content-Type -> RequestDecoders
var decoders map[string]RequestDecoder = map[string]RequestDecoder{
	"application/json":                  new(JsonRequestDecoder),
	"application/xml":                   new(XmlRequestDecoder),
	"text/xml":                          new(XmlRequestDecoder),
	"application/x-www-form-urlencoded": new(FormRequestDecoder),
	"text/plain":                        new(FormRequestDecoder),
	"multipart/form-data":               new(MultiFormRequestDecoder),
}

// goweb.Context Helper function to fill a variable with the contents
// of the request body. The body will be decoded based
// on the content-type and an apropriate RequestDecoder
// automatically selected
// If you want to use md5 function for the specified field, please add
// md5 tag for it. AND the md5 tag must be the last one of the tags, so
// if you have no other tag, please add ',' before md5
func (cx *Context) Fill(v interface{}, fills ...bool) error {
	// get content type
	ct := cx.request.Header.Get("Content-Type")
	// default to urlencoded
	if ct == "" {
		if cx.Server.Basic.DefaultType != "" {
			ct = cx.Server.Basic.DefaultType
		} else {
			ct = "application/x-www-form-urlencoded"
		}
	} else if strings.HasPrefix(ct, "text/plain") && cx.Server.Basic.DefaultType != "" {
		ct = cx.Server.Basic.DefaultType
	}
	autofill := true
	if len(fills) > 0 {
		autofill = fills[0]
	}
	return cx.FillAs(v, autofill, ct)
}

func (cx *Context) FillAs(v interface{}, autofill bool, ct string) error {
	// ignore charset (after ';')
	ct = strings.Split(ct, ";")[0]
	// get request decoder
	decoder, ok := decoders[ct]
	if ok != true {
		return fmt.Errorf("Cannot decode request for %s data", ct)
	}
	// decode
	err := decoder.Unmarshal(cx, v, autofill)
	if err != nil {
		fmt.Println(err)
		return err
	}
	// all clear
	return nil
}

func concatPrefix(prefix, tag string) string {
	return prefix + "[" + tag + "]"
}
<|endoftext|>


<commit_before>package bongo

import (
	"github.com/maxwellhealth/mgo"
	"math"
)

type ResultSet struct {
	Query      *mgo.Query
	Iter       *mgo.Iter
	loadedIter bool
	Collection *Collection
	Error      error
}

type PaginationInfo struct {
	Current       int `json:"current"`
	TotalPages    int `json:"totalPages"`
	PerPage       int `json:"perPage"`
	TotalRecords  int `json:"totalRecords"`
	RecordsOnPage int `json:"recordsOnPage"`
}

func (r *ResultSet) Next(mod interface{}) bool {

	// Check if the iter has been instantiated yet
	if !r.loadedIter {
		r.Iter = r.Query.Iter()
		r.loadedIter = true
	}

	gotResult := r.Iter.Next(mod)

	if gotResult {

		if hook, ok := mod.(interface {
			AfterFind(*Collection)
		}); ok {
			hook.AfterFind(r.Collection)
		}

		if newt, ok := mod.(NewTracker); ok {
			newt.SetIsNew(false)
		}
		return true
	}

	err := r.Iter.Err()
	if err != nil {
		r.Error = err
	}

	return false
}

func (r *ResultSet) Free() error {
	if r.loadedIter {
		if err := r.Iter.Close(); err != nil {
			return err
		}
	}

	return nil
}

// Set skip + limit on the current query and generates a PaginationInfo struct with info for your front end
func (r *ResultSet) Paginate(perPage, page int) (*PaginationInfo, error) {

	info := new(PaginationInfo)

	// Get count of current query
	count, err := r.Query.Count()

	if err != nil {
		return info, err
	}

	// Calculate how many pages
	totalPages := int(math.Ceil(float64(count) / float64(perPage)))

	if page < 1 {
		page = 1
	} else if page > totalPages {
		page = totalPages
	}

	skip := (page - 1) * perPage

	r.Query.Skip(skip).Limit(perPage)

	info.TotalPages = totalPages
	info.PerPage = perPage
	info.Current = page
	info.TotalRecords = count

	if info.Current < info.TotalPages {
		info.RecordsOnPage = info.PerPage
	} else {

		info.RecordsOnPage = int(math.Mod(float64(count), float64(perPage)))

		if info.RecordsOnPage == 0 && count > 0 {
			info.RecordsOnPage = perPage
		}

	}

	return info, nil
}
<commit_msg>Started concurrency fix on paginate<commit_after>package bongo

import (
	"github.com/maxwellhealth/mgo"
	"log"
	"math"
	"time"
)

type ResultSet struct {
	Query      *mgo.Query
	Iter       *mgo.Iter
	loadedIter bool
	Collection *Collection
	Error      error
}

type PaginationInfo struct {
	Current       int `json:"current"`
	TotalPages    int `json:"totalPages"`
	PerPage       int `json:"perPage"`
	TotalRecords  int `json:"totalRecords"`
	RecordsOnPage int `json:"recordsOnPage"`
}

func (r *ResultSet) Next(mod interface{}) bool {

	// Check if the iter has been instantiated yet
	if !r.loadedIter {
		r.Iter = r.Query.Iter()
		r.loadedIter = true
	}

	gotResult := r.Iter.Next(mod)

	if gotResult {

		if hook, ok := mod.(interface {
			AfterFind(*Collection)
		}); ok {
			hook.AfterFind(r.Collection)
		}

		if newt, ok := mod.(NewTracker); ok {
			newt.SetIsNew(false)
		}
		return true
	}

	err := r.Iter.Err()
	if err != nil {
		r.Error = err
	}

	return false
}

func (r *ResultSet) Free() error {
	if r.loadedIter {
		if err := r.Iter.Close(); err != nil {
			return err
		}
	}

	return nil
}

// Set skip + limit on the current query and generates a PaginationInfo struct with info for your front end
func (r *ResultSet) Paginate(perPage, page int) (*PaginationInfo, error) {

	start := time.Now()
	elapsed := time.Since(start)
	log.Printf("Time at 74: %s\n", elapsed)
	info := new(PaginationInfo)

	// Get count of current query
	// count, err := r.Query.Count()

	sess := r.Collection.Connection.Session.Clone()
	defer sess.Close()

	count, err := sess.DB(r.Collection.Connection.Config.Database).C(r.Collection.Name).Count()
	// count, err := r.Collection.Collection().Count()

	if err != nil {
		return info, err
	}

	elapsed = time.Since(start)
	log.Printf("Time at 85: %s\n", elapsed)

	// Calculate how many pages
	totalPages := int(math.Ceil(float64(count) / float64(perPage)))

	if page < 1 {
		page = 1
	} else if page > totalPages {
		page = totalPages
	}

	elapsed = time.Since(start)
	log.Printf("Time at 96: %s\n", elapsed)

	skip := (page - 1) * perPage

	r.Query.Skip(skip).Limit(perPage)

	info.TotalPages = totalPages
	info.PerPage = perPage
	info.Current = page
	info.TotalRecords = count

	if info.Current < info.TotalPages {
		info.RecordsOnPage = info.PerPage
	} else {

		info.RecordsOnPage = int(math.Mod(float64(count), float64(perPage)))

		if info.RecordsOnPage == 0 && count > 0 {
			info.RecordsOnPage = perPage
		}

	}
	elapsed = time.Since(start)
	log.Printf("Time at 120: %s\n", elapsed)

	return info, nil
}
<|endoftext|>