<commit_before>// Copyright 2013 com authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package com

import (
	"errors"
	"fmt"
	"os"
	"path"
	"strings"
)

// IsDir returns true if given path is a directory,
// or returns false when it's a file or does not exist.
func IsDir(dir string) bool {
	f, e := os.Stat(dir)
	if e != nil {
		return false
	}
	return f.IsDir()
}

func statDir(dirPath, recPath string, includeDir, isDirOnly bool) ([]string, error) {
	dir, err := os.Open(dirPath)
	if err != nil {
		return nil, err
	}
	defer dir.Close()

	fis, err := dir.Readdir(0)
	if err != nil {
		return nil, err
	}

	statList := make([]string, 0)
	for _, fi := range fis {
		if strings.Contains(fi.Name(), ".DS_Store") {
			continue
		}

		relPath := path.Join(recPath, fi.Name())
		curPath := path.Join(dirPath, fi.Name())
		if fi.IsDir() {
			if includeDir {
				statList = append(statList, relPath+"/")
			}
			s, err := statDir(curPath, relPath, includeDir, isDirOnly)
			if err != nil {
				return nil, err
			}
			statList = append(statList, s...)
		} else if !isDirOnly {
			statList = append(statList, relPath)
		}
	}
	return statList, nil
}

// StatDir gathers information of given directory by depth-first.
// It returns slice of file list and includes subdirectories if enabled;
// it returns error and nil slice when error occurs in underlying functions,
// or given path is not a directory or does not exist.
//
// Slice does not include given path itself.
// If subdirectories is enabled, they will have suffix '/'.
func StatDir(rootPath string, includeDir ...bool) ([]string, error) {
	if !IsDir(rootPath) {
		return nil, errors.New("not a directory or does not exist: " + rootPath)
	}

	isIncludeDir := false
	if len(includeDir) >= 1 {
		isIncludeDir = includeDir[0]
	}
	return statDir(rootPath, "", isIncludeDir, false)
}

// GetAllSubDirs returns all subdirectories of given root path.
// Slice does not include given path itself.
func GetAllSubDirs(rootPath string) ([]string, error) {
	if !IsDir(rootPath) {
		return nil, errors.New("not a directory or does not exist: " + rootPath)
	}
	return statDir(rootPath, "", true, true)
}

// GetFileListBySuffix returns an ordered list of file paths.
// It recognize if given path is a file, and don't do recursive find.
func GetFileListBySuffix(dirPath, suffix string) ([]string, error) {
	if !IsExist(dirPath) {
		return nil, fmt.Errorf("given path does not exist: %s", dirPath)
	} else if IsFile(dirPath) {
		return []string{dirPath}, nil
	}

	// Given path is a directory.
	dir, err := os.Open(dirPath)
	if err != nil {
		return nil, err
	}

	fis, err := dir.Readdir(0)
	if err != nil {
		return nil, err
	}

	files := make([]string, 0, len(fis))
	for _, fi := range fis {
		if strings.HasSuffix(fi.Name(), suffix) {
			files = append(files, path.Join(dirPath, fi.Name()))
		}
	}

	return files, nil
}

// CopyDir copy files recursively from source to target directory.
//
// The filter accepts a function that process the path info.
// and should return true for need to filter.
//
// It returns error when error occurs in underlying functions.
func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) error {
	// Check if target directory exists.
	if IsExist(destPath) {
		return errors.New("file or directory alreay exists: " + destPath)
	}

	err := os.MkdirAll(destPath, os.ModePerm)
	if err != nil {
		return err
	}

	// Gather directory info.
	infos, err := StatDir(srcPath, true)
	if err != nil {
		return err
	}

	var filter func(filePath string) bool
	if len(filters) > 0 {
		filter = filters[0]
	}

	for _, info := range infos {
		if filter != nil && filter(info) {
			continue
		}

		curPath := path.Join(destPath, info)
		if strings.HasSuffix(info, "/") {
			err = os.MkdirAll(curPath, os.ModePerm)
		} else {
			err = Copy(path.Join(srcPath, info), curPath)
		}
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>support for following symbolic links in GetAllSubDirs and StatDir (#18)<commit_after>// Copyright 2013 com authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package com

import (
	"errors"
	"fmt"
	"os"
	"path"
	"strings"
)

// IsDir returns true if given path is a directory,
// or returns false when it's a file or does not exist.
func IsDir(dir string) bool {
	f, e := os.Stat(dir)
	if e != nil {
		return false
	}
	return f.IsDir()
}

func statDir(dirPath, recPath string, includeDir, isDirOnly, followSymlinks bool) ([]string, error) {
	dir, err := os.Open(dirPath)
	if err != nil {
		return nil, err
	}
	defer dir.Close()

	fis, err := dir.Readdir(0)
	if err != nil {
		return nil, err
	}

	statList := make([]string, 0)
	for _, fi := range fis {
		if strings.Contains(fi.Name(), ".DS_Store") {
			continue
		}

		relPath := path.Join(recPath, fi.Name())
		curPath := path.Join(dirPath, fi.Name())
		if fi.IsDir() {
			if includeDir {
				statList = append(statList, relPath+"/")
			}
			s, err := statDir(curPath, relPath, includeDir, isDirOnly, followSymlinks)
			if err != nil {
				return nil, err
			}
			statList = append(statList, s...)
		} else if !isDirOnly {
			statList = append(statList, relPath)
		} else if followSymlinks && fi.Mode()&os.ModeSymlink != 0 {
			link, err := os.Readlink(curPath)
			if err != nil {
				return nil, err
			}

			if IsDir(link) {
				if includeDir {
					statList = append(statList, relPath+"/")
				}
				s, err := statDir(curPath, relPath, includeDir, isDirOnly, followSymlinks)
				if err != nil {
					return nil, err
				}
				statList = append(statList, s...)
			}
		}
	}
	return statList, nil
}

// StatDir gathers information of given directory by depth-first.
// It returns slice of file list and includes subdirectories if enabled;
// it returns error and nil slice when error occurs in underlying functions,
// or given path is not a directory or does not exist.
//
// Slice does not include given path itself.
// If subdirectories is enabled, they will have suffix '/'.
func StatDir(rootPath string, includeDir ...bool) ([]string, error) {
	if !IsDir(rootPath) {
		return nil, errors.New("not a directory or does not exist: " + rootPath)
	}

	isIncludeDir := false
	if len(includeDir) >= 1 {
		isIncludeDir = includeDir[0]
	}
	return statDir(rootPath, "", isIncludeDir, false, false)
}

// LstatDir gathers information of given directory by depth-first.
// It returns slice of file list, follows symbolic links and includes subdirectories if enabled;
// it returns error and nil slice when error occurs in underlying functions,
// or given path is not a directory or does not exist.
//
// Slice does not include given path itself.
// If subdirectories is enabled, they will have suffix '/'.
func LstatDir(rootPath string, includeDir ...bool) ([]string, error) {
	if !IsDir(rootPath) {
		return nil, errors.New("not a directory or does not exist: " + rootPath)
	}

	isIncludeDir := false
	if len(includeDir) >= 1 {
		isIncludeDir = includeDir[0]
	}
	return statDir(rootPath, "", isIncludeDir, false, true)
}

// GetAllSubDirs returns all subdirectories of given root path.
// Slice does not include given path itself.
func GetAllSubDirs(rootPath string) ([]string, error) {
	if !IsDir(rootPath) {
		return nil, errors.New("not a directory or does not exist: " + rootPath)
	}
	return statDir(rootPath, "", true, true, false)
}

// LgetAllSubDirs returns all subdirectories of given root path, including
// following symbolic links, if any.
// Slice does not include given path itself.
func LgetAllSubDirs(rootPath string) ([]string, error) {
	if !IsDir(rootPath) {
		return nil, errors.New("not a directory or does not exist: " + rootPath)
	}
	return statDir(rootPath, "", true, true, true)
}

// GetFileListBySuffix returns an ordered list of file paths.
// It recognize if given path is a file, and don't do recursive find.
func GetFileListBySuffix(dirPath, suffix string) ([]string, error) {
	if !IsExist(dirPath) {
		return nil, fmt.Errorf("given path does not exist: %s", dirPath)
	} else if IsFile(dirPath) {
		return []string{dirPath}, nil
	}

	// Given path is a directory.
	dir, err := os.Open(dirPath)
	if err != nil {
		return nil, err
	}

	fis, err := dir.Readdir(0)
	if err != nil {
		return nil, err
	}

	files := make([]string, 0, len(fis))
	for _, fi := range fis {
		if strings.HasSuffix(fi.Name(), suffix) {
			files = append(files, path.Join(dirPath, fi.Name()))
		}
	}

	return files, nil
}

// CopyDir copy files recursively from source to target directory.
//
// The filter accepts a function that process the path info.
// and should return true for need to filter.
//
// It returns error when error occurs in underlying functions.
func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) error {
	// Check if target directory exists.
	if IsExist(destPath) {
		return errors.New("file or directory alreay exists: " + destPath)
	}

	err := os.MkdirAll(destPath, os.ModePerm)
	if err != nil {
		return err
	}

	// Gather directory info.
	infos, err := StatDir(srcPath, true)
	if err != nil {
		return err
	}

	var filter func(filePath string) bool
	if len(filters) > 0 {
		filter = filters[0]
	}

	for _, info := range infos {
		if filter != nil && filter(info) {
			continue
		}

		curPath := path.Join(destPath, info)
		if strings.HasSuffix(info, "/") {
			err = os.MkdirAll(curPath, os.ModePerm)
		} else {
			err = Copy(path.Join(srcPath, info), curPath)
		}
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"net/http/pprof"

	"github.com/animenotifier/arn"
)

func init() {
	if !arn.IsDevelopment() {
		return
	}

	app.Router.HandlerFunc("GET", "/debug/pprof/", http.HandlerFunc(pprof.Index))
	app.Router.HandlerFunc("GET", "/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	app.Router.HandlerFunc("GET", "/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	app.Router.HandlerFunc("GET", "/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	app.Router.HandlerFunc("GET", "/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	app.Router.Handler("GET", "/debug/pprof/goroutine", pprof.Handler("goroutine"))
	app.Router.Handler("GET", "/debug/pprof/heap", pprof.Handler("heap"))
	app.Router.Handler("GET", "/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
	app.Router.Handler("GET", "/debug/pprof/block", pprof.Handler("block"))
}
<commit_msg>Activate profiling on all versions<commit_after>package main

import (
	"net/http"
	"net/http/pprof"
)

func init() {
	// if !arn.IsDevelopment() {
	// 	return
	// }

	app.Router.HandlerFunc("GET", "/debug/pprof/", http.HandlerFunc(pprof.Index))
	app.Router.HandlerFunc("GET", "/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	app.Router.HandlerFunc("GET", "/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	app.Router.HandlerFunc("GET", "/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	app.Router.HandlerFunc("GET", "/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	app.Router.Handler("GET", "/debug/pprof/goroutine", pprof.Handler("goroutine"))
	app.Router.Handler("GET", "/debug/pprof/heap", pprof.Handler("heap"))
	app.Router.Handler("GET", "/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
	app.Router.Handler("GET", "/debug/pprof/block", pprof.Handler("block"))
	app.Router.Handler("GET", "/debug/pprof/allocs", pprof.Handler("allocs"))
	app.Router.Handler("GET", "/debug/pprof/mutex", pprof.Handler("mutex"))
}
<|endoftext|>


<commit_before>package main

import(
    "github.com/spf13/cobra"
    "fmt"
    "./iwscanner"
)

func main() {

    var cmdScan = &cobra.Command{
        Use:   "scan",
        Short: "Scan current location for APs",
        Long:  `Turns on WiFi and scans for AP around.`,
        Run: func(cmd *cobra.Command, args []string) {
                fmt.Println("Turning wifi on")

                if e := iwscanner.TurnWifi("wlan0", "on"); e != nil {
                        panic(e)
                }

                fmt.Println("Scanning")
                aps, err := iwscanner.GetAPs("wlan0")
                if err != nil {
                        panic(err);
                } else {
                        fmt.Println(aps)
                }
        },
    }

    var cmdProfile = &cobra.Command{
        Use:   "profile [name of the profile]",
        Short: "Manage profile",
        Long:  `Create, update, delete or differently manage the given profile.`,
        Run: func(cmd *cobra.Command, args []string) {
                fmt.Println("Managing")
        },
    }
    var rootCmd = &cobra.Command{Use: "proflock"}
    rootCmd.AddCommand(cmdScan, cmdProfile)
    rootCmd.Execute()
}

<commit_msg>turn-wifi command<commit_after>package main

import(
    "github.com/spf13/cobra"
    "fmt"
    "./iwscanner"
)

func main() {

    var cmdScan = &cobra.Command{
        Use:   "scan",
        Short: "Scan current location for APs",
        Long:  `Turns on WiFi and scans for AP around.`,
        Run: func(cmd *cobra.Command, args []string) {
                fmt.Println("Turning wifi on")

                if e := iwscanner.TurnWifi("wlan0", "on"); e != nil {
                        panic(e)
                }

                fmt.Println("Scanning")
                aps, err := iwscanner.GetAPs("wlan0")
                if err != nil {
                        panic(err);
                } else {
                        fmt.Println(aps)
                }
        },
    }

    var cmdProfile = &cobra.Command{
        Use:   "profile [name of the profile]",
        Short: "Manage profile",
        Long:  `Create, update, delete or differently manage the given profile.`,
        Run: func(cmd *cobra.Command, args []string) {
                fmt.Println("Managing")
        },
    }

    var cmdTurnWifi = &cobra.Command{
        Use:   "turn-wifi [on|off]",
        Short: "Turns Wifi On or Off",
        Long:  `Turns Wifi On or Off.`,
        Run: func(cmd *cobra.Command, args []string) {
                if len(args) == 0 {
                        iwscanner.TurnWifi("wlan0", "on")
                } else {
                        iwscanner.TurnWifi("wlan0", args[0])
                }
        },
    }
    var rootCmd = &cobra.Command{Use: "proflock"}
    rootCmd.AddCommand(cmdScan, cmdProfile, cmdTurnWifi)
    rootCmd.Execute()
}

<|endoftext|>


<commit_before>package mpb

import (
	"bytes"
	"container/heap"
	"context"
	"fmt"
	"io"
	"math"
	"os"
	"sync"
	"time"

	"github.com/vbauerster/mpb/v8/cwriter"
)

const (
	defaultRefreshRate = 150 * time.Millisecond
)

// DoneError represents an error when `*mpb.Progress` is done but its functionality is requested.
var DoneError = fmt.Errorf("%T instance can't be reused after it's done!", (*Progress)(nil))

// Progress represents a container that renders one or more progress bars.
type Progress struct {
	ctx          context.Context
	uwg          *sync.WaitGroup
	bwg          *sync.WaitGroup
	operateState chan func(*pState)
	interceptIo  chan func(io.Writer)
	done         chan struct{}
	shutdown     chan struct{}
	cancel       func()
}

// pState holds bars in its priorityQueue, it gets passed to (*Progress).serve monitor goroutine.
type pState struct {
	bHeap       priorityQueue
	heapUpdated bool
	pMatrix     map[int][]chan int
	aMatrix     map[int][]chan int

	// for reuse purposes
	rows []io.Reader
	pool []*Bar

	// following are provided/overrided by user
	refreshRate        time.Duration
	idCount            int
	reqWidth           int
	popPriority        int
	popCompleted       bool
	outputDiscarded    bool
	disableAutoRefresh bool
	manualRefresh      chan interface{}
	renderDelay        <-chan struct{}
	shutdownNotifier   chan struct{}
	queueBars          map[*Bar]*Bar
	output             io.Writer
	debugOut           io.Writer
	uwg                *sync.WaitGroup
}

// New creates new Progress container instance. It's not possible to
// reuse instance after (*Progress).Wait method has been called.
func New(options ...ContainerOption) *Progress {
	return NewWithContext(context.Background(), options...)
}

// NewWithContext creates new Progress container instance with provided
// context. It's not possible to reuse instance after (*Progress).Wait
// method has been called.
func NewWithContext(ctx context.Context, options ...ContainerOption) *Progress {
	s := &pState{
		rows:          make([]io.Reader, 0, 64),
		pool:          make([]*Bar, 0, 64),
		refreshRate:   defaultRefreshRate,
		popPriority:   math.MinInt32,
		manualRefresh: make(chan interface{}),
		queueBars:     make(map[*Bar]*Bar),
		output:        os.Stdout,
		debugOut:      io.Discard,
	}

	for _, opt := range options {
		if opt != nil {
			opt(s)
		}
	}

	ctx, cancel := context.WithCancel(ctx)
	p := &Progress{
		ctx:          ctx,
		uwg:          s.uwg,
		bwg:          new(sync.WaitGroup),
		operateState: make(chan func(*pState)),
		interceptIo:  make(chan func(io.Writer)),
		done:         make(chan struct{}),
		cancel:       cancel,
	}

	if s.shutdownNotifier != nil {
		p.shutdown = s.shutdownNotifier
		s.shutdownNotifier = nil
	} else {
		p.shutdown = make(chan struct{})
	}

	go p.serve(s, cwriter.New(s.output))
	return p
}

// AddBar creates a bar with default bar filler.
func (p *Progress) AddBar(total int64, options ...BarOption) *Bar {
	return p.New(total, BarStyle(), options...)
}

// AddSpinner creates a bar with default spinner filler.
func (p *Progress) AddSpinner(total int64, options ...BarOption) *Bar {
	return p.New(total, SpinnerStyle(), options...)
}

// New creates a bar by calling `Build` method on provided `BarFillerBuilder`.
func (p *Progress) New(total int64, builder BarFillerBuilder, options ...BarOption) *Bar {
	return p.AddFiller(total, builder.Build(), options...)
}

// AddFiller creates a bar which renders itself by provided filler.
// If `total <= 0` triggering complete event by increment methods is disabled.
// Panics if *Progress instance is done, i.e. called after (*Progress).Wait().
func (p *Progress) AddFiller(total int64, filler BarFiller, options ...BarOption) *Bar {
	if filler == nil {
		filler = NopStyle().Build()
	}
	p.bwg.Add(1)
	result := make(chan *Bar)
	select {
	case p.operateState <- func(ps *pState) {
		bs := ps.makeBarState(total, filler, options...)
		bar := newBar(p, bs)
		if bs.wait.bar != nil {
			ps.queueBars[bs.wait.bar] = bar
		} else {
			heap.Push(&ps.bHeap, bar)
			ps.heapUpdated = true
		}
		ps.idCount++
		result <- bar
	}:
		bar := <-result
		return bar
	case <-p.done:
		p.bwg.Done()
		panic(DoneError)
	}
}

func (p *Progress) traverseBars(cb func(b *Bar) bool) {
	sync := make(chan struct{})
	select {
	case p.operateState <- func(s *pState) {
		for i := 0; i < s.bHeap.Len(); i++ {
			bar := s.bHeap[i]
			if !cb(bar) {
				break
			}
		}
		close(sync)
	}:
		<-sync
	case <-p.done:
	}
}

// UpdateBarPriority same as *Bar.SetPriority(int).
func (p *Progress) UpdateBarPriority(b *Bar, priority int) {
	select {
	case p.operateState <- func(s *pState) {
		if b.index < 0 {
			return
		}
		b.priority = priority
		heap.Fix(&s.bHeap, b.index)
	}:
	case <-p.done:
	}
}

// BarCount returns bars count.
func (p *Progress) BarCount() int {
	result := make(chan int)
	select {
	case p.operateState <- func(s *pState) { result <- s.bHeap.Len() }:
		return <-result
	case <-p.done:
		return 0
	}
}

// Write is implementation of io.Writer.
// Writing to `*mpb.Progress` will print lines above a running bar.
// Writes aren't flushed immediately, but at next refresh cycle.
// If Write is called after `*mpb.Progress` is done, `mpb.DoneError`
// is returned.
func (p *Progress) Write(b []byte) (int, error) {
	type result struct {
		n   int
		err error
	}
	ch := make(chan *result)
	select {
	case p.interceptIo <- func(w io.Writer) {
		n, err := w.Write(b)
		ch <- &result{n, err}
	}:
		res := <-ch
		return res.n, res.err
	case <-p.done:
		return 0, DoneError
	}
}

// Wait waits for all bars to complete and finally shutdowns container. After
// this method has been called, there is no way to reuse (*Progress) instance.
func (p *Progress) Wait() {
	// wait for user wg, if any
	if p.uwg != nil {
		p.uwg.Wait()
	}

	p.bwg.Wait()
	p.Shutdown()
}

// Shutdown cancels any running bar immediately and then shutdowns (*Progress)
// instance. Normally this method shouldn't be called unless you know what you
// are doing. Proper way to shutdown is to call (*Progress).Wait() instead.
func (p *Progress) Shutdown() {
	p.cancel()
	<-p.shutdown
}

func (p *Progress) newTicker(s *pState) chan time.Time {
	ch := make(chan time.Time)
	go func() {
		var autoRefresh <-chan time.Time
		if !s.disableAutoRefresh && !s.outputDiscarded {
			if s.renderDelay != nil {
				<-s.renderDelay
			}
			ticker := time.NewTicker(s.refreshRate)
			defer ticker.Stop()
			autoRefresh = ticker.C
		}
		for {
			select {
			case t := <-autoRefresh:
				ch <- t
			case x := <-s.manualRefresh:
				if t, ok := x.(time.Time); ok {
					ch <- t
				} else {
					ch <- time.Now()
				}
			case <-p.ctx.Done():
				close(p.done)
				return
			}
		}
	}()
	return ch
}

func (p *Progress) serve(s *pState, cw *cwriter.Writer) {
	defer close(p.shutdown)

	render := func() error {
		if s.bHeap.Len() == 0 {
			return nil
		}
		return s.render(cw)
	}

	refreshCh := p.newTicker(s)

	for {
		select {
		case op := <-p.operateState:
			op(s)
		case fn := <-p.interceptIo:
			fn(cw)
		case <-refreshCh:
			err := render()
			if err != nil {
				s.heapUpdated = false
				render = func() error { return nil }
				_, _ = fmt.Fprintln(s.debugOut, err)
				p.cancel() // cancel all bars
			}
		case <-p.done:
			for s.heapUpdated {
				err := render()
				if err != nil {
					_, _ = fmt.Fprintln(s.debugOut, err)
					return
				}
			}
			return
		}
	}
}

func (s *pState) render(cw *cwriter.Writer) error {
	var wg sync.WaitGroup
	if s.heapUpdated {
		s.updateSyncMatrix()
		s.heapUpdated = false
	}
	syncWidth(&wg, s.pMatrix)
	syncWidth(&wg, s.aMatrix)

	width, height, err := cw.GetTermSize()
	if err != nil {
		width = s.reqWidth
		height = s.bHeap.Len()
	}
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := s.bHeap[i]
		go bar.render(width)
	}

	err = s.flush(&wg, cw, height)
	wg.Wait()
	return err
}

func (s *pState) flush(wg *sync.WaitGroup, cw *cwriter.Writer, height int) error {
	var popCount int

	for s.bHeap.Len() > 0 {
		b := heap.Pop(&s.bHeap).(*Bar)
		frame := <-b.frameCh
		if frame.err != nil {
			s.rows = s.rows[:0]
			return frame.err
		}
		var usedRows int
		for i := len(frame.rows) - 1; i >= 0; i-- {
			if row := frame.rows[i]; len(s.rows) < height {
				s.rows = append(s.rows, row)
				usedRows++
			} else {
				wg.Add(1)
				go func() {
					_, _ = io.Copy(io.Discard, row)
					wg.Done()
				}()
			}
		}
		if frame.shutdown {
			b.Wait() // waiting for b.done, so it's safe to read b.bs
			if qb, ok := s.queueBars[b]; ok {
				delete(s.queueBars, b)
				qb.priority = b.priority
				s.pool = append(s.pool, qb)
				s.heapUpdated = true
				continue
			}
			if s.popCompleted && !b.bs.noPop {
				switch b.bs.shutdown++; b.bs.shutdown {
				case 1:
					b.priority = s.popPriority
					s.popPriority++
				default:
					if b.bs.dropOnComplete {
						popCount += usedRows
						s.heapUpdated = true
						continue
					}
				}
			} else if b.bs.dropOnComplete {
				s.heapUpdated = true
				continue
			}
		}
		s.pool = append(s.pool, b)
	}

	switch len(s.pool) {
	case 0:
		if s.heapUpdated {
			s.updateSyncMatrix()
			s.heapUpdated = false
		}
	case 1:
		heap.Push(&s.bHeap, s.pool[0])
		s.pool = s.pool[:0]
	default:
		wg.Add(1)
		go func() {
			for _, b := range s.pool {
				heap.Push(&s.bHeap, b)
			}
			s.pool = s.pool[:0]
			wg.Done()
		}()
	}

	for i := len(s.rows) - 1; i >= 0; i-- {
		_, err := cw.ReadFrom(s.rows[i])
		if err != nil {
			s.rows = s.rows[:0]
			return err
		}
	}

	err := cw.Flush(len(s.rows) - popCount)
	s.rows = s.rows[:0]
	return err
}

func (s *pState) updateSyncMatrix() {
	s.pMatrix = make(map[int][]chan int)
	s.aMatrix = make(map[int][]chan int)
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := s.bHeap[i]
		table := bar.wSyncTable()
		pRow, aRow := table[0], table[1]

		for i, ch := range pRow {
			s.pMatrix[i] = append(s.pMatrix[i], ch)
		}

		for i, ch := range aRow {
			s.aMatrix[i] = append(s.aMatrix[i], ch)
		}
	}
}

func (s *pState) makeBarState(total int64, filler BarFiller, options ...BarOption) *bState {
	bs := &bState{
		id:            s.idCount,
		priority:      s.idCount,
		reqWidth:      s.reqWidth,
		total:         total,
		filler:        filler,
		manualRefresh: s.manualRefresh,
	}

	if total > 0 {
		bs.triggerComplete = true
	}

	for _, opt := range options {
		if opt != nil {
			opt(bs)
		}
	}

	if bs.middleware != nil {
		bs.filler = bs.middleware(filler)
		bs.middleware = nil
	}

	for i := 0; i < len(bs.buffers); i++ {
		bs.buffers[i] = bytes.NewBuffer(make([]byte, 0, 512))
	}

	bs.subscribeDecorators()

	return bs
}

func syncWidth(wg *sync.WaitGroup, matrix map[int][]chan int) {
	wg.Add(len(matrix))
	for _, column := range matrix {
		go maxWidthDistributor(wg, column)
	}
}

func maxWidthDistributor(wg *sync.WaitGroup, column []chan int) {
	var maxWidth int
	for _, ch := range column {
		if w := <-ch; w > maxWidth {
			maxWidth = w
		}
	}
	for _, ch := range column {
		ch <- maxWidth
	}
	wg.Done()
}
<commit_msg>minor: wg.Add(1)<commit_after>package mpb

import (
	"bytes"
	"container/heap"
	"context"
	"fmt"
	"io"
	"math"
	"os"
	"sync"
	"time"

	"github.com/vbauerster/mpb/v8/cwriter"
)

const (
	defaultRefreshRate = 150 * time.Millisecond
)

// DoneError represents an error when `*mpb.Progress` is done but its functionality is requested.
var DoneError = fmt.Errorf("%T instance can't be reused after it's done!", (*Progress)(nil))

// Progress represents a container that renders one or more progress bars.
type Progress struct {
	ctx          context.Context
	uwg          *sync.WaitGroup
	bwg          *sync.WaitGroup
	operateState chan func(*pState)
	interceptIo  chan func(io.Writer)
	done         chan struct{}
	shutdown     chan struct{}
	cancel       func()
}

// pState holds bars in its priorityQueue, it gets passed to (*Progress).serve monitor goroutine.
type pState struct {
	bHeap       priorityQueue
	heapUpdated bool
	pMatrix     map[int][]chan int
	aMatrix     map[int][]chan int

	// for reuse purposes
	rows []io.Reader
	pool []*Bar

	// following are provided/overrided by user
	refreshRate        time.Duration
	idCount            int
	reqWidth           int
	popPriority        int
	popCompleted       bool
	outputDiscarded    bool
	disableAutoRefresh bool
	manualRefresh      chan interface{}
	renderDelay        <-chan struct{}
	shutdownNotifier   chan struct{}
	queueBars          map[*Bar]*Bar
	output             io.Writer
	debugOut           io.Writer
	uwg                *sync.WaitGroup
}

// New creates new Progress container instance. It's not possible to
// reuse instance after (*Progress).Wait method has been called.
func New(options ...ContainerOption) *Progress {
	return NewWithContext(context.Background(), options...)
}

// NewWithContext creates new Progress container instance with provided
// context. It's not possible to reuse instance after (*Progress).Wait
// method has been called.
func NewWithContext(ctx context.Context, options ...ContainerOption) *Progress {
	s := &pState{
		rows:          make([]io.Reader, 0, 64),
		pool:          make([]*Bar, 0, 64),
		refreshRate:   defaultRefreshRate,
		popPriority:   math.MinInt32,
		manualRefresh: make(chan interface{}),
		queueBars:     make(map[*Bar]*Bar),
		output:        os.Stdout,
		debugOut:      io.Discard,
	}

	for _, opt := range options {
		if opt != nil {
			opt(s)
		}
	}

	ctx, cancel := context.WithCancel(ctx)
	p := &Progress{
		ctx:          ctx,
		uwg:          s.uwg,
		bwg:          new(sync.WaitGroup),
		operateState: make(chan func(*pState)),
		interceptIo:  make(chan func(io.Writer)),
		done:         make(chan struct{}),
		cancel:       cancel,
	}

	if s.shutdownNotifier != nil {
		p.shutdown = s.shutdownNotifier
		s.shutdownNotifier = nil
	} else {
		p.shutdown = make(chan struct{})
	}

	go p.serve(s, cwriter.New(s.output))
	return p
}

// AddBar creates a bar with default bar filler.
func (p *Progress) AddBar(total int64, options ...BarOption) *Bar {
	return p.New(total, BarStyle(), options...)
}

// AddSpinner creates a bar with default spinner filler.
func (p *Progress) AddSpinner(total int64, options ...BarOption) *Bar {
	return p.New(total, SpinnerStyle(), options...)
}

// New creates a bar by calling `Build` method on provided `BarFillerBuilder`.
func (p *Progress) New(total int64, builder BarFillerBuilder, options ...BarOption) *Bar {
	return p.AddFiller(total, builder.Build(), options...)
}

// AddFiller creates a bar which renders itself by provided filler.
// If `total <= 0` triggering complete event by increment methods is disabled.
// Panics if *Progress instance is done, i.e. called after (*Progress).Wait().
func (p *Progress) AddFiller(total int64, filler BarFiller, options ...BarOption) *Bar {
	if filler == nil {
		filler = NopStyle().Build()
	}
	p.bwg.Add(1)
	result := make(chan *Bar)
	select {
	case p.operateState <- func(ps *pState) {
		bs := ps.makeBarState(total, filler, options...)
		bar := newBar(p, bs)
		if bs.wait.bar != nil {
			ps.queueBars[bs.wait.bar] = bar
		} else {
			heap.Push(&ps.bHeap, bar)
			ps.heapUpdated = true
		}
		ps.idCount++
		result <- bar
	}:
		bar := <-result
		return bar
	case <-p.done:
		p.bwg.Done()
		panic(DoneError)
	}
}

func (p *Progress) traverseBars(cb func(b *Bar) bool) {
	sync := make(chan struct{})
	select {
	case p.operateState <- func(s *pState) {
		for i := 0; i < s.bHeap.Len(); i++ {
			bar := s.bHeap[i]
			if !cb(bar) {
				break
			}
		}
		close(sync)
	}:
		<-sync
	case <-p.done:
	}
}

// UpdateBarPriority same as *Bar.SetPriority(int).
func (p *Progress) UpdateBarPriority(b *Bar, priority int) {
	select {
	case p.operateState <- func(s *pState) {
		if b.index < 0 {
			return
		}
		b.priority = priority
		heap.Fix(&s.bHeap, b.index)
	}:
	case <-p.done:
	}
}

// BarCount returns bars count.
func (p *Progress) BarCount() int {
	result := make(chan int)
	select {
	case p.operateState <- func(s *pState) { result <- s.bHeap.Len() }:
		return <-result
	case <-p.done:
		return 0
	}
}

// Write is implementation of io.Writer.
// Writing to `*mpb.Progress` will print lines above a running bar.
// Writes aren't flushed immediately, but at next refresh cycle.
// If Write is called after `*mpb.Progress` is done, `mpb.DoneError`
// is returned.
func (p *Progress) Write(b []byte) (int, error) {
	type result struct {
		n   int
		err error
	}
	ch := make(chan *result)
	select {
	case p.interceptIo <- func(w io.Writer) {
		n, err := w.Write(b)
		ch <- &result{n, err}
	}:
		res := <-ch
		return res.n, res.err
	case <-p.done:
		return 0, DoneError
	}
}

// Wait waits for all bars to complete and finally shutdowns container. After
// this method has been called, there is no way to reuse (*Progress) instance.
func (p *Progress) Wait() {
	// wait for user wg, if any
	if p.uwg != nil {
		p.uwg.Wait()
	}

	p.bwg.Wait()
	p.Shutdown()
}

// Shutdown cancels any running bar immediately and then shutdowns (*Progress)
// instance. Normally this method shouldn't be called unless you know what you
// are doing. Proper way to shutdown is to call (*Progress).Wait() instead.
func (p *Progress) Shutdown() {
	p.cancel()
	<-p.shutdown
}

func (p *Progress) newTicker(s *pState) chan time.Time {
	ch := make(chan time.Time)
	go func() {
		var autoRefresh <-chan time.Time
		if !s.disableAutoRefresh && !s.outputDiscarded {
			if s.renderDelay != nil {
				<-s.renderDelay
			}
			ticker := time.NewTicker(s.refreshRate)
			defer ticker.Stop()
			autoRefresh = ticker.C
		}
		for {
			select {
			case t := <-autoRefresh:
				ch <- t
			case x := <-s.manualRefresh:
				if t, ok := x.(time.Time); ok {
					ch <- t
				} else {
					ch <- time.Now()
				}
			case <-p.ctx.Done():
				close(p.done)
				return
			}
		}
	}()
	return ch
}

func (p *Progress) serve(s *pState, cw *cwriter.Writer) {
	defer close(p.shutdown)

	render := func() error {
		if s.bHeap.Len() == 0 {
			return nil
		}
		return s.render(cw)
	}

	refreshCh := p.newTicker(s)

	for {
		select {
		case op := <-p.operateState:
			op(s)
		case fn := <-p.interceptIo:
			fn(cw)
		case <-refreshCh:
			err := render()
			if err != nil {
				s.heapUpdated = false
				render = func() error { return nil }
				_, _ = fmt.Fprintln(s.debugOut, err)
				p.cancel() // cancel all bars
			}
		case <-p.done:
			for s.heapUpdated {
				err := render()
				if err != nil {
					_, _ = fmt.Fprintln(s.debugOut, err)
					return
				}
			}
			return
		}
	}
}

func (s *pState) render(cw *cwriter.Writer) error {
	var wg sync.WaitGroup
	if s.heapUpdated {
		s.updateSyncMatrix()
		s.heapUpdated = false
	}
	syncWidth(&wg, s.pMatrix)
	syncWidth(&wg, s.aMatrix)

	width, height, err := cw.GetTermSize()
	if err != nil {
		width = s.reqWidth
		height = s.bHeap.Len()
	}
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := s.bHeap[i]
		go bar.render(width)
	}

	err = s.flush(&wg, cw, height)
	wg.Wait()
	return err
}

func (s *pState) flush(wg *sync.WaitGroup, cw *cwriter.Writer, height int) error {
	var popCount int

	for s.bHeap.Len() > 0 {
		b := heap.Pop(&s.bHeap).(*Bar)
		frame := <-b.frameCh
		if frame.err != nil {
			s.rows = s.rows[:0]
			return frame.err
		}
		var usedRows int
		for i := len(frame.rows) - 1; i >= 0; i-- {
			if row := frame.rows[i]; len(s.rows) < height {
				s.rows = append(s.rows, row)
				usedRows++
			} else {
				wg.Add(1)
				go func() {
					_, _ = io.Copy(io.Discard, row)
					wg.Done()
				}()
			}
		}
		if frame.shutdown {
			b.Wait() // waiting for b.done, so it's safe to read b.bs
			if qb, ok := s.queueBars[b]; ok {
				delete(s.queueBars, b)
				qb.priority = b.priority
				s.pool = append(s.pool, qb)
				s.heapUpdated = true
				continue
			}
			if s.popCompleted && !b.bs.noPop {
				switch b.bs.shutdown++; b.bs.shutdown {
				case 1:
					b.priority = s.popPriority
					s.popPriority++
				default:
					if b.bs.dropOnComplete {
						popCount += usedRows
						s.heapUpdated = true
						continue
					}
				}
			} else if b.bs.dropOnComplete {
				s.heapUpdated = true
				continue
			}
		}
		s.pool = append(s.pool, b)
	}

	switch len(s.pool) {
	case 0:
		if s.heapUpdated {
			s.updateSyncMatrix()
			s.heapUpdated = false
		}
	case 1:
		heap.Push(&s.bHeap, s.pool[0])
		s.pool = s.pool[:0]
	default:
		wg.Add(1)
		go func() {
			for _, b := range s.pool {
				heap.Push(&s.bHeap, b)
			}
			s.pool = s.pool[:0]
			wg.Done()
		}()
	}

	for i := len(s.rows) - 1; i >= 0; i-- {
		_, err := cw.ReadFrom(s.rows[i])
		if err != nil {
			s.rows = s.rows[:0]
			return err
		}
	}

	err := cw.Flush(len(s.rows) - popCount)
	s.rows = s.rows[:0]
	return err
}

func (s *pState) updateSyncMatrix() {
	s.pMatrix = make(map[int][]chan int)
	s.aMatrix = make(map[int][]chan int)
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := s.bHeap[i]
		table := bar.wSyncTable()
		pRow, aRow := table[0], table[1]

		for i, ch := range pRow {
			s.pMatrix[i] = append(s.pMatrix[i], ch)
		}

		for i, ch := range aRow {
			s.aMatrix[i] = append(s.aMatrix[i], ch)
		}
	}
}

func (s *pState) makeBarState(total int64, filler BarFiller, options ...BarOption) *bState {
	bs := &bState{
		id:            s.idCount,
		priority:      s.idCount,
		reqWidth:      s.reqWidth,
		total:         total,
		filler:        filler,
		manualRefresh: s.manualRefresh,
	}

	if total > 0 {
		bs.triggerComplete = true
	}

	for _, opt := range options {
		if opt != nil {
			opt(bs)
		}
	}

	if bs.middleware != nil {
		bs.filler = bs.middleware(filler)
		bs.middleware = nil
	}

	for i := 0; i < len(bs.buffers); i++ {
		bs.buffers[i] = bytes.NewBuffer(make([]byte, 0, 512))
	}

	bs.subscribeDecorators()

	return bs
}

func syncWidth(wg *sync.WaitGroup, matrix map[int][]chan int) {
	for _, column := range matrix {
		wg.Add(1)
		go maxWidthDistributor(wg, column)
	}
}

func maxWidthDistributor(wg *sync.WaitGroup, column []chan int) {
	var maxWidth int
	for _, ch := range column {
		if w := <-ch; w > maxWidth {
			maxWidth = w
		}
	}
	for _, ch := range column {
		ch <- maxWidth
	}
	wg.Done()
}
<|endoftext|>


<commit_before>package kcp

import (
	"encoding/binary"
	"sync/atomic"

	"github.com/klauspost/reedsolomon"
)

const (
	fecHeaderSize      = 6
	fecHeaderSizePlus2 = fecHeaderSize + 2 // plus 2B data size
	typeData           = 0xf1
	typeParity         = 0xf2
	fecExpire          = 60000
	rxFECMulti         = 3 // FEC keeps rxFECMulti* (dataShard+parityShard) ordered packets in memory
)

// fecPacket is a decoded FEC packet
type fecPacket []byte

func (bts fecPacket) seqid() uint32 { return binary.LittleEndian.Uint32(bts) }
func (bts fecPacket) flag() uint16  { return binary.LittleEndian.Uint16(bts[4:]) }
func (bts fecPacket) data() []byte  { return bts[6:] }

// fecElement has auxcilliary time field
type fecElement struct {
	fecPacket
	ts uint32
}

// fecDecoder for decoding incoming packets
type fecDecoder struct {
	rxlimit      int // queue size limit
	dataShards   int
	parityShards int
	shardSize    int
	rx           []fecElement // ordered receive queue

	// caches
	decodeCache [][]byte
	flagCache   []bool

	// zeros
	zeros []byte

	// RS decoder
	codec reedsolomon.Encoder

	// auto tune fec parameter
	autoTune autoTune
}

func newFECDecoder(dataShards, parityShards int) *fecDecoder {
	if dataShards <= 0 || parityShards <= 0 {
		return nil
	}

	dec := new(fecDecoder)
	dec.dataShards = dataShards
	dec.parityShards = parityShards
	dec.shardSize = dataShards + parityShards
	dec.rxlimit = rxFECMulti * dec.shardSize
	codec, err := reedsolomon.New(dataShards, parityShards)
	if err != nil {
		return nil
	}
	dec.codec = codec
	dec.decodeCache = make([][]byte, dec.shardSize)
	dec.flagCache = make([]bool, dec.shardSize)
	dec.zeros = make([]byte, mtuLimit)
	return dec
}

// decode a fec packet
func (dec *fecDecoder) decode(in fecPacket) (recovered [][]byte) {
	// sample to auto FEC tuner
	if in.flag() == typeData {
		dec.autoTune.Sample(true, in.seqid())
	} else {
		dec.autoTune.Sample(false, in.seqid())
	}

	// check if FEC parameters is out of sync
	var shouldTune bool
	if int(in.seqid())%dec.shardSize < dec.dataShards {
		if in.flag() != typeData { // expect typeData
			shouldTune = true
		}
	} else {
		if in.flag() != typeParity {
			shouldTune = true
		}
	}

	if shouldTune {
		autoDS := dec.autoTune.FindPeriod(true)
		autoPS := dec.autoTune.FindPeriod(false)

		// edges found, we can tune parameters now
		if autoDS > 0 && autoPS > 0 && autoDS < 256 && autoPS < 256 {
			dec.dataShards = autoDS
			dec.parityShards = autoPS
			dec.shardSize = autoDS + autoPS
			dec.rxlimit = rxFECMulti * dec.shardSize
			codec, err := reedsolomon.New(autoDS, autoPS)
			if err != nil {
				return nil
			}
			dec.codec = codec
			dec.decodeCache = make([][]byte, dec.shardSize)
			dec.flagCache = make([]bool, dec.shardSize)
			//log.Println("autotune to :", dec.dataShards, dec.parityShards)
		}
	}

	// insertion
	n := len(dec.rx) - 1
	insertIdx := 0
	for i := n; i >= 0; i-- {
		if in.seqid() == dec.rx[i].seqid() { // de-duplicate
			return nil
		} else if _itimediff(in.seqid(), dec.rx[i].seqid()) > 0 { // insertion
			insertIdx = i + 1
			break
		}
	}

	// make a copy
	pkt := fecPacket(xmitBuf.Get().([]byte)[:len(in)])
	copy(pkt, in)
	elem := fecElement{pkt, currentMs()}

	// insert into ordered rx queue
	if insertIdx == n+1 {
		dec.rx = append(dec.rx, elem)
	} else {
		dec.rx = append(dec.rx, fecElement{})
		copy(dec.rx[insertIdx+1:], dec.rx[insertIdx:]) // shift right
		dec.rx[insertIdx] = elem
	}

	// shard range for current packet
	shardBegin := pkt.seqid() - pkt.seqid()%uint32(dec.shardSize)
	shardEnd := shardBegin + uint32(dec.shardSize) - 1

	// max search range in ordered queue for current shard
	searchBegin := insertIdx - int(pkt.seqid()%uint32(dec.shardSize))
	if searchBegin < 0 {
		searchBegin = 0
	}
	searchEnd := searchBegin + dec.shardSize - 1
	if searchEnd >= len(dec.rx) {
		searchEnd = len(dec.rx) - 1
	}

	// re-construct datashards
	if searchEnd-searchBegin+1 >= dec.dataShards {
		var numshard, numDataShard, first, maxlen int

		// zero caches
		shards := dec.decodeCache
		shardsflag := dec.flagCache
		for k := range dec.decodeCache {
			shards[k] = nil
			shardsflag[k] = false
		}

		// shard assembly
		for i := searchBegin; i <= searchEnd; i++ {
			seqid := dec.rx[i].seqid()
			if _itimediff(seqid, shardEnd) > 0 {
				break
			} else if _itimediff(seqid, shardBegin) >= 0 {
				shards[seqid%uint32(dec.shardSize)] = dec.rx[i].data()
				shardsflag[seqid%uint32(dec.shardSize)] = true
				numshard++
				if dec.rx[i].flag() == typeData {
					numDataShard++
				}
				if numshard == 1 {
					first = i
				}
				if len(dec.rx[i].data()) > maxlen {
					maxlen = len(dec.rx[i].data())
				}
			}
		}

		if numDataShard == dec.dataShards {
			// case 1: no loss on data shards
			dec.rx = dec.freeRange(first, numshard, dec.rx)
		} else if numshard >= dec.dataShards {
			// case 2: loss on data shards, but it's recoverable from parity shards
			for k := range shards {
				if shards[k] != nil {
					dlen := len(shards[k])
					shards[k] = shards[k][:maxlen]
					copy(shards[k][dlen:], dec.zeros)
				} else if k < dec.dataShards {
					shards[k] = xmitBuf.Get().([]byte)[:0]
				}
			}
			if err := dec.codec.ReconstructData(shards); err == nil {
				for k := range shards[:dec.dataShards] {
					if !shardsflag[k] {
						// recovered data should be recycled
						recovered = append(recovered, shards[k])
					}
				}
			}
			dec.rx = dec.freeRange(first, numshard, dec.rx)
		}
	}

	// keep rxlimit
	if len(dec.rx) > dec.rxlimit {
		if dec.rx[0].flag() == typeData { // track the unrecoverable data
			atomic.AddUint64(&DefaultSnmp.FECShortShards, 1)
		}
		dec.rx = dec.freeRange(0, 1, dec.rx)
	}

	// timeout policy
	current := currentMs()
	numExpired := 0
	for k := range dec.rx {
		if _itimediff(current, dec.rx[k].ts) > fecExpire {
			numExpired++
			continue
		}
		break
	}
	if numExpired > 0 {
		dec.rx = dec.freeRange(0, numExpired, dec.rx)
	}
	return
}

// free a range of fecPacket
func (dec *fecDecoder) freeRange(first, n int, q []fecElement) []fecElement {
	for i := first; i < first+n; i++ { // recycle buffer
		xmitBuf.Put([]byte(q[i].fecPacket))
	}

	if first == 0 && n < cap(q)/2 {
		return q[n:]
	}
	copy(q[first:], q[first+n:])
	return q[:len(q)-n]
}

// release all segments back to xmitBuf
func (dec *fecDecoder) release() {
	if n := len(dec.rx); n > 0 {
		dec.rx = dec.freeRange(0, n, dec.rx)
	}
}

type (
	// fecEncoder for encoding outgoing packets
	fecEncoder struct {
		dataShards   int
		parityShards int
		shardSize    int
		paws         uint32 // Protect Against Wrapped Sequence numbers
		next         uint32 // next seqid

		shardCount int // count the number of datashards collected
		maxSize    int // track maximum data length in datashard

		headerOffset  int // FEC header offset
		payloadOffset int // FEC payload offset

		// caches
		shardCache  [][]byte
		encodeCache [][]byte

		// zeros
		zeros []byte

		// RS encoder
		codec reedsolomon.Encoder
	}
)

func newFECEncoder(dataShards, parityShards, offset int) *fecEncoder {
	if dataShards <= 0 || parityShards <= 0 {
		return nil
	}
	enc := new(fecEncoder)
	enc.dataShards = dataShards
	enc.parityShards = parityShards
	enc.shardSize = dataShards + parityShards
	enc.paws = 0xffffffff / uint32(enc.shardSize) * uint32(enc.shardSize)
	enc.headerOffset = offset
	enc.payloadOffset = enc.headerOffset + fecHeaderSize

	codec, err := reedsolomon.New(dataShards, parityShards)
	if err != nil {
		return nil
	}
	enc.codec = codec

	// caches
	enc.encodeCache = make([][]byte, enc.shardSize)
	enc.shardCache = make([][]byte, enc.shardSize)
	for k := range enc.shardCache {
		enc.shardCache[k] = make([]byte, mtuLimit)
	}
	enc.zeros = make([]byte, mtuLimit)
	return enc
}

// encodes the packet, outputs parity shards if we have collected quorum datashards
// notice: the contents of 'ps' will be re-written in successive calling
func (enc *fecEncoder) encode(b []byte) (ps [][]byte) {
	// The header format:
	// | FEC SEQID(4B) | FEC TYPE(2B) | SIZE (2B) | PAYLOAD(SIZE-2) |
	// |<-headerOffset                |<-payloadOffset
	enc.markData(b[enc.headerOffset:])
	binary.LittleEndian.PutUint16(b[enc.payloadOffset:], uint16(len(b[enc.payloadOffset:])))

	// copy data from payloadOffset to fec shard cache
	sz := len(b)
	enc.shardCache[enc.shardCount] = enc.shardCache[enc.shardCount][:sz]
	copy(enc.shardCache[enc.shardCount][enc.payloadOffset:], b[enc.payloadOffset:])
	enc.shardCount++

	// track max datashard length
	if sz > enc.maxSize {
		enc.maxSize = sz
	}

	//  Generation of Reed-Solomon Erasure Code
	if enc.shardCount == enc.dataShards {
		// fill '0' into the tail of each datashard
		for i := 0; i < enc.dataShards; i++ {
			shard := enc.shardCache[i]
			slen := len(shard)
			copy(shard[slen:enc.maxSize], enc.zeros)
		}

		// construct equal-sized slice with stripped header
		cache := enc.encodeCache
		for k := range cache {
			cache[k] = enc.shardCache[k][enc.payloadOffset:enc.maxSize]
		}

		// encoding
		if err := enc.codec.Encode(cache); err == nil {
			ps = enc.shardCache[enc.dataShards:]
			for k := range ps {
				enc.markParity(ps[k][enc.headerOffset:])
				ps[k] = ps[k][:enc.maxSize]
			}
		}

		// counters resetting
		enc.shardCount = 0
		enc.maxSize = 0
	}

	return
}

func (enc *fecEncoder) markData(data []byte) {
	binary.LittleEndian.PutUint32(data, enc.next)
	binary.LittleEndian.PutUint16(data[4:], typeData)
	enc.next++
}

func (enc *fecEncoder) markParity(data []byte) {
	binary.LittleEndian.PutUint32(data, enc.next)
	binary.LittleEndian.PutUint16(data[4:], typeParity)
	// sequence wrap will only happen at parity shard
	enc.next = (enc.next + 1) % enc.paws
}
<commit_msg>add extra check in auto tune<commit_after>package kcp

import (
	"encoding/binary"
	"sync/atomic"

	"github.com/klauspost/reedsolomon"
)

const (
	fecHeaderSize      = 6
	fecHeaderSizePlus2 = fecHeaderSize + 2 // plus 2B data size
	typeData           = 0xf1
	typeParity         = 0xf2
	fecExpire          = 60000
	rxFECMulti         = 3 // FEC keeps rxFECMulti* (dataShard+parityShard) ordered packets in memory
)

// fecPacket is a decoded FEC packet
type fecPacket []byte

func (bts fecPacket) seqid() uint32 { return binary.LittleEndian.Uint32(bts) }
func (bts fecPacket) flag() uint16  { return binary.LittleEndian.Uint16(bts[4:]) }
func (bts fecPacket) data() []byte  { return bts[6:] }

// fecElement has auxcilliary time field
type fecElement struct {
	fecPacket
	ts uint32
}

// fecDecoder for decoding incoming packets
type fecDecoder struct {
	rxlimit      int // queue size limit
	dataShards   int
	parityShards int
	shardSize    int
	rx           []fecElement // ordered receive queue

	// caches
	decodeCache [][]byte
	flagCache   []bool

	// zeros
	zeros []byte

	// RS decoder
	codec reedsolomon.Encoder

	// auto tune fec parameter
	autoTune autoTune
}

func newFECDecoder(dataShards, parityShards int) *fecDecoder {
	if dataShards <= 0 || parityShards <= 0 {
		return nil
	}

	dec := new(fecDecoder)
	dec.dataShards = dataShards
	dec.parityShards = parityShards
	dec.shardSize = dataShards + parityShards
	dec.rxlimit = rxFECMulti * dec.shardSize
	codec, err := reedsolomon.New(dataShards, parityShards)
	if err != nil {
		return nil
	}
	dec.codec = codec
	dec.decodeCache = make([][]byte, dec.shardSize)
	dec.flagCache = make([]bool, dec.shardSize)
	dec.zeros = make([]byte, mtuLimit)
	return dec
}

// decode a fec packet
func (dec *fecDecoder) decode(in fecPacket) (recovered [][]byte) {
	// sample to auto FEC tuner
	if in.flag() == typeData {
		dec.autoTune.Sample(true, in.seqid())
	} else {
		dec.autoTune.Sample(false, in.seqid())
	}

	// check if FEC parameters is out of sync
	var shouldTune bool
	if int(in.seqid())%dec.shardSize < dec.dataShards {
		if in.flag() != typeData { // expect typeData
			shouldTune = true
		}
	} else {
		if in.flag() != typeParity {
			shouldTune = true
		}
	}

	if shouldTune {
		autoDS := dec.autoTune.FindPeriod(true)
		autoPS := dec.autoTune.FindPeriod(false)

		// edges found, we can tune parameters now
		if autoDS > 0 && autoPS > 0 && autoDS < 256 && autoPS < 256 {
			// and make sure it's different
			if autoDS != dec.dataShards || autoPS != dec.parityShards {
				dec.dataShards = autoDS
				dec.parityShards = autoPS
				dec.shardSize = autoDS + autoPS
				dec.rxlimit = rxFECMulti * dec.shardSize
				codec, err := reedsolomon.New(autoDS, autoPS)
				if err != nil {
					return nil
				}
				dec.codec = codec
				dec.decodeCache = make([][]byte, dec.shardSize)
				dec.flagCache = make([]bool, dec.shardSize)
				//log.Println("autotune to :", dec.dataShards, dec.parityShards)
			}
		}
	}

	// insertion
	n := len(dec.rx) - 1
	insertIdx := 0
	for i := n; i >= 0; i-- {
		if in.seqid() == dec.rx[i].seqid() { // de-duplicate
			return nil
		} else if _itimediff(in.seqid(), dec.rx[i].seqid()) > 0 { // insertion
			insertIdx = i + 1
			break
		}
	}

	// make a copy
	pkt := fecPacket(xmitBuf.Get().([]byte)[:len(in)])
	copy(pkt, in)
	elem := fecElement{pkt, currentMs()}

	// insert into ordered rx queue
	if insertIdx == n+1 {
		dec.rx = append(dec.rx, elem)
	} else {
		dec.rx = append(dec.rx, fecElement{})
		copy(dec.rx[insertIdx+1:], dec.rx[insertIdx:]) // shift right
		dec.rx[insertIdx] = elem
	}

	// shard range for current packet
	shardBegin := pkt.seqid() - pkt.seqid()%uint32(dec.shardSize)
	shardEnd := shardBegin + uint32(dec.shardSize) - 1

	// max search range in ordered queue for current shard
	searchBegin := insertIdx - int(pkt.seqid()%uint32(dec.shardSize))
	if searchBegin < 0 {
		searchBegin = 0
	}
	searchEnd := searchBegin + dec.shardSize - 1
	if searchEnd >= len(dec.rx) {
		searchEnd = len(dec.rx) - 1
	}

	// re-construct datashards
	if searchEnd-searchBegin+1 >= dec.dataShards {
		var numshard, numDataShard, first, maxlen int

		// zero caches
		shards := dec.decodeCache
		shardsflag := dec.flagCache
		for k := range dec.decodeCache {
			shards[k] = nil
			shardsflag[k] = false
		}

		// shard assembly
		for i := searchBegin; i <= searchEnd; i++ {
			seqid := dec.rx[i].seqid()
			if _itimediff(seqid, shardEnd) > 0 {
				break
			} else if _itimediff(seqid, shardBegin) >= 0 {
				shards[seqid%uint32(dec.shardSize)] = dec.rx[i].data()
				shardsflag[seqid%uint32(dec.shardSize)] = true
				numshard++
				if dec.rx[i].flag() == typeData {
					numDataShard++
				}
				if numshard == 1 {
					first = i
				}
				if len(dec.rx[i].data()) > maxlen {
					maxlen = len(dec.rx[i].data())
				}
			}
		}

		if numDataShard == dec.dataShards {
			// case 1: no loss on data shards
			dec.rx = dec.freeRange(first, numshard, dec.rx)
		} else if numshard >= dec.dataShards {
			// case 2: loss on data shards, but it's recoverable from parity shards
			for k := range shards {
				if shards[k] != nil {
					dlen := len(shards[k])
					shards[k] = shards[k][:maxlen]
					copy(shards[k][dlen:], dec.zeros)
				} else if k < dec.dataShards {
					shards[k] = xmitBuf.Get().([]byte)[:0]
				}
			}
			if err := dec.codec.ReconstructData(shards); err == nil {
				for k := range shards[:dec.dataShards] {
					if !shardsflag[k] {
						// recovered data should be recycled
						recovered = append(recovered, shards[k])
					}
				}
			}
			dec.rx = dec.freeRange(first, numshard, dec.rx)
		}
	}

	// keep rxlimit
	if len(dec.rx) > dec.rxlimit {
		if dec.rx[0].flag() == typeData { // track the unrecoverable data
			atomic.AddUint64(&DefaultSnmp.FECShortShards, 1)
		}
		dec.rx = dec.freeRange(0, 1, dec.rx)
	}

	// timeout policy
	current := currentMs()
	numExpired := 0
	for k := range dec.rx {
		if _itimediff(current, dec.rx[k].ts) > fecExpire {
			numExpired++
			continue
		}
		break
	}
	if numExpired > 0 {
		dec.rx = dec.freeRange(0, numExpired, dec.rx)
	}
	return
}

// free a range of fecPacket
func (dec *fecDecoder) freeRange(first, n int, q []fecElement) []fecElement {
	for i := first; i < first+n; i++ { // recycle buffer
		xmitBuf.Put([]byte(q[i].fecPacket))
	}

	if first == 0 && n < cap(q)/2 {
		return q[n:]
	}
	copy(q[first:], q[first+n:])
	return q[:len(q)-n]
}

// release all segments back to xmitBuf
func (dec *fecDecoder) release() {
	if n := len(dec.rx); n > 0 {
		dec.rx = dec.freeRange(0, n, dec.rx)
	}
}

type (
	// fecEncoder for encoding outgoing packets
	fecEncoder struct {
		dataShards   int
		parityShards int
		shardSize    int
		paws         uint32 // Protect Against Wrapped Sequence numbers
		next         uint32 // next seqid

		shardCount int // count the number of datashards collected
		maxSize    int // track maximum data length in datashard

		headerOffset  int // FEC header offset
		payloadOffset int // FEC payload offset

		// caches
		shardCache  [][]byte
		encodeCache [][]byte

		// zeros
		zeros []byte

		// RS encoder
		codec reedsolomon.Encoder
	}
)

func newFECEncoder(dataShards, parityShards, offset int) *fecEncoder {
	if dataShards <= 0 || parityShards <= 0 {
		return nil
	}
	enc := new(fecEncoder)
	enc.dataShards = dataShards
	enc.parityShards = parityShards
	enc.shardSize = dataShards + parityShards
	enc.paws = 0xffffffff / uint32(enc.shardSize) * uint32(enc.shardSize)
	enc.headerOffset = offset
	enc.payloadOffset = enc.headerOffset + fecHeaderSize

	codec, err := reedsolomon.New(dataShards, parityShards)
	if err != nil {
		return nil
	}
	enc.codec = codec

	// caches
	enc.encodeCache = make([][]byte, enc.shardSize)
	enc.shardCache = make([][]byte, enc.shardSize)
	for k := range enc.shardCache {
		enc.shardCache[k] = make([]byte, mtuLimit)
	}
	enc.zeros = make([]byte, mtuLimit)
	return enc
}

// encodes the packet, outputs parity shards if we have collected quorum datashards
// notice: the contents of 'ps' will be re-written in successive calling
func (enc *fecEncoder) encode(b []byte) (ps [][]byte) {
	// The header format:
	// | FEC SEQID(4B) | FEC TYPE(2B) | SIZE (2B) | PAYLOAD(SIZE-2) |
	// |<-headerOffset                |<-payloadOffset
	enc.markData(b[enc.headerOffset:])
	binary.LittleEndian.PutUint16(b[enc.payloadOffset:], uint16(len(b[enc.payloadOffset:])))

	// copy data from payloadOffset to fec shard cache
	sz := len(b)
	enc.shardCache[enc.shardCount] = enc.shardCache[enc.shardCount][:sz]
	copy(enc.shardCache[enc.shardCount][enc.payloadOffset:], b[enc.payloadOffset:])
	enc.shardCount++

	// track max datashard length
	if sz > enc.maxSize {
		enc.maxSize = sz
	}

	//  Generation of Reed-Solomon Erasure Code
	if enc.shardCount == enc.dataShards {
		// fill '0' into the tail of each datashard
		for i := 0; i < enc.dataShards; i++ {
			shard := enc.shardCache[i]
			slen := len(shard)
			copy(shard[slen:enc.maxSize], enc.zeros)
		}

		// construct equal-sized slice with stripped header
		cache := enc.encodeCache
		for k := range cache {
			cache[k] = enc.shardCache[k][enc.payloadOffset:enc.maxSize]
		}

		// encoding
		if err := enc.codec.Encode(cache); err == nil {
			ps = enc.shardCache[enc.dataShards:]
			for k := range ps {
				enc.markParity(ps[k][enc.headerOffset:])
				ps[k] = ps[k][:enc.maxSize]
			}
		}

		// counters resetting
		enc.shardCount = 0
		enc.maxSize = 0
	}

	return
}

func (enc *fecEncoder) markData(data []byte) {
	binary.LittleEndian.PutUint32(data, enc.next)
	binary.LittleEndian.PutUint16(data[4:], typeData)
	enc.next++
}

func (enc *fecEncoder) markParity(data []byte) {
	binary.LittleEndian.PutUint32(data, enc.next)
	binary.LittleEndian.PutUint16(data[4:], typeParity)
	// sequence wrap will only happen at parity shard
	enc.next = (enc.next + 1) % enc.paws
}
<|endoftext|>


<commit_before>package fuse

import (
	"io/ioutil"
	"log"
	"os"
	"testing"
	"time"
)

var _ = log.Println

type NotifyFs struct {
	DefaultFileSystem
	size  uint64
	exist bool
}

func (fs *NotifyFs) GetAttr(name string, context *Context) (*Attr, Status) {
	if name == "" {
		return &Attr{Mode: S_IFDIR | 0755}, OK
	}
	if name == "file" || (name == "dir/file" && fs.exist) {
		return &Attr{Mode: S_IFREG | 0644, Size: fs.size}, OK
	}
	if name == "dir" {
		return &Attr{Mode: S_IFDIR | 0755}, OK
	}
	return nil, ENOENT
}

func (fs *NotifyFs) Open(name string, f uint32, context *Context) (File, Status) {
	return NewDataFile([]byte{42}), OK
}

type NotifyTest struct {
	fs        *NotifyFs
	pathfs    *PathNodeFs
	connector *FileSystemConnector
	dir       string
	state     *MountState
}

func NewNotifyTest() *NotifyTest {
	me := &NotifyTest{}
	me.fs = &NotifyFs{}
	var err error
	me.dir, err = ioutil.TempDir("", "go-fuse")
	CheckSuccess(err)
	entryTtl := 100 * time.Millisecond
	opts := &FileSystemOptions{
		EntryTimeout:    entryTtl,
		AttrTimeout:     entryTtl,
		NegativeTimeout: entryTtl,
	}

	me.pathfs = NewPathNodeFs(me.fs, nil)
	me.state, me.connector, err = MountNodeFileSystem(me.dir, me.pathfs, opts)
	CheckSuccess(err)
	me.state.Debug = VerboseTest()
	go me.state.Loop()

	return me
}

func (t *NotifyTest) Clean() {
	err := t.state.Unmount()
	if err == nil {
		os.RemoveAll(t.dir)
	}
}

func TestInodeNotify(t *testing.T) {
	test := NewNotifyTest()
	defer test.Clean()

	fs := test.fs
	dir := test.dir

	fs.size = 42
	fi, err := os.Lstat(dir + "/file")
	CheckSuccess(err)
	if fi.Mode()&os.ModeType != 0 || fi.Size() != 42 {
		t.Error(fi)
	}

	fs.size = 666
	fi, err = os.Lstat(dir + "/file")
	CheckSuccess(err)
	if fi.Mode()&os.ModeType != 0 || fi.Size() == 666 {
		t.Error(fi)
	}

	code := test.pathfs.FileNotify("file", -1, 0)
	if !code.Ok() {
		t.Error(code)
	}

	fi, err = os.Lstat(dir + "/file")
	CheckSuccess(err)
	if fi.Mode()&os.ModeType != 0 || fi.Size() != 666 {
		t.Error(fi)
	}
}

func TestEntryNotify(t *testing.T) {
	test := NewNotifyTest()
	defer test.Clean()

	dir := test.dir
	test.fs.size = 42
	test.fs.exist = false
	fn := dir + "/dir/file"
	fi, _ := os.Lstat(fn)
	if fi != nil {
		t.Errorf("File should not exist, %#v", fi)
	}

	test.fs.exist = true
	fi, _ = os.Lstat(fn)
	if fi != nil {
		t.Errorf("negative entry should have been cached: %#v", fi)
	}

	code := test.pathfs.EntryNotify("dir", "file")
	if !code.Ok() {
		t.Errorf("EntryNotify returns error: %v", code)
	}

	fi, err := os.Lstat(fn)
	CheckSuccess(err)
}
<commit_msg>Fix data race in fuse.TestEntryNotify.<commit_after>package fuse

import (
	"io/ioutil"
	"log"
	"os"
	"testing"
	"time"
)

var _ = log.Println

type NotifyFs struct {
	DefaultFileSystem
	size  uint64
	exist bool
}

func (fs *NotifyFs) GetAttr(name string, context *Context) (*Attr, Status) {
	if name == "" {
		return &Attr{Mode: S_IFDIR | 0755}, OK
	}
	if name == "file" || (name == "dir/file" && fs.exist) {
		return &Attr{Mode: S_IFREG | 0644, Size: fs.size}, OK
	}
	if name == "dir" {
		return &Attr{Mode: S_IFDIR | 0755}, OK
	}
	return nil, ENOENT
}

func (fs *NotifyFs) Open(name string, f uint32, context *Context) (File, Status) {
	return NewDataFile([]byte{42}), OK
}

type NotifyTest struct {
	fs        *NotifyFs
	pathfs    *PathNodeFs
	connector *FileSystemConnector
	dir       string
	state     *MountState
}

func NewNotifyTest() *NotifyTest {
	me := &NotifyTest{}
	me.fs = &NotifyFs{}
	var err error
	me.dir, err = ioutil.TempDir("", "go-fuse")
	CheckSuccess(err)
	entryTtl := 100 * time.Millisecond
	opts := &FileSystemOptions{
		EntryTimeout:    entryTtl,
		AttrTimeout:     entryTtl,
		NegativeTimeout: entryTtl,
	}

	me.pathfs = NewPathNodeFs(me.fs, nil)
	me.state, me.connector, err = MountNodeFileSystem(me.dir, me.pathfs, opts)
	CheckSuccess(err)
	me.state.Debug = VerboseTest()
	go me.state.Loop()

	return me
}

func (t *NotifyTest) Clean() {
	err := t.state.Unmount()
	if err == nil {
		os.RemoveAll(t.dir)
	}
}

func TestInodeNotify(t *testing.T) {
	test := NewNotifyTest()
	defer test.Clean()

	fs := test.fs
	dir := test.dir

	fs.size = 42
	fi, err := os.Lstat(dir + "/file")
	CheckSuccess(err)
	if fi.Mode()&os.ModeType != 0 || fi.Size() != 42 {
		t.Error(fi)
	}

	fs.size = 666
	fi, err = os.Lstat(dir + "/file")
	CheckSuccess(err)
	if fi.Mode()&os.ModeType != 0 || fi.Size() == 666 {
		t.Error(fi)
	}

	code := test.pathfs.FileNotify("file", -1, 0)
	if !code.Ok() {
		t.Error(code)
	}

	fi, err = os.Lstat(dir + "/file")
	CheckSuccess(err)
	if fi.Mode()&os.ModeType != 0 || fi.Size() != 666 {
		t.Error(fi)
	}
}

func TestEntryNotify(t *testing.T) {
	test := NewNotifyTest()
	defer test.Clean()

	dir := test.dir
	test.fs.size = 42
	test.fs.exist = false
	test.state.ThreadSanitizerSync()

	fn := dir + "/dir/file"
	fi, _ := os.Lstat(fn)
	if fi != nil {
		t.Errorf("File should not exist, %#v", fi)
	}

	test.fs.exist = true
	test.state.ThreadSanitizerSync()
	fi, _ = os.Lstat(fn)
	if fi != nil {
		t.Errorf("negative entry should have been cached: %#v", fi)
	}

	code := test.pathfs.EntryNotify("dir", "file")
	if !code.Ok() {
		t.Errorf("EntryNotify returns error: %v", code)
	}

	fi, err := os.Lstat(fn)
	CheckSuccess(err)
}
<|endoftext|>


<commit_before><commit_msg>close fd after program close<commit_after><|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"log"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

type Commit struct {
	Author  string
	Date    time.Time
	File    string
	Hash    string
	Subject string
}

func (c Commit) Diff() ([]byte, error) {
	return Diff(c.File, c.Hash)
}

func (c Commit) FileNoExt() string {
	return strings.TrimSuffix(c.File, filepath.Ext(c.File))
}

func Diff(file, hash string) ([]byte, error) {
	var out bytes.Buffer

	git := exec.Command("git", "-C", options.Dir, "show", "--oneline", "--no-color", hash, file)

	// Prune diff stats from output with tail
	tail := exec.Command("tail", "-n", "+8")

	var err error
	tail.Stdin, err = git.StdoutPipe()
	if err != nil {
		log.Println("ERROR", err)
	}

	tail.Stdout = &out

	err = tail.Start()
	if err != nil {
		log.Println("ERROR", err)
	}

	err = git.Run()
	if err != nil {
		log.Println("ERROR", err)
	}

	err = tail.Wait()
	if err != nil {
		log.Println("ERROR", err)
	}

	return out.Bytes(), err
}

func Commits(filename string, n int) ([]Commit, error) {
	var commits []Commit

	// abbreviated commit hash|author name|author date, strict ISO 8601 format|subject
	logFormat := "--pretty=%h|%an|%aI|%s"

	cmd := exec.Command("git", "-C", options.Dir, "log", "-n", strconv.Itoa(n), logFormat, filename)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Println("ERROR", err)
		return commits, err
	}

	defer stdout.Close()

	err = cmd.Start()
	if err != nil {
		log.Println("ERROR", err)
		return commits, err
	}

	out := bufio.NewScanner(stdout)
	for out.Scan() {
		fields := strings.Split(out.Text(), "|")

		commit := Commit{
			Author:  fields[1],
			File:    filename,
			Hash:    fields[0],
			Subject: fields[3],
		}

		commit.Date, err = time.Parse(time.RFC3339Nano, fields[2])
		if err != nil {
			log.Println("ERROR", err)
		}

		commits = append(commits, commit)
	}

	return commits, nil
}

// Check if a path contains a Git repository
func IsGitRepository(path string) bool {
	var out bytes.Buffer
	cmd := exec.Command("git", "-C", options.Dir, "rev-parse", "--is-inside-work-tree")
	cmd.Stdout = &out

	err := cmd.Run()
	if err != nil {
		log.Println("ERROR", err)
		return false
	}

	var val bool
	_, err = fmt.Sscanf(out.String(), "%t", &val)
	if err != nil {
		log.Println("ERROR", err)
		return false
	}

	return val
}
<commit_msg>Use unix time instead of ISO 8601.<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"log"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

type Commit struct {
	Author  string
	Date    time.Time
	File    string
	Hash    string
	Subject string
}

func (c Commit) Diff() ([]byte, error) {
	return Diff(c.File, c.Hash)
}

func (c Commit) FileNoExt() string {
	return strings.TrimSuffix(c.File, filepath.Ext(c.File))
}

func Diff(file, hash string) ([]byte, error) {
	var out bytes.Buffer

	git := exec.Command("git", "-C", options.Dir, "show", "--oneline", "--no-color", hash, file)

	// Prune diff stats from output with tail
	tail := exec.Command("tail", "-n", "+8")

	var err error
	tail.Stdin, err = git.StdoutPipe()
	if err != nil {
		log.Println("ERROR", err)
	}

	tail.Stdout = &out

	err = tail.Start()
	if err != nil {
		log.Println("ERROR", err)
	}

	err = git.Run()
	if err != nil {
		log.Println("ERROR", err)
	}

	err = tail.Wait()
	if err != nil {
		log.Println("ERROR", err)
	}

	return out.Bytes(), err
}

func Commits(filename string, n int) ([]Commit, error) {
	var commits []Commit

	// abbreviated commit hash|author name|author date, strict ISO 8601 format|subject
	logFormat := "--pretty=%h|%an|%at|%s"

	cmd := exec.Command("git", "-C", options.Dir, "log", "-n", strconv.Itoa(n), logFormat, filename)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Println("ERROR", err)
		return commits, err
	}

	defer stdout.Close()

	err = cmd.Start()
	if err != nil {
		log.Println("ERROR", err)
		return commits, err
	}

	out := bufio.NewScanner(stdout)
	for out.Scan() {
		fields := strings.Split(out.Text(), "|")

		commit := Commit{
			Author:  fields[1],
			File:    filename,
			Hash:    fields[0],
			Subject: fields[3],
		}

		unix, err := strconv.ParseInt(fields[2], 10, 64)
		if err != nil {
			log.Println("ERROR", err)
		}
		commit.Date = time.Unix(unix, 0)

		commits = append(commits, commit)
	}

	return commits, nil
}

// Check if a path contains a Git repository
func IsGitRepository(path string) bool {
	var out bytes.Buffer
	cmd := exec.Command("git", "-C", options.Dir, "rev-parse", "--is-inside-work-tree")
	cmd.Stdout = &out

	err := cmd.Run()
	if err != nil {
		log.Println("ERROR", err)
		return false
	}

	var val bool
	_, err = fmt.Sscanf(out.String(), "%t", &val)
	if err != nil {
		log.Println("ERROR", err)
		return false
	}

	return val
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/jzipfler/htw-ava/server"
	"github.com/jzipfler/htw-ava/utils"
)

var (
	filename    string
	managerName string
	logFile     string
	ipAddress   string
	port        int
)

func init() {
	flag.StringVar(&filename, "filename", "path/to/file.txt", "A file that is managed by this process.")
	flag.StringVar(&managerName, "name", "Manager A", "Define the name of this manager.")
	flag.StringVar(&logFile, "logFile", "path/to/logfile.txt", "This parameter can be used to print the logging output to the given file.")
	flag.StringVar(&ipAddress, "ipAddress", "127.0.0.1", "The ip address of the actual starting node.")
	flag.IntVar(&port, "port", 15100, "The port of the actual starting node.")
}

func main() {

	if filename == "path/to/file.txt" {
		log.Printf("A filename is required.\n%s\n\n", utils.ERROR_FOOTER)
		flag.Usage()
		os.Exit(0)
	}

	flag.Parse()

	if exists := utils.CheckIfFileExists(filename); exists {
		if writable, err := utils.CheckIfFileIsReadableAndWritebale(filename); !writable {
			log.Fatalf("%s\n%s\n", err.Error(), utils.ERROR_FOOTER)
		}
	} else {
		os.Create(filename)
	}

	utils.InitializeLogger(logFile, "")
	utils.PrintMessage(fmt.Sprintf("File \"%s\" is now managed by this process.", filename))

	serverObject := server.New()

	serverObject.SetClientName(managerName)
	serverObject.SetIpAddressAsString(ipAddress)
	serverObject.SetPort(port)
	serverObject.SetUsedProtocol("tcp")

	if err := server.StartServer(serverObject, nil); err != nil {
		log.Fatalln("Could not start server. --> Exit.")
		os.Exit(1)
	}
	defer server.StopServer()
}
<commit_msg>Take care that the file is recreated if it already exists and test the new Increase and Decrease functions.<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/jzipfler/htw-ava/server"
	"github.com/jzipfler/htw-ava/utils"
)

var (
	filename    string
	managerName string
	logFile     string
	ipAddress   string
	port        int
	managedFile *os.File
	force       bool
)

func init() {
	flag.StringVar(&filename, "filename", "path/to/file.txt", "A file that is managed by this process.")
	flag.StringVar(&managerName, "name", "Manager A", "Define the name of this manager.")
	flag.StringVar(&logFile, "logFile", "path/to/logfile.txt", "This parameter can be used to print the logging output to the given file.")
	flag.StringVar(&ipAddress, "ipAddress", "127.0.0.1", "The ip address of the actual starting node.")
	flag.IntVar(&port, "port", 15100, "The port of the actual starting node.")
	flag.BoolVar(&force, "force", false, "If force is enabled, the programm removes a existing management file and creates a new one without asking.")
}

func main() {

	flag.Parse()

	if filename == "path/to/file.txt" {
		log.Printf("A filename is required.\n%s\n\n", utils.ERROR_FOOTER)
		flag.Usage()
		os.Exit(0)
	}

	utils.InitializeLogger(logFile, "")
	utils.PrintMessage(fmt.Sprintf("File \"%s\" is now managed by this process.", filename))

	if exists := utils.CheckIfFileExists(filename); exists {
		if !force {
			if deleteIt := askForToDeleteFile(); !deleteIt {
				fmt.Println("Do not delete the file and exit the program.")
				utils.PrintMessage(fmt.Sprintf("The file \"%s\" already exists and should not be deleted.", filename))
				os.Exit(0)
			}
		}
		if err := os.Remove(filename); err != nil {
			log.Fatalf("%s\n%s\n", err.Error(), utils.ERROR_FOOTER)
		}
		utils.PrintMessage(fmt.Sprintf("Removed the file \"%s\"", filename))
	}

	managedFile, err := os.Create(filename)
	utils.PrintMessage(fmt.Sprintf("Created the file \"%s\"", filename))
	defer managedFile.Close()
	if err != nil {
		log.Fatalf("%s\n%s\n", err.Error(), utils.ERROR_FOOTER)
	}

	managedFile.WriteString("000000\n")
	utils.PrintMessage("Wrote 000000 to the file.")

	for i := 0; i <= 100; i++ {
		if numbers, err := utils.IncreaseNumbersFromFirstLine(managedFile, 6); err != nil {
			log.Fatalln(err.Error())
		} else {
			fmt.Println(numbers)
		}
	}

	for i := 0; i <= 101; i++ {
		if numbers, err := utils.DecreaseNumbersFromFirstLine(managedFile, 6); err != nil {
			log.Fatalln(err.Error())
		} else {
			fmt.Println(numbers)
		}
	}

	os.Exit(0)

	serverObject := server.New()

	serverObject.SetClientName(managerName)
	serverObject.SetIpAddressAsString(ipAddress)
	serverObject.SetPort(port)
	serverObject.SetUsedProtocol("tcp")

	if err := server.StartServer(serverObject, nil); err != nil {
		log.Fatalln("Could not start server. --> Exit.")
		os.Exit(1)
	}
	defer server.StopServer()
}

func askForToDeleteFile() bool {
	var input string
	fmt.Printf("Would you like to delete the file \"%s\"? (y/j/n)", filename)
	fmt.Print("\nInput: ")
	if _, err := fmt.Scanln(&input); err == nil {
		switch input {
		case "y", "j":
			fmt.Println("File gets deleted.")
			return true
		case "n":
			fmt.Println(input)
			return false
		default:
			fmt.Println("Please only insert y/j for \"YES\" or n for \"NO\".\n" + utils.ERROR_FOOTER)
			fmt.Println("Assume a \"n\" as input.")
			return false
		}
	} else {
		fmt.Println("Please only insert y/j for \"YES\" or n for \"NO\".\n" + utils.ERROR_HEADER)
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
)

//VERSION of the program
var version = "undefined-autogenerated"

var globalScanRange string
var globalScanIntervall int

//Config data struct to read the config file
type Config struct {
	NMAPRange     string
	HTTPPort      int
	ScanIntervall int //seconds
}

//ReadConfig reads the config file
func ReadConfig(configfile string) Config {
	_, err := os.Stat(configfile)
	if err != nil {
		log.Fatal("Config file is missing: ", configfile)
	}

	var config Config
	if _, err := toml.DecodeFile(configfile, &config); err != nil {
		log.Fatal(err)
	}
	return config
}

func callNMAP() {
	log.Println("Starting nmap caller")
	var Counter = 1
	var tempScanFileName = "temp_scan.xml"
	var scanResultsFileName = "scan.xml"
	for {
		log.Println("Init NMAP scan no:", Counter)
		cmd := exec.Command("nmap", "-p", "22,80", "-oX", tempScanFileName, globalScanRange)
		cmd.Stdin = strings.NewReader("some input")
		var out bytes.Buffer
		cmd.Stdout = &out
		err := cmd.Run()
		if err != nil {
			log.Println(err)
		}
		log.Println("Scan no.", Counter, "complete")
		//log.Printf("in all caps: %q\n", out.String())
		Counter = Counter + 1

		//copy to the scan.xml
		r, err := os.Open(tempScanFileName)
		if err != nil {
			panic(err)
		}
		defer r.Close()

		w, err := os.Create(scanResultsFileName)
		if err != nil {
			panic(err)
		}
		defer w.Close()

		// do the actual work
		n, err := io.Copy(w, r)
		if err != nil {
			panic(err)
		}
		log.Printf("Scan results saved %v bytes\n", n)
		<-time.After(time.Duration(globalScanIntervall) * time.Second)
	}
}

func pageHandler(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path[1:]
	log.Println("URL path: " + path)

	//in case we have no path refer/redirect to index.html
	if len(path) == 0 {
		path = "index.html"
	}

	f, err := os.Open(path)
	if err == nil {
		Reader := bufio.NewReader(f)

		var contentType string

		if strings.HasSuffix(path, "css") {
			contentType = "text/css"
		} else if strings.HasSuffix(path, ".html") {
			contentType = "text/html"
		} else if strings.HasSuffix(path, ".js") {
			contentType = "application/javascript"
		} else if strings.HasSuffix(path, ".png") {
			contentType = "image/png"
		} else if strings.HasSuffix(path, ".svg") {
			contentType = "image/svg+xml"
		} else {
			contentType = "text/plain"
		}

		w.Header().Add("Content Type", contentType)
		Reader.WriteTo(w)
	} else {
		w.WriteHeader(404)
		fmt.Fprintln(w, "404 - Page not found"+http.StatusText(404))
	}
}

func main() {
	log.Println("Starting lan-monitor-server ver: " + version)

	//process the config
	//1st the config file is read and set parameters applied
	//2nd the command line parameters are interpreted,
	//if they are set they will overrule the config file
	//3rd if none of the above is applied the program reverts to the hardcoded defaults

	//defaults
	var config Config
	defaultConfigFileLocation := "/etc/lan-monitor/config"
	config.HTTPPort = 8080
	config.NMAPRange = "192.168.1.1/24"
	config.ScanIntervall = 120 //seconds

	displayVersion := flag.Bool("version", false, "Prints the version number")
	cmdlineHTTPPort := flag.Int("port", config.HTTPPort, "HTTP port for the webserver")
	cmdlineNMAPScanRange := flag.String("range", config.NMAPRange, "The range NMAP should scan e.g. 192.168.1.1/24 it has to be nmap compatible")
	cmdlineScanIntervall := flag.Int("scan-rate", config.ScanIntervall, "The intervall of the scans in seconds")
	configFileLocation := flag.String("config-file", defaultConfigFileLocation, "Location of the config file")
	flag.Parse()

	//read the configfile
	config = ReadConfig(*configFileLocation)

	//if no range is defined in the config file
	if config.NMAPRange == "" {
		globalScanRange = *cmdlineNMAPScanRange
	} else {
		globalScanRange = config.NMAPRange
	}

	//if no port is defined in the config file
	if config.HTTPPort == 0 {
		config.HTTPPort = *cmdlineHTTPPort
	}

	//if no scan intervall is defined in the config file
	if config.ScanIntervall == 0 {
		globalScanIntervall = *cmdlineScanIntervall
	} else {
		globalScanIntervall = config.ScanIntervall
	}

	log.Println("Config - range:", globalScanRange, "port:", config.HTTPPort, "intervall:", globalScanIntervall, "sec")

	if *displayVersion == true {
		fmt.Println("Version: " + version)
		return
	}

	//changing working dir
	log.Println("Changing working dir to: ")
	err := os.Chdir("../www")
	if err != nil {
		log.Fatalln("Unable to switch working dir")
	}

	workingDir, _ := os.Getwd()
	log.Println("Dir:" + workingDir)

	//init the scanning routine
	go callNMAP()

	//starting the webserver
	http.HandleFunc("/", pageHandler)
	err = http.ListenAndServe(":"+strconv.Itoa(config.HTTPPort), nil)
	if err != nil {
		log.Println("Server error - " + err.Error())
	}
}
<commit_msg>default location config file<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
)

//VERSION of the program
var version = "undefined-autogenerated"

var globalScanRange string
var globalScanIntervall int

//Config data struct to read the config file
type Config struct {
	NMAPRange     string
	HTTPPort      int
	ScanIntervall int //seconds
}

//ReadConfig reads the config file
func ReadConfig(configfile string) Config {
	_, err := os.Stat(configfile)
	if err != nil {
		log.Fatal("Config file is missing: ", configfile)
	}

	var config Config
	if _, err := toml.DecodeFile(configfile, &config); err != nil {
		log.Fatal(err)
	}
	return config
}

func callNMAP() {
	log.Println("Starting nmap caller")
	var Counter = 1
	var tempScanFileName = "temp_scan.xml"
	var scanResultsFileName = "scan.xml"
	for {
		log.Println("Init NMAP scan no:", Counter)
		cmd := exec.Command("nmap", "-p", "22,80", "-oX", tempScanFileName, globalScanRange)
		cmd.Stdin = strings.NewReader("some input")
		var out bytes.Buffer
		cmd.Stdout = &out
		err := cmd.Run()
		if err != nil {
			log.Println(err)
		}
		log.Println("Scan no.", Counter, "complete")
		//log.Printf("in all caps: %q\n", out.String())
		Counter = Counter + 1

		//copy to the scan.xml
		r, err := os.Open(tempScanFileName)
		if err != nil {
			panic(err)
		}
		defer r.Close()

		w, err := os.Create(scanResultsFileName)
		if err != nil {
			panic(err)
		}
		defer w.Close()

		// do the actual work
		n, err := io.Copy(w, r)
		if err != nil {
			panic(err)
		}
		log.Printf("Scan results saved %v bytes\n", n)
		<-time.After(time.Duration(globalScanIntervall) * time.Second)
	}
}

func pageHandler(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path[1:]
	log.Println("URL path: " + path)

	//in case we have no path refer/redirect to index.html
	if len(path) == 0 {
		path = "index.html"
	}

	f, err := os.Open(path)
	if err == nil {
		Reader := bufio.NewReader(f)

		var contentType string

		if strings.HasSuffix(path, "css") {
			contentType = "text/css"
		} else if strings.HasSuffix(path, ".html") {
			contentType = "text/html"
		} else if strings.HasSuffix(path, ".js") {
			contentType = "application/javascript"
		} else if strings.HasSuffix(path, ".png") {
			contentType = "image/png"
		} else if strings.HasSuffix(path, ".svg") {
			contentType = "image/svg+xml"
		} else {
			contentType = "text/plain"
		}

		w.Header().Add("Content Type", contentType)
		Reader.WriteTo(w)
	} else {
		w.WriteHeader(404)
		fmt.Fprintln(w, "404 - Page not found"+http.StatusText(404))
	}
}

func main() {
	log.Println("Starting lan-monitor-server ver: " + version)

	//process the config
	//1st the config file is read and set parameters applied
	//2nd the command line parameters are interpreted,
	//if they are set they will overrule the config file
	//3rd if none of the above is applied the program reverts to the hardcoded defaults

	//defaults
	var config Config
	defaultConfigFileLocation := "/etc/lan-monitor.conf"
	config.HTTPPort = 8080
	config.NMAPRange = "192.168.1.1/24"
	config.ScanIntervall = 120 //seconds

	displayVersion := flag.Bool("version", false, "Prints the version number")
	cmdlineHTTPPort := flag.Int("port", config.HTTPPort, "HTTP port for the webserver")
	cmdlineNMAPScanRange := flag.String("range", config.NMAPRange, "The range NMAP should scan e.g. 192.168.1.1/24 it has to be nmap compatible")
	cmdlineScanIntervall := flag.Int("scan-rate", config.ScanIntervall, "The intervall of the scans in seconds")
	configFileLocation := flag.String("config-file", defaultConfigFileLocation, "Location of the config file")
	flag.Parse()

	//read the configfile
	config = ReadConfig(*configFileLocation)

	//if no range is defined in the config file
	if config.NMAPRange == "" {
		globalScanRange = *cmdlineNMAPScanRange
	} else {
		globalScanRange = config.NMAPRange
	}

	//if no port is defined in the config file
	if config.HTTPPort == 0 {
		config.HTTPPort = *cmdlineHTTPPort
	}

	//if no scan intervall is defined in the config file
	if config.ScanIntervall == 0 {
		globalScanIntervall = *cmdlineScanIntervall
	} else {
		globalScanIntervall = config.ScanIntervall
	}

	log.Println("Config - range:", globalScanRange, "port:", config.HTTPPort, "intervall:", globalScanIntervall, "sec")

	if *displayVersion == true {
		fmt.Println("Version: " + version)
		return
	}

	//changing working dir
	log.Println("Changing working dir to: ")
	err := os.Chdir("../www")
	if err != nil {
		log.Fatalln("Unable to switch working dir")
	}

	workingDir, _ := os.Getwd()
	log.Println("Dir:" + workingDir)

	//init the scanning routine
	go callNMAP()

	//starting the webserver
	http.HandleFunc("/", pageHandler)
	err = http.ListenAndServe(":"+strconv.Itoa(config.HTTPPort), nil)
	if err != nil {
		log.Println("Server error - " + err.Error())
	}
}
<|endoftext|>