<commit_before>package glock

import (
	"log"
	"net"
	"time"

	"github.com/stathat/consistent"
)

func initServersPool(endpoints []string) *consistent.Consistent {
	cons := consistent.New()
	addEndpoints(cons, endpoints)
	return cons
}

func addEndpoints(cons *consistent.Consistent, endpoints []string) {
	for _, endpoint := range endpoints {
		conn, err := net.Dial("tcp", endpoint)
		if err == nil {
			cons.Add(endpoint)
			conn.Close()
		}
	}
}

func (c *Client) CheckServerStatus() {
	ticker := time.Tick(5 * time.Second)
	go func() {
		for _ = range ticker {
			down := downServers(c.endpoints, c.consistent.Members())
			if len(down) > 0 {
				addEndpoints(c.consistent, down)
			}

			log.Println("Connection Pool Size: ")
			for server, pool := range c.connectionPools {
				log.Println("Glock Server - ", server, ": ", len(pool))
			}
		}
	}()
}

func downServers(endpoints, upServers []string) (downServers []string) {
	for _, endpoint := range endpoints {
		isUp := false
		for _, member := range upServers {
			if endpoint == member {
				isUp = true
			}
		}
		if !isUp {
			downServers = append(downServers, endpoint)
		}
	}
	return downServers
}
<commit_msg>Better Loggin<commit_after>package glock

import (
	"log"
	"net"
	"time"

	"github.com/stathat/consistent"
)

func initServersPool(endpoints []string) *consistent.Consistent {
	cons := consistent.New()
	addEndpoints(cons, endpoints)
	return cons
}

func addEndpoints(cons *consistent.Consistent, endpoints []string) {
	for _, endpoint := range endpoints {
		conn, err := net.Dial("tcp", endpoint)
		if err == nil {
			log.Println("Adding Endpoint to glock servers: ", endpoint)
			cons.Add(endpoint)
			conn.Close()
		}
	}
}

func (c *Client) CheckServerStatus() {
	ticker := time.Tick(1 * time.Second)
	go func() {
		for _ = range ticker {
			down := downServers(c.endpoints, c.consistent.Members())
			if len(down) > 0 {
				addEndpoints(c.consistent, down)
			}

			for _, server := range c.consistent.Members() {
				log.Println("Glock Server - ", server, ": ", len(c.connectionPools[server]))
			}
		}
	}()
}

func downServers(endpoints, upServers []string) (downServers []string) {
	for _, endpoint := range endpoints {
		isUp := false
		for _, member := range upServers {
			if endpoint == member {
				isUp = true
			}
		}
		if !isUp {
			downServers = append(downServers, endpoint)
		}
	}
	return downServers
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"github.com/pellaeon/goas/v3/logger"
	"github.com/pellaeon/skicka/gdrive"
	"golang.org/x/net/context"
)

var _ fs.Node = (*File)(nil)

type File struct {
	sk_file *gdrive.File
	fs      *FS
}

func (n File) Attr(ctx context.Context, attr *fuse.Attr) error {
	logger.Debugf("%s.Attr()", n.sk_file.Path)
	go SingleUpdateMetadataCache()
	sum := uint64(0)
	for _, c := range n.sk_file.Id {
		sum += uint64(c)
	}
	attr.Inode = sum
	attr.Size = uint64(n.sk_file.FileSize)
	attr.Blocks = uint64(n.sk_file.FileSize / 1024) // XXX: block size 1024 bytes
	attr.Atime = n.sk_file.ModTime
	attr.Mtime = n.sk_file.ModTime
	attr.Ctime = n.sk_file.ModTime
	attr.Crtime = n.sk_file.ModTime
	attr.Mode = 0600
	attr.Nlink = 0

	return nil
}

var _ fs.NodeOpener = (*File)(nil)

func (n File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
	resp.Flags |= fuse.OpenNonSeekable
	return &FileHandle{
		sk_file:       n.sk_file,
		url:           "",
		urlLastUpdate: time.Unix(0, 0),
	}, nil
}

var _ fs.Handle = (*FileHandle)(nil)

type FileHandle struct {
	sk_file       *gdrive.File
	url           string // file download URL
	urlLastUpdate time.Time
}

var _ fs.HandleReleaser = (*FileHandle)(nil)

func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
	return nil
}

func (fh *FileHandle) GetContent() (io.ReadCloser, error) {
	if time.Since(fh.urlLastUpdate) > 3*time.Hour {
		logger.Debugf("fh.GetContent url expired: %s", fh.sk_file.Path)
		driveFile, err := gd.GetFileById(fh.sk_file.Id)
		if err != nil {
			return nil, err
		}

		url := driveFile.DownloadUrl
		if url == "" {
			// Google Docs files can't be downloaded directly via DownloadUrl,
			// but can be exported to another format that can be downloaded.

			// Docs, Sheets, and Slides can be downloaded into .docx, .xls,
			// and .pptx formats, respectively. This may be a bit confusing since
			// they won't have that suffix locally.
			for mt, u := range driveFile.ExportLinks {
				if strings.HasPrefix(mt, "application/vnd.openxmlformats-officedocument") {
					url = u
					break
				}
			}
			// Google Drawings can be downloaded in SVG form.
			if url == "" {
				if u, ok := driveFile.ExportLinks["image/svg+xml"]; ok {
					url = u
				}
			}
			// Otherwise we seem to be out of luck.
			if url == "" {
				return nil, fmt.Errorf("%s: unable to download Google Docs file", fh.sk_file.Path)
			}
		}
		fh.url = url
		fh.urlLastUpdate = time.Now()
	}
	for try := 0; ; try++ {
		request, err := http.NewRequest("GET", fh.url, nil)
		if err != nil {
			return nil, err
		}

		resp, err := gd.Client.Do(request)

		switch gd.HandleHTTPResponse(resp, err, try) {
		case gdrive.Success:
			// Rate-limit the download, if required.
			return gdrive.MakeLimitedDownloadReader(resp.Body), nil
		case gdrive.Fail:
			return nil, err
		case gdrive.Retry:
		}
	}
}

var _ fs.HandleReader = (*FileHandle)(nil)

func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
	go SingleUpdateMetadataCache()
	buf := make([]byte, req.Size)
	reader, err := fh.GetContent()
	if err != nil {
		logger.Errorf("FileHandle Read GetContent failed: %v", err)
		return err
	}
	n, err := reader.Read(buf)
	reader.Close()
	if err != nil && n == 0 {
		logger.Errorf("Filehandle Read failed: %v", err)
		return err
	} else {
		resp.Data = buf
		logger.Debugf("FileHandle Read() %d bytes", n)
		if int64(n) != fh.sk_file.FileSize {
			logger.Warningf("FileHandle Read() %d bytes but file has %d bytes", n, fh.sk_file.FileSize)
		}
		return nil
	}
}

var _ fs.NodeAccesser = (*File)(nil)

func (f *File) Access(ctx context.Context, req *fuse.AccessRequest) error {
	logger.Debugf(f.sk_file.Path + " Access()")
	return nil
}
<commit_msg>Update# [WIP] add FileCache struct, which 1. handles mem and disk caches 2. grab data from gdrive transparently when cache miss<commit_after>package main

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"strings"
	"time"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"github.com/pellaeon/goas/v3/logger"
	"github.com/pellaeon/skicka/gdrive"
	"golang.org/x/net/context"
)

const CACHE_THRESHOLD_SIZE = 4096

var _ fs.Node = (*File)(nil)

type File struct {
	sk_file *gdrive.File
	fs      *FS
}

func (n File) Attr(ctx context.Context, attr *fuse.Attr) error {
	logger.Debugf("%s.Attr()", n.sk_file.Path)
	go SingleUpdateMetadataCache()
	sum := uint64(0)
	for _, c := range n.sk_file.Id {
		sum += uint64(c)
	}
	attr.Inode = sum
	attr.Size = uint64(n.sk_file.FileSize)
	attr.Blocks = uint64(n.sk_file.FileSize / 1024) // XXX: block size 1024 bytes
	attr.Atime = n.sk_file.ModTime
	attr.Mtime = n.sk_file.ModTime
	attr.Ctime = n.sk_file.ModTime
	attr.Crtime = n.sk_file.ModTime
	attr.Mode = 0600
	attr.Nlink = 0

	return nil
}

var _ fs.NodeOpener = (*File)(nil)

func (n File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
	resp.Flags |= fuse.OpenNonSeekable
	return &FileHandle{
		sk_file:       n.sk_file,
		url:           "",
		urlLastUpdate: time.Unix(0, 0),
		cache:         NewFileCache(n.sk_file.Id),
	}, nil
}

var _ fs.Handle = (*FileHandle)(nil)

type FileHandle struct {
	sk_file       *gdrive.File
	url           string // file download URL
	urlLastUpdate time.Time
	cache         *FileCache
}

var _ fs.HandleReleaser = (*FileHandle)(nil)

func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
	return nil
}

func (fh *FileHandle) GetContent() (io.ReadCloser, error) {
	if time.Since(fh.urlLastUpdate) > 3*time.Hour {
		logger.Debugf("fh.GetContent url expired: %s", fh.sk_file.Path)
		driveFile, err := gd.GetFileById(fh.sk_file.Id)
		if err != nil {
			return nil, err
		}

		url := driveFile.DownloadUrl
		if url == "" {
			// Google Docs files can't be downloaded directly via DownloadUrl,
			// but can be exported to another format that can be downloaded.

			// Docs, Sheets, and Slides can be downloaded into .docx, .xls,
			// and .pptx formats, respectively. This may be a bit confusing since
			// they won't have that suffix locally.
			for mt, u := range driveFile.ExportLinks {
				if strings.HasPrefix(mt, "application/vnd.openxmlformats-officedocument") {
					url = u
					break
				}
			}
			// Google Drawings can be downloaded in SVG form.
			if url == "" {
				if u, ok := driveFile.ExportLinks["image/svg+xml"]; ok {
					url = u
				}
			}
			// Otherwise we seem to be out of luck.
			if url == "" {
				return nil, fmt.Errorf("%s: unable to download Google Docs file", fh.sk_file.Path)
			}
		}
		fh.url = url
		fh.urlLastUpdate = time.Now()
	}
	for try := 0; ; try++ {
		request, err := http.NewRequest("GET", fh.url, nil)
		if err != nil {
			return nil, err
		}

		resp, err := gd.Client.Do(request)

		switch gd.HandleHTTPResponse(resp, err, try) {
		case gdrive.Success:
			// Rate-limit the download, if required.
			return gdrive.MakeLimitedDownloadReader(resp.Body), nil
		case gdrive.Fail:
			return nil, err
		case gdrive.Retry:
		}
	}
}

var _ fs.HandleReader = (*FileHandle)(nil)

func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
	go SingleUpdateMetadataCache()
	if req.Offset == 0 {
		doneFirstReq := make(chan bool)
		fh.doneReadToCache = make(chan bool)
		go fh.LaunchBackgroundGetContent(req.Size, doneFirstReq, fh.doneReadToCache)
		<-doneFirstReq
		logger.Debugf("========got first block")
		resp.Data = fh.fileCache[:req.Size]
		req.Respond(resp)
		return nil
	} else {
		<-fh.doneReadToCache
		logger.Debugf("====read from %d to %d", req.Offset, len(fh.fileCache))
		resp.Data = fh.fileCache[req.Offset:len(fh.fileCache)]
		req.Respond(resp)
		return nil
	}
}
func (fh *FileHandle) LaunchBackgroundGetContent(firstReqSize int, doneFirstReq, done chan<- bool) error {
	//var cacheSize int64
	//if CACHE_THRESHOLD_SIZE <= fh.sk_file.FileSize {
	//	cacheSize = CACHE_THRESHOLD_SIZE
	//} else {
	//	cacheSize = fh.sk_file.FileSize
	//}
	fh.fileCache = make([]byte, fh.sk_file.FileSize)
	reader, err := fh.GetContent()
	if err != nil {
		logger.Errorf("FileHandle LaunchBackgroundGetContent GetContent failed: %v", err)
		return err
	}
	n, err := io.ReadFull(reader, fh.fileCache[:firstReqSize])
	if err != nil && n != firstReqSize {
		logger.Errorf("Filehandle LaunchBackgroundGetContent read first block failed: %v", err)
		return err
	} else {
		doneFirstReq <- true
		close(doneFirstReq)
	}
	logger.Debugf("==========start reading %d", len(fh.fileCache[firstReqSize:]))
	n, err = io.ReadFull(reader, fh.fileCache[firstReqSize:])
	logger.Debugf("==========end reading %d", n)
	if err != nil && n != int(fh.sk_file.FileSize)-firstReqSize { //TODO possible int overflow
		logger.Errorf("Filehandle LaunchBackgroundGetContent read remaining failed: %v", err)
		return err
	} else if err != nil {
		logger.Debugf("======%+v", err)
	} else {
		done <- true
		close(done)
	}
	logger.Debugf("==============background getcontent terminating")
	return nil
}

type FileCache struct {
	fileId           string
	inMemorySegments []int64
	onDiskSegments   []int64
	memSegments      map[int64][]byte
}

func NewFileCache(id string) *FileCache {
	return &FileCache{
		fileId:           id,
		inMemorySegments: make([]int64, 100), //XXX
		onDiskSegments:   make([]int64, 100), //TODO: check segment files on disk and populate information
		memSegments:      make(map[int64][]byte),
	}
}

func (fc *FileCache) Read(start int64, p []byte) (int, error) {
	end, n, segmentno := int64(0)
	// search memory segments
	for i := 0; i <= len(fc.inMemorySegments)-1; i += 2 {
		if fc.inMemorySegments[i] <= start {
			if fc.inMemorySegments[i+1] > start+len(p) {
				logger.Warningf("MemorySegment smaller than requested")
				end = fc.inMemorySegments[i+1]
				n = fc.inMemorySegments[i+1] - fc.inMemorySegments[i]
			} else {
				end = start + len(p)
				n = len(p)
			}
			segmentno = fc.inMemorySegments[i]
			break
		}
	}
	if end != 0 {
		p = memSegments[segmentno][start-segmentno : end-segmentno]
		return n, nil
	} else {
		// TODO: search disk segments
		logger.Debugf("Reading not in memory segent")
		for i := 0; i <= len(fc.onDiskSegments)-1; i += 2 {
			if fc.onDiskSegments[i] <= start {
				if fc.onDiskSegments[i+1] > start+len(p) {
					logger.Warningf("MemorySegment smaller than requested")
					end = fc.onDiskSegments[i+1]
					n = fc.onDiskSegments[i+1] - fc.onDiskSegments[i]
				} else {
					end = start + len(p)
					n = len(p)
				}
				segmentno = fc.onDiskSegments[i]
				break
			}
		}
		// open on disk cache file
		cacheFileName := CACHEDIR + fc.fileId + "_" + string(segmentno)
		cacheFile, err := os.Open(cacheFileName)
		if err != nil {
			logger.Errorf("Opening cache file %s failed: %+v", cacheFileName, err)
			//TODO: handle error
		}
		n, err = cacheFile.ReadAt(p, start-segmentno) //XXX: len(p) != [start-segmentno : end-segmentno]
		if err != nil && n == 0 {
			logger.Errorf("Reading cache file %s failed: %+v", cacheFileName, err)
		}
		return n, nil
	}
	// no data found in mem nor disk, start download
	//TODO
}

func (fc *FileCache) Write(start int64, p []byte) (int, error) {
	//TODO
	append(fc.inMemorySegments, start)
	append(fc.inMemorySegments, start+len(p))
	memSegments[start] = p
}

var _ fs.NodeAccesser = (*File)(nil)

func (f *File) Access(ctx context.Context, req *fuse.AccessRequest) error {
	logger.Debugf(f.sk_file.Path + " Access()")
	return nil
}
<|endoftext|>


<commit_before>package torus

import (
	"errors"
	"io"
	"os"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/coreos/pkg/capnslog"
	"github.com/prometheus/client_golang/prometheus"

	"github.com/coreos/torus/models"
)

var (
	promOpenINodes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "torus_server_open_inodes",
		Help: "Number of open inodes reported on last update to mds",
	}, []string{"volume"})
	promOpenFiles = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "torus_server_open_files",
		Help: "Number of open files",
	}, []string{"volume"})
	promFileSyncs = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "torus_server_file_syncs",
		Help: "Number of times a file has been synced on this server",
	}, []string{"volume"})
	promFileChangedSyncs = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "torus_server_file_changed_syncs",
		Help: "Number of times a file has been synced on this server, and the file has changed underneath it",
	}, []string{"volume"})
	promFileWrittenBytes = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "torus_server_file_written_bytes",
		Help: "Number of bytes written to a file on this server",
	}, []string{"volume"})
	promFileBlockRead = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "torus_server_file_block_read_us",
		Help:    "Histogram of ms taken to read a block through the layers and into the file abstraction",
		Buckets: prometheus.ExponentialBuckets(50.0, 2, 20),
	})
	promFileBlockWrite = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "torus_server_file_block_write_us",
		Help:    "Histogram of ms taken to write a block through the layers and into the file abstraction",
		Buckets: prometheus.ExponentialBuckets(50.0, 2, 20),
	})
)

func init() {
	prometheus.MustRegister(promOpenINodes)
	prometheus.MustRegister(promOpenFiles)
	prometheus.MustRegister(promFileSyncs)
	prometheus.MustRegister(promFileChangedSyncs)
	prometheus.MustRegister(promFileWrittenBytes)
	prometheus.MustRegister(promFileBlockRead)
	prometheus.MustRegister(promFileBlockWrite)
}

type File struct {
	// globals
	mut      sync.RWMutex
	srv      *Server
	blkSize  int64
	offset   int64
	ReadOnly bool

	// file metadata
	volume   *models.Volume
	inode    *models.INode
	blocks   Blockset
	replaces uint64
	changed  map[string]bool
	cache    fileCache

	writeINodeRef INodeRef
	writeOpen     bool
}

func (f *File) WriteOpen() bool {
	return f.writeOpen
}

func (f *File) Replaces() uint64 {
	return f.replaces
}

func (s *Server) CreateFile(volume *models.Volume, inode *models.INode, blocks Blockset) (*File, error) {
	md, err := s.MDS.GlobalMetadata()
	if err != nil {
		return nil, err
	}
	clog.Tracef("Creating File For Inode %d:%d", inode.Volume, inode.INode)
	return &File{
		volume:  volume,
		inode:   inode,
		srv:     s,
		blocks:  blocks,
		blkSize: int64(md.BlockSize),
		cache:   newSingleBlockCache(blocks, md.BlockSize),
	}, nil
}

func (f *File) openWrite() error {
	if f.ReadOnly {
		return ErrLocked
	}
	if f.writeOpen {
		return nil
	}
	f.srv.writeableLock.RLock()
	defer f.srv.writeableLock.RUnlock()
	vid := VolumeID(f.volume.Id)
	newINode, err := f.srv.MDS.CommitINodeIndex(vid)
	if err != nil {
		if err == ErrAgain {
			return f.openWrite()
		}
		return err
	}
	f.writeINodeRef = NewINodeRef(VolumeID(vid), newINode)
	if f.inode != nil {
		f.replaces = f.inode.INode
		f.inode.INode = uint64(newINode)
	}
	f.writeOpen = true
	f.cache.newINode(f.writeINodeRef)
	return nil
}

func (f *File) writeToBlock(i, from, to int, data []byte) (int, error) {
	return f.cache.writeToBlock(f.getContext(), i, from, to, data)
}

func (f *File) getContext() context.Context {
	return f.srv.getContext()
}

func (f *File) Write(b []byte) (n int, err error) {
	n, err = f.WriteAt(b, f.offset)
	f.offset += int64(n)
	return
}

func (f *File) WriteAt(b []byte, off int64) (n int, err error) {
	f.mut.Lock()
	defer f.mut.Unlock()
	err = f.openWrite()
	if err != nil {
		return 0, err
	}

	if clog.LevelAt(capnslog.TRACE) {
		clog.Trace("begin write: offset ", off, " size ", len(b))
	}
	toWrite := len(b)

	defer func() {
		if off > int64(f.inode.Filesize) {
			clog.Tracef("updating filesize: %d", off)
			f.inode.Filesize = uint64(off)
		}
	}()

	// Write the front matter, which may dangle from a byte offset
	blkIndex := int(off / f.blkSize)

	if f.blocks.Length()+1 < blkIndex {
		if clog.LevelAt(capnslog.DEBUG) {
			clog.Debug("begin write: offset ", off, " size ", len(b))
			clog.Debug("end of file ", f.blocks.Length(), " blkIndex ", blkIndex)
		}
		promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
		err := f.Truncate(off)
		if err != nil {
			return n, err
		}
		//return n, errors.New("Can't write past the end of a file")
	}

	blkOff := off - int64(int(f.blkSize)*blkIndex)
	if blkOff != 0 {
		frontlen := int(f.blkSize - blkOff)
		if frontlen > toWrite {
			frontlen = toWrite
		}
		wrote, err := f.writeToBlock(blkIndex, int(blkOff), int(blkOff)+frontlen, b[:frontlen])
		clog.Tracef("head writing block at index %d, inoderef %s", blkIndex, f.writeINodeRef)
		if err != nil {
			return n, err
		} else if wrote != frontlen {
			promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
			return n, errors.New("Couldn't write all of the first block at the offset")
		}
		b = b[frontlen:]
		n += wrote
		off += int64(wrote)
	}

	toWrite = len(b)
	if toWrite == 0 {
		// We're done
		promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
		return n, nil
	}

	// Bulk Write! We'd rather be here.
	if off%f.blkSize != 0 {
		panic("Offset not equal to a block boundary")
	}

	for toWrite >= int(f.blkSize) {
		blkIndex := int(off / f.blkSize)
		if clog.LevelAt(capnslog.TRACE) {
			clog.Tracef("bulk writing block at index %d, inoderef %s", blkIndex, f.writeINodeRef)
		}
		start := time.Now()
		err = f.blocks.PutBlock(f.getContext(), f.writeINodeRef, blkIndex, b[:f.blkSize])
		if err != nil {
			promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
			return n, err
		}
		delta := time.Now().Sub(start)
		promFileBlockWrite.Observe(float64(delta.Nanoseconds()) / 1000)
		b = b[f.blkSize:]
		n += int(f.blkSize)
		off += int64(f.blkSize)
		toWrite = len(b)
	}

	if toWrite == 0 {
		// We're done
		promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
		return n, nil
	}

	// Trailing matter. This sucks too.
	if off%f.blkSize != 0 {
		panic("Offset not equal to a block boundary after bulk")
	}
	blkIndex = int(off / f.blkSize)
	wrote, err := f.writeToBlock(blkIndex, 0, toWrite, b)
	if clog.LevelAt(capnslog.TRACE) {
		clog.Tracef("tail writing block at index %d, inoderef %s", blkIndex, f.writeINodeRef)
	}
	if err != nil {
		promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
		return n, err
	} else if wrote != toWrite {
		promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
		return n, errors.New("Couldn't write all of the last block")
	}
	n += wrote
	off += int64(wrote)
	promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
	return n, nil
}

func (f *File) Read(b []byte) (n int, err error) {
	n, err = f.ReadAt(b, f.offset)
	f.offset += int64(n)
	return
}

func (f *File) ReadAt(b []byte, off int64) (n int, ferr error) {
	f.mut.RLock()
	defer f.mut.RUnlock()
	toRead := len(b)
	if clog.LevelAt(capnslog.TRACE) {
		clog.Tracef("begin read @ %x of size %d", off, toRead)
	}
	n = 0
	if int64(toRead)+off > int64(f.inode.Filesize) {
		toRead = int(int64(f.inode.Filesize) - off)
		ferr = io.EOF
		clog.Tracef("read is longer than file")
	}
	for toRead > n {
		blkIndex := int(off / f.blkSize)
		blkOff := off - int64(int(f.blkSize)*blkIndex)
		if clog.LevelAt(capnslog.TRACE) {
			clog.Tracef("getting block index %d", blkIndex)
		}
		blk, err := f.cache.getBlock(f.getContext(), blkIndex)
		if err != nil {
			return n, err
		}
		thisRead := f.blkSize - blkOff
		if int64(toRead-n) < thisRead {
			thisRead = int64(toRead - n)
		}
		count := copy(b[n:], blk[blkOff:blkOff+thisRead])
		n += count
		off += int64(count)
	}
	if toRead != n {
		//panic("Read more than n bytes?")
	}
	return n, ferr
}

func (f *File) Seek(offset int64, whence int) (int64, error) {
	// TODO(mischief): validate offset
	switch whence {
	case os.SEEK_SET:
		f.offset = offset
	case os.SEEK_CUR:
		f.offset += offset
	case os.SEEK_END:
		//f.offset = int64(f.inode.Filesize) - offset
		fallthrough
	default:
		return 0, errors.New("invalid whence")
	}

	return offset, nil
}

func (f *File) Close() error {
	if f == nil {
		return ErrInvalid
	}
	var err error
	promOpenFiles.WithLabelValues(f.volume.Name).Dec()
	return err
}

func (f *File) Truncate(size int64) error {
	err := f.openWrite()
	if err != nil {
		return err
	}
	nBlocks := (size / f.blkSize)
	if size%f.blkSize != 0 {
		nBlocks++
	}
	clog.Tracef("truncate to %d %d", size, nBlocks)
	f.blocks.Truncate(int(nBlocks), uint64(f.blkSize))
	f.inode.Filesize = uint64(size)
	return nil
}

// Trim zeroes data in the middle of a file.
func (f *File) Trim(offset, length int64) error {
	clog.Debugf("trimming %d %d", offset, length)
	err := f.openWrite()
	if err != nil {
		return err
	}
	// find the block edges
	blkFrom := offset / f.blkSize
	if offset%f.blkSize != 0 {
		blkFrom += 1
	}
	blkTo := (offset + length) / f.blkSize
	return f.blocks.Trim(int(blkFrom), int(blkTo))
}

func (f *File) SyncAllWrites() (INodeRef, error) {
	err := f.SyncBlocks()
	if err != nil {
		return ZeroINode(), err
	}
	return f.SyncINode(f.getContext())
}

func (f *File) SyncINode(ctx context.Context) (INodeRef, error) {
	ref := f.writeINodeRef
	blkdata, err := MarshalBlocksetToProto(f.blocks)
	if err != nil {
		clog.Error("sync: couldn't marshal proto")
		return ZeroINode(), err
	}
	f.inode.Blocks = blkdata
	if f.inode.Volume != f.volume.Id {
		panic("mismatched volume and inode volume")
	}
	err = f.srv.INodes.WriteINode(ctx, ref, f.inode)
	if err != nil {
		return ZeroINode(), err
	}
	f.writeOpen = false
	return ref, nil
}

func (f *File) SyncBlocks() error {
	err := f.cache.sync(f.getContext())
	if err != nil {
		clog.Error("sync: couldn't sync block")
		return err
	}
	return f.srv.Blocks.Flush()
}

func (f *File) Size() uint64 {
	return f.inode.Filesize
}
<commit_msg>file: tiny: remove ErrAgain and retry<commit_after>package torus

import (
	"errors"
	"io"
	"os"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/coreos/pkg/capnslog"
	"github.com/prometheus/client_golang/prometheus"

	"github.com/coreos/torus/models"
)

var (
	promOpenINodes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "torus_server_open_inodes",
		Help: "Number of open inodes reported on last update to mds",
	}, []string{"volume"})
	promOpenFiles = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "torus_server_open_files",
		Help: "Number of open files",
	}, []string{"volume"})
	promFileSyncs = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "torus_server_file_syncs",
		Help: "Number of times a file has been synced on this server",
	}, []string{"volume"})
	promFileChangedSyncs = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "torus_server_file_changed_syncs",
		Help: "Number of times a file has been synced on this server, and the file has changed underneath it",
	}, []string{"volume"})
	promFileWrittenBytes = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "torus_server_file_written_bytes",
		Help: "Number of bytes written to a file on this server",
	}, []string{"volume"})
	promFileBlockRead = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "torus_server_file_block_read_us",
		Help:    "Histogram of ms taken to read a block through the layers and into the file abstraction",
		Buckets: prometheus.ExponentialBuckets(50.0, 2, 20),
	})
	promFileBlockWrite = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "torus_server_file_block_write_us",
		Help:    "Histogram of ms taken to write a block through the layers and into the file abstraction",
		Buckets: prometheus.ExponentialBuckets(50.0, 2, 20),
	})
)

func init() {
	prometheus.MustRegister(promOpenINodes)
	prometheus.MustRegister(promOpenFiles)
	prometheus.MustRegister(promFileSyncs)
	prometheus.MustRegister(promFileChangedSyncs)
	prometheus.MustRegister(promFileWrittenBytes)
	prometheus.MustRegister(promFileBlockRead)
	prometheus.MustRegister(promFileBlockWrite)
}

type File struct {
	// globals
	mut      sync.RWMutex
	srv      *Server
	blkSize  int64
	offset   int64
	ReadOnly bool

	// file metadata
	volume   *models.Volume
	inode    *models.INode
	blocks   Blockset
	replaces uint64
	changed  map[string]bool
	cache    fileCache

	writeINodeRef INodeRef
	writeOpen     bool
}

func (f *File) WriteOpen() bool {
	return f.writeOpen
}

func (f *File) Replaces() uint64 {
	return f.replaces
}

func (s *Server) CreateFile(volume *models.Volume, inode *models.INode, blocks Blockset) (*File, error) {
	md, err := s.MDS.GlobalMetadata()
	if err != nil {
		return nil, err
	}
	clog.Tracef("Creating File For Inode %d:%d", inode.Volume, inode.INode)
	return &File{
		volume:  volume,
		inode:   inode,
		srv:     s,
		blocks:  blocks,
		blkSize: int64(md.BlockSize),
		cache:   newSingleBlockCache(blocks, md.BlockSize),
	}, nil
}

func (f *File) openWrite() error {
	if f.ReadOnly {
		return ErrLocked
	}
	if f.writeOpen {
		return nil
	}
	f.srv.writeableLock.RLock()
	defer f.srv.writeableLock.RUnlock()
	vid := VolumeID(f.volume.Id)
	newINode, err := f.srv.MDS.CommitINodeIndex(vid)
	if err != nil {
		return err
	}
	f.writeINodeRef = NewINodeRef(VolumeID(vid), newINode)
	if f.inode != nil {
		f.replaces = f.inode.INode
		f.inode.INode = uint64(newINode)
	}
	f.writeOpen = true
	f.cache.newINode(f.writeINodeRef)
	return nil
}

func (f *File) writeToBlock(i, from, to int, data []byte) (int, error) {
	return f.cache.writeToBlock(f.getContext(), i, from, to, data)
}

func (f *File) getContext() context.Context {
	return f.srv.getContext()
}

func (f *File) Write(b []byte) (n int, err error) {
	n, err = f.WriteAt(b, f.offset)
	f.offset += int64(n)
	return
}

func (f *File) WriteAt(b []byte, off int64) (n int, err error) {
	f.mut.Lock()
	defer f.mut.Unlock()
	err = f.openWrite()
	if err != nil {
		return 0, err
	}

	if clog.LevelAt(capnslog.TRACE) {
		clog.Trace("begin write: offset ", off, " size ", len(b))
	}
	toWrite := len(b)

	defer func() {
		if off > int64(f.inode.Filesize) {
			clog.Tracef("updating filesize: %d", off)
			f.inode.Filesize = uint64(off)
		}
	}()

	// Write the front matter, which may dangle from a byte offset
	blkIndex := int(off / f.blkSize)

	if f.blocks.Length()+1 < blkIndex {
		if clog.LevelAt(capnslog.DEBUG) {
			clog.Debug("begin write: offset ", off, " size ", len(b))
			clog.Debug("end of file ", f.blocks.Length(), " blkIndex ", blkIndex)
		}
		promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
		err := f.Truncate(off)
		if err != nil {
			return n, err
		}
		//return n, errors.New("Can't write past the end of a file")
	}

	blkOff := off - int64(int(f.blkSize)*blkIndex)
	if blkOff != 0 {
		frontlen := int(f.blkSize - blkOff)
		if frontlen > toWrite {
			frontlen = toWrite
		}
		wrote, err := f.writeToBlock(blkIndex, int(blkOff), int(blkOff)+frontlen, b[:frontlen])
		clog.Tracef("head writing block at index %d, inoderef %s", blkIndex, f.writeINodeRef)
		if err != nil {
			return n, err
		} else if wrote != frontlen {
			promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
			return n, errors.New("Couldn't write all of the first block at the offset")
		}
		b = b[frontlen:]
		n += wrote
		off += int64(wrote)
	}

	toWrite = len(b)
	if toWrite == 0 {
		// We're done
		promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
		return n, nil
	}

	// Bulk Write! We'd rather be here.
	if off%f.blkSize != 0 {
		panic("Offset not equal to a block boundary")
	}

	for toWrite >= int(f.blkSize) {
		blkIndex := int(off / f.blkSize)
		if clog.LevelAt(capnslog.TRACE) {
			clog.Tracef("bulk writing block at index %d, inoderef %s", blkIndex, f.writeINodeRef)
		}
		start := time.Now()
		err = f.blocks.PutBlock(f.getContext(), f.writeINodeRef, blkIndex, b[:f.blkSize])
		if err != nil {
			promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
			return n, err
		}
		delta := time.Now().Sub(start)
		promFileBlockWrite.Observe(float64(delta.Nanoseconds()) / 1000)
		b = b[f.blkSize:]
		n += int(f.blkSize)
		off += int64(f.blkSize)
		toWrite = len(b)
	}

	if toWrite == 0 {
		// We're done
		promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
		return n, nil
	}

	// Trailing matter. This sucks too.
	if off%f.blkSize != 0 {
		panic("Offset not equal to a block boundary after bulk")
	}
	blkIndex = int(off / f.blkSize)
	wrote, err := f.writeToBlock(blkIndex, 0, toWrite, b)
	if clog.LevelAt(capnslog.TRACE) {
		clog.Tracef("tail writing block at index %d, inoderef %s", blkIndex, f.writeINodeRef)
	}
	if err != nil {
		promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
		return n, err
	} else if wrote != toWrite {
		promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
		return n, errors.New("Couldn't write all of the last block")
	}
	n += wrote
	off += int64(wrote)
	promFileWrittenBytes.WithLabelValues(f.volume.Name).Add(float64(n))
	return n, nil
}

func (f *File) Read(b []byte) (n int, err error) {
	n, err = f.ReadAt(b, f.offset)
	f.offset += int64(n)
	return
}

func (f *File) ReadAt(b []byte, off int64) (n int, ferr error) {
	f.mut.RLock()
	defer f.mut.RUnlock()
	toRead := len(b)
	if clog.LevelAt(capnslog.TRACE) {
		clog.Tracef("begin read @ %x of size %d", off, toRead)
	}
	n = 0
	if int64(toRead)+off > int64(f.inode.Filesize) {
		toRead = int(int64(f.inode.Filesize) - off)
		ferr = io.EOF
		clog.Tracef("read is longer than file")
	}
	for toRead > n {
		blkIndex := int(off / f.blkSize)
		blkOff := off - int64(int(f.blkSize)*blkIndex)
		if clog.LevelAt(capnslog.TRACE) {
			clog.Tracef("getting block index %d", blkIndex)
		}
		blk, err := f.cache.getBlock(f.getContext(), blkIndex)
		if err != nil {
			return n, err
		}
		thisRead := f.blkSize - blkOff
		if int64(toRead-n) < thisRead {
			thisRead = int64(toRead - n)
		}
		count := copy(b[n:], blk[blkOff:blkOff+thisRead])
		n += count
		off += int64(count)
	}
	if toRead != n {
		//panic("Read more than n bytes?")
	}
	return n, ferr
}

func (f *File) Seek(offset int64, whence int) (int64, error) {
	// TODO(mischief): validate offset
	switch whence {
	case os.SEEK_SET:
		f.offset = offset
	case os.SEEK_CUR:
		f.offset += offset
	case os.SEEK_END:
		//f.offset = int64(f.inode.Filesize) - offset
		fallthrough
	default:
		return 0, errors.New("invalid whence")
	}

	return offset, nil
}

func (f *File) Close() error {
	if f == nil {
		return ErrInvalid
	}
	var err error
	promOpenFiles.WithLabelValues(f.volume.Name).Dec()
	return err
}

func (f *File) Truncate(size int64) error {
	err := f.openWrite()
	if err != nil {
		return err
	}
	nBlocks := (size / f.blkSize)
	if size%f.blkSize != 0 {
		nBlocks++
	}
	clog.Tracef("truncate to %d %d", size, nBlocks)
	f.blocks.Truncate(int(nBlocks), uint64(f.blkSize))
	f.inode.Filesize = uint64(size)
	return nil
}

// Trim zeroes data in the middle of a file.
func (f *File) Trim(offset, length int64) error {
	clog.Debugf("trimming %d %d", offset, length)
	err := f.openWrite()
	if err != nil {
		return err
	}
	// find the block edges
	blkFrom := offset / f.blkSize
	if offset%f.blkSize != 0 {
		blkFrom += 1
	}
	blkTo := (offset + length) / f.blkSize
	return f.blocks.Trim(int(blkFrom), int(blkTo))
}

func (f *File) SyncAllWrites() (INodeRef, error) {
	err := f.SyncBlocks()
	if err != nil {
		return ZeroINode(), err
	}
	return f.SyncINode(f.getContext())
}

func (f *File) SyncINode(ctx context.Context) (INodeRef, error) {
	ref := f.writeINodeRef
	blkdata, err := MarshalBlocksetToProto(f.blocks)
	if err != nil {
		clog.Error("sync: couldn't marshal proto")
		return ZeroINode(), err
	}
	f.inode.Blocks = blkdata
	if f.inode.Volume != f.volume.Id {
		panic("mismatched volume and inode volume")
	}
	err = f.srv.INodes.WriteINode(ctx, ref, f.inode)
	if err != nil {
		return ZeroINode(), err
	}
	f.writeOpen = false
	return ref, nil
}

func (f *File) SyncBlocks() error {
	err := f.cache.sync(f.getContext())
	if err != nil {
		clog.Error("sync: couldn't sync block")
		return err
	}
	return f.srv.Blocks.Flush()
}

func (f *File) Size() uint64 {
	return f.inode.Filesize
}
<|endoftext|>


<commit_before>package cli

import (
	"bytes"
	"fmt"
	"io"
	"strings"
	"text/template"
)

// ToFishCompletion creates a fish completion string for the `*App`
// The function errors if either parsing or writing of the string fails.
func (a *App) ToFishCompletion() (string, error) {
	var w bytes.Buffer
	if err := a.writeFishCompletionTemplate(&w); err != nil {
		return "", err
	}
	return w.String(), nil
}

type fishCompletionTemplate struct {
	App         *App
	Completions []string
	AllCommands []string
}

func (a *App) writeFishCompletionTemplate(w io.Writer) error {
	const name = "cli"
	t, err := template.New(name).Parse(FishCompletionTemplate)
	if err != nil {
		return err
	}
	allCommands := []string{}

	// Add global flags
	completions := a.prepareFishFlags(a.VisibleFlags(), allCommands)

	// Add help flag
	if !a.HideHelp {
		completions = append(
			completions,
			a.prepareFishFlags([]Flag{HelpFlag}, allCommands)...,
		)
	}

	// Add version flag
	if !a.HideVersion {
		completions = append(
			completions,
			a.prepareFishFlags([]Flag{VersionFlag}, allCommands)...,
		)
	}

	// Add commands and their flags
	completions = append(
		completions,
		a.prepareFishCommands(a.VisibleCommands(), &allCommands, []string{})...,
	)

	return t.ExecuteTemplate(w, name, &fishCompletionTemplate{
		App:         a,
		Completions: completions,
		AllCommands: allCommands,
	})
}

func (a *App) prepareFishCommands(
	commands []Command,
	allCommands *[]string,
	previousCommands []string,
) []string {
	completions := []string{}
	for i := range commands {
		command := &commands[i]

		var completion strings.Builder
		completion.WriteString(fmt.Sprintf(
			"complete -r -c %s -n '%s' -a '%s'",
			a.Name,
			a.fishSubcommandHelper(previousCommands),
			strings.Join(command.Names(), " "),
		))

		if command.Usage != "" {
			completion.WriteString(fmt.Sprintf(" -d '%s'", command.Usage))
		}

		if !command.HideHelp {
			completions = append(
				completions,
				a.prepareFishFlags([]Flag{HelpFlag}, command.Names())...,
			)
		}

		*allCommands = append(*allCommands, command.Names()...)
		completions = append(completions, completion.String())
		completions = append(
			completions,
			a.prepareFishFlags(command.Flags, command.Names())...,
		)

		// recursevly iterate subcommands
		if len(command.Subcommands) > 0 {
			completions = append(
				completions,
				a.prepareFishCommands(
					command.Subcommands, allCommands, command.Names(),
				)...,
			)
		}
	}

	return completions
}

func (a *App) prepareFishFlags(
	flags []Flag,
	previousCommands []string,
) []string {
	completions := []string{}
	for _, f := range flags {
		flag, ok := f.(DocGenerationFlag)
		if !ok {
			continue
		}

		completion := &strings.Builder{}
		completion.WriteString(fmt.Sprintf(
			"complete -c %s -n '%s'",
			a.Name,
			a.fishSubcommandHelper(previousCommands),
		))

		fishAddFileFlag(f, completion)

		for idx, opt := range strings.Split(flag.GetName(), ",") {
			if idx == 0 {
				completion.WriteString(fmt.Sprintf(
					" -l %s", strings.TrimSpace(opt),
				))
			} else {
				completion.WriteString(fmt.Sprintf(
					" -s %s", strings.TrimSpace(opt),
				))

			}
		}

		if flag.TakesValue() {
			completion.WriteString(" -r")
		}

		if flag.GetUsage() != "" {
			completion.WriteString(fmt.Sprintf(" -d '%s'", flag.GetUsage()))
		}

		completions = append(completions, completion.String())
	}

	return completions
}

func fishAddFileFlag(
	flag Flag,
	completion *strings.Builder,
) {
	addFileExclusionFlag := true
	if f, ok := flag.(GenericFlag); ok && f.TakesFile {
		addFileExclusionFlag = false
	}
	if f, ok := flag.(StringFlag); ok && f.TakesFile {
		addFileExclusionFlag = false
	}
	if f, ok := flag.(StringSliceFlag); ok && f.TakesFile {
		addFileExclusionFlag = false
	}
	if addFileExclusionFlag {
		completion.WriteString(" -f")
	}
}

func (a *App) fishSubcommandHelper(allCommands []string) string {
	fishHelper := fmt.Sprintf("__fish_%s_no_subcommand", a.Name)
	if len(allCommands) > 0 {
		fishHelper = fmt.Sprintf(
			"__fish_seen_subcommand_from %s",
			strings.Join(allCommands, " "),
		)
	}
	return fishHelper

}
<commit_msg>Add type switch<commit_after>package cli

import (
	"bytes"
	"fmt"
	"io"
	"strings"
	"text/template"
)

// ToFishCompletion creates a fish completion string for the `*App`
// The function errors if either parsing or writing of the string fails.
func (a *App) ToFishCompletion() (string, error) {
	var w bytes.Buffer
	if err := a.writeFishCompletionTemplate(&w); err != nil {
		return "", err
	}
	return w.String(), nil
}

type fishCompletionTemplate struct {
	App         *App
	Completions []string
	AllCommands []string
}

func (a *App) writeFishCompletionTemplate(w io.Writer) error {
	const name = "cli"
	t, err := template.New(name).Parse(FishCompletionTemplate)
	if err != nil {
		return err
	}
	allCommands := []string{}

	// Add global flags
	completions := a.prepareFishFlags(a.VisibleFlags(), allCommands)

	// Add help flag
	if !a.HideHelp {
		completions = append(
			completions,
			a.prepareFishFlags([]Flag{HelpFlag}, allCommands)...,
		)
	}

	// Add version flag
	if !a.HideVersion {
		completions = append(
			completions,
			a.prepareFishFlags([]Flag{VersionFlag}, allCommands)...,
		)
	}

	// Add commands and their flags
	completions = append(
		completions,
		a.prepareFishCommands(a.VisibleCommands(), &allCommands, []string{})...,
	)

	return t.ExecuteTemplate(w, name, &fishCompletionTemplate{
		App:         a,
		Completions: completions,
		AllCommands: allCommands,
	})
}

func (a *App) prepareFishCommands(
	commands []Command,
	allCommands *[]string,
	previousCommands []string,
) []string {
	completions := []string{}
	for i := range commands {
		command := &commands[i]

		var completion strings.Builder
		completion.WriteString(fmt.Sprintf(
			"complete -r -c %s -n '%s' -a '%s'",
			a.Name,
			a.fishSubcommandHelper(previousCommands),
			strings.Join(command.Names(), " "),
		))

		if command.Usage != "" {
			completion.WriteString(fmt.Sprintf(" -d '%s'", command.Usage))
		}

		if !command.HideHelp {
			completions = append(
				completions,
				a.prepareFishFlags([]Flag{HelpFlag}, command.Names())...,
			)
		}

		*allCommands = append(*allCommands, command.Names()...)
		completions = append(completions, completion.String())
		completions = append(
			completions,
			a.prepareFishFlags(command.Flags, command.Names())...,
		)

		// recursevly iterate subcommands
		if len(command.Subcommands) > 0 {
			completions = append(
				completions,
				a.prepareFishCommands(
					command.Subcommands, allCommands, command.Names(),
				)...,
			)
		}
	}

	return completions
}

func (a *App) prepareFishFlags(
	flags []Flag,
	previousCommands []string,
) []string {
	completions := []string{}
	for _, f := range flags {
		flag, ok := f.(DocGenerationFlag)
		if !ok {
			continue
		}

		completion := &strings.Builder{}
		completion.WriteString(fmt.Sprintf(
			"complete -c %s -n '%s'",
			a.Name,
			a.fishSubcommandHelper(previousCommands),
		))

		fishAddFileFlag(f, completion)

		for idx, opt := range strings.Split(flag.GetName(), ",") {
			if idx == 0 {
				completion.WriteString(fmt.Sprintf(
					" -l %s", strings.TrimSpace(opt),
				))
			} else {
				completion.WriteString(fmt.Sprintf(
					" -s %s", strings.TrimSpace(opt),
				))

			}
		}

		if flag.TakesValue() {
			completion.WriteString(" -r")
		}

		if flag.GetUsage() != "" {
			completion.WriteString(fmt.Sprintf(" -d '%s'", flag.GetUsage()))
		}

		completions = append(completions, completion.String())
	}

	return completions
}

func fishAddFileFlag(
	flag Flag,
	completion *strings.Builder,
) {
	switch f := flag.(type) {
	case GenericFlag:
		if f.TakesFile {
			return
		}
	case StringFlag:
		if f.TakesFile {
			return
		}
	case StringSliceFlag:
		if f.TakesFile {
			return
		}
	}
	completion.WriteString(" -f")
}

func (a *App) fishSubcommandHelper(allCommands []string) string {
	fishHelper := fmt.Sprintf("__fish_%s_no_subcommand", a.Name)
	if len(allCommands) > 0 {
		fishHelper = fmt.Sprintf(
			"__fish_seen_subcommand_from %s",
			strings.Join(allCommands, " "),
		)
	}
	return fishHelper

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/stevencorona/elastic-haproxy/elb"
	"github.com/stevencorona/elastic-haproxy/haproxy"
	"github.com/stevencorona/elastic-haproxy/statsd"
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"
)

var defaultConfigFile = "config/elastic.toml"
var flagConfigFile string

func main() {

	haproxy.Transform()
	os.Exit(1)

	flag.StringVar(&flagConfigFile, "configFile", defaultConfigFile, "Path to toml file")
	flag.Parse()

	conf := LoadConfig(flagConfigFile)

	server := new(haproxy.Server)

	notificationChan := make(chan haproxy.Event)
	shouldReloadChan := make(chan haproxy.Action)

	go gracefulSignals(server)
	go server.Start(notificationChan, shouldReloadChan)
	go elb.SetupApiHandlers()

	for {
		<-notificationChan
		fmt.Println("Got notification")
		time.Sleep(2 * time.Second)

		server.Socket = conf.Haproxy.Socket
		serverInfo := server.GetInfo()
		fmt.Println(serverInfo)

		if conf.Statsd.Enabled {
			go statsd.SendMetrics(server)
		}
	}
}

func gracefulSignals(server *haproxy.Server) {
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGQUIT)

	for {
		s := <-signals
		log.Println("Got signal:", s)

		if s == syscall.SIGQUIT {
			fmt.Println("caught sigquit")
			server.ActionChan <- haproxy.WantsStop
			os.Exit(1)
		}

		server.ActionChan <- haproxy.WantsReload
	}
}
<commit_msg>use action channel<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/stevencorona/elastic-haproxy/elb"
	"github.com/stevencorona/elastic-haproxy/haproxy"
	"github.com/stevencorona/elastic-haproxy/statsd"
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"
)

var defaultConfigFile = "config/elastic.toml"
var flagConfigFile string

func main() {

	haproxy.Transform()
	os.Exit(1)

	flag.StringVar(&flagConfigFile, "configFile", defaultConfigFile, "Path to toml file")
	flag.Parse()

	conf := LoadConfig(flagConfigFile)

	server := new(haproxy.Server)

	notificationChan := make(chan haproxy.Event)
	actionChan := make(chan haproxy.Action)

	go gracefulSignals(server)
	go server.Start(notificationChan, actionChan)
	go elb.SetupApiHandlers()

	for {
		<-notificationChan
		fmt.Println("Got notification")
		time.Sleep(2 * time.Second)

		server.Socket = conf.Haproxy.Socket
		serverInfo := server.GetInfo()
		fmt.Println(serverInfo)

		if conf.Statsd.Enabled {
			go statsd.SendMetrics(server)
		}
	}
}

func gracefulSignals(server *haproxy.Server) {
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGQUIT)

	for {
		s := <-signals
		log.Println("Got signal:", s)

		if s == syscall.SIGQUIT {
			fmt.Println("caught sigquit")
			server.ActionChan <- haproxy.WantsStop
			os.Exit(1)
		}

		server.ActionChan <- haproxy.WantsReload
	}
}
<|endoftext|>


<commit_before>// Package common contains interfaces and non-specific protocol entities
package common

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"regexp"
	"strings"

	"gopkg.in/src-d/go-git.v4/core"
	"gopkg.in/src-d/go-git.v4/formats/packp"
	"gopkg.in/src-d/go-git.v4/formats/packp/advrefs"
	"gopkg.in/src-d/go-git.v4/formats/packp/pktline"
	"gopkg.in/src-d/go-git.v4/storage/memory"
)

var (
	ErrRepositoryNotFound    = errors.New("repository not found")
	ErrAuthorizationRequired = errors.New("authorization required")
	ErrEmptyGitUploadPack    = errors.New("empty git-upload-pack given")
	ErrInvalidAuthMethod     = errors.New("invalid auth method")
)

const GitUploadPackServiceName = "git-upload-pack"

type GitUploadPackService interface {
	Connect() error
	SetAuth(AuthMethod) error
	Info() (*GitUploadPackInfo, error)
	Fetch(*GitUploadPackRequest) (io.ReadCloser, error)
	Disconnect() error
}

type AuthMethod interface {
	Name() string
	String() string
}

type Endpoint url.URL

var (
	isSchemeRegExp   = regexp.MustCompile("^[^:]+://")
	scpLikeUrlRegExp = regexp.MustCompile("^(?P<user>[^@]+@)?(?P<host>[^:]+):/?(?P<path>.+)$")
)

func NewEndpoint(endpoint string) (Endpoint, error) {
	endpoint = transformSCPLikeIfNeeded(endpoint)

	u, err := url.Parse(endpoint)
	if err != nil {
		return Endpoint{}, core.NewPermanentError(err)
	}

	if !u.IsAbs() {
		return Endpoint{}, core.NewPermanentError(fmt.Errorf(
			"invalid endpoint: %s", endpoint,
		))
	}

	return Endpoint(*u), nil
}

func transformSCPLikeIfNeeded(endpoint string) string {
	if !isSchemeRegExp.MatchString(endpoint) && scpLikeUrlRegExp.MatchString(endpoint) {
		m := scpLikeUrlRegExp.FindStringSubmatch(endpoint)
		return fmt.Sprintf("ssh://%s%s/%s", m[1], m[2], m[3])
	}

	return endpoint
}

func (e *Endpoint) String() string {
	u := url.URL(*e)
	return u.String()
}

type GitUploadPackInfo struct {
	Capabilities *packp.Capabilities
	Refs         memory.ReferenceStorage
}

func NewGitUploadPackInfo() *GitUploadPackInfo {
	return &GitUploadPackInfo{Capabilities: packp.NewCapabilities()}
}

func (i *GitUploadPackInfo) Decode(r io.Reader) error {
	d := advrefs.NewDecoder(r)
	ar := advrefs.New()
	if err := d.Decode(ar); err != nil {
		if err == advrefs.ErrEmpty {
			return core.NewPermanentError(err)
		}
		return core.NewUnexpectedError(err)
	}

	i.Capabilities = ar.Capabilities

	if err := i.addRefs(ar); err != nil {
		return core.NewUnexpectedError(err)
	}

	return nil
}

func (i *GitUploadPackInfo) addRefs(ar *advrefs.AdvRefs) error {
	i.Refs = make(memory.ReferenceStorage, 0)
	for name, hash := range ar.References {
		ref := core.NewReferenceFromStrings(name, hash.String())
		i.Refs.Set(ref)
	}

	return i.addSymbolicRefs(ar)
}

func (i *GitUploadPackInfo) addSymbolicRefs(ar *advrefs.AdvRefs) error {
	if !hasSymrefs(ar) {
		return nil
	}

	for _, symref := range ar.Capabilities.Get("symref").Values {
		chunks := strings.Split(symref, ":")
		if len(chunks) != 2 {
			err := fmt.Errorf("bad number of `:` in symref value (%q)", symref)
			return core.NewUnexpectedError(err)
		}
		name := core.ReferenceName(chunks[0])
		target := core.ReferenceName(chunks[1])
		ref := core.NewSymbolicReference(name, target)
		i.Refs.Set(ref)
	}

	return nil
}

func hasSymrefs(ar *advrefs.AdvRefs) bool {
	return ar.Capabilities.Supports("symref")
}

func (i *GitUploadPackInfo) Head() *core.Reference {
	ref, _ := core.ResolveReference(i.Refs, core.HEAD)
	return ref
}

func (i *GitUploadPackInfo) String() string {
	return string(i.Bytes())
}

func (i *GitUploadPackInfo) Bytes() []byte {
	var buf bytes.Buffer
	e := pktline.NewEncoder(&buf)

	_ = e.EncodeString("# service=git-upload-pack\n")

	// inserting a flush-pkt here violates the protocol spec, but some
	// servers do it, like Github.com
	e.Flush()

	_ = e.Encodef("%s HEAD\x00%s\n", i.Head().Hash(), i.Capabilities.String())

	for _, ref := range i.Refs {
		if ref.Type() != core.HashReference {
			continue
		}

		_ = e.Encodef("%s %s\n", ref.Hash(), ref.Name())
	}

	e.Flush()

	return buf.Bytes()
}

type GitUploadPackRequest struct {
	Wants []core.Hash
	Haves []core.Hash
	Depth int
}

func (r *GitUploadPackRequest) Want(h ...core.Hash) {
	r.Wants = append(r.Wants, h...)
}

func (r *GitUploadPackRequest) Have(h ...core.Hash) {
	r.Haves = append(r.Haves, h...)
}

func (r *GitUploadPackRequest) String() string {
	b, _ := ioutil.ReadAll(r.Reader())
	return string(b)
}

func (r *GitUploadPackRequest) Reader() *strings.Reader {
	var buf bytes.Buffer
	e := pktline.NewEncoder(&buf)

	for _, want := range r.Wants {
		_ = e.Encodef("want %s\n", want)
	}

	for _, have := range r.Haves {
		_ = e.Encodef("have %s\n", have)
	}

	if r.Depth != 0 {
		_ = e.Encodef("deepen %d\n", r.Depth)
	}

	_ = e.Flush()
	_ = e.EncodeString("done\n")

	return strings.NewReader(buf.String())
}
<commit_msg>clients: common, initialize Refs in GitUploadPackInfo<commit_after>// Package common contains interfaces and non-specific protocol entities
package common

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"regexp"
	"strings"

	"gopkg.in/src-d/go-git.v4/core"
	"gopkg.in/src-d/go-git.v4/formats/packp"
	"gopkg.in/src-d/go-git.v4/formats/packp/advrefs"
	"gopkg.in/src-d/go-git.v4/formats/packp/pktline"
	"gopkg.in/src-d/go-git.v4/storage/memory"
)

var (
	ErrRepositoryNotFound    = errors.New("repository not found")
	ErrAuthorizationRequired = errors.New("authorization required")
	ErrEmptyGitUploadPack    = errors.New("empty git-upload-pack given")
	ErrInvalidAuthMethod     = errors.New("invalid auth method")
)

const GitUploadPackServiceName = "git-upload-pack"

type GitUploadPackService interface {
	Connect() error
	SetAuth(AuthMethod) error
	Info() (*GitUploadPackInfo, error)
	Fetch(*GitUploadPackRequest) (io.ReadCloser, error)
	Disconnect() error
}

type AuthMethod interface {
	Name() string
	String() string
}

type Endpoint url.URL

var (
	isSchemeRegExp   = regexp.MustCompile("^[^:]+://")
	scpLikeUrlRegExp = regexp.MustCompile("^(?P<user>[^@]+@)?(?P<host>[^:]+):/?(?P<path>.+)$")
)

func NewEndpoint(endpoint string) (Endpoint, error) {
	endpoint = transformSCPLikeIfNeeded(endpoint)

	u, err := url.Parse(endpoint)
	if err != nil {
		return Endpoint{}, core.NewPermanentError(err)
	}

	if !u.IsAbs() {
		return Endpoint{}, core.NewPermanentError(fmt.Errorf(
			"invalid endpoint: %s", endpoint,
		))
	}

	return Endpoint(*u), nil
}

func transformSCPLikeIfNeeded(endpoint string) string {
	if !isSchemeRegExp.MatchString(endpoint) && scpLikeUrlRegExp.MatchString(endpoint) {
		m := scpLikeUrlRegExp.FindStringSubmatch(endpoint)
		return fmt.Sprintf("ssh://%s%s/%s", m[1], m[2], m[3])
	}

	return endpoint
}

func (e *Endpoint) String() string {
	u := url.URL(*e)
	return u.String()
}

type GitUploadPackInfo struct {
	Capabilities *packp.Capabilities
	Refs         memory.ReferenceStorage
}

func NewGitUploadPackInfo() *GitUploadPackInfo {
	return &GitUploadPackInfo{
		Capabilities: packp.NewCapabilities(),
		Refs:         make(memory.ReferenceStorage, 0),
	}
}

func (i *GitUploadPackInfo) Decode(r io.Reader) error {
	d := advrefs.NewDecoder(r)
	ar := advrefs.New()
	if err := d.Decode(ar); err != nil {
		if err == advrefs.ErrEmpty {
			return core.NewPermanentError(err)
		}
		return core.NewUnexpectedError(err)
	}

	i.Capabilities = ar.Capabilities

	if err := i.addRefs(ar); err != nil {
		return core.NewUnexpectedError(err)
	}

	return nil
}

func (i *GitUploadPackInfo) addRefs(ar *advrefs.AdvRefs) error {
	i.Refs = make(memory.ReferenceStorage, 0)
	for name, hash := range ar.References {
		ref := core.NewReferenceFromStrings(name, hash.String())
		i.Refs.Set(ref)
	}

	return i.addSymbolicRefs(ar)
}

func (i *GitUploadPackInfo) addSymbolicRefs(ar *advrefs.AdvRefs) error {
	if !hasSymrefs(ar) {
		return nil
	}

	for _, symref := range ar.Capabilities.Get("symref").Values {
		chunks := strings.Split(symref, ":")
		if len(chunks) != 2 {
			err := fmt.Errorf("bad number of `:` in symref value (%q)", symref)
			return core.NewUnexpectedError(err)
		}
		name := core.ReferenceName(chunks[0])
		target := core.ReferenceName(chunks[1])
		ref := core.NewSymbolicReference(name, target)
		i.Refs.Set(ref)
	}

	return nil
}

func hasSymrefs(ar *advrefs.AdvRefs) bool {
	return ar.Capabilities.Supports("symref")
}

func (i *GitUploadPackInfo) Head() *core.Reference {
	ref, _ := core.ResolveReference(i.Refs, core.HEAD)
	return ref
}

func (i *GitUploadPackInfo) String() string {
	return string(i.Bytes())
}

func (i *GitUploadPackInfo) Bytes() []byte {
	var buf bytes.Buffer
	e := pktline.NewEncoder(&buf)

	_ = e.EncodeString("# service=git-upload-pack\n")

	// inserting a flush-pkt here violates the protocol spec, but some
	// servers do it, like Github.com
	e.Flush()

	_ = e.Encodef("%s HEAD\x00%s\n", i.Head().Hash(), i.Capabilities.String())

	for _, ref := range i.Refs {
		if ref.Type() != core.HashReference {
			continue
		}

		_ = e.Encodef("%s %s\n", ref.Hash(), ref.Name())
	}

	e.Flush()

	return buf.Bytes()
}

type GitUploadPackRequest struct {
	Wants []core.Hash
	Haves []core.Hash
	Depth int
}

func (r *GitUploadPackRequest) Want(h ...core.Hash) {
	r.Wants = append(r.Wants, h...)
}

func (r *GitUploadPackRequest) Have(h ...core.Hash) {
	r.Haves = append(r.Haves, h...)
}

func (r *GitUploadPackRequest) String() string {
	b, _ := ioutil.ReadAll(r.Reader())
	return string(b)
}

func (r *GitUploadPackRequest) Reader() *strings.Reader {
	var buf bytes.Buffer
	e := pktline.NewEncoder(&buf)

	for _, want := range r.Wants {
		_ = e.Encodef("want %s\n", want)
	}

	for _, have := range r.Haves {
		_ = e.Encodef("have %s\n", have)
	}

	if r.Depth != 0 {
		_ = e.Encodef("deepen %d\n", r.Depth)
	}

	_ = e.Flush()
	_ = e.EncodeString("done\n")

	return strings.NewReader(buf.String())
}
<|endoftext|>


<commit_before>package encodes

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha256"
	"fmt"
	"net"
	"time"
)

const (
	IVLength = 1400
)

type AESConn struct {
	conn net.Conn
	r    *cipher.StreamReader
	w    *cipher.StreamWriter
}

func (ac *AESConn) Read(dst []byte) (n int, err error) {
	n, err = ac.r.Read(dst)
	return n, err
}

func (ac *AESConn) Write(src []byte) (n int, err error) {
	n, err = ac.w.Write(src)
	return n, err
}

func (ac *AESConn) Close() error {
	return ac.conn.Close()
}

func (ac *AESConn) LocalAddr() net.Addr {
	return ac.conn.LocalAddr()
}

func (ac *AESConn) RemoteAddr() net.Addr {
	return ac.conn.RemoteAddr()
}

func (ac *AESConn) SetDeadline(t time.Time) error {
	return ac.conn.SetDeadline(t)
}

func (ac *AESConn) SetReadDeadline(t time.Time) error {
	return ac.conn.SetReadDeadline(t)
}

func (ac *AESConn) SetWriteDeadline(t time.Time) error {
	return ac.conn.SetWriteDeadline(t)
}

func NewAESConn(key string, iv [aes.BlockSize]byte, conn net.Conn) (aesconn *AESConn, err error) {
	keybin := []byte(key)

	hashkey := sha256.Sum256(keybin)
	salt := sha256.Sum256([]byte(fmt.Sprintf("%x", key)))

	hashsalt := append(hashkey[:], salt[:]...)

	aeskey := sha256.Sum256(hashsalt)

	rblock, err := aes.NewCipher(aeskey[:])
	wblock, err := aes.NewCipher(aeskey[:])
	if err != nil {
		return nil, err
	}

	rstream := cipher.NewOFB(rblock, iv[:])
	wstream := cipher.NewOFB(wblock, iv[:])

	aesconn = &AESConn{
		conn: conn,
		r:    &cipher.StreamReader{S: rstream, R: conn},
		w:    &cipher.StreamWriter{S: wstream, W: conn},
	}
	return aesconn, err
}

type Listener struct {
	netl net.Listener
	key  string
}

func (l *Listener) Accept() (aesconn net.Conn, err error) {
	conn, err := l.netl.Accept()
	if err != nil {
		return nil, err
	}

	ivvector := make([]byte, IVLength)
	var ri = 0
	for ri < IVLength {
		r, err := conn.Read(ivvector[ri:])
		if err != nil {
			return nil, err
		}
		ri += r
	}

	passwd := sha256.Sum256([]byte(l.key))
	longiv := append(passwd[:], ivvector...)

	hash := sha256.Sum256(longiv)

	var iv [aes.BlockSize]byte
	copy(iv[:], hash[:aes.BlockSize])

	aesconn, err = NewAESConn(l.key, iv, conn)

	return aesconn, err
}

func (l *Listener) Close() error {
	return l.netl.Close()
}

func (l *Listener) Addr() net.Addr {
	return l.netl.Addr()
}

func Listen(laddr string, key string) (l *Listener, err error) {
	netl, err := net.Listen("tcp", laddr)
	if err != nil {
		return nil, err
	}

	l = &Listener{
		netl: netl,
		key:  key,
	}
	return l, nil
}

func Dial(address string, key string) (aesconn net.Conn, err error) {
	conn, err := net.Dial("tcp", address)
	if err != nil {
		return nil, err
	}

	ivvector := make([]byte, IVLength)
	_, err = rand.Read(ivvector[:])

	if err != nil {
		return nil, err
	}

	var wi = 0
	for wi < IVLength {
		r, err := conn.Write(ivvector[wi:])
		if err != nil {
			return nil, err
		}
		wi += r
	}

	passwd := sha256.Sum256([]byte(key))
	longiv := append(passwd[:], ivvector...)

	hash := sha256.Sum256(longiv)

	var iv [aes.BlockSize]byte
	copy(iv[:], hash[:aes.BlockSize])

	aesconn, err = NewAESConn(key, iv, conn)
	return aesconn, err
}
<commit_msg>add read time out<commit_after>package encodes

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha256"
	"fmt"
	"net"
	"time"
)

const (
	IVLength = 1400
)

type AESConn struct {
	conn net.Conn
	r    *cipher.StreamReader
	w    *cipher.StreamWriter
}

func (ac *AESConn) Read(dst []byte) (n int, err error) {
	n, err = ac.r.Read(dst)
	return n, err
}

func (ac *AESConn) Write(src []byte) (n int, err error) {
	n, err = ac.w.Write(src)
	return n, err
}

func (ac *AESConn) Close() error {
	return ac.conn.Close()
}

func (ac *AESConn) LocalAddr() net.Addr {
	return ac.conn.LocalAddr()
}

func (ac *AESConn) RemoteAddr() net.Addr {
	return ac.conn.RemoteAddr()
}

func (ac *AESConn) SetDeadline(t time.Time) error {
	return ac.conn.SetDeadline(t)
}

func (ac *AESConn) SetReadDeadline(t time.Time) error {
	return ac.conn.SetReadDeadline(t)
}

func (ac *AESConn) SetWriteDeadline(t time.Time) error {
	return ac.conn.SetWriteDeadline(t)
}

func NewAESConn(key string, iv [aes.BlockSize]byte, conn net.Conn) (aesconn *AESConn, err error) {
	keybin := []byte(key)

	hashkey := sha256.Sum256(keybin)
	salt := sha256.Sum256([]byte(fmt.Sprintf("%x", key)))

	hashsalt := append(hashkey[:], salt[:]...)

	aeskey := sha256.Sum256(hashsalt)

	rblock, err := aes.NewCipher(aeskey[:])
	wblock, err := aes.NewCipher(aeskey[:])
	if err != nil {
		return nil, err
	}

	rstream := cipher.NewOFB(rblock, iv[:])
	wstream := cipher.NewOFB(wblock, iv[:])

	aesconn = &AESConn{
		conn: conn,
		r:    &cipher.StreamReader{S: rstream, R: conn},
		w:    &cipher.StreamWriter{S: wstream, W: conn},
	}
	return aesconn, err
}

type Listener struct {
	netl net.Listener
	key  string
}

func (l *Listener) Accept() (aesconn net.Conn, err error) {
	conn, err := l.netl.Accept()
	if err != nil {
		return nil, err
	}

	ivvector := make([]byte, IVLength)
	var ri = 0
	for ri < IVLength {
		conn.SetReadDeadline(time.Now().Add(30 * time.Microsecond))
		r, err := conn.Read(ivvector[ri:])
		if err != nil {
			return nil, err
		}
		ri += r
	}

	passwd := sha256.Sum256([]byte(l.key))
	longiv := append(passwd[:], ivvector...)

	hash := sha256.Sum256(longiv)

	var iv [aes.BlockSize]byte
	copy(iv[:], hash[:aes.BlockSize])

	aesconn, err = NewAESConn(l.key, iv, conn)

	return aesconn, err
}

func (l *Listener) Close() error {
	return l.netl.Close()
}

func (l *Listener) Addr() net.Addr {
	return l.netl.Addr()
}

func Listen(laddr string, key string) (l *Listener, err error) {
	netl, err := net.Listen("tcp", laddr)
	if err != nil {
		return nil, err
	}

	l = &Listener{
		netl: netl,
		key:  key,
	}
	return l, nil
}

func Dial(address string, key string) (aesconn net.Conn, err error) {
	conn, err := net.Dial("tcp", address)
	if err != nil {
		return nil, err
	}

	ivvector := make([]byte, IVLength)
	_, err = rand.Read(ivvector[:])

	if err != nil {
		return nil, err
	}

	var wi = 0
	for wi < IVLength {
		r, err := conn.Write(ivvector[wi:])
		if err != nil {
			return nil, err
		}
		wi += r
	}

	passwd := sha256.Sum256([]byte(key))
	longiv := append(passwd[:], ivvector...)

	hash := sha256.Sum256(longiv)

	var iv [aes.BlockSize]byte
	copy(iv[:], hash[:aes.BlockSize])

	aesconn, err = NewAESConn(key, iv, conn)
	return aesconn, err
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/estafette/estafette-ci-contracts"

	"github.com/rs/zerolog/log"
	"github.com/sethgrid/pester"
)

// EndOfLifeHelper has methods to shutdown the runner after a fatal or successful run
type EndOfLifeHelper interface {
	handleFatal(contracts.BuildLog, error, string)
	sendBuildFinishedEvent(buildStatus string) error
	sendBuildCleanEvent(buildStatus string) error
	sendBuildJobLogEvent(buildLog contracts.BuildLog) error
}

type endOfLifeHelperImpl struct {
	runAsJob bool
	config   contracts.BuilderConfig
}

// NewEndOfLifeHelper returns a new EndOfLifeHelper
func NewEndOfLifeHelper(runAsJob bool, config contracts.BuilderConfig) EndOfLifeHelper {
	return &endOfLifeHelperImpl{
		runAsJob: runAsJob,
		config:   config,
	}
}

func (elh *endOfLifeHelperImpl) handleFatal(buildLog contracts.BuildLog, err error, message string) {

	// add error messages as step to show in logs
	fatalStep := contracts.BuildLogStep{
		Step:     "init",
		LogLines: []contracts.BuildLogLine{},
		ExitCode: -1,
		Status:   "failed",
	}
	lineNumber := 1

	if err != nil {
		fatalStep.LogLines = append(fatalStep.LogLines, contracts.BuildLogLine{
			LineNumber: lineNumber,
			Timestamp:  time.Now().UTC(),
			StreamType: "stderr",
			Text:       err.Error(),
		})
		lineNumber++
	}
	if message != "" {
		fatalStep.LogLines = append(fatalStep.LogLines, contracts.BuildLogLine{
			LineNumber: lineNumber,
			Timestamp:  time.Now().UTC(),
			StreamType: "stderr",
			Text:       message,
		})
		lineNumber++
	}

	buildLog.Steps = append(buildLog.Steps, fatalStep)

	_ = elh.sendBuildJobLogEvent(buildLog)
	_ = elh.sendBuildFinishedEvent("failed")
	_ = elh.sendBuildCleanEvent("failed")

	if elh.runAsJob {
		log.Error().Err(err).Msg(message)
		os.Exit(0)
	} else {
		log.Fatal().Err(err).Msg(message)
	}
}

func (elh *endOfLifeHelperImpl) sendBuildJobLogEvent(buildLog contracts.BuildLog) (err error) {

	ciServerBuilderPostLogsURL := elh.config.CIServer.PostLogsURL
	ciAPIKey := elh.config.CIServer.APIKey
	jobName := *elh.config.JobName

	if ciServerBuilderPostLogsURL != "" && ciAPIKey != "" && jobName != "" {

		// convert BuildJobLogs to json
		var requestBody io.Reader

		var data []byte
		if *elh.config.Action == "release" {
			// copy buildLog to releaseLog and marshal that
			releaseLog := contracts.ReleaseLog{
				ID:         buildLog.ID,
				RepoSource: buildLog.RepoSource,
				RepoOwner:  buildLog.RepoOwner,
				RepoName:   buildLog.RepoName,
				ReleaseID:  strconv.Itoa(elh.config.ReleaseParams.ReleaseID),
				Steps:      buildLog.Steps,
				InsertedAt: buildLog.InsertedAt,
			}
			data, err = json.Marshal(releaseLog)
			if err != nil {
				log.Error().Err(err).Msgf("Failed marshalling ReleaseLog for job %v", jobName)
				return
			}
		} else {
			data, err = json.Marshal(buildLog)
			if err != nil {
				log.Error().Err(err).Msgf("Failed marshalling BuildLog for job %v", jobName)
				return
			}
		}

		requestBody = bytes.NewReader(data)

		// create client, in order to add headers
		client := pester.New()
		client.MaxRetries = 3
		client.Backoff = pester.ExponentialJitterBackoff
		client.KeepLog = true
		request, err := http.NewRequest("POST", ciServerBuilderPostLogsURL, requestBody)
		if err != nil {
			log.Error().Err(err).Msgf("Failed creating http client for job %v", jobName)
			return err
		}

		// add headers
		request.Header.Add("Authorization", fmt.Sprintf("Bearer %v", ciAPIKey))
		request.Header.Add("Content-Type", "application/json")

		// perform actual request
		response, err := client.Do(request)
		if err != nil {
			log.Error().Err(err).Str("logs", client.LogString()).Msgf("Failed performing http request to %v for job %v", ciServerBuilderPostLogsURL, jobName)
			return err
		}

		defer response.Body.Close()

		log.Debug().Str("logs", client.LogString()).Str("url", ciServerBuilderPostLogsURL).Msg("Sent ci-builder logs v2")
	}

	return nil
}

func (elh *endOfLifeHelperImpl) sendBuildFinishedEvent(buildStatus string) error {
	return elh.sendBuilderEvent(buildStatus, fmt.Sprintf("builder:%v", buildStatus))
}

func (elh *endOfLifeHelperImpl) sendBuildCleanEvent(buildStatus string) error {
	return elh.sendBuilderEvent(buildStatus, "builder:clean")
}

func (elh *endOfLifeHelperImpl) sendBuilderEvent(buildStatus, event string) (err error) {

	ciServerBuilderEventsURL := elh.config.CIServer.BuilderEventsURL
	ciAPIKey := elh.config.CIServer.APIKey
	jobName := *elh.config.JobName

	if ciServerBuilderEventsURL != "" && ciAPIKey != "" && jobName != "" {
		// convert EstafetteCiBuilderEvent to json
		var requestBody io.Reader

		buildID := ""
		if elh.config.BuildParams != nil {
			buildID = strconv.Itoa(elh.config.BuildParams.BuildID)
		}

		releaseID := ""
		if elh.config.ReleaseParams != nil {
			releaseID = strconv.Itoa(elh.config.ReleaseParams.ReleaseID)
		}

		ciBuilderEvent := EstafetteCiBuilderEvent{
			JobName:      jobName,
			RepoSource:   elh.config.Git.RepoSource,
			RepoOwner:    elh.config.Git.RepoOwner,
			RepoName:     elh.config.Git.RepoName,
			RepoBranch:   elh.config.Git.RepoBranch,
			RepoRevision: elh.config.Git.RepoRevision,
			ReleaseID:    releaseID,
			BuildID:      buildID,
			BuildStatus:  buildStatus,
		}
		data, err := json.Marshal(ciBuilderEvent)
		if err != nil {
			log.Error().Err(err).Msgf("Failed marshalling EstafetteCiBuilderEvent for job %v", jobName)
			return err
		}
		requestBody = bytes.NewReader(data)

		// create client, in order to add headers
		client := pester.New()
		client.MaxRetries = 3
		client.Backoff = pester.ExponentialJitterBackoff
		client.KeepLog = true
		request, err := http.NewRequest("POST", ciServerBuilderEventsURL, requestBody)
		if err != nil {
			log.Error().Err(err).Msgf("Failed creating http client for job %v", jobName)
			return err
		}

		// add headers
		request.Header.Add("X-Estafette-Event", event)
		request.Header.Add("X-Estafette-Event-Job-Name", jobName)
		request.Header.Add("Authorization", fmt.Sprintf("Bearer %v", ciAPIKey))

		// perform actual request
		response, err := client.Do(request)
		if err != nil {
			log.Error().Err(err).Str("logs", client.LogString()).Msgf("Failed performing http request to %v for job %v", ciServerBuilderEventsURL, jobName)
			return err
		}

		defer response.Body.Close()

		log.Debug().Str("logs", client.LogString()).Str("url", ciServerBuilderEventsURL).Msg("Notified ci-api that ci-builder has finished")
	}

	return nil
}
<commit_msg>update log message when finished sending event to api<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/estafette/estafette-ci-contracts"

	"github.com/rs/zerolog/log"
	"github.com/sethgrid/pester"
)

// EndOfLifeHelper has methods to shutdown the runner after a fatal or successful run
type EndOfLifeHelper interface {
	handleFatal(contracts.BuildLog, error, string)
	sendBuildFinishedEvent(buildStatus string) error
	sendBuildCleanEvent(buildStatus string) error
	sendBuildJobLogEvent(buildLog contracts.BuildLog) error
}

type endOfLifeHelperImpl struct {
	runAsJob bool
	config   contracts.BuilderConfig
}

// NewEndOfLifeHelper returns a new EndOfLifeHelper
func NewEndOfLifeHelper(runAsJob bool, config contracts.BuilderConfig) EndOfLifeHelper {
	return &endOfLifeHelperImpl{
		runAsJob: runAsJob,
		config:   config,
	}
}

func (elh *endOfLifeHelperImpl) handleFatal(buildLog contracts.BuildLog, err error, message string) {

	// add error messages as step to show in logs
	fatalStep := contracts.BuildLogStep{
		Step:     "init",
		LogLines: []contracts.BuildLogLine{},
		ExitCode: -1,
		Status:   "failed",
	}
	lineNumber := 1

	if err != nil {
		fatalStep.LogLines = append(fatalStep.LogLines, contracts.BuildLogLine{
			LineNumber: lineNumber,
			Timestamp:  time.Now().UTC(),
			StreamType: "stderr",
			Text:       err.Error(),
		})
		lineNumber++
	}
	if message != "" {
		fatalStep.LogLines = append(fatalStep.LogLines, contracts.BuildLogLine{
			LineNumber: lineNumber,
			Timestamp:  time.Now().UTC(),
			StreamType: "stderr",
			Text:       message,
		})
		lineNumber++
	}

	buildLog.Steps = append(buildLog.Steps, fatalStep)

	_ = elh.sendBuildJobLogEvent(buildLog)
	_ = elh.sendBuildFinishedEvent("failed")
	_ = elh.sendBuildCleanEvent("failed")

	if elh.runAsJob {
		log.Error().Err(err).Msg(message)
		os.Exit(0)
	} else {
		log.Fatal().Err(err).Msg(message)
	}
}

func (elh *endOfLifeHelperImpl) sendBuildJobLogEvent(buildLog contracts.BuildLog) (err error) {

	ciServerBuilderPostLogsURL := elh.config.CIServer.PostLogsURL
	ciAPIKey := elh.config.CIServer.APIKey
	jobName := *elh.config.JobName

	if ciServerBuilderPostLogsURL != "" && ciAPIKey != "" && jobName != "" {

		// convert BuildJobLogs to json
		var requestBody io.Reader

		var data []byte
		if *elh.config.Action == "release" {
			// copy buildLog to releaseLog and marshal that
			releaseLog := contracts.ReleaseLog{
				ID:         buildLog.ID,
				RepoSource: buildLog.RepoSource,
				RepoOwner:  buildLog.RepoOwner,
				RepoName:   buildLog.RepoName,
				ReleaseID:  strconv.Itoa(elh.config.ReleaseParams.ReleaseID),
				Steps:      buildLog.Steps,
				InsertedAt: buildLog.InsertedAt,
			}
			data, err = json.Marshal(releaseLog)
			if err != nil {
				log.Error().Err(err).Msgf("Failed marshalling ReleaseLog for job %v", jobName)
				return
			}
		} else {
			data, err = json.Marshal(buildLog)
			if err != nil {
				log.Error().Err(err).Msgf("Failed marshalling BuildLog for job %v", jobName)
				return
			}
		}

		requestBody = bytes.NewReader(data)

		// create client, in order to add headers
		client := pester.New()
		client.MaxRetries = 3
		client.Backoff = pester.ExponentialJitterBackoff
		client.KeepLog = true
		request, err := http.NewRequest("POST", ciServerBuilderPostLogsURL, requestBody)
		if err != nil {
			log.Error().Err(err).Msgf("Failed creating http client for job %v", jobName)
			return err
		}

		// add headers
		request.Header.Add("Authorization", fmt.Sprintf("Bearer %v", ciAPIKey))
		request.Header.Add("Content-Type", "application/json")

		// perform actual request
		response, err := client.Do(request)
		if err != nil {
			log.Error().Err(err).Str("logs", client.LogString()).Msgf("Failed performing http request to %v for job %v", ciServerBuilderPostLogsURL, jobName)
			return err
		}

		defer response.Body.Close()

		log.Debug().Str("logs", client.LogString()).Str("url", ciServerBuilderPostLogsURL).Msg("Sent ci-builder logs v2")
	}

	return nil
}

func (elh *endOfLifeHelperImpl) sendBuildFinishedEvent(buildStatus string) error {
	return elh.sendBuilderEvent(buildStatus, fmt.Sprintf("builder:%v", buildStatus))
}

func (elh *endOfLifeHelperImpl) sendBuildCleanEvent(buildStatus string) error {
	return elh.sendBuilderEvent(buildStatus, "builder:clean")
}

func (elh *endOfLifeHelperImpl) sendBuilderEvent(buildStatus, event string) (err error) {

	ciServerBuilderEventsURL := elh.config.CIServer.BuilderEventsURL
	ciAPIKey := elh.config.CIServer.APIKey
	jobName := *elh.config.JobName

	if ciServerBuilderEventsURL != "" && ciAPIKey != "" && jobName != "" {
		// convert EstafetteCiBuilderEvent to json
		var requestBody io.Reader

		buildID := ""
		if elh.config.BuildParams != nil {
			buildID = strconv.Itoa(elh.config.BuildParams.BuildID)
		}

		releaseID := ""
		if elh.config.ReleaseParams != nil {
			releaseID = strconv.Itoa(elh.config.ReleaseParams.ReleaseID)
		}

		ciBuilderEvent := EstafetteCiBuilderEvent{
			JobName:      jobName,
			RepoSource:   elh.config.Git.RepoSource,
			RepoOwner:    elh.config.Git.RepoOwner,
			RepoName:     elh.config.Git.RepoName,
			RepoBranch:   elh.config.Git.RepoBranch,
			RepoRevision: elh.config.Git.RepoRevision,
			ReleaseID:    releaseID,
			BuildID:      buildID,
			BuildStatus:  buildStatus,
		}
		data, err := json.Marshal(ciBuilderEvent)
		if err != nil {
			log.Error().Err(err).Msgf("Failed marshalling EstafetteCiBuilderEvent for job %v", jobName)
			return err
		}
		requestBody = bytes.NewReader(data)

		// create client, in order to add headers
		client := pester.New()
		client.MaxRetries = 3
		client.Backoff = pester.ExponentialJitterBackoff
		client.KeepLog = true
		request, err := http.NewRequest("POST", ciServerBuilderEventsURL, requestBody)
		if err != nil {
			log.Error().Err(err).Msgf("Failed creating http client for job %v", jobName)
			return err
		}

		// add headers
		request.Header.Add("X-Estafette-Event", event)
		request.Header.Add("X-Estafette-Event-Job-Name", jobName)
		request.Header.Add("Authorization", fmt.Sprintf("Bearer %v", ciAPIKey))

		// perform actual request
		response, err := client.Do(request)
		if err != nil {
			log.Error().Err(err).Str("logs", client.LogString()).Msgf("Failed performing http request to %v for job %v", ciServerBuilderEventsURL, jobName)
			return err
		}

		defer response.Body.Close()

		log.Debug().Str("url", ciServerBuilderEventsURL).Msgf("Succesfully sent %v event to api", event)
	}

	return nil
}
<|endoftext|>


<commit_before>// +build !windows,!nacl,!plan9

package main

import (
	"flag"
	"log"
	"log/syslog"
)

var noSyslog = flag.Bool("disable-syslog", false, "disables logging to syslog")

func init() {
	mains = append(mains, setSyslog)
}

func setSyslog() {
	if *noSyslog || *flagDev {
		return
	}
	w, err := syslog.New(syslog.LOG_LOCAL6, "bosun")
	if err != nil {
		log.Println("could not open syslog: %v", err)
		return
	}
	log.Println("enabling syslog")
	log.SetOutput(w)
}
<commit_msg>bosun: Set default log level of INFO for syslog_unix.<commit_after>// +build !windows,!nacl,!plan9

package main

import (
	"flag"
	"log"
	"log/syslog"
)

var noSyslog = flag.Bool("disable-syslog", false, "disables logging to syslog")

func init() {
	mains = append(mains, setSyslog)
}

func setSyslog() {
	if *noSyslog || *flagDev {
		return
	}
	w, err := syslog.New(syslog.LOG_LOCAL6 | syslog.LOG_INFO, "bosun")
	if err != nil {
		log.Println("could not open syslog: %v", err)
		return
	}
	log.Println("enabling syslog")
	log.SetOutput(w)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Frank Braun <frank@cryptogroup.net>
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// ccprices prints current currency prices in ledger format.
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"time"
)

var coins = []string{"BTC", "DCR", "NMC"}

type result struct {
	symbol string
	price  float64
	err    error
}

func getCoinPrice(ch chan<- *result, sym string) {
	resp, err := http.Get("http://coinmarketcap.northpole.ro/api/v5/" + sym + ".json")
	if err != nil {
		ch <- &result{err: err}
		return
	}
	b, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		ch <- &result{err: err}
		return
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		ch <- &result{err: err}
		return
	}
	s, _ := json.MarshalIndent(jsn, "", "  ")
	fmt.Println(string(s))
	f := jsn["price"].(map[string]interface{})["eur"].(string)
	p, err := strconv.ParseFloat(f, 64)
	if err != nil {
		ch <- &result{err: err}
		return
	}
	ch <- &result{symbol: sym, price: p}
}

func fatal(err error) {
	fmt.Fprintf(os.Stderr, "%s: error: %s\n", os.Args[0], err)
	os.Exit(1)
}

func main() {
	ch := make(chan *result)
	for _, sym := range coins {
		go getCoinPrice(ch, sym)
	}
	prices := make(map[string]float64)
	for range coins {
		r := <-ch
		if r.err != nil {
			fatal(r.err)
		}
		prices[r.symbol] = r.price
	}
	t := time.Now().Format("2006/01/02 15:04:05")
	for _, sym := range coins {
		fmt.Printf("P %s %s %7.2f EUR\n", t, sym, prices[sym])
	}
}
<commit_msg>make sure we get correct Decred price<commit_after>// Copyright (c) 2016 Frank Braun <frank@cryptogroup.net>
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// ccprices prints current currency prices in ledger format.
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"time"
)

var coins = []string{"Bitcoin", "Decred", "Namecoin"}

type result struct {
	symbol string
	price  float64
}

func getCoinPrices() ([]interface{}, error) {
	resp, err := http.Get("http://coinmarketcap.northpole.ro/api/v5/all.json")
	if err != nil {
		return nil, err
	}
	b, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn["markets"].([]interface{}), nil
}

func fatal(err error) {
	fmt.Fprintf(os.Stderr, "%s: error: %s\n", os.Args[0], err)
	os.Exit(1)
}

func main() {
	// get all coin prices
	all, err := getCoinPrices()
	if err != nil {
		fatal(err)
	}
	// construct map of coin names we are interested in
	names := make(map[string]struct{})
	for _, name := range coins {
		names[name] = struct{}{}
	}
	prices := make(map[string]*result)
	// iterate over all coin informations
	for _, info := range all {
		coin := info.(map[string]interface{})
		name := coin["name"].(string)
		_, ok := names[name]
		if ok {
			// we are interested in this coin -> store price and symbol
			f := coin["price"].(map[string]interface{})["eur"].(string)
			p, err := strconv.ParseFloat(f, 64)
			if err != nil {
				fatal(err)
			}
			prices[name] = &result{symbol: coin["symbol"].(string), price: p}
		}
	}
	// output coin prices
	t := time.Now().Format("2006/01/02 15:04:05")
	for _, name := range coins {
		fmt.Printf("P %s %s %7.2f EUR\n", t, prices[name].symbol,
			prices[name].price)
	}
}
<|endoftext|>


<commit_before>package command

import (
	"bufio"
	"flag"
	"fmt"
	"strings"

	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/pipestream"
)

type Ext4fs struct {
	Ui  cli.Ui
	Cmd string
}

func (this *Ext4fs) Run(args []string) (exitCode int) {
	var noJournal string
	cmdFlags := flag.NewFlagSet("ext4fs", flag.ContinueOnError)
	cmdFlags.StringVar(&noJournal, "nojournal", "", "")
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if noJournal {
		this.genNoJournalScript(noJournal)
		return
	}

	this.Ui.Output(fmt.Sprintf(`
dumpe2fs /dev/XXX | grep 'Filesystem features' | grep 'has_journal'

mount -t ext4 -o remount,rw,barrier=0,commit=60,noatime,nodiratime  /dev/XXX /dataYYY

umount /dataYYY
tune2fs -O ^has_journal /dev/XXX
fsck.ext4 -f /dev/XXX
mount -t ext4 -o rw,barrier=0,commit=60,noatime,nodiratime /dev/XXX /dataYYY
		`))

	return
}

func (this *Ext4fs) genNoJournalScript(mountPointPattern string) {
	cmd := pipestream.New("df")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)

	fileSystems := make(map[string]string) // dev:mountPoint
	for scanner.Scan() {
		tuples := strings.Fields(scanner.Text())
		if !patternMatched(tuples[5], mountPointPattern) {
			continue
		}

		fileSystems[tuples[0]] = tuples[5]
	}
	swallow(scanner.Err())

	for dev, mp := range fileSystems {
		this.Ui.Output(fmt.Sprintf(`umount %s
tune2fs -O ^has_journal %s
fsck.ext4 -f %s`, mp, dev, dev))
	}

}

func (*Ext4fs) Synopsis() string {
	return "Tune ext4 file system for localhost"
}

func (this *Ext4fs) Help() string {
	help := fmt.Sprintf(`
Usage: %s ext4 [options]

    %s

Options:

    -nojournal mount point pattern
      Generate script to turn off journal feature for ext4

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<commit_msg>fix compile err<commit_after>package command

import (
	"bufio"
	"flag"
	"fmt"
	"strings"

	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/pipestream"
)

type Ext4fs struct {
	Ui  cli.Ui
	Cmd string
}

func (this *Ext4fs) Run(args []string) (exitCode int) {
	var noJournal string
	cmdFlags := flag.NewFlagSet("ext4fs", flag.ContinueOnError)
	cmdFlags.StringVar(&noJournal, "nojournal", "", "")
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if noJournal != "" {
		this.genNoJournalScript(noJournal)
		return
	}

	this.Ui.Output(fmt.Sprintf(`
dumpe2fs /dev/XXX | grep 'Filesystem features' | grep 'has_journal'

mount -t ext4 -o remount,rw,barrier=0,commit=60,noatime,nodiratime  /dev/XXX /dataYYY

umount /dataYYY
tune2fs -O ^has_journal /dev/XXX
fsck.ext4 -f /dev/XXX
mount -t ext4 -o rw,barrier=0,commit=60,noatime,nodiratime /dev/XXX /dataYYY
		`))

	return
}

func (this *Ext4fs) genNoJournalScript(mountPointPattern string) {
	cmd := pipestream.New("df")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)

	fileSystems := make(map[string]string) // dev:mountPoint
	for scanner.Scan() {
		tuples := strings.Fields(scanner.Text())
		if !patternMatched(tuples[5], mountPointPattern) {
			continue
		}

		fileSystems[tuples[0]] = tuples[5]
	}
	swallow(scanner.Err())

	for dev, mp := range fileSystems {
		this.Ui.Output(fmt.Sprintf(`umount %s
tune2fs -O ^has_journal %s
fsck.ext4 -f %s`, mp, dev, dev))
	}

}

func (*Ext4fs) Synopsis() string {
	return "Tune ext4 file system for localhost"
}

func (this *Ext4fs) Help() string {
	help := fmt.Sprintf(`
Usage: %s ext4 [options]

    %s

Options:

    -nojournal mount point pattern
      Generate script to turn off journal feature for ext4

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/grafana/metrictank/clock"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stats"
	log "github.com/sirupsen/logrus"
	"math"
	"net/http"
	"strconv"
	"time"
)

var (
	httpError    = stats.NewCounter32("parrot.monitoring.error;error=http")
	decodeError = stats.NewCounter32("parrot.monitoring.error;error=decode")
	invalidError = stats.NewCounter32("parrot.monitoring.error;error=invalid")
)

type seriesStats struct {
	lastTs uint32
	//the partition currently being checked
	nans int32
	//the sum of abs(value - ts) across the time series
	deltaSum float64
	//the number of timestamps where value != ts
	numNonMatching int32
	//tracks the last seen non-NaN time stamp (useful for lag
	lastSeen uint32
}

type partitionMetrics struct {
	//number of missing values for each series
	nanCount *stats.Gauge32
	//time since the last value was recorded
	lag *stats.Gauge32
	//total amount of drift between expected value and actual values
	deltaSum *stats.Gauge32
	//total number of entries where drift occurred
	nonMatching *stats.Gauge32
}

func monitor() {
	metricsBySeries := initMetricsBySeries()
	for tick := range clock.AlignedTickLossless(queryInterval) {

		query := graphite.ExecuteRenderQuery(buildRequest(tick))
		if query.HTTPErr != nil {
			httpError.Inc()
		}
		if query.DecodeErr != nil {
			decodeError.Inc()
		}

		for _, s := range query.Decoded {
			log.Infof("%d - %d", s.Datapoints[0].Ts, s.Datapoints[len(s.Datapoints)-1].Ts)
			partition, err := strconv.Atoi(s.Target)
			if err != nil {
				log.Debug("unable to parse partition", err)
				invalidError.Inc()
				continue
			}
			serStats := seriesStats{}
			serStats.lastTs = s.Datapoints[len(s.Datapoints)-1].Ts

			for _, dp := range s.Datapoints {

				if math.IsNaN(dp.Val) {
					serStats.nans += 1
					continue
				}
				serStats.lastSeen = dp.Ts
				if diff := dp.Val - float64(dp.Ts); diff != 0 {
					log.Debugf("partition=%d dp.Val=%f dp.Ts=%d diff=%f", partition, dp.Val, dp.Ts, diff)
					serStats.deltaSum += diff
					serStats.numNonMatching += 1
				}
			}

			metrics := metricsBySeries[partition]
			metrics.nanCount.Set(int(serStats.nans))
			metrics.lag.Set(int(serStats.lastTs - serStats.lastSeen))
			metrics.deltaSum.Set(int(serStats.deltaSum))
			metrics.nonMatching.Set(int(serStats.numNonMatching))
		}
	}
}

func initMetricsBySeries() []partitionMetrics {
	var metricsBySeries []partitionMetrics
	for p := 0; p < int(partitionCount); p++ {
		metrics := partitionMetrics{
			nanCount:    stats.NewGauge32(fmt.Sprintf("parrot.monitoring.nancount;partition=%d", p)),
			lag:         stats.NewGauge32(fmt.Sprintf("parrot.monitoring.lag;partition=%d", p)),
			deltaSum:    stats.NewGauge32(fmt.Sprintf("parrot.monitoring.deltaSum;partition=%d", p)),
			nonMatching: stats.NewGauge32(fmt.Sprintf("parrot.monitoring.nonMatching;partition=%d", p)),
		}
		metricsBySeries = append(metricsBySeries, metrics)
	}
	return metricsBySeries
}

func buildRequest(now time.Time) *http.Request {
	req, _ := http.NewRequest("GET", fmt.Sprintf("%s/render", gatewayAddress), nil)
	q := req.URL.Query()
	q.Set("target", "aliasByNode(parrot.testdata.*.generated.*, 2)")
	q.Set("from", strconv.Itoa(int(now.Add(-5*time.Minute).Unix())))
	q.Set("until", strconv.Itoa(int(now.Unix())))
	q.Set("format", "json")
	q.Set("X-Org-Id", strconv.Itoa(orgId))
	req.URL.RawQuery = q.Encode()
	if len(gatewayKey) != 0 {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", gatewayKey))
	}
	return req
}
<commit_msg>refactor, pull series metrics gathering into a function<commit_after>package main

import (
	"fmt"
	"github.com/grafana/metrictank/clock"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stats"
	log "github.com/sirupsen/logrus"
	"math"
	"net/http"
	"strconv"
	"time"
)

var (
	httpError    = stats.NewCounter32("parrot.monitoring.error;error=http")
	decodeError  = stats.NewCounter32("parrot.monitoring.error;error=decode")
	invalidError = stats.NewCounter32("parrot.monitoring.error;error=invalid")
)

var metricsBySeries []partitionMetrics

type seriesStats struct {
	lastTs uint32
	//the partition currently being checked
	nans int32
	//the sum of abs(value - ts) across the time series
	deltaSum float64
	//the number of timestamps where value != ts
	numNonMatching int32
	//tracks the last seen non-NaN time stamp (useful for lag
	lastSeen uint32
}

type partitionMetrics struct {
	//number of missing values for each series
	nanCount *stats.Gauge32
	//time since the last value was recorded
	lag *stats.Gauge32
	//total amount of drift between expected value and actual values
	deltaSum *stats.Gauge32
	//total number of entries where drift occurred
	nonMatching *stats.Gauge32
}

func monitor() {
	initMetricsBySeries()
	for tick := range clock.AlignedTickLossless(queryInterval) {

		query := graphite.ExecuteRenderQuery(buildRequest(tick))
		if query.HTTPErr != nil {
			httpError.Inc()
		}
		if query.DecodeErr != nil {
			decodeError.Inc()
		}

		for _, s := range query.Decoded {
			processPartitionSeries(s)
		}
	}
}

func processPartitionSeries(s graphite.Series) {
	log.Infof("%d - %d", s.Datapoints[0].Ts, s.Datapoints[len(s.Datapoints)-1].Ts)
	partition, err := strconv.Atoi(s.Target)
	if err != nil {
		log.Debug("unable to parse partition", err)
		invalidError.Inc()
		return
	}
	serStats := seriesStats{}
	serStats.lastTs = s.Datapoints[len(s.Datapoints)-1].Ts

	for _, dp := range s.Datapoints {

		if math.IsNaN(dp.Val) {
			serStats.nans += 1
			continue
		}
		serStats.lastSeen = dp.Ts
		if diff := dp.Val - float64(dp.Ts); diff != 0 {
			log.Debugf("partition=%d dp.Val=%f dp.Ts=%d diff=%f", partition, dp.Val, dp.Ts, diff)
			serStats.deltaSum += diff
			serStats.numNonMatching += 1
		}
	}

	metrics := metricsBySeries[partition]
	metrics.nanCount.Set(int(serStats.nans))
	metrics.lag.Set(int(serStats.lastTs - serStats.lastSeen))
	metrics.deltaSum.Set(int(serStats.deltaSum))
	metrics.nonMatching.Set(int(serStats.numNonMatching))
}

func initMetricsBySeries() {
	for p := 0; p < int(partitionCount); p++ {
		metrics := partitionMetrics{
			nanCount:    stats.NewGauge32(fmt.Sprintf("parrot.monitoring.nancount;partition=%d", p)),
			lag:         stats.NewGauge32(fmt.Sprintf("parrot.monitoring.lag;partition=%d", p)),
			deltaSum:    stats.NewGauge32(fmt.Sprintf("parrot.monitoring.deltaSum;partition=%d", p)),
			nonMatching: stats.NewGauge32(fmt.Sprintf("parrot.monitoring.nonMatching;partition=%d", p)),
		}
		metricsBySeries = append(metricsBySeries, metrics)
	}
}

func buildRequest(now time.Time) *http.Request {
	req, _ := http.NewRequest("GET", fmt.Sprintf("%s/render", gatewayAddress), nil)
	q := req.URL.Query()
	q.Set("target", "aliasByNode(parrot.testdata.*.generated.*, 2)")
	q.Set("from", strconv.Itoa(int(now.Add(-5*time.Minute).Unix())))
	q.Set("until", strconv.Itoa(int(now.Unix())))
	q.Set("format", "json")
	q.Set("X-Org-Id", strconv.Itoa(orgId))
	req.URL.RawQuery = q.Encode()
	if len(gatewayKey) != 0 {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", gatewayKey))
	}
	return req
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"

	"github.com/scjalliance/resourceful/lease"
	"github.com/scjalliance/resourceful/lease/leaseutil"
	"github.com/scjalliance/resourceful/policy"
)

// A StatRecipient is capable of receiving resource statistics.
type StatRecipient interface {
	SendResource(resource string, stats ResourceStats) error
	SendUser(resource, user string, count uint, t time.Time) error
}

// ResourceStatsMap holds resource statistics for a set of resources.
type ResourceStatsMap map[string]ResourceStats

// ResourceStats hold statistics for a particular resource
type ResourceStats struct {
	Time     time.Time
	Consumed uint
	Limit    uint
	Active   uint
	Released uint
	Queued   uint
	Users    map[string]uint
}

// StatManager manages resources statistics.
type StatManager struct {
	recipient StatRecipient
	last      ResourceStatsMap // The last set of statistics that were collected
}

// NewStatManager returns a new statistics manager.
func NewStatManager(r StatRecipient) *StatManager {
	return &StatManager{recipient: r}
}

// Init initializes the stat manager.
func (m *StatManager) Init(polProv policy.Provider, leaseProv lease.Provider) error {
	stats, err := m.collect(polProv, leaseProv, false)
	if err != nil {
		return err
	}
	m.last = stats
	return nil
}

// CollectAndSend collects statistics from the given providers and sends them
// to the manager's stat recipient.
func (m *StatManager) CollectAndSend(polProv policy.Provider, leaseProv lease.Provider) error {
	// Collect the current values
	current, err := m.collect(polProv, leaseProv, true)
	if err != nil {
		return err
	}

	// Any resources or users that are no longer present need to have a final
	// set of zeroed values sent
	for resource, last := range m.last {
		if current, exists := current[resource]; !exists {
			removal := ResourceStats{
				Limit: last.Limit,
				Users: make(map[string]uint, len(last.Users)),
			}
			for user := range last.Users {
				removal.Users[user] = 0
			}
			if err := m.recipient.SendResource(resource, removal); err != nil {
				return fmt.Errorf("failed to remove expired stats for %s: %v", resource, err)
			}
		} else {
			for user, count := range last.Users {
				if count == 0 {
					// Already zeroed in the last submission
					continue
				}
				if _, found := current.Users[user]; found {
					// User is still active
					continue
				}
				current.Users[user] = 0
			}
		}
	}

	// Any users that were previously absent need to have a zero value sent
	// prior to the current value
	for resource, current := range current {
		if last, exists := m.last[resource]; exists {
			for user := range current.Users {
				if _, found := last.Users[user]; found {
					continue // Not previously absent
				}
				m.recipient.SendUser(resource, user, 0, current.Time.Add(-time.Minute))
			}
		}
	}

	// Send the current values
	for resource, stats := range current {
		if err := m.recipient.SendResource(resource, stats); err != nil {
			return fmt.Errorf("failed to send stats for %s: %v", resource, err)
		}
	}

	m.last = current

	return nil
}

func (m *StatManager) collect(polProv policy.Provider, leaseProv lease.Provider, refresh bool) (stats map[string]ResourceStats, err error) {
	policies, err := polProv.Policies()
	if err != nil {
		return nil, err
	}

	resources, err := leaseProv.LeaseResources()
	if err != nil {
		return nil, err
	}

	stats = make(map[string]ResourceStats, len(resources))

	for _, resource := range resources {
		// Collect policy settings for the resource
		policies := policies.MatchResource(resource)
		strat := policies.Strategy()
		limit := policies.Limit()

		// Collect a view of the current lease set
		now := time.Now()
		revision, leases, err := leaseProv.LeaseView(resource)
		if err != nil {
			return nil, err
		}

		if refresh {
			// Purge expired leases
			tx := lease.NewTx(resource, revision, leases)
			leaseutil.Refresh(tx, now)

			// Use the refreshed lease set
			leases = tx.Leases()
		}

		// Collect statistics
		data := leases.Stats()

		// Translate users into user account names
		users := make(map[string]uint)
		for user, count := range data.Users(strat) {
			if props := leases.User(user).Property("user.account"); len(props) > 0 {
				name := props[0]
				users[name] = count
			} else {
				users[user] = count
			}
		}

		// Include the assembled stats
		stats[resource] = ResourceStats{
			Time:     now,
			Consumed: data.Consumed(strat),
			Limit:    limit,
			Active:   data.Active(strat),
			Released: data.Released(strat),
			Queued:   data.Queued(strat),
			Users:    users,
		}
	}

	return stats, nil
}
<commit_msg>cmd: server: Bugfix for user stat zeroing<commit_after>package main

import (
	"fmt"
	"time"

	"github.com/scjalliance/resourceful/lease"
	"github.com/scjalliance/resourceful/lease/leaseutil"
	"github.com/scjalliance/resourceful/policy"
)

// A StatRecipient is capable of receiving resource statistics.
type StatRecipient interface {
	SendResource(resource string, stats ResourceStats) error
	SendUser(resource, user string, count uint, t time.Time) error
}

// ResourceStatsMap holds resource statistics for a set of resources.
type ResourceStatsMap map[string]ResourceStats

// ResourceStats hold statistics for a particular resource
type ResourceStats struct {
	Time     time.Time
	Consumed uint
	Limit    uint
	Active   uint
	Released uint
	Queued   uint
	Users    UserStatsMap
}

// UserStatsMap holds resource statistics for a user.
type UserStatsMap map[string]uint

// Contains returns true if m contains user.
func (m UserStatsMap) Contains(user string) bool {
	if m == nil {
		return false
	}
	_, found := m[user]
	return found
}

// StatManager manages resources statistics.
type StatManager struct {
	recipient StatRecipient
	last      ResourceStatsMap // The last set of statistics that were collected
}

// NewStatManager returns a new statistics manager.
func NewStatManager(r StatRecipient) *StatManager {
	return &StatManager{recipient: r}
}

// Init initializes the stat manager.
func (m *StatManager) Init(polProv policy.Provider, leaseProv lease.Provider) error {
	stats, err := m.collect(polProv, leaseProv, false)
	if err != nil {
		return err
	}
	m.last = stats
	return nil
}

// CollectAndSend collects statistics from the given providers and sends them
// to the manager's stat recipient.
func (m *StatManager) CollectAndSend(polProv policy.Provider, leaseProv lease.Provider) error {
	// Collect the current values
	current, err := m.collect(polProv, leaseProv, true)
	if err != nil {
		return err
	}

	// Any resources or users that are no longer present need to have a final
	// set of zeroed values sent
	for resource, last := range m.last {
		if current, exists := current[resource]; !exists {
			removal := ResourceStats{
				Limit: last.Limit,
				Users: make(UserStatsMap, len(last.Users)),
			}
			for user := range last.Users {
				removal.Users[user] = 0
			}
			if err := m.recipient.SendResource(resource, removal); err != nil {
				return fmt.Errorf("failed to remove expired stats for %s: %v", resource, err)
			}
		} else {
			for user, count := range last.Users {
				if count == 0 {
					// Already zeroed in the last submission
					continue
				}
				if _, found := current.Users[user]; found {
					// User is still active
					continue
				}
				current.Users[user] = 0
			}
		}
	}

	// Any users that were previously absent need to have a zero value sent
	// prior to the current value
	for resource, current := range current {
		last, exists := m.last[resource]
		for user := range current.Users {
			if exists && last.Users.Contains(user) {
				continue // Not previously absent
			}
			m.recipient.SendUser(resource, user, 0, current.Time.Add(-time.Minute))
		}
	}

	// Send the current values
	for resource, stats := range current {
		if err := m.recipient.SendResource(resource, stats); err != nil {
			return fmt.Errorf("failed to send stats for %s: %v", resource, err)
		}
	}

	m.last = current

	return nil
}

func (m *StatManager) collect(polProv policy.Provider, leaseProv lease.Provider, refresh bool) (stats ResourceStatsMap, err error) {
	policies, err := polProv.Policies()
	if err != nil {
		return nil, err
	}

	resources, err := leaseProv.LeaseResources()
	if err != nil {
		return nil, err
	}

	stats = make(ResourceStatsMap, len(resources))

	for _, resource := range resources {
		// Collect policy settings for the resource
		policies := policies.MatchResource(resource)
		strat := policies.Strategy()
		limit := policies.Limit()

		// Collect a view of the current lease set
		now := time.Now()
		revision, leases, err := leaseProv.LeaseView(resource)
		if err != nil {
			return nil, err
		}

		if refresh {
			// Purge expired leases
			tx := lease.NewTx(resource, revision, leases)
			leaseutil.Refresh(tx, now)

			// Use the refreshed lease set
			leases = tx.Leases()
		}

		// Collect statistics
		data := leases.Stats()

		// Translate users into user account names
		users := make(UserStatsMap)
		for user, count := range data.Users(strat) {
			if props := leases.User(user).Property("user.account"); len(props) > 0 {
				name := props[0]
				users[name] = count
			} else {
				users[user] = count
			}
		}

		// Include the assembled stats
		stats[resource] = ResourceStats{
			Time:     now,
			Consumed: data.Consumed(strat),
			Limit:    limit,
			Active:   data.Active(strat),
			Released: data.Released(strat),
			Queued:   data.Queued(strat),
			Users:    users,
		}
	}

	return stats, nil
}
<|endoftext|>