<commit_before>package channel

import (
	log "github.com/Sirupsen/logrus"
	"github.com/fatih/structs"
	"github.com/jaksi/sshesame/request"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/terminal"
	"io"
	"net"
)

// RFC 4254
type x11 struct {
	SourceAddress string
	SourcePort    uint32
}
type tcpip struct {
	DestinationAddress string
	DestinationPort    uint32
	SourceAddress      string
	SourcePort         uint32
}

func Handle(remoteAddr net.Addr, newChannel ssh.NewChannel) {
	payload := map[string]interface{}{
		"data": newChannel.ExtraData(),
	}
	switch newChannel.ChannelType() {
	case "x11":
		parsedPayload := x11{}
		err := ssh.Unmarshal(newChannel.ExtraData(), &parsedPayload)
		if err != nil {
			log.Warning("Failed to parse payload:", err.Error())
			break
		}
		payload = structs.Map(parsedPayload)
	case "forwarded-tcpip":
		// Server initiated forwarding
		fallthrough
	case "direct-tcpip":
		// Client initiated forwarding
		parsedPayload := tcpip{}
		err := ssh.Unmarshal(newChannel.ExtraData(), &parsedPayload)
		if err != nil {
			log.Warning("Failed to parse payload:", err.Error())
			break
		}
		payload = structs.Map(parsedPayload)
	}
	logData := payload
	logData["client"] = remoteAddr
	logData["channel"] = newChannel.ChannelType()
	log.WithFields(logData).Info("Channel requested")
	channel, channelRequests, err := newChannel.Accept()
	if err != nil {
		log.Warning("Failed to accept channel:", err.Error())
		return
	}
	defer channel.Close()
	go request.Handle(remoteAddr, newChannel.ChannelType(), channelRequests)
	if newChannel.ChannelType() == "session" {
		terminal := terminal.NewTerminal(channel, "$ ")
		for {
			line, err := terminal.ReadLine()
			if err != nil {
				if err == io.EOF {
					log.WithFields(log.Fields{
						"client":  remoteAddr,
						"channel": newChannel.ChannelType(),
					}).Info("Terminal closed")
					request.SendExitStatus(channel)
				} else {
					log.Warning("Failed to read from terminal:", err.Error())
				}
				break
			}
			log.WithFields(log.Fields{
				"client":  remoteAddr,
				"channel": newChannel.ChannelType(),
				"line":    line,
			}).Info("Channel input received")
		}
	} else {
		data := make([]byte, 256)
		for {
			length, err := channel.Read(data)
			if err != nil {
				if err == io.EOF {
					log.WithFields(log.Fields{
						"client":  remoteAddr,
						"channel": newChannel.ChannelType(),
					}).Info("Channel closed")
					request.SendExitStatus(channel)
				} else {
					log.Warning("Failed to read from channel:", err.Error())
				}
				break
			}
			log.WithFields(log.Fields{
				"client":  remoteAddr,
				"channel": newChannel.ChannelType(),
				"data":    string(data[:length]),
			}).Info("Channel input received")
		}
	}
}
<commit_msg>Don't send exit status requests on channels with types other than session<commit_after>package channel

import (
	log "github.com/Sirupsen/logrus"
	"github.com/fatih/structs"
	"github.com/jaksi/sshesame/request"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/terminal"
	"io"
	"net"
)

// RFC 4254
type x11 struct {
	SourceAddress string
	SourcePort    uint32
}
type tcpip struct {
	DestinationAddress string
	DestinationPort    uint32
	SourceAddress      string
	SourcePort         uint32
}

func Handle(remoteAddr net.Addr, newChannel ssh.NewChannel) {
	payload := map[string]interface{}{
		"data": newChannel.ExtraData(),
	}
	switch newChannel.ChannelType() {
	case "x11":
		parsedPayload := x11{}
		err := ssh.Unmarshal(newChannel.ExtraData(), &parsedPayload)
		if err != nil {
			log.Warning("Failed to parse payload:", err.Error())
			break
		}
		payload = structs.Map(parsedPayload)
	case "forwarded-tcpip":
		// Server initiated forwarding
		fallthrough
	case "direct-tcpip":
		// Client initiated forwarding
		parsedPayload := tcpip{}
		err := ssh.Unmarshal(newChannel.ExtraData(), &parsedPayload)
		if err != nil {
			log.Warning("Failed to parse payload:", err.Error())
			break
		}
		payload = structs.Map(parsedPayload)
	}
	logData := payload
	logData["client"] = remoteAddr
	logData["channel"] = newChannel.ChannelType()
	log.WithFields(logData).Info("Channel requested")
	channel, channelRequests, err := newChannel.Accept()
	if err != nil {
		log.Warning("Failed to accept channel:", err.Error())
		return
	}
	defer channel.Close()
	go request.Handle(remoteAddr, newChannel.ChannelType(), channelRequests)
	if newChannel.ChannelType() == "session" {
		terminal := terminal.NewTerminal(channel, "$ ")
		for {
			line, err := terminal.ReadLine()
			if err != nil {
				if err == io.EOF {
					log.WithFields(log.Fields{
						"client":  remoteAddr,
						"channel": newChannel.ChannelType(),
					}).Info("Terminal closed")
					request.SendExitStatus(channel)
				} else {
					log.Warning("Failed to read from terminal:", err.Error())
				}
				break
			}
			log.WithFields(log.Fields{
				"client":  remoteAddr,
				"channel": newChannel.ChannelType(),
				"line":    line,
			}).Info("Channel input received")
		}
	} else {
		data := make([]byte, 256)
		for {
			length, err := channel.Read(data)
			if err != nil {
				if err == io.EOF {
					log.WithFields(log.Fields{
						"client":  remoteAddr,
						"channel": newChannel.ChannelType(),
					}).Info("Channel closed")
				} else {
					log.Warning("Failed to read from channel:", err.Error())
				}
				break
			}
			log.WithFields(log.Fields{
				"client":  remoteAddr,
				"channel": newChannel.ChannelType(),
				"data":    string(data[:length]),
			}).Info("Channel input received")
		}
	}
}
<|endoftext|>


<commit_before>package global

import (
	"bytes"
	"fmt"
	"log"
	"time"
)

const (
	NumLatestLogEntries = 384  // Keep this number of latest log entries in memory
	MaxLogMessageLen    = 1024 // Truncate long log messages to this length
)

var LatestLogs = NewRingBuffer(NumLatestLogEntries)     // Keep latest log entry of all kinds in the buffer
var LatestWarnings = NewRingBuffer(NumLatestLogEntries) // Keep latest warnings and log entries that come with error in the buffer

// Help to write log messages in a regular format.
type Logger struct {
	ComponentName string // Component name, such as HTTPD, SMTPD.
	ComponentID   string // Clue of which component instance the message comes from, e.g. "0.0.0.0:25"
}

// Format a log message and return, but do not print it.
func (logger *Logger) Format(functionName, actorName string, err error, template string, values ...interface{}) string {
	// Message is going to look like this:
	// ComponentName[ID].FunctionName(actorName): Error "no such file" - failed to start component
	var msg bytes.Buffer
	if logger.ComponentName != "" {
		msg.WriteString(logger.ComponentName)
	}
	if logger.ComponentID != "" {
		msg.WriteString(fmt.Sprintf("[%s]", logger.ComponentID))
	}
	if msg.Len() > 0 {
		msg.WriteRune('.')
	}
	if functionName != "" {
		msg.WriteString(fmt.Sprintf("%s", functionName))
	}
	if actorName != "" {
		msg.WriteString(fmt.Sprintf("(%s)", actorName))
	}
	if msg.Len() > 0 {
		msg.WriteString(": ")
	}
	if err != nil {
		msg.WriteString(fmt.Sprintf("Error \"%v\" - ", err))
	}
	msg.WriteString(fmt.Sprintf(template, values...))
	if msg.Len() > MaxLogMessageLen {
		msg.Truncate(MaxLogMessageLen)
	}
	return msg.String()
}

// Print a log message and keep the message in warnings buffer.
func (logger *Logger) Warningf(functionName, actorName string, err error, template string, values ...interface{}) {
	msg := logger.Format(functionName, actorName, err, template, values...)
	msg = time.Now().Format("2006-01-02 15:04:05 ") + msg
	LatestLogs.Push(msg)
	LatestWarnings.Push(msg)
	log.Print(msg)
}

// Print a log message and keep the message in latest log buffer. If there is an error, also keep the message in warnings buffer.
func (logger *Logger) Printf(functionName, actorName string, err error, template string, values ...interface{}) {
	msg := logger.Format(functionName, actorName, err, template, values...)
	msg = time.Now().Format("2006-01-02 15:04:05 ") + msg
	LatestLogs.Push(msg)
	if err != nil {
		LatestWarnings.Push(msg)
	}
	log.Print(msg)
}

func (logger *Logger) Fatalf(functionName, actorName string, err error, template string, values ...interface{}) {
	log.Fatal(logger.Format(functionName, actorName, err, template, values...))
}

func (logger *Logger) Panicf(functionName, actorName string, err error, template string, values ...interface{}) {
	log.Panic(logger.Format(functionName, actorName, err, template, values...))
}
<commit_msg>do not double print log message timestamp<commit_after>package global

import (
	"bytes"
	"fmt"
	"log"
	"time"
)

const (
	NumLatestLogEntries = 384  // Keep this number of latest log entries in memory
	MaxLogMessageLen    = 1024 // Truncate long log messages to this length
)

var LatestLogs = NewRingBuffer(NumLatestLogEntries)     // Keep latest log entry of all kinds in the buffer
var LatestWarnings = NewRingBuffer(NumLatestLogEntries) // Keep latest warnings and log entries that come with error in the buffer

// Help to write log messages in a regular format.
type Logger struct {
	ComponentName string // Component name, such as HTTPD, SMTPD.
	ComponentID   string // Clue of which component instance the message comes from, e.g. "0.0.0.0:25"
}

// Format a log message and return, but do not print it.
func (logger *Logger) Format(functionName, actorName string, err error, template string, values ...interface{}) string {
	// Message is going to look like this:
	// ComponentName[ID].FunctionName(actorName): Error "no such file" - failed to start component
	var msg bytes.Buffer
	if logger.ComponentName != "" {
		msg.WriteString(logger.ComponentName)
	}
	if logger.ComponentID != "" {
		msg.WriteString(fmt.Sprintf("[%s]", logger.ComponentID))
	}
	if msg.Len() > 0 {
		msg.WriteRune('.')
	}
	if functionName != "" {
		msg.WriteString(fmt.Sprintf("%s", functionName))
	}
	if actorName != "" {
		msg.WriteString(fmt.Sprintf("(%s)", actorName))
	}
	if msg.Len() > 0 {
		msg.WriteString(": ")
	}
	if err != nil {
		msg.WriteString(fmt.Sprintf("Error \"%v\" - ", err))
	}
	msg.WriteString(fmt.Sprintf(template, values...))
	if msg.Len() > MaxLogMessageLen {
		msg.Truncate(MaxLogMessageLen)
	}
	return msg.String()
}

// Print a log message and keep the message in warnings buffer.
func (logger *Logger) Warningf(functionName, actorName string, err error, template string, values ...interface{}) {
	msg := logger.Format(functionName, actorName, err, template, values...)
	msgWithTime := time.Now().Format("2006-01-02 15:04:05 ") + msg
	LatestLogs.Push(msgWithTime)
	LatestWarnings.Push(msgWithTime)
	log.Print(msg)
}

// Print a log message and keep the message in latest log buffer. If there is an error, also keep the message in warnings buffer.
func (logger *Logger) Printf(functionName, actorName string, err error, template string, values ...interface{}) {
	msg := logger.Format(functionName, actorName, err, template, values...)
	msgWithTime := time.Now().Format("2006-01-02 15:04:05 ") + msg
	LatestLogs.Push(msgWithTime)
	if err != nil {
		LatestWarnings.Push(msgWithTime)
	}
	log.Print(msg)
}

func (logger *Logger) Fatalf(functionName, actorName string, err error, template string, values ...interface{}) {
	log.Fatal(logger.Format(functionName, actorName, err, template, values...))
}

func (logger *Logger) Panicf(functionName, actorName string, err error, template string, values ...interface{}) {
	log.Panic(logger.Format(functionName, actorName, err, template, values...))
}
<|endoftext|>


<commit_before>// Hellofs implements a simple "hello world" file system.
package main

import (
	"bytes"
	"flag"
	"log"
	"os"
	"path/filepath"
	"time"

	"golang.org/x/net/context"

	proto "rpc/project2/proto"

	"google.golang.org/grpc"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	_ "bazil.org/fuse/fs/fstestutil"
)

func NewGrpcClient(ip string) proto.BasicFileServiceClient {
	conn, err := grpc.Dial(ip, grpc.WithInsecure())
	if err != nil {
		log.Fatalf("could not connect to %v: %v", ip, err)
	}
	//defer conn.Close()
	c := proto.NewBasicFileServiceClient(conn)
	return c
}

func recover(path string, info os.FileInfo, err error) error {
	log.Println(path, err, "need recover")
	return nil
}

func diagnose(cachePath string) error {
	log.Printf("Checking cache folder: %v...", cachePath)
	err := filepath.Walk(cachePath, recover)
	log.Println("Recovery check finished")
	return err
}

func main() {
	var mountPoint, cachePath, ip string
	flag.StringVar(&mountPoint, "m", "root", "path you want to mount AFS")
	flag.StringVar(&cachePath, "c", "cache", "cache files for write operations")
	flag.StringVar(&ip, "server", "localhost:61512", "server address")
	flag.Parse()

	c, err := fuse.Mount(
		mountPoint,
		fuse.FSName("helloworld"),
		fuse.Subtype("hellofs"),
		fuse.LocalVolume(),
		fuse.VolumeName("Hello world!"),
	)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()
	diagnose(cachePath)
	log.Printf("MountPath: %s, CachePath: %s\n", mountPoint, cachePath)

	err = fs.Serve(c, &MyFS{
		Client: NewGrpcClient(ip),
		Files:  make(map[string]proto.File)})
	if err != nil {
		log.Fatal(err)
	}

	// check if the mount process has an error to report
	<-c.Ready
	if err := c.MountError; err != nil {
		log.Fatal(err)
	}
}

// FS implements the hello world file system.
type MyFS struct {
	Client proto.BasicFileServiceClient
	Files  map[string]proto.File
}

func (f *MyFS) Root() (fs.Node, error) {
	return &Dir{Name: "", Client: f.Client}, nil
}

// Dir implements both Node and Handle for the root directory.
type Dir struct {
	Name   string
	Client proto.BasicFileServiceClient
}

func MyGetAttr(Client proto.BasicFileServiceClient, name string, a *fuse.Attr) error {
	log.Println("MyGetAttr:", name)
	//a.Inode = 2
	path := proto.Path{Data: name}
	fileInfo, err := Client.GetFileInfo(context.Background(), &path)
	if err != nil {
		log.Println(err)
		return nil
	}

	if name == "" { // hack for server did not return mode
		a.Mode = os.ModeDir | 0644
	} else {
		a.Mode = 0644
	}
	a.Size = fileInfo.Size
	a.Atime = time.Unix(int64(fileInfo.AccessTime), 0)
	a.Mtime = time.Unix(int64(fileInfo.ModificationTime), 0)
	a.Ctime = time.Unix(int64(fileInfo.CreationTime), 0)

	return nil
}

func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) error {
	log.Println("DirAttr:", d.Name)
	return MyGetAttr(d.Client, d.Name, a)
}

func (d *Dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
	log.Println("Lookup:", name)
	return &File{Name: name, Client: d.Client}, nil
}

var dirDirs = []fuse.Dirent{
	{Inode: 2, Name: "hello", Type: fuse.DT_File},
}

func (d *Dir) ReadDirAll(ctx context.Context) (c []fuse.Dirent, err error) {
	log.Println("ReadDirAll:", d.Name)
	path := proto.Path{Data: d.Name}
	dirInfo, err := d.Client.GetDirectoryContents(context.Background(), &path)

	if err != nil {
		log.Fatal(err)
	}
	for _, fileInfo := range dirInfo.Contents {
		//log.Println(name, fileInfo)
		c = append(c, fuse.Dirent{Name: fileInfo})
	}
	return c, err
}

//func (d *Dir) ReadDirAll(ctx context.Context) (c []fuse.Dirent, err error) {
//	log.Println(ctx, d)
//	return dirDirs, nil
//}

// File implements both Node and Handle for the hello file.
type File struct {
	Name   string
	Client proto.BasicFileServiceClient
}

const greeting = "hello, world\n"

func (f *File) Attr(ctx context.Context, a *fuse.Attr) error {
	log.Println("FileAttr:", f.Name)
	return MyGetAttr(f.Client, f.Name, a)
}

func (f *File) ReadAll(ctx context.Context) ([]byte, error) {
	log.Println("FileReadAll:", f.Name)
	path := proto.Path{Data: f.Name}
	myfile, err := f.Client.DownloadFile(context.Background(), &path)
	if err != nil {
		log.Println(err)
		return nil, fuse.ENOENT
	}

	var buf bytes.Buffer
	for _, d := range myfile.Contents {
		buf.WriteString(d)
	}
	return buf.Bytes(), nil
}

func (f *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {
	log.Println("FileWrite", f.Name)
	var in proto.FileData
	in.Path = &proto.Path{Data: f.Name}
	content := string(req.Data)
	in.Contents = append(in.Contents, content)
	fileInfo, err := f.Client.UploadFile(context.Background(), &in)
	if err != nil {
		log.Println(err)
		return err
	}
	resp.Size = int(fileInfo.Size)
	return nil
}

//func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
//	log.Println("FileOpen", f.Name)
//	return nil, nil
//}

func (f *File) Flush(ctx context.Context, req *fuse.FlushRequest) error {
	log.Println("FileFlush", f.Name)
	return nil
}
<commit_msg>finish read write<commit_after>// Hellofs implements a simple "hello world" file system.
package main

import (
	"bytes"
	"flag"
	"log"
	"os"
	"path/filepath"
	"time"

	"golang.org/x/net/context"

	proto "rpc/project2/proto"

	"google.golang.org/grpc"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	_ "bazil.org/fuse/fs/fstestutil"
)

func NewGrpcClient(ip string) proto.BasicFileServiceClient {
	conn, err := grpc.Dial(ip, grpc.WithInsecure())
	if err != nil {
		log.Fatalf("could not connect to %v: %v", ip, err)
	}
	//defer conn.Close()
	c := proto.NewBasicFileServiceClient(conn)
	return c
}

func recover(path string, info os.FileInfo, err error) error {
	log.Println(path, err, "need recover")
	return nil
}

func diagnose(cachePath string) error {
	log.Printf("Checking cache folder: %v...", cachePath)
	err := filepath.Walk(cachePath, recover)
	log.Println("Recovery check finished")
	return err
}

func main() {
	var mountPoint, cachePath, ip string
	flag.StringVar(&mountPoint, "m", "root", "path you want to mount AFS")
	flag.StringVar(&cachePath, "c", "cache", "cache files for write operations")
	flag.StringVar(&ip, "server", "localhost:61512", "server address")
	flag.Parse()

	c, err := fuse.Mount(
		mountPoint,
		fuse.FSName("helloworld"),
		fuse.Subtype("hellofs"),
		fuse.LocalVolume(),
		fuse.VolumeName("Hello world!"),
	)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()
	diagnose(cachePath)
	log.Printf("MountPath: %s, CachePath: %s\n", mountPoint, cachePath)

	err = fs.Serve(c, &MyFS{
		Client:     NewGrpcClient(ip),
		FileCaches: make(map[string]*MyFile)})
	if err != nil {
		log.Fatal(err)
	}

	// check if the mount process has an error to report
	<-c.Ready
	if err := c.MountError; err != nil {
		log.Fatal(err)
	}
}

// FS implements the hello world file system.
type MyFS struct {
	Client     proto.BasicFileServiceClient
	FileCaches map[string]*MyFile
}

func (f *MyFS) Root() (fs.Node, error) {
	return &Dir{Name: "", fs: f}, nil
}

// Dir implements both Node and Handle for the root directory.
type Dir struct {
	Name string
	fs   *MyFS
}

func MyGetAttr(Client proto.BasicFileServiceClient, name string, a *fuse.Attr) error {
	log.Println("MyGetAttr:", name)
	//a.Inode = 2
	path := proto.Path{Data: name}
	fileInfo, err := Client.GetFileInfo(context.Background(), &path)
	if err != nil {
		log.Println(err)
		return nil
	}

	if name == "" { // hack for server did not return mode
		a.Mode = os.ModeDir | 0644
	} else {
		a.Mode = 0644
	}
	a.Size = fileInfo.Size
	a.Atime = time.Unix(int64(fileInfo.AccessTime), 0)
	a.Mtime = time.Unix(int64(fileInfo.ModificationTime), 0)
	a.Ctime = time.Unix(int64(fileInfo.CreationTime), 0)

	return nil
}

func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) error {
	log.Println("DirAttr:", d.Name)
	return MyGetAttr(d.fs.Client, d.Name, a)
}

func (d *Dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
	log.Println("Lookup:", name)
	return &MyFile{Name: name, fs: d.fs}, nil
}

var dirDirs = []fuse.Dirent{
	{Inode: 2, Name: "hello", Type: fuse.DT_File},
}

func (d *Dir) ReadDirAll(ctx context.Context) (c []fuse.Dirent, err error) {
	log.Println("ReadDirAll:", d.Name)
	path := proto.Path{Data: d.Name}
	dirInfo, err := d.fs.Client.GetDirectoryContents(context.Background(), &path)

	if err != nil {
		log.Fatal(err)
	}
	for _, fileInfo := range dirInfo.Contents {
		//log.Println(name, fileInfo)
		c = append(c, fuse.Dirent{Name: fileInfo})
	}
	return c, err
}

//func (d *Dir) ReadDirAll(ctx context.Context) (c []fuse.Dirent, err error) {
//	log.Println(ctx, d)
//	return dirDirs, nil
//}

// File implements both Node and Handle for the hello file.
type MyFile struct {
	*proto.File
	Name string
	fs   *MyFS
	buf  bytes.Buffer
}

const greeting = "hello, world\n"

func (f *MyFile) Attr(ctx context.Context, a *fuse.Attr) error {
	log.Println("FileAttr:", f.Name)
	return MyGetAttr(f.fs.Client, f.Name, a)
}

// only for opened file or directory
func (f *MyFile) ReadAll(ctx context.Context) ([]byte, error) {
	log.Println("FileReadAll:", f.Name, f.buf.Len())
	return f.buf.Bytes(), nil
}

func (f *MyFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
	log.Println("FileRead:", f.Name)
	return nil
}

// only for opened file or directory
func (f *MyFile) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {
	log.Println("FileWrite", f.Name, len(req.Data))
	b := f.buf.Bytes()
	tmp := append(b[:req.Offset], req.Data...)
	if int(req.Offset)+len(req.Data) < len(b) {
		tmp = append(tmp, b[int(req.Offset)+len(req.Data)+1:]...)
	}
	b = tmp
	resp.Size = len(b)
	return nil
}

func (f *MyFile) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
	log.Println("FileOpen", f.Name)
	path := proto.Path{Data: f.Name}
	myfile, err := f.fs.Client.DownloadFile(context.Background(), &path)
	if err != nil {
		log.Println(err)
		return nil, fuse.ENOENT
	}

	fileCache := &MyFile{File: myfile, Name: f.Name, fs: f.fs}
	for _, d := range myfile.Contents {
		fileCache.buf.WriteString(d)
	}
	f.fs.FileCaches[f.Name] = fileCache
	return fileCache, nil
}

func (f *MyFile) Flush(ctx context.Context, req *fuse.FlushRequest) error {
	log.Println("FileFlush", f.Name, f.buf.Len())
	var in proto.FileData
	in.Path = &proto.Path{Data: f.Name}
	in.Contents = append(in.Contents, f.buf.String())
	_, err := f.fs.Client.UploadFile(context.Background(), &in)
	if err != nil {
		log.Println(err)
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2019 The OpenSDS Authors.
//
//    Licensed under the Apache License, Version 2.0 (the "License"); you may
//    not use this file except in compliance with the License. You may obtain
//    a copy of the License at
//
//         http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
//    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
//    License for the specific language governing permissions and limitations
//    under the License.
package lvm

import (
	"regexp"
	"strings"

	log "github.com/golang/glog"
	"github.com/opensds/opensds/pkg/utils/exec"
)

const (
	sarNotEnabledOut = "Please check if data collecting is enabled"
	cmdNotFound      = "No such file or directory"
)

type MetricCli struct {
	// Command executer
	BaseExecuter exec.Executer
	// Command Root executer
	RootExecuter exec.Executer
}

func NewMetricCli() (*MetricCli, error) {
	return &MetricCli{
		BaseExecuter: exec.NewBaseExecuter(),
		RootExecuter: exec.NewRootExecuter(),
	}, nil
}

func (c *MetricCli) execute(cmd ...string) (string, error) {
	return c.RootExecuter.Run(cmd[0], cmd[1:]...)
}

func isSarEnabled(out string) bool {

	if strings.Contains(string(out), sarNotEnabledOut) || strings.Contains(string(out), cmdNotFound) {

		return false
	}

	return true
}

// Function to parse sar and iostat command output
// metricList -> metrics to be collected
// metricMap	-> metric to command output column mapping
// out 		-> command output
// returnMap	-> metric to value map to be returned
func (c *MetricCli) parseCommandOutput(metricList []string, returnMap map[string]map[string]string, labelMap map[string]map[string]string, metricMap map[string]int, out string) {

	tableRows := strings.Split(string(out), "\n")
	for _, row := range tableRows[3:] {
		if row != "" {
			tokens := regexp.MustCompile(" ")
			cols := tokens.Split(row, -1)
			// remove all empty space
			var columns = make([]string, 0, 0)
			for _, v := range cols {
				if v != "" {
					columns = append(columns, v)
				}
			}
			// map the values
			aVolMap := make(map[string]string)
			aLabMap := make(map[string]string)
			for _, metric := range metricList {
				val := columns[metricMap[metric]]
				aVolMap[metric] = val
				aVolMap["InstanceName"] = columns[metricMap["InstanceID"]]
				aLabMap["device"] = columns[metricMap["InstanceID"]]
			}
			returnMap[columns[1]] = aVolMap
			labelMap[columns[1]] = aLabMap
		}
	}

}

// CollectMetrics function is to call the cli for metrics collection. This will be invoked  by lvm metric driver
// metricList	-> metrics to be collected
// returnMap	-> metrics to value map
func (cli *MetricCli) CollectMetrics(metricList []string) ( /*returnMAp*/ map[string]map[string]string /*labelMap*/, map[string]map[string]string, error) {

	returnMap := make(map[string]map[string]string)
	labelMap := make(map[string]map[string]string)
	var err error

	cmd := []string{"env", "LC_ALL=C", "sar", "-dp", "1", "1"}
	out, err := cli.execute(cmd...)
	if err != nil {
		log.Errorf("cmd.Run() failed with %s\n", err)
		err = nil

	}
	//check whether sar collection is enabled ?
	//If not use iostat command
	if isSarEnabled(out) {
		// sar command output mapping
		metricMap := make(map[string]int)
		metricMap["InstanceID"] = 1
		metricMap["iops"] = 2
		metricMap["read_throughput"] = 3
		metricMap["write_throughput"] = 4
		metricMap["response_time"] = 7
		metricMap["service_time"] = 8
		metricMap["utilization_prcnt"] = 9
		//call parser
		cli.parseCommandOutput(metricList, returnMap, labelMap, metricMap, out)
	} else {
		cmd := []string{"env", "LC_ALL=C", "iostat", "-N"}
		out, err := cli.execute(cmd...)
		if strings.Contains(string(out), cmdNotFound) {
			log.Errorf("iostat is not available: cmd.Run() failed with %s\n", err)
			err = nil
		} else if err != nil {
			log.Errorf("cmd.Run() failed with %s\n", err)
			return nil, nil, err
		}
		metricMap := make(map[string]int)
		// iostat command output mapping
		metricMap["iops"] = 1
		metricMap["read_throughput"] = 2
		metricMap["write_throughput"] = 3
		cli.parseCommandOutput(metricList, returnMap, labelMap, metricMap, out)

	}
	return returnMap, labelMap, err
}

// Discover LVM volumes
func (c *MetricCli) DiscoverVolumes() ([]string, error) {
	cmd := []string{"env", "LC_ALL=C", "lvs"}
	out, err := c.execute(cmd...)
	tableRows := strings.Split(string(out), "\n")
	var volumes []string
	for _, row := range tableRows[1:] {
		if row != "" {
			tokens := regexp.MustCompile(" ")
			cols := tokens.Split(row, -1)
			var columns = make([]string, 0, 0)
			for _, v := range cols {
				if v != "" {
					columns = append(columns, v)
				}
			}
			volumes = append(volumes, columns[0])

		}
	}
	return volumes, err
}

// Discover LVM Disks
func (c *MetricCli) DiscoverDisks() ([]string, error) {
	cmd := []string{"env", "LC_ALL=C", "pvs"}
	out, err := c.execute(cmd...)
	tableRows := strings.Split(string(out), "\n")
	var volumes []string
	for _, row := range tableRows[1:] {
		if row != "" {
			tokens := regexp.MustCompile(" ")
			cols := tokens.Split(row, -1)
			var columns = make([]string, 0, 0)
			for _, v := range cols {
				if v != "" {
					columns = append(columns, v)
				}
			}
			volumes = append(volumes, columns[0])

		}
	}
	return volumes, err
}
<commit_msg> systat not installed bug fix (#783)<commit_after>// Copyright (c) 2019 The OpenSDS Authors.
//
//    Licensed under the Apache License, Version 2.0 (the "License"); you may
//    not use this file except in compliance with the License. You may obtain
//    a copy of the License at
//
//         http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
//    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
//    License for the specific language governing permissions and limitations
//    under the License.
package lvm

import (
	"regexp"
	"strings"

	log "github.com/golang/glog"
	"github.com/opensds/opensds/pkg/utils/exec"
)

const (
	sarNotEnabledOut = "Please check if data collecting is enabled"
	cmdNotFound      = "No such file or directory"
	sarNotFound      = "Command 'sar' not found"
	iostatNotFound   = "Command 'iostat' not found"
)

type MetricCli struct {
	// Command executer
	BaseExecuter exec.Executer
	// Command Root executer
	RootExecuter exec.Executer
}

func NewMetricCli() (*MetricCli, error) {
	return &MetricCli{
		BaseExecuter: exec.NewBaseExecuter(),
		RootExecuter: exec.NewRootExecuter(),
	}, nil
}

func (c *MetricCli) execute(cmd ...string) (string, error) {
	return c.RootExecuter.Run(cmd[0], cmd[1:]...)
}

func isSarEnabled(out string) bool {

	if strings.Contains(string(out), sarNotEnabledOut) || strings.Contains(string(out), cmdNotFound) || strings.Contains(string(out), sarNotFound) {

		return false
	}

	return true
}

// Function to parse sar and iostat command output
// metricList -> metrics to be collected
// metricMap	-> metric to command output column mapping
// out 		-> command output
// returnMap	-> metric to value map to be returned
func (c *MetricCli) parseCommandOutput(metricList []string, returnMap map[string]map[string]string, labelMap map[string]map[string]string, metricMap map[string]int, out string) {

	tableRows := strings.Split(string(out), "\n")
	for _, row := range tableRows[3:] {
		if row != "" {
			tokens := regexp.MustCompile(" ")
			cols := tokens.Split(row, -1)
			// remove all empty space
			var columns = make([]string, 0, 0)
			for _, v := range cols {
				if v != "" {
					columns = append(columns, v)
				}
			}
			// map the values
			aVolMap := make(map[string]string)
			aLabMap := make(map[string]string)
			for _, metric := range metricList {
				val := columns[metricMap[metric]]
				aVolMap[metric] = val
				aVolMap["InstanceName"] = columns[metricMap["InstanceID"]]
				aLabMap["device"] = columns[metricMap["InstanceID"]]
			}
			returnMap[columns[1]] = aVolMap
			labelMap[columns[1]] = aLabMap
		}
	}

}

// CollectMetrics function is to call the cli for metrics collection. This will be invoked  by lvm metric driver
// metricList	-> metrics to be collected
// returnMap	-> metrics to value map
func (cli *MetricCli) CollectMetrics(metricList []string) ( /*returnMAp*/ map[string]map[string]string /*labelMap*/, map[string]map[string]string, error) {

	returnMap := make(map[string]map[string]string)
	labelMap := make(map[string]map[string]string)
	var err error

	cmd := []string{"env", "LC_ALL=C", "sar", "-dp", "1", "1"}
	out, err := cli.execute(cmd...)
	if err != nil {
		log.Errorf("cmd.Run() failed with %s\n", err)
		err = nil

	}
	//check whether sar collection is enabled ?
	//If not use iostat command
	if isSarEnabled(out) {
		// sar command output mapping
		metricMap := make(map[string]int)
		metricMap["InstanceID"] = 1
		metricMap["iops"] = 2
		metricMap["read_throughput"] = 3
		metricMap["write_throughput"] = 4
		metricMap["response_time"] = 7
		metricMap["service_time"] = 8
		metricMap["utilization_prcnt"] = 9
		//call parser
		cli.parseCommandOutput(metricList, returnMap, labelMap, metricMap, out)
	} else {
		cmd := []string{"env", "LC_ALL=C", "iostat", "-N"}
		out, err := cli.execute(cmd...)
		if strings.Contains(string(out), cmdNotFound) || strings.Contains(string(out), iostatNotFound) {
			log.Errorf("iostat is not available: cmd.Run() failed with %s\n", err)
			return nil, nil, err
		} else if err != nil {
			log.Errorf("cmd.Run() failed with %s\n", err)
			return nil, nil, err
		}
		metricMap := make(map[string]int)
		// iostat command output mapping
		metricMap["iops"] = 1
		metricMap["read_throughput"] = 2
		metricMap["write_throughput"] = 3
		cli.parseCommandOutput(metricList, returnMap, labelMap, metricMap, out)

	}
	return returnMap, labelMap, err
}

// Discover LVM volumes
func (c *MetricCli) DiscoverVolumes() ([]string, error) {
	cmd := []string{"env", "LC_ALL=C", "lvs"}
	out, err := c.execute(cmd...)
	tableRows := strings.Split(string(out), "\n")
	var volumes []string
	for _, row := range tableRows[1:] {
		if row != "" {
			tokens := regexp.MustCompile(" ")
			cols := tokens.Split(row, -1)
			var columns = make([]string, 0, 0)
			for _, v := range cols {
				if v != "" {
					columns = append(columns, v)
				}
			}
			volumes = append(volumes, columns[0])

		}
	}
	return volumes, err
}

// Discover LVM Disks
func (c *MetricCli) DiscoverDisks() ([]string, error) {
	cmd := []string{"env", "LC_ALL=C", "pvs"}
	out, err := c.execute(cmd...)
	tableRows := strings.Split(string(out), "\n")
	var volumes []string
	for _, row := range tableRows[1:] {
		if row != "" {
			tokens := regexp.MustCompile(" ")
			cols := tokens.Split(row, -1)
			var columns = make([]string, 0, 0)
			for _, v := range cols {
				if v != "" {
					columns = append(columns, v)
				}
			}
			volumes = append(volumes, columns[0])

		}
	}
	return volumes, err
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2019 IBM, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy ofthe License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specificlanguage governing permissions and
 * limitations under the License.
 *
 */

package core

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/spf13/viper"

	"github.com/IBM/ibm-cos-sdk-go/aws"

	"github.com/skydive-project/skydive/logging"
)

// stream is a series of consecutive persisted objects
type stream struct {
	// ID holds the time when the stream was created
	ID time.Time
	// SeqNumber holds the next available sequence number of this stream
	SeqNumber int
}

// StoreS3 allows writing flows to an object storage service
type StoreS3 struct {
	bucket            string
	objectPrefix      string
	currentStream     map[Tag]stream
	maxFlowsPerObject int
	maxFlowArraySize  int
	maxStreamDuration time.Duration
	maxObjectDuration time.Duration
	client            objectStoreClient
	pipeline          *Pipeline
	flows             map[Tag][]interface{}
	lastFlushTime     map[Tag]time.Time
	flushTimers       map[Tag]*time.Timer
}

// SetPipeline setup
func (s *StoreS3) SetPipeline(pipeline *Pipeline) {
	s.pipeline = pipeline
}

// ListObjects lists all stored objects
func (s *StoreS3) ListObjects() ([]*string, error) {
	objectKeys, err := s.client.ListObjects(s.bucket, s.objectPrefix)
	if err != nil {
		logging.GetLogger().Error("Failed to list objects: ", err)
		return nil, err
	}

	return objectKeys, nil
}

// ReadObjectFlows reads flows from object
func (s *StoreS3) ReadObjectFlows(objectKey *string, objectFlows interface{}) error {
	objectBytes, err := s.client.ReadObject(s.bucket, *objectKey)
	if err != nil {
		logging.GetLogger().Error("Failed to read object: ", err)
		return err
	}

	if err = json.Unmarshal(objectBytes, objectFlows); err != nil {
		logging.GetLogger().Error("Failed to JSON-decode object: ", err)
		return err
	}

	return nil
}

// DeleteObject deletes an object
func (s *StoreS3) DeleteObject(objectKey *string) error {
	if err := s.client.DeleteObject(s.bucket, *objectKey); err != nil {
		logging.GetLogger().Error("Failed to delete object: ", err)
		return err
	}
	return nil
}

// StoreFlows store flows in memory, before being written to the object store
func (s *StoreS3) StoreFlows(in map[Tag][]interface{}) error {
	endTime := time.Now()

	s.flows = in

	// check which flows needs to be flushed to the object store
	flushedTags := make(map[Tag]bool)
	tagsToTimerReset := make(map[Tag]bool)
	for t := range s.flows {
		if _, ok := s.lastFlushTime[t]; !ok {
			s.lastFlushTime[t] = endTime
			tagsToTimerReset[t] = true
		}
		for {
			flowsLength := len(s.flows[t])
			if flowsLength == 0 {
				break
			}
			if flowsLength < s.maxFlowsPerObject && time.Since(s.lastFlushTime[t]) < s.maxObjectDuration {
				break
			}

			if s.flushFlowsToObject(t, endTime) != nil {
				break
			}

			flushedTags[t] = true
			tagsToTimerReset[t] = true
		}

		// check if need to trim flow array
		overflowCount := len(s.flows[t]) - s.maxFlowArraySize
		if overflowCount > 0 {
			logging.GetLogger().Warningf("Flow array for tag '%s' overflowed maximum size. Discarding %d flows", string(t), overflowCount)
			s.flows[t] = s.flows[t][overflowCount:]
		}
	}

	for t := range flushedTags {
		// copy slice to free up memory of already flushed flows
		newArray := make([]interface{}, 0, len(s.flows[t]))
		copy(newArray, s.flows[t])
		s.flows[t] = newArray
	}

	for t := range tagsToTimerReset {
		// stop old flush timers
		oldTimer, ok := s.flushTimers[t]
		if ok {
			oldTimer.Stop()
		}
	}

	// set a timer for next flush
	if s.maxObjectDuration > 0 {
		for t := range tagsToTimerReset {
			s.flushTimers[t] = time.AfterFunc(s.maxObjectDuration-time.Since(endTime), func() {
				s.StoreFlows(nil)
			})

			// a single timer will cover all of the flushed tags
			break
		}
	}

	return nil
}

func (s *StoreS3) flushFlowsToObject(t Tag, endTime time.Time) error {
	flows := s.flows[t]
	if len(flows) == 0 {
		return nil
	}

	if len(flows) > s.maxFlowsPerObject {
		flows = flows[:s.maxFlowsPerObject]
	}

	startTime := s.lastFlushTime[t]
	startTimeString := strconv.FormatInt(startTime.UTC().UnixNano()/int64(time.Millisecond), 10)
	endTimeString := strconv.FormatInt(endTime.UTC().UnixNano()/int64(time.Millisecond), 10)

	metadata := map[string]*string{
		"first-timestamp": aws.String(startTimeString),
		"last-timestamp":  aws.String(endTimeString),
		"num-records":     aws.String(strconv.Itoa(len(flows))),
	}

	currentStream := s.currentStream[t]
	if endTime.Sub(currentStream.ID) >= s.maxStreamDuration {
		currentStream = stream{ID: endTime}
	}

	encodedFlows, err := s.pipeline.Encoder.Encode(flows)
	if err != nil {
		logging.GetLogger().Error("Failed to encode object: ", err)
		return err
	}

	b, err := s.pipeline.Compressor.Compress(encodedFlows)
	if err != nil {
		logging.GetLogger().Error("Failed to compress object: ", err)
		return err
	}

	objectKey := strings.Join([]string{s.objectPrefix, string(t), currentStream.ID.UTC().Format("20060102T150405Z"), fmt.Sprintf("%08d.gz", currentStream.SeqNumber)}, "/")
	err = s.client.WriteObject(s.bucket, objectKey, string(b.Bytes()), "application/json", "gzip", metadata)

	if err != nil {
		logging.GetLogger().Error("Failed to write object: ", err)
		return err
	}

	// flush was successful, update state
	s.lastFlushTime[t] = endTime
	currentStream.SeqNumber++
	s.currentStream[t] = currentStream
	s.flows[t] = s.flows[t][len(flows):]

	return nil
}

// NewStoreS3 returns a new storage interface for storing flows to object store
func NewStoreS3(cfg *viper.Viper) (interface{}, error) {
	bucket := cfg.GetString(CfgRoot + "store.s3.bucket")
	objectPrefix := cfg.GetString(CfgRoot + "store.s3.object_prefix")
	maxFlowArraySize := cfg.GetInt(CfgRoot + "store.s3.max_flow_array_size")
	maxFlowsPerObject := cfg.GetInt(CfgRoot + "store.s3.max_flows_per_object")
	maxSecondsPerObject := cfg.GetInt(CfgRoot + "store.s3.max_seconds_per_object")
	maxSecondsPerStream := cfg.GetInt(CfgRoot + "store.s3.max_seconds_per_stream")

	client := newClient(cfg)

	store := &StoreS3{
		bucket:            bucket,
		objectPrefix:      objectPrefix,
		maxFlowsPerObject: maxFlowsPerObject,
		maxFlowArraySize:  maxFlowArraySize,
		maxObjectDuration: time.Second * time.Duration(maxSecondsPerObject),
		maxStreamDuration: time.Second * time.Duration(maxSecondsPerStream),
		client:            client,
		currentStream:     make(map[Tag]stream),
		flows:             make(map[Tag][]interface{}),
		lastFlushTime:     make(map[Tag]time.Time),
		flushTimers:       make(map[Tag]*time.Timer),
	}

	return store, nil
}
<commit_msg>exporters/core: StoreS3: Serialize access to StoreFlows<commit_after>/*
 * Copyright (C) 2019 IBM, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy ofthe License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specificlanguage governing permissions and
 * limitations under the License.
 *
 */

package core

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/spf13/viper"

	"github.com/IBM/ibm-cos-sdk-go/aws"

	"github.com/skydive-project/skydive/logging"
)

// stream is a series of consecutive persisted objects
type stream struct {
	// ID holds the time when the stream was created
	ID time.Time
	// SeqNumber holds the next available sequence number of this stream
	SeqNumber int
}

// StoreS3 allows writing flows to an object storage service
type StoreS3 struct {
	bucket            string
	objectPrefix      string
	currentStream     map[Tag]stream
	maxFlowsPerObject int
	maxFlowArraySize  int
	maxStreamDuration time.Duration
	maxObjectDuration time.Duration
	client            objectStoreClient
	pipeline          *Pipeline
	flows             map[Tag][]interface{}
	lastFlushTime     map[Tag]time.Time
	flushTimers       map[Tag]*time.Timer
	flowsMutex        sync.Mutex
}

// SetPipeline setup
func (s *StoreS3) SetPipeline(pipeline *Pipeline) {
	s.pipeline = pipeline
}

// ListObjects lists all stored objects
func (s *StoreS3) ListObjects() ([]*string, error) {
	objectKeys, err := s.client.ListObjects(s.bucket, s.objectPrefix)
	if err != nil {
		logging.GetLogger().Error("Failed to list objects: ", err)
		return nil, err
	}

	return objectKeys, nil
}

// ReadObjectFlows reads flows from object
func (s *StoreS3) ReadObjectFlows(objectKey *string, objectFlows interface{}) error {
	objectBytes, err := s.client.ReadObject(s.bucket, *objectKey)
	if err != nil {
		logging.GetLogger().Error("Failed to read object: ", err)
		return err
	}

	if err = json.Unmarshal(objectBytes, objectFlows); err != nil {
		logging.GetLogger().Error("Failed to JSON-decode object: ", err)
		return err
	}

	return nil
}

// DeleteObject deletes an object
func (s *StoreS3) DeleteObject(objectKey *string) error {
	if err := s.client.DeleteObject(s.bucket, *objectKey); err != nil {
		logging.GetLogger().Error("Failed to delete object: ", err)
		return err
	}
	return nil
}

// StoreFlows store flows in memory, before being written to the object store
func (s *StoreS3) StoreFlows(in map[Tag][]interface{}) error {
	s.flowsMutex.Lock()
	defer s.flowsMutex.Unlock()

	endTime := time.Now()

	s.flows = in

	// check which flows needs to be flushed to the object store
	flushedTags := make(map[Tag]bool)
	tagsToTimerReset := make(map[Tag]bool)
	for t := range s.flows {
		if _, ok := s.lastFlushTime[t]; !ok {
			s.lastFlushTime[t] = endTime
			tagsToTimerReset[t] = true
		}
		for {
			flowsLength := len(s.flows[t])
			if flowsLength == 0 {
				break
			}
			if flowsLength < s.maxFlowsPerObject && time.Since(s.lastFlushTime[t]) < s.maxObjectDuration {
				break
			}

			if s.flushFlowsToObject(t, endTime) != nil {
				break
			}

			flushedTags[t] = true
			tagsToTimerReset[t] = true
		}

		// check if need to trim flow array
		overflowCount := len(s.flows[t]) - s.maxFlowArraySize
		if overflowCount > 0 {
			logging.GetLogger().Warningf("Flow array for tag '%s' overflowed maximum size. Discarding %d flows", string(t), overflowCount)
			s.flows[t] = s.flows[t][overflowCount:]
		}
	}

	for t := range flushedTags {
		// copy slice to free up memory of already flushed flows
		newArray := make([]interface{}, 0, len(s.flows[t]))
		copy(newArray, s.flows[t])
		s.flows[t] = newArray
	}

	for t := range tagsToTimerReset {
		// stop old flush timers
		oldTimer, ok := s.flushTimers[t]
		if ok {
			oldTimer.Stop()
		}
	}

	// set a timer for next flush
	if s.maxObjectDuration > 0 {
		for t := range tagsToTimerReset {
			s.flushTimers[t] = time.AfterFunc(s.maxObjectDuration-time.Since(endTime), func() {
				s.StoreFlows(nil)
			})

			// a single timer will cover all of the flushed tags
			break
		}
	}

	return nil
}

func (s *StoreS3) flushFlowsToObject(t Tag, endTime time.Time) error {
	flows := s.flows[t]
	if len(flows) == 0 {
		return nil
	}

	if len(flows) > s.maxFlowsPerObject {
		flows = flows[:s.maxFlowsPerObject]
	}

	startTime := s.lastFlushTime[t]
	startTimeString := strconv.FormatInt(startTime.UTC().UnixNano()/int64(time.Millisecond), 10)
	endTimeString := strconv.FormatInt(endTime.UTC().UnixNano()/int64(time.Millisecond), 10)

	metadata := map[string]*string{
		"first-timestamp": aws.String(startTimeString),
		"last-timestamp":  aws.String(endTimeString),
		"num-records":     aws.String(strconv.Itoa(len(flows))),
	}

	currentStream := s.currentStream[t]
	if endTime.Sub(currentStream.ID) >= s.maxStreamDuration {
		currentStream = stream{ID: endTime}
	}

	encodedFlows, err := s.pipeline.Encoder.Encode(flows)
	if err != nil {
		logging.GetLogger().Error("Failed to encode object: ", err)
		return err
	}

	b, err := s.pipeline.Compressor.Compress(encodedFlows)
	if err != nil {
		logging.GetLogger().Error("Failed to compress object: ", err)
		return err
	}

	objectKey := strings.Join([]string{s.objectPrefix, string(t), currentStream.ID.UTC().Format("20060102T150405Z"), fmt.Sprintf("%08d.gz", currentStream.SeqNumber)}, "/")
	err = s.client.WriteObject(s.bucket, objectKey, string(b.Bytes()), "application/json", "gzip", metadata)

	if err != nil {
		logging.GetLogger().Error("Failed to write object: ", err)
		return err
	}

	// flush was successful, update state
	s.lastFlushTime[t] = endTime
	currentStream.SeqNumber++
	s.currentStream[t] = currentStream
	s.flows[t] = s.flows[t][len(flows):]

	return nil
}

// NewStoreS3 returns a new storage interface for storing flows to object store
func NewStoreS3(cfg *viper.Viper) (interface{}, error) {
	bucket := cfg.GetString(CfgRoot + "store.s3.bucket")
	objectPrefix := cfg.GetString(CfgRoot + "store.s3.object_prefix")
	maxFlowArraySize := cfg.GetInt(CfgRoot + "store.s3.max_flow_array_size")
	maxFlowsPerObject := cfg.GetInt(CfgRoot + "store.s3.max_flows_per_object")
	maxSecondsPerObject := cfg.GetInt(CfgRoot + "store.s3.max_seconds_per_object")
	maxSecondsPerStream := cfg.GetInt(CfgRoot + "store.s3.max_seconds_per_stream")

	client := newClient(cfg)

	store := &StoreS3{
		bucket:            bucket,
		objectPrefix:      objectPrefix,
		maxFlowsPerObject: maxFlowsPerObject,
		maxFlowArraySize:  maxFlowArraySize,
		maxObjectDuration: time.Second * time.Duration(maxSecondsPerObject),
		maxStreamDuration: time.Second * time.Duration(maxSecondsPerStream),
		client:            client,
		currentStream:     make(map[Tag]stream),
		flows:             make(map[Tag][]interface{}),
		lastFlushTime:     make(map[Tag]time.Time),
		flushTimers:       make(map[Tag]*time.Timer),
	}

	return store, nil
}
<|endoftext|>


<commit_before>package grep

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"strings"
)

type line struct {
	number   int
	contents string
	filename string
}

type configuration struct {
	// -n Print the line numbers of each matching line.
	prefixLineNumbers bool
	// -l Print only the names of files that contain at least one matching line.
	printFileNames bool
	// -i Match line using a case-insensitive comparison.
	matchCaseInsensitive bool
	// -v Invert the program -- collect all lines that fail to match the pattern.
	invertMatch bool
	// -x Only match entire lines, instead of lines that contain a match.
	matchEntireLine bool
	// No flag exists for this option. Prefix each matching line with the filename it was found in.
	prefixFileName bool
}

func NewConfiguration(flags []string, files []string) (c configuration) {
	c = configuration{
		prefixLineNumbers:    false,
		printFileNames:       false,
		matchCaseInsensitive: false,
		invertMatch:          false,
		matchEntireLine:      false,
		prefixFileName:       false,
	}

	for _, flag := range flags {
		switch flag {
		case "-n":
			c.prefixLineNumbers = true
		case "-l":
			c.printFileNames = true
		case "-i":
			c.matchCaseInsensitive = true
		case "-v":
			c.invertMatch = true
		case "-x":
			c.matchEntireLine = true
		default:
			log.Fatalf("unrecognized flag %v", flag)
		}
	}

	if len(files) > 1 {
		c.prefixFileName = true
	}

	return c
}

func Search(pattern string, flags, files []string) (result []string) {
	lines := readFiles(files)
	config := NewConfiguration(flags, files)
	matches := search(pattern, config, lines)

	temp := format(matches, config)
	fmt.Printf("result %v\n", temp)
	return temp
}

func search(pattern string, config configuration, lines []line) (matches []line) {
	for _, line := range lines {
		if isMatch(pattern, config, line) {
			matches = append(matches, line)
		}
	}
	return matches
}

func isMatch(pattern string, config configuration, line line) bool {
	if config.matchCaseInsensitive {
		lowerContents := strings.ToLower(line.contents)
		lowerPattern := strings.ToLower(pattern)
		return strings.Contains(lowerContents, lowerPattern)
	} else if config.matchEntireLine && config.invertMatch {
		return pattern != line.contents
	} else if config.matchEntireLine {
		return pattern == line.contents
	} else if config.invertMatch {
		return !strings.Contains(line.contents, pattern)
	} else {
		return strings.Contains(line.contents, pattern)
	}
}

func format(matches []line, config configuration) (result []string) {
	result = []string{}
	for _, match := range matches {
		if config.printFileNames {
			result = append(result, match.filename)
		} else if config.prefixFileName && config.prefixLineNumbers {
			result = append(result, fmt.Sprintf("%s:%d:%s", match.filename, match.number, match.contents))
		} else if config.prefixFileName {
			result = append(result, fmt.Sprintf("%s:%s", match.filename, match.contents))
		} else if config.prefixLineNumbers {
			result = append(result, fmt.Sprintf("%d:%s", match.number, match.contents))
		} else {
			result = append(result, match.contents)
		}
	}

	if config.printFileNames {
		return uniq(result)
	}
	return result
}

func readFiles(files []string) (lines []line) {
	for _, file := range files {
		lines = append(lines, readLines(file)...)
	}
	return lines
}

func readLines(filename string) (lines []line) {
	file, err := os.Open(filename)

	if err != nil {
		log.Fatal(err)
	}

	scanner := bufio.NewScanner(file)
	lineNumber := 1
	for scanner.Scan() {
		contents := scanner.Text()
		line := line{lineNumber, contents, filename}

		lines = append(lines, line)
		lineNumber += 1
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}

	return lines
}

func uniq(items []string) (result []string) {
	result = []string{}
	for _, item := range items {
		if !contains(result, item) {
			result = append(result, item)
		}
	}
	return result
}

func contains(list []string, value string) bool {
	for _, item := range list {
		if item == value {
			return true
		}
	}
	return false
}
<commit_msg>Extract formatMatch<commit_after>package grep

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"strings"
)

type line struct {
	number   int
	contents string
	filename string
}

type configuration struct {
	// -n Print the line numbers of each matching line.
	prefixLineNumbers bool
	// -l Print only the names of files that contain at least one matching line.
	printFileNames bool
	// -i Match line using a case-insensitive comparison.
	matchCaseInsensitive bool
	// -v Invert the program -- collect all lines that fail to match the pattern.
	invertMatch bool
	// -x Only match entire lines, instead of lines that contain a match.
	matchEntireLine bool
	// No flag exists for this option. Prefix each matching line with the filename it was found in.
	prefixFileName bool
}

func NewConfiguration(flags []string, files []string) (c configuration) {
	c = configuration{
		prefixLineNumbers:    false,
		printFileNames:       false,
		matchCaseInsensitive: false,
		invertMatch:          false,
		matchEntireLine:      false,
		prefixFileName:       false,
	}

	for _, flag := range flags {
		switch flag {
		case "-n":
			c.prefixLineNumbers = true
		case "-l":
			c.printFileNames = true
		case "-i":
			c.matchCaseInsensitive = true
		case "-v":
			c.invertMatch = true
		case "-x":
			c.matchEntireLine = true
		default:
			log.Fatalf("unrecognized flag %v", flag)
		}
	}

	if len(files) > 1 {
		c.prefixFileName = true
	}

	return c
}

func Search(pattern string, flags, files []string) (result []string) {
	lines := readFiles(files)
	config := NewConfiguration(flags, files)
	matches := search(pattern, config, lines)

	temp := format(matches, config)
	fmt.Printf("result %v\n", temp)
	return temp
}

func search(pattern string, config configuration, lines []line) (matches []line) {
	for _, line := range lines {
		if isMatch(pattern, config, line) {
			matches = append(matches, line)
		}
	}
	return matches
}

func isMatch(pattern string, config configuration, line line) bool {
	if config.matchCaseInsensitive {
		lowerContents := strings.ToLower(line.contents)
		lowerPattern := strings.ToLower(pattern)
		return strings.Contains(lowerContents, lowerPattern)
	} else if config.matchEntireLine && config.invertMatch {
		return pattern != line.contents
	} else if config.matchEntireLine {
		return pattern == line.contents
	} else if config.invertMatch {
		return !strings.Contains(line.contents, pattern)
	} else {
		return strings.Contains(line.contents, pattern)
	}
}

func format(matches []line, config configuration) (result []string) {
	result = []string{}
	for _, match := range matches {
		result = append(result, formatMatch(match, config))
	}

	if config.printFileNames {
		return uniq(result)
	}
	return result
}

func formatMatch(match line, config configuration) string {
	if config.printFileNames {
		return match.filename
	} else if config.prefixFileName && config.prefixLineNumbers {
		return fmt.Sprintf("%s:%d:%s", match.filename, match.number, match.contents)
	} else if config.prefixFileName {
		return fmt.Sprintf("%s:%s", match.filename, match.contents)
	} else if config.prefixLineNumbers {
		return fmt.Sprintf("%d:%s", match.number, match.contents)
	} else {
		return match.contents
	}
}

func readFiles(files []string) (lines []line) {
	for _, file := range files {
		lines = append(lines, readLines(file)...)
	}
	return lines
}

func readLines(filename string) (lines []line) {
	file, err := os.Open(filename)

	if err != nil {
		log.Fatal(err)
	}

	scanner := bufio.NewScanner(file)
	lineNumber := 1
	for scanner.Scan() {
		contents := scanner.Text()
		line := line{lineNumber, contents, filename}

		lines = append(lines, line)
		lineNumber += 1
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}

	return lines
}

func uniq(items []string) (result []string) {
	result = []string{}
	for _, item := range items {
		if !contains(result, item) {
			result = append(result, item)
		}
	}
	return result
}

func contains(list []string, value string) bool {
	for _, item := range list {
		if item == value {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
	"os"
	"os/signal"
    "github.com/fuzxxl/nfc/2.0/nfc"
    "github.com/fuzxxl/freefare/0.3/freefare"
    "code.google.com/p/go-sqlite/go1/sqlite3"
    "time"
    "github.com/davecheney/gpio"
    "gopkg.in/yaml.v2"
    "io/ioutil"
)

func heartbeat() {
    for {
        time.Sleep(2000 * time.Millisecond)
        fmt.Println("Dunka-dunk")
    }
}

func pulse_gpio(pin gpio.Pin, ms int) {
    pin.Set()
    time.Sleep(time.Duration(ms) * time.Millisecond)
    pin.Clear()
}

func clear_and_close(pin gpio.Pin) {
    pin.Clear()
    pin.Close()
}

func main() {
    d, err := nfc.Open("");
    if err != nil {
        panic(err);
    }

    gpiodata, err := ioutil.ReadFile("gpio.yaml");
    if err != nil {
        panic(err);
    }
    gpiomap := make(map[interface{}]interface{});
    err = yaml.Unmarshal([]byte(gpiodata), &gpiomap);
    if err != nil {
        panic(err);
    }

    c, err := sqlite3.Open("keys.db")
    if err != nil {
        panic(err);
    }
    row := make(sqlite3.RowMap)

    go heartbeat()

	green_led, err := gpio.OpenPin(gpiomap["green_led"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening green_led! %s\n", err)
		return
	}
	red_led, err := gpio.OpenPin(gpiomap["red_led"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening green_led! %s\n", err)
		return
	}
	relay, err := gpio.OpenPin(gpiomap["relay"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening relay! %s\n", err)
		return
	}
	// turn the leds off on exit
	exit_ch := make(chan os.Signal, 1)
	signal.Notify(exit_ch, os.Interrupt)
	go func() {
		for _ = range exit_ch {
			fmt.Printf("\nClearing and unexporting the pins.\n")
			go clear_and_close(green_led)
			go clear_and_close(red_led)
			go clear_and_close(relay)
			os.Exit(0)
		}
	}()

    // mainloop
    for {
        // Poll for tags
        var tags []freefare.Tag
        for {
            tags, err = freefare.GetTags(d);
            if err != nil {
                panic(err);
            }
            if len(tags) > 0 {
                break;
            }
            time.Sleep(100 * time.Millisecond)
            //fmt.Println("...polling")
        }
        valid_found := false
        for i := 0; i < len(tags); i++ {
            tag := tags[i]
            uidstr := tag.UID()
            fmt.Println("Found tag", uidstr);
            sql := "SELECT rowid, * FROM keys where uid=?"
            for s, err := c.Query(sql, uidstr); err == nil; err = s.Next() {
                var rowid int64
                s.Scan(&rowid, row)     // Assigns 1st column to rowid, the rest to row
                fmt.Println(rowid, row)
                valid_found = true
                fmt.Println("Access GRANTED to ", uidstr)
                go pulse_gpio(green_led, gpiomap["green_led"].(map[interface{}]interface{})["time"].(int))
                go pulse_gpio(relay, gpiomap["relay"].(map[interface{}]interface{})["time"].(int))
            }
        }
        if !valid_found {
            fmt.Println("Access DENIED")
            go pulse_gpio(red_led, gpiomap["red_led"].(map[interface{}]interface{})["time"].(int))
        }
        // Wait a moment before continuing with fast polling
        time.Sleep(500 * time.Millisecond)
    }
}
<commit_msg>cleanup on kill too<commit_after>package main

import (
    "fmt"
	"os"
	"os/signal"
    "github.com/fuzxxl/nfc/2.0/nfc"
    "github.com/fuzxxl/freefare/0.3/freefare"
    "code.google.com/p/go-sqlite/go1/sqlite3"
    "time"
    "github.com/davecheney/gpio"
    "gopkg.in/yaml.v2"
    "io/ioutil"
)

func heartbeat() {
    for {
        time.Sleep(2000 * time.Millisecond)
        fmt.Println("Dunka-dunk")
    }
}

func pulse_gpio(pin gpio.Pin, ms int) {
    pin.Set()
    time.Sleep(time.Duration(ms) * time.Millisecond)
    pin.Clear()
}

func clear_and_close(pin gpio.Pin) {
    pin.Clear()
    pin.Close()
}

func main() {
    d, err := nfc.Open("");
    if err != nil {
        panic(err);
    }

    gpiodata, err := ioutil.ReadFile("gpio.yaml");
    if err != nil {
        panic(err);
    }
    gpiomap := make(map[interface{}]interface{});
    err = yaml.Unmarshal([]byte(gpiodata), &gpiomap);
    if err != nil {
        panic(err);
    }

    c, err := sqlite3.Open("keys.db")
    if err != nil {
        panic(err);
    }
    row := make(sqlite3.RowMap)

    go heartbeat()

	green_led, err := gpio.OpenPin(gpiomap["green_led"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening green_led! %s\n", err)
		return
	}
	red_led, err := gpio.OpenPin(gpiomap["red_led"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening green_led! %s\n", err)
		return
	}
	relay, err := gpio.OpenPin(gpiomap["relay"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening relay! %s\n", err)
		return
	}
	// turn the leds off on exit
	exit_ch := make(chan os.Signal, 1)
	signal.Notify(exit_ch, os.Interrupt)
	signal.Notify(exit_ch, os.Kill)
	go func() {
		for _ = range exit_ch {
			fmt.Printf("\nClearing and unexporting the pins.\n")
			go clear_and_close(green_led)
			go clear_and_close(red_led)
			go clear_and_close(relay)
			os.Exit(0)
		}
	}()

    // mainloop
    for {
        // Poll for tags
        var tags []freefare.Tag
        for {
            tags, err = freefare.GetTags(d);
            if err != nil {
                panic(err);
            }
            if len(tags) > 0 {
                break;
            }
            time.Sleep(100 * time.Millisecond)
            //fmt.Println("...polling")
        }
        valid_found := false
        for i := 0; i < len(tags); i++ {
            tag := tags[i]
            uidstr := tag.UID()
            fmt.Println("Found tag", uidstr);
            sql := "SELECT rowid, * FROM keys where uid=?"
            for s, err := c.Query(sql, uidstr); err == nil; err = s.Next() {
                var rowid int64
                s.Scan(&rowid, row)     // Assigns 1st column to rowid, the rest to row
                fmt.Println(rowid, row)
                valid_found = true
                fmt.Println("Access GRANTED to ", uidstr)
                go pulse_gpio(green_led, gpiomap["green_led"].(map[interface{}]interface{})["time"].(int))
                go pulse_gpio(relay, gpiomap["relay"].(map[interface{}]interface{})["time"].(int))
            }
        }
        if !valid_found {
            fmt.Println("Access DENIED")
            go pulse_gpio(red_led, gpiomap["red_led"].(map[interface{}]interface{})["time"].(int))
        }
        // Wait a moment before continuing with fast polling
        time.Sleep(500 * time.Millisecond)
    }
}
<|endoftext|>


<commit_before>/*
gopm (Go Package Manager)
Copyright (c) 2012 cailei (dancercl@gmail.com)

The MIT License (MIT)

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package main

import (
    "flag"
    "fmt"
    "github.com/cailei/gopm_index/gopm/index"
    "log"
    "strings"
)

func cmd_install(args []string) {
    var help bool
    f := flag.NewFlagSet("install_flags", flag.ExitOnError)
    f.BoolVar(&help, "help", false, "show help info")
    f.BoolVar(&help, "h", false, "show help info")
    f.Usage = print_install_help
    f.Parse(args)

    if help {
        print_install_help()
        return
    }

    db := openLocalDB()
    names := f.Args()

    var exact_matches []*index.PackageMeta
    var partial_matches []*index.PackageMeta

    for pkg, err := db.FirstPackage(); pkg != nil; pkg, err = db.NextPackage() {
        if err != nil {
            log.Fatalln(err)
        }

        low_pkg_name := strings.ToLower(pkg.Name)

        for _, name := range names {
            low_name := strings.ToLower(name)
            if low_pkg_name == low_name {
                exact_matches = append(exact_matches, pkg)
            } else if strings.Contains(low_pkg_name, low_name) {
                partial_matches = append(partial_matches, pkg)
            }
        }
    }

    if len(partial_matches) > 0 {
        fmt.Println("Some packages cannot find a match, did you mean:")
        for _, p := range partial_matches {
            fmt.Printf("\t%v\n", p.Name)
        }
        return
    }

    if len(exact_matches) == 0 {
        fmt.Printf("Found nothing!\n")
    }

    for _, p := range exact_matches {
        install_package(p)
    }
}

func install_package(pkg *index.PackageMeta) bool {
    for _, repo := range pkg.Repositories {
        fmt.Printf("go get %v\n", repo)
        break
    }
    return true
}

func print_install_help() {
    fmt.Print(`
gopm install <pkg1> [pkg2...]:
    install packages.

options:
    -h, -help       show help info

`)
}
<commit_msg>improve help info for 'gopm install'<commit_after>/*
gopm (Go Package Manager)
Copyright (c) 2012 cailei (dancercl@gmail.com)

The MIT License (MIT)

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package main

import (
    "flag"
    "fmt"
    "github.com/cailei/gopm_index/gopm/index"
    "log"
    "strings"
)

func cmd_install(args []string) {
    var help bool
    f := flag.NewFlagSet("install_flags", flag.ExitOnError)
    f.BoolVar(&help, "help", false, "show help info")
    f.BoolVar(&help, "h", false, "show help info")
    f.Usage = print_install_help
    f.Parse(args)

    if help {
        print_install_help()
        return
    }

    names := f.Args()
    if len(names) == 0 {
        fmt.Println("Please specify the packages you want to install.")
        return
    }

    db := openLocalDB()

    var exact_matches []*index.PackageMeta
    var partial_matches []*index.PackageMeta

    for pkg, err := db.FirstPackage(); pkg != nil; pkg, err = db.NextPackage() {
        if err != nil {
            log.Fatalln(err)
        }

        low_pkg_name := strings.ToLower(pkg.Name)

        for _, name := range names {
            low_name := strings.ToLower(name)
            if low_pkg_name == low_name {
                exact_matches = append(exact_matches, pkg)
            } else if strings.Contains(low_pkg_name, low_name) {
                partial_matches = append(partial_matches, pkg)
            }
        }
    }

    if len(partial_matches) > 0 {
        fmt.Println("Some packages cannot find a match, did you mean:")
        for _, p := range partial_matches {
            fmt.Printf("\t%v\n", p.Name)
        }
        return
    }

    if len(exact_matches) == 0 {
        fmt.Printf("Found nothing!\n")
    }

    for _, p := range exact_matches {
        install_package(p)
    }
}

func install_package(pkg *index.PackageMeta) bool {
    for _, repo := range pkg.Repositories {
        fmt.Printf("go get %v\n", repo)
        break
    }
    return true
}

func print_install_help() {
    fmt.Print(`
gopm install <pkg1> [pkg2...]:
    install packages.

    this command will invoke 'go get' to get the job done, you may specify any valid 'go get' flags (see 'go get -h') in the command line, and they will be passed directly to the 'go get'.

e.g.
    'gopm install -u <package>'     install or upgrade a package
    'gopm install -a <package>'     install or upgrade a package

options:
    -h, -help       show help info

`)
}
<|endoftext|>


<commit_before>// Package gosamplerate is a golang binding for libsamplerate (audio sample rate converter)
package gosamplerate

/*

#cgo pkg-config: samplerate

#include <samplerate.h>
#include <stdlib.h>

SRC_DATA *alloc_src_data() {
    return (SRC_DATA*)malloc(sizeof(SRC_DATA));
}

void free_src_data(SRC_DATA *p){
    free(p);
}

int run_src_simple(float *data_in, float *data_out,
                   long input_frames, long output_frames,
                   double src_ratio, int converter_type, int channels,
                   long *output_frames_gen) {
    SRC_DATA src_data;
    src_data.data_in = data_in;
    src_data.data_out = data_out;
    src_data.input_frames = input_frames;
    src_data.output_frames = output_frames;
    src_data.src_ratio = src_ratio;
    int res = src_simple(&src_data, converter_type, channels);
    *output_frames_gen = src_data.output_frames_gen;
    return res;
}

*/
import "C"

import (
	"errors"
	"fmt"
	"math"
)

// Src struct holding the data for the full API
type Src struct {
	srcState     *C.SRC_STATE
	srcData      *C.SRC_DATA
	channels     C.long
	inputBuffer  []C.float
	outputBuffer []C.float
}

const (
	SRC_SINC_BEST_QUALITY   = C.SRC_SINC_BEST_QUALITY
	SRC_SINC_MEDIUM_QUALITY = C.SRC_SINC_MEDIUM_QUALITY
	SRC_SINC_FASTEST        = C.SRC_SINC_FASTEST
	SRC_ZERO_ORDER_HOLD     = C.SRC_ZERO_ORDER_HOLD
	SRC_LINEAR              = C.SRC_LINEAR
)

//New initializes the converter object and returns a reference to it.
func New(converterType int, channels int, bufferLen int) (Src, error) {
	cConverter := C.int(converterType)
	cChannels := C.int(channels)
	var cErr *C.int

	src_state := C.src_new(cConverter, cChannels, cErr)
	if src_state == nil {
		return Src{}, errors.New("Could not initialize samplerate converter object")
	}

	inputBuffer := make([]C.float, bufferLen)
	outputBuffer := make([]C.float, bufferLen)

	cData := C.alloc_src_data()
	cData.data_in = &inputBuffer[0]
	cData.data_out = &outputBuffer[0]
	cData.output_frames = C.long(len(outputBuffer) / channels)
	cData.src_ratio = 1

	src := Src{
		srcState:     src_state,
		srcData:      cData,
		channels:     C.long(cChannels),
		inputBuffer:  inputBuffer,
		outputBuffer: outputBuffer,
	}

	return src, nil
}

// Delete cleans up all internal allocations.
func Delete(src Src) error {
	srcState := C.src_delete(src.srcState)
	C.free_src_data(src.srcData)
	if srcState == nil {
		return nil
	}
	return errors.New("Could not delete object; It did not exist")
}

// GetChannels gets the current channel count.
func (src *Src) GetChannels() (int, error) {

	// for version < 1.9
	return int(src.channels), nil

	// with version 1.9 src_get_channels was added
	// cChannels := C.src_get_channels(src.srcState)
	// if cChannels < 0 {
	// 	return int(cChannels), errors.New("invalid channel count")
	// }
	// return int(cChannels), nil
}

// Reset the internal SRC state. It does not modify the quality settings.
// It does not free any memory allocations.
func (src *Src) Reset() error {
	res := C.src_reset(src.srcState)
	if res < 0 {
		errMsg := fmt.Sprintf("Could not reset samplerate converter state: %s",
			Error(int(res)))
		return errors.New(errMsg)
	}
	return nil
}

// Error Convert the error number into a string.
func Error(errNo int) string {
	err := C.src_strerror(C.int(errNo))
	return C.GoString(err)
}

//ErrorNo return an error number
func (src *Src) ErrorNo() int {
	errNo := C.src_error(src.srcState)
	return int(errNo)
}

// Simple converts a single block of samples (one or more channels) in one go. The simple API is less
// capable than the full API (Process()). It must not be used if Audio shall be converted in chunks. For full documentation see: http://www.mega-nerd.com/SRC/api_simple.html
func Simple(dataIn []float32, ratio float64, channels int, converterType int) ([]float32, error) {
	if ratio <= 0 {
		return nil, fmt.Errorf("Error code: 6; SRC ratio outside [1/256, 256] range.")
	}
	outputBuffer := make([]float32, len(dataIn)*int(math.Ceil(ratio)))
	var outputFramesGen C.long
	res := C.run_src_simple(
		(*C.float)(&dataIn[0]),
		(*C.float)(&outputBuffer[0]),
		C.long(len(dataIn)/channels),
		C.long(cap(outputBuffer)/channels),
		C.double(ratio),
		C.int(converterType),
		C.int(channels),
		&outputFramesGen)
	if res != 0 {
		errMsg := fmt.Sprintf("Error code: %d; %s", res, Error(int(res)))
		return nil, errors.New(errMsg)
	}
	return outputBuffer[:int(outputFramesGen)*channels], nil
}

// Process is known as the full API. It allows time varying sample rate conversion on streaming data on one or more channels. For full documentation see: http://www.mega-nerd.com/SRC/api_full.html
func (src *Src) Process(dataIn []float32, ratio float64, endOfInput bool) ([]float32, error) {

	inputLength := len(dataIn)

	if inputLength > len(src.inputBuffer) {
		return nil, errors.New("data slice is larger than buffer")
	}

	// copy data into input buffer
	for i, el := range dataIn {
		src.inputBuffer[i] = C.float(el)
	}

	var cEndOfInput = C.int(0)
	if endOfInput {
		cEndOfInput = 1
	} else {
		cEndOfInput = 0
	}

	src.srcData.input_frames = C.long(inputLength) / src.channels
	src.srcData.end_of_input = cEndOfInput
	src.srcData.src_ratio = C.double(ratio)

	res := C.src_process(src.srcState, src.srcData)

	if res != 0 {
		errMsg := fmt.Sprintf("Error code: %d; %s", res, Error(int(res)))
		return nil, errors.New(errMsg)
	}

	output := make([]float32, 0, src.srcData.output_frames_gen)

	// fmt.Println("channels:", src.channels)
	// fmt.Println("input frames", cSrcData.input_frames)
	// fmt.Println("input used", cSrcData.input_frames_used)
	// fmt.Println("output generated", cSrcData.output_frames_gen)

	for i := 0; i < int(src.srcData.output_frames_gen*src.channels); i++ {
		output = append(output, float32(src.outputBuffer[i]))
	}

	return output, nil
}

// SetRatio sets the samplerate conversion ratio between input and output samples.
// Normally, when using (src *SRC) Process or the callback, the library will try to smoothly
// transition between the conversion ratio of the last call and the conversion ratio of the next
// call. This function bypasses this behaviour and achieves a step response in the conversion rate.
func (src *Src) SetRatio(ratio float64) error {

	res := C.src_set_ratio(src.srcState, C.double(ratio))
	if res != 0 {
		return errors.New(Error(int(res)))
	}
	return nil
}

// IsValidRatio returns True is ratio is a valid conversion ratio, False otherwise.
func IsValidRatio(ratio float64) bool {
	res := C.src_is_valid_ratio(C.double(ratio))
	if res == 1 {
		return true
	}
	return false
}

// GetName returns the name of a sample rate converter
func GetName(converter C.int) (string, error) {
	cConverterName := C.src_get_name(converter)
	if cConverterName == nil {
		return "", errors.New("unknown samplerate converter")
	}
	return C.GoString(cConverterName), nil
}

// GetDescription returns the description of a sample rate converter
func GetDescription(converter C.int) (string, error) {
	cConverterDescription := C.src_get_description(converter)
	if cConverterDescription == nil {
		return "", errors.New("unknown samplerate converter")
	}
	return C.GoString(cConverterDescription), nil
}

// GetVersion returns the version number of libsamplerate
func GetVersion() string {

	cVersion := C.src_get_version()
	return C.GoString(cVersion)
}
<commit_msg>fix more possible dangling C pointers<commit_after>// Package gosamplerate is a golang binding for libsamplerate (audio sample rate converter)
package gosamplerate

/*

#cgo pkg-config: samplerate

#include <samplerate.h>
#include <stdlib.h>

SRC_DATA *alloc_src_data(float *data_in, float *data_out,
					long output_frames, double src_ratio) {

	SRC_DATA *src_data = malloc(sizeof(SRC_DATA));
    src_data->data_in = data_in;
    src_data->data_out = data_out;
    src_data->output_frames = output_frames;
    src_data->src_ratio = src_ratio;
	return src_data;
}

void free_src_data(SRC_DATA *p){
    free(p);
}

int run_src_simple(float *data_in, float *data_out,
                   long input_frames, long output_frames,
                   double src_ratio, int converter_type, int channels,
                   long *output_frames_gen) {
    SRC_DATA src_data;
    src_data.data_in = data_in;
    src_data.data_out = data_out;
    src_data.input_frames = input_frames;
    src_data.output_frames = output_frames;
    src_data.src_ratio = src_ratio;
    int res = src_simple(&src_data, converter_type, channels);
    *output_frames_gen = src_data.output_frames_gen;
    return res;
}

*/
import "C"

import (
	"errors"
	"fmt"
	"math"
)

// Src struct holding the data for the full API
type Src struct {
	srcState     *C.SRC_STATE
	srcData      *C.SRC_DATA
	channels     C.long
	inputBuffer  []C.float
	outputBuffer []C.float
}

const (
	SRC_SINC_BEST_QUALITY   = C.SRC_SINC_BEST_QUALITY
	SRC_SINC_MEDIUM_QUALITY = C.SRC_SINC_MEDIUM_QUALITY
	SRC_SINC_FASTEST        = C.SRC_SINC_FASTEST
	SRC_ZERO_ORDER_HOLD     = C.SRC_ZERO_ORDER_HOLD
	SRC_LINEAR              = C.SRC_LINEAR
)

//New initializes the converter object and returns a reference to it.
func New(converterType int, channels int, bufferLen int) (Src, error) {
	cConverter := C.int(converterType)
	cChannels := C.int(channels)
	var cErr *C.int

	src_state := C.src_new(cConverter, cChannels, cErr)
	if src_state == nil {
		return Src{}, errors.New("Could not initialize samplerate converter object")
	}

	inputBuffer := make([]C.float, bufferLen)
	outputBuffer := make([]C.float, bufferLen)

	cData := C.alloc_src_data(
		&inputBuffer[0],
		&outputBuffer[0],
		C.long(len(outputBuffer)/channels),
		1,
	)

	src := Src{
		srcState:     src_state,
		srcData:      cData,
		channels:     C.long(cChannels),
		inputBuffer:  inputBuffer,
		outputBuffer: outputBuffer,
	}

	return src, nil
}

// Delete cleans up all internal allocations.
func Delete(src Src) error {
	srcState := C.src_delete(src.srcState)
	C.free_src_data(src.srcData)
	if srcState == nil {
		return nil
	}
	return errors.New("Could not delete object; It did not exist")
}

// GetChannels gets the current channel count.
func (src *Src) GetChannels() (int, error) {

	// for version < 1.9
	return int(src.channels), nil

	// with version 1.9 src_get_channels was added
	// cChannels := C.src_get_channels(src.srcState)
	// if cChannels < 0 {
	// 	return int(cChannels), errors.New("invalid channel count")
	// }
	// return int(cChannels), nil
}

// Reset the internal SRC state. It does not modify the quality settings.
// It does not free any memory allocations.
func (src *Src) Reset() error {
	res := C.src_reset(src.srcState)
	if res < 0 {
		errMsg := fmt.Sprintf("Could not reset samplerate converter state: %s",
			Error(int(res)))
		return errors.New(errMsg)
	}
	return nil
}

// Error Convert the error number into a string.
func Error(errNo int) string {
	err := C.src_strerror(C.int(errNo))
	return C.GoString(err)
}

//ErrorNo return an error number
func (src *Src) ErrorNo() int {
	errNo := C.src_error(src.srcState)
	return int(errNo)
}

// Simple converts a single block of samples (one or more channels) in one go. The simple API is less
// capable than the full API (Process()). It must not be used if Audio shall be converted in chunks. For full documentation see: http://www.mega-nerd.com/SRC/api_simple.html
func Simple(dataIn []float32, ratio float64, channels int, converterType int) ([]float32, error) {
	if ratio <= 0 {
		return nil, fmt.Errorf("Error code: 6; SRC ratio outside [1/256, 256] range.")
	}
	outputBuffer := make([]float32, len(dataIn)*int(math.Ceil(ratio)))
	var outputFramesGen C.long
	res := C.run_src_simple(
		(*C.float)(&dataIn[0]),
		(*C.float)(&outputBuffer[0]),
		C.long(len(dataIn)/channels),
		C.long(cap(outputBuffer)/channels),
		C.double(ratio),
		C.int(converterType),
		C.int(channels),
		&outputFramesGen)
	if res != 0 {
		errMsg := fmt.Sprintf("Error code: %d; %s", res, Error(int(res)))
		return nil, errors.New(errMsg)
	}
	return outputBuffer[:int(outputFramesGen)*channels], nil
}

// Process is known as the full API. It allows time varying sample rate conversion on streaming data on one or more channels. For full documentation see: http://www.mega-nerd.com/SRC/api_full.html
func (src *Src) Process(dataIn []float32, ratio float64, endOfInput bool) ([]float32, error) {

	inputLength := len(dataIn)

	if inputLength > len(src.inputBuffer) {
		return nil, errors.New("data slice is larger than buffer")
	}

	// copy data into input buffer
	for i, el := range dataIn {
		src.inputBuffer[i] = C.float(el)
	}

	var cEndOfInput = C.int(0)
	if endOfInput {
		cEndOfInput = 1
	} else {
		cEndOfInput = 0
	}

	src.srcData.input_frames = C.long(inputLength) / src.channels
	src.srcData.end_of_input = cEndOfInput
	src.srcData.src_ratio = C.double(ratio)

	res := C.src_process(src.srcState, src.srcData)

	if res != 0 {
		errMsg := fmt.Sprintf("Error code: %d; %s", res, Error(int(res)))
		return nil, errors.New(errMsg)
	}

	output := make([]float32, 0, src.srcData.output_frames_gen)

	// fmt.Println("channels:", src.channels)
	// fmt.Println("input frames", cSrcData.input_frames)
	// fmt.Println("input used", cSrcData.input_frames_used)
	// fmt.Println("output generated", cSrcData.output_frames_gen)

	for i := 0; i < int(src.srcData.output_frames_gen*src.channels); i++ {
		output = append(output, float32(src.outputBuffer[i]))
	}

	return output, nil
}

// SetRatio sets the samplerate conversion ratio between input and output samples.
// Normally, when using (src *SRC) Process or the callback, the library will try to smoothly
// transition between the conversion ratio of the last call and the conversion ratio of the next
// call. This function bypasses this behaviour and achieves a step response in the conversion rate.
func (src *Src) SetRatio(ratio float64) error {

	res := C.src_set_ratio(src.srcState, C.double(ratio))
	if res != 0 {
		return errors.New(Error(int(res)))
	}
	return nil
}

// IsValidRatio returns True is ratio is a valid conversion ratio, False otherwise.
func IsValidRatio(ratio float64) bool {
	res := C.src_is_valid_ratio(C.double(ratio))
	if res == 1 {
		return true
	}
	return false
}

// GetName returns the name of a sample rate converter
func GetName(converter C.int) (string, error) {
	cConverterName := C.src_get_name(converter)
	if cConverterName == nil {
		return "", errors.New("unknown samplerate converter")
	}
	return C.GoString(cConverterName), nil
}

// GetDescription returns the description of a sample rate converter
func GetDescription(converter C.int) (string, error) {
	cConverterDescription := C.src_get_description(converter)
	if cConverterDescription == nil {
		return "", errors.New("unknown samplerate converter")
	}
	return C.GoString(cConverterDescription), nil
}

// GetVersion returns the version number of libsamplerate
func GetVersion() string {

	cVersion := C.src_get_version()
	return C.GoString(cVersion)
}
<|endoftext|>


<commit_before>/*-
 * Copyright 2014 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package josecipher

import (
	"bytes"
	"crypto/cipher"
	"crypto/hmac"
	"crypto/sha256"
	"crypto/sha512"
	"crypto/subtle"
	"encoding/binary"
	"errors"
	"hash"
)

const (
	nonceBytes = 16
)

// NewCBCHMAC instantiates a new AEAD based on CBC+HMAC.
func NewCBCHMAC(key []byte, newBlockCipher func([]byte) (cipher.Block, error)) (cipher.AEAD, error) {
	keySize := len(key) / 2
	integrityKey := key[:keySize]
	encryptionKey := key[keySize:]

	blockCipher, err := newBlockCipher(encryptionKey)
	if err != nil {
		return nil, err
	}

	return &cbcAEAD{
		encryptionKey: encryptionKey,
		integrityKey:  integrityKey,
		blockCipher:   blockCipher,
		authtagBytes:  keySize,
	}, nil
}

// An AEAD based on CBC+HMAC
type cbcAEAD struct {
	encryptionKey []byte
	integrityKey  []byte
	authtagBytes  int
	blockCipher   cipher.Block
}

func (ctx *cbcAEAD) NonceSize() int {
	return nonceBytes
}

func (ctx *cbcAEAD) Overhead() int {
	// Maximum overhead is block size (for padding) plus auth tag length
	return ctx.blockCipher.BlockSize() + ctx.authtagBytes
}

// Seal encrypts and authenticates the plaintext.
func (ctx *cbcAEAD) Seal(dst, nonce, plaintext, data []byte) []byte {
	// Output buffer -- must take care not to mangle plaintext input.
	ciphertext := make([]byte, len(plaintext)+ctx.Overhead())[:len(plaintext)]
	copy(ciphertext, plaintext)
	ciphertext = padBuffer(ciphertext, ctx.blockCipher.BlockSize())

	cbc := cipher.NewCBCEncrypter(ctx.blockCipher, nonce)

	cbc.CryptBlocks(ciphertext, ciphertext)
	authtag := ctx.computeAuthTag(data, nonce, ciphertext)

	ret, out := resize(dst, len(dst)+len(ciphertext)+len(authtag))
	copy(out, ciphertext)
	copy(out[len(ciphertext):], authtag)

	return ret
}

// Open decrypts and authenticates the ciphertext.
func (ctx *cbcAEAD) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) {
	if len(ciphertext) < ctx.authtagBytes {
		return nil, errors.New("square/go-jose: invalid ciphertext (too short)")
	}

	offset := len(ciphertext) - ctx.authtagBytes
	expectedTag := ctx.computeAuthTag(data, nonce, ciphertext[:offset])
	match := subtle.ConstantTimeCompare(expectedTag, ciphertext[offset:])
	if match != 1 {
		return nil, errors.New("square/go-jose: invalid ciphertext (auth tag mismatch)")
	}

	cbc := cipher.NewCBCDecrypter(ctx.blockCipher, nonce)
	buffer := []byte(ciphertext[:offset])
	cbc.CryptBlocks(buffer, buffer)

	// Remove padding
	plaintext, err := unpadBuffer(buffer, ctx.blockCipher.BlockSize())
	if err != nil {
		return nil, err
	}

	ret, out := resize(dst, len(dst)+len(plaintext))
	copy(out, plaintext)

	return ret, nil
}

// Compute an authentication tag
func (ctx *cbcAEAD) computeAuthTag(aad, nonce, ciphertext []byte) []byte {
	buffer := []byte(aad)
	buffer = append(buffer, nonce...)
	buffer = append(buffer, ciphertext...)
	buffer = append(buffer, bitLen(aad)...)

	var hash func() hash.Hash
	switch len(ctx.integrityKey) {
	case 16:
		hash = sha256.New
	case 24:
		hash = sha512.New384
	case 32:
		hash = sha512.New
	}

	hmac := hmac.New(hash, ctx.integrityKey)

	// According to documentation, Write() on hash.Hash never fails.
	_, _ = hmac.Write(buffer)

	return hmac.Sum(nil)[:ctx.authtagBytes]
}

// Helper function for serializing bit length into array
func bitLen(input []byte) []byte {
	encodedLen := make([]byte, 8)
	binary.BigEndian.PutUint64(encodedLen, uint64(len(input)*8))
	return encodedLen
}

// resize ensures the the given slice has a capacity of at least n bytes.
// If the capacity of the slice is less than n, a new slice is allocated
// and the existing data will be copied.
func resize(in []byte, n int) (head, tail []byte) {
	if cap(in) >= n {
		head = in[:n]
	} else {
		head = make([]byte, n)
		copy(head, in)
	}

	tail = head[len(in):]
	return
}

// Apply padding
func padBuffer(buffer []byte, blockSize int) []byte {
	missing := blockSize - (len(buffer) % blockSize)
	ret, out := resize(buffer, len(buffer)+missing)
	padding := bytes.Repeat([]byte{byte(missing)}, missing)
	copy(out, padding)
	return ret
}

// Remove padding
func unpadBuffer(buffer []byte, blockSize int) ([]byte, error) {
	if len(buffer)%blockSize != 0 {
		return nil, errors.New("square/go-jose: invalid padding")
	}

	last := buffer[len(buffer)-1]
	count := int(last)

	if count > blockSize || count > len(buffer) {
		return nil, errors.New("square/go-jose: invalid padding")
	}

	padding := bytes.Repeat([]byte{last}, count)
	if bytes.Compare(buffer[len(buffer)-count:], padding) != 0 {
		return nil, errors.New("square/go-jose: invalid padding")
	}

	return buffer[:len(buffer)-count], nil
}
<commit_msg>Use bytes.HasSuffix to check padding<commit_after>/*-
 * Copyright 2014 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package josecipher

import (
	"bytes"
	"crypto/cipher"
	"crypto/hmac"
	"crypto/sha256"
	"crypto/sha512"
	"crypto/subtle"
	"encoding/binary"
	"errors"
	"hash"
)

const (
	nonceBytes = 16
)

// NewCBCHMAC instantiates a new AEAD based on CBC+HMAC.
func NewCBCHMAC(key []byte, newBlockCipher func([]byte) (cipher.Block, error)) (cipher.AEAD, error) {
	keySize := len(key) / 2
	integrityKey := key[:keySize]
	encryptionKey := key[keySize:]

	blockCipher, err := newBlockCipher(encryptionKey)
	if err != nil {
		return nil, err
	}

	return &cbcAEAD{
		encryptionKey: encryptionKey,
		integrityKey:  integrityKey,
		blockCipher:   blockCipher,
		authtagBytes:  keySize,
	}, nil
}

// An AEAD based on CBC+HMAC
type cbcAEAD struct {
	encryptionKey []byte
	integrityKey  []byte
	authtagBytes  int
	blockCipher   cipher.Block
}

func (ctx *cbcAEAD) NonceSize() int {
	return nonceBytes
}

func (ctx *cbcAEAD) Overhead() int {
	// Maximum overhead is block size (for padding) plus auth tag length
	return ctx.blockCipher.BlockSize() + ctx.authtagBytes
}

// Seal encrypts and authenticates the plaintext.
func (ctx *cbcAEAD) Seal(dst, nonce, plaintext, data []byte) []byte {
	// Output buffer -- must take care not to mangle plaintext input.
	ciphertext := make([]byte, len(plaintext)+ctx.Overhead())[:len(plaintext)]
	copy(ciphertext, plaintext)
	ciphertext = padBuffer(ciphertext, ctx.blockCipher.BlockSize())

	cbc := cipher.NewCBCEncrypter(ctx.blockCipher, nonce)

	cbc.CryptBlocks(ciphertext, ciphertext)
	authtag := ctx.computeAuthTag(data, nonce, ciphertext)

	ret, out := resize(dst, len(dst)+len(ciphertext)+len(authtag))
	copy(out, ciphertext)
	copy(out[len(ciphertext):], authtag)

	return ret
}

// Open decrypts and authenticates the ciphertext.
func (ctx *cbcAEAD) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) {
	if len(ciphertext) < ctx.authtagBytes {
		return nil, errors.New("square/go-jose: invalid ciphertext (too short)")
	}

	offset := len(ciphertext) - ctx.authtagBytes
	expectedTag := ctx.computeAuthTag(data, nonce, ciphertext[:offset])
	match := subtle.ConstantTimeCompare(expectedTag, ciphertext[offset:])
	if match != 1 {
		return nil, errors.New("square/go-jose: invalid ciphertext (auth tag mismatch)")
	}

	cbc := cipher.NewCBCDecrypter(ctx.blockCipher, nonce)
	buffer := []byte(ciphertext[:offset])
	cbc.CryptBlocks(buffer, buffer)

	// Remove padding
	plaintext, err := unpadBuffer(buffer, ctx.blockCipher.BlockSize())
	if err != nil {
		return nil, err
	}

	ret, out := resize(dst, len(dst)+len(plaintext))
	copy(out, plaintext)

	return ret, nil
}

// Compute an authentication tag
func (ctx *cbcAEAD) computeAuthTag(aad, nonce, ciphertext []byte) []byte {
	buffer := []byte(aad)
	buffer = append(buffer, nonce...)
	buffer = append(buffer, ciphertext...)
	buffer = append(buffer, bitLen(aad)...)

	var hash func() hash.Hash
	switch len(ctx.integrityKey) {
	case 16:
		hash = sha256.New
	case 24:
		hash = sha512.New384
	case 32:
		hash = sha512.New
	}

	hmac := hmac.New(hash, ctx.integrityKey)

	// According to documentation, Write() on hash.Hash never fails.
	_, _ = hmac.Write(buffer)

	return hmac.Sum(nil)[:ctx.authtagBytes]
}

// Helper function for serializing bit length into array
func bitLen(input []byte) []byte {
	encodedLen := make([]byte, 8)
	binary.BigEndian.PutUint64(encodedLen, uint64(len(input)*8))
	return encodedLen
}

// resize ensures the the given slice has a capacity of at least n bytes.
// If the capacity of the slice is less than n, a new slice is allocated
// and the existing data will be copied.
func resize(in []byte, n int) (head, tail []byte) {
	if cap(in) >= n {
		head = in[:n]
	} else {
		head = make([]byte, n)
		copy(head, in)
	}

	tail = head[len(in):]
	return
}

// Apply padding
func padBuffer(buffer []byte, blockSize int) []byte {
	missing := blockSize - (len(buffer) % blockSize)
	ret, out := resize(buffer, len(buffer)+missing)
	padding := bytes.Repeat([]byte{byte(missing)}, missing)
	copy(out, padding)
	return ret
}

// Remove padding
func unpadBuffer(buffer []byte, blockSize int) ([]byte, error) {
	if len(buffer)%blockSize != 0 {
		return nil, errors.New("square/go-jose: invalid padding")
	}

	last := buffer[len(buffer)-1]
	count := int(last)

	if count > blockSize || count > len(buffer) {
		return nil, errors.New("square/go-jose: invalid padding")
	}

	padding := bytes.Repeat([]byte{last}, count)
	if !bytes.HasSuffix(buffer, padding) {
		return nil, errors.New("square/go-jose: invalid padding")
	}

	return buffer[:len(buffer)-count], nil
}
<|endoftext|>