<commit_before>package grpcutil

import (
	"sync"

	"github.com/pachyderm/pachyderm/src/client/pkg/tracing"
	"google.golang.org/grpc"
)

// Dialer defines a grpc.ClientConn connection dialer.
type Dialer interface {
	Dial(address string) (*grpc.ClientConn, error)
	CloseConns() error
}

// NewDialer creates a Dialer.
func NewDialer(opts ...grpc.DialOption) Dialer {
	return newDialer(opts...)
}

type dialer struct {
	opts []grpc.DialOption
	// A map from addresses to connections
	connMap map[string]*grpc.ClientConn
	lock    sync.Mutex
}

func newDialer(opts ...grpc.DialOption) *dialer {
	return &dialer{
		opts:    opts,
		connMap: make(map[string]*grpc.ClientConn),
	}
}

func (d *dialer) Dial(addr string) (*grpc.ClientConn, error) {
	d.lock.Lock()
	defer d.lock.Unlock()

	if conn, ok := d.connMap[addr]; ok {
		return conn, nil
	}
	opts := append(d.opts,
		grpc.WithUnaryInterceptor(tracing.UnaryClientInterceptor()),
		grpc.WithStreamInterceptor(tracing.StreamClientInterceptor()),
	)
	if (strings.Index(addr, "dns:///") == -1) {
	    addr = "dns:///"+addr
	}
	conn, err := grpc.Dial(addr, opts...)
	if err != nil {
		return nil, err
	}
	d.connMap[addr] = conn
	return conn, nil
}

func (d *dialer) CloseConns() error {
	d.lock.Lock()
	defer d.lock.Unlock()
	for addr, conn := range d.connMap {
		if err := conn.Close(); err != nil {
			return err
		}
		delete(d.connMap, addr)
	}
	return nil
}
<commit_msg>Fix import<commit_after>package grpcutil

import (
	"sync"
	"strings"

	"github.com/pachyderm/pachyderm/src/client/pkg/tracing"
	"google.golang.org/grpc"
)

// Dialer defines a grpc.ClientConn connection dialer.
type Dialer interface {
	Dial(address string) (*grpc.ClientConn, error)
	CloseConns() error
}

// NewDialer creates a Dialer.
func NewDialer(opts ...grpc.DialOption) Dialer {
	return newDialer(opts...)
}

type dialer struct {
	opts []grpc.DialOption
	// A map from addresses to connections
	connMap map[string]*grpc.ClientConn
	lock    sync.Mutex
}

func newDialer(opts ...grpc.DialOption) *dialer {
	return &dialer{
		opts:    opts,
		connMap: make(map[string]*grpc.ClientConn),
	}
}

func (d *dialer) Dial(addr string) (*grpc.ClientConn, error) {
	d.lock.Lock()
	defer d.lock.Unlock()

	if conn, ok := d.connMap[addr]; ok {
		return conn, nil
	}
	opts := append(d.opts,
		grpc.WithUnaryInterceptor(tracing.UnaryClientInterceptor()),
		grpc.WithStreamInterceptor(tracing.StreamClientInterceptor()),
	)
	if (strings.Index(addr, "dns:///") == -1) {
	    addr = "dns:///"+addr
	}
	conn, err := grpc.Dial(addr, opts...)
	if err != nil {
		return nil, err
	}
	d.connMap[addr] = conn
	return conn, nil
}

func (d *dialer) CloseConns() error {
	d.lock.Lock()
	defer d.lock.Unlock()
	for addr, conn := range d.connMap {
		if err := conn.Close(); err != nil {
			return err
		}
		delete(d.connMap, addr)
	}
	return nil
}
<|endoftext|>


<commit_before>package reseed

import (
	"bytes"
	"crypto/tls"
	"io"
	"net/http"
	"os"
	"strconv"

	"github.com/PuerkitoBio/throttled"
	"github.com/PuerkitoBio/throttled/store"
	"github.com/gorilla/handlers"
	"github.com/justinas/alice"
)

const (
	I2P_USER_AGENT = "Wget/1.11.4"
)

type Server struct {
	*http.Server
	Reseeder Reseeder
}

func NewServer(prefix string, trustProxy bool) *Server {
	config := &tls.Config{MinVersion: tls.VersionTLS10}
	h := &http.Server{TLSConfig: config}
	server := Server{h, nil}

	th := throttled.RateLimit(throttled.PerHour(120), &throttled.VaryBy{RemoteAddr: true}, store.NewMemStore(10000))

	middlewareChain := alice.New()
	if trustProxy {
		middlewareChain.Append(proxiedMiddleware)
	}
	middlewareChain = middlewareChain.Append(loggingMiddleware, verifyMiddleware, th.Throttle)

	mux := http.NewServeMux()
	mux.Handle(prefix+"/i2pseeds.su3", middlewareChain.Then(http.HandlerFunc(server.reseedHandler)))
	server.Handler = mux

	return &server
}

func (s *Server) reseedHandler(w http.ResponseWriter, r *http.Request) {
	peer := s.Reseeder.Peer(r)

	su3, err := s.Reseeder.PeerSu3Bytes(peer)
	if nil != err {
		http.Error(w, "500 Unable to get SU3", http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Disposition", "attachment; filename=i2pseeds.su3")
	w.Header().Set("Content-Type", "application/octet-stream")
	w.Header().Set("Content-Length", strconv.FormatInt(int64(len(su3)), 10))

	io.Copy(w, bytes.NewReader(su3))
}

func loggingMiddleware(next http.Handler) http.Handler {
	return handlers.CombinedLoggingHandler(os.Stdout, next)
}

func verifyMiddleware(next http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		if I2P_USER_AGENT != r.UserAgent() {
			http.Error(w, "403 Forbidden", http.StatusForbidden)
			return
		}

		next.ServeHTTP(w, r)
	}
	return http.HandlerFunc(fn)
}

func proxiedMiddleware(next http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		if prior, ok := r.Header["X-Forwarded-For"]; ok {
			r.RemoteAddr = prior[0]
		}

		next.ServeHTTP(w, r)
	}
	return http.HandlerFunc(fn)
}
<commit_msg>add a handler to log failed requests<commit_after>package reseed

import (
	"bytes"
	"crypto/tls"
	"io"
	"net/http"
	"os"
	"strconv"

	"github.com/PuerkitoBio/throttled"
	"github.com/PuerkitoBio/throttled/store"
	"github.com/gorilla/handlers"
	"github.com/justinas/alice"
)

const (
	I2P_USER_AGENT = "Wget/1.11.4"
)

type Server struct {
	*http.Server
	Reseeder Reseeder
}

func NewServer(prefix string, trustProxy bool) *Server {
	config := &tls.Config{MinVersion: tls.VersionTLS10}
	h := &http.Server{TLSConfig: config}
	server := Server{h, nil}

	th := throttled.RateLimit(throttled.PerHour(120), &throttled.VaryBy{RemoteAddr: true}, store.NewMemStore(10000))

	middlewareChain := alice.New()
	if trustProxy {
		middlewareChain.Append(proxiedMiddleware)
	}

	errorHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusNotFound)
	})

	mux := http.NewServeMux()
	mux.Handle("/", middlewareChain.Append(loggingMiddleware).Then(errorHandler))
	mux.Handle(prefix+"/i2pseeds.su3", middlewareChain.Append(loggingMiddleware, verifyMiddleware, th.Throttle).Then(http.HandlerFunc(server.reseedHandler)))
	server.Handler = mux

	return &server
}

func (s *Server) reseedHandler(w http.ResponseWriter, r *http.Request) {
	peer := s.Reseeder.Peer(r)

	su3, err := s.Reseeder.PeerSu3Bytes(peer)
	if nil != err {
		http.Error(w, "500 Unable to get SU3", http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Disposition", "attachment; filename=i2pseeds.su3")
	w.Header().Set("Content-Type", "application/octet-stream")
	w.Header().Set("Content-Length", strconv.FormatInt(int64(len(su3)), 10))

	io.Copy(w, bytes.NewReader(su3))
}

func loggingMiddleware(next http.Handler) http.Handler {
	return handlers.CombinedLoggingHandler(os.Stdout, next)
}

func verifyMiddleware(next http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		if I2P_USER_AGENT != r.UserAgent() {
			http.Error(w, "403 Forbidden", http.StatusForbidden)
			return
		}

		next.ServeHTTP(w, r)
	}
	return http.HandlerFunc(fn)
}

func proxiedMiddleware(next http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		if prior, ok := r.Header["X-Forwarded-For"]; ok {
			r.RemoteAddr = prior[0]
		}

		next.ServeHTTP(w, r)
	}
	return http.HandlerFunc(fn)
}
<|endoftext|>


<commit_before>package resource

import (
	"fmt"
	"io"
	"os"
	"os/user"
	"path/filepath"

	"github.com/dnaeon/gru/utils"
	"github.com/hashicorp/hcl"
	"github.com/hashicorp/hcl/hcl/ast"
	"github.com/imdario/mergo"
)

// Name and description of the resource
const fileResourceType = "file"
const fileResourceDesc = "manages files"

// BaseFileResource is the base resource for managing files
type BaseFileResource struct {
	// Path to the file
	Path string `hcl:"path"`

	// Permission bits to set on the file
	Mode int `hcl:"mode"`

	// Owner of the file
	Owner string `hcl:"owner"`

	// Group of the file
	Group string `hcl:"group"`

	// Source file to use when creating/updating the file
	Source string `hcl:"source"`

	// The destination file we manage
	dstFile *utils.FileUtil
}

// permissionsChanged returns a boolean indicating whether the
// permissions of the file managed by the resource is different than the
// permissions defined by the resource
func (bfr *BaseFileResource) permissionsChanged() (bool, error) {
	m, err := bfr.dstFile.Mode()
	if err != nil {
		return false, err
	}

	return m.Perm() != os.FileMode(bfr.Mode), nil
}

// ownerChanged returns a boolean indicating whether the
// owner/group of the file managed by the resource is different than the
// owner/group defined by the resource
func (bfr *BaseFileResource) ownerChanged() (bool, error) {
	owner, err := bfr.dstFile.Owner()
	if err != nil {
		return false, err
	}

	if bfr.Owner != owner.User.Username || bfr.Group != owner.Group.Name {
		return true, nil
	}

	return false, nil
}

// contentChanged returns a boolean indicating whether the
// content of the file managed by the resource is different than the
// content of the source file defined by the resource
func (bfr *BaseFileResource) contentChanged(srcPath string) (bool, error) {
	if bfr.Source == "" {
		return false, nil
	}

	srcFile := utils.NewFileUtil(srcPath)
	srcMd5, err := srcFile.Md5()
	if err != nil {
		return false, err
	}

	dstMd5, err := bfr.dstFile.Md5()
	if err != nil {
		return false, err
	}

	return srcMd5 != dstMd5, nil
}

// FileResource is a resource which manages files
type FileResource struct {
	BaseResource     `hcl:",squash"`
	BaseFileResource `hcl:",squash"`
}

// NewFileResource creates a new resource for managing files
func NewFileResource(title string, obj *ast.ObjectItem) (Resource, error) {
	// Defaults for owner and group
	currentUser, err := user.Current()
	if err != nil {
		return nil, err
	}

	currentGroup, err := user.LookupGroupId(currentUser.Gid)
	if err != nil {
		return nil, err
	}

	defaultOwner := currentUser.Username
	defaultGroup := currentGroup.Name

	// Resource defaults
	defaults := FileResource{
		BaseResource: BaseResource{
			Title: title,
			Type:  fileResourceType,
			State: StatePresent,
		},
		BaseFileResource: BaseFileResource{
			Path:  title,
			Mode:  0644,
			Owner: defaultOwner,
			Group: defaultGroup,
		},
	}

	var fr FileResource
	err = hcl.DecodeObject(&fr, obj)
	if err != nil {
		return nil, err
	}

	// Merge the decoded object with the resource defaults
	err = mergo.Merge(&fr, defaults)

	// The destination file we manage
	fr.dstFile = utils.NewFileUtil(fr.Path)

	return &fr, err
}

// Evaluate evaluates the file resource
func (fr *FileResource) Evaluate(w io.Writer, opts *Options) (State, error) {
	rs := State{
		Current: StateUnknown,
		Want:    fr.State,
		Update:  false,
	}

	// File does not exist
	fi, err := os.Stat(fr.Path)
	if os.IsNotExist(err) {
		rs.Current = StateAbsent
		return rs, nil
	}

	// Ensure that the file we manage is a regular file
	rs.Current = StatePresent
	if !fi.Mode().IsRegular() {
		return rs, fmt.Errorf("%s is not a regular file", fr.Path)
	}

	// Check file content
	srcPath := filepath.Join(opts.SiteDir, "data", fr.Source)
	changed, err := fr.contentChanged(srcPath)
	if err != nil {
		return rs, err
	}

	if changed {
		fr.Printf(w, "content is out of date\n")
		rs.Update = true
	}

	// Check file permissions
	changed, err = fr.permissionsChanged()
	if err != nil {
		return rs, err
	}

	if changed {
		fr.Printf(w, "permissions are out of date\n")
		rs.Update = true
	}

	// Check ownership
	changed, err = fr.ownerChanged()
	if err != nil {
		return rs, err
	}

	if changed {
		fr.Printf(w, "owner is out of date\n")
		rs.Update = true
	}

	return rs, nil
}

// Create creates the file
func (fr *FileResource) Create(w io.Writer, opts *Options) error {
	fr.Printf(w, "creating file\n")

	// Set file content
	switch {
	case fr.Source == "" && fr.dstFile.Exists():
		// Do nothing
		break
	case fr.Source == "" && !fr.dstFile.Exists():
		// Create an empty file
		if _, err := os.Create(fr.Path); err != nil {
			return err
		}
	case fr.Source != "" && fr.dstFile.Exists():
		// File exists and we have a source file
		srcPath := filepath.Join(opts.SiteDir, "data", fr.Source)
		if err := fr.dstFile.CopyFrom(srcPath); err != nil {
			return err
		}
	}

	// Set file owner
	if err := fr.dstFile.SetOwner(fr.Owner, fr.Group); err != nil {
		return err
	}

	// Set file permissions
	return fr.dstFile.Chmod(os.FileMode(fr.Mode))
}

// Delete deletes the file
func (fr *FileResource) Delete(w io.Writer, opts *Options) error {
	fr.Printf(w, "removing file\n")

	return fr.dstFile.Remove()
}

// Update updates the file managed by the resource
func (fr *FileResource) Update(w io.Writer, opts *Options) error {
	// Update file content if needed
	srcPath := filepath.Join(opts.SiteDir, "data", fr.Source)
	changed, err := fr.contentChanged(srcPath)
	if err != nil {
		return err
	}

	if changed {
		srcFile := utils.NewFileUtil(srcPath)
		srcMd5, err := srcFile.Md5()
		if err != nil {
			return err
		}

		fr.Printf(w, "updating content to md5:%s\n", srcMd5)
		if err := fr.dstFile.CopyFrom(srcFile.Path); err != nil {
			return err
		}
	}

	// Fix permissions if needed
	changed, err = fr.permissionsChanged()
	if err != nil {
		return err
	}

	if changed {
		fr.Printf(w, "setting permissions to %#o\n", fr.Mode)
		fr.dstFile.Chmod(os.FileMode(fr.Mode))
	}

	// Fix ownership if needed
	changed, err = fr.ownerChanged()
	if err != nil {
		return err
	}

	if changed {
		fr.Printf(w, "setting owner to %s:%s\n", fr.Owner, fr.Group)
		if err := fr.dstFile.SetOwner(fr.Owner, fr.Group); err != nil {
			return err
		}
	}

	return nil
}

func init() {
	item := RegistryItem{
		Name:        fileResourceType,
		Description: fileResourceDesc,
		Provider:    NewFileResource,
	}

	Register(item)
}
<commit_msg>resource: refactor resource.FileResource type<commit_after>package resource

import (
	"fmt"
	"io"
	"os"
	"os/user"
	"path/filepath"

	"github.com/dnaeon/gru/utils"
	"github.com/hashicorp/hcl"
	"github.com/hashicorp/hcl/hcl/ast"
	"github.com/imdario/mergo"
)

// Name and description of the resource
const fileResourceType = "file"
const fileResourceDesc = "manages files"

// BaseFileResource is the base resource for managing files
type BaseFileResource struct {
	// Path to the file
	Path string `hcl:"path"`

	// Permission bits to set on the file
	Mode int `hcl:"mode"`

	// Owner of the file
	Owner string `hcl:"owner"`

	// Group of the file
	Group string `hcl:"group"`

	// Source file to use when creating/updating the file
	Source string `hcl:"source"`
}

// FileResource is a resource which manages files
type FileResource struct {
	BaseResource     `hcl:",squash"`
	BaseFileResource `hcl:",squash"`
}

// NewFileResource creates a new resource for managing files
func NewFileResource(title string, obj *ast.ObjectItem) (Resource, error) {
	// Defaults for owner and group
	currentUser, err := user.Current()
	if err != nil {
		return nil, err
	}

	currentGroup, err := user.LookupGroupId(currentUser.Gid)
	if err != nil {
		return nil, err
	}

	defaultOwner := currentUser.Username
	defaultGroup := currentGroup.Name

	// Resource defaults
	defaults := FileResource{
		BaseResource: BaseResource{
			Title: title,
			Type:  fileResourceType,
			State: StatePresent,
		},
		BaseFileResource: BaseFileResource{
			Path:  title,
			Mode:  0644,
			Owner: defaultOwner,
			Group: defaultGroup,
		},
	}

	var fr FileResource
	err = hcl.DecodeObject(&fr, obj)
	if err != nil {
		return nil, err
	}

	// Merge the decoded object with the resource defaults
	err = mergo.Merge(&fr, defaults)

	return &fr, err
}

// Evaluate evaluates the file resource
func (fr *FileResource) Evaluate(w io.Writer, opts *Options) (State, error) {
	rs := State{
		Current: StateUnknown,
		Want:    fr.State,
		Update:  false,
	}

	// The file we manage
	dst := utils.NewFileUtil(fr.Path)

	// File does not exist
	fi, err := os.Stat(fr.Path)
	if os.IsNotExist(err) {
		rs.Current = StateAbsent
		return rs, nil
	} else {
		rs.Current = StatePresent
	}

	// Ensure that the file we manage is a regular file
	if !fi.Mode().IsRegular() {
		return rs, fmt.Errorf("%s is not a regular file", fr.Path)
	}

	// Check file content
	if fr.Source != "" {
		srcPath := filepath.Join(opts.SiteDir, "data", fr.Source)
		same, err := dst.SameContentWith(srcPath)
		if err != nil {
			return rs, err
		}
		if !same {
			fr.Printf(w, "content is out of date\n")
			rs.Update = true
		}
	}

	// Check file permissions
	mode, err := dst.Mode()
	if err != nil {
		return rs, err
	}

	if mode.Perm() != os.FileMode(fr.Mode) {
		fr.Printf(w, "permissions are out of date\n")
		rs.Update = true
	}

	// Check ownership
	owner, err := dst.Owner()
	if err != nil {
		return rs, err
	}

	if fr.Owner != owner.User.Username || fr.Group != owner.Group.Name {
		fr.Printf(w, "owner is out of date\n")
		rs.Update = true
	}

	return rs, nil
}

// Create creates the file managed by the resource
func (fr *FileResource) Create(w io.Writer, opts *Options) error {
	dst := utils.NewFileUtil(fr.Path)
	fr.Printf(w, "creating file\n")

	// Set file content
	switch {
	case fr.Source == "" && dst.Exists():
		// Do nothing
		break
	case fr.Source == "" && !dst.Exists():
		// Create an empty file
		if _, err := os.Create(fr.Path); err != nil {
			return err
		}
	case fr.Source != "" && dst.Exists():
		// File exists and we have a source file
		srcPath := filepath.Join(opts.SiteDir, "data", fr.Source)
		if err := dst.CopyFrom(srcPath); err != nil {
			return err
		}
	}

	// Set file owner
	if err := dst.SetOwner(fr.Owner, fr.Group); err != nil {
		return err
	}

	// Set file permissions
	return dst.Chmod(os.FileMode(fr.Mode))
}

// Delete deletes the file
func (fr *FileResource) Delete(w io.Writer, opts *Options) error {
	fr.Printf(w, "removing file\n")
	dst := utils.NewFileUtil(fr.Path)

	return dst.Remove()
}

// Update updates the file managed by the resource
func (fr *FileResource) Update(w io.Writer, opts *Options) error {
	dst := utils.NewFileUtil(fr.Path)

	// Update file content if needed
	if fr.Source != "" {
		srcPath := filepath.Join(opts.SiteDir, "data", fr.Source)
		same, err := dst.SameContentWith(srcPath)
		if err != nil {
			return err
		}

		if !same {
			srcFile := utils.NewFileUtil(srcPath)
			srcMd5, err := srcFile.Md5()
			if err != nil {
				return err
			}

			fr.Printf(w, "updating content to md5:%s\n", srcMd5)
			if err := dst.CopyFrom(srcPath); err != nil {
				return err
			}
		}
	}

	// Fix permissions if needed
	mode, err := dst.Mode()
	if err != nil {
		return err
	}

	if mode.Perm() != os.FileMode(fr.Mode) {
		fr.Printf(w, "setting permissions to %#o\n", fr.Mode)
		dst.Chmod(os.FileMode(fr.Mode))
	}

	// Fix ownership if needed
	owner, err := dst.Owner()
	if err != nil {
		return err
	}

	if fr.Owner != owner.User.Username || fr.Group != owner.Group.Name {
		fr.Printf(w, "setting owner to %s:%s\n", fr.Owner, fr.Group)
		if err := dst.SetOwner(fr.Owner, fr.Group); err != nil {
			return err
		}
	}

	return nil
}

func init() {
	item := RegistryItem{
		Name:        fileResourceType,
		Description: fileResourceDesc,
		Provider:    NewFileResource,
	}

	Register(item)
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package rest

import (
	"encoding/json"
	"net/http"

	"github.com/couchbase/cbgt"
)

// TODO: Need to give the codebase a scrub of its log
// messages and fmt.Errorf()'s.

// LogGetHandler is a REST handler that retrieves recent log messages.
type LogGetHandler struct {
	mgr *cbgt.Manager
	mr  *cbgt.MsgRing
}

func NewLogGetHandler(
	mgr *cbgt.Manager, mr *cbgt.MsgRing) *LogGetHandler {
	return &LogGetHandler{mgr: mgr, mr: mr}
}

func (h *LogGetHandler) ServeHTTP(
	w http.ResponseWriter, req *http.Request) {
	w.Write([]byte(`{"messages":[`))
	for i, message := range h.mr.Messages() {
		buf, err := json.Marshal(string(message))
		if err == nil {
			if i > 0 {
				w.Write(cbgt.JsonComma)
			}
			w.Write(buf)
		}
	}
	w.Write([]byte(`],"events":[`))
	if h.mgr != nil {
		first := true
		h.mgr.Lock()
		p := h.mgr.Events().Front()
		for p != nil {
			if !first {
				w.Write(cbgt.JsonComma)
			}
			first = false
			w.Write(p.Value.([]byte))
			p = p.Next()
		}
		h.mgr.Unlock()
	}
	w.Write([]byte(`]}`))
}
<commit_msg>fixed #3 - check nil MsgRing in REST log handler<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package rest

import (
	"encoding/json"
	"net/http"

	"github.com/couchbase/cbgt"
)

// TODO: Need to give the codebase a scrub of its log
// messages and fmt.Errorf()'s.

// LogGetHandler is a REST handler that retrieves recent log messages.
type LogGetHandler struct {
	mgr *cbgt.Manager
	mr  *cbgt.MsgRing
}

func NewLogGetHandler(
	mgr *cbgt.Manager, mr *cbgt.MsgRing) *LogGetHandler {
	return &LogGetHandler{mgr: mgr, mr: mr}
}

func (h *LogGetHandler) ServeHTTP(
	w http.ResponseWriter, req *http.Request) {
	w.Write([]byte(`{"messages":[`))
	if h.mr != nil {
		for i, message := range h.mr.Messages() {
			buf, err := json.Marshal(string(message))
			if err == nil {
				if i > 0 {
					w.Write(cbgt.JsonComma)
				}
				w.Write(buf)
			}
		}
	}
	w.Write([]byte(`],"events":[`))
	if h.mgr != nil {
		first := true
		h.mgr.Lock()
		p := h.mgr.Events().Front()
		for p != nil {
			if !first {
				w.Write(cbgt.JsonComma)
			}
			first = false
			w.Write(p.Value.([]byte))
			p = p.Next()
		}
		h.mgr.Unlock()
	}
	w.Write([]byte(`]}`))
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/procfs"
	"k8s.io/klog"
	"os"
	"time"
)

var processStartTime = prometheus.NewGaugeVec(
	prometheus.GaugeOpts{
		Name: "process_start_time_seconds",
		Help: "Start time of the process since unix epoch in seconds.",
	},
	[]string{},
)

// RegisterProcessStartTime registers the process_start_time_seconds to
// a prometheus registry. This metric needs to be included to ensure counter
// data fidelity.
func RegisterProcessStartTime(registerer prometheus.Registerer) error {
	start, err := getProcessStart()
	if err != nil {
		klog.Errorf("Could not get process start time, %v", err)
		start = float64(time.Now().Unix())
	}
	processStartTime.WithLabelValues().Set(start)
	return registerer.Register(processStartTime)
}

func getProcessStart() (float64, error) {
	pid := os.Getpid()
	p, err := procfs.NewProc(pid)
	if err != nil {
		return 0, err
	}

	if stat, err := p.NewStat(); err == nil {
		return stat.StartTime()
	}
	return 0, err
}
<commit_msg>migrate kubelet's metrics/probes & metrics endpoint to metrics stability framework<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/procfs"
	"k8s.io/klog"
	"os"
	"time"
)

var processStartTime = prometheus.NewGaugeVec(
	prometheus.GaugeOpts{
		Name: "process_start_time_seconds",
		Help: "Start time of the process since unix epoch in seconds.",
	},
	[]string{},
)

// Registerer is an interface expected by RegisterProcessStartTime in order to register the metric
type Registerer interface {
	Register(prometheus.Collector) error
}

// RegisterProcessStartTime registers the process_start_time_seconds to
// a prometheus registry. This metric needs to be included to ensure counter
// data fidelity.
func RegisterProcessStartTime(registrationFunc func(prometheus.Collector) error) error {
	start, err := getProcessStart()
	if err != nil {
		klog.Errorf("Could not get process start time, %v", err)
		start = float64(time.Now().Unix())
	}
	processStartTime.WithLabelValues().Set(start)
	return registrationFunc(processStartTime)
}

func getProcessStart() (float64, error) {
	pid := os.Getpid()
	p, err := procfs.NewProc(pid)
	if err != nil {
		return 0, err
	}

	if stat, err := p.NewStat(); err == nil {
		return stat.StartTime()
	}
	return 0, err
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"flag"
	"encoding/json"
	"io/ioutil"
	"log"
	ph "github.com/advptr/reverse-proxy/proxyhandler"
	"crypto/x509"
)

// Main config
type Config struct {
	TrustFiles []string `json:"trust-files"`
	Routes []ph.Route `json:"routes""`
}

// Program  Options
type Options struct {
	ServerAddress string
	ConfigFile    string
}

// Main Bootstrap
func main() {
	args := args()
	config := config(args.ConfigFile)

	for _, r := range config.Routes {
		handler := ph.NewWSHandler(r, config.trustedCertPool())
		http.HandleFunc(handler.Path(), handler.Handle)
	}
	log.Printf("reverse-proxy on %v\n", args.ServerAddress)

	err := http.ListenAndServe(args.ServerAddress, nil)
	if err != nil {
		panic(err)
	}
}

// Parse command args
func args() Options {
	const (
		defaultServerAddress = ":80"
		serverAddressUsage   = "server address: ':80', '0.0.0.0:8080'..."
		defaultRouteConfig   = "config.json"
		routeConfigUsage     = "configuration file: 'config.json'"
	)
	address := flag.String("address", defaultServerAddress, serverAddressUsage)
	config := flag.String("config", defaultRouteConfig, routeConfigUsage)
	flag.Parse()

	return Options{*address, *config}
}

//
func (c *Config) trustedCertPool() (*x509.CertPool) {
	trustedCertPool := x509.NewCertPool()
	for _, file := range c.TrustFiles {
		trustedCert, err := ioutil.ReadFile(file)
		if err != nil {
			log.Fatal(err)
		}
		trustedCertPool.AppendCertsFromPEM(trustedCert);
	}

	return trustedCertPool
}

// Unmarshal routes from JSON configuration file
func config(configFile string) Config {
	file, err := ioutil.ReadFile(configFile)
	if err != nil {
		log.Fatal(err)
	}
	var config Config
	err = json.Unmarshal(file, &config)
	if err != nil {
		log.Fatal(err)
	}
	return config
}

<commit_msg>one cert pool is good enough.<commit_after>package main

import (
	"net/http"
	"flag"
	"encoding/json"
	"io/ioutil"
	"log"
	ph "github.com/advptr/reverse-proxy/proxyhandler"
	"crypto/x509"
)

// Main config
type Config struct {
	TrustFiles []string `json:"trust-files"`
	Routes []ph.Route `json:"routes""`
}

// Program  Options
type Options struct {
	ServerAddress string
	ConfigFile    string
}

// Main Bootstrap
func main() {
	args := args()
	config := config(args.ConfigFile)
	trustedCertPool := config.trustedCertPool()
	for _, r := range config.Routes {
		handler := ph.NewWSHandler(r, trustedCertPool)
		http.HandleFunc(handler.Path(), handler.Handle)
	}
	log.Printf("reverse-proxy on %v\n", args.ServerAddress)

	err := http.ListenAndServe(args.ServerAddress, nil)
	if err != nil {
		panic(err)
	}
}

// Parse command args
func args() Options {
	const (
		defaultServerAddress = ":80"
		serverAddressUsage   = "server address: ':80', '0.0.0.0:8080'..."
		defaultRouteConfig   = "config.json"
		routeConfigUsage     = "configuration file: 'config.json'"
	)
	address := flag.String("address", defaultServerAddress, serverAddressUsage)
	config := flag.String("config", defaultRouteConfig, routeConfigUsage)
	flag.Parse()

	return Options{*address, *config}
}

//
func (c *Config) trustedCertPool() (*x509.CertPool) {
	trustedCertPool := x509.NewCertPool()
	for _, file := range c.TrustFiles {
		trustedCert, err := ioutil.ReadFile(file)
		if err != nil {
			log.Fatal(err)
		}
		trustedCertPool.AppendCertsFromPEM(trustedCert);
	}

	return trustedCertPool
}

// Unmarshal routes from JSON configuration file
func config(configFile string) Config {
	file, err := ioutil.ReadFile(configFile)
	if err != nil {
		log.Fatal(err)
	}
	var config Config
	err = json.Unmarshal(file, &config)
	if err != nil {
		log.Fatal(err)
	}
	return config
}

<|endoftext|>


<commit_before>package main

type Point struct {
	X int
	Y int
}

type Direction int

const (
	North Direction = iota
	South
	East
	West
)

func (d Direction) Reverse() Direction {
	switch d {
	case North:
		return South
	case South:
		return North
	case East:
		return West
	case West:
		return East
	default:
		return North
	}
}

type Exits map[Direction]*Room

type Room struct {
	Name        string
	Description string
	Position    Point
	Exits       Exits
	Dungeon     *Dungeon
}

type Dungeon struct {
	Rooms map[Point]*Room
}

// OpenExit creates a connection from the receiver's Room to another Room. If
// a Room already exists in the Direction specified, Exits on both Rooms are
// linked. If no Room exists, a new one is created and added to the Dungeon.
// If an exit is already open in that Direction, returns false.
func (r *Room) OpenExit(direction Direction) bool {
	if _, present := r.Exits[direction]; present {
		return false
	}

	changeX := 0
	changeY := 0

	switch direction {
	case North:
		changeY = -1
	case South:
		changeY = 1
	case East:
		changeX = 1
	case West:
		changeX = -1
	default:
		return false
	}

	destinationPosition := Point{
		X: r.Position.X + changeX,
		Y: r.Position.Y + changeY,
	}

	if destinationRoom, roomPresent := r.Dungeon.Rooms[destinationPosition]; roomPresent {
		r.Exits[direction] = destinationRoom
		destinationRoom.Exits[direction.Reverse()] = r
	}

	return true
}
<commit_msg>Add Dungeon initializer NewDungeon<commit_after>package main

type Point struct {
	X int
	Y int
}

type Direction int

const (
	North Direction = iota
	South
	East
	West
)

func (d Direction) Reverse() Direction {
	switch d {
	case North:
		return South
	case South:
		return North
	case East:
		return West
	case West:
		return East
	default:
		return North
	}
}

type Exits map[Direction]*Room

type Room struct {
	Name        string
	Description string
	Position    Point
	Exits       Exits
	Dungeon     *Dungeon
}

type Dungeon struct {
	Rooms map[Point]*Room
}

// OpenExit creates a connection from the receiver's Room to another Room. If
// a Room already exists in the Direction specified, Exits on both Rooms are
// linked. If no Room exists, a new one is created and added to the Dungeon.
// If an exit is already open in that Direction, returns false.
func (r *Room) OpenExit(direction Direction) bool {
	if _, present := r.Exits[direction]; present {
		return false
	}

	changeX := 0
	changeY := 0

	switch direction {
	case North:
		changeY = -1
	case South:
		changeY = 1
	case East:
		changeX = 1
	case West:
		changeX = -1
	default:
		return false
	}

	destinationPosition := Point{
		X: r.Position.X + changeX,
		Y: r.Position.Y + changeY,
	}

	if destinationRoom, roomPresent := r.Dungeon.Rooms[destinationPosition]; roomPresent {
		r.Exits[direction] = destinationRoom
		destinationRoom.Exits[direction.Reverse()] = r
	}

	return true
}

// NewDungeon returns a new Dungeon type initialized with one empty Room.
func NewDungeon() *Dungeon {
	firstRoom := &Room{}
	rooms := make(map[Point]*Room)
	rooms[Point{X: 0, Y: 0}] = firstRoom

	return &Dungeon{
		Rooms: rooms,
	}
}
<|endoftext|>


<commit_before>// Package durafmt formats time.Duration into a human readable format.
package durafmt

import (
	"errors"
	"strconv"
	"strings"
	"time"
)

var (
	units = []string{"years", "weeks", "days", "hours", "minutes", "seconds", "milliseconds"}
)

// Durafmt holds the parsed duration and the original input duration.
type Durafmt struct {
	duration time.Duration
	input    string // Used as reference.
}

// Parse creates a new *Durafmt struct, returns error if input is invalid.
func Parse(dinput time.Duration) *Durafmt {
	input := dinput.String()
	return &Durafmt{dinput, input}
}

// ParseString creates a new *Durafmt struct from a string.
// returns an error if input is invalid.
func ParseString(input string) (*Durafmt, error) {
	if input == "0" || input == "-0" {
		return nil, errors.New("durafmt: missing unit in duration " + input)
	}
	duration, err := time.ParseDuration(input)
	if err != nil {
		return nil, err
	}
	return &Durafmt{duration, input}, nil
}

// String parses d *Durafmt into a human readable duration.
func (d *Durafmt) String() string {
	var duration string

	// Check for minus durations.
	if string(d.input[0]) == "-" {
		duration += "-"
		d.duration = -d.duration
	}

	// Convert duration.
	seconds := int64(d.duration.Seconds()) % 60
	minutes := int64(d.duration.Minutes()) % 60
	hours := int64(d.duration.Hours()) % 24
	days := int64(d.duration/(24*time.Hour)) % 365 % 7
	weeks := int64(d.duration/(24*time.Hour)) / 7 % 52
	years := int64(d.duration/(24*time.Hour)) / 365
	milliseconds := int64(d.duration/time.Millisecond) -
		(seconds * 1000) - (minutes * 60000) - (hours * 3600000) -
		(days * 86400000) - (weeks * 604800000) - (years * 31536000000)

	// Create a map of the converted duration time.
	durationMap := map[string]int64{
		"milliseconds": milliseconds,
		"seconds":      seconds,
		"minutes":      minutes,
		"hours":        hours,
		"days":         days,
		"weeks":        weeks,
		"years":        years,
	}

	// Construct duration string.
	for _, u := range units {
		v := durationMap[u]
		strval := strconv.FormatInt(v, 10)
		switch {
		// add to the duration string if v > 1.
		case v > 1:
			duration += strval + " " + u + " "
		// remove the plural 's', if v is 1.
		case v == 1:
			duration += strval + " " + strings.TrimRight(u, "s") + " "
		// omit any value with 0s or 0.
		case d.duration.String() == "0" || d.duration.String() == "0s":
			// note: milliseconds and minutes have the same suffix (m)
			// so we have to check if the units match with the suffix.

			// check for a suffix that is NOT the milliseconds suffix.
			if strings.HasSuffix(d.input, string(u[0])) && !strings.Contains(d.input, "ms") {
				// if it happens that the units are milliseconds, skip.
				if u == "milliseconds" {
					continue
				}
				duration += strval + " " + u
			}
			// process milliseconds here.
			if u == "milliseconds" {
				if strings.Contains(d.input, "ms") {
					duration += strval + " " + u
					break
				}
			}
			break
		// omit any value with 0.
		case v == 0:
			continue
		}
	}
	// trim any remaining spaces.
	duration = strings.TrimSpace(duration)
	return duration
}
<commit_msg>solving the edge case between 364 and 365 days tracked as https://github.com/hako/durafmt/issues/7<commit_after>// Package durafmt formats time.Duration into a human readable format.
package durafmt

import (
	"errors"
	"strconv"
	"strings"
	"time"
)

var (
	units = []string{"years", "weeks", "days", "hours", "minutes", "seconds", "milliseconds"}
)

// Durafmt holds the parsed duration and the original input duration.
type Durafmt struct {
	duration time.Duration
	input    string // Used as reference.
}

// Parse creates a new *Durafmt struct, returns error if input is invalid.
func Parse(dinput time.Duration) *Durafmt {
	input := dinput.String()
	return &Durafmt{dinput, input}
}

// ParseString creates a new *Durafmt struct from a string.
// returns an error if input is invalid.
func ParseString(input string) (*Durafmt, error) {
	if input == "0" || input == "-0" {
		return nil, errors.New("durafmt: missing unit in duration " + input)
	}
	duration, err := time.ParseDuration(input)
	if err != nil {
		return nil, err
	}
	return &Durafmt{duration, input}, nil
}

// String parses d *Durafmt into a human readable duration.
func (d *Durafmt) String() string {
	var duration string

	// Check for minus durations.
	if string(d.input[0]) == "-" {
		duration += "-"
		d.duration = -d.duration
	}

	// Convert duration.
	seconds := int64(d.duration.Seconds()) % 60
	minutes := int64(d.duration.Minutes()) % 60
	hours := int64(d.duration.Hours()) % 24
	days := int64(d.duration/(24*time.Hour)) % 365 % 7
	weeks := int64(d.duration/(24*time.Hour)) / 7 % 52

	// edge case between 364 and 365 days
	total_days := int64(d.duration/(24*time.Hour))
        if total_days >= 364 && total_days < 365 {
		weeks = 52
	}

	years := int64(d.duration/(24*time.Hour)) / 365
	milliseconds := int64(d.duration/time.Millisecond) -
		(seconds * 1000) - (minutes * 60000) - (hours * 3600000) -
		(days * 86400000) - (weeks * 604800000) - (years * 31536000000)

	// Create a map of the converted duration time.
	durationMap := map[string]int64{
		"milliseconds": milliseconds,
		"seconds":      seconds,
		"minutes":      minutes,
		"hours":        hours,
		"days":         days,
		"weeks":        weeks,
		"years":        years,
	}

	// Construct duration string.
	for _, u := range units {
		v := durationMap[u]
		strval := strconv.FormatInt(v, 10)
		switch {
		// add to the duration string if v > 1.
		case v > 1:
			duration += strval + " " + u + " "
		// remove the plural 's', if v is 1.
		case v == 1:
			duration += strval + " " + strings.TrimRight(u, "s") + " "
		// omit any value with 0s or 0.
		case d.duration.String() == "0" || d.duration.String() == "0s":
			// note: milliseconds and minutes have the same suffix (m)
			// so we have to check if the units match with the suffix.

			// check for a suffix that is NOT the milliseconds suffix.
			if strings.HasSuffix(d.input, string(u[0])) && !strings.Contains(d.input, "ms") {
				// if it happens that the units are milliseconds, skip.
				if u == "milliseconds" {
					continue
				}
				duration += strval + " " + u
			}
			// process milliseconds here.
			if u == "milliseconds" {
				if strings.Contains(d.input, "ms") {
					duration += strval + " " + u
					break
				}
			}
			break
		// omit any value with 0.
		case v == 0:
			continue
		}
	}
	// trim any remaining spaces.
	duration = strings.TrimSpace(duration)
	return duration
}
<|endoftext|>


<commit_before>package logger

import (
	"strconv"
	"time"
	"strings"
	"sync"

	"github.com/kataras/iris"
	"github.com/iris-contrib/logger"
	"github.com/reconquest/loreley"
)

type loggerMiddleware struct {
	*logger.Logger
	config Config
}

var mapLock       sync.RWMutex

// Serve serves the middleware
func (l *loggerMiddleware) Serve(ctx *iris.Context) {
	//all except latency to string
	var date, timed, latency, status, ip, method, path string
	var startTime, endTime time.Time
	path = ctx.PathString()
	method = ctx.MethodString()

	startTime = time.Now()

	ctx.Next()
	//no time.Since in order to format it well after
	endTime = time.Now()
	timed = rightPad2Len(endTime.Format("15:04:05.999999"), "0", 15)
	date = endTime.Format("02/01/2006")
	latency = endTime.Sub(startTime).String()
	parts := strings.Split(latency,".")
	if(len(parts) == 2) {
	    	if(!strings.Contains(parts[1],"ms")) {
			parts[1] = leftPad2Len(parts[1],"0",6)
		}
		latency = parts[0] + "." + parts[1]
	}
	latency = leftPad2Len(latency," ",14)

	if l.config.Status {
		status = strconv.Itoa(ctx.Response.StatusCode())
	}

	if l.config.IP {
		ip = leftPad2Len(ctx.RemoteAddr()," ",15)
	}

	if !l.config.Method {
		method = ""
	}

	if !l.config.Path {
		path = ""
	}

	getText, _ := loreley.CompileAndExecuteToString(
		`{bold}{fg 15}{bg 40} GET  {from "" 0}{reset}`,
		nil,
		nil,
	)
	postText, _ := loreley.CompileAndExecuteToString(
		`{bold}{fg 15}{bg 21} POST {from "" 0}{reset}`,
		nil,
		nil,
	)
	headText, _ := loreley.CompileAndExecuteToString(
		`{bold}{fg 15}{bg 53} HEAD {from "" 0}{reset}`,
		nil,
		nil,
	)
	putText, _ := loreley.CompileAndExecuteToString(
		`{bold}{fg 15}{bg 208} PUT  {from "" 0}{reset}`,
		nil,
		nil,
	)
	delText, _ := loreley.CompileAndExecuteToString(
		`{bold}{fg 15}{bg 160} DEL  {from "" 0}{reset}`,
		nil,
		nil,
	)

	//finally print the logs
	if(method == "GET") {
		mapLock.RLock()
		l.printf("%s %s - %s | %v | %4v | %s | %s \n", getText, timed, date, status, latency, ip, path)
		mapLock.RUnlock()
	} else if method == "POST" {
		mapLock.RLock()
		l.printf("%s %s - %s | %v | %4v | %s | %s \n", postText, timed, date, status, latency, ip, path)
		mapLock.RUnlock()
	} else if method == "PUT" {
		mapLock.RLock()
		l.printf("%s %s - %s | %v | %4v | %s | %s \n", putText, timed, date, status, latency, ip, path)
		mapLock.RUnlock()
	} else if method == "HEAD" {
		mapLock.RLock()
		l.printf("%s %s - %s | %v | %4v | %s | %s \n", headText, timed, date, status, latency, ip, path)
		mapLock.RUnlock()
	} else if method == "DELETE" {
		mapLock.RLock()
		l.printf("%s %s - %s | %v | %4v | %s | %s \n", delText, timed, date, status, latency, ip, path)
		mapLock.RUnlock()
	} else {
		mapLock.RLock()
		l.printf("%s - %s %v %4v %s %s %s \n", timed, date, status, latency, ip, method, path)
		mapLock.RUnlock()
	}

}

func rightPad2Len(s string, padStr string, overallLen int) string{
	var padCountInt int
	padCountInt = 1 + ((overallLen-len(padStr))/len(padStr))
	var retStr =  s + strings.Repeat(padStr, padCountInt)
	return retStr[:overallLen]
}

func leftPad2Len(s string, padStr string, overallLen int) string{
	var padCountInt int
	padCountInt = 1 + ((overallLen-len(padStr))/len(padStr))
	var retStr = strings.Repeat(padStr, padCountInt) + s
	return retStr[(len(retStr)-overallLen):]
}

func (l *loggerMiddleware) printf(format string, a ...interface{}) {
	if l.config.EnableColors {
		l.Logger.Otherf(format, a...)
	} else {
		l.Logger.Printf(format, a...)
	}
}

// New returns the logger middleware
// receives two parameters, both of them optionals
// first is the logger, which normally you set to the 'iris.Logger'
// if logger is nil then the middlewares makes one with the default configs.
// second is optional configs(logger.Config)
func New(theLogger *logger.Logger, cfg ...Config) iris.HandlerFunc {
	if theLogger == nil {
		theLogger = logger.New(logger.DefaultConfig())
	}
	c := DefaultConfig().Merge(cfg)
	l := &loggerMiddleware{Logger: theLogger, config: c}

	return l.Serve
}
<commit_msg>Update logger.go<commit_after>package logger

import (
	"strconv"
	"time"
	"strings"
	"sync"

	"github.com/kataras/iris"
	"github.com/iris-contrib/logger"
	"github.com/reconquest/loreley"
	"github.com/fatih/color"
)

type loggerMiddleware struct {
	*logger.Logger
	config Config
}

var mapLock       sync.RWMutex

// Serve serves the middleware
func (l *loggerMiddleware) Serve(ctx *iris.Context) {
	//all except latency to string
	var date, timed, latency, status, ip, method, path string
	var startTime, endTime time.Time
	path = ctx.PathString()
	method = ctx.MethodString()

	startTime = time.Now()

	ctx.Next()
	//no time.Since in order to format it well after
	endTime = time.Now()
	timed = rightPad2Len(endTime.Format("15:04:05.999999"), "0", 15)
	date = endTime.Format("02/01/2006")
	latency = endTime.Sub(startTime).String()
	parts := strings.Split(latency,".")
	if(len(parts) == 2) {
	    if(!strings.Contains(parts[1],"ms")) {
			parts[1] = leftPad2Len(parts[1],"0",6)
		}
		latency = parts[0] + "." + parts[1]
		latency = leftPad2Len(latency," ",15)
	} else {
		latency = leftPad2Len(latency," ",14)
	}

	if l.config.Status {
		status = strconv.Itoa(ctx.Response.StatusCode())
	}

	if l.config.IP {
		ip = leftPad2Len(ctx.RemoteAddr()," ",15)
	}

	if !l.config.Method {
		method = ""
	}

	if !l.config.Path {
		path = ""
	}

	getText, _ := loreley.CompileAndExecuteToString(
		`{bold}{fg 15}{bg 40} GET  {from "" 0}{reset}`,
		nil,
		nil,
	)
	postText, _ := loreley.CompileAndExecuteToString(
		`{bold}{fg 15}{bg 21} POST {from "" 0}{reset}`,
		nil,
		nil,
	)
	headText, _ := loreley.CompileAndExecuteToString(
		`{bold}{fg 15}{bg 53} HEAD {from "" 0}{reset}`,
		nil,
		nil,
	)
	putText, _ := loreley.CompileAndExecuteToString(
		`{bold}{fg 15}{bg 208} PUT  {from "" 0}{reset}`,
		nil,
		nil,
	)
	delText, _ := loreley.CompileAndExecuteToString(
		`{bold}{fg 15}{bg 160} DEL  {from "" 0}{reset}`,
		nil,
		nil,
	)

	if(status == "200" || status == "201") {
		status = color.GreenString(status)
	} else if(status == "404" || status == "500" || status == "403") {
		status = color.RedString(status)
	}

	//finally print the logs
	if(method == "GET") {
		mapLock.RLock()
		l.printf("%s %s - %s | %v | %4v | %s | %s \n", getText, timed, date, status, latency, ip, path)
		mapLock.RUnlock()
	} else if method == "POST" {
		mapLock.RLock()
		l.printf("%s %s - %s | %v | %4v | %s | %s \n", postText, timed, date, status, latency, ip, path)
		mapLock.RUnlock()
	} else if method == "PUT" {
		mapLock.RLock()
		l.printf("%s %s - %s | %v | %4v | %s | %s \n", putText, timed, date, status, latency, ip, path)
		mapLock.RUnlock()
	} else if method == "HEAD" {
		mapLock.RLock()
		l.printf("%s %s - %s | %v | %4v | %s | %s \n", headText, timed, date, status, latency, ip, path)
		mapLock.RUnlock()
	} else if method == "DELETE" {
		mapLock.RLock()
		l.printf("%s %s - %s | %v | %4v | %s | %s \n", delText, timed, date, status, latency, ip, path)
		mapLock.RUnlock()
	} else {
		mapLock.RLock()
		l.printf("%s - %s %v %4v %s %s %s \n", timed, date, status, latency, ip, method, path)
		mapLock.RUnlock()
	}

}

func rightPad2Len(s string, padStr string, overallLen int) string{
	var padCountInt int
	padCountInt = 1 + ((overallLen-len(padStr))/len(padStr))
	var retStr =  s + strings.Repeat(padStr, padCountInt)
	return retStr[:overallLen]
}

func leftPad2Len(s string, padStr string, overallLen int) string{
	var padCountInt int
	padCountInt = 1 + ((overallLen-len(padStr))/len(padStr))
	var retStr = strings.Repeat(padStr, padCountInt) + s
	return retStr[(len(retStr)-overallLen):]
}

func (l *loggerMiddleware) printf(format string, a ...interface{}) {
	if l.config.EnableColors {
		l.Logger.Otherf(format, a...)
	} else {
		l.Logger.Printf(format, a...)
	}
}

// New returns the logger middleware
// receives two parameters, both of them optionals
// first is the logger, which normally you set to the 'iris.Logger'
// if logger is nil then the middlewares makes one with the default configs.
// second is optional configs(logger.Config)
func New(theLogger *logger.Logger, cfg ...Config) iris.HandlerFunc {
	if theLogger == nil {
		theLogger = logger.New(logger.DefaultConfig())
	}
	c := DefaultConfig().Merge(cfg)
	l := &loggerMiddleware{Logger: theLogger, config: c}

	return l.Serve
}
<|endoftext|>


<commit_before>package goble

import (
	"log"

	"github.com/dim13/goble/xpc"
)

const ALL = "__allEvents__"

// Event generated by blued, with associated data
type Event struct {
	Name               string
	State              string
	DeviceUUID         xpc.UUID
	ServiceUuid        string
	CharacteristicUuid string
	Peripheral         Peripheral
	Data               []byte
	Mtu                int
	IsNotification     bool
}

// The event handler function.
// Return true to terminate
type EventHandlerFunc func(Event) bool

// Emitter is an object to emit and handle Event(s)
type Emitter struct {
	handlers map[string]EventHandlerFunc
	event    chan Event
	verbose  bool
}

// Init initialize the emitter and start a goroutine to execute the event handlers
func (e *Emitter) Init() {
	e.handlers = make(map[string]EventHandlerFunc)
	e.event = make(chan Event)

	// event handler
	go func() {
		defer close(e.event) // FIXME: this causes new "emits" to panic.
		for ev := range e.event {
			if fn, ok := e.handlers[ev.Name]; ok {
				if fn(ev) {
					return
				}
			} else if fn, ok := e.handlers[ALL]; ok {
				if fn(ev) {
					return
				}
			} else {
				if e.verbose {
					log.Println("unhandled Emit", ev)
				}
			}
		}
	}()
}

func (e *Emitter) SetVerbose(v bool) {
	e.verbose = v
}

// Emit sends the event on the 'event' channel
func (e *Emitter) Emit(ev Event) {
	e.event <- ev
}

// Register an handler for the specified event
func (e *Emitter) Register(event string, fn EventHandlerFunc) {
	e.handlers[event] = fn
}

// Deregister an handler for the specified event
func (e *Emitter) Deregister(event string) {
	delete(e.handlers, event)
}
<commit_msg>Don't panic<commit_after>package goble

import (
	"log"

	"github.com/dim13/goble/xpc"
)

const ALL = "__allEvents__"

// Event generated by blued, with associated data
type Event struct {
	Name               string
	State              string
	DeviceUUID         xpc.UUID
	ServiceUuid        string
	CharacteristicUuid string
	Peripheral         Peripheral
	Data               []byte
	Mtu                int
	IsNotification     bool
}

// The event handler function.
// Return true to terminate
type EventHandlerFunc func(Event) bool

// Emitter is an object to emit and handle Event(s)
type Emitter struct {
	handlers map[string]EventHandlerFunc
	event    chan Event
	verbose  bool
}

// Init initialize the emitter and start a goroutine to execute the event handlers
func (e *Emitter) Init() {
	e.handlers = make(map[string]EventHandlerFunc)
	e.event = make(chan Event)

	// event handler
	go func() {
		for ev := range e.event {
			fn, ok := e.handlers[ev.Name]
			if !ok {
				fn, ok = e.handlers[ALL]
			}
			if ok {
				go func(ev Event) {
					if fn(ev) {
						close(e.event)
					}
				}(ev)
			} else if e.verbose {
				log.Println("unhandled Emit", ev)
			}
		}
	}()
}

func (e *Emitter) SetVerbose(v bool) {
	e.verbose = v
}

// Emit sends the event on the 'event' channel
func (e *Emitter) Emit(ev Event) {
	select {
	case e.event <- ev:
	default:
		log.Println("skip Event", ev)
	}
}

// Register an handler for the specified event
func (e *Emitter) Register(event string, fn EventHandlerFunc) {
	e.handlers[event] = fn
}

// Deregister an handler for the specified event
func (e *Emitter) Deregister(event string) {
	delete(e.handlers, event)
}
<|endoftext|>


<commit_before>package eventemitter

import (
	"sync"
)

// Emitter is the base struct which manages event subscriptions and calls all registered handlers on event emits.
type Emitter struct {
	sync.RWMutex

	async         bool
	capturers     []*Capturer
	listeners     map[EventType][]*Listener
	listenersOnce map[EventType][]*Listener
}

// NewEmitter creates a new event emitter that implements the Observable interface.
// Async determines whether events listeners fire in separate goroutines or not.
func NewEmitter(async bool) (em *Emitter) {
	em = &Emitter{
		async:         async,
		listeners:     make(map[EventType][]*Listener),
		listenersOnce: make(map[EventType][]*Listener),
	}
	return em
}

// EmitEvent emits the given event to all listeners and capturers
func (em *Emitter) EmitEvent(event EventType, arguments ...interface{}) {
	// If we have no single listeners for this event, skip
	if len(em.listenersOnce) > 0 {
		// Get a full lock, we are changing a map
		em.Lock()
		// Copy the slice
		listenersOnce := em.listenersOnce[event]
		// Create new empty slice
		em.listenersOnce[event] = make([]*Listener, 0)
		em.Unlock()

		// No lock needed, we are working with an inaccessible copy
		em.emitListenerEvents(listenersOnce, arguments)
	}

	// If we have no listeners for this event, skip
	if len(em.listeners[event]) > 0 {
		em.RLock()
		em.emitListenerEvents(em.listeners[event], arguments)
		em.RUnlock()
	}

	// If we have no capturers, skip
	if len(em.capturers) > 0 {
		em.RLock()
		em.emitCapturerEvents(em.capturers, event, arguments)
		em.RUnlock()
	}
}

func (em *Emitter) emitListenerEvents(listeners []*Listener, arguments []interface{}) {
	for _, listener := range listeners {
		if em.async {
			go listener.handler(arguments...)
			continue
		}
		listener.handler(arguments...)
	}
}

func (em *Emitter) emitCapturerEvents(capturers []*Capturer, event EventType, arguments []interface{}) {
	for _, capturer := range capturers {
		if em.async {
			go capturer.handler(event, arguments...)
			continue
		}
		capturer.handler(event, arguments...)
	}
}

// AddListener adds a listener for the given event type
func (em *Emitter) AddListener(event EventType, handler HandleFunc) (listener *Listener) {
	em.Lock()
	defer em.Unlock()

	listener = &Listener{
		handler: handler,
	}
	em.listeners[event] = append(em.listeners[event], listener)
	return listener
}

// ListenOnce adds a listener for the given event type that removes itself after it has been fired once
func (em *Emitter) ListenOnce(event EventType, handler HandleFunc) (listener *Listener) {
	em.Lock()
	defer em.Unlock()

	listener = &Listener{
		handler: handler,
	}
	em.listenersOnce[event] = append(em.listenersOnce[event], listener)
	return listener
}

// AddCapturer adds an event capturer for all events
func (em *Emitter) AddCapturer(handler CaptureFunc) (capturer *Capturer) {
	em.Lock()
	defer em.Unlock()

	capturer = &Capturer{
		handler: handler,
	}
	em.capturers = append(em.capturers, capturer)
	return capturer
}

// RemoveListener removes the registered given listener for the given event
func (em *Emitter) RemoveListener(event EventType, listener *Listener) {
	em.Lock()
	defer em.Unlock()

	for index, list := range em.listeners[event] {
		if list == listener {
			em.removeListenerAt(event, index)
			return
		}
	}

	// If it hasnt been found yet, remove from listeners once if present there
	for index, list := range em.listenersOnce[event] {
		if list == listener {
			em.removeOnceListenerAt(event, index)
			return
		}
	}
}

// RemoveAllListenersForEvent removes all registered listeners for a given event type
func (em *Emitter) RemoveAllListenersForEvent(event EventType) {
	em.Lock()
	defer em.Unlock()

	em.listeners[event] = make([]*Listener, 0)
}

// RemoveAllListeners removes all registered listeners for all event types
func (em *Emitter) RemoveAllListeners() {
	em.Lock()
	defer em.Unlock()

	em.listeners = make(map[EventType][]*Listener)
	em.listenersOnce = make(map[EventType][]*Listener)
}

// RemoveCapturer removes the given capturer
func (em *Emitter) RemoveCapturer(capturer *Capturer) {
	em.Lock()
	defer em.Unlock()

	for index, capt := range em.capturers {
		if capt == capturer {
			em.removeCapturerAt(index)
			return
		}
	}
}

// RemoveAllCapturers removes all registered capturers
func (em *Emitter) RemoveAllCapturers() {
	em.Lock()
	defer em.Unlock()

	em.capturers = make([]*Capturer, 0)
}

func (em *Emitter) removeListenerAt(event EventType, index int) {
	copy(em.listeners[event][index:], em.listeners[event][index+1:])
	em.listeners[event][len(em.listeners[event])-1] = nil
	em.listeners[event] = em.listeners[event][:len(em.listeners[event])-1]
}

func (em *Emitter) removeOnceListenerAt(event EventType, index int) {
	copy(em.listenersOnce[event][index:], em.listenersOnce[event][index+1:])
	em.listenersOnce[event][len(em.listenersOnce[event])-1] = nil
	em.listenersOnce[event] = em.listenersOnce[event][:len(em.listenersOnce[event])-1]
}

func (em *Emitter) removeCapturerAt(index int) {
	copy(em.capturers[index:], em.capturers[index+1:])
	em.capturers[len(em.capturers)-1] = nil
	em.capturers = em.capturers[:len(em.capturers)-1]
}
<commit_msg>Make the emitter lock private and fix data races when doing len() on maps<commit_after>package eventemitter

import (
	"sync"
)

// Emitter is the base struct which manages event subscriptions and calls all registered handlers on event emits.
type Emitter struct {
	mu            sync.RWMutex
	async         bool
	capturers     []*Capturer
	listeners     map[EventType][]*Listener
	listenersOnce map[EventType][]*Listener
}

// NewEmitter creates a new event emitter that implements the Observable interface.
// Async determines whether events listeners fire in separate goroutines or not.
func NewEmitter(async bool) (em *Emitter) {
	em = &Emitter{
		async:         async,
		listeners:     make(map[EventType][]*Listener),
		listenersOnce: make(map[EventType][]*Listener),
	}
	return em
}

// EmitEvent emits the given event to all listeners and capturers
func (em *Emitter) EmitEvent(event EventType, arguments ...interface{}) {
	// If we have no single listeners for this event, skip
	em.mu.RLock()
	if len(em.listenersOnce) > 0 {
		em.mu.RUnlock()
		// Get a full lock, we are changing a map
		em.mu.Lock()
		// Copy the slice
		listenersOnce := em.listenersOnce[event]
		// Create new empty slice
		em.listenersOnce[event] = make([]*Listener, 0)
		em.mu.Unlock()

		// No lock needed, we are working with an inaccessible copy
		em.emitListenerEvents(listenersOnce, arguments)
	} else {
		em.mu.RUnlock()
	}

	em.mu.RLock()
	// If we have no listeners for this event, skip
	if len(em.listeners[event]) > 0 {
		em.emitListenerEvents(em.listeners[event], arguments)
	}
	em.mu.RUnlock()

	em.mu.RLock()
	// If we have no capturers, skip
	if len(em.capturers) > 0 {
		em.emitCapturerEvents(em.capturers, event, arguments)
	}
	em.mu.RUnlock()
}

func (em *Emitter) emitListenerEvents(listeners []*Listener, arguments []interface{}) {
	for _, listener := range listeners {
		if em.async {
			go listener.handler(arguments...)
			continue
		}
		listener.handler(arguments...)
	}
}

func (em *Emitter) emitCapturerEvents(capturers []*Capturer, event EventType, arguments []interface{}) {
	for _, capturer := range capturers {
		if em.async {
			go capturer.handler(event, arguments...)
			continue
		}
		capturer.handler(event, arguments...)
	}
}

// AddListener adds a listener for the given event type
func (em *Emitter) AddListener(event EventType, handler HandleFunc) (listener *Listener) {
	em.mu.Lock()
	defer em.mu.Unlock()

	listener = &Listener{
		handler: handler,
	}
	em.listeners[event] = append(em.listeners[event], listener)
	return listener
}

// ListenOnce adds a listener for the given event type that removes itself after it has been fired once
func (em *Emitter) ListenOnce(event EventType, handler HandleFunc) (listener *Listener) {
	em.mu.Lock()
	defer em.mu.Unlock()

	listener = &Listener{
		handler: handler,
	}
	em.listenersOnce[event] = append(em.listenersOnce[event], listener)
	return listener
}

// AddCapturer adds an event capturer for all events
func (em *Emitter) AddCapturer(handler CaptureFunc) (capturer *Capturer) {
	em.mu.Lock()
	defer em.mu.Unlock()

	capturer = &Capturer{
		handler: handler,
	}
	em.capturers = append(em.capturers, capturer)
	return capturer
}

// RemoveListener removes the registered given listener for the given event
func (em *Emitter) RemoveListener(event EventType, listener *Listener) {
	em.mu.Lock()
	defer em.mu.Unlock()

	for index, list := range em.listeners[event] {
		if list == listener {
			em.removeListenerAt(event, index)
			return
		}
	}

	// If it hasnt been found yet, remove from listeners once if present there
	for index, list := range em.listenersOnce[event] {
		if list == listener {
			em.removeOnceListenerAt(event, index)
			return
		}
	}
}

// RemoveAllListenersForEvent removes all registered listeners for a given event type
func (em *Emitter) RemoveAllListenersForEvent(event EventType) {
	em.mu.Lock()
	defer em.mu.Unlock()

	em.listeners[event] = make([]*Listener, 0)
}

// RemoveAllListeners removes all registered listeners for all event types
func (em *Emitter) RemoveAllListeners() {
	em.mu.Lock()
	defer em.mu.Unlock()

	em.listeners = make(map[EventType][]*Listener)
	em.listenersOnce = make(map[EventType][]*Listener)
}

// RemoveCapturer removes the given capturer
func (em *Emitter) RemoveCapturer(capturer *Capturer) {
	em.mu.Lock()
	defer em.mu.Unlock()

	for index, capt := range em.capturers {
		if capt == capturer {
			em.removeCapturerAt(index)
			return
		}
	}
}

// RemoveAllCapturers removes all registered capturers
func (em *Emitter) RemoveAllCapturers() {
	em.mu.Lock()
	defer em.mu.Unlock()

	em.capturers = make([]*Capturer, 0)
}

func (em *Emitter) removeListenerAt(event EventType, index int) {
	copy(em.listeners[event][index:], em.listeners[event][index+1:])
	em.listeners[event][len(em.listeners[event])-1] = nil
	em.listeners[event] = em.listeners[event][:len(em.listeners[event])-1]
}

func (em *Emitter) removeOnceListenerAt(event EventType, index int) {
	copy(em.listenersOnce[event][index:], em.listenersOnce[event][index+1:])
	em.listenersOnce[event][len(em.listenersOnce[event])-1] = nil
	em.listenersOnce[event] = em.listenersOnce[event][:len(em.listenersOnce[event])-1]
}

func (em *Emitter) removeCapturerAt(index int) {
	copy(em.capturers[index:], em.capturers[index+1:])
	em.capturers[len(em.capturers)-1] = nil
	em.capturers = em.capturers[:len(em.capturers)-1]
}
<|endoftext|>


<commit_before>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package s3

import (
	"errors"
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/mendersoftware/artifacts/models/fileservice"
)

const (
	ExpireMaxLimit = 7 * 24 * time.Hour
	ExpireMinLimit = 1 * time.Minute
)

// AWS S3 client. Implements FileServiceModelI
type SimpleStorageService struct {
	client *s3.S3
	bucket string
}

// NewSimpleStorageService create new S3 client model.
// AWS authentication keys are automatically reloaded from env variables.
func NewSimpleStorageServiceStatic(bucket, key, secret, region, token string) *SimpleStorageService {

	credentials := credentials.NewStaticCredentials(key, secret, token)
	config := aws.NewConfig().WithCredentials(credentials).WithRegion(region)

	sess := session.New(config)

	return &SimpleStorageService{
		client: s3.New(sess),
		bucket: bucket,
	}
}

// NewSimpleStorageService create new S3 client model.
// Use default authentication provides which looks at env variables,
// aws profile file and ec2 iam role
func NewSimpleStorageServiceDefaults(bucket, region string) *SimpleStorageService {

	sess := session.New(aws.NewConfig().WithRegion(region))

	return &SimpleStorageService{
		client: s3.New(sess),
		bucket: bucket,
	}
}

// makeFileId creates file s3 path based on object id and customer id.
// Current structure used is directory per customer id: <customerId>/<objectId>
func (s *SimpleStorageService) makeFileId(customerId, objectId string) string {
	return customerId + "/" + objectId
}

func (s *SimpleStorageService) Delete(customerId, objectId string) error {

	id := s.makeFileId(customerId, objectId)

	params := &s3.DeleteObjectInput{
		// Required
		Bucket: aws.String(s.bucket),
		Key:    aws.String(id),

		// Optional
		RequestPayer: aws.String(s3.RequestPayerRequester),
	}

	// ignore return response which contains charing info
	// and file versioning data which are not in interest
	_, err := s.client.DeleteObject(params)
	if err != nil {
		return err
	}

	return nil
}

func (s *SimpleStorageService) Exists(customerId, objectId string) (bool, error) {

	id := s.makeFileId(customerId, objectId)

	params := &s3.ListObjectsInput{
		// Required
		Bucket: aws.String(s.bucket),

		// Optional
		MaxKeys: aws.Int64(1),
		Prefix:  aws.String(id),
	}

	resp, err := s.client.ListObjects(params)
	if err != nil {
		return false, err
	}

	if len(resp.Contents) == 0 {
		return false, nil
	}

	// Note: Response should contain max 1 object (MaxKetys=1)
	// Double check if it's exact match as object search matches prefix.
	if *resp.Contents[0].Key == id {
		return true, nil
	}

	return false, nil
}

// PutRequest duration is limited to 7 days (AWS limitation)
func (s *SimpleStorageService) PutRequest(customerId, objectId string, duration time.Duration) (*fileservice.Link, error) {

	if err := s.validateDurationLimits(duration); err != nil {
		return nil, err
	}

	id := s.makeFileId(customerId, objectId)

	params := &s3.PutObjectInput{
		// Required
		Bucket: aws.String(s.bucket),
		Key:    aws.String(id),
	}

	// Ignore out object
	req, _ := s.client.PutObjectRequest(params)

	uri, err := req.Presign(duration)
	if err != nil {
		return nil, err
	}

	return fileservice.NewLink(uri, req.Time), nil
}

// GetRequest duration is limited to 7 days (AWS limitation)
func (s *SimpleStorageService) GetRequest(customerId, objectId string, duration time.Duration) (*fileservice.Link, error) {

	if err := s.validateDurationLimits(duration); err != nil {
		return nil, err
	}

	id := s.makeFileId(customerId, objectId)

	params := &s3.GetObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(id),
	}

	// Ignore out object
	req, _ := s.client.GetObjectRequest(params)

	uri, err := req.Presign(duration)
	if err != nil {
		return nil, err
	}

	return fileservice.NewLink(uri, req.Time), nil
}

func (s *SimpleStorageService) validateDurationLimits(duration time.Duration) error {
	if duration > ExpireMaxLimit || duration < ExpireMinLimit {
		return errors.New(fmt.Sprintf("Expire duration out of range: %d[ns] allowed %d-%d[ns]",
			duration, ExpireMinLimit, ExpireMaxLimit))
	}

	return nil
}

func (s *SimpleStorageService) LastModified(customerId, objectId string) (time.Time, error) {

	id := s.makeFileId(customerId, objectId)

	params := &s3.ListObjectsInput{
		// Required
		Bucket: aws.String(s.bucket),

		// Optional
		MaxKeys: aws.Int64(1),
		Prefix:  aws.String(id),
	}

	resp, err := s.client.ListObjects(params)
	if err != nil {
		return time.Time{}, err
	}

	if len(resp.Contents) == 0 {
		return time.Time{}, fileservice.ErrNotFound
	}

	// Note: Response should contain max 1 object (MaxKetys=1)
	// Double check if it's exact match as object search matches prefix.
	if *resp.Contents[0].Key != id {
		return time.Time{}, fileservice.ErrNotFound
	}

	return *resp.Contents[0].LastModified, nil
}
<commit_msg>fileservice/s3: use fmt.Errorf where applicable<commit_after>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package s3

import (
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/mendersoftware/artifacts/models/fileservice"
)

const (
	ExpireMaxLimit = 7 * 24 * time.Hour
	ExpireMinLimit = 1 * time.Minute
)

// AWS S3 client. Implements FileServiceModelI
type SimpleStorageService struct {
	client *s3.S3
	bucket string
}

// NewSimpleStorageService create new S3 client model.
// AWS authentication keys are automatically reloaded from env variables.
func NewSimpleStorageServiceStatic(bucket, key, secret, region, token string) *SimpleStorageService {

	credentials := credentials.NewStaticCredentials(key, secret, token)
	config := aws.NewConfig().WithCredentials(credentials).WithRegion(region)

	sess := session.New(config)

	return &SimpleStorageService{
		client: s3.New(sess),
		bucket: bucket,
	}
}

// NewSimpleStorageService create new S3 client model.
// Use default authentication provides which looks at env variables,
// aws profile file and ec2 iam role
func NewSimpleStorageServiceDefaults(bucket, region string) *SimpleStorageService {

	sess := session.New(aws.NewConfig().WithRegion(region))

	return &SimpleStorageService{
		client: s3.New(sess),
		bucket: bucket,
	}
}

// makeFileId creates file s3 path based on object id and customer id.
// Current structure used is directory per customer id: <customerId>/<objectId>
func (s *SimpleStorageService) makeFileId(customerId, objectId string) string {
	return customerId + "/" + objectId
}

func (s *SimpleStorageService) Delete(customerId, objectId string) error {

	id := s.makeFileId(customerId, objectId)

	params := &s3.DeleteObjectInput{
		// Required
		Bucket: aws.String(s.bucket),
		Key:    aws.String(id),

		// Optional
		RequestPayer: aws.String(s3.RequestPayerRequester),
	}

	// ignore return response which contains charing info
	// and file versioning data which are not in interest
	_, err := s.client.DeleteObject(params)
	if err != nil {
		return err
	}

	return nil
}

func (s *SimpleStorageService) Exists(customerId, objectId string) (bool, error) {

	id := s.makeFileId(customerId, objectId)

	params := &s3.ListObjectsInput{
		// Required
		Bucket: aws.String(s.bucket),

		// Optional
		MaxKeys: aws.Int64(1),
		Prefix:  aws.String(id),
	}

	resp, err := s.client.ListObjects(params)
	if err != nil {
		return false, err
	}

	if len(resp.Contents) == 0 {
		return false, nil
	}

	// Note: Response should contain max 1 object (MaxKetys=1)
	// Double check if it's exact match as object search matches prefix.
	if *resp.Contents[0].Key == id {
		return true, nil
	}

	return false, nil
}

// PutRequest duration is limited to 7 days (AWS limitation)
func (s *SimpleStorageService) PutRequest(customerId, objectId string, duration time.Duration) (*fileservice.Link, error) {

	if err := s.validateDurationLimits(duration); err != nil {
		return nil, err
	}

	id := s.makeFileId(customerId, objectId)

	params := &s3.PutObjectInput{
		// Required
		Bucket: aws.String(s.bucket),
		Key:    aws.String(id),
	}

	// Ignore out object
	req, _ := s.client.PutObjectRequest(params)

	uri, err := req.Presign(duration)
	if err != nil {
		return nil, err
	}

	return fileservice.NewLink(uri, req.Time), nil
}

// GetRequest duration is limited to 7 days (AWS limitation)
func (s *SimpleStorageService) GetRequest(customerId, objectId string, duration time.Duration) (*fileservice.Link, error) {

	if err := s.validateDurationLimits(duration); err != nil {
		return nil, err
	}

	id := s.makeFileId(customerId, objectId)

	params := &s3.GetObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(id),
	}

	// Ignore out object
	req, _ := s.client.GetObjectRequest(params)

	uri, err := req.Presign(duration)
	if err != nil {
		return nil, err
	}

	return fileservice.NewLink(uri, req.Time), nil
}

func (s *SimpleStorageService) validateDurationLimits(duration time.Duration) error {
	if duration > ExpireMaxLimit || duration < ExpireMinLimit {
		return fmt.Errorf("Expire duration out of range: %d[ns] allowed %d-%d[ns]",
			duration, ExpireMinLimit, ExpireMaxLimit)
	}

	return nil
}

func (s *SimpleStorageService) LastModified(customerId, objectId string) (time.Time, error) {

	id := s.makeFileId(customerId, objectId)

	params := &s3.ListObjectsInput{
		// Required
		Bucket: aws.String(s.bucket),

		// Optional
		MaxKeys: aws.Int64(1),
		Prefix:  aws.String(id),
	}

	resp, err := s.client.ListObjects(params)
	if err != nil {
		return time.Time{}, err
	}

	if len(resp.Contents) == 0 {
		return time.Time{}, fileservice.ErrNotFound
	}

	// Note: Response should contain max 1 object (MaxKetys=1)
	// Double check if it's exact match as object search matches prefix.
	if *resp.Contents[0].Key != id {
		return time.Time{}, fileservice.ErrNotFound
	}

	return *resp.Contents[0].LastModified, nil
}
<|endoftext|>


<commit_before>package models_test

import (
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/util/testutil"
	"github.com/stretchr/testify/assert"
	"strings"
	"testing"
)

const digest = "12345678901234567890123456789012"
const md5_digest = "md5:12345678901234567890123456789012"
const sha256_digest = "sha256:12345678901234567890123456789012"

func TestEventTypeValid(t *testing.T) {
	for _, eventType := range constants.EventTypes {
		premisEvent := &models.PremisEvent{
			EventType: eventType,
		}
		if premisEvent.EventTypeValid() == false {
			t.Errorf("EventType '%s' should be valid", eventType)
		}
	}
	premisEvent := &models.PremisEvent{
		EventType: "pub_crawl",
	}
	if premisEvent.EventTypeValid() == true {
		t.Errorf("EventType 'pub_crawl' should not be valid")
	}
}

func TestNewEventObjectIngest(t *testing.T) {
	event, err := models.NewEventObjectIngest(300)
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "ingestion", event.EventType)
	assert.False(t, event.DateTime.IsZero())
	assert.Equal(t, "Copied all files to perservation bucket", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, "300 files copied", event.OutcomeDetail)
	assert.Equal(t, "goamz S3 client", event.Object)
	assert.Equal(t, "https://github.com/crowdmob/goamz", event.Agent)
	assert.Equal(t, "Multipart put using md5 checksum", event.OutcomeInformation)
}

func TestNewEventObjectIdentifierAssignment(t *testing.T) {
	event, err := models.NewEventObjectIdentifierAssignment("test.edu/object001")
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "identifier assignment", event.EventType)
	assert.False(t, event.DateTime.IsZero())
	assert.Equal(t, "Assigned bag identifier", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, "test.edu/object001", event.OutcomeDetail)
	assert.Equal(t, "APTrust exchange", event.Object)
	assert.Equal(t, "https://github.com/APTrust/exchange", event.Agent)
	assert.Equal(t, "Institution domain + tar file name", event.OutcomeInformation)
}

func TestNewEventObjectRights(t *testing.T) {
	event, err := models.NewEventObjectRights("institution")
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "access assignment", event.EventType)
	assert.False(t, event.DateTime.IsZero())
	assert.Equal(t, "Assigned bag access rights", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, "institution", event.OutcomeDetail)
	assert.Equal(t, "APTrust exchange", event.Object)
	assert.Equal(t, "https://github.com/APTrust/exchange", event.Agent)
	assert.Equal(t, "Set access to institution", event.OutcomeInformation)
}

func TestNewEventGenericFileIngest(t *testing.T) {
	event, err := models.NewEventGenericFileIngest(testutil.TEST_TIMESTAMP, digest)
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "ingestion", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Completed copy to S3", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, md5_digest, event.OutcomeDetail)
	assert.Equal(t, "exchange + goamz S3 client", event.Object)
	assert.Equal(t, "https://github.com/APTrust/exchange", event.Agent)
	assert.Equal(t, "Put using md5 checksum", event.OutcomeInformation)
}

func TestNewEventGenericFileFixityCheck(t *testing.T) {
	event, err := models.NewEventGenericFileFixityCheck(testutil.TEST_TIMESTAMP, constants.AlgMd5,
		digest, true)
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "fixity check", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Fixity check against registered hash", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, md5_digest, event.OutcomeDetail)
	assert.Equal(t, "Go language crypto/md5", event.Object)
	assert.Equal(t, "http://golang.org/pkg/crypto/md5/", event.Agent)
	assert.Equal(t, "Fixity matches", event.OutcomeInformation)

	event, err = models.NewEventGenericFileFixityCheck(testutil.TEST_TIMESTAMP, constants.AlgSha256,
		digest, false)
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "fixity check", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Fixity check against registered hash", event.Detail)
	assert.Equal(t, "Failed", event.Outcome)
	assert.Equal(t, sha256_digest, event.OutcomeDetail)
	assert.Equal(t, "Go language crypto/sha256", event.Object)
	assert.Equal(t, "http://golang.org/pkg/crypto/sha256/", event.Agent)
	assert.Equal(t, "Fixity did not match", event.OutcomeInformation)
}

func TestNewEventGenericFileDigestCalculation(t *testing.T) {
	event, err := models.NewEventGenericFileDigestCalculation(testutil.TEST_TIMESTAMP,
		constants.AlgMd5, digest)
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "message digest calculation", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Calculated fixity value", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, md5_digest, event.OutcomeDetail)
	assert.Equal(t, "Go language crypto/md5", event.Object)
	assert.Equal(t, "http://golang.org/pkg/crypto/md5/", event.Agent)
	assert.Equal(t, "", event.OutcomeInformation)

	event, err = models.NewEventGenericFileDigestCalculation(testutil.TEST_TIMESTAMP, constants.AlgSha256, digest)
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "message digest calculation", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Calculated fixity value", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, sha256_digest, event.OutcomeDetail)
	assert.Equal(t, "Go language crypto/sha256", event.Object)
	assert.Equal(t, "http://golang.org/pkg/crypto/sha256/", event.Agent)
	assert.Equal(t, "", event.OutcomeInformation)
}

func TestNewEventGenericFileIdentifierAssignment(t *testing.T) {
	event, err := models.NewEventGenericFileIdentifierAssignment(testutil.TEST_TIMESTAMP, constants.IdTypeBagAndPath, "blah.edu/blah/blah.txt")
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "identifier assignment", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Assigned new institution.bag/path identifier", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, "blah.edu/blah/blah.txt", event.OutcomeDetail)
	assert.Equal(t, "APTrust exchange/ingest processor", event.Object)
	assert.Equal(t, "https://github.com/APTrust/exchange", event.Agent)
	assert.Equal(t, "", event.OutcomeInformation)

	event, err = models.NewEventGenericFileIdentifierAssignment(testutil.TEST_TIMESTAMP, constants.IdTypeStorageURL, "https://example.com/000-000-999")
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "identifier assignment", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.True(t, strings.HasPrefix(event.Detail, "Assigned new storage URL identifier"))
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, "https://example.com/000-000-999", event.OutcomeDetail)
	assert.Equal(t, "Go uuid library + goamz S3 library", event.Object)
	assert.Equal(t, "http://github.com/nu7hatch/gouuid", event.Agent)
	assert.Equal(t, "", event.OutcomeInformation)

}

func TestNewEventGenericFileReplication(t *testing.T) {
	event, err := models.NewEventGenericFileReplication(testutil.TEST_TIMESTAMP, "https://example.com/123456789")
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "replication", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Copied to replication storage and assigned replication URL identifier", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, "https://example.com/123456789", event.OutcomeDetail)
	assert.Equal(t, "Go uuid library + goamz S3 library", event.Object)
	assert.Equal(t, "http://github.com/nu7hatch/gouuid", event.Agent)
	assert.Equal(t, "", event.OutcomeInformation)
}
<commit_msg>Test required Premis event params<commit_after>package models_test

import (
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/util/testutil"
	"github.com/stretchr/testify/assert"
	"strings"
	"testing"
	"time"
)

const digest = "12345678901234567890123456789012"
const md5_digest = "md5:12345678901234567890123456789012"
const sha256_digest = "sha256:12345678901234567890123456789012"

func TestEventTypeValid(t *testing.T) {
	for _, eventType := range constants.EventTypes {
		premisEvent := &models.PremisEvent{
			EventType: eventType,
		}
		if premisEvent.EventTypeValid() == false {
			t.Errorf("EventType '%s' should be valid", eventType)
		}
	}
	premisEvent := &models.PremisEvent{
		EventType: "pub_crawl",
	}
	if premisEvent.EventTypeValid() == true {
		t.Errorf("EventType 'pub_crawl' should not be valid")
	}
}

func TestNewEventObjectIngest(t *testing.T) {
	// Test with required params missing
	_, err := models.NewEventObjectIngest(0)
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}

	event, err := models.NewEventObjectIngest(300)
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "ingestion", event.EventType)
	assert.False(t, event.DateTime.IsZero())
	assert.Equal(t, "Copied all files to perservation bucket", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, "300 files copied", event.OutcomeDetail)
	assert.Equal(t, "goamz S3 client", event.Object)
	assert.Equal(t, "https://github.com/crowdmob/goamz", event.Agent)
	assert.Equal(t, "Multipart put using md5 checksum", event.OutcomeInformation)
}

func TestNewEventObjectIdentifierAssignment(t *testing.T) {
	// Test with required params missing
	_, err := models.NewEventObjectIdentifierAssignment("")
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}

	event, err := models.NewEventObjectIdentifierAssignment("test.edu/object001")
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "identifier assignment", event.EventType)
	assert.False(t, event.DateTime.IsZero())
	assert.Equal(t, "Assigned bag identifier", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, "test.edu/object001", event.OutcomeDetail)
	assert.Equal(t, "APTrust exchange", event.Object)
	assert.Equal(t, "https://github.com/APTrust/exchange", event.Agent)
	assert.Equal(t, "Institution domain + tar file name", event.OutcomeInformation)
}

func TestNewEventObjectRights(t *testing.T) {
	// Test with required params missing
	_, err := models.NewEventObjectRights("")
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}

	event, err := models.NewEventObjectRights("institution")
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "access assignment", event.EventType)
	assert.False(t, event.DateTime.IsZero())
	assert.Equal(t, "Assigned bag access rights", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, "institution", event.OutcomeDetail)
	assert.Equal(t, "APTrust exchange", event.Object)
	assert.Equal(t, "https://github.com/APTrust/exchange", event.Agent)
	assert.Equal(t, "Set access to institution", event.OutcomeInformation)
}

func TestNewEventGenericFileIngest(t *testing.T) {
	// Test with required params missing
	_, err := models.NewEventGenericFileIngest(time.Time{}, digest)
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}
	_, err = models.NewEventGenericFileIngest(testutil.TEST_TIMESTAMP, "")
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}

	event, err := models.NewEventGenericFileIngest(testutil.TEST_TIMESTAMP, digest)
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "ingestion", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Completed copy to S3", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, md5_digest, event.OutcomeDetail)
	assert.Equal(t, "exchange + goamz S3 client", event.Object)
	assert.Equal(t, "https://github.com/APTrust/exchange", event.Agent)
	assert.Equal(t, "Put using md5 checksum", event.OutcomeInformation)
}

func TestNewEventGenericFileFixityCheck(t *testing.T) {
	// Test with required params missing
	_, err := models.NewEventGenericFileFixityCheck(time.Time{}, constants.AlgMd5, digest, true)
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}
	_, err = models.NewEventGenericFileFixityCheck(testutil.TEST_TIMESTAMP, "", digest, true)
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}
	_, err = models.NewEventGenericFileFixityCheck(testutil.TEST_TIMESTAMP, constants.AlgMd5, "", true)
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}

	event, err := models.NewEventGenericFileFixityCheck(testutil.TEST_TIMESTAMP, constants.AlgMd5,
		digest, true)
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "fixity check", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Fixity check against registered hash", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, md5_digest, event.OutcomeDetail)
	assert.Equal(t, "Go language crypto/md5", event.Object)
	assert.Equal(t, "http://golang.org/pkg/crypto/md5/", event.Agent)
	assert.Equal(t, "Fixity matches", event.OutcomeInformation)

	event, err = models.NewEventGenericFileFixityCheck(testutil.TEST_TIMESTAMP, constants.AlgSha256,
		digest, false)
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "fixity check", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Fixity check against registered hash", event.Detail)
	assert.Equal(t, "Failed", event.Outcome)
	assert.Equal(t, sha256_digest, event.OutcomeDetail)
	assert.Equal(t, "Go language crypto/sha256", event.Object)
	assert.Equal(t, "http://golang.org/pkg/crypto/sha256/", event.Agent)
	assert.Equal(t, "Fixity did not match", event.OutcomeInformation)
}

func TestNewEventGenericFileDigestCalculation(t *testing.T) {
	// Test with required params missing
	_, err := models.NewEventGenericFileDigestCalculation(time.Time{}, constants.AlgMd5, digest)
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}
	_, err = models.NewEventGenericFileDigestCalculation(testutil.TEST_TIMESTAMP, "", digest)
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}
	_, err = models.NewEventGenericFileDigestCalculation(testutil.TEST_TIMESTAMP, constants.AlgMd5, "")
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}

	event, err := models.NewEventGenericFileDigestCalculation(testutil.TEST_TIMESTAMP,
		constants.AlgMd5, digest)
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "message digest calculation", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Calculated fixity value", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, md5_digest, event.OutcomeDetail)
	assert.Equal(t, "Go language crypto/md5", event.Object)
	assert.Equal(t, "http://golang.org/pkg/crypto/md5/", event.Agent)
	assert.Equal(t, "", event.OutcomeInformation)

	event, err = models.NewEventGenericFileDigestCalculation(testutil.TEST_TIMESTAMP, constants.AlgSha256, digest)
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "message digest calculation", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Calculated fixity value", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, sha256_digest, event.OutcomeDetail)
	assert.Equal(t, "Go language crypto/sha256", event.Object)
	assert.Equal(t, "http://golang.org/pkg/crypto/sha256/", event.Agent)
	assert.Equal(t, "", event.OutcomeInformation)
}

func TestNewEventGenericFileIdentifierAssignment(t *testing.T) {
	// Test with required params missing
	_, err := models.NewEventGenericFileIdentifierAssignment(time.Time{}, constants.AlgMd5, "abc/123")
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}
	_, err = models.NewEventGenericFileIdentifierAssignment(testutil.TEST_TIMESTAMP, "", "abc/123")
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}
	_, err = models.NewEventGenericFileIdentifierAssignment(testutil.TEST_TIMESTAMP, constants.AlgMd5, "")
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}

	event, err := models.NewEventGenericFileIdentifierAssignment(testutil.TEST_TIMESTAMP, constants.IdTypeBagAndPath, "blah.edu/blah/blah.txt")
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "identifier assignment", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Assigned new institution.bag/path identifier", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, "blah.edu/blah/blah.txt", event.OutcomeDetail)
	assert.Equal(t, "APTrust exchange/ingest processor", event.Object)
	assert.Equal(t, "https://github.com/APTrust/exchange", event.Agent)
	assert.Equal(t, "", event.OutcomeInformation)

	event, err = models.NewEventGenericFileIdentifierAssignment(testutil.TEST_TIMESTAMP, constants.IdTypeStorageURL, "https://example.com/000-000-999")
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "identifier assignment", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.True(t, strings.HasPrefix(event.Detail, "Assigned new storage URL identifier"))
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, "https://example.com/000-000-999", event.OutcomeDetail)
	assert.Equal(t, "Go uuid library + goamz S3 library", event.Object)
	assert.Equal(t, "http://github.com/nu7hatch/gouuid", event.Agent)
	assert.Equal(t, "", event.OutcomeInformation)

}

func TestNewEventGenericFileReplication(t *testing.T) {
	// Test with required params missing
	_, err := models.NewEventGenericFileReplication(time.Time{}, "https://example.com/123456789")
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}
	_, err = models.NewEventGenericFileReplication(testutil.TEST_TIMESTAMP, "")
	assert.NotNil(t, err)
	if err != nil {
		assert.True(t, strings.HasPrefix(err.Error(), "Param"))
	}

	event, err := models.NewEventGenericFileReplication(testutil.TEST_TIMESTAMP, "https://example.com/123456789")
	if err != nil {
		t.Errorf("Error creating PremisEvent: %v", err)
		return
	}
	assert.Len(t, event.Identifier, 36)
	assert.Equal(t, "replication", event.EventType)
	assert.Equal(t, testutil.TEST_TIMESTAMP, event.DateTime)
	assert.Equal(t, "Copied to replication storage and assigned replication URL identifier", event.Detail)
	assert.Equal(t, "Success", event.Outcome)
	assert.Equal(t, "https://example.com/123456789", event.OutcomeDetail)
	assert.Equal(t, "Go uuid library + goamz S3 library", event.Object)
	assert.Equal(t, "http://github.com/nu7hatch/gouuid", event.Agent)
	assert.Equal(t, "", event.OutcomeInformation)
}
<|endoftext|>


<commit_before>package models_test<commit_msg>add TODO<commit_after>package models_test

// TODO: add tests
<|endoftext|>


<commit_before>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package stats

import (
	"fmt"

	repo_model "code.gitea.io/gitea/models/repo"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/graceful"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/process"
)

// DBIndexer implements Indexer interface to use database's like search
type DBIndexer struct{}

// Index repository status function
func (db *DBIndexer) Index(id int64) error {
	ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().ShutdownContext(), fmt.Sprintf("Stats.DB Index Repo[%d]", id))
	defer finished()

	repo, err := repo_model.GetRepositoryByID(id)
	if err != nil {
		return err
	}
	if repo.IsEmpty {
		return nil
	}

	status, err := repo_model.GetIndexerStatus(repo, repo_model.RepoIndexerTypeStats)
	if err != nil {
		return err
	}

	gitRepo, err := git.OpenRepositoryCtx(ctx, repo.RepoPath())
	if err != nil {
		return err
	}
	defer gitRepo.Close()

	// Get latest commit for default branch
	commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch)
	if err != nil {
		if git.IsErrBranchNotExist(err) || git.IsErrNotExist(err) {
			log.Debug("Unable to get commit ID for default branch %s in %s ... skipping this repository", repo.DefaultBranch, repo.RepoPath())
			return nil
		}
		log.Error("Unable to get commit ID for default branch %s in %s. Error: %v", repo.DefaultBranch, repo.RepoPath(), err)
		return err
	}

	// Do not recalculate stats if already calculated for this commit
	if status.CommitSha == commitID {
		return nil
	}

	// Calculate and save language statistics to database
	stats, err := gitRepo.GetLanguageStats(commitID)
	if err != nil {
		log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
		return err
	}
	err = repo_model.UpdateLanguageStats(repo, commitID, stats)
	if err != nil {
		log.Error("Unable to update language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
		return err
	}

	log.Debug("DBIndexer completed language stats for ID %s for default branch %s in %s. stats count: %d", commitID, repo.DefaultBranch, repo.RepoPath(), len(stats))
	return nil
}

// Close dummy function
func (db *DBIndexer) Close() {
}
<commit_msg>Prevent Stats Indexer reporting error if repo dir missing (#18870)<commit_after>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package stats

import (
	"fmt"

	repo_model "code.gitea.io/gitea/models/repo"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/graceful"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/process"
)

// DBIndexer implements Indexer interface to use database's like search
type DBIndexer struct{}

// Index repository status function
func (db *DBIndexer) Index(id int64) error {
	ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().ShutdownContext(), fmt.Sprintf("Stats.DB Index Repo[%d]", id))
	defer finished()

	repo, err := repo_model.GetRepositoryByID(id)
	if err != nil {
		return err
	}
	if repo.IsEmpty {
		return nil
	}

	status, err := repo_model.GetIndexerStatus(repo, repo_model.RepoIndexerTypeStats)
	if err != nil {
		return err
	}

	gitRepo, err := git.OpenRepositoryCtx(ctx, repo.RepoPath())
	if err != nil {
		if err.Error() == "no such file or directory" {
			return nil
		}
		return err
	}
	defer gitRepo.Close()

	// Get latest commit for default branch
	commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch)
	if err != nil {
		if git.IsErrBranchNotExist(err) || git.IsErrNotExist(err) {
			log.Debug("Unable to get commit ID for default branch %s in %s ... skipping this repository", repo.DefaultBranch, repo.RepoPath())
			return nil
		}
		log.Error("Unable to get commit ID for default branch %s in %s. Error: %v", repo.DefaultBranch, repo.RepoPath(), err)
		return err
	}

	// Do not recalculate stats if already calculated for this commit
	if status.CommitSha == commitID {
		return nil
	}

	// Calculate and save language statistics to database
	stats, err := gitRepo.GetLanguageStats(commitID)
	if err != nil {
		log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
		return err
	}
	err = repo_model.UpdateLanguageStats(repo, commitID, stats)
	if err != nil {
		log.Error("Unable to update language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
		return err
	}

	log.Debug("DBIndexer completed language stats for ID %s for default branch %s in %s. stats count: %d", commitID, repo.DefaultBranch, repo.RepoPath(), len(stats))
	return nil
}

// Close dummy function
func (db *DBIndexer) Close() {
}
<|endoftext|>