<commit_before>package rest

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/gorilla/websocket"
	"github.com/lxc/lxd/shared"
	"github.com/pborman/uuid"
	"gopkg.in/inconshreveable/log15.v2"

	"github.com/nsec/askgod/api"
	"github.com/nsec/askgod/internal/utils"
)

var eventHostname string
var eventsLock sync.Mutex
var eventListeners = make(map[string]*eventListener)

type eventListener struct {
	connection   *websocket.Conn
	messageTypes []string

	active  chan bool
	id      string
	msgLock sync.Mutex
	peer    bool
}

func (r *rest) injectEvents(writer http.ResponseWriter, request *http.Request, logger log15.Logger) {
	// Access control
	if !r.isPeer(request) {
		logger.Warn("Unauthorized attempt to send events")
		r.errorResponse(403, "Forbidden", writer, request)
	}

	// Setup websocket
	conn, err := shared.WebsocketUpgrader.Upgrade(writer, request, nil)
	if err != nil {
		logger.Error("Failed to setup websocket", log15.Ctx{"error": err})
		r.errorResponse(500, fmt.Sprintf("%v", err), writer, request)
		return
	}

	// Process messages
	for {
		_, data, err := conn.ReadMessage()
		if err != nil {
			break
		}

		var event interface{}
		err = json.Unmarshal(data, &event)
		if err != nil {
			logger.Error("Received a broken event from peer", log15.Ctx{"error": err})
			continue
		}

		err = eventSendRaw(event)
		if err != nil {
			logger.Error("Failed to relay event from peer", log15.Ctx{"error": err})
			continue
		}
	}
}

func (r *rest) getEvents(writer http.ResponseWriter, request *http.Request, logger log15.Logger) {
	listener := eventListener{}

	// Get the provided event type
	typeStr := request.FormValue("type")
	if typeStr == "" {
		logger.Warn("Events request without a type")
		r.errorResponse(400, "Missing event type", writer, request)
		return
	}

	if typeStr == "cluster" {
		r.injectEvents(writer, request, logger)
		return
	}

	// Valid the provided types
	eventTypes := strings.Split(typeStr, ",")
	for _, entry := range eventTypes {
		// Make sure that all types are valid
		if !utils.StringInSlice(entry, []string{"timeline", "teams", "logging", "flags"}) {
			logger.Warn("Invalid event type", log15.Ctx{"type": entry})
			r.errorResponse(400, "Invalid event type", writer, request)
			return
		}

		// Admin access control
		if utils.StringInSlice(entry, []string{"logging", "flags"}) && !r.hasAccess("admin", request) {
			logger.Warn("Unauthorized attempt to get events", log15.Ctx{"type": entry})
			r.errorResponse(403, "Forbidden", writer, request)
			return
		}
	}

	// Setup websocket
	c, err := shared.WebsocketUpgrader.Upgrade(writer, request, nil)
	if err != nil {
		logger.Error("Failed to setup websocket", log15.Ctx{"error": err})
		r.errorResponse(500, fmt.Sprintf("%v", err), writer, request)
		return
	}

	// Prepare the listener
	listener.active = make(chan bool, 1)
	listener.connection = c
	listener.id = uuid.NewRandom().String()
	listener.messageTypes = eventTypes

	// Add it to the set
	eventsLock.Lock()
	eventListeners[listener.id] = &listener
	eventsLock.Unlock()

	r.logger.Debug("New events listener", log15.Ctx{"uuid": listener.id})

	<-listener.active

	eventsLock.Lock()
	delete(eventListeners, listener.id)
	eventsLock.Unlock()

	listener.connection.Close()
	r.logger.Debug("Disconnected events listener", log15.Ctx{"uuid": listener.id})
}

func eventSend(eventType string, eventMessage interface{}) error {
	event := map[string]interface{}{}
	event["type"] = eventType
	event["timestamp"] = time.Now()
	event["metadata"] = eventMessage

	if eventHostname == "" {
		hostname, err := os.Hostname()
		if err != nil {
			return err
		}
		eventHostname = hostname
	}
	event["server"] = eventHostname

	return eventSendRaw(event)
}

func eventSendRaw(event interface{}) error {
	body, err := json.Marshal(event)
	if err != nil {
		return err
	}

	eventsLock.Lock()
	listeners := eventListeners
	for _, listener := range listeners {
		if event.(map[string]interface{})["server"].(string) != eventHostname && listener.peer {
			continue
		}

		if listener.messageTypes != nil && !utils.StringInSlice(event.(map[string]interface{})["type"].(string), listener.messageTypes) {
			continue
		}

		go func(listener *eventListener, body []byte) {
			if listener == nil {
				return
			}

			listener.msgLock.Lock()
			err = listener.connection.WriteMessage(websocket.TextMessage, body)
			listener.msgLock.Unlock()

			if err != nil {
				listener.active <- false
			}
		}(listener, body)
	}
	eventsLock.Unlock()

	return nil
}

func logContextMap(ctx []interface{}) map[string]string {
	var key string
	ctxMap := map[string]string{}

	for _, entry := range ctx {
		if key == "" {
			key = entry.(string)
		} else {
			ctxMap[key] = fmt.Sprintf("%v", entry)
			key = ""
		}
	}

	return ctxMap
}

func (r *rest) forwardEvents(peer string) {
	var peerURL string
	if strings.HasPrefix(peer, "https://") {
		peerURL = fmt.Sprintf("wss://%s/1.0/events?type=cluster", strings.TrimPrefix(peer, "https://"))
	} else {
		peerURL = fmt.Sprintf("ws://%s/1.0/events?type=cluster", strings.TrimPrefix(peer, "http://"))
	}

	tlsConfig := &tls.Config{
		MinVersion: tls.VersionTLS12,
		MaxVersion: tls.VersionTLS12,
		CipherSuites: []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA},
		PreferServerCipherSuites: true,
	}

	if r.config.Daemon.HTTPSCertificate != "" {
		cert := r.config.Daemon.HTTPSCertificate
		if !strings.Contains(cert, "\n") && utils.PathExists(cert) {
			content, err := ioutil.ReadFile(cert)
			if err != nil {
				r.logger.Error("Failed to read cluster certificate", log15.Ctx{"error": err, "peer": peer})
				return
			}

			cert = string(content)
		}

		caCertPool := tlsConfig.RootCAs
		if caCertPool == nil {
			caCertPool = x509.NewCertPool()
		}

		content := cert
		for content != "" {
			block, remainder := pem.Decode([]byte(content))
			if block == nil {
				r.logger.Error("Failed to decode cluster certificate", log15.Ctx{"peer": peer})
				return
			}

			crt, err := x509.ParseCertificate(block.Bytes)
			if err != nil {
				r.logger.Error("Failed to parse cluster certificate", log15.Ctx{"error": err, "peer": peer})
				return
			}

			if !crt.IsCA {
				// Override the ServerName
				if crt.DNSNames != nil {
					tlsConfig.ServerName = crt.DNSNames[0]
				} else {
					tlsConfig.ServerName = crt.Subject.CommonName
				}
			}

			caCertPool.AddCert(crt)

			content = string(remainder)
		}

		// Setup the pool
		tlsConfig.RootCAs = caCertPool
		tlsConfig.BuildNameToCertificate()
	}

	dialer := websocket.Dialer{
		TLSClientConfig: tlsConfig,
	}

	for i := 0; i < 20; i++ {
		r.logger.Debug("Connecting to cluster peer", log15.Ctx{"peer": peer})

		conn, _, err := dialer.Dial(peerURL, nil)
		if err != nil {
			r.logger.Warn("Failed to connect to cluster peer", log15.Ctx{"error": err, "peer": peer})
		} else {
			listener := eventListener{
				connection: conn,
				active:     make(chan bool, 1),
				id:         uuid.NewRandom().String(),
				peer:       true,
			}

			eventsLock.Lock()
			eventListeners[listener.id] = &listener
			eventsLock.Unlock()
			r.logger.Info("Connected to cluster peer", log15.Ctx{"peer": peer})

			<-listener.active
			r.logger.Warn("Lost connection with cluster peer", log15.Ctx{"peer": peer})
		}

		time.Sleep(3 * time.Second)
	}
	r.logger.Error("Giving up on cluster peer", log15.Ctx{"peer": peer})
}

// EventsLogHandler represents a log15 handler for the /1.0/events API
type EventsLogHandler struct {
}

// Log send a log message through websocket
func (h EventsLogHandler) Log(r *log15.Record) error {
	eventSend("logging", api.EventLogging{
		Message: r.Msg,
		Level:   r.Lvl.String(),
		Context: logContextMap(r.Ctx)})
	return nil
}
<commit_msg>askgod-server: Reset cluster timeout on success<commit_after>package rest

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/gorilla/websocket"
	"github.com/lxc/lxd/shared"
	"github.com/pborman/uuid"
	"gopkg.in/inconshreveable/log15.v2"

	"github.com/nsec/askgod/api"
	"github.com/nsec/askgod/internal/utils"
)

var eventHostname string
var eventsLock sync.Mutex
var eventListeners = make(map[string]*eventListener)

type eventListener struct {
	connection   *websocket.Conn
	messageTypes []string

	active  chan bool
	id      string
	msgLock sync.Mutex
	peer    bool
}

func (r *rest) injectEvents(writer http.ResponseWriter, request *http.Request, logger log15.Logger) {
	// Access control
	if !r.isPeer(request) {
		logger.Warn("Unauthorized attempt to send events")
		r.errorResponse(403, "Forbidden", writer, request)
	}

	// Setup websocket
	conn, err := shared.WebsocketUpgrader.Upgrade(writer, request, nil)
	if err != nil {
		logger.Error("Failed to setup websocket", log15.Ctx{"error": err})
		r.errorResponse(500, fmt.Sprintf("%v", err), writer, request)
		return
	}

	// Process messages
	for {
		_, data, err := conn.ReadMessage()
		if err != nil {
			break
		}

		var event interface{}
		err = json.Unmarshal(data, &event)
		if err != nil {
			logger.Error("Received a broken event from peer", log15.Ctx{"error": err})
			continue
		}

		err = eventSendRaw(event)
		if err != nil {
			logger.Error("Failed to relay event from peer", log15.Ctx{"error": err})
			continue
		}
	}
}

func (r *rest) getEvents(writer http.ResponseWriter, request *http.Request, logger log15.Logger) {
	listener := eventListener{}

	// Get the provided event type
	typeStr := request.FormValue("type")
	if typeStr == "" {
		logger.Warn("Events request without a type")
		r.errorResponse(400, "Missing event type", writer, request)
		return
	}

	if typeStr == "cluster" {
		r.injectEvents(writer, request, logger)
		return
	}

	// Valid the provided types
	eventTypes := strings.Split(typeStr, ",")
	for _, entry := range eventTypes {
		// Make sure that all types are valid
		if !utils.StringInSlice(entry, []string{"timeline", "teams", "logging", "flags"}) {
			logger.Warn("Invalid event type", log15.Ctx{"type": entry})
			r.errorResponse(400, "Invalid event type", writer, request)
			return
		}

		// Admin access control
		if utils.StringInSlice(entry, []string{"logging", "flags"}) && !r.hasAccess("admin", request) {
			logger.Warn("Unauthorized attempt to get events", log15.Ctx{"type": entry})
			r.errorResponse(403, "Forbidden", writer, request)
			return
		}
	}

	// Setup websocket
	c, err := shared.WebsocketUpgrader.Upgrade(writer, request, nil)
	if err != nil {
		logger.Error("Failed to setup websocket", log15.Ctx{"error": err})
		r.errorResponse(500, fmt.Sprintf("%v", err), writer, request)
		return
	}

	// Prepare the listener
	listener.active = make(chan bool, 1)
	listener.connection = c
	listener.id = uuid.NewRandom().String()
	listener.messageTypes = eventTypes

	// Add it to the set
	eventsLock.Lock()
	eventListeners[listener.id] = &listener
	eventsLock.Unlock()

	r.logger.Debug("New events listener", log15.Ctx{"uuid": listener.id})

	<-listener.active

	eventsLock.Lock()
	delete(eventListeners, listener.id)
	eventsLock.Unlock()

	listener.connection.Close()
	r.logger.Debug("Disconnected events listener", log15.Ctx{"uuid": listener.id})
}

func eventSend(eventType string, eventMessage interface{}) error {
	event := map[string]interface{}{}
	event["type"] = eventType
	event["timestamp"] = time.Now()
	event["metadata"] = eventMessage

	if eventHostname == "" {
		hostname, err := os.Hostname()
		if err != nil {
			return err
		}
		eventHostname = hostname
	}
	event["server"] = eventHostname

	return eventSendRaw(event)
}

func eventSendRaw(event interface{}) error {
	body, err := json.Marshal(event)
	if err != nil {
		return err
	}

	eventsLock.Lock()
	listeners := eventListeners
	for _, listener := range listeners {
		if event.(map[string]interface{})["server"].(string) != eventHostname && listener.peer {
			continue
		}

		if listener.messageTypes != nil && !utils.StringInSlice(event.(map[string]interface{})["type"].(string), listener.messageTypes) {
			continue
		}

		go func(listener *eventListener, body []byte) {
			if listener == nil {
				return
			}

			listener.msgLock.Lock()
			err = listener.connection.WriteMessage(websocket.TextMessage, body)
			listener.msgLock.Unlock()

			if err != nil {
				listener.active <- false
			}
		}(listener, body)
	}
	eventsLock.Unlock()

	return nil
}

func logContextMap(ctx []interface{}) map[string]string {
	var key string
	ctxMap := map[string]string{}

	for _, entry := range ctx {
		if key == "" {
			key = entry.(string)
		} else {
			ctxMap[key] = fmt.Sprintf("%v", entry)
			key = ""
		}
	}

	return ctxMap
}

func (r *rest) forwardEvents(peer string) {
	var peerURL string
	if strings.HasPrefix(peer, "https://") {
		peerURL = fmt.Sprintf("wss://%s/1.0/events?type=cluster", strings.TrimPrefix(peer, "https://"))
	} else {
		peerURL = fmt.Sprintf("ws://%s/1.0/events?type=cluster", strings.TrimPrefix(peer, "http://"))
	}

	tlsConfig := &tls.Config{
		MinVersion: tls.VersionTLS12,
		MaxVersion: tls.VersionTLS12,
		CipherSuites: []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA},
		PreferServerCipherSuites: true,
	}

	if r.config.Daemon.HTTPSCertificate != "" {
		cert := r.config.Daemon.HTTPSCertificate
		if !strings.Contains(cert, "\n") && utils.PathExists(cert) {
			content, err := ioutil.ReadFile(cert)
			if err != nil {
				r.logger.Error("Failed to read cluster certificate", log15.Ctx{"error": err, "peer": peer})
				return
			}

			cert = string(content)
		}

		caCertPool := tlsConfig.RootCAs
		if caCertPool == nil {
			caCertPool = x509.NewCertPool()
		}

		content := cert
		for content != "" {
			block, remainder := pem.Decode([]byte(content))
			if block == nil {
				r.logger.Error("Failed to decode cluster certificate", log15.Ctx{"peer": peer})
				return
			}

			crt, err := x509.ParseCertificate(block.Bytes)
			if err != nil {
				r.logger.Error("Failed to parse cluster certificate", log15.Ctx{"error": err, "peer": peer})
				return
			}

			if !crt.IsCA {
				// Override the ServerName
				if crt.DNSNames != nil {
					tlsConfig.ServerName = crt.DNSNames[0]
				} else {
					tlsConfig.ServerName = crt.Subject.CommonName
				}
			}

			caCertPool.AddCert(crt)

			content = string(remainder)
		}

		// Setup the pool
		tlsConfig.RootCAs = caCertPool
		tlsConfig.BuildNameToCertificate()
	}

	dialer := websocket.Dialer{
		TLSClientConfig: tlsConfig,
	}

	for i := 0; i < 20; i++ {
		r.logger.Debug("Connecting to cluster peer", log15.Ctx{"peer": peer})

		conn, _, err := dialer.Dial(peerURL, nil)
		if err != nil {
			r.logger.Warn("Failed to connect to cluster peer", log15.Ctx{"error": err, "peer": peer})
		} else {
			listener := eventListener{
				connection: conn,
				active:     make(chan bool, 1),
				id:         uuid.NewRandom().String(),
				peer:       true,
			}

			eventsLock.Lock()
			eventListeners[listener.id] = &listener
			eventsLock.Unlock()
			r.logger.Info("Connected to cluster peer", log15.Ctx{"peer": peer})

			i = 0
			<-listener.active

			r.logger.Warn("Lost connection with cluster peer", log15.Ctx{"peer": peer})
		}

		time.Sleep(3 * time.Second)
	}
	r.logger.Error("Giving up on cluster peer", log15.Ctx{"peer": peer})
}

// EventsLogHandler represents a log15 handler for the /1.0/events API
type EventsLogHandler struct {
}

// Log send a log message through websocket
func (h EventsLogHandler) Log(r *log15.Record) error {
	eventSend("logging", api.EventLogging{
		Message: r.Msg,
		Level:   r.Lvl.String(),
		Context: logContextMap(r.Ctx)})
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package restore

import (
	"log"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/comeback/internal/dag"
)

// Create a dag.Visitor for *node.
//
// For each node n, the visitor does the following:
//
//  *  Ensure that the directory path.Dir(n.RelPath) exists and is writeable.
//  *  <Perform type-specific action.>
//  *  Set the appropriate permissions, times, and owners for n.RelPath.
//
// The type-specific actions are as follows:
//
//  *  Files: create the file with the contents described by n.Info.Scores.
//  *  Directories: ensure that the directory n.RelPath exists.
//  *  Symlinks: create a symlink pointing at n.Info.Target.
//
func newVisitor(
	basePath string,
	blobStore blob.Store,
	logger *log.Logger) (v dag.Visitor) {
	panic("TODO")
}
<commit_msg>newVisitor<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package restore

import (
	"errors"
	"log"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/comeback/internal/dag"
)

// Create a dag.Visitor for *node.
//
// For each node n, the visitor does the following:
//
//  *  Ensure that the directory path.Dir(n.RelPath) exists and is writeable.
//  *  <Perform type-specific action.>
//  *  Set the appropriate permissions, times, and owners for n.RelPath.
//
// The type-specific actions are as follows:
//
//  *  Files: create the file with the contents described by n.Info.Scores.
//  *  Directories: ensure that the directory n.RelPath exists.
//  *  Symlinks: create a symlink pointing at n.Info.Target.
//
func newVisitor(
	basePath string,
	blobStore blob.Store,
	logger *log.Logger) (v dag.Visitor) {
	v = &visitor{
		basePath:  basePath,
		blobStore: blobStore,
		logger:    logger,
	}

	return
}

type visitor struct {
	basePath  string
	blobStore blob.Store
	logger    *log.Logger
}

func (v *visitor) Visit(ctx context.Context, untyped dag.Node) (err error) {
	err = errors.New("TODO")
	return
}
<|endoftext|>


<commit_before>package routers

import (
	"io"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/go-macaron/binding"
	"github.com/go-macaron/gzip"
	"github.com/go-macaron/toolbox"
	"github.com/ouqiang/gocron/internal/modules/app"
	"github.com/ouqiang/gocron/internal/modules/logger"
	"github.com/ouqiang/gocron/internal/modules/utils"
	"github.com/ouqiang/gocron/internal/routers/host"
	"github.com/ouqiang/gocron/internal/routers/install"
	"github.com/ouqiang/gocron/internal/routers/loginlog"
	"github.com/ouqiang/gocron/internal/routers/manage"
	"github.com/ouqiang/gocron/internal/routers/task"
	"github.com/ouqiang/gocron/internal/routers/tasklog"
	"github.com/ouqiang/gocron/internal/routers/user"
	"github.com/rakyll/statik/fs"
	"gopkg.in/macaron.v1"

	_ "github.com/ouqiang/gocron/internal/statik"
)

const (
	urlPrefix = "/api"
	staticDir = "public"
)

var statikFS http.FileSystem

func init() {
	var err error
	statikFS, err = fs.New()
	if err != nil {
		log.Fatal(err)
	}
}

// Register 路由注册
func Register(m *macaron.Macaron) {
	m.SetURLPrefix(urlPrefix)
	// 所有GET方法，自动注册HEAD方法
	m.SetAutoHead(true)
	m.Get("/", func(ctx *macaron.Context) {
		file, err := statikFS.Open("/index.html")
		if err != nil {
			logger.Error("读取首页文件失败: %s", err)
			ctx.WriteHeader(http.StatusInternalServerError)
			return
		}

		io.Copy(ctx.Resp, file)

	})
	// 系统安装
	m.Group("/install", func() {
		m.Post("/store", binding.Bind(install.InstallForm{}), install.Store)
		m.Get("/status", func(ctx *macaron.Context) string {
			jsonResp := utils.JsonResponse{}
			return jsonResp.Success("", app.Installed)
		})
	})

	// 用户
	m.Group("/user", func() {
		m.Get("", user.Index)
		m.Get("/:id", user.Detail)
		m.Post("/store", binding.Bind(user.UserForm{}), user.Store)
		m.Post("/remove/:id", user.Remove)
		m.Post("/login", user.ValidateLogin)
		m.Post("/enable/:id", user.Enable)
		m.Post("/disable/:id", user.Disable)
		m.Post("/editMyPassword", user.UpdateMyPassword)
		m.Post("/editPassword/:id", user.UpdatePassword)
	})

	// 定时任务
	m.Group("/task", func() {
		m.Post("/store", binding.Bind(task.TaskForm{}), task.Store)
		m.Get("/:id", task.Detail)
		m.Get("", task.Index)
		m.Get("/log", tasklog.Index)
		m.Post("/log/clear", tasklog.Clear)
		m.Post("/log/stop", tasklog.Stop)
		m.Post("/remove/:id", task.Remove)
		m.Post("/enable/:id", task.Enable)
		m.Post("/disable/:id", task.Disable)
		m.Get("/run/:id", task.Run)
	})

	// 主机
	m.Group("/host", func() {
		m.Get("/:id", host.Detail)
		m.Post("/store", binding.Bind(host.HostForm{}), host.Store)
		m.Get("", host.Index)
		m.Get("/all", host.All)
		m.Get("/ping/:id", host.Ping)
		m.Post("/remove/:id", host.Remove)
	})

	// 管理
	m.Group("/system", func() {
		m.Group("/slack", func() {
			m.Get("", manage.Slack)
			m.Post("/update", manage.UpdateSlack)
			m.Post("/channel", manage.CreateSlackChannel)
			m.Post("/channel/remove/:id", manage.RemoveSlackChannel)
		})
		m.Group("/mail", func() {
			m.Get("", manage.Mail)
			m.Post("/update", binding.Bind(manage.MailServerForm{}), manage.UpdateMail)
			m.Post("/user", manage.CreateMailUser)
			m.Post("/user/remove/:id", manage.RemoveMailUser)
		})
		m.Group("/webhook", func() {
			m.Get("", manage.WebHook)
			m.Post("/update", manage.UpdateWebHook)
		})
		m.Get("/login-log", loginlog.Index)
	})

	// API
	m.Group("/v1", func() {
		m.Post("/tasklog/remove/:id", tasklog.Remove)
		m.Post("/task/enable/:id", task.Enable)
		m.Post("/task/disable/:id", task.Disable)
	}, apiAuth)

	// 404错误
	m.NotFound(func(ctx *macaron.Context) string {
		jsonResp := utils.JsonResponse{}

		return jsonResp.Failure(utils.NotFound, "您访问的页面不存在")
	})
	// 50x错误
	m.InternalServerError(func(ctx *macaron.Context) string {
		jsonResp := utils.JsonResponse{}

		return jsonResp.Failure(utils.ServerError, "服务器内部错误, 请稍后再试")
	})
}

// 中间件注册
func RegisterMiddleware(m *macaron.Macaron) {
	m.Use(macaron.Logger())
	m.Use(macaron.Recovery())
	if macaron.Env != macaron.DEV {
		m.Use(gzip.Gziper())
	}
	m.Use(
		macaron.Static(
			"",
			macaron.StaticOptions{
				Prefix:     staticDir,
				FileSystem: statikFS,
			},
		),
	)
	if macaron.Env == macaron.DEV {
		m.Use(toolbox.Toolboxer(m))
	}
	m.Use(macaron.Renderer())
	m.Use(checkAppInstall)
	m.Use(ipAuth)
	m.Use(userAuth)
	m.Use(urlAuth)
}

// region 自定义中间件

/** 检测应用是否已安装 **/
func checkAppInstall(ctx *macaron.Context) {
	if app.Installed {
		return
	}
	if strings.HasPrefix(ctx.Req.URL.Path, "/install") || ctx.Req.URL.Path == "/" {
		return
	}
	jsonResp := utils.JsonResponse{}

	data := jsonResp.Failure(utils.AppNotInstall, "应用未安装")
	ctx.Write([]byte(data))
}

// IP验证, 通过反向代理访问gocron，需设置Header X-Real-IP才能获取到客户端真实IP
func ipAuth(ctx *macaron.Context) {
	if !app.Installed {
		return
	}
	allowIpsStr := app.Setting.AllowIps
	if allowIpsStr == "" {
		return
	}
	clientIp := ctx.RemoteAddr()
	allowIps := strings.Split(allowIpsStr, ",")
	if utils.InStringSlice(allowIps, clientIp) {
		return
	}
	logger.Warnf("非法IP访问-%s", clientIp)
	jsonResp := utils.JsonResponse{}

	data := jsonResp.Failure(utils.UnauthorizedError, "您无权限访问")

	ctx.Write([]byte(data))
}

// 用户认证
func userAuth(ctx *macaron.Context) {
	if !app.Installed {
		return
	}
	user.RestoreToken(ctx)
	if user.IsLogin(ctx) {
		return
	}
	uri := strings.TrimRight(ctx.Req.URL.Path, "/")
	if strings.HasPrefix(uri, "/v1") {
		return
	}
	excludePaths := []string{"", "/user/login", "/install/status"}
	for _, path := range excludePaths {
		if uri == path {
			return
		}
	}
	jsonResp := utils.JsonResponse{}
	data := jsonResp.Failure(utils.AuthError, "认证失败")
	ctx.Write([]byte(data))

}

// URL权限验证
func urlAuth(ctx *macaron.Context) {
	if !app.Installed {
		return
	}
	if user.IsAdmin(ctx) {
		return
	}
	uri := strings.TrimRight(ctx.Req.URL.Path, "/")
	if strings.HasPrefix(uri, "/v1") {
		return
	}
	// 普通用户允许访问的URL地址
	allowPaths := []string{
		"",
		"/install/status",
		"/task",
		"/task/log",
		"/host",
		"/user/login",
		"/user/editMyPassword",
	}
	for _, path := range allowPaths {
		if path == uri {
			return
		}
	}

	jsonResp := utils.JsonResponse{}

	data := jsonResp.Failure(utils.UnauthorizedError, "您无权限访问")
	ctx.Write([]byte(data))
}

/** API接口签名验证 **/
func apiAuth(ctx *macaron.Context) {
	if !app.Installed {
		return
	}
	if !app.Setting.ApiSignEnable {
		return
	}
	apiKey := strings.TrimSpace(app.Setting.ApiKey)
	apiSecret := strings.TrimSpace(app.Setting.ApiSecret)
	json := utils.JsonResponse{}
	if apiKey == "" || apiSecret == "" {
		msg := json.CommonFailure("使用API前, 请先配置密钥")
		ctx.Write([]byte(msg))
		return
	}
	currentTimestamp := time.Now().Unix()
	time := ctx.QueryInt64("time")
	if time <= 0 {
		msg := json.CommonFailure("参数time不能为空")
		ctx.Write([]byte(msg))
		return
	}
	if time < (currentTimestamp - 1800) {
		msg := json.CommonFailure("time无效")
		ctx.Write([]byte(msg))
		return
	}
	sign := ctx.QueryTrim("sign")
	if sign == "" {
		msg := json.CommonFailure("参数sign不能为空")
		ctx.Write([]byte(msg))
		return
	}
	raw := apiKey + strconv.FormatInt(time, 10) + strings.TrimSpace(ctx.Req.URL.Path) + apiSecret
	realSign := utils.Md5(raw)
	if sign != realSign {
		msg := json.CommonFailure("签名验证失败")
		ctx.Write([]byte(msg))
		return
	}
}

// endregion
<commit_msg>close #94 修复普通用户无权限查看任务列表<commit_after>package routers

import (
	"io"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/go-macaron/binding"
	"github.com/go-macaron/gzip"
	"github.com/go-macaron/toolbox"
	"github.com/ouqiang/gocron/internal/modules/app"
	"github.com/ouqiang/gocron/internal/modules/logger"
	"github.com/ouqiang/gocron/internal/modules/utils"
	"github.com/ouqiang/gocron/internal/routers/host"
	"github.com/ouqiang/gocron/internal/routers/install"
	"github.com/ouqiang/gocron/internal/routers/loginlog"
	"github.com/ouqiang/gocron/internal/routers/manage"
	"github.com/ouqiang/gocron/internal/routers/task"
	"github.com/ouqiang/gocron/internal/routers/tasklog"
	"github.com/ouqiang/gocron/internal/routers/user"
	"github.com/rakyll/statik/fs"
	"gopkg.in/macaron.v1"

	_ "github.com/ouqiang/gocron/internal/statik"
)

const (
	urlPrefix = "/api"
	staticDir = "public"
)

var statikFS http.FileSystem

func init() {
	var err error
	statikFS, err = fs.New()
	if err != nil {
		log.Fatal(err)
	}
}

// Register 路由注册
func Register(m *macaron.Macaron) {
	m.SetURLPrefix(urlPrefix)
	// 所有GET方法，自动注册HEAD方法
	m.SetAutoHead(true)
	m.Get("/", func(ctx *macaron.Context) {
		file, err := statikFS.Open("/index.html")
		if err != nil {
			logger.Error("读取首页文件失败: %s", err)
			ctx.WriteHeader(http.StatusInternalServerError)
			return
		}

		io.Copy(ctx.Resp, file)

	})
	// 系统安装
	m.Group("/install", func() {
		m.Post("/store", binding.Bind(install.InstallForm{}), install.Store)
		m.Get("/status", func(ctx *macaron.Context) string {
			jsonResp := utils.JsonResponse{}
			return jsonResp.Success("", app.Installed)
		})
	})

	// 用户
	m.Group("/user", func() {
		m.Get("", user.Index)
		m.Get("/:id", user.Detail)
		m.Post("/store", binding.Bind(user.UserForm{}), user.Store)
		m.Post("/remove/:id", user.Remove)
		m.Post("/login", user.ValidateLogin)
		m.Post("/enable/:id", user.Enable)
		m.Post("/disable/:id", user.Disable)
		m.Post("/editMyPassword", user.UpdateMyPassword)
		m.Post("/editPassword/:id", user.UpdatePassword)
	})

	// 定时任务
	m.Group("/task", func() {
		m.Post("/store", binding.Bind(task.TaskForm{}), task.Store)
		m.Get("/:id", task.Detail)
		m.Get("", task.Index)
		m.Get("/log", tasklog.Index)
		m.Post("/log/clear", tasklog.Clear)
		m.Post("/log/stop", tasklog.Stop)
		m.Post("/remove/:id", task.Remove)
		m.Post("/enable/:id", task.Enable)
		m.Post("/disable/:id", task.Disable)
		m.Get("/run/:id", task.Run)
	})

	// 主机
	m.Group("/host", func() {
		m.Get("/:id", host.Detail)
		m.Post("/store", binding.Bind(host.HostForm{}), host.Store)
		m.Get("", host.Index)
		m.Get("/all", host.All)
		m.Get("/ping/:id", host.Ping)
		m.Post("/remove/:id", host.Remove)
	})

	// 管理
	m.Group("/system", func() {
		m.Group("/slack", func() {
			m.Get("", manage.Slack)
			m.Post("/update", manage.UpdateSlack)
			m.Post("/channel", manage.CreateSlackChannel)
			m.Post("/channel/remove/:id", manage.RemoveSlackChannel)
		})
		m.Group("/mail", func() {
			m.Get("", manage.Mail)
			m.Post("/update", binding.Bind(manage.MailServerForm{}), manage.UpdateMail)
			m.Post("/user", manage.CreateMailUser)
			m.Post("/user/remove/:id", manage.RemoveMailUser)
		})
		m.Group("/webhook", func() {
			m.Get("", manage.WebHook)
			m.Post("/update", manage.UpdateWebHook)
		})
		m.Get("/login-log", loginlog.Index)
	})

	// API
	m.Group("/v1", func() {
		m.Post("/tasklog/remove/:id", tasklog.Remove)
		m.Post("/task/enable/:id", task.Enable)
		m.Post("/task/disable/:id", task.Disable)
	}, apiAuth)

	// 404错误
	m.NotFound(func(ctx *macaron.Context) string {
		jsonResp := utils.JsonResponse{}

		return jsonResp.Failure(utils.NotFound, "您访问的页面不存在")
	})
	// 50x错误
	m.InternalServerError(func(ctx *macaron.Context) string {
		jsonResp := utils.JsonResponse{}

		return jsonResp.Failure(utils.ServerError, "服务器内部错误, 请稍后再试")
	})
}

// 中间件注册
func RegisterMiddleware(m *macaron.Macaron) {
	m.Use(macaron.Logger())
	m.Use(macaron.Recovery())
	if macaron.Env != macaron.DEV {
		m.Use(gzip.Gziper())
	}
	m.Use(
		macaron.Static(
			"",
			macaron.StaticOptions{
				Prefix:     staticDir,
				FileSystem: statikFS,
			},
		),
	)
	if macaron.Env == macaron.DEV {
		m.Use(toolbox.Toolboxer(m))
	}
	m.Use(macaron.Renderer())
	m.Use(checkAppInstall)
	m.Use(ipAuth)
	m.Use(userAuth)
	m.Use(urlAuth)
}

// region 自定义中间件

/** 检测应用是否已安装 **/
func checkAppInstall(ctx *macaron.Context) {
	if app.Installed {
		return
	}
	if strings.HasPrefix(ctx.Req.URL.Path, "/install") || ctx.Req.URL.Path == "/" {
		return
	}
	jsonResp := utils.JsonResponse{}

	data := jsonResp.Failure(utils.AppNotInstall, "应用未安装")
	ctx.Write([]byte(data))
}

// IP验证, 通过反向代理访问gocron，需设置Header X-Real-IP才能获取到客户端真实IP
func ipAuth(ctx *macaron.Context) {
	if !app.Installed {
		return
	}
	allowIpsStr := app.Setting.AllowIps
	if allowIpsStr == "" {
		return
	}
	clientIp := ctx.RemoteAddr()
	allowIps := strings.Split(allowIpsStr, ",")
	if utils.InStringSlice(allowIps, clientIp) {
		return
	}
	logger.Warnf("非法IP访问-%s", clientIp)
	jsonResp := utils.JsonResponse{}

	data := jsonResp.Failure(utils.UnauthorizedError, "您无权限访问")

	ctx.Write([]byte(data))
}

// 用户认证
func userAuth(ctx *macaron.Context) {
	if !app.Installed {
		return
	}
	user.RestoreToken(ctx)
	if user.IsLogin(ctx) {
		return
	}
	uri := strings.TrimRight(ctx.Req.URL.Path, "/")
	if strings.HasPrefix(uri, "/v1") {
		return
	}
	excludePaths := []string{"", "/user/login", "/install/status"}
	for _, path := range excludePaths {
		if uri == path {
			return
		}
	}
	jsonResp := utils.JsonResponse{}
	data := jsonResp.Failure(utils.AuthError, "认证失败")
	ctx.Write([]byte(data))

}

// URL权限验证
func urlAuth(ctx *macaron.Context) {
	if !app.Installed {
		return
	}
	if user.IsAdmin(ctx) {
		return
	}
	uri := strings.TrimRight(ctx.Req.URL.Path, "/")
	if strings.HasPrefix(uri, "/v1") {
		return
	}
	// 普通用户允许访问的URL地址
	allowPaths := []string{
		"",
		"/install/status",
		"/task",
		"/task/log",
		"/host",
		"/host/all",
		"/user/login",
		"/user/editMyPassword",
	}
	for _, path := range allowPaths {
		if path == uri {
			return
		}
	}

	jsonResp := utils.JsonResponse{}

	data := jsonResp.Failure(utils.UnauthorizedError, "您无权限访问")
	ctx.Write([]byte(data))
}

/** API接口签名验证 **/
func apiAuth(ctx *macaron.Context) {
	if !app.Installed {
		return
	}
	if !app.Setting.ApiSignEnable {
		return
	}
	apiKey := strings.TrimSpace(app.Setting.ApiKey)
	apiSecret := strings.TrimSpace(app.Setting.ApiSecret)
	json := utils.JsonResponse{}
	if apiKey == "" || apiSecret == "" {
		msg := json.CommonFailure("使用API前, 请先配置密钥")
		ctx.Write([]byte(msg))
		return
	}
	currentTimestamp := time.Now().Unix()
	time := ctx.QueryInt64("time")
	if time <= 0 {
		msg := json.CommonFailure("参数time不能为空")
		ctx.Write([]byte(msg))
		return
	}
	if time < (currentTimestamp - 1800) {
		msg := json.CommonFailure("time无效")
		ctx.Write([]byte(msg))
		return
	}
	sign := ctx.QueryTrim("sign")
	if sign == "" {
		msg := json.CommonFailure("参数sign不能为空")
		ctx.Write([]byte(msg))
		return
	}
	raw := apiKey + strconv.FormatInt(time, 10) + strings.TrimSpace(ctx.Req.URL.Path) + apiSecret
	realSign := utils.Md5(raw)
	if sign != realSign {
		msg := json.CommonFailure("签名验证失败")
		ctx.Write([]byte(msg))
		return
	}
}

// endregion
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/infobloxopen/themis/pdpctrl-client"

	log "github.com/Sirupsen/logrus"
)

func main() {
	log.SetLevel(log.InfoLevel)

	f, policy := openFile()
	defer f.Close()

	hosts := []*pdpcc.Client{}

	for _, addr := range conf.addresses {
		h := pdpcc.NewClient(addr, conf.chunkSize)
		if err := h.Connect(conf.timeout); err != nil {
			panic(err)
		}

		hosts = append(hosts, h)
		defer h.Close()
	}

	log.Infof("Requesting data upload to PDP servers...")

	uids := make([]int32, len(hosts))
	errors := 0
	for i, h := range hosts {
		var (
			ID  int32
			err error
		)
		if policy {
			ID, err = h.RequestPoliciesUpload(conf.fromTag, conf.toTag)
		} else {
			ID, err = h.RequestContentUpload(conf.contentID, conf.fromTag, conf.toTag)
		}

		if err != nil {
			log.Errorf("Failed to upload data: %v", err)
			uids[i] = -1
			errors++
		} else {
			uids[i] = ID
		}
	}

	if errors >= len(hosts) {
		panic(fmt.Errorf("no hosts accepted upload requests"))
	}

	log.Infof("Uploading data to PDP servers...")

	rem := 0
	for _, id := range uids {
		if id == -1 {
			continue
		}

		rem++
	}

	errors = 0
	for i, h := range hosts {
		id := uids[i]
		if id == -1 {
			continue
		}

		f.Seek(0, 0)
		nid, err := h.Upload(id, f)
		if err != nil {
			uids[i] = -1
			errors++
			log.Errorf("Failed to upload data: %v", err)
		} else {
			uids[i] = nid
		}
	}

	if errors >= rem {
		panic(fmt.Errorf("no hosts got data"))
	}

	for i, h := range hosts {
		id := uids[i]
		if id == -1 {
			continue
		}

		if err := h.Apply(id); err != nil {
			log.Errorf("Failed to apply: %v", err)
		}
	}
}

func openFile() (*os.File, bool) {
	pOk := len(conf.policy) > 0
	cOk := len(conf.content) > 0

	if pOk && cOk {
		panic(fmt.Errorf("both policy and content are specified. Please choose only one"))
	}

	if !pOk && !cOk {
		panic(fmt.Errorf("neither policy nor content are specified. Please secifiy any"))
	}

	path := conf.content
	if pOk {
		path = conf.policy
	}

	f, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	return f, pOk
}
<commit_msg>fixing NGP-423: adding 'NotifyReady' call after 'Apply' (#97)<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/infobloxopen/themis/pdpctrl-client"

	log "github.com/Sirupsen/logrus"
)

func main() {
	log.SetLevel(log.InfoLevel)

	f, policy := openFile()
	defer f.Close()

	hosts := []*pdpcc.Client{}

	for _, addr := range conf.addresses {
		h := pdpcc.NewClient(addr, conf.chunkSize)
		if err := h.Connect(conf.timeout); err != nil {
			panic(err)
		}

		hosts = append(hosts, h)
		defer h.Close()
	}

	log.Infof("Requesting data upload to PDP servers...")

	uids := make([]int32, len(hosts))
	errors := 0
	for i, h := range hosts {
		var (
			ID  int32
			err error
		)
		if policy {
			ID, err = h.RequestPoliciesUpload(conf.fromTag, conf.toTag)
		} else {
			ID, err = h.RequestContentUpload(conf.contentID, conf.fromTag, conf.toTag)
		}

		if err != nil {
			log.Errorf("Failed to upload data: %v", err)
			uids[i] = -1
			errors++
		} else {
			uids[i] = ID
		}
	}

	if errors >= len(hosts) {
		panic(fmt.Errorf("no hosts accepted upload requests"))
	}

	log.Infof("Uploading data to PDP servers...")

	rem := 0
	for _, id := range uids {
		if id == -1 {
			continue
		}

		rem++
	}

	errors = 0
	for i, h := range hosts {
		id := uids[i]
		if id == -1 {
			continue
		}

		f.Seek(0, 0)
		nid, err := h.Upload(id, f)
		if err != nil {
			uids[i] = -1
			errors++
			log.Errorf("Failed to upload data: %v", err)
		} else {
			uids[i] = nid
		}
	}

	if errors >= rem {
		panic(fmt.Errorf("no hosts got data"))
	}

	for i, h := range hosts {
		id := uids[i]
		if id == -1 {
			continue
		}

		if err := h.Apply(id); err != nil {
			log.Errorf("Failed to apply: %v", err)
		} else if err := h.NotifyReady(); err != nil {
			log.Errorf("Failed to signal readiness status to the PDP server: %v", err)
		}
	}
}

func openFile() (*os.File, bool) {
	pOk := len(conf.policy) > 0
	cOk := len(conf.content) > 0

	if pOk && cOk {
		panic(fmt.Errorf("both policy and content are specified. Please choose only one"))
	}

	if !pOk && !cOk {
		panic(fmt.Errorf("neither policy nor content are specified. Please secifiy any"))
	}

	path := conf.content
	if pOk {
		path = conf.policy
	}

	f, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	return f, pOk
}
<|endoftext|>


<commit_before>package ioutilmore

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"github.com/grokify/gotilla/encoding/jsonutil"
	"github.com/grokify/gotilla/type/maputil"
)

type FileType int

const (
	File FileType = iota
	Directory
	Any
)

func CopyFile(src, dst string) (err error) {
	r, err := os.Open(src)
	if err != nil {
		return
	}
	defer r.Close()

	w, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		if e := w.Close(); e != nil {
			err = e
		}
	}()

	_, err = io.Copy(w, r)
	if err != nil {
		return
	}

	err = w.Sync()
	if err != nil {
		return
	}

	si, err := os.Stat(src)
	if err != nil {
		return
	}
	err = os.Chmod(dst, si.Mode())
	if err != nil {
		return
	}

	return
}

func ReadDirSplit(dirname string, inclDotDirs bool) ([]os.FileInfo, []os.FileInfo, error) {
	all, err := ioutil.ReadDir(dirname)
	if err != nil {
		return []os.FileInfo{}, []os.FileInfo{}, err
	}
	subdirs, regular := FileInfosSplit(all, inclDotDirs)
	return subdirs, regular, nil
}

func FileInfosSplit(all []os.FileInfo, inclDotDirs bool) ([]os.FileInfo, []os.FileInfo) {
	subdirs := []os.FileInfo{}
	regular := []os.FileInfo{}
	for _, f := range all {
		if f.Mode().IsDir() {
			if f.Name() == "." && f.Name() == ".." {
				if inclDotDirs {
					subdirs = append(subdirs, f)
				}
			} else {
				subdirs = append(subdirs, f)
			}
		} else {
			regular = append(regular, f)
		}
	}
	return subdirs, regular
}

// DirEntriesNameRxVarFirsts returns a slice of the first
// regexp match encountered.
func DirEntriesNameRxVarFirsts(dir string, rx1 *regexp.Regexp) ([]string, error) {
	vars := map[string]int{}
	varsMatch := []string{}
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return varsMatch, err
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." {
			continue
		}
		if f.Size() > int64(0) {
			rs1 := rx1.FindStringSubmatch(f.Name())
			if len(rs1) > 1 { // len = 2+
				vars[rs1[1]] = 1
				//filesMatch = append(filesMatch, f)
			}
		}
	}
	for varVal := range vars {
		varsMatch = append(varsMatch, varVal)
	}
	return varsMatch, nil
}

func DirEntriesPathsReNotEmpty(dir string, rx1 *regexp.Regexp) ([]string, error) {
	paths := []string{}
	files, err := DirEntriesReNotEmpty(dir, rx1)
	if err != nil {
		return paths, err
	}
	for _, fi := range files {
		paths = append(paths, filepath.Join(dir, fi.Name()))
	}
	return paths, nil
}

// DirEntriesReNotEmpty returns a slide of files for non-empty files
// matching regular expression. It was formerly `DirEntriesReSizeGt0`.
func DirEntriesReNotEmpty(dir string, rx1 *regexp.Regexp) ([]os.FileInfo, error) {
	filesMatch := []os.FileInfo{}
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return filesMatch, err
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." {
			continue
		}
		if f.Size() > int64(0) {
			rs1 := rx1.FindStringSubmatch(f.Name())
			if len(rs1) > 0 {
				filesMatch = append(filesMatch, f)
			}
		}
	}
	return filesMatch, nil
}

func DirEntriesRxSizeGt0Filepaths(dir string, fileFilter FileType, rx *regexp.Regexp) ([]string, error) {
	fileinfos, err := DirEntriesRxSizeGt0(dir, fileFilter, rx)
	if err != nil {
		return []string{}, err
	}
	filepaths := []string{}
	for _, fi := range fileinfos {
		filepaths = append(filepaths, filepath.Join(dir, fi.Name()))
	}
	return filepaths, nil
}

func DirEntriesRxSizeGt0(dir string, fileFilter FileType, rx1 *regexp.Regexp) ([]os.FileInfo, error) {
	filesMatch := []os.FileInfo{}
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return filesMatch, err
	}
	for _, fi := range filesAll {
		if fi.Name() == "." || fi.Name() == ".." {
			continue
		} else if fileFilter == Directory && !fi.Mode().IsDir() {
			continue
		} else if fileFilter == File && !fi.Mode().IsRegular() {
			continue
		} else if fi.Size() <= int64(0) {
			continue
		}
		rs1 := rx1.FindStringSubmatch(fi.Name())
		if len(rs1) > 0 {
			filesMatch = append(filesMatch, fi)
		}
	}
	return filesMatch, nil
}

// DirEntriesRegexpGreatest takes a directory, regular expression and boolean to indicate
// whether to include zero size files and returns the greatest of a single match in the
// regular expression.
func DirFilesRegexpSubmatchGreatest(dir string, rx1 *regexp.Regexp, nonZeroFilesOnly bool) ([]os.FileInfo, error) {
	files := map[string][]os.FileInfo{}

	filesAll, e := ioutil.ReadDir(dir)
	if e != nil {
		return []os.FileInfo{}, e
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." ||
			(nonZeroFilesOnly && f.Size() <= int64(0)) {
			continue
		}

		if rs1 := rx1.FindStringSubmatch(f.Name()); len(rs1) > 1 {
			extract := rs1[1]
			if _, ok := files[extract]; !ok {
				files[extract] = []os.FileInfo{}
			}
			files[extract] = append(files[extract], f)
		}
	}
	keysSorted := maputil.StringKeysSorted(files)
	greatest := keysSorted[len(keysSorted)-1]
	return files[greatest], nil
}

// DirFilesRegexpSubmatchGreatestSubmatch takes a directory, regular expression and boolean to indicate
// whether to include zero size files and returns the greatest of a single match in the
// regular expression.
func DirFilesRegexpSubmatchGreatestSubmatch(dir string, rx1 *regexp.Regexp, nonZeroFilesOnly bool) (string, error) {
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", err
	}
	strs := []string{}
	for _, f := range filesAll {
		if nonZeroFilesOnly && f.Size() <= int64(0) {
			continue
		}
		rs1 := rx1.FindStringSubmatch(f.Name())
		if len(rs1) > 1 {
			strs = append(strs, rs1[1])
		}
	}
	sort.Strings(strs)
	if len(strs) == 0 {
		return "", fmt.Errorf("No matches found")
	}
	return strs[len(strs)-1], nil
}

func DirFromPath(path string) (string, error) {
	path = strings.TrimRight(path, "/\\")
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()
	fi, err := f.Stat()
	if err != nil {
		return "", err
	}
	isFile := false
	switch mode := fi.Mode(); {
	case mode.IsDir():
		return path, nil
	case mode.IsRegular():
		isFile = true
	}
	if isFile == false {
		return "", nil
	}
	rx1 := regexp.MustCompile(`^(.+)[/\\][^/\\]+`)
	rs1 := rx1.FindStringSubmatch(path)
	dir := ""
	if len(rs1) > 1 {
		dir = rs1[1]
	}
	return dir, nil
}

func IsDir(name string) (bool, error) {
	if fi, err := os.Stat(name); err != nil {
		return false, err
	} else if !fi.Mode().IsDir() {
		return false, nil
	}
	return true, nil
}

func IsFile(name string) (bool, error) {
	if fi, err := os.Stat(name); err != nil {
		return false, err
	} else if !fi.Mode().IsRegular() {
		return false, nil
	}
	return true, nil
}

func Exists(name string) (bool, error) {
	_, err := os.Stat(name)
	if os.IsNotExist(err) {
		return false, nil
	}
	return err != nil, err
}

// IsFileWithSizeGtZero verifies a path exists, is a file and is not empty,
// returning an error otherwise. An os file not exists check can be done
// with os.IsNotExist(err) which acts on error from os.Stat()
func IsFileWithSizeGtZero(name string) (bool, error) {
	if fi, err := os.Stat(name); err != nil {
		return false, err
	} else if !fi.Mode().IsRegular() {
		return false, nil
		// return fmt.Errorf("Filepath [%v] exists but is not a file.", name)
	} else if fi.Size() <= 0 {
		return false, nil
		// return fmt.Errorf("Filepath [%v] exists but is empty with size [%v].", name, fi.Size())
	}
	return true, nil
}

func SplitBetter(path string) (dir, file string) {
	isDir, err := IsDir(path)
	if err != nil && isDir {
		return dir, ""
	}
	return filepath.Split(path)
}

func SplitBest(path string) (dir, file string, err error) {
	isDir, err := IsDir(path)
	if err != nil {
		return "", "", err
	} else if isDir {
		return path, "", nil
	}
	isFile, err := IsFile(path)
	if err != nil {
		return "", "", err
	} else if isFile {
		dir, file := filepath.Split(path)
		return dir, file, nil
	}
	return "", "", fmt.Errorf("Path is valid but not file or directory: [%v]", path)
}

func FileinfosToFilepaths(dir string, fileInfos []os.FileInfo) []string {
	dir = strings.TrimSpace(dir)
	paths := []string{}
	for _, fi := range fileInfos {
		if len(dir) > 0 {
			paths = append(paths, filepath.Join(dir, fi.Name()))
		} else {
			paths = append(paths, fi.Name())
		}
	}
	return paths
}

func FilterFilenamesSizeGtZero(filepaths ...string) []string {
	filepathsExist := []string{}

	for _, envPathVal := range filepaths {
		envPathVals := strings.Split(envPathVal, ",")
		for _, envPath := range envPathVals {
			envPath = strings.TrimSpace(envPath)

			if isFile, err := IsFileWithSizeGtZero(envPath); isFile && err == nil {
				filepathsExist = append(filepathsExist, envPath)
			}
		}
	}
	return filepathsExist
}

func RemoveAllChildren(dir string) error {
	isDir, err := IsDir(dir)
	if err != nil {
		return err
	}
	if isDir == false {
		err = errors.New("400: Path Is Not Directory")
		return err
	}
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}
	for _, fi := range filesAll {
		if fi.Name() == "." || fi.Name() == ".." {
			continue
		}
		filepath := path.Join(dir, fi.Name())
		if fi.IsDir() {
			err = os.RemoveAll(filepath)
			if err != nil {
				return err
			}
		} else {
			err = os.Remove(filepath)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func FileinfosNames(fis []os.FileInfo) []string {
	s := []string{}
	for _, e := range fis {
		s = append(s, e.Name())
	}
	return s
}

// ReaderToBytes reads from an io.Reader, e.g. io.ReadCloser
func ReaderToBytes(ior io.Reader) []byte {
	buf := new(bytes.Buffer)
	buf.ReadFrom(ior)
	return buf.Bytes()
}

// ReadFileJSON reads and unmarshals a file.
func ReadFileJSON(file string, v interface{}) error {
	bytes, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	return json.Unmarshal(bytes, v)
}

func WriteFileJSON(filepath string, data interface{}, perm os.FileMode, prefix, indent string) error {
	bytes, err := jsonutil.MarshalSimple(data, prefix, indent)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filepath, bytes, perm)
}

type FileWriter struct {
	File   *os.File
	Writer *bufio.Writer
}

func NewFileWriter(path string) (FileWriter, error) {
	fw := FileWriter{}
	file, err := os.Create(path)
	if err != nil {
		return fw, err
	}

	fw.File = file
	fw.Writer = bufio.NewWriter(file)

	return fw, nil
}

func (f *FileWriter) Close() {
	f.Writer.Flush()
	f.File.Close()
}
<commit_msg>add ioutilmore.ReadDirRegexp()<commit_after>package ioutilmore

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"github.com/grokify/gotilla/encoding/jsonutil"
	"github.com/grokify/gotilla/type/maputil"
)

type FileType int

const (
	File FileType = iota
	Directory
	Any
)

func CopyFile(src, dst string) (err error) {
	r, err := os.Open(src)
	if err != nil {
		return
	}
	defer r.Close()

	w, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		if e := w.Close(); e != nil {
			err = e
		}
	}()

	_, err = io.Copy(w, r)
	if err != nil {
		return
	}

	err = w.Sync()
	if err != nil {
		return
	}

	si, err := os.Stat(src)
	if err != nil {
		return
	}
	err = os.Chmod(dst, si.Mode())
	if err != nil {
		return
	}

	return
}

func ReadDirSplit(dirname string, inclDotDirs bool) ([]os.FileInfo, []os.FileInfo, error) {
	all, err := ioutil.ReadDir(dirname)
	if err != nil {
		return []os.FileInfo{}, []os.FileInfo{}, err
	}
	subdirs, regular := FileInfosSplit(all, inclDotDirs)
	return subdirs, regular, nil
}

func FileInfosSplit(all []os.FileInfo, inclDotDirs bool) ([]os.FileInfo, []os.FileInfo) {
	subdirs := []os.FileInfo{}
	regular := []os.FileInfo{}
	for _, f := range all {
		if f.Mode().IsDir() {
			if f.Name() == "." && f.Name() == ".." {
				if inclDotDirs {
					subdirs = append(subdirs, f)
				}
			} else {
				subdirs = append(subdirs, f)
			}
		} else {
			regular = append(regular, f)
		}
	}
	return subdirs, regular
}

// DirEntriesNameRxVarFirsts returns a slice of the first
// regexp match encountered.
func DirEntriesNameRxVarFirsts(dir string, rx1 *regexp.Regexp) ([]string, error) {
	vars := map[string]int{}
	varsMatch := []string{}
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return varsMatch, err
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." {
			continue
		}
		if f.Size() > int64(0) {
			rs1 := rx1.FindStringSubmatch(f.Name())
			if len(rs1) > 1 { // len = 2+
				vars[rs1[1]] = 1
				//filesMatch = append(filesMatch, f)
			}
		}
	}
	for varVal := range vars {
		varsMatch = append(varsMatch, varVal)
	}
	return varsMatch, nil
}

func DirEntriesPathsReNotEmpty(dir string, rx1 *regexp.Regexp) ([]string, error) {
	paths := []string{}
	files, err := DirEntriesReNotEmpty(dir, rx1)
	if err != nil {
		return paths, err
	}
	for _, fi := range files {
		paths = append(paths, filepath.Join(dir, fi.Name()))
	}
	return paths, nil
}

func ReadDirRegexp(dir string, rx1 *regexp.Regexp, skipEmpty bool) ([]os.FileInfo, []string, error) {
	filesMatch := []os.FileInfo{}
	filenames := []string{}
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return filesMatch, filenames, err
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." {
			continue
		}
		if (skipEmpty && f.Size() > int64(0)) || !skipEmpty {
			rs1 := rx1.FindStringSubmatch(f.Name())
			if len(rs1) > 0 {
				filesMatch = append(filesMatch, f)
				filenames = append(filenames, filepath.Join(dir, f.Name()))
			}
		}
	}
	return filesMatch, filenames, nil
}

// DirEntriesReNotEmpty returns a slide of files for non-empty files
// matching regular expression. It was formerly `DirEntriesReSizeGt0`.
func DirEntriesReNotEmpty(dir string, rx1 *regexp.Regexp) ([]os.FileInfo, error) {
	files, _, err := ReadDirRegexp(dir, rx1, true)
	return files, err
	/* filesMatch := []os.FileInfo{}
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return filesMatch, err
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." {
			continue
		}
		if f.Size() > int64(0) {
			rs1 := rx1.FindStringSubmatch(f.Name())
			if len(rs1) > 0 {
				filesMatch = append(filesMatch, f)
			}
		}
	}
	return filesMatch, nil */
}

func DirEntriesRxSizeGt0Filepaths(dir string, fileFilter FileType, rx *regexp.Regexp) ([]string, error) {
	fileinfos, err := DirEntriesRxSizeGt0(dir, fileFilter, rx)
	if err != nil {
		return []string{}, err
	}
	filepaths := []string{}
	for _, fi := range fileinfos {
		filepaths = append(filepaths, filepath.Join(dir, fi.Name()))
	}
	return filepaths, nil
}

func DirEntriesRxSizeGt0(dir string, fileFilter FileType, rx1 *regexp.Regexp) ([]os.FileInfo, error) {
	filesMatch := []os.FileInfo{}
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return filesMatch, err
	}
	for _, fi := range filesAll {
		if fi.Name() == "." || fi.Name() == ".." {
			continue
		} else if fileFilter == Directory && !fi.Mode().IsDir() {
			continue
		} else if fileFilter == File && !fi.Mode().IsRegular() {
			continue
		} else if fi.Size() <= int64(0) {
			continue
		}
		rs1 := rx1.FindStringSubmatch(fi.Name())
		if len(rs1) > 0 {
			filesMatch = append(filesMatch, fi)
		}
	}
	return filesMatch, nil
}

// DirEntriesRegexpGreatest takes a directory, regular expression and boolean to indicate
// whether to include zero size files and returns the greatest of a single match in the
// regular expression.
func DirFilesRegexpSubmatchGreatest(dir string, rx1 *regexp.Regexp, nonZeroFilesOnly bool) ([]os.FileInfo, error) {
	files := map[string][]os.FileInfo{}

	filesAll, e := ioutil.ReadDir(dir)
	if e != nil {
		return []os.FileInfo{}, e
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." ||
			(nonZeroFilesOnly && f.Size() <= int64(0)) {
			continue
		}

		if rs1 := rx1.FindStringSubmatch(f.Name()); len(rs1) > 1 {
			extract := rs1[1]
			if _, ok := files[extract]; !ok {
				files[extract] = []os.FileInfo{}
			}
			files[extract] = append(files[extract], f)
		}
	}
	keysSorted := maputil.StringKeysSorted(files)
	greatest := keysSorted[len(keysSorted)-1]
	return files[greatest], nil
}

// DirFilesRegexpSubmatchGreatestSubmatch takes a directory, regular expression and boolean to indicate
// whether to include zero size files and returns the greatest of a single match in the
// regular expression.
func DirFilesRegexpSubmatchGreatestSubmatch(dir string, rx1 *regexp.Regexp, nonZeroFilesOnly bool) (string, error) {
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", err
	}
	strs := []string{}
	for _, f := range filesAll {
		if nonZeroFilesOnly && f.Size() <= int64(0) {
			continue
		}
		rs1 := rx1.FindStringSubmatch(f.Name())
		if len(rs1) > 1 {
			strs = append(strs, rs1[1])
		}
	}
	sort.Strings(strs)
	if len(strs) == 0 {
		return "", fmt.Errorf("No matches found")
	}
	return strs[len(strs)-1], nil
}

func DirFromPath(path string) (string, error) {
	path = strings.TrimRight(path, "/\\")
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()
	fi, err := f.Stat()
	if err != nil {
		return "", err
	}
	isFile := false
	switch mode := fi.Mode(); {
	case mode.IsDir():
		return path, nil
	case mode.IsRegular():
		isFile = true
	}
	if isFile == false {
		return "", nil
	}
	rx1 := regexp.MustCompile(`^(.+)[/\\][^/\\]+`)
	rs1 := rx1.FindStringSubmatch(path)
	dir := ""
	if len(rs1) > 1 {
		dir = rs1[1]
	}
	return dir, nil
}

func IsDir(name string) (bool, error) {
	if fi, err := os.Stat(name); err != nil {
		return false, err
	} else if !fi.Mode().IsDir() {
		return false, nil
	}
	return true, nil
}

func IsFile(name string) (bool, error) {
	if fi, err := os.Stat(name); err != nil {
		return false, err
	} else if !fi.Mode().IsRegular() {
		return false, nil
	}
	return true, nil
}

func Exists(name string) (bool, error) {
	_, err := os.Stat(name)
	if os.IsNotExist(err) {
		return false, nil
	}
	return err != nil, err
}

// IsFileWithSizeGtZero verifies a path exists, is a file and is not empty,
// returning an error otherwise. An os file not exists check can be done
// with os.IsNotExist(err) which acts on error from os.Stat()
func IsFileWithSizeGtZero(name string) (bool, error) {
	if fi, err := os.Stat(name); err != nil {
		return false, err
	} else if !fi.Mode().IsRegular() {
		return false, nil
		// return fmt.Errorf("Filepath [%v] exists but is not a file.", name)
	} else if fi.Size() <= 0 {
		return false, nil
		// return fmt.Errorf("Filepath [%v] exists but is empty with size [%v].", name, fi.Size())
	}
	return true, nil
}

func SplitBetter(path string) (dir, file string) {
	isDir, err := IsDir(path)
	if err != nil && isDir {
		return dir, ""
	}
	return filepath.Split(path)
}

func SplitBest(path string) (dir, file string, err error) {
	isDir, err := IsDir(path)
	if err != nil {
		return "", "", err
	} else if isDir {
		return path, "", nil
	}
	isFile, err := IsFile(path)
	if err != nil {
		return "", "", err
	} else if isFile {
		dir, file := filepath.Split(path)
		return dir, file, nil
	}
	return "", "", fmt.Errorf("Path is valid but not file or directory: [%v]", path)
}

func FileinfosToFilepaths(dir string, fileInfos []os.FileInfo) []string {
	dir = strings.TrimSpace(dir)
	paths := []string{}
	for _, fi := range fileInfos {
		if len(dir) > 0 {
			paths = append(paths, filepath.Join(dir, fi.Name()))
		} else {
			paths = append(paths, fi.Name())
		}
	}
	return paths
}

func FilterFilenamesSizeGtZero(filepaths ...string) []string {
	filepathsExist := []string{}

	for _, envPathVal := range filepaths {
		envPathVals := strings.Split(envPathVal, ",")
		for _, envPath := range envPathVals {
			envPath = strings.TrimSpace(envPath)

			if isFile, err := IsFileWithSizeGtZero(envPath); isFile && err == nil {
				filepathsExist = append(filepathsExist, envPath)
			}
		}
	}
	return filepathsExist
}

func RemoveAllChildren(dir string) error {
	isDir, err := IsDir(dir)
	if err != nil {
		return err
	}
	if isDir == false {
		err = errors.New("400: Path Is Not Directory")
		return err
	}
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}
	for _, fi := range filesAll {
		if fi.Name() == "." || fi.Name() == ".." {
			continue
		}
		filepath := path.Join(dir, fi.Name())
		if fi.IsDir() {
			err = os.RemoveAll(filepath)
			if err != nil {
				return err
			}
		} else {
			err = os.Remove(filepath)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func FileinfosNames(fis []os.FileInfo) []string {
	s := []string{}
	for _, e := range fis {
		s = append(s, e.Name())
	}
	return s
}

// ReaderToBytes reads from an io.Reader, e.g. io.ReadCloser
func ReaderToBytes(ior io.Reader) []byte {
	buf := new(bytes.Buffer)
	buf.ReadFrom(ior)
	return buf.Bytes()
}

// ReadFileJSON reads and unmarshals a file.
func ReadFileJSON(file string, v interface{}) error {
	bytes, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	return json.Unmarshal(bytes, v)
}

func WriteFileJSON(filepath string, data interface{}, perm os.FileMode, prefix, indent string) error {
	bytes, err := jsonutil.MarshalSimple(data, prefix, indent)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filepath, bytes, perm)
}

type FileWriter struct {
	File   *os.File
	Writer *bufio.Writer
}

func NewFileWriter(path string) (FileWriter, error) {
	fw := FileWriter{}
	file, err := os.Create(path)
	if err != nil {
		return fw, err
	}

	fw.File = file
	fw.Writer = bufio.NewWriter(file)

	return fw, nil
}

func (f *FileWriter) Close() {
	f.Writer.Flush()
	f.File.Close()
}
<|endoftext|>


<commit_before>package ioutilmore

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"sort"
	"strings"

	"github.com/grokify/gotilla/type/maputil"
)

func CopyFile(src, dst string) (err error) {
	r, err := os.Open(src)
	if err != nil {
		return
	}
	defer r.Close()

	w, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		if e := w.Close(); e != nil {
			err = e
		}
	}()

	_, err = io.Copy(w, r)
	if err != nil {
		return
	}

	err = w.Sync()
	if err != nil {
		return
	}

	si, err := os.Stat(src)
	if err != nil {
		return
	}
	err = os.Chmod(dst, si.Mode())
	if err != nil {
		return
	}

	return
}

func ReadDirSplit(dirname string, skipDotDirs bool) ([]os.FileInfo, []os.FileInfo, error) {
	all, err := ioutil.ReadDir(dirname)
	if err != nil {
		return []os.FileInfo{}, []os.FileInfo{}, err
	}
	subdirs, regular := FileInfosSplit(all, skipDotDirs)
	return subdirs, regular, nil
}

func FileInfosSplit(all []os.FileInfo, skipDotDirs bool) ([]os.FileInfo, []os.FileInfo) {
	subdirs := []os.FileInfo{}
	regular := []os.FileInfo{}
	for _, f := range all {
		if f.Mode().IsDir() {
			if !skipDotDirs || (f.Name() != "." && f.Name() != "..") {
				subdirs = append(subdirs, f)
			}
		} else {
			regular = append(regular, f)
		}
	}
	return subdirs, regular
}

func DirEntriesReSizeGt0(dir string, rx1 *regexp.Regexp) ([]os.FileInfo, error) {
	filesMatch := []os.FileInfo{}
	filesAll, e := ioutil.ReadDir(dir)
	if e != nil {
		return filesMatch, e
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." {
			continue
		}
		if f.Size() > int64(0) {
			rs1 := rx1.FindStringSubmatch(f.Name())
			if len(rs1) > 0 {
				filesMatch = append(filesMatch, f)
			}
		}
	}
	return filesMatch, nil
}

// DirEntriesRegexpGreatest takes a directory, regular expression and boolean to indicate
// whether to include zero size files and returns the greatest of a single match in the
// regular expression.
func DirFilesRegexpSubmatchGreatest(dir string, rx1 *regexp.Regexp, nonZeroFilesOnly bool) ([]os.FileInfo, error) {
	files := map[string][]os.FileInfo{}

	filesAll, e := ioutil.ReadDir(dir)
	if e != nil {
		return []os.FileInfo{}, e
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." ||
			(nonZeroFilesOnly && f.Size() <= int64(0)) {
			continue
		}

		if rs1 := rx1.FindStringSubmatch(f.Name()); len(rs1) > 1 {
			extract := rs1[1]
			if _, ok := files[extract]; !ok {
				files[extract] = []os.FileInfo{}
			}
			files[extract] = append(files[extract], f)
		}
	}
	keysSorted := maputil.StringKeysSorted(files)
	greatest := keysSorted[len(keysSorted)-1]
	return files[greatest], nil
}

// DirFilesRegexpSubmatchGreatestSubmatch takes a directory, regular expression and boolean to indicate
// whether to include zero size files and returns the greatest of a single match in the
// regular expression.
func DirFilesRegexpSubmatchGreatestSubmatch(dir string, rx1 *regexp.Regexp, nonZeroFilesOnly bool) (string, error) {
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", err
	}
	strs := []string{}
	for _, f := range filesAll {
		if nonZeroFilesOnly && f.Size() <= int64(0) {
			continue
		}
		rs1 := rx1.FindStringSubmatch(f.Name())
		if len(rs1) > 1 {
			strs = append(strs, rs1[1])
		}
	}
	sort.Strings(strs)
	if len(strs) == 0 {
		return "", fmt.Errorf("No matches found")
	}
	return strs[len(strs)-1], nil
}

func DirFromPath(path string) (string, error) {
	path = strings.TrimRight(path, "/\\")
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()
	fi, err := f.Stat()
	if err != nil {
		return "", err
	}
	isFile := false
	switch mode := fi.Mode(); {
	case mode.IsDir():
		return path, nil
	case mode.IsRegular():
		isFile = true
	}
	if isFile == false {
		return "", nil
	}
	rx1 := regexp.MustCompile(`^(.+)[/\\][^/\\]+`)
	rs1 := rx1.FindStringSubmatch(path)
	dir := ""
	if len(rs1) > 1 {
		dir = rs1[1]
	}
	return dir, nil
}

func GetFileInfo(path string) (os.FileInfo, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return f.Stat()
}

func IsDir(path string) (bool, error) {
	fi, err := GetFileInfo(path)
	if err != nil {
		return false, err
	}
	switch mode := fi.Mode(); {
	case mode.IsDir():
		return true, nil
	case mode.IsRegular():
		return false, nil
	}
	return false, nil
}

func IsFile(path string) (bool, error) {
	fi, err := GetFileInfo(path)
	if err != nil {
		return false, err
	}
	switch mode := fi.Mode(); {
	case mode.IsDir():
		return false, nil
	case mode.IsRegular():
		return true, nil
	}
	return false, nil
}

// IsFileWithSizeGtZero verifies a path exists, is a file and is not empty,
// returning an error otherwise. An os file not exists check can be done
// with os.IsNotExist(err) which acts on error from os.Stat()
func IsFileWithSizeGtZero(path string) error {
	fi, err := os.Stat(path)
	if err != nil {
		return err
	}
	if fi.Mode().IsRegular() == false {
		return fmt.Errorf("Filepath [%v] exists but is not a file.", path)
	} else if fi.Size() <= 0 {
		return fmt.Errorf("Filepath [%v] exists but is empty with size [%v].", path, fi.Size())
	}
	return nil
}

func FilterFilenamesSizeGtZero(filepaths ...string) []string {
	filepathsExist := []string{}

	for _, envPathVal := range filepaths {
		envPathVals := strings.Split(envPathVal, ",")
		for _, envPath := range envPathVals {
			envPath = strings.TrimSpace(envPath)

			if err := IsFileWithSizeGtZero(envPath); err == nil {
				filepathsExist = append(filepathsExist, envPath)
			}
		}
	}
	return filepathsExist
}

func RemoveAllChildren(dir string) error {
	isDir, err := IsDir(dir)
	if err != nil {
		return err
	}
	if isDir == false {
		err = errors.New("400: Path Is Not Directory")
		return err
	}
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}
	for _, fi := range filesAll {
		if fi.Name() == "." || fi.Name() == ".." {
			continue
		}
		filepath := path.Join(dir, fi.Name())
		if fi.IsDir() {
			err = os.RemoveAll(filepath)
			if err != nil {
				return err
			}
		} else {
			err = os.Remove(filepath)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// ReaderToBytes reads from an io.Reader, e.g. io.ReadCloser
func ReaderToBytes(ior io.Reader) []byte {
	buf := new(bytes.Buffer)
	buf.ReadFrom(ior)
	return buf.Bytes()
}

// ReadFileJSON reads and unmarshals a file.
func ReadFileJSON(file string, v interface{}) error {
	bytes, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	return json.Unmarshal(bytes, v)
}

func WriteFileJSON(filepath string, data interface{}, perm os.FileMode, wantPretty bool) error {
	var bytes []byte
	if wantPretty {
		bytesTry, err := json.MarshalIndent(data, "", "  ")
		if err != nil {
			return err
		}
		bytes = bytesTry
	} else {
		bytesTry, err := json.Marshal(data)
		if err != nil {
			return err
		}
		bytes = bytesTry
	}
	return ioutil.WriteFile(filepath, bytes, perm)
}

type FileWriter struct {
	File   *os.File
	Writer *bufio.Writer
}

func NewFileWriter(path string) (FileWriter, error) {
	fw := FileWriter{}
	file, err := os.Create(path)
	if err != nil {
		return fw, err
	}

	fw.File = file
	fw.Writer = bufio.NewWriter(file)

	return fw, nil
}

func (f *FileWriter) Close() {
	f.Writer.Flush()
	f.File.Close()
}
<commit_msg>refactor iom.IsFileWithSizeGtZero<commit_after>package ioutilmore

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"sort"
	"strings"

	"github.com/grokify/gotilla/type/maputil"
)

func CopyFile(src, dst string) (err error) {
	r, err := os.Open(src)
	if err != nil {
		return
	}
	defer r.Close()

	w, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		if e := w.Close(); e != nil {
			err = e
		}
	}()

	_, err = io.Copy(w, r)
	if err != nil {
		return
	}

	err = w.Sync()
	if err != nil {
		return
	}

	si, err := os.Stat(src)
	if err != nil {
		return
	}
	err = os.Chmod(dst, si.Mode())
	if err != nil {
		return
	}

	return
}

func ReadDirSplit(dirname string, skipDotDirs bool) ([]os.FileInfo, []os.FileInfo, error) {
	all, err := ioutil.ReadDir(dirname)
	if err != nil {
		return []os.FileInfo{}, []os.FileInfo{}, err
	}
	subdirs, regular := FileInfosSplit(all, skipDotDirs)
	return subdirs, regular, nil
}

func FileInfosSplit(all []os.FileInfo, skipDotDirs bool) ([]os.FileInfo, []os.FileInfo) {
	subdirs := []os.FileInfo{}
	regular := []os.FileInfo{}
	for _, f := range all {
		if f.Mode().IsDir() {
			if !skipDotDirs || (f.Name() != "." && f.Name() != "..") {
				subdirs = append(subdirs, f)
			}
		} else {
			regular = append(regular, f)
		}
	}
	return subdirs, regular
}

func DirEntriesReSizeGt0(dir string, rx1 *regexp.Regexp) ([]os.FileInfo, error) {
	filesMatch := []os.FileInfo{}
	filesAll, e := ioutil.ReadDir(dir)
	if e != nil {
		return filesMatch, e
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." {
			continue
		}
		if f.Size() > int64(0) {
			rs1 := rx1.FindStringSubmatch(f.Name())
			if len(rs1) > 0 {
				filesMatch = append(filesMatch, f)
			}
		}
	}
	return filesMatch, nil
}

// DirEntriesRegexpGreatest takes a directory, regular expression and boolean to indicate
// whether to include zero size files and returns the greatest of a single match in the
// regular expression.
func DirFilesRegexpSubmatchGreatest(dir string, rx1 *regexp.Regexp, nonZeroFilesOnly bool) ([]os.FileInfo, error) {
	files := map[string][]os.FileInfo{}

	filesAll, e := ioutil.ReadDir(dir)
	if e != nil {
		return []os.FileInfo{}, e
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." ||
			(nonZeroFilesOnly && f.Size() <= int64(0)) {
			continue
		}

		if rs1 := rx1.FindStringSubmatch(f.Name()); len(rs1) > 1 {
			extract := rs1[1]
			if _, ok := files[extract]; !ok {
				files[extract] = []os.FileInfo{}
			}
			files[extract] = append(files[extract], f)
		}
	}
	keysSorted := maputil.StringKeysSorted(files)
	greatest := keysSorted[len(keysSorted)-1]
	return files[greatest], nil
}

// DirFilesRegexpSubmatchGreatestSubmatch takes a directory, regular expression and boolean to indicate
// whether to include zero size files and returns the greatest of a single match in the
// regular expression.
func DirFilesRegexpSubmatchGreatestSubmatch(dir string, rx1 *regexp.Regexp, nonZeroFilesOnly bool) (string, error) {
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", err
	}
	strs := []string{}
	for _, f := range filesAll {
		if nonZeroFilesOnly && f.Size() <= int64(0) {
			continue
		}
		rs1 := rx1.FindStringSubmatch(f.Name())
		if len(rs1) > 1 {
			strs = append(strs, rs1[1])
		}
	}
	sort.Strings(strs)
	if len(strs) == 0 {
		return "", fmt.Errorf("No matches found")
	}
	return strs[len(strs)-1], nil
}

func DirFromPath(path string) (string, error) {
	path = strings.TrimRight(path, "/\\")
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()
	fi, err := f.Stat()
	if err != nil {
		return "", err
	}
	isFile := false
	switch mode := fi.Mode(); {
	case mode.IsDir():
		return path, nil
	case mode.IsRegular():
		isFile = true
	}
	if isFile == false {
		return "", nil
	}
	rx1 := regexp.MustCompile(`^(.+)[/\\][^/\\]+`)
	rs1 := rx1.FindStringSubmatch(path)
	dir := ""
	if len(rs1) > 1 {
		dir = rs1[1]
	}
	return dir, nil
}

func GetFileInfo(path string) (os.FileInfo, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return f.Stat()
}

func IsDir(path string) (bool, error) {
	fi, err := GetFileInfo(path)
	if err != nil {
		return false, err
	}
	switch mode := fi.Mode(); {
	case mode.IsDir():
		return true, nil
	case mode.IsRegular():
		return false, nil
	}
	return false, nil
}

func IsFile(path string) (bool, error) {
	fi, err := GetFileInfo(path)
	if err != nil {
		return false, err
	}
	switch mode := fi.Mode(); {
	case mode.IsDir():
		return false, nil
	case mode.IsRegular():
		return true, nil
	}
	return false, nil
}

// IsFileWithSizeGtZero verifies a path exists, is a file and is not empty,
// returning an error otherwise. An os file not exists check can be done
// with os.IsNotExist(err) which acts on error from os.Stat()
func IsFileWithSizeGtZero(path string) error {
	if fi, err := os.Stat(path); err != nil {
		return err
	} else if fi.Mode().IsRegular() == false {
		return fmt.Errorf("Filepath [%v] exists but is not a file.", path)
	} else if fi.Size() <= 0 {
		return fmt.Errorf("Filepath [%v] exists but is empty with size [%v].", path, fi.Size())
	}
	return nil
}

func FilterFilenamesSizeGtZero(filepaths ...string) []string {
	filepathsExist := []string{}

	for _, envPathVal := range filepaths {
		envPathVals := strings.Split(envPathVal, ",")
		for _, envPath := range envPathVals {
			envPath = strings.TrimSpace(envPath)

			if err := IsFileWithSizeGtZero(envPath); err == nil {
				filepathsExist = append(filepathsExist, envPath)
			}
		}
	}
	return filepathsExist
}

func RemoveAllChildren(dir string) error {
	isDir, err := IsDir(dir)
	if err != nil {
		return err
	}
	if isDir == false {
		err = errors.New("400: Path Is Not Directory")
		return err
	}
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}
	for _, fi := range filesAll {
		if fi.Name() == "." || fi.Name() == ".." {
			continue
		}
		filepath := path.Join(dir, fi.Name())
		if fi.IsDir() {
			err = os.RemoveAll(filepath)
			if err != nil {
				return err
			}
		} else {
			err = os.Remove(filepath)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// ReaderToBytes reads from an io.Reader, e.g. io.ReadCloser
func ReaderToBytes(ior io.Reader) []byte {
	buf := new(bytes.Buffer)
	buf.ReadFrom(ior)
	return buf.Bytes()
}

// ReadFileJSON reads and unmarshals a file.
func ReadFileJSON(file string, v interface{}) error {
	bytes, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	return json.Unmarshal(bytes, v)
}

func WriteFileJSON(filepath string, data interface{}, perm os.FileMode, wantPretty bool) error {
	var bytes []byte
	if wantPretty {
		bytesTry, err := json.MarshalIndent(data, "", "  ")
		if err != nil {
			return err
		}
		bytes = bytesTry
	} else {
		bytesTry, err := json.Marshal(data)
		if err != nil {
			return err
		}
		bytes = bytesTry
	}
	return ioutil.WriteFile(filepath, bytes, perm)
}

type FileWriter struct {
	File   *os.File
	Writer *bufio.Writer
}

func NewFileWriter(path string) (FileWriter, error) {
	fw := FileWriter{}
	file, err := os.Create(path)
	if err != nil {
		return fw, err
	}

	fw.File = file
	fw.Writer = bufio.NewWriter(file)

	return fw, nil
}

func (f *FileWriter) Close() {
	f.Writer.Flush()
	f.File.Close()
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, version 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package logger

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/c4milo/handlers/internal"
	"github.com/satori/go.uuid"
)

// http://commandcenter.blogspot.com/2014/01/self-referential-functions-and-design.html
type option func(*handler)

// Internal handler
type handler struct {
	name   string
	format string
	flags  int
	out    io.Writer
}

// AppName allows to set the application name to log.
func AppName(name string) option {
	return func(l *handler) {
		l.name = name
	}
}

// Format allows to set a custom log format. Although, the timestamp is always logged at the beginning.
// This handler is a bit opinionated.
//
// Directives:
//
// {remote_user}		: Remote user if Basic Auth credentials were sent
// {remote_ip}			: Remote IP address.
// {latency}			: The time taken to serve the request, in microseconds.
// {latency_human}		: The time taken to serve the request, human readable.
// {id}					: The request ID.
// {host}				: The Host header sent to the server
// {method}				: The request method. Ex: GET, POST, DELETE, etc.
// {url}				: The URL path requested.
// {query}				: Request's query string
// {rxbytes}			: Bytes received without headers
// {txbytes}			: Bytes sent, excluding HTTP headers.
// {status}				: Status sent to the client
// {useragent}			: User Agent
// {referer}			: The site from where the request came from
//
func Format(format string) option {
	return func(l *handler) {
		l.format = format
	}
}

// Flags allows to set logging flags using Go's standard log flags.
//
// Example: log.LstdFlags | log.shortfile
// Keep in mind that log.shortfile and log.Llongfile are expensive flags
func Flags(flags int) option {
	return func(l *handler) {
		l.flags = flags
	}
}

// Output allows setting an output writer for logging to be written to
func Output(out io.Writer) option {
	return func(l *handler) {
		l.out = out
	}
}

// Handler does HTTP request logging
func Handler(h http.Handler, opts ...option) http.Handler {
	// Default options
	handler := &handler{
		name:   "unknown",
		format: `{id} remote_ip={remote_ip} {method} "{host}{url}?{query}" rxbytes={rxbytes} status={status} latency_human={latency_human} latency={latency} txbytes={txbytes}`,
		out:    os.Stdout,
		flags:  log.LstdFlags,
	}

	for _, opt := range opts {
		opt(handler)
	}

	l := log.New(handler.out, fmt.Sprintf("[%s] ", handler.name), handler.flags)

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()

		// If there is a request ID already, we use it to keep the transaction
		// traceable. If not, we generate a new request ID.
		reqID := w.Header().Get("Request-ID")
		if reqID == "" {
			reqID = uuid.NewV4().String()
		}

		w.Header().Set("Request-ID", reqID)

		l.Print(applyLogFormat(handler.format, -1, w, r))

		res := internal.NewResponseWriter(w)
		h.ServeHTTP(res, r)

		latency := time.Since(start)
		l.Print(applyLogFormat(handler.format, latency, res, r))
	})
}

func applyLogFormat(format string, latency time.Duration, w http.ResponseWriter, r *http.Request) string {
	reqID := w.Header().Get("Request-ID")

	if strings.Index(format, "{remote_ip}") > -1 {
		format = strings.Replace(format, "{remote_ip}", strings.Split(r.RemoteAddr, ":")[0], -1)
	}

	if strings.Index(format, "{remote_user}") > -1 {
		user, _, _ := r.BasicAuth()
		if user == "" {
			user = r.URL.User.Username()
		}
		format = strings.Replace(format, "{remote_user}", user, -1)
	}

	if strings.Index(format, "{latency_human}") > -1 {
		l := "..."
		if latency > -1 {
			l = latency.String()
		}
		format = strings.Replace(format, "{latency_human}", l, -1)
	}

	if strings.Index(format, "{latency}") > -1 {
		l := "..."
		if latency > -1 {
			l = strconv.FormatInt(latency.Nanoseconds(), 10)
		}
		format = strings.Replace(format, "{latency}", l, -1)
	}

	if strings.Index(format, "{id}") > -1 {
		format = strings.Replace(format, "{id}", reqID, -1)
	}

	if strings.Index(format, "{method}") > -1 {
		format = strings.Replace(format, "{method}", r.Method, -1)
	}

	if strings.Index(format, "{url}") > -1 {
		format = strings.Replace(format, "{url}", r.URL.Path, -1)
	}

	if strings.Index(format, "{query}") > -1 {
		format = strings.Replace(format, "{query}", r.URL.RawQuery, -1)
	}

	if strings.Index(format, "{rxbytes}") > -1 {
		format = strings.Replace(format, "{rxbytes}", strconv.FormatInt(r.ContentLength, 10), -1)
	}

	if strings.Index(format, "{txbytes}") > -1 {
		size := "..."
		if v, ok := w.(internal.ResponseWriter); ok {
			size = strconv.Itoa(v.Size())
		}
		format = strings.Replace(format, "{txbytes}", size, -1)
	}

	if strings.Index(format, "{status}") > -1 {
		status := "..."
		if v, ok := w.(internal.ResponseWriter); ok {
			status = strconv.Itoa(v.Status())
		}
		format = strings.Replace(format, "{status}", status, -1)
	}

	if strings.Index(format, "{useragent}") > -1 {
		format = strings.Replace(format, "{useragent}", r.UserAgent(), -1)
	}

	if strings.Index(format, "{host}") > -1 {
		format = strings.Replace(format, "{host}", r.Host, -1)
	}

	if strings.Index(format, "{referer}") > -1 {
		format = strings.Replace(format, "{referer}", r.Referer(), -1)
	}

	return format
}
<commit_msg>Reorders log output to give priority to the HTTP status response code.<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, version 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package logger

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/c4milo/handlers/internal"
	"github.com/satori/go.uuid"
)

// http://commandcenter.blogspot.com/2014/01/self-referential-functions-and-design.html
type option func(*handler)

// Internal handler
type handler struct {
	name   string
	format string
	flags  int
	out    io.Writer
}

// AppName allows to set the application name to log.
func AppName(name string) option {
	return func(l *handler) {
		l.name = name
	}
}

// Format allows to set a custom log format. Although, the timestamp is always logged at the beginning.
// This handler is a bit opinionated.
//
// Directives:
//
// {remote_user}		: Remote user if Basic Auth credentials were sent
// {remote_ip}			: Remote IP address.
// {latency}			: The time taken to serve the request, in microseconds.
// {latency_human}		: The time taken to serve the request, human readable.
// {id}					: The request ID.
// {host}				: The Host header sent to the server
// {method}				: The request method. Ex: GET, POST, DELETE, etc.
// {url}				: The URL path requested.
// {query}				: Request's query string
// {rxbytes}			: Bytes received without headers
// {txbytes}			: Bytes sent, excluding HTTP headers.
// {status}				: Status sent to the client
// {useragent}			: User Agent
// {referer}			: The site from where the request came from
//
func Format(format string) option {
	return func(l *handler) {
		l.format = format
	}
}

// Flags allows to set logging flags using Go's standard log flags.
//
// Example: log.LstdFlags | log.shortfile
// Keep in mind that log.shortfile and log.Llongfile are expensive flags
func Flags(flags int) option {
	return func(l *handler) {
		l.flags = flags
	}
}

// Output allows setting an output writer for logging to be written to
func Output(out io.Writer) option {
	return func(l *handler) {
		l.out = out
	}
}

// Handler does HTTP request logging
func Handler(h http.Handler, opts ...option) http.Handler {
	// Default options
	handler := &handler{
		name:   "unknown",
		format: `{id} remote_ip={remote_ip} {method} "{host}{url}?{query}" status={status} latency_human={latency_human} latency={latency} rxbytes={rxbytes} txbytes={txbytes}`,
		out:    os.Stdout,
		flags:  log.LstdFlags,
	}

	for _, opt := range opts {
		opt(handler)
	}

	l := log.New(handler.out, fmt.Sprintf("[%s] ", handler.name), handler.flags)

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()

		// If there is a request ID already, we use it to keep the transaction
		// traceable. If not, we generate a new request ID.
		reqID := w.Header().Get("Request-ID")
		if reqID == "" {
			reqID = uuid.NewV4().String()
		}

		w.Header().Set("Request-ID", reqID)

		l.Print(applyLogFormat(handler.format, -1, w, r))

		res := internal.NewResponseWriter(w)
		h.ServeHTTP(res, r)

		latency := time.Since(start)
		l.Print(applyLogFormat(handler.format, latency, res, r))
	})
}

func applyLogFormat(format string, latency time.Duration, w http.ResponseWriter, r *http.Request) string {
	reqID := w.Header().Get("Request-ID")

	if strings.Index(format, "{remote_ip}") > -1 {
		format = strings.Replace(format, "{remote_ip}", strings.Split(r.RemoteAddr, ":")[0], -1)
	}

	if strings.Index(format, "{remote_user}") > -1 {
		user, _, _ := r.BasicAuth()
		if user == "" {
			user = r.URL.User.Username()
		}
		format = strings.Replace(format, "{remote_user}", user, -1)
	}

	if strings.Index(format, "{latency_human}") > -1 {
		l := "..."
		if latency > -1 {
			l = latency.String()
		}
		format = strings.Replace(format, "{latency_human}", l, -1)
	}

	if strings.Index(format, "{latency}") > -1 {
		l := "..."
		if latency > -1 {
			l = strconv.FormatInt(latency.Nanoseconds(), 10)
		}
		format = strings.Replace(format, "{latency}", l, -1)
	}

	if strings.Index(format, "{id}") > -1 {
		format = strings.Replace(format, "{id}", reqID, -1)
	}

	if strings.Index(format, "{method}") > -1 {
		format = strings.Replace(format, "{method}", r.Method, -1)
	}

	if strings.Index(format, "{url}") > -1 {
		format = strings.Replace(format, "{url}", r.URL.Path, -1)
	}

	if strings.Index(format, "{query}") > -1 {
		format = strings.Replace(format, "{query}", r.URL.RawQuery, -1)
	}

	if strings.Index(format, "{rxbytes}") > -1 {
		format = strings.Replace(format, "{rxbytes}", strconv.FormatInt(r.ContentLength, 10), -1)
	}

	if strings.Index(format, "{txbytes}") > -1 {
		size := "..."
		if v, ok := w.(internal.ResponseWriter); ok {
			size = strconv.Itoa(v.Size())
		}
		format = strings.Replace(format, "{txbytes}", size, -1)
	}

	if strings.Index(format, "{status}") > -1 {
		status := "..."
		if v, ok := w.(internal.ResponseWriter); ok {
			status = strconv.Itoa(v.Status())
		}
		format = strings.Replace(format, "{status}", status, -1)
	}

	if strings.Index(format, "{useragent}") > -1 {
		format = strings.Replace(format, "{useragent}", r.UserAgent(), -1)
	}

	if strings.Index(format, "{host}") > -1 {
		format = strings.Replace(format, "{host}", r.Host, -1)
	}

	if strings.Index(format, "{referer}") > -1 {
		format = strings.Replace(format, "{referer}", r.Referer(), -1)
	}

	return format
}
<|endoftext|>


<commit_before>package logger

import (
	"fmt"
	"log"
	"sort"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/honeybadger-io/honeybadger-go"
	"github.com/satori/go.uuid"
)

const (
	maxBatchByteSize = 1048576
	maxBatchLength   = 10000
	logEventOverhead = 26
)

type Logger interface {
	Log(t time.Time, s string)
}

type logBatch []*cloudwatchlogs.InputLogEvent

func (b logBatch) Len() int {
	return len(b)
}

func (b logBatch) Less(i, j int) bool {
	return *b[i].Timestamp < *b[j].Timestamp
}

func (b logBatch) Swap(i, j int) {
	b[i], b[j] = b[j], b[i]
}

type CloudWatchLogger struct {
	logGroupName  string
	logStreamName string
	sequenceToken *string
	retention     int
	logs          chan *cloudwatchlogs.InputLogEvent
	batch         logBatch
	batchByteSize int
	timeout       <-chan time.Time
	client        *cloudwatchlogs.CloudWatchLogs
}

func NewCloudWatchLogger(logGroupName string, retention int) (*CloudWatchLogger, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, fmt.Errorf("failed to create AWS session: %s", err)
	}

	client := cloudwatchlogs.New(sess)

	cwl := &CloudWatchLogger{
		logGroupName:  logGroupName,
		logStreamName: uuid.NewV4().String(),
		retention:     retention,
		logs:          make(chan *cloudwatchlogs.InputLogEvent, 100),
		client:        client,
	}
	go cwl.worker()
	return cwl, nil
}

func (cwl *CloudWatchLogger) Log(t time.Time, s string) {
	cwl.logs <- &cloudwatchlogs.InputLogEvent{
		Message:   aws.String(s),
		Timestamp: aws.Int64(t.UnixNano() / int64(time.Millisecond)),
	}
}

func (cwl *CloudWatchLogger) worker() {
	cwl.resetBatch()
	for {
		select {
		case logEvent := <-cwl.logs:
			cwl.addToBatch(logEvent)
		case <-cwl.timeout:
			cwl.flush()
		}
	}
}

func (cwl *CloudWatchLogger) addToBatch(logEvent *cloudwatchlogs.InputLogEvent) {
	logEventSize := len(*logEvent.Message) + logEventOverhead

	if logEventSize+cwl.batchByteSize > maxBatchByteSize || len(cwl.batch) == maxBatchLength {
		cwl.flush()
	}

	if cwl.timeout == nil {
		cwl.timeout = time.After(time.Second)
	}

	cwl.batch = append(cwl.batch, logEvent)
	cwl.batchByteSize += logEventSize
}

func (cwl *CloudWatchLogger) flush() {
	batch := cwl.batch
	cwl.resetBatch()
	sort.Sort(batch)
	if err := cwl.sendToCloudWatchLogs(batch); err != nil {
		if honeybadger.Config.APIKey == "" {
			honeybadger.Notify(err)
		}
		log.Println(err)
	}
}

func (cwl *CloudWatchLogger) resetBatch() {
	cwl.batch = logBatch{}
	cwl.batchByteSize = 0
	cwl.timeout = nil
}

func (cwl *CloudWatchLogger) sendToCloudWatchLogs(batch logBatch) error {
	s := time.Now()
	params := &cloudwatchlogs.PutLogEventsInput{
		LogEvents:     batch,
		LogGroupName:  aws.String(cwl.logGroupName),
		LogStreamName: aws.String(cwl.logStreamName),
		SequenceToken: cwl.sequenceToken,
	}
	resp, err := cwl.client.PutLogEvents(params)

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				if err = cwl.createLogStream(); err != nil {
					return err
				}
				return cwl.sendToCloudWatchLogs(batch)
			}
		}
		return fmt.Errorf("PutLogEvents failed: %s", err)
	}
	log.Printf("wrote %d log events (%d bytes) in %s\n", len(cwl.batch), cwl.batchByteSize, time.Since(s))

	cwl.sequenceToken = resp.NextSequenceToken
	return nil
}

func (cwl *CloudWatchLogger) createLogStream() error {
	params := &cloudwatchlogs.CreateLogStreamInput{
		LogGroupName:  aws.String(cwl.logGroupName),
		LogStreamName: aws.String(cwl.logStreamName),
	}
	if _, err := cwl.client.CreateLogStream(params); err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				if err = cwl.createLogGroup(); err != nil {
					return err
				}
				return cwl.createLogStream()
			}
		}
		return fmt.Errorf("CreateLogStream failed: %s", err)
	}
	return nil
}

func (cwl *CloudWatchLogger) createLogGroup() error {
	params := &cloudwatchlogs.CreateLogGroupInput{
		LogGroupName: aws.String(cwl.logGroupName),
	}
	if _, err := cwl.client.CreateLogGroup(params); err != nil {
		return fmt.Errorf("CreateLogGroup failed: %s", err)
	}
	return cwl.putRetentionPolicy()
}

func (cwl *CloudWatchLogger) putRetentionPolicy() error {
	if cwl.retention == 0 {
		return nil
	}
	params := &cloudwatchlogs.PutRetentionPolicyInput{
		LogGroupName:    aws.String(cwl.logGroupName),
		RetentionInDays: aws.Int64(int64(cwl.retention)),
	}
	_, err := cwl.client.PutRetentionPolicy(params)
	if err != nil {
		return fmt.Errorf("PutRetentionPolicy failed: %s", err)
	}
	return nil
}
<commit_msg>Fixed the events count<commit_after>package logger

import (
	"fmt"
	"log"
	"sort"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/honeybadger-io/honeybadger-go"
	"github.com/satori/go.uuid"
)

const (
	maxBatchByteSize = 1048576
	maxBatchLength   = 10000
	logEventOverhead = 26
)

type Logger interface {
	Log(t time.Time, s string)
}

type logBatch []*cloudwatchlogs.InputLogEvent

func (b logBatch) Len() int {
	return len(b)
}

func (b logBatch) Less(i, j int) bool {
	return *b[i].Timestamp < *b[j].Timestamp
}

func (b logBatch) Swap(i, j int) {
	b[i], b[j] = b[j], b[i]
}

type CloudWatchLogger struct {
	logGroupName  string
	logStreamName string
	sequenceToken *string
	retention     int
	logs          chan *cloudwatchlogs.InputLogEvent
	batch         logBatch
	batchByteSize int
	timeout       <-chan time.Time
	client        *cloudwatchlogs.CloudWatchLogs
}

func NewCloudWatchLogger(logGroupName string, retention int) (*CloudWatchLogger, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, fmt.Errorf("failed to create AWS session: %s", err)
	}

	client := cloudwatchlogs.New(sess)

	cwl := &CloudWatchLogger{
		logGroupName:  logGroupName,
		logStreamName: uuid.NewV4().String(),
		retention:     retention,
		logs:          make(chan *cloudwatchlogs.InputLogEvent, 100),
		client:        client,
	}
	go cwl.worker()
	return cwl, nil
}

func (cwl *CloudWatchLogger) Log(t time.Time, s string) {
	cwl.logs <- &cloudwatchlogs.InputLogEvent{
		Message:   aws.String(s),
		Timestamp: aws.Int64(t.UnixNano() / int64(time.Millisecond)),
	}
}

func (cwl *CloudWatchLogger) worker() {
	cwl.resetBatch()
	for {
		select {
		case logEvent := <-cwl.logs:
			cwl.addToBatch(logEvent)
		case <-cwl.timeout:
			cwl.flush()
		}
	}
}

func (cwl *CloudWatchLogger) addToBatch(logEvent *cloudwatchlogs.InputLogEvent) {
	logEventSize := len(*logEvent.Message) + logEventOverhead

	if logEventSize+cwl.batchByteSize > maxBatchByteSize || len(cwl.batch) == maxBatchLength {
		cwl.flush()
	}

	if cwl.timeout == nil {
		cwl.timeout = time.After(time.Second)
	}

	cwl.batch = append(cwl.batch, logEvent)
	cwl.batchByteSize += logEventSize
}

func (cwl *CloudWatchLogger) flush() {
	batch := cwl.batch
	cwl.resetBatch()
	sort.Sort(batch)
	if err := cwl.sendToCloudWatchLogs(batch); err != nil {
		if honeybadger.Config.APIKey == "" {
			honeybadger.Notify(err)
		}
		log.Println(err)
	}
}

func (cwl *CloudWatchLogger) resetBatch() {
	cwl.batch = logBatch{}
	cwl.batchByteSize = 0
	cwl.timeout = nil
}

func (cwl *CloudWatchLogger) sendToCloudWatchLogs(batch logBatch) error {
	s := time.Now()
	params := &cloudwatchlogs.PutLogEventsInput{
		LogEvents:     batch,
		LogGroupName:  aws.String(cwl.logGroupName),
		LogStreamName: aws.String(cwl.logStreamName),
		SequenceToken: cwl.sequenceToken,
	}
	resp, err := cwl.client.PutLogEvents(params)

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				if err = cwl.createLogStream(); err != nil {
					return err
				}
				return cwl.sendToCloudWatchLogs(batch)
			}
		}
		return fmt.Errorf("PutLogEvents failed: %s", err)
	}
	log.Printf("wrote %d log events (%d bytes) in %s\n", len(batch), cwl.batchByteSize, time.Since(s))

	cwl.sequenceToken = resp.NextSequenceToken
	return nil
}

func (cwl *CloudWatchLogger) createLogStream() error {
	params := &cloudwatchlogs.CreateLogStreamInput{
		LogGroupName:  aws.String(cwl.logGroupName),
		LogStreamName: aws.String(cwl.logStreamName),
	}
	if _, err := cwl.client.CreateLogStream(params); err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				if err = cwl.createLogGroup(); err != nil {
					return err
				}
				return cwl.createLogStream()
			}
		}
		return fmt.Errorf("CreateLogStream failed: %s", err)
	}
	return nil
}

func (cwl *CloudWatchLogger) createLogGroup() error {
	params := &cloudwatchlogs.CreateLogGroupInput{
		LogGroupName: aws.String(cwl.logGroupName),
	}
	if _, err := cwl.client.CreateLogGroup(params); err != nil {
		return fmt.Errorf("CreateLogGroup failed: %s", err)
	}
	return cwl.putRetentionPolicy()
}

func (cwl *CloudWatchLogger) putRetentionPolicy() error {
	if cwl.retention == 0 {
		return nil
	}
	params := &cloudwatchlogs.PutRetentionPolicyInput{
		LogGroupName:    aws.String(cwl.logGroupName),
		RetentionInDays: aws.Int64(int64(cwl.retention)),
	}
	_, err := cwl.client.PutRetentionPolicy(params)
	if err != nil {
		return fmt.Errorf("PutRetentionPolicy failed: %s", err)
	}
	return nil
}
<|endoftext|>