<commit_before>package http

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	fm "github.com/hacdias/filemanager"
	"github.com/hacdias/fileutils"
)

// sanitizeURL sanitizes the URL to prevent path transversal
// using fileutils.SlashClean and adds the trailing slash bar.
func sanitizeURL(url string) string {
	path := fileutils.SlashClean(url)
	if strings.HasSuffix(url, "/") && path != "/" {
		return path + "/"
	}
	return path
}

func resourceHandler(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	r.URL.Path = sanitizeURL(r.URL.Path)

	switch r.Method {
	case http.MethodGet:
		return resourceGetHandler(c, w, r)
	case http.MethodDelete:
		return resourceDeleteHandler(c, w, r)
	case http.MethodPut:
		// Before save command handler.
		path := filepath.Join(c.User.Scope, r.URL.Path)
		if err := c.Runner("before_save", path); err != nil {
			return http.StatusInternalServerError, err
		}

		code, err := resourcePostPutHandler(c, w, r)
		if code != http.StatusOK {
			return code, err
		}

		// After save command handler.
		if err := c.Runner("after_save", path); err != nil {
			return http.StatusInternalServerError, err
		}

		return code, err
	case http.MethodPatch:
		return resourcePatchHandler(c, w, r)
	case http.MethodPost:
		return resourcePostPutHandler(c, w, r)
	}

	return http.StatusNotImplemented, nil
}

func resourceGetHandler(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	// Gets the information of the directory/file.
	f, err := fm.GetInfo(r.URL, c.FileManager, c.User)
	if err != nil {
		return ErrorToHTTP(err, false), err
	}

	// If it's a dir and the path doesn't end with a trailing slash,
	// add a trailing slash to the path.
	if f.IsDir && !strings.HasSuffix(r.URL.Path, "/") {
		r.URL.Path = r.URL.Path + "/"
	}

	// If it is a dir, go and serve the listing.
	if f.IsDir {
		c.File = f
		return listingHandler(c, w, r)
	}

	// Tries to get the file type.
	if err = f.GetFileType(true); err != nil {
		return ErrorToHTTP(err, true), err
	}

	// Serve a preview if the file can't be edited or the
	// user has no permission to edit this file. Otherwise,
	// just serve the editor.
	if !f.CanBeEdited() || !c.User.AllowEdit {
		f.Kind = "preview"
		return renderJSON(w, f)
	}

	f.Kind = "editor"

	// Tries to get the editor data.
	if err = f.GetEditor(); err != nil {
		return http.StatusInternalServerError, err
	}

	return renderJSON(w, f)
}

func listingHandler(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	f := c.File
	f.Kind = "listing"

	// Tries to get the listing data.
	if err := f.GetListing(c.User, r); err != nil {
		return ErrorToHTTP(err, true), err
	}

	listing := f.Listing

	// Defines the cookie scope.
	cookieScope := c.RootURL()
	if cookieScope == "" {
		cookieScope = "/"
	}

	// Copy the query values into the Listing struct
	if sort, order, err := handleSortOrder(w, r, cookieScope); err == nil {
		listing.Sort = sort
		listing.Order = order
	} else {
		return http.StatusBadRequest, err
	}

	listing.ApplySort()
	listing.Display = displayMode(w, r, cookieScope)

	return renderJSON(w, f)
}

func resourceDeleteHandler(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	// Prevent the removal of the root directory.
	if r.URL.Path == "/" || !c.User.AllowEdit {
		return http.StatusForbidden, nil
	}

	// Remove the file or folder.
	err := c.User.FileSystem.RemoveAll(r.URL.Path)
	if err != nil {
		return ErrorToHTTP(err, true), err
	}

	return http.StatusOK, nil
}

func resourcePostPutHandler(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	if !c.User.AllowNew && r.Method == http.MethodPost {
		return http.StatusForbidden, nil
	}

	if !c.User.AllowEdit && r.Method == http.MethodPut {
		return http.StatusForbidden, nil
	}

	// Discard any invalid upload before returning to avoid connection
	// reset fm.Error.
	defer func() {
		io.Copy(ioutil.Discard, r.Body)
	}()

	// Checks if the current request is for a directory and not a file.
	if strings.HasSuffix(r.URL.Path, "/") {
		// If the method is PUT, we return 405 Method not Allowed, because
		// POST should be used instead.
		if r.Method == http.MethodPut {
			return http.StatusMethodNotAllowed, nil
		}

		// Otherwise we try to create the directory.
		err := c.User.FileSystem.Mkdir(r.URL.Path, 0776)
		return ErrorToHTTP(err, false), err
	}

	// If using POST method, we are trying to create a new file so it is not
	// desirable to ovfm.Erride an already existent file. Thus, we check
	// if the file already exists. If so, we just return a 409 Conflict.
	if r.Method == http.MethodPost && r.Header.Get("Action") != "ovfm.Erride" {
		if _, err := c.User.FileSystem.Stat(r.URL.Path); err == nil {
			return http.StatusConflict, errors.New("There is already a file on that path")
		}
	}

	// Create/Open the file.
	f, err := c.User.FileSystem.OpenFile(r.URL.Path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0776)
	if err != nil {
		return ErrorToHTTP(err, false), err
	}
	defer f.Close()

	// Copies the new content for the file.
	_, err = io.Copy(f, r.Body)
	if err != nil {
		return ErrorToHTTP(err, false), err
	}

	// Gets the info about the file.
	fi, err := f.Stat()
	if err != nil {
		return ErrorToHTTP(err, false), err
	}

	// Check if this instance has a Static Generator and handles publishing
	// or scheduling if it's the case.
	if c.StaticGen != nil {
		code, err := resourcePublishSchedule(c, w, r)
		if code != 0 {
			return code, err
		}
	}

	// Writes the ETag Header.
	etag := fmt.Sprintf(`"%x%x"`, fi.ModTime().UnixNano(), fi.Size())
	w.Header().Set("ETag", etag)
	return http.StatusOK, nil
}

func resourcePublishSchedule(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	publish := r.Header.Get("Publish")
	schedule := r.Header.Get("Schedule")

	if publish != "true" && schedule == "" {
		return 0, nil
	}

	if !c.User.AllowPublish {
		return http.StatusForbidden, nil
	}

	if publish == "true" {
		return resourcePublish(c, w, r)
	}

	t, err := time.Parse("2006-01-02T15:04", schedule)
	if err != nil {
		return http.StatusInternalServerError, err
	}

	c.Cron.AddFunc(t.Format("05 04 15 02 01 *"), func() {
		_, err := resourcePublish(c, w, r)
		if err != nil {
			log.Print(err)
		}
	})

	return http.StatusOK, nil
}

func resourcePublish(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	path := filepath.Join(c.User.Scope, r.URL.Path)

	// Before save command handler.
	if err := c.Runner("before_publish", path); err != nil {
		return http.StatusInternalServerError, err
	}

	code, err := c.StaticGen.Publish(c, w, r)
	if err != nil {
		return code, err
	}

	// Executed the before publish command.
	if err := c.Runner("before_publish", path); err != nil {
		return http.StatusInternalServerError, err
	}

	return code, nil
}

// resourcePatchHandler is the entry point for resource handler.
func resourcePatchHandler(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	if !c.User.AllowEdit {
		return http.StatusForbidden, nil
	}

	dst := r.Header.Get("Destination")
	action := r.Header.Get("Action")
	dst, err := url.QueryUnescape(dst)
	if err != nil {
		return ErrorToHTTP(err, true), err
	}

	src := r.URL.Path

	if dst == "/" || src == "/" {
		return http.StatusForbidden, nil
	}

	if action == "copy" {
		err = c.User.FileSystem.Copy(src, dst)
	} else {
		err = c.User.FileSystem.Rename(src, dst)
	}

	return ErrorToHTTP(err, true), err
}

// displayMode obtains the display mode from the Cookie.
func displayMode(w http.ResponseWriter, r *http.Request, scope string) string {
	var displayMode string

	// Checks the cookie.
	if displayCookie, err := r.Cookie("display"); err == nil {
		displayMode = displayCookie.Value
	}

	// If it's invalid, set it to mosaic, which is the default.
	if displayMode == "" || (displayMode != "mosaic" && displayMode != "list") {
		displayMode = "mosaic"
	}

	// Set the cookie.
	http.SetCookie(w, &http.Cookie{
		Name:   "display",
		Value:  displayMode,
		MaxAge: 31536000,
		Path:   scope,
		Secure: r.TLS != nil,
	})

	return displayMode
}

// handleSortOrder gets and stores for a Listing the 'sort' and 'order',
// and reads 'limit' if given. The latter is 0 if not given. Sets cookies.
func handleSortOrder(w http.ResponseWriter, r *http.Request, scope string) (sort string, order string, err error) {
	sort = r.URL.Query().Get("sort")
	order = r.URL.Query().Get("order")

	// If the query 'sort' or 'order' is empty, use defaults or any values
	// previously saved in Cookies.
	switch sort {
	case "":
		sort = "name"
		if sortCookie, sortErr := r.Cookie("sort"); sortErr == nil {
			sort = sortCookie.Value
		}
	case "name", "size":
		http.SetCookie(w, &http.Cookie{
			Name:   "sort",
			Value:  sort,
			MaxAge: 31536000,
			Path:   scope,
			Secure: r.TLS != nil,
		})
	}

	switch order {
	case "":
		order = "asc"
		if orderCookie, orderErr := r.Cookie("order"); orderErr == nil {
			order = orderCookie.Value
		}
	case "asc", "desc":
		http.SetCookie(w, &http.Cookie{
			Name:   "order",
			Value:  order,
			MaxAge: 31536000,
			Path:   scope,
			Secure: r.TLS != nil,
		})
	}

	return
}
<commit_msg>Fix override.<commit_after>package http

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	fm "github.com/hacdias/filemanager"
	"github.com/hacdias/fileutils"
)

// sanitizeURL sanitizes the URL to prevent path transversal
// using fileutils.SlashClean and adds the trailing slash bar.
func sanitizeURL(url string) string {
	path := fileutils.SlashClean(url)
	if strings.HasSuffix(url, "/") && path != "/" {
		return path + "/"
	}
	return path
}

func resourceHandler(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	r.URL.Path = sanitizeURL(r.URL.Path)

	switch r.Method {
	case http.MethodGet:
		return resourceGetHandler(c, w, r)
	case http.MethodDelete:
		return resourceDeleteHandler(c, w, r)
	case http.MethodPut:
		// Before save command handler.
		path := filepath.Join(c.User.Scope, r.URL.Path)
		if err := c.Runner("before_save", path); err != nil {
			return http.StatusInternalServerError, err
		}

		code, err := resourcePostPutHandler(c, w, r)
		if code != http.StatusOK {
			return code, err
		}

		// After save command handler.
		if err := c.Runner("after_save", path); err != nil {
			return http.StatusInternalServerError, err
		}

		return code, err
	case http.MethodPatch:
		return resourcePatchHandler(c, w, r)
	case http.MethodPost:
		return resourcePostPutHandler(c, w, r)
	}

	return http.StatusNotImplemented, nil
}

func resourceGetHandler(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	// Gets the information of the directory/file.
	f, err := fm.GetInfo(r.URL, c.FileManager, c.User)
	if err != nil {
		return ErrorToHTTP(err, false), err
	}

	// If it's a dir and the path doesn't end with a trailing slash,
	// add a trailing slash to the path.
	if f.IsDir && !strings.HasSuffix(r.URL.Path, "/") {
		r.URL.Path = r.URL.Path + "/"
	}

	// If it is a dir, go and serve the listing.
	if f.IsDir {
		c.File = f
		return listingHandler(c, w, r)
	}

	// Tries to get the file type.
	if err = f.GetFileType(true); err != nil {
		return ErrorToHTTP(err, true), err
	}

	// Serve a preview if the file can't be edited or the
	// user has no permission to edit this file. Otherwise,
	// just serve the editor.
	if !f.CanBeEdited() || !c.User.AllowEdit {
		f.Kind = "preview"
		return renderJSON(w, f)
	}

	f.Kind = "editor"

	// Tries to get the editor data.
	if err = f.GetEditor(); err != nil {
		return http.StatusInternalServerError, err
	}

	return renderJSON(w, f)
}

func listingHandler(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	f := c.File
	f.Kind = "listing"

	// Tries to get the listing data.
	if err := f.GetListing(c.User, r); err != nil {
		return ErrorToHTTP(err, true), err
	}

	listing := f.Listing

	// Defines the cookie scope.
	cookieScope := c.RootURL()
	if cookieScope == "" {
		cookieScope = "/"
	}

	// Copy the query values into the Listing struct
	if sort, order, err := handleSortOrder(w, r, cookieScope); err == nil {
		listing.Sort = sort
		listing.Order = order
	} else {
		return http.StatusBadRequest, err
	}

	listing.ApplySort()
	listing.Display = displayMode(w, r, cookieScope)

	return renderJSON(w, f)
}

func resourceDeleteHandler(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	// Prevent the removal of the root directory.
	if r.URL.Path == "/" || !c.User.AllowEdit {
		return http.StatusForbidden, nil
	}

	// Remove the file or folder.
	err := c.User.FileSystem.RemoveAll(r.URL.Path)
	if err != nil {
		return ErrorToHTTP(err, true), err
	}

	return http.StatusOK, nil
}

func resourcePostPutHandler(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	if !c.User.AllowNew && r.Method == http.MethodPost {
		return http.StatusForbidden, nil
	}

	if !c.User.AllowEdit && r.Method == http.MethodPut {
		return http.StatusForbidden, nil
	}

	// Discard any invalid upload before returning to avoid connection
	// reset error.
	defer func() {
		io.Copy(ioutil.Discard, r.Body)
	}()

	// Checks if the current request is for a directory and not a file.
	if strings.HasSuffix(r.URL.Path, "/") {
		// If the method is PUT, we return 405 Method not Allowed, because
		// POST should be used instead.
		if r.Method == http.MethodPut {
			return http.StatusMethodNotAllowed, nil
		}

		// Otherwise we try to create the directory.
		err := c.User.FileSystem.Mkdir(r.URL.Path, 0776)
		return ErrorToHTTP(err, false), err
	}

	// If using POST method, we are trying to create a new file so it is not
	// desirable to override an already existent file. Thus, we check
	// if the file already exists. If so, we just return a 409 Conflict.
	if r.Method == http.MethodPost && r.Header.Get("Action") != "override" {
		if _, err := c.User.FileSystem.Stat(r.URL.Path); err == nil {
			return http.StatusConflict, errors.New("There is already a file on that path")
		}
	}

	// Create/Open the file.
	f, err := c.User.FileSystem.OpenFile(r.URL.Path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0776)
	if err != nil {
		return ErrorToHTTP(err, false), err
	}
	defer f.Close()

	// Copies the new content for the file.
	_, err = io.Copy(f, r.Body)
	if err != nil {
		return ErrorToHTTP(err, false), err
	}

	// Gets the info about the file.
	fi, err := f.Stat()
	if err != nil {
		return ErrorToHTTP(err, false), err
	}

	// Check if this instance has a Static Generator and handles publishing
	// or scheduling if it's the case.
	if c.StaticGen != nil {
		code, err := resourcePublishSchedule(c, w, r)
		if code != 0 {
			return code, err
		}
	}

	// Writes the ETag Header.
	etag := fmt.Sprintf(`"%x%x"`, fi.ModTime().UnixNano(), fi.Size())
	w.Header().Set("ETag", etag)
	return http.StatusOK, nil
}

func resourcePublishSchedule(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	publish := r.Header.Get("Publish")
	schedule := r.Header.Get("Schedule")

	if publish != "true" && schedule == "" {
		return 0, nil
	}

	if !c.User.AllowPublish {
		return http.StatusForbidden, nil
	}

	if publish == "true" {
		return resourcePublish(c, w, r)
	}

	t, err := time.Parse("2006-01-02T15:04", schedule)
	if err != nil {
		return http.StatusInternalServerError, err
	}

	c.Cron.AddFunc(t.Format("05 04 15 02 01 *"), func() {
		_, err := resourcePublish(c, w, r)
		if err != nil {
			log.Print(err)
		}
	})

	return http.StatusOK, nil
}

func resourcePublish(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	path := filepath.Join(c.User.Scope, r.URL.Path)

	// Before save command handler.
	if err := c.Runner("before_publish", path); err != nil {
		return http.StatusInternalServerError, err
	}

	code, err := c.StaticGen.Publish(c, w, r)
	if err != nil {
		return code, err
	}

	// Executed the before publish command.
	if err := c.Runner("before_publish", path); err != nil {
		return http.StatusInternalServerError, err
	}

	return code, nil
}

// resourcePatchHandler is the entry point for resource handler.
func resourcePatchHandler(c *fm.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	if !c.User.AllowEdit {
		return http.StatusForbidden, nil
	}

	dst := r.Header.Get("Destination")
	action := r.Header.Get("Action")
	dst, err := url.QueryUnescape(dst)
	if err != nil {
		return ErrorToHTTP(err, true), err
	}

	src := r.URL.Path

	if dst == "/" || src == "/" {
		return http.StatusForbidden, nil
	}

	if action == "copy" {
		err = c.User.FileSystem.Copy(src, dst)
	} else {
		err = c.User.FileSystem.Rename(src, dst)
	}

	return ErrorToHTTP(err, true), err
}

// displayMode obtains the display mode from the Cookie.
func displayMode(w http.ResponseWriter, r *http.Request, scope string) string {
	var displayMode string

	// Checks the cookie.
	if displayCookie, err := r.Cookie("display"); err == nil {
		displayMode = displayCookie.Value
	}

	// If it's invalid, set it to mosaic, which is the default.
	if displayMode == "" || (displayMode != "mosaic" && displayMode != "list") {
		displayMode = "mosaic"
	}

	// Set the cookie.
	http.SetCookie(w, &http.Cookie{
		Name:   "display",
		Value:  displayMode,
		MaxAge: 31536000,
		Path:   scope,
		Secure: r.TLS != nil,
	})

	return displayMode
}

// handleSortOrder gets and stores for a Listing the 'sort' and 'order',
// and reads 'limit' if given. The latter is 0 if not given. Sets cookies.
func handleSortOrder(w http.ResponseWriter, r *http.Request, scope string) (sort string, order string, err error) {
	sort = r.URL.Query().Get("sort")
	order = r.URL.Query().Get("order")

	// If the query 'sort' or 'order' is empty, use defaults or any values
	// previously saved in Cookies.
	switch sort {
	case "":
		sort = "name"
		if sortCookie, sortErr := r.Cookie("sort"); sortErr == nil {
			sort = sortCookie.Value
		}
	case "name", "size":
		http.SetCookie(w, &http.Cookie{
			Name:   "sort",
			Value:  sort,
			MaxAge: 31536000,
			Path:   scope,
			Secure: r.TLS != nil,
		})
	}

	switch order {
	case "":
		order = "asc"
		if orderCookie, orderErr := r.Cookie("order"); orderErr == nil {
			order = orderCookie.Value
		}
	case "asc", "desc":
		http.SetCookie(w, &http.Cookie{
			Name:   "order",
			Value:  order,
			MaxAge: 31536000,
			Path:   scope,
			Secure: r.TLS != nil,
		})
	}

	return
}
<|endoftext|>


<commit_before>package main

// Chat is a private or group chat.
type Chat struct {
	ID    uint64
	Users []User
}

// User is a mobile user.
type User struct {
	ID      uint32
	Devices []Device
	Name    string
	Picture []byte
}

// Device is an NBusy installed device.
type Device interface {
	Send()
}

// Android device.
type Android struct {
	GCMRegID string
	Phone    uint64
}

// iOS device.
type iOS struct {
	APNSDeviceToken string
	Phone           uint64
}

var chats = make(map[string]Chat)

// user -> id (user or chat id) -> message
// delivery status -> user
// read status -> user
<commit_msg>add note<commit_after>package main

// Chat is a private or group chat.
type Chat struct {
	ID    uint64
	Users []User
}

// User is a mobile user.
type User struct {
	ID      uint32
	Devices []Device
	Name    string
	Picture []byte
}

// Device is an NBusy installed device.
type Device interface {
	Send(data map[string]string) error // note: not adding SendMessage/SendNotification/etc. like fine grained methods to keep this library more low level
}

// Android device.
type Android struct {
	GCMRegID    string
	PhoneNumber uint64
}

// iOS device.
type iOS struct {
	APNSDeviceToken string
	PhoneNumber     uint64
}

var chats = make(map[string]Chat)

// user -> id (user or chat id) -> message
// delivery status -> user
// read status -> user
<|endoftext|>


<commit_before>package slack

import (
	"encoding/json"
	"errors"
	"net/url"
	"strings"
)

const (
	DEFAULT_MESSAGE_USERNAME         = ""
	DEFAULT_MESSAGE_THREAD_TIMESTAMP = ""
	DEFAULT_MESSAGE_ASUSER           = false
	DEFAULT_MESSAGE_PARSE            = ""
	DEFAULT_MESSAGE_LINK_NAMES       = 0
	DEFAULT_MESSAGE_UNFURL_LINKS     = false
	DEFAULT_MESSAGE_UNFURL_MEDIA     = true
	DEFAULT_MESSAGE_ICON_URL         = ""
	DEFAULT_MESSAGE_ICON_EMOJI       = ""
	DEFAULT_MESSAGE_MARKDOWN         = true
	DEFAULT_MESSAGE_ESCAPE_TEXT      = true
)

type chatResponseFull struct {
	Channel   string `json:"channel"`
	Timestamp string `json:"ts"`
	Text      string `json:"text"`
	SlackResponse
}

// PostMessageParameters contains all the parameters necessary (including the optional ones) for a PostMessage() request
type PostMessageParameters struct {
	Text            string       `json:"text"`
	Username        string       `json:"user_name"`
	AsUser          bool         `json:"as_user"`
	Parse           string       `json:"parse"`
	ThreadTimestamp string       `json:"thread_ts"`
	LinkNames       int          `json:"link_names"`
	Attachments     []Attachment `json:"attachments"`
	UnfurlLinks     bool         `json:"unfurl_links"`
	UnfurlMedia     bool         `json:"unfurl_media"`
	IconURL         string       `json:"icon_url"`
	IconEmoji       string       `json:"icon_emoji"`
	Markdown        bool         `json:"mrkdwn,omitempty"`
	EscapeText      bool         `json:"escape_text"`
}

// NewPostMessageParameters provides an instance of PostMessageParameters with all the sane default values set
func NewPostMessageParameters() PostMessageParameters {
	return PostMessageParameters{
		Username:    DEFAULT_MESSAGE_USERNAME,
		AsUser:      DEFAULT_MESSAGE_ASUSER,
		Parse:       DEFAULT_MESSAGE_PARSE,
		LinkNames:   DEFAULT_MESSAGE_LINK_NAMES,
		Attachments: nil,
		UnfurlLinks: DEFAULT_MESSAGE_UNFURL_LINKS,
		UnfurlMedia: DEFAULT_MESSAGE_UNFURL_MEDIA,
		IconURL:     DEFAULT_MESSAGE_ICON_URL,
		IconEmoji:   DEFAULT_MESSAGE_ICON_EMOJI,
		Markdown:    DEFAULT_MESSAGE_MARKDOWN,
		EscapeText:  DEFAULT_MESSAGE_ESCAPE_TEXT,
	}
}

// DeleteMessage deletes a message in a channel
func (api *Client) DeleteMessage(channel, messageTimestamp string) (string, string, error) {
	respChannel, respTimestamp, _, err := api.SendMessage(channel, MsgOptionDelete(messageTimestamp))
	return respChannel, respTimestamp, err
}

// PostMessage sends a message to a channel.
// Message is escaped by default according to https://api.slack.com/docs/formatting
// Use http://davestevens.github.io/slack-message-builder/ to help crafting your message.
func (api *Client) PostMessage(channel, text string, params PostMessageParameters) (string, string, error) {
	respChannel, respTimestamp, _, err := api.SendMessage(
		channel,
		MsgOptionText(text, params.EscapeText),
		MsgOptionAttachments(params.Attachments...),
		MsgOptionPostMessageParameters(params),
	)
	return respChannel, respTimestamp, err
}

// UpdateMessage updates a message in a channel
func (api *Client) UpdateMessage(channel, timestamp, text string) (string, string, string, error) {
	return api.SendMessage(channel, MsgOptionUpdate(timestamp), MsgOptionText(text, true))
}

// SendMessage more flexible method for configuring messages.
func (api *Client) SendMessage(channel string, options ...MsgOption) (string, string, string, error) {
	channel, values, err := ApplyMsgOptions(api.config.token, channel, options...)
	if err != nil {
		return "", "", "", err
	}

	response, err := chatRequest(channel, values, api.debug)
	if err != nil {
		return "", "", "", err
	}

	return response.Channel, response.Timestamp, response.Text, nil
}

// ApplyMsgOptions utility function for debugging/testing chat requests.
func ApplyMsgOptions(token, channel string, options ...MsgOption) (string, url.Values, error) {
	config := sendConfig{
		mode: chatPostMessage,
		values: url.Values{
			"token":   {token},
			"channel": {channel},
		},
	}

	for _, opt := range options {
		if err := opt(&config); err != nil {
			return string(config.mode), config.values, err
		}
	}

	return string(config.mode), config.values, nil
}

func escapeMessage(message string) string {
	replacer := strings.NewReplacer("&", "&", "<", "<", ">", ">")
	return replacer.Replace(message)
}

func chatRequest(path string, values url.Values, debug bool) (*chatResponseFull, error) {
	response := &chatResponseFull{}
	err := post(path, values, response, debug)
	if err != nil {
		return nil, err
	}
	if !response.Ok {
		return nil, errors.New(response.Error)
	}
	return response, nil
}

type sendMode string

const (
	chatUpdate      sendMode = "chat.update"
	chatPostMessage sendMode = "chat.postMessage"
	chatDelete      sendMode = "chat.delete"
)

type sendConfig struct {
	mode   sendMode
	values url.Values
}

// MsgOption option provided when sending a message.
type MsgOption func(*sendConfig) error

// MsgOptionPost posts a messages, this is the default.
func MsgOptionPost() MsgOption {
	return func(config *sendConfig) error {
		config.mode = chatPostMessage
		config.values.Del("ts")
		return nil
	}
}

// MsgOptionUpdate updates a message based on the timestamp.
func MsgOptionUpdate(timestamp string) MsgOption {
	return func(config *sendConfig) error {
		config.mode = chatUpdate
		config.values.Add("ts", timestamp)
		return nil
	}
}

// MsgOptionDelete deletes a message based on the timestamp.
func MsgOptionDelete(timestamp string) MsgOption {
	return func(config *sendConfig) error {
		config.mode = chatDelete
		config.values.Add("ts", timestamp)
		return nil
	}
}

// MsgOptionAsUser whether or not to send the message as the user.
func MsgOptionAsUser(b bool) MsgOption {
	return func(config *sendConfig) error {
		if b != DEFAULT_MESSAGE_ASUSER {
			config.values.Set("as_user", "true")
		}
		return nil
	}
}

// MsgOptionText provide the text for the message, optionally escape the provided
// text.
func MsgOptionText(text string, escape bool) MsgOption {
	return func(config *sendConfig) error {
		if escape {
			text = escapeMessage(text)
		}
		config.values.Add("text", text)
		return nil
	}
}

// MsgOptionAttachments provide attachments for the message.
func MsgOptionAttachments(attachments ...Attachment) MsgOption {
	return func(config *sendConfig) error {
		if attachments == nil {
			return nil
		}

		attachments, err := json.Marshal(attachments)
		if err == nil {
			config.values.Set("attachments", string(attachments))
		}
		return err
	}
}

// MsgOptionEnableLinkUnfurl enables link unfurling
func MsgOptionEnableLinkUnfurl() MsgOption {
	return func(config *sendConfig) error {
		config.values.Set("unfurl_links", "true")
		return nil
	}
}

// MsgOptionDisableMediaUnfurl disables media unfurling.
func MsgOptionDisableMediaUnfurl() MsgOption {
	return func(config *sendConfig) error {
		config.values.Set("unfurl_media", "false")
		return nil
	}
}

// MsgOptionDisableMarkdown disables markdown.
func MsgOptionDisableMarkdown() MsgOption {
	return func(config *sendConfig) error {
		config.values.Set("mrkdwn", "false")
		return nil
	}
}

// MsgOptionPostMessageParameters maintain backwards compatibility.
func MsgOptionPostMessageParameters(params PostMessageParameters) MsgOption {
	return func(config *sendConfig) error {
		if params.Username != DEFAULT_MESSAGE_USERNAME {
			config.values.Set("username", string(params.Username))
		}

		// never generates an error.
		_ = MsgOptionAsUser(params.AsUser)

		if params.Parse != DEFAULT_MESSAGE_PARSE {
			config.values.Set("parse", string(params.Parse))
		}
		if params.LinkNames != DEFAULT_MESSAGE_LINK_NAMES {
			config.values.Set("link_names", "1")
		}

		if params.UnfurlLinks != DEFAULT_MESSAGE_UNFURL_LINKS {
			config.values.Set("unfurl_links", "true")
		}

		// I want to send a message with explicit `as_user` `true` and `unfurl_links` `false` in request.
		// Because setting `as_user` to `true` will change the default value for `unfurl_links` to `true` on Slack API side.
		if params.AsUser != DEFAULT_MESSAGE_ASUSER && params.UnfurlLinks == DEFAULT_MESSAGE_UNFURL_LINKS {
			config.values.Set("unfurl_links", "false")
		}
		if params.UnfurlMedia != DEFAULT_MESSAGE_UNFURL_MEDIA {
			config.values.Set("unfurl_media", "false")
		}
		if params.IconURL != DEFAULT_MESSAGE_ICON_URL {
			config.values.Set("icon_url", params.IconURL)
		}
		if params.IconEmoji != DEFAULT_MESSAGE_ICON_EMOJI {
			config.values.Set("icon_emoji", params.IconEmoji)
		}
		if params.Markdown != DEFAULT_MESSAGE_MARKDOWN {
			config.values.Set("mrkdwn", "false")
		}

		if params.ThreadTimestamp != DEFAULT_MESSAGE_THREAD_TIMESTAMP {
			config.values.Set("thread_ts", params.ThreadTimestamp)
		}

		return nil
	}
}
<commit_msg>correctly apply the AsUser parameter<commit_after>package slack

import (
	"encoding/json"
	"errors"
	"net/url"
	"strings"
)

const (
	DEFAULT_MESSAGE_USERNAME         = ""
	DEFAULT_MESSAGE_THREAD_TIMESTAMP = ""
	DEFAULT_MESSAGE_ASUSER           = false
	DEFAULT_MESSAGE_PARSE            = ""
	DEFAULT_MESSAGE_LINK_NAMES       = 0
	DEFAULT_MESSAGE_UNFURL_LINKS     = false
	DEFAULT_MESSAGE_UNFURL_MEDIA     = true
	DEFAULT_MESSAGE_ICON_URL         = ""
	DEFAULT_MESSAGE_ICON_EMOJI       = ""
	DEFAULT_MESSAGE_MARKDOWN         = true
	DEFAULT_MESSAGE_ESCAPE_TEXT      = true
)

type chatResponseFull struct {
	Channel   string `json:"channel"`
	Timestamp string `json:"ts"`
	Text      string `json:"text"`
	SlackResponse
}

// PostMessageParameters contains all the parameters necessary (including the optional ones) for a PostMessage() request
type PostMessageParameters struct {
	Text            string       `json:"text"`
	Username        string       `json:"user_name"`
	AsUser          bool         `json:"as_user"`
	Parse           string       `json:"parse"`
	ThreadTimestamp string       `json:"thread_ts"`
	LinkNames       int          `json:"link_names"`
	Attachments     []Attachment `json:"attachments"`
	UnfurlLinks     bool         `json:"unfurl_links"`
	UnfurlMedia     bool         `json:"unfurl_media"`
	IconURL         string       `json:"icon_url"`
	IconEmoji       string       `json:"icon_emoji"`
	Markdown        bool         `json:"mrkdwn,omitempty"`
	EscapeText      bool         `json:"escape_text"`
}

// NewPostMessageParameters provides an instance of PostMessageParameters with all the sane default values set
func NewPostMessageParameters() PostMessageParameters {
	return PostMessageParameters{
		Username:    DEFAULT_MESSAGE_USERNAME,
		AsUser:      DEFAULT_MESSAGE_ASUSER,
		Parse:       DEFAULT_MESSAGE_PARSE,
		LinkNames:   DEFAULT_MESSAGE_LINK_NAMES,
		Attachments: nil,
		UnfurlLinks: DEFAULT_MESSAGE_UNFURL_LINKS,
		UnfurlMedia: DEFAULT_MESSAGE_UNFURL_MEDIA,
		IconURL:     DEFAULT_MESSAGE_ICON_URL,
		IconEmoji:   DEFAULT_MESSAGE_ICON_EMOJI,
		Markdown:    DEFAULT_MESSAGE_MARKDOWN,
		EscapeText:  DEFAULT_MESSAGE_ESCAPE_TEXT,
	}
}

// DeleteMessage deletes a message in a channel
func (api *Client) DeleteMessage(channel, messageTimestamp string) (string, string, error) {
	respChannel, respTimestamp, _, err := api.SendMessage(channel, MsgOptionDelete(messageTimestamp))
	return respChannel, respTimestamp, err
}

// PostMessage sends a message to a channel.
// Message is escaped by default according to https://api.slack.com/docs/formatting
// Use http://davestevens.github.io/slack-message-builder/ to help crafting your message.
func (api *Client) PostMessage(channel, text string, params PostMessageParameters) (string, string, error) {
	respChannel, respTimestamp, _, err := api.SendMessage(
		channel,
		MsgOptionText(text, params.EscapeText),
		MsgOptionAttachments(params.Attachments...),
		MsgOptionPostMessageParameters(params),
	)
	return respChannel, respTimestamp, err
}

// UpdateMessage updates a message in a channel
func (api *Client) UpdateMessage(channel, timestamp, text string) (string, string, string, error) {
	return api.SendMessage(channel, MsgOptionUpdate(timestamp), MsgOptionText(text, true))
}

// SendMessage more flexible method for configuring messages.
func (api *Client) SendMessage(channel string, options ...MsgOption) (string, string, string, error) {
	channel, values, err := ApplyMsgOptions(api.config.token, channel, options...)
	if err != nil {
		return "", "", "", err
	}

	response, err := chatRequest(channel, values, api.debug)
	if err != nil {
		return "", "", "", err
	}

	return response.Channel, response.Timestamp, response.Text, nil
}

// ApplyMsgOptions utility function for debugging/testing chat requests.
func ApplyMsgOptions(token, channel string, options ...MsgOption) (string, url.Values, error) {
	config := sendConfig{
		mode: chatPostMessage,
		values: url.Values{
			"token":   {token},
			"channel": {channel},
		},
	}

	for _, opt := range options {
		if err := opt(&config); err != nil {
			return string(config.mode), config.values, err
		}
	}

	return string(config.mode), config.values, nil
}

func escapeMessage(message string) string {
	replacer := strings.NewReplacer("&", "&", "<", "<", ">", ">")
	return replacer.Replace(message)
}

func chatRequest(path string, values url.Values, debug bool) (*chatResponseFull, error) {
	response := &chatResponseFull{}
	err := post(path, values, response, debug)
	if err != nil {
		return nil, err
	}
	if !response.Ok {
		return nil, errors.New(response.Error)
	}
	return response, nil
}

type sendMode string

const (
	chatUpdate      sendMode = "chat.update"
	chatPostMessage sendMode = "chat.postMessage"
	chatDelete      sendMode = "chat.delete"
)

type sendConfig struct {
	mode   sendMode
	values url.Values
}

// MsgOption option provided when sending a message.
type MsgOption func(*sendConfig) error

// MsgOptionPost posts a messages, this is the default.
func MsgOptionPost() MsgOption {
	return func(config *sendConfig) error {
		config.mode = chatPostMessage
		config.values.Del("ts")
		return nil
	}
}

// MsgOptionUpdate updates a message based on the timestamp.
func MsgOptionUpdate(timestamp string) MsgOption {
	return func(config *sendConfig) error {
		config.mode = chatUpdate
		config.values.Add("ts", timestamp)
		return nil
	}
}

// MsgOptionDelete deletes a message based on the timestamp.
func MsgOptionDelete(timestamp string) MsgOption {
	return func(config *sendConfig) error {
		config.mode = chatDelete
		config.values.Add("ts", timestamp)
		return nil
	}
}

// MsgOptionAsUser whether or not to send the message as the user.
func MsgOptionAsUser(b bool) MsgOption {
	return func(config *sendConfig) error {
		if b != DEFAULT_MESSAGE_ASUSER {
			config.values.Set("as_user", "true")
		}
		return nil
	}
}

// MsgOptionText provide the text for the message, optionally escape the provided
// text.
func MsgOptionText(text string, escape bool) MsgOption {
	return func(config *sendConfig) error {
		if escape {
			text = escapeMessage(text)
		}
		config.values.Add("text", text)
		return nil
	}
}

// MsgOptionAttachments provide attachments for the message.
func MsgOptionAttachments(attachments ...Attachment) MsgOption {
	return func(config *sendConfig) error {
		if attachments == nil {
			return nil
		}

		attachments, err := json.Marshal(attachments)
		if err == nil {
			config.values.Set("attachments", string(attachments))
		}
		return err
	}
}

// MsgOptionEnableLinkUnfurl enables link unfurling
func MsgOptionEnableLinkUnfurl() MsgOption {
	return func(config *sendConfig) error {
		config.values.Set("unfurl_links", "true")
		return nil
	}
}

// MsgOptionDisableMediaUnfurl disables media unfurling.
func MsgOptionDisableMediaUnfurl() MsgOption {
	return func(config *sendConfig) error {
		config.values.Set("unfurl_media", "false")
		return nil
	}
}

// MsgOptionDisableMarkdown disables markdown.
func MsgOptionDisableMarkdown() MsgOption {
	return func(config *sendConfig) error {
		config.values.Set("mrkdwn", "false")
		return nil
	}
}

// MsgOptionPostMessageParameters maintain backwards compatibility.
func MsgOptionPostMessageParameters(params PostMessageParameters) MsgOption {
	return func(config *sendConfig) error {
		if params.Username != DEFAULT_MESSAGE_USERNAME {
			config.values.Set("username", string(params.Username))
		}

		// never generates an error.
		MsgOptionAsUser(params.AsUser)(config)

		if params.Parse != DEFAULT_MESSAGE_PARSE {
			config.values.Set("parse", string(params.Parse))
		}
		if params.LinkNames != DEFAULT_MESSAGE_LINK_NAMES {
			config.values.Set("link_names", "1")
		}

		if params.UnfurlLinks != DEFAULT_MESSAGE_UNFURL_LINKS {
			config.values.Set("unfurl_links", "true")
		}

		// I want to send a message with explicit `as_user` `true` and `unfurl_links` `false` in request.
		// Because setting `as_user` to `true` will change the default value for `unfurl_links` to `true` on Slack API side.
		if params.AsUser != DEFAULT_MESSAGE_ASUSER && params.UnfurlLinks == DEFAULT_MESSAGE_UNFURL_LINKS {
			config.values.Set("unfurl_links", "false")
		}
		if params.UnfurlMedia != DEFAULT_MESSAGE_UNFURL_MEDIA {
			config.values.Set("unfurl_media", "false")
		}
		if params.IconURL != DEFAULT_MESSAGE_ICON_URL {
			config.values.Set("icon_url", params.IconURL)
		}
		if params.IconEmoji != DEFAULT_MESSAGE_ICON_EMOJI {
			config.values.Set("icon_emoji", params.IconEmoji)
		}
		if params.Markdown != DEFAULT_MESSAGE_MARKDOWN {
			config.values.Set("mrkdwn", "false")
		}

		if params.ThreadTimestamp != DEFAULT_MESSAGE_THREAD_TIMESTAMP {
			config.values.Set("thread_ts", params.ThreadTimestamp)
		}

		return nil
	}
}
<|endoftext|>


<commit_before>package makecoe

import (
	"fmt"
	"net/http"

	"strings"

	"time"

	"github.com/getcarina/carina/common"
	"github.com/getcarina/libcarina"
	"github.com/pkg/errors"
)

// MakeCOE is an adapter between the cli and Carina (make-coe)
type MakeCOE struct {
	client           *libcarina.CarinaClient
	clusterTypeCache map[int]*libcarina.ClusterType
	Account          *Account
}

func (carina *MakeCOE) init() error {
	if carina.client == nil {
		carinaClient, err := carina.Account.Authenticate()
		if err != nil {
			return err
		}
		carina.client = carinaClient
	}
	return nil
}

// GetQuotas retrieves the quotas set for the account
func (carina *MakeCOE) GetQuotas() (common.Quotas, error) {
	return &Quotas{
		&libcarina.Quotas{
			MaxClusters:        3,
			MaxNodesPerCluster: 1,
		},
	}, nil
}

// CreateCluster creates a new cluster and prints the cluster information
func (carina *MakeCOE) CreateCluster(name string, template string, nodes int) (common.Cluster, error) {
	if template == "" {
		return nil, errors.New("--template is required")
	}

	if nodes > 1 {
		common.Log.WriteWarning("Using --nodes=1. Multi-node cluster support is coming soon!")
	}

	err := carina.init()
	if err != nil {
		return nil, err
	}

	clusterType, err := carina.lookupClusterTypeByName(template)
	if err != nil {
		return nil, err
	}

	common.Log.WriteDebug("[make-coe] Creating a %d-node %s cluster hosted on %s named %s", nodes, clusterType.COE, clusterType.HostType, name)
	createOpts := &libcarina.CreateClusterOpts{
		Name:          name,
		ClusterTypeID: clusterType.ID,
		Nodes:         nodes,
	}

	result, err := carina.client.Create(createOpts)
	if err != nil {
		return nil, errors.Wrap(err, "[make-coe] Unable to create cluster")
	}

	cluster := &Cluster{Cluster: result}

	return cluster, nil
}

// GetClusterCredentials retrieves the TLS certificates and configuration scripts for a cluster by its id or name (if unique)
func (carina *MakeCOE) GetClusterCredentials(token string) (*libcarina.CredentialsBundle, error) {
	err := carina.init()
	if err != nil {
		return nil, err
	}

	common.Log.WriteDebug("[make-coe] Retrieving cluster credentials (%s)", token)
	creds, err := carina.client.GetCredentials(token)
	if err != nil {
		return nil, errors.Wrap(err, "[make-coe] Unable to retrieve the cluster credentials")
	}

	return creds, nil
}

// ListClusters prints out a list of the user's clusters to the console
func (carina *MakeCOE) ListClusters() ([]common.Cluster, error) {
	var clusters []common.Cluster

	err := carina.init()
	if err != nil {
		return clusters, err
	}

	common.Log.WriteDebug("[make-coe] Listing clusters")
	results, err := carina.client.List()
	if err != nil {
		return clusters, errors.Wrap(err, "[make-coe] Unable to list clusters")
	}

	for _, result := range results {
		cluster := &Cluster{Cluster: result}
		clusters = append(clusters, cluster)
	}

	return clusters, err
}

// ListClusterTemplates retrieves available templates for creating a new cluster
func (carina *MakeCOE) ListClusterTemplates() ([]common.ClusterTemplate, error) {
	err := carina.init()
	if err != nil {
		return nil, err
	}

	results, err := carina.listClusterTypes()
	if err != nil {
		return nil, err
	}

	var templates []common.ClusterTemplate
	for _, result := range results {
		template := &ClusterTemplate{ClusterType: result}
		templates = append(templates, template)
	}

	return templates, err
}

// RebuildCluster destroys and recreates the cluster by its id or name (if unique)
func (carina *MakeCOE) RebuildCluster(token string) (common.Cluster, error) {
	return nil, errors.New("[make-coe] Rebuilding clusters from the carina cli is not supported yet")
}

// GetCluster prints out a cluster's information to the console by its id or name (if unique)
func (carina *MakeCOE) GetCluster(token string) (common.Cluster, error) {
	err := carina.init()
	if err != nil {
		return nil, err
	}

	common.Log.WriteDebug("[make-coe] Retrieving cluster (%s)", token)
	result, err := carina.client.Get(token)
	if err != nil {
		return nil, errors.Wrap(err, fmt.Sprintf("[make-coe] Unable to retrieve cluster (%s)", token))
	}
	cluster := &Cluster{Cluster: result}

	return cluster, nil
}

// DeleteCluster permanently deletes a cluster by its id or name (if unique)
func (carina *MakeCOE) DeleteCluster(token string) (common.Cluster, error) {
	err := carina.init()
	if err != nil {
		return nil, err
	}

	common.Log.WriteDebug("[make-coe] Deleting cluster (%s)", token)
	result, err := carina.client.Delete(token)
	if err != nil {
		if httpErr, ok := err.(libcarina.HTTPErr); ok {
			if httpErr.StatusCode == http.StatusNotFound {
				common.Log.WriteWarning("Could not find the cluster (%s) to delete", token)
				cluster := newCluster()
				cluster.Status = "deleted"
				return cluster, nil
			}
		}

		return nil, errors.Wrap(err, fmt.Sprintf("[make-coe] Unable to delete cluster (%s)", token))
	}

	cluster := &Cluster{Cluster: result}

	return cluster, nil
}

// GrowCluster adds nodes to a cluster by its id or name (if unique)
func (carina *MakeCOE) GrowCluster(token string, nodes int) (common.Cluster, error) {
	return nil, errors.New("[make-coe] Growing clusters from the carina cli is not supported yet")
}

// SetAutoScale is not supported
func (carina *MakeCOE) SetAutoScale(token string, value bool) (common.Cluster, error) {
	return nil, errors.New("make-coe does not support autoscaling")
}

// WaitUntilClusterIsActive waits until the prior cluster operation is completed
func (carina *MakeCOE) WaitUntilClusterIsActive(cluster common.Cluster) (common.Cluster, error) {
	return nil, errors.New("Not implemented")
}

// WaitUntilClusterIsDeleted polls the cluster status until either the cluster is gone or an error state is hit
func (carina *MakeCOE) WaitUntilClusterIsDeleted(cluster common.Cluster) error {
	isDone := func(cluster common.Cluster) bool {
		status := strings.ToUpper(cluster.GetStatus())
		return status == "deleted"
	}

	if isDone(cluster) {
		return nil
	}

	pollingInterval := 5 * time.Second
	for {
		cluster, err := carina.GetCluster(cluster.GetID())

		if err != nil {
			err = errors.Cause(err)

			// Gracefully handle a 404 Not Found when the cluster is deleted quickly
			if httpErr, ok := err.(libcarina.HTTPErr); ok {
				if httpErr.StatusCode == http.StatusNotFound {
					return nil
				}
			}

			return err
		}

		if isDone(cluster) {
			return nil
		}

		common.Log.WriteDebug("[make-coe] Waiting until cluster (%s) is deleted, currently in %s", cluster.GetName(), cluster.GetStatus())
		time.Sleep(pollingInterval)
	}
}

func (carina *MakeCOE) listClusterTypes() ([]*libcarina.ClusterType, error) {
	common.Log.WriteDebug("[make-coe] Listing cluster types")
	clusterTypes, err := carina.client.ListClusterTypes()
	if err != nil {
		return nil, errors.Wrap(err, "[make-coe] Unabe to list cluster types")
	}

	return clusterTypes, err
}

func (carina *MakeCOE) getClusterTypeCache() (map[int]*libcarina.ClusterType, error) {
	if carina.clusterTypeCache == nil {
		clusterTypes, err := carina.listClusterTypes()
		if err != nil {
			return nil, err
		}

		carina.clusterTypeCache = make(map[int]*libcarina.ClusterType)
		for _, clusterType := range clusterTypes {
			carina.clusterTypeCache[clusterType.ID] = clusterType
		}
	}

	return carina.clusterTypeCache, nil
}

func (carina *MakeCOE) lookupClusterTypeByName(name string) (*libcarina.ClusterType, error) {
	cache, err := carina.getClusterTypeCache()
	if err != nil {
		return nil, err
	}

	name = strings.ToLower(name)
	var clusterType *libcarina.ClusterType
	for _, m := range cache {
		if strings.ToLower(m.Name) == name {
			clusterType = m
			break
		}
	}

	if clusterType == nil {
		return nil, fmt.Errorf("Could not find template named %s", name)
	}

	return clusterType, nil
}
<commit_msg>Support --wait for make-coe's get and create<commit_after>package makecoe

import (
	"fmt"
	"net/http"

	"strings"

	"time"

	"github.com/getcarina/carina/common"
	"github.com/getcarina/libcarina"
	"github.com/pkg/errors"
)

// MakeCOE is an adapter between the cli and Carina (make-coe)
type MakeCOE struct {
	client           *libcarina.CarinaClient
	clusterTypeCache map[int]*libcarina.ClusterType
	Account          *Account
}

func (carina *MakeCOE) init() error {
	if carina.client == nil {
		carinaClient, err := carina.Account.Authenticate()
		if err != nil {
			return err
		}
		carina.client = carinaClient
	}
	return nil
}

// GetQuotas retrieves the quotas set for the account
func (carina *MakeCOE) GetQuotas() (common.Quotas, error) {
	return &Quotas{
		&libcarina.Quotas{
			MaxClusters:        3,
			MaxNodesPerCluster: 1,
		},
	}, nil
}

// CreateCluster creates a new cluster and prints the cluster information
func (carina *MakeCOE) CreateCluster(name string, template string, nodes int) (common.Cluster, error) {
	if template == "" {
		return nil, errors.New("--template is required")
	}

	if nodes > 1 {
		common.Log.WriteWarning("Using --nodes=1. Multi-node cluster support is coming soon!")
	}

	err := carina.init()
	if err != nil {
		return nil, err
	}

	clusterType, err := carina.lookupClusterTypeByName(template)
	if err != nil {
		return nil, err
	}

	common.Log.WriteDebug("[make-coe] Creating a %d-node %s cluster hosted on %s named %s", nodes, clusterType.COE, clusterType.HostType, name)
	createOpts := &libcarina.CreateClusterOpts{
		Name:          name,
		ClusterTypeID: clusterType.ID,
		Nodes:         nodes,
	}

	result, err := carina.client.Create(createOpts)
	if err != nil {
		return nil, errors.Wrap(err, "[make-coe] Unable to create cluster")
	}

	cluster := &Cluster{Cluster: result}

	return cluster, nil
}

// GetClusterCredentials retrieves the TLS certificates and configuration scripts for a cluster by its id or name (if unique)
func (carina *MakeCOE) GetClusterCredentials(token string) (*libcarina.CredentialsBundle, error) {
	err := carina.init()
	if err != nil {
		return nil, err
	}

	common.Log.WriteDebug("[make-coe] Retrieving cluster credentials (%s)", token)
	creds, err := carina.client.GetCredentials(token)
	if err != nil {
		return nil, errors.Wrap(err, "[make-coe] Unable to retrieve the cluster credentials")
	}

	return creds, nil
}

// ListClusters prints out a list of the user's clusters to the console
func (carina *MakeCOE) ListClusters() ([]common.Cluster, error) {
	var clusters []common.Cluster

	err := carina.init()
	if err != nil {
		return clusters, err
	}

	common.Log.WriteDebug("[make-coe] Listing clusters")
	results, err := carina.client.List()
	if err != nil {
		return clusters, errors.Wrap(err, "[make-coe] Unable to list clusters")
	}

	for _, result := range results {
		cluster := &Cluster{Cluster: result}
		clusters = append(clusters, cluster)
	}

	return clusters, err
}

// ListClusterTemplates retrieves available templates for creating a new cluster
func (carina *MakeCOE) ListClusterTemplates() ([]common.ClusterTemplate, error) {
	err := carina.init()
	if err != nil {
		return nil, err
	}

	results, err := carina.listClusterTypes()
	if err != nil {
		return nil, err
	}

	var templates []common.ClusterTemplate
	for _, result := range results {
		template := &ClusterTemplate{ClusterType: result}
		templates = append(templates, template)
	}

	return templates, err
}

// RebuildCluster destroys and recreates the cluster by its id or name (if unique)
func (carina *MakeCOE) RebuildCluster(token string) (common.Cluster, error) {
	return nil, errors.New("[make-coe] Rebuilding clusters from the carina cli is not supported yet")
}

// GetCluster prints out a cluster's information to the console by its id or name (if unique)
func (carina *MakeCOE) GetCluster(token string) (common.Cluster, error) {
	err := carina.init()
	if err != nil {
		return nil, err
	}

	common.Log.WriteDebug("[make-coe] Retrieving cluster (%s)", token)
	result, err := carina.client.Get(token)
	if err != nil {
		return nil, errors.Wrap(err, fmt.Sprintf("[make-coe] Unable to retrieve cluster (%s)", token))
	}
	cluster := &Cluster{Cluster: result}

	return cluster, nil
}

// DeleteCluster permanently deletes a cluster by its id or name (if unique)
func (carina *MakeCOE) DeleteCluster(token string) (common.Cluster, error) {
	err := carina.init()
	if err != nil {
		return nil, err
	}

	common.Log.WriteDebug("[make-coe] Deleting cluster (%s)", token)
	result, err := carina.client.Delete(token)
	if err != nil {
		if httpErr, ok := err.(libcarina.HTTPErr); ok {
			if httpErr.StatusCode == http.StatusNotFound {
				common.Log.WriteWarning("Could not find the cluster (%s) to delete", token)
				cluster := newCluster()
				cluster.Status = "deleted"
				return cluster, nil
			}
		}

		return nil, errors.Wrap(err, fmt.Sprintf("[make-coe] Unable to delete cluster (%s)", token))
	}

	cluster := &Cluster{Cluster: result}

	return cluster, nil
}

// GrowCluster adds nodes to a cluster by its id or name (if unique)
func (carina *MakeCOE) GrowCluster(token string, nodes int) (common.Cluster, error) {
	return nil, errors.New("[make-coe] Growing clusters from the carina cli is not supported yet")
}

// SetAutoScale is not supported
func (carina *MakeCOE) SetAutoScale(token string, value bool) (common.Cluster, error) {
	return nil, errors.New("make-coe does not support autoscaling")
}

// WaitUntilClusterIsActive waits until the prior cluster operation is completed
func (carina *MakeCOE) WaitUntilClusterIsActive(cluster common.Cluster) (common.Cluster, error) {
	isDone := func(cluster common.Cluster) bool {
		status := strings.ToLower(cluster.GetStatus())
		return status == "active"
	}

	if isDone(cluster) {
		return cluster, nil
	}

	pollingInterval := 5 * time.Second
	for {
		cluster, err := carina.GetCluster(cluster.GetID())
		if err != nil {
			err = errors.Cause(err)
			return nil, err
		}

		if isDone(cluster) {
			return cluster, nil
		}

		common.Log.WriteDebug("[make-coe] Waiting until cluster (%s) is active, currently in %s", cluster.GetName(), cluster.GetStatus())
		time.Sleep(pollingInterval)
	}
}

// WaitUntilClusterIsDeleted polls the cluster status until either the cluster is gone or an error state is hit
func (carina *MakeCOE) WaitUntilClusterIsDeleted(cluster common.Cluster) error {
	isDone := func(cluster common.Cluster) bool {
		status := strings.ToUpper(cluster.GetStatus())
		return status == "deleted"
	}

	if isDone(cluster) {
		return nil
	}

	pollingInterval := 5 * time.Second
	for {
		cluster, err := carina.GetCluster(cluster.GetID())
		if err != nil {
			err = errors.Cause(err)

			// Gracefully handle a 404 Not Found when the cluster is deleted quickly
			if httpErr, ok := err.(libcarina.HTTPErr); ok {
				if httpErr.StatusCode == http.StatusNotFound {
					return nil
				}
			}

			return err
		}

		if isDone(cluster) {
			return nil
		}

		common.Log.WriteDebug("[make-coe] Waiting until cluster (%s) is deleted, currently in %s", cluster.GetName(), cluster.GetStatus())
		time.Sleep(pollingInterval)
	}
}

func (carina *MakeCOE) listClusterTypes() ([]*libcarina.ClusterType, error) {
	common.Log.WriteDebug("[make-coe] Listing cluster types")
	clusterTypes, err := carina.client.ListClusterTypes()
	if err != nil {
		return nil, errors.Wrap(err, "[make-coe] Unabe to list cluster types")
	}

	return clusterTypes, err
}

func (carina *MakeCOE) getClusterTypeCache() (map[int]*libcarina.ClusterType, error) {
	if carina.clusterTypeCache == nil {
		clusterTypes, err := carina.listClusterTypes()
		if err != nil {
			return nil, err
		}

		carina.clusterTypeCache = make(map[int]*libcarina.ClusterType)
		for _, clusterType := range clusterTypes {
			carina.clusterTypeCache[clusterType.ID] = clusterType
		}
	}

	return carina.clusterTypeCache, nil
}

func (carina *MakeCOE) lookupClusterTypeByName(name string) (*libcarina.ClusterType, error) {
	cache, err := carina.getClusterTypeCache()
	if err != nil {
		return nil, err
	}

	name = strings.ToLower(name)
	var clusterType *libcarina.ClusterType
	for _, m := range cache {
		if strings.ToLower(m.Name) == name {
			clusterType = m
			break
		}
	}

	if clusterType == nil {
		return nil, fmt.Errorf("Could not find template named %s", name)
	}

	return clusterType, nil
}
<|endoftext|>


<commit_before>/*

*/
package mandrake

import (
	"log"
	"encoding/json"

	"golang.org/x/exp/inotify"
	"github.com/hosom/gomandrake/config"
	"github.com/hosom/gomandrake/filemeta"
	"github.com/hosom/gomandrake/plugin"
)

// Mandrake is a wrapper struct for the bulk of the application logic
type Mandrake struct {
	AnalysisPipeline	chan string
	MonitoredDirectory	string
	Analyzers			[]plugin.AnalyzerCaller
	AnalyzerFilter		map[string][]plugin.AnalyzerCaller
	Loggers				[]plugin.LoggerCaller
}

// NewMandrake creates and returns a Mandrake struct utilizing a passed
// parsed configuration file to create the correct fields.
func NewMandrake(c config.Config) Mandrake {
	analyzers := []plugin.AnalyzerCaller{}
	filter := make(map[string][]plugin.AnalyzerCaller)
	for _, plug := range c.Analyzers {
		analyzer := plugin.NewAnalyzerCaller(plug)
		// Build a slice of all AnalyzerCaller structs
		analyzers = append(analyzers, analyzer)

		// Create a map to function as a mime_type filter for analyzers
		for _, mime := range analyzer.MimeFilter {
			filter[mime] = append(filter[mime], analyzer)
		}
	}

	loggers := []plugin.LoggerCaller{}
	for _, plug := range c.Loggers {
		logger := plugin.NewLoggerCaller(plug)
		loggers = append(loggers, logger)
	}

	return Mandrake{make(chan string), c.MonitoredDirectory, analyzers, filter, loggers}
}

// ListenAndServe starts the goroutines that perform all of the heavy lifting
// including Monitor() and DispatchAnalysis().
func (m Mandrake) ListenAndServe() {
	log.SetPrefix("[mandrake] ")
	log.Println(m.Analyzers[0])
	go m.DispatchAnalysis()
	m.Monitor()
}

// DispatchAnalysis intelligently sends a new file to registered plugins so
// that it can be analyzed.
func (m Mandrake) DispatchAnalysis() {
	for fpath := range m.AnalysisPipeline {
		go m.Analysis(fpath)
	}
}

// Analysis is the method that kicks off all of the analysis plugins
// this is utilized so that each file can be analyzed in a goroutine
func (m Mandrake) Analysis(fpath string) {
	fmeta, err := filemeta.NewFileMeta(fpath)
	if err != nil {
		log.Println(err)
	}

	// Create JSON filemeta object to pass to plugins so that plugins
	// receive basic contextual information about the file.
	fs, err := json.Marshal(fmeta)
	// Finalize string form of JSON filemeta to pass to plugins
	fstring := string(fs)

	var analysis []map[string]interface{}

	for _, analyzer := range m.AnalyzerFilter["all"] {
		result, err := analyzer.Analyze(fstring)
		if err != nil {
			log.Print(err)
		}
		analysis = append(analysis, MapFromJSON(result))
	}

	for _, analyzer := range m.AnalyzerFilter[fmeta.Mime] {
		result, err := analyzer.Analyze(fstring)
		if err != nil {
			log.Print(err)
		}
		analysis = append(analysis, MapFromJSON(result))
	}

	report := MapFromJSON(fstring)
	report["analysis"] = analysis

	r, _ := json.Marshal(report)
	log.Println(string(r))
	log.Println(string(fs))
	log.Printf("%s", fpath)
}

// Monitor uses inotify to monitor the MonitoredDirectory for IN_CLOSE_WRITE
// events. Files written to the MonitoredDirectory will be sent to the
// analysis pipeline to be analyzed.
func (m Mandrake) Monitor() {
	log.Println("starting inotify watcher")
	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("adding watcher to %s directory", m.MonitoredDirectory)
	err = watcher.AddWatch(m.MonitoredDirectory, inotify.IN_CLOSE_WRITE)
	if err != nil {
		log.Fatal(err)
	}

	for {
		select {
		case ev := <- watcher.Event:
			m.AnalysisPipeline <- ev.Name
		case err := <- watcher.Error:
			log.Printf("inotify error: %s", err)
		}
	}
}

// MapFromJSON accepts an anonymous JSON object as a string and returns the
// resulting Map
func MapFromJSON(s string) map[string]interface{} {
	log.Printf("Performing mapping with string: %s", s)
	var f interface{}
	json.Unmarshal([]byte(s), &f)
	m := f.(map[string]interface{})
	log.Printf("Mapping complete: %s", m)
	return m
}<commit_msg>update mandrake<commit_after>/*

*/
package mandrake

import (
	"log"
	"encoding/json"

	"golang.org/x/exp/inotify"
	"github.com/hosom/gomandrake/config"
	"github.com/hosom/gomandrake/filemeta"
	"github.com/hosom/gomandrake/plugin"
)

// Mandrake is a wrapper struct for the bulk of the application logic
type Mandrake struct {
	AnalysisPipeline	chan string
	LoggingPipeline		chan string
	MonitoredDirectory	string
	Analyzers			[]plugin.AnalyzerCaller
	AnalyzerFilter		map[string][]plugin.AnalyzerCaller
	Loggers				[]plugin.LoggerCaller
}

// NewMandrake creates and returns a Mandrake struct utilizing a passed
// parsed configuration file to create the correct fields.
func NewMandrake(c config.Config) Mandrake {
	analyzers := []plugin.AnalyzerCaller{}
	filter := make(map[string][]plugin.AnalyzerCaller)
	for _, plug := range c.Analyzers {
		analyzer := plugin.NewAnalyzerCaller(plug)
		// Build a slice of all AnalyzerCaller structs
		analyzers = append(analyzers, analyzer)

		// Create a map to function as a mime_type filter for analyzers
		for _, mime := range analyzer.MimeFilter {
			filter[mime] = append(filter[mime], analyzer)
		}
	}

	loggers := []plugin.LoggerCaller{}
	for _, plug := range c.Loggers {
		logger := plugin.NewLoggerCaller(plug)
		loggers = append(loggers, logger)
	}

	return Mandrake{make(chan string), c.MonitoredDirectory, analyzers, filter, loggers}
}

// ListenAndServe starts the goroutines that perform all of the heavy lifting
// including Monitor() and DispatchAnalysis().
func (m Mandrake) ListenAndServe() {
	log.SetPrefix("[mandrake] ")
	log.Println(m.Analyzers[0])
	go m.DispatchAnalysis()
	m.Monitor()
}

// DispatchAnalysis intelligently sends a new file to registered plugins so
// that it can be analyzed.
func (m Mandrake) DispatchAnalysis() {
	for fpath := range m.AnalysisPipeline {
		go m.Analysis(fpath)
	}
}

// Analysis is the method that kicks off all of the analysis plugins
// this is utilized so that each file can be analyzed in a goroutine
func (m Mandrake) Analysis(fpath string) {
	fmeta, err := filemeta.NewFileMeta(fpath)
	if err != nil {
		log.Println(err)
	}

	// Create JSON filemeta object to pass to plugins so that plugins
	// receive basic contextual information about the file.
	fs, err := json.Marshal(fmeta)
	// Finalize string form of JSON filemeta to pass to plugins
	fstring := string(fs)

	var analysis []map[string]interface{}

	for _, analyzer := range m.AnalyzerFilter["all"] {
		result, err := analyzer.Analyze(fstring)
		if err != nil {
			log.Print(err)
		}
		analysis = append(analysis, MapFromJSON(result))
	}

	for _, analyzer := range m.AnalyzerFilter[fmeta.Mime] {
		result, err := analyzer.Analyze(fstring)
		if err != nil {
			log.Print(err)
		}
		analysis = append(analysis, MapFromJSON(result))
	}

	report := MapFromJSON(fstring)
	report["analysis"] = analysis

	r, _ := json.Marshal(report)

	log.Printf("Analysis of %s complete", fpath)
	m.LoggingPipeline <- string(report)
	log.Printf("File analysis sent to logging pipeline.")
}

// DispatcheLogging sends the call to the Logger plugins to log the completed
// record of analysis performed by Mandrake
func (m Mandrake) DispatchLogging() {
	for record := range m.LoggingPipeline {
		for _, logger := range m.Loggers {
			result, err := logger.Log(record)
			if err != nil {
				log.Print(err)
			}
		}
	}
}

// Monitor uses inotify to monitor the MonitoredDirectory for IN_CLOSE_WRITE
// events. Files written to the MonitoredDirectory will be sent to the
// analysis pipeline to be analyzed.
func (m Mandrake) Monitor() {
	log.Println("starting inotify watcher")
	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("adding watcher to %s directory", m.MonitoredDirectory)
	err = watcher.AddWatch(m.MonitoredDirectory, inotify.IN_CLOSE_WRITE)
	if err != nil {
		log.Fatal(err)
	}

	for {
		select {
		case ev := <- watcher.Event:
			m.AnalysisPipeline <- ev.Name
		case err := <- watcher.Error:
			log.Printf("inotify error: %s", err)
		}
	}
}

// MapFromJSON accepts an anonymous JSON object as a string and returns the
// resulting Map
func MapFromJSON(s string) map[string]interface{} {
	log.Printf("Performing mapping with string: %s", s)
	var f interface{}
	json.Unmarshal([]byte(s), &f)
	m := f.(map[string]interface{})
	log.Printf("Mapping complete: %s", m)
	return m
}<|endoftext|>


<commit_before>package kv

import (
	"encoding/json"
	"fmt"
	"strings"

	"log"

	"golang.org/x/net/context"
	elastic "gopkg.in/olivere/elastic.v5"

	"github.com/movio/kasper/util"
)

const indexSettings = `{
	"index.translog.durability": "sync"
}`

const indexMapping = `{
	"_all" : {
		"enabled" : false
	},
	"dynamic_templates": [{
		"no_index": {
			"mapping": {
				"index": "no"
			},
			"match": "*"
		}
	}]
}`

type indexAndType struct {
	indexName string
	indexType string
}

// ElasticsearchKeyValueStore is a key-value storage that uses ElasticSearch.
// In this key-value store, all keys must have the format "<index>/<type>/<_id>".
type ElasticsearchKeyValueStore struct {
	witness         *util.StructPtrWitness
	client          *elastic.Client
	context         context.Context
	existingIndexes []indexAndType
}

// NewESKeyValueStore creates new ElasticsearchKeyValueStore instance.
// Host must of the format hostname:port.
// StructPtr should be a pointer to struct type that is used.
// for serialization and deserialization of store values.
func NewESKeyValueStore(url string, structPtr interface{}) *ElasticsearchKeyValueStore {
	client, err := elastic.NewClient(
		elastic.SetURL(url),
		elastic.SetSniff(false), // FIXME: workaround for issues with ES in docker
	)
	if err != nil {
		panic(fmt.Sprintf("Cannot create ElasticSearch Client to '%s': %s", url, err))
	}
	return &ElasticsearchKeyValueStore{
		witness:         util.NewStructPtrWitness(structPtr),
		client:          client,
		context:         context.Background(),
		existingIndexes: nil,
	}
}

func (s *ElasticsearchKeyValueStore) checkOrCreateIndex(indexName string, indexType string) {
	for _, existing := range s.existingIndexes {
		if existing.indexName == indexName && existing.indexType == indexType {
			return
		}
	}
	exists, err := s.client.IndexExists(indexName).Do(s.context)
	if err != nil {
		panic(fmt.Sprintf("Failed to check if index exists: %s", err))
	}
	if !exists {
		_, err = s.client.CreateIndex(indexName).BodyString(indexSettings).Do(s.context)
		if err != nil {
			panic(fmt.Sprintf("Failed to create index: %s", err))
		}
		s.putMapping(indexName, indexType)
	}

	s.existingIndexes = append(s.existingIndexes, indexAndType{indexName, indexType})
}

func (s *ElasticsearchKeyValueStore) putMapping(indexName string, indexType string) {
	resp, err := s.client.PutMapping().Index(indexName).Type(indexType).BodyString(indexMapping).Do(s.context)
	if err != nil {
		panic(fmt.Sprintf("Failed to put mapping for index: %s/%s: %s", indexName, indexType, err))
	}
	if resp == nil {
		panic(fmt.Sprintf("Expected put mapping response; got: %v", resp))
	}
	if !resp.Acknowledged {
		panic(fmt.Sprintf("Expected put mapping ack; got: %v", resp.Acknowledged))
	}
}

// Get gets value by key from store
func (s *ElasticsearchKeyValueStore) Get(key string) (interface{}, error) {
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return nil, fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	rawValue, err := s.client.Get().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if fmt.Sprintf("%s", err) == "elastic: Error 404 (Not Found)" {
		return s.witness.Nil(), nil
	}

	if err != nil {
		return s.witness.Nil(), err
	}

	if !rawValue.Found {
		return s.witness.Nil(), nil
	}

	structPtr := s.witness.Allocate()
	err = json.Unmarshal(*rawValue.Source, structPtr)
	if err != nil {
		return s.witness.Nil(), err
	}
	return structPtr, nil
}

// TBD
func (s *ElasticsearchKeyValueStore) GetAll(keys []string) ([]*Entry, error) {
	multiGet := s.client.MultiGet()
	for _, key := range keys {
		keyParts := strings.Split(key, "/")
		if len(keyParts) != 3 {
			return nil, fmt.Errorf("invalid key: '%s'", key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.checkOrCreateIndex(indexName, indexType)

		item := elastic.NewMultiGetItem().
			Index(indexName).
			Type(indexType).
			Id(valueID)

		multiGet.Add(item)
	}
	response, err := multiGet.Do(s.context)
	if err != nil {
		return nil, err
	}
	entries := make([]*Entry, len(keys))
	for i, doc := range response.Docs {
		var structPtr interface{}
		if !doc.Found {
			structPtr = s.witness.Nil()
		} else {
			structPtr = s.witness.Allocate()
			err = json.Unmarshal(*doc.Source, structPtr)
			if err != nil {
				return nil, err
			}
		}
		entries[i] = &Entry{keys[i], structPtr}
	}
	return entries, nil
}

// Put updates key in store with serialized value
func (s *ElasticsearchKeyValueStore) Put(key string, structPtr interface{}) error {
	s.witness.Assert(structPtr)
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	_, err := s.client.Index().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		BodyJson(structPtr).
		Do(s.context)

	return err
}

// PutAll bulk executes Put operation for several entries
func (s *ElasticsearchKeyValueStore) PutAll(entries []*Entry) error {
	if len(entries) == 0 {
		return nil
	}
	bulk := s.client.Bulk()
	for _, entry := range entries {
		keyParts := strings.Split(entry.Key, "/")
		if len(keyParts) != 3 {
			return fmt.Errorf("invalid key: '%s'", entry.Key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.witness.Assert(entry.Value)
		s.checkOrCreateIndex(indexName, indexType)

		bulk.Add(elastic.NewBulkIndexRequest().
			Index(indexName).
			Type(indexType).
			Id(valueID).
			Doc(entry.Value),
		)
	}
	_, err := bulk.Do(s.context)
	return err
}

// Delete removes key from store
func (s *ElasticsearchKeyValueStore) Delete(key string) error {
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	response, err := s.client.Delete().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if response != nil && !response.Found {
		return nil
	}

	return err
}

// Flush the Elasticsearch translog to disk
func (s *ElasticsearchKeyValueStore) Flush() error {
	log.Println("Flusing ES indexes...")
	_, err := s.client.Flush("_all").
		WaitIfOngoing(true).
		Do(s.context)
	log.Println("Done flusing ES indexes.")
	return err
}
<commit_msg>s/sync/request<commit_after>package kv

import (
	"encoding/json"
	"fmt"
	"strings"

	"log"

	"golang.org/x/net/context"
	elastic "gopkg.in/olivere/elastic.v5"

	"github.com/movio/kasper/util"
)

const indexSettings = `{
	"index.translog.durability": "request"
}`

const indexMapping = `{
	"_all" : {
		"enabled" : false
	},
	"dynamic_templates": [{
		"no_index": {
			"mapping": {
				"index": "no"
			},
			"match": "*"
		}
	}]
}`

type indexAndType struct {
	indexName string
	indexType string
}

// ElasticsearchKeyValueStore is a key-value storage that uses ElasticSearch.
// In this key-value store, all keys must have the format "<index>/<type>/<_id>".
type ElasticsearchKeyValueStore struct {
	witness         *util.StructPtrWitness
	client          *elastic.Client
	context         context.Context
	existingIndexes []indexAndType
}

// NewESKeyValueStore creates new ElasticsearchKeyValueStore instance.
// Host must of the format hostname:port.
// StructPtr should be a pointer to struct type that is used.
// for serialization and deserialization of store values.
func NewESKeyValueStore(url string, structPtr interface{}) *ElasticsearchKeyValueStore {
	client, err := elastic.NewClient(
		elastic.SetURL(url),
		elastic.SetSniff(false), // FIXME: workaround for issues with ES in docker
	)
	if err != nil {
		panic(fmt.Sprintf("Cannot create ElasticSearch Client to '%s': %s", url, err))
	}
	return &ElasticsearchKeyValueStore{
		witness:         util.NewStructPtrWitness(structPtr),
		client:          client,
		context:         context.Background(),
		existingIndexes: nil,
	}
}

func (s *ElasticsearchKeyValueStore) checkOrCreateIndex(indexName string, indexType string) {
	for _, existing := range s.existingIndexes {
		if existing.indexName == indexName && existing.indexType == indexType {
			return
		}
	}
	exists, err := s.client.IndexExists(indexName).Do(s.context)
	if err != nil {
		panic(fmt.Sprintf("Failed to check if index exists: %s", err))
	}
	if !exists {
		_, err = s.client.CreateIndex(indexName).BodyString(indexSettings).Do(s.context)
		if err != nil {
			panic(fmt.Sprintf("Failed to create index: %s", err))
		}
		s.putMapping(indexName, indexType)
	}

	s.existingIndexes = append(s.existingIndexes, indexAndType{indexName, indexType})
}

func (s *ElasticsearchKeyValueStore) putMapping(indexName string, indexType string) {
	resp, err := s.client.PutMapping().Index(indexName).Type(indexType).BodyString(indexMapping).Do(s.context)
	if err != nil {
		panic(fmt.Sprintf("Failed to put mapping for index: %s/%s: %s", indexName, indexType, err))
	}
	if resp == nil {
		panic(fmt.Sprintf("Expected put mapping response; got: %v", resp))
	}
	if !resp.Acknowledged {
		panic(fmt.Sprintf("Expected put mapping ack; got: %v", resp.Acknowledged))
	}
}

// Get gets value by key from store
func (s *ElasticsearchKeyValueStore) Get(key string) (interface{}, error) {
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return nil, fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	rawValue, err := s.client.Get().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if fmt.Sprintf("%s", err) == "elastic: Error 404 (Not Found)" {
		return s.witness.Nil(), nil
	}

	if err != nil {
		return s.witness.Nil(), err
	}

	if !rawValue.Found {
		return s.witness.Nil(), nil
	}

	structPtr := s.witness.Allocate()
	err = json.Unmarshal(*rawValue.Source, structPtr)
	if err != nil {
		return s.witness.Nil(), err
	}
	return structPtr, nil
}

// TBD
func (s *ElasticsearchKeyValueStore) GetAll(keys []string) ([]*Entry, error) {
	multiGet := s.client.MultiGet()
	for _, key := range keys {
		keyParts := strings.Split(key, "/")
		if len(keyParts) != 3 {
			return nil, fmt.Errorf("invalid key: '%s'", key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.checkOrCreateIndex(indexName, indexType)

		item := elastic.NewMultiGetItem().
			Index(indexName).
			Type(indexType).
			Id(valueID)

		multiGet.Add(item)
	}
	response, err := multiGet.Do(s.context)
	if err != nil {
		return nil, err
	}
	entries := make([]*Entry, len(keys))
	for i, doc := range response.Docs {
		var structPtr interface{}
		if !doc.Found {
			structPtr = s.witness.Nil()
		} else {
			structPtr = s.witness.Allocate()
			err = json.Unmarshal(*doc.Source, structPtr)
			if err != nil {
				return nil, err
			}
		}
		entries[i] = &Entry{keys[i], structPtr}
	}
	return entries, nil
}

// Put updates key in store with serialized value
func (s *ElasticsearchKeyValueStore) Put(key string, structPtr interface{}) error {
	s.witness.Assert(structPtr)
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	_, err := s.client.Index().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		BodyJson(structPtr).
		Do(s.context)

	return err
}

// PutAll bulk executes Put operation for several entries
func (s *ElasticsearchKeyValueStore) PutAll(entries []*Entry) error {
	if len(entries) == 0 {
		return nil
	}
	bulk := s.client.Bulk()
	for _, entry := range entries {
		keyParts := strings.Split(entry.Key, "/")
		if len(keyParts) != 3 {
			return fmt.Errorf("invalid key: '%s'", entry.Key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.witness.Assert(entry.Value)
		s.checkOrCreateIndex(indexName, indexType)

		bulk.Add(elastic.NewBulkIndexRequest().
			Index(indexName).
			Type(indexType).
			Id(valueID).
			Doc(entry.Value),
		)
	}
	_, err := bulk.Do(s.context)
	return err
}

// Delete removes key from store
func (s *ElasticsearchKeyValueStore) Delete(key string) error {
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	response, err := s.client.Delete().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if response != nil && !response.Found {
		return nil
	}

	return err
}

// Flush the Elasticsearch translog to disk
func (s *ElasticsearchKeyValueStore) Flush() error {
	log.Println("Flusing ES indexes...")
	_, err := s.client.Flush("_all").
		WaitIfOngoing(true).
		Do(s.context)
	log.Println("Done flusing ES indexes.")
	return err
}
<|endoftext|>


<commit_before>package date

import (
	"time"
)

func DateNowStringYMD() string {
	return time.Now().Format("2006-01-02")
}

func DateNowNanosecond() int64 {
	return time.Now().UnixNano()
}

func DateNowMillisecond() int64 {
	return time.Now().UnixNano() / int64(time.Millisecond/time.Nanosecond)
}

func DateNowSecond() int64 {
	return time.Now().Unix()
}

func DateDurationFrom(t time.Time) time.Duration {
	return time.Now().Sub(t)
}

func DateUnix(date string) int64 {
	t, err := time.Parse("2006-01-02", date)
	if err != nil {
		return 0
	}
	return t.Unix()
}

func DateUnixByLocation(date string, location string) int64 {
	loc, err := time.LoadLocation(location)
	var t time.Time
	var pErr error
	if err != nil {
		t, pErr = time.Parse("2006-01-02", date)
	} else {
		t, pErr = time.ParseInLocation("2006-01-02", date, loc)
	}
	if pErr != nil {
		return 0
	}
	return t.Unix()
}

func UnixMilli(t time.Time) int64 {
	return t.UnixNano() / int64(time.Millisecond/time.Nanosecond)
}

func BeginOfDate(t time.Time) time.Time {
	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
}

func Today() time.Time {
	now := time.Now()
	return BeginOfDate(now)
}

func Yesterday() time.Time {
	now := time.Now()
	diff := now.Add(-time.Hour * 24)
	return BeginOfDate(diff)
}

func DaysBeforeNow(days int64) time.Time {
	now := time.Now()
	diff := now.Add(-time.Hour * 24 * time.Duration(days))
	return BeginOfDate(diff)
}
<commit_msg>add DateMillisecond<commit_after>package date

import (
	"time"
)

func DateNowStringYMD() string {
	return time.Now().Format("2006-01-02")
}

func DateNowNanosecond() int64 {
	return time.Now().UnixNano()
}

func DateNowMillisecond() int64 {
	return time.Now().UnixNano() / int64(time.Millisecond/time.Nanosecond)
}

func DateNowSecond() int64 {
	return time.Now().Unix()
}

func DateDurationFrom(t time.Time) time.Duration {
	return time.Now().Sub(t)
}

func DateUnix(date string) int64 {
	t, err := time.Parse("2006-01-02", date)
	if err != nil {
		return 0
	}
	return t.Unix()
}

func DateMillisecond(date string) int64 {
	t, err := time.Parse("2006-01-02", date)
	if err != nil {
		return 0
	}
	return t.UnixNano() / int64(time.Millisecond/time.Nanosecond)
}

func DateUnixByLocation(date string, location string) int64 {
	loc, err := time.LoadLocation(location)
	var t time.Time
	var pErr error
	if err != nil {
		t, pErr = time.Parse("2006-01-02", date)
	} else {
		t, pErr = time.ParseInLocation("2006-01-02", date, loc)
	}
	if pErr != nil {
		return 0
	}
	return t.Unix()
}

func UnixMilli(t time.Time) int64 {
	return t.UnixNano() / int64(time.Millisecond/time.Nanosecond)
}

func BeginOfDate(t time.Time) time.Time {
	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
}

func Today() time.Time {
	now := time.Now()
	return BeginOfDate(now)
}

func Yesterday() time.Time {
	now := time.Now()
	diff := now.Add(-time.Hour * 24)
	return BeginOfDate(diff)
}

func DaysBeforeNow(days int64) time.Time {
	now := time.Now()
	diff := now.Add(-time.Hour * 24 * time.Duration(days))
	return BeginOfDate(diff)
}
<|endoftext|>


<commit_before>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reservefs.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package storage

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/syndtr/goleveldb/leveldb/util"
)

var errFileOpen = errors.New("leveldb/storage: file still open")

type fileLock interface {
	release() error
}

type fileStorageLock struct {
	fs *fileStorage
}

func (lock *fileStorageLock) Release() {
	fs := lock.fs
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.slock == lock {
		fs.slock = nil
	}
	return
}

// fileStorage is a file-system backed storage.
type fileStorage struct {
	path string

	mu    sync.Mutex
	flock fileLock
	slock *fileStorageLock
	logw  *os.File
	buf   []byte
	// Opened file counter; if open < 0 means closed.
	open int
}

// OpenFile returns a new filesytem-backed storage implementation with the given
// path. This also hold a file lock, so any subsequent attempt to open the same
// path will fail.
//
// The storage must be closed after use, by calling Close method.
func OpenFile(path string) (Storage, error) {
	if err := os.MkdirAll(path, 0755); err != nil {
		return nil, err
	}

	flock, err := newFileLock(filepath.Join(path, "LOCK"))
	if err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			flock.release()
		}
	}()

	rename(filepath.Join(path, "LOG"), filepath.Join(path, "LOG.old"))
	logw, err := os.OpenFile(filepath.Join(path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return nil, err
	}

	fs := &fileStorage{path: path, flock: flock, logw: logw}
	runtime.SetFinalizer(fs, (*fileStorage).Close)
	return fs, nil
}

func (fs *fileStorage) Lock() (util.Releaser, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	if fs.slock != nil {
		return nil, ErrLocked
	}
	fs.slock = &fileStorageLock{fs: fs}
	return fs.slock, nil
}

func itoa(buf []byte, i int, wid int) []byte {
	var u uint = uint(i)
	if u == 0 && wid <= 1 {
		return append(buf, '0')
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; u > 0 || wid > 0; u /= 10 {
		bp--
		wid--
		b[bp] = byte(u%10) + '0'
	}
	return append(buf, b[bp:]...)
}

func (fs *fileStorage) doLog(t time.Time, str string) {
	year, month, day := t.Date()
	hour, min, sec := t.Clock()
	msec := t.Nanosecond() / 1e3
	// date
	fs.buf = itoa(fs.buf[:0], year, 4)
	fs.buf = append(fs.buf, '/')
	fs.buf = itoa(fs.buf, int(month), 2)
	fs.buf = append(fs.buf, '/')
	fs.buf = itoa(fs.buf, day, 4)
	fs.buf = append(fs.buf, ' ')
	// time
	fs.buf = itoa(fs.buf, hour, 2)
	fs.buf = append(fs.buf, ':')
	fs.buf = itoa(fs.buf, min, 2)
	fs.buf = append(fs.buf, ':')
	fs.buf = itoa(fs.buf, sec, 2)
	fs.buf = append(fs.buf, '.')
	fs.buf = itoa(fs.buf, msec, 6)
	fs.buf = append(fs.buf, ' ')
	// write
	fs.buf = append(fs.buf, []byte(str)...)
	fs.buf = append(fs.buf, '\n')
	fs.logw.Write(fs.buf)
}

func (fs *fileStorage) Log(str string) {
	t := time.Now()
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return
	}
	fs.doLog(t, str)
}

func (fs *fileStorage) log(str string) {
	fs.doLog(time.Now(), str)
}

func (fs *fileStorage) GetFile(num uint64, t FileType) File {
	return &file{fs: fs, num: num, t: t}
}

func (fs *fileStorage) GetFiles(t FileType) (ff []File, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	dir, err := os.Open(fs.path)
	if err != nil {
		return
	}
	fnn, err := dir.Readdirnames(0)
	// Close the dir first before checking for Readdirnames error.
	if err := dir.Close(); err != nil {
		fs.log(fmt.Sprintf("close dir: %v", err))
	}
	if err != nil {
		return
	}
	f := &file{fs: fs}
	for _, fn := range fnn {
		if f.parse(fn) && (f.t&t) != 0 {
			ff = append(ff, f)
			f = &file{fs: fs}
		}
	}
	return
}

func (fs *fileStorage) GetManifest() (f File, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	dir, err := os.Open(fs.path)
	if err != nil {
		return
	}
	fnn, err := dir.Readdirnames(0)
	// Close the dir first before checking for Readdirnames error.
	if err := dir.Close(); err != nil {
		fs.log(fmt.Sprintf("close dir: %v", err))
	}
	if err != nil {
		return
	}
	// Find latest CURRENT file.
	var rem []string
	var pend bool
	var cerr error
	for _, fn := range fnn {
		if strings.HasPrefix(fn, "CURRENT") {
			pend1 := len(fn) > 7
			// Make sure it is valid name for a CURRENT file, otherwise skip it.
			if pend1 {
				if fn[7] != '.' || len(fn) < 9 {
					fs.log(fmt.Sprintf("skipping %s: invalid file name", fn))
					continue
				}
				if _, e1 := strconv.ParseUint(fn[7:], 10, 0); e1 != nil {
					fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", fn, e1))
					continue
				}
			}
			path := filepath.Join(fs.path, fn)
			r, e1 := os.OpenFile(path, os.O_RDONLY, 0)
			if e1 != nil {
				return nil, e1
			}
			b, e1 := ioutil.ReadAll(r)
			if e1 != nil {
				r.Close()
				return nil, e1
			}
			f1 := &file{fs: fs}
			if len(b) < 1 || b[len(b)-1] != '\n' || !f1.parse(string(b[:len(b)-1])) {
				fs.log(fmt.Sprintf("skipping %s: corrupted or incomplete", fn))
				if pend1 {
					rem = append(rem, fn)
				}
				if !pend1 || cerr == nil {
					cerr = fmt.Errorf("leveldb/storage: corrupted or incomplete %s file", fn)
				}
			} else if f != nil && f1.Num() < f.Num() {
				fs.log(fmt.Sprintf("skipping %s: obsolete", fn))
				if pend1 {
					rem = append(rem, fn)
				}
			} else {
				f = f1
				pend = pend1
			}
			if err := r.Close(); err != nil {
				fs.log(fmt.Sprintf("close %s: %v", fn, err))
			}
		}
	}
	// Don't remove any files if there is no valid CURRENT file.
	if f == nil {
		if cerr != nil {
			err = cerr
		} else {
			err = os.ErrNotExist
		}
		return
	}
	// Rename pending CURRENT file to an effective CURRENT.
	if pend {
		path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f.Num())
		if err := rename(path, filepath.Join(fs.path, "CURRENT")); err != nil {
			fs.log(fmt.Sprintf("CURRENT.%d -> CURRENT: %d", f.Num(), err))
		}
	}
	// Remove obsolete or incomplete pending CURRENT files.
	for _, fn := range rem {
		path := filepath.Join(fs.path, fn)
		if err := os.Remove(path); err != nil {
			fs.log(fmt.Sprintf("remove %s: %v", fn, err))
		}
	}
	return
}

func (fs *fileStorage) SetManifest(f File) (err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return ErrClosed
	}
	f2, ok := f.(*file)
	if !ok || f2.t != TypeManifest {
		return ErrInvalidFile
	}
	defer func() {
		if err != nil {
			fs.log(fmt.Sprintf("CURRENT: %v", err))
		}
	}()
	path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f2.Num())
	w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintln(w, f2.name())
	// Close the file first.
	if err := w.Close(); err != nil {
		fs.log(fmt.Sprintf("close CURRENT.%d: %v", f2.num, err))
	}
	if err != nil {
		return err
	}
	err = rename(path, filepath.Join(fs.path, "CURRENT"))
	return
}

func (fs *fileStorage) Close() error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return ErrClosed
	}
	// Clear the finalizer.
	runtime.SetFinalizer(fs, nil)

	if fs.open > 0 {
		fs.log(fmt.Sprintf("refuse to close, %d files still open", fs.open))
		return fmt.Errorf("leveldb/storage: cannot close, %d files still open", fs.open)
	}
	fs.open = -1
	e1 := fs.logw.Close()
	err := fs.flock.release()
	if err == nil {
		err = e1
	}
	return err
}

type fileWrap struct {
	*os.File
	f *file
}

func (fw fileWrap) Close() error {
	f := fw.f
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if !f.open {
		return ErrClosed
	}
	f.open = false
	f.fs.open--
	err := fw.File.Close()
	if err != nil {
		f.fs.log(fmt.Sprint("close %s.%d: %v", f.Type(), f.Num(), err))
	}
	return err
}

type file struct {
	fs   *fileStorage
	num  uint64
	t    FileType
	open bool
}

func (f *file) Open() (Reader, error) {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return nil, ErrClosed
	}
	if f.open {
		return nil, errFileOpen
	}
	of, err := os.OpenFile(f.path(), os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	f.open = true
	f.fs.open++
	return fileWrap{of, f}, nil
}

func (f *file) Create() (Writer, error) {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return nil, ErrClosed
	}
	if f.open {
		return nil, errFileOpen
	}
	of, err := os.OpenFile(f.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return nil, err
	}
	f.open = true
	f.fs.open++
	return fileWrap{of, f}, nil
}

func (f *file) Type() FileType {
	return f.t
}

func (f *file) Num() uint64 {
	return f.num
}

func (f *file) Remove() error {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return ErrClosed
	}
	if f.open {
		return errFileOpen
	}
	err := os.Remove(f.path())
	if err != nil {
		f.fs.log(fmt.Sprint("remove %s.%d: %v", f.Type(), f.Num(), err))
	}
	return err
}

func (f *file) name() string {
	switch f.t {
	case TypeManifest:
		return fmt.Sprintf("MANIFEST-%06d", f.num)
	case TypeJournal:
		return fmt.Sprintf("%06d.log", f.num)
	case TypeTable:
		return fmt.Sprintf("%06d.sst", f.num)
	default:
		panic("invalid file type")
	}
	return ""
}

func (f *file) path() string {
	return filepath.Join(f.fs.path, f.name())
}

func (f *file) parse(name string) bool {
	var num uint64
	var tail string
	_, err := fmt.Sscanf(name, "%d.%s", &num, &tail)
	if err == nil {
		switch tail {
		case "log":
			f.t = TypeJournal
		case "sst":
			f.t = TypeTable
		default:
			return false
		}
		f.num = num
		return true
	}
	n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &num, &tail)
	if n == 1 {
		f.t = TypeManifest
		f.num = num
		return true
	}

	return false
}
<commit_msg>storage: fileStorage: Shorter logging prefix<commit_after>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reservefs.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package storage

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/syndtr/goleveldb/leveldb/util"
)

var errFileOpen = errors.New("leveldb/storage: file still open")

type fileLock interface {
	release() error
}

type fileStorageLock struct {
	fs *fileStorage
}

func (lock *fileStorageLock) Release() {
	fs := lock.fs
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.slock == lock {
		fs.slock = nil
	}
	return
}

// fileStorage is a file-system backed storage.
type fileStorage struct {
	path string

	mu    sync.Mutex
	flock fileLock
	slock *fileStorageLock
	logw  *os.File
	buf   []byte
	// Opened file counter; if open < 0 means closed.
	open int
	day  int
}

// OpenFile returns a new filesytem-backed storage implementation with the given
// path. This also hold a file lock, so any subsequent attempt to open the same
// path will fail.
//
// The storage must be closed after use, by calling Close method.
func OpenFile(path string) (Storage, error) {
	if err := os.MkdirAll(path, 0755); err != nil {
		return nil, err
	}

	flock, err := newFileLock(filepath.Join(path, "LOCK"))
	if err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			flock.release()
		}
	}()

	rename(filepath.Join(path, "LOG"), filepath.Join(path, "LOG.old"))
	logw, err := os.OpenFile(filepath.Join(path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return nil, err
	}

	fs := &fileStorage{path: path, flock: flock, logw: logw}
	runtime.SetFinalizer(fs, (*fileStorage).Close)
	return fs, nil
}

func (fs *fileStorage) Lock() (util.Releaser, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	if fs.slock != nil {
		return nil, ErrLocked
	}
	fs.slock = &fileStorageLock{fs: fs}
	return fs.slock, nil
}

func itoa(buf []byte, i int, wid int) []byte {
	var u uint = uint(i)
	if u == 0 && wid <= 1 {
		return append(buf, '0')
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; u > 0 || wid > 0; u /= 10 {
		bp--
		wid--
		b[bp] = byte(u%10) + '0'
	}
	return append(buf, b[bp:]...)
}

func (fs *fileStorage) printDay(t time.Time) {
	if fs.day == t.Day() {
		return
	}
	fs.day = t.Day()
	fs.logw.Write([]byte("=============== " + t.Format("Jan 2, 2006 (MST)") + " ===============\n"))
}

func (fs *fileStorage) doLog(t time.Time, str string) {
	fs.printDay(t)
	hour, min, sec := t.Clock()
	msec := t.Nanosecond() / 1e3
	// time
	fs.buf = itoa(fs.buf[:0], hour, 2)
	fs.buf = append(fs.buf, ':')
	fs.buf = itoa(fs.buf, min, 2)
	fs.buf = append(fs.buf, ':')
	fs.buf = itoa(fs.buf, sec, 2)
	fs.buf = append(fs.buf, '.')
	fs.buf = itoa(fs.buf, msec, 6)
	fs.buf = append(fs.buf, ' ')
	// write
	fs.buf = append(fs.buf, []byte(str)...)
	fs.buf = append(fs.buf, '\n')
	fs.logw.Write(fs.buf)
}

func (fs *fileStorage) Log(str string) {
	t := time.Now()
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return
	}
	fs.doLog(t, str)
}

func (fs *fileStorage) log(str string) {
	fs.doLog(time.Now(), str)
}

func (fs *fileStorage) GetFile(num uint64, t FileType) File {
	return &file{fs: fs, num: num, t: t}
}

func (fs *fileStorage) GetFiles(t FileType) (ff []File, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	dir, err := os.Open(fs.path)
	if err != nil {
		return
	}
	fnn, err := dir.Readdirnames(0)
	// Close the dir first before checking for Readdirnames error.
	if err := dir.Close(); err != nil {
		fs.log(fmt.Sprintf("close dir: %v", err))
	}
	if err != nil {
		return
	}
	f := &file{fs: fs}
	for _, fn := range fnn {
		if f.parse(fn) && (f.t&t) != 0 {
			ff = append(ff, f)
			f = &file{fs: fs}
		}
	}
	return
}

func (fs *fileStorage) GetManifest() (f File, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	dir, err := os.Open(fs.path)
	if err != nil {
		return
	}
	fnn, err := dir.Readdirnames(0)
	// Close the dir first before checking for Readdirnames error.
	if err := dir.Close(); err != nil {
		fs.log(fmt.Sprintf("close dir: %v", err))
	}
	if err != nil {
		return
	}
	// Find latest CURRENT file.
	var rem []string
	var pend bool
	var cerr error
	for _, fn := range fnn {
		if strings.HasPrefix(fn, "CURRENT") {
			pend1 := len(fn) > 7
			// Make sure it is valid name for a CURRENT file, otherwise skip it.
			if pend1 {
				if fn[7] != '.' || len(fn) < 9 {
					fs.log(fmt.Sprintf("skipping %s: invalid file name", fn))
					continue
				}
				if _, e1 := strconv.ParseUint(fn[7:], 10, 0); e1 != nil {
					fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", fn, e1))
					continue
				}
			}
			path := filepath.Join(fs.path, fn)
			r, e1 := os.OpenFile(path, os.O_RDONLY, 0)
			if e1 != nil {
				return nil, e1
			}
			b, e1 := ioutil.ReadAll(r)
			if e1 != nil {
				r.Close()
				return nil, e1
			}
			f1 := &file{fs: fs}
			if len(b) < 1 || b[len(b)-1] != '\n' || !f1.parse(string(b[:len(b)-1])) {
				fs.log(fmt.Sprintf("skipping %s: corrupted or incomplete", fn))
				if pend1 {
					rem = append(rem, fn)
				}
				if !pend1 || cerr == nil {
					cerr = fmt.Errorf("leveldb/storage: corrupted or incomplete %s file", fn)
				}
			} else if f != nil && f1.Num() < f.Num() {
				fs.log(fmt.Sprintf("skipping %s: obsolete", fn))
				if pend1 {
					rem = append(rem, fn)
				}
			} else {
				f = f1
				pend = pend1
			}
			if err := r.Close(); err != nil {
				fs.log(fmt.Sprintf("close %s: %v", fn, err))
			}
		}
	}
	// Don't remove any files if there is no valid CURRENT file.
	if f == nil {
		if cerr != nil {
			err = cerr
		} else {
			err = os.ErrNotExist
		}
		return
	}
	// Rename pending CURRENT file to an effective CURRENT.
	if pend {
		path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f.Num())
		if err := rename(path, filepath.Join(fs.path, "CURRENT")); err != nil {
			fs.log(fmt.Sprintf("CURRENT.%d -> CURRENT: %d", f.Num(), err))
		}
	}
	// Remove obsolete or incomplete pending CURRENT files.
	for _, fn := range rem {
		path := filepath.Join(fs.path, fn)
		if err := os.Remove(path); err != nil {
			fs.log(fmt.Sprintf("remove %s: %v", fn, err))
		}
	}
	return
}

func (fs *fileStorage) SetManifest(f File) (err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return ErrClosed
	}
	f2, ok := f.(*file)
	if !ok || f2.t != TypeManifest {
		return ErrInvalidFile
	}
	defer func() {
		if err != nil {
			fs.log(fmt.Sprintf("CURRENT: %v", err))
		}
	}()
	path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f2.Num())
	w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintln(w, f2.name())
	// Close the file first.
	if err := w.Close(); err != nil {
		fs.log(fmt.Sprintf("close CURRENT.%d: %v", f2.num, err))
	}
	if err != nil {
		return err
	}
	err = rename(path, filepath.Join(fs.path, "CURRENT"))
	return
}

func (fs *fileStorage) Close() error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return ErrClosed
	}
	// Clear the finalizer.
	runtime.SetFinalizer(fs, nil)

	if fs.open > 0 {
		fs.log(fmt.Sprintf("refuse to close, %d files still open", fs.open))
		return fmt.Errorf("leveldb/storage: cannot close, %d files still open", fs.open)
	}
	fs.open = -1
	e1 := fs.logw.Close()
	err := fs.flock.release()
	if err == nil {
		err = e1
	}
	return err
}

type fileWrap struct {
	*os.File
	f *file
}

func (fw fileWrap) Close() error {
	f := fw.f
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if !f.open {
		return ErrClosed
	}
	f.open = false
	f.fs.open--
	err := fw.File.Close()
	if err != nil {
		f.fs.log(fmt.Sprint("close %s.%d: %v", f.Type(), f.Num(), err))
	}
	return err
}

type file struct {
	fs   *fileStorage
	num  uint64
	t    FileType
	open bool
}

func (f *file) Open() (Reader, error) {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return nil, ErrClosed
	}
	if f.open {
		return nil, errFileOpen
	}
	of, err := os.OpenFile(f.path(), os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	f.open = true
	f.fs.open++
	return fileWrap{of, f}, nil
}

func (f *file) Create() (Writer, error) {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return nil, ErrClosed
	}
	if f.open {
		return nil, errFileOpen
	}
	of, err := os.OpenFile(f.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return nil, err
	}
	f.open = true
	f.fs.open++
	return fileWrap{of, f}, nil
}

func (f *file) Type() FileType {
	return f.t
}

func (f *file) Num() uint64 {
	return f.num
}

func (f *file) Remove() error {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return ErrClosed
	}
	if f.open {
		return errFileOpen
	}
	err := os.Remove(f.path())
	if err != nil {
		f.fs.log(fmt.Sprint("remove %s.%d: %v", f.Type(), f.Num(), err))
	}
	return err
}

func (f *file) name() string {
	switch f.t {
	case TypeManifest:
		return fmt.Sprintf("MANIFEST-%06d", f.num)
	case TypeJournal:
		return fmt.Sprintf("%06d.log", f.num)
	case TypeTable:
		return fmt.Sprintf("%06d.sst", f.num)
	default:
		panic("invalid file type")
	}
	return ""
}

func (f *file) path() string {
	return filepath.Join(f.fs.path, f.name())
}

func (f *file) parse(name string) bool {
	var num uint64
	var tail string
	_, err := fmt.Sscanf(name, "%d.%s", &num, &tail)
	if err == nil {
		switch tail {
		case "log":
			f.t = TypeJournal
		case "sst":
			f.t = TypeTable
		default:
			return false
		}
		f.num = num
		return true
	}
	n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &num, &tail)
	if n == 1 {
		f.t = TypeManifest
		f.num = num
		return true
	}

	return false
}
<|endoftext|>