<commit_before>package weed_server

import (
	"code.google.com/p/weed-fs/go/glog"
	"code.google.com/p/weed-fs/go/images"
	"code.google.com/p/weed-fs/go/operation"
	"code.google.com/p/weed-fs/go/stats"
	"code.google.com/p/weed-fs/go/storage"
	"code.google.com/p/weed-fs/go/topology"
	"io"
	"mime"
	"mime/multipart"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

func (vs *VolumeServer) storeHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		stats.ReadRequest()
		vs.GetOrHeadHandler(w, r)
	case "HEAD":
		stats.ReadRequest()
		vs.GetOrHeadHandler(w, r)
	case "DELETE":
		stats.DeleteRequest()
		secure(vs.whiteList, vs.DeleteHandler)(w, r)
	case "PUT":
		stats.WriteRequest()
		secure(vs.whiteList, vs.PostHandler)(w, r)
	case "POST":
		stats.WriteRequest()
		secure(vs.whiteList, vs.PostHandler)(w, r)
	}
}

func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, filename, ext, _ := parseURLPath(r.URL.Path)
	volumeId, err := storage.NewVolumeId(vid)
	if err != nil {
		glog.V(2).Infoln("parsing error:", err, r.URL.Path)
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	err = n.ParsePath(fid)
	if err != nil {
		glog.V(2).Infoln("parsing fid error:", err, r.URL.Path)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	glog.V(4).Infoln("volume", volumeId, "reading", n)
	if !vs.store.HasVolume(volumeId) {
		lookupResult, err := operation.Lookup(vs.masterNode, volumeId.String())
		glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err)
		if err == nil && len(lookupResult.Locations) > 0 {
			http.Redirect(w, r, "http://"+lookupResult.Locations[0].PublicUrl+r.URL.Path, http.StatusMovedPermanently)
		} else {
			glog.V(2).Infoln("lookup error:", err, r.URL.Path)
			w.WriteHeader(http.StatusNotFound)
		}
		return
	}
	cookie := n.Cookie
	count, e := vs.store.Read(volumeId, n)
	glog.V(4).Infoln("read bytes", count, "error", e)
	if e != nil || count <= 0 {
		glog.V(0).Infoln("read error:", e, r.URL.Path)
		w.WriteHeader(http.StatusNotFound)
		return
	}
	if n.Cookie != cookie {
		glog.V(0).Infoln("request", r.URL.Path, "with unmaching cookie seen:", cookie, "expected:", n.Cookie, "from", r.RemoteAddr, "agent", r.UserAgent())
		w.WriteHeader(http.StatusNotFound)
		return
	}
	if n.LastModified != 0 {
		w.Header().Set("Last-Modified", time.Unix(int64(n.LastModified), 0).UTC().Format(http.TimeFormat))
		if r.Header.Get("If-Modified-Since") != "" {
			if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil {
				if t.Unix() >= int64(n.LastModified) {
					w.WriteHeader(http.StatusNotModified)
					return
				}
			}
		}
	}
	if n.NameSize > 0 && filename == "" {
		filename = string(n.Name)
		dotIndex := strings.LastIndex(filename, ".")
		if dotIndex > 0 {
			ext = filename[dotIndex:]
		}
	}
	mtype := ""
	if ext != "" {
		mtype = mime.TypeByExtension(ext)
	}
	if n.MimeSize > 0 {
		mtype = string(n.Mime)
	}
	if mtype != "" {
		w.Header().Set("Content-Type", mtype)
	}
	if filename != "" {
		w.Header().Set("Content-Disposition", "filename=\""+fileNameEscaper.Replace(filename)+"\"")
	}
	if ext != ".gz" {
		if n.IsGzipped() {
			if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
				w.Header().Set("Content-Encoding", "gzip")
			} else {
				if n.Data, err = storage.UnGzipData(n.Data); err != nil {
					glog.V(0).Infoln("lookup error:", err, r.URL.Path)
				}
			}
		}
	}
	if ext == ".png" || ext == ".jpg" || ext == ".gif" {
		width, height := 0, 0
		if r.FormValue("width") != "" {
			width, _ = strconv.Atoi(r.FormValue("width"))
		}
		if r.FormValue("height") != "" {
			height, _ = strconv.Atoi(r.FormValue("height"))
		}
		n.Data = images.Resized(ext, n.Data, width, height)
	}

	w.Header().Set("Accept-Ranges", "bytes")
	if r.Method == "HEAD" {
		w.Header().Set("Content-Length", strconv.Itoa(len(n.Data)))
		return
	}
	rangeReq := r.Header.Get("Range")
	if rangeReq == "" {
		w.Header().Set("Content-Length", strconv.Itoa(len(n.Data)))
		if _, e = w.Write(n.Data); e != nil {
			glog.V(0).Infoln("response write error:", e)
		}
		return
	}

	//the rest is dealing with partial content request
	//mostly copy from src/pkg/net/http/fs.go
	size := int64(len(n.Data))
	ranges, err := parseRange(rangeReq, size)
	if err != nil {
		http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
		return
	}
	if sumRangesSize(ranges) > size {
		// The total number of bytes in all the ranges
		// is larger than the size of the file by
		// itself, so this is probably an attack, or a
		// dumb client.  Ignore the range request.
		ranges = nil
		return
	}
	if len(ranges) == 0 {
		return
	}
	if len(ranges) == 1 {
		// RFC 2616, Section 14.16:
		// "When an HTTP message includes the content of a single
		// range (for example, a response to a request for a
		// single range, or to a request for a set of ranges
		// that overlap without any holes), this content is
		// transmitted with a Content-Range header, and a
		// Content-Length header showing the number of bytes
		// actually transferred.
		// ...
		// A response to a request for a single range MUST NOT
		// be sent using the multipart/byteranges media type."
		ra := ranges[0]
		w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
		w.Header().Set("Content-Range", ra.contentRange(size))
		w.WriteHeader(http.StatusPartialContent)
		if _, e = w.Write(n.Data[ra.start : ra.start+ra.length]); e != nil {
			glog.V(0).Infoln("response write error:", e)
		}
		return
	}
	// process mulitple ranges
	for _, ra := range ranges {
		if ra.start > size {
			http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable)
			return
		}
	}
	sendSize := rangesMIMESize(ranges, mtype, size)
	pr, pw := io.Pipe()
	mw := multipart.NewWriter(pw)
	w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
	sendContent := pr
	defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
	go func() {
		for _, ra := range ranges {
			part, err := mw.CreatePart(ra.mimeHeader(mtype, size))
			if err != nil {
				pw.CloseWithError(err)
				return
			}
			if _, err = part.Write(n.Data[ra.start : ra.start+ra.length]); err != nil {
				pw.CloseWithError(err)
				return
			}
		}
		mw.Close()
		pw.Close()
	}()
	if w.Header().Get("Content-Encoding") == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
	}
	w.WriteHeader(http.StatusPartialContent)
	io.CopyN(w, sendContent, sendSize)

}

func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
	if e := r.ParseForm(); e != nil {
		glog.V(0).Infoln("form parse error:", e)
		writeJsonError(w, r, e)
		return
	}
	vid, _, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, ve := storage.NewVolumeId(vid)
	if ve != nil {
		glog.V(0).Infoln("NewVolumeId error:", ve)
		writeJsonError(w, r, ve)
		return
	}
	needle, ne := storage.NewNeedle(r, vs.FixJpgOrientation)
	if ne != nil {
		writeJsonError(w, r, ne)
		return
	}

	ret := operation.UploadResult{}
	size, errorStatus := topology.ReplicatedWrite(vs.masterNode, vs.store, volumeId, needle, r)
	if errorStatus == "" {
		w.WriteHeader(http.StatusCreated)
	} else {
		w.WriteHeader(http.StatusInternalServerError)
		ret.Error = errorStatus
	}
	if needle.HasName() {
		ret.Name = string(needle.Name)
	}
	ret.Size = size
	writeJsonQuiet(w, r, ret)
}

func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, _ := storage.NewVolumeId(vid)
	n.ParsePath(fid)

	glog.V(2).Infoln("deleting", n)

	cookie := n.Cookie
	count, ok := vs.store.Read(volumeId, n)

	if ok != nil {
		m := make(map[string]uint32)
		m["size"] = 0
		writeJsonQuiet(w, r, m)
		return
	}

	if n.Cookie != cookie {
		glog.V(0).Infoln("delete", r.URL.Path, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		return
	}

	n.Size = 0
	ret := topology.ReplicatedDelete(vs.masterNode, vs.store, volumeId, n, r)

	if ret != 0 {
		w.WriteHeader(http.StatusAccepted)
	} else {
		w.WriteHeader(http.StatusInternalServerError)
	}

	m := make(map[string]uint32)
	m["size"] = uint32(count)
	writeJsonQuiet(w, r, m)
}

//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	var ret []operation.DeleteResult
	for _, fid := range r.Form["fid"] {
		vid, id_cookie, err := operation.ParseFileId(fid)
		if err != nil {
			ret = append(ret, operation.DeleteResult{Fid: fid, Error: err.Error()})
			continue
		}
		n := new(storage.Needle)
		volumeId, _ := storage.NewVolumeId(vid)
		n.ParsePath(id_cookie)
		glog.V(4).Infoln("batch deleting", n)
		cookie := n.Cookie
		if _, err := vs.store.Read(volumeId, n); err != nil {
			ret = append(ret, operation.DeleteResult{Fid: fid, Error: err.Error()})
			continue
		}
		if n.Cookie != cookie {
			ret = append(ret, operation.DeleteResult{Fid: fid, Error: "File Random Cookie does not match."})
			glog.V(0).Infoln("deleting", fid, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
			return
		}
		if size, err := vs.store.Delete(volumeId, n); err != nil {
			ret = append(ret, operation.DeleteResult{Fid: fid, Error: err.Error()})
		} else {
			ret = append(ret, operation.DeleteResult{Fid: fid, Size: int(size)})
		}
	}

	w.WriteHeader(http.StatusAccepted)

	writeJsonQuiet(w, r, ret)
}
<commit_msg>Better guessing mime type with octstream<commit_after>package weed_server

import (
	"code.google.com/p/weed-fs/go/glog"
	"code.google.com/p/weed-fs/go/images"
	"code.google.com/p/weed-fs/go/operation"
	"code.google.com/p/weed-fs/go/stats"
	"code.google.com/p/weed-fs/go/storage"
	"code.google.com/p/weed-fs/go/topology"
	"io"
	"mime"
	"mime/multipart"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

func (vs *VolumeServer) storeHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		stats.ReadRequest()
		vs.GetOrHeadHandler(w, r)
	case "HEAD":
		stats.ReadRequest()
		vs.GetOrHeadHandler(w, r)
	case "DELETE":
		stats.DeleteRequest()
		secure(vs.whiteList, vs.DeleteHandler)(w, r)
	case "PUT":
		stats.WriteRequest()
		secure(vs.whiteList, vs.PostHandler)(w, r)
	case "POST":
		stats.WriteRequest()
		secure(vs.whiteList, vs.PostHandler)(w, r)
	}
}

func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, filename, ext, _ := parseURLPath(r.URL.Path)
	volumeId, err := storage.NewVolumeId(vid)
	if err != nil {
		glog.V(2).Infoln("parsing error:", err, r.URL.Path)
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	err = n.ParsePath(fid)
	if err != nil {
		glog.V(2).Infoln("parsing fid error:", err, r.URL.Path)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	glog.V(4).Infoln("volume", volumeId, "reading", n)
	if !vs.store.HasVolume(volumeId) {
		lookupResult, err := operation.Lookup(vs.masterNode, volumeId.String())
		glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err)
		if err == nil && len(lookupResult.Locations) > 0 {
			http.Redirect(w, r, "http://"+lookupResult.Locations[0].PublicUrl+r.URL.Path, http.StatusMovedPermanently)
		} else {
			glog.V(2).Infoln("lookup error:", err, r.URL.Path)
			w.WriteHeader(http.StatusNotFound)
		}
		return
	}
	cookie := n.Cookie
	count, e := vs.store.Read(volumeId, n)
	glog.V(4).Infoln("read bytes", count, "error", e)
	if e != nil || count <= 0 {
		glog.V(0).Infoln("read error:", e, r.URL.Path)
		w.WriteHeader(http.StatusNotFound)
		return
	}
	if n.Cookie != cookie {
		glog.V(0).Infoln("request", r.URL.Path, "with unmaching cookie seen:", cookie, "expected:", n.Cookie, "from", r.RemoteAddr, "agent", r.UserAgent())
		w.WriteHeader(http.StatusNotFound)
		return
	}
	if n.LastModified != 0 {
		w.Header().Set("Last-Modified", time.Unix(int64(n.LastModified), 0).UTC().Format(http.TimeFormat))
		if r.Header.Get("If-Modified-Since") != "" {
			if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil {
				if t.Unix() >= int64(n.LastModified) {
					w.WriteHeader(http.StatusNotModified)
					return
				}
			}
		}
	}
	if n.NameSize > 0 && filename == "" {
		filename = string(n.Name)
		dotIndex := strings.LastIndex(filename, ".")
		if dotIndex > 0 {
			ext = filename[dotIndex:]
		}
	}
	mtype := ""
	if ext != "" {
		mtype = mime.TypeByExtension(ext)
	}
	if n.MimeSize > 0 {
		mt := string(n.Mime)
		if mt != "application/octet-stream" {
			mtype = mt
		}
	}
	if mtype != "" {
		w.Header().Set("Content-Type", mtype)
	}
	if filename != "" {
		w.Header().Set("Content-Disposition", "filename=\""+fileNameEscaper.Replace(filename)+"\"")
	}
	if ext != ".gz" {
		if n.IsGzipped() {
			if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
				w.Header().Set("Content-Encoding", "gzip")
			} else {
				if n.Data, err = storage.UnGzipData(n.Data); err != nil {
					glog.V(0).Infoln("lookup error:", err, r.URL.Path)
				}
			}
		}
	}
	if ext == ".png" || ext == ".jpg" || ext == ".gif" {
		width, height := 0, 0
		if r.FormValue("width") != "" {
			width, _ = strconv.Atoi(r.FormValue("width"))
		}
		if r.FormValue("height") != "" {
			height, _ = strconv.Atoi(r.FormValue("height"))
		}
		n.Data = images.Resized(ext, n.Data, width, height)
	}

	w.Header().Set("Accept-Ranges", "bytes")
	if r.Method == "HEAD" {
		w.Header().Set("Content-Length", strconv.Itoa(len(n.Data)))
		return
	}
	rangeReq := r.Header.Get("Range")
	if rangeReq == "" {
		w.Header().Set("Content-Length", strconv.Itoa(len(n.Data)))
		if _, e = w.Write(n.Data); e != nil {
			glog.V(0).Infoln("response write error:", e)
		}
		return
	}

	//the rest is dealing with partial content request
	//mostly copy from src/pkg/net/http/fs.go
	size := int64(len(n.Data))
	ranges, err := parseRange(rangeReq, size)
	if err != nil {
		http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
		return
	}
	if sumRangesSize(ranges) > size {
		// The total number of bytes in all the ranges
		// is larger than the size of the file by
		// itself, so this is probably an attack, or a
		// dumb client.  Ignore the range request.
		ranges = nil
		return
	}
	if len(ranges) == 0 {
		return
	}
	if len(ranges) == 1 {
		// RFC 2616, Section 14.16:
		// "When an HTTP message includes the content of a single
		// range (for example, a response to a request for a
		// single range, or to a request for a set of ranges
		// that overlap without any holes), this content is
		// transmitted with a Content-Range header, and a
		// Content-Length header showing the number of bytes
		// actually transferred.
		// ...
		// A response to a request for a single range MUST NOT
		// be sent using the multipart/byteranges media type."
		ra := ranges[0]
		w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
		w.Header().Set("Content-Range", ra.contentRange(size))
		w.WriteHeader(http.StatusPartialContent)
		if _, e = w.Write(n.Data[ra.start : ra.start+ra.length]); e != nil {
			glog.V(0).Infoln("response write error:", e)
		}
		return
	}
	// process mulitple ranges
	for _, ra := range ranges {
		if ra.start > size {
			http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable)
			return
		}
	}
	sendSize := rangesMIMESize(ranges, mtype, size)
	pr, pw := io.Pipe()
	mw := multipart.NewWriter(pw)
	w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
	sendContent := pr
	defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
	go func() {
		for _, ra := range ranges {
			part, err := mw.CreatePart(ra.mimeHeader(mtype, size))
			if err != nil {
				pw.CloseWithError(err)
				return
			}
			if _, err = part.Write(n.Data[ra.start : ra.start+ra.length]); err != nil {
				pw.CloseWithError(err)
				return
			}
		}
		mw.Close()
		pw.Close()
	}()
	if w.Header().Get("Content-Encoding") == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
	}
	w.WriteHeader(http.StatusPartialContent)
	io.CopyN(w, sendContent, sendSize)

}

func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
	if e := r.ParseForm(); e != nil {
		glog.V(0).Infoln("form parse error:", e)
		writeJsonError(w, r, e)
		return
	}
	vid, _, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, ve := storage.NewVolumeId(vid)
	if ve != nil {
		glog.V(0).Infoln("NewVolumeId error:", ve)
		writeJsonError(w, r, ve)
		return
	}
	needle, ne := storage.NewNeedle(r, vs.FixJpgOrientation)
	if ne != nil {
		writeJsonError(w, r, ne)
		return
	}

	ret := operation.UploadResult{}
	size, errorStatus := topology.ReplicatedWrite(vs.masterNode, vs.store, volumeId, needle, r)
	if errorStatus == "" {
		w.WriteHeader(http.StatusCreated)
	} else {
		w.WriteHeader(http.StatusInternalServerError)
		ret.Error = errorStatus
	}
	if needle.HasName() {
		ret.Name = string(needle.Name)
	}
	ret.Size = size
	writeJsonQuiet(w, r, ret)
}

func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, _ := storage.NewVolumeId(vid)
	n.ParsePath(fid)

	glog.V(2).Infoln("deleting", n)

	cookie := n.Cookie
	count, ok := vs.store.Read(volumeId, n)

	if ok != nil {
		m := make(map[string]uint32)
		m["size"] = 0
		writeJsonQuiet(w, r, m)
		return
	}

	if n.Cookie != cookie {
		glog.V(0).Infoln("delete", r.URL.Path, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		return
	}

	n.Size = 0
	ret := topology.ReplicatedDelete(vs.masterNode, vs.store, volumeId, n, r)

	if ret != 0 {
		w.WriteHeader(http.StatusAccepted)
	} else {
		w.WriteHeader(http.StatusInternalServerError)
	}

	m := make(map[string]uint32)
	m["size"] = uint32(count)
	writeJsonQuiet(w, r, m)
}

//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	var ret []operation.DeleteResult
	for _, fid := range r.Form["fid"] {
		vid, id_cookie, err := operation.ParseFileId(fid)
		if err != nil {
			ret = append(ret, operation.DeleteResult{Fid: fid, Error: err.Error()})
			continue
		}
		n := new(storage.Needle)
		volumeId, _ := storage.NewVolumeId(vid)
		n.ParsePath(id_cookie)
		glog.V(4).Infoln("batch deleting", n)
		cookie := n.Cookie
		if _, err := vs.store.Read(volumeId, n); err != nil {
			ret = append(ret, operation.DeleteResult{Fid: fid, Error: err.Error()})
			continue
		}
		if n.Cookie != cookie {
			ret = append(ret, operation.DeleteResult{Fid: fid, Error: "File Random Cookie does not match."})
			glog.V(0).Infoln("deleting", fid, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
			return
		}
		if size, err := vs.store.Delete(volumeId, n); err != nil {
			ret = append(ret, operation.DeleteResult{Fid: fid, Error: err.Error()})
		} else {
			ret = append(ret, operation.DeleteResult{Fid: fid, Size: int(size)})
		}
	}

	w.WriteHeader(http.StatusAccepted)

	writeJsonQuiet(w, r, ret)
}
<|endoftext|>


<commit_before>package events

import (
	"log"

	"github.com/marpaia/graphite-golang"
	"github.com/wanelo/image-server/core"
	"github.com/wanelo/image-server/uploader"
)

// InitializeEventListeners activates all workers that listen to events
func InitializeEventListeners(sc *core.ServerConfiguration) {
	uwc := uploader.UploadWorkers(sc.UploaderConcurrency)
	g := initializeGraphite(sc)
	go handleImageProcessed(sc, uwc, g)
	go handleImageProcessedWithErrors(sc, g)
	go handleOriginalDownloaded(sc, uwc, g)
	go handleOriginalDownloadUnavailable(sc, g)
}

func handleImageProcessed(sc *core.ServerConfiguration, uwc chan *uploader.UploadWork, g *graphite.Graphite) {
	for {
		ic := <-sc.Events.ImageProcessed
		resizedPath := ic.LocalResizedImagePath()
		log.Printf("Processed image: %s", resizedPath)

		uwc <- &uploader.UploadWork{ic, sc.Adapters.Uploader.Upload}
		g.SimpleSend("stats.image_server.image_request", "1")
		g.SimpleSend("stats.image_server.image_request."+ic.Format, "1")
	}
}

func handleImageProcessedWithErrors(sc *core.ServerConfiguration, g *graphite.Graphite) {
	for {
		_ = <-sc.Events.ImageProcessedWithErrors
		g.SimpleSend("stats.image_server.image_request_fail", "1")
	}
}

func handleOriginalDownloaded(sc *core.ServerConfiguration, uwc chan *uploader.UploadWork, g *graphite.Graphite) {
	for {
		ic := <-sc.Events.OriginalDownloaded
		uwc <- &uploader.UploadWork{ic, sc.Adapters.Uploader.UploadOriginal}
		g.SimpleSend("stats.image_server.original_downloaded", "1")
	}
}

func handleOriginalDownloadUnavailable(sc *core.ServerConfiguration, g *graphite.Graphite) {
	for {
		_ = <-sc.Events.OriginalDownloaded
		g.SimpleSend("stats.image_server.original_unavailable", "1")
	}
}
<commit_msg>Makes graphite package explicit on import<commit_after>package events

import (
	"log"

	graphite "github.com/marpaia/graphite-golang"
	"github.com/wanelo/image-server/core"
	"github.com/wanelo/image-server/uploader"
)

// InitializeEventListeners activates all workers that listen to events
func InitializeEventListeners(sc *core.ServerConfiguration) {
	uwc := uploader.UploadWorkers(sc.UploaderConcurrency)
	g := initializeGraphite(sc)
	go handleImageProcessed(sc, uwc, g)
	go handleImageProcessedWithErrors(sc, g)
	go handleOriginalDownloaded(sc, uwc, g)
	go handleOriginalDownloadUnavailable(sc, g)
}

func handleImageProcessed(sc *core.ServerConfiguration, uwc chan *uploader.UploadWork, g *graphite.Graphite) {
	for {
		ic := <-sc.Events.ImageProcessed
		resizedPath := ic.LocalResizedImagePath()
		log.Printf("Processed image: %s", resizedPath)

		uwc <- &uploader.UploadWork{ic, sc.Adapters.Uploader.Upload}
		g.SimpleSend("stats.image_server.image_request", "1")
		g.SimpleSend("stats.image_server.image_request."+ic.Format, "1")
	}
}

func handleImageProcessedWithErrors(sc *core.ServerConfiguration, g *graphite.Graphite) {
	for {
		_ = <-sc.Events.ImageProcessedWithErrors
		g.SimpleSend("stats.image_server.image_request_fail", "1")
	}
}

func handleOriginalDownloaded(sc *core.ServerConfiguration, uwc chan *uploader.UploadWork, g *graphite.Graphite) {
	for {
		ic := <-sc.Events.OriginalDownloaded
		uwc <- &uploader.UploadWork{ic, sc.Adapters.Uploader.UploadOriginal}
		g.SimpleSend("stats.image_server.original_downloaded", "1")
	}
}

func handleOriginalDownloadUnavailable(sc *core.ServerConfiguration, g *graphite.Graphite) {
	for {
		_ = <-sc.Events.OriginalDownloaded
		g.SimpleSend("stats.image_server.original_unavailable", "1")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"time"
)

// Service : Model representing ernest.io service json responses
type Service struct {
	ID         string    `json:"id"`
	Name       string    `json:"name"`
	Datacenter int       `json:"datacenter_id"`
	Version    time.Time `json:"version"`
	Status     string    `json:"status"`
	Definition string    `json:"definition"`
	Result     string    `json:"result"`
	Endpoint   string    `json:"endpoint"`
	Networks   []struct {
		Name   string `json:"name"`
		Subnet string `json:"network_aws_id"`
	} `json:"networks"`
	Instances []struct {
		Name          string `json:"name"`
		InstanceAWSID string `json:"instance_aws_id"`
		PublicIP      string `json:"public_ip"`
		IP            string `json:"ip"`
	} `json:"instances"`
	Nats []struct {
		Name            string `json:"name"`
		NatGatewayAWSID string `json:"nat_gateway_aws_id"`
	} `json:"nats"`
	SecurityGroups []struct {
		Name               string `json:"name"`
		SecurityGroupAWSID string `json:"security_group_aws_id"`
	} `json:"nats"`
}
<commit_msg>correct json field for security_groups<commit_after>package main

import (
	"time"
)

// Service : Model representing ernest.io service json responses
type Service struct {
	ID         string    `json:"id"`
	Name       string    `json:"name"`
	Datacenter int       `json:"datacenter_id"`
	Version    time.Time `json:"version"`
	Status     string    `json:"status"`
	Definition string    `json:"definition"`
	Result     string    `json:"result"`
	Endpoint   string    `json:"endpoint"`
	Networks   []struct {
		Name   string `json:"name"`
		Subnet string `json:"network_aws_id"`
	} `json:"networks"`
	Instances []struct {
		Name          string `json:"name"`
		InstanceAWSID string `json:"instance_aws_id"`
		PublicIP      string `json:"public_ip"`
		IP            string `json:"ip"`
	} `json:"instances"`
	Nats []struct {
		Name            string `json:"name"`
		NatGatewayAWSID string `json:"nat_gateway_aws_id"`
	} `json:"nats"`
	SecurityGroups []struct {
		Name               string `json:"name"`
		SecurityGroupAWSID string `json:"security_group_aws_id"`
	} `json:"security_groups"`
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"

	"image"
	"image/draw"
	"image/png"
)

type Size struct {
	x, y int
}

func (a Size) Equal(b Size) bool {
	return a.x == b.x && a.y == b.y
}

func (a Size) Larger(b Size) bool {
	return b.x < a.x && b.y < a.y
}

func (a Size) Smaller(b Size) bool {
	return !a.Larger(b)
}

type sprite struct {
	name string
	img  image.Image
	rect image.Rectangle
	area int
	size Size
}

type ByArea []sprite

func (a ByArea) Len() int           { return len(a) }
func (a ByArea) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByArea) Less(i, j int) bool { return a[i].area < a[j].area }

type Node struct {
	child [2]*Node
	rect  image.Rectangle
	img   *sprite
}

func (n *Node) size() Size {
	return Size{n.rect.Dx(), n.rect.Dy()}
}

func (n *Node) print() {
	fmt.Println(n)
	if n.child[0] != nil {
		n.child[0].print()
	}
	if n.child[1] != nil {
		n.child[1].print()
	}
}

func (n *Node) isLeaf() bool {
	return n.child[0] == nil || n.child[1] == nil
}

func (n *Node) insert(img *sprite) *Node {
	if !n.isLeaf() {
		node := n.child[0].insert(img)

		if node != nil {
			return node
		} else {
			return n.child[1].insert(img)
		}
	}

	// there is already an image in this node
	if n.img != nil {
		return nil
	}

	// space too small
	if n.rect.Dx() < img.size.x || n.rect.Dy() < img.size.y {
		return nil
	}

	// just right
	if n.rect.Dx() == img.size.x && n.rect.Dy() == img.size.y {
		n.img = img
		return n
	}

	// the space that is left will be large enough to split
	n.split(img)

	return n.child[0].insert(img)
}

func (n *Node) split(img *sprite) {
	dx := n.rect.Dx() - img.size.x
	dy := n.rect.Dy() - img.size.y

	rc := n.rect

	if dx > dy {
		n.child[0] = &Node{rect: image.Rect(
			rc.Min.X,
			rc.Min.Y,
			rc.Min.X+img.size.x,
			rc.Max.Y)}
		n.child[1] = &Node{rect: image.Rect(
			rc.Min.X+img.size.x,
			rc.Min.Y,
			rc.Max.X,
			rc.Max.Y)}
	} else {
		n.child[0] = &Node{rect: image.Rect(
			rc.Min.X,
			rc.Min.Y,
			rc.Max.X,
			rc.Min.Y+img.size.y)}
		n.child[1] = &Node{rect: image.Rect(
			rc.Min.X,
			rc.Min.Y+img.size.y,
			rc.Max.X,
			rc.Max.Y)}
	}

}

func main() {
	flag.Parse()

	args := flag.Args()

	inputDir := args[0]

	files, _ := ioutil.ReadDir(inputDir)
	sprites := make([]sprite, len(files))

	totalX := 0
	totalY := 0

	for i := range sprites {
		s := readSprite(inputDir, files[i].Name())
		sprites[i] = s
		totalX += s.size.x
		totalY += s.size.y
	}

	// we want to place the largest sprite first
	sort.Sort(sort.Reverse(ByArea(sprites)))

	// the final image
	dst := image.NewRGBA(image.Rect(0, 0, 2048, 2048))

	n := Node{rect: image.Rect(0, 0, 2048, 2048)}

	for i := range sprites {
		s := &sprites[i]
		node := n.insert(s)
		if node != nil {
			draw.Draw(dst, node.rect, s.img, image.ZP, draw.Src)
		}
	}

	//n.print()

	writer, err := os.Create("test.png")
	err = png.Encode(writer, dst)
	if err != nil {
		log.Fatal(err)
	}
}

func readSprite(dir, name string) (s sprite) {
	path := path.Join(dir, name)
	reader, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()

	img, err := png.Decode(reader)
	if err != nil {
		log.Fatal(err)
	}

	s.name = name
	s.img = img
	s.rect = img.Bounds()
	s.size = Size{s.rect.Dx(), s.rect.Dy()}
	s.area = s.size.x * s.size.y
	return
}
<commit_msg>Some code cleanup<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"

	"image"
	"image/draw"
	"image/png"
)

type Size struct {
	x, y int
}

func (a Size) Equal(b Size) bool {
	return a.x == b.x && a.y == b.y
}

func (a Size) Larger(b Size) bool {
	return b.x < a.x && b.y < a.y
}

func (a Size) Smaller(b Size) bool {
	return !a.Larger(b)
}

type sprite struct {
	name string
	img  image.Image
	rect image.Rectangle
	area int
	size Size
}

type ByArea []sprite

func (a ByArea) Len() int           { return len(a) }
func (a ByArea) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByArea) Less(i, j int) bool { return a[i].area < a[j].area }

type Node struct {
	child [2]*Node
	rect  image.Rectangle
	img   *sprite
}

func (n *Node) size() Size {
	return Size{n.rect.Dx(), n.rect.Dy()}
}

func (n *Node) print() {
	fmt.Println(n)
	if n.child[0] != nil {
		n.child[0].print()
	}
	if n.child[1] != nil {
		n.child[1].print()
	}
}

func (n *Node) isLeaf() bool {
	return n.child[0] == nil || n.child[1] == nil
}

func (n *Node) insert(img *sprite) *Node {
	if !n.isLeaf() {
		node := n.child[0].insert(img)

		if node != nil {
			return node
		} else {
			return n.child[1].insert(img)
		}
	}

	// there is already an image in this node
	if n.img != nil {
		return nil
	}

	// space too small
	if n.rect.Dx() < img.size.x || n.rect.Dy() < img.size.y {
		return nil
	}

	// just right
	if n.rect.Dx() == img.size.x && n.rect.Dy() == img.size.y {
		n.img = img
		return n
	}

	// the space that is left will be large enough to split
	n.split(img)

	return n.child[0].insert(img)
}

func (n *Node) split(img *sprite) {
	dx := n.rect.Dx() - img.size.x
	dy := n.rect.Dy() - img.size.y

	if dx > dy {
		n.child[0] = &Node{rect: image.Rect(
			n.rect.Min.X,
			n.rect.Min.Y,
			n.rect.Min.X+img.size.x,
			n.rect.Max.Y)}
		n.child[1] = &Node{rect: image.Rect(
			n.rect.Min.X+img.size.x,
			n.rect.Min.Y,
			n.rect.Max.X,
			n.rect.Max.Y)}
	} else {
		n.child[0] = &Node{rect: image.Rect(
			n.rect.Min.X,
			n.rect.Min.Y,
			n.rect.Max.X,
			n.rect.Min.Y+img.size.y)}
		n.child[1] = &Node{rect: image.Rect(
			n.rect.Min.X,
			n.rect.Min.Y+img.size.y,
			n.rect.Max.X,
			n.rect.Max.Y)}
	}

}

func main() {
	flag.Parse()

	args := flag.Args()

	inputDir := args[0]

	files, _ := ioutil.ReadDir(inputDir)
	sprites := make([]sprite, len(files))

	for i := range sprites {
		s := readSprite(inputDir, files[i].Name())
		sprites[i] = s
	}

	// we want to place the largest sprite first
	sort.Sort(sort.Reverse(ByArea(sprites)))

	// the final image
	dst := image.NewRGBA(image.Rect(0, 0, 2048, 2048))

	n := Node{rect: image.Rect(0, 0, 2048, 2048)}

	for i := range sprites {
		s := &sprites[i]
		node := n.insert(s)
		if node != nil {
			draw.Draw(dst, node.rect, s.img, image.ZP, draw.Src)
		} else {
			fmt.Printf("could not place %s\n", s.name)
		}

	}

	writer, err := os.Create("test.png")
	err = png.Encode(writer, dst)
	if err != nil {
		log.Fatal(err)
	}
}

func readSprite(dir, name string) (s sprite) {
	path := path.Join(dir, name)
	reader, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()

	img, err := png.Decode(reader)
	if err != nil {
		log.Fatal(err)
	}

	s.name = name
	s.img = img
	s.rect = img.Bounds()
	s.size = Size{s.rect.Dx(), s.rect.Dy()}
	s.area = s.size.x * s.size.y
	return
}
<|endoftext|>


<commit_before>package provision

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"strings"
	"text/template"

	"github.com/docker/machine/drivers"
	"github.com/docker/machine/libmachine/auth"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/provision/pkgaction"
	"github.com/docker/machine/libmachine/swarm"
	"github.com/docker/machine/log"
	"github.com/docker/machine/utils"
)

func init() {
	Register("Debian", &RegisteredProvisioner{
		New: NewDebianProvisioner,
	})
}

func NewDebianProvisioner(d drivers.Driver) Provisioner {
	return &DebianProvisioner{
		GenericProvisioner{
			DockerOptionsDir:  "/etc/docker",
			DaemonOptionsFile: "/etc/systemd/system/docker.service",
			OsReleaseId:       "debian",
			Packages: []string{
				"curl",
			},
			Driver: d,
		},
	}
}

type DebianProvisioner struct {
	GenericProvisioner
}

func (provisioner *DebianProvisioner) Service(name string, action pkgaction.ServiceAction) error {
	// daemon-reload to catch config updates; systemd -- ugh
	if _, err := provisioner.SSHCommand("sudo systemctl daemon-reload"); err != nil {
		return err
	}

	command := fmt.Sprintf("sudo systemctl %s %s", action.String(), name)

	if _, err := provisioner.SSHCommand(command); err != nil {
		return err
	}

	return nil
}

func (provisioner *DebianProvisioner) Package(name string, action pkgaction.PackageAction) error {
	var packageAction string

	updateMetadata := true

	switch action {
	case pkgaction.Install:
		packageAction = "install"
	case pkgaction.Remove:
		packageAction = "remove"
		updateMetadata = false
	case pkgaction.Upgrade:
		packageAction = "upgrade"
	}

	switch name {
	case "docker":
		name = "lxc-docker"
	}

	if updateMetadata {
		if _, err := provisioner.SSHCommand("sudo apt-get update"); err != nil {
			return err
		}
	}

	command := fmt.Sprintf("DEBIAN_FRONTEND=noninteractive sudo -E apt-get %s -y  %s", packageAction, name)

	log.Debugf("package: action=%s name=%s", action.String(), name)

	if _, err := provisioner.SSHCommand(command); err != nil {
		return err
	}

	return nil
}

func (provisioner *DebianProvisioner) dockerDaemonResponding() bool {
	if _, err := provisioner.SSHCommand("sudo docker version"); err != nil {
		log.Warnf("Error getting SSH command to check if the daemon is up: %s", err)
		return false
	}

	// The daemon is up if the command worked.  Carry on.
	return true
}

func (provisioner *DebianProvisioner) Provision(swarmOptions swarm.SwarmOptions, authOptions auth.AuthOptions, engineOptions engine.EngineOptions) error {
	provisioner.SwarmOptions = swarmOptions
	provisioner.AuthOptions = authOptions
	provisioner.EngineOptions = engineOptions

	if provisioner.EngineOptions.StorageDriver == "" {
		provisioner.EngineOptions.StorageDriver = "aufs"
	}

	// HACK: since debian does not come with sudo by default,
	// we check if present and install first
	out, err := provisioner.SSHCommand("which sudo")
	if err != nil {
		log.Debug("sudo not found; attempting to install")
		// HACK: command not found which is what we are looking for;
		// since we only get back the string we check it :(
		if strings.Index(err.Error(), "Process exited with: 1") != -1 {
			res, err := ioutil.ReadAll(out.Stdout)
			if err != nil {
				return err
			}

			if string(res) == "" {
				log.Debug("installing sudo")
				if _, err := provisioner.SSHCommand("apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y sudo"); err != nil {
					return err
				}
			}
		} else {
			return err
		}
	}

	log.Debug("setting hostname")
	if err := provisioner.SetHostname(provisioner.Driver.GetMachineName()); err != nil {
		return err
	}

	log.Debug("installing base packages")
	for _, pkg := range provisioner.Packages {
		if err := provisioner.Package(pkg, pkgaction.Install); err != nil {
			return err
		}
	}

	log.Debug("installing docker")
	if err := installDockerGeneric(provisioner); err != nil {
		return err
	}

	log.Debug("waiting for docker daemon")
	if err := utils.WaitFor(provisioner.dockerDaemonResponding); err != nil {
		return err
	}

	provisioner.AuthOptions = setRemoteAuthOptions(provisioner)

	log.Debug("configuring auth")
	if err := ConfigureAuth(provisioner); err != nil {
		return err
	}

	log.Debug("configuring swarm")
	if err := configureSwarm(provisioner, swarmOptions); err != nil {
		return err
	}

	// enable in systemd
	log.Debug("enabling docker in systemd")
	if err := provisioner.Service("docker", pkgaction.Enable); err != nil {
		return err
	}

	return nil
}

func (provisioner *DebianProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) {
	var (
		engineCfg bytes.Buffer
	)

	driverNameLabel := fmt.Sprintf("provider=%s", provisioner.Driver.DriverName())
	provisioner.EngineOptions.Labels = append(provisioner.EngineOptions.Labels, driverNameLabel)

	engineConfigTmpl := `[Service]
ExecStart=/usr/bin/docker -d -H tcp://0.0.0.0:{{.DockerPort}} -H unix:///var/run/docker.sock --storage-driver {{.EngineOptions.StorageDriver}} --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}} {{ range .EngineOptions.Labels }}--label {{.}} {{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}} {{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}} {{ end }}
MountFlags=slave
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
`
	t, err := template.New("engineConfig").Parse(engineConfigTmpl)
	if err != nil {
		return nil, err
	}

	engineConfigContext := EngineConfigContext{
		DockerPort:    dockerPort,
		AuthOptions:   provisioner.AuthOptions,
		EngineOptions: provisioner.EngineOptions,
	}

	t.Execute(&engineCfg, engineConfigContext)

	return &DockerOptions{
		EngineOptions:     engineCfg.String(),
		EngineOptionsPath: provisioner.DaemonOptionsFile,
	}, nil
}
<commit_msg>debian: rebase and streamline sudo install<commit_after>package provision

import (
	"bytes"
	"fmt"
	"text/template"

	"github.com/docker/machine/drivers"
	"github.com/docker/machine/libmachine/auth"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/provision/pkgaction"
	"github.com/docker/machine/libmachine/swarm"
	"github.com/docker/machine/log"
	"github.com/docker/machine/utils"
)

func init() {
	Register("Debian", &RegisteredProvisioner{
		New: NewDebianProvisioner,
	})
}

func NewDebianProvisioner(d drivers.Driver) Provisioner {
	return &DebianProvisioner{
		GenericProvisioner{
			DockerOptionsDir:  "/etc/docker",
			DaemonOptionsFile: "/etc/systemd/system/docker.service",
			OsReleaseId:       "debian",
			Packages: []string{
				"curl",
			},
			Driver: d,
		},
	}
}

type DebianProvisioner struct {
	GenericProvisioner
}

func (provisioner *DebianProvisioner) Service(name string, action pkgaction.ServiceAction) error {
	// daemon-reload to catch config updates; systemd -- ugh
	if _, err := provisioner.SSHCommand("sudo systemctl daemon-reload"); err != nil {
		return err
	}

	command := fmt.Sprintf("sudo systemctl %s %s", action.String(), name)

	if _, err := provisioner.SSHCommand(command); err != nil {
		return err
	}

	return nil
}

func (provisioner *DebianProvisioner) Package(name string, action pkgaction.PackageAction) error {
	var packageAction string

	updateMetadata := true

	switch action {
	case pkgaction.Install:
		packageAction = "install"
	case pkgaction.Remove:
		packageAction = "remove"
		updateMetadata = false
	case pkgaction.Upgrade:
		packageAction = "upgrade"
	}

	switch name {
	case "docker":
		name = "lxc-docker"
	}

	if updateMetadata {
		if _, err := provisioner.SSHCommand("sudo apt-get update"); err != nil {
			return err
		}
	}

	command := fmt.Sprintf("DEBIAN_FRONTEND=noninteractive sudo -E apt-get %s -y  %s", packageAction, name)

	log.Debugf("package: action=%s name=%s", action.String(), name)

	if _, err := provisioner.SSHCommand(command); err != nil {
		return err
	}

	return nil
}

func (provisioner *DebianProvisioner) dockerDaemonResponding() bool {
	if _, err := provisioner.SSHCommand("sudo docker version"); err != nil {
		log.Warnf("Error getting SSH command to check if the daemon is up: %s", err)
		return false
	}

	// The daemon is up if the command worked.  Carry on.
	return true
}

func (provisioner *DebianProvisioner) Provision(swarmOptions swarm.SwarmOptions, authOptions auth.AuthOptions, engineOptions engine.EngineOptions) error {
	provisioner.SwarmOptions = swarmOptions
	provisioner.AuthOptions = authOptions
	provisioner.EngineOptions = engineOptions

	if provisioner.EngineOptions.StorageDriver == "" {
		provisioner.EngineOptions.StorageDriver = "aufs"
	}

	// HACK: since debian does not come with sudo by default we install
	log.Debug("installing sudo")
	if _, err := provisioner.SSHCommand("if ! type sudo; then apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y sudo; fi"); err != nil {
		return err
	}

	log.Debug("setting hostname")
	if err := provisioner.SetHostname(provisioner.Driver.GetMachineName()); err != nil {
		return err
	}

	log.Debug("installing base packages")
	for _, pkg := range provisioner.Packages {
		if err := provisioner.Package(pkg, pkgaction.Install); err != nil {
			return err
		}
	}

	log.Debug("installing docker")
	if err := installDockerGeneric(provisioner); err != nil {
		return err
	}

	log.Debug("waiting for docker daemon")
	if err := utils.WaitFor(provisioner.dockerDaemonResponding); err != nil {
		return err
	}

	provisioner.AuthOptions = setRemoteAuthOptions(provisioner)

	log.Debug("configuring auth")
	if err := ConfigureAuth(provisioner); err != nil {
		return err
	}

	log.Debug("configuring swarm")
	if err := configureSwarm(provisioner, swarmOptions); err != nil {
		return err
	}

	// enable in systemd
	log.Debug("enabling docker in systemd")
	if err := provisioner.Service("docker", pkgaction.Enable); err != nil {
		return err
	}

	return nil
}

func (provisioner *DebianProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) {
	var (
		engineCfg bytes.Buffer
	)

	driverNameLabel := fmt.Sprintf("provider=%s", provisioner.Driver.DriverName())
	provisioner.EngineOptions.Labels = append(provisioner.EngineOptions.Labels, driverNameLabel)

	engineConfigTmpl := `[Service]
ExecStart=/usr/bin/docker -d -H tcp://0.0.0.0:{{.DockerPort}} -H unix:///var/run/docker.sock --storage-driver {{.EngineOptions.StorageDriver}} --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}} {{ range .EngineOptions.Labels }}--label {{.}} {{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}} {{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}} {{ end }}
MountFlags=slave
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
`
	t, err := template.New("engineConfig").Parse(engineConfigTmpl)
	if err != nil {
		return nil, err
	}

	engineConfigContext := EngineConfigContext{
		DockerPort:    dockerPort,
		AuthOptions:   provisioner.AuthOptions,
		EngineOptions: provisioner.EngineOptions,
	}

	t.Execute(&engineCfg, engineConfigContext)

	return &DockerOptions{
		EngineOptions:     engineCfg.String(),
		EngineOptionsPath: provisioner.DaemonOptionsFile,
	}, nil
}
<|endoftext|>


<commit_before>package stripe

import (
	"context"
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"net/http"
	"net/url"
	"time"

	"github.com/stripe/stripe-go/form"
)

const (
	EndingBefore  = "ending_before"
	StartingAfter = "starting_after"
)

type ParamsContainer interface {
	GetParams() *Params
}

type ListParamsContainer interface {
	GetListParams() *ListParams
}

// Params is the structure that contains the common properties
// of any *Params structure.
type Params struct {
	// Context used for request. It may carry deadlines, cancelation signals,
	// and other request-scoped values across API boundaries and between
	// processes.
	//
	// Note that a cancelled or timed out context does not provide any
	// guarantee whether the operation was or was not completed on Stripe's API
	// servers. For certainty, you must either retry with the same idempotency
	// key or query the state of the API.
	Context context.Context `form:"-"`

	Expand []*string    `form:"expand"`
	Extra  *ExtraValues `form:"*"`

	// Headers may be used to provide extra header lines on the HTTP request.
	Headers http.Header `form:"-"`

	IdempotencyKey *string           `form:"-"` // Passed as header
	Metadata       map[string]string `form:"metadata"`

	// StripeAccount may contain the ID of a connected account. By including
	// this field, the request is made as if it originated from the connected
	// account instead of under the account of the owner of the configured
	// Stripe key.
	StripeAccount *string `form:"-"` // Passed as header
}

// GetParams returns a Params struct (itself). It exists because any structs
// that embed Params will inherit it, and thus implement the ParamsContainer
// interface.
func (p *Params) GetParams() *Params {
	return p
}

// ExtraValues are extra parameters that are attached to an API request.
// They're implemented as a custom type so that they can have their own
// AppendTo implementation.
type ExtraValues struct {
	url.Values `form:"-"` // See custom AppendTo implementation
}

// AppendTo implements custom form encoding for extra parameter values.
func (v ExtraValues) AppendTo(body *form.Values, keyParts []string) {
	for k, vs := range v.Values {
		for _, v := range vs {
			body.Add(form.FormatKey(append(keyParts, k)), v)
		}
	}
}

// ListParams is the structure that contains the common properties
// of any *ListParams structure.
type ListParams struct {
	// Context used for request. It may carry deadlines, cancelation signals,
	// and other request-scoped values across API boundaries and between
	// processes.
	//
	// Note that a cancelled or timed out context does not provide any
	// guarantee whether the operation was or was not completed on Stripe's API
	// servers. For certainty, you must either retry with the same idempotency
	// key or query the state of the API.
	Context context.Context `form:"-"`

	EndingBefore *string   `form:"ending_before"`
	Expand       []*string `form:"expand"`
	Filters      Filters   `form:"*"`
	Limit        *int64    `form:"limit"`

	// Single specifies whether this is a single page iterator. By default,
	// listing through an iterator will automatically grab additional pages as
	// the query progresses. To change this behavior and just load a single
	// page, set this to true.
	Single bool `form:"-"` // Not an API parameter

	StartingAfter *string `form:"starting_after"`

	// StripeAccount may contain the ID of a connected account. By including
	// this field, the request is made as if it originated from the connected
	// account instead of under the account of the owner of the configured
	// Stripe key.
	StripeAccount *string `form:"-"` // Passed as header
}

// ListMeta is the structure that contains the common properties
// of List iterators. The Count property is only populated if the
// total_count include option is passed in (see tests for example).
type ListMeta struct {
	HasMore    bool   `json:"has_more"`
	TotalCount uint32 `json:"total_count"`
	URL        string `json:"url"`
}

// RangeQueryParams are a set of generic request parameters that are used on
// list endpoints to filter their results by some timestamp.
type RangeQueryParams struct {
	// GreaterThan specifies that values should be a greater than this
	// timestamp.
	GreaterThan int64 `form:"gt"`

	// GreaterThanOrEqual specifies that values should be greater than or equal
	// to this timestamp.
	GreaterThanOrEqual int64 `form:"gte"`

	// LesserThan specifies that values should be lesser than this timetamp.
	LesserThan int64 `form:"lt"`

	// LesserThanOrEqual specifies that values should be lesser than or
	// equalthis timetamp.
	LesserThanOrEqual int64 `form:"lte"`
}

// Filters is a structure that contains a collection of filters for list-related APIs.
type Filters struct {
	f []*filter `form:"-"` // See custom AppendTo implementation
}

// AddFilter adds a new filter with a given key, op and value.
func (f *Filters) AddFilter(key, op, value string) {
	filter := &filter{Key: key, Op: op, Val: value}
	f.f = append(f.f, filter)
}

// AppendTo implements custom form encoding for filters.
func (f Filters) AppendTo(body *form.Values, keyParts []string) {
	if len(f.f) > 0 {
		for _, v := range f.f {
			if len(v.Op) > 0 {
				body.Add(form.FormatKey(append(keyParts, v.Key, v.Op)), v.Val)
			} else {
				body.Add(form.FormatKey(append(keyParts, v.Key)), v.Val)
			}
		}
	}
}

// filter is the structure that contains a filter for list-related APIs.
// It ends up passing query string parameters in the format key[op]=value.
type filter struct {
	Key, Op, Val string
}

// NewIdempotencyKey generates a new idempotency key that
// can be used on a request.
func NewIdempotencyKey() string {
	now := time.Now().UnixNano()
	buf := make([]byte, 4)
	rand.Read(buf)
	return fmt.Sprintf("%v_%v", now, base64.URLEncoding.EncodeToString(buf)[:6])
}

// SetIdempotencyKey sets a value for the Idempotency-Key header.
func (p *Params) SetIdempotencyKey(val string) {
	p.IdempotencyKey = &val
}

// SetStripeAccount sets a value for the Stripe-Account header.
func (p *Params) SetStripeAccount(val string) {
	p.StripeAccount = &val
}

// AddExpand appends a new field to expand.
func (p *Params) AddExpand(f string) {
	p.Expand = append(p.Expand, &f)
}

// AddMetadata adds a new key-value pair to the Metadata.
func (p *Params) AddMetadata(key, value string) {
	if p.Metadata == nil {
		p.Metadata = make(map[string]string)
	}

	p.Metadata[key] = value
}

// AddExtra adds a new arbitrary key-value pair to the request data
func (p *Params) AddExtra(key, value string) {
	if p.Extra == nil {
		p.Extra = &ExtraValues{Values: make(url.Values)}
	}

	p.Extra.Add(key, value)
}

// AddExpand appends a new field to expand.
func (p *ListParams) AddExpand(f string) {
	p.Expand = append(p.Expand, &f)
}

// GetListParams returns a ListParams struct (itself). It exists because any
// structs that embed ListParams will inherit it, and thus implement the
// ListParamsContainer interface.
func (p *ListParams) GetListParams() *ListParams {
	return p
}

// GetParams returns ListParams as a Params struct. It exists because any
// structs that embed Params will inherit it, and thus implement the
// ParamsContainer interface.
func (p *ListParams) GetParams() *Params {
	return p.ToParams()
}

// SetStripeAccount sets a value for the Stripe-Account header.
func (p *ListParams) SetStripeAccount(val string) {
	p.StripeAccount = &val
}

// ToParams converts a ListParams to a Params by moving over any fields that
// have valid targets in the new type. This is useful because fields in
// Params can be injected directly into an http.Request while generally
// ListParams is only used to build a set of parameters.
func (p *ListParams) ToParams() *Params {
	return &Params{
		Context:       p.Context,
		StripeAccount: p.StripeAccount,
	}
}
<commit_msg>Add a little more documentation for containers<commit_after>package stripe

import (
	"context"
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"net/http"
	"net/url"
	"time"

	"github.com/stripe/stripe-go/form"
)

const (
	EndingBefore  = "ending_before"
	StartingAfter = "starting_after"
)

// ParamsContainer is a general interface for which all parameter structs
// should comply. They achieve this by embedding a Params struct and inheriting
// its implementation of this interface.
type ParamsContainer interface {
	GetParams() *Params
}

// ListParamsContainer is a general interface for which all list parameter
// structs should comply. They achieve this by embedding a ListParams struct
// and inheriting its implementation of this interface.
type ListParamsContainer interface {
	GetListParams() *ListParams
}

// Params is the structure that contains the common properties
// of any *Params structure.
type Params struct {
	// Context used for request. It may carry deadlines, cancelation signals,
	// and other request-scoped values across API boundaries and between
	// processes.
	//
	// Note that a cancelled or timed out context does not provide any
	// guarantee whether the operation was or was not completed on Stripe's API
	// servers. For certainty, you must either retry with the same idempotency
	// key or query the state of the API.
	Context context.Context `form:"-"`

	Expand []*string    `form:"expand"`
	Extra  *ExtraValues `form:"*"`

	// Headers may be used to provide extra header lines on the HTTP request.
	Headers http.Header `form:"-"`

	IdempotencyKey *string           `form:"-"` // Passed as header
	Metadata       map[string]string `form:"metadata"`

	// StripeAccount may contain the ID of a connected account. By including
	// this field, the request is made as if it originated from the connected
	// account instead of under the account of the owner of the configured
	// Stripe key.
	StripeAccount *string `form:"-"` // Passed as header
}

// GetParams returns a Params struct (itself). It exists because any structs
// that embed Params will inherit it, and thus implement the ParamsContainer
// interface.
func (p *Params) GetParams() *Params {
	return p
}

// ExtraValues are extra parameters that are attached to an API request.
// They're implemented as a custom type so that they can have their own
// AppendTo implementation.
type ExtraValues struct {
	url.Values `form:"-"` // See custom AppendTo implementation
}

// AppendTo implements custom form encoding for extra parameter values.
func (v ExtraValues) AppendTo(body *form.Values, keyParts []string) {
	for k, vs := range v.Values {
		for _, v := range vs {
			body.Add(form.FormatKey(append(keyParts, k)), v)
		}
	}
}

// ListParams is the structure that contains the common properties
// of any *ListParams structure.
type ListParams struct {
	// Context used for request. It may carry deadlines, cancelation signals,
	// and other request-scoped values across API boundaries and between
	// processes.
	//
	// Note that a cancelled or timed out context does not provide any
	// guarantee whether the operation was or was not completed on Stripe's API
	// servers. For certainty, you must either retry with the same idempotency
	// key or query the state of the API.
	Context context.Context `form:"-"`

	EndingBefore *string   `form:"ending_before"`
	Expand       []*string `form:"expand"`
	Filters      Filters   `form:"*"`
	Limit        *int64    `form:"limit"`

	// Single specifies whether this is a single page iterator. By default,
	// listing through an iterator will automatically grab additional pages as
	// the query progresses. To change this behavior and just load a single
	// page, set this to true.
	Single bool `form:"-"` // Not an API parameter

	StartingAfter *string `form:"starting_after"`

	// StripeAccount may contain the ID of a connected account. By including
	// this field, the request is made as if it originated from the connected
	// account instead of under the account of the owner of the configured
	// Stripe key.
	StripeAccount *string `form:"-"` // Passed as header
}

// ListMeta is the structure that contains the common properties
// of List iterators. The Count property is only populated if the
// total_count include option is passed in (see tests for example).
type ListMeta struct {
	HasMore    bool   `json:"has_more"`
	TotalCount uint32 `json:"total_count"`
	URL        string `json:"url"`
}

// RangeQueryParams are a set of generic request parameters that are used on
// list endpoints to filter their results by some timestamp.
type RangeQueryParams struct {
	// GreaterThan specifies that values should be a greater than this
	// timestamp.
	GreaterThan int64 `form:"gt"`

	// GreaterThanOrEqual specifies that values should be greater than or equal
	// to this timestamp.
	GreaterThanOrEqual int64 `form:"gte"`

	// LesserThan specifies that values should be lesser than this timetamp.
	LesserThan int64 `form:"lt"`

	// LesserThanOrEqual specifies that values should be lesser than or
	// equalthis timetamp.
	LesserThanOrEqual int64 `form:"lte"`
}

// Filters is a structure that contains a collection of filters for list-related APIs.
type Filters struct {
	f []*filter `form:"-"` // See custom AppendTo implementation
}

// AddFilter adds a new filter with a given key, op and value.
func (f *Filters) AddFilter(key, op, value string) {
	filter := &filter{Key: key, Op: op, Val: value}
	f.f = append(f.f, filter)
}

// AppendTo implements custom form encoding for filters.
func (f Filters) AppendTo(body *form.Values, keyParts []string) {
	if len(f.f) > 0 {
		for _, v := range f.f {
			if len(v.Op) > 0 {
				body.Add(form.FormatKey(append(keyParts, v.Key, v.Op)), v.Val)
			} else {
				body.Add(form.FormatKey(append(keyParts, v.Key)), v.Val)
			}
		}
	}
}

// filter is the structure that contains a filter for list-related APIs.
// It ends up passing query string parameters in the format key[op]=value.
type filter struct {
	Key, Op, Val string
}

// NewIdempotencyKey generates a new idempotency key that
// can be used on a request.
func NewIdempotencyKey() string {
	now := time.Now().UnixNano()
	buf := make([]byte, 4)
	rand.Read(buf)
	return fmt.Sprintf("%v_%v", now, base64.URLEncoding.EncodeToString(buf)[:6])
}

// SetIdempotencyKey sets a value for the Idempotency-Key header.
func (p *Params) SetIdempotencyKey(val string) {
	p.IdempotencyKey = &val
}

// SetStripeAccount sets a value for the Stripe-Account header.
func (p *Params) SetStripeAccount(val string) {
	p.StripeAccount = &val
}

// AddExpand appends a new field to expand.
func (p *Params) AddExpand(f string) {
	p.Expand = append(p.Expand, &f)
}

// AddMetadata adds a new key-value pair to the Metadata.
func (p *Params) AddMetadata(key, value string) {
	if p.Metadata == nil {
		p.Metadata = make(map[string]string)
	}

	p.Metadata[key] = value
}

// AddExtra adds a new arbitrary key-value pair to the request data
func (p *Params) AddExtra(key, value string) {
	if p.Extra == nil {
		p.Extra = &ExtraValues{Values: make(url.Values)}
	}

	p.Extra.Add(key, value)
}

// AddExpand appends a new field to expand.
func (p *ListParams) AddExpand(f string) {
	p.Expand = append(p.Expand, &f)
}

// GetListParams returns a ListParams struct (itself). It exists because any
// structs that embed ListParams will inherit it, and thus implement the
// ListParamsContainer interface.
func (p *ListParams) GetListParams() *ListParams {
	return p
}

// GetParams returns ListParams as a Params struct. It exists because any
// structs that embed Params will inherit it, and thus implement the
// ParamsContainer interface.
func (p *ListParams) GetParams() *Params {
	return p.ToParams()
}

// SetStripeAccount sets a value for the Stripe-Account header.
func (p *ListParams) SetStripeAccount(val string) {
	p.StripeAccount = &val
}

// ToParams converts a ListParams to a Params by moving over any fields that
// have valid targets in the new type. This is useful because fields in
// Params can be injected directly into an http.Request while generally
// ListParams is only used to build a set of parameters.
func (p *ListParams) ToParams() *Params {
	return &Params{
		Context:       p.Context,
		StripeAccount: p.StripeAccount,
	}
}
<|endoftext|>


<commit_before>package router

import (
	"fmt"
	"strconv"
	"strings"
	"time"
)

// Params is similar to url.Values, but with a few more utility functions
type Params map[string][]string

// Map gets the params as a flat map[string]string, discarding any multiple values.
func (p Params) Map() map[string]string {
	flat := make(map[string]string)

	for k, v := range p {
		flat[k] = v[0]
	}

	return flat
}

// Clean returns the params as a  map[string]string, discarding any multiple values, with any params not in the accepted list removed
func (p Params) Clean(accepted []string) map[string]string {
	flat := make(map[string]string)

	for k, v := range p {
		if containsString(accepted, k) {
			flat[k] = v[0]
		}
	}

	return flat
}

// Flatten deflates a set of params (of any type) to a comma separated list (only for simple params)
func (p Params) Flatten(k string) string {
	flat := ""

	for i, v := range p[k] {
		if i > 0 {
			flat = fmt.Sprintf("%s,%s", flat, v)
		} else {
			flat = v
		}
	}

	// replace this key with an array containing only flat
	p[k] = []string{flat}

	return flat
}

// GetDate returns the first value associated with a given key as a time, using the given time format.
func (p Params) GetDate(key string, format string) (time.Time, error) {
	v := p.Get(key)
	return time.Parse(format, v)
}

// GetInt returns the first value associated with the given key as an integer. If there is no value or a parse error, it returns 0
// If the string contains non-numeric characters, they are first stripped
func (p Params) GetInt(key string) int64 {
	var i int64
	v := p.Get(key)
	// We truncate the string at the first non-numeric character
	v = v[0 : strings.LastIndexAny(v, "0123456789")+1]
	i, err := strconv.ParseInt(v, 10, 64)
	if err != nil {
		return 0
	}
	return i
}

// GetInts returns all values associated with the given key as an array of integers.
func (p Params) GetInts(key string) []int64 {
	ints := []int64{}

	for _, v := range p.GetAll(key) {
		vi, err := strconv.ParseInt(v, 10, 64)
		if err != nil {
			vi = 0
		}
		ints = append(ints, vi)
	}

	return ints
}

// GetUniqueInts returns all unique non-zero int values
// associated with the given key as an array of integers
func (p Params) GetUniqueInts(key string) []int64 {
	ints := []int64{}

	for _, v := range p.GetAll(key) {
		if string(v) == "" {
			continue // ignore blank ints
		}
		vi, err := strconv.ParseInt(v, 10, 64)
		if err != nil {
			vi = 0
		}

		// Do not insert 0, or duplicate entries
		if vi > 0 && !contains(ints, vi) {
			ints = append(ints, vi)
		}
	}

	return ints
}

// GetIntsString returns all values associated with the given key as a comma separated string
func (p Params) GetIntsString(key string) string {
	ints := ""

	for _, v := range p.GetAll(key) {
		if "" == string(v) {
			continue // ignore blank ints
		}

		if len(ints) > 0 {
			ints += "," + string(v)
		} else {
			ints += string(v)
		}

	}

	return ints
}

// GetAll returns all values associated with the given key - equivalent to params[key].
func (p Params) GetAll(key string) []string {
	return p[key]
}

// Get gets the first value associated with the given key.
// If there are no values returns the empty string.
func (p Params) Get(key string) string {
	if p == nil {
		return ""
	}

	v := p[key]

	if v == nil || len(v) == 0 {
		return ""
	}

	return v[0]
}

// Blank returns true if the value corresponding to key is an empty string
func (p Params) Blank(key string) bool {
	v := p.Get(key)
	return v == ""
}

// Set sets the key to a string value replacing any existing values.
func (p Params) Set(key, value string) {
	p[key] = []string{value}
}

// SetInt sets the key to a single int value as string replacing any existing values.
func (p Params) SetInt(key string, value int64) {
	p[key] = []string{fmt.Sprintf("%d", value)}
}

// Add adds the value, if necessary appending to any existing values associated with key.
func (p Params) Add(key, value string) {
	p[key] = append(p[key], value)
}

// Remove deletes the values associated with key.
func (p Params) Remove(key string) {
	delete(p, key)
}

// Contains returns true if this array of ints contains the given int
func contains(list []int64, item int64) bool {
	for _, b := range list {
		if b == item {
			return true
		}
	}
	return false
}

// containsString returns true if the string is in this list
func containsString(allowed []string, p string) bool {
	for _, v := range allowed {
		if p == v {
			return true
		}
	}
	return false
}
<commit_msg>Add GetFloats and GetFloat to params<commit_after>package router

import (
	"fmt"
	"strconv"
	"strings"
	"time"
)

// Params is similar to url.Values, but with a few more utility functions
type Params map[string][]string

// Map gets the params as a flat map[string]string, discarding any multiple values.
func (p Params) Map() map[string]string {
	flat := make(map[string]string)

	for k, v := range p {
		flat[k] = v[0]
	}

	return flat
}

// Clean returns the params as a  map[string]string, discarding any multiple values, with any params not in the accepted list removed
func (p Params) Clean(accepted []string) map[string]string {
	flat := make(map[string]string)

	for k, v := range p {
		if containsString(accepted, k) {
			flat[k] = v[0]
		}
	}

	return flat
}

// Flatten deflates a set of params (of any type) to a comma separated list (only for simple params)
func (p Params) Flatten(k string) string {
	flat := ""

	for i, v := range p[k] {
		if i > 0 {
			flat = fmt.Sprintf("%s,%s", flat, v)
		} else {
			flat = v
		}
	}

	// replace this key with an array containing only flat
	p[k] = []string{flat}

	return flat
}

// GetDate returns the first value associated with a given key as a time, using the given time format.
func (p Params) GetDate(key string, format string) (time.Time, error) {
	v := p.Get(key)
	return time.Parse(format, v)
}

// GetInt returns the first value associated with the given key as an integer. If there is no value or a parse error, it returns 0
// If the string contains non-numeric characters, they are first stripped
func (p Params) GetInt(key string) int64 {
	var i int64
	v := p.Get(key)
	// We truncate the string at the first non-numeric character
	v = v[0 : strings.LastIndexAny(v, "0123456789")+1]
	i, err := strconv.ParseInt(v, 10, 64)
	if err != nil {
		return 0
	}
	return i
}

// GetInts returns all values associated with the given key as an array of integers.
func (p Params) GetInts(key string) []int64 {
	ints := []int64{}

	for _, v := range p.GetAll(key) {
		vi, err := strconv.ParseInt(v, 10, 64)
		if err != nil {
			vi = 0
		}
		ints = append(ints, vi)
	}

	return ints
}

// GetUniqueInts returns all unique non-zero int values
// associated with the given key as an array of integers
func (p Params) GetUniqueInts(key string) []int64 {
	ints := []int64{}

	for _, v := range p.GetAll(key) {
		if string(v) == "" {
			continue // ignore blank ints
		}
		vi, err := strconv.ParseInt(v, 10, 64)
		if err != nil {
			vi = 0
		}

		// Do not insert 0, or duplicate entries
		if vi > 0 && !contains(ints, vi) {
			ints = append(ints, vi)
		}
	}

	return ints
}

// GetIntsString returns all values associated with the given key as a comma separated string
func (p Params) GetIntsString(key string) string {
	ints := ""

	for _, v := range p.GetAll(key) {
		if "" == string(v) {
			continue // ignore blank ints
		}

		if len(ints) > 0 {
			ints += "," + string(v)
		} else {
			ints += string(v)
		}

	}

	return ints
}

// GetFloat returns the first value associated with the given key as an integer. If there is no value or a parse error, it returns 0.0
func (p Params) GetFloat(key string) float64 {
	var value float64
	v := p.Get(key)
	value, err := strconv.ParseFloat(v, 64)
	if err != nil {
		return 0.0
	}
	return value
}

// GetFloats returns all values associated with the given key as an array of floats.
func (p Params) GetFloats(key string) []float64 {
	var values []float64
	for _, v := range p.GetAll(key) {
		value, err := strconv.ParseFloat(v, 64)
		if err != nil {
			value = 0.0
		}
		values = append(values, value)
	}
	return values
}

// GetAll returns all values associated with the given key - equivalent to params[key].
func (p Params) GetAll(key string) []string {
	return p[key]
}

// Get gets the first value associated with the given key.
// If there are no values returns the empty string.
func (p Params) Get(key string) string {
	if p == nil {
		return ""
	}

	v := p[key]

	if v == nil || len(v) == 0 {
		return ""
	}

	return v[0]
}

// Blank returns true if the value corresponding to key is an empty string
func (p Params) Blank(key string) bool {
	v := p.Get(key)
	return v == ""
}

// Set sets the key to a string value replacing any existing values.
func (p Params) Set(key, value string) {
	p[key] = []string{value}
}

// SetInt sets the key to a single int value as string replacing any existing values.
func (p Params) SetInt(key string, value int64) {
	p[key] = []string{fmt.Sprintf("%d", value)}
}

// Add adds the value, if necessary appending to any existing values associated with key.
func (p Params) Add(key, value string) {
	p[key] = append(p[key], value)
}

// Remove deletes the values associated with key.
func (p Params) Remove(key string) {
	delete(p, key)
}

// Contains returns true if this array of ints contains the given int
func contains(list []int64, item int64) bool {
	for _, b := range list {
		if b == item {
			return true
		}
	}
	return false
}

// containsString returns true if the string is in this list
func containsString(allowed []string, p string) bool {
	for _, v := range allowed {
		if p == v {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package gojison

import (
	"fmt"
	"strings"
)

type Params map[string]interface{}

func (p Params) Add(key string, value interface{}) {
	p[key] = value
}

func (p Params) GetP(key string) Params {
	if val, ok := p[key]; ok {
		return Params(val.(map[string]interface{}))
	}

	return Params{}
}

func (p Params) Get(key string) string {
	if val, ok := p[key]; ok {
		return fmt.Sprintf("%v", val)
	}

	return ""
}

func (p Params) GetString(key string) string {
	if val, ok := p[key]; ok {
		if vs, ok := val.(string); ok {
			return vs
		}
	}
	return ""
}

func (p Params) GetInt(key string) int {
	if val, ok := p[key]; ok {
		if vi, ok := val.(int); ok {
			return vi
		}
	}

	return 0
}

func (p Params) GetFloat(key string) float64 {
	if val, ok := p[key]; ok {
		if vf, ok := val.(float64); ok {
			return vf
		}
	}

	return 0
}

func (p Params) GetI(key string) interface{} {
	if val, ok := p[key]; ok {
		return val
	}

	return nil
}

func (p Params) GetSlice(key string) []interface{} {
	if val, ok := p[key]; ok {
		return val.([]interface{})
	}

	return nil
}

func (p Params) GetSliceStrings(key string) []string {
	result := make([]string, 0)
	if val, ok := p[key]; ok {
		if val, ok := val.([]interface{}); ok {
			for _, v := range val {
				if vs, ok := v.(string); ok {
					result = append(result, vs)
				}
			}
		}
		return result
	}

	return nil
}

func (p Params) Required(keys ...string) error {
	for _, key := range keys {
		if ok := p.exists(p, key); !ok {
			return fmt.Errorf("The parameter %s is required!", key)
		}
	}

	return nil
}

func (p Params) exists(input map[string]interface{}, key string) (ok bool) {
	if input == nil {
		return false
	}

	if index := strings.Index(key, "."); index != -1 {
		pair := strings.SplitN(key, ".", 2)
		params, asserted := input[pair[0]].(map[string]interface{})
		if !asserted {
			return false
		}
		ok = p.exists(params, pair[1])
	} else {
		var v interface{}
		v, ok = input[key]
		if s, isS := v.(string); isS && ok {
			ok = (s != "")
		}
	}

	return ok
}
<commit_msg>Add GetTime.<commit_after>package gojison

import (
	"fmt"
	"strings"
	"time"
)

type Params map[string]interface{}

func (p Params) Add(key string, value interface{}) {
	p[key] = value
}

func (p Params) GetP(key string) Params {
	if val, ok := p[key]; ok {
		return Params(val.(map[string]interface{}))
	}

	return Params{}
}

func (p Params) Get(key string) string {
	if val, ok := p[key]; ok {
		return fmt.Sprintf("%v", val)
	}

	return ""
}

func (p Params) GetString(key string) string {
	if val, ok := p[key]; ok {
		if vs, ok := val.(string); ok {
			return vs
		}
	}
	return ""
}

func (p Params) GetInt(key string) int {
	if val, ok := p[key]; ok {
		if vi, ok := val.(int); ok {
			return vi
		}
	}

	return 0
}

func (p Params) GetFloat(key string) float64 {
	if val, ok := p[key]; ok {
		if vf, ok := val.(float64); ok {
			return vf
		}
	}

	return 0
}

func (p Params) GetTime(key string) time.Time {

	if time, err := time.Parse(time.RFC3339, p.Get(key)); !err {
		return time
	}

	return time.Time{}
}

func (p Params) GetI(key string) interface{} {
	if val, ok := p[key]; ok {
		return val
	}

	return nil
}

func (p Params) GetSlice(key string) []interface{} {
	if val, ok := p[key]; ok {
		return val.([]interface{})
	}

	return nil
}

func (p Params) GetSliceStrings(key string) []string {
	result := make([]string, 0)
	if val, ok := p[key]; ok {
		if val, ok := val.([]interface{}); ok {
			for _, v := range val {
				if vs, ok := v.(string); ok {
					result = append(result, vs)
				}
			}
		}
		return result
	}

	return nil
}

func (p Params) Required(keys ...string) error {
	for _, key := range keys {
		if ok := p.exists(p, key); !ok {
			return fmt.Errorf("The parameter %s is required!", key)
		}
	}

	return nil
}

func (p Params) exists(input map[string]interface{}, key string) (ok bool) {
	if input == nil {
		return false
	}

	if index := strings.Index(key, "."); index != -1 {
		pair := strings.SplitN(key, ".", 2)
		params, asserted := input[pair[0]].(map[string]interface{})
		if !asserted {
			return false
		}
		ok = p.exists(params, pair[1])
	} else {
		var v interface{}
		v, ok = input[key]
		if s, isS := v.(string); isS && ok {
			ok = (s != "")
		}
	}

	return ok
}
<|endoftext|>


<commit_before>package gophercloud

import (
	"fmt"
	"net/url"
	"reflect"
	"strconv"
	"strings"
	"time"
)

// MaybeString takes a string that might be a zero-value, and either returns a
// pointer to its address or a nil value (i.e. empty pointer). This is useful
// for converting zero values in options structs when the end-user hasn't
// defined values. Those zero values need to be nil in order for the JSON
// serialization to ignore them.
func MaybeString(original string) *string {
	if original != "" {
		return &original
	}
	return nil
}

// MaybeInt takes an int that might be a zero-value, and either returns a
// pointer to its address or a nil value (i.e. empty pointer).
func MaybeInt(original int) *int {
	if original != 0 {
		return &original
	}
	return nil
}

var t time.Time

func isZero(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.Func, reflect.Map, reflect.Slice:
		return v.IsNil()
	case reflect.Array:
		z := true
		for i := 0; i < v.Len(); i++ {
			z = z && isZero(v.Index(i))
		}
		return z
	case reflect.Struct:
		if v.Type() == reflect.TypeOf(t) {
			if v.Interface().(time.Time).IsZero() {
				return true
			}
			return false
		}
		z := true
		for i := 0; i < v.NumField(); i++ {
			z = z && isZero(v.Field(i))
		}
		return z
	}
	// Compare other types directly:
	z := reflect.Zero(v.Type())
	return v.Interface() == z.Interface()
}

/*
BuildQueryString accepts a generic structure and parses it URL struct. It
converts field names into query names based on "q" tags. So for example, this
type:

	struct {
	   Bar string `q:"x_bar"`
	   Baz int    `q:"lorem_ipsum"`
	}{
	   Bar: "XXX",
	   Baz: "YYY",
	}

will be converted into ?x_bar=XXX&lorem_ipsum=YYYY
*/
func BuildQueryString(opts interface{}) (*url.URL, error) {
	optsValue := reflect.ValueOf(opts)
	if optsValue.Kind() == reflect.Ptr {
		optsValue = optsValue.Elem()
	}

	optsType := reflect.TypeOf(opts)
	if optsType.Kind() == reflect.Ptr {
		optsType = optsType.Elem()
	}

	var optsSlice []string
	if optsValue.Kind() == reflect.Struct {
		for i := 0; i < optsValue.NumField(); i++ {
			v := optsValue.Field(i)
			f := optsType.Field(i)
			qTag := f.Tag.Get("q")

			// if the field has a 'q' tag, it goes in the query string
			if qTag != "" {
				tags := strings.Split(qTag, ",")

				// if the field is set, add it to the slice of query pieces
				if !isZero(v) {
					switch v.Kind() {
					case reflect.String:
						optsSlice = append(optsSlice, tags[0]+"="+v.String())
					case reflect.Int:
						optsSlice = append(optsSlice, tags[0]+"="+strconv.FormatInt(v.Int(), 10))
					case reflect.Bool:
						optsSlice = append(optsSlice, tags[0]+"="+strconv.FormatBool(v.Bool()))
					}
				} else {
					// Otherwise, the field is not set.
					if len(tags) == 2 && tags[1] == "required" {
						// And the field is required. Return an error.
						return nil, fmt.Errorf("Required query parameter [%s] not set.", f.Name)
					}
				}
			}

		}
		// URL encode the string for safety.
		s := strings.Join(optsSlice, "&")
		if s != "" {
			s = "?" + s
		}
		u, err := url.Parse(s)
		if err != nil {
			return nil, err
		}
		return u, nil
	}
	// Return an error if the underlying type of 'opts' isn't a struct.
	return nil, fmt.Errorf("Options type is not a struct.")
}

// BuildHeaders accepts a generic structure and parses it into a string map. It
// converts field names into header names based on "h" tags, and field values
// into header values by a simple one-to-one mapping.
func BuildHeaders(opts interface{}) (map[string]string, error) {
	optsValue := reflect.ValueOf(opts)
	if optsValue.Kind() == reflect.Ptr {
		optsValue = optsValue.Elem()
	}

	optsType := reflect.TypeOf(opts)
	if optsType.Kind() == reflect.Ptr {
		optsType = optsType.Elem()
	}

	optsMap := make(map[string]string)
	if optsValue.Kind() == reflect.Struct {
		for i := 0; i < optsValue.NumField(); i++ {
			v := optsValue.Field(i)
			f := optsType.Field(i)
			hTag := f.Tag.Get("h")

			// if the field has a 'h' tag, it goes in the header
			if hTag != "" {
				tags := strings.Split(hTag, ",")

				// if the field is set, add it to the slice of query pieces
				if !isZero(v) {
					switch v.Kind() {
					case reflect.String:
						optsMap[tags[0]] = v.String()
					case reflect.Int:
						optsMap[tags[0]] = strconv.FormatInt(v.Int(), 10)
					case reflect.Bool:
						optsMap[tags[0]] = strconv.FormatBool(v.Bool())
					}
				} else {
					// Otherwise, the field is not set.
					if len(tags) == 2 && tags[1] == "required" {
						// And the field is required. Return an error.
						return optsMap, fmt.Errorf("Required header not set.")
					}
				}
			}

		}
		return optsMap, nil
	}
	// Return an error if the underlying type of 'opts' isn't a struct.
	return optsMap, fmt.Errorf("Options type is not a struct.")
}
<commit_msg>Expand documentation for things in params.go<commit_after>package gophercloud

import (
	"fmt"
	"net/url"
	"reflect"
	"strconv"
	"strings"
	"time"
)

/*
MaybeString is an internal function to be used by request methods in individual
resource packages.

It takes a string that might be a zero value and returns either a pointer to its
address or nil. This is useful for allowing users to conveniently omit values
from an options struct, but still pass nil to the JSON serializer to omit them
from a request body.
*/
func MaybeString(original string) *string {
	if original != "" {
		return &original
	}
	return nil
}

/*
MaybeInt is an internal function to be used by request methods in individual
resource packages.

Like MaybeString, it accepts an int that may be a zero value and returns either
a pointer to its address or nil to hint to the JSON serializer to omit its
field.
*/
func MaybeInt(original int) *int {
	if original != 0 {
		return &original
	}
	return nil
}

var t time.Time

func isZero(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.Func, reflect.Map, reflect.Slice:
		return v.IsNil()
	case reflect.Array:
		z := true
		for i := 0; i < v.Len(); i++ {
			z = z && isZero(v.Index(i))
		}
		return z
	case reflect.Struct:
		if v.Type() == reflect.TypeOf(t) {
			if v.Interface().(time.Time).IsZero() {
				return true
			}
			return false
		}
		z := true
		for i := 0; i < v.NumField(); i++ {
			z = z && isZero(v.Field(i))
		}
		return z
	}
	// Compare other types directly:
	z := reflect.Zero(v.Type())
	return v.Interface() == z.Interface()
}

/*
BuildQueryString is an internal function to be used by request methods in
individual resource packages.

It accepts a tagged structure and expands it into a URL struct. Field names are
converted into query parameters based on a "q" tag. For example:

	type struct Something {
	   Bar string `q:"x_bar"`
	   Baz int    `q:"lorem_ipsum"`
	}

	instance := Something{
	   Bar: "AAA",
	   Baz: "BBB",
	}

will be converted into "?x_bar=AAA&lorem_ipsum=BBB".

The struct's fields may be strings, integers, or boolean values. Fields left at
their type's zero value will be omittted from the query.
*/
func BuildQueryString(opts interface{}) (*url.URL, error) {
	optsValue := reflect.ValueOf(opts)
	if optsValue.Kind() == reflect.Ptr {
		optsValue = optsValue.Elem()
	}

	optsType := reflect.TypeOf(opts)
	if optsType.Kind() == reflect.Ptr {
		optsType = optsType.Elem()
	}

	var optsSlice []string
	if optsValue.Kind() == reflect.Struct {
		for i := 0; i < optsValue.NumField(); i++ {
			v := optsValue.Field(i)
			f := optsType.Field(i)
			qTag := f.Tag.Get("q")

			// if the field has a 'q' tag, it goes in the query string
			if qTag != "" {
				tags := strings.Split(qTag, ",")

				// if the field is set, add it to the slice of query pieces
				if !isZero(v) {
					switch v.Kind() {
					case reflect.String:
						optsSlice = append(optsSlice, tags[0]+"="+v.String())
					case reflect.Int:
						optsSlice = append(optsSlice, tags[0]+"="+strconv.FormatInt(v.Int(), 10))
					case reflect.Bool:
						optsSlice = append(optsSlice, tags[0]+"="+strconv.FormatBool(v.Bool()))
					}
				} else {
					// Otherwise, the field is not set.
					if len(tags) == 2 && tags[1] == "required" {
						// And the field is required. Return an error.
						return nil, fmt.Errorf("Required query parameter [%s] not set.", f.Name)
					}
				}
			}

		}
		// URL encode the string for safety.
		s := strings.Join(optsSlice, "&")
		if s != "" {
			s = "?" + s
		}
		u, err := url.Parse(s)
		if err != nil {
			return nil, err
		}
		return u, nil
	}
	// Return an error if the underlying type of 'opts' isn't a struct.
	return nil, fmt.Errorf("Options type is not a struct.")
}

/*
BuildHeaders is an internal function to be used by request methods in
individual resource packages.

It accepts a arbitrary tagged structure and produces a string map that's
suitable for use as the HTTP headers of an outgoing request. Field names are
mapped to header names based in "h" tags.

  type struct Something {
    Bar string `h:"x_bar"`
    Baz int    `h:"lorem_ipsum"`
  }

  instance := Something{
    Bar: "AAA",
    Baz: "BBB",
  }

will be converted into:

  map[string]string{
    "x_bar": "AAA",
    "lorem_ipsum": "BBB",
  }

Untagged fields and fields left at their zero values are skipped. Integers,
booleans and string values are supported.
*/
func BuildHeaders(opts interface{}) (map[string]string, error) {
	optsValue := reflect.ValueOf(opts)
	if optsValue.Kind() == reflect.Ptr {
		optsValue = optsValue.Elem()
	}

	optsType := reflect.TypeOf(opts)
	if optsType.Kind() == reflect.Ptr {
		optsType = optsType.Elem()
	}

	optsMap := make(map[string]string)
	if optsValue.Kind() == reflect.Struct {
		for i := 0; i < optsValue.NumField(); i++ {
			v := optsValue.Field(i)
			f := optsType.Field(i)
			hTag := f.Tag.Get("h")

			// if the field has a 'h' tag, it goes in the header
			if hTag != "" {
				tags := strings.Split(hTag, ",")

				// if the field is set, add it to the slice of query pieces
				if !isZero(v) {
					switch v.Kind() {
					case reflect.String:
						optsMap[tags[0]] = v.String()
					case reflect.Int:
						optsMap[tags[0]] = strconv.FormatInt(v.Int(), 10)
					case reflect.Bool:
						optsMap[tags[0]] = strconv.FormatBool(v.Bool())
					}
				} else {
					// Otherwise, the field is not set.
					if len(tags) == 2 && tags[1] == "required" {
						// And the field is required. Return an error.
						return optsMap, fmt.Errorf("Required header not set.")
					}
				}
			}

		}
		return optsMap, nil
	}
	// Return an error if the underlying type of 'opts' isn't a struct.
	return optsMap, fmt.Errorf("Options type is not a struct.")
}
<|endoftext|>


<commit_before>package goparsify

import (
	"errors"
	"fmt"
	"regexp"
	"strings"
	"unicode/utf8"
)

// Result is the output of a parser. Usually only one of its fields will be set and should be though of
// more as a union type. having it avoids interface{} littered all through the parsing code and makes
// the it easy to do the two most common operations, getting a token and finding a child.
type Result struct {
	Token  string
	Child  []Result
	Result interface{}
}

// Parser is the workhorse of parsify. A parser takes a State and returns a result, consuming some
// of the State in the process.
// Given state is shared there are a few rules that should be followed:
//  - A parser that errors must set state.Error
//  - A parser that errors must not change state.Pos
//  - A parser that consumed some input should advance state.Pos
type Parser func(*State) Result

// VoidParser is a special type of parser that never returns anything but can still consume input
type VoidParser func(*State)

// Parserish types are any type that can be turned into a Parser by Parsify
// These currently include *Parser and string literals.
//
// This makes recursive grammars cleaner and allows string literals to be used directly in most contexts.
// eg, matching balanced paren:
// ```go
// var group Parser
// group = Seq("(", Maybe(&group), ")")
// ```
// vs
// ```go
// var group ParserPtr{}
// group.P = Seq(Exact("("), Maybe(group.Parse), Exact(")"))
// ```
type Parserish interface{}

// Parsify takes a Parserish and makes a Parser out of it. It should be called by
// any Parser that accepts a Parser as an argument. It should never be called during
// instead call it during parser creation so there is no runtime cost.
//
// See Parserish for details.
func Parsify(p Parserish) Parser {
	switch p := p.(type) {
	case func(*State) Result:
		return p
	case Parser:
		return p
	case *Parser:
		// Todo: Maybe capture this stack and on nil show it? Is there a good error library to do this?
		return func(ptr *State) Result {
			return (*p)(ptr)
		}
	case string:
		return Exact(p)
	default:
		panic(fmt.Errorf("cant turn a `%T` into a parser", p))
	}
}

// ParsifyAll calls Parsify on all parsers
func ParsifyAll(parsers ...Parserish) []Parser {
	ret := make([]Parser, len(parsers))
	for i, parser := range parsers {
		ret[i] = Parsify(parser)
	}
	return ret
}

// Run applies some input to a parser and returns the result, failing if the input isnt fully consumed.
// It is a convenience method for the most common way to invoke a parser.
func Run(parser Parserish, input string, ws ...VoidParser) (result interface{}, err error) {
	p := Parsify(parser)
	ps := NewState(input)
	if len(ws) > 0 {
		ps.WS = ws[0]
	}

	ret := p(ps)
	ps.AutoWS()

	if ps.Error.expected != "" {
		return ret.Result, ps.Error
	}

	if ps.Get() != "" {
		return ret.Result, errors.New("left unparsed: " + ps.Get())
	}

	return ret.Result, nil
}

// WS will consume whitespace, it should only be needed when AutoWS is turned off
func WS() Parser {
	return NewParser("AutoWS", func(ps *State) Result {
		ps.WS(ps)
		return Result{}
	})
}

// Cut prevents backtracking beyond this point. Usually used after keywords when you
// are sure this is the correct path. Improves performance and error reporting.
func Cut() Parser {
	return func(ps *State) Result {
		ps.Cut = ps.Pos
		return Result{}
	}
}

// Regex returns a match if the regex successfully matches
func Regex(pattern string) Parser {
	re := regexp.MustCompile("^" + pattern)
	return NewParser(pattern, func(ps *State) Result {
		ps.AutoWS()
		if match := re.FindString(ps.Get()); match != "" {
			ps.Advance(len(match))
			return Result{Token: match}
		}
		ps.ErrorHere(pattern)
		return Result{}
	})
}

// Exact will fully match the exact string supplied, or error. The match will be stored in .Token
func Exact(match string) Parser {
	if len(match) == 1 {
		matchByte := match[0]
		return NewParser(match, func(ps *State) Result {
			ps.AutoWS()
			if ps.Pos >= len(ps.Input) || ps.Input[ps.Pos] != matchByte {
				ps.ErrorHere(match)
				return Result{}
			}

			ps.Advance(1)

			return Result{Token: match}
		})
	}

	return NewParser(match, func(ps *State) Result {
		ps.AutoWS()
		if !strings.HasPrefix(ps.Get(), match) {
			ps.ErrorHere(match)
			return Result{}
		}

		ps.Advance(len(match))

		return Result{Token: match}
	})
}

func parseRepetition(defaultMin, defaultMax int, repetition ...int) (min int, max int) {
	min = defaultMin
	max = defaultMax
	switch len(repetition) {
	case 0:
	case 1:
		min = repetition[0]
	case 2:
		min = repetition[0]
		max = repetition[1]
	default:
		panic(fmt.Errorf("Dont know what %d repetition args mean", len(repetition)))
	}
	return min, max
}

// parseMatcher turns a string in the format a-f01234A-F into:
//   - an alphabet of matches string(01234)
//   - a set of ranges [][]rune{{'a', 'f'}, {'A', 'F'}}
func parseMatcher(matcher string) (alphabet string, ranges [][]rune) {
	runes := []rune(matcher)

	for i := 0; i < len(runes); i++ {

		if i+2 < len(runes) && runes[i+1] == '-' && runes[i] != '\\' {
			start := runes[i]
			end := runes[i+2]
			if start <= end {
				ranges = append(ranges, []rune{start, end})
			} else {
				ranges = append(ranges, []rune{end, start})
			}
		} else if i+1 < len(runes) && runes[i] == '\\' {
			alphabet += string(runes[i+1])
		} else {
			alphabet += string(runes[i])
		}

	}

	return alphabet, ranges
}

// Chars is the swiss army knife of character matches. It can match:
//  - ranges: Chars("a-z") will match one or more lowercase letter
//  - alphabets: Chars("abcd") will match one or more of the letters abcd in any order
//  - min and max: Chars("a-z0-9", 4, 6) will match 4-6 lowercase alphanumeric characters
// the above can be combined in any order
func Chars(matcher string, repetition ...int) Parser {
	return NewParser("["+matcher+"]", charsImpl(matcher, false, repetition...))
}

// NotChars accepts the full range of input from Chars, but it will stop when any
// character matches.
func NotChars(matcher string, repetition ...int) Parser {
	return NewParser("!["+matcher+"]", charsImpl(matcher, true, repetition...))
}

func charsImpl(matcher string, stopOn bool, repetition ...int) Parser {
	min, max := parseRepetition(1, -1, repetition...)
	alphabet, ranges := parseMatcher(matcher)

	return func(ps *State) Result {
		ps.AutoWS()
		matched := 0
		for ps.Pos+matched < len(ps.Input) {
			if max != -1 && matched >= max {
				break
			}

			r, w := utf8.DecodeRuneInString(ps.Input[ps.Pos+matched:])

			anyMatched := strings.ContainsRune(alphabet, r)
			if !anyMatched {
				for _, rng := range ranges {
					if r >= rng[0] && r <= rng[1] {
						anyMatched = true
					}
				}
			}

			if anyMatched == stopOn {
				break
			}

			matched += w
		}

		if matched < min {
			ps.ErrorHere(matcher)
			return Result{}
		}

		result := ps.Input[ps.Pos : ps.Pos+matched]
		ps.Advance(matched)
		return Result{Token: result}
	}
}
<commit_msg>Fix godoc<commit_after>package goparsify

import (
	"errors"
	"fmt"
	"regexp"
	"strings"
	"unicode/utf8"
)

// Result is the output of a parser. Usually only one of its fields will be set and should be though of
// more as a union type. having it avoids interface{} littered all through the parsing code and makes
// the it easy to do the two most common operations, getting a token and finding a child.
type Result struct {
	Token  string
	Child  []Result
	Result interface{}
}

// Parser is the workhorse of parsify. A parser takes a State and returns a result, consuming some
// of the State in the process.
// Given state is shared there are a few rules that should be followed:
//  - A parser that errors must set state.Error
//  - A parser that errors must not change state.Pos
//  - A parser that consumed some input should advance state.Pos
type Parser func(*State) Result

// VoidParser is a special type of parser that never returns anything but can still consume input
type VoidParser func(*State)

// Parserish types are any type that can be turned into a Parser by Parsify
// These currently include *Parser and string literals.
//
// This makes recursive grammars cleaner and allows string literals to be used directly in most contexts.
// eg, matching balanced paren:
//  var group Parser
//  group = Seq("(", Maybe(&group), ")")
// vs
//  var group ParserPtr{}
//  group.P = Seq(Exact("("), Maybe(group.Parse), Exact(")"))
type Parserish interface{}

// Parsify takes a Parserish and makes a Parser out of it. It should be called by
// any Parser that accepts a Parser as an argument. It should never be called during
// instead call it during parser creation so there is no runtime cost.
//
// See Parserish for details.
func Parsify(p Parserish) Parser {
	switch p := p.(type) {
	case func(*State) Result:
		return p
	case Parser:
		return p
	case *Parser:
		// Todo: Maybe capture this stack and on nil show it? Is there a good error library to do this?
		return func(ptr *State) Result {
			return (*p)(ptr)
		}
	case string:
		return Exact(p)
	default:
		panic(fmt.Errorf("cant turn a `%T` into a parser", p))
	}
}

// ParsifyAll calls Parsify on all parsers
func ParsifyAll(parsers ...Parserish) []Parser {
	ret := make([]Parser, len(parsers))
	for i, parser := range parsers {
		ret[i] = Parsify(parser)
	}
	return ret
}

// Run applies some input to a parser and returns the result, failing if the input isnt fully consumed.
// It is a convenience method for the most common way to invoke a parser.
func Run(parser Parserish, input string, ws ...VoidParser) (result interface{}, err error) {
	p := Parsify(parser)
	ps := NewState(input)
	if len(ws) > 0 {
		ps.WS = ws[0]
	}

	ret := p(ps)
	ps.AutoWS()

	if ps.Error.expected != "" {
		return ret.Result, ps.Error
	}

	if ps.Get() != "" {
		return ret.Result, errors.New("left unparsed: " + ps.Get())
	}

	return ret.Result, nil
}

// WS will consume whitespace, it should only be needed when AutoWS is turned off
func WS() Parser {
	return NewParser("AutoWS", func(ps *State) Result {
		ps.WS(ps)
		return Result{}
	})
}

// Cut prevents backtracking beyond this point. Usually used after keywords when you
// are sure this is the correct path. Improves performance and error reporting.
func Cut() Parser {
	return func(ps *State) Result {
		ps.Cut = ps.Pos
		return Result{}
	}
}

// Regex returns a match if the regex successfully matches
func Regex(pattern string) Parser {
	re := regexp.MustCompile("^" + pattern)
	return NewParser(pattern, func(ps *State) Result {
		ps.AutoWS()
		if match := re.FindString(ps.Get()); match != "" {
			ps.Advance(len(match))
			return Result{Token: match}
		}
		ps.ErrorHere(pattern)
		return Result{}
	})
}

// Exact will fully match the exact string supplied, or error. The match will be stored in .Token
func Exact(match string) Parser {
	if len(match) == 1 {
		matchByte := match[0]
		return NewParser(match, func(ps *State) Result {
			ps.AutoWS()
			if ps.Pos >= len(ps.Input) || ps.Input[ps.Pos] != matchByte {
				ps.ErrorHere(match)
				return Result{}
			}

			ps.Advance(1)

			return Result{Token: match}
		})
	}

	return NewParser(match, func(ps *State) Result {
		ps.AutoWS()
		if !strings.HasPrefix(ps.Get(), match) {
			ps.ErrorHere(match)
			return Result{}
		}

		ps.Advance(len(match))

		return Result{Token: match}
	})
}

func parseRepetition(defaultMin, defaultMax int, repetition ...int) (min int, max int) {
	min = defaultMin
	max = defaultMax
	switch len(repetition) {
	case 0:
	case 1:
		min = repetition[0]
	case 2:
		min = repetition[0]
		max = repetition[1]
	default:
		panic(fmt.Errorf("Dont know what %d repetition args mean", len(repetition)))
	}
	return min, max
}

// parseMatcher turns a string in the format a-f01234A-F into:
//   - an alphabet of matches string(01234)
//   - a set of ranges [][]rune{{'a', 'f'}, {'A', 'F'}}
func parseMatcher(matcher string) (alphabet string, ranges [][]rune) {
	runes := []rune(matcher)

	for i := 0; i < len(runes); i++ {

		if i+2 < len(runes) && runes[i+1] == '-' && runes[i] != '\\' {
			start := runes[i]
			end := runes[i+2]
			if start <= end {
				ranges = append(ranges, []rune{start, end})
			} else {
				ranges = append(ranges, []rune{end, start})
			}
		} else if i+1 < len(runes) && runes[i] == '\\' {
			alphabet += string(runes[i+1])
		} else {
			alphabet += string(runes[i])
		}

	}

	return alphabet, ranges
}

// Chars is the swiss army knife of character matches. It can match:
//  - ranges: Chars("a-z") will match one or more lowercase letter
//  - alphabets: Chars("abcd") will match one or more of the letters abcd in any order
//  - min and max: Chars("a-z0-9", 4, 6) will match 4-6 lowercase alphanumeric characters
// the above can be combined in any order
func Chars(matcher string, repetition ...int) Parser {
	return NewParser("["+matcher+"]", charsImpl(matcher, false, repetition...))
}

// NotChars accepts the full range of input from Chars, but it will stop when any
// character matches.
func NotChars(matcher string, repetition ...int) Parser {
	return NewParser("!["+matcher+"]", charsImpl(matcher, true, repetition...))
}

func charsImpl(matcher string, stopOn bool, repetition ...int) Parser {
	min, max := parseRepetition(1, -1, repetition...)
	alphabet, ranges := parseMatcher(matcher)

	return func(ps *State) Result {
		ps.AutoWS()
		matched := 0
		for ps.Pos+matched < len(ps.Input) {
			if max != -1 && matched >= max {
				break
			}

			r, w := utf8.DecodeRuneInString(ps.Input[ps.Pos+matched:])

			anyMatched := strings.ContainsRune(alphabet, r)
			if !anyMatched {
				for _, rng := range ranges {
					if r >= rng[0] && r <= rng[1] {
						anyMatched = true
					}
				}
			}

			if anyMatched == stopOn {
				break
			}

			matched += w
		}

		if matched < min {
			ps.ErrorHere(matcher)
			return Result{}
		}

		result := ps.Input[ps.Pos : ps.Pos+matched]
		ps.Advance(matched)
		return Result{Token: result}
	}
}
<|endoftext|>