<commit_before>package main

import (
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"github.com/gorilla/mux"
	"github.com/minotar/minecraft"
)

type Router struct {
	Mux *mux.Router
}

// Middleware function to manipulate our request and response.
func imgdHandler(router http.Handler) http.Handler {
	return metricChain(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Methods", "GET")
		w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding")
		router.ServeHTTP(w, r)
	}))
}

func metricChain(router http.Handler) http.Handler {
	return promhttp.InstrumentHandlerInFlight(inFlightGauge,
		promhttp.InstrumentHandlerDuration(requestDuration,
			promhttp.InstrumentHandlerResponseSize(responseSize, router),
		),
	)
}

type NotFoundHandler struct{}

// Handles 404 errors
func (h NotFoundHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusNotFound)
	fmt.Fprintf(w, "404 not found")
	log.Infof("%s %s 404", r.RemoteAddr, r.RequestURI)
}

// GetWidth converts and sanitizes the string for the avatar width.
func (router *Router) GetWidth(inp string) uint {
	out64, err := strconv.ParseUint(inp, 10, 0)
	out := uint(out64)
	if err != nil {
		return DefaultWidth
	} else if out > MaxWidth {
		return MaxWidth
	} else if out < MinWidth {
		return MinWidth
	}
	return out

}

// SkinPage shows only the user's skin.
func (router *Router) SkinPage(w http.ResponseWriter, r *http.Request) {
	stats.Requested("Skin")
	vars := mux.Vars(r)
	username := vars["username"]
	skin := fetchSkin(username)

	if r.Header.Get("If-None-Match") == skin.Skin.Hash {
		w.WriteHeader(http.StatusNotModified)
		log.Infof("%s %s 304 %s", r.RemoteAddr, r.RequestURI, skin.Skin.Source)
		return
	}

	w.Header().Add("Cache-Control", fmt.Sprintf("public, max-age=%d", config.Server.Ttl))
	w.Header().Add("ETag", skin.Hash)
	w.Header().Add("Content-Type", "image/png")
	skin.WriteSkin(w)
	log.Infof("%s %s 200 %s", r.RemoteAddr, r.RequestURI, skin.Skin.Source)
}

// DownloadPage shows the skin and tells the browser to attempt to download it.
func (router *Router) DownloadPage(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Disposition", "attachment; filename=\"skin.png\"")
	router.SkinPage(w, r)
}

// ResolveMethod pulls the Get<resource> method from the skin. Originally this used
// reflection, but that was slow.
func (router *Router) ResolveMethod(skin *mcSkin, resource string) func(int) error {
	switch resource {
	case "Avatar":
		return skin.GetHead
	case "Helm":
		return skin.GetHelm
	case "Cube":
		return skin.GetCube
	case "Bust":
		return skin.GetBust
	case "Body":
		return skin.GetBody
	case "Armor/Bust":
		return skin.GetArmorBust
	case "Armour/Bust":
		return skin.GetArmorBust
	case "Armor/Body":
		return skin.GetArmorBody
	case "Armour/Body":
		return skin.GetArmorBody
	default:
		return skin.GetHelm
	}
}

func (router *Router) getResizeMode(ext string) string {
	switch ext {
	case ".svg":
		return "None"
	default:
		return "Normal"
	}
}

func (router *Router) writeType(ext string, skin *mcSkin, w http.ResponseWriter) {
	w.Header().Add("Cache-Control", fmt.Sprintf("public, max-age=%d", config.Server.Ttl))
	w.Header().Add("ETag", skin.Hash)
	switch ext {
	case ".svg":
		w.Header().Add("Content-Type", "image/svg+xml")
		skin.WriteSVG(w)
	default:
		w.Header().Add("Content-Type", "image/png")
		skin.WritePNG(w)
	}
}

// Serve binds the route and makes a handler function for the requested resource.
func (router *Router) Serve(resource string) {
	fn := func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		width := router.GetWidth(vars["width"])
		skin := fetchSkin(vars["username"])
		skin.Mode = router.getResizeMode(vars["extension"])
		stats.Requested(resource)

		if r.Header.Get("If-None-Match") == skin.Skin.Hash {
			w.WriteHeader(http.StatusNotModified)
			log.Infof("%s %s 304 %s", r.RemoteAddr, r.RequestURI, skin.Skin.Source)
			return
		}

		processingTimer := prometheus.NewTimer(processingDuration.WithLabelValues(resource))
		err := router.ResolveMethod(skin, resource)(int(width))
		processingTimer.ObserveDuration()
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "500 internal server error")
			log.Infof("%s %s 500 %s", r.RemoteAddr, r.RequestURI, skin.Skin.Source)
			stats.Errored("InternalServerError")
			return
		}
		router.writeType(vars["extension"], skin, w)
		log.Infof("%s %s 200 %s", r.RemoteAddr, r.RequestURI, skin.Skin.Source)
	}

	router.Mux.HandleFunc("/"+strings.ToLower(resource)+"/{username:"+minecraft.ValidUsernameRegex+"}{extension:(?:\\..*)?}", fn)
	router.Mux.HandleFunc("/"+strings.ToLower(resource)+"/{username:"+minecraft.ValidUsernameRegex+"}/{width:[0-9]+}{extension:(?:\\..*)?}", fn)
}

// Bind routes to the ServerMux.
func (router *Router) Bind() {

	router.Mux.NotFoundHandler = NotFoundHandler{}

	router.Serve("Avatar")
	router.Serve("Helm")
	router.Serve("Cube")
	router.Serve("Bust")
	router.Serve("Body")
	router.Serve("Armor/Bust")
	router.Serve("Armour/Bust")
	router.Serve("Armor/Body")
	router.Serve("Armour/Body")

	router.Mux.HandleFunc("/download/{username:"+minecraft.ValidUsernameRegex+"}{extension:(?:.png)?}", router.DownloadPage)
	router.Mux.HandleFunc("/skin/{username:"+minecraft.ValidUsernameRegex+"}{extension:(?:.png)?}", router.SkinPage)

	router.Mux.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "%s\n", ImgdVersion)
		log.Infof("%s %s 200", r.RemoteAddr, r.RequestURI)
	})

	router.Mux.Handle("/metrics", promhttp.Handler())

	router.Mux.HandleFunc("/stats", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		w.Write(stats.ToJSON())
		log.Infof("%s %s 200", r.RemoteAddr, r.RequestURI)
	})

	router.Mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, config.Server.URL, http.StatusFound)
		log.Infof("%s %s 200", r.RemoteAddr, r.RequestURI)
	})
}

func fetchSkin(username string) *mcSkin {
	if username == "char" || username == "MHF_Steve" {
		skin, _ := minecraft.FetchSkinForSteve()
		return &mcSkin{Skin: skin}
	}

	hasTimer := prometheus.NewTimer(cacheDuration.WithLabelValues("has"))
	if cache.has(strings.ToLower(username)) {
		hasTimer.ObserveDuration()
		pullTimer := prometheus.NewTimer(cacheDuration.WithLabelValues("pull"))
		defer pullTimer.ObserveDuration()
		stats.HitCache()
		return &mcSkin{Processed: nil, Skin: cache.pull(strings.ToLower(username))}
	}
	hasTimer.ObserveDuration()

	uuid, err := minecraft.NormalizePlayerForUUID(username)
	if err != nil {
		log.Debugf("Failed UUID lookup: %s (%s)", username, err.Error())
		skin, _ := minecraft.FetchSkinForSteve()
		stats.Errored("LookupUUID")
		return &mcSkin{Skin: skin}
	}

	sPTimer := prometheus.NewTimer(getDuration.WithLabelValues("SessionProfile"))
	skin, err := minecraft.FetchSkinUUID(uuid)
	sPTimer.ObserveDuration()
	if err != nil {
		log.Debugf("Failed Skin SessionProfile: %s (%s)", username, err.Error())
		// Let's fallback to S3 and try and serve at least an old skin...
		s3Timer := prometheus.NewTimer(getDuration.WithLabelValues("S3"))
		skin, err = minecraft.FetchSkinUsernameS3(username)
		s3Timer.ObserveDuration()
		if err != nil {
			log.Debugf("Failed Skin S3: %s (%s)", username, err.Error())
			// Well, looks like they don't exist after all.
			skin, _ = minecraft.FetchSkinForSteve()
			stats.Errored("FallbackSteve")
		} else {
			stats.Errored("FallbackUsernameS3")
		}
	}

	stats.MissCache()
	addTimer := prometheus.NewTimer(cacheDuration.WithLabelValues("add"))
	cache.add(strings.ToLower(username), skin)
	addTimer.ObserveDuration()
	return &mcSkin{Processed: nil, Skin: skin}
}
<commit_msg>UUID fallback logic for ratelimit<commit_after>package main

import (
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"github.com/gorilla/mux"
	"github.com/minotar/minecraft"
)

type Router struct {
	Mux *mux.Router
}

// Middleware function to manipulate our request and response.
func imgdHandler(router http.Handler) http.Handler {
	return metricChain(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Methods", "GET")
		w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding")
		router.ServeHTTP(w, r)
	}))
}

func metricChain(router http.Handler) http.Handler {
	return promhttp.InstrumentHandlerInFlight(inFlightGauge,
		promhttp.InstrumentHandlerDuration(requestDuration,
			promhttp.InstrumentHandlerResponseSize(responseSize, router),
		),
	)
}

type NotFoundHandler struct{}

// Handles 404 errors
func (h NotFoundHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusNotFound)
	fmt.Fprintf(w, "404 not found")
	log.Infof("%s %s 404", r.RemoteAddr, r.RequestURI)
}

// GetWidth converts and sanitizes the string for the avatar width.
func (router *Router) GetWidth(inp string) uint {
	out64, err := strconv.ParseUint(inp, 10, 0)
	out := uint(out64)
	if err != nil {
		return DefaultWidth
	} else if out > MaxWidth {
		return MaxWidth
	} else if out < MinWidth {
		return MinWidth
	}
	return out

}

// SkinPage shows only the user's skin.
func (router *Router) SkinPage(w http.ResponseWriter, r *http.Request) {
	stats.Requested("Skin")
	vars := mux.Vars(r)
	username := vars["username"]
	skin := fetchSkin(username)

	if r.Header.Get("If-None-Match") == skin.Skin.Hash {
		w.WriteHeader(http.StatusNotModified)
		log.Infof("%s %s 304 %s", r.RemoteAddr, r.RequestURI, skin.Skin.Source)
		return
	}

	w.Header().Add("Cache-Control", fmt.Sprintf("public, max-age=%d", config.Server.Ttl))
	w.Header().Add("ETag", skin.Hash)
	w.Header().Add("Content-Type", "image/png")
	skin.WriteSkin(w)
	log.Infof("%s %s 200 %s", r.RemoteAddr, r.RequestURI, skin.Skin.Source)
}

// DownloadPage shows the skin and tells the browser to attempt to download it.
func (router *Router) DownloadPage(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Disposition", "attachment; filename=\"skin.png\"")
	router.SkinPage(w, r)
}

// ResolveMethod pulls the Get<resource> method from the skin. Originally this used
// reflection, but that was slow.
func (router *Router) ResolveMethod(skin *mcSkin, resource string) func(int) error {
	switch resource {
	case "Avatar":
		return skin.GetHead
	case "Helm":
		return skin.GetHelm
	case "Cube":
		return skin.GetCube
	case "Bust":
		return skin.GetBust
	case "Body":
		return skin.GetBody
	case "Armor/Bust":
		return skin.GetArmorBust
	case "Armour/Bust":
		return skin.GetArmorBust
	case "Armor/Body":
		return skin.GetArmorBody
	case "Armour/Body":
		return skin.GetArmorBody
	default:
		return skin.GetHelm
	}
}

func (router *Router) getResizeMode(ext string) string {
	switch ext {
	case ".svg":
		return "None"
	default:
		return "Normal"
	}
}

func (router *Router) writeType(ext string, skin *mcSkin, w http.ResponseWriter) {
	w.Header().Add("Cache-Control", fmt.Sprintf("public, max-age=%d", config.Server.Ttl))
	w.Header().Add("ETag", skin.Hash)
	switch ext {
	case ".svg":
		w.Header().Add("Content-Type", "image/svg+xml")
		skin.WriteSVG(w)
	default:
		w.Header().Add("Content-Type", "image/png")
		skin.WritePNG(w)
	}
}

// Serve binds the route and makes a handler function for the requested resource.
func (router *Router) Serve(resource string) {
	fn := func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		width := router.GetWidth(vars["width"])
		skin := fetchSkin(vars["username"])
		skin.Mode = router.getResizeMode(vars["extension"])
		stats.Requested(resource)

		if r.Header.Get("If-None-Match") == skin.Skin.Hash {
			w.WriteHeader(http.StatusNotModified)
			log.Infof("%s %s 304 %s", r.RemoteAddr, r.RequestURI, skin.Skin.Source)
			return
		}

		processingTimer := prometheus.NewTimer(processingDuration.WithLabelValues(resource))
		err := router.ResolveMethod(skin, resource)(int(width))
		processingTimer.ObserveDuration()
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "500 internal server error")
			log.Infof("%s %s 500 %s", r.RemoteAddr, r.RequestURI, skin.Skin.Source)
			stats.Errored("InternalServerError")
			return
		}
		router.writeType(vars["extension"], skin, w)
		log.Infof("%s %s 200 %s", r.RemoteAddr, r.RequestURI, skin.Skin.Source)
	}

	router.Mux.HandleFunc("/"+strings.ToLower(resource)+"/{username:"+minecraft.ValidUsernameRegex+"}{extension:(?:\\..*)?}", fn)
	router.Mux.HandleFunc("/"+strings.ToLower(resource)+"/{username:"+minecraft.ValidUsernameRegex+"}/{width:[0-9]+}{extension:(?:\\..*)?}", fn)
}

// Bind routes to the ServerMux.
func (router *Router) Bind() {

	router.Mux.NotFoundHandler = NotFoundHandler{}

	router.Serve("Avatar")
	router.Serve("Helm")
	router.Serve("Cube")
	router.Serve("Bust")
	router.Serve("Body")
	router.Serve("Armor/Bust")
	router.Serve("Armour/Bust")
	router.Serve("Armor/Body")
	router.Serve("Armour/Body")

	router.Mux.HandleFunc("/download/{username:"+minecraft.ValidUsernameRegex+"}{extension:(?:.png)?}", router.DownloadPage)
	router.Mux.HandleFunc("/skin/{username:"+minecraft.ValidUsernameRegex+"}{extension:(?:.png)?}", router.SkinPage)

	router.Mux.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "%s\n", ImgdVersion)
		log.Infof("%s %s 200", r.RemoteAddr, r.RequestURI)
	})

	router.Mux.Handle("/metrics", promhttp.Handler())

	router.Mux.HandleFunc("/stats", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		w.Write(stats.ToJSON())
		log.Infof("%s %s 200", r.RemoteAddr, r.RequestURI)
	})

	router.Mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, config.Server.URL, http.StatusFound)
		log.Infof("%s %s 200", r.RemoteAddr, r.RequestURI)
	})
}

func fetchSkin(username string) *mcSkin {
	if username == "char" || username == "MHF_Steve" {
		skin, _ := minecraft.FetchSkinForSteve()
		return &mcSkin{Skin: skin}
	}

	hasTimer := prometheus.NewTimer(cacheDuration.WithLabelValues("has"))
	if cache.has(strings.ToLower(username)) {
		hasTimer.ObserveDuration()
		pullTimer := prometheus.NewTimer(cacheDuration.WithLabelValues("pull"))
		defer pullTimer.ObserveDuration()
		stats.HitCache()
		return &mcSkin{Processed: nil, Skin: cache.pull(strings.ToLower(username))}
	}
	hasTimer.ObserveDuration()
	stats.MissCache()

	// Everyone loves nested if statements, right?
	var skin minecraft.Skin
	uuid, err := minecraft.NormalizePlayerForUUID(username)
	if err != nil && err.Error() == "unable to GetAPIProfile: user not found" {
		log.Debugf("Failed UUID lookup: %s (%s)", username, err.Error())
		skin, _ = minecraft.FetchSkinForSteve()
		stats.Errored("UnknownUser")
		// Don't return yet to ensure we cache the failure
	} else {
		var catchErr error
		// Either no error, or there is one (eg. rate limit or network etc.), but they do possibly still exist
		if err != nil && err.Error() == "unable to GetAPIProfile: rate limited" {
			log.Noticef("Failed UUID lookup: %s (%s)", username, err.Error())
			stats.Errored("LookupUUIDRateLimit")
			catchErr = err
		} else if err != nil {
			// Other generic issues with looking up UUID, but still worth trying S3
			log.Infof("Failed UUID lookup: %s (%s)", username, err.Error())
			stats.Errored("LookupUUID")
			catchErr = err
		} else {
			// We have a UUID, so let's get a skin!
			sPTimer := prometheus.NewTimer(getDuration.WithLabelValues("SessionProfile"))
			skin, catchErr = minecraft.FetchSkinUUID(uuid)
			sPTimer.ObserveDuration()
			if catchErr != nil {
				log.Noticef("Failed Skin SessionProfile: %s (%s)", username, catchErr.Error())
				stats.Errored("SkinSessionProfile")
			}
		}
		if catchErr != nil {
			// Let's fallback to S3 and try and serve at least an old skin...
			s3Timer := prometheus.NewTimer(getDuration.WithLabelValues("S3"))
			skin, err = minecraft.FetchSkinUsernameS3(username)
			s3Timer.ObserveDuration()
			if err != nil {
				log.Debugf("Failed Skin S3: %s (%s)", username, err.Error())
				// Well, looks like they don't exist after all.
				skin, _ = minecraft.FetchSkinForSteve()
				stats.Errored("FallbackSteve")
			} else {
				stats.Errored("FallbackUsernameS3")
			}
		}
	}

	addTimer := prometheus.NewTimer(cacheDuration.WithLabelValues("add"))
	cache.add(strings.ToLower(username), skin)
	addTimer.ObserveDuration()
	return &mcSkin{Processed: nil, Skin: skin}
}
<|endoftext|>


<commit_before>package errors

import "fmt"

type HTTP struct {
	*primitive
	code int
}

func (h HTTP) Code() int {
	return h.code
}

func NewHTTP(cause error, code int, message string) error {
	return &HTTP{
		primitive: newPrimitive(cause, message),
		code:      code,
	}
}

func HTTPf(cause error, code int, format string, args ...interface{}) error {
	return &HTTP{
		primitive: newPrimitive(cause, fmt.Sprintf(format, args...)),
		code:      code,
	}
}
<commit_msg>Make HTTP error type an interface<commit_after>package errors

import "fmt"

type HTTP interface {
	error
	Code() int
}

type http struct {
	*primitive
	code int
}

func (h http) Code() int {
	return h.code
}

func NewHTTP(cause error, code int, message string) error {
	return &http{
		primitive: newPrimitive(cause, message),
		code:      code,
	}
}

func HTTPf(cause error, code int, format string, args ...interface{}) error {
	return &http{
		primitive: newPrimitive(cause, fmt.Sprintf(format, args...)),
		code:      code,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"log"
	"net/http"

	"github.com/elazarl/goproxy"
	"h12.me/socks"
)

// 启动本地 http(s) 代理
func startHttpProxy(listen string) {
	log.Printf("try start HTTP(s) %s, no backend\n", listen)

	proxy := goproxy.NewProxyHttpServer()
	proxy.Verbose = true
	log.Fatal(http.ListenAndServe(listen, proxy))
}

// 启动 http(s) 代理，使用一个 socks v5 服务器作为后端
func startHttpProxyByBackend(listen string, backend string) {
	log.Printf("try start HTTP(s) %s, use backend %s\n", listen, backend)

	var tlsClientSkipVerify = &tls.Config{InsecureSkipVerify: true}
	dialSocksProxy := socks.DialSocksProxy(socks.SOCKS5, backend)

	proxy := goproxy.NewProxyHttpServer()
	proxy.Tr = &http.Transport{
		TLSClientConfig: tlsClientSkipVerify,
		Proxy:           http.ProxyFromEnvironment,
		Dial:            dialSocksProxy}
	proxy.Verbose = true
	log.Fatal(http.ListenAndServe(listen, proxy))
}
<commit_msg>fix new address<commit_after>package main

import (
	"crypto/tls"
	"log"
	"net/http"

	"github.com/elazarl/goproxy"
	"h12.io/socks"
)

// 启动本地 http(s) 代理
func startHttpProxy(listen string) {
	log.Printf("try start HTTP(s) %s, no backend\n", listen)

	proxy := goproxy.NewProxyHttpServer()
	proxy.Verbose = true
	log.Fatal(http.ListenAndServe(listen, proxy))
}

// 启动 http(s) 代理，使用一个 socks v5 服务器作为后端
func startHttpProxyByBackend(listen string, backend string) {
	log.Printf("try start HTTP(s) %s, use backend %s\n", listen, backend)

	var tlsClientSkipVerify = &tls.Config{InsecureSkipVerify: true}
	dialSocksProxy := socks.DialSocksProxy(socks.SOCKS5, backend)

	proxy := goproxy.NewProxyHttpServer()
	proxy.Tr = &http.Transport{
		TLSClientConfig: tlsClientSkipVerify,
		Proxy:           http.ProxyFromEnvironment,
		Dial:            dialSocksProxy}
	proxy.Verbose = true
	log.Fatal(http.ListenAndServe(listen, proxy))
}
<|endoftext|>


<commit_before>// Copyright 2016 FullStory, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package solrmanapi

// TODO: rename 'NodeStatuses'
type SolrCloudStatus map[string]*SolrNodeStatus // keys are hostnames

type SolrNodeStatus struct {
	Hostname string                     // the node's hostname, as determined by gethostbyaddr
	NodeName string                     // the node's identifier within SolrCloud (e.g. "1.1.1.1:8983_solr")
	Cores    map[string]*SolrCoreStatus // keys are core names
}

type SolrCoreStatus struct {
	Name         string
	NodeName     string // the node's identifier within SolrCloud (e.g. "1.1.1.1:8983_solr")
	Collection   string // collection name (e.g. "1A00E" or "thefullstory.com")
	Shard        string // shard name (e.g. "shard1")
	ShardState   string // e.g. "active", "inactive"
	Range        string // // e.g. "80000000-b332ffff"
	Replica      string // replica name (e.g. "core_node_2")
	ReplicaState string // e.g. "active", "inactive", "recovering", "down"
	IsLeader     bool   // whether this replica is the leader of its shard
	HasStats     bool   // if false, core status could not be queried and following attributes are expected to be -1
	NumDocs      int64  // total number of indexed documents
	IndexSize    int64  // in bytes
}
<commit_msg>adding type property to SolrCoreStatus (#42)<commit_after>// Copyright 2016 FullStory, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package solrmanapi

// TODO: rename 'NodeStatuses'
type SolrCloudStatus map[string]*SolrNodeStatus // keys are hostnames

type SolrNodeStatus struct {
	Hostname string                     // the node's hostname, as determined by gethostbyaddr
	NodeName string                     // the node's identifier within SolrCloud (e.g. "1.1.1.1:8983_solr")
	Cores    map[string]*SolrCoreStatus // keys are core names
}

type SolrCoreStatus struct {
	Name         string
	NodeName     string // the node's identifier within SolrCloud (e.g. "1.1.1.1:8983_solr")
	Collection   string // collection name (e.g. "1A00E" or "thefullstory.com")
	Shard        string // shard name (e.g. "shard1")
	ShardState   string // e.g. "active", "inactive"
	Range        string // // e.g. "80000000-b332ffff"
	Replica      string // replica name (e.g. "core_node_2")
	ReplicaState string // e.g. "active", "inactive", "recovering", "down"
	IsLeader     bool   // whether this replica is the leader of its shard
	HasStats     bool   // if false, core status could not be queried and following attributes are expected to be -1
	NumDocs      int64  // total number of indexed documents
	IndexSize    int64  // in bytes
	Type         string // e.g. "NRT", "PULL", "TLOG"
}
<|endoftext|>


<commit_before>// Copyright 2013 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package i18n is for app Internationalization and Localization.
package i18n

import (
	"errors"
	"fmt"
	"reflect"
	"strings"

	"gopkg.in/ini.v1"
)

var (
	ErrLangAlreadyExist = errors.New("Lang already exists")

	locales = &localeStore{store: make(map[string]*locale)}
)

type locale struct {
	id       int
	lang     string
	langDesc string
	message  *ini.File
}

type localeStore struct {
	langs     []string
	langDescs []string
	store     map[string]*locale
}

// Get target language string
func (d *localeStore) Get(lang, section, format string) (string, bool) {
	if locale, ok := d.store[lang]; ok {
		if key, err := locale.message.Section(section).GetKey(format); err == nil {
			return key.Value(), true
		}
	}

	return "", false
}

func (d *localeStore) Add(lc *locale) bool {
	if _, ok := d.store[lc.lang]; ok {
		return false
	}

	lc.id = len(d.langs)
	d.langs = append(d.langs, lc.lang)
	d.langDescs = append(d.langDescs, lc.langDesc)
	d.store[lc.lang] = lc

	return true
}

func (d *localeStore) Reload(langs ...string) (err error) {
	if len(langs) == 0 {
		for _, lc := range d.store {
			if err = lc.message.Reload(); err != nil {
				return err
			}
		}
	} else {
		for _, lang := range langs {
			if lc, ok := d.store[lang]; ok {
				if err = lc.message.Reload(); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// ReloadLangs reloads locale files.
func ReloadLangs(langs ...string) error {
	return locales.Reload(langs...)
}

// Count returns number of languages that are registered.
func Count() int {
	return len(locales.langs)
}

// ListLangs returns list of all locale languages.
func ListLangs() []string {
	langs := make([]string, len(locales.langs))
	copy(langs, locales.langs)
	return langs
}

func ListLangDescs() []string {
	langDescs := make([]string, len(locales.langDescs))
	copy(langDescs, locales.langDescs)
	return langDescs
}

// IsExist returns true if given language locale exists.
func IsExist(lang string) bool {
	_, ok := locales.store[lang]
	return ok
}

// IndexLang returns index of language locale,
// it returns -1 if locale not exists.
func IndexLang(lang string) int {
	if lc, ok := locales.store[lang]; ok {
		return lc.id
	}
	return -1
}

// GetLangByIndex return language by given index.
func GetLangByIndex(index int) string {
	if index < 0 || index >= len(locales.langs) {
		return ""
	}
	return locales.langs[index]
}

func GetDescriptionByIndex(index int) string {
	if index < 0 || index >= len(locales.langDescs) {
		return ""
	}

	return locales.langDescs[index]
}

func GetDescriptionByLang(lang string) string {
	return GetDescriptionByIndex(IndexLang(lang))
}

func SetMessageWithDesc(lang, langDesc string, localeFile interface{}, otherLocaleFiles ...interface{}) error {
	message, err := ini.Load(localeFile, otherLocaleFiles...)
	if err == nil {
		message.BlockMode = false
		lc := new(locale)
		lc.lang = lang
		lc.langDesc = langDesc
		lc.message = message

		if locales.Add(lc) == false {
			return ErrLangAlreadyExist
		}
	}
	return err
}

// SetMessage sets the message file for localization.
func SetMessage(lang string, localeFile interface{}, otherLocaleFiles ...interface{}) error {
	return SetMessageWithDesc(lang, lang, localeFile, otherLocaleFiles...)
}

// Locale represents the information of localization.
type Locale struct {
	Lang string
}

// Tr translates content to target language.
func (l Locale) Tr(format string, args ...interface{}) string {
	return Tr(l.Lang, format, args...)
}

// Index returns lang index of LangStore.
func (l Locale) Index() int {
	return IndexLang(l.Lang)
}

// Tr translates content to target language.
func Tr(lang, format string, args ...interface{}) string {
	var section string
	parts := strings.SplitN(format, ".", 2)
	if len(parts) == 2 {
		section = parts[0]
		format = parts[1]
	}

	value, ok := locales.Get(lang, section, format)
	if ok {
		format = value
	}

	if len(args) > 0 {
		params := make([]interface{}, 0, len(args))
		for _, arg := range args {
			if arg != nil {
				val := reflect.ValueOf(arg)
				if val.Kind() == reflect.Slice {
					for i := 0; i < val.Len(); i++ {
						params = append(params, val.Index(i).Interface())
					}
				} else {
					params = append(params, arg)
				}
			}
		}
		return fmt.Sprintf(format, params...)
	}
	return format
}
<commit_msg>able to set default language<commit_after>// Copyright 2013 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package i18n is for app Internationalization and Localization.
package i18n

import (
	"errors"
	"fmt"
	"reflect"
	"strings"

	"gopkg.in/ini.v1"
)

var (
	ErrLangAlreadyExist = errors.New("Lang already exists")

	locales = &localeStore{store: make(map[string]*locale)}
)

type locale struct {
	id       int
	lang     string
	langDesc string
	message  *ini.File
}

type localeStore struct {
	langs       []string
	langDescs   []string
	store       map[string]*locale
	defaultLang string
}

// Get target language string
func (d *localeStore) Get(lang, section, format string) (string, bool) {
	if locale, ok := d.store[lang]; ok {
		if key, err := locale.message.Section(section).GetKey(format); err == nil {
			return key.Value(), true
		}
	}

	if len(d.defaultLang) > 0 && lang != d.defaultLang {
		return d.Get(d.defaultLang, section, format)
	}

	return "", false
}

func (d *localeStore) Add(lc *locale) bool {
	if _, ok := d.store[lc.lang]; ok {
		return false
	}

	lc.id = len(d.langs)
	d.langs = append(d.langs, lc.lang)
	d.langDescs = append(d.langDescs, lc.langDesc)
	d.store[lc.lang] = lc

	return true
}

func (d *localeStore) Reload(langs ...string) (err error) {
	if len(langs) == 0 {
		for _, lc := range d.store {
			if err = lc.message.Reload(); err != nil {
				return err
			}
		}
	} else {
		for _, lang := range langs {
			if lc, ok := d.store[lang]; ok {
				if err = lc.message.Reload(); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// SetDefaultLang sets default language which is a indicator that
// when target language is not found, try find in default language again.
func SetDefaultLang(lang string) {
	locales.defaultLang = lang
}

// ReloadLangs reloads locale files.
func ReloadLangs(langs ...string) error {
	return locales.Reload(langs...)
}

// Count returns number of languages that are registered.
func Count() int {
	return len(locales.langs)
}

// ListLangs returns list of all locale languages.
func ListLangs() []string {
	langs := make([]string, len(locales.langs))
	copy(langs, locales.langs)
	return langs
}

func ListLangDescs() []string {
	langDescs := make([]string, len(locales.langDescs))
	copy(langDescs, locales.langDescs)
	return langDescs
}

// IsExist returns true if given language locale exists.
func IsExist(lang string) bool {
	_, ok := locales.store[lang]
	return ok
}

// IndexLang returns index of language locale,
// it returns -1 if locale not exists.
func IndexLang(lang string) int {
	if lc, ok := locales.store[lang]; ok {
		return lc.id
	}
	return -1
}

// GetLangByIndex return language by given index.
func GetLangByIndex(index int) string {
	if index < 0 || index >= len(locales.langs) {
		return ""
	}
	return locales.langs[index]
}

func GetDescriptionByIndex(index int) string {
	if index < 0 || index >= len(locales.langDescs) {
		return ""
	}

	return locales.langDescs[index]
}

func GetDescriptionByLang(lang string) string {
	return GetDescriptionByIndex(IndexLang(lang))
}

func SetMessageWithDesc(lang, langDesc string, localeFile interface{}, otherLocaleFiles ...interface{}) error {
	message, err := ini.Load(localeFile, otherLocaleFiles...)
	if err == nil {
		message.BlockMode = false
		lc := new(locale)
		lc.lang = lang
		lc.langDesc = langDesc
		lc.message = message

		if locales.Add(lc) == false {
			return ErrLangAlreadyExist
		}
	}
	return err
}

// SetMessage sets the message file for localization.
func SetMessage(lang string, localeFile interface{}, otherLocaleFiles ...interface{}) error {
	return SetMessageWithDesc(lang, lang, localeFile, otherLocaleFiles...)
}

// Locale represents the information of localization.
type Locale struct {
	Lang string
}

// Tr translates content to target language.
func (l Locale) Tr(format string, args ...interface{}) string {
	return Tr(l.Lang, format, args...)
}

// Index returns lang index of LangStore.
func (l Locale) Index() int {
	return IndexLang(l.Lang)
}

// Tr translates content to target language.
func Tr(lang, format string, args ...interface{}) string {
	var section string
	parts := strings.SplitN(format, ".", 2)
	if len(parts) == 2 {
		section = parts[0]
		format = parts[1]
	}

	value, ok := locales.Get(lang, section, format)
	if ok {
		format = value
	}

	if len(args) > 0 {
		params := make([]interface{}, 0, len(args))
		for _, arg := range args {
			if arg != nil {
				val := reflect.ValueOf(arg)
				if val.Kind() == reflect.Slice {
					for i := 0; i < val.Len(); i++ {
						params = append(params, val.Index(i).Interface())
					}
				} else {
					params = append(params, arg)
				}
			}
		}
		return fmt.Sprintf(format, params...)
	}
	return format
}
<|endoftext|>


<commit_before>package parser

import (
	"github.com/fxnn/gowatch/logentry"
	"github.com/stretchr/testify/require"
	"testing"
	"time"
)

func TestGrok_MultipleFields(t *testing.T) {
	linesource := givenLineSource(t, "WARNING This is the message")

	parser := grokParserWithLinesourceAndPattern(linesource, "%{LOGLEVEL:Level} %{DATA:Message}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, logentry.WARNING, resultEntry.Level)
	require.Equal(t, "This is the message", resultEntry.Message)
}

func TestGrok_SingleMessage(t *testing.T) {
	linesource := givenLineSource(t, "abc")

	parser := grokParserWithLinesourceAndPattern(linesource, "^%{DATA:Message}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, "abc", resultEntry.Message)
}

func TestGrok_SingleTag(t *testing.T) {
	linesource := givenLineSource(t, "abc")

	parser := grokParserWithLinesourceAndPattern(linesource, "^%{DATA:Tags}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, []string{"abc"}, resultEntry.Tags)
}

func TestGrok_MultipleTags(t *testing.T) {
	linesource := givenLineSource(t, "abc def")

	parser := grokParserWithLinesourceAndPattern(linesource, "^%{DATA:Tags} %{DATA:Tags}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, []string{"abc", "def"}, resultEntry.Tags)
}

func TestGrok_SingleLogLevel(t *testing.T) {
	linesource := givenLineSource(t, "DEBUG")

	parser := grokParserWithLinesourceAndPattern(linesource, "^%{LOGLEVEL:Level}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, logentry.DEBUG, resultEntry.Level)
}

func TestGrok_SingleCustomEntry(t *testing.T) {
	linesource := givenLineSource(t, "abc")

	parser := grokParserWithLinesourceAndPattern(linesource, "^%{DATA:MyCustomEntry}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, "abc", resultEntry.Custom["MyCustomEntry"])
}

func TestGrok_MultipleCustomEntries(t *testing.T) {
	linesource := givenLineSource(t, "28.03.2015 abc")

	parser := grokParserWithLinesourceAndPattern(linesource, "^%{DATE:CustomDate} %{USER:MyCustomEntry}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, "28.03.2015", resultEntry.Custom["CustomDate"])
	require.Equal(t, "abc", resultEntry.Custom["MyCustomEntry"])
}

func TestGrok_Predicate(t *testing.T) {
	linesource := givenLineSource(t, "abc")

	parser := grokParserWithLinesourceAndPredicate(linesource, &logentry.ContainsPredicate{FieldName: "Message", ToBeContained: "xyz"})
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.Equal(t, logentry.LogEntry{}, resultEntry) // zero value --> no element in channel
}

func TestGrok_TimeLayout(t *testing.T) {
	linesource := givenLineSource(t, "Tue, 10 Nov 2009 23:00:00 +0000")

	parser := grokParserWithLinesourceAndPattern(linesource, time.RFC1123Z)
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.Equal(t, time.Parse(time.RFC1123Z, "Tue, 10 Nov 2009 23:00:00 +0000"), resultEntry.Timestamp)
}

func grokParserWithLinesourceAndTimeLayout(linesource LineSource, timeLayout string) *GrokParser {
	return NewGrokParser(linesource, "^%{DATA:timestamp}$", timeLayout, acceptAllPredicate())
}

func grokParserWithLinesourceAndPredicate(linesource LineSource, predicate logentry.Predicate) *GrokParser {
	return NewGrokParser(linesource, "^%{DATA:Message}$", "", predicate)
}

func grokParserWithLinesourceAndPattern(linesource LineSource, pattern string) *GrokParser {
	return NewGrokParser(linesource, pattern, "", acceptAllPredicate())
}

func acceptAllPredicate() logentry.Predicate {
	return &logentry.AcceptAllPredicate{}
}
<commit_msg>fixed unit test<commit_after>package parser

import (
	"github.com/fxnn/gowatch/logentry"
	"github.com/stretchr/testify/require"
	"testing"
	"time"
)

func TestGrok_MultipleFields(t *testing.T) {
	linesource := givenLineSource(t, "WARNING This is the message")

	parser := grokParserWithLinesourceAndPattern(linesource, "%{LOGLEVEL:Level} %{DATA:Message}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, logentry.WARNING, resultEntry.Level)
	require.Equal(t, "This is the message", resultEntry.Message)
}

func TestGrok_SingleMessage(t *testing.T) {
	linesource := givenLineSource(t, "abc")

	parser := grokParserWithLinesourceAndPattern(linesource, "^%{DATA:Message}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, "abc", resultEntry.Message)
}

func TestGrok_SingleTag(t *testing.T) {
	linesource := givenLineSource(t, "abc")

	parser := grokParserWithLinesourceAndPattern(linesource, "^%{DATA:Tags}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, []string{"abc"}, resultEntry.Tags)
}

func TestGrok_MultipleTags(t *testing.T) {
	linesource := givenLineSource(t, "abc def")

	parser := grokParserWithLinesourceAndPattern(linesource, "^%{DATA:Tags} %{DATA:Tags}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, []string{"abc", "def"}, resultEntry.Tags)
}

func TestGrok_SingleLogLevel(t *testing.T) {
	linesource := givenLineSource(t, "DEBUG")

	parser := grokParserWithLinesourceAndPattern(linesource, "^%{LOGLEVEL:Level}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, logentry.DEBUG, resultEntry.Level)
}

func TestGrok_SingleCustomEntry(t *testing.T) {
	linesource := givenLineSource(t, "abc")

	parser := grokParserWithLinesourceAndPattern(linesource, "^%{DATA:MyCustomEntry}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, "abc", resultEntry.Custom["MyCustomEntry"])
}

func TestGrok_MultipleCustomEntries(t *testing.T) {
	linesource := givenLineSource(t, "28.03.2015 abc")

	parser := grokParserWithLinesourceAndPattern(linesource, "^%{DATE:CustomDate} %{USER:MyCustomEntry}$")
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.NotNil(t, resultEntry)
	require.Equal(t, "28.03.2015", resultEntry.Custom["CustomDate"])
	require.Equal(t, "abc", resultEntry.Custom["MyCustomEntry"])
}

func TestGrok_Predicate(t *testing.T) {
	linesource := givenLineSource(t, "abc")

	parser := grokParserWithLinesourceAndPredicate(linesource, &logentry.ContainsPredicate{FieldName: "Message", ToBeContained: "xyz"})
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.Equal(t, logentry.LogEntry{}, resultEntry) // zero value --> no element in channel
}

func TestGrok_TimeLayout(t *testing.T) {
	linesource := givenLineSource(t, "Tue, 10 Nov 2009 23:00:00 +0000")
	expectedTime, _ := time.Parse(time.RFC1123Z, "Tue, 10 Nov 2009 23:00:00 +0000")

	parser := grokParserWithLinesourceAndTimeLayout(linesource, time.RFC1123Z)
	result := parser.Parse()

	require.NotNil(t, result)

	resultEntry := <-result
	require.Equal(t, expectedTime, resultEntry.Timestamp)
}

func grokParserWithLinesourceAndTimeLayout(linesource LineSource, timeLayout string) *GrokParser {
	return NewGrokParser(linesource, "^%{DATA:timestamp}$", timeLayout, acceptAllPredicate())
}

func grokParserWithLinesourceAndPredicate(linesource LineSource, predicate logentry.Predicate) *GrokParser {
	return NewGrokParser(linesource, "^%{DATA:Message}$", "", predicate)
}

func grokParserWithLinesourceAndPattern(linesource LineSource, pattern string) *GrokParser {
	return NewGrokParser(linesource, pattern, "", acceptAllPredicate())
}

func acceptAllPredicate() logentry.Predicate {
	return &logentry.AcceptAllPredicate{}
}
<|endoftext|>


<commit_before>package upload

import (
	"net/url"
	"strings"
	"testing"

	"github.com/webx-top/echo/testing/test"
)

func TestUploadURL(t *testing.T) {
	urls := BackendUploadURL(`/manager/upload/movie`, `refid`, `123`)
	values, err := url.ParseQuery(strings.SplitN(urls, `?`, 2)[1])
	if err != nil {
		t.Fatal(err)
	}
	//com.Dump(values)
	test.True(t, strings.HasPrefix(urls, `/manager/upload//manager/upload/movie?refid=123&time=`))
	token := values.Get(`token`)
	values.Del(`token`)
	test.Eq(t, token, Token(values))
	test.Eq(t, token, Token(`refid`, values.Get(`refid`), `time`, values.Get(`time`)))
}
<commit_msg>update<commit_after>package upload

import (
	"fmt"
	"net/url"
	"strings"
	"testing"

	"github.com/webx-top/echo/testing/test"
)

func TestUploadURL(t *testing.T) {
	urls := BackendUploadURL(`movie`, `refid`, `123`)
	values, err := url.ParseQuery(strings.SplitN(urls, `?`, 2)[1])
	if err != nil {
		t.Fatal(err)
	}
	fmt.Println(urls)
	//com.Dump(values)
	test.True(t, strings.HasPrefix(urls, `/manager/upload/movie?refid=123&time=`))
	token := values.Get(`token`)
	values.Del(`token`)
	test.Eq(t, token, Token(values))
	test.Eq(t, token, Token(`refid`, values.Get(`refid`), `time`, values.Get(`time`)))
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fsnotify

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"sync"
	"syscall"
	"unsafe"
)

const (
	// Options for inotify_init() are not exported
	// sys_IN_CLOEXEC    uint32 = syscall.IN_CLOEXEC
	// sys_IN_NONBLOCK   uint32 = syscall.IN_NONBLOCK

	// Options for AddWatch
	sys_IN_DONT_FOLLOW uint32 = syscall.IN_DONT_FOLLOW
	sys_IN_ONESHOT     uint32 = syscall.IN_ONESHOT
	sys_IN_ONLYDIR     uint32 = syscall.IN_ONLYDIR

	// The "sys_IN_MASK_ADD" option is not exported, as AddWatch
	// adds it automatically, if there is already a watch for the given path
	// sys_IN_MASK_ADD      uint32 = syscall.IN_MASK_ADD

	// Events
	sys_IN_ACCESS        uint32 = syscall.IN_ACCESS
	sys_IN_ALL_EVENTS    uint32 = syscall.IN_ALL_EVENTS
	sys_IN_ATTRIB        uint32 = syscall.IN_ATTRIB
	sys_IN_CLOSE         uint32 = syscall.IN_CLOSE
	sys_IN_CLOSE_NOWRITE uint32 = syscall.IN_CLOSE_NOWRITE
	sys_IN_CLOSE_WRITE   uint32 = syscall.IN_CLOSE_WRITE
	sys_IN_CREATE        uint32 = syscall.IN_CREATE
	sys_IN_DELETE        uint32 = syscall.IN_DELETE
	sys_IN_DELETE_SELF   uint32 = syscall.IN_DELETE_SELF
	sys_IN_MODIFY        uint32 = syscall.IN_MODIFY
	sys_IN_MOVE          uint32 = syscall.IN_MOVE
	sys_IN_MOVED_FROM    uint32 = syscall.IN_MOVED_FROM
	sys_IN_MOVED_TO      uint32 = syscall.IN_MOVED_TO
	sys_IN_MOVE_SELF     uint32 = syscall.IN_MOVE_SELF
	sys_IN_OPEN          uint32 = syscall.IN_OPEN

	sys_AGNOSTIC_EVENTS = sys_IN_MOVED_TO | sys_IN_MOVED_FROM | sys_IN_CREATE | sys_IN_ATTRIB | sys_IN_MODIFY | sys_IN_MOVE_SELF | sys_IN_DELETE | sys_IN_DELETE_SELF

	// Special events
	sys_IN_ISDIR      uint32 = syscall.IN_ISDIR
	sys_IN_IGNORED    uint32 = syscall.IN_IGNORED
	sys_IN_Q_OVERFLOW uint32 = syscall.IN_Q_OVERFLOW
	sys_IN_UNMOUNT    uint32 = syscall.IN_UNMOUNT
)

func newEvent(name string, mask uint32) *Event {
	e := &Event{Name: name}
	if mask&sys_IN_CREATE == sys_IN_CREATE || mask&sys_IN_MOVED_TO == sys_IN_MOVED_TO {
		e.Op |= Create
	}
	if mask&sys_IN_DELETE_SELF == sys_IN_DELETE_SELF || mask&sys_IN_DELETE == sys_IN_DELETE {
		e.Op |= Remove
	}
	if mask&sys_IN_MODIFY == sys_IN_MODIFY || mask&sys_IN_ATTRIB == sys_IN_ATTRIB {
		e.Op |= Write
	}
	if mask&sys_IN_MOVE_SELF == sys_IN_MOVE_SELF || mask&sys_IN_MOVED_FROM == sys_IN_MOVED_FROM {
		e.Op |= Rename
	}
	if mask&sys_IN_ATTRIB == sys_IN_ATTRIB {
		e.Op |= Chmod
	}
	return e
}

type watch struct {
	wd    uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall)
	flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags)
}

type Watcher struct {
	mu       sync.Mutex        // Map access
	fd       int               // File descriptor (as returned by the inotify_init() syscall)
	watches  map[string]*watch // Map of inotify watches (key: path)
	paths    map[int]string    // Map of watched paths (key: watch descriptor)
	Errors   chan error        // Errors are sent on this channel
	Events   chan *Event       // Events are returned on this channel
	done     chan bool         // Channel for sending a "quit message" to the reader goroutine
	isClosed bool              // Set to true when Close() is first called
}

// NewWatcher creates and returns a new inotify instance using inotify_init(2)
func NewWatcher() (*Watcher, error) {
	fd, errno := syscall.InotifyInit()
	if fd == -1 {
		return nil, os.NewSyscallError("inotify_init", errno)
	}
	w := &Watcher{
		fd:      fd,
		watches: make(map[string]*watch),
		paths:   make(map[int]string),
		Events:  make(chan *Event),
		Errors:  make(chan error),
		done:    make(chan bool, 1),
	}

	go w.readEvents()
	return w, nil
}

// Close closes an inotify watcher instance
// It sends a message to the reader goroutine to quit and removes all watches
// associated with the inotify instance
func (w *Watcher) Close() error {
	if w.isClosed {
		return nil
	}
	w.isClosed = true

	// Remove all watches
	for path := range w.watches {
		w.Remove(path)
	}

	// Send "quit" message to the reader goroutine
	w.done <- true

	return nil
}

// AddWatch adds path to the watched file set.
// The flags are interpreted as described in inotify_add_watch(2).
func (w *Watcher) addWatch(path string, flags uint32) error {
	if w.isClosed {
		return errors.New("inotify instance already closed")
	}

	w.mu.Lock()
	watchEntry, found := w.watches[path]
	w.mu.Unlock()
	if found {
		watchEntry.flags |= flags
		flags |= syscall.IN_MASK_ADD
	}
	wd, errno := syscall.InotifyAddWatch(w.fd, path, flags)
	if wd == -1 {
		return errno
	}

	w.mu.Lock()
	w.watches[path] = &watch{wd: uint32(wd), flags: flags}
	w.paths[wd] = path
	w.mu.Unlock()

	return nil
}

// Add starts watching on the named file.
func (w *Watcher) Add(path string) error {
	return w.addWatch(path, sys_AGNOSTIC_EVENTS)
}

// Remove stops watching on the named file.
func (w *Watcher) Remove(path string) error {
	w.mu.Lock()
	defer w.mu.Unlock()
	watch, ok := w.watches[path]
	if !ok {
		return errors.New(fmt.Sprintf("can't remove non-existent inotify watch for: %s", path))
	}
	success, errno := syscall.InotifyRmWatch(w.fd, watch.wd)
	if success == -1 {
		return os.NewSyscallError("inotify_rm_watch", errno)
	}
	delete(w.watches, path)
	return nil
}

// readEvents reads from the inotify file descriptor, converts the
// received events into Event objects and sends them via the Events channel
func (w *Watcher) readEvents() {
	var (
		buf   [syscall.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events
		n     int                                     // Number of bytes read with read()
		errno error                                   // Syscall errno
	)

	for {
		// See if there is a message on the "done" channel
		select {
		case <-w.done:
			syscall.Close(w.fd)
			close(w.Events)
			close(w.Errors)
			return
		default:
		}

		n, errno = syscall.Read(w.fd, buf[:])

		// If EOF is received
		if n == 0 {
			syscall.Close(w.fd)
			close(w.Events)
			close(w.Errors)
			return
		}

		if n < 0 {
			w.Errors <- os.NewSyscallError("read", errno)
			continue
		}
		if n < syscall.SizeofInotifyEvent {
			w.Errors <- errors.New("inotify: short read in readEvents()")
			continue
		}

		var offset uint32 = 0
		// We don't know how many events we just read into the buffer
		// While the offset points to at least one whole event...
		for offset <= uint32(n-syscall.SizeofInotifyEvent) {
			// Point "raw" to the event in the buffer
			raw := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[offset]))

			mask := uint32(raw.Mask)
			nameLen := uint32(raw.Len)
			// If the event happened to the watched directory or the watched file, the kernel
			// doesn't append the filename to the event, but we would like to always fill the
			// the "Name" field with a valid filename. We retrieve the path of the watch from
			// the "paths" map.
			w.mu.Lock()
			name := w.paths[int(raw.Wd)]
			w.mu.Unlock()
			if nameLen > 0 {
				// Point "bytes" at the first byte of the filename
				bytes := (*[syscall.PathMax]byte)(unsafe.Pointer(&buf[offset+syscall.SizeofInotifyEvent]))
				// The filename is padded with NULL bytes. TrimRight() gets rid of those.
				name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000")
			}

			event := newEvent(name, mask)

			// Send the events that are not ignored on the events channel
			if !event.ignoreLinux(mask) {
				w.Events <- event
			}

			// Move to the next event in the buffer
			offset += syscall.SizeofInotifyEvent + nameLen
		}
	}
}

// Certain types of events can be "ignored" and not sent over the Events
// channel. Such as events marked ignore by the kernel, or MODIFY events
// against files that do not exist.
func (e *Event) ignoreLinux(mask uint32) bool {
	// Ignore anything the inotify API says to ignore
	if mask&sys_IN_IGNORED == sys_IN_IGNORED {
		return true
	}

	// If the event is not a DELETE or RENAME, the file must exist.
	// Otherwise the event is ignored.
	// *Note*: this was put in place because it was seen that a MODIFY
	// event was sent after the DELETE. This ignores that MODIFY and
	// assumes a DELETE will come or has come if the file doesn't exist.
	if !(e.Op&Remove == Remove || e.Op&Rename == Rename) {
		_, statErr := os.Lstat(e.Name)
		return os.IsNotExist(statErr)
	}
	return false
}
<commit_msg>inotify: use syscall constants directly<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fsnotify

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"sync"
	"syscall"
	"unsafe"
)

const (
	sys_AGNOSTIC_EVENTS = syscall.IN_MOVED_TO | syscall.IN_MOVED_FROM |
		syscall.IN_CREATE | syscall.IN_ATTRIB | syscall.IN_MODIFY |
		syscall.IN_MOVE_SELF | syscall.IN_DELETE | syscall.IN_DELETE_SELF
)

func newEvent(name string, mask uint32) *Event {
	e := &Event{Name: name}
	if mask&syscall.IN_CREATE == syscall.IN_CREATE || mask&syscall.IN_MOVED_TO == syscall.IN_MOVED_TO {
		e.Op |= Create
	}
	if mask&syscall.IN_DELETE_SELF == syscall.IN_DELETE_SELF || mask&syscall.IN_DELETE == syscall.IN_DELETE {
		e.Op |= Remove
	}
	if mask&syscall.IN_MODIFY == syscall.IN_MODIFY || mask&syscall.IN_ATTRIB == syscall.IN_ATTRIB {
		e.Op |= Write
	}
	if mask&syscall.IN_MOVE_SELF == syscall.IN_MOVE_SELF || mask&syscall.IN_MOVED_FROM == syscall.IN_MOVED_FROM {
		e.Op |= Rename
	}
	if mask&syscall.IN_ATTRIB == syscall.IN_ATTRIB {
		e.Op |= Chmod
	}
	return e
}

type watch struct {
	wd    uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall)
	flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags)
}

type Watcher struct {
	mu       sync.Mutex        // Map access
	fd       int               // File descriptor (as returned by the inotify_init() syscall)
	watches  map[string]*watch // Map of inotify watches (key: path)
	paths    map[int]string    // Map of watched paths (key: watch descriptor)
	Errors   chan error        // Errors are sent on this channel
	Events   chan *Event       // Events are returned on this channel
	done     chan bool         // Channel for sending a "quit message" to the reader goroutine
	isClosed bool              // Set to true when Close() is first called
}

// NewWatcher creates and returns a new inotify instance using inotify_init(2)
func NewWatcher() (*Watcher, error) {
	fd, errno := syscall.InotifyInit()
	if fd == -1 {
		return nil, os.NewSyscallError("inotify_init", errno)
	}
	w := &Watcher{
		fd:      fd,
		watches: make(map[string]*watch),
		paths:   make(map[int]string),
		Events:  make(chan *Event),
		Errors:  make(chan error),
		done:    make(chan bool, 1),
	}

	go w.readEvents()
	return w, nil
}

// Close closes an inotify watcher instance
// It sends a message to the reader goroutine to quit and removes all watches
// associated with the inotify instance
func (w *Watcher) Close() error {
	if w.isClosed {
		return nil
	}
	w.isClosed = true

	// Remove all watches
	for path := range w.watches {
		w.Remove(path)
	}

	// Send "quit" message to the reader goroutine
	w.done <- true

	return nil
}

// AddWatch adds path to the watched file set.
// The flags are interpreted as described in inotify_add_watch(2).
func (w *Watcher) addWatch(path string, flags uint32) error {
	if w.isClosed {
		return errors.New("inotify instance already closed")
	}

	w.mu.Lock()
	watchEntry, found := w.watches[path]
	w.mu.Unlock()
	if found {
		watchEntry.flags |= flags
		flags |= syscall.IN_MASK_ADD
	}
	wd, errno := syscall.InotifyAddWatch(w.fd, path, flags)
	if wd == -1 {
		return errno
	}

	w.mu.Lock()
	w.watches[path] = &watch{wd: uint32(wd), flags: flags}
	w.paths[wd] = path
	w.mu.Unlock()

	return nil
}

// Add starts watching on the named file.
func (w *Watcher) Add(path string) error {
	return w.addWatch(path, sys_AGNOSTIC_EVENTS)
}

// Remove stops watching on the named file.
func (w *Watcher) Remove(path string) error {
	w.mu.Lock()
	defer w.mu.Unlock()
	watch, ok := w.watches[path]
	if !ok {
		return errors.New(fmt.Sprintf("can't remove non-existent inotify watch for: %s", path))
	}
	success, errno := syscall.InotifyRmWatch(w.fd, watch.wd)
	if success == -1 {
		return os.NewSyscallError("inotify_rm_watch", errno)
	}
	delete(w.watches, path)
	return nil
}

// readEvents reads from the inotify file descriptor, converts the
// received events into Event objects and sends them via the Events channel
func (w *Watcher) readEvents() {
	var (
		buf   [syscall.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events
		n     int                                     // Number of bytes read with read()
		errno error                                   // Syscall errno
	)

	for {
		// See if there is a message on the "done" channel
		select {
		case <-w.done:
			syscall.Close(w.fd)
			close(w.Events)
			close(w.Errors)
			return
		default:
		}

		n, errno = syscall.Read(w.fd, buf[:])

		// If EOF is received
		if n == 0 {
			syscall.Close(w.fd)
			close(w.Events)
			close(w.Errors)
			return
		}

		if n < 0 {
			w.Errors <- os.NewSyscallError("read", errno)
			continue
		}
		if n < syscall.SizeofInotifyEvent {
			w.Errors <- errors.New("inotify: short read in readEvents()")
			continue
		}

		var offset uint32 = 0
		// We don't know how many events we just read into the buffer
		// While the offset points to at least one whole event...
		for offset <= uint32(n-syscall.SizeofInotifyEvent) {
			// Point "raw" to the event in the buffer
			raw := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[offset]))

			mask := uint32(raw.Mask)
			nameLen := uint32(raw.Len)
			// If the event happened to the watched directory or the watched file, the kernel
			// doesn't append the filename to the event, but we would like to always fill the
			// the "Name" field with a valid filename. We retrieve the path of the watch from
			// the "paths" map.
			w.mu.Lock()
			name := w.paths[int(raw.Wd)]
			w.mu.Unlock()
			if nameLen > 0 {
				// Point "bytes" at the first byte of the filename
				bytes := (*[syscall.PathMax]byte)(unsafe.Pointer(&buf[offset+syscall.SizeofInotifyEvent]))
				// The filename is padded with NULL bytes. TrimRight() gets rid of those.
				name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000")
			}

			event := newEvent(name, mask)

			// Send the events that are not ignored on the events channel
			if !event.ignoreLinux(mask) {
				w.Events <- event
			}

			// Move to the next event in the buffer
			offset += syscall.SizeofInotifyEvent + nameLen
		}
	}
}

// Certain types of events can be "ignored" and not sent over the Events
// channel. Such as events marked ignore by the kernel, or MODIFY events
// against files that do not exist.
func (e *Event) ignoreLinux(mask uint32) bool {
	// Ignore anything the inotify API says to ignore
	if mask&syscall.IN_IGNORED == syscall.IN_IGNORED {
		return true
	}

	// If the event is not a DELETE or RENAME, the file must exist.
	// Otherwise the event is ignored.
	// *Note*: this was put in place because it was seen that a MODIFY
	// event was sent after the DELETE. This ignores that MODIFY and
	// assumes a DELETE will come or has come if the file doesn't exist.
	if !(e.Op&Remove == Remove || e.Op&Rename == Rename) {
		_, statErr := os.Lstat(e.Name)
		return os.IsNotExist(statErr)
	}
	return false
}
<|endoftext|>


<commit_before>package fusis

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/luizbafilho/fusis/config"
	"github.com/luizbafilho/fusis/engine"
	"github.com/luizbafilho/fusis/ipvs"
	fusis_net "github.com/luizbafilho/fusis/net"
	_ "github.com/luizbafilho/fusis/provider/none" // to intialize

	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/hashicorp/serf/serf"
)

const (
	retainSnapshotCount   = 2
	raftTimeout           = 10 * time.Second
	raftRemoveGracePeriod = 5 * time.Second
)

// Balancer represents the Load Balancer
type Balancer struct {
	sync.Mutex
	eventCh chan serf.Event

	serf          *serf.Serf
	raft          *raft.Raft // The consensus mechanism
	raftPeers     raft.PeerStore
	raftStore     *raftboltdb.BoltStore
	raftTransport *raft.NetworkTransport

	engine     *engine.Engine
	shutdownCh chan bool
}

// NewBalancer initializes a new balancer
//TODO: Graceful shutdown on initialization errors
func NewBalancer() (*Balancer, error) {
	engine, err := engine.New()
	if err != nil {
		return nil, err
	}

	balancer := &Balancer{
		eventCh: make(chan serf.Event, 64),
		engine:  engine,
	}

	if err = balancer.setupRaft(); err != nil {
		log.Fatalf("Setuping Raft", err)
	}

	if err = balancer.setupSerf(); err != nil {
		log.Fatalf("Setuping Serf", err)
	}

	// Flushing all VIPs on the network interface
	if err := fusis_net.DelVips(config.Balancer.Provider.Params["interface"]); err != nil {
		log.Fatalf("Fusis wasn't capable of cleanup network vips. Err: %v", err)
	}

	go balancer.watchLeaderChanges()

	return balancer, nil
}

// Start starts the balancer
func (b *Balancer) setupSerf() error {
	conf := serf.DefaultConfig()
	conf.Init()
	conf.Tags["role"] = "balancer"
	conf.Tags["raft-port"] = strconv.Itoa(config.Balancer.RaftPort)

	bindAddr, err := config.Balancer.GetIpByInterface()
	if err != nil {
		return err
	}

	conf.MemberlistConfig.BindAddr = bindAddr
	conf.EventCh = b.eventCh

	serf, err := serf.Create(conf)
	if err != nil {
		return err
	}

	b.serf = serf

	go b.handleEvents()

	return nil
}

func (b *Balancer) setupRaft() error {
	// Setup Raft configuration.
	raftConfig := raft.DefaultConfig()

	raftConfig.ShutdownOnRemove = false
	// Check for any existing peers.
	peers, err := readPeersJSON(filepath.Join(config.Balancer.ConfigPath, "peers.json"))
	if err != nil {
		return err
	}

	// Allow the node to entry single-mode, potentially electing itself, if
	// explicitly enabled and there is only 1 node in the cluster already.
	if config.Balancer.Single && len(peers) <= 1 {
		log.Info("enabling single-node mode")
		raftConfig.EnableSingleNode = true
		raftConfig.DisableBootstrapAfterElect = false
	}

	ip, err := config.Balancer.GetIpByInterface()
	if err != nil {
		return err
	}

	// Setup Raft communication.
	raftAddr := &net.TCPAddr{IP: net.ParseIP(ip), Port: config.Balancer.RaftPort}
	transport, err := raft.NewTCPTransport(raftAddr.String(), raftAddr, 3, 10*time.Second, os.Stderr)
	if err != nil {
		return err
	}
	b.raftTransport = transport

	// Create peer storage.
	peerStore := raft.NewJSONPeers(config.Balancer.ConfigPath, transport)
	b.raftPeers = peerStore

	// Create the snapshot store. This allows the Raft to truncate the log.
	snapshots, err := raft.NewFileSnapshotStore(config.Balancer.ConfigPath, retainSnapshotCount, os.Stderr)
	if err != nil {
		return fmt.Errorf("file snapshot store: %s", err)
	}

	// Create the log store and stable store.
	logStore, err := raftboltdb.NewBoltStore(filepath.Join(config.Balancer.ConfigPath, "raft.db"))
	if err != nil {
		return fmt.Errorf("new bolt store: %s", err)
	}
	b.raftStore = logStore

	// Instantiate the Raft systems.
	ra, err := raft.NewRaft(raftConfig, b.engine, logStore, logStore, snapshots, peerStore, transport)
	if err != nil {
		return fmt.Errorf("new raft: %s", err)
	}
	b.raft = ra

	go b.watchCommands()

	return nil
}

func (b *Balancer) watchCommands() {
	for {
		select {
		case c := <-b.engine.CommandCh:
			switch c.Op {
			case engine.AddServiceOp:
				b.AssignVIP(c.Service)
			case engine.DelServiceOp:
				b.UnassignVIP(c.Service)
			}
		}
	}
}

func (b *Balancer) UnassignVIP(svc *ipvs.Service) {
	if b.isLeader() {
		if err := b.engine.UnassignVIP(svc); err != nil {
			log.Errorf("Unassigning VIP to Service: %#v. Err: %#v", svc, err)
		}
	}
}

func (b *Balancer) AssignVIP(svc *ipvs.Service) {
	if b.isLeader() {
		if err := b.engine.AssignVIP(svc); err != nil {
			log.Errorf("Assigning VIP to Service: %#v. Err: %#v", svc, err)
		}
	}
}

func (b *Balancer) isLeader() bool {
	return b.raft.State() == raft.Leader
}

// JoinPool joins the Fusis Serf cluster
func (b *Balancer) JoinPool() error {
	log.Infof("Balancer: joining: %v ignore: %v", config.Balancer.Join)

	_, err := b.serf.Join([]string{config.Balancer.Join}, true)
	if err != nil {
		log.Errorf("Balancer: error joining: %v", err)
		return err
	}

	return nil
}

func (b *Balancer) watchLeaderChanges() {
	log.Infof("Watching to Leader changes")

	for {
		if <-b.raft.LeaderCh() {
			b.flushVips()
			b.setVips()
		} else {
			b.flushVips()
		}
	}
}

func (b *Balancer) handleEvents() {
	for {
		select {
		case e := <-b.eventCh:
			switch e.EventType() {
			case serf.EventMemberJoin:
				me := e.(serf.MemberEvent)
				b.handleMemberJoin(me)
			case serf.EventMemberFailed:
				memberEvent := e.(serf.MemberEvent)
				b.handleMemberLeave(memberEvent)
			case serf.EventMemberLeave:
				memberEvent := e.(serf.MemberEvent)
				b.handleMemberLeave(memberEvent)
			// case serf.EventQuery:
			// 	query := e.(*serf.Query)
			// 	b.handleQuery(query)
			default:
				log.Warnf("Balancer: unhandled Serf Event: %#v", e)
			}
		}
	}
}

func (b *Balancer) setVips() {
	svcs := b.engine.State.GetServices()

	for _, s := range *svcs {
		err := b.engine.AssignVIP(&s)
		if err != nil {
			//TODO: Remove balancer from cluster when error occurs
			log.Error(err)
		}
	}
}

func (b *Balancer) flushVips() {
	if err := fusis_net.DelVips(config.Balancer.Provider.Params["interface"]); err != nil {
		panic(err)
	}
}

func (b *Balancer) handleMemberJoin(event serf.MemberEvent) {
	log.Infof("handleMemberJoin: %s", event)

	if !b.isLeader() {
		return
	}

	for _, m := range event.Members {
		if isBalancer(m) {
			b.addMemberToPool(m)
		}
	}
}

func (b *Balancer) addMemberToPool(m serf.Member) {
	remoteAddr := fmt.Sprintf("%s:%v", m.Addr.String(), config.Balancer.RaftPort)

	log.Infof("Adding Balancer to Pool", remoteAddr)
	f := b.raft.AddPeer(remoteAddr)
	if f.Error() != nil {
		log.Errorf("node at %s joined failure. err: %s", remoteAddr, f.Error())
	}
}

func isBalancer(m serf.Member) bool {
	return m.Tags["role"] == "balancer"
}

func (b *Balancer) handleMemberLeave(memberEvent serf.MemberEvent) {
	log.Infof("handleMemberLeave: %s", memberEvent)
	for _, m := range memberEvent.Members {
		if isBalancer(m) {
			b.handleBalancerLeave(m)
		} else {
			b.handleAgentLeave(m)
		}
	}
}

func (b *Balancer) handleBalancerLeave(m serf.Member) {
	log.Info("Removing left balancer from raft")
	if !b.isLeader() {
		log.Info("Member is not leader")
		return
	}

	raftPort, err := strconv.Atoi(m.Tags["raft-port"])
	if err != nil {
		log.Errorln("handle balancer leaver failed", err)
	}

	peer := &net.TCPAddr{IP: m.Addr, Port: raftPort}
	log.Infof("Removing %v peer from raft", peer)
	future := b.raft.RemovePeer(peer.String())
	if err := future.Error(); err != nil && err != raft.ErrUnknownPeer {
		log.Errorf("balancer: failed to remove raft peer '%v': %v", peer, err)
	} else if err == nil {
		log.Infof("balancer: removed balancer '%s' as peer", m.Name)
	}
}

func (b *Balancer) Leave() {
	log.Info("balancer: server starting leave")
	// s.left = true

	// Check the number of known peers
	numPeers, err := b.numOtherPeers()
	if err != nil {
		log.Errorf("balancer: failed to check raft peers: %v", err)
		return
	}

	// If we are the current leader, and we have any other peers (cluster has multiple
	// servers), we should do a RemovePeer to safely reduce the quorum size. If we are
	// not the leader, then we should issue our leave intention and wait to be removed
	// for some sane period of time.
	isLeader := b.isLeader()
	// if isLeader && numPeers > 0 {
	// 	future := b.raft.RemovePeer(b.raftTransport.LocalAddr())
	// 	if err := future.Error(); err != nil && err != raft.ErrUnknownPeer {
	// 		log.Errorf("balancer: failed to remove ourself as raft peer: %v", err)
	// 	}
	// }

	// Leave the LAN pool
	if b.serf != nil {
		if err := b.serf.Leave(); err != nil {
			log.Errorf("balancer: failed to leave LAN Serf cluster: %v", err)
		}
	}

	// If we were not leader, wait to be safely removed from the cluster.
	// We must wait to allow the raft replication to take place, otherwise
	// an immediate shutdown could cause a loss of quorum.
	if !isLeader {
		limit := time.Now().Add(raftRemoveGracePeriod)
		for numPeers > 0 && time.Now().Before(limit) {
			// Update the number of peers
			numPeers, err = b.numOtherPeers()
			if err != nil {
				log.Errorf("balancer: failed to check raft peers: %v", err)
				break
			}

			// Avoid the sleep if we are done
			if numPeers == 0 {
				break
			}

			// Sleep a while and check again
			time.Sleep(50 * time.Millisecond)
		}
		if numPeers != 0 {
			log.Warnln("balancer: failed to leave raft peer set gracefully, timeout")
		}
	}
}

// numOtherPeers is used to check on the number of known peers
// excluding the local node
func (b *Balancer) numOtherPeers() (int, error) {
	peers, err := b.raftPeers.Peers()
	if err != nil {
		return 0, err
	}
	otherPeers := raft.ExcludePeer(peers, b.raftTransport.LocalAddr())
	return len(otherPeers), nil
}

func (b *Balancer) Shutdown() {
	b.Leave()
	b.serf.Shutdown()

	future := b.raft.Shutdown()
	if err := future.Error(); err != nil {
		log.Errorf("balancer: Error shutting down raft: %s", err)
	}

	if b.raftStore != nil {
		b.raftStore.Close()
	}

	b.raftPeers.SetPeers(nil)
}

func (b *Balancer) handleAgentLeave(m serf.Member) {
	dst, err := b.GetDestination(m.Name)
	if err != nil {
		log.Errorln("handleAgenteLeave failed", err)
		return
	}

	b.DeleteDestination(dst)
}

func readPeersJSON(path string) ([]string, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	if len(b) == 0 {
		return nil, nil
	}

	var peers []string
	dec := json.NewDecoder(bytes.NewReader(b))
	if err := dec.Decode(&peers); err != nil {
		return nil, err
	}

	return peers, nil
}
<commit_msg>adding looger to balancer<commit_after>package fusis

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/luizbafilho/fusis/config"
	"github.com/luizbafilho/fusis/engine"
	"github.com/luizbafilho/fusis/ipvs"
	fusis_net "github.com/luizbafilho/fusis/net"
	_ "github.com/luizbafilho/fusis/provider/none" // to intialize

	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/hashicorp/serf/serf"
)

const (
	retainSnapshotCount   = 2
	raftTimeout           = 10 * time.Second
	raftRemoveGracePeriod = 5 * time.Second
)

// Balancer represents the Load Balancer
type Balancer struct {
	sync.Mutex
	eventCh chan serf.Event

	serf          *serf.Serf
	raft          *raft.Raft // The consensus mechanism
	raftPeers     raft.PeerStore
	raftStore     *raftboltdb.BoltStore
	raftTransport *raft.NetworkTransport
	logger        *logrus.Logger

	engine     *engine.Engine
	shutdownCh chan bool
}

// NewBalancer initializes a new balancer
//TODO: Graceful shutdown on initialization errors
func NewBalancer() (*Balancer, error) {
	engine, err := engine.New()
	if err != nil {
		return nil, err
	}

	balancer := &Balancer{
		eventCh: make(chan serf.Event, 64),
		engine:  engine,
		logger:  logrus.New(),
	}

	if err = balancer.setupRaft(); err != nil {
		log.Fatalf("Setuping Raft", err)
	}

	if err = balancer.setupSerf(); err != nil {
		log.Fatalf("Setuping Serf", err)
	}

	// Flushing all VIPs on the network interface
	if err := fusis_net.DelVips(config.Balancer.Provider.Params["interface"]); err != nil {
		log.Fatalf("Fusis wasn't capable of cleanup network vips. Err: %v", err)
	}

	go balancer.watchLeaderChanges()

	return balancer, nil
}

// Start starts the balancer
func (b *Balancer) setupSerf() error {
	conf := serf.DefaultConfig()
	conf.Init()
	conf.Tags["role"] = "balancer"
	conf.Tags["raft-port"] = strconv.Itoa(config.Balancer.RaftPort)

	bindAddr, err := config.Balancer.GetIpByInterface()
	if err != nil {
		return err
	}

	conf.MemberlistConfig.BindAddr = bindAddr
	conf.EventCh = b.eventCh

	serf, err := serf.Create(conf)
	if err != nil {
		return err
	}

	b.serf = serf

	go b.handleEvents()

	return nil
}

func (b *Balancer) newStdLogger() *log.Logger {
	return log.New(b.logger.Writer(), "", 0)
}

func (b *Balancer) setupRaft() error {
	// Setup Raft configuration.
	raftConfig := raft.DefaultConfig()
	raftConfig.Logger = b.newStdLogger()

	raftConfig.ShutdownOnRemove = false
	// Check for any existing peers.
	peers, err := readPeersJSON(filepath.Join(config.Balancer.ConfigPath, "peers.json"))
	if err != nil {
		return err
	}

	// Allow the node to entry single-mode, potentially electing itself, if
	// explicitly enabled and there is only 1 node in the cluster already.
	if config.Balancer.Single && len(peers) <= 1 {
		b.logger.Infof("enabling single-node mode")
		raftConfig.EnableSingleNode = true
		raftConfig.DisableBootstrapAfterElect = false
	}

	ip, err := config.Balancer.GetIpByInterface()
	if err != nil {
		return err
	}

	// Setup Raft communication.
	raftAddr := &net.TCPAddr{IP: net.ParseIP(ip), Port: config.Balancer.RaftPort}
	transport, err := raft.NewTCPTransport(raftAddr.String(), raftAddr, 3, 10*time.Second, os.Stderr)
	if err != nil {
		return err
	}
	b.raftTransport = transport

	// Create peer storage.
	peerStore := raft.NewJSONPeers(config.Balancer.ConfigPath, transport)
	b.raftPeers = peerStore

	// Create the snapshot store. This allows the Raft to truncate the log.
	snapshots, err := raft.NewFileSnapshotStore(config.Balancer.ConfigPath, retainSnapshotCount, os.Stderr)
	if err != nil {
		return fmt.Errorf("file snapshot store: %s", err)
	}

	// Create the log store and stable store.
	logStore, err := raftboltdb.NewBoltStore(filepath.Join(config.Balancer.ConfigPath, "raft.db"))
	if err != nil {
		return fmt.Errorf("new bolt store: %s", err)
	}
	b.raftStore = logStore

	// Instantiate the Raft systems.
	ra, err := raft.NewRaft(raftConfig, b.engine, logStore, logStore, snapshots, peerStore, transport)
	if err != nil {
		return fmt.Errorf("new raft: %s", err)
	}
	b.raft = ra

	go b.watchCommands()

	return nil
}

func (b *Balancer) watchCommands() {
	for {
		select {
		case c := <-b.engine.CommandCh:
			switch c.Op {
			case engine.AddServiceOp:
				b.AssignVIP(c.Service)
			case engine.DelServiceOp:
				b.UnassignVIP(c.Service)
			}
		}
	}
}

func (b *Balancer) UnassignVIP(svc *ipvs.Service) {
	if b.isLeader() {
		if err := b.engine.UnassignVIP(svc); err != nil {
			b.logger.Errorf("Unassigning VIP to Service: %#v. Err: %#v", svc, err)
		}
	}
}

func (b *Balancer) AssignVIP(svc *ipvs.Service) {
	if b.isLeader() {
		if err := b.engine.AssignVIP(svc); err != nil {
			b.logger.Errorf("Assigning VIP to Service: %#v. Err: %#v", svc, err)
		}
	}
}

func (b *Balancer) isLeader() bool {
	return b.raft.State() == raft.Leader
}

// JoinPool joins the Fusis Serf cluster
func (b *Balancer) JoinPool() error {
	b.logger.Infof("Balancer: joining: %v ignore: %v", config.Balancer.Join)

	_, err := b.serf.Join([]string{config.Balancer.Join}, true)
	if err != nil {
		b.logger.Errorf("Balancer: error joining: %v", err)
		return err
	}

	return nil
}

func (b *Balancer) watchLeaderChanges() {
	b.logger.Infof("Watching to Leader changes")

	for {
		if <-b.raft.LeaderCh() {
			b.flushVips()
			b.setVips()
		} else {
			b.flushVips()
		}
	}
}

func (b *Balancer) handleEvents() {
	for {
		select {
		case e := <-b.eventCh:
			switch e.EventType() {
			case serf.EventMemberJoin:
				me := e.(serf.MemberEvent)
				b.handleMemberJoin(me)
			case serf.EventMemberFailed:
				memberEvent := e.(serf.MemberEvent)
				b.handleMemberLeave(memberEvent)
			case serf.EventMemberLeave:
				memberEvent := e.(serf.MemberEvent)
				b.handleMemberLeave(memberEvent)
			// case serf.EventQuery:
			// 	query := e.(*serf.Query)
			// 	b.handleQuery(query)
			default:
				b.logger.Warnf("Balancer: unhandled Serf Event: %#v", e)
			}
		}
	}
}

func (b *Balancer) setVips() {
	svcs := b.engine.State.GetServices()

	for _, s := range *svcs {
		err := b.engine.AssignVIP(&s)
		if err != nil {
			//TODO: Remove balancer from cluster when error occurs
			b.logger.Error(err)
		}
	}
}

func (b *Balancer) flushVips() {
	if err := fusis_net.DelVips(config.Balancer.Provider.Params["interface"]); err != nil {
		panic(err)
	}
}

func (b *Balancer) handleMemberJoin(event serf.MemberEvent) {
	b.logger.Infof("handleMemberJoin: %s", event)

	if !b.isLeader() {
		return
	}

	for _, m := range event.Members {
		if isBalancer(m) {
			b.addMemberToPool(m)
		}
	}
}

func (b *Balancer) addMemberToPool(m serf.Member) {
	remoteAddr := fmt.Sprintf("%s:%v", m.Addr.String(), config.Balancer.RaftPort)

	b.logger.Infof("Adding Balancer to Pool", remoteAddr)
	f := b.raft.AddPeer(remoteAddr)
	if f.Error() != nil {
		b.logger.Errorf("node at %s joined failure. err: %s", remoteAddr, f.Error())
	}
}

func isBalancer(m serf.Member) bool {
	return m.Tags["role"] == "balancer"
}

func (b *Balancer) handleMemberLeave(memberEvent serf.MemberEvent) {
	b.logger.Infof("handleMemberLeave: %s", memberEvent)
	for _, m := range memberEvent.Members {
		if isBalancer(m) {
			b.handleBalancerLeave(m)
		} else {
			b.handleAgentLeave(m)
		}
	}
}

func (b *Balancer) handleBalancerLeave(m serf.Member) {
	b.logger.Info("Removing left balancer from raft")
	if !b.isLeader() {
		b.logger.Info("Member is not leader")
		return
	}

	raftPort, err := strconv.Atoi(m.Tags["raft-port"])
	if err != nil {
		b.logger.Errorln("handle balancer leaver failed", err)
	}

	peer := &net.TCPAddr{IP: m.Addr, Port: raftPort}
	b.logger.Infof("Removing %v peer from raft", peer)
	future := b.raft.RemovePeer(peer.String())
	if err := future.Error(); err != nil && err != raft.ErrUnknownPeer {
		b.logger.Errorf("balancer: failed to remove raft peer '%v': %v", peer, err)
	} else if err == nil {
		b.logger.Infof("balancer: removed balancer '%s' as peer", m.Name)
	}
}

func (b *Balancer) Leave() {
	b.logger.Info("balancer: server starting leave")
	// s.left = true

	// Check the number of known peers
	numPeers, err := b.numOtherPeers()
	if err != nil {
		b.logger.Errorf("balancer: failed to check raft peers: %v", err)
		return
	}

	// If we are the current leader, and we have any other peers (cluster has multiple
	// servers), we should do a RemovePeer to safely reduce the quorum size. If we are
	// not the leader, then we should issue our leave intention and wait to be removed
	// for some sane period of time.
	isLeader := b.isLeader()
	// if isLeader && numPeers > 0 {
	// 	future := b.raft.RemovePeer(b.raftTransport.LocalAddr())
	// 	if err := future.Error(); err != nil && err != raft.ErrUnknownPeer {
	// 		b.logger.Errorf("balancer: failed to remove ourself as raft peer: %v", err)
	// 	}
	// }

	// Leave the LAN pool
	if b.serf != nil {
		if err := b.serf.Leave(); err != nil {
			b.logger.Errorf("balancer: failed to leave LAN Serf cluster: %v", err)
		}
	}

	// If we were not leader, wait to be safely removed from the cluster.
	// We must wait to allow the raft replication to take place, otherwise
	// an immediate shutdown could cause a loss of quorum.
	if !isLeader {
		limit := time.Now().Add(raftRemoveGracePeriod)
		for numPeers > 0 && time.Now().Before(limit) {
			// Update the number of peers
			numPeers, err = b.numOtherPeers()
			if err != nil {
				b.logger.Errorf("balancer: failed to check raft peers: %v", err)
				break
			}

			// Avoid the sleep if we are done
			if numPeers == 0 {
				break
			}

			// Sleep a while and check again
			time.Sleep(50 * time.Millisecond)
		}
		if numPeers != 0 {
			b.logger.Warnln("balancer: failed to leave raft peer set gracefully, timeout")
		}
	}
}

// numOtherPeers is used to check on the number of known peers
// excluding the local node
func (b *Balancer) numOtherPeers() (int, error) {
	peers, err := b.raftPeers.Peers()
	if err != nil {
		return 0, err
	}
	otherPeers := raft.ExcludePeer(peers, b.raftTransport.LocalAddr())
	return len(otherPeers), nil
}

func (b *Balancer) Shutdown() {
	b.Leave()
	b.serf.Shutdown()

	future := b.raft.Shutdown()
	if err := future.Error(); err != nil {
		b.logger.Errorf("balancer: Error shutting down raft: %s", err)
	}

	if b.raftStore != nil {
		b.raftStore.Close()
	}

	b.raftPeers.SetPeers(nil)
}

func (b *Balancer) handleAgentLeave(m serf.Member) {
	dst, err := b.GetDestination(m.Name)
	if err != nil {
		b.logger.Errorln("handleAgenteLeave failed", err)
		return
	}

	b.DeleteDestination(dst)
}

func readPeersJSON(path string) ([]string, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	if len(b) == 0 {
		return nil, nil
	}

	var peers []string
	dec := json.NewDecoder(bytes.NewReader(b))
	if err := dec.Decode(&peers); err != nil {
		return nil, err
	}

	return peers, nil
}
<|endoftext|>


<commit_before>package azure

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"log"

	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/go-uuid"
	"github.com/hashicorp/terraform/state"
	"github.com/hashicorp/terraform/state/remote"
	"github.com/hashicorp/terraform/states"
)

const (
	leaseHeader = "x-ms-lease-id"
	// Must be lower case
	lockInfoMetaKey = "terraformlockid"
)

type RemoteClient struct {
	blobClient    storage.BlobStorageClient
	containerName string
	keyName       string
	leaseID       string
}

func (c *RemoteClient) Get() (*remote.Payload, error) {
	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)
	options := &storage.GetBlobOptions{}

	if c.leaseID != "" {
		options.LeaseID = c.leaseID
	}

	blob, err := blobReference.Get(options)
	if err != nil {
		if storErr, ok := err.(storage.AzureStorageServiceError); ok {
			if storErr.Code == "BlobNotFound" {
				return nil, nil
			}
		}
		return nil, err
	}

	defer blob.Close()

	buf := bytes.NewBuffer(nil)
	if _, err := io.Copy(buf, blob); err != nil {
		return nil, fmt.Errorf("Failed to read remote state: %s", err)
	}

	payload := &remote.Payload{
		Data: buf.Bytes(),
	}

	// If there was no data, then return nil
	if len(payload.Data) == 0 {
		return nil, nil
	}

	return payload, nil
}

func (c *RemoteClient) Put(data []byte) error {
	getOptions := &storage.GetBlobMetadataOptions{}
	setOptions := &storage.SetBlobPropertiesOptions{}
	putOptions := &storage.PutBlobOptions{}

	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)

	blobReference.Properties.ContentType = "application/json"
	blobReference.Properties.ContentLength = int64(len(data))

	if c.leaseID != "" {
		getOptions.LeaseID = c.leaseID
		setOptions.LeaseID = c.leaseID
		putOptions.LeaseID = c.leaseID
	}

	exists, err := blobReference.Exists()
	if err != nil {
		return err
	}

	if exists {
		err = blobReference.GetMetadata(getOptions)
		if err != nil {
			return err
		}
	}

	reader := bytes.NewReader(data)

	err = blobReference.CreateBlockBlobFromReader(reader, putOptions)
	if err != nil {
		return err
	}

	return blobReference.SetProperties(setOptions)
}

func (c *RemoteClient) Delete() error {
	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)
	options := &storage.DeleteBlobOptions{}

	if c.leaseID != "" {
		options.LeaseID = c.leaseID
	}

	return blobReference.Delete(options)
}

func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) {
	stateName := fmt.Sprintf("%s/%s", c.containerName, c.keyName)
	info.Path = stateName

	if info.ID == "" {
		lockID, err := uuid.GenerateUUID()
		if err != nil {
			return "", err
		}

		info.ID = lockID
	}

	getLockInfoErr := func(err error) error {
		lockInfo, infoErr := c.getLockInfo()
		if infoErr != nil {
			err = multierror.Append(err, infoErr)
		}

		return &state.LockError{
			Err:  err,
			Info: lockInfo,
		}
	}

	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)
	leaseID, err := blobReference.AcquireLease(-1, info.ID, &storage.LeaseOptions{})
	if err != nil {
		if storErr, ok := err.(storage.AzureStorageServiceError); ok && storErr.Code != "BlobNotFound" {
			return "", getLockInfoErr(err)
		}

		// failed to lock as there was no state blob, write empty state
		stateMgr := &remote.State{Client: c}

		// ensure state is actually empty
		if err := stateMgr.RefreshState(); err != nil {
			return "", fmt.Errorf("Failed to refresh state before writing empty state for locking: %s", err)
		}

		log.Print("[DEBUG] Could not lock as state blob did not exist, creating with empty state")

		if v := stateMgr.State(); v == nil {
			if err := stateMgr.WriteState(states.NewState()); err != nil {
				return "", fmt.Errorf("Failed to write empty state for locking: %s", err)
			}
			if err := stateMgr.PersistState(); err != nil {
				return "", fmt.Errorf("Failed to persist empty state for locking: %s", err)
			}
		}

		leaseID, err = blobReference.AcquireLease(-1, info.ID, &storage.LeaseOptions{})
		if err != nil {
			return "", getLockInfoErr(err)
		}
	}

	info.ID = leaseID
	c.leaseID = leaseID

	if err := c.writeLockInfo(info); err != nil {
		return "", err
	}

	return info.ID, nil
}

func (c *RemoteClient) getLockInfo() (*state.LockInfo, error) {
	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)
	err := blobReference.GetMetadata(&storage.GetBlobMetadataOptions{})
	if err != nil {
		return nil, err
	}

	raw := blobReference.Metadata[lockInfoMetaKey]
	if raw == "" {
		return nil, fmt.Errorf("blob metadata %q was empty", lockInfoMetaKey)
	}

	data, err := base64.StdEncoding.DecodeString(raw)
	if err != nil {
		return nil, err
	}

	lockInfo := &state.LockInfo{}
	err = json.Unmarshal(data, lockInfo)
	if err != nil {
		return nil, err
	}

	return lockInfo, nil
}

// writes info to blob meta data, deletes metadata entry if info is nil
func (c *RemoteClient) writeLockInfo(info *state.LockInfo) error {
	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)
	err := blobReference.GetMetadata(&storage.GetBlobMetadataOptions{
		LeaseID: c.leaseID,
	})
	if err != nil {
		return err
	}

	if info == nil {
		delete(blobReference.Metadata, lockInfoMetaKey)
	} else {
		value := base64.StdEncoding.EncodeToString(info.Marshal())
		blobReference.Metadata[lockInfoMetaKey] = value
	}

	opts := &storage.SetBlobMetadataOptions{
		LeaseID: c.leaseID,
	}
	return blobReference.SetMetadata(opts)
}

func (c *RemoteClient) Unlock(id string) error {
	lockErr := &state.LockError{}

	lockInfo, err := c.getLockInfo()
	if err != nil {
		lockErr.Err = fmt.Errorf("failed to retrieve lock info: %s", err)
		return lockErr
	}
	lockErr.Info = lockInfo

	if lockInfo.ID != id {
		lockErr.Err = fmt.Errorf("lock id %q does not match existing lock", id)
		return lockErr
	}

	if err := c.writeLockInfo(nil); err != nil {
		lockErr.Err = fmt.Errorf("failed to delete lock info from metadata: %s", err)
		return lockErr
	}

	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)
	err = blobReference.ReleaseLease(id, &storage.LeaseOptions{})
	if err != nil {
		lockErr.Err = err
		return lockErr
	}

	c.leaseID = ""

	return nil
}
<commit_msg>backend/azurerm: fixing a bug where locks couldn't be unlocked (#19441)<commit_after>package azure

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"log"

	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/go-uuid"
	"github.com/hashicorp/terraform/state"
	"github.com/hashicorp/terraform/state/remote"
	"github.com/hashicorp/terraform/states"
)

const (
	leaseHeader = "x-ms-lease-id"
	// Must be lower case
	lockInfoMetaKey = "terraformlockid"
)

type RemoteClient struct {
	blobClient    storage.BlobStorageClient
	containerName string
	keyName       string
	leaseID       string
}

func (c *RemoteClient) Get() (*remote.Payload, error) {
	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)
	options := &storage.GetBlobOptions{}

	if c.leaseID != "" {
		options.LeaseID = c.leaseID
	}

	blob, err := blobReference.Get(options)
	if err != nil {
		if storErr, ok := err.(storage.AzureStorageServiceError); ok {
			if storErr.Code == "BlobNotFound" {
				return nil, nil
			}
		}
		return nil, err
	}

	defer blob.Close()

	buf := bytes.NewBuffer(nil)
	if _, err := io.Copy(buf, blob); err != nil {
		return nil, fmt.Errorf("Failed to read remote state: %s", err)
	}

	payload := &remote.Payload{
		Data: buf.Bytes(),
	}

	// If there was no data, then return nil
	if len(payload.Data) == 0 {
		return nil, nil
	}

	return payload, nil
}

func (c *RemoteClient) Put(data []byte) error {
	getOptions := &storage.GetBlobMetadataOptions{}
	setOptions := &storage.SetBlobPropertiesOptions{}
	putOptions := &storage.PutBlobOptions{}

	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)

	blobReference.Properties.ContentType = "application/json"
	blobReference.Properties.ContentLength = int64(len(data))

	if c.leaseID != "" {
		getOptions.LeaseID = c.leaseID
		setOptions.LeaseID = c.leaseID
		putOptions.LeaseID = c.leaseID
	}

	exists, err := blobReference.Exists()
	if err != nil {
		return err
	}

	if exists {
		err = blobReference.GetMetadata(getOptions)
		if err != nil {
			return err
		}
	}

	reader := bytes.NewReader(data)

	err = blobReference.CreateBlockBlobFromReader(reader, putOptions)
	if err != nil {
		return err
	}

	return blobReference.SetProperties(setOptions)
}

func (c *RemoteClient) Delete() error {
	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)
	options := &storage.DeleteBlobOptions{}

	if c.leaseID != "" {
		options.LeaseID = c.leaseID
	}

	return blobReference.Delete(options)
}

func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) {
	stateName := fmt.Sprintf("%s/%s", c.containerName, c.keyName)
	info.Path = stateName

	if info.ID == "" {
		lockID, err := uuid.GenerateUUID()
		if err != nil {
			return "", err
		}

		info.ID = lockID
	}

	getLockInfoErr := func(err error) error {
		lockInfo, infoErr := c.getLockInfo()
		if infoErr != nil {
			err = multierror.Append(err, infoErr)
		}

		return &state.LockError{
			Err:  err,
			Info: lockInfo,
		}
	}

	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)
	leaseID, err := blobReference.AcquireLease(-1, info.ID, &storage.LeaseOptions{})
	if err != nil {
		if storErr, ok := err.(storage.AzureStorageServiceError); ok && storErr.Code != "BlobNotFound" {
			return "", getLockInfoErr(err)
		}

		// failed to lock as there was no state blob, write empty state
		stateMgr := &remote.State{Client: c}

		// ensure state is actually empty
		if err := stateMgr.RefreshState(); err != nil {
			return "", fmt.Errorf("Failed to refresh state before writing empty state for locking: %s", err)
		}

		log.Print("[DEBUG] Could not lock as state blob did not exist, creating with empty state")

		if v := stateMgr.State(); v == nil {
			if err := stateMgr.WriteState(states.NewState()); err != nil {
				return "", fmt.Errorf("Failed to write empty state for locking: %s", err)
			}
			if err := stateMgr.PersistState(); err != nil {
				return "", fmt.Errorf("Failed to persist empty state for locking: %s", err)
			}
		}

		leaseID, err = blobReference.AcquireLease(-1, info.ID, &storage.LeaseOptions{})
		if err != nil {
			return "", getLockInfoErr(err)
		}
	}

	info.ID = leaseID
	c.leaseID = leaseID

	if err := c.writeLockInfo(info); err != nil {
		return "", err
	}

	return info.ID, nil
}

func (c *RemoteClient) getLockInfo() (*state.LockInfo, error) {
	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)
	err := blobReference.GetMetadata(&storage.GetBlobMetadataOptions{})
	if err != nil {
		return nil, err
	}

	raw := blobReference.Metadata[lockInfoMetaKey]
	if raw == "" {
		return nil, fmt.Errorf("blob metadata %q was empty", lockInfoMetaKey)
	}

	data, err := base64.StdEncoding.DecodeString(raw)
	if err != nil {
		return nil, err
	}

	lockInfo := &state.LockInfo{}
	err = json.Unmarshal(data, lockInfo)
	if err != nil {
		return nil, err
	}

	return lockInfo, nil
}

// writes info to blob meta data, deletes metadata entry if info is nil
func (c *RemoteClient) writeLockInfo(info *state.LockInfo) error {
	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)
	err := blobReference.GetMetadata(&storage.GetBlobMetadataOptions{
		LeaseID: c.leaseID,
	})
	if err != nil {
		return err
	}

	if info == nil {
		delete(blobReference.Metadata, lockInfoMetaKey)
	} else {
		value := base64.StdEncoding.EncodeToString(info.Marshal())
		blobReference.Metadata[lockInfoMetaKey] = value
	}

	opts := &storage.SetBlobMetadataOptions{
		LeaseID: c.leaseID,
	}
	return blobReference.SetMetadata(opts)
}

func (c *RemoteClient) Unlock(id string) error {
	lockErr := &state.LockError{}

	lockInfo, err := c.getLockInfo()
	if err != nil {
		lockErr.Err = fmt.Errorf("failed to retrieve lock info: %s", err)
		return lockErr
	}
	lockErr.Info = lockInfo

	if lockInfo.ID != id {
		lockErr.Err = fmt.Errorf("lock id %q does not match existing lock", id)
		return lockErr
	}

	c.leaseID = lockInfo.ID
	if err := c.writeLockInfo(nil); err != nil {
		lockErr.Err = fmt.Errorf("failed to delete lock info from metadata: %s", err)
		return lockErr
	}

	containerReference := c.blobClient.GetContainerReference(c.containerName)
	blobReference := containerReference.GetBlobReference(c.keyName)
	err = blobReference.ReleaseLease(id, &storage.LeaseOptions{})
	if err != nil {
		lockErr.Err = err
		return lockErr
	}

	c.leaseID = ""

	return nil
}
<|endoftext|>


<commit_before>package http

import (
	//	"github.com/siddontang/ledisdb/ledis"
	"testing"
)

func TestBgetCommand(t *testing.T) {
	db := getTestDB()
	db.BSetBit([]byte("test_bget"), 0, 1)
	db.BSetBit([]byte("test_bget"), 1, 1)
	db.BSetBit([]byte("test_bget"), 2, 1)

	_, err := bgetCommand(db, "test_bget", "a", "b", "c")
	if err == nil || err.Error() != "ERR wrong number of arguments for 'bget' command" {
		t.Fatal("invalid err %v", err)
	}

	r, err := bgetCommand(db, "test_bget")
	if err != nil {
		t.Fatal(err.Error())
	}
	str := r.(string)
	if str != "\x07" {
		t.Fatal("wrong result of 'bget': %v", []byte(str))
	}
}

func TestBDeleteCommand(t *testing.T) {
	db := getTestDB()

	_, err := bdeleteCommand(db, "test_bdelete", "a", "b", "c")
	if err == nil || err.Error() != "ERR wrong number of arguments for 'bdelete' command" {
		t.Fatalf("invalid err %v", err)
	}

	db.BSetBit([]byte("test_bdelete"), 0, 1)
	db.BSetBit([]byte("test_bdelete"), 1, 1)
	db.BSetBit([]byte("test_bdelete"), 2, 1)
	n, err := bdeleteCommand(db, "test_bdelete")
	if err != nil {
		t.Fatal(err.Error())
	}
	if n.(int64) != 1 {
		t.Fatalf("wrong result: %v", n)
	}

	n, err = bdeleteCommand(db, "test_bdelete_not_exit")
	if err != nil {
		t.Fatal(err.Error())
	}
	if n.(int64) != 0 {
		t.Fatalf("wrong result: %v", n)
	}
}

func TestBSetbitCommand(t *testing.T) {
	db := getTestDB()
	_, err := bsetbitCommand(db, "test_bsetbit", "a", "b", "c")
	if err == nil || err.Error() != "ERR wrong number of arguments for 'bsetbit' command" {
		t.Fatalf("invalid err %v", err)
	}
	n, err := bsetbitCommand(db, "test_bsetbit", "1", "1")
	if err != nil {
		t.Fatal(err.Error())
	}
	if n.(uint8) != 0 {
		t.Fatal("wrong result: %v", n)
	}
	n, err = db.BGetBit([]byte("test_bsetbit"), 1)
	if err != nil {
		t.Fatal(err.Error())
	}
	if n.(uint8) != 1 {
		t.Fatalf("wrong result: %v", n)
	}
}

func TestBMsetbitCommand(t *testing.T) {
	db := getTestDB()
	_, err := bmsetbitCommand(db, "test_bmsetbit", "a", "b", "c")

	if err == nil || err.Error() != "ERR wrong number of arguments for 'bmsetbit' command" {
		t.Fatalf("invalid err %v", err)
	}
	n, err := bmsetbitCommand(db, "test_bmsetbit", "1", "1", "3", "1")
	if err != nil {
		t.Fatal(err.Error())
	}
	if n.(int64) != 2 {
		t.Fatalf("wrong result: %v", n)
	}
}
<commit_msg>modify unit test<commit_after>package http

import (
	//	"github.com/siddontang/ledisdb/ledis"
	"testing"
)

func TestBgetCommand(t *testing.T) {
	db := getTestDB()
	db.BSetBit([]byte("test_bget"), 0, 1)
	db.BSetBit([]byte("test_bget"), 1, 1)
	db.BSetBit([]byte("test_bget"), 2, 1)

	_, err := bgetCommand(db, "test_bget", "a", "b", "c")
	if err == nil || err.Error() != "wrong number of arguments for 'bget' command" {
		t.Fatal("invalid err %v", err)
	}

	r, err := bgetCommand(db, "test_bget")
	if err != nil {
		t.Fatal(err.Error())
	}
	str := r.(string)
	if str != "\x07" {
		t.Fatal("wrong result of 'bget': %v", []byte(str))
	}
}

func TestBDeleteCommand(t *testing.T) {
	db := getTestDB()

	_, err := bdeleteCommand(db, "test_bdelete", "a", "b", "c")
	if err == nil || err.Error() != "wrong number of arguments for 'bdelete' command" {
		t.Fatalf("invalid err %v", err)
	}

	db.BSetBit([]byte("test_bdelete"), 0, 1)
	db.BSetBit([]byte("test_bdelete"), 1, 1)
	db.BSetBit([]byte("test_bdelete"), 2, 1)
	n, err := bdeleteCommand(db, "test_bdelete")
	if err != nil {
		t.Fatal(err.Error())
	}
	if n.(int64) != 1 {
		t.Fatalf("wrong result: %v", n)
	}

	n, err = bdeleteCommand(db, "test_bdelete_not_exit")
	if err != nil {
		t.Fatal(err.Error())
	}
	if n.(int64) != 0 {
		t.Fatalf("wrong result: %v", n)
	}
}

func TestBSetbitCommand(t *testing.T) {
	db := getTestDB()
	_, err := bsetbitCommand(db, "test_bsetbit", "a", "b", "c")
	if err == nil || err.Error() != "wrong number of arguments for 'bsetbit' command" {
		t.Fatalf("invalid err %v", err)
	}
	n, err := bsetbitCommand(db, "test_bsetbit", "1", "1")
	if err != nil {
		t.Fatal(err.Error())
	}
	if n.(uint8) != 0 {
		t.Fatal("wrong result: %v", n)
	}
	n, err = db.BGetBit([]byte("test_bsetbit"), 1)
	if err != nil {
		t.Fatal(err.Error())
	}
	if n.(uint8) != 1 {
		t.Fatalf("wrong result: %v", n)
	}
}

func TestBMsetbitCommand(t *testing.T) {
	db := getTestDB()
	_, err := bmsetbitCommand(db, "test_bmsetbit", "a", "b", "c")

	if err == nil || err.Error() != "wrong number of arguments for 'bmsetbit' command" {
		t.Fatalf("invalid err %v", err)
	}
	n, err := bmsetbitCommand(db, "test_bmsetbit", "1", "1", "3", "1")
	if err != nil {
		t.Fatal(err.Error())
	}
	if n.(int64) != 2 {
		t.Fatalf("wrong result: %v", n)
	}
}

func TestBCountCommand(t *testing.T) {
	db := getTestDB()
	_, err := bcountCommand(db, "test_bcount", "a", "b", "c")
	if err == nil || err.Error() != "wrong number of arguments for 'bcount' command" {
		t.Fatalf("invalid err %v", err)
	}
}
<|endoftext|>