<commit_before>package scbe

import (
	"fmt"
	"github.com/IBM/ubiquity/resources"
)

type serviceDoesntExistError struct {
	volName     string
	serviceName string
	scbeName    string
}

func (e *serviceDoesntExistError) Error() string {
	return fmt.Sprintf("Cannot create volume [%s] on service [%s]. Reason : Service does not exist or not delegated to the Ubiquity interface in [%s]",
		e.volName, e.serviceName, e.scbeName)
}

type mappingResponseError struct {
	mapping ScbeResponseMappings
}

func (e *mappingResponseError) Error() string {
	return fmt.Sprintf("Mapping operation succeed but response is missing the mapping details. %#v", e.mapping)
}

type volumeNotFoundError struct {
	volName string
}

func (e *volumeNotFoundError) Error() string {
	return fmt.Sprintf("Volume [%s] was not found", e.volName)
}

type hostNotFoundvolumeNotFoundError struct {
	volName   string
	arrayName string
	hostName  string
}

func (e *hostNotFoundvolumeNotFoundError) Error() string {
	return fmt.Sprintf("Host name [%s] was not found on the storage system [%s](according to SCBE caching) that related to volume with WWN [%s]",
		e.hostName, e.arrayName, e.volName)
}

type activateDefaultServiceError struct {
	serviceName string
	scbeName    string
}

func (e *activateDefaultServiceError) Error() string {
	return fmt.Sprintf("Error in activate SCBE backend. The default service %s does not exist in SCBE %s",
		e.serviceName, e.scbeName)
}

type volAlreadyExistsError struct {
	volName string
}

func (e *volAlreadyExistsError) Error() string {
	return fmt.Sprintf("Volume [%s] already exists.", e.volName)
}

type provisionParamMissingError struct {
	volName string
	param   string
}

func (e *provisionParamMissingError) Error() string {
	return fmt.Sprintf("Fail to provision a volume [%s] because the [%s] option is missing", e.volName, e.param)
}

type FsTypeNotSupportedError struct {
	volName        string
	wrongFStype    string
	supportedTypes string
}

func (e *FsTypeNotSupportedError) Error() string {
	return fmt.Sprintf("Fail to provision a volume [%s]. Supported filesystem types are [%s] (but given [%s])",
		e.volName, e.supportedTypes, e.supportedTypes)
}

type provisionParamIsNotNumberError struct {
	volName string
	param   string
}

func (e *provisionParamIsNotNumberError) Error() string {
	return fmt.Sprintf("Fail to provision a volume [%s] because the [%s] option is not a number", e.volName, e.param)
}

type volAlreadyAttachedError struct {
	volName  string
	hostName string
}

func (e *volAlreadyAttachedError) Error() string {
	return fmt.Sprintf("Volume [%s] already attached to [%s]", e.volName, e.hostName)
}

type CannotDeleteVolWhichAttachedToHostError struct {
	volName  string
	hostName string
}

func (e *CannotDeleteVolWhichAttachedToHostError) Error() string {
	return fmt.Sprintf("Cannot delete a volume that is attached to a host. The volume [%s] currently attached to host [%s]", e.volName, e.hostName)
}

type volNotAttachedError struct {
	volName string
}

func (e *volNotAttachedError) Error() string {
	return fmt.Sprintf("Volume [%s] not attached", e.volName)
}

type ConfigDefaultSizeNotNumError struct {
	size string
}

func (e *ConfigDefaultSizeNotNumError) Error() string {
	return fmt.Sprintf("Error in config file. The parameter [%s] must be a number",
		"ScbeConfig.DefaultVolumeSize")
}

type ConfigDefaultFilesystemTypeNotSupported struct {
	wrongFStype    string
	supportedTypes string
}

func (e *ConfigDefaultFilesystemTypeNotSupported) Error() string {
	return fmt.Sprintf("Error in config file. The parameter [%s] can be the following values [%s] (given [%s])",
		"ScbeConfig.DefaultFileSystemType", e.supportedTypes, e.wrongFStype)
}

type ConfigScbeUbiquityInstanceNameWrongSize struct {
}

func (e *ConfigScbeUbiquityInstanceNameWrongSize) Error() string {
	return fmt.Sprintf("Error in config file. The parameter [%s] max length is [%d]",
		"ScbeConfig.UbiquityInstanceName", resources.UbiquityInstanceNameMaxSize)
}

type VolumeNameExceededMaxLengthError struct {
	volName         string
	maxVolumeLength int
}

func (e *VolumeNameExceededMaxLengthError) Error() string {
	return fmt.Sprintf("Volume name [%s] is too long(len=%d). Max length should be [%d]",
		e.volName, len(e.volName), e.maxVolumeLength)
}

type InValidRequestError struct {
	requestType       string
	badParam          string
	paramCurrentValue string
	paramExpectedToBe string
}

func (e *InValidRequestError) Error() string {
	return fmt.Sprintf(
		"Request type [%s] not valid. Param [%s]=[%s] but expacted to be [%s]",
		e.requestType,
		e.badParam,
		e.paramCurrentValue,
		e.paramExpectedToBe)
}
<commit_msg>#61 : Support xfs filesystem (improve error message)<commit_after>package scbe

import (
	"fmt"
	"github.com/IBM/ubiquity/resources"
)

type serviceDoesntExistError struct {
	volName     string
	serviceName string
	scbeName    string
}

func (e *serviceDoesntExistError) Error() string {
	return fmt.Sprintf("Cannot create volume [%s] on service [%s]. Reason : Service does not exist or not delegated to the Ubiquity interface in [%s]",
		e.volName, e.serviceName, e.scbeName)
}

type mappingResponseError struct {
	mapping ScbeResponseMappings
}

func (e *mappingResponseError) Error() string {
	return fmt.Sprintf("Mapping operation succeed but response is missing the mapping details. %#v", e.mapping)
}

type volumeNotFoundError struct {
	volName string
}

func (e *volumeNotFoundError) Error() string {
	return fmt.Sprintf("Volume [%s] was not found", e.volName)
}

type hostNotFoundvolumeNotFoundError struct {
	volName   string
	arrayName string
	hostName  string
}

func (e *hostNotFoundvolumeNotFoundError) Error() string {
	return fmt.Sprintf("Host name [%s] was not found on the storage system [%s](according to SCBE caching) that related to volume with WWN [%s]",
		e.hostName, e.arrayName, e.volName)
}

type activateDefaultServiceError struct {
	serviceName string
	scbeName    string
}

func (e *activateDefaultServiceError) Error() string {
	return fmt.Sprintf("Error in activate SCBE backend. The default service %s does not exist in SCBE %s",
		e.serviceName, e.scbeName)
}

type volAlreadyExistsError struct {
	volName string
}

func (e *volAlreadyExistsError) Error() string {
	return fmt.Sprintf("Volume [%s] already exists.", e.volName)
}

type provisionParamMissingError struct {
	volName string
	param   string
}

func (e *provisionParamMissingError) Error() string {
	return fmt.Sprintf("Fail to provision a volume [%s] because the [%s] option is missing", e.volName, e.param)
}

type FsTypeNotSupportedError struct {
	volName        string
	wrongFStype    string
	supportedTypes string
}

func (e *FsTypeNotSupportedError) Error() string {
	return fmt.Sprintf("Fail to provision a volume [%s]. Supported filesystem types are [%s] (but given [%s])",
		e.volName, e.supportedTypes, e.wrongFStype)
}

type provisionParamIsNotNumberError struct {
	volName string
	param   string
}

func (e *provisionParamIsNotNumberError) Error() string {
	return fmt.Sprintf("Fail to provision a volume [%s] because the [%s] option is not a number", e.volName, e.param)
}

type volAlreadyAttachedError struct {
	volName  string
	hostName string
}

func (e *volAlreadyAttachedError) Error() string {
	return fmt.Sprintf("Volume [%s] already attached to [%s]", e.volName, e.hostName)
}

type CannotDeleteVolWhichAttachedToHostError struct {
	volName  string
	hostName string
}

func (e *CannotDeleteVolWhichAttachedToHostError) Error() string {
	return fmt.Sprintf("Cannot delete a volume that is attached to a host. The volume [%s] currently attached to host [%s]", e.volName, e.hostName)
}

type volNotAttachedError struct {
	volName string
}

func (e *volNotAttachedError) Error() string {
	return fmt.Sprintf("Volume [%s] not attached", e.volName)
}

type ConfigDefaultSizeNotNumError struct {
	size string
}

func (e *ConfigDefaultSizeNotNumError) Error() string {
	return fmt.Sprintf("Error in config file. The parameter [%s] must be a number",
		"ScbeConfig.DefaultVolumeSize")
}

type ConfigDefaultFilesystemTypeNotSupported struct {
	wrongFStype    string
	supportedTypes string
}

func (e *ConfigDefaultFilesystemTypeNotSupported) Error() string {
	return fmt.Sprintf("Error in config file. The parameter [%s] can be the following values [%s] (given [%s])",
		"ScbeConfig.DefaultFileSystemType", e.supportedTypes, e.wrongFStype)
}

type ConfigScbeUbiquityInstanceNameWrongSize struct {
}

func (e *ConfigScbeUbiquityInstanceNameWrongSize) Error() string {
	return fmt.Sprintf("Error in config file. The parameter [%s] max length is [%d]",
		"ScbeConfig.UbiquityInstanceName", resources.UbiquityInstanceNameMaxSize)
}

type VolumeNameExceededMaxLengthError struct {
	volName         string
	maxVolumeLength int
}

func (e *VolumeNameExceededMaxLengthError) Error() string {
	return fmt.Sprintf("Volume name [%s] is too long(len=%d). Max length should be [%d]",
		e.volName, len(e.volName), e.maxVolumeLength)
}

type InValidRequestError struct {
	requestType       string
	badParam          string
	paramCurrentValue string
	paramExpectedToBe string
}

func (e *InValidRequestError) Error() string {
	return fmt.Sprintf(
		"Request type [%s] not valid. Param [%s]=[%s] but expacted to be [%s]",
		e.requestType,
		e.badParam,
		e.paramCurrentValue,
		e.paramExpectedToBe)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"log"
	"math/rand"
	"mime"
	"net/http"
	"os"
	"pkg/operation"
	"pkg/storage"
	"strconv"
	"strings"
	"time"
)

func init() {
	cmdVolume.Run = runVolume // break init cycle
	IsDebug = cmdVolume.Flag.Bool("debug", false, "enable debug mode")
}

var cmdVolume = &Command{
	UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
	Short:     "start a volume server",
	Long: `start a volume server to provide storage spaces

  `,
}

var (
	vport          = cmdVolume.Flag.Int("port", 8080, "http listen port")
	volumeFolder   = cmdVolume.Flag.String("dir", "/tmp", "directory to store data files")
	ip             = cmdVolume.Flag.String("ip", "localhost", "ip or server name")
	publicUrl      = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible <ip|server_name>:<port>")
	masterNode     = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location")
	vpulse         = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	maxVolumeCount = cmdVolume.Flag.Int("max", 5, "maximum number of volumes")

	store *storage.Store
)

func statusHandler(w http.ResponseWriter, r *http.Request) {
	writeJson(w, r, store.Status())
}
func assignVolumeHandler(w http.ResponseWriter, r *http.Request) {
	err := store.AddVolume(r.FormValue("volume"), r.FormValue("replicationType"))
	if err == nil {
		writeJson(w, r, map[string]string{"error": ""})
	} else {
		writeJson(w, r, map[string]string{"error": err.Error()})
	}
	if *IsDebug {
		log.Println("volume =", r.FormValue("volume"), ", replicationType =", r.FormValue("replicationType"), ", error =", err)
	}
}
func storeHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		GetHandler(w, r)
	case "DELETE":
		DeleteHandler(w, r)
	case "POST":
		PostHandler(w, r)
	}
}
func GetHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, ext := parseURLPath(r.URL.Path)
	volumeId, err := storage.NewVolumeId(vid)
	if err != nil {
		if *IsDebug {
			log.Println("parsing error:", err, r.URL.Path)
		}
		return
	}
	n.ParsePath(fid)

	if *IsDebug {
		log.Println("volume", volumeId, "reading", n)
	}
	if !store.HasVolume(volumeId) {
		lookupResult, err := operation.Lookup(*masterNode, volumeId)
		if *IsDebug {
			log.Println("volume", volumeId, "found on", lookupResult, "error", err)
		}
		if err == nil {
			http.Redirect(w, r, "http://"+lookupResult.Locations[0].PublicUrl+r.URL.Path, http.StatusMovedPermanently)
		} else {
			if *IsDebug {
				log.Println("lookup error:", err, r.URL.Path)
			}
			w.WriteHeader(http.StatusNotFound)
		}
		return
	}
	cookie := n.Cookie
	count, e := store.Read(volumeId, n)
	if *IsDebug {
		log.Println("read bytes", count, "error", e)
	}
	if e != nil || count <= 0 {
		log.Println("read error:", e)
		w.WriteHeader(http.StatusNotFound)
		return
	}
	if n.Cookie != cookie {
		log.Println("request with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		w.WriteHeader(http.StatusNotFound)
		return
	}
	if ext != "" {
		mtype := mime.TypeByExtension(ext)
		w.Header().Set("Content-Type", mtype)
		if storage.IsCompressable(ext, mtype) {
			if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
				w.Header().Set("Content-Encoding", "gzip")
			} else {
				n.Data = storage.UnGzipData(n.Data)
			}
		}
	}
	w.Write(n.Data)
}
func PostHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	vid, _, _ := parseURLPath(r.URL.Path)
	volumeId, e := storage.NewVolumeId(vid)
	if e != nil {
		writeJson(w, r, e)
	} else {
		needle, filename, ne := storage.NewNeedle(r)
		if ne != nil {
			writeJson(w, r, ne)
		} else {
			ret := store.Write(volumeId, needle)
			if ret > 0 || !store.HasVolume(volumeId) { //send to other replica locations
				if r.FormValue("type") != "standard" {
					waitTime, err := strconv.Atoi(r.FormValue("wait"))
					distributedOperation(volumeId, func(location operation.Location) bool {
						operation.Upload("http://"+location.Url+r.URL.Path+"?type=standard", filename, bytes.NewReader(needle.Data))
						return true
					}, err == nil && waitTime > 0)
				}
				w.WriteHeader(http.StatusCreated)
			}
			m := make(map[string]uint32)
			m["size"] = ret
			writeJson(w, r, m)
		}
	}
}
func DeleteHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, _ := parseURLPath(r.URL.Path)
	volumeId, _ := storage.NewVolumeId(vid)
	n.ParsePath(fid)

	if *IsDebug {
		log.Println("deleting", n)
	}

	cookie := n.Cookie
	count, ok := store.Read(volumeId, n)

	if ok != nil {
		m := make(map[string]uint32)
		m["size"] = 0
		writeJson(w, r, m)
		return
	}

	if n.Cookie != cookie {
		log.Println("delete with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		return
	}

	n.Size = 0
	ret := store.Delete(volumeId, n)

	if ret > 0 || !store.HasVolume(volumeId) { //send to other replica locations
		if r.FormValue("type") != "standard" {
			waitTime, err := strconv.Atoi(r.FormValue("wait"))
			distributedOperation(volumeId, func(location operation.Location) bool {
				operation.Delete("http://"+location.Url+r.URL.Path+"?type=standard")
				return true
			}, err == nil && waitTime > 0)
		}
		w.WriteHeader(http.StatusCreated)
	}

	m := make(map[string]uint32)
	m["size"] = uint32(count)
	writeJson(w, r, m)
}
func parseURLPath(path string) (vid, fid, ext string) {

	sepIndex := strings.LastIndex(path, "/")
	commaIndex := strings.LastIndex(path[sepIndex:], ",")
	if commaIndex <= 0 {
		if "favicon.ico" != path[sepIndex+1:] {
			log.Println("unknown file id", path[sepIndex+1:])
		}
		return
	}
	dotIndex := strings.LastIndex(path[sepIndex:], ".")
	vid = path[sepIndex+1 : commaIndex]
	fid = path[commaIndex+1:]
	ext = ""
	if dotIndex > 0 {
		fid = path[commaIndex+1 : dotIndex]
		ext = path[dotIndex:]
	}
	return
}

type distributedFunction func(location operation.Location) bool

func distributedOperation(volumeId storage.VolumeId, op distributedFunction, wait bool) {
	if lookupResult, lookupErr := operation.Lookup(*masterNode, volumeId); lookupErr == nil {
		sendFunc := func(background bool) {
			for _, location := range lookupResult.Locations {
				if location.Url != (*ip + ":" + strconv.Itoa(*vport)) {
					if background {
						go op(location)
					} else {
						op(location)
					}
				}
			}
		}
		sendFunc(wait)
	} else {
		log.Println("Failed to lookup for", volumeId, lookupErr.Error())
	}
}

func runVolume(cmd *Command, args []string) bool {
	fileInfo, err := os.Stat(*volumeFolder)
	//TODO: now default to 1G, this value should come from server?
	if err != nil {
		log.Fatalf("No Existing Folder:%s", *volumeFolder)
	}
	if !fileInfo.IsDir() {
		log.Fatalf("Volume Folder should not be a file:%s", *volumeFolder)
	}
	perm := fileInfo.Mode().Perm()
	log.Println("Volume Folder permission:", perm)

	if *publicUrl == "" {
		*publicUrl = *ip + ":" + strconv.Itoa(*vport)
	}

	store = storage.NewStore(*vport, *ip, *publicUrl, *volumeFolder, *maxVolumeCount)
	defer store.Close()
	http.HandleFunc("/", storeHandler)
	http.HandleFunc("/status", statusHandler)
	http.HandleFunc("/admin/assign_volume", assignVolumeHandler)

	go func() {
		for {
			store.Join(*masterNode)
			time.Sleep(time.Duration(float32(*vpulse*1e3)*(1+rand.Float32())) * time.Millisecond)
		}
	}()
	log.Println("store joined at", *masterNode)

	log.Println("Start storage service at http://" + *ip + ":" + strconv.Itoa(*vport))
	e := http.ListenAndServe(":"+strconv.Itoa(*vport), nil)
	if e != nil {
		log.Fatalf("Fail to start:%s", e.Error())
	}
	return true
}
<commit_msg>remove old comments<commit_after>package main

import (
	"bytes"
	"log"
	"math/rand"
	"mime"
	"net/http"
	"os"
	"pkg/operation"
	"pkg/storage"
	"strconv"
	"strings"
	"time"
)

func init() {
	cmdVolume.Run = runVolume // break init cycle
	IsDebug = cmdVolume.Flag.Bool("debug", false, "enable debug mode")
}

var cmdVolume = &Command{
	UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
	Short:     "start a volume server",
	Long: `start a volume server to provide storage spaces

  `,
}

var (
	vport          = cmdVolume.Flag.Int("port", 8080, "http listen port")
	volumeFolder   = cmdVolume.Flag.String("dir", "/tmp", "directory to store data files")
	ip             = cmdVolume.Flag.String("ip", "localhost", "ip or server name")
	publicUrl      = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible <ip|server_name>:<port>")
	masterNode     = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location")
	vpulse         = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	maxVolumeCount = cmdVolume.Flag.Int("max", 5, "maximum number of volumes")

	store *storage.Store
)

func statusHandler(w http.ResponseWriter, r *http.Request) {
	writeJson(w, r, store.Status())
}
func assignVolumeHandler(w http.ResponseWriter, r *http.Request) {
	err := store.AddVolume(r.FormValue("volume"), r.FormValue("replicationType"))
	if err == nil {
		writeJson(w, r, map[string]string{"error": ""})
	} else {
		writeJson(w, r, map[string]string{"error": err.Error()})
	}
	if *IsDebug {
		log.Println("volume =", r.FormValue("volume"), ", replicationType =", r.FormValue("replicationType"), ", error =", err)
	}
}
func storeHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		GetHandler(w, r)
	case "DELETE":
		DeleteHandler(w, r)
	case "POST":
		PostHandler(w, r)
	}
}
func GetHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, ext := parseURLPath(r.URL.Path)
	volumeId, err := storage.NewVolumeId(vid)
	if err != nil {
		if *IsDebug {
			log.Println("parsing error:", err, r.URL.Path)
		}
		return
	}
	n.ParsePath(fid)

	if *IsDebug {
		log.Println("volume", volumeId, "reading", n)
	}
	if !store.HasVolume(volumeId) {
		lookupResult, err := operation.Lookup(*masterNode, volumeId)
		if *IsDebug {
			log.Println("volume", volumeId, "found on", lookupResult, "error", err)
		}
		if err == nil {
			http.Redirect(w, r, "http://"+lookupResult.Locations[0].PublicUrl+r.URL.Path, http.StatusMovedPermanently)
		} else {
			if *IsDebug {
				log.Println("lookup error:", err, r.URL.Path)
			}
			w.WriteHeader(http.StatusNotFound)
		}
		return
	}
	cookie := n.Cookie
	count, e := store.Read(volumeId, n)
	if *IsDebug {
		log.Println("read bytes", count, "error", e)
	}
	if e != nil || count <= 0 {
		log.Println("read error:", e)
		w.WriteHeader(http.StatusNotFound)
		return
	}
	if n.Cookie != cookie {
		log.Println("request with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		w.WriteHeader(http.StatusNotFound)
		return
	}
	if ext != "" {
		mtype := mime.TypeByExtension(ext)
		w.Header().Set("Content-Type", mtype)
		if storage.IsCompressable(ext, mtype) {
			if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
				w.Header().Set("Content-Encoding", "gzip")
			} else {
				n.Data = storage.UnGzipData(n.Data)
			}
		}
	}
	w.Write(n.Data)
}
func PostHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	vid, _, _ := parseURLPath(r.URL.Path)
	volumeId, e := storage.NewVolumeId(vid)
	if e != nil {
		writeJson(w, r, e)
	} else {
		needle, filename, ne := storage.NewNeedle(r)
		if ne != nil {
			writeJson(w, r, ne)
		} else {
			ret := store.Write(volumeId, needle)
			if ret > 0 || !store.HasVolume(volumeId) { //send to other replica locations
				if r.FormValue("type") != "standard" {
					waitTime, err := strconv.Atoi(r.FormValue("wait"))
					distributedOperation(volumeId, func(location operation.Location) bool {
						operation.Upload("http://"+location.Url+r.URL.Path+"?type=standard", filename, bytes.NewReader(needle.Data))
						return true
					}, err == nil && waitTime > 0)
				}
				w.WriteHeader(http.StatusCreated)
			}
			m := make(map[string]uint32)
			m["size"] = ret
			writeJson(w, r, m)
		}
	}
}
func DeleteHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, _ := parseURLPath(r.URL.Path)
	volumeId, _ := storage.NewVolumeId(vid)
	n.ParsePath(fid)

	if *IsDebug {
		log.Println("deleting", n)
	}

	cookie := n.Cookie
	count, ok := store.Read(volumeId, n)

	if ok != nil {
		m := make(map[string]uint32)
		m["size"] = 0
		writeJson(w, r, m)
		return
	}

	if n.Cookie != cookie {
		log.Println("delete with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		return
	}

	n.Size = 0
	ret := store.Delete(volumeId, n)

	if ret > 0 || !store.HasVolume(volumeId) { //send to other replica locations
		if r.FormValue("type") != "standard" {
			waitTime, err := strconv.Atoi(r.FormValue("wait"))
			distributedOperation(volumeId, func(location operation.Location) bool {
				operation.Delete("http://"+location.Url+r.URL.Path+"?type=standard")
				return true
			}, err == nil && waitTime > 0)
		}
		w.WriteHeader(http.StatusCreated)
	}

	m := make(map[string]uint32)
	m["size"] = uint32(count)
	writeJson(w, r, m)
}
func parseURLPath(path string) (vid, fid, ext string) {

	sepIndex := strings.LastIndex(path, "/")
	commaIndex := strings.LastIndex(path[sepIndex:], ",")
	if commaIndex <= 0 {
		if "favicon.ico" != path[sepIndex+1:] {
			log.Println("unknown file id", path[sepIndex+1:])
		}
		return
	}
	dotIndex := strings.LastIndex(path[sepIndex:], ".")
	vid = path[sepIndex+1 : commaIndex]
	fid = path[commaIndex+1:]
	ext = ""
	if dotIndex > 0 {
		fid = path[commaIndex+1 : dotIndex]
		ext = path[dotIndex:]
	}
	return
}

type distributedFunction func(location operation.Location) bool

func distributedOperation(volumeId storage.VolumeId, op distributedFunction, wait bool) {
	if lookupResult, lookupErr := operation.Lookup(*masterNode, volumeId); lookupErr == nil {
		sendFunc := func(background bool) {
			for _, location := range lookupResult.Locations {
				if location.Url != (*ip + ":" + strconv.Itoa(*vport)) {
					if background {
						go op(location)
					} else {
						op(location)
					}
				}
			}
		}
		sendFunc(wait)
	} else {
		log.Println("Failed to lookup for", volumeId, lookupErr.Error())
	}
}

func runVolume(cmd *Command, args []string) bool {
	fileInfo, err := os.Stat(*volumeFolder)
	if err != nil {
		log.Fatalf("No Existing Folder:%s", *volumeFolder)
	}
	if !fileInfo.IsDir() {
		log.Fatalf("Volume Folder should not be a file:%s", *volumeFolder)
	}
	perm := fileInfo.Mode().Perm()
	log.Println("Volume Folder permission:", perm)

	if *publicUrl == "" {
		*publicUrl = *ip + ":" + strconv.Itoa(*vport)
	}

	store = storage.NewStore(*vport, *ip, *publicUrl, *volumeFolder, *maxVolumeCount)
	defer store.Close()
	http.HandleFunc("/", storeHandler)
	http.HandleFunc("/status", statusHandler)
	http.HandleFunc("/admin/assign_volume", assignVolumeHandler)

	go func() {
		for {
			store.Join(*masterNode)
			time.Sleep(time.Duration(float32(*vpulse*1e3)*(1+rand.Float32())) * time.Millisecond)
		}
	}()
	log.Println("store joined at", *masterNode)

	log.Println("Start storage service at http://" + *ip + ":" + strconv.Itoa(*vport))
	e := http.ListenAndServe(":"+strconv.Itoa(*vport), nil)
	if e != nil {
		log.Fatalf("Fail to start:%s", e.Error())
	}
	return true
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"strings"
	"testing"

	"k8s.io/kops/util/pkg/ui"
)

// TestConfirmation attempts to test the majority of the ui.GetConfirm function used in the 'kogs delete' commands
func TestConfirmation(t *testing.T) {
	var out bytes.Buffer
	c := &ui.ConfirmArgs{
		Message: "Are you sure you want to remove?",
		Out:     &out,
		TestVal: "no",
		Default: "no",
	}

	answer, err := ui.GetConfirm(c)
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(out.String(), "Are you sure") {
		t.Fatal("Confirmation not in output")
	}
	if !strings.Contains(out.String(), "y/N") {
		t.Fatal("Default 'No' was not set")
	}
	if answer == true {
		t.Fatal("Confirmation should have been denied.")
	}

	c.Default = "yes"
	answer, err = ui.GetConfirm(c)
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(out.String(), "Y/n") {
		t.Fatal("Default 'Yes' was not set")
	}

	c.TestVal = "yes"
	answer, err = ui.GetConfirm(c)
	if err != nil {
		t.Fatal(err)
	}
	if answer != true {
		t.Fatal("Confirmation should have been approved.")
	}

}
<commit_msg>Assign unused test variable to _<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"strings"
	"testing"

	"k8s.io/kops/util/pkg/ui"
)

// TestConfirmation attempts to test the majority of the ui.GetConfirm function used in the 'kogs delete' commands
func TestConfirmation(t *testing.T) {
	var out bytes.Buffer
	c := &ui.ConfirmArgs{
		Message: "Are you sure you want to remove?",
		Out:     &out,
		TestVal: "no",
		Default: "no",
	}

	answer, err := ui.GetConfirm(c)
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(out.String(), "Are you sure") {
		t.Fatal("Confirmation not in output")
	}
	if !strings.Contains(out.String(), "y/N") {
		t.Fatal("Default 'No' was not set")
	}
	if answer == true {
		t.Fatal("Confirmation should have been denied.")
	}

	c.Default = "yes"
	_, err = ui.GetConfirm(c)
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(out.String(), "Y/n") {
		t.Fatal("Default 'Yes' was not set")
	}

	c.TestVal = "yes"
	answer, err = ui.GetConfirm(c)
	if err != nil {
		t.Fatal(err)
	}
	if answer != true {
		t.Fatal("Confirmation should have been approved.")
	}

}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"net/url"
	"os"
	"strings"

	"github.com/google/webpackager/internal/customflag"
	multierror "github.com/hashicorp/go-multierror"
)

var (
	flagURL     = customflag.MultiString("url", `URL of an HTML page. Ignored when --url_file is given. (repeatable)`)
	flagURLFile = flag.String("url_file", "", `File to read the URL list from, or "-" to read it from stdin.`)
)

func getURLListFromFlags() ([]*url.URL, error) {
	unparsed, err := getURLStringList()
	if err != nil {
		return nil, err
	}
	if len(unparsed) == 0 {
		return nil, errors.New("no urls to process")
	}

	errs := new(multierror.Error)

	urls := make([]*url.URL, len(unparsed))
	for i, s := range unparsed {
		urls[i], err = url.Parse(s)
		if err != nil {
			if uerr, ok := err.(*url.Error); ok {
				err = uerr.Err
			}
			errs = multierror.Append(errs, fmt.Errorf("malformed url %q: %v", s, err))
		}
	}
	if err := errs.ErrorOrNil(); err != nil {
		return nil, err
	}
	return urls, nil
}

func getURLStringList() ([]string, error) {
	if *flagURLFile == "" {
		return *flagURL, nil
	}
	if len(*flagURL) != 0 {
		return nil, errors.New("--url and --url_file may not be used together")
	}
	if *flagURLFile == "-" {
		return readLines(bufio.NewScanner(os.Stdin))
	} else {
		f, err := os.Open(*flagURLFile)
		if err != nil {
			return nil, err
		}
		defer f.Close()
		return readLines(bufio.NewScanner(f))
	}
}

func removeComment(s string) string {
	return strings.SplitN(s, "#", 2)[0]
}

func readLines(scanner *bufio.Scanner) ([]string, error) {
	lines := []string{}
	for scanner.Scan() {
		if s := strings.TrimSpace(removeComment(scanner.Text())); s != "" {
			lines = append(lines, s)
		}
	}
	return lines, scanner.Err()
}
<commit_msg>Small change to make golint slightly happier. (#36)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"strings"

	"github.com/google/webpackager/internal/customflag"
	multierror "github.com/hashicorp/go-multierror"
)

var (
	flagURL     = customflag.MultiString("url", `URL of an HTML page. Ignored when --url_file is given. (repeatable)`)
	flagURLFile = flag.String("url_file", "", `File to read the URL list from, or "-" to read it from stdin.`)
)

func getURLListFromFlags() ([]*url.URL, error) {
	unparsed, err := getURLStringList()
	if err != nil {
		return nil, err
	}
	if len(unparsed) == 0 {
		return nil, errors.New("no urls to process")
	}

	errs := new(multierror.Error)

	urls := make([]*url.URL, len(unparsed))
	for i, s := range unparsed {
		urls[i], err = url.Parse(s)
		if err != nil {
			if uerr, ok := err.(*url.Error); ok {
				err = uerr.Err
			}
			errs = multierror.Append(errs, fmt.Errorf("malformed url %q: %v", s, err))
		}
	}
	if err := errs.ErrorOrNil(); err != nil {
		return nil, err
	}
	return urls, nil
}

func getURLStringList() ([]string, error) {
	if *flagURLFile == "" {
		return *flagURL, nil
	}
	if len(*flagURL) != 0 {
		return nil, errors.New("--url and --url_file may not be used together")
	}
	f, err := openFile(*flagURLFile)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return readLines(bufio.NewScanner(f))
}

func openFile(filename string) (io.ReadCloser, error) {
	if filename == "-" {
		return ioutil.NopCloser(os.Stdin), nil
	}
	return os.Open(filename)
}

func removeComment(s string) string {
	return strings.SplitN(s, "#", 2)[0]
}

func readLines(scanner *bufio.Scanner) ([]string, error) {
	lines := []string{}
	for scanner.Scan() {
		if s := strings.TrimSpace(removeComment(scanner.Text())); s != "" {
			lines = append(lines, s)
		}
	}
	return lines, scanner.Err()
}
<|endoftext|>


<commit_before>package application

import (
	"net/http"
	"strings"

	"github.com/carbocation/interpose"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"github.com/jmoiron/sqlx"
	"github.com/justinas/alice"
	_ "github.com/lib/pq"
	"github.com/mattes/migrate/migrate"
	"github.com/resourced/resourced-master/config"
	"github.com/resourced/resourced-master/dal"
	"github.com/resourced/resourced-master/handlers"
	"github.com/resourced/resourced-master/libtime"
	"github.com/resourced/resourced-master/middlewares"
	"github.com/resourced/resourced-master/wstrafficker"
)

// New is the constructor for Application struct.
func New(configDir string) (*Application, error) {
	generalConfig, err := config.NewGeneralConfig(configDir)
	if err != nil {
		return nil, err
	}

	db, err := sqlx.Connect("postgres", generalConfig.DSN)
	if err != nil {
		return nil, err
	}

	app := &Application{}
	app.GeneralConfig = generalConfig
	app.db = db
	app.cookieStore = sessions.NewCookieStore([]byte(app.GeneralConfig.CookieSecret))
	app.WSTraffickers = wstrafficker.NewWSTraffickers()

	return app, err
}

// Application is the application object that runs HTTP server.
type Application struct {
	GeneralConfig config.GeneralConfig
	db            *sqlx.DB
	cookieStore   *sessions.CookieStore
	WSTraffickers *wstrafficker.WSTraffickers
}

func (app *Application) MiddlewareStruct() (*interpose.Middleware, error) {
	middle := interpose.New()
	middle.Use(middlewares.SetAddr(app.GeneralConfig.Addr))
	middle.Use(middlewares.SetDB(app.db))
	middle.Use(middlewares.SetCookieStore(app.cookieStore))
	middle.Use(middlewares.SetWSTraffickers(app.WSTraffickers))

	middle.UseHandler(app.mux())

	return middle, nil
}

func (app *Application) mux() *mux.Router {
	MustLogin := middlewares.MustLogin
	MustLoginApi := middlewares.MustLoginApi
	SetClusters := middlewares.SetClusters

	router := mux.NewRouter()

	router.HandleFunc("/signup", handlers.GetSignup).Methods("GET")
	router.HandleFunc("/signup", handlers.PostSignup).Methods("POST")
	router.HandleFunc("/login", handlers.GetLogin).Methods("GET")
	router.HandleFunc("/login", handlers.PostLogin).Methods("POST")
	router.HandleFunc("/logout", handlers.GetLogout).Methods("GET")

	router.Handle("/", alice.New(MustLogin, SetClusters).ThenFunc(handlers.GetHosts)).Methods("GET")

	router.Handle("/metadata", alice.New(MustLogin, SetClusters).ThenFunc(handlers.GetMetadata)).Methods("GET")
	router.Handle("/metadata", alice.New(MustLogin, SetClusters).ThenFunc(handlers.PostMetadata)).Methods("POST")
	router.Handle(`/metadata/{key}`, alice.New(MustLogin, SetClusters).ThenFunc(handlers.DeleteMetadataKey)).Methods("POST", "DELETE")

	router.Handle("/watchers", alice.New(MustLogin, SetClusters).ThenFunc(handlers.GetWatchers)).Methods("GET")
	router.Handle("/watchers", alice.New(MustLogin, SetClusters).ThenFunc(handlers.PostWatchers)).Methods("POST")
	router.Handle("/watchers/{id:[0-9]+}", alice.New(MustLogin, SetClusters).ThenFunc(handlers.PostPutDeleteWatcherID)).Methods("POST", "PUT", "DELETE")

	router.Handle("/users/{id:[0-9]+}", alice.New(MustLogin).ThenFunc(handlers.PostPutDeleteUsersID)).Methods("POST", "PUT", "DELETE")

	router.Handle("/clusters", alice.New(MustLogin, SetClusters).ThenFunc(handlers.GetClusters)).Methods("GET")
	router.Handle("/clusters", alice.New(MustLogin).ThenFunc(handlers.PostClusters)).Methods("POST")

	router.Handle("/clusters/current", alice.New(MustLogin, SetClusters).ThenFunc(handlers.PostClustersCurrent)).Methods("POST")

	router.Handle("/clusters/{id:[0-9]+}/access-tokens", alice.New(MustLogin).ThenFunc(handlers.PostAccessTokens)).Methods("POST")

	router.Handle("/access-tokens/{id:[0-9]+}/level", alice.New(MustLogin).ThenFunc(handlers.PostAccessTokensLevel)).Methods("POST")
	router.Handle("/access-tokens/{id:[0-9]+}/enabled", alice.New(MustLogin).ThenFunc(handlers.PostAccessTokensEnabled)).Methods("POST")

	router.Handle("/saved-queries", alice.New(MustLogin).ThenFunc(handlers.PostSavedQueries)).Methods("POST")
	router.Handle("/saved-queries/{id:[0-9]+}", alice.New(MustLogin).ThenFunc(handlers.PostPutDeleteSavedQueriesID)).Methods("POST", "PUT", "DELETE")

	router.HandleFunc("/api/ws/access-tokens/{id}", handlers.ApiWSAccessToken)

	router.Handle("/api/hosts", alice.New(MustLoginApi).ThenFunc(handlers.GetApiHosts)).Methods("GET")
	router.Handle("/api/hosts", alice.New(MustLoginApi).ThenFunc(handlers.PostApiHosts)).Methods("POST")

	router.Handle("/api/metadata", alice.New(MustLoginApi).ThenFunc(handlers.GetApiMetadata)).Methods("GET")
	router.Handle(`/api/metadata/{key}`, alice.New(MustLoginApi).ThenFunc(handlers.PostApiMetadataKey)).Methods("POST")
	router.Handle(`/api/metadata/{key}`, alice.New(MustLoginApi).ThenFunc(handlers.DeleteApiMetadataKey)).Methods("DELETE")
	router.Handle(`/api/metadata/{key}`, alice.New(MustLoginApi).ThenFunc(handlers.GetApiMetadataKey)).Methods("GET")

	// Path of static files must be last!
	router.PathPrefix("/").Handler(http.FileServer(http.Dir("static")))

	return router
}

func (app *Application) MigrateUp() (err []error, ok bool) {
	return migrate.UpSync(app.GeneralConfig.DSN, "./migrations")
}

func (app *Application) WatchAll() {
	// hostname, err := os.Hostname()
	// if err != nil {
	// 	return
	// }

	clusterRows, err := dal.NewCluster(app.db).AllClusters(nil)
	if err != nil {
		return
	}

	for _, clusterRow := range clusterRows {
		// TODO(didip): We don't wan't to fetch all watchers. Use modulo!
		watcherRows, err := dal.NewWatcher(app.db).AllByClusterID(nil, clusterRow.ID)
		if err != nil {
			return
		}

		watcherResultRows, err := dal.NewWatcherResult(app.db).AllByClusterID(nil, clusterRow.ID)
		if err != nil {
			return
		}

		for _, watcherRow := range watcherRows {
			go func() {

				var watcherResultRow *dal.WatcherResultRow

				for _, wr := range watcherResultRows {
					if wr.ClusterID == clusterRow.ID {
						watcherResultRow = wr
						break
					}
				}

				if watcherResultRow != nil {
					app.WatchOnce(clusterRow, watcherRow, watcherResultRow)
				}

				libtime.SleepString(watcherRow.CheckInterval)
			}()
		}
	}
}

func (app *Application) WatchOnce(clusterRow *dal.ClusterRow, watcherRow *dal.WatcherRow, watcherResultRow *dal.WatcherResultRow) error {
	var err error

	lastUpdated := strings.Replace(watcherRow.HostsLastUpdated, " ago", "", 1)

	numAffectedHosts, err := dal.NewHost(app.db).CountByClusterIDQueryAndUpdatedInterval(nil, clusterRow.ID, watcherRow.SavedQuery, lastUpdated)
	if err != nil {
		return err
	}

	watcherResult := dal.NewWatcherResult(app.db)

	if numAffectedHosts == 0 || numAffectedHosts < watcherRow.LowThreshold {
		watcherResultRow, err = watcherResult.ResetCountByClusterIDAndWatcherID(nil, clusterRow.ID, watcherRow.ID)
		if err != nil {
			return err
		}
		return nil
	}

	if watcherRow.LowThreshold <= numAffectedHosts && numAffectedHosts <= watcherRow.HighThreshold {
		watcherResultRow, err = watcherResult.IncrementCountByClusterIDAndWatcherID(nil, clusterRow.ID, watcherRow.ID, 1)
		if err != nil {
			return err
		}

		// TODO(didip): perform action
	}

	return nil
}
<commit_msg>Application now stores hostname info.<commit_after>package application

import (
	"net/http"
	"os"
	"strings"

	"github.com/carbocation/interpose"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"github.com/jmoiron/sqlx"
	"github.com/justinas/alice"
	_ "github.com/lib/pq"
	"github.com/mattes/migrate/migrate"
	"github.com/resourced/resourced-master/config"
	"github.com/resourced/resourced-master/dal"
	"github.com/resourced/resourced-master/handlers"
	"github.com/resourced/resourced-master/libtime"
	"github.com/resourced/resourced-master/middlewares"
	"github.com/resourced/resourced-master/wstrafficker"
)

// New is the constructor for Application struct.
func New(configDir string) (*Application, error) {
	generalConfig, err := config.NewGeneralConfig(configDir)
	if err != nil {
		return nil, err
	}

	db, err := sqlx.Connect("postgres", generalConfig.DSN)
	if err != nil {
		return nil, err
	}

	hostname, _ := os.Hostname()

	app := &Application{}
	app.Hostname = hostname
	app.GeneralConfig = generalConfig
	app.DB = db
	app.cookieStore = sessions.NewCookieStore([]byte(app.GeneralConfig.CookieSecret))
	app.WSTraffickers = wstrafficker.NewWSTraffickers()

	return app, err
}

// Application is the application object that runs HTTP server.
type Application struct {
	Hostname      string
	GeneralConfig config.GeneralConfig
	DB            *sqlx.DB
	cookieStore   *sessions.CookieStore
	WSTraffickers *wstrafficker.WSTraffickers
}

func (app *Application) MiddlewareStruct() (*interpose.Middleware, error) {
	middle := interpose.New()
	middle.Use(middlewares.SetAddr(app.GeneralConfig.Addr))
	middle.Use(middlewares.SetDB(app.DB))
	middle.Use(middlewares.SetCookieStore(app.cookieStore))
	middle.Use(middlewares.SetWSTraffickers(app.WSTraffickers))

	middle.UseHandler(app.mux())

	return middle, nil
}

func (app *Application) mux() *mux.Router {
	MustLogin := middlewares.MustLogin
	MustLoginApi := middlewares.MustLoginApi
	SetClusters := middlewares.SetClusters

	router := mux.NewRouter()

	router.HandleFunc("/signup", handlers.GetSignup).Methods("GET")
	router.HandleFunc("/signup", handlers.PostSignup).Methods("POST")
	router.HandleFunc("/login", handlers.GetLogin).Methods("GET")
	router.HandleFunc("/login", handlers.PostLogin).Methods("POST")
	router.HandleFunc("/logout", handlers.GetLogout).Methods("GET")

	router.Handle("/", alice.New(MustLogin, SetClusters).ThenFunc(handlers.GetHosts)).Methods("GET")

	router.Handle("/metadata", alice.New(MustLogin, SetClusters).ThenFunc(handlers.GetMetadata)).Methods("GET")
	router.Handle("/metadata", alice.New(MustLogin, SetClusters).ThenFunc(handlers.PostMetadata)).Methods("POST")
	router.Handle(`/metadata/{key}`, alice.New(MustLogin, SetClusters).ThenFunc(handlers.DeleteMetadataKey)).Methods("POST", "DELETE")

	router.Handle("/watchers", alice.New(MustLogin, SetClusters).ThenFunc(handlers.GetWatchers)).Methods("GET")
	router.Handle("/watchers", alice.New(MustLogin, SetClusters).ThenFunc(handlers.PostWatchers)).Methods("POST")
	router.Handle("/watchers/{id:[0-9]+}", alice.New(MustLogin, SetClusters).ThenFunc(handlers.PostPutDeleteWatcherID)).Methods("POST", "PUT", "DELETE")

	router.Handle("/users/{id:[0-9]+}", alice.New(MustLogin).ThenFunc(handlers.PostPutDeleteUsersID)).Methods("POST", "PUT", "DELETE")

	router.Handle("/clusters", alice.New(MustLogin, SetClusters).ThenFunc(handlers.GetClusters)).Methods("GET")
	router.Handle("/clusters", alice.New(MustLogin).ThenFunc(handlers.PostClusters)).Methods("POST")

	router.Handle("/clusters/current", alice.New(MustLogin, SetClusters).ThenFunc(handlers.PostClustersCurrent)).Methods("POST")

	router.Handle("/clusters/{id:[0-9]+}/access-tokens", alice.New(MustLogin).ThenFunc(handlers.PostAccessTokens)).Methods("POST")

	router.Handle("/access-tokens/{id:[0-9]+}/level", alice.New(MustLogin).ThenFunc(handlers.PostAccessTokensLevel)).Methods("POST")
	router.Handle("/access-tokens/{id:[0-9]+}/enabled", alice.New(MustLogin).ThenFunc(handlers.PostAccessTokensEnabled)).Methods("POST")

	router.Handle("/saved-queries", alice.New(MustLogin).ThenFunc(handlers.PostSavedQueries)).Methods("POST")
	router.Handle("/saved-queries/{id:[0-9]+}", alice.New(MustLogin).ThenFunc(handlers.PostPutDeleteSavedQueriesID)).Methods("POST", "PUT", "DELETE")

	router.HandleFunc("/api/ws/access-tokens/{id}", handlers.ApiWSAccessToken)

	router.Handle("/api/hosts", alice.New(MustLoginApi).ThenFunc(handlers.GetApiHosts)).Methods("GET")
	router.Handle("/api/hosts", alice.New(MustLoginApi).ThenFunc(handlers.PostApiHosts)).Methods("POST")

	router.Handle("/api/metadata", alice.New(MustLoginApi).ThenFunc(handlers.GetApiMetadata)).Methods("GET")
	router.Handle(`/api/metadata/{key}`, alice.New(MustLoginApi).ThenFunc(handlers.PostApiMetadataKey)).Methods("POST")
	router.Handle(`/api/metadata/{key}`, alice.New(MustLoginApi).ThenFunc(handlers.DeleteApiMetadataKey)).Methods("DELETE")
	router.Handle(`/api/metadata/{key}`, alice.New(MustLoginApi).ThenFunc(handlers.GetApiMetadataKey)).Methods("GET")

	// Path of static files must be last!
	router.PathPrefix("/").Handler(http.FileServer(http.Dir("static")))

	return router
}

func (app *Application) MigrateUp() (err []error, ok bool) {
	return migrate.UpSync(app.GeneralConfig.DSN, "./migrations")
}

func (app *Application) WatchAll() {
	// hostname, err := os.Hostname()
	// if err != nil {
	// 	return
	// }

	clusterRows, err := dal.NewCluster(app.DB).AllClusters(nil)
	if err != nil {
		return
	}

	for _, clusterRow := range clusterRows {
		// TODO(didip): We don't wan't to fetch all watchers. Use modulo!
		watcherRows, err := dal.NewWatcher(app.DB).AllByClusterID(nil, clusterRow.ID)
		if err != nil {
			return
		}

		watcherResultRows, err := dal.NewWatcherResult(app.DB).AllByClusterID(nil, clusterRow.ID)
		if err != nil {
			return
		}

		for _, watcherRow := range watcherRows {
			go func() {

				var watcherResultRow *dal.WatcherResultRow

				for _, wr := range watcherResultRows {
					if wr.ClusterID == clusterRow.ID {
						watcherResultRow = wr
						break
					}
				}

				if watcherResultRow != nil {
					app.WatchOnce(clusterRow, watcherRow, watcherResultRow)
				}

				libtime.SleepString(watcherRow.CheckInterval)
			}()
		}
	}
}

func (app *Application) WatchOnce(clusterRow *dal.ClusterRow, watcherRow *dal.WatcherRow, watcherResultRow *dal.WatcherResultRow) error {
	var err error

	lastUpdated := strings.Replace(watcherRow.HostsLastUpdated, " ago", "", 1)

	numAffectedHosts, err := dal.NewHost(app.DB).CountByClusterIDQueryAndUpdatedInterval(nil, clusterRow.ID, watcherRow.SavedQuery, lastUpdated)
	if err != nil {
		return err
	}

	watcherResult := dal.NewWatcherResult(app.DB)

	if numAffectedHosts == 0 || numAffectedHosts < watcherRow.LowThreshold {
		watcherResultRow, err = watcherResult.ResetCountByClusterIDAndWatcherID(nil, clusterRow.ID, watcherRow.ID)
		if err != nil {
			return err
		}
		return nil
	}

	if watcherRow.LowThreshold <= numAffectedHosts && numAffectedHosts <= watcherRow.HighThreshold {
		watcherResultRow, err = watcherResult.IncrementCountByClusterIDAndWatcherID(nil, clusterRow.ID, watcherRow.ID, 1)
		if err != nil {
			return err
		}

		// TODO(didip): perform action
	}

	return nil
}
<|endoftext|>


<commit_before>package hukabuk

import (
	"errors"
	"net/http"

	"appengine/user"

	"github.com/GoogleCloudPlatform/go-endpoints/endpoints"
)

var (
	scopes    = []string{endpoints.EmailScope}
	clientIds = []string{ClientID, endpoints.APIExplorerClientID}
	// in case we'll want to use Huk-A-Buk API from an Android app
	audiences = []string{ClientID}
)

type EmptyRequest struct {
}

// Huk-A-Buk API service
type HukABukApi struct {
}

func (hapi *HukABukApi) GetCards(r *http.Request,
	req *EmptyRequest, resp *Hand) error {

	c := endpoints.NewContext(r)
	u, err := getCurrentUser(c) // Not Used
	if err != nil {
		return err
	}
	c.Infof("%v", u)

	var card *Card
	card, err = NewCard('H', '2')
	resp.Cards = append(resp.Cards, *card)

	card, err = NewCard('S', '7')
	resp.Cards = append(resp.Cards, *card)

	card, err = NewCard('C', 'T')
	resp.Cards = append(resp.Cards, *card)

	card, err = NewCard('D', 'K')
	resp.Cards = append(resp.Cards, *card)

	card, err = NewCard('H', 'A')
	resp.Cards = append(resp.Cards, *card)

	if err == nil {
		err = StoreHand(c, u, resp)
	}

	return err
}

// getCurrentUser retrieves a user associated with the request.
// If there's no user (e.g. no auth info present in the request) returns
// an "unauthorized" error.
func getCurrentUser(c endpoints.Context) (*user.User, error) {
	u, err := endpoints.CurrentUser(c, scopes, audiences, clientIds)
	if err != nil {
		return nil, err
	}
	if u == nil {
		return nil, errors.New("Unauthorized: Please, sign in.")
	}
	c.Debugf("Current user: %#v", u)
	return u, nil
}

// RegisterService exposes HukABukApi methods as API endpoints.
//
// The registration/initialization during startup is not performed here.
func RegisterService() (*endpoints.RPCService, error) {
	api := &HukABukApi{}
	rpcService, err := endpoints.RegisterService(
		api,
		"hukabuk",
		"v1beta",
		"Huk-A-Buk API",
		true,
	)
	if err != nil {
		return nil, err
	}

	info := rpcService.MethodByName("GetCards").Info()
	info.Path, info.HTTPMethod, info.Name = "cards", "GET", "cards.list"
	info.Scopes, info.ClientIds, info.Audiences = scopes, clientIds, audiences

	return rpcService, nil
}

func init() {
	if _, err := RegisterService(); err != nil {
		panic(err.Error())
	}
	endpoints.HandleHTTP()
}
<commit_msg>Using local version of endpoints auth before tweaking.<commit_after>package hukabuk

import (
	"errors"
	"net/http"

	"appengine/user"

	"github.com/GoogleCloudPlatform/go-endpoints/endpoints"
)

var (
	scopes    = []string{endpoints.EmailScope}
	clientIds = []string{ClientID, endpoints.APIExplorerClientID}
	// in case we'll want to use Huk-A-Buk API from an Android app
	audiences = []string{ClientID}
)

type EmptyRequest struct {
}

// Huk-A-Buk API service
type HukABukApi struct {
}

func (hapi *HukABukApi) GetCards(r *http.Request,
	req *EmptyRequest, resp *Hand) error {

	c := NewContext(r)
	u, err := getCurrentUser(c) // Not Used
	if err != nil {
		return err
	}
	c.Infof("%v", u)

	var card *Card
	card, err = NewCard('H', '2')
	resp.Cards = append(resp.Cards, *card)

	card, err = NewCard('S', '7')
	resp.Cards = append(resp.Cards, *card)

	card, err = NewCard('C', 'T')
	resp.Cards = append(resp.Cards, *card)

	card, err = NewCard('D', 'K')
	resp.Cards = append(resp.Cards, *card)

	card, err = NewCard('H', 'A')
	resp.Cards = append(resp.Cards, *card)

	if err == nil {
		err = StoreHand(c, u, resp)
	}

	return err
}

// getCurrentUser retrieves a user associated with the request.
// If there's no user (e.g. no auth info present in the request) returns
// an "unauthorized" error.
func getCurrentUser(c Context) (*user.User, error) {
	u, err := CurrentUser(c, scopes, audiences, clientIds)
	if err != nil {
		return nil, err
	}
	if u == nil {
		return nil, errors.New("Unauthorized: Please, sign in.")
	}
	c.Debugf("Current user: %#v", u)
	return u, nil
}

// RegisterService exposes HukABukApi methods as API endpoints.
//
// The registration/initialization during startup is not performed here.
func RegisterService() (*endpoints.RPCService, error) {
	api := &HukABukApi{}
	rpcService, err := endpoints.RegisterService(
		api,
		"hukabuk",
		"v1beta",
		"Huk-A-Buk API",
		true,
	)
	if err != nil {
		return nil, err
	}

	info := rpcService.MethodByName("GetCards").Info()
	info.Path, info.HTTPMethod, info.Name = "cards", "GET", "cards.list"
	info.Scopes, info.ClientIds, info.Audiences = scopes, clientIds, audiences

	return rpcService, nil
}

func init() {
	if _, err := RegisterService(); err != nil {
		panic(err.Error())
	}
	endpoints.HandleHTTP()
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package k8s abstracts all Kubernetes specific behaviour
package k8s

import (
	"os"
	"strings"
)

var (
	// config is the configuration of Kubernetes related settings
	config configuration
)

type configuration struct {
	// APIServerURL is the URL address of the API server
	APIServerURL string

	// KubeconfigPath is the local path to the kubeconfig configuration
	// file on the filesystem
	KubeconfigPath string

	// QPS is the QPS to pass to the kubernetes client configuration.
	QPS float32

	// Burst is the burst to pass to the kubernetes client configuration.
	Burst int
}

// GetAPIServerURL returns the configured API server URL address
func GetAPIServerURL() string {
	return config.APIServerURL
}

// GetKubeconfigPath returns the configured path to the kubeconfig
// configuration file
func GetKubeconfigPath() string {
	return config.KubeconfigPath
}

// GetQPS gets the QPS of the K8s configuration.
func GetQPS() float32 {
	return config.QPS
}

// GetBurst gets the burst limit of the K8s configuration.
func GetBurst() int {
	return config.Burst
}

// Configure sets the parameters of the Kubernetes package
func Configure(apiServerURL, kubeconfigPath string, qps float32, burst int) {
	config.APIServerURL = apiServerURL
	config.KubeconfigPath = kubeconfigPath
	config.QPS = qps
	config.Burst = burst

	if IsEnabled() &&
		config.APIServerURL != "" &&
		!strings.HasPrefix(apiServerURL, "http") {
		config.APIServerURL = "http://" + apiServerURL // default to HTTP
	}
}

// IsEnabled checks if Cilium is being used in tandem with Kubernetes.
func IsEnabled() bool {
	return config.APIServerURL != "" ||
		config.KubeconfigPath != "" ||
		(os.Getenv("KUBERNETES_SERVICE_HOST") != "" &&
			os.Getenv("KUBERNETES_SERVICE_PORT") != "")
}
<commit_msg>pkg/k8s: detect k8s mode if env variable K8S_NODE_NAME is set<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package k8s abstracts all Kubernetes specific behaviour
package k8s

import (
	"os"
	"strings"
)

var (
	// config is the configuration of Kubernetes related settings
	config configuration
)

type configuration struct {
	// APIServerURL is the URL address of the API server
	APIServerURL string

	// KubeconfigPath is the local path to the kubeconfig configuration
	// file on the filesystem
	KubeconfigPath string

	// QPS is the QPS to pass to the kubernetes client configuration.
	QPS float32

	// Burst is the burst to pass to the kubernetes client configuration.
	Burst int
}

// GetAPIServerURL returns the configured API server URL address
func GetAPIServerURL() string {
	return config.APIServerURL
}

// GetKubeconfigPath returns the configured path to the kubeconfig
// configuration file
func GetKubeconfigPath() string {
	return config.KubeconfigPath
}

// GetQPS gets the QPS of the K8s configuration.
func GetQPS() float32 {
	return config.QPS
}

// GetBurst gets the burst limit of the K8s configuration.
func GetBurst() int {
	return config.Burst
}

// Configure sets the parameters of the Kubernetes package
func Configure(apiServerURL, kubeconfigPath string, qps float32, burst int) {
	config.APIServerURL = apiServerURL
	config.KubeconfigPath = kubeconfigPath
	config.QPS = qps
	config.Burst = burst

	if IsEnabled() &&
		config.APIServerURL != "" &&
		!strings.HasPrefix(apiServerURL, "http") {
		config.APIServerURL = "http://" + apiServerURL // default to HTTP
	}
}

// IsEnabled checks if Cilium is being used in tandem with Kubernetes.
func IsEnabled() bool {
	return config.APIServerURL != "" ||
		config.KubeconfigPath != "" ||
		(os.Getenv("KUBERNETES_SERVICE_HOST") != "" &&
			os.Getenv("KUBERNETES_SERVICE_PORT") != "") ||
		os.Getenv("K8S_NODE_NAME") != ""
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package kubecli

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"net/url"

	"github.com/gorilla/websocket"

	k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/rest"

	"k8s.io/apimachinery/pkg/types"

	"kubevirt.io/kubevirt/pkg/api/v1"
)

const (
	WebsocketMessageBufferSize = 10240
)

func (k *kubevirt) VM(namespace string) VMInterface {
	return &vms{
		restClient: k.restClient,
		config:     k.config,
		clientSet:  k.Clientset,
		namespace:  namespace,
		resource:   "virtualmachines",
	}
}

type vms struct {
	restClient *rest.RESTClient
	config     *rest.Config
	clientSet  *kubernetes.Clientset
	namespace  string
	resource   string
	master     string
	kubeconfig string
}

type BinaryReadWriter struct {
	Conn *websocket.Conn
}

func (s *BinaryReadWriter) Write(p []byte) (int, error) {
	wsFrameHeaderSize := 2 + 8 + 4 // Fixed header + length + mask (RFC 6455)
	// our websocket package has an issue where it truncates messages
	// when the message+header is greater than the buffer size we allocate.
	// because of this, we have to chunk messages
	chunkSize := WebsocketMessageBufferSize - wsFrameHeaderSize
	bytesWritten := 0

	for i := 0; i < len(p); i += chunkSize {
		w, err := s.Conn.NextWriter(websocket.BinaryMessage)
		if err != nil {
			return bytesWritten, s.err(err)
		}
		defer w.Close()

		end := i + chunkSize
		if end > len(p) {
			end = len(p)
		}
		n, err := w.Write(p[i:end])
		if err != nil {
			return bytesWritten, err
		}

		bytesWritten = n + bytesWritten
	}
	return bytesWritten, nil

}

func (s *BinaryReadWriter) Read(p []byte) (int, error) {
	for {
		msgType, r, err := s.Conn.NextReader()
		if err != nil {
			return 0, s.err(err)
		}

		switch msgType {
		case websocket.BinaryMessage:
			n, err := r.Read(p)
			return n, s.err(err)

		case websocket.CloseMessage:
			return 0, io.EOF
		}
	}
}

func (s *BinaryReadWriter) err(err error) error {
	if err == nil {
		return nil
	}
	if e, ok := err.(*websocket.CloseError); ok {
		if e.Code == websocket.CloseNormalClosure {
			return io.EOF
		}
	}
	return err
}

type RoundTripCallback func(conn *websocket.Conn, resp *http.Response, err error) error

type WebsocketRoundTripper struct {
	Dialer *websocket.Dialer
	Do     RoundTripCallback
}

func (d *WebsocketRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
	conn, resp, err := d.Dialer.Dial(r.URL.String(), r.Header)
	if err == nil {
		defer conn.Close()
	}
	return resp, d.Do(conn, resp, err)
}

type asyncWSRoundTripper struct {
	Done       chan struct{}
	Connection chan *websocket.Conn
}

func (aws *asyncWSRoundTripper) WebsocketCallback(ws *websocket.Conn, resp *http.Response, err error) error {

	if err != nil {
		if resp != nil && resp.StatusCode != http.StatusOK {
			buf := new(bytes.Buffer)
			buf.ReadFrom(resp.Body)
			return fmt.Errorf("Can't connect to websocket (%d): %s\n", resp.StatusCode, buf.String())
		}
		return fmt.Errorf("Can't connect to websocket: %s\n", err.Error())
	}
	aws.Connection <- ws

	// Keep the roundtripper open until we are done with the stream
	<-aws.Done
	return nil
}

func roundTripperFromConfig(config *rest.Config, callback RoundTripCallback) (http.RoundTripper, error) {

	// Configure TLS
	tlsConfig, err := rest.TLSConfigFor(config)
	if err != nil {
		return nil, err
	}

	// Configure the websocket dialer
	dialer := &websocket.Dialer{
		Proxy:           http.ProxyFromEnvironment,
		TLSClientConfig: tlsConfig,
		WriteBufferSize: WebsocketMessageBufferSize,
		ReadBufferSize:  WebsocketMessageBufferSize,
	}

	// Create a roundtripper which will pass in the final underlying websocket connection to a callback
	rt := &WebsocketRoundTripper{
		Do:     callback,
		Dialer: dialer,
	}

	// Make sure we inherit all relevant security headers
	return rest.HTTPWrappersForConfig(config, rt)
}

func RequestFromConfig(config *rest.Config, vm string, namespace string, resource string) (*http.Request, error) {

	u, err := url.Parse(config.Host)
	if err != nil {
		return nil, err
	}

	switch u.Scheme {
	case "https":
		u.Scheme = "wss"
	case "http":
		u.Scheme = "ws"
	default:
		return nil, fmt.Errorf("Unsupported Protocol %s", u.Scheme)
	}

	u.Path = fmt.Sprintf("/apis/subresources.kubevirt.io/v1alpha1/namespaces/%s/virtualmachines/%s/%s", namespace, vm, resource)
	req := &http.Request{
		Method: http.MethodGet,
		URL:    u,
	}

	return req, nil
}

type wsStreamer struct {
	conn *websocket.Conn
	done chan struct{}
}

func (ws *wsStreamer) Stream(options StreamOptions) error {
	wsReadWriter := &BinaryReadWriter{Conn: ws.conn}

	copyErr := make(chan error)

	go func() {
		_, err := io.Copy(wsReadWriter, options.In)
		copyErr <- err
	}()

	go func() {
		_, err := io.Copy(options.Out, wsReadWriter)
		copyErr <- err
	}()

	close(ws.done)
	return <-copyErr
}

func (v *vms) VNC(name string) (StreamInterface, error) {
	return v.asyncSubresourceHelper(name, "vnc")
}
func (v *vms) SerialConsole(name string) (StreamInterface, error) {
	return v.asyncSubresourceHelper(name, "console")
}

func (v *vms) asyncSubresourceHelper(name string, resource string) (StreamInterface, error) {

	done := make(chan struct{})

	aws := &asyncWSRoundTripper{
		Connection: make(chan *websocket.Conn),
		Done:       done,
	}
	// Create a round tripper with all necessary kubernetes security details
	wrappedRoundTripper, err := roundTripperFromConfig(v.config, aws.WebsocketCallback)
	if err != nil {
		return nil, fmt.Errorf("unable to create round tripper for remote execution: %v", err)
	}

	// Create a request out of config and the query parameters
	req, err := RequestFromConfig(v.config, name, v.namespace, resource)
	if err != nil {
		return nil, fmt.Errorf("unable to create request for remote execution: %v", err)
	}

	errChan := make(chan error)

	go func() {
		// Send the request and let the callback do its work
		response, err := wrappedRoundTripper.RoundTrip(req)

		if err != nil {
			errChan <- err
			return
		}

		if response != nil {
			switch response.StatusCode {
			case http.StatusOK:
			case http.StatusNotFound:
				err = fmt.Errorf("Virtual Machine not found.")
			case http.StatusInternalServerError:
				err = fmt.Errorf("Websocket failed due to internal server error.")
			default:
				err = fmt.Errorf("Websocket failed with http status: %s", response.Status)
			}
		} else {
			err = fmt.Errorf("no response received")
		}
		errChan <- err
	}()

	select {
	case err = <-errChan:
		return nil, err
	case ws := <-aws.Connection:
		return &wsStreamer{
			conn: ws,
			done: done,
		}, nil
	}
}

func (v *vms) Get(name string, options k8smetav1.GetOptions) (vm *v1.VirtualMachine, err error) {
	vm = &v1.VirtualMachine{}
	err = v.restClient.Get().
		Resource(v.resource).
		Namespace(v.namespace).
		Name(name).
		VersionedParams(&options, scheme.ParameterCodec).
		Do().
		Into(vm)
	vm.SetGroupVersionKind(v1.VirtualMachineGroupVersionKind)
	return
}

func (v *vms) List(options k8smetav1.ListOptions) (vmList *v1.VirtualMachineList, err error) {
	vmList = &v1.VirtualMachineList{}
	err = v.restClient.Get().
		Resource(v.resource).
		Namespace(v.namespace).
		VersionedParams(&options, scheme.ParameterCodec).
		Do().
		Into(vmList)
	for _, vm := range vmList.Items {
		vm.SetGroupVersionKind(v1.VirtualMachineGroupVersionKind)
	}

	return
}

func (v *vms) Create(vm *v1.VirtualMachine) (result *v1.VirtualMachine, err error) {
	result = &v1.VirtualMachine{}
	err = v.restClient.Post().
		Namespace(v.namespace).
		Resource(v.resource).
		Body(vm).
		Do().
		Into(result)
	result.SetGroupVersionKind(v1.VirtualMachineGroupVersionKind)
	return
}

func (v *vms) Update(vm *v1.VirtualMachine) (result *v1.VirtualMachine, err error) {
	result = &v1.VirtualMachine{}
	err = v.restClient.Put().
		Name(vm.ObjectMeta.Name).
		Namespace(v.namespace).
		Resource(v.resource).
		Body(vm).
		Do().
		Into(result)
	result.SetGroupVersionKind(v1.VirtualMachineGroupVersionKind)
	return
}

func (v *vms) Delete(name string, options *k8smetav1.DeleteOptions) error {
	return v.restClient.Delete().
		Namespace(v.namespace).
		Resource(v.resource).
		Name(name).
		Body(options).
		Do().
		Error()
}

func (v *vms) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.VirtualMachine, err error) {
	result = &v1.VirtualMachine{}
	err = v.restClient.Patch(pt).
		Namespace(v.namespace).
		Resource(v.resource).
		SubResource(subresources...).
		Name(name).
		Body(data).
		Do().
		Into(result)
	return
}
<commit_msg>Streamer: don't signal done too early<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package kubecli

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"net/url"

	"github.com/gorilla/websocket"

	k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/rest"

	"k8s.io/apimachinery/pkg/types"

	"kubevirt.io/kubevirt/pkg/api/v1"
)

const (
	WebsocketMessageBufferSize = 10240
)

func (k *kubevirt) VM(namespace string) VMInterface {
	return &vms{
		restClient: k.restClient,
		config:     k.config,
		clientSet:  k.Clientset,
		namespace:  namespace,
		resource:   "virtualmachines",
	}
}

type vms struct {
	restClient *rest.RESTClient
	config     *rest.Config
	clientSet  *kubernetes.Clientset
	namespace  string
	resource   string
	master     string
	kubeconfig string
}

type BinaryReadWriter struct {
	Conn *websocket.Conn
}

func (s *BinaryReadWriter) Write(p []byte) (int, error) {
	wsFrameHeaderSize := 2 + 8 + 4 // Fixed header + length + mask (RFC 6455)
	// our websocket package has an issue where it truncates messages
	// when the message+header is greater than the buffer size we allocate.
	// because of this, we have to chunk messages
	chunkSize := WebsocketMessageBufferSize - wsFrameHeaderSize
	bytesWritten := 0

	for i := 0; i < len(p); i += chunkSize {
		w, err := s.Conn.NextWriter(websocket.BinaryMessage)
		if err != nil {
			return bytesWritten, s.err(err)
		}
		defer w.Close()

		end := i + chunkSize
		if end > len(p) {
			end = len(p)
		}
		n, err := w.Write(p[i:end])
		if err != nil {
			return bytesWritten, err
		}

		bytesWritten = n + bytesWritten
	}
	return bytesWritten, nil

}

func (s *BinaryReadWriter) Read(p []byte) (int, error) {
	for {
		msgType, r, err := s.Conn.NextReader()
		if err != nil {
			return 0, s.err(err)
		}

		switch msgType {
		case websocket.BinaryMessage:
			n, err := r.Read(p)
			return n, s.err(err)

		case websocket.CloseMessage:
			return 0, io.EOF
		}
	}
}

func (s *BinaryReadWriter) err(err error) error {
	if err == nil {
		return nil
	}
	if e, ok := err.(*websocket.CloseError); ok {
		if e.Code == websocket.CloseNormalClosure {
			return io.EOF
		}
	}
	return err
}

type RoundTripCallback func(conn *websocket.Conn, resp *http.Response, err error) error

type WebsocketRoundTripper struct {
	Dialer *websocket.Dialer
	Do     RoundTripCallback
}

func (d *WebsocketRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
	conn, resp, err := d.Dialer.Dial(r.URL.String(), r.Header)
	if err == nil {
		defer conn.Close()
	}
	return resp, d.Do(conn, resp, err)
}

type asyncWSRoundTripper struct {
	Done       chan struct{}
	Connection chan *websocket.Conn
}

func (aws *asyncWSRoundTripper) WebsocketCallback(ws *websocket.Conn, resp *http.Response, err error) error {

	if err != nil {
		if resp != nil && resp.StatusCode != http.StatusOK {
			buf := new(bytes.Buffer)
			buf.ReadFrom(resp.Body)
			return fmt.Errorf("Can't connect to websocket (%d): %s\n", resp.StatusCode, buf.String())
		}
		return fmt.Errorf("Can't connect to websocket: %s\n", err.Error())
	}
	aws.Connection <- ws

	// Keep the roundtripper open until we are done with the stream
	<-aws.Done
	return nil
}

func roundTripperFromConfig(config *rest.Config, callback RoundTripCallback) (http.RoundTripper, error) {

	// Configure TLS
	tlsConfig, err := rest.TLSConfigFor(config)
	if err != nil {
		return nil, err
	}

	// Configure the websocket dialer
	dialer := &websocket.Dialer{
		Proxy:           http.ProxyFromEnvironment,
		TLSClientConfig: tlsConfig,
		WriteBufferSize: WebsocketMessageBufferSize,
		ReadBufferSize:  WebsocketMessageBufferSize,
	}

	// Create a roundtripper which will pass in the final underlying websocket connection to a callback
	rt := &WebsocketRoundTripper{
		Do:     callback,
		Dialer: dialer,
	}

	// Make sure we inherit all relevant security headers
	return rest.HTTPWrappersForConfig(config, rt)
}

func RequestFromConfig(config *rest.Config, vm string, namespace string, resource string) (*http.Request, error) {

	u, err := url.Parse(config.Host)
	if err != nil {
		return nil, err
	}

	switch u.Scheme {
	case "https":
		u.Scheme = "wss"
	case "http":
		u.Scheme = "ws"
	default:
		return nil, fmt.Errorf("Unsupported Protocol %s", u.Scheme)
	}

	u.Path = fmt.Sprintf("/apis/subresources.kubevirt.io/v1alpha1/namespaces/%s/virtualmachines/%s/%s", namespace, vm, resource)
	req := &http.Request{
		Method: http.MethodGet,
		URL:    u,
	}

	return req, nil
}

type wsStreamer struct {
	conn *websocket.Conn
	done chan struct{}
}

func (ws *wsStreamer) streamDone() {
	close(ws.done)
}

func (ws *wsStreamer) Stream(options StreamOptions) error {
	wsReadWriter := &BinaryReadWriter{Conn: ws.conn}

	copyErr := make(chan error)

	go func() {
		_, err := io.Copy(wsReadWriter, options.In)
		copyErr <- err
	}()

	go func() {
		_, err := io.Copy(options.Out, wsReadWriter)
		copyErr <- err
	}()

	defer ws.streamDone()
	return <-copyErr
}

func (v *vms) VNC(name string) (StreamInterface, error) {
	return v.asyncSubresourceHelper(name, "vnc")
}
func (v *vms) SerialConsole(name string) (StreamInterface, error) {
	return v.asyncSubresourceHelper(name, "console")
}

func (v *vms) asyncSubresourceHelper(name string, resource string) (StreamInterface, error) {

	done := make(chan struct{})

	aws := &asyncWSRoundTripper{
		Connection: make(chan *websocket.Conn),
		Done:       done,
	}
	// Create a round tripper with all necessary kubernetes security details
	wrappedRoundTripper, err := roundTripperFromConfig(v.config, aws.WebsocketCallback)
	if err != nil {
		return nil, fmt.Errorf("unable to create round tripper for remote execution: %v", err)
	}

	// Create a request out of config and the query parameters
	req, err := RequestFromConfig(v.config, name, v.namespace, resource)
	if err != nil {
		return nil, fmt.Errorf("unable to create request for remote execution: %v", err)
	}

	errChan := make(chan error)

	go func() {
		// Send the request and let the callback do its work
		response, err := wrappedRoundTripper.RoundTrip(req)

		if err != nil {
			errChan <- err
			return
		}

		if response != nil {
			switch response.StatusCode {
			case http.StatusOK:
			case http.StatusNotFound:
				err = fmt.Errorf("Virtual Machine not found.")
			case http.StatusInternalServerError:
				err = fmt.Errorf("Websocket failed due to internal server error.")
			default:
				err = fmt.Errorf("Websocket failed with http status: %s", response.Status)
			}
		} else {
			err = fmt.Errorf("no response received")
		}
		errChan <- err
	}()

	select {
	case err = <-errChan:
		return nil, err
	case ws := <-aws.Connection:
		return &wsStreamer{
			conn: ws,
			done: done,
		}, nil
	}
}

func (v *vms) Get(name string, options k8smetav1.GetOptions) (vm *v1.VirtualMachine, err error) {
	vm = &v1.VirtualMachine{}
	err = v.restClient.Get().
		Resource(v.resource).
		Namespace(v.namespace).
		Name(name).
		VersionedParams(&options, scheme.ParameterCodec).
		Do().
		Into(vm)
	vm.SetGroupVersionKind(v1.VirtualMachineGroupVersionKind)
	return
}

func (v *vms) List(options k8smetav1.ListOptions) (vmList *v1.VirtualMachineList, err error) {
	vmList = &v1.VirtualMachineList{}
	err = v.restClient.Get().
		Resource(v.resource).
		Namespace(v.namespace).
		VersionedParams(&options, scheme.ParameterCodec).
		Do().
		Into(vmList)
	for _, vm := range vmList.Items {
		vm.SetGroupVersionKind(v1.VirtualMachineGroupVersionKind)
	}

	return
}

func (v *vms) Create(vm *v1.VirtualMachine) (result *v1.VirtualMachine, err error) {
	result = &v1.VirtualMachine{}
	err = v.restClient.Post().
		Namespace(v.namespace).
		Resource(v.resource).
		Body(vm).
		Do().
		Into(result)
	result.SetGroupVersionKind(v1.VirtualMachineGroupVersionKind)
	return
}

func (v *vms) Update(vm *v1.VirtualMachine) (result *v1.VirtualMachine, err error) {
	result = &v1.VirtualMachine{}
	err = v.restClient.Put().
		Name(vm.ObjectMeta.Name).
		Namespace(v.namespace).
		Resource(v.resource).
		Body(vm).
		Do().
		Into(result)
	result.SetGroupVersionKind(v1.VirtualMachineGroupVersionKind)
	return
}

func (v *vms) Delete(name string, options *k8smetav1.DeleteOptions) error {
	return v.restClient.Delete().
		Namespace(v.namespace).
		Resource(v.resource).
		Name(name).
		Body(options).
		Do().
		Error()
}

func (v *vms) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.VirtualMachine, err error) {
	result = &v1.VirtualMachine{}
	err = v.restClient.Patch(pt).
		Namespace(v.namespace).
		Resource(v.resource).
		SubResource(subresources...).
		Name(name).
		Body(data).
		Do().
		Into(result)
	return
}
<|endoftext|>


<commit_before>package store

import (
	"gob"
	"strconv"
	"strings"
)

var root = node{v:"", ds:make(map[string]node), cas:Dir}

var errPath = "/store/error"

// This structure should be kept immutable.
type node struct {
	v string
	cas string
	ds map[string]node
}

func (n node) readdir() string {
	names := make([]string, len(n.ds))
	i := 0
	for name, _ := range n.ds {
		names[i] = name + "\n"
		i++
	}
	return strings.Join(names, "")
}

func (n node) get(parts []string) (string, string) {
	switch len(parts) {
	case 0:
		if len(n.ds) > 0 {
			return n.readdir(), n.cas
		} else {
			return n.v, n.cas
		}
	default:
		if m, ok := n.ds[parts[0]]; ok {
			return m.get(parts[1:])
		}
		return "", Missing
	}
	panic("can't happen")
}

func (n node) getp(path string) (string, string) {
	if err := checkPath(path); err != nil {
		return "", Missing
	}

	return n.get(split(path))
}

func copyMap(a map[string]node) map[string]node {
	b := make(map[string]node)
	for k,v := range a {
		b[k] = v
	}
	return b
}

// Return value is replacement node
func (n node) set(parts []string, v, cas string, keep bool) (node, bool) {
	if len(parts) == 0 {
		return node{v, cas, n.ds}, keep
	}

	n.ds = copyMap(n.ds)
	p, ok := n.ds[parts[0]].set(parts[1:], v, cas, keep)
	n.ds[parts[0]] = p, ok
	n.cas = Dir
	return n, len(n.ds) > 0
}

func (n node) setp(k, v, cas string, keep bool) node {
	if err := checkPath(k); err != nil {
		return n
	}

	n, _ = n.set(split(k), v, cas, keep)
	return n
}

func (n node) apply(seqn uint64, mut string) (rep node, ev Event) {
	ev.Seqn, ev.Cas, ev.Mut = seqn, strconv.Uitoa64(seqn), mut
	if seqn == 1 {
		d := gob.NewDecoder(strings.NewReader(mut))
		if d.Decode(&ev.Seqn) == nil {
			ev.Cas = ""
			ev.Err = d.Decode(&rep)
			if ev.Err != nil {
				ev.Seqn = seqn + 1
				rep = n
			}
			return
		}
	}

	cas, keep := "", false
	ev.Path, ev.Body, cas, keep, ev.Err = decode(mut)
	if ev.Err != nil {
		ev.Path, ev.Body, cas, keep = errPath, ev.Err.String(), Clobber, true
	}

	if cas != Clobber {
		_, curCas := n.getp(ev.Path)
		if cas != curCas {
			ev.Err = CasMismatchError
			ev.Path, ev.Body, cas, keep = errPath, ev.Err.String(), Clobber, true
		}
	}

	if !keep {
		ev.Cas = Missing
	}

	rep = n.setp(ev.Path, ev.Body, ev.Cas, keep)
	return
}
<commit_msg>more DRY<commit_after>package store

import (
	"gob"
	"strconv"
	"strings"
)

var root = node{v:"", ds:make(map[string]node), cas:Dir}

var errPath = "/store/error"

// This structure should be kept immutable.
type node struct {
	v string
	cas string
	ds map[string]node
}

func (n node) readdir() string {
	names := make([]string, len(n.ds))
	i := 0
	for name, _ := range n.ds {
		names[i] = name + "\n"
		i++
	}
	return strings.Join(names, "")
}

func (n node) get(parts []string) (string, string) {
	switch len(parts) {
	case 0:
		if len(n.ds) > 0 {
			return n.readdir(), n.cas
		} else {
			return n.v, n.cas
		}
	default:
		if m, ok := n.ds[parts[0]]; ok {
			return m.get(parts[1:])
		}
		return "", Missing
	}
	panic("can't happen")
}

func (n node) getp(path string) (string, string) {
	if err := checkPath(path); err != nil {
		return "", Missing
	}

	return n.get(split(path))
}

func copyMap(a map[string]node) map[string]node {
	b := make(map[string]node)
	for k,v := range a {
		b[k] = v
	}
	return b
}

// Return value is replacement node
func (n node) set(parts []string, v, cas string, keep bool) (node, bool) {
	if len(parts) == 0 {
		return node{v, cas, n.ds}, keep
	}

	n.ds = copyMap(n.ds)
	p, ok := n.ds[parts[0]].set(parts[1:], v, cas, keep)
	n.ds[parts[0]] = p, ok
	n.cas = Dir
	return n, len(n.ds) > 0
}

func (n node) setp(k, v, cas string, keep bool) node {
	if err := checkPath(k); err != nil {
		return n
	}

	n, _ = n.set(split(k), v, cas, keep)
	return n
}

func (n node) apply(seqn uint64, mut string) (rep node, ev Event) {
	ev.Seqn, ev.Cas, ev.Mut = seqn, strconv.Uitoa64(seqn), mut
	if seqn == 1 {
		d := gob.NewDecoder(strings.NewReader(mut))
		if d.Decode(&ev.Seqn) == nil {
			ev.Cas = ""
			ev.Err = d.Decode(&rep)
			if ev.Err != nil {
				ev.Seqn = seqn + 1
				rep = n
			}
			return
		}
	}

	cas, keep := "", false
	ev.Path, ev.Body, cas, keep, ev.Err = decode(mut)

	if ev.Err == nil && cas != Clobber {
		_, curCas := n.getp(ev.Path)
		if cas != curCas {
			ev.Err = CasMismatchError
		}
	}

	if ev.Err != nil {
		ev.Path, ev.Body, cas, keep = errPath, ev.Err.String(), Clobber, true
	}

	if !keep {
		ev.Cas = Missing
	}

	rep = n.setp(ev.Path, ev.Body, ev.Cas, keep)
	return
}
<|endoftext|>


<commit_before>package sobjects

import (
	"encoding/json"
	"encoding/xml"
	"io/ioutil"
)

type CampaignMemberSet struct {
	IdSet      IdSet                     `xml:"-"`
	Records    []CampaignMember          `xml:"records"`
	RecordsMap map[string]CampaignMember `xml:"-"`
}

func NewCampaignMemberSet() CampaignMemberSet {
	set := CampaignMemberSet{
		IdSet:      NewIdSet(),
		Records:    []CampaignMember{},
		RecordsMap: map[string]CampaignMember{}}
	return set
}

func NewCampaignMemberSetFromXml(bytes []byte) (CampaignMemberSet, error) {
	set := NewCampaignMemberSet()
	err := xml.Unmarshal(bytes, &set)
	set.Inflate()
	return set, err
}

func NewCampaignMemberSetFromXmlFile(filepath string) (CampaignMemberSet, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return CampaignMemberSet{}, err
	}
	return NewCampaignMemberSetFromXml(bytes)
}

func (set *CampaignMemberSet) Inflate() {
	for _, record := range set.Records {
		if len(record.Id) > 0 {
			set.IdSet.AddId(record.Id)
			set.RecordsMap[record.Id] = record
		}
		if len(record.ContactId) > 0 {
			set.IdSet.AddId(record.ContactId)
		}
		if len(record.LeadId) > 0 {
			set.IdSet.AddId(record.LeadId)
		}
	}
}

type CampaignMember struct {
	Id                 string
	CampaignId         string
	ContactId          string
	CreatedDate        string
	CurrencyIsoCode    string
	FirstRespondedDate string
	HasResponded       bool
	LeadId             string
	RecordTypeId       string
	Name               string
	Status             string
}

func NewCampaignMemberFromJson(bytes []byte) (CampaignMember, error) {
	obj := CampaignMember{}
	err := json.Unmarshal(bytes, &obj)
	return obj, err
}

func NewCampaignMemberFromJsonFile(filepath string) (CampaignMember, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return CampaignMember{}, err
	}
	return NewCampaignMemberFromJson(bytes)
}
<commit_msg>update campaign-member<commit_after>package sobjects

import (
	"encoding/json"
	"encoding/xml"
	"io/ioutil"
)

type CampaignMemberSet struct {
	IdSet      IdSet                     `xml:"-"`
	Records    []CampaignMember          `xml:"records"`
	RecordsMap map[string]CampaignMember `xml:"-"`
}

func NewCampaignMemberSet() CampaignMemberSet {
	set := CampaignMemberSet{
		IdSet:      NewIdSet(),
		Records:    []CampaignMember{},
		RecordsMap: map[string]CampaignMember{}}
	return set
}

func NewCampaignMemberSetFromXml(bytes []byte) (CampaignMemberSet, error) {
	set := NewCampaignMemberSet()
	err := xml.Unmarshal(bytes, &set)
	set.Inflate()
	return set, err
}

func NewCampaignMemberSetFromXmlFile(filepath string) (CampaignMemberSet, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return CampaignMemberSet{}, err
	}
	return NewCampaignMemberSetFromXml(bytes)
}

func (set *CampaignMemberSet) Inflate() {
	for _, record := range set.Records {
		if len(record.Id) > 0 {
			set.IdSet.AddId(record.Id)
			set.RecordsMap[record.Id] = record
		}
		if len(record.ContactId) > 0 {
			set.IdSet.AddId(record.ContactId)
		}
		if len(record.LeadId) > 0 {
			set.IdSet.AddId(record.LeadId)
		}
	}
}

type CampaignMember struct {
	Id                 string
	CampaignId         string
	ContactId          string
	CreatedDate        string
	CurrencyIsoCode    string
	FirstRespondedDate string
	HasResponded       bool
	LeadId             string
	Name               string
	Status             string
}

func NewCampaignMemberFromJson(bytes []byte) (CampaignMember, error) {
	obj := CampaignMember{}
	err := json.Unmarshal(bytes, &obj)
	return obj, err
}

func NewCampaignMemberFromJsonFile(filepath string) (CampaignMember, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return CampaignMember{}, err
	}
	return NewCampaignMemberFromJson(bytes)
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "sort"
import "math/rand"
import "time"

func main() {
    // Seed
    rand.Seed( time.Now().UTC().UnixNano())

    var slice []int
    var n = 10000000

    fmt.Printf("Start creating %v random integers...\n", n)
    for i := 0; i < n; i++ {
        slice = append(slice, rand.Intn(1000000000))
    }

    fmt.Printf("Start sorting %v random integers...\n", n)

    start := time.Now()
    sort.Sort(sort.IntSlice(slice))
    elapsed := time.Since(start)

    fmt.Printf("%s\n", elapsed)
}
<commit_msg>sorting/integer-array: Updated Go example<commit_after>package main

import (
	"fmt"
	"math/rand"
	"sort"
	"time"
)

func main() {
	// Seed
	rand.Seed(time.Now().UTC().UnixNano())

	var slice []int
	var n = 10000000

	fmt.Printf("Start creating %v random integers...\n", n)
	for i := 0; i < n; i++ {
		slice = append(slice, rand.Intn(1000000000))
	}

	fmt.Printf("Start sorting %v random integers...\n", n)

	start := time.Now()
	sort.Sort(sort.IntSlice(slice))
	elapsed := time.Since(start)

	fmt.Printf("%s\n", elapsed)
}
<|endoftext|>


<commit_before>/*

*/
package mandrake

import (
	"log"
	"net/rpc/jsonrpc"
	"github.com/natefinch/pie"
)

type Mandrake struct {
	consumers		chan string
}

func NewMandrake() (*Mandrake, error) {
	return *Mandrake{make(chan string)}, nil
}

func (m *Mandrake) ListenAndServe() {
	p := pie.NewProvider()
	if err := p.RegisterName("mandrake", m); err != nil {
		log.Fatalf("Failed to register plugin: %s", err)
	}

	go p.ServeCodec(jsonrpc.NewServerCodec)
}

func (m *Mandrake) Analyze(fname string, response *string) error {
	log.Printf("Beginning analysis: %s", fname)

	m.consumers <- fname

	return nil
}<commit_msg>fix typo<commit_after>/*

*/
package mandrake

import (
	"log"
	"net/rpc/jsonrpc"
	"github.com/natefinch/pie"
)

type Mandrake struct {
	consumers		chan string
}

func NewMandrake() (*Mandrake, error) {
	return &Mandrake{make(chan string)}, nil
}

func (m *Mandrake) ListenAndServe() {
	p := pie.NewProvider()
	if err := p.RegisterName("mandrake", m); err != nil {
		log.Fatalf("Failed to register plugin: %s", err)
	}

	go p.ServeCodec(jsonrpc.NewServerCodec)
}

func (m *Mandrake) Analyze(fname string, response *string) error {
	log.Printf("Beginning analysis: %s", fname)

	m.consumers <- fname

	return nil
}<|endoftext|>


<commit_before>package mediafile

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"time"
)

// HashString takes a string and returns the MD5 of that string
func HashString(s string) string {
	h := md5.New()
	io.WriteString(h, s)
	return fmt.Sprintf("%x", h.Sum(nil))
}

// MonitorDirectory takes a local directory and a iterates through it
// placing newly found files into a channel
//
// It is not meant to be returned from
func MonitorDirectory(directory string, additions chan (*File)) {
	if err := os.Chdir(directory); err != nil {
		log.Fatalln(err)
	}
	fileList := make(map[string]*File)
	for {
		files, err := ioutil.ReadDir(directory)
		if err != nil {
			log.Fatalln("Error reading")
		}
		for _, file := range files {
			sha := HashString(file.Name())
			if _, ok := fileList[sha]; ok {
				continue
			}
			fileList[sha] = &File{
				file,
				filepath.Join(directory, file.Name()),
				sha,
			}
			log.Println("Adding:", file.Name())
			additions <- fileList[sha]
		}
		time.Sleep(time.Second * 60)
	}
}
<commit_msg>Simplify checksum computation<commit_after>package mediafile

import (
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"time"
)

// HashString takes a string and returns the MD5 of that string
func HashString(s string) string {
	checksum := md5.Sum([]byte(s))
	return fmt.Sprintf("%x", checksum)
}

// MonitorDirectory takes a local directory and a iterates through it
// placing newly found files into a channel
//
// It is not meant to be returned from
func MonitorDirectory(directory string, additions chan (*File)) {
	if err := os.Chdir(directory); err != nil {
		log.Fatalln(err)
	}
	fileList := make(map[string]*File)
	for {
		files, err := ioutil.ReadDir(directory)
		if err != nil {
			log.Fatalln("Error reading")
		}
		for _, file := range files {
			sha := HashString(file.Name())
			if _, ok := fileList[sha]; ok {
				continue
			}
			fileList[sha] = &File{
				file,
				filepath.Join(directory, file.Name()),
				sha,
			}
			log.Println("Adding:", file.Name())
			additions <- fileList[sha]
		}
		time.Sleep(time.Second * 60)
	}
}
<|endoftext|>