<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

//execute commands only from this directory
var cmdDir = flag.String("c", "./cmds", "Commands dir")

// return a list of avaiable jobs that can be run
func jobEntries() ([]string, error) {
	var entries []string
	dir, err := os.Open(*cmdDir)
	if err != nil {
		return entries, err
	}
	defer dir.Close()

	dirInfoSlice, _ := dir.Readdir(-1)
	for _, fileinfo := range dirInfoSlice {
		entries = append(entries, fileinfo.Name())
	}
	return entries, nil
}

// validates the command to be run and returns it's execution path
func validateCmd(cmd string) (string, error) {
	jobs, err := jobEntries()
	if err != nil {
		return "", err
	}
	for _, job := range jobs {
		if cmd == job {
			return filepath.Join(*cmdDir, cmd), nil
		}
	}
	return "", errors.New("Command not found")
}

/* Run the command and send back the results on channels */
func runCommand(command string, outChan chan string, errChan chan error) {
	// no matter what happens, close the channel
	defer close(errChan)

	cmdPath, err := validateCmd(command)
	if err != nil {
		errChan <- err
		log.Print("Invalid command: ", err)
		return
	}

	cmd := exec.Command(cmdPath)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		errChan <- err
		return
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		errChan <- err
		return
	}

	// run the command, but don't block
	if err := cmd.Start(); err != nil {
		errChan <- err
		return
	}

	stdoutBuf := make([]byte, 1024)
	stderrBuf := make([]byte, 1024)
	// read from the stdout to the buffer

	_, err = stdout.Read(stdoutBuf)
	_, err = stderr.Read(stderrBuf)
	// while we have stuff to read from the output
	for err == nil {
		// send the output of  the command to the channel
		outChan <- string(stdoutBuf)
		outChan <- string(stderrBuf)
		// read some more
		_, err = stdout.Read(stdoutBuf)
		_, err = stderr.Read(stderrBuf)
	}
	// nothing more to send.. we can close the channel here
	close(outChan)

	// report any errors including the exit code of the command
	err = cmd.Wait()
	errChan <- err
}

/* /run/ - this handler will send the output of a running command */
func runHandler(response http.ResponseWriter, request *http.Request) {
	// forcing the output content type
	header := response.Header()

	// for some reason if text/plain is passed
	// Chrome thinks it's a application/octet-stream and tries
	// to download the log.
	header["Content-Type"] = []string{"text/html; charset=UTF-8"}
	header["Connection"] = []string{"close"}
	header["Vary"] = []string{"User-Agent"}

	jobName := request.URL.Path[len("/run/"):]
	// TODO: fix this and use the HTTP headers with user authentication
	userName := "Anonymous"
	startTime := time.Now().UTC()

	logEntry := JobLogEntry{
		Name: jobName,
		User: userName,
		Time: startTime.Unix(),
	}

	firstLine := "Started at " + startTime.Format("Mon Jan 2 15:04:05 -0700 MST 2006") + " by " + userName
	firstLine = firstLine + "\n==========================\n\n"
	// Adding a
 here because we want pretty output
	// the browser. We close it at end when we finished reading
	// from the command's output
	fmt.Fprintf(response, "
"+firstLine)
	logFilePath, err := NewLogEntry(logEntry, firstLine)
	if err != nil {
		log.Print("Failed to create a new log entry: ", err)
	}

	log.Print("Started job: " + jobName)
	// We'll use two channels. The error and the command output channel
	// the error channel is used to get the errors thrown while running
	// the job and the output channel is for returning the output of
	// the command
	outChan := make(chan string)
	errChan := make(chan error)
	go runCommand(jobName, outChan, errChan)

	// TODO: perhaps implement some kind of timeout?
	for {
		select {
		case content, closed := <-outChan:
			AppendLog(logFilePath, content)
			fmt.Fprintf(response, content)
			response.(http.Flusher).Flush()

			if !closed {
				secondsSince := time.Since(startTime).Seconds()
				logFilePath = RenameLogFile(logFilePath, secondsSince, "0")
				fmt.Fprintf(response, "
") // close

				log.Print("Finished job: " + jobName)
			}
		case err, _ := <-errChan:
			log.Print("Received an error: ", err)
			errStr := ""
			if err != nil {
				errStr = err.Error()
				// TODO: maybe there is another way to get the
				// exit status?
				if strings.Contains(errStr, "exit status") {
					// Ex: exit status 0
					msgparts := strings.Split(err.Error(), " ")
					// Ex: we get "0" here
					status := msgparts[2]
					// Rename the log file to store
					// the exit status
					logFilePath = RenameLogFile(logFilePath, time.Since(startTime).Seconds(), status)
				}
			}

			AppendLog(logFilePath, errStr)

			fmt.Fprintf(response, errStr)
			response.(http.Flusher).Flush()
			return
		}
	}
}

// update the log filename with the new duration and
func RenameLogFile(filepath string, duration float64, status string) string {
	parts := strings.Split(filepath, "-")
	durationStr := strconv.FormatFloat(duration, 'f', 0, 64)

	parts[4] = durationStr
	parts[5] = status
	newLogPath := strings.Join(parts, "-")
	err := os.Rename(filepath, newLogPath)
	if err != nil {
		log.Print("Failed to rename log file ", err)
	}
	return newLogPath
}

/* /logs will return the latest logs ordered by date from the logs folder */
func logsHandler(w http.ResponseWriter, r *http.Request) {
	// forcing the output content type
	header := w.Header()
	header["Content-Type"] = []string{"application/json"}
	// if we have a name of the log then we should get the contents of the log
	var dataJson []byte
	var err error
	if r.FormValue("name") != "" {
		data := make(map[string]string, 1)
		body, err := LogEntryBody(r.FormValue("name"))
		if err != nil {
			log.Print("Failed to get log entry body ", err)
			fmt.Fprintf(w, err.Error())
		}

		data["body"] = string(body)
		dataJson, err = json.Marshal(data)
		if err != nil {
			log.Print("Failed to encode json: ", err)
		}
	} else {
		job := r.FormValue("job")
		data := LogEntries(job)
		dataJson, err = json.Marshal(data)
		if err != nil {
			log.Print("Failed to encode json: ", err)
		}
	}
	fmt.Fprintf(w, string(dataJson))
}

/* /listJobs will display the available jobs that can be run */
func jobsHandler(w http.ResponseWriter, r *http.Request) {
	// forcing the output content type
	header := w.Header()
	header["Content-Type"] = []string{"application/json"}

	entries, err := jobEntries()
	if err != nil {
		log.Print("Error loading available jobs: ", err)
	}
	dataJson, err := json.Marshal(entries)
	if err != nil {
		log.Print("Failed to encode json")
	}
	fmt.Fprintf(w, string(dataJson))
}

/* This is a helper wrapper. Allows us to log some stuff */
func DefaultWrapper(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
		handler.ServeHTTP(w, r)
	})
}

func main() {
	http.HandleFunc("/run/", runHandler)

	http.HandleFunc("/logs", logsHandler)
	http.HandleFunc("/jobs", jobsHandler)

	// serve other static stuff
	http.Handle("/", http.StripPrefix("/",
		http.FileServer(http.Dir("./static"))))

	// command running handler
	// server index.html at the end
	port := ":8000"
	log.Printf("Starting on " + port)
	err := http.ListenAndServe(port, DefaultWrapper(http.DefaultServeMux))
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Added command lock.<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

// execute commands only from this directory
var cmdDir = flag.String("c", "./cmds", "Commands dir")
// this lock is used to not allow 2 commands to run at once
var commandLock *sync.Mutex

// return a list of avaiable jobs that can be run
func jobEntries() ([]string, error) {
	var entries []string
	dir, err := os.Open(*cmdDir)
	if err != nil {
		return entries, err
	}
	defer dir.Close()

	dirInfoSlice, _ := dir.Readdir(-1)
	for _, fileinfo := range dirInfoSlice {
		entries = append(entries, fileinfo.Name())
	}
	return entries, nil
}

// validates the command to be run and returns it's execution path
func validateCmd(cmd string) (string, error) {
	jobs, err := jobEntries()
	if err != nil {
		return "", err
	}
	for _, job := range jobs {
		if cmd == job {
			return filepath.Join(*cmdDir, cmd), nil
		}
	}
	return "", errors.New("Command not found")
}

/* Run the command and send back the results on channels */
func runCommand(command string, outChan chan string, errChan chan error) {
	// no matter what happens, close the channel
	defer close(errChan)

	cmdPath, err := validateCmd(command)
	if err != nil {
		errChan <- err
		log.Print("Invalid command: ", err)
		return
	}

	cmd := exec.Command(cmdPath)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		errChan <- err
		return
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		errChan <- err
		return
	}

	// run the command, but don't block
	if err := cmd.Start(); err != nil {
		errChan <- err
		return
	}

	stdoutBuf := make([]byte, 1024)
	stderrBuf := make([]byte, 1024)
	// read from the stdout to the buffer

	_, errStdout := stdout.Read(stdoutBuf)
	_, errStderr := stderr.Read(stderrBuf)
	// while we have stuff to read from the output
	for errStdout == nil || errStderr == nil {
		// send the output of  the command to the channel
		if errStdout == nil {
			log.Print("sending stdout")
			outChan <- string(stdoutBuf)
		}
		if errStderr == nil {
			log.Print("sending stderr")
			outChan <- string(stderrBuf)
		}

		// read some more
		_, errStdout = stdout.Read(stdoutBuf)
		_, errStderr = stderr.Read(stderrBuf)
	}
	// nothing more to send.. we can close the channel here
	close(outChan)

	// report any errors including the exit code of the command
	err = cmd.Wait()
	errChan <- err
}

/* /run/ - this handler will send the output of a running command */
func runHandler(response http.ResponseWriter, request *http.Request) {
	// forcing the output content type
	header := response.Header()

	// for some reason if text/plain is passed
	// Chrome thinks it's a application/octet-stream and tries
	// to download the log.
	header["Content-Type"] = []string{"text/html; charset=UTF-8"}
	header["Connection"] = []string{"close"}
	header["Vary"] = []string{"User-Agent"}

	jobName := request.URL.Path[len("/run/"):]
	// TODO: fix this and use the HTTP headers with user authentication
	userName := "Anonymous"
	startTime := time.Now().UTC()

	logEntry := JobLogEntry{
		Name: jobName,
		User: userName,
		Time: startTime.Unix(),
	}

	firstLine := "Started at " + startTime.Format("Mon Jan 2 15:04:05 -0700 MST 2006") + " by " + userName
	firstLine = firstLine + "\n==========================\n\n"
	// Adding a
 here because we want pretty output
	// the browser. We close it at end when we finished reading
	// from the command's output
	fmt.Fprintf(response, "
"+firstLine)
	response.(http.Flusher).Flush()

	logFilePath, err := NewLogEntry(logEntry, firstLine)
	if err != nil {
		log.Print("Failed to create a new log entry: ", err)
	}
	// aquiring the lock. This will block the execution
	commandLock.Lock()
	defer commandLock.Unlock()

	log.Print("Started job: " + jobName)
	// We'll use two channels. The error and the command output channel
	// the error channel is used to get the errors thrown while running
	// the job and the output channel is for returning the output of
	// the command
	outChan := make(chan string)
	errChan := make(chan error)
	go runCommand(jobName, outChan, errChan)

	// TODO: perhaps implement some kind of timeout?
	for {
		select {
		case content, closed := <-outChan:
			AppendLog(logFilePath, content)
			fmt.Fprintf(response, content)
			response.(http.Flusher).Flush()

			if !closed {
				secondsSince := time.Since(startTime).Seconds()
				logFilePath = RenameLogFile(logFilePath, secondsSince, "0")
				fmt.Fprintf(response, "
") // close

				log.Print("Finished job: " + jobName)
			}
		case err, _ := <-errChan:
			log.Print("Received an error: ", err)
			errStr := ""
			if err != nil {
				errStr = err.Error()
				// TODO: maybe there is another way to get the
				// exit status?
				if strings.Contains(errStr, "exit status") {
					// Ex: exit status 0
					msgparts := strings.Split(err.Error(), " ")
					// Ex: we get "0" here
					status := msgparts[2]
					// Rename the log file to store
					// the exit status
					logFilePath = RenameLogFile(logFilePath, time.Since(startTime).Seconds(), status)
				}
			}

			AppendLog(logFilePath, errStr)

			fmt.Fprintf(response, errStr)
			response.(http.Flusher).Flush()
			return
		}
	}
}

// update the log filename with the new duration and
func RenameLogFile(filepath string, duration float64, status string) string {
	parts := strings.Split(filepath, "-")
	durationStr := strconv.FormatFloat(duration, 'f', 0, 64)

	parts[4] = durationStr
	parts[5] = status
	newLogPath := strings.Join(parts, "-")
	err := os.Rename(filepath, newLogPath)
	if err != nil {
		log.Print("Failed to rename log file ", err)
	}
	return newLogPath
}

/* /logs will return the latest logs ordered by date from the logs folder */
func logsHandler(w http.ResponseWriter, r *http.Request) {
	// forcing the output content type
	header := w.Header()
	header["Content-Type"] = []string{"application/json"}
	// if we have a name of the log then we should get the contents of the log
	var dataJson []byte
	var err error
	if r.FormValue("name") != "" {
		data := make(map[string]string, 1)
		body, err := LogEntryBody(r.FormValue("name"))
		if err != nil {
			log.Print("Failed to get log entry body ", err)
			fmt.Fprintf(w, err.Error())
		}

		data["body"] = string(body)
		dataJson, err = json.Marshal(data)
		if err != nil {
			log.Print("Failed to encode json: ", err)
		}
	} else {
		job := r.FormValue("job")
		data := LogEntries(job)
		dataJson, err = json.Marshal(data)
		if err != nil {
			log.Print("Failed to encode json: ", err)
		}
	}
	fmt.Fprintf(w, string(dataJson))
}

/* /listJobs will display the available jobs that can be run */
func jobsHandler(w http.ResponseWriter, r *http.Request) {
	// forcing the output content type
	header := w.Header()
	header["Content-Type"] = []string{"application/json"}

	entries, err := jobEntries()
	if err != nil {
		log.Print("Error loading available jobs: ", err)
	}
	dataJson, err := json.Marshal(entries)
	if err != nil {
		log.Print("Failed to encode json")
	}
	fmt.Fprintf(w, string(dataJson))
}

/* This is a helper wrapper. Allows us to log some stuff */
func DefaultWrapper(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
		handler.ServeHTTP(w, r)
	})
}

func main() {
	// create the lock
	commandLock = new(sync.Mutex)
	http.HandleFunc("/run/", runHandler)

	http.HandleFunc("/logs", logsHandler)
	http.HandleFunc("/jobs", jobsHandler)

	// serve other static stuff
	http.Handle("/", http.StripPrefix("/",
		http.FileServer(http.Dir("./static"))))

	// command running handler
	// server index.html at the end
	port := ":8000"
	log.Printf("Starting on " + port)
	err := http.ListenAndServe(port, DefaultWrapper(http.DefaultServeMux))
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

// execute commands only from this directory
var CmdDir = flag.String("dir", "./cmds", "Commands dir")
var ExcludePat = flag.String("exclude", "", "Exclude shell file name pattern. Ex: *.pyc,a.out")

// this lock is used to not allow 2 commands to run at once
var commandLock *sync.Mutex

// return a list of avaiable jobs that can be run
func jobEntries() ([]string, error) {
	var entries []string
	dir, err := os.Open(*CmdDir)
	if err != nil {
		return entries, err
	}
	defer dir.Close()

	dirInfoSlice, _ := dir.Readdir(-1)
	for _, fileinfo := range dirInfoSlice {
		fileName := fileinfo.Name()
		excludes := strings.Split(*ExcludePat, ",")
		matched := false
		for _, exclude := range excludes {
			matched, _ = filepath.Match(exclude, fileName)
			if matched {
				break
			}
		}
		if !matched {
			entries = append(entries, fileName)
		}
	}
	return entries, nil
}

// validates the command to be run and returns it's execution path
func validateCmd(cmd string) (string, error) {
	jobs, err := jobEntries()
	if err != nil {
		return "", err
	}
	for _, job := range jobs {
		if cmd == job {
			return filepath.Join(*CmdDir, cmd), nil
		}
	}
	return "", errors.New("Command not found")
}

// this is a combined output channel used for both stdout and stderr pipes
type combinedOutput struct {
	data []byte
	exit bool // use this to signal that we ended reading from the pipe
}

// read from pipe (stdout/stderr) and send back using channel
func readPipe(pipe io.ReadCloser, pipeChan chan combinedOutput) {
	buf := make([]byte, 1024)
	_, err := pipe.Read(buf)

	var out combinedOutput
	for err == nil {
		// send the output of  the command to the channel
		out.data = buf
		out.exit = false
		pipeChan <- out

		// read some more
		_, err = pipe.Read(buf)
	}
	out.data = nil
	out.exit = true
	pipeChan <- out
}

/* Run the command and send back the results on channels */
func runCommand(command string, outChan chan string, errChan chan error) {
	// no matter what happens, close the channel
	defer close(errChan)

	cmdPath, err := validateCmd(command)
	if err != nil {
		errChan <- err
		log.Print("Invalid command: ", err)
		return
	}

	cmd := exec.Command(cmdPath)
	cmd.Dir = *CmdDir
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		errChan <- err
		return
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		errChan <- err
		return
	}

	// run the command, but don't block
	if err := cmd.Start(); err != nil {
		errChan <- err
		return
	}

	// combine the out from both pipes
	comChan := make(chan combinedOutput)
	go readPipe(stdout, comChan)
	go readPipe(stderr, comChan)

	count := 0
	for out := range comChan {
		if out.exit == true {
			if count == 1 {
				//close(comChan)
				break
			}
			count++
		}
		outChan <- string(out.data)
	}
	// nothing more to send.. we can close the channel here
	close(outChan)

	// report any errors including the exit code of the command
	err = cmd.Wait()
	errChan <- err
}

/* /run/ - this handler will send the output of a running command */
func runHandler(response http.ResponseWriter, request *http.Request) {
	// forcing the output content type
	header := response.Header()

	// for some reason if text/plain is passed
	// Chrome thinks it's a application/octet-stream and tries
	// to download the log.
	header["Content-Type"] = []string{"text/html; charset=UTF-8"}
	header["Connection"] = []string{"close"}
	header["Vary"] = []string{"User-Agent"}

	// aquiring the lock. Should block here
	commandLock.Lock()
	defer commandLock.Unlock()

	jobName := request.URL.Path[len("/run/"):]
	// TODO: fix this and use the HTTP headers with user authentication
	userName := "Anonymous"

	// start counting
	startTime := time.Now().UTC()

	logEntry := JobLogEntry{
		Name: jobName,
		User: userName,
		Time: startTime.Unix(),
	}

	firstLine := "Started at " + startTime.Format("Mon Jan 2 15:04:05 -0700 MST 2006") + " by " + userName
	firstLine = firstLine + "\n==========================\n\n"
	// Adding a
 here because we want pretty output
	// the browser. We close it at end when we finished reading
	// from the command's output
	fmt.Fprintf(response, "
"+firstLine)
	response.(http.Flusher).Flush()

	logFilePath, err := NewLogEntry(logEntry, firstLine)
	if err != nil {
		log.Print("Failed to create a new log entry: ", err)
	}

	log.Print("Started job: " + jobName)
	// We'll use two channels. The error and the command output channel
	// the error channel is used to get the errors thrown while running
	// the job and the output channel is for returning the output of
	// the command
	outChan := make(chan string)
	errChan := make(chan error)
	go runCommand(jobName, outChan, errChan)

	// TODO: perhaps implement some kind of timeout?
	for {
		select {
		case content, closed := <-outChan:
			AppendLog(logFilePath, content)
			fmt.Fprintf(response, content)

			if !closed {
				secondsSince := time.Since(startTime).Seconds()
				logFilePath = RenameLogFile(logFilePath, secondsSince, "0")
				fmt.Fprintf(response, "
") // close

				log.Print("Finished job: " + jobName)
				return
			}
			response.(http.Flusher).Flush()
		case err, _ := <-errChan:
			errStr := ""
			if err != nil {
				errStr = err.Error()
				// TODO: maybe there is another way to get the
				// exit status?
				if strings.Contains(errStr, "exit status") {
					// Ex: exit status 0
					msgparts := strings.Split(err.Error(), " ")
					// Ex: we get "0" here
					status := msgparts[2]
					// Rename the log file to store
					// the exit status
					secondsSince := time.Since(startTime).Seconds()
					logFilePath = RenameLogFile(logFilePath,
						secondsSince, status)

					log.Print("Finished job: " + jobName)
					return
				}
			}

			AppendLog(logFilePath, errStr)

			fmt.Fprintf(response, errStr)
			response.(http.Flusher).Flush()
		}
	}

}

// update the log filename with the new duration and
func RenameLogFile(filepath string, duration float64, status string) string {
	parts := strings.Split(filepath, "-")
	durationStr := strconv.FormatFloat(duration, 'f', 0, 64)

	parts[4] = durationStr
	parts[5] = status
	newLogPath := strings.Join(parts, "-")
	err := os.Rename(filepath, newLogPath)
	if err != nil {
		log.Print("Failed to rename log file ", err)
	}
	return newLogPath
}

// This will be useful for some pagination
type ResponseLogs struct {
	Entries JobLogEntries
	Length  int
}

/* /logs will return the latest logs ordered by date from the logs folder */
func logsHandler(w http.ResponseWriter, r *http.Request) {
	// forcing the output content type
	header := w.Header()
	header["Content-Type"] = []string{"application/json"}
	// if we have a name of the log then we should get the contents of the log
	var dataJson []byte
	if r.FormValue("name") != "" {
		data := make(map[string]string, 1)
		body, err := LogEntryBody(r.FormValue("name"))
		if err != nil {
			log.Print("Failed to get log entry body ", err)
			fmt.Fprintf(w, err.Error())
		}

		data["body"] = string(body)
		dataJson, err = json.Marshal(data)
		if err != nil {
			log.Print("Failed to encode json: ", err)
		}
	} else {
		start := 0
		offset := 50              // number of items per page
		job := r.FormValue("job") // filter by job is any
		logEntries := LogEntries(job)
		logEntriesLen := len(logEntries)
		if offset >= logEntriesLen {
			offset = logEntriesLen
		}

		resp := ResponseLogs{
			Entries: logEntries[start:offset],
			Length:  logEntriesLen,
		}

		page, err := strconv.ParseInt(r.FormValue("page"), 10, 32)
		if err == nil {
			start = int(page) * offset
			if start >= resp.Length {
				start = resp.Length - offset
			}
			offset += start
			if offset >= resp.Length {
				offset = resp.Length
			}
			resp.Entries = resp.Entries[start:offset]
		}

		dataJson, err = json.Marshal(resp)
		if err != nil {
			log.Print("Failed to encode json: ", err)
		}
	}
	fmt.Fprintf(w, string(dataJson))
}

/* /listJobs will display the available jobs that can be run */
func jobsHandler(w http.ResponseWriter, r *http.Request) {
	// forcing the output content type
	header := w.Header()
	header["Content-Type"] = []string{"application/json"}

	entries, err := jobEntries()
	if err != nil {
		log.Print("Error loading available jobs: ", err)
	}
	dataJson, err := json.Marshal(entries)
	if err != nil {
		log.Print("Failed to encode json")
	}
	fmt.Fprintf(w, string(dataJson))
}

/* This is a helper wrapper. Allows us to log some stuff */
func DefaultWrapper(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
		handler.ServeHTTP(w, r)
	})
}

func main() {
	flag.Parse()

	// create the lock
	commandLock = new(sync.Mutex)
	http.HandleFunc("/run/", runHandler)

	http.HandleFunc("/logs", logsHandler)
	http.HandleFunc("/jobs", jobsHandler)

	// serve other static stuff
	http.Handle("/", http.StripPrefix("/",
		http.FileServer(http.Dir("./static"))))

	// command running handler
	// server index.html at the end
	port := ":8000"
	log.Printf("Starting on " + port)
	err := http.ListenAndServe(port, DefaultWrapper(http.DefaultServeMux))
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Formatting<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

// execute commands only from this directory
var CmdDir = flag.String("dir", "./cmds", "Commands dir")
var ExcludePat = flag.String("exclude", "",
	"Exclude shell file name pattern. Ex: *.pyc,a.out")

// this lock is used to not allow 2 commands to run at once
var commandLock *sync.Mutex

// return a list of avaiable jobs that can be run
func jobEntries() ([]string, error) {
	var entries []string
	dir, err := os.Open(*CmdDir)
	if err != nil {
		return entries, err
	}
	defer dir.Close()

	dirInfoSlice, _ := dir.Readdir(-1)
	for _, fileinfo := range dirInfoSlice {
		fileName := fileinfo.Name()
		excludes := strings.Split(*ExcludePat, ",")
		matched := false
		for _, exclude := range excludes {
			matched, _ = filepath.Match(exclude, fileName)
			if matched {
				break
			}
		}
		if !matched {
			entries = append(entries, fileName)
		}
	}
	return entries, nil
}

// validates the command to be run and returns it's execution path
func validateCmd(cmd string) (string, error) {
	jobs, err := jobEntries()
	if err != nil {
		return "", err
	}
	for _, job := range jobs {
		if cmd == job {
			return filepath.Join(*CmdDir, cmd), nil
		}
	}
	return "", errors.New("Command not found")
}

// this is a combined output channel used for both stdout and stderr pipes
type combinedOutput struct {
	data []byte
	exit bool // use this to signal that we ended reading from the pipe
}

// read from pipe (stdout/stderr) and send back using channel
func readPipe(pipe io.ReadCloser, pipeChan chan combinedOutput) {
	buf := make([]byte, 1024)
	_, err := pipe.Read(buf)

	var out combinedOutput
	for err == nil {
		// send the output of  the command to the channel
		out.data = buf
		out.exit = false
		pipeChan <- out

		// read some more
		_, err = pipe.Read(buf)
	}
	out.data = nil
	out.exit = true
	pipeChan <- out
}

/* Run the command and send back the results on channels */
func runCommand(command string, outChan chan string, errChan chan error) {
	// no matter what happens, close the channel
	defer close(errChan)

	cmdPath, err := validateCmd(command)
	if err != nil {
		errChan <- err
		log.Print("Invalid command: ", err)
		return
	}

	cmd := exec.Command(cmdPath)
	cmd.Dir = *CmdDir
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		errChan <- err
		return
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		errChan <- err
		return
	}

	// run the command, but don't block
	if err := cmd.Start(); err != nil {
		errChan <- err
		return
	}

	// combine the out from both pipes
	comChan := make(chan combinedOutput)
	go readPipe(stdout, comChan)
	go readPipe(stderr, comChan)

	count := 0
	for out := range comChan {
		if out.exit == true {
			if count == 1 {
				//close(comChan)
				break
			}
			count++
		}
		outChan <- string(out.data)
	}
	// nothing more to send.. we can close the channel here
	close(outChan)

	// report any errors including the exit code of the command
	err = cmd.Wait()
	errChan <- err
}

/* /run/ - this handler will send the output of a running command */
func runHandler(response http.ResponseWriter, request *http.Request) {
	// forcing the output content type
	header := response.Header()

	// for some reason if text/plain is passed
	// Chrome thinks it's a application/octet-stream and tries
	// to download the log.
	header["Content-Type"] = []string{"text/html; charset=UTF-8"}
	header["Connection"] = []string{"close"}
	header["Vary"] = []string{"User-Agent"}

	// aquiring the lock. Should block here
	commandLock.Lock()
	defer commandLock.Unlock()

	jobName := request.URL.Path[len("/run/"):]
	// TODO: fix this and use the HTTP headers with user authentication
	userName := "Anonymous"

	// start counting
	startTime := time.Now().UTC()

	logEntry := JobLogEntry{
		Name: jobName,
		User: userName,
		Time: startTime.Unix(),
	}

	firstLine := "Started at " + startTime.Format("Mon Jan 2 15:04:05 -0700 MST 2006") + " by " + userName
	firstLine = firstLine + "\n==========================\n\n"
	// Adding a
 here because we want pretty output
	// the browser. We close it at end when we finished reading
	// from the command's output
	fmt.Fprintf(response, "
"+firstLine)
	response.(http.Flusher).Flush()

	logFilePath, err := NewLogEntry(logEntry, firstLine)
	if err != nil {
		log.Print("Failed to create a new log entry: ", err)
	}

	log.Print("Started job: " + jobName)
	// We'll use two channels. The error and the command output channel
	// the error channel is used to get the errors thrown while running
	// the job and the output channel is for returning the output of
	// the command
	outChan := make(chan string)
	errChan := make(chan error)
	go runCommand(jobName, outChan, errChan)

	// TODO: perhaps implement some kind of timeout?
	for {
		select {
		case content, closed := <-outChan:
			AppendLog(logFilePath, content)
			fmt.Fprintf(response, content)

			if !closed {
				secondsSince := time.Since(startTime).Seconds()
				logFilePath = RenameLogFile(logFilePath, secondsSince, "0")
				fmt.Fprintf(response, "
") // close

				log.Print("Finished job: " + jobName)
				return
			}
			response.(http.Flusher).Flush()
		case err, _ := <-errChan:
			errStr := ""
			if err != nil {
				errStr = err.Error()
				// TODO: maybe there is another way to get the
				// exit status?
				if strings.Contains(errStr, "exit status") {
					// Ex: exit status 0
					msgparts := strings.Split(err.Error(), " ")
					// Ex: we get "0" here
					status := msgparts[2]
					// Rename the log file to store
					// the exit status
					secondsSince := time.Since(startTime).Seconds()
					logFilePath = RenameLogFile(logFilePath,
						secondsSince, status)

					log.Print("Finished job: " + jobName)
					return
				}
			}

			AppendLog(logFilePath, errStr)

			fmt.Fprintf(response, errStr)
			response.(http.Flusher).Flush()
		}
	}

}

// update the log filename with the new duration and
func RenameLogFile(filepath string, duration float64, status string) string {
	parts := strings.Split(filepath, "-")
	durationStr := strconv.FormatFloat(duration, 'f', 0, 64)

	parts[4] = durationStr
	parts[5] = status
	newLogPath := strings.Join(parts, "-")
	err := os.Rename(filepath, newLogPath)
	if err != nil {
		log.Print("Failed to rename log file ", err)
	}
	return newLogPath
}

// This will be useful for some pagination
type ResponseLogs struct {
	Entries JobLogEntries
	Length  int
}

/* /logs will return the latest logs ordered by date from the logs folder */
func logsHandler(w http.ResponseWriter, r *http.Request) {
	// forcing the output content type
	header := w.Header()
	header["Content-Type"] = []string{"application/json"}
	// if we have a name of the log then we should get the contents of the log
	var dataJson []byte
	if r.FormValue("name") != "" {
		data := make(map[string]string, 1)
		body, err := LogEntryBody(r.FormValue("name"))
		if err != nil {
			log.Print("Failed to get log entry body ", err)
			fmt.Fprintf(w, err.Error())
		}

		data["body"] = string(body)
		dataJson, err = json.Marshal(data)
		if err != nil {
			log.Print("Failed to encode json: ", err)
		}
	} else {
		start := 0
		offset := 50              // number of items per page
		job := r.FormValue("job") // filter by job is any
		logEntries := LogEntries(job)
		logEntriesLen := len(logEntries)
		if offset >= logEntriesLen {
			offset = logEntriesLen
		}

		resp := ResponseLogs{
			Entries: logEntries[start:offset],
			Length:  logEntriesLen,
		}

		page, err := strconv.ParseInt(r.FormValue("page"), 10, 32)
		if err == nil {
			start = int(page) * offset
			if start >= resp.Length {
				start = resp.Length - offset
			}
			offset += start
			if offset >= resp.Length {
				offset = resp.Length
			}
			resp.Entries = resp.Entries[start:offset]
		}

		dataJson, err = json.Marshal(resp)
		if err != nil {
			log.Print("Failed to encode json: ", err)
		}
	}
	fmt.Fprintf(w, string(dataJson))
}

/* /listJobs will display the available jobs that can be run */
func jobsHandler(w http.ResponseWriter, r *http.Request) {
	// forcing the output content type
	header := w.Header()
	header["Content-Type"] = []string{"application/json"}

	entries, err := jobEntries()
	if err != nil {
		log.Print("Error loading available jobs: ", err)
	}
	dataJson, err := json.Marshal(entries)
	if err != nil {
		log.Print("Failed to encode json")
	}
	fmt.Fprintf(w, string(dataJson))
}

/* This is a helper wrapper. Allows us to log some stuff */
func DefaultWrapper(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
		handler.ServeHTTP(w, r)
	})
}

func main() {
	flag.Parse()

	// create the lock
	commandLock = new(sync.Mutex)
	http.HandleFunc("/run/", runHandler)

	http.HandleFunc("/logs", logsHandler)
	http.HandleFunc("/jobs", jobsHandler)

	// serve other static stuff
	http.Handle("/", http.StripPrefix("/",
		http.FileServer(http.Dir("./static"))))

	// command running handler
	// server index.html at the end
	port := ":8000"
	log.Printf("Starting on " + port)
	err := http.ListenAndServe(port, DefaultWrapper(http.DefaultServeMux))
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>/*
Package Goria provides the functionality of an LRU Cache with an eye to JSR 107
*/
package goria

import (
	"container/list"
	"errors"
	"sync"
)

type EvictionCallback func(key interface{}, value interface{})

type GoriaLRU struct {
	Name         string
	Size         int
	items        map[interface{}]*list.Element
	evictionList *list.List
	onEvict      EvictionCallback
	statsEnabled bool
	stats        CacheStats
	rwMutex      sync.RWMutex
}

type CacheStats struct {
	Items     int64
	Gets      int64
	Hits      int64
	Evictions int64
	Miss      int64
}

type entry struct {
	key   interface{}
	value interface{}
}

func newGoriaLRU(name string, size int, evictionC EvictionCallback, statsEnabled bool) (*GoriaLRU, error) {
	if size <= 0 {
		return nil, errors.New("The Goria Cache need a positive value as size")
	}
	c := &GoriaLRU{
		Name:         name,
		Size:         size,
		evictionList: list.New(),
		items:        make(map[interface{}]*list.Element),
		onEvict:      evictionC,
		statsEnabled: statsEnabled,
		stats: CacheStats{
			Items:     0,
			Evictions: 0,
			Gets:      0,
			Hits:      0,
			Miss:      0,
		},
	}
	return c, nil
}

func (c *GoriaLRU) Put(key, value interface{}) {
	if item, ok := c.items[key]; ok {
		c.rwMutex.Lock()
		defer c.rwMutex.Unlock()
		c.evictionList.MoveToFront(item)
		item.Value.(*entry).value = value
		return
	}

	c.rwMutex.Lock()
	defer c.rwMutex.Unlock()
	item := &entry{key, value}
	element := c.evictionList.PushFront(item)
	c.items[key] = element

	if c.evictionList.Len() > c.Size {
		c.removeFromTail()
	}

	if c.IsStatsEnabled() {
		c.stats.Items++
	}
}

func (c *GoriaLRU) PutAll(m map[interface{}]interface{}) {
	for key, value := range m {
		c.Put(key, value)
	}
}

func (c *GoriaLRU) PutIfAbsent(key, value interface{}) bool {
	var element, exists = c.items[key]
	if !exists && element == nil {
		item := &entry{key, value}
		element := c.evictionList.PushFront(item)
		c.items[key] = element

		if c.evictionList.Len() > c.Size {
			c.removeFromTail()
		}
		if c.IsStatsEnabled() {
			c.stats.Items++
		}
		return true
	}
	return false
}

func (c *GoriaLRU) Get(key interface{}) (value interface{}, exists bool) {
	if c.IsStatsEnabled() {
		c.stats.Gets++
	}
	if item, exists := c.items[key]; exists {
		c.evictionList.MoveToFront(item)
		if c.IsStatsEnabled() {
			c.stats.Hits++
		}
		return item.Value.(*entry).value, true
	}
	if c.IsStatsEnabled() {
		c.stats.Miss++
	}
	return
}

func (c *GoriaLRU) GetAll(m map[interface{}]interface{}) map[interface{}]interface{} {
	returnedMap := make(map[interface{}]interface{})

	for k := range m {
		value, exists := c.Get(k)
		if exists {
			returnedMap[k] = value
		}
	}
	return returnedMap
}

func (c *GoriaLRU) Replace(key, oldValue interface{}, newValue interface{}) bool {
	var element, exists = c.items[key]
	if exists && element.Value.(*entry).value == oldValue {
		c.evictionList.MoveToFront(element)
		element.Value.(*entry).value = newValue
		return true
	}
	return false
}

func (c *GoriaLRU) ReplaceWithKeyOnly(key, newValue interface{}) bool {
	var element, exists = c.items[key]
	if exists && element != nil {
		c.evictionList.MoveToFront(element)
		element.Value.(*entry).value = newValue
		return true
	}
	return false
}

func (c *GoriaLRU) GetAndReplace(key interface{}, newValue interface{}) interface{} {
	v, ok := c.Get(key)
	if ok {
		c.ReplaceWithKeyOnly(key, newValue)
		return v
	}
	return nil
}

func (c *GoriaLRU) RemoveWithKeyOnly(key interface{}) bool {
	if element, exists := c.items[key]; exists {
		c.removeElement(element)
		return true
	}
	return false
}

func (c *GoriaLRU) Remove(key interface{}, oldValue interface{}) bool {
	var element, exists = c.items[key]
	if exists && element.Value.(*entry).value == oldValue {
		c.removeElement(element)
		return true
	}
	return false
}

func (c *GoriaLRU) RemoveAll(m map[interface{}]interface{}) {
	for key, value := range m {
		c.Remove(key, value)
	}
}

func (c *GoriaLRU) RemoveAllWithoutParameters() {
	var keys = c.Keys()
	for i := 0; i < len(keys); i++ {
		c.RemoveWithKeyOnly(keys[i])
	}
}

func (c *GoriaLRU) GetAndRemove(key interface{}) interface{} {
	v, ok := c.Get(key)
	if ok {
		c.RemoveWithKeyOnly(key)
		return v
	}
	return nil
}

func (c *GoriaLRU) Keys() []interface{} {
	keys := make([]interface{}, len(c.items))
	i := 0
	for ent := c.evictionList.Back(); ent != nil; ent = ent.Prev() {
		keys[i] = ent.Value.(*entry).key
		i++
	}
	return keys
}

func (c *GoriaLRU) ContainsKey(key interface{}) bool {
	for ent := c.evictionList.Back(); ent != nil; ent = ent.Prev() {
		if ent.Value.(*entry).key == key {
			return true
		}
	}
	return false
}

func (c *GoriaLRU) Len() int {
	return c.evictionList.Len()
}

func (c *GoriaLRU) GetName() string {
	return c.Name
}

func (c *GoriaLRU) IsStatsEnabled() bool {
	return c.statsEnabled
}

func (c *GoriaLRU) GetStats() CacheStats {
	return c.stats
}

func (c *GoriaLRU) removeFromTail() {
	element := c.evictionList.Back()

	if element != nil {
		c.removeElement(element)
	}
}

func (c *GoriaLRU) removeElement(el *list.Element) {
	c.evictionList.Remove(el)
	entry := el.Value.(*entry)
	delete(c.items, entry.key)

	if c.onEvict != nil {
		c.onEvict(entry.key, entry.value)
	}

	if c.IsStatsEnabled() {
		c.stats.Evictions++
		c.stats.Items--
	}
}
<commit_msg>Update RWMutex usage in Put Operation<commit_after>/*
Package Goria provides the functionality of an LRU Cache with an eye to JSR 107
*/
package goria

import (
	"container/list"
	"errors"
	"sync"
)

type EvictionCallback func(key interface{}, value interface{})

type GoriaLRU struct {
	Name         string
	Size         int
	items        map[interface{}]*list.Element
	evictionList *list.List
	onEvict      EvictionCallback
	statsEnabled bool
	stats        CacheStats
	rwMutex      sync.RWMutex
}

type CacheStats struct {
	Items     int64
	Gets      int64
	Hits      int64
	Evictions int64
	Miss      int64
}

type entry struct {
	key   interface{}
	value interface{}
}

func newGoriaLRU(name string, size int, evictionC EvictionCallback, statsEnabled bool) (*GoriaLRU, error) {
	if size <= 0 {
		return nil, errors.New("The Goria Cache need a positive value as size")
	}
	c := &GoriaLRU{
		Name:         name,
		Size:         size,
		evictionList: list.New(),
		items:        make(map[interface{}]*list.Element),
		onEvict:      evictionC,
		statsEnabled: statsEnabled,
		stats: CacheStats{
			Items:     0,
			Evictions: 0,
			Gets:      0,
			Hits:      0,
			Miss:      0,
		},
	}
	return c, nil
}

func (c *GoriaLRU) Put(key, value interface{}) {
	if item, ok := c.items[key]; ok {
		c.rwMutex.Lock()
		c.evictionList.MoveToFront(item)
		item.Value.(*entry).value = value
		c.rwMutex.Unlock()
		return
	}

	c.rwMutex.Lock()
	item := &entry{key, value}
	element := c.evictionList.PushFront(item)
	c.items[key] = element

	if c.evictionList.Len() > c.Size {
		c.removeFromTail()
	}

	if c.IsStatsEnabled() {
		c.stats.Items++
	}
	c.rwMutex.Unlock()
}

func (c *GoriaLRU) PutAll(m map[interface{}]interface{}) {
	for key, value := range m {
		c.Put(key, value)
	}
}

func (c *GoriaLRU) PutIfAbsent(key, value interface{}) bool {
	var element, exists = c.items[key]
	if !exists && element == nil {
		item := &entry{key, value}
		element := c.evictionList.PushFront(item)
		c.items[key] = element

		if c.evictionList.Len() > c.Size {
			c.removeFromTail()
		}
		if c.IsStatsEnabled() {
			c.stats.Items++
		}
		return true
	}
	return false
}

func (c *GoriaLRU) Get(key interface{}) (value interface{}, exists bool) {
	if c.IsStatsEnabled() {
		c.stats.Gets++
	}
	if item, exists := c.items[key]; exists {
		c.evictionList.MoveToFront(item)
		if c.IsStatsEnabled() {
			c.stats.Hits++
		}
		return item.Value.(*entry).value, true
	}
	if c.IsStatsEnabled() {
		c.stats.Miss++
	}
	return
}

func (c *GoriaLRU) GetAll(m map[interface{}]interface{}) map[interface{}]interface{} {
	returnedMap := make(map[interface{}]interface{})

	for k := range m {
		value, exists := c.Get(k)
		if exists {
			returnedMap[k] = value
		}
	}
	return returnedMap
}

func (c *GoriaLRU) Replace(key, oldValue interface{}, newValue interface{}) bool {
	var element, exists = c.items[key]
	if exists && element.Value.(*entry).value == oldValue {
		c.evictionList.MoveToFront(element)
		element.Value.(*entry).value = newValue
		return true
	}
	return false
}

func (c *GoriaLRU) ReplaceWithKeyOnly(key, newValue interface{}) bool {
	var element, exists = c.items[key]
	if exists && element != nil {
		c.evictionList.MoveToFront(element)
		element.Value.(*entry).value = newValue
		return true
	}
	return false
}

func (c *GoriaLRU) GetAndReplace(key interface{}, newValue interface{}) interface{} {
	v, ok := c.Get(key)
	if ok {
		c.ReplaceWithKeyOnly(key, newValue)
		return v
	}
	return nil
}

func (c *GoriaLRU) RemoveWithKeyOnly(key interface{}) bool {
	if element, exists := c.items[key]; exists {
		c.removeElement(element)
		return true
	}
	return false
}

func (c *GoriaLRU) Remove(key interface{}, oldValue interface{}) bool {
	var element, exists = c.items[key]
	if exists && element.Value.(*entry).value == oldValue {
		c.removeElement(element)
		return true
	}
	return false
}

func (c *GoriaLRU) RemoveAll(m map[interface{}]interface{}) {
	for key, value := range m {
		c.Remove(key, value)
	}
}

func (c *GoriaLRU) RemoveAllWithoutParameters() {
	var keys = c.Keys()
	for i := 0; i < len(keys); i++ {
		c.RemoveWithKeyOnly(keys[i])
	}
}

func (c *GoriaLRU) GetAndRemove(key interface{}) interface{} {
	v, ok := c.Get(key)
	if ok {
		c.RemoveWithKeyOnly(key)
		return v
	}
	return nil
}

func (c *GoriaLRU) Keys() []interface{} {
	keys := make([]interface{}, len(c.items))
	i := 0
	for ent := c.evictionList.Back(); ent != nil; ent = ent.Prev() {
		keys[i] = ent.Value.(*entry).key
		i++
	}
	return keys
}

func (c *GoriaLRU) ContainsKey(key interface{}) bool {
	for ent := c.evictionList.Back(); ent != nil; ent = ent.Prev() {
		if ent.Value.(*entry).key == key {
			return true
		}
	}
	return false
}

func (c *GoriaLRU) Len() int {
	return c.evictionList.Len()
}

func (c *GoriaLRU) GetName() string {
	return c.Name
}

func (c *GoriaLRU) IsStatsEnabled() bool {
	return c.statsEnabled
}

func (c *GoriaLRU) GetStats() CacheStats {
	return c.stats
}

func (c *GoriaLRU) removeFromTail() {
	element := c.evictionList.Back()

	if element != nil {
		c.removeElement(element)
	}
}

func (c *GoriaLRU) removeElement(el *list.Element) {
	c.evictionList.Remove(el)
	entry := el.Value.(*entry)
	delete(c.items, entry.key)

	if c.onEvict != nil {
		c.onEvict(entry.key, entry.value)
	}

	if c.IsStatsEnabled() {
		c.stats.Evictions++
		c.stats.Items--
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

type Tree struct {
	Root           *Node
	MaxDepth       int
	directoryCount int
	fileCount      int
}

func NewTree(directory string, maxDepth int) *Tree {
	year, month, day := GetYMD(directory)
	dirPath, err := filepath.Abs(directory)
	if err != nil {
		log.Fatal(err)
	}
	return &Tree{
		Root:           NewNode(dirPath, year, month, day),
		MaxDepth:       maxDepth,
		directoryCount: 0,
		fileCount:      0,
	}
}

func (t *Tree) Build() error {
	file, err := os.Open(t.Root.FileName)

	if err != nil {
		return err
	}

	var regularExpression *regexp.Regexp
	if filterPattern != "" {
		regularExpression, err = regexp.Compile(filterPattern)
		if err != nil {
			return groupbyError("Invalid regular expression specified in -e/-pattern")
		}
	}

	files, err := file.Readdir(-1)

	if err != nil {
		return err
	}

	if files == nil {
		return groupbyError("Directory is empty or cannot be read")
	}

	for _, f := range files {
		if !regularExpression.MatchString(f.Name()) {
			continue
		}
		if ignoreDirectories && f.IsDir() {
			continue
		} else {
			t.AddEntry(f)
		}
	}
	return nil
}

func (t *Tree) AddEntry(file os.FileInfo) {

	if strings.HasPrefix(file.Name(), ".") && !includeHidden {
		return
	}

	if file.IsDir() {
		t.directoryCount++
	} else {
		t.fileCount++
	}

	year, month, day := GetFileInfoYMD(file)
	var node = NewNode(file.Name(), year, month, day)

	yearStr, monthStr, dayStr := fmt.Sprintf("%d", year), fmt.Sprintf("%d", month), fmt.Sprintf("%d", day)
	var yearNode = t.Root.Search(yearStr)

	// year node
	if yearNode == nil {
		yearNode = NewNode(yearStr, year, month, day)
		t.Root.AddChild(yearNode)
	}

	if t.MaxDepth == 1 {
		yearNode.AddChild(node)
	}

	if t.MaxDepth >= 2 {
		// month node
		var monthNode = yearNode.Search(monthStr)

		if monthNode == nil {
			monthNode = NewNode(monthStr, year, month, day)
			yearNode.AddChild(monthNode)
		}

		if t.MaxDepth == 3 {
			var dayNode = monthNode.Search(dayStr)

			if dayNode == nil {
				dayNode = NewNode(dayStr, year, month, day)
				monthNode.AddChild(dayNode)
			}

			dayNode.AddChild(node)

		} else {
			monthNode.AddChild(node)
		}
	}
}

func (t *Tree) Visit(visitor NodeVisitor) {
	t.Root.Visit(visitor, 0)
}

// Directories returns number of directories in the tree
func (t *Tree) Directories() int {
	return t.directoryCount
}

// Files returns number of files in the tree
func (t *Tree) Files() int {
	return t.fileCount
}
<commit_msg>Check for nil regular expression<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

type Tree struct {
	Root           *Node
	MaxDepth       int
	directoryCount int
	fileCount      int
}

func NewTree(directory string, maxDepth int) *Tree {
	year, month, day := GetYMD(directory)
	dirPath, err := filepath.Abs(directory)
	if err != nil {
		log.Fatal(err)
	}
	return &Tree{
		Root:           NewNode(dirPath, year, month, day),
		MaxDepth:       maxDepth,
		directoryCount: 0,
		fileCount:      0,
	}
}

func (t *Tree) Build() error {
	file, err := os.Open(t.Root.FileName)

	if err != nil {
		return err
	}

	var regularExpression *regexp.Regexp
	if filterPattern != "" {
		regularExpression, err = regexp.Compile(filterPattern)
		if err != nil {
			return groupbyError("Invalid regular expression specified in -e/-pattern")
		}
	}

	files, err := file.Readdir(-1)

	if err != nil {
		return err
	}

	if files == nil {
		return groupbyError("Directory is empty or cannot be read")
	}

	for _, f := range files {
		if regularExpression != nil && !regularExpression.MatchString(f.Name()) {
			continue
		}
		if ignoreDirectories && f.IsDir() {
			continue
		} else {
			t.AddEntry(f)
		}
	}
	return nil
}

func (t *Tree) AddEntry(file os.FileInfo) {

	if strings.HasPrefix(file.Name(), ".") && !includeHidden {
		return
	}

	if file.IsDir() {
		t.directoryCount++
	} else {
		t.fileCount++
	}

	year, month, day := GetFileInfoYMD(file)
	var node = NewNode(file.Name(), year, month, day)

	yearStr, monthStr, dayStr := fmt.Sprintf("%d", year), fmt.Sprintf("%d", month), fmt.Sprintf("%d", day)
	var yearNode = t.Root.Search(yearStr)

	// year node
	if yearNode == nil {
		yearNode = NewNode(yearStr, year, month, day)
		t.Root.AddChild(yearNode)
	}

	if t.MaxDepth == 1 {
		yearNode.AddChild(node)
	}

	if t.MaxDepth >= 2 {
		// month node
		var monthNode = yearNode.Search(monthStr)

		if monthNode == nil {
			monthNode = NewNode(monthStr, year, month, day)
			yearNode.AddChild(monthNode)
		}

		if t.MaxDepth == 3 {
			var dayNode = monthNode.Search(dayStr)

			if dayNode == nil {
				dayNode = NewNode(dayStr, year, month, day)
				monthNode.AddChild(dayNode)
			}

			dayNode.AddChild(node)

		} else {
			monthNode.AddChild(node)
		}
	}
}

func (t *Tree) Visit(visitor NodeVisitor) {
	t.Root.Visit(visitor, 0)
}

// Directories returns number of directories in the tree
func (t *Tree) Directories() int {
	return t.directoryCount
}

// Files returns number of files in the tree
func (t *Tree) Files() int {
	return t.fileCount
}
<|endoftext|>


<commit_before>package tree

import "fmt"

var limit = 3 // maximum amount of childs for each nodes

type Tree struct {
	root *Node
}

type Node struct {
	parent *Node
	child []*Node
	data Data
}

type Data struct {
	Id int
}

type Result struct {
	Parent int
	Child []int
	Data Data
}

func addNode (parent *Node, data Data) *Node {
	return &Node{parent, make([]*Node, limit), data}
}

func (tree *Tree) Insert(data Data) (node *Node) {
	if tree.root == nil {
		node = addNode(nil, data)
		tree.root = node

		return
	}

	return
}

func (node *Node) findFreeNode() *Node {
	stack := make([]*Node, 0, 32)
	stack = append(stack, node) // push root to stack
	var n *Node = nil

	for stack != nil {
		n, stack = stack[0], stack[1:] //pop first node

		for index := 0; index < len(n.child); index++ {
			if (n.child[index] == nil) {
				return n
			}
			stack = append(stack, n.child[index]) // push
		}
	}

	return nil
}

func (node *Node) Insert(data Data) (newNode *Node) {
	for i := 0; i< len(node.child); i++ {
		if (node.child[i] == nil) {
			newNode = addNode(node, data)
			node.child[i] = newNode

			return
		}
	}

	freeNode := node.findFreeNode()
	newNode = freeNode.Insert(data)

	return
}

func (node *Node) Count(c chan int) (counter int) {
	if node == nil {
		return
	}

	c <- 1
	for i := 0; i < len(node.child); i++ {
		node.child[i].Count(c)
	}

	return
}

func (node *Node) Print() {
	if node == nil {
		return
	}

	fmt.Print(node.data)
	fmt.Print("\n")
	for i := 0; i < len(node.child); i++ {
		node.child[i].Print()
	}
}

func (node *Node) Search(id int) (n *Node) {
	stack := make([]*Node, 0, 32)
	stack = append(stack, node) // push root to stack
	if (node.data.Id == id) {
		return node
	}

	for stack != nil {
		n, stack = stack[0], stack[1:] //shift first node

		for index := 0; index < len(n.child); index++ {
			if (n.child[index] == nil) {
				continue
			}
			if (n.child[index].data.Id == id) {
				return n.child[index]
			}
			stack = append(stack, n.child[index]) // push
		}
	}

	return nil
}

func (node *Node) GetData() (result Result) {
	var childs []int = make([]int, 0)
	for _, value := range node.child {
		if value != nil {
			childs = append(childs, value.data.Id)
		} else {
			childs = append(childs, 0)
		}
	}

	var parentNode int = 0

	if node.parent != nil {
		parentNode = node.parent.data.Id
	}

	return Result{
		Parent: parentNode,
		Child: childs,
		Data: node.data,
	}
}<commit_msg>add array of indexes and usability of it<commit_after>package tree

import "fmt"

var limit = 3 // maximum amount of childs for each nodes

type Tree struct {
	root *Node
}

type Node struct {
	parent *Node
	child []*Node
	data Data
}

type Data struct {
	Id int
}

type Result struct {
	Parent int
	Child []int
	Data Data
}

var treeIndex map[int]*Node

func addNode (parent *Node, data Data) *Node {
	return &Node{parent, make([]*Node, limit), data}
}

func (tree *Tree) Insert(data Data) (node *Node) {
	treeIndex = make(map[int]*Node)
	if tree.root == nil {
		node = addNode(nil, data)
		treeIndex[data.Id] = node
		tree.root = node

		return
	}

	return
}

func (node *Node) SetAllNodes(results []*Result) {
	var newNode, currentNode *Node
	for _, result := range results {
		if result.Parent != 0 {
			currentNode = node.Search(result.Parent)
		} else {
			currentNode = node
		}
		newNode = currentNode.Insert(result.Data)
		treeIndex[result.Data.Id] = newNode
	}
}

func (node *Node) findFreeNode() *Node {
	stack := make([]*Node, 0, 32)
	stack = append(stack, node) // push root to stack
	var n *Node = nil

	for stack != nil {
		n, stack = stack[0], stack[1:] //pop first node

		for index := 0; index < len(n.child); index++ {
			if (n.child[index] == nil) {
				return n
			}
			stack = append(stack, n.child[index]) // push
		}
	}

	return nil
}

func (node *Node) Insert(data Data) (newNode *Node) {
	for i := 0; i< len(node.child); i++ {
		if (node.child[i] == nil) {
			newNode = addNode(node, data)
			treeIndex[data.Id] = node
			node.child[i] = newNode

			return
		}
	}

	freeNode := node.findFreeNode()
	newNode = freeNode.Insert(data)

	return
}

func (node *Node) Print() {
	if node == nil {
		return
	}

	fmt.Print(node.data)
	fmt.Print("\n")
	for i := 0; i < len(node.child); i++ {
		node.child[i].Print()
	}
}

func (node *Node) Search(id int) (n *Node) {

	n = treeIndex[id]
	if n == nil {
		return nil
	}

	return
}

func (node *Node) GetData() (result *Result) {
	var childs []int = make([]int, 0)
	for _, value := range node.child {
		if value != nil {
			childs = append(childs, value.data.Id)
		} else {
			childs = append(childs, 0)
		}
	}

	var parentNode int = 0

	if node.parent != nil {
		parentNode = node.parent.data.Id
	}

	return &Result{
		Parent: parentNode,
		Child: childs,
		Data: node.data,
	}
}

func getLevel(nodes []*Node) (results []*Node) {
	for _, value := range nodes {
		for index := 0; index < len(value.child); index++ {
			if (value.child[index] == nil) {
				continue
			}
			results = append(results, value.child[index])
		}
	}

	return
}

func (node *Node) GetNodes(level int) (results []*Result) {
	nodes := make([]*Node, 0)
	nodes = append(nodes, node)
	results = append(results, node.GetData())

	for index := 0; index < level; index++ {
		nodes = getLevel(nodes)
		for _, value := range nodes {
			results = append(results, value.GetData())
		}
	}

	return
}
<|endoftext|>


<commit_before>package eGaugeDecoders

import (
	"encoding/xml"
)

type Cname struct {
	Type string `xml:"t,attr"`
	Name string `xml:",chardata"`
}

type Row struct {
	Columns []string `xml:"c"`
}

type Data struct {
	Columns   int     `xml:"columns,attr"`
	Timestamp string  `xml:"time_stamp,attr"`
	Delta     int     `xml:"time_delta,attr"`
	Rows      []Row   `xml:"r"`
	Cnames    []Cname `xml:"cname"`
}

type Packet struct {
	Serial string `xml:"serial,attr"`
	Data   []Data `xml:"data"`
}

func DecodePacket(raw []byte) (packet Packet, err error) {
	err = xml.Unmarshal(raw, &packet)
	return
}
<commit_msg>Try automated xml hex to int64<commit_after>package eGaugeDecoders

import (
	"encoding/xml"
)

type Cname struct {
	Type string `xml:"t,attr"`
	Name string `xml:",chardata"`
}

type Row struct {
	Columns []string `xml:"c"`
}

type Data struct {
	Columns   int     `xml:"columns,attr"`
	Timestamp int64   `xml:"time_stamp,attr"`
	Delta     int     `xml:"time_delta,attr"`
	Rows      []Row   `xml:"r"`
	Cnames    []Cname `xml:"cname"`
}

type Packet struct {
	Serial int64  `xml:"serial,attr"`
	Data   []Data `xml:"data"`
}

func DecodePacket(raw []byte) (packet Packet, err error) {
	err = xml.Unmarshal(raw, &packet)
	return
}
<|endoftext|>


<commit_before>// +build solaris
package tty

import (
	"golang.org/x/sys/unix"
)

const (
	ioctlReadTermios  = unix.TCGETS
	ioctlWriteTermios = unix.TCSETS
)
<commit_msg>conditional compile fix<commit_after>// +build solaris

package tty

import (
	"golang.org/x/sys/unix"
)

const (
	ioctlReadTermios  = unix.TCGETS
	ioctlWriteTermios = unix.TCSETS
)
<|endoftext|>


<commit_before>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package drive

import (
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sort"
	"sync"

	"github.com/odeke-em/statos"
)

const (
	maxNumOfConcPullTasks = 4
)

type urlMimeTypeExt struct {
	ext      string
	mimeType string
	url      string
}

type downloadArg struct {
	id              string
	path            string
	exportURL       string
	ackByteProgress bool
}

// Pull from remote if remote path exists and in a god context. If path is a
// directory, it recursively pulls from the remote if there are remote changes.
// It doesn't check if there are remote changes if isForce is set.
func (g *Commands) Pull() (err error) {
	var cl []*Change

	g.log.Logln("Resolving...")

	spin := g.playabler()
	spin.play()

	for _, relToRootPath := range g.opts.Sources {
		fsPath := g.context.AbsPathOf(relToRootPath)
		ccl, cErr := g.changeListResolve(relToRootPath, fsPath, false)
		if cErr != nil {
			return cErr
		}
		if len(ccl) > 0 {
			cl = append(cl, ccl...)
		}
	}

	spin.stop()

	nonConflictsPtr, conflictsPtr := g.resolveConflicts(cl, false)
	if conflictsPtr != nil {
		warnConflictsPersist(g.log, *conflictsPtr)
		return fmt.Errorf("conflicts have prevented a pull operation")
	}

	nonConflicts := *nonConflictsPtr

	ok, opMap := printChangeList(g.log, nonConflicts, !g.opts.canPrompt(), g.opts.NoClobber)
	if !ok {
		return
	}

	return g.playPullChanges(nonConflicts, g.opts.Exports, opMap)
}

func (g *Commands) PullMatches() (err error) {
	var cl []*Change
	matches, err := g.rem.FindMatches(g.opts.Path, g.opts.Sources, false)

	if err != nil {
		return err
	}

	p := g.opts.Path
	if p == "/" {
		p = ""
	}

	for match := range matches {
		if match == nil {
			continue
		}
		relToRoot := "/" + match.Name
		fsPath := g.context.AbsPathOf(relToRoot)

		ccl, cErr := g.byRemoteResolve(relToRoot, fsPath, match, false)
		if cErr != nil {
			return cErr
		}

		cl = append(cl, ccl...)
	}

	if len(cl) < 1 {
		return fmt.Errorf("no changes detected!")
	}

	nonConflictsPtr, conflictsPtr := g.resolveConflicts(cl, false)
	if conflictsPtr != nil {
		warnConflictsPersist(g.log, *conflictsPtr)
		return fmt.Errorf("conflicts have prevented a pull operation")
	}

	nonConflicts := *nonConflictsPtr

	ok, opMap := printChangeList(g.log, nonConflicts, !g.opts.canPrompt(), g.opts.NoClobber)
	if !ok {
		return nil
	}

	return g.playPullChanges(nonConflicts, g.opts.Exports, opMap)
}

func (g *Commands) PullPiped() (err error) {
	// Cannot pull asynchronously because the pull order must be maintained
	for _, relToRootPath := range g.opts.Sources {
		rem, err := g.rem.FindByPath(relToRootPath)
		if err != nil {
			return fmt.Errorf("%s: %v", relToRootPath, err)
		}
		if rem == nil {
			continue
		}

		if hasExportLinks(rem) {
			g.log.LogErrf("'%s' is a GoogleDoc/Sheet document cannot be pulled from raw, only exported.\n", relToRootPath)
			continue
		}
		blobHandle, dlErr := g.rem.Download(rem.Id, "")
		if dlErr != nil {
			return dlErr
		}
		if blobHandle == nil {
			continue
		}
		_, err = io.Copy(os.Stdout, blobHandle)
		blobHandle.Close()
	}
	return
}

func (g *Commands) playPullChanges(cl []*Change, exports []string, opMap *map[Operation]sizeCounter) (err error) {
	var next []*Change

	if opMap == nil {
		result := opChangeCount(cl)
		opMap = &result
	}

	totalSize := int64(0)
	ops := *opMap

	for _, counter := range ops {
		totalSize += counter.src
	}

	g.taskStart(totalSize)

	defer close(g.rem.progressChan)

	// TODO: Only provide precedence ordering if all the other options are allowed
	// Currently noop on sorting by precedence
	if false && !g.opts.NoClobber {
		sort.Sort(ByPrecedence(cl))
	}

	go func() {
		for n := range g.rem.progressChan {
			g.taskAdd(int64(n))
		}
	}()

	for {
		if len(cl) > maxNumOfConcPullTasks {
			next, cl = cl[:maxNumOfConcPullTasks], cl[maxNumOfConcPullTasks:len(cl)]
		} else {
			next, cl = cl, []*Change{}
		}
		if len(next) == 0 {
			break
		}
		var wg sync.WaitGroup
		wg.Add(len(next))
		// play the changes
		// TODO: add timeouts
		for _, c := range next {
			switch c.Op() {
			case OpMod:
				go g.localMod(&wg, c, exports)
			case OpModConflict:
				go g.localMod(&wg, c, exports)
			case OpAdd:
				go g.localAdd(&wg, c, exports)
			case OpDelete:
				go g.localDelete(&wg, c)
			}
		}
		wg.Wait()
	}

	g.taskFinish()
	return err
}

func (g *Commands) localMod(wg *sync.WaitGroup, change *Change, exports []string) (err error) {
	defer func() {
		if err == nil {
			src := change.Src
			index := src.ToIndex()
			wErr := g.context.SerializeIndex(index, g.context.AbsPathOf(""))

			// TODO: Should indexing errors be reported?
			if wErr != nil {
				g.log.LogErrf("serializeIndex %s: %v\n", src.Name, wErr)
			}
		}
		wg.Done()
	}()

	destAbsPath := g.context.AbsPathOf(change.Path)

	downloadPerformed := false

	// Simple heuristic to avoid downloading all the
	// content yet it could just be a modTime difference
	mask := fileDifferences(change.Src, change.Dest, change.IgnoreChecksum)
	if checksumDiffers(mask) {
		// download and replace
		if err = g.download(change, exports); err != nil {
			return
		}
		downloadPerformed = true
	}
	err = os.Chtimes(destAbsPath, change.Src.ModTime, change.Src.ModTime)

	// Update progress for the case in which you are only Chtime-ing
	// since progress for downloaded files is already handled separately
	if !downloadPerformed {
		chunks := chunkInt64(change.Src.Size)
		for n := range chunks {
			g.rem.progressChan <- n
		}
	}

	return
}

func (g *Commands) localAdd(wg *sync.WaitGroup, change *Change, exports []string) (err error) {
	defer func() {
		if err == nil {
			src := change.Src
			index := src.ToIndex()
			sErr := g.context.SerializeIndex(index, g.context.AbsPathOf(""))

			// TODO: Should indexing errors be reported?
			if sErr != nil {
				g.log.LogErrf("serializeIndex %s: %v\n", src.Name, sErr)
			}
		}
		wg.Done()
	}()

	destAbsPath := g.context.AbsPathOf(change.Path)

	// make parent's dir if not exists
	destAbsDir := g.context.AbsPathOf(change.Parent)

	if destAbsDir != destAbsPath {
		err = os.MkdirAll(destAbsDir, os.ModeDir|0755)
		if err != nil {
			return err
		}
	}

	if change.Src.IsDir {
		return os.Mkdir(destAbsPath, os.ModeDir|0755)
	}

	// download and create
	if err = g.download(change, exports); err != nil {
		return
	}

	err = os.Chtimes(destAbsPath, change.Src.ModTime, change.Src.ModTime)
	return
}

func (g *Commands) localDelete(wg *sync.WaitGroup, change *Change) (err error) {
	defer func() {
		if err == nil {
			chunks := chunkInt64(change.Dest.Size)
			for n := range chunks {
				g.rem.progressChan <- n
			}
		}
		wg.Done()
	}()
	err = os.RemoveAll(change.Dest.BlobAt)
	return
}

func touchFile(path string) (err error) {
	var ef *os.File
	defer func() {
		if err == nil && ef != nil {
			ef.Close()
		}
	}()
	ef, err = os.Create(path)
	return
}

func (g *Commands) export(f *File, destAbsPath string, exports []string) (manifest []string, err error) {
	if len(exports) < 1 || f == nil {
		return
	}

	dirPath := sepJoin("_", destAbsPath, "exports")
	if err = os.MkdirAll(dirPath, os.ModeDir|0755); err != nil {
		return
	}

	var ok bool
	var mimeType, exportURL string

	waitables := []*urlMimeTypeExt{}

	for _, ext := range exports {
		mimeType = mimeTypeFromExt(ext)
		exportURL, ok = f.ExportLinks[mimeType]
		if !ok {
			continue
		}

		waitables = append(waitables, &urlMimeTypeExt{
			mimeType: mimeType,
			url:      exportURL,
			ext:      ext,
		})
	}

	var wg sync.WaitGroup
	wg.Add(len(waitables))

	basePath := filepath.Base(f.Name)
	baseDir := path.Join(dirPath, basePath)

	for _, exportee := range waitables {
		go func(wg *sync.WaitGroup, baseDirPath, id string, urlMExt *urlMimeTypeExt) error {
			defer func() {
				wg.Done()
			}()

			exportPath := sepJoin(".", baseDirPath, urlMExt.ext)

			// TODO: Decide if users should get to make *.desktop users even for exports
			if runtime.GOOS == OSLinuxKey && false {
				desktopEntryPath := sepJoin(".", exportPath, DesktopExtension)

				_, dentErr := f.serializeAsDesktopEntry(desktopEntryPath, urlMExt)
				if dentErr != nil {
					g.log.LogErrf("desktopEntry: %s %v\n", desktopEntryPath, dentErr)
				}
			}

			dlArg := downloadArg{
				ackByteProgress: false,
				path:            exportPath,
				id:              id,
				exportURL:       urlMExt.url,
			}

			err := g.singleDownload(&dlArg)
			if err == nil {
				manifest = append(manifest, exportPath)
			}
			return err
		}(&wg, baseDir, f.Id, exportee)
	}
	wg.Wait()
	return
}

func isLocalFile(f *File) bool {
	// TODO: Better check
	return f != nil && f.Etag == ""
}

func (g *Commands) download(change *Change, exports []string) (err error) {
	if change.Src == nil {
		return fmt.Errorf("tried to download nil change.Src")
	}

	destAbsPath := g.context.AbsPathOf(change.Path)
	if change.Src.BlobAt != "" {
		dlArg := downloadArg{
			path:            destAbsPath,
			id:              change.Src.Id,
			ackByteProgress: true,
		}

		return g.singleDownload(&dlArg)
	}

	// We need to touch the empty file to
	// ensure consistency during a push.
	if runtime.GOOS != OSLinuxKey {
		err = touchFile(destAbsPath)
		if err != nil {
			return err
		}
	} else {
		// For those our Linux kin that need .desktop files
		dirPath := g.opts.ExportsDir
		if dirPath == "" {
			dirPath = filepath.Dir(destAbsPath)
		}

		f := change.Src

		urlMExt := urlMimeTypeExt{
			url:      f.AlternateLink,
			ext:      DesktopExtension,
			mimeType: f.MimeType,
		}

		dirPath = filepath.Join(dirPath, f.Name)
		desktopEntryPath := sepJoin(".", dirPath, urlMExt.ext)

		_, dentErr := f.serializeAsDesktopEntry(desktopEntryPath, &urlMExt)
		if dentErr != nil {
			g.log.LogErrf("desktopEntry: %s %v\n", desktopEntryPath, dentErr)
		}
	}

	if len(exports) >= 1 && hasExportLinks(change.Src) {
		exportDirPath := destAbsPath
		if g.opts.ExportsDir != "" {
			exportDirPath = path.Join(g.opts.ExportsDir, change.Src.Name)
		}

		manifest, exportErr := g.export(change.Src, exportDirPath, exports)
		if exportErr == nil {
			for _, exportPath := range manifest {
				g.log.Logf("Exported '%s' to '%s'\n", destAbsPath, exportPath)
			}
		}
		return exportErr
	}
	return
}

func (g *Commands) singleDownload(dlArg *downloadArg) (err error) {
	var fo *os.File
	fo, err = os.Create(dlArg.path)
	if err != nil {
		g.log.LogErrf("create: %s %v\n", dlArg.path, err)
		return
	}

	// close fo on exit and check for its returned error
	defer func() {
		fErr := fo.Close()
		if fErr != nil {
			g.log.LogErrf("fErr", fErr)
			err = fErr
		}
	}()

	var blob io.ReadCloser
	defer func() {
		if blob != nil {
			blob.Close()
		}
	}()

	blob, err = g.rem.Download(dlArg.id, dlArg.exportURL)
	if err != nil {
		return err
	}

	ws := statos.NewWriter(fo)

	go func() {
		commChan := ws.ProgressChan()
		if dlArg.ackByteProgress {
			for n := range commChan {
				g.rem.progressChan <- n
			}
		} else { // Just drain the progress channel
			for _ = range commChan {
				g.rem.progressChan <- 0
			}
		}
	}()

	_, err = io.Copy(ws, blob)

	return
}
<commit_msg>desktopEntry: omit -desktop in entry name<commit_after>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package drive

import (
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sort"
	"sync"

	"github.com/odeke-em/statos"
)

const (
	maxNumOfConcPullTasks = 4
)

type urlMimeTypeExt struct {
	ext      string
	mimeType string
	url      string
}

type downloadArg struct {
	id              string
	path            string
	exportURL       string
	ackByteProgress bool
}

// Pull from remote if remote path exists and in a god context. If path is a
// directory, it recursively pulls from the remote if there are remote changes.
// It doesn't check if there are remote changes if isForce is set.
func (g *Commands) Pull() (err error) {
	var cl []*Change

	g.log.Logln("Resolving...")

	spin := g.playabler()
	spin.play()

	for _, relToRootPath := range g.opts.Sources {
		fsPath := g.context.AbsPathOf(relToRootPath)
		ccl, cErr := g.changeListResolve(relToRootPath, fsPath, false)
		if cErr != nil {
			return cErr
		}
		if len(ccl) > 0 {
			cl = append(cl, ccl...)
		}
	}

	spin.stop()

	nonConflictsPtr, conflictsPtr := g.resolveConflicts(cl, false)
	if conflictsPtr != nil {
		warnConflictsPersist(g.log, *conflictsPtr)
		return fmt.Errorf("conflicts have prevented a pull operation")
	}

	nonConflicts := *nonConflictsPtr

	ok, opMap := printChangeList(g.log, nonConflicts, !g.opts.canPrompt(), g.opts.NoClobber)
	if !ok {
		return
	}

	return g.playPullChanges(nonConflicts, g.opts.Exports, opMap)
}

func (g *Commands) PullMatches() (err error) {
	var cl []*Change
	matches, err := g.rem.FindMatches(g.opts.Path, g.opts.Sources, false)

	if err != nil {
		return err
	}

	p := g.opts.Path
	if p == "/" {
		p = ""
	}

	for match := range matches {
		if match == nil {
			continue
		}
		relToRoot := "/" + match.Name
		fsPath := g.context.AbsPathOf(relToRoot)

		ccl, cErr := g.byRemoteResolve(relToRoot, fsPath, match, false)
		if cErr != nil {
			return cErr
		}

		cl = append(cl, ccl...)
	}

	if len(cl) < 1 {
		return fmt.Errorf("no changes detected!")
	}

	nonConflictsPtr, conflictsPtr := g.resolveConflicts(cl, false)
	if conflictsPtr != nil {
		warnConflictsPersist(g.log, *conflictsPtr)
		return fmt.Errorf("conflicts have prevented a pull operation")
	}

	nonConflicts := *nonConflictsPtr

	ok, opMap := printChangeList(g.log, nonConflicts, !g.opts.canPrompt(), g.opts.NoClobber)
	if !ok {
		return nil
	}

	return g.playPullChanges(nonConflicts, g.opts.Exports, opMap)
}

func (g *Commands) PullPiped() (err error) {
	// Cannot pull asynchronously because the pull order must be maintained
	for _, relToRootPath := range g.opts.Sources {
		rem, err := g.rem.FindByPath(relToRootPath)
		if err != nil {
			return fmt.Errorf("%s: %v", relToRootPath, err)
		}
		if rem == nil {
			continue
		}

		if hasExportLinks(rem) {
			g.log.LogErrf("'%s' is a GoogleDoc/Sheet document cannot be pulled from raw, only exported.\n", relToRootPath)
			continue
		}
		blobHandle, dlErr := g.rem.Download(rem.Id, "")
		if dlErr != nil {
			return dlErr
		}
		if blobHandle == nil {
			continue
		}
		_, err = io.Copy(os.Stdout, blobHandle)
		blobHandle.Close()
	}
	return
}

func (g *Commands) playPullChanges(cl []*Change, exports []string, opMap *map[Operation]sizeCounter) (err error) {
	var next []*Change

	if opMap == nil {
		result := opChangeCount(cl)
		opMap = &result
	}

	totalSize := int64(0)
	ops := *opMap

	for _, counter := range ops {
		totalSize += counter.src
	}

	g.taskStart(totalSize)

	defer close(g.rem.progressChan)

	// TODO: Only provide precedence ordering if all the other options are allowed
	// Currently noop on sorting by precedence
	if false && !g.opts.NoClobber {
		sort.Sort(ByPrecedence(cl))
	}

	go func() {
		for n := range g.rem.progressChan {
			g.taskAdd(int64(n))
		}
	}()

	for {
		if len(cl) > maxNumOfConcPullTasks {
			next, cl = cl[:maxNumOfConcPullTasks], cl[maxNumOfConcPullTasks:len(cl)]
		} else {
			next, cl = cl, []*Change{}
		}
		if len(next) == 0 {
			break
		}
		var wg sync.WaitGroup
		wg.Add(len(next))
		// play the changes
		// TODO: add timeouts
		for _, c := range next {
			switch c.Op() {
			case OpMod:
				go g.localMod(&wg, c, exports)
			case OpModConflict:
				go g.localMod(&wg, c, exports)
			case OpAdd:
				go g.localAdd(&wg, c, exports)
			case OpDelete:
				go g.localDelete(&wg, c)
			}
		}
		wg.Wait()
	}

	g.taskFinish()
	return err
}

func (g *Commands) localMod(wg *sync.WaitGroup, change *Change, exports []string) (err error) {
	defer func() {
		if err == nil {
			src := change.Src
			index := src.ToIndex()
			wErr := g.context.SerializeIndex(index, g.context.AbsPathOf(""))

			// TODO: Should indexing errors be reported?
			if wErr != nil {
				g.log.LogErrf("serializeIndex %s: %v\n", src.Name, wErr)
			}
		}
		wg.Done()
	}()

	destAbsPath := g.context.AbsPathOf(change.Path)

	downloadPerformed := false

	// Simple heuristic to avoid downloading all the
	// content yet it could just be a modTime difference
	mask := fileDifferences(change.Src, change.Dest, change.IgnoreChecksum)
	if checksumDiffers(mask) {
		// download and replace
		if err = g.download(change, exports); err != nil {
			return
		}
		downloadPerformed = true
	}
	err = os.Chtimes(destAbsPath, change.Src.ModTime, change.Src.ModTime)

	// Update progress for the case in which you are only Chtime-ing
	// since progress for downloaded files is already handled separately
	if !downloadPerformed {
		chunks := chunkInt64(change.Src.Size)
		for n := range chunks {
			g.rem.progressChan <- n
		}
	}

	return
}

func (g *Commands) localAdd(wg *sync.WaitGroup, change *Change, exports []string) (err error) {
	defer func() {
		if err == nil {
			src := change.Src
			index := src.ToIndex()
			sErr := g.context.SerializeIndex(index, g.context.AbsPathOf(""))

			// TODO: Should indexing errors be reported?
			if sErr != nil {
				g.log.LogErrf("serializeIndex %s: %v\n", src.Name, sErr)
			}
		}
		wg.Done()
	}()

	destAbsPath := g.context.AbsPathOf(change.Path)

	// make parent's dir if not exists
	destAbsDir := g.context.AbsPathOf(change.Parent)

	if destAbsDir != destAbsPath {
		err = os.MkdirAll(destAbsDir, os.ModeDir|0755)
		if err != nil {
			return err
		}
	}

	if change.Src.IsDir {
		return os.Mkdir(destAbsPath, os.ModeDir|0755)
	}

	// download and create
	if err = g.download(change, exports); err != nil {
		return
	}

	err = os.Chtimes(destAbsPath, change.Src.ModTime, change.Src.ModTime)
	return
}

func (g *Commands) localDelete(wg *sync.WaitGroup, change *Change) (err error) {
	defer func() {
		if err == nil {
			chunks := chunkInt64(change.Dest.Size)
			for n := range chunks {
				g.rem.progressChan <- n
			}
		}
		wg.Done()
	}()
	err = os.RemoveAll(change.Dest.BlobAt)
	return
}

func touchFile(path string) (err error) {
	var ef *os.File
	defer func() {
		if err == nil && ef != nil {
			ef.Close()
		}
	}()
	ef, err = os.Create(path)
	return
}

func (g *Commands) export(f *File, destAbsPath string, exports []string) (manifest []string, err error) {
	if len(exports) < 1 || f == nil {
		return
	}

	dirPath := sepJoin("_", destAbsPath, "exports")
	if err = os.MkdirAll(dirPath, os.ModeDir|0755); err != nil {
		return
	}

	var ok bool
	var mimeType, exportURL string

	waitables := []*urlMimeTypeExt{}

	for _, ext := range exports {
		mimeType = mimeTypeFromExt(ext)
		exportURL, ok = f.ExportLinks[mimeType]
		if !ok {
			continue
		}

		waitables = append(waitables, &urlMimeTypeExt{
			mimeType: mimeType,
			url:      exportURL,
			ext:      ext,
		})
	}

	var wg sync.WaitGroup
	wg.Add(len(waitables))

	basePath := filepath.Base(f.Name)
	baseDir := path.Join(dirPath, basePath)

	for _, exportee := range waitables {
		go func(wg *sync.WaitGroup, baseDirPath, id string, urlMExt *urlMimeTypeExt) error {
			defer func() {
				wg.Done()
			}()

			exportPath := sepJoin(".", baseDirPath, urlMExt.ext)

			// TODO: Decide if users should get to make *.desktop users even for exports
			if runtime.GOOS == OSLinuxKey && false {
				desktopEntryPath := sepJoin(".", exportPath, DesktopExtension)

				_, dentErr := f.serializeAsDesktopEntry(desktopEntryPath, urlMExt)
				if dentErr != nil {
					g.log.LogErrf("desktopEntry: %s %v\n", desktopEntryPath, dentErr)
				}
			}

			dlArg := downloadArg{
				ackByteProgress: false,
				path:            exportPath,
				id:              id,
				exportURL:       urlMExt.url,
			}

			err := g.singleDownload(&dlArg)
			if err == nil {
				manifest = append(manifest, exportPath)
			}
			return err
		}(&wg, baseDir, f.Id, exportee)
	}
	wg.Wait()
	return
}

func isLocalFile(f *File) bool {
	// TODO: Better check
	return f != nil && f.Etag == ""
}

func (g *Commands) download(change *Change, exports []string) (err error) {
	if change.Src == nil {
		return fmt.Errorf("tried to download nil change.Src")
	}

	destAbsPath := g.context.AbsPathOf(change.Path)
	if change.Src.BlobAt != "" {
		dlArg := downloadArg{
			path:            destAbsPath,
			id:              change.Src.Id,
			ackByteProgress: true,
		}

		return g.singleDownload(&dlArg)
	}

	// We need to touch the empty file to
	// ensure consistency during a push.
	if runtime.GOOS != OSLinuxKey {
		err = touchFile(destAbsPath)
		if err != nil {
			return err
		}
	} else {
		// For those our Linux kin that need .desktop files
		dirPath := g.opts.ExportsDir
		if dirPath == "" {
			dirPath = filepath.Dir(destAbsPath)
		}

		f := change.Src

		urlMExt := urlMimeTypeExt{
			url:      f.AlternateLink,
			ext:      "",
			mimeType: f.MimeType,
		}

		dirPath = filepath.Join(dirPath, f.Name)
		desktopEntryPath := sepJoin(".", dirPath, DesktopExtension)

		_, dentErr := f.serializeAsDesktopEntry(desktopEntryPath, &urlMExt)
		if dentErr != nil {
			g.log.LogErrf("desktopEntry: %s %v\n", desktopEntryPath, dentErr)
		}
	}

	if len(exports) >= 1 && hasExportLinks(change.Src) {
		exportDirPath := destAbsPath
		if g.opts.ExportsDir != "" {
			exportDirPath = path.Join(g.opts.ExportsDir, change.Src.Name)
		}

		manifest, exportErr := g.export(change.Src, exportDirPath, exports)
		if exportErr == nil {
			for _, exportPath := range manifest {
				g.log.Logf("Exported '%s' to '%s'\n", destAbsPath, exportPath)
			}
		}
		return exportErr
	}
	return
}

func (g *Commands) singleDownload(dlArg *downloadArg) (err error) {
	var fo *os.File
	fo, err = os.Create(dlArg.path)
	if err != nil {
		g.log.LogErrf("create: %s %v\n", dlArg.path, err)
		return
	}

	// close fo on exit and check for its returned error
	defer func() {
		fErr := fo.Close()
		if fErr != nil {
			g.log.LogErrf("fErr", fErr)
			err = fErr
		}
	}()

	var blob io.ReadCloser
	defer func() {
		if blob != nil {
			blob.Close()
		}
	}()

	blob, err = g.rem.Download(dlArg.id, dlArg.exportURL)
	if err != nil {
		return err
	}

	ws := statos.NewWriter(fo)

	go func() {
		commChan := ws.ProgressChan()
		if dlArg.ackByteProgress {
			for n := range commChan {
				g.rem.progressChan <- n
			}
		} else { // Just drain the progress channel
			for _ = range commChan {
				g.rem.progressChan <- 0
			}
		}
	}()

	_, err = io.Copy(ws, blob)

	return
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

// Package quotasbat is a placeholder package for the basic BAT tests.
package quotasbat

import (
	"context"
	"strconv"
	"testing"
	"time"

	"github.com/ciao-project/ciao/bat"
)

const standardTimeout = time.Second * 300

func findQuota(qds []bat.QuotaDetails, name string) *bat.QuotaDetails {
	for i := range qds {
		if qds[i].Name == name {
			return &qds[i]
		}
	}
	return nil
}

func restoreQuotas(ctx context.Context, tenantID string, origQuotas []bat.QuotaDetails, currentQuotas []bat.QuotaDetails) error {
	for i := range currentQuotas {
		qd := findQuota(origQuotas, currentQuotas[i].Name)
		if qd != nil && qd.Value != currentQuotas[i].Value {
			err := bat.UpdateQuota(ctx, "", tenantID, qd.Name, qd.Value)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Test getting and setting quotas
//
// Tests retrieving and setting quotas.
//
// Gets the current quotas, sets several, gets them again checks they've
// changed, restores the original and checks the restoration.
func TestQuotas(t *testing.T) {
	qn := "tenant-vcpu-quota"
	ctx, cancelFunc := context.WithTimeout(context.Background(), standardTimeout)
	defer cancelFunc()

	tenants, err := bat.GetUserTenants(ctx)
	if err != nil {
		t.Fatal(err)
	}

	if len(tenants) < 1 {
		t.Fatal("Expected user to have access to at least one tenant")
	}

	tenantID := tenants[0].ID
	origQuotas, err := bat.ListQuotas(ctx, tenantID, "")
	if err != nil {
		t.Fatal(err)
	}

	err = bat.UpdateQuota(ctx, "", tenantID, qn, "10")
	if err != nil {
		t.Fatal(err)
	}

	defer cleanupQuotas(ctx, t, tenantID, origQuotas)

	updatedQuotas, err := bat.ListQuotas(ctx, tenantID, "")
	if err != nil {
		t.Error(err)
	}

	qd := findQuota(updatedQuotas, qn)
	if qd.Value != "10" {
		t.Error("Quota not expected value")
	}
}

func getUsage(qds []bat.QuotaDetails, name string) int {
	qd := findQuota(qds, name)
	if qd == nil {
		return -1
	}

	value, err := strconv.Atoi(qd.Usage)
	if err != nil {
		return -1
	}

	return value
}

func checkUsage(qds []bat.QuotaDetails, wl bat.Workload, count int) bool {
	expectedMem := wl.Mem * count
	expectedCPU := wl.CPUs * count
	expectedInstances := count

	actualMem := getUsage(qds, "tenant-mem-quota")
	actualCPU := getUsage(qds, "tenant-vcpu-quota")
	actualInstances := getUsage(qds, "tenant-instances-quota")
	return actualMem == expectedMem &&
		actualCPU == expectedCPU &&
		actualInstances == expectedInstances
}

func getContainerWorkload(ctx context.Context, tenantID string) (bat.Workload, error) {
	const testContainerWorkload = "Debian latest test container"
	return bat.GetWorkloadByName(ctx, tenantID, testContainerWorkload)
}

func cleanupQuotas(ctx context.Context, t *testing.T, tenantID string, origQuotas []bat.QuotaDetails) {
	updatedQuotas, err := bat.ListQuotas(ctx, tenantID, "")
	if err != nil {
		t.Error(err)
	}

	err = restoreQuotas(ctx, tenantID, origQuotas, updatedQuotas)
	if err != nil {
		t.Error(err)
	}
}

// Test reporting of instance usage
//
// Starts 3 copies of a workload and checks that the usage matches
// 3 * workload defaults
//
// Gets a workload, launches 3 instances from that workloads, gets the
// quota and usage information and then checks that the usage is as
// expected. Deletes the instances and checks the usage reflects that.
func TestInstanceUsage(t *testing.T) {
	ctx, cancelFunc := context.WithTimeout(context.Background(), standardTimeout)
	defer cancelFunc()
	tenants, err := bat.GetUserTenants(ctx)
	if err != nil {
		t.Fatal(err)
	}

	if len(tenants) < 1 {
		t.Fatal("Expected user to have access to at least one tenant")
	}
	tenantID := tenants[0].ID

	wl, err := getContainerWorkload(ctx, tenantID)
	if err != nil {
		t.Skip()
	}

	instances, err := bat.LaunchInstances(ctx, "", wl.ID, 3)
	if err != nil {
		t.Error(err)
	}

	scheduled, err := bat.WaitForInstancesLaunch(ctx, "", instances, false)
	if err != nil {
		t.Errorf("Instances failed to launch: %v", err)
	}

	updatedQuotas, err := bat.ListQuotas(ctx, tenantID, "")
	if err != nil {
		t.Error(err)
	}

	if !checkUsage(updatedQuotas, wl, 3) {
		t.Error("Usage not recorded correctly")
	}

	for _, i := range scheduled {
		err = bat.DeleteInstanceAndWait(ctx, "", i)
		if err != nil {
			t.Errorf("Failed to delete instances: %v", err)
		}
	}
	updatedQuotas, err = bat.ListQuotas(ctx, tenantID, "")
	if err != nil {
		t.Error(err)
	}
	if !checkUsage(updatedQuotas, wl, 0) {
		t.Error("Usage not recorded correctly")
	}
}

// Workaround for https://github.com/ciao-project/ciao/issues/1203
func launchMultipleInstances(ctx context.Context, t *testing.T, tenantID string, count int) error {
	wl, err := getContainerWorkload(ctx, tenantID)
	if err != nil {
		t.Skip()
	}

	for i := 0; i < count; i++ {
		instances, err := bat.LaunchInstances(ctx, "", wl.ID, 1)
		if err != nil {
			return err
		}

		scheduled, err := bat.WaitForInstancesLaunch(ctx, "", instances, false)
		defer func() {
			_, err := bat.DeleteInstances(ctx, "", scheduled)
			if err != nil {
				t.Errorf("Failed to delete instances: %v", err)
			}
		}()
		if err != nil {
			return err
		}
	}

	return nil
}

// Try launching instances with a quota.
//
// Sets a quota on the number of instances that can be launched and exceeds
// that.
//
// Set a quota of two instances, "tenant-instances-quota", and then try and
// start a set of three instances and check that the launch fails.
func TestInstanceLimited(t *testing.T) {
	ctx, cancelFunc := context.WithTimeout(context.Background(), standardTimeout)
	defer cancelFunc()
	tenants, err := bat.GetUserTenants(ctx)
	if err != nil {
		t.Fatal(err)
	}

	if len(tenants) < 1 {
		t.Fatal("Expected user to have access to at least one tenant")
	}
	tenantID := tenants[0].ID
	origQuotas, err := bat.ListQuotas(ctx, "", "")
	if err != nil {
		t.Fatal(err)
	}

	err = bat.UpdateQuota(ctx, "", tenantID, "tenant-instances-quota", "2")
	if err != nil {
		t.Fatal(err)
	}
	defer cleanupQuotas(ctx, t, tenantID, origQuotas)

	err = launchMultipleInstances(ctx, t, tenantID, 3)
	if err == nil {
		t.Errorf("Expected launch of instances to fail")
	}
}

// Try launching instance that is over quota and check usage
//
// Sets a quota that would deny the instance from being created and
// then check the usage reflects the instance failed to start.
//
// Set an memory quota limit < workload usage. Try and create instance
// of workload and check that instance launch fails and that the usage
// reflects that.
func TestInstanceUsageAfterDenial(t *testing.T) {
	ctx, cancelFunc := context.WithTimeout(context.Background(), standardTimeout)
	defer cancelFunc()
	tenants, err := bat.GetUserTenants(ctx)
	if err != nil {
		t.Fatal(err)
	}

	if len(tenants) < 1 {
		t.Fatal("Expected user to have access to at least one tenant")
	}

	tenantID := tenants[0].ID
	origQuotas, err := bat.ListQuotas(ctx, "", "")
	if err != nil {
		t.Fatal(err)
	}

	wl, err := getContainerWorkload(ctx, tenantID)
	if err != nil {
		t.Skip()
	}

	err = bat.UpdateQuota(ctx, "", tenantID, "tenant-mem-quota", "0")
	if err != nil {
		t.Fatal(err)
	}
	defer cleanupQuotas(ctx, t, tenantID, origQuotas)

	instances, err := bat.LaunchInstances(ctx, "", wl.ID, 1)
	if err == nil {
		t.Error("Expected instance launch to be denied")
	}

	scheduled, err := bat.WaitForInstancesLaunch(ctx, "", instances, false)
	if err != nil {
		t.Error(err)
	}

	updatedQuotas, err := bat.ListQuotas(ctx, tenantID, "")
	if err != nil {
		t.Error(err)
	}

	if !checkUsage(updatedQuotas, wl, 0) {
		t.Error("Usage not recorded correctly")
	}

	_, err = bat.DeleteInstances(ctx, "", scheduled)
	if err != nil {
		t.Error(err)
	}
}
<commit_msg>BAT: Fix a race condition in TestInstanceLimited<commit_after>//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

// Package quotasbat is a placeholder package for the basic BAT tests.
package quotasbat

import (
	"context"
	"strconv"
	"testing"
	"time"

	"github.com/ciao-project/ciao/bat"
)

const standardTimeout = time.Second * 300

func findQuota(qds []bat.QuotaDetails, name string) *bat.QuotaDetails {
	for i := range qds {
		if qds[i].Name == name {
			return &qds[i]
		}
	}
	return nil
}

func restoreQuotas(ctx context.Context, tenantID string, origQuotas []bat.QuotaDetails, currentQuotas []bat.QuotaDetails) error {
	for i := range currentQuotas {
		qd := findQuota(origQuotas, currentQuotas[i].Name)
		if qd != nil && qd.Value != currentQuotas[i].Value {
			err := bat.UpdateQuota(ctx, "", tenantID, qd.Name, qd.Value)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Test getting and setting quotas
//
// Tests retrieving and setting quotas.
//
// Gets the current quotas, sets several, gets them again checks they've
// changed, restores the original and checks the restoration.
func TestQuotas(t *testing.T) {
	qn := "tenant-vcpu-quota"
	ctx, cancelFunc := context.WithTimeout(context.Background(), standardTimeout)
	defer cancelFunc()

	tenants, err := bat.GetUserTenants(ctx)
	if err != nil {
		t.Fatal(err)
	}

	if len(tenants) < 1 {
		t.Fatal("Expected user to have access to at least one tenant")
	}

	tenantID := tenants[0].ID
	origQuotas, err := bat.ListQuotas(ctx, tenantID, "")
	if err != nil {
		t.Fatal(err)
	}

	err = bat.UpdateQuota(ctx, "", tenantID, qn, "10")
	if err != nil {
		t.Fatal(err)
	}

	defer cleanupQuotas(ctx, t, tenantID, origQuotas)

	updatedQuotas, err := bat.ListQuotas(ctx, tenantID, "")
	if err != nil {
		t.Error(err)
	}

	qd := findQuota(updatedQuotas, qn)
	if qd.Value != "10" {
		t.Error("Quota not expected value")
	}
}

func getUsage(qds []bat.QuotaDetails, name string) int {
	qd := findQuota(qds, name)
	if qd == nil {
		return -1
	}

	value, err := strconv.Atoi(qd.Usage)
	if err != nil {
		return -1
	}

	return value
}

func checkUsage(qds []bat.QuotaDetails, wl bat.Workload, count int) bool {
	expectedMem := wl.Mem * count
	expectedCPU := wl.CPUs * count
	expectedInstances := count

	actualMem := getUsage(qds, "tenant-mem-quota")
	actualCPU := getUsage(qds, "tenant-vcpu-quota")
	actualInstances := getUsage(qds, "tenant-instances-quota")
	return actualMem == expectedMem &&
		actualCPU == expectedCPU &&
		actualInstances == expectedInstances
}

func getContainerWorkload(ctx context.Context, tenantID string) (bat.Workload, error) {
	const testContainerWorkload = "Debian latest test container"
	return bat.GetWorkloadByName(ctx, tenantID, testContainerWorkload)
}

func cleanupQuotas(ctx context.Context, t *testing.T, tenantID string, origQuotas []bat.QuotaDetails) {
	updatedQuotas, err := bat.ListQuotas(ctx, tenantID, "")
	if err != nil {
		t.Error(err)
	}

	err = restoreQuotas(ctx, tenantID, origQuotas, updatedQuotas)
	if err != nil {
		t.Error(err)
	}
}

// Test reporting of instance usage
//
// Starts 3 copies of a workload and checks that the usage matches
// 3 * workload defaults
//
// Gets a workload, launches 3 instances from that workloads, gets the
// quota and usage information and then checks that the usage is as
// expected. Deletes the instances and checks the usage reflects that.
func TestInstanceUsage(t *testing.T) {
	ctx, cancelFunc := context.WithTimeout(context.Background(), standardTimeout)
	defer cancelFunc()
	tenants, err := bat.GetUserTenants(ctx)
	if err != nil {
		t.Fatal(err)
	}

	if len(tenants) < 1 {
		t.Fatal("Expected user to have access to at least one tenant")
	}
	tenantID := tenants[0].ID

	wl, err := getContainerWorkload(ctx, tenantID)
	if err != nil {
		t.Skip()
	}

	instances, err := bat.LaunchInstances(ctx, "", wl.ID, 3)
	if err != nil {
		t.Error(err)
	}

	scheduled, err := bat.WaitForInstancesLaunch(ctx, "", instances, false)
	if err != nil {
		t.Errorf("Instances failed to launch: %v", err)
	}

	updatedQuotas, err := bat.ListQuotas(ctx, tenantID, "")
	if err != nil {
		t.Error(err)
	}

	if !checkUsage(updatedQuotas, wl, 3) {
		t.Error("Usage not recorded correctly")
	}

	for _, i := range scheduled {
		err = bat.DeleteInstanceAndWait(ctx, "", i)
		if err != nil {
			t.Errorf("Failed to delete instances: %v", err)
		}
	}
	updatedQuotas, err = bat.ListQuotas(ctx, tenantID, "")
	if err != nil {
		t.Error(err)
	}
	if !checkUsage(updatedQuotas, wl, 0) {
		t.Error("Usage not recorded correctly")
	}
}

// Workaround for https://github.com/ciao-project/ciao/issues/1203
func launchMultipleInstances(ctx context.Context, t *testing.T, tenantID string, count int) error {
	wl, err := getContainerWorkload(ctx, tenantID)
	if err != nil {
		t.Skip()
	}

	for i := 0; i < count; i++ {
		instances, err := bat.LaunchInstances(ctx, "", wl.ID, 1)
		if err != nil {
			return err
		}

		scheduled, err := bat.WaitForInstancesLaunch(ctx, "", instances, false)
		defer func() {
			for _, s := range scheduled {
				err := bat.DeleteInstanceAndWait(ctx, "", s)
				if err != nil {
					t.Errorf("Failed to delete instances: %v", err)
				}
			}
		}()
		if err != nil {
			return err
		}
	}

	return nil
}

// Try launching instances with a quota.
//
// Sets a quota on the number of instances that can be launched and exceeds
// that.
//
// Set a quota of two instances, "tenant-instances-quota", and then try and
// start a set of three instances and check that the launch fails.
func TestInstanceLimited(t *testing.T) {
	ctx, cancelFunc := context.WithTimeout(context.Background(), standardTimeout)
	defer cancelFunc()
	tenants, err := bat.GetUserTenants(ctx)
	if err != nil {
		t.Fatal(err)
	}

	if len(tenants) < 1 {
		t.Fatal("Expected user to have access to at least one tenant")
	}
	tenantID := tenants[0].ID
	origQuotas, err := bat.ListQuotas(ctx, "", "")
	if err != nil {
		t.Fatal(err)
	}

	err = bat.UpdateQuota(ctx, "", tenantID, "tenant-instances-quota", "2")
	if err != nil {
		t.Fatal(err)
	}
	defer cleanupQuotas(ctx, t, tenantID, origQuotas)

	err = launchMultipleInstances(ctx, t, tenantID, 3)
	if err == nil {
		t.Errorf("Expected launch of instances to fail")
	}
}

// Try launching instance that is over quota and check usage
//
// Sets a quota that would deny the instance from being created and
// then check the usage reflects the instance failed to start.
//
// Set an memory quota limit < workload usage. Try and create instance
// of workload and check that instance launch fails and that the usage
// reflects that.
func TestInstanceUsageAfterDenial(t *testing.T) {
	ctx, cancelFunc := context.WithTimeout(context.Background(), standardTimeout)
	defer cancelFunc()
	tenants, err := bat.GetUserTenants(ctx)
	if err != nil {
		t.Fatal(err)
	}

	if len(tenants) < 1 {
		t.Fatal("Expected user to have access to at least one tenant")
	}

	tenantID := tenants[0].ID
	origQuotas, err := bat.ListQuotas(ctx, "", "")
	if err != nil {
		t.Fatal(err)
	}

	wl, err := getContainerWorkload(ctx, tenantID)
	if err != nil {
		t.Skip()
	}

	err = bat.UpdateQuota(ctx, "", tenantID, "tenant-mem-quota", "0")
	if err != nil {
		t.Fatal(err)
	}
	defer cleanupQuotas(ctx, t, tenantID, origQuotas)

	instances, err := bat.LaunchInstances(ctx, "", wl.ID, 1)
	if err == nil {
		t.Error("Expected instance launch to be denied")
	}

	scheduled, err := bat.WaitForInstancesLaunch(ctx, "", instances, false)
	if err != nil {
		t.Error(err)
	}

	updatedQuotas, err := bat.ListQuotas(ctx, tenantID, "")
	if err != nil {
		t.Error(err)
	}

	if !checkUsage(updatedQuotas, wl, 0) {
		t.Error("Usage not recorded correctly")
	}

	_, err = bat.DeleteInstances(ctx, "", scheduled)
	if err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package mount // import "github.com/docker/docker/pkg/mount"

/*
#include <sys/param.h>
#include <sys/ucred.h>
#include <sys/mount.h>
*/
import "C"

import (
	"fmt"
	"reflect"
	"unsafe"
)

//parseMountTable returns information about mounted filesystems
func parseMountTable(filter FilterFunc) ([]*Info, error) {
	var rawEntries *C.struct_statfs

	count := int(C.getmntinfo(&rawEntries, C.MNT_WAIT))
	if count == 0 {
		return nil, fmt.Errorf("Failed to call getmntinfo")
	}

	var entries []C.struct_statfs
	header := (*reflect.SliceHeader)(unsafe.Pointer(&entries))
	header.Cap = count
	header.Len = count
	header.Data = uintptr(unsafe.Pointer(rawEntries))

	var out []*Info
	for _, entry := range entries {
		var mountinfo Info
		var skip, stop bool
		mountinfo.Mountpoint = C.GoString(&entry.f_mntonname[0])

		if filter != nil {
			// filter out entries we're not interested in
			skip, stop = filter(&mountinfo)
			if skip {
				continue
			}
		}

		mountinfo.Source = C.GoString(&entry.f_mntfromname[0])
		mountinfo.Fstype = C.GoString(&entry.f_fstypename[0])

		out = append(out, &mountinfo)
		if stop {
			break
		}
	}
	return out, nil
}
<commit_msg>Fix to gofmt/goimports errors<commit_after>package mount // import "github.com/docker/docker/pkg/mount"

/*
#include <sys/param.h>
#include <sys/ucred.h>
#include <sys/mount.h>
*/
import "C"

import (
	"fmt"
	"reflect"
	"unsafe"
)

//parseMountTable returns information about mounted filesystems
func parseMountTable(filter FilterFunc) ([]*Info, error) {
	var rawEntries *C.struct_statfs

	count := int(C.getmntinfo(&rawEntries, C.MNT_WAIT))
	if count == 0 {
		return nil, fmt.Errorf("Failed to call getmntinfo")
	}

	var entries []C.struct_statfs
	header := (*reflect.SliceHeader)(unsafe.Pointer(&entries))
	header.Cap = count
	header.Len = count
	header.Data = uintptr(unsafe.Pointer(rawEntries))

	var out []*Info
	for _, entry := range entries {
		var mountinfo Info
		var skip, stop bool
		mountinfo.Mountpoint = C.GoString(&entry.f_mntonname[0])

		if filter != nil {
			// filter out entries we're not interested in
			skip, stop = filter(&mountinfo)
			if skip {
				continue
			}
		}

		mountinfo.Source = C.GoString(&entry.f_mntfromname[0])
		mountinfo.Fstype = C.GoString(&entry.f_fstypename[0])

		out = append(out, &mountinfo)
		if stop {
			break
		}
	}
	return out, nil
}
<|endoftext|>