<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"internal/testlog"
	"runtime"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

// Process stores the information about a process created by StartProcess.
type Process struct {
	Pid    int
	handle uintptr      // handle is accessed atomically on Windows
	isdone uint32       // process has been successfully waited on, non zero if true
	sigMu  sync.RWMutex // avoid race between wait and signal
}

func newProcess(pid int, handle uintptr) *Process {
	p := &Process{Pid: pid, handle: handle}
	runtime.SetFinalizer(p, (*Process).Release)
	return p
}

func (p *Process) setDone() {
	atomic.StoreUint32(&p.isdone, 1)
}

func (p *Process) done() bool {
	return atomic.LoadUint32(&p.isdone) > 0
}

// ProcAttr holds the attributes that will be applied to a new process
// started by StartProcess.
type ProcAttr struct {
	// If Dir is non-empty, the child changes into the directory before
	// creating the process.
	Dir string
	// If Env is non-nil, it gives the environment variables for the
	// new process in the form returned by Environ.
	// If it is nil, the result of Environ will be used.
	Env []string
	// Files specifies the open files inherited by the new process. The
	// first three entries correspond to standard input, standard output, and
	// standard error. An implementation may support additional entries,
	// depending on the underlying operating system. A nil entry corresponds
	// to that file being closed when the process starts.
	Files []*File

	// Operating system-specific process creation attributes.
	// Note that setting this field means that your program
	// may not execute properly or even compile on some
	// operating systems.
	Sys *syscall.SysProcAttr
}

// A Signal represents an operating system signal.
// The usual underlying implementation is operating system-dependent:
// on Unix it is syscall.Signal.
type Signal interface {
	String() string
	Signal() // to distinguish from other Stringers
}

// Getpid returns the process id of the caller.
func Getpid() int { return syscall.Getpid() }

// Getppid returns the process id of the caller's parent.
func Getppid() int { return syscall.Getppid() }

// FindProcess looks for a running process by its pid.
//
// The Process it returns can be used to obtain information
// about the underlying operating system process.
//
// On Unix systems, FindProcess always succeeds and returns a Process
// for the given pid, regardless of whether the process exists.
func FindProcess(pid int) (*Process, error) {
	return findProcess(pid)
}

// StartProcess starts a new process with the program, arguments and attributes
// specified by name, argv and attr.
//
// StartProcess is a low-level interface. The os/exec package provides
// higher-level interfaces.
//
// If there is an error, it will be of type *PathError.
func StartProcess(name string, argv []string, attr *ProcAttr) (*Process, error) {
	testlog.Open(name)
	return startProcess(name, argv, attr)
}

// Release releases any resources associated with the Process p,
// rendering it unusable in the future.
// Release only needs to be called if Wait is not.
func (p *Process) Release() error {
	return p.release()
}

// Kill causes the Process to exit immediately.
func (p *Process) Kill() error {
	return p.kill()
}

// Wait waits for the Process to exit, and then returns a
// ProcessState describing its status and an error, if any.
// Wait releases any resources associated with the Process.
// On most operating systems, the Process must be a child
// of the current process or an error will be returned.
func (p *Process) Wait() (*ProcessState, error) {
	return p.wait()
}

// Signal sends a signal to the Process.
// Sending Interrupt on Windows is not implemented.
func (p *Process) Signal(sig Signal) error {
	return p.signal(sig)
}

// UserTime returns the user CPU time of the exited process and its children.
func (p *ProcessState) UserTime() time.Duration {
	return p.userTime()
}

// SystemTime returns the system CPU time of the exited process and its children.
func (p *ProcessState) SystemTime() time.Duration {
	return p.systemTime()
}

// Exited reports whether the program has exited.
func (p *ProcessState) Exited() bool {
	return p.exited()
}

// Success reports whether the program exited successfully,
// such as with exit status 0 on Unix.
func (p *ProcessState) Success() bool {
	return p.success()
}

// Sys returns system-dependent exit information about
// the process. Convert it to the appropriate underlying
// type, such as syscall.WaitStatus on Unix, to access its contents.
func (p *ProcessState) Sys() interface{} {
	return p.sys()
}

// SysUsage returns system-dependent resource usage information about
// the exited process. Convert it to the appropriate underlying
// type, such as *syscall.Rusage on Unix, to access its contents.
// (On Unix, *syscall.Rusage matches struct rusage as defined in the
// getrusage(2) manual page.)
func (p *ProcessState) SysUsage() interface{} {
	return p.sysUsage()
}
<commit_msg>os: document that StartProcess's argv starts with the binary name<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"internal/testlog"
	"runtime"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

// Process stores the information about a process created by StartProcess.
type Process struct {
	Pid    int
	handle uintptr      // handle is accessed atomically on Windows
	isdone uint32       // process has been successfully waited on, non zero if true
	sigMu  sync.RWMutex // avoid race between wait and signal
}

func newProcess(pid int, handle uintptr) *Process {
	p := &Process{Pid: pid, handle: handle}
	runtime.SetFinalizer(p, (*Process).Release)
	return p
}

func (p *Process) setDone() {
	atomic.StoreUint32(&p.isdone, 1)
}

func (p *Process) done() bool {
	return atomic.LoadUint32(&p.isdone) > 0
}

// ProcAttr holds the attributes that will be applied to a new process
// started by StartProcess.
type ProcAttr struct {
	// If Dir is non-empty, the child changes into the directory before
	// creating the process.
	Dir string
	// If Env is non-nil, it gives the environment variables for the
	// new process in the form returned by Environ.
	// If it is nil, the result of Environ will be used.
	Env []string
	// Files specifies the open files inherited by the new process. The
	// first three entries correspond to standard input, standard output, and
	// standard error. An implementation may support additional entries,
	// depending on the underlying operating system. A nil entry corresponds
	// to that file being closed when the process starts.
	Files []*File

	// Operating system-specific process creation attributes.
	// Note that setting this field means that your program
	// may not execute properly or even compile on some
	// operating systems.
	Sys *syscall.SysProcAttr
}

// A Signal represents an operating system signal.
// The usual underlying implementation is operating system-dependent:
// on Unix it is syscall.Signal.
type Signal interface {
	String() string
	Signal() // to distinguish from other Stringers
}

// Getpid returns the process id of the caller.
func Getpid() int { return syscall.Getpid() }

// Getppid returns the process id of the caller's parent.
func Getppid() int { return syscall.Getppid() }

// FindProcess looks for a running process by its pid.
//
// The Process it returns can be used to obtain information
// about the underlying operating system process.
//
// On Unix systems, FindProcess always succeeds and returns a Process
// for the given pid, regardless of whether the process exists.
func FindProcess(pid int) (*Process, error) {
	return findProcess(pid)
}

// StartProcess starts a new process with the program, arguments and attributes
// specified by name, argv and attr. The argv slice will become os.Args in the
// new process, and normally starts with the program name.
//
// StartProcess is a low-level interface. The os/exec package provides
// higher-level interfaces.
//
// If there is an error, it will be of type *PathError.
func StartProcess(name string, argv []string, attr *ProcAttr) (*Process, error) {
	testlog.Open(name)
	return startProcess(name, argv, attr)
}

// Release releases any resources associated with the Process p,
// rendering it unusable in the future.
// Release only needs to be called if Wait is not.
func (p *Process) Release() error {
	return p.release()
}

// Kill causes the Process to exit immediately.
func (p *Process) Kill() error {
	return p.kill()
}

// Wait waits for the Process to exit, and then returns a
// ProcessState describing its status and an error, if any.
// Wait releases any resources associated with the Process.
// On most operating systems, the Process must be a child
// of the current process or an error will be returned.
func (p *Process) Wait() (*ProcessState, error) {
	return p.wait()
}

// Signal sends a signal to the Process.
// Sending Interrupt on Windows is not implemented.
func (p *Process) Signal(sig Signal) error {
	return p.signal(sig)
}

// UserTime returns the user CPU time of the exited process and its children.
func (p *ProcessState) UserTime() time.Duration {
	return p.userTime()
}

// SystemTime returns the system CPU time of the exited process and its children.
func (p *ProcessState) SystemTime() time.Duration {
	return p.systemTime()
}

// Exited reports whether the program has exited.
func (p *ProcessState) Exited() bool {
	return p.exited()
}

// Success reports whether the program exited successfully,
// such as with exit status 0 on Unix.
func (p *ProcessState) Success() bool {
	return p.success()
}

// Sys returns system-dependent exit information about
// the process. Convert it to the appropriate underlying
// type, such as syscall.WaitStatus on Unix, to access its contents.
func (p *ProcessState) Sys() interface{} {
	return p.sys()
}

// SysUsage returns system-dependent resource usage information about
// the exited process. Convert it to the appropriate underlying
// type, such as *syscall.Rusage on Unix, to access its contents.
// (On Unix, *syscall.Rusage matches struct rusage as defined in the
// getrusage(2) manual page.)
func (p *ProcessState) SysUsage() interface{} {
	return p.sysUsage()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
)

type Response struct {
	Text        string        `json:"text"`
	Attachments []Attachments `json:"attachments"`
}

type Attachments struct {
	Title      string `json:"title"`
	Title_link string `json:"title_link"`
	Thumb_url  string `json:"thumb_url"`
}

type jsonData struct {
	Status  int64
	Message string
	Data    []Data
}

type Data struct {
	Id      string
	Caption string
	Images  struct {
		Small  string
		Cover  string
		Normal string
		Large  string
	}
	Media interface{}
	Link  string
	Votes struct {
		Count int64
	}
	Comments struct {
		Count int64
	}
}

func main() {
	http.HandleFunc("/", handler)
	fmt.Println("listening...")
	err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
	if err != nil {
		panic(err)
	}
}

func handler(w http.ResponseWriter, r *http.Request) {

	url := "http://infinigag.k3min.eu"
	// Read the Request Parameter "command"
	command := r.FormValue("command")

	if command == "/9gag" {
		// Read the Request Parameter "text"
		text := r.FormValue("text")
		s := strings.Split(text, " ")
		var section string
		var subsection string
		if len(s) == 2 {
			section = s[0]
			subsection = s[1]
		} else if len(s) == 1 {
			section = s[0]
		}
		switch section {
		case "":
		case "cute":
			url += "/cute"
		case "cosplay":
			url += "/cosplay"
		case "design":
			url += "/design"
		case "food":
			url += "/food"
		case "funny":
			url += "/funny"
		case "geeky":
			url += "/geeky"
		case "gif":
			url += "/gif"
		case "girl":
			url += "/girl"
		case "meme":
			url += "/meme"
		case "nsfw":
			url += "/nsfw"
		case "timely":
			url += "/timely"
		case "wtf":
			url += "/wtf"
		default:
			fmt.Fprint(w, "I do not understand your command.")
			return
		}

		switch subsection {
		case "":
		case "fresh":
			url += "/fresh"
		case "trending":
			url += "trending"
		default:
			fmt.Fprint(w, "I do not understand your command.")
			return
		}

		fmt.Println(url)
		r, err := http.Get(url)
		if err != nil {
			fmt.Println("Error requesting data")
			return
		}
		body, err := ioutil.ReadAll(r.Body)
		if err != nil {
			fmt.Println("Error while opening file", err)
			return
		}
		x := new(jsonData)
		err = json.Unmarshal(body, &x)
		if err != nil {
			fmt.Println("Error while parsing file", err)
			return
		}
		jsonResp(w, x)
	} else {
		fmt.Fprint(w, "I do not understand your command.")
	}
}

func jsonResp(w http.ResponseWriter, x *jsonData) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")

	attachments := make([]Attachments, len(x.Data))
	for i := 0; i < len(x.Data); i++ {
		attachments[i] = Attachments{
			Title:      x.Data[i].Caption,
			Title_link: x.Data[i].Link,
			Thumb_url:  x.Data[i].Images.Small,
		}
	}

	resp := Response{
		Text:        "lorem ipsum",
		Attachments: attachments,
	}

	r, err := json.Marshal(resp)
	if err != nil {
		fmt.Println("Couldn't marshal hook response:", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	w.Write(r)
}
<commit_msg>Fix subsection to 'hot' from 'trending'.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
)

type Response struct {
	Text        string        `json:"text"`
	Attachments []Attachments `json:"attachments"`
}

type Attachments struct {
	Title      string `json:"title"`
	Title_link string `json:"title_link"`
	Thumb_url  string `json:"thumb_url"`
}

type jsonData struct {
	Status  int64
	Message string
	Data    []Data
}

type Data struct {
	Id      string
	Caption string
	Images  struct {
		Small  string
		Cover  string
		Normal string
		Large  string
	}
	Media interface{}
	Link  string
	Votes struct {
		Count int64
	}
	Comments struct {
		Count int64
	}
}

func main() {
	http.HandleFunc("/", handler)
	fmt.Println("listening...")
	err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
	if err != nil {
		panic(err)
	}
}

func handler(w http.ResponseWriter, r *http.Request) {

	url := "http://infinigag.k3min.eu"
	// Read the Request Parameter "command"
	command := r.FormValue("command")

	if command == "/9gag" {
		// Read the Request Parameter "text"
		text := r.FormValue("text")
		s := strings.Split(text, " ")
		var section string
		var subsection string
		if len(s) == 2 {
			section = s[0]
			subsection = s[1]
		} else if len(s) == 1 {
			section = s[0]
		}
		switch section {
		case "":
		case "cute":
			url += "/cute"
		case "cosplay":
			url += "/cosplay"
		case "design":
			url += "/design"
		case "food":
			url += "/food"
		case "funny":
			url += "/funny"
		case "geeky":
			url += "/geeky"
		case "gif":
			url += "/gif"
		case "girl":
			url += "/girl"
		case "meme":
			url += "/meme"
		case "nsfw":
			url += "/nsfw"
		case "timely":
			url += "/timely"
		case "wtf":
			url += "/wtf"
		default:
			fmt.Fprint(w, "I do not understand your command.")
			return
		}

		switch subsection {
		case "":
		case "fresh":
			url += "/fresh"
		case "hot":
			url += "/hot"
		default:
			fmt.Fprint(w, "I do not understand your command.")
			return
		}

		fmt.Println(url)
		r, err := http.Get(url)
		if err != nil {
			fmt.Println("Error requesting data")
			return
		}
		body, err := ioutil.ReadAll(r.Body)
		if err != nil {
			fmt.Println("Error while opening file", err)
			return
		}
		x := new(jsonData)
		err = json.Unmarshal(body, &x)
		if err != nil {
			fmt.Println("Error while parsing file", err)
			return
		}
		jsonResp(w, x)
	} else {
		fmt.Fprint(w, "I do not understand your command.")
	}
}

func jsonResp(w http.ResponseWriter, x *jsonData) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")

	attachments := make([]Attachments, len(x.Data))
	for i := 0; i < len(x.Data); i++ {
		attachments[i] = Attachments{
			Title:      x.Data[i].Caption,
			Title_link: x.Data[i].Link,
			Thumb_url:  x.Data[i].Images.Small,
		}
	}

	resp := Response{
		Text:        "lorem ipsum",
		Attachments: attachments,
	}

	r, err := json.Marshal(resp)
	if err != nil {
		fmt.Println("Couldn't marshal hook response:", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	w.Write(r)
}
<|endoftext|>


<commit_before>package main

import (
	"container/heap"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/tj/go-dropbox"
	"golang.org/x/text/unicode/norm"
	"math"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

/*
TODO
- Performance improvements:
	- Test if buffered channels improve performance in the parallel local file processing
	- Profile to find other bottlenecks?
	- Could printing progress for each local file result slow things down? (When processing lots of small files)
- Print I/O usage? i.e. how many MB/s are we processing
- Clean up output formatting
- Ignore more file names in skipLocalFile - see https://www.dropbox.com/help/syncing-uploads/files-not-syncing
- Do a real retry + backoff for Dropbox API errors (do we have access to the Retry-After header?)
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileError records a local file that could not be read due to an error
type FileError struct {
	Path  string
	Error error
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Errored         []*FileError
	Matches         int
	Misses          int
}

type progressType int

const (
	remoteProgress progressType = iota
	localProgress
	errorProgress
)

type scanProgressUpdate struct {
	Type  progressType
	Count int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose          bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot       string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:"/"`
		LocalRoot        string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		CheckContentHash bool   `long:"check" description:"Check content hash of local files"`
		WorkerCount      int    `short:"w" long:"workers" description:"Number of worker threads to use (defaults to 8)" default:"8"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	if opts.RemoteRoot == "" {
		opts.RemoteRoot = "/"
	}
	if opts.RemoteRoot[0] != '/' {
		opts.RemoteRoot = "/" + opts.RemoteRoot
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", opts.RemoteRoot, localRoot)
	if opts.CheckContentHash {
		fmt.Println("Checking content hashes.")
	}
	fmt.Println("")

	progressChan := make(chan *scanProgressUpdate)
	var wg sync.WaitGroup
	wg.Add(2)

	var dropboxManifest *FileHeap
	var dropboxErr error
	go func() {
		dropboxManifest, dropboxErr = getDropboxManifest(progressChan, dbxClient, opts.RemoteRoot)
		wg.Done()
	}()

	var localManifest *FileHeap
	var errored []*FileError
	var localErr error
	go func() {
		localManifest, errored, localErr = getLocalManifest(progressChan, localRoot, opts.CheckContentHash, opts.WorkerCount)
		wg.Done()
	}()

	go func() {
		remoteCount := 0
		localCount := 0
		errorCount := 0
		for update := range progressChan {
			switch update.Type {
			case remoteProgress:
				remoteCount = update.Count
			case localProgress:
				localCount = update.Count
			case errorProgress:
				errorCount = update.Count
			}

			if opts.Verbose {
				fmt.Fprintf(os.Stderr, "Scanning: %d (remote) %d (local) %d (errored)\r", remoteCount, localCount, errorCount)
			}
		}
		fmt.Fprintf(os.Stderr, "\n")
	}()

	// wait until remote and local scans are complete, then close progress reporting channel
	wg.Wait()
	close(progressChan)
	fmt.Printf("\nGenerated manifests for %d remote files, %d local files, with %d local errors\n\n", dropboxManifest.Len(), localManifest.Len(), len(errored))

	// check for fatal errors
	if dropboxErr != nil {
		panic(dropboxErr)
	}
	if localErr != nil {
		panic(localErr)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest, errored)

	fmt.Println("")

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")

	fmt.Printf("Errored: %d\n\n", len(manifestComparison.Errored))
	if len(manifestComparison.Errored) > 0 {
		for _, rec := range manifestComparison.Errored {
			fmt.Printf("%s: %s\n", rec.Path, rec.Error)
		}
		if len(manifestComparison.Errored) > 0 {
			fmt.Print("\n\n")
		}
	}

	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func getDropboxManifest(progressChan chan<- *scanProgressUpdate, dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			if strings.HasPrefix(err.Error(), "too_many_requests") {
				fmt.Fprintf(os.Stderr, "\n[%s] Dropbox returned too many requests error, sleeping 60 seconds\n", time.Now().Format("15:04:05"))
				// fmt.Fprintf(os.Stderr, "Error: %v\n", err)
				// fmt.Fprintf(os.Stderr, "Response: %v\n", resp)
				time.Sleep(60 * time.Second)
				continue
			}
			return
		}
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathLower)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore

		progressChan <- &scanProgressUpdate{Type: remoteProgress, Count: manifest.Len()}
	}

	return
}

func getLocalManifest(progressChan chan<- *scanProgressUpdate, localRoot string, contentHash bool, workerCount int) (manifest *FileHeap, errored []*FileError, err error) {
	localRootLowercase := strings.ToLower(localRoot)
	manifest = &FileHeap{}
	heap.Init(manifest)
	if workerCount <= 0 {
		workerCount = int(math.Max(1, float64(runtime.NumCPU())))
	}
	processChan := make(chan string)
	resultChan := make(chan *File)
	errorChan := make(chan *FileError)
	var wg sync.WaitGroup

	for i := 0; i < workerCount; i++ {
		// spin up workers
		wg.Add(1)
		go handleLocalFile(localRootLowercase, contentHash, processChan, resultChan, errorChan, &wg)
	}

	// walk in separate goroutine so that sends to errorChan don't block
	go func() {
		filepath.Walk(localRoot, func(entryPath string, info os.FileInfo, err error) error {
			if err != nil {
				errorChan <- &FileError{Path: entryPath, Error: err}
				return nil
			}

			if info.Mode().IsDir() && skipLocalDir(entryPath) {
				return filepath.SkipDir
			}

			if info.Mode().IsRegular() && !skipLocalFile(entryPath) {
				processChan <- entryPath
			}

			return nil
		})

		close(processChan)
	}()

	// Once processing goroutines are done, close result and error channels to indicate no more results streaming in
	go func() {
		wg.Wait()
		close(resultChan)
		close(errorChan)
	}()

	for {
		select {
		case result, ok := <-resultChan:
			if ok {
				heap.Push(manifest, result)
				progressChan <- &scanProgressUpdate{Type: localProgress, Count: manifest.Len()}
			} else {
				resultChan = nil
			}

		case e, ok := <-errorChan:
			if ok {
				errored = append(errored, e)
				progressChan <- &scanProgressUpdate{Type: errorProgress, Count: len(errored)}
			} else {
				errorChan = nil
			}
		}

		if resultChan == nil && errorChan == nil {
			break
		}
	}

	return
}

// fill in args etc
func handleLocalFile(localRootLowercase string, contentHash bool, processChan <-chan string, resultChan chan<- *File, errorChan chan<- *FileError, wg *sync.WaitGroup) {
	for entryPath := range processChan {

		relPath, err := normalizePath(localRootLowercase, strings.ToLower(entryPath))
		if err != nil {
			errorChan <- &FileError{Path: entryPath, Error: err}
			continue
		}

		hash := ""
		if contentHash {
			hash, err = dropbox.FileContentHash(entryPath)
			if err != nil {
				errorChan <- &FileError{Path: relPath, Error: err}
				continue
			}
		}

		resultChan <- &File{
			Path:        relPath,
			ContentHash: hash,
		}
	}
	wg.Done()
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}
	if relPath[0:3] == "../" {
		// try lowercase root instead
		relPath, err = filepath.Rel(strings.ToLower(root), entryPath)
		if err != nil {
			return "", err
		}
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func skipLocalFile(path string) bool {
	if filepath.Base(path) == ".DS_Store" {
		return true
	}
	return false
}

func skipLocalDir(path string) bool {
	if filepath.Base(path) == "@eaDir" {
		return true
	}
	return false
}

func compareManifests(remoteManifest, localManifest *FileHeap, errored []*FileError) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{Errored: errored}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else {
			// this must mean that remote.Path == local.Path
			if compareFileContents(remote, local) {
				comparison.Matches++
			} else {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<commit_msg>Add todo<commit_after>package main

import (
	"container/heap"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/tj/go-dropbox"
	"golang.org/x/text/unicode/norm"
	"math"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

/*
TODO
- Performance improvements:
	- Test if buffered channels improve performance in the parallel local file processing
	- Profile to find other bottlenecks?
	- Could printing progress for each local file result slow things down? (When processing lots of small files)
- Print I/O usage? i.e. how many MB/s are we processing
- Clean up output formatting
- Ignore more file names in skipLocalFile - see https://www.dropbox.com/help/syncing-uploads/files-not-syncing
- Do a real retry + backoff for Dropbox API errors (do we have access to the Retry-After header?)
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileError records a local file that could not be read due to an error
type FileError struct {
	Path  string
	Error error
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Errored         []*FileError
	Matches         int
	Misses          int
}

type progressType int

const (
	remoteProgress progressType = iota
	localProgress
	errorProgress
)

type scanProgressUpdate struct {
	Type  progressType
	Count int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose          bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot       string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:"/"`
		LocalRoot        string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		CheckContentHash bool   `long:"check" description:"Check content hash of local files"`
		WorkerCount      int    `short:"w" long:"workers" description:"Number of worker threads to use (defaults to 8)" default:"8"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	if opts.RemoteRoot == "" {
		opts.RemoteRoot = "/"
	}
	if opts.RemoteRoot[0] != '/' {
		opts.RemoteRoot = "/" + opts.RemoteRoot
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", opts.RemoteRoot, localRoot)
	if opts.CheckContentHash {
		fmt.Println("Checking content hashes.")
	}
	fmt.Println("")

	progressChan := make(chan *scanProgressUpdate)
	var wg sync.WaitGroup
	wg.Add(2)

	var dropboxManifest *FileHeap
	var dropboxErr error
	go func() {
		dropboxManifest, dropboxErr = getDropboxManifest(progressChan, dbxClient, opts.RemoteRoot)
		wg.Done()
	}()

	var localManifest *FileHeap
	var errored []*FileError
	var localErr error
	go func() {
		localManifest, errored, localErr = getLocalManifest(progressChan, localRoot, opts.CheckContentHash, opts.WorkerCount)
		wg.Done()
	}()

	go func() {
		remoteCount := 0
		localCount := 0
		errorCount := 0
		for update := range progressChan {
			switch update.Type {
			case remoteProgress:
				remoteCount = update.Count
			case localProgress:
				localCount = update.Count
			case errorProgress:
				errorCount = update.Count
			}

			if opts.Verbose {
				fmt.Fprintf(os.Stderr, "Scanning: %d (remote) %d (local) %d (errored)\r", remoteCount, localCount, errorCount)
			}
		}
		fmt.Fprintf(os.Stderr, "\n")
	}()

	// wait until remote and local scans are complete, then close progress reporting channel
	wg.Wait()
	close(progressChan)
	fmt.Printf("\nGenerated manifests for %d remote files, %d local files, with %d local errors\n\n", dropboxManifest.Len(), localManifest.Len(), len(errored))

	// check for fatal errors
	if dropboxErr != nil {
		panic(dropboxErr)
	}
	if localErr != nil {
		panic(localErr)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest, errored)

	fmt.Println("")

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")

	fmt.Printf("Errored: %d\n\n", len(manifestComparison.Errored))
	if len(manifestComparison.Errored) > 0 {
		for _, rec := range manifestComparison.Errored {
			fmt.Printf("%s: %s\n", rec.Path, rec.Error)
		}
		if len(manifestComparison.Errored) > 0 {
			fmt.Print("\n\n")
		}
	}

	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func getDropboxManifest(progressChan chan<- *scanProgressUpdate, dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			// TODO: submit feature request for dropbox client to expose retry_after param
			if strings.HasPrefix(err.Error(), "too_many_requests") {
				fmt.Fprintf(os.Stderr, "\n[%s] Dropbox returned too many requests error, sleeping 60 seconds\n", time.Now().Format("15:04:05"))
				// fmt.Fprintf(os.Stderr, "Error: %v\n", err)
				// fmt.Fprintf(os.Stderr, "Response: %v\n", resp)
				time.Sleep(60 * time.Second)
				continue
			}
			return
		}
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathLower)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore

		progressChan <- &scanProgressUpdate{Type: remoteProgress, Count: manifest.Len()}
	}

	return
}

func getLocalManifest(progressChan chan<- *scanProgressUpdate, localRoot string, contentHash bool, workerCount int) (manifest *FileHeap, errored []*FileError, err error) {
	localRootLowercase := strings.ToLower(localRoot)
	manifest = &FileHeap{}
	heap.Init(manifest)
	if workerCount <= 0 {
		workerCount = int(math.Max(1, float64(runtime.NumCPU())))
	}
	processChan := make(chan string)
	resultChan := make(chan *File)
	errorChan := make(chan *FileError)
	var wg sync.WaitGroup

	for i := 0; i < workerCount; i++ {
		// spin up workers
		wg.Add(1)
		go handleLocalFile(localRootLowercase, contentHash, processChan, resultChan, errorChan, &wg)
	}

	// walk in separate goroutine so that sends to errorChan don't block
	go func() {
		filepath.Walk(localRoot, func(entryPath string, info os.FileInfo, err error) error {
			if err != nil {
				errorChan <- &FileError{Path: entryPath, Error: err}
				return nil
			}

			if info.Mode().IsDir() && skipLocalDir(entryPath) {
				return filepath.SkipDir
			}

			if info.Mode().IsRegular() && !skipLocalFile(entryPath) {
				processChan <- entryPath
			}

			return nil
		})

		close(processChan)
	}()

	// Once processing goroutines are done, close result and error channels to indicate no more results streaming in
	go func() {
		wg.Wait()
		close(resultChan)
		close(errorChan)
	}()

	for {
		select {
		case result, ok := <-resultChan:
			if ok {
				heap.Push(manifest, result)
				progressChan <- &scanProgressUpdate{Type: localProgress, Count: manifest.Len()}
			} else {
				resultChan = nil
			}

		case e, ok := <-errorChan:
			if ok {
				errored = append(errored, e)
				progressChan <- &scanProgressUpdate{Type: errorProgress, Count: len(errored)}
			} else {
				errorChan = nil
			}
		}

		if resultChan == nil && errorChan == nil {
			break
		}
	}

	return
}

// fill in args etc
func handleLocalFile(localRootLowercase string, contentHash bool, processChan <-chan string, resultChan chan<- *File, errorChan chan<- *FileError, wg *sync.WaitGroup) {
	for entryPath := range processChan {

		relPath, err := normalizePath(localRootLowercase, strings.ToLower(entryPath))
		if err != nil {
			errorChan <- &FileError{Path: entryPath, Error: err}
			continue
		}

		hash := ""
		if contentHash {
			hash, err = dropbox.FileContentHash(entryPath)
			if err != nil {
				errorChan <- &FileError{Path: relPath, Error: err}
				continue
			}
		}

		resultChan <- &File{
			Path:        relPath,
			ContentHash: hash,
		}
	}
	wg.Done()
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}
	if relPath[0:3] == "../" {
		// try lowercase root instead
		relPath, err = filepath.Rel(strings.ToLower(root), entryPath)
		if err != nil {
			return "", err
		}
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func skipLocalFile(path string) bool {
	if filepath.Base(path) == ".DS_Store" {
		return true
	}
	return false
}

func skipLocalDir(path string) bool {
	if filepath.Base(path) == "@eaDir" {
		return true
	}
	return false
}

func compareManifests(remoteManifest, localManifest *FileHeap, errored []*FileError) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{Errored: errored}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else {
			// this must mean that remote.Path == local.Path
			if compareFileContents(remote, local) {
				comparison.Matches++
			} else {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<|endoftext|>


<commit_before>package clusterconfigs

import (
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"
	"github.com/radanalyticsio/oshinko-rest/models"
)

var defaultConfig models.NewClusterConfig = models.NewClusterConfig{
								MasterCount: 1,
	                                                        WorkerCount: 1,
								Name: "",
								SparkMasterConfig: "",
								SparkWorkerConfig: ""}
var configpath, globpath string

const Defaultname = "default"
const failOnMissing = true
const allowMissing = false
const DefaultConfigPath = "/etc/oshinko-cluster-configs/"

const MasterCountMustBeOne = "Cluster configuration must have a masterCount of 1"
const WorkerCountMustBeAtLeastOne = "Cluster configuration may not have a workerCount less than 1"
const NamedConfigDoesNotExist = "Named config '%s' does not exist"
const ErrorWhileProcessing = "Error while processing %s: %s"

func init() {
	SetConfigPath(DefaultConfigPath)
}

// This function is meant to supoprt testability
func SetConfigPath(dir string) {
	configpath = dir
	globpath = path.Join(configpath, "%s\\.*")
}

// This function is meant to support testability
func GetConfigPath() string {
	return configpath
}

// This function is meant to support testability
func GetDefaultConfig() models.NewClusterConfig {
	return defaultConfig
}

func assignConfig(res *models.NewClusterConfig, src models.NewClusterConfig) {
	if src.MasterCount != 0 {
		res.MasterCount = src.MasterCount
	}
	if src.WorkerCount != 0 {
		res.WorkerCount = src.WorkerCount
	}
	sparkMasterConfig, _ := src.SparkMasterConfig.(string)
	if sparkMasterConfig != "" {
		res.SparkMasterConfig = sparkMasterConfig
	}
	sparkWorkerConfig, _ := src.SparkWorkerConfig.(string)
	if sparkWorkerConfig != "" {
		res.SparkWorkerConfig = sparkWorkerConfig
	}
}

func checkConfiguration(config models.NewClusterConfig) error {
	var err error
	if config.MasterCount != 1 {
		err = errors.New(MasterCountMustBeOne)
	} else if config.WorkerCount < 1 {
		err = errors.New(WorkerCountMustBeAtLeastOne)
	}
	return err
}

func getInt(filename string) (res int64, err error) {
	fd, err := os.Open(filename)
	if err == nil {
		_, err = fmt.Fscanf(fd, "%d", &res)
		fd.Close()
		if err != nil {
			err = errors.New(fmt.Sprintf(ErrorWhileProcessing, filename, err.Error()))
		}
	}
	return res, err
}

func process(config *models.NewClusterConfig, nameElements []string, filename string) error {

	var err error

	// At present we only have a single level of configs, but if/when we have
	// nested configs then we would descend through the levels beginning here with
	// the first element in the name
	switch nameElements[0] {
	case "mastercount":
		config.MasterCount, err = getInt(filename)
	case "workercount":
		config.WorkerCount, err = getInt(filename)
	}
	// TODO (tmckay) have to allow the named configs to set sparkworkerconfig and sparkmasterconfig here
	return err
}

func readConfig(name string, res *models.NewClusterConfig, failOnMissing bool) (err error) {

	filelist, err := filepath.Glob(fmt.Sprintf(globpath, name))
	if err == nil {
		if failOnMissing == true && len(filelist) == 0 {
			return errors.New(fmt.Sprintf(NamedConfigDoesNotExist, name))
		}
		for _, v := range (filelist) {
			// Break up each filename into elements by "."
			// The first element of every filename will be the config name, dump it
			elements := strings.Split(filepath.Base(v), ".")[1:]
			err = process(res, elements, v)
			if err != nil {
				break
			}
		}
	}
	return
}

func loadConfig(name string) (res models.NewClusterConfig, err error) {
	// If the default config has been modified use those mods.
	// This can probably be smarter, assuming file timestamps
	// work for ConfigMap volumes.
	res = defaultConfig
	err = readConfig(Defaultname, &res, allowMissing)
	if err == nil && name != "" && name != Defaultname {
		err = readConfig(name, &res, failOnMissing)
	}
	return res, err
}

func GetClusterConfig(config *models.NewClusterConfig) (res models.NewClusterConfig, err error) {
	var name string = ""
	if config != nil {
		name, _ = config.Name.(string)
	}
	res, err = loadConfig(name)
	if err == nil && config != nil {
		assignConfig(&res, *config)
	}

	// Check that the final configuration is valid
	if err == nil {
		err = checkConfiguration(res)
	}
	return res, err
}
<commit_msg>Allow named configs to specify master/worker spark configurations<commit_after>package clusterconfigs

import (
	"io/ioutil"
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"
	"github.com/radanalyticsio/oshinko-rest/models"
)

var defaultConfig models.NewClusterConfig = models.NewClusterConfig{
								MasterCount: 1,
	                                                        WorkerCount: 1,
								Name: "",
								SparkMasterConfig: "",
								SparkWorkerConfig: ""}
var configpath, globpath string

const Defaultname = "default"
const failOnMissing = true
const allowMissing = false
const DefaultConfigPath = "/etc/oshinko-cluster-configs/"

const MasterCountMustBeOne = "Cluster configuration must have a masterCount of 1"
const WorkerCountMustBeAtLeastOne = "Cluster configuration may not have a workerCount less than 1"
const NamedConfigDoesNotExist = "Named config '%s' does not exist"
const ErrorWhileProcessing = "Error while processing %s: %s"

func init() {
	SetConfigPath(DefaultConfigPath)
}

// This function is meant to supoprt testability
func SetConfigPath(dir string) {
	configpath = dir
	globpath = path.Join(configpath, "%s\\.*")
}

// This function is meant to support testability
func GetConfigPath() string {
	return configpath
}

// This function is meant to support testability
func GetDefaultConfig() models.NewClusterConfig {
	return defaultConfig
}

func assignConfig(res *models.NewClusterConfig, src models.NewClusterConfig) {
	if src.MasterCount != 0 {
		res.MasterCount = src.MasterCount
	}
	if src.WorkerCount != 0 {
		res.WorkerCount = src.WorkerCount
	}
	sparkMasterConfig, _ := src.SparkMasterConfig.(string)
	if sparkMasterConfig != "" {
		res.SparkMasterConfig = sparkMasterConfig
	}
	sparkWorkerConfig, _ := src.SparkWorkerConfig.(string)
	if sparkWorkerConfig != "" {
		res.SparkWorkerConfig = sparkWorkerConfig
	}
}

func checkConfiguration(config models.NewClusterConfig) error {
	var err error
	if config.MasterCount != 1 {
		err = errors.New(MasterCountMustBeOne)
	} else if config.WorkerCount < 1 {
		err = errors.New(WorkerCountMustBeAtLeastOne)
	}
	return err
}

func getInt(filename string) (res int64, err error) {
	fd, err := os.Open(filename)
	if err == nil {
		_, err = fmt.Fscanf(fd, "%d", &res)
		fd.Close()
		if err != nil {
			err = errors.New(fmt.Sprintf(ErrorWhileProcessing, filename, err.Error()))
		}
	}
	return res, err
}
func getStr(filename string) (res string, err error) {
	buff, err := ioutil.ReadFile(filename)
	if err != nil {
		err = errors.New(fmt.Sprintf(ErrorWhileProcessing, filename, err.Error()))
	} else {
		// In case there is a carriage return
		res = strings.Trim(string(buff), "\n")
	}
	return res, err
}

func process(config *models.NewClusterConfig, nameElements []string, filename string) error {

	var err error

	// At present we only have a single level of configs, but if/when we have
	// nested configs then we would descend through the levels beginning here with
	// the first element in the name
	switch nameElements[0] {
	case "mastercount":
		config.MasterCount, err = getInt(filename)
	case "workercount":
		config.WorkerCount, err = getInt(filename)
	case "sparkmasterconfig":
		str, err := getStr(filename)
		if err == nil {
			config.SparkMasterConfig = str
		}
	case "sparkworkerconfig":
		str, err := getStr(filename)
		if err == nil {
			config.SparkWorkerConfig = str
		}
	}
	return err
}

func readConfig(name string, res *models.NewClusterConfig, failOnMissing bool) (err error) {

	filelist, err := filepath.Glob(fmt.Sprintf(globpath, name))
	if err == nil {
		if failOnMissing == true && len(filelist) == 0 {
			return errors.New(fmt.Sprintf(NamedConfigDoesNotExist, name))
		}
		for _, v := range (filelist) {
			// Break up each filename into elements by "."
			// The first element of every filename will be the config name, dump it
			elements := strings.Split(filepath.Base(v), ".")[1:]
			err = process(res, elements, v)
			if err != nil {
				break
			}
		}
	}
	return
}

func loadConfig(name string) (res models.NewClusterConfig, err error) {
	// If the default config has been modified use those mods.
	// This can probably be smarter, assuming file timestamps
	// work for ConfigMap volumes.
	res = defaultConfig
	err = readConfig(Defaultname, &res, allowMissing)
	if err == nil && name != "" && name != Defaultname {
		err = readConfig(name, &res, failOnMissing)
	}
	return res, err
}

func GetClusterConfig(config *models.NewClusterConfig) (res models.NewClusterConfig, err error) {
	var name string = ""
	if config != nil {
		name, _ = config.Name.(string)
	}
	res, err = loadConfig(name)
	if err == nil && config != nil {
		assignConfig(&res, *config)
	}

	// Check that the final configuration is valid
	if err == nil {
		err = checkConfiguration(res)
	}
	return res, err
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "gopkg.in/alecthomas/kingpin.v1"
)

var (
    app = kingpin.New("substenv", "Substitute environment variables into templates")
    input = kingpin.Arg("input", "Input file or stdin if not given").File()
)

func main() {
    kingpin.Version("0.0.1")
    kingpin.Parse()
    fmt.Printf("Hello, World!\n")
}
<commit_msg>Read and output stdin<commit_after>package main

import (
    "bufio"
    "fmt"
    "io"
    "log"
    "os"
    "gopkg.in/alecthomas/kingpin.v1"
)

var (
    app = kingpin.New("substenv", "Substitute environment variables into templates")
    input = kingpin.Arg("input", "Input file or stdin if not given").File()
)

func main() {
    kingpin.Version("0.0.1")
    kingpin.Parse()

    var bio = bufio.NewReader(os.Stdin)
    for {
        var line, _, err = bio.ReadLine()
        if err != nil {
            if err != io.EOF {
                log.Fatal(err)
            }
            break
        }
        fmt.Printf("%s\n", line)
    }
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"net/url"

	"testing"

	"github.com/motemen/ghq/cmdutil"
	. "github.com/onsi/gomega"
)

func mustParseURL(urlString string) *url.URL {
	u, err := url.Parse(urlString)
	if err != nil {
		panic(err)
	}
	return u
}

func TestNewRemoteRepository(t *testing.T) {
	testCases := []struct {
		url        string
		valid      bool
		vcsBackend *VCSBackend
	}{
		{
			url:        "https://github.com/motemen/pusheen-explorer",
			valid:      true,
			vcsBackend: GitBackend,
		},
		{
			url:        "https://github.com/motemen/pusheen-explorer/",
			valid:      true,
			vcsBackend: GitBackend,
		},
		{
			url:        "https://github.com/motemen/pusheen-explorer/blob/master/README.md",
			valid:      false,
			vcsBackend: GitBackend,
		},
		{
			url:        "https://example.com/motemen/pusheen-explorer/",
			valid:      true,
			vcsBackend: nil,
		},
		{
			url:        "https://gist.github.com/motemen/9733745",
			valid:      true,
			vcsBackend: GitBackend,
		},
		{
			url:        "http://hub.darcs.net/foo/bar",
			valid:      true,
			vcsBackend: DarcsBackend,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.url, func(t *testing.T) {
			repo, err := NewRemoteRepository(mustParseURL(tc.url))
			if err != nil {
				t.Errorf("error should be nil but: %s", err)
			}
			if repo.IsValid() != tc.valid {
				t.Errorf("repo.IsValid() should be %v, but %v", tc.valid, repo.IsValid())
			}
			vcs, _ := repo.VCS()
			if vcs != tc.vcsBackend {
				t.Errorf("got: %+v, expect: %+v", vcs, tc.vcsBackend)
			}
		})
	}
}

func TestNewRemoteRepositoryGoogleCode(t *testing.T) {
	RegisterTestingT(t)

	var (
		repo RemoteRepository
		err  error
	)

	repo, err = NewRemoteRepository(mustParseURL("https://code.google.com/p/vim/"))
	Expect(err).To(BeNil())
	Expect(repo.IsValid()).To(Equal(true))
	cmdutil.CommandRunner = NewFakeRunner(map[string]error{
		"hg identify":   nil,
		"git ls-remote": errors.New(""),
	})
	vcs, _ := repo.VCS()
	Expect(vcs).To(Equal(MercurialBackend))

	repo, err = NewRemoteRepository(mustParseURL("https://code.google.com/p/git-core"))
	Expect(err).To(BeNil())
	Expect(repo.IsValid()).To(Equal(true))
	cmdutil.CommandRunner = NewFakeRunner(map[string]error{
		"hg identify":   errors.New(""),
		"git ls-remote": nil,
	})
	vcs, _ = repo.VCS()
	Expect(vcs).To(Equal(GitBackend))
}
<commit_msg>remove test around code.google.com<commit_after>package main

import (
	"net/url"
	"testing"
)

func mustParseURL(urlString string) *url.URL {
	u, err := url.Parse(urlString)
	if err != nil {
		panic(err)
	}
	return u
}

func TestNewRemoteRepository(t *testing.T) {
	testCases := []struct {
		url        string
		valid      bool
		vcsBackend *VCSBackend
	}{
		{
			url:        "https://github.com/motemen/pusheen-explorer",
			valid:      true,
			vcsBackend: GitBackend,
		},
		{
			url:        "https://github.com/motemen/pusheen-explorer/",
			valid:      true,
			vcsBackend: GitBackend,
		},
		{
			url:        "https://github.com/motemen/pusheen-explorer/blob/master/README.md",
			valid:      false,
			vcsBackend: GitBackend,
		},
		{
			url:        "https://example.com/motemen/pusheen-explorer/",
			valid:      true,
			vcsBackend: nil,
		},
		{
			url:        "https://gist.github.com/motemen/9733745",
			valid:      true,
			vcsBackend: GitBackend,
		},
		{
			url:        "http://hub.darcs.net/foo/bar",
			valid:      true,
			vcsBackend: DarcsBackend,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.url, func(t *testing.T) {
			repo, err := NewRemoteRepository(mustParseURL(tc.url))
			if err != nil {
				t.Errorf("error should be nil but: %s", err)
			}
			if repo.IsValid() != tc.valid {
				t.Errorf("repo.IsValid() should be %v, but %v", tc.valid, repo.IsValid())
			}
			vcs, _ := repo.VCS()
			if vcs != tc.vcsBackend {
				t.Errorf("got: %+v, expect: %+v", vcs, tc.vcsBackend)
			}
		})
	}
}
<|endoftext|>


<commit_before>package file

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"

	"github.com/src-d/go-git-fixtures"

	. "gopkg.in/check.v1"
)

type ServerSuite struct {
	CommonSuite
	RemoteName string
	SrcPath    string
	DstPath    string
	DstURL     string
}

var _ = Suite(&ServerSuite{})

func (s *ServerSuite) SetUpSuite(c *C) {
	s.CommonSuite.SetUpSuite(c)

	s.RemoteName = "test"

	fixture := fixtures.Basic().One()
	s.SrcPath = fixture.DotGit().Base()

	fixture = fixtures.ByTag("empty").One()
	s.DstPath = fixture.DotGit().Base()
	s.DstURL = fmt.Sprintf("file://%s", s.DstPath)

	cmd := exec.Command("git", "remote", "add", s.RemoteName, s.DstURL)
	cmd.Dir = s.SrcPath
	c.Assert(cmd.Run(), IsNil)
}

func (s *ServerSuite) TestPush(c *C) {
	// git <2.0 cannot push to an empty repository without a refspec.
	cmd := exec.Command("git", "push",
		"--receive-pack", s.ReceivePackBin,
		s.RemoteName, "refs/heads/*:refs/heads/*",
	)
	cmd.Dir = s.SrcPath
	cmd.Env = os.Environ()
	cmd.Env = append(cmd.Env, "GIT_TRACE=true", "GIT_TRACE_PACKET=true")
	stdout, stderr, err := execAndGetOutput(c, cmd)
	c.Assert(err, IsNil, Commentf("STDOUT:\n%s\nSTDERR:\n%s\n", stdout, stderr))
}

func (s *ServerSuite) TestClone(c *C) {
	pathToClone := c.MkDir()

	cmd := exec.Command("git", "clone",
		"--upload-pack", s.UploadPackBin,
		s.SrcPath, pathToClone,
	)
	cmd.Env = os.Environ()
	cmd.Env = append(cmd.Env, "GIT_TRACE=true", "GIT_TRACE_PACKET=true")
	stdout, stderr, err := execAndGetOutput(c, cmd)
	c.Assert(err, IsNil, Commentf("STDOUT:\n%s\nSTDERR:\n%s\n", stdout, stderr))
}

func execAndGetOutput(c *C, cmd *exec.Cmd) (stdout, stderr string, err error) {
	sout, err := cmd.StdoutPipe()
	c.Assert(err, IsNil)
	serr, err := cmd.StderrPipe()
	c.Assert(err, IsNil)

	outChan, outErr := readAllAsync(sout)
	errChan, errErr := readAllAsync(serr)

	c.Assert(cmd.Start(), IsNil)

	if err = cmd.Wait(); err != nil {
		return <-outChan, <-errChan, err
	}

	if err := <-outErr; err != nil {
		return <-outChan, <-errChan, err
	}

	return <-outChan, <-errChan, <-errErr
}

func readAllAsync(r io.Reader) (out chan string, err chan error) {
	out = make(chan string, 1)
	err = make(chan error, 1)
	go func() {
		b, e := ioutil.ReadAll(r)
		if e != nil {
			err <- e
		} else {
			err <- nil
		}

		out <- string(b)
	}()

	return out, err
}
<commit_msg>transport/file: fix race condition on test (#267)<commit_after>package file

import (
	"fmt"
	"os"
	"os/exec"

	"github.com/src-d/go-git-fixtures"

	. "gopkg.in/check.v1"
)

type ServerSuite struct {
	CommonSuite
	RemoteName string
	SrcPath    string
	DstPath    string
	DstURL     string
}

var _ = Suite(&ServerSuite{})

func (s *ServerSuite) SetUpSuite(c *C) {
	s.CommonSuite.SetUpSuite(c)

	s.RemoteName = "test"

	fixture := fixtures.Basic().One()
	s.SrcPath = fixture.DotGit().Base()

	fixture = fixtures.ByTag("empty").One()
	s.DstPath = fixture.DotGit().Base()
	s.DstURL = fmt.Sprintf("file://%s", s.DstPath)

	cmd := exec.Command("git", "remote", "add", s.RemoteName, s.DstURL)
	cmd.Dir = s.SrcPath
	c.Assert(cmd.Run(), IsNil)
}

func (s *ServerSuite) TestPush(c *C) {
	// git <2.0 cannot push to an empty repository without a refspec.
	cmd := exec.Command("git", "push",
		"--receive-pack", s.ReceivePackBin,
		s.RemoteName, "refs/heads/*:refs/heads/*",
	)
	cmd.Dir = s.SrcPath
	cmd.Env = os.Environ()
	cmd.Env = append(cmd.Env, "GIT_TRACE=true", "GIT_TRACE_PACKET=true")
	out, err := cmd.CombinedOutput()
	c.Assert(err, IsNil, Commentf("combined stdout and stderr:\n%s\n", out))
}

func (s *ServerSuite) TestClone(c *C) {
	pathToClone := c.MkDir()

	cmd := exec.Command("git", "clone",
		"--upload-pack", s.UploadPackBin,
		s.SrcPath, pathToClone,
	)
	cmd.Env = os.Environ()
	cmd.Env = append(cmd.Env, "GIT_TRACE=true", "GIT_TRACE_PACKET=true")
	out, err := cmd.CombinedOutput()
	c.Assert(err, IsNil, Commentf("combined stdout and stderr:\n%s\n", out))
}
<|endoftext|>


<commit_before>package fsrepo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"

	repo "github.com/ipfs/go-ipfs/repo"

	measure "gx/ipfs/QmSb95iHExSSb47zpmyn5CyY5PZidVWSjyKyDqgYQrnKor/go-ds-measure"
	flatfs "gx/ipfs/QmUTshC2PP4ZDqkrFfDU4JGJFMWjYnunxPgkQ6ZCA2hGqh/go-ds-flatfs"

	ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore"
	mount "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/syncmount"

	badgerds "gx/ipfs/QmNWbaGdPCA3anCcvh4jm3VAahAbmmAsU58sp8Ti4KTJkL/go-ds-badger"
	levelds "gx/ipfs/QmPdvXuXWAR6gtxxqZw42RtSADMwz4ijVmYHGS542b6cMz/go-ds-leveldb"
	ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
)

// ConfigFromMap creates a new datastore config from a map
type ConfigFromMap func(map[string]interface{}) (DatastoreConfig, error)

// DatastoreConfig is an abstraction of a datastore config.  A "spec"
// is first converted to a DatastoreConfig and then Create() is called
// to instantiate a new datastore
type DatastoreConfig interface {
	// DiskSpec returns a minimal configuration of the datastore
	// represting what is stored on disk.  Run time values are
	// excluded.
	DiskSpec() DiskSpec

	// Create instantiate a new datastore from this config
	Create(path string) (repo.Datastore, error)
}

// DiskSpec is the type returned by the DatastoreConfig's DiskSpec method
type DiskSpec map[string]interface{}

// Bytes returns a minimal JSON encoding of the DiskSpec
func (spec DiskSpec) Bytes() []byte {
	b, err := json.Marshal(spec)
	if err != nil {
		// should not happen
		panic(err)
	}
	return bytes.TrimSpace(b)
}

// String returns a minimal JSON encoding of the DiskSpec
func (spec DiskSpec) String() string {
	return string(spec.Bytes())
}

var datastores map[string]ConfigFromMap

func init() {
	datastores = map[string]ConfigFromMap{
		"mount":    MountDatastoreConfig,
		"flatfs":   FlatfsDatastoreConfig,
		"levelds":  LeveldsDatastoreConfig,
		"badgerds": BadgerdsDatastoreConfig,
		"mem":      MemDatastoreConfig,
		"log":      LogDatastoreConfig,
		"measure":  MeasureDatastoreConfig,
	}
}

// AnyDatastoreConfig returns a DatastoreConfig from a spec based on
// the "type" parameter
func AnyDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	which, ok := params["type"].(string)
	if !ok {
		return nil, fmt.Errorf("'type' field missing or not a string")
	}
	fun, ok := datastores[which]
	if !ok {
		return nil, fmt.Errorf("unknown datastore type: %s", which)
	}
	return fun(params)
}

type mountDatastoreConfig struct {
	mounts []premount
}

type premount struct {
	ds     DatastoreConfig
	prefix ds.Key
}

// MountDatastoreConfig returns a mount DatastoreConfig from a spec
func MountDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	var res mountDatastoreConfig
	mounts, ok := params["mounts"].([]interface{})
	if !ok {
		return nil, fmt.Errorf("'mounts' field is missing or not an array")
	}
	for _, iface := range mounts {
		cfg, ok := iface.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("expected map for mountpoint")
		}

		child, err := AnyDatastoreConfig(cfg)
		if err != nil {
			return nil, err
		}

		prefix, found := cfg["mountpoint"]
		if !found {
			return nil, fmt.Errorf("no 'mountpoint' on mount")
		}

		res.mounts = append(res.mounts, premount{
			ds:     child,
			prefix: ds.NewKey(prefix.(string)),
		})
	}
	sort.Slice(res.mounts,
		func(i, j int) bool {
			return res.mounts[i].prefix.String() > res.mounts[j].prefix.String()
		})

	return &res, nil
}

func (c *mountDatastoreConfig) DiskSpec() DiskSpec {
	cfg := map[string]interface{}{"type": "mount"}
	mounts := make([]interface{}, len(c.mounts))
	for i, m := range c.mounts {
		c := m.ds.DiskSpec()
		if c == nil {
			c = make(map[string]interface{})
		}
		c["mountpoint"] = m.prefix.String()
		mounts[i] = c
	}
	cfg["mounts"] = mounts
	return cfg
}

func (c *mountDatastoreConfig) Create(path string) (repo.Datastore, error) {
	mounts := make([]mount.Mount, len(c.mounts))
	for i, m := range c.mounts {
		ds, err := m.ds.Create(path)
		if err != nil {
			return nil, err
		}
		mounts[i].Datastore = ds
		mounts[i].Prefix = m.prefix
	}
	return mount.New(mounts), nil
}

type flatfsDatastoreConfig struct {
	path      string
	shardFun  *flatfs.ShardIdV1
	syncField bool
}

// FlatfsDatastoreConfig returns a flatfs DatastoreConfig from a spec
func FlatfsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	var c flatfsDatastoreConfig
	var ok bool
	var err error

	c.path, ok = params["path"].(string)
	if !ok {
		return nil, fmt.Errorf("'path' field is missing or not boolean")
	}

	sshardFun, ok := params["shardFunc"].(string)
	if !ok {
		return nil, fmt.Errorf("'shardFunc' field is missing or not a string")
	}
	c.shardFun, err = flatfs.ParseShardFunc(sshardFun)
	if err != nil {
		return nil, err
	}

	c.syncField, ok = params["sync"].(bool)
	if !ok {
		return nil, fmt.Errorf("'sync' field is missing or not boolean")
	}
	return &c, nil
}

func (c *flatfsDatastoreConfig) DiskSpec() DiskSpec {
	return map[string]interface{}{
		"type":      "flatfs",
		"path":      c.path,
		"shardFunc": c.shardFun.String(),
	}
}

func (c *flatfsDatastoreConfig) Create(path string) (repo.Datastore, error) {
	p := c.path
	if !filepath.IsAbs(p) {
		p = filepath.Join(path, p)
	}

	return flatfs.CreateOrOpen(p, c.shardFun, c.syncField)
}

type leveldsDatastoreConfig struct {
	path        string
	compression ldbopts.Compression
}

// LeveldsDatastoreConfig returns a levelds DatastoreConfig from a spec
func LeveldsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	var c leveldsDatastoreConfig
	var ok bool

	c.path, ok = params["path"].(string)
	if !ok {
		return nil, fmt.Errorf("'path' field is missing or not string")
	}

	switch cm := params["compression"].(string); cm {
	case "none":
		c.compression = ldbopts.NoCompression
	case "snappy":
		c.compression = ldbopts.SnappyCompression
	case "":
		c.compression = ldbopts.DefaultCompression
	default:
		return nil, fmt.Errorf("unrecognized value for compression: %s", cm)
	}

	return &c, nil
}

func (c *leveldsDatastoreConfig) DiskSpec() DiskSpec {
	return map[string]interface{}{
		"type": "levelds",
		"path": c.path,
	}
}

func (c *leveldsDatastoreConfig) Create(path string) (repo.Datastore, error) {
	p := c.path
	if !filepath.IsAbs(p) {
		p = filepath.Join(path, p)
	}

	return levelds.NewDatastore(p, &levelds.Options{
		Compression: c.compression,
	})
}

type memDatastoreConfig struct {
	cfg map[string]interface{}
}

// MemDatastoreConfig returns a memory DatastoreConfig from a spec
func MemDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	return &memDatastoreConfig{params}, nil
}

func (c *memDatastoreConfig) DiskSpec() DiskSpec {
	return nil
}

func (c *memDatastoreConfig) Create(string) (repo.Datastore, error) {
	return ds.NewMapDatastore(), nil
}

type logDatastoreConfig struct {
	child DatastoreConfig
	name  string
}

// LogDatastoreConfig returns a log DatastoreConfig from a spec
func LogDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	childField, ok := params["child"].(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("'child' field is missing or not a map")
	}
	child, err := AnyDatastoreConfig(childField)
	if err != nil {
		return nil, err
	}
	name, ok := params["name"].(string)
	if !ok {
		return nil, fmt.Errorf("'name' field was missing or not a string")
	}
	return &logDatastoreConfig{child, name}, nil

}

func (c *logDatastoreConfig) Create(path string) (repo.Datastore, error) {
	child, err := c.child.Create(path)
	if err != nil {
		return nil, err
	}
	return ds.NewLogDatastore(child, c.name), nil
}

func (c *logDatastoreConfig) DiskSpec() DiskSpec {
	return c.child.DiskSpec()
}

type measureDatastoreConfig struct {
	child  DatastoreConfig
	prefix string
}

// MeasureDatastoreConfig returns a measure DatastoreConfig from a spec
func MeasureDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	childField, ok := params["child"].(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("'child' field is missing or not a map")
	}
	child, err := AnyDatastoreConfig(childField)
	if err != nil {
		return nil, err
	}
	prefix, ok := params["prefix"].(string)
	if !ok {
		return nil, fmt.Errorf("'prefix' field was missing or not a string")
	}
	return &measureDatastoreConfig{child, prefix}, nil
}

func (c *measureDatastoreConfig) DiskSpec() DiskSpec {
	return c.child.DiskSpec()
}

func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) {
	child, err := c.child.Create(path)
	if err != nil {
		return nil, err
	}
	return measure.New(c.prefix, child), nil
}

type badgerdsDatastoreConfig struct {
	path string
}

func BadgerdsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	var c badgerdsDatastoreConfig
	var ok bool

	c.path, ok = params["path"].(string)
	if !ok {
		return nil, fmt.Errorf("'path' field is missing or not string")
	}

	return &c, nil
}

func (c *badgerdsDatastoreConfig) DiskSpec() DiskSpec {
	return map[string]interface{}{
		"type": "badgerds",
		"path": c.path,
	}
}

func (c *badgerdsDatastoreConfig) Create(path string) (repo.Datastore, error) {
	p := c.path
	if !filepath.IsAbs(p) {
		p = filepath.Join(path, p)
	}

	err := os.MkdirAll(p, 0755)
	if err != nil {
		return nil, err
	}

	return badgerds.NewDatastore(p, nil)
}
<commit_msg>add godoc<commit_after>package fsrepo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"

	repo "github.com/ipfs/go-ipfs/repo"

	measure "gx/ipfs/QmSb95iHExSSb47zpmyn5CyY5PZidVWSjyKyDqgYQrnKor/go-ds-measure"
	flatfs "gx/ipfs/QmUTshC2PP4ZDqkrFfDU4JGJFMWjYnunxPgkQ6ZCA2hGqh/go-ds-flatfs"

	ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore"
	mount "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/syncmount"

	badgerds "gx/ipfs/QmNWbaGdPCA3anCcvh4jm3VAahAbmmAsU58sp8Ti4KTJkL/go-ds-badger"
	levelds "gx/ipfs/QmPdvXuXWAR6gtxxqZw42RtSADMwz4ijVmYHGS542b6cMz/go-ds-leveldb"
	ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
)

// ConfigFromMap creates a new datastore config from a map
type ConfigFromMap func(map[string]interface{}) (DatastoreConfig, error)

// DatastoreConfig is an abstraction of a datastore config.  A "spec"
// is first converted to a DatastoreConfig and then Create() is called
// to instantiate a new datastore
type DatastoreConfig interface {
	// DiskSpec returns a minimal configuration of the datastore
	// represting what is stored on disk.  Run time values are
	// excluded.
	DiskSpec() DiskSpec

	// Create instantiate a new datastore from this config
	Create(path string) (repo.Datastore, error)
}

// DiskSpec is the type returned by the DatastoreConfig's DiskSpec method
type DiskSpec map[string]interface{}

// Bytes returns a minimal JSON encoding of the DiskSpec
func (spec DiskSpec) Bytes() []byte {
	b, err := json.Marshal(spec)
	if err != nil {
		// should not happen
		panic(err)
	}
	return bytes.TrimSpace(b)
}

// String returns a minimal JSON encoding of the DiskSpec
func (spec DiskSpec) String() string {
	return string(spec.Bytes())
}

var datastores map[string]ConfigFromMap

func init() {
	datastores = map[string]ConfigFromMap{
		"mount":    MountDatastoreConfig,
		"flatfs":   FlatfsDatastoreConfig,
		"levelds":  LeveldsDatastoreConfig,
		"badgerds": BadgerdsDatastoreConfig,
		"mem":      MemDatastoreConfig,
		"log":      LogDatastoreConfig,
		"measure":  MeasureDatastoreConfig,
	}
}

// AnyDatastoreConfig returns a DatastoreConfig from a spec based on
// the "type" parameter
func AnyDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	which, ok := params["type"].(string)
	if !ok {
		return nil, fmt.Errorf("'type' field missing or not a string")
	}
	fun, ok := datastores[which]
	if !ok {
		return nil, fmt.Errorf("unknown datastore type: %s", which)
	}
	return fun(params)
}

type mountDatastoreConfig struct {
	mounts []premount
}

type premount struct {
	ds     DatastoreConfig
	prefix ds.Key
}

// MountDatastoreConfig returns a mount DatastoreConfig from a spec
func MountDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	var res mountDatastoreConfig
	mounts, ok := params["mounts"].([]interface{})
	if !ok {
		return nil, fmt.Errorf("'mounts' field is missing or not an array")
	}
	for _, iface := range mounts {
		cfg, ok := iface.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("expected map for mountpoint")
		}

		child, err := AnyDatastoreConfig(cfg)
		if err != nil {
			return nil, err
		}

		prefix, found := cfg["mountpoint"]
		if !found {
			return nil, fmt.Errorf("no 'mountpoint' on mount")
		}

		res.mounts = append(res.mounts, premount{
			ds:     child,
			prefix: ds.NewKey(prefix.(string)),
		})
	}
	sort.Slice(res.mounts,
		func(i, j int) bool {
			return res.mounts[i].prefix.String() > res.mounts[j].prefix.String()
		})

	return &res, nil
}

func (c *mountDatastoreConfig) DiskSpec() DiskSpec {
	cfg := map[string]interface{}{"type": "mount"}
	mounts := make([]interface{}, len(c.mounts))
	for i, m := range c.mounts {
		c := m.ds.DiskSpec()
		if c == nil {
			c = make(map[string]interface{})
		}
		c["mountpoint"] = m.prefix.String()
		mounts[i] = c
	}
	cfg["mounts"] = mounts
	return cfg
}

func (c *mountDatastoreConfig) Create(path string) (repo.Datastore, error) {
	mounts := make([]mount.Mount, len(c.mounts))
	for i, m := range c.mounts {
		ds, err := m.ds.Create(path)
		if err != nil {
			return nil, err
		}
		mounts[i].Datastore = ds
		mounts[i].Prefix = m.prefix
	}
	return mount.New(mounts), nil
}

type flatfsDatastoreConfig struct {
	path      string
	shardFun  *flatfs.ShardIdV1
	syncField bool
}

// FlatfsDatastoreConfig returns a flatfs DatastoreConfig from a spec
func FlatfsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	var c flatfsDatastoreConfig
	var ok bool
	var err error

	c.path, ok = params["path"].(string)
	if !ok {
		return nil, fmt.Errorf("'path' field is missing or not boolean")
	}

	sshardFun, ok := params["shardFunc"].(string)
	if !ok {
		return nil, fmt.Errorf("'shardFunc' field is missing or not a string")
	}
	c.shardFun, err = flatfs.ParseShardFunc(sshardFun)
	if err != nil {
		return nil, err
	}

	c.syncField, ok = params["sync"].(bool)
	if !ok {
		return nil, fmt.Errorf("'sync' field is missing or not boolean")
	}
	return &c, nil
}

func (c *flatfsDatastoreConfig) DiskSpec() DiskSpec {
	return map[string]interface{}{
		"type":      "flatfs",
		"path":      c.path,
		"shardFunc": c.shardFun.String(),
	}
}

func (c *flatfsDatastoreConfig) Create(path string) (repo.Datastore, error) {
	p := c.path
	if !filepath.IsAbs(p) {
		p = filepath.Join(path, p)
	}

	return flatfs.CreateOrOpen(p, c.shardFun, c.syncField)
}

type leveldsDatastoreConfig struct {
	path        string
	compression ldbopts.Compression
}

// LeveldsDatastoreConfig returns a levelds DatastoreConfig from a spec
func LeveldsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	var c leveldsDatastoreConfig
	var ok bool

	c.path, ok = params["path"].(string)
	if !ok {
		return nil, fmt.Errorf("'path' field is missing or not string")
	}

	switch cm := params["compression"].(string); cm {
	case "none":
		c.compression = ldbopts.NoCompression
	case "snappy":
		c.compression = ldbopts.SnappyCompression
	case "":
		c.compression = ldbopts.DefaultCompression
	default:
		return nil, fmt.Errorf("unrecognized value for compression: %s", cm)
	}

	return &c, nil
}

func (c *leveldsDatastoreConfig) DiskSpec() DiskSpec {
	return map[string]interface{}{
		"type": "levelds",
		"path": c.path,
	}
}

func (c *leveldsDatastoreConfig) Create(path string) (repo.Datastore, error) {
	p := c.path
	if !filepath.IsAbs(p) {
		p = filepath.Join(path, p)
	}

	return levelds.NewDatastore(p, &levelds.Options{
		Compression: c.compression,
	})
}

type memDatastoreConfig struct {
	cfg map[string]interface{}
}

// MemDatastoreConfig returns a memory DatastoreConfig from a spec
func MemDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	return &memDatastoreConfig{params}, nil
}

func (c *memDatastoreConfig) DiskSpec() DiskSpec {
	return nil
}

func (c *memDatastoreConfig) Create(string) (repo.Datastore, error) {
	return ds.NewMapDatastore(), nil
}

type logDatastoreConfig struct {
	child DatastoreConfig
	name  string
}

// LogDatastoreConfig returns a log DatastoreConfig from a spec
func LogDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	childField, ok := params["child"].(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("'child' field is missing or not a map")
	}
	child, err := AnyDatastoreConfig(childField)
	if err != nil {
		return nil, err
	}
	name, ok := params["name"].(string)
	if !ok {
		return nil, fmt.Errorf("'name' field was missing or not a string")
	}
	return &logDatastoreConfig{child, name}, nil

}

func (c *logDatastoreConfig) Create(path string) (repo.Datastore, error) {
	child, err := c.child.Create(path)
	if err != nil {
		return nil, err
	}
	return ds.NewLogDatastore(child, c.name), nil
}

func (c *logDatastoreConfig) DiskSpec() DiskSpec {
	return c.child.DiskSpec()
}

type measureDatastoreConfig struct {
	child  DatastoreConfig
	prefix string
}

// MeasureDatastoreConfig returns a measure DatastoreConfig from a spec
func MeasureDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	childField, ok := params["child"].(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("'child' field is missing or not a map")
	}
	child, err := AnyDatastoreConfig(childField)
	if err != nil {
		return nil, err
	}
	prefix, ok := params["prefix"].(string)
	if !ok {
		return nil, fmt.Errorf("'prefix' field was missing or not a string")
	}
	return &measureDatastoreConfig{child, prefix}, nil
}

func (c *measureDatastoreConfig) DiskSpec() DiskSpec {
	return c.child.DiskSpec()
}

func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) {
	child, err := c.child.Create(path)
	if err != nil {
		return nil, err
	}
	return measure.New(c.prefix, child), nil
}

type badgerdsDatastoreConfig struct {
	path string
}

// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
// from the given parameters
func BadgerdsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
	var c badgerdsDatastoreConfig
	var ok bool

	c.path, ok = params["path"].(string)
	if !ok {
		return nil, fmt.Errorf("'path' field is missing or not string")
	}

	return &c, nil
}

func (c *badgerdsDatastoreConfig) DiskSpec() DiskSpec {
	return map[string]interface{}{
		"type": "badgerds",
		"path": c.path,
	}
}

func (c *badgerdsDatastoreConfig) Create(path string) (repo.Datastore, error) {
	p := c.path
	if !filepath.IsAbs(p) {
		p = filepath.Join(path, p)
	}

	err := os.MkdirAll(p, 0755)
	if err != nil {
		return nil, err
	}

	return badgerds.NewDatastore(p, nil)
}
<|endoftext|>


<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package itl

import (
	"fmt"
	"html"
	"io"
	"net/url"
	"reflect"
	"strconv"
	"strings"
	"time"

	rawitl "github.com/dhowden/itl"
	"tchaik.com/index"
)

// ReadFrom creates a Tchaik Library implementation from an iTunes Music Library passed through
// an io.Reader.
func ReadFrom(r io.Reader) (index.Library, error) {
	l, err := rawitl.ReadFromXML(r)
	if err != nil {
		return nil, err
	}
	return &itlLibrary{&l}, nil
}

type itlLibrary struct {
	*rawitl.Library
}

// Implements Library.
func (l *itlLibrary) Tracks() []index.Track {
	tracks := make([]index.Track, 0, len(l.Library.Tracks))
	for _, t := range l.Library.Tracks {
		if strings.HasSuffix(t.Kind, "audio file") {
			x := t
			tracks = append(tracks, &itlTrack{&x})
		}
	}
	return tracks
}

// Implements Library.
func (l *itlLibrary) Track(id string) (index.Track, bool) {
	t, ok := l.Library.Tracks[id]
	if ok {
		return &itlTrack{&t}, true
	}
	return nil, false
}

// itlTrack is a wrapper type which implements Track for an rawitl.Track.
type itlTrack struct {
	*rawitl.Track
}

func decodeLocation(l string) (string, error) {
	u, err := url.ParseRequestURI(l)
	if err != nil {
		return "", err
	}
	// Annoyingly this doesn't replace & (&)
	path := strings.Replace(u.Path, "&", "&", -1)
	return path, nil
}

// GetString fetches the given string field in the Track, panics if field doesn't
// exist.
func (t *itlTrack) GetString(name string) string {
	defer func() {
		if r := recover(); r != nil {
			panic(fmt.Sprintf("panic attempting to read string field '%v': %v", name, r))
		}
	}()

	switch name {
	case "Location":
		loc, err := decodeLocation(html.UnescapeString(t.Location))
		if err != nil {
			panic(fmt.Sprintf("error in decodeLocation: %v", err))
		}
		return loc
	case "ID":
		return strconv.Itoa(t.TrackID)
	case "Name":
		return html.UnescapeString(t.Name)
	case "Artist":
		return html.UnescapeString(t.Artist)
	case "Album":
		return html.UnescapeString(t.Album)
	case "AlbumArtist":
		return html.UnescapeString(t.AlbumArtist)
	case "Composer":
		return html.UnescapeString(t.Composer)
	case "Genre":
		return html.UnescapeString(t.Genre)
	case "Kind":
		return html.UnescapeString(t.Kind)
	}

	tt := reflect.TypeOf(t)
	ft, ok := tt.FieldByName(name)
	if !ok {
		panic(fmt.Sprintf("invalid field '%v'", name))
	}
	if ft.Type.Kind() != reflect.String {
		panic(fmt.Sprintf("field '%v' is not a string", name))
	}

	v := reflect.ValueOf(t)
	f := v.FieldByName(name)
	return html.UnescapeString(f.String())
}

// GetStrings implements index.Track (will panic if the field doesn't exist as a list
// of strings).
func (t *itlTrack) GetStrings(name string) []string {
	switch name {
	case "Artist", "AlbumArtist", "Composer":
		return index.DefaultGetStrings(t, name)
	}
	panic(fmt.Sprintf("field '%v' is not a []string", name))
}

// GetInt fetches the given int field in the Track, panics if field doesn't exist.
func (t *itlTrack) GetInt(name string) int {
	switch name {
	case "ID": // NB: This should really be read as a string
		return t.TrackID
	case "DiscNumber":
		return t.DiscNumber
	case "DiscCount":
		return t.DiscCount
	case "TrackNumber":
		return t.TrackNumber
	case "TrackCount":
		return t.TrackCount
	case "Year":
		return t.Year
	case "TotalTime":
		return t.TotalTime
	case "BitRate":
		return t.BitRate
	}

	tt := reflect.TypeOf(t)
	ft, ok := tt.FieldByName(name)
	if !ok {
		panic(fmt.Sprintf("invalid field '%v'", name))
	}
	if ft.Type.Kind() != reflect.Int {
		panic(fmt.Sprintf("field '%v' is not an int", name))
	}

	v := reflect.ValueOf(t)
	f := v.FieldByName(name)
	return int(f.Int())
}

// GetTime fetches the given time field in the Track, panics if field doesn't exist.
func (t *itlTrack) GetTime(name string) time.Time {
	switch name {
	case "DateAdded":
		return t.DateAdded
	case "DateModified":
		return t.DateModified
	}
	panic(fmt.Sprintf("field '%v' is not a time value", name))
}
<commit_msg>Fix: iTunes library import when library has "Remote" tracks.<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package itl

import (
	"fmt"
	"html"
	"io"
	"net/url"
	"reflect"
	"strconv"
	"strings"
	"time"

	rawitl "github.com/dhowden/itl"
	"tchaik.com/index"
)

// ReadFrom creates a Tchaik Library implementation from an iTunes Music Library passed through
// an io.Reader.
func ReadFrom(r io.Reader) (index.Library, error) {
	l, err := rawitl.ReadFromXML(r)
	if err != nil {
		return nil, err
	}
	return &itlLibrary{&l}, nil
}

type itlLibrary struct {
	*rawitl.Library
}

// Implements Library.
func (l *itlLibrary) Tracks() []index.Track {
	tracks := make([]index.Track, 0, len(l.Library.Tracks))
	for _, t := range l.Library.Tracks {
		if t.TrackType == "File" && strings.HasSuffix(t.Kind, "audio file") {
			x := t
			tracks = append(tracks, &itlTrack{&x})
		}
	}
	return tracks
}

// Implements Library.
func (l *itlLibrary) Track(id string) (index.Track, bool) {
	t, ok := l.Library.Tracks[id]
	if ok {
		return &itlTrack{&t}, true
	}
	return nil, false
}

// itlTrack is a wrapper type which implements Track for an rawitl.Track.
type itlTrack struct {
	*rawitl.Track
}

func decodeLocation(l string) (string, error) {
	u, err := url.ParseRequestURI(l)
	if err != nil {
		return "", err
	}
	// Annoyingly this doesn't replace & (&)
	path := strings.Replace(u.Path, "&", "&", -1)
	return path, nil
}

// GetString fetches the given string field in the Track, panics if field doesn't
// exist.
func (t *itlTrack) GetString(name string) string {
	defer func() {
		if r := recover(); r != nil {
			panic(fmt.Sprintf("panic attempting to read string field '%v': %v", name, r))
		}
	}()

	switch name {
	case "Location":
		loc, err := decodeLocation(html.UnescapeString(t.Location))
		if err != nil {
			panic(fmt.Sprintf("error in decodeLocation: %v", err))
		}
		return loc
	case "ID":
		return strconv.Itoa(t.TrackID)
	case "Name":
		return html.UnescapeString(t.Name)
	case "Artist":
		return html.UnescapeString(t.Artist)
	case "Album":
		return html.UnescapeString(t.Album)
	case "AlbumArtist":
		return html.UnescapeString(t.AlbumArtist)
	case "Composer":
		return html.UnescapeString(t.Composer)
	case "Genre":
		return html.UnescapeString(t.Genre)
	case "Kind":
		return html.UnescapeString(t.Kind)
	}

	tt := reflect.TypeOf(t)
	ft, ok := tt.FieldByName(name)
	if !ok {
		panic(fmt.Sprintf("invalid field '%v'", name))
	}
	if ft.Type.Kind() != reflect.String {
		panic(fmt.Sprintf("field '%v' is not a string", name))
	}

	v := reflect.ValueOf(t)
	f := v.FieldByName(name)
	return html.UnescapeString(f.String())
}

// GetStrings implements index.Track (will panic if the field doesn't exist as a list
// of strings).
func (t *itlTrack) GetStrings(name string) []string {
	switch name {
	case "Artist", "AlbumArtist", "Composer":
		return index.DefaultGetStrings(t, name)
	}
	panic(fmt.Sprintf("field '%v' is not a []string", name))
}

// GetInt fetches the given int field in the Track, panics if field doesn't exist.
func (t *itlTrack) GetInt(name string) int {
	switch name {
	case "ID": // NB: This should really be read as a string
		return t.TrackID
	case "DiscNumber":
		return t.DiscNumber
	case "DiscCount":
		return t.DiscCount
	case "TrackNumber":
		return t.TrackNumber
	case "TrackCount":
		return t.TrackCount
	case "Year":
		return t.Year
	case "TotalTime":
		return t.TotalTime
	case "BitRate":
		return t.BitRate
	}

	tt := reflect.TypeOf(t)
	ft, ok := tt.FieldByName(name)
	if !ok {
		panic(fmt.Sprintf("invalid field '%v'", name))
	}
	if ft.Type.Kind() != reflect.Int {
		panic(fmt.Sprintf("field '%v' is not an int", name))
	}

	v := reflect.ValueOf(t)
	f := v.FieldByName(name)
	return int(f.Int())
}

// GetTime fetches the given time field in the Track, panics if field doesn't exist.
func (t *itlTrack) GetTime(name string) time.Time {
	switch name {
	case "DateAdded":
		return t.DateAdded
	case "DateModified":
		return t.DateModified
	}
	panic(fmt.Sprintf("field '%v' is not a time value", name))
}
<|endoftext|>