<commit_before>// Basic tool to talk to the rangeserver over http

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
)

//globals
var debug bool
var help bool
var timing bool
var vip string

func init() {
	parseFlags()
	if help == true {
		printHelp()
		os.Exit(0)
	}
	return
}

func parseFlags() {
	flag.BoolVar(&debug, "debug", false, "enable debug")
	flag.BoolVar(&help, "help", false, "enable help")
	flag.BoolVar(&timing, "timing", false, "enable timing")
	flag.StringVar(&vip, "vip", "localhost", "vip endpoint")
	flag.Parse()
	return
}

func printHelp() {
	fmt.Println(
		`	Usage: rangerclient [OPTIONS] <query>
	eg: yr %RANGE
	--debug .................... Run client in debug mode
	--help ..................... Prints this documentation
	--timing ................... profiling of execution time
	--vip ...................... Range vip endpoint`)
	os.Exit(0)
	return
}

func main() {
	query := ""
	if len(os.Args) == 1 {
		printHelp()
	}
	query = os.Args[len(os.Args)-1] //trying to get the last element of an array
	if vip == "localhost" {
		vip = "localhost:9999"
	}
	res, err := http.Get(fmt.Sprintf("http://%s/v1/range/list?%s", vip, url.QueryEscape(query)))
	if err != nil {
		log.Fatal(err)
	}
	results, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal("%s", err)
	}
	if res.Header.Get("Range-Err-Count") != "" {
		fmt.Printf("%s", results)
	} else {
		fmt.Printf("%s\n", results)
	}
	if timing == true {
		fmt.Printf("Range-Expand-Microsecond : %sms \n", res.Header.Get("Range-Expand-Microsecond"))
	}
}
<commit_msg>adding more error handling + some cleanup<commit_after>// Basic tool to talk to the rangeserver over http

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
)

//globals
var debug bool
var help bool
var timing bool
var vip string

func init() {
	parseFlags()
	if help == true {
		printHelp()
		os.Exit(0)
	}
	return
}

func parseFlags() {
	flag.BoolVar(&debug, "debug", false, "enable debug")
	flag.BoolVar(&help, "help", false, "Help")
	flag.BoolVar(&timing, "timing", false, "display timing")
	flag.StringVar(&vip, "vip", "localhost", "VIP endpoint")
	flag.Parse()
	return
}

func printHelp() {
	fmt.Printf(
		`	Usage: %s [OPTIONS] <query>
	eg: %s %%RANGE
	--debug .................... Debug
	--help ..................... Good Ol' Help
	--timing ................... Execution Time as provided by rangeserver
	--vip ...................... Range VIP Endpoint (default: localhost:9999)
`, os.Args[0], os.Args[0])
	os.Exit(0)
	return
}

func main() {
	query := ""
	if len(os.Args) == 1 {
		printHelp()
	}
	query = os.Args[len(os.Args)-1] //trying to get the last element of an array
	if vip == "localhost" {
		vip = "localhost:9999"
	}

	url := fmt.Sprintf("http://%s/v1/range/list?%s", vip, url.QueryEscape(query))
	if debug {
		fmt.Println("Range URL: ", url)
	}
	res, err := http.Get(url)
	// fatal out if we have error
	if err != nil || res.StatusCode != 200 {
		log.Fatalf("Http Error, URL: (%s) Error: (%v)\n", url, err)
	}

	results, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}

	res.Body.Close()

	// show the time taken for range expansion (as per the rangeserver)
	if timing || debug {
		fmt.Printf("Range-Expand-Microsecond: %s \n", res.Header.Get("Range-Expand-Microsecond"))
	}

	// if error, print the error
	if res.Header.Get("Range-Err-Count") != "" {
		fmt.Printf("%s", results)
	} else { // else print the result
		fmt.Printf("%s\n", results)
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"
)

type Console int
type Profiler int
type FullQuery Group
type PartQuery Group

const QueryPage = `<!doctype html>

    <title>Comp Console</title>

Query

    <input id="query" type="text" spellcheck="false" size="120"></input>
    <input type="button" value="Run" onclick="query();"></input>

`

func webFail(w http.ResponseWriter, msg string, args ...interface{}) {
	msg = fmt.Sprintf(msg, args...)
	http.Error(w, msg, http.StatusInternalServerError)
	log.Print(msg)
}

func badReq(w http.ResponseWriter, json string, args ...interface{}) {
	json = fmt.Sprintf(json, args...)
	http.Error(w, json, http.StatusBadRequest)
	log.Print(json)
}

func (c Console) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	io.WriteString(w, QueryPage)
}

func (fq FullQuery) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		dec := json.NewDecoder(r.Body)

		var req struct {
			Query string `json:"query"`
			Limit int    `json:"limit"`
		}
		if err := dec.Decode(&req); err != nil {
			badReq(w, `{"error": "invalid request object: %v"}`, strconv.Quote(err.Error()))
			return
		}

		// TODO: increase the limits after testing
		if req.Limit < 0 || req.Limit > 100 {
			req.Limit = 100
		}

		if err := Group(fq).FullRun(w, req.Query, req.Limit); err != nil {
			badReq(w, err.Error())
			return
		}
	} else {
		badReq(w, `{"error": "%v unsupported method %v"}`, r.URL, r.Method)
	}
}

func (pq PartQuery) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		query, err := ioutil.ReadAll(r.Body)
		if err != nil {
			badReq(w, `{"error": "failed to read query: %v"}`, strconv.Quote(err.Error()))
			return
		}

		limit := 0
		if str := r.URL.Query().Get("limit"); str != "" {
			num, err := strconv.ParseInt(str, 10, 64)
			if err == nil {
				limit = int(num)
			}
		}

		if err := Group(pq).PartRun(w, string(query), limit); err != nil {
			badReq(w, err.Error())
		}
	} else {
		badReq(w, `{"error": "%v unsupported method %v"}`, r.URL, r.Method)
	}
}

// See pprof_remote_servers.html bundled with the gperftools.
func (p Profiler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	switch r.URL.Path {
	case "cmdline":
		for _, arg := range os.Args {
			fmt.Fprintf(w, "%v\n", arg)
		}
	case "profile":
		sec := r.URL.Query()["seconds"]
		if len(sec) > 0 {
			dur, _ := strconv.Atoi(sec[0])
			buf := new(bytes.Buffer)
			pprof.StartCPUProfile(buf)
			time.Sleep(time.Duration(dur) * time.Second)
			pprof.StopCPUProfile()

			buf.WriteTo(w)
		} else {
			webFail(w, "invalid profile request, expected seconds=XX")
		}
	case "memstats":
		var m runtime.MemStats
		runtime.ReadMemStats(&m)
		buf, err := json.MarshalIndent(m, "", "  ")
		if err != nil {
			webFail(w, "failed to marshal object: %v", err)
		} else {
			w.Write(buf)
		}
	case "symbol":
		if r.Method == "GET" {
			fmt.Fprintf(w, "num_symbols: 1")
			return
		}

		buf, err := ioutil.ReadAll(r.Body)
		if err != nil {
			webFail(w, "failed to read request body: %v", err)
			return
		}

		for _, strAddr := range strings.Split(string(buf), "+") {
			strAddr = strings.Trim(strAddr, " \r\n\t")
			desc := "unknownFunc"
			addr, err := strconv.ParseUint(strAddr, 0, 64)
			if err == nil {
				fn := runtime.FuncForPC(uintptr(addr))
				if fn != nil {
					file, line := fn.FileLine(uintptr(addr))
					desc = fmt.Sprintf("%v:%v:%v", path.Base(file), line, fn.Name())
				}
			}
			fmt.Fprintf(w, "%v\t%v\n", strAddr, desc)
		}
	case "":
		for _, p := range pprof.Profiles() {
			fmt.Fprintf(w, "%v\n", p.Name())
		}
	default:
		for _, p := range pprof.Profiles() {
			if p.Name() == r.URL.Path {
				p.WriteTo(w, 0)
				return
			}
		}
		webFail(w, "unknown profile: %v", r.URL.Path)
	}
}
<commit_msg>fixing double qouting on errors<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"
)

type Console int
type Profiler int
type FullQuery Group
type PartQuery Group

const QueryPage = `<!doctype html>

    <title>Comp Console</title>

Query

    <input id="query" type="text" spellcheck="false" size="120"></input>
    <input type="button" value="Run" onclick="query();"></input>

`

func webFail(w http.ResponseWriter, msg string, args ...interface{}) {
	msg = fmt.Sprintf(msg, args...)
	http.Error(w, msg, http.StatusInternalServerError)
	log.Print(msg)
}

func badReq(w http.ResponseWriter, json string, args ...interface{}) {
	json = fmt.Sprintf(json, args...)
	http.Error(w, json, http.StatusBadRequest)
	log.Print(json)
}

func (c Console) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	io.WriteString(w, QueryPage)
}

func (fq FullQuery) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		dec := json.NewDecoder(r.Body)

		var req struct {
			Query string `json:"query"`
			Limit int    `json:"limit"`
		}
		if err := dec.Decode(&req); err != nil {
			badReq(w, `{"error": %v}`, strconv.Quote("invalid request object: "+err.Error()))
			return
		}

		// TODO: increase the limits after testing
		if req.Limit < 0 || req.Limit > 100 {
			req.Limit = 100
		}

		if err := Group(fq).FullRun(w, req.Query, req.Limit); err != nil {
			badReq(w, err.Error())
			return
		}
	} else {
		badReq(w, `{"error": "%v unsupported method %v"}`, r.URL, r.Method)
	}
}

func (pq PartQuery) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		query, err := ioutil.ReadAll(r.Body)
		if err != nil {
			badReq(w, `{"error": %v}`, strconv.Quote("failed to read query: "+err.Error()))
			return
		}

		limit := 0
		if str := r.URL.Query().Get("limit"); str != "" {
			num, err := strconv.ParseInt(str, 10, 64)
			if err == nil {
				limit = int(num)
			}
		}

		if err := Group(pq).PartRun(w, string(query), limit); err != nil {
			badReq(w, err.Error())
		}
	} else {
		badReq(w, `{"error": "%v unsupported method %v"}`, r.URL, r.Method)
	}
}

// See pprof_remote_servers.html bundled with the gperftools.
func (p Profiler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	switch r.URL.Path {
	case "cmdline":
		for _, arg := range os.Args {
			fmt.Fprintf(w, "%v\n", arg)
		}
	case "profile":
		sec := r.URL.Query()["seconds"]
		if len(sec) > 0 {
			dur, _ := strconv.Atoi(sec[0])
			buf := new(bytes.Buffer)
			pprof.StartCPUProfile(buf)
			time.Sleep(time.Duration(dur) * time.Second)
			pprof.StopCPUProfile()

			buf.WriteTo(w)
		} else {
			webFail(w, "invalid profile request, expected seconds=XX")
		}
	case "memstats":
		var m runtime.MemStats
		runtime.ReadMemStats(&m)
		buf, err := json.MarshalIndent(m, "", "  ")
		if err != nil {
			webFail(w, "failed to marshal object: %v", err)
		} else {
			w.Write(buf)
		}
	case "symbol":
		if r.Method == "GET" {
			fmt.Fprintf(w, "num_symbols: 1")
			return
		}

		buf, err := ioutil.ReadAll(r.Body)
		if err != nil {
			webFail(w, "failed to read request body: %v", err)
			return
		}

		for _, strAddr := range strings.Split(string(buf), "+") {
			strAddr = strings.Trim(strAddr, " \r\n\t")
			desc := "unknownFunc"
			addr, err := strconv.ParseUint(strAddr, 0, 64)
			if err == nil {
				fn := runtime.FuncForPC(uintptr(addr))
				if fn != nil {
					file, line := fn.FileLine(uintptr(addr))
					desc = fmt.Sprintf("%v:%v:%v", path.Base(file), line, fn.Name())
				}
			}
			fmt.Fprintf(w, "%v\t%v\n", strAddr, desc)
		}
	case "":
		for _, p := range pprof.Profiles() {
			fmt.Fprintf(w, "%v\n", p.Name())
		}
	default:
		for _, p := range pprof.Profiles() {
			if p.Name() == r.URL.Path {
				p.WriteTo(w, 0)
				return
			}
		}
		webFail(w, "unknown profile: %v", r.URL.Path)
	}
}
<|endoftext|>


<commit_before>package builder

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/Unknwon/com"
	"github.com/go-xiaohei/pugo/app/helper"
	"github.com/go-xiaohei/pugo/app/model"
	"gopkg.in/inconshreveable/log15.v2"
)

type (
	// Source include all sources data
	Source struct {
		Meta      *model.Meta
		Nav       model.NavGroup
		Owner     *model.Author
		Authors   map[string]*model.Author
		Comment   *model.Comment
		Analytics *model.Analytics
		I18n      map[string]*helper.I18n

		Posts    model.Posts
		PostPage int
		Archive  []*model.Archive
		Pages    model.Pages
		Tags     map[string]*model.Tag
		tagPosts map[string]model.Posts
	}
)

// NewSource return new *Source from *MetaAll,
// it returns general *Source object without posts and pages,
// but meta, navigation, authors,comment and analytics are loaded.
func NewSource(all *model.MetaAll) *Source {
	s := &Source{
		Meta:      all.Meta,
		Nav:       all.NavGroup,
		Owner:     all.AuthorGroup[0],
		Comment:   all.Comment,
		Analytics: all.Analytics,
		Authors:   make(map[string]*model.Author),
	}
	for _, a := range all.AuthorGroup {
		s.Authors[a.Name] = a
	}
	return s
}

// ReadSource read source with *Context.
// parse *Context.From and read data to *Context.Source
func ReadSource(ctx *Context) {
	ctx.parseDir()
	if ctx.Err != nil {
		return
	}

	// read meta
	// then read languages,posts and pages together
	metaAll, err := ReadMeta(ctx.srcDir)
	if err != nil {
		ctx.Err = err
		return
	}
	ctx.Source = NewSource(metaAll)

	ctx.GoGroup.Wrap("ReadLang", func() {
		ctx.Source.I18n, ctx.Err = ReadLang(ctx.srcDir)
	})
	ctx.GoGroup.Wrap("ReadPosts", func() {
		ctx.Source.Posts, ctx.Err = ReadPosts(ctx.srcDir)
	})
	ctx.GoGroup.Wrap("ReadPages", func() {
		ctx.Source.Pages, ctx.Err = ReadPages(ctx.srcDir)
	})
	ctx.GoGroup.Wait()
}

// ReadMeta read meta file in srcDir
func ReadMeta(srcDir string) (*model.MetaAll, error) {
	var metaFile string
	for t, f := range model.ShouldMetaFiles() {
		metaFile = filepath.Join(srcDir, f)
		if !com.IsFile(metaFile) {
			continue
		}
		log15.Debug("Build|Load|%s", metaFile)
		bytes, err := ioutil.ReadFile(metaFile)
		if err != nil {
			return nil, err
		}
		meta, err := model.NewMetaAll(bytes, t)
		if err != nil {
			return nil, err
		}
		if meta != nil {
			return meta, nil
		}
	}
	return nil, fmt.Errorf("meta file is missing")
}

// ReadLang read languages in srcDir
func ReadLang(srcDir string) (map[string]*helper.I18n, error) {
	srcDir = filepath.Join(srcDir, "lang")
	if !com.IsDir(srcDir) {
		return nil, nil
	}
	langs := make(map[string]*helper.I18n)
	err := filepath.Walk(srcDir, func(p string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if fi.IsDir() {
			return nil
		}
		ext := filepath.Ext(p)
		if ext == ".toml" || ext == ".ini" {
			log15.Debug("Build|Load|%s", p)
			b, err := ioutil.ReadFile(p)
			if err != nil {
				return err
			}
			lang := strings.TrimSuffix(filepath.Base(p), ext)
			i18n, err := helper.NewI18n(lang, b, ext)
			if err != nil {
				return err
			}
			langs[lang] = i18n
		}
		return nil
	})
	return langs, err
}

// ReadPosts read posts files in srcDir/post
func ReadPosts(srcDir string) ([]*model.Post, error) {
	srcDir = filepath.Join(srcDir, "post")
	if !com.IsDir(srcDir) {
		return nil, fmt.Errorf("posts directory '%s' is missing", srcDir)
	}

	var posts []*model.Post
	err := filepath.Walk(srcDir, func(p string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if fi.IsDir() {
			return nil
		}
		if filepath.Ext(p) == ".md" {
			log15.Debug("Build|Load|%s", p)
			post, err := model.NewPostOfMarkdown(p)
			if err != nil {
				return err
			}
			if post != nil {
				posts = append(posts, post)
			}
		}
		return nil
	})
	sort.Sort(model.Posts(posts))
	return posts, err
}

// ReadPages read pages files in srcDir/page
func ReadPages(srcDir string) ([]*model.Page, error) {
	srcDir = filepath.Join(srcDir, "page")
	if !com.IsDir(srcDir) {
		return nil, fmt.Errorf("pages directory '%s' is missing", srcDir)
	}

	var pages []*model.Page
	err := filepath.Walk(srcDir, func(p string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if fi.IsDir() {
			return nil
		}
		if filepath.Ext(p) == ".md" {
			log15.Debug("Build|Load|%s", p)
			rel, _ := filepath.Rel(srcDir, p)
			rel = strings.TrimSuffix(rel, filepath.Ext(rel))
			page, err := model.NewPageOfMarkdown(p, filepath.ToSlash(rel))
			if err != nil {
				return err
			}
			if page != nil {
				pages = append(pages, page)
			}
		}
		return nil
	})
	return pages, err
}
<commit_msg>when read languages, print error<commit_after>package builder

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/Unknwon/com"
	"github.com/go-xiaohei/pugo/app/helper"
	"github.com/go-xiaohei/pugo/app/model"
	"gopkg.in/inconshreveable/log15.v2"
)

type (
	// Source include all sources data
	Source struct {
		Meta      *model.Meta
		Nav       model.NavGroup
		Owner     *model.Author
		Authors   map[string]*model.Author
		Comment   *model.Comment
		Analytics *model.Analytics
		I18n      map[string]*helper.I18n

		Posts    model.Posts
		PostPage int
		Archive  []*model.Archive
		Pages    model.Pages
		Tags     map[string]*model.Tag
		tagPosts map[string]model.Posts
	}
)

// NewSource return new *Source from *MetaAll,
// it returns general *Source object without posts and pages,
// but meta, navigation, authors,comment and analytics are loaded.
func NewSource(all *model.MetaAll) *Source {
	s := &Source{
		Meta:      all.Meta,
		Nav:       all.NavGroup,
		Owner:     all.AuthorGroup[0],
		Comment:   all.Comment,
		Analytics: all.Analytics,
		Authors:   make(map[string]*model.Author),
	}
	for _, a := range all.AuthorGroup {
		s.Authors[a.Name] = a
	}
	return s
}

// ReadSource read source with *Context.
// parse *Context.From and read data to *Context.Source
func ReadSource(ctx *Context) {
	ctx.parseDir()
	if ctx.Err != nil {
		return
	}

	// read meta
	// then read languages,posts and pages together
	metaAll, err := ReadMeta(ctx.srcDir)
	if err != nil {
		ctx.Err = err
		return
	}
	ctx.Source = NewSource(metaAll)

	ctx.GoGroup.Wrap("ReadLang", func() {
		ctx.Source.I18n = ReadLang(ctx.srcDir)
	})
	ctx.GoGroup.Wrap("ReadPosts", func() {
		ctx.Source.Posts, ctx.Err = ReadPosts(ctx.srcDir)
	})
	ctx.GoGroup.Wrap("ReadPages", func() {
		ctx.Source.Pages, ctx.Err = ReadPages(ctx.srcDir)
	})
	ctx.GoGroup.Wait()
}

// ReadMeta read meta file in srcDir
func ReadMeta(srcDir string) (*model.MetaAll, error) {
	var metaFile string
	for t, f := range model.ShouldMetaFiles() {
		metaFile = filepath.Join(srcDir, f)
		if !com.IsFile(metaFile) {
			continue
		}
		log15.Debug("Build|Load|%s", metaFile)
		bytes, err := ioutil.ReadFile(metaFile)
		if err != nil {
			return nil, err
		}
		meta, err := model.NewMetaAll(bytes, t)
		if err != nil {
			return nil, err
		}
		if meta != nil {
			return meta, nil
		}
	}
	return nil, fmt.Errorf("meta file is missing")
}

// ReadLang read languages in srcDir
func ReadLang(srcDir string) map[string]*helper.I18n {
	srcDir = filepath.Join(srcDir, "lang")
	if !com.IsDir(srcDir) {
		return nil
	}
	langs := make(map[string]*helper.I18n)
	filepath.Walk(srcDir, func(p string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if fi.IsDir() {
			return nil
		}
		ext := filepath.Ext(p)
		if ext == ".toml" || ext == ".ini" {
			log15.Debug("Build|Load|%s", p)
			b, err := ioutil.ReadFile(p)
			if err != nil {
				log15.Warn("Build|Lang|%s|%v", p, err)
				return err
			}
			lang := strings.TrimSuffix(filepath.Base(p), ext)
			i18n, err := helper.NewI18n(lang, b, ext)
			if err != nil {
				log15.Warn("Build|Lang|%s|%v", p, err)
				return err
			}
			langs[lang] = i18n
		}
		return nil
	})
	return langs
}

// ReadPosts read posts files in srcDir/post
func ReadPosts(srcDir string) ([]*model.Post, error) {
	srcDir = filepath.Join(srcDir, "post")
	if !com.IsDir(srcDir) {
		return nil, fmt.Errorf("posts directory '%s' is missing", srcDir)
	}

	var posts []*model.Post
	err := filepath.Walk(srcDir, func(p string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if fi.IsDir() {
			return nil
		}
		if filepath.Ext(p) == ".md" {
			log15.Debug("Build|Load|%s", p)
			post, err := model.NewPostOfMarkdown(p)
			if err != nil {
				return err
			}
			if post != nil {
				posts = append(posts, post)
			}
		}
		return nil
	})
	sort.Sort(model.Posts(posts))
	return posts, err
}

// ReadPages read pages files in srcDir/page
func ReadPages(srcDir string) ([]*model.Page, error) {
	srcDir = filepath.Join(srcDir, "page")
	if !com.IsDir(srcDir) {
		return nil, fmt.Errorf("pages directory '%s' is missing", srcDir)
	}

	var pages []*model.Page
	err := filepath.Walk(srcDir, func(p string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if fi.IsDir() {
			return nil
		}
		if filepath.Ext(p) == ".md" {
			log15.Debug("Build|Load|%s", p)
			rel, _ := filepath.Rel(srcDir, p)
			rel = strings.TrimSuffix(rel, filepath.Ext(rel))
			page, err := model.NewPageOfMarkdown(p, filepath.ToSlash(rel))
			if err != nil {
				return err
			}
			if page != nil {
				pages = append(pages, page)
			}
		}
		return nil
	})
	return pages, err
}
<|endoftext|>


<commit_before>package analyzer

import (
	"fmt"
	"github.com/cloudfoundry/hm9000/config"
	"github.com/cloudfoundry/hm9000/helpers/logger"
	"github.com/cloudfoundry/hm9000/models"
	"strconv"
	"time"
)

type appAnalyzer struct {
	app                          *models.App
	conf                         config.Config
	existingPendingStartMessages map[string]models.PendingStartMessage
	existingPendingStopMessages  map[string]models.PendingStopMessage
	currentTime                  time.Time
	logger                       logger.Logger

	startMessages map[string]models.PendingStartMessage
	stopMessages  map[string]models.PendingStopMessage
	crashCounts   []models.CrashCount
}

func newAppAnalyzer(app *models.App, currentTime time.Time, existingPendingStartMessages map[string]models.PendingStartMessage, existingPendingStopMessages map[string]models.PendingStopMessage, logger logger.Logger, conf config.Config) *appAnalyzer {
	return &appAnalyzer{
		app:  app,
		conf: conf,
		existingPendingStartMessages: existingPendingStartMessages,
		existingPendingStopMessages:  existingPendingStopMessages,
		currentTime:                  currentTime,
		logger:                       logger,
		startMessages:                make(map[string]models.PendingStartMessage, 0),
		stopMessages:                 make(map[string]models.PendingStopMessage, 0),
		crashCounts:                  make([]models.CrashCount, 0),
	}
}

func (a *appAnalyzer) analyzeApp() (map[string]models.PendingStartMessage, map[string]models.PendingStopMessage, []models.CrashCount) {
	priority := a.computePendingStartMessagePriority()
	a.generatePendingStartsForMissingInstances(priority)
	a.generatePendingStartsForCrashedInstances(priority)
	a.generatePendingStartsAndStopsForEvacuatingInstances()

	if len(a.startMessages) == 0 {
		a.generatePendingStopsForExtraInstances()
		a.generatePendingStopsForDuplicateInstances()
	}

	return a.startMessages, a.stopMessages, a.crashCounts
}

func (a *appAnalyzer) generatePendingStartsForMissingInstances(priority float64) {
	for index := 0; a.app.IsIndexDesired(index); index++ {
		if !a.app.HasStartingOrRunningInstanceAtIndex(index) && !a.app.HasCrashedInstanceAtIndex(index) {
			message := models.NewPendingStartMessage(a.currentTime, a.conf.GracePeriod(), 0, a.app.AppGuid, a.app.AppVersion, index, priority)

			a.appendStartMessageIfNotDuplicate(message, "Identified missing instance", map[string]string{
				"Desired # of Instances": strconv.Itoa(a.app.NumberOfDesiredInstances()),
			})
		}
	}

	return
}

func (a *appAnalyzer) generatePendingStartsForCrashedInstances(priority float64) (crashCounts []models.CrashCount) {
	for index := 0; a.app.IsIndexDesired(index); index++ {
		if !a.app.HasStartingOrRunningInstanceAtIndex(index) && a.app.HasCrashedInstanceAtIndex(index) {
			if index != 0 && !a.app.HasStartingOrRunningInstances() {
				continue
			}

			crashCount := a.app.CrashCountAtIndex(index, a.currentTime)
			delay := a.computeDelayForCrashCount(crashCount)
			message := models.NewPendingStartMessage(a.currentTime, delay, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, index, priority)

			didAppend := a.appendStartMessageIfNotDuplicate(message, "Identified crashed instance", map[string]string{
				"Desired # of Instances": strconv.Itoa(a.app.NumberOfDesiredInstances()),
			})

			if didAppend {
				crashCount.CrashCount += 1
				a.crashCounts = append(a.crashCounts, crashCount)
			}
		}
	}

	return
}

func (a *appAnalyzer) generatePendingStopsForExtraInstances() {
	for _, extraInstance := range a.app.ExtraStartingOrRunningInstances() {
		message := models.NewPendingStopMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, extraInstance.InstanceGuid)

		a.appendStopMessageIfNotDuplicate(message, "Identified extra running instance", map[string]string{
			"InstanceIndex":          strconv.Itoa(extraInstance.InstanceIndex),
			"Desired # of Instances": strconv.Itoa(a.app.NumberOfDesiredInstances()),
		})
	}

	return
}

func (a *appAnalyzer) generatePendingStopsForDuplicateInstances() {
	//stop duplicate instances at indices < numDesired
	//this works by scheduling stops for *all* duplicate instances at increasing delays
	//the sender will process the stops one at a time and only send stops that don't put
	//the system in an invalid state
	for index := 0; a.app.IsIndexDesired(index); index++ {
		instances := a.app.StartingOrRunningInstancesAtIndex(index)
		if len(instances) > 1 {
			for i, instance := range instances {
				delay := (i + 1) * a.conf.GracePeriod()
				message := models.NewPendingStopMessage(a.currentTime, delay, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, instance.InstanceGuid)

				a.appendStopMessageIfNotDuplicate(message, "Identified duplicate running instance", map[string]string{
					"InstanceIndex": strconv.Itoa(instance.InstanceIndex),
				})
			}
		}
	}

	return
}

func (a *appAnalyzer) generatePendingStartsAndStopsForEvacuatingInstances() {
	heartbeatsByIndex := a.app.HeartbeatsByIndex()

	for index, _ := range heartbeatsByIndex {
		evacuatingInstances := a.app.EvacuatingInstancesAtIndex(index)

		if len(evacuatingInstances) > 0 {
			startMessage := models.NewPendingStartMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, index, 2.0)
			addStopMessages := func(reason string) {
				for _, evacuatingInstance := range evacuatingInstances {
					stopMessage := models.NewPendingStopMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, evacuatingInstance.InstanceGuid)
					a.appendStopMessageIfNotDuplicate(stopMessage, reason, map[string]string{})
				}
			}

			if !a.app.IsIndexDesired(index) {
				addStopMessages("Identified undesired evacuating instance.")
				continue
			}

			if a.app.HasRunningInstanceAtIndex(index) {
				addStopMessages("Stopping an evacuating instance that has started running elsewhere.")
				continue
			}

			if a.app.HasStartingInstanceAtIndex(index) {
				continue
			}

			a.appendStartMessageIfNotDuplicate(startMessage, "An instance is evacuating.  Starting it elsewhere.", map[string]string{})

			if a.app.CrashCountAtIndex(index, a.currentTime).CrashCount >= a.conf.NumberOfCrashesBeforeBackoffBegins {
				addStopMessages("Stopping an unstable evacuating instance.")
			}
		}
	}
}

func (a *appAnalyzer) appendStartMessageIfNotDuplicate(message models.PendingStartMessage, loggingMessage string, additionalDetails map[string]string) (didAppend bool) {
	existingMessage, alreadyQueued := a.existingPendingStartMessages[message.StoreKey()]
	if !alreadyQueued {
		a.logger.Info(fmt.Sprintf("Enqueuing Start Message: %s", loggingMessage), message.LogDescription(), additionalDetails)
		a.startMessages[message.StoreKey()] = message
		return true
	} else {
		a.logger.Info(fmt.Sprintf("Skipping Already Enqueued Start Message: %s", loggingMessage), existingMessage.LogDescription(), additionalDetails)
		return false
	}
}

func (a *appAnalyzer) appendStopMessageIfNotDuplicate(message models.PendingStopMessage, loggingMessage string, additionalDetails map[string]string) {
	existingMessage, alreadyQueued := a.existingPendingStopMessages[message.StoreKey()]
	if !alreadyQueued {
		a.logger.Info(fmt.Sprintf("Enqueuing Stop Message: %s", loggingMessage), message.LogDescription(), additionalDetails)
		a.stopMessages[message.StoreKey()] = message
	} else {
		a.logger.Info(fmt.Sprintf("Skipping Already Enqueued Stop Message: %s", loggingMessage), existingMessage.LogDescription(), additionalDetails)
	}
}

func (a *appAnalyzer) computePendingStartMessagePriority() float64 {
	numberOfMissingIndices := a.app.NumberOfDesiredInstances() - a.app.NumberOfDesiredIndicesWithAStartingOrRunningInstance()

	return float64(numberOfMissingIndices) / float64(a.app.NumberOfDesiredInstances())
}

func (a *appAnalyzer) computeDelayForCrashCount(crashCount models.CrashCount) (delay int) {
	startingBackoffDelay := int(a.conf.StartingBackoffDelay().Seconds())
	maximumBackoffDelay := int(a.conf.MaximumBackoffDelay().Seconds())
	return ComputeCrashDelay(crashCount.CrashCount, a.conf.NumberOfCrashesBeforeBackoffBegins, startingBackoffDelay, maximumBackoffDelay)
}
<commit_msg>log crash count when enqueing a start message for a crash<commit_after>package analyzer

import (
	"fmt"
	"github.com/cloudfoundry/hm9000/config"
	"github.com/cloudfoundry/hm9000/helpers/logger"
	"github.com/cloudfoundry/hm9000/models"
	"strconv"
	"time"
)

type appAnalyzer struct {
	app                          *models.App
	conf                         config.Config
	existingPendingStartMessages map[string]models.PendingStartMessage
	existingPendingStopMessages  map[string]models.PendingStopMessage
	currentTime                  time.Time
	logger                       logger.Logger

	startMessages map[string]models.PendingStartMessage
	stopMessages  map[string]models.PendingStopMessage
	crashCounts   []models.CrashCount
}

func newAppAnalyzer(app *models.App, currentTime time.Time, existingPendingStartMessages map[string]models.PendingStartMessage, existingPendingStopMessages map[string]models.PendingStopMessage, logger logger.Logger, conf config.Config) *appAnalyzer {
	return &appAnalyzer{
		app:  app,
		conf: conf,
		existingPendingStartMessages: existingPendingStartMessages,
		existingPendingStopMessages:  existingPendingStopMessages,
		currentTime:                  currentTime,
		logger:                       logger,
		startMessages:                make(map[string]models.PendingStartMessage, 0),
		stopMessages:                 make(map[string]models.PendingStopMessage, 0),
		crashCounts:                  make([]models.CrashCount, 0),
	}
}

func (a *appAnalyzer) analyzeApp() (map[string]models.PendingStartMessage, map[string]models.PendingStopMessage, []models.CrashCount) {
	priority := a.computePendingStartMessagePriority()
	a.generatePendingStartsForMissingInstances(priority)
	a.generatePendingStartsForCrashedInstances(priority)
	a.generatePendingStartsAndStopsForEvacuatingInstances()

	if len(a.startMessages) == 0 {
		a.generatePendingStopsForExtraInstances()
		a.generatePendingStopsForDuplicateInstances()
	}

	return a.startMessages, a.stopMessages, a.crashCounts
}

func (a *appAnalyzer) generatePendingStartsForMissingInstances(priority float64) {
	for index := 0; a.app.IsIndexDesired(index); index++ {
		if !a.app.HasStartingOrRunningInstanceAtIndex(index) && !a.app.HasCrashedInstanceAtIndex(index) {
			message := models.NewPendingStartMessage(a.currentTime, a.conf.GracePeriod(), 0, a.app.AppGuid, a.app.AppVersion, index, priority)

			a.appendStartMessageIfNotDuplicate(message, "Identified missing instance", map[string]string{
				"Desired # of Instances": strconv.Itoa(a.app.NumberOfDesiredInstances()),
			})
		}
	}

	return
}

func (a *appAnalyzer) generatePendingStartsForCrashedInstances(priority float64) (crashCounts []models.CrashCount) {
	for index := 0; a.app.IsIndexDesired(index); index++ {
		if !a.app.HasStartingOrRunningInstanceAtIndex(index) && a.app.HasCrashedInstanceAtIndex(index) {
			if index != 0 && !a.app.HasStartingOrRunningInstances() {
				continue
			}

			crashCount := a.app.CrashCountAtIndex(index, a.currentTime)
			delay := a.computeDelayForCrashCount(crashCount)
			message := models.NewPendingStartMessage(a.currentTime, delay, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, index, priority)

			didAppend := a.appendStartMessageIfNotDuplicate(message, "Identified crashed instance", map[string]string{
				"Desired # of Instances": strconv.Itoa(a.app.NumberOfDesiredInstances()),
				"Crash Count":            strconv.Itoa(crashCount.CrashCount),
			})

			if didAppend {
				crashCount.CrashCount += 1
				a.crashCounts = append(a.crashCounts, crashCount)
			}
		}
	}

	return
}

func (a *appAnalyzer) generatePendingStopsForExtraInstances() {
	for _, extraInstance := range a.app.ExtraStartingOrRunningInstances() {
		message := models.NewPendingStopMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, extraInstance.InstanceGuid)

		a.appendStopMessageIfNotDuplicate(message, "Identified extra running instance", map[string]string{
			"InstanceIndex":          strconv.Itoa(extraInstance.InstanceIndex),
			"Desired # of Instances": strconv.Itoa(a.app.NumberOfDesiredInstances()),
		})
	}

	return
}

func (a *appAnalyzer) generatePendingStopsForDuplicateInstances() {
	//stop duplicate instances at indices < numDesired
	//this works by scheduling stops for *all* duplicate instances at increasing delays
	//the sender will process the stops one at a time and only send stops that don't put
	//the system in an invalid state
	for index := 0; a.app.IsIndexDesired(index); index++ {
		instances := a.app.StartingOrRunningInstancesAtIndex(index)
		if len(instances) > 1 {
			for i, instance := range instances {
				delay := (i + 1) * a.conf.GracePeriod()
				message := models.NewPendingStopMessage(a.currentTime, delay, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, instance.InstanceGuid)

				a.appendStopMessageIfNotDuplicate(message, "Identified duplicate running instance", map[string]string{
					"InstanceIndex": strconv.Itoa(instance.InstanceIndex),
				})
			}
		}
	}

	return
}

func (a *appAnalyzer) generatePendingStartsAndStopsForEvacuatingInstances() {
	heartbeatsByIndex := a.app.HeartbeatsByIndex()

	for index, _ := range heartbeatsByIndex {
		evacuatingInstances := a.app.EvacuatingInstancesAtIndex(index)

		if len(evacuatingInstances) > 0 {
			startMessage := models.NewPendingStartMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, index, 2.0)
			addStopMessages := func(reason string) {
				for _, evacuatingInstance := range evacuatingInstances {
					stopMessage := models.NewPendingStopMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, evacuatingInstance.InstanceGuid)
					a.appendStopMessageIfNotDuplicate(stopMessage, reason, map[string]string{})
				}
			}

			if !a.app.IsIndexDesired(index) {
				addStopMessages("Identified undesired evacuating instance.")
				continue
			}

			if a.app.HasRunningInstanceAtIndex(index) {
				addStopMessages("Stopping an evacuating instance that has started running elsewhere.")
				continue
			}

			if a.app.HasStartingInstanceAtIndex(index) {
				continue
			}

			a.appendStartMessageIfNotDuplicate(startMessage, "An instance is evacuating.  Starting it elsewhere.", map[string]string{})

			if a.app.CrashCountAtIndex(index, a.currentTime).CrashCount >= a.conf.NumberOfCrashesBeforeBackoffBegins {
				addStopMessages("Stopping an unstable evacuating instance.")
			}
		}
	}
}

func (a *appAnalyzer) appendStartMessageIfNotDuplicate(message models.PendingStartMessage, loggingMessage string, additionalDetails map[string]string) (didAppend bool) {
	existingMessage, alreadyQueued := a.existingPendingStartMessages[message.StoreKey()]
	if !alreadyQueued {
		a.logger.Info(fmt.Sprintf("Enqueuing Start Message: %s", loggingMessage), message.LogDescription(), additionalDetails)
		a.startMessages[message.StoreKey()] = message
		return true
	} else {
		a.logger.Info(fmt.Sprintf("Skipping Already Enqueued Start Message: %s", loggingMessage), existingMessage.LogDescription(), additionalDetails)
		return false
	}
}

func (a *appAnalyzer) appendStopMessageIfNotDuplicate(message models.PendingStopMessage, loggingMessage string, additionalDetails map[string]string) {
	existingMessage, alreadyQueued := a.existingPendingStopMessages[message.StoreKey()]
	if !alreadyQueued {
		a.logger.Info(fmt.Sprintf("Enqueuing Stop Message: %s", loggingMessage), message.LogDescription(), additionalDetails)
		a.stopMessages[message.StoreKey()] = message
	} else {
		a.logger.Info(fmt.Sprintf("Skipping Already Enqueued Stop Message: %s", loggingMessage), existingMessage.LogDescription(), additionalDetails)
	}
}

func (a *appAnalyzer) computePendingStartMessagePriority() float64 {
	numberOfMissingIndices := a.app.NumberOfDesiredInstances() - a.app.NumberOfDesiredIndicesWithAStartingOrRunningInstance()

	return float64(numberOfMissingIndices) / float64(a.app.NumberOfDesiredInstances())
}

func (a *appAnalyzer) computeDelayForCrashCount(crashCount models.CrashCount) (delay int) {
	startingBackoffDelay := int(a.conf.StartingBackoffDelay().Seconds())
	maximumBackoffDelay := int(a.conf.MaximumBackoffDelay().Seconds())
	return ComputeCrashDelay(crashCount.CrashCount, a.conf.NumberOfCrashesBeforeBackoffBegins, startingBackoffDelay, maximumBackoffDelay)
}
<|endoftext|>


<commit_before>package dpn_test

import (
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/APTrust/bagman/dpn"
	"os"
	"path/filepath"
	"testing"
)

// Many of the variables and functions used in this file
// come from dpnrestclient_test.go.

// Make sure that the DPNHome and DPNStaging directories exist,
// and that the files that we expect to be present in testing
// are there.
func recorderTestEnsureFiles(t *testing.T, procUtil *bagman.ProcessUtil) {
	dirs := []string {
		procUtil.Config.DPNStagingDirectory,
		filepath.Join(procUtil.Config.DPNHomeDirectory, "dpn.aptrust", "outbound"),
		filepath.Join(procUtil.Config.DPNHomeDirectory, "dpn.chron", "outbound"),
		filepath.Join(procUtil.Config.DPNHomeDirectory, "dpn.hathi", "outbound"),
		filepath.Join(procUtil.Config.DPNHomeDirectory, "dpn.sdr", "outbound"),
		filepath.Join(procUtil.Config.DPNHomeDirectory, "dpn.tdr", "outbound"),
	}
	for _, dir := range dirs {
		if !bagman.FileExists(dir) {
			os.MkdirAll(dir, 0755)
		}
	}
}

func getRecorder(t *testing.T) (*dpn.Recorder) {
	procUtil := bagman.NewProcessUtil(&testConfig)
	dpnConfig := loadConfig(t, configFile)

	recorder, err := dpn.NewRecorder(procUtil, dpnConfig)
	if err != nil {
		t.Error(err)
		return nil
	}
	return recorder
}

func buildResultWithTransfer(t *testing.T, recorder *dpn.Recorder) (*dpn.DPNResult) {
	dpnBag, err := recorder.LocalRESTClient.DPNBagGet(aptrustBagIdentifier)
	if err != nil {
		t.Error(err)
		return nil
	}
	// In our test data fixtures for the local DPN REST cluster,
	// the transfers with id <namespace>-13 to <namespace>-18 are
	// transfers to APTrust. So tdr-18, sdr-18, chron-18, etc. are
	// all bound for APTrust.
	xfer, err := recorder.RemoteClients["hathi"].ReplicationTransferGet("hathi-18")
	if err != nil {
		t.Error(err)
		return nil
	}
	bag, err := recorder.RemoteClients["hathi"].DPNBagGet(xfer.UUID)
	if err != nil {
		t.Error(err)
		return nil
	}
	if len(bag.Fixities) < 1 {
		t.Errorf("Bag %s has no fixity value!", bag.UUID)
		return nil
	}
	result := dpn.NewDPNResult("")
	result.DPNBag = dpnBag
	result.TransferRequest = xfer
	result.BagSha256Digest = bag.Fixities[0].Sha256
	result.BagMd5Digest = "SomeFakeValue"
	return result
}

// This DPNResult has a BagIdentifier and no TransferRequest,
// mimicking a result for a locally-ingested bag.
func buildLocalResult(t *testing.T, recorder *dpn.Recorder) (*dpn.DPNResult) {
	result := dpn.NewDPNResult("test.edu/test.edu.bag6")
	result.DPNBag = makeBag() // defined in dpnrestclient_test.go
	result.DPNBag.LocalId = "test.edu/test.edu.bag6"
	result.StorageURL = fmt.Sprintf("http://fakeurl.kom/%s", result.DPNBag.UUID)
	return result
}

func TestLocalBag(t *testing.T) {
	if runRestTests(t) == false {
		// Local DPN REST not running.
		return
	}
	if canRunTests() == false {
		// Local Fluctus not running.
		return
	}
	recorder := getRecorder(t)
	dpnResult := buildLocalResult(t, recorder)
	recorderTestEnsureFiles(t, recorder.ProcUtil)

	// Make a dummy file so the symlink operation in
	// recorder.go doesn't fail.
	filePath := filepath.Join(
		recorder.ProcUtil.Config.DPNStagingDirectory,
		dpnResult.DPNBag.UUID + ".tar")
	_, err := os.Create(filePath)
	if err != nil {
		t.Errorf("Error creating empty file %s: %v",
			filePath, err)
		return
	}
	defer os.Remove(filePath)

	// Run the test
	recorder.RunTest(dpnResult)
	if dpnResult.ErrorMessage != "" {
		t.Errorf(dpnResult.ErrorMessage)
	}

	if dpnResult.RecordResult.DPNBagCreatedAt.IsZero() {
		t.Errorf("DPNBagCreatedAt was not set")
	}
	// Note that this test will fail if the DPN REST service
	// is configured so that APTrust replicates to fewer than
	// two nodes. This shouldn't happen, since the whole point
	// of DPN is to replicate to at least two other nodes. But
	// if you get a failure here, check the ReplicateTo property
	// of the node entry for APTrust on the local REST service.
	if len(dpnResult.RecordResult.DPNReplicationRequests) < recorder.DPNConfig.ReplicateToNumNodes {
		t.Errorf("Replication requests generated for %d nodes, expected %d",
			dpnResult.RecordResult.DPNReplicationRequests,  recorder.DPNConfig.ReplicateToNumNodes)
	}
	if dpnResult.RecordResult.PremisIngestEventId == "" {
		t.Errorf("PremisIngestEventId was not set")
	}
	if dpnResult.RecordResult.PremisIdentifierEventId == "" {
		t.Errorf("PremisIdentifierEventId was not set")
	}
}

func TestReplicatedBag(t *testing.T) {
	if runRestTests(t) == false {
		return
	}
	recorder := getRecorder(t)
	dpnResult := buildResultWithTransfer(t, recorder)
	recorderTestEnsureFiles(t, recorder.ProcUtil)

	// Test a bag that was copied and validated
	// but not stored.
	filePath := filepath.Join(
		recorder.ProcUtil.Config.DPNStagingDirectory,
		dpnResult.DPNBag.UUID + ".tar")
	dpnResult.CopyResult.LocalPath = filePath
	dpnResult.ValidationResult = &dpn.ValidationResult{
		TarFilePath: filePath,
	}

	// Run the test...
	recorder.RunTest(dpnResult)
	if dpnResult.ErrorMessage != "" {
		t.Errorf(dpnResult.ErrorMessage)
	}

	// Make sure RecordResult items were set correctly.
	if dpnResult.RecordResult.CopyReceiptSentAt.IsZero() {
		t.Errorf("CopyReceiptSentAt was not set")
	}
	if !dpnResult.RecordResult.StorageResultSentAt.IsZero() {
		t.Errorf("StorageResultSentAt was set when it should not have been")
	}

	// Test a bag that was stored
	dpnResult.StorageURL = "https://www.yahoo.com"

	// Run the test again
	recorder.RunTest(dpnResult)
	if dpnResult.ErrorMessage != "" {
		t.Errorf(dpnResult.ErrorMessage)
	}

	// Check status...
	if dpnResult.RecordResult.StorageResultSentAt.IsZero() {
		t.Errorf("StorageResultSentAt was not set")
	}
}
<commit_msg>Add ProcessStatus to result in recorder_test.go to prevent seg fault<commit_after>package dpn_test

import (
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/APTrust/bagman/dpn"
	"os"
	"path/filepath"
	"testing"
	"time"
)

// Many of the variables and functions used in this file
// come from dpnrestclient_test.go.

// Make sure that the DPNHome and DPNStaging directories exist,
// and that the files that we expect to be present in testing
// are there.
func recorderTestEnsureFiles(t *testing.T, procUtil *bagman.ProcessUtil) {
	dirs := []string {
		procUtil.Config.DPNStagingDirectory,
		filepath.Join(procUtil.Config.DPNHomeDirectory, "dpn.aptrust", "outbound"),
		filepath.Join(procUtil.Config.DPNHomeDirectory, "dpn.chron", "outbound"),
		filepath.Join(procUtil.Config.DPNHomeDirectory, "dpn.hathi", "outbound"),
		filepath.Join(procUtil.Config.DPNHomeDirectory, "dpn.sdr", "outbound"),
		filepath.Join(procUtil.Config.DPNHomeDirectory, "dpn.tdr", "outbound"),
	}
	for _, dir := range dirs {
		if !bagman.FileExists(dir) {
			os.MkdirAll(dir, 0755)
		}
	}
}

func getRecorder(t *testing.T) (*dpn.Recorder) {
	procUtil := bagman.NewProcessUtil(&testConfig)
	dpnConfig := loadConfig(t, configFile)

	recorder, err := dpn.NewRecorder(procUtil, dpnConfig)
	if err != nil {
		t.Error(err)
		return nil
	}
	return recorder
}

func buildResultWithTransfer(t *testing.T, recorder *dpn.Recorder) (*dpn.DPNResult) {
	dpnBag, err := recorder.LocalRESTClient.DPNBagGet(aptrustBagIdentifier)
	if err != nil {
		t.Error(err)
		return nil
	}
	// In our test data fixtures for the local DPN REST cluster,
	// the transfers with id <namespace>-13 to <namespace>-18 are
	// transfers to APTrust. So tdr-18, sdr-18, chron-18, etc. are
	// all bound for APTrust.
	xfer, err := recorder.RemoteClients["hathi"].ReplicationTransferGet("hathi-18")
	if err != nil {
		t.Error(err)
		return nil
	}
	bag, err := recorder.RemoteClients["hathi"].DPNBagGet(xfer.UUID)
	if err != nil {
		t.Error(err)
		return nil
	}
	if len(bag.Fixities) < 1 {
		t.Errorf("Bag %s has no fixity value!", bag.UUID)
		return nil
	}
	result := dpn.NewDPNResult("")
	result.DPNBag = dpnBag
	result.TransferRequest = xfer
	result.BagSha256Digest = bag.Fixities[0].Sha256
	result.BagMd5Digest = "SomeFakeValue"
	return result
}

// This DPNResult has a BagIdentifier and no TransferRequest,
// mimicking a result for a locally-ingested bag.
func buildLocalResult(t *testing.T, recorder *dpn.Recorder) (*dpn.DPNResult) {
	result := dpn.NewDPNResult("test.edu/test.edu.bag6")
	result.DPNBag = makeBag() // defined in dpnrestclient_test.go
	result.DPNBag.LocalId = "test.edu/test.edu.bag6"
	result.StorageURL = fmt.Sprintf("http://fakeurl.kom/%s", result.DPNBag.UUID)

	ps := &bagman.ProcessStatus{
		ObjectIdentifier: "test.edu/test.edu.bag6",
		Name: "Test Bag Six",
		Bucket: "bukkety-poo",
		ETag: "12345678",
		BagDate: time.Now(),
		Institution: "test.edu",
		Action: "DPN",
		Stage: "Requested",
		Status: "Pending",
		Note: "Requested...",
		Outcome: "Requested...",
		Retry: true,
	}
	err := recorder.ProcUtil.FluctusClient.UpdateProcessedItem(ps)
	if err != nil {
		t.Errorf("Could not create Fluctus ProcessedItem to test DPN ingest: %v", err)
		return nil
	}
	result.FluctusProcessStatus = ps
	return result
}

func TestLocalBag(t *testing.T) {
	if runRestTests(t) == false {
		// Local DPN REST not running.
		return
	}
	if canRunTests() == false {
		// Local Fluctus not running.
		return
	}
	recorder := getRecorder(t)
	dpnResult := buildLocalResult(t, recorder)
	if dpnResult == nil {
		t.Errorf("Cannot perform TestLocalBag due to previous errors")
		return
	}
	recorderTestEnsureFiles(t, recorder.ProcUtil)

	// Make a dummy file so the symlink operation in
	// recorder.go doesn't fail.
	filePath := filepath.Join(
		recorder.ProcUtil.Config.DPNStagingDirectory,
		dpnResult.DPNBag.UUID + ".tar")
	_, err := os.Create(filePath)
	if err != nil {
		t.Errorf("Error creating empty file %s: %v",
			filePath, err)
		return
	}
	defer os.Remove(filePath)

	// Run the test
	recorder.RunTest(dpnResult)
	if dpnResult.ErrorMessage != "" {
		t.Errorf(dpnResult.ErrorMessage)
	}

	if dpnResult.RecordResult.DPNBagCreatedAt.IsZero() {
		t.Errorf("DPNBagCreatedAt was not set")
	}
	// Note that this test will fail if the DPN REST service
	// is configured so that APTrust replicates to fewer than
	// two nodes. This shouldn't happen, since the whole point
	// of DPN is to replicate to at least two other nodes. But
	// if you get a failure here, check the ReplicateTo property
	// of the node entry for APTrust on the local REST service.
	if len(dpnResult.RecordResult.DPNReplicationRequests) < recorder.DPNConfig.ReplicateToNumNodes {
		t.Errorf("Replication requests generated for %d nodes, expected %d",
			dpnResult.RecordResult.DPNReplicationRequests,  recorder.DPNConfig.ReplicateToNumNodes)
	}
	if dpnResult.RecordResult.PremisIngestEventId == "" {
		t.Errorf("PremisIngestEventId was not set")
	}
	if dpnResult.RecordResult.PremisIdentifierEventId == "" {
		t.Errorf("PremisIdentifierEventId was not set")
	}
}

func TestReplicatedBag(t *testing.T) {
	if runRestTests(t) == false {
		return
	}
	recorder := getRecorder(t)
	dpnResult := buildResultWithTransfer(t, recorder)
	recorderTestEnsureFiles(t, recorder.ProcUtil)

	// Test a bag that was copied and validated
	// but not stored.
	filePath := filepath.Join(
		recorder.ProcUtil.Config.DPNStagingDirectory,
		dpnResult.DPNBag.UUID + ".tar")
	dpnResult.CopyResult.LocalPath = filePath
	dpnResult.ValidationResult = &dpn.ValidationResult{
		TarFilePath: filePath,
	}

	// Run the test...
	recorder.RunTest(dpnResult)
	if dpnResult.ErrorMessage != "" {
		t.Errorf(dpnResult.ErrorMessage)
	}

	// Make sure RecordResult items were set correctly.
	if dpnResult.RecordResult.CopyReceiptSentAt.IsZero() {
		t.Errorf("CopyReceiptSentAt was not set")
	}
	if !dpnResult.RecordResult.StorageResultSentAt.IsZero() {
		t.Errorf("StorageResultSentAt was set when it should not have been")
	}

	// Test a bag that was stored
	dpnResult.StorageURL = "https://www.yahoo.com"

	// Run the test again
	recorder.RunTest(dpnResult)
	if dpnResult.ErrorMessage != "" {
		t.Errorf(dpnResult.ErrorMessage)
	}

	// Check status...
	if dpnResult.RecordResult.StorageResultSentAt.IsZero() {
		t.Errorf("StorageResultSentAt was not set")
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"
	"path"
	"runtime"

	"github.com/BurntSushi/toml"
	"github.com/spf13/viper"
	"github.com/youngpm/gbdx"
)

// ensureGBDXDir will create the gbdx directory if it doesn't already exist.
func ensureGBDXDir() (string, error) {
	gbdxPath := path.Join(userHomeDir(), ".gbdx")
	err := os.MkdirAll(gbdxPath, 0600)
	return gbdxPath, err
}

// userHomeDir returns the home directory of the user.  I've borrowed
// this from https://github.com/spf13/viper/blob/master/util.go .
func userHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}
	return os.Getenv("HOME")
}

func apiFromConfig() (*gbdx.Api, error) {
	var profile GBDXProfile
	if err := viper.Unmarshal(&profile); err != nil {
		return nil, err
	}
	return gbdx.NewApi(profile.ActiveConfig)
}

// cacheToken updates an existing configuration file with the
// provided one.  Note that we only update the profile as stored in
// viper.
func cacheToken(api *gbdx.Api) error {

	// Read in configuration file if it exists.
	confFile := viper.ConfigFileUsed()
	profilesOut := make(map[string]gbdx.Config)
	_, err := toml.DecodeFile(confFile, &profilesOut)
	if err != nil {
		return fmt.Errorf("failed to parse the configurtion: %v", err)
	}

	// Update/add the token.
	c := profilesOut[viper.GetString("profile")]
	c.Token, err = api.Token()
	if err != nil {
		return fmt.Errorf("failed to fetch a token to cache: %v", err)
	}
	profilesOut[viper.GetString("profile")] = c

	// Save to the GBDX config file.
	file, err := os.Create(confFile)
	if err != nil {
		return fmt.Errorf("failed to write updated configuration to disk: %v", err)
	}
	defer file.Close()
	return toml.NewEncoder(file).Encode(profilesOut)
}
<commit_msg>.gbdx dir permissions were wrong.<commit_after>package cmd

import (
	"fmt"
	"os"
	"path"
	"runtime"

	"github.com/BurntSushi/toml"
	"github.com/spf13/viper"
	"github.com/youngpm/gbdx"
)

// ensureGBDXDir will create the gbdx directory if it doesn't already exist.
func ensureGBDXDir() (string, error) {
	gbdxPath := path.Join(userHomeDir(), ".gbdx")
	err := os.MkdirAll(gbdxPath, 0700)
	return gbdxPath, err
}

// userHomeDir returns the home directory of the user.  I've borrowed
// this from https://github.com/spf13/viper/blob/master/util.go .
func userHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}
	return os.Getenv("HOME")
}

func apiFromConfig() (*gbdx.Api, error) {
	var profile GBDXProfile
	if err := viper.Unmarshal(&profile); err != nil {
		return nil, err
	}
	return gbdx.NewApi(profile.ActiveConfig)
}

// cacheToken updates an existing configuration file with the
// provided one.  Note that we only update the profile as stored in
// viper.
func cacheToken(api *gbdx.Api) error {

	// Read in configuration file if it exists.
	confFile := viper.ConfigFileUsed()
	profilesOut := make(map[string]gbdx.Config)
	_, err := toml.DecodeFile(confFile, &profilesOut)
	if err != nil {
		return fmt.Errorf("failed to parse the configurtion: %v", err)
	}

	// Update/add the token.
	c := profilesOut[viper.GetString("profile")]
	c.Token, err = api.Token()
	if err != nil {
		return fmt.Errorf("failed to fetch a token to cache: %v", err)
	}
	profilesOut[viper.GetString("profile")] = c

	// Save to the GBDX config file.
	file, err := os.Create(confFile)
	if err != nil {
		return fmt.Errorf("failed to write updated configuration to disk: %v", err)
	}
	defer file.Close()
	return toml.NewEncoder(file).Encode(profilesOut)
}
<|endoftext|>


<commit_before>//
// An interactive shell for DynamoDB
//
package main

import (
	"../../dynago"

	"code.google.com/p/gcfg"
	"github.com/gobs/args"
	"github.com/gobs/cmd"
	"github.com/gobs/pretty"

	"fmt"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
)

const (
	CONFIG_FILE  = ".dynagorc"
	HISTORY_FILE = ".dynago_history"
)

// the configuration should look like the following
// (with multiple profiles and a selected one)
//
// [dynago]
// profile=xxx
//
// [profile "xxx"]
// region=us-west-1
// accessKey=XXXXXXXX
// secretKey=YYYYYYYY

type Config struct {
	Dynago struct {
		// define default profile
		Profile string
	}

	// list of named profiles
	Profile map[string]*struct {
		Region    string
		AccessKey string
		SecretKey string
	}
}

// Look for configFile in current directory or home directory.
// No configuration file is NOT an error.
// A malformed configuration file is a FATAL error.

func ReadConfig(configFile string, config *Config) *Config {
	if config == nil {
		config = &Config{}
	}

	// configFile in current directory or full path
	if _, err := os.Stat(configFile); err != nil {
		if strings.Contains(configFile, "/") {
			return config
		}

		// configFile in home directory
		configFile = path.Join(os.Getenv("HOME"), configFile)
		if _, err := os.Stat(configFile); err != nil {
			return config
		}
	}

	err := gcfg.ReadFileInto(config, configFile)
	if err != nil {
		log.Fatal(err)
	}

	return config
}

var (
	// this hold the current list of table names, to be used by the CompletionFunction
	table_list []string
)

func add_to_list(table string) {
	table_list = append(table_list, table)
}

func remove_from_list(table string) {
	for i, t := range table_list {
		if t == table {
			table_list = append(table_list[:i], table_list[i+1:]...)
			return
		}
	}
}

// return list of table names that match the input pattern (table name starts with "text")
func CompletionFunction(text string, line string, start, stop int) []string {
	if len(table_list) > 0 {
		matches := make([]string, 0, len(table_list))

		for _, w := range table_list {
			if strings.HasPrefix(w, text) {
				matches = append(matches, w)
			}
		}

		return matches
	}

	return nil
}

func main() {
	config := ReadConfig(CONFIG_FILE, nil)

	selected := config.Dynago.Profile

	if len(os.Args) > 1 {
		// there is at least one parameter:
		// override the selected profile
		selected = os.Args[1]
	}

	profile := config.Profile[selected]
	if profile == nil {
		log.Fatal("no profile for ", selected)
	}

	db := dynago.NewDBClient()

	if len(profile.Region) > 0 {
		db.WithRegion(profile.Region)
	}

	if len(profile.AccessKey) > 0 {
		db.WithCredentials(profile.AccessKey, profile.SecretKey)
	}

	commander := &cmd.Cmd{HistoryFile: HISTORY_FILE, Complete: CompletionFunction, EnableShell: true}
	commander.Prompt = "dynagosh> "
	commander.Init()

	commander.Add(cmd.Command{"config",
		`
		config : display current configuration
		`,
		func(string) (stop bool) {
			pretty.PrettyPrint(config)
			return
		}})

	commander.Add(cmd.Command{"list",
		`
                list : display list of available tables
                `,
		func(string) (stop bool) {
			tables, err := db.ListTables()

			if err != nil {
				fmt.Println(err)
				return
			}

			fmt.Println("Available tables")

			for _, tableName := range tables {
				fmt.Println("  ", tableName)
			}

			if len(tables) > 0 {
				table_list = tables
			}

			return
		}})

	commander.Add(cmd.Command{"describe",
		`
                describe {table} : display table configuration
                `,
		func(line string) (stop bool) {
			tableName := line
			table, err := db.DescribeTable(tableName)
			if err != nil {
				fmt.Println(err)
			} else {
				pretty.PrettyPrint(table)
			}

			return
		}})

	commander.Add(cmd.Command{"create",
		`
		create {tablename} hashKey:hashType [rangeKey:rangeType] [readCapacity] [writeCapacity]
		`,
		func(line string) (stop bool) {
			args := args.GetArgs(line)

			if len(args) < 2 {
				fmt.Println("not enough arguments")
				return
			}

			tableName := args[0]

			hashKey := &dynago.AttributeDefinition{AttributeType: dynago.STRING_ATTRIBUTE}
			var rangeKey *dynago.AttributeDefinition
			rc := 5
			wc := 5

			if strings.Contains(args[1], ":") {
				parts := strings.Split(args[1], ":")
				hashKey.AttributeName = parts[0]
				hashKey.AttributeType = parts[1]
			} else {
				hashKey.AttributeName = args[1]
			}

			if len(args) > 2 {
				rangeKey := &dynago.AttributeDefinition{AttributeType: dynago.STRING_ATTRIBUTE}

				if strings.Contains(args[2], ":") {
					parts := strings.Split(args[2], ":")
					rangeKey.AttributeName = parts[0]
					rangeKey.AttributeType = parts[1]
				} else {
					rangeKey.AttributeName = args[2]
				}
			}

			if table, err := db.CreateTable(tableName, hashKey, rangeKey, rc, wc); err != nil {
				fmt.Println(err)
			} else {
				pretty.PrettyPrint(table)
				add_to_list(tableName)
			}

			return
		}})

	commander.Add(cmd.Command{"delete",
		`
                delete {table} : delete table
                `,
		func(line string) (stop bool) {
			tableName := line
			table, err := db.DeleteTable(tableName)
			if err != nil {
				fmt.Println(err)
			} else {
				pretty.PrettyPrint(table)
				remove_from_list(tableName)
			}

			return
		}})

	commander.Add(cmd.Command{"update",
		`
		update {tablename} readCapacity writeCapacity
		`,
		func(line string) (stop bool) {
			args := args.GetArgs(line)

			if len(args) < 2 {
				fmt.Println("not enough arguments")
				return
			}

			tableName := args[0]
			table, err := db.DescribeTable(tableName)

			if err != nil {
				fmt.Println(err)
				return
			}

			rc := -1 // table.ProvisionedThroughput.ReadCapacityUnits
			wc := -1 // table.ProvisionedThroughput.WriteCapacityUnits

			if v, err := strconv.Atoi(args[1]); err == nil {
				rc = v
			}

			if len(args) > 2 {
				if v, err := strconv.Atoi(args[2]); err == nil {
					wc = v
				}
			}

			if rc <= 0 && wc <= 0 {
				fmt.Println("no valid value for rc or wc")
				return
			}

			if rc <= 0 {
				rc = table.ProvisionedThroughput.ReadCapacityUnits
			}

			if wc <= 0 {
				wc = table.ProvisionedThroughput.WriteCapacityUnits
			}

			if table, err := db.UpdateTable(tableName, rc, wc); err != nil {
				fmt.Println(err)
			} else {
				pretty.PrettyPrint(table)
			}

			return
		}})

	commander.Add(cmd.Command{"get",
		`
		get {tablename} {hashKey} [rangeKey]
		`,
		func(line string) (stop bool) {
			args := args.GetArgs(line)

			if len(args) < 2 {
				fmt.Println("not enough arguments")
				return
			}

			tableName := args[0]
			table, err := db.DescribeTable(tableName)

			if err != nil {
				fmt.Println(err)
				return
			}

			hashKey := &dynago.KeyValue{table.GetHashKey(), args[1]}
			var rangeKey *dynago.KeyValue

			if len(args) > 2 {
				rangeKey = &dynago.KeyValue{table.GetRangeKey(), args[2]}
			}

			if item, _, err := db.GetItem(table.TableName, hashKey, rangeKey, nil, false, false); err != nil {
				fmt.Println(err)
			} else {
				pretty.PrettyPrint(item)
			}

			return
		}})

	commander.Commands["ls"] = commander.Commands["list"]
	commander.Commands["drop"] = commander.Commands["delete"]

	commander.CmdLoop()
}
<commit_msg>- make sure we pass rangeKey, when requested - added debug flag (in config) to log requests/responses<commit_after>//
// An interactive shell for DynamoDB
//
package main

import (
	"../../dynago"

	"code.google.com/p/gcfg"
	"github.com/gobs/args"
	"github.com/gobs/cmd"
	"github.com/gobs/httpclient"
	"github.com/gobs/pretty"

	"fmt"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
)

const (
	CONFIG_FILE  = ".dynagorc"
	HISTORY_FILE = ".dynago_history"
)

// the configuration should look like the following
// (with multiple profiles and a selected one)
//
// [dynago]
// profile=xxx
//
// [profile "xxx"]
// region=us-west-1
// accessKey=XXXXXXXX
// secretKey=YYYYYYYY

type Config struct {
	Dynago struct {
		// define default profile
		Profile string
		// enable request debugging
		Debug bool
	}

	// list of named profiles
	Profile map[string]*struct {
		Region    string
		AccessKey string
		SecretKey string
	}
}

// Look for configFile in current directory or home directory.
// No configuration file is NOT an error.
// A malformed configuration file is a FATAL error.

func ReadConfig(configFile string, config *Config) *Config {
	if config == nil {
		config = &Config{}
	}

	// configFile in current directory or full path
	if _, err := os.Stat(configFile); err != nil {
		if strings.Contains(configFile, "/") {
			return config
		}

		// configFile in home directory
		configFile = path.Join(os.Getenv("HOME"), configFile)
		if _, err := os.Stat(configFile); err != nil {
			return config
		}
	}

	err := gcfg.ReadFileInto(config, configFile)
	if err != nil {
		log.Fatal(err)
	}

	return config
}

var (
	// this hold the current list of table names, to be used by the CompletionFunction
	table_list []string
)

func add_to_list(table string) {
	table_list = append(table_list, table)
}

func remove_from_list(table string) {
	for i, t := range table_list {
		if t == table {
			table_list = append(table_list[:i], table_list[i+1:]...)
			return
		}
	}
}

// return list of table names that match the input pattern (table name starts with "text")
func CompletionFunction(text string, line string, start, stop int) []string {
	if len(table_list) > 0 {
		matches := make([]string, 0, len(table_list))

		for _, w := range table_list {
			if strings.HasPrefix(w, text) {
				matches = append(matches, w)
			}
		}

		return matches
	}

	return nil
}

func main() {
	config := ReadConfig(CONFIG_FILE, nil)

	selected := config.Dynago.Profile

	if len(os.Args) > 1 {
		// there is at least one parameter:
		// override the selected profile
		selected = os.Args[1]
	}

	profile := config.Profile[selected]
	if profile == nil {
		log.Fatal("no profile for ", selected)
	}

	if config.Dynago.Debug {
		httpclient.StartLogging(true, true)
	}

	db := dynago.NewDBClient()

	if len(profile.Region) > 0 {
		db.WithRegion(profile.Region)
	}

	if len(profile.AccessKey) > 0 {
		db.WithCredentials(profile.AccessKey, profile.SecretKey)
	}

	commander := &cmd.Cmd{HistoryFile: HISTORY_FILE, Complete: CompletionFunction, EnableShell: true}
	commander.Prompt = "dynagosh> "
	commander.Init()

	commander.Add(cmd.Command{"config",
		`
		config : display current configuration
		`,
		func(string) (stop bool) {
			pretty.PrettyPrint(config)
			return
		}})

	commander.Add(cmd.Command{"list",
		`
                list : display list of available tables
                `,
		func(string) (stop bool) {
			tables, err := db.ListTables()

			if err != nil {
				fmt.Println(err)
				return
			}

			fmt.Println("Available tables")

			for _, tableName := range tables {
				fmt.Println("  ", tableName)
			}

			if len(tables) > 0 {
				table_list = tables
			}

			return
		}})

	commander.Add(cmd.Command{"describe",
		`
                describe {table} : display table configuration
                `,
		func(line string) (stop bool) {
			tableName := line
			table, err := db.DescribeTable(tableName)
			if err != nil {
				fmt.Println(err)
			} else {
				pretty.PrettyPrint(table)
			}

			return
		}})

	commander.Add(cmd.Command{"create",
		`
		create {tablename} hashKey:hashType [rangeKey:rangeType] [readCapacity] [writeCapacity]
		`,
		func(line string) (stop bool) {
			args := args.GetArgs(line)

			if len(args) < 2 {
				fmt.Println("not enough arguments")
				return
			}

			tableName := args[0]

			hashKey := &dynago.AttributeDefinition{AttributeType: dynago.STRING_ATTRIBUTE}
			var rangeKey *dynago.AttributeDefinition
			rc := 5
			wc := 5

			if strings.Contains(args[1], ":") {
				parts := strings.Split(args[1], ":")
				hashKey.AttributeName = parts[0]
				hashKey.AttributeType = parts[1]
			} else {
				hashKey.AttributeName = args[1]
			}

			if len(args) > 2 {
				rangeKey := &dynago.AttributeDefinition{AttributeType: dynago.STRING_ATTRIBUTE}

				if strings.Contains(args[2], ":") {
					parts := strings.Split(args[2], ":")
					rangeKey.AttributeName = parts[0]
					rangeKey.AttributeType = parts[1]
				} else {
					rangeKey.AttributeName = args[2]
				}
			}

			if table, err := db.CreateTable(tableName, hashKey, rangeKey, rc, wc); err != nil {
				fmt.Println(err)
			} else {
				pretty.PrettyPrint(table)
				add_to_list(tableName)
			}

			return
		}})

	commander.Add(cmd.Command{"delete",
		`
                delete {table} : delete table
                `,
		func(line string) (stop bool) {
			tableName := line
			table, err := db.DeleteTable(tableName)
			if err != nil {
				fmt.Println(err)
			} else {
				pretty.PrettyPrint(table)
				remove_from_list(tableName)
			}

			return
		}})

	commander.Add(cmd.Command{"update",
		`
		update {tablename} readCapacity writeCapacity
		`,
		func(line string) (stop bool) {
			args := args.GetArgs(line)

			if len(args) < 2 {
				fmt.Println("not enough arguments")
				return
			}

			tableName := args[0]
			table, err := db.DescribeTable(tableName)

			if err != nil {
				fmt.Println(err)
				return
			}

			rc := -1 // table.ProvisionedThroughput.ReadCapacityUnits
			wc := -1 // table.ProvisionedThroughput.WriteCapacityUnits

			if v, err := strconv.Atoi(args[1]); err == nil {
				rc = v
			}

			if len(args) > 2 {
				if v, err := strconv.Atoi(args[2]); err == nil {
					wc = v
				}
			}

			if rc <= 0 && wc <= 0 {
				fmt.Println("no valid value for rc or wc")
				return
			}

			if rc <= 0 {
				rc = table.ProvisionedThroughput.ReadCapacityUnits
			}

			if wc <= 0 {
				wc = table.ProvisionedThroughput.WriteCapacityUnits
			}

			if table, err := db.UpdateTable(tableName, rc, wc); err != nil {
				fmt.Println(err)
			} else {
				pretty.PrettyPrint(table)
			}

			return
		}})

	commander.Add(cmd.Command{"get",
		`
		get {tablename} {hashKey} [rangeKey] [attributes]
		`,
		func(line string) (stop bool) {
			args := args.GetArgs(line)

			if len(args) < 2 {
				fmt.Println("not enough arguments")
				return
			}

			tableName := args[0]
			table, err := db.DescribeTable(tableName)

			if err != nil {
				fmt.Println(err)
				return
			}

			hashKey := &dynago.KeyValue{table.GetHashKey(), args[1]}
			var rangeKey *dynago.KeyValue

			if len(args) > 2 {
				rangeKey = &dynago.KeyValue{table.GetRangeKey(), args[2]}
			} else if table.GetRangeKey() != "" {
				fmt.Println("required rangeKey value")
				return
			}

			var attributes []string

			if len(args) > 3 {
				attributes = args[3:]
			}

			if item, _, err := db.GetItem(table.TableName, hashKey, rangeKey, attributes, false, true); err != nil {
				fmt.Println(err)
			} else {
				pretty.PrettyPrint(item)
			}

			return
		}})

	commander.Commands["ls"] = commander.Commands["list"]
	commander.Commands["drop"] = commander.Commands["delete"]

	commander.CmdLoop()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path"
	"strings"
	"text/template"
)

var bootstrapText = `package {{.Package}}_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
)

func Test{{.PackageTitleCase}}(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "{{.PackageTitleCase}} Suite")
}
`

var specText = `package {{.Package}}_test

import (
	. "{{.PackageImportPath}}"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("{{.Subject}}", func() {

})
`

type bootstrapData struct {
	Package          string
	PackageTitleCase string
}

type specData struct {
	Package           string
	Subject           string
	PackageImportPath string
}

func generateBootstrap() {
	packageName := getPackage()
	data := bootstrapData{
		Package:          packageName,
		PackageTitleCase: strings.Title(packageName),
	}

	targetFile := fmt.Sprintf("%s_suite_test.go", packageName)
	if fileExists(targetFile) {
		fmt.Printf("%s already exists.\n\n", targetFile)
		os.Exit(1)
	} else {
		fmt.Printf("Generating ginkgo test suite bootstrap for %s in:\n\t%s\n\n", packageName, targetFile)
	}

	f, err := os.Create(targetFile)
	if err != nil {
		panic(err.Error())
	}
	defer f.Close()

	bootstrapTemplate, err := template.New("bootstrap").Parse(bootstrapText)
	if err != nil {
		panic(err.Error())
	}

	bootstrapTemplate.Execute(f, data)
}

func generateSpec(subject string) {
	packageName := getPackage()
	if subject == "" {
		subject = packageName
	}

	formattedSubject := strings.Replace(strings.Title(strings.Replace(subject, "_", " ", -1)), " ", "", -1)

	data := specData{
		Package:           packageName,
		Subject:           formattedSubject,
		PackageImportPath: getPackageImportPath(),
	}

	targetFile := fmt.Sprintf("%s_test.go", subject)
	if fileExists(targetFile) {
		fmt.Printf("%s already exists.\n\n", targetFile)
		os.Exit(1)
	} else {
		fmt.Printf("Generating ginkgo test for %s in:\n\t%s\n\n", data.Subject, targetFile)
	}

	f, err := os.Create(targetFile)
	if err != nil {
		panic(err.Error())
	}
	defer f.Close()

	specTemplate, err := template.New("spec").Parse(specText)
	if err != nil {
		panic(err.Error())
	}

	specTemplate.Execute(f, data)
}

func getPackage() string {
	workingDir, err := os.Getwd()
	if err != nil {
		panic(err.Error())
	}
	return path.Base(workingDir)
}

func getPackageImportPath() string {
	workingDir, err := os.Getwd()
	if err != nil {
		panic(err.Error())
	}
	paths := strings.Split(workingDir, "/src/")
	if len(paths) == 1 {
		fmt.Printf("\nCouldn't identify package import path.\n\n\tginkgo generate\n\nMust be run within a package directory under $GOPATH/src/...\nYou're going to had to change UNKOWN_PACKAGE_PATH in the generated file...\n\n")
		return "UNKOWN_PACKAGE_PATH"
	}
	return paths[len(paths)-1]
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	if os.IsNotExist(err) {
		return false
	}
	return false
}
<commit_msg>You should use `path/filepath` instead of `path`. Because os.Getwd returns operating system-defined file path. Windows path separator is not / in \.<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

var bootstrapText = `package {{.Package}}_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
)

func Test{{.PackageTitleCase}}(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "{{.PackageTitleCase}} Suite")
}
`

var specText = `package {{.Package}}_test

import (
	. "{{.PackageImportPath}}"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("{{.Subject}}", func() {

})
`

type bootstrapData struct {
	Package          string
	PackageTitleCase string
}

type specData struct {
	Package           string
	Subject           string
	PackageImportPath string
}

func generateBootstrap() {
	packageName := getPackage()
	data := bootstrapData{
		Package:          packageName,
		PackageTitleCase: strings.Title(packageName),
	}

	targetFile := fmt.Sprintf("%s_suite_test.go", packageName)
	if fileExists(targetFile) {
		fmt.Printf("%s already exists.\n\n", targetFile)
		os.Exit(1)
	} else {
		fmt.Printf("Generating ginkgo test suite bootstrap for %s in:\n\t%s\n\n", packageName, targetFile)
	}

	f, err := os.Create(targetFile)
	if err != nil {
		panic(err.Error())
	}
	defer f.Close()

	bootstrapTemplate, err := template.New("bootstrap").Parse(bootstrapText)
	if err != nil {
		panic(err.Error())
	}

	bootstrapTemplate.Execute(f, data)
}

func generateSpec(subject string) {
	packageName := getPackage()
	if subject == "" {
		subject = packageName
	}

	formattedSubject := strings.Replace(strings.Title(strings.Replace(subject, "_", " ", -1)), " ", "", -1)

	data := specData{
		Package:           packageName,
		Subject:           formattedSubject,
		PackageImportPath: getPackageImportPath(),
	}

	targetFile := fmt.Sprintf("%s_test.go", subject)
	if fileExists(targetFile) {
		fmt.Printf("%s already exists.\n\n", targetFile)
		os.Exit(1)
	} else {
		fmt.Printf("Generating ginkgo test for %s in:\n\t%s\n\n", data.Subject, targetFile)
	}

	f, err := os.Create(targetFile)
	if err != nil {
		panic(err.Error())
	}
	defer f.Close()

	specTemplate, err := template.New("spec").Parse(specText)
	if err != nil {
		panic(err.Error())
	}

	specTemplate.Execute(f, data)
}

func getPackage() string {
	workingDir, err := os.Getwd()
	if err != nil {
		panic(err.Error())
	}
	return filepath.Base(workingDir)
}

func getPackageImportPath() string {
	workingDir, err := os.Getwd()
	if err != nil {
		panic(err.Error())
	}
	paths := strings.Split(workingDir, "/src/")
	if len(paths) == 1 {
		fmt.Printf("\nCouldn't identify package import path.\n\n\tginkgo generate\n\nMust be run within a package directory under $GOPATH/src/...\nYou're going to had to change UNKOWN_PACKAGE_PATH in the generated file...\n\n")
		return "UNKOWN_PACKAGE_PATH"
	}
	return paths[len(paths)-1]
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	if os.IsNotExist(err) {
		return false
	}
	return false
}
<|endoftext|>


<commit_before>package depbleed

import (
	"fmt"
	"go/token"
	"go/types"
)

// Leak represents a leaking type.
type Leak struct {
	Object   types.Object
	Position token.Position
	err      error
}

// Leaks represent a slice of Leak
type Leaks []Leak

//Len give the length of the leak slice
func (slice Leaks) Len() int {
	return len(slice)
}

//Less returns if slice[i] should be before slice[j]
func (slice Leaks) Less(i, j int) bool {
	return slice[i].Position.Filename < slice[j].Position.Filename ||
		(slice[i].Position.Filename == slice[j].Position.Filename &&
			slice[i].Position.Line < slice[j].Position.Line) ||
		(slice[i].Position.Filename == slice[j].Position.Filename &&
			slice[i].Position.Line == slice[j].Position.Line &&
			slice[i].Position.Column == slice[j].Position.Column)
}

//Swap swaps two elements
func (slice Leaks) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

//Error construt an error string
func (l Leak) Error() string {
	return fmt.Sprintf("%s %s: %s", GetObjectKind(l.Object), l.Object.Name(), l.err)
}
<commit_msg>fix typo in Leaks sort<commit_after>package depbleed

import (
	"fmt"
	"go/token"
	"go/types"
)

// Leak represents a leaking type.
type Leak struct {
	Object   types.Object
	Position token.Position
	err      error
}

// Leaks represent a slice of Leak
type Leaks []Leak

//Len give the length of the leak slice
func (slice Leaks) Len() int {
	return len(slice)
}

//Less returns if slice[i] should be before slice[j]
func (slice Leaks) Less(i, j int) bool {
	return slice[i].Position.Filename < slice[j].Position.Filename ||
		(slice[i].Position.Filename == slice[j].Position.Filename &&
			slice[i].Position.Line < slice[j].Position.Line) ||
		(slice[i].Position.Filename == slice[j].Position.Filename &&
			slice[i].Position.Line == slice[j].Position.Line &&
			slice[i].Position.Column < slice[j].Position.Column)
}

//Swap swaps two elements
func (slice Leaks) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

//Error construt an error string
func (l Leak) Error() string {
	return fmt.Sprintf("%s %s: %s", GetObjectKind(l.Object), l.Object.Name(), l.err)
}
<|endoftext|>


<commit_before>package engine

import (
	"errors"
	"fmt"

	"github.com/keybase/client/go/libkb"
	keybase_1 "github.com/keybase/client/protocol/go"
)

type Doctor struct {
	user       *libkb.User
	signingKey libkb.GenericKey
	devName    string
	libkb.Contextified
}

func NewDoctor() *Doctor {
	return &Doctor{}
}

func (d *Doctor) GetPrereqs() EnginePrereqs { return EnginePrereqs{} }

func (d *Doctor) Name() string {
	return "Doctor"
}

func (d *Doctor) RequiredUIs() []libkb.UIKind {
	return []libkb.UIKind{
		libkb.LogUIKind,
		libkb.DoctorUIKind,
		libkb.GPGUIKind,
		libkb.SecretUIKind,
	}
}

func (d *Doctor) SubConsumers() []libkb.UIConsumer {
	return []libkb.UIConsumer{
		NewDeviceEngine(nil, nil),
		NewDetKeyEngine(nil),
	}
}

func (d *Doctor) LoginCheckup(ctx *Context, u *libkb.User) error {
	d.user = u
	d.SetGlobalContext(ctx.GlobalContext)

	// This can fail, but we'll warn if it does.
	d.syncSecrets()

	if err := d.checkKeys(ctx); err != nil {
		return err
	}
	return nil
}

func (d *Doctor) syncSecrets() (err error) {
	if err = libkb.RunSyncer(d.G().SecretSyncer, d.user.GetUid().P()); err != nil {
		d.G().Log.Warning("Problem syncing secrets from server: %s", err)
	}
	return err
}

func (d *Doctor) checkKeys(ctx *Context) error {
	kf := d.user.GetKeyFamily()
	if kf == nil {
		return d.addBasicKeys(ctx)
	}
	if kf.GetEldest() == nil {
		return d.addBasicKeys(ctx)
	}

	// they have at least one key

	if d.user.HasDeviceInCurrentInstall() {
		// they have a device sibkey for this device
		return nil
	}

	// make sure secretsyncer loaded --- likely not needed since we
	// already did this about
	d.syncSecrets()

	hasPGP := len(d.user.GetActivePgpKeys(false)) > 0

	if d.G().SecretSyncer.HasActiveDevice() {
		// they have at least one device, just not this device...
		return d.deviceSign(ctx, hasPGP)
	}

	// they don't have any devices.

	dk, err := d.detkey(ctx)
	if err != nil {
		if _, ok := err.(libkb.NotFoundError); ok {
			// they don't have a detkey.
			//
			// they can get to this point if they sign up on the web,
			// add a pgp key.  With that situation, they have keys
			// so the check above for a nil key family doesn't apply.

			// make sure we have pgp
			if !hasPGP {
				return fmt.Errorf("unknown state:  no detkey, no pgpkey, no devices, but have some key(s).\nOne way to get here is to create a web user, create a pgp key via web but don't store private key on web.  Then login.  When issue #174 is done (fixes a bug with the user's computedkeyfamily, activepgpkeys), then that scenario should work fine.")
			}

			// deviceSign will handle the rest...
			return d.deviceSign(ctx, true)
		}
		return err
	}

	// use their detkey to sign this device
	return d.addDeviceKeyWithSigner(ctx, dk, dk.GetKid())
}

// addBasicKeys is used for accounts that have no device or det
// keys.
func (d *Doctor) addBasicKeys(ctx *Context) error {
	if err := d.addDeviceKey(ctx); err != nil {
		return err
	}

	if err := d.addDetKey(ctx, d.signingKey.GetKid()); err != nil {
		return err
	}

	return nil
}

func (d *Doctor) addDeviceKey(ctx *Context) error {
	devname, err := d.deviceName(ctx)
	if err != nil {
		return err
	}
	tk, err := d.tspkey(ctx)
	if err != nil {
		return err
	}
	args := DeviceEngineArgs{
		Name:          devname,
		LksClientHalf: tk.LksClientHalf(),
	}
	eng := NewDeviceEngine(d.user, &args)
	if err := RunEngine(eng, ctx); err != nil {
		return err
	}

	// XXX get this from reply?
	d.signingKey = eng.EldestKey()
	return nil
}

func (d *Doctor) addDeviceKeyWithSigner(ctx *Context, signer libkb.GenericKey, eldestKID libkb.KID) error {
	devname, err := d.deviceName(ctx)
	if err != nil {
		return err
	}
	tk, err := d.tspkey(ctx)
	if err != nil {
		return err
	}
	args := DeviceEngineArgs{
		Name:          devname,
		LksClientHalf: tk.LksClientHalf(),
		Signer:        signer,
		EldestKID:     eldestKID,
	}
	eng := NewDeviceEngine(d.user, &args)
	if err := RunEngine(eng, ctx); err != nil {
		return err
	}

	d.signingKey = signer
	return nil
}

func (d *Doctor) addDetKey(ctx *Context, eldest libkb.KID) error {
	tk, err := d.tspkey(ctx)
	if err != nil {
		return err
	}
	arg := &DetKeyArgs{
		Tsp:         tk,
		Me:          d.user,
		SigningKey:  d.signingKey,
		EldestKeyID: eldest,
	}
	eng := NewDetKeyEngine(arg)
	return RunEngine(eng, ctx)
}

var ErrNotYetImplemented = errors.New("not yet implemented")

// deviceSign is used to sign a new installation of keybase on a
// new device.  It happens when the user has keys already, either
// a device key, pgp key, or both.
func (d *Doctor) deviceSign(ctx *Context, withPGPOption bool) error {
	devname, err := d.deviceName(ctx)
	if err != nil {
		return err
	}

	devs, err := d.G().SecretSyncer.ActiveDevices()
	if err != nil {
		return err
	}

	var arg keybase_1.SelectSignerArg
	for k, v := range devs {
		if v.Type != libkb.DEVICE_TYPE_WEB {
			arg.Devices = append(arg.Devices, keybase_1.Device{Type: v.Type, Name: v.Description, DeviceID: k})
		}
	}
	arg.HasPGP = withPGPOption

	res, err := ctx.DoctorUI.SelectSigner(arg)
	if err != nil {
		return err
	}

	if res.Action == keybase_1.SelectSignerAction_CANCEL {
		// XXX another way to bail besides returning an error?
		return fmt.Errorf("cancel requested by user")
	}
	if res.Action == keybase_1.SelectSignerAction_RESET_ACCOUNT {
		d.G().Log.Info("reset account action not yet implemented")
		return ErrNotYetImplemented
	}

	if res.Action != keybase_1.SelectSignerAction_SIGN {
		return fmt.Errorf("unknown action value: %d", res.Action)
	}

	// sign action:

	if res.Signer.Kind == keybase_1.DeviceSignerKind_PGP {
		return d.deviceSignPGP(ctx)
	}

	if res.Signer.Kind == keybase_1.DeviceSignerKind_DEVICE {
		return d.deviceSignExistingDevice(ctx, *res.Signer.DeviceID, devname, libkb.DEVICE_TYPE_DESKTOP)
	}

	return fmt.Errorf("unknown signer kind: %d", res.Signer.Kind)
}

func (d *Doctor) deviceSignPGP(ctx *Context) error {
	pgpKeys := d.user.GetActivePgpKeys(false)
	var selected *libkb.PgpKeyBundle
	if len(pgpKeys) > 1 {
		// show a list of pgp keys and let them select which one to use
		var err error
		selected, err = d.selectPGPKey(ctx, pgpKeys)
		if err != nil {
			return err
		}
		if selected == nil {
			return fmt.Errorf("no key selected")
		}
	} else {
		selected = pgpKeys[0]
	}

	d.G().Log.Info("selected pgp key: %s", selected.VerboseDescription())
	d.G().Log.Info("selected pgp key kid: %s", selected.GetKid())
	if pk, ok := d.G().SecretSyncer.FindPrivateKey(selected.GetKid().String()); ok {
		skb, err := pk.ToSKB()
		if err != nil {
			return err
		}

		pgpk, err := skb.PromptAndUnlock("pgp sign", "keybase", ctx.SecretUI)
		if err != nil {
			return err
		}
		return d.deviceSignPGPNext(ctx, pgpk)
	}

	// use gpg to unlock it
	gpg := d.G().GetGpgClient()
	if _, err := gpg.Configure(); err != nil {
		return err
	}

	bundle, err := gpg.ImportKey(true, selected.GetFingerprint())
	if err != nil {
		return fmt.Errorf("ImportKey error: %s", err)
	}

	if err := bundle.Unlock("Import of key into keybase keyring", ctx.SecretUI); err != nil {
		return fmt.Errorf("bundle Unlock error: %s", err)
	}

	return d.deviceSignPGPNext(ctx, bundle)
}

func (d *Doctor) deviceSignPGPNext(ctx *Context, pgpk libkb.GenericKey) error {
	if pgpk.CanSign() == false {
		return fmt.Errorf("pgp key can't sign")
	}

	eldest := d.user.GetEldestFOKID().Kid
	d.G().Log.Info("eldest kid from user: %s", eldest)
	if err := d.addDeviceKeyWithSigner(ctx, pgpk, eldest); err != nil {
		return err
	}

	if err := d.addDetKey(ctx, eldest); err != nil {
		return err
	}

	return nil
}

func (d *Doctor) deviceSignExistingDevice(ctx *Context, id, devName, devType string) error {
	d.G().Log.Info("device sign with existing device [%s]", id)

	src, err := libkb.NewDeviceID()
	if err != nil {
		return err
	}

	dst, err := libkb.ImportDeviceID(id)
	if err != nil {
		return err
	}

	tk, err := d.tspkey(ctx)
	if err != nil {
		return err
	}

	kargs := &KexFwdArgs{
		User:    d.user,
		Src:     src,
		Dst:     *dst,
		DevType: devType,
		DevDesc: devName,
	}
	k := NewKexFwd(tk.LksClientHalf(), kargs)
	return RunEngine(k, ctx)
}

func (d *Doctor) selectPGPKey(ctx *Context, keys []*libkb.PgpKeyBundle) (*libkb.PgpKeyBundle, error) {
	var gks []keybase_1.GPGKey
	for _, key := range keys {
		algo, kid, creation := key.KeyInfo()
		gk := keybase_1.GPGKey{
			Algorithm:  algo,
			KeyID:      kid,
			Creation:   creation,
			Identities: key.IdentityNames(),
		}
		gks = append(gks, gk)
	}

	keyid, err := ctx.GPGUI.SelectKey(keybase_1.SelectKeyArg{Keys: gks})
	if err != nil {
		return nil, err
	}
	d.G().Log.Info("SelectKey result: %+v", keyid)

	var selected *libkb.PgpKeyBundle
	for _, key := range keys {
		if key.GetFingerprint().ToKeyId() == keyid {
			selected = key
			break
		}
	}

	return selected, nil
}

func (d *Doctor) tspkey(ctx *Context) (libkb.PassphraseStream, error) {
	return d.G().LoginState.GetPassphraseStream(ctx.SecretUI)
}

func (d *Doctor) detkey(ctx *Context) (libkb.GenericKey, error) {
	// get server half of detkey via ss
	half, err := d.G().SecretSyncer.FindDetKeySrvHalf(libkb.KEY_TYPE_KB_NACL_EDDSA_SERVER_HALF)
	if err != nil {
		return nil, err
	}

	// regenerate the detkey
	tk, err := d.tspkey(ctx)
	if err != nil {
		return nil, err
	}

	detkey, err := GenSigningDetKey(tk, half)
	if err != nil {
		return nil, err
	}
	return detkey, nil
}

func (d *Doctor) deviceName(ctx *Context) (string, error) {
	if len(d.devName) == 0 {
		name, err := ctx.DoctorUI.PromptDeviceName(0)
		if err != nil {
			return "", err
		}
		d.devName = name
	}
	return d.devName, nil
}
<commit_msg>more debug<commit_after>package engine

import (
	"errors"
	"fmt"

	"github.com/keybase/client/go/libkb"
	keybase_1 "github.com/keybase/client/protocol/go"
)

type Doctor struct {
	user       *libkb.User
	signingKey libkb.GenericKey
	devName    string
	libkb.Contextified
}

func NewDoctor() *Doctor {
	return &Doctor{}
}

func (d *Doctor) GetPrereqs() EnginePrereqs { return EnginePrereqs{} }

func (d *Doctor) Name() string {
	return "Doctor"
}

func (d *Doctor) RequiredUIs() []libkb.UIKind {
	return []libkb.UIKind{
		libkb.LogUIKind,
		libkb.DoctorUIKind,
		libkb.GPGUIKind,
		libkb.SecretUIKind,
	}
}

func (d *Doctor) SubConsumers() []libkb.UIConsumer {
	return []libkb.UIConsumer{
		NewDeviceEngine(nil, nil),
		NewDetKeyEngine(nil),
	}
}

func (d *Doctor) LoginCheckup(ctx *Context, u *libkb.User) error {
	d.user = u
	d.SetGlobalContext(ctx.GlobalContext)

	// This can fail, but we'll warn if it does.
	d.syncSecrets()

	if err := d.checkKeys(ctx); err != nil {
		return err
	}
	return nil
}

func (d *Doctor) syncSecrets() (err error) {
	if err = libkb.RunSyncer(d.G().SecretSyncer, d.user.GetUid().P()); err != nil {
		d.G().Log.Warning("Problem syncing secrets from server: %s", err)
	}
	return err
}

func (d *Doctor) checkKeys(ctx *Context) error {
	d.G().Log.Debug("+ Doctor::checkKeys()")
	defer func() {
		d.G().Log.Debug("- Doctor::checkKeys()")
	}()

	kf := d.user.GetKeyFamily()
	if kf == nil {
		d.G().Log.Debug("| User didn't have a key family")
		return d.addBasicKeys(ctx)
	}
	if kf.GetEldest() == nil {
		d.G().Log.Debug("| User didn't have an eldest key")
		return d.addBasicKeys(ctx)
	}

	// they have at least one key

	if d.user.HasDeviceInCurrentInstall() {
		// they have a device sibkey for this device
		d.G().Log.Debug("| User has a device in the current install; all done")
		return nil
	}

	// make sure secretsyncer loaded --- likely not needed since we
	// already did this about
	d.G().Log.Debug("| Syncing secrets")
	d.syncSecrets()

	hasPGP := len(d.user.GetActivePgpKeys(false)) > 0

	if d.G().SecretSyncer.HasActiveDevice() {
		// they have at least one device, just not this device...
		d.G().Log.Debug("| User has an active device, just not this one")
		return d.deviceSign(ctx, hasPGP)
	}

	// they don't have any devices.
	d.G().Log.Debug("| the user doesn't have any devices")

	dk, err := d.detkey(ctx)

	if err == nil {

		d.G().Log.Debug("| The user has a detkey")
		// use their detkey to sign this device
		err = d.addDeviceKeyWithSigner(ctx, dk, dk.GetKid())

	} else if _, ok := err.(libkb.NotFoundError); ok {

		d.G().Log.Debug("| The user doesn't have a detkey")
		// they don't have a detkey.
		//
		// they can get to this point if they sign up on the web,
		// add a pgp key.  With that situation, they have keys
		// so the check above for a nil key family doesn't apply.

		// make sure we have pgp
		if !hasPGP {
			return fmt.Errorf("unknown state:  no detkey, no pgpkey, no devices, but have some key(s).\nOne way to get here is to create a web user, create a pgp key via web but don't store private key on web.  Then login.  When issue #174 is done (fixes a bug with the user's computedkeyfamily, activepgpkeys), then that scenario should work fine.")
		}

		// deviceSign will handle the rest...
		err = d.deviceSign(ctx, true)
	} else {
		d.G().Log.Debug("| The user doesn't have a detkey")
	}

	return err
}

// addBasicKeys is used for accounts that have no device or det
// keys.
func (d *Doctor) addBasicKeys(ctx *Context) error {
	if err := d.addDeviceKey(ctx); err != nil {
		return err
	}

	if err := d.addDetKey(ctx, d.signingKey.GetKid()); err != nil {
		return err
	}

	return nil
}

func (d *Doctor) addDeviceKey(ctx *Context) error {
	devname, err := d.deviceName(ctx)
	if err != nil {
		return err
	}
	tk, err := d.tspkey(ctx)
	if err != nil {
		return err
	}
	args := DeviceEngineArgs{
		Name:          devname,
		LksClientHalf: tk.LksClientHalf(),
	}
	eng := NewDeviceEngine(d.user, &args)
	if err := RunEngine(eng, ctx); err != nil {
		return err
	}

	// XXX get this from reply?
	d.signingKey = eng.EldestKey()
	return nil
}

func (d *Doctor) addDeviceKeyWithSigner(ctx *Context, signer libkb.GenericKey, eldestKID libkb.KID) error {
	devname, err := d.deviceName(ctx)
	if err != nil {
		return err
	}
	tk, err := d.tspkey(ctx)
	if err != nil {
		return err
	}
	args := DeviceEngineArgs{
		Name:          devname,
		LksClientHalf: tk.LksClientHalf(),
		Signer:        signer,
		EldestKID:     eldestKID,
	}
	eng := NewDeviceEngine(d.user, &args)
	if err := RunEngine(eng, ctx); err != nil {
		return err
	}

	d.signingKey = signer
	return nil
}

func (d *Doctor) addDetKey(ctx *Context, eldest libkb.KID) error {
	tk, err := d.tspkey(ctx)
	if err != nil {
		return err
	}
	arg := &DetKeyArgs{
		Tsp:         tk,
		Me:          d.user,
		SigningKey:  d.signingKey,
		EldestKeyID: eldest,
	}
	eng := NewDetKeyEngine(arg)
	return RunEngine(eng, ctx)
}

var ErrNotYetImplemented = errors.New("not yet implemented")

// deviceSign is used to sign a new installation of keybase on a
// new device.  It happens when the user has keys already, either
// a device key, pgp key, or both.
func (d *Doctor) deviceSign(ctx *Context, withPGPOption bool) error {
	devname, err := d.deviceName(ctx)
	if err != nil {
		return err
	}

	devs, err := d.G().SecretSyncer.ActiveDevices()
	if err != nil {
		return err
	}

	var arg keybase_1.SelectSignerArg
	for k, v := range devs {
		if v.Type != libkb.DEVICE_TYPE_WEB {
			arg.Devices = append(arg.Devices, keybase_1.Device{Type: v.Type, Name: v.Description, DeviceID: k})
		}
	}
	arg.HasPGP = withPGPOption

	res, err := ctx.DoctorUI.SelectSigner(arg)
	if err != nil {
		return err
	}

	if res.Action == keybase_1.SelectSignerAction_CANCEL {
		// XXX another way to bail besides returning an error?
		return fmt.Errorf("cancel requested by user")
	}
	if res.Action == keybase_1.SelectSignerAction_RESET_ACCOUNT {
		d.G().Log.Info("reset account action not yet implemented")
		return ErrNotYetImplemented
	}

	if res.Action != keybase_1.SelectSignerAction_SIGN {
		return fmt.Errorf("unknown action value: %d", res.Action)
	}

	// sign action:

	if res.Signer.Kind == keybase_1.DeviceSignerKind_PGP {
		return d.deviceSignPGP(ctx)
	}

	if res.Signer.Kind == keybase_1.DeviceSignerKind_DEVICE {
		return d.deviceSignExistingDevice(ctx, *res.Signer.DeviceID, devname, libkb.DEVICE_TYPE_DESKTOP)
	}

	return fmt.Errorf("unknown signer kind: %d", res.Signer.Kind)
}

func (d *Doctor) deviceSignPGP(ctx *Context) error {
	pgpKeys := d.user.GetActivePgpKeys(false)
	var selected *libkb.PgpKeyBundle
	if len(pgpKeys) > 1 {
		// show a list of pgp keys and let them select which one to use
		var err error
		selected, err = d.selectPGPKey(ctx, pgpKeys)
		if err != nil {
			return err
		}
		if selected == nil {
			return fmt.Errorf("no key selected")
		}
	} else {
		selected = pgpKeys[0]
	}

	d.G().Log.Info("selected pgp key: %s", selected.VerboseDescription())
	d.G().Log.Info("selected pgp key kid: %s", selected.GetKid())
	if pk, ok := d.G().SecretSyncer.FindPrivateKey(selected.GetKid().String()); ok {
		skb, err := pk.ToSKB()
		if err != nil {
			return err
		}

		pgpk, err := skb.PromptAndUnlock("pgp sign", "keybase", ctx.SecretUI)
		if err != nil {
			return err
		}
		return d.deviceSignPGPNext(ctx, pgpk)
	}

	// use gpg to unlock it
	gpg := d.G().GetGpgClient()
	if _, err := gpg.Configure(); err != nil {
		return err
	}

	bundle, err := gpg.ImportKey(true, selected.GetFingerprint())
	if err != nil {
		return fmt.Errorf("ImportKey error: %s", err)
	}

	if err := bundle.Unlock("Import of key into keybase keyring", ctx.SecretUI); err != nil {
		return fmt.Errorf("bundle Unlock error: %s", err)
	}

	return d.deviceSignPGPNext(ctx, bundle)
}

func (d *Doctor) deviceSignPGPNext(ctx *Context, pgpk libkb.GenericKey) error {
	if pgpk.CanSign() == false {
		return fmt.Errorf("pgp key can't sign")
	}

	eldest := d.user.GetEldestFOKID().Kid
	d.G().Log.Info("eldest kid from user: %s", eldest)
	if err := d.addDeviceKeyWithSigner(ctx, pgpk, eldest); err != nil {
		return err
	}

	if err := d.addDetKey(ctx, eldest); err != nil {
		return err
	}

	return nil
}

func (d *Doctor) deviceSignExistingDevice(ctx *Context, id, devName, devType string) error {
	d.G().Log.Info("device sign with existing device [%s]", id)

	src, err := libkb.NewDeviceID()
	if err != nil {
		return err
	}

	dst, err := libkb.ImportDeviceID(id)
	if err != nil {
		return err
	}

	tk, err := d.tspkey(ctx)
	if err != nil {
		return err
	}

	kargs := &KexFwdArgs{
		User:    d.user,
		Src:     src,
		Dst:     *dst,
		DevType: devType,
		DevDesc: devName,
	}
	k := NewKexFwd(tk.LksClientHalf(), kargs)
	return RunEngine(k, ctx)
}

func (d *Doctor) selectPGPKey(ctx *Context, keys []*libkb.PgpKeyBundle) (*libkb.PgpKeyBundle, error) {
	var gks []keybase_1.GPGKey
	for _, key := range keys {
		algo, kid, creation := key.KeyInfo()
		gk := keybase_1.GPGKey{
			Algorithm:  algo,
			KeyID:      kid,
			Creation:   creation,
			Identities: key.IdentityNames(),
		}
		gks = append(gks, gk)
	}

	keyid, err := ctx.GPGUI.SelectKey(keybase_1.SelectKeyArg{Keys: gks})
	if err != nil {
		return nil, err
	}
	d.G().Log.Info("SelectKey result: %+v", keyid)

	var selected *libkb.PgpKeyBundle
	for _, key := range keys {
		if key.GetFingerprint().ToKeyId() == keyid {
			selected = key
			break
		}
	}

	return selected, nil
}

func (d *Doctor) tspkey(ctx *Context) (libkb.PassphraseStream, error) {
	return d.G().LoginState.GetPassphraseStream(ctx.SecretUI)
}

func (d *Doctor) detkey(ctx *Context) (libkb.GenericKey, error) {
	// get server half of detkey via ss
	half, err := d.G().SecretSyncer.FindDetKeySrvHalf(libkb.KEY_TYPE_KB_NACL_EDDSA_SERVER_HALF)
	if err != nil {
		return nil, err
	}

	// regenerate the detkey
	tk, err := d.tspkey(ctx)
	if err != nil {
		return nil, err
	}

	detkey, err := GenSigningDetKey(tk, half)
	if err != nil {
		return nil, err
	}
	return detkey, nil
}

func (d *Doctor) deviceName(ctx *Context) (string, error) {
	if len(d.devName) == 0 {
		name, err := ctx.DoctorUI.PromptDeviceName(0)
		if err != nil {
			return "", err
		}
		d.devName = name
	}
	return d.devName, nil
}
<|endoftext|>