<commit_before>package docs

import (
	"fmt"
	"pygmentize"
	"strings"
	"time"

	"github.com/blang/semver"
	"github.com/vito/booklit"
)

var flyBinariesVersion = semver.MustParse("2.2.0")

func init() {
	booklit.RegisterPlugin("concourse-docs", NewPlugin)
}

type Plugin struct {
	section *booklit.Section
}

func NewPlugin(section *booklit.Section) booklit.Plugin {
	return Plugin{
		section: section,
	}
}

func (p Plugin) FontAwesome(class string) booklit.Content {
	return booklit.Styled{
		Style:   "font-awesome",
		Content: booklit.String(class),
	}
}

func (p Plugin) Codeblock(language string, code booklit.Content) (booklit.Content, error) {
	code, err := pygmentize.Block(language, code.String())
	if err != nil {
		return nil, err
	}

	return booklit.Styled{
		Style:   "codeblock",
		Content: code,
	}, nil
}

func (p Plugin) TitledCodeblock(title booklit.Content, language string, code booklit.Content) (booklit.Content, error) {
	codeblock, err := p.Codeblock(language, code)
	if err != nil {
		return nil, err
	}

	return booklit.Styled{
		Style: "titled-codeblock",

		Content: codeblock,

		Partials: booklit.Partials{
			"Title": booklit.Styled{
				Style:   booklit.StyleVerbatim,
				Content: title,
			},
		},
	}, nil
}

func (p Plugin) Warn(content booklit.Content) booklit.Content {
	return booklit.Styled{
		Style:   "warning",
		Content: content,
	}
}

func (p Plugin) DefineAttribute(attribute string, content booklit.Content, tags ...string) (booklit.Content, error) {
	attrSplit := strings.SplitN(attribute, ":", 2)

	attrName := attrSplit[0]
	if len(tags) == 0 {
		tags = []string{attrName}
	}

	display := booklit.Styled{
		Style: booklit.StyleVerbatim,
		Content: booklit.Styled{
			Style:   booklit.StyleBold,
			Content: booklit.String(attrName),
		},
	}

	targets := booklit.Sequence{}
	for _, t := range tags {
		targets = append(targets, booklit.Target{
			TagName: t,
			Display: display,
		})
	}

	return booklit.Styled{
		Style:   "definition",
		Content: content,
		Partials: booklit.Partials{
			"Targets": targets,
			"Thumb": booklit.Styled{
				Style: booklit.StyleVerbatim,
				Content: booklit.Preformatted{
					booklit.Sequence{
						&booklit.Reference{
							TagName: tags[0],
							Content: booklit.Styled{
								Style:   booklit.StyleBold,
								Content: booklit.String(attrName),
							},
						},
						booklit.String(":" + attrSplit[1]),
					},
				},
			},
		},
	}, nil
}

func (p Plugin) DefineMetric(metric string, content booklit.Content) booklit.Content {
	return booklit.Styled{
		Style:   "definition",
		Content: content,
		Partials: booklit.Partials{
			"Targets": booklit.Target{
				TagName: metric,
				Display: booklit.String(metric),
			},
			"Thumb": booklit.Styled{
				Style:   booklit.StyleVerbatim,
				Content: booklit.Preformatted{booklit.String(metric)},
			},
		},
	}
}

func (p Plugin) Boshprop(job string, target string) booklit.Content {
	return booklit.Link{
		Target: fmt.Sprintf("http://bosh.io/jobs/%s?source=github.com/concourse/concourse#p=%s", job, target),
		Content: booklit.Styled{
			Style:   booklit.StyleVerbatim,
			Content: booklit.String(target),
		},
	}
}

func (p Plugin) Ghuser(user string) booklit.Content {
	return booklit.Link{
		Target: fmt.Sprintf("http://github.com/%s", user),
		Content: booklit.Styled{
			Style:   booklit.StyleBold,
			Content: booklit.String(user),
		},
	}
}

func (p Plugin) Ghissue(number string, optionalRepo ...string) booklit.Content {
	repo := "concourse"
	if len(optionalRepo) > 0 {
		repo = optionalRepo[0]
	}

	return booklit.Link{
		Target: fmt.Sprintf("http://github.com/concourse/%s/issues/%s", repo, number),
		Content: booklit.Styled{
			Style:   booklit.StyleBold,
			Content: booklit.String("#" + number),
		},
	}
}

func (p Plugin) Resource(resource string, optionalName ...string) booklit.Content {
	name := ""
	if len(optionalName) > 0 {
		name = optionalName[0]
	} else {
		for _, word := range strings.Split(resource, "-") {
			if name != "" {
				name += " "
			}

			name += strings.Title(word)
		}
	}

	return booklit.Link{
		Target:  fmt.Sprintf("http://github.com/concourse/%s-resource", resource),
		Content: booklit.String(fmt.Sprintf("%s resource", name)),
	}
}

func (p Plugin) TutorialImage(path string) booklit.Content {
	return booklit.Styled{
		Style: "classed-image",
		Content: booklit.Image{
			Path:        path,
			Description: "tutorial image",
		},
		Partials: booklit.Partials{"Class": booklit.String("tutorial-image")},
	}
}

func (p Plugin) LiterateSegment(parasAndFinalCode ...booklit.Content) (booklit.Content, error) {
	if len(parasAndFinalCode) == 0 {
		return nil, fmt.Errorf("no paragraphs or code given")
	}

	paras := parasAndFinalCode[0 : len(parasAndFinalCode)-1]
	code := parasAndFinalCode[len(parasAndFinalCode)-1]

	if len(paras) == 0 {
		paras = []booklit.Content{code}
		code = booklit.Empty
	}

	return booklit.Styled{
		Style:   "literate-segment",
		Content: booklit.Sequence(paras),
		Partials: booklit.Partials{
			"Code": code,
		},
	}, nil
}

func (p Plugin) PipelineImage(path string) booklit.Content {
	return booklit.Styled{
		Style: "pipeline-image",
		Content: booklit.Image{
			Path:        path,
			Description: "pipeline",
		},
	}
}

func (p Plugin) ReleaseWithGardenLinux(date string, concourseVersion string, gardenLinuxVersion string, content booklit.Content) (booklit.Content, error) {
	p.section.SetPartial("GardenReleaseFilename", booklit.String("garden-linux"))
	p.section.SetPartial("GardenReleaseName", booklit.String("Garden Linux"))
	return p.release(date, concourseVersion, gardenLinuxVersion, content)
}

func (p Plugin) Release(date string, concourseVersion string, gardenRunCVersion string, content booklit.Content) (booklit.Content, error) {
	p.section.SetPartial("GardenReleaseFilename", booklit.String("garden-runc"))
	p.section.SetPartial("GardenReleaseName", booklit.String("Garden runC"))
	return p.release(date, concourseVersion, gardenRunCVersion, content)
}

func (p Plugin) release(
	date string,
	concourseVersion string,
	gardenVersion string,
	content booklit.Content,
) (booklit.Content, error) {
	t, err := time.Parse("2006-1-2", date)
	if err != nil {
		return nil, err
	}

	p.section.SetTitle(booklit.String("v" + concourseVersion))

	p.section.SetPartial("Version", booklit.String(concourseVersion))
	p.section.SetPartial("VersionLabel", booklit.Styled{
		Style:   "release-version-number",
		Content: booklit.String("v" + concourseVersion),
	})

	p.section.SetPartial("GardenVersion", booklit.String(gardenVersion))
	p.section.SetPartial("GardenVersionLabel", booklit.Styled{
		Style:   "release-version-number",
		Content: booklit.String("v" + gardenVersion),
	})

	p.section.SetPartial("ReleaseDate", booklit.Styled{
		Style:   "release-date",
		Content: booklit.String(t.Format("January 2, 2006")),
	})

	cv, err := semver.Parse(concourseVersion)
	if err != nil {
		return nil, err
	}

	if cv.GTE(flyBinariesVersion) {
		p.section.SetPartial("HasFlyBinaries", booklit.Empty)
	}

	return content, nil
}

func (p Plugin) Note(commaSeparatedTags string, content booklit.Content) booklit.Content {
	tags := strings.Split(commaSeparatedTags, ",")

	tagNotes := []booklit.Content{}
	for _, t := range tags {
		tagNotes = append(tagNotes, booklit.Styled{
			Style:   "release-note-tag",
			Content: booklit.String(t),
		})
	}

	return booklit.Styled{
		Style:   "release-note",
		Content: content,
		Partials: booklit.Partials{
			"Tags": booklit.List{Items: tagNotes},
		},
	}
}
<commit_msg>put @ in front of github usernames<commit_after>package docs

import (
	"fmt"
	"pygmentize"
	"strings"
	"time"

	"github.com/blang/semver"
	"github.com/vito/booklit"
)

var flyBinariesVersion = semver.MustParse("2.2.0")

func init() {
	booklit.RegisterPlugin("concourse-docs", NewPlugin)
}

type Plugin struct {
	section *booklit.Section
}

func NewPlugin(section *booklit.Section) booklit.Plugin {
	return Plugin{
		section: section,
	}
}

func (p Plugin) FontAwesome(class string) booklit.Content {
	return booklit.Styled{
		Style:   "font-awesome",
		Content: booklit.String(class),
	}
}

func (p Plugin) Codeblock(language string, code booklit.Content) (booklit.Content, error) {
	code, err := pygmentize.Block(language, code.String())
	if err != nil {
		return nil, err
	}

	return booklit.Styled{
		Style:   "codeblock",
		Content: code,
	}, nil
}

func (p Plugin) TitledCodeblock(title booklit.Content, language string, code booklit.Content) (booklit.Content, error) {
	codeblock, err := p.Codeblock(language, code)
	if err != nil {
		return nil, err
	}

	return booklit.Styled{
		Style: "titled-codeblock",

		Content: codeblock,

		Partials: booklit.Partials{
			"Title": booklit.Styled{
				Style:   booklit.StyleVerbatim,
				Content: title,
			},
		},
	}, nil
}

func (p Plugin) Warn(content booklit.Content) booklit.Content {
	return booklit.Styled{
		Style:   "warning",
		Content: content,
	}
}

func (p Plugin) DefineAttribute(attribute string, content booklit.Content, tags ...string) (booklit.Content, error) {
	attrSplit := strings.SplitN(attribute, ":", 2)

	attrName := attrSplit[0]
	if len(tags) == 0 {
		tags = []string{attrName}
	}

	display := booklit.Styled{
		Style: booklit.StyleVerbatim,
		Content: booklit.Styled{
			Style:   booklit.StyleBold,
			Content: booklit.String(attrName),
		},
	}

	targets := booklit.Sequence{}
	for _, t := range tags {
		targets = append(targets, booklit.Target{
			TagName: t,
			Display: display,
		})
	}

	return booklit.Styled{
		Style:   "definition",
		Content: content,
		Partials: booklit.Partials{
			"Targets": targets,
			"Thumb": booklit.Styled{
				Style: booklit.StyleVerbatim,
				Content: booklit.Preformatted{
					booklit.Sequence{
						&booklit.Reference{
							TagName: tags[0],
							Content: booklit.Styled{
								Style:   booklit.StyleBold,
								Content: booklit.String(attrName),
							},
						},
						booklit.String(":" + attrSplit[1]),
					},
				},
			},
		},
	}, nil
}

func (p Plugin) DefineMetric(metric string, content booklit.Content) booklit.Content {
	return booklit.Styled{
		Style:   "definition",
		Content: content,
		Partials: booklit.Partials{
			"Targets": booklit.Target{
				TagName: metric,
				Display: booklit.String(metric),
			},
			"Thumb": booklit.Styled{
				Style:   booklit.StyleVerbatim,
				Content: booklit.Preformatted{booklit.String(metric)},
			},
		},
	}
}

func (p Plugin) Boshprop(job string, target string) booklit.Content {
	return booklit.Link{
		Target: fmt.Sprintf("http://bosh.io/jobs/%s?source=github.com/concourse/concourse#p=%s", job, target),
		Content: booklit.Styled{
			Style:   booklit.StyleVerbatim,
			Content: booklit.String(target),
		},
	}
}

func (p Plugin) Ghuser(user string) booklit.Content {
	return booklit.Link{
		Target: fmt.Sprintf("http://github.com/%s", user),
		Content: booklit.Styled{
			Style:   booklit.StyleBold,
			Content: booklit.String("@" + user),
		},
	}
}

func (p Plugin) Ghissue(number string, optionalRepo ...string) booklit.Content {
	repo := "concourse"
	if len(optionalRepo) > 0 {
		repo = optionalRepo[0]
	}

	return booklit.Link{
		Target: fmt.Sprintf("http://github.com/concourse/%s/issues/%s", repo, number),
		Content: booklit.Styled{
			Style:   booklit.StyleBold,
			Content: booklit.String("#" + number),
		},
	}
}

func (p Plugin) Resource(resource string, optionalName ...string) booklit.Content {
	name := ""
	if len(optionalName) > 0 {
		name = optionalName[0]
	} else {
		for _, word := range strings.Split(resource, "-") {
			if name != "" {
				name += " "
			}

			name += strings.Title(word)
		}
	}

	return booklit.Link{
		Target:  fmt.Sprintf("http://github.com/concourse/%s-resource", resource),
		Content: booklit.String(fmt.Sprintf("%s resource", name)),
	}
}

func (p Plugin) TutorialImage(path string) booklit.Content {
	return booklit.Styled{
		Style: "classed-image",
		Content: booklit.Image{
			Path:        path,
			Description: "tutorial image",
		},
		Partials: booklit.Partials{"Class": booklit.String("tutorial-image")},
	}
}

func (p Plugin) LiterateSegment(parasAndFinalCode ...booklit.Content) (booklit.Content, error) {
	if len(parasAndFinalCode) == 0 {
		return nil, fmt.Errorf("no paragraphs or code given")
	}

	paras := parasAndFinalCode[0 : len(parasAndFinalCode)-1]
	code := parasAndFinalCode[len(parasAndFinalCode)-1]

	if len(paras) == 0 {
		paras = []booklit.Content{code}
		code = booklit.Empty
	}

	return booklit.Styled{
		Style:   "literate-segment",
		Content: booklit.Sequence(paras),
		Partials: booklit.Partials{
			"Code": code,
		},
	}, nil
}

func (p Plugin) PipelineImage(path string) booklit.Content {
	return booklit.Styled{
		Style: "pipeline-image",
		Content: booklit.Image{
			Path:        path,
			Description: "pipeline",
		},
	}
}

func (p Plugin) ReleaseWithGardenLinux(date string, concourseVersion string, gardenLinuxVersion string, content booklit.Content) (booklit.Content, error) {
	p.section.SetPartial("GardenReleaseFilename", booklit.String("garden-linux"))
	p.section.SetPartial("GardenReleaseName", booklit.String("Garden Linux"))
	return p.release(date, concourseVersion, gardenLinuxVersion, content)
}

func (p Plugin) Release(date string, concourseVersion string, gardenRunCVersion string, content booklit.Content) (booklit.Content, error) {
	p.section.SetPartial("GardenReleaseFilename", booklit.String("garden-runc"))
	p.section.SetPartial("GardenReleaseName", booklit.String("Garden runC"))
	return p.release(date, concourseVersion, gardenRunCVersion, content)
}

func (p Plugin) release(
	date string,
	concourseVersion string,
	gardenVersion string,
	content booklit.Content,
) (booklit.Content, error) {
	t, err := time.Parse("2006-1-2", date)
	if err != nil {
		return nil, err
	}

	p.section.SetTitle(booklit.String("v" + concourseVersion))

	p.section.SetPartial("Version", booklit.String(concourseVersion))
	p.section.SetPartial("VersionLabel", booklit.Styled{
		Style:   "release-version-number",
		Content: booklit.String("v" + concourseVersion),
	})

	p.section.SetPartial("GardenVersion", booklit.String(gardenVersion))
	p.section.SetPartial("GardenVersionLabel", booklit.Styled{
		Style:   "release-version-number",
		Content: booklit.String("v" + gardenVersion),
	})

	p.section.SetPartial("ReleaseDate", booklit.Styled{
		Style:   "release-date",
		Content: booklit.String(t.Format("January 2, 2006")),
	})

	cv, err := semver.Parse(concourseVersion)
	if err != nil {
		return nil, err
	}

	if cv.GTE(flyBinariesVersion) {
		p.section.SetPartial("HasFlyBinaries", booklit.Empty)
	}

	return content, nil
}

func (p Plugin) Note(commaSeparatedTags string, content booklit.Content) booklit.Content {
	tags := strings.Split(commaSeparatedTags, ",")

	tagNotes := []booklit.Content{}
	for _, t := range tags {
		tagNotes = append(tagNotes, booklit.Styled{
			Style:   "release-note-tag",
			Content: booklit.String(t),
		})
	}

	return booklit.Styled{
		Style:   "release-note",
		Content: content,
		Partials: booklit.Partials{
			"Tags": booklit.List{Items: tagNotes},
		},
	}
}
<|endoftext|>


<commit_before>package index

import (
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node"
	"github.com/MG-RAST/Shock/shock-server/node/file/index"
	"github.com/MG-RAST/Shock/shock-server/request"
	"github.com/MG-RAST/Shock/shock-server/responder"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/stretchr/goweb/context"
	"net/http"
	"os"
)

type getRes struct {
	I interface{} `json:"indexes"`
	A interface{} `json:"available_indexers"`
}

type m map[string]string

// GET, PUT, DELETE: /node/{nid}/index/{idxType}
func IndexTypedRequest(ctx context.Context) {
	nid := ctx.PathValue("nid")
	idxType := ctx.PathValue("idxType")

	u, err := request.Authenticate(ctx.HttpRequest())
	if err != nil && err.Error() != e.NoAuth {
		request.AuthError(err, ctx)
		return
	}

	// Fake public user
	if u == nil {
		u = &user.User{Uuid: ""}
	}

	// Load node and handle user unauthorized
	n, err := node.Load(nid, u.Uuid)
	if err != nil {
		if err.Error() == e.UnAuth {
			responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
			return
		} else if err.Error() == e.MongoDocNotFound {
			responder.RespondWithError(ctx, http.StatusNotFound, "Node not found.")
			return
		} else {
			// In theory the db connection could be lost between
			// checking user and load but seems unlikely.
			err_msg := "Err@index:LoadNode: " + err.Error()
			logger.Error(err_msg)
			responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
			return
		}
	}

	switch ctx.HttpRequest().Method {
	case "GET":
		if v, has := n.Indexes[idxType]; has {
			responder.RespondWithData(ctx, map[string]interface{}{idxType: v})
		} else {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, idxType))
		}

	case "PUT":
		if !n.HasFile() {
			responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file.")
			return
		} else if idxType == "" {
			responder.RespondWithError(ctx, http.StatusBadRequest, "Index create requires type.")
			return
		}
		if _, ok := index.Indexers[idxType]; !ok && idxType != "bai" && idxType != "subset" {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Index type %s unavailable.", idxType))
			return
		}
		if idxType == "size" {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Index type size is a virtual index and does not require index building."))
			return
		}

		if conf.Bool(conf.Conf["perf-log"]) {
			logger.Perf("START indexing: " + nid)
		}

		if idxType == "bai" {
			//bam index is created by the command-line tool samtools
			if ext := n.FileExt(); ext == ".bam" {
				if err := index.CreateBamIndex(n.FilePath()); err != nil {
					responder.RespondWithError(ctx, http.StatusBadRequest, "Error while creating bam index.")
					return
				}
				responder.RespondOK(ctx)
				return
			} else {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type bai requires .bam file.")
				return
			}
		}

		count := int64(0)
		if idxType == "subset" {
			// Utilizing the multipart form parser since we need to upload a file.
			params, files, err := request.ParseMultipartForm(ctx.HttpRequest())
			if err != nil {
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
				return
			}

			parentIndex, hasParent := params["parent_index"]
			if !hasParent {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires parent_index param.")
				return
			} else if _, has := n.Indexes[parentIndex]; !has {
				responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, parentIndex))
				return
			}

			newIndex, hasName := params["index_name"]
			if !hasName {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires index_name param.")
				return
			} else if _, reservedName := index.Indexers[newIndex]; reservedName || newIndex == "bai" {
				responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("%s is a reserved index name and cannot be used to create a custom subset index.", newIndex))
				return
			}

			subsetIndices, hasFile := files["subset_indices"]
			if !hasFile {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires subset_indices file.")
				return
			}

			f, _ := os.Open(subsetIndices.Path)
			defer f.Close()
			idxer := index.NewSubsetIndexer(f)
			count, err = index.CreateSubsetIndex(&idxer, n.IndexPath()+"/"+newIndex+".idx", n.IndexPath()+"/"+parentIndex+".idx")
			if err != nil {
				logger.Error("err " + err.Error())
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
			}

		} else {
			newIndexer := index.Indexers[idxType]
			f, _ := os.Open(n.FilePath())
			defer f.Close()
			idxer := newIndexer(f)
			count, err = idxer.Create(n.IndexPath() + "/" + idxType + ".idx")
			if err != nil {
				logger.Error("err " + err.Error())
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
				return
			}
		}

		if count == 0 {
			responder.RespondWithError(ctx, http.StatusBadRequest, "Index empty.")
			return
		}

		idxInfo := node.IdxInfo{
			Type:        idxType,
			TotalUnits:  count,
			AvgUnitSize: n.File.Size / count,
		}

		//if idxType == "chunkrecord" {
		//	idxInfo.AvgUnitSize = conf.CHUNK_SIZE
		//}

		if idxType == "subset" {
			idxInfo.AvgUnitSize = -1
		}

		if err := n.SetIndexInfo(idxType, idxInfo); err != nil {
			logger.Error("err@node.SetIndexInfo: " + err.Error())
		}

		if conf.Bool(conf.Conf["perf-log"]) {
			logger.Perf("END indexing: " + nid)
		}

		responder.RespondOK(ctx)
		return

	default:
		responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.")
	}
	return
}

func contains(list []string, s string) bool {
	for _, i := range list {
		if i == s {
			return true
		}
	}
	return false
}

func availIndexers() (indexers []string) {
	for name, _ := range index.Indexers {
		indexers = append(indexers, name)
	}
	return
}
<commit_msg>Need to die if indexing returns an error.<commit_after>package index

import (
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node"
	"github.com/MG-RAST/Shock/shock-server/node/file/index"
	"github.com/MG-RAST/Shock/shock-server/request"
	"github.com/MG-RAST/Shock/shock-server/responder"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/stretchr/goweb/context"
	"net/http"
	"os"
)

type getRes struct {
	I interface{} `json:"indexes"`
	A interface{} `json:"available_indexers"`
}

type m map[string]string

// GET, PUT, DELETE: /node/{nid}/index/{idxType}
func IndexTypedRequest(ctx context.Context) {
	nid := ctx.PathValue("nid")
	idxType := ctx.PathValue("idxType")

	u, err := request.Authenticate(ctx.HttpRequest())
	if err != nil && err.Error() != e.NoAuth {
		request.AuthError(err, ctx)
		return
	}

	// Fake public user
	if u == nil {
		u = &user.User{Uuid: ""}
	}

	// Load node and handle user unauthorized
	n, err := node.Load(nid, u.Uuid)
	if err != nil {
		if err.Error() == e.UnAuth {
			responder.RespondWithError(ctx, http.StatusUnauthorized, e.UnAuth)
			return
		} else if err.Error() == e.MongoDocNotFound {
			responder.RespondWithError(ctx, http.StatusNotFound, "Node not found.")
			return
		} else {
			// In theory the db connection could be lost between
			// checking user and load but seems unlikely.
			err_msg := "Err@index:LoadNode: " + err.Error()
			logger.Error(err_msg)
			responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
			return
		}
	}

	switch ctx.HttpRequest().Method {
	case "GET":
		if v, has := n.Indexes[idxType]; has {
			responder.RespondWithData(ctx, map[string]interface{}{idxType: v})
		} else {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, idxType))
		}

	case "PUT":
		if !n.HasFile() {
			responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file.")
			return
		} else if idxType == "" {
			responder.RespondWithError(ctx, http.StatusBadRequest, "Index create requires type.")
			return
		}
		if _, ok := index.Indexers[idxType]; !ok && idxType != "bai" && idxType != "subset" {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Index type %s unavailable.", idxType))
			return
		}
		if idxType == "size" {
			responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Index type size is a virtual index and does not require index building."))
			return
		}

		if conf.Bool(conf.Conf["perf-log"]) {
			logger.Perf("START indexing: " + nid)
		}

		if idxType == "bai" {
			//bam index is created by the command-line tool samtools
			if ext := n.FileExt(); ext == ".bam" {
				if err := index.CreateBamIndex(n.FilePath()); err != nil {
					responder.RespondWithError(ctx, http.StatusBadRequest, "Error while creating bam index.")
					return
				}
				responder.RespondOK(ctx)
				return
			} else {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type bai requires .bam file.")
				return
			}
		}

		count := int64(0)
		if idxType == "subset" {
			// Utilizing the multipart form parser since we need to upload a file.
			params, files, err := request.ParseMultipartForm(ctx.HttpRequest())
			if err != nil {
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
				return
			}

			parentIndex, hasParent := params["parent_index"]
			if !hasParent {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires parent_index param.")
				return
			} else if _, has := n.Indexes[parentIndex]; !has {
				responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("Node %s does not have index of type %s.", n.Id, parentIndex))
				return
			}

			newIndex, hasName := params["index_name"]
			if !hasName {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires index_name param.")
				return
			} else if _, reservedName := index.Indexers[newIndex]; reservedName || newIndex == "bai" {
				responder.RespondWithError(ctx, http.StatusBadRequest, fmt.Sprintf("%s is a reserved index name and cannot be used to create a custom subset index.", newIndex))
				return
			}

			subsetIndices, hasFile := files["subset_indices"]
			if !hasFile {
				responder.RespondWithError(ctx, http.StatusBadRequest, "Index type subset requires subset_indices file.")
				return
			}

			f, _ := os.Open(subsetIndices.Path)
			defer f.Close()
			idxer := index.NewSubsetIndexer(f)
			count, err = index.CreateSubsetIndex(&idxer, n.IndexPath()+"/"+newIndex+".idx", n.IndexPath()+"/"+parentIndex+".idx")
			if err != nil {
				logger.Error("err " + err.Error())
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
				return
			}

		} else {
			newIndexer := index.Indexers[idxType]
			f, _ := os.Open(n.FilePath())
			defer f.Close()
			idxer := newIndexer(f)
			count, err = idxer.Create(n.IndexPath() + "/" + idxType + ".idx")
			if err != nil {
				logger.Error("err " + err.Error())
				responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
				return
			}
		}

		if count == 0 {
			responder.RespondWithError(ctx, http.StatusBadRequest, "Index empty.")
			return
		}

		idxInfo := node.IdxInfo{
			Type:        idxType,
			TotalUnits:  count,
			AvgUnitSize: n.File.Size / count,
		}

		//if idxType == "chunkrecord" {
		//	idxInfo.AvgUnitSize = conf.CHUNK_SIZE
		//}

		if idxType == "subset" {
			idxInfo.AvgUnitSize = -1
		}

		if err := n.SetIndexInfo(idxType, idxInfo); err != nil {
			logger.Error("err@node.SetIndexInfo: " + err.Error())
		}

		if conf.Bool(conf.Conf["perf-log"]) {
			logger.Perf("END indexing: " + nid)
		}

		responder.RespondOK(ctx)
		return

	default:
		responder.RespondWithError(ctx, http.StatusNotImplemented, "This request type is not implemented.")
	}
	return
}

func contains(list []string, s string) bool {
	for _, i := range list {
		if i == s {
			return true
		}
	}
	return false
}

func availIndexers() (indexers []string) {
	for name, _ := range index.Indexers {
		indexers = append(indexers, name)
	}
	return
}
<|endoftext|>


<commit_before>package volume

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/nu7hatch/gouuid"
	"github.com/pivotal-golang/lager"
)

type Strategy map[string]string

const (
	StrategyEmpty       = "empty"
	StrategyCopyOnWrite = "cow"

	propertiesFileName = "properties.json"
)

type Volume struct {
	GUID       string     `json:"guid"`
	Path       string     `json:"path"`
	Properties Properties `json:"properties"`
}

type Volumes []Volume

var ErrMissingStrategy = errors.New("missing strategy")
var ErrUnrecognizedStrategy = errors.New("unrecognized strategy")
var ErrCreateVolumeFailed = errors.New("failed to create volume")
var ErrSetPropertyFailed = errors.New("failed to set property on volume")
var ErrListVolumesFailed = errors.New("failed to list volumes")
var ErrNoParentVolumeProvided = errors.New("no parent volume provided")
var ErrParentVolumeNotFound = errors.New("parent volume not found")

type Repository struct {
	volumeDir string
	driver    Driver

	logger lager.Logger
}

type propertiesFile struct {
	path string
}

func (pf *propertiesFile) WriteProperties(properties Properties) error {
	file, err := os.OpenFile(
		pf.path,
		os.O_WRONLY|os.O_CREATE,
		0644,
	)
	if err != nil {
		return err
	}
	defer file.Close()

	return json.NewEncoder(file).Encode(properties)
}

func (pf *propertiesFile) Properties() (Properties, error) {
	var properties Properties

	file, err := os.Open(pf.path)
	if err != nil {
		return Properties{}, err
	}
	defer file.Close()

	if err := json.NewDecoder(file).Decode(&properties); err != nil {
		return Properties{}, err
	}

	return properties, nil
}

func NewRepository(logger lager.Logger, volumeDir string, driver Driver) *Repository {
	return &Repository{
		volumeDir: volumeDir,
		logger:    logger,
		driver:    driver,
	}
}

type Driver interface {
	CreateVolume(path string) error
	CreateCopyOnWriteLayer(path string, parent string) error
}

func (repo *Repository) CreateVolume(strategy Strategy, properties Properties) (Volume, error) {
	strategyName, found := strategy["type"]
	if !found {
		return Volume{}, ErrMissingStrategy
	}

	logger := repo.logger.Session("create-volume", lager.Data{
		"strategy": strategyName,
	})

	guid := repo.createUuid()
	newVolumeMetadataPath := repo.metadataPath(guid)
	err := os.Mkdir(newVolumeMetadataPath, 0755)
	if err != nil {
		return repo.handleError(err, "failed-to-create-metadata-dir", ErrCreateVolumeFailed)
	}

	err = repo.propertiesFile(guid).WriteProperties(properties)
	if err != nil {
		return repo.handleError(err, "failed-to-create-properties-file", ErrCreateVolumeFailed)
	}

	newVolumeDataPath := repo.dataPath(guid)
	err = repo.doStrategy(strategyName, newVolumeDataPath, strategy, logger)
	if err != nil {
		repo.deleteVolumeMetadataDir(guid)
		return Volume{}, err
	}

	return Volume{
		Path:       newVolumeDataPath,
		GUID:       guid,
		Properties: properties,
	}, nil
}

func (repo *Repository) handleError(internalError error, errorMsg string, externalError error) (Volume, error) {
	repo.logger.Error(errorMsg, internalError)
	return Volume{}, externalError
}

func (repo *Repository) ListVolumes(queryProperties Properties) (Volumes, error) {
	volumeDirs, err := ioutil.ReadDir(repo.volumeDir)
	if err != nil {
		repo.logger.Error("failed-to-list-dirs", err, lager.Data{
			"volume-dir": repo.volumeDir,
		})

		return Volumes{}, ErrListVolumesFailed
	}

	response := make(Volumes, 0, len(volumeDirs))

	for _, volumeDir := range volumeDirs {
		volumeProperties, err := repo.propertiesFile(volumeDir.Name()).Properties()
		if err != nil {
			repo.logger.Error("failed-to-read-properties", err, lager.Data{
				"volume": volumeDir.Name(),
			})
			return nil, err
		}

		if volumeProperties.HasProperties(queryProperties) {
			response = append(response, Volume{
				GUID:       volumeDir.Name(),
				Path:       repo.dataPath(volumeDir.Name()),
				Properties: volumeProperties,
			})
		}
	}

	return response, nil
}

func (repo *Repository) SetProperty(volumeGUID string, propertyName string, propertyValue string) error {
	pf := repo.propertiesFile(volumeGUID)

	properties, err := pf.Properties()
	if err != nil {
		repo.logger.Error("failed-to-read-properties", err, lager.Data{
			"volume": volumeGUID,
		})
		return err
	}

	properties = properties.UpdateProperty(propertyName, propertyValue)

	err = pf.WriteProperties(properties)
	if err != nil {
		_, err = repo.handleError(err, "failed-to-write-properties", ErrSetPropertyFailed)
		return err
	}

	return nil
}

func (repo *Repository) doStrategy(strategyName string, newVolumeDataPath string, strategy Strategy, logger lager.Logger) error {
	switch strategyName {
	case StrategyEmpty:
		err := repo.createEmptyVolume(newVolumeDataPath)
		if err != nil {
			logger.Error("failed-to-create-volume", err, lager.Data{
				"path": newVolumeDataPath,
			})
			return ErrCreateVolumeFailed
		}

	case StrategyCopyOnWrite:
		parentGUID, found := strategy["volume"]
		if !found {
			logger.Error("no-parent-volume-provided", nil)
			return ErrNoParentVolumeProvided
		}

		if !repo.volumeExists(parentGUID) {
			logger.Error("parent-volume-not-found", nil)
			return ErrParentVolumeNotFound
		}

		parentDataPath := repo.dataPath(parentGUID)
		err := repo.createCowVolume(parentDataPath, newVolumeDataPath)
		if err != nil {
			logger.Error("failed-to-copy-volume", err)
			return ErrCreateVolumeFailed
		}

	default:
		logger.Error("unrecognized-strategy", nil, lager.Data{
			"strategy": strategyName,
		})
		return ErrUnrecognizedStrategy
	}

	return nil
}

func (repo *Repository) metadataPath(id string) string {
	return filepath.Join(repo.volumeDir, id)
}

func (repo *Repository) propertiesPath(id string) string {
	return filepath.Join(repo.metadataPath(id), propertiesFileName)
}

func (repo *Repository) propertiesFile(id string) *propertiesFile {
	return &propertiesFile{path: repo.propertiesPath(id)}
}

func (repo *Repository) dataPath(id string) string {
	return filepath.Join(repo.metadataPath(id), "volume")
}

func (repo *Repository) deleteVolumeMetadataDir(id string) {
	err := os.RemoveAll(repo.metadataPath(id))
	if err != nil {
		repo.logger.Error("failed-to-cleanup", err, lager.Data{
			"guid": id,
		})
	}
}

func (repo *Repository) createEmptyVolume(volumePath string) error {
	err := repo.driver.CreateVolume(volumePath)
	if err != nil {
		return err
	}

	return nil
}

func (repo *Repository) createCowVolume(parentPath string, newPath string) error {
	err := repo.driver.CreateCopyOnWriteLayer(newPath, parentPath)
	if err != nil {
		return err
	}

	return nil
}

func (repo *Repository) createUuid() string {
	guid, err := uuid.NewV4()
	if err != nil {
		repo.logger.Fatal("failed-to-generate-guid", err)
	}

	return guid.String()
}

func (repo *Repository) volumeExists(guid string) bool {
	info, err := os.Stat(repo.metadataPath(guid))
	if err != nil {
		return false
	}

	return info.IsDir()
}
<commit_msg>add locking to setting properties<commit_after>package volume

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"

	"github.com/nu7hatch/gouuid"
	"github.com/pivotal-golang/lager"
)

type Strategy map[string]string

const (
	StrategyEmpty       = "empty"
	StrategyCopyOnWrite = "cow"

	propertiesFileName = "properties.json"
)

type Volume struct {
	GUID       string     `json:"guid"`
	Path       string     `json:"path"`
	Properties Properties `json:"properties"`
}

type Volumes []Volume

var ErrMissingStrategy = errors.New("missing strategy")
var ErrUnrecognizedStrategy = errors.New("unrecognized strategy")
var ErrCreateVolumeFailed = errors.New("failed to create volume")
var ErrSetPropertyFailed = errors.New("failed to set property on volume")
var ErrListVolumesFailed = errors.New("failed to list volumes")
var ErrNoParentVolumeProvided = errors.New("no parent volume provided")
var ErrParentVolumeNotFound = errors.New("parent volume not found")

type Repository struct {
	volumeDir string
	driver    Driver

	logger lager.Logger

	setPropertyLock *sync.Mutex
}

type propertiesFile struct {
	path string
}

func (pf *propertiesFile) WriteProperties(properties Properties) error {
	file, err := os.OpenFile(
		pf.path,
		os.O_WRONLY|os.O_CREATE,
		0644,
	)
	if err != nil {
		return err
	}
	defer file.Close()

	return json.NewEncoder(file).Encode(properties)
}

func (pf *propertiesFile) Properties() (Properties, error) {
	var properties Properties

	file, err := os.Open(pf.path)
	if err != nil {
		return Properties{}, err
	}
	defer file.Close()

	if err := json.NewDecoder(file).Decode(&properties); err != nil {
		return Properties{}, err
	}

	return properties, nil
}

func NewRepository(logger lager.Logger, volumeDir string, driver Driver) *Repository {
	return &Repository{
		volumeDir:       volumeDir,
		logger:          logger,
		driver:          driver,
		setPropertyLock: &sync.Mutex{},
	}
}

type Driver interface {
	CreateVolume(path string) error
	CreateCopyOnWriteLayer(path string, parent string) error
}

func (repo *Repository) CreateVolume(strategy Strategy, properties Properties) (Volume, error) {
	strategyName, found := strategy["type"]
	if !found {
		return Volume{}, ErrMissingStrategy
	}

	logger := repo.logger.Session("create-volume", lager.Data{
		"strategy": strategyName,
	})

	guid := repo.createUuid()
	newVolumeMetadataPath := repo.metadataPath(guid)
	err := os.Mkdir(newVolumeMetadataPath, 0755)
	if err != nil {
		return repo.handleError(err, "failed-to-create-metadata-dir", ErrCreateVolumeFailed)
	}

	err = repo.propertiesFile(guid).WriteProperties(properties)
	if err != nil {
		return repo.handleError(err, "failed-to-create-properties-file", ErrCreateVolumeFailed)
	}

	newVolumeDataPath := repo.dataPath(guid)
	err = repo.doStrategy(strategyName, newVolumeDataPath, strategy, logger)
	if err != nil {
		repo.deleteVolumeMetadataDir(guid)
		return Volume{}, err
	}

	return Volume{
		Path:       newVolumeDataPath,
		GUID:       guid,
		Properties: properties,
	}, nil
}

func (repo *Repository) handleError(internalError error, errorMsg string, externalError error) (Volume, error) {
	repo.logger.Error(errorMsg, internalError)
	return Volume{}, externalError
}

func (repo *Repository) ListVolumes(queryProperties Properties) (Volumes, error) {
	volumeDirs, err := ioutil.ReadDir(repo.volumeDir)
	if err != nil {
		repo.logger.Error("failed-to-list-dirs", err, lager.Data{
			"volume-dir": repo.volumeDir,
		})

		return Volumes{}, ErrListVolumesFailed
	}

	response := make(Volumes, 0, len(volumeDirs))

	for _, volumeDir := range volumeDirs {
		volumeProperties, err := repo.propertiesFile(volumeDir.Name()).Properties()
		if err != nil {
			repo.logger.Error("failed-to-read-properties", err, lager.Data{
				"volume": volumeDir.Name(),
			})
			return nil, err
		}

		if volumeProperties.HasProperties(queryProperties) {
			response = append(response, Volume{
				GUID:       volumeDir.Name(),
				Path:       repo.dataPath(volumeDir.Name()),
				Properties: volumeProperties,
			})
		}
	}

	return response, nil
}

func (repo *Repository) SetProperty(volumeGUID string, propertyName string, propertyValue string) error {
	repo.setPropertyLock.Lock()
	defer repo.setPropertyLock.Unlock()

	pf := repo.propertiesFile(volumeGUID)

	properties, err := pf.Properties()
	if err != nil {
		repo.logger.Error("failed-to-read-properties", err, lager.Data{
			"volume": volumeGUID,
		})
		return err
	}

	properties = properties.UpdateProperty(propertyName, propertyValue)

	err = pf.WriteProperties(properties)
	if err != nil {
		_, err = repo.handleError(err, "failed-to-write-properties", ErrSetPropertyFailed)
		return err
	}

	return nil
}

func (repo *Repository) doStrategy(strategyName string, newVolumeDataPath string, strategy Strategy, logger lager.Logger) error {
	switch strategyName {
	case StrategyEmpty:
		err := repo.createEmptyVolume(newVolumeDataPath)
		if err != nil {
			logger.Error("failed-to-create-volume", err, lager.Data{
				"path": newVolumeDataPath,
			})
			return ErrCreateVolumeFailed
		}

	case StrategyCopyOnWrite:
		parentGUID, found := strategy["volume"]
		if !found {
			logger.Error("no-parent-volume-provided", nil)
			return ErrNoParentVolumeProvided
		}

		if !repo.volumeExists(parentGUID) {
			logger.Error("parent-volume-not-found", nil)
			return ErrParentVolumeNotFound
		}

		parentDataPath := repo.dataPath(parentGUID)
		err := repo.createCowVolume(parentDataPath, newVolumeDataPath)
		if err != nil {
			logger.Error("failed-to-copy-volume", err)
			return ErrCreateVolumeFailed
		}

	default:
		logger.Error("unrecognized-strategy", nil, lager.Data{
			"strategy": strategyName,
		})
		return ErrUnrecognizedStrategy
	}

	return nil
}

func (repo *Repository) metadataPath(id string) string {
	return filepath.Join(repo.volumeDir, id)
}

func (repo *Repository) propertiesPath(id string) string {
	return filepath.Join(repo.metadataPath(id), propertiesFileName)
}

func (repo *Repository) propertiesFile(id string) *propertiesFile {
	return &propertiesFile{path: repo.propertiesPath(id)}
}

func (repo *Repository) dataPath(id string) string {
	return filepath.Join(repo.metadataPath(id), "volume")
}

func (repo *Repository) deleteVolumeMetadataDir(id string) {
	err := os.RemoveAll(repo.metadataPath(id))
	if err != nil {
		repo.logger.Error("failed-to-cleanup", err, lager.Data{
			"guid": id,
		})
	}
}

func (repo *Repository) createEmptyVolume(volumePath string) error {
	err := repo.driver.CreateVolume(volumePath)
	if err != nil {
		return err
	}

	return nil
}

func (repo *Repository) createCowVolume(parentPath string, newPath string) error {
	err := repo.driver.CreateCopyOnWriteLayer(newPath, parentPath)
	if err != nil {
		return err
	}

	return nil
}

func (repo *Repository) createUuid() string {
	guid, err := uuid.NewV4()
	if err != nil {
		repo.logger.Fatal("failed-to-generate-guid", err)
	}

	return guid.String()
}

func (repo *Repository) volumeExists(guid string) bool {
	info, err := os.Stat(repo.metadataPath(guid))
	if err != nil {
		return false
	}

	return info.IsDir()
}
<|endoftext|>


<commit_before>package main

type sourceWalker struct {
	Directory string
}

func (self *sourceWalker) Walk(c *ctx) error {
	return nil
}

var _ walker = (*sourceWalker)(nil)
<commit_msg>only walk for future implementation<commit_after>package main

import (
	"os"
	"path/filepath"
)

type sourceWalker struct {
	Directory string
}

func (self *sourceWalker) Walk(c *ctx) error {
	return filepath.Walk(self.Directory, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		// TODO: waiting for implementing java parser
		return nil
	})
}

var _ walker = (*sourceWalker)(nil)
<|endoftext|>


<commit_before>package submatview

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/hashicorp/go-hclog"
	"github.com/stretchr/testify/require"

	"github.com/hashicorp/consul/agent/cache"
	"github.com/hashicorp/consul/lib/ttlcache"
	"github.com/hashicorp/consul/proto/pbcommon"
	"github.com/hashicorp/consul/proto/pbservice"
	"github.com/hashicorp/consul/proto/pbsubscribe"
	"github.com/hashicorp/consul/sdk/testutil/retry"
)

func TestStore_Get(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	store := NewStore(hclog.New(nil))
	go store.Run(ctx)

	req := &fakeRequest{
		client: NewTestStreamingClient(pbcommon.DefaultEnterpriseMeta.Namespace),
	}
	req.client.QueueEvents(
		newEndOfSnapshotEvent(2),
		newEventServiceHealthRegister(10, 1, "srv1"),
		newEventServiceHealthRegister(22, 2, "srv1"))

	runStep(t, "from empty store, starts materializer", func(t *testing.T) {
		result, err := store.Get(ctx, req)
		require.NoError(t, err)
		require.Equal(t, uint64(22), result.Index)

		r, ok := result.Value.(fakeResult)
		require.True(t, ok)
		require.Len(t, r.srvs, 2)
		require.Equal(t, uint64(22), r.index)

		store.lock.Lock()
		defer store.lock.Unlock()
		require.Len(t, store.byKey, 1)
		e := store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
		require.Equal(t, 0, e.expiry.Index())
		require.Equal(t, 0, e.requests)

		require.Equal(t, store.expiryHeap.Next().Entry, e.expiry)
	})

	runStep(t, "with an index that already exists in the view", func(t *testing.T) {
		req.index = 21
		result, err := store.Get(ctx, req)
		require.NoError(t, err)
		require.Equal(t, uint64(22), result.Index)

		r, ok := result.Value.(fakeResult)
		require.True(t, ok)
		require.Len(t, r.srvs, 2)
		require.Equal(t, uint64(22), r.index)

		store.lock.Lock()
		defer store.lock.Unlock()
		require.Len(t, store.byKey, 1)
		e := store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
		require.Equal(t, 0, e.expiry.Index())
		require.Equal(t, 0, e.requests)

		require.Equal(t, store.expiryHeap.Next().Entry, e.expiry)
	})

	runStep(t, "blocks with an index that is not yet in the view", func(t *testing.T) {
		req.index = 23

		chResult := make(chan resultOrError, 1)
		go func() {
			result, err := store.Get(ctx, req)
			chResult <- resultOrError{Result: result, Err: err}
		}()

		select {
		case <-chResult:
			t.Fatalf("expected Get to block")
		case <-time.After(50 * time.Millisecond):
		}

		store.lock.Lock()
		e := store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
		store.lock.Unlock()
		require.Equal(t, 1, e.requests)

		req.client.QueueEvents(newEventServiceHealthRegister(24, 1, "srv1"))

		var getResult resultOrError
		select {
		case getResult = <-chResult:
		case <-time.After(100 * time.Millisecond):
			t.Fatalf("expected Get to unblock when new events are received")
		}

		require.NoError(t, getResult.Err)
		require.Equal(t, uint64(24), getResult.Result.Index)

		r, ok := getResult.Result.Value.(fakeResult)
		require.True(t, ok)
		require.Len(t, r.srvs, 2)
		require.Equal(t, uint64(24), r.index)

		store.lock.Lock()
		defer store.lock.Unlock()
		require.Len(t, store.byKey, 1)
		e = store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
		require.Equal(t, 0, e.expiry.Index())
		require.Equal(t, 0, e.requests)

		require.Equal(t, store.expiryHeap.Next().Entry, e.expiry)
	})
}

type resultOrError struct {
	Result Result
	Err    error
}

type fakeRequest struct {
	index  uint64
	key    string
	client *TestStreamingClient
}

func (r *fakeRequest) CacheInfo() cache.RequestInfo {
	key := r.key
	if key == "" {
		key = "key"
	}
	return cache.RequestInfo{
		Key:        key,
		Token:      "abcd",
		Datacenter: "dc1",
		Timeout:    4 * time.Second,
		MinIndex:   r.index,
	}
}

func (r *fakeRequest) NewMaterializer() *Materializer {
	return NewMaterializer(Deps{
		View:   &fakeView{srvs: make(map[string]*pbservice.CheckServiceNode)},
		Client: r.client,
		Logger: hclog.New(nil),
		Request: func(index uint64) pbsubscribe.SubscribeRequest {
			req := pbsubscribe.SubscribeRequest{
				Topic:      pbsubscribe.Topic_ServiceHealth,
				Key:        "key",
				Token:      "abcd",
				Datacenter: "dc1",
				Index:      index,
				Namespace:  pbcommon.DefaultEnterpriseMeta.Namespace,
			}
			return req
		},
	})
}

func (r *fakeRequest) Type() string {
	return fmt.Sprintf("%T", r)
}

type fakeView struct {
	srvs map[string]*pbservice.CheckServiceNode
}

func (f *fakeView) Update(events []*pbsubscribe.Event) error {
	for _, event := range events {
		serviceHealth := event.GetServiceHealth()
		if serviceHealth == nil {
			return fmt.Errorf("unexpected event type for service health view: %T",
				event.GetPayload())
		}

		id := serviceHealth.CheckServiceNode.UniqueID()
		switch serviceHealth.Op {
		case pbsubscribe.CatalogOp_Register:
			f.srvs[id] = serviceHealth.CheckServiceNode

		case pbsubscribe.CatalogOp_Deregister:
			delete(f.srvs, id)
		}
	}
	return nil
}

func (f *fakeView) Result(index uint64) interface{} {
	srvs := make([]*pbservice.CheckServiceNode, 0, len(f.srvs))
	for _, srv := range f.srvs {
		srvs = append(srvs, srv)
	}
	return fakeResult{srvs: srvs, index: index}
}

type fakeResult struct {
	srvs  []*pbservice.CheckServiceNode
	index uint64
}

func (f *fakeView) Reset() {
	f.srvs = make(map[string]*pbservice.CheckServiceNode)
}

func TestStore_Notify(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	store := NewStore(hclog.New(nil))
	go store.Run(ctx)

	req := &fakeRequest{
		client: NewTestStreamingClient(pbcommon.DefaultEnterpriseMeta.Namespace),
	}
	req.client.QueueEvents(
		newEndOfSnapshotEvent(2),
		newEventServiceHealthRegister(10, 1, "srv1"),
		newEventServiceHealthRegister(22, 2, "srv1"))

	cID := "correlate"
	ch := make(chan cache.UpdateEvent)

	err := store.Notify(ctx, req, cID, ch)
	require.NoError(t, err)

	runStep(t, "from empty store, starts materializer", func(t *testing.T) {
		store.lock.Lock()
		defer store.lock.Unlock()
		require.Len(t, store.byKey, 1)
		e := store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
		require.Equal(t, ttlcache.NotIndexed, e.expiry.Index())
		require.Equal(t, 1, e.requests)
	})

	runStep(t, "updates are received", func(t *testing.T) {
		select {
		case update := <-ch:
			require.NoError(t, update.Err)
			require.Equal(t, cID, update.CorrelationID)
			require.Equal(t, uint64(22), update.Meta.Index)
			require.Equal(t, uint64(22), update.Result.(fakeResult).index)
		case <-time.After(100 * time.Millisecond):
			t.Fatalf("expected Get to unblock when new events are received")
		}

		req.client.QueueEvents(newEventServiceHealthRegister(24, 2, "srv1"))

		select {
		case update := <-ch:
			require.NoError(t, update.Err)
			require.Equal(t, cID, update.CorrelationID)
			require.Equal(t, uint64(24), update.Meta.Index)
			require.Equal(t, uint64(24), update.Result.(fakeResult).index)
		case <-time.After(100 * time.Millisecond):
			t.Fatalf("expected Get to unblock when new events are received")
		}
	})

	runStep(t, "closing the notify starts the expiry counter", func(t *testing.T) {
		cancel()

		retry.Run(t, func(r *retry.R) {
			store.lock.Lock()
			defer store.lock.Unlock()
			e := store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
			require.Equal(r, 0, e.expiry.Index())
			require.Equal(r, 0, e.requests)
			require.Equal(r, store.expiryHeap.Next().Entry, e.expiry)
		})
	})
}

func TestStore_Notify_ManyRequests(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	store := NewStore(hclog.New(nil))
	go store.Run(ctx)

	req := &fakeRequest{
		client: NewTestStreamingClient(pbcommon.DefaultEnterpriseMeta.Namespace),
	}
	req.client.QueueEvents(newEndOfSnapshotEvent(2))

	cID := "correlate"
	ch1 := make(chan cache.UpdateEvent)
	ch2 := make(chan cache.UpdateEvent)

	require.NoError(t, store.Notify(ctx, req, cID, ch1))
	assertRequestCount(t, store, req, 1)

	require.NoError(t, store.Notify(ctx, req, cID, ch2))
	assertRequestCount(t, store, req, 2)

	req.index = 15

	go func() {
		_, _ = store.Get(ctx, req)
	}()

	retry.Run(t, func(r *retry.R) {
		assertRequestCount(r, store, req, 3)
	})

	go func() {
		_, _ = store.Get(ctx, req)
	}()

	retry.Run(t, func(r *retry.R) {
		assertRequestCount(r, store, req, 4)
	})

	var req2 *fakeRequest

	runStep(t, "Get and Notify with a different key", func(t *testing.T) {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		req2 = &fakeRequest{client: req.client, key: "key2"}

		require.NoError(t, store.Notify(ctx, req2, cID, ch1))
		go func() {
			_, _ = store.Get(ctx, req2)
		}()

		// the original entry should still be at count 4
		assertRequestCount(t, store, req, 4)
		// the new entry should be at count 2
		retry.Run(t, func(r *retry.R) {
			assertRequestCount(r, store, req2, 2)
		})
	})

	runStep(t, "end all the requests", func(t *testing.T) {
		req.client.QueueEvents(
			newEventServiceHealthRegister(10, 1, "srv1"),
			newEventServiceHealthRegister(12, 2, "srv1"),
			newEventServiceHealthRegister(13, 1, "srv2"),
			newEventServiceHealthRegister(16, 3, "srv2"))

		// The two Get requests should exit now that the index has been updated
		retry.Run(t, func(r *retry.R) {
			assertRequestCount(r, store, req, 2)
		})

		// Cancel the context so all requests terminate
		cancel()
		retry.Run(t, func(r *retry.R) {
			assertRequestCount(r, store, req, 0)
		})
	})

	runStep(t, "the expiry heap should contain two entries", func(t *testing.T) {
		store.lock.Lock()
		defer store.lock.Unlock()
		e := store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
		e2 := store.byKey[makeEntryKey(req2.Type(), req2.CacheInfo())]
		require.Equal(t, 0, e2.expiry.Index())
		require.Equal(t, 1, e.expiry.Index())

		require.Equal(t, store.expiryHeap.Next().Entry, e2.expiry)
	})
}

type testingT interface {
	Helper()
	Fatalf(string, ...interface{})
}

func assertRequestCount(t testingT, s *Store, req Request, expected int) {
	t.Helper()

	key := makeEntryKey(req.Type(), req.CacheInfo())

	s.lock.Lock()
	defer s.lock.Unlock()
	actual := s.byKey[key].requests
	if actual != expected {
		t.Fatalf("expected request count to be %d, got %d", expected, actual)
	}
}

// TODO: test expiration

func runStep(t *testing.T, name string, fn func(t *testing.T)) {
	t.Helper()
	if !t.Run(name, fn) {
		t.FailNow()
	}
}
<commit_msg>submatview: test Store.Run<commit_after>package submatview

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/hashicorp/go-hclog"
	"github.com/stretchr/testify/require"

	"github.com/hashicorp/consul/agent/cache"
	"github.com/hashicorp/consul/lib/ttlcache"
	"github.com/hashicorp/consul/proto/pbcommon"
	"github.com/hashicorp/consul/proto/pbservice"
	"github.com/hashicorp/consul/proto/pbsubscribe"
	"github.com/hashicorp/consul/sdk/testutil/retry"
)

func TestStore_Get(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	store := NewStore(hclog.New(nil))
	go store.Run(ctx)

	req := &fakeRequest{
		client: NewTestStreamingClient(pbcommon.DefaultEnterpriseMeta.Namespace),
	}
	req.client.QueueEvents(
		newEndOfSnapshotEvent(2),
		newEventServiceHealthRegister(10, 1, "srv1"),
		newEventServiceHealthRegister(22, 2, "srv1"))

	runStep(t, "from empty store, starts materializer", func(t *testing.T) {
		result, err := store.Get(ctx, req)
		require.NoError(t, err)
		require.Equal(t, uint64(22), result.Index)

		r, ok := result.Value.(fakeResult)
		require.True(t, ok)
		require.Len(t, r.srvs, 2)
		require.Equal(t, uint64(22), r.index)

		store.lock.Lock()
		defer store.lock.Unlock()
		require.Len(t, store.byKey, 1)
		e := store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
		require.Equal(t, 0, e.expiry.Index())
		require.Equal(t, 0, e.requests)

		require.Equal(t, store.expiryHeap.Next().Entry, e.expiry)
	})

	runStep(t, "with an index that already exists in the view", func(t *testing.T) {
		req.index = 21
		result, err := store.Get(ctx, req)
		require.NoError(t, err)
		require.Equal(t, uint64(22), result.Index)

		r, ok := result.Value.(fakeResult)
		require.True(t, ok)
		require.Len(t, r.srvs, 2)
		require.Equal(t, uint64(22), r.index)

		store.lock.Lock()
		defer store.lock.Unlock()
		require.Len(t, store.byKey, 1)
		e := store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
		require.Equal(t, 0, e.expiry.Index())
		require.Equal(t, 0, e.requests)

		require.Equal(t, store.expiryHeap.Next().Entry, e.expiry)
	})

	runStep(t, "blocks with an index that is not yet in the view", func(t *testing.T) {
		req.index = 23

		chResult := make(chan resultOrError, 1)
		go func() {
			result, err := store.Get(ctx, req)
			chResult <- resultOrError{Result: result, Err: err}
		}()

		select {
		case <-chResult:
			t.Fatalf("expected Get to block")
		case <-time.After(50 * time.Millisecond):
		}

		store.lock.Lock()
		e := store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
		store.lock.Unlock()
		require.Equal(t, 1, e.requests)

		req.client.QueueEvents(newEventServiceHealthRegister(24, 1, "srv1"))

		var getResult resultOrError
		select {
		case getResult = <-chResult:
		case <-time.After(100 * time.Millisecond):
			t.Fatalf("expected Get to unblock when new events are received")
		}

		require.NoError(t, getResult.Err)
		require.Equal(t, uint64(24), getResult.Result.Index)

		r, ok := getResult.Result.Value.(fakeResult)
		require.True(t, ok)
		require.Len(t, r.srvs, 2)
		require.Equal(t, uint64(24), r.index)

		store.lock.Lock()
		defer store.lock.Unlock()
		require.Len(t, store.byKey, 1)
		e = store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
		require.Equal(t, 0, e.expiry.Index())
		require.Equal(t, 0, e.requests)

		require.Equal(t, store.expiryHeap.Next().Entry, e.expiry)
	})
}

type resultOrError struct {
	Result Result
	Err    error
}

type fakeRequest struct {
	index  uint64
	key    string
	client *TestStreamingClient
}

func (r *fakeRequest) CacheInfo() cache.RequestInfo {
	key := r.key
	if key == "" {
		key = "key"
	}
	return cache.RequestInfo{
		Key:        key,
		Token:      "abcd",
		Datacenter: "dc1",
		Timeout:    4 * time.Second,
		MinIndex:   r.index,
	}
}

func (r *fakeRequest) NewMaterializer() *Materializer {
	return NewMaterializer(Deps{
		View:   &fakeView{srvs: make(map[string]*pbservice.CheckServiceNode)},
		Client: r.client,
		Logger: hclog.New(nil),
		Request: func(index uint64) pbsubscribe.SubscribeRequest {
			req := pbsubscribe.SubscribeRequest{
				Topic:      pbsubscribe.Topic_ServiceHealth,
				Key:        "key",
				Token:      "abcd",
				Datacenter: "dc1",
				Index:      index,
				Namespace:  pbcommon.DefaultEnterpriseMeta.Namespace,
			}
			return req
		},
	})
}

func (r *fakeRequest) Type() string {
	return fmt.Sprintf("%T", r)
}

type fakeView struct {
	srvs map[string]*pbservice.CheckServiceNode
}

func (f *fakeView) Update(events []*pbsubscribe.Event) error {
	for _, event := range events {
		serviceHealth := event.GetServiceHealth()
		if serviceHealth == nil {
			return fmt.Errorf("unexpected event type for service health view: %T",
				event.GetPayload())
		}

		id := serviceHealth.CheckServiceNode.UniqueID()
		switch serviceHealth.Op {
		case pbsubscribe.CatalogOp_Register:
			f.srvs[id] = serviceHealth.CheckServiceNode

		case pbsubscribe.CatalogOp_Deregister:
			delete(f.srvs, id)
		}
	}
	return nil
}

func (f *fakeView) Result(index uint64) interface{} {
	srvs := make([]*pbservice.CheckServiceNode, 0, len(f.srvs))
	for _, srv := range f.srvs {
		srvs = append(srvs, srv)
	}
	return fakeResult{srvs: srvs, index: index}
}

type fakeResult struct {
	srvs  []*pbservice.CheckServiceNode
	index uint64
}

func (f *fakeView) Reset() {
	f.srvs = make(map[string]*pbservice.CheckServiceNode)
}

func TestStore_Notify(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	store := NewStore(hclog.New(nil))
	go store.Run(ctx)

	req := &fakeRequest{
		client: NewTestStreamingClient(pbcommon.DefaultEnterpriseMeta.Namespace),
	}
	req.client.QueueEvents(
		newEndOfSnapshotEvent(2),
		newEventServiceHealthRegister(22, 2, "srv1"))

	cID := "correlate"
	ch := make(chan cache.UpdateEvent)

	err := store.Notify(ctx, req, cID, ch)
	require.NoError(t, err)

	runStep(t, "from empty store, starts materializer", func(t *testing.T) {
		store.lock.Lock()
		defer store.lock.Unlock()
		require.Len(t, store.byKey, 1)
		e := store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
		require.Equal(t, ttlcache.NotIndexed, e.expiry.Index())
		require.Equal(t, 1, e.requests)
	})

	runStep(t, "updates are received", func(t *testing.T) {
		select {
		case update := <-ch:
			require.NoError(t, update.Err)
			require.Equal(t, cID, update.CorrelationID)
			require.Equal(t, uint64(22), update.Meta.Index)
			require.Equal(t, uint64(22), update.Result.(fakeResult).index)
		case <-time.After(100 * time.Millisecond):
			t.Fatalf("expected Get to unblock when new events are received")
		}

		req.client.QueueEvents(newEventServiceHealthRegister(24, 2, "srv1"))

		select {
		case update := <-ch:
			require.NoError(t, update.Err)
			require.Equal(t, cID, update.CorrelationID)
			require.Equal(t, uint64(24), update.Meta.Index)
			require.Equal(t, uint64(24), update.Result.(fakeResult).index)
		case <-time.After(100 * time.Millisecond):
			t.Fatalf("expected Get to unblock when new events are received")
		}
	})

	runStep(t, "closing the notify starts the expiry counter", func(t *testing.T) {
		cancel()

		retry.Run(t, func(r *retry.R) {
			store.lock.Lock()
			defer store.lock.Unlock()
			e := store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
			require.Equal(r, 0, e.expiry.Index())
			require.Equal(r, 0, e.requests)
			require.Equal(r, store.expiryHeap.Next().Entry, e.expiry)
		})
	})
}

func TestStore_Notify_ManyRequests(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	store := NewStore(hclog.New(nil))
	go store.Run(ctx)

	req := &fakeRequest{
		client: NewTestStreamingClient(pbcommon.DefaultEnterpriseMeta.Namespace),
	}
	req.client.QueueEvents(newEndOfSnapshotEvent(2))

	cID := "correlate"
	ch1 := make(chan cache.UpdateEvent)
	ch2 := make(chan cache.UpdateEvent)

	require.NoError(t, store.Notify(ctx, req, cID, ch1))
	assertRequestCount(t, store, req, 1)

	require.NoError(t, store.Notify(ctx, req, cID, ch2))
	assertRequestCount(t, store, req, 2)

	req.index = 15

	go func() {
		_, _ = store.Get(ctx, req)
	}()

	retry.Run(t, func(r *retry.R) {
		assertRequestCount(r, store, req, 3)
	})

	go func() {
		_, _ = store.Get(ctx, req)
	}()

	retry.Run(t, func(r *retry.R) {
		assertRequestCount(r, store, req, 4)
	})

	var req2 *fakeRequest

	runStep(t, "Get and Notify with a different key", func(t *testing.T) {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		req2 = &fakeRequest{client: req.client, key: "key2"}

		require.NoError(t, store.Notify(ctx, req2, cID, ch1))
		go func() {
			_, _ = store.Get(ctx, req2)
		}()

		// the original entry should still be at count 4
		assertRequestCount(t, store, req, 4)
		// the new entry should be at count 2
		retry.Run(t, func(r *retry.R) {
			assertRequestCount(r, store, req2, 2)
		})
	})

	runStep(t, "end all the requests", func(t *testing.T) {
		req.client.QueueEvents(
			newEventServiceHealthRegister(10, 1, "srv1"),
			newEventServiceHealthRegister(12, 2, "srv1"),
			newEventServiceHealthRegister(13, 1, "srv2"),
			newEventServiceHealthRegister(16, 3, "srv2"))

		// The two Get requests should exit now that the index has been updated
		retry.Run(t, func(r *retry.R) {
			assertRequestCount(r, store, req, 2)
		})

		// Cancel the context so all requests terminate
		cancel()
		retry.Run(t, func(r *retry.R) {
			assertRequestCount(r, store, req, 0)
		})
	})

	runStep(t, "the expiry heap should contain two entries", func(t *testing.T) {
		store.lock.Lock()
		defer store.lock.Unlock()
		e := store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
		e2 := store.byKey[makeEntryKey(req2.Type(), req2.CacheInfo())]
		require.Equal(t, 0, e2.expiry.Index())
		require.Equal(t, 1, e.expiry.Index())

		require.Equal(t, store.expiryHeap.Next().Entry, e2.expiry)
	})
}

type testingT interface {
	Helper()
	Fatalf(string, ...interface{})
}

func assertRequestCount(t testingT, s *Store, req Request, expected int) {
	t.Helper()

	key := makeEntryKey(req.Type(), req.CacheInfo())

	s.lock.Lock()
	defer s.lock.Unlock()
	actual := s.byKey[key].requests
	if actual != expected {
		t.Fatalf("expected request count to be %d, got %d", expected, actual)
	}
}

func TestStore_Run_ExpiresEntries(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	ttl := 10 * time.Millisecond
	patchIdleTTL(t, ttl)

	store := NewStore(hclog.New(nil))
	go store.Run(ctx)

	req := &fakeRequest{
		client: NewTestStreamingClient(pbcommon.DefaultEnterpriseMeta.Namespace),
	}
	req.client.QueueEvents(newEndOfSnapshotEvent(2))

	cID := "correlate"
	ch1 := make(chan cache.UpdateEvent)

	reqCtx, reqCancel := context.WithCancel(context.Background())
	defer reqCancel()

	require.NoError(t, store.Notify(reqCtx, req, cID, ch1))
	assertRequestCount(t, store, req, 1)

	// Get a copy of the entry so that we can check it was expired later
	store.lock.Lock()
	e := store.byKey[makeEntryKey(req.Type(), req.CacheInfo())]
	store.lock.Unlock()

	reqCancel()
	retry.Run(t, func(r *retry.R) {
		assertRequestCount(r, store, req, 0)
	})

	// wait for the entry to expire, with lots of buffer
	time.Sleep(3 * ttl)

	store.lock.Lock()
	defer store.lock.Unlock()
	require.Len(t, store.byKey, 0)
	require.Equal(t, ttlcache.NotIndexed, e.expiry.Index())
}

func patchIdleTTL(t *testing.T, ttl time.Duration) {
	orig := idleTTL
	idleTTL = ttl
	t.Cleanup(func() {
		idleTTL = orig
	})
}

func runStep(t *testing.T, name string, fn func(t *testing.T)) {
	t.Helper()
	if !t.Run(name, fn) {
		t.FailNow()
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 The btcsuite developers
// Copyright (c) 2015 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package blockchain

import (
	"fmt"

	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrutil"
)

// BehaviorFlags is a bitmask defining tweaks to the normal behavior when
// performing chain processing and consensus rules checks.
type BehaviorFlags uint32

const (
	// BFFastAdd may be set to indicate that several checks can be avoided
	// for the block since it is already known to fit into the chain due to
	// already proving it correct links into the chain up to a known
	// checkpoint.  This is primarily used for headers-first mode.
	BFFastAdd BehaviorFlags = 1 << iota

	// BFNoPoWCheck may be set to indicate the proof of work check which
	// ensures a block hashes to a value less than the required target will
	// not be performed.
	BFNoPoWCheck

	// BFDryRun may be set to indicate the block should not modify the chain
	// or memory chain index.  This is useful to test that a block is valid
	// without modifying the current state.
	BFDryRun

	// BFNone is a convenience value to specifically indicate no flags.
	BFNone BehaviorFlags = 0
)

// blockExists determines whether a block with the given hash exists either in
// the main chain or any side chains.
func (b *BlockChain) blockExists(hash *chainhash.Hash) (bool, error) {
	// Check memory chain first (could be main chain or side chain blocks).
	if _, ok := b.index[*hash]; ok {
		return true, nil
	}

	// Check in database (rest of main chain not in memory).
	return b.db.ExistsSha(hash)
}

// processOrphans determines if there are any orphans which depend on the passed
// block hash (they are no longer orphans if true) and potentially accepts them.
// It repeats the process for the newly accepted blocks (to detect further
// orphans which may no longer be orphans) until there are no more.
//
// The flags do not modify the behavior of this function directly, however they
// are needed to pass along to maybeAcceptBlock.
func (b *BlockChain) processOrphans(hash *chainhash.Hash, flags BehaviorFlags) error {
	// Start with processing at least the passed hash.  Leave a little room
	// for additional orphan blocks that need to be processed without
	// needing to grow the array in the common case.
	processHashes := make([]*chainhash.Hash, 0, 10)
	processHashes = append(processHashes, hash)
	for len(processHashes) > 0 {
		// Pop the first hash to process from the slice.
		processHash := processHashes[0]
		processHashes[0] = nil // Prevent GC leak.
		processHashes = processHashes[1:]

		// Look up all orphans that are parented by the block we just
		// accepted.  This will typically only be one, but it could
		// be multiple if multiple blocks are mined and broadcast
		// around the same time.  The one with the most proof of work
		// will eventually win out.  An indexing for loop is
		// intentionally used over a range here as range does not
		// reevaluate the slice on each iteration nor does it adjust the
		// index for the modified slice.
		for i := 0; i < len(b.prevOrphans[*processHash]); i++ {
			orphan := b.prevOrphans[*processHash][i]
			if orphan == nil {
				log.Warnf("Found a nil entry at index %d in the "+
					"orphan dependency list for block %v", i,
					processHash)
				continue
			}

			// Remove the orphan from the orphan pool.
			orphanHash := orphan.block.Sha()
			b.removeOrphanBlock(orphan)
			i--

			// Potentially accept the block into the block chain.
			_, err := b.maybeAcceptBlock(orphan.block, flags)
			if err != nil {
				return err
			}

			// Add this block to the list of blocks to process so
			// any orphan blocks that depend on this block are
			// handled too.
			processHashes = append(processHashes, orphanHash)
		}
	}
	return nil
}

// ProcessBlock is the main workhorse for handling insertion of new blocks into
// the block chain.  It includes functionality such as rejecting duplicate
// blocks, ensuring blocks follow all rules, orphan handling, and insertion into
// the block chain along with best chain selection and reorganization.
//
// It returns a first bool specifying whether or not the block is on on a fork
// or on a side chain. True means it's on the main chain.
//
// It returns a second bool which indicates whether or not the block is an orphan
// and any errors that occurred during processing.  The returned bool is only
// valid when the error is nil.
func (b *BlockChain) ProcessBlock(block *dcrutil.Block,
	timeSource MedianTimeSource, flags BehaviorFlags) (bool, bool, error) {
	fastAdd := flags&BFFastAdd == BFFastAdd
	dryRun := flags&BFDryRun == BFDryRun

	blockHash := block.Sha()
	log.Tracef("Processing block %v", blockHash)

	// The block must not already exist in the main chain or side chains.
	exists, err := b.blockExists(blockHash)
	if err != nil {
		return false, false, err
	}
	if exists {
		str := fmt.Sprintf("already have block %v", blockHash)
		return false, false, ruleError(ErrDuplicateBlock, str)
	}

	// The block must not already exist as an orphan.
	if _, exists := b.orphans[*blockHash]; exists {
		str := fmt.Sprintf("already have block (orphan) %v", blockHash)
		return false, false, ruleError(ErrDuplicateBlock, str)
	}

	// Perform preliminary sanity checks on the block and its transactions.
	err = checkBlockSanity(block, timeSource, flags, b.chainParams)
	if err != nil {
		return false, false, err
	}

	// Find the previous checkpoint and perform some additional checks based
	// on the checkpoint.  This provides a few nice properties such as
	// preventing old side chain blocks before the last checkpoint,
	// rejecting easy to mine, but otherwise bogus, blocks that could be
	// used to eat memory, and ensuring expected (versus claimed) proof of
	// work requirements since the previous checkpoint are met.
	blockHeader := &block.MsgBlock().Header
	checkpointBlock, err := b.findPreviousCheckpoint()
	if err != nil {
		return false, false, err
	}
	if checkpointBlock != nil {
		// Ensure the block timestamp is after the checkpoint timestamp.
		checkpointHeader := &checkpointBlock.MsgBlock().Header
		checkpointTime := checkpointHeader.Timestamp
		if blockHeader.Timestamp.Before(checkpointTime) {
			str := fmt.Sprintf("block %v has timestamp %v before "+
				"last checkpoint timestamp %v", blockHash,
				blockHeader.Timestamp, checkpointTime)
			return false, false, ruleError(ErrCheckpointTimeTooOld, str)
		}
		if !fastAdd {
			// Even though the checks prior to now have already ensured the
			// proof of work exceeds the claimed amount, the claimed amount
			// is a field in the block header which could be forged.  This
			// check ensures the proof of work is at least the minimum
			// expected based on elapsed time since the last checkpoint and
			// maximum adjustment allowed by the retarget rules.
			duration := blockHeader.Timestamp.Sub(checkpointTime)
			requiredTarget := CompactToBig(b.calcEasiestDifficulty(
				checkpointHeader.Bits, duration))
			currentTarget := CompactToBig(blockHeader.Bits)
			if currentTarget.Cmp(requiredTarget) > 0 {
				str := fmt.Sprintf("block target difficulty of %064x "+
					"is too low when compared to the previous "+
					"checkpoint", currentTarget)
				return false, false, ruleError(ErrDifficultyTooLow, str)
			}
		}
	}

	// Handle orphan blocks.
	prevHash := &blockHeader.PrevBlock
	if !prevHash.IsEqual(zeroHash) {
		prevHashExists, err := b.blockExists(prevHash)
		if err != nil {
			return false, false, err
		}
		if !prevHashExists {
			if !dryRun {
				log.Infof("Adding orphan block %v with parent %v",
					blockHash, prevHash)
				b.addOrphanBlock(block)
			}

			return false, true, err
		}
	}

	// The block has passed all context independent checks and appears sane
	// enough to potentially accept it into the block chain.
	var onMainChain bool
	onMainChain, err = b.maybeAcceptBlock(block, flags)
	if err != nil {
		return false, false, err
	}

	// Don't process any orphans or log when the dry run flag is set.
	if !dryRun {
		// Accept any orphan blocks that depend on this block (they are
		// no longer orphans) and repeat for those accepted blocks until
		// there are no more.
		err := b.processOrphans(blockHash, flags)
		if err != nil {
			return false, false, err
		}

		log.Debugf("Accepted block %v", blockHash)
	}

	return onMainChain, false, err
}
<commit_msg>Log block processing time in CHAN with debug on<commit_after>// Copyright (c) 2013-2014 The btcsuite developers
// Copyright (c) 2015 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package blockchain

import (
	"fmt"
	"time"

	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrutil"
)

// BehaviorFlags is a bitmask defining tweaks to the normal behavior when
// performing chain processing and consensus rules checks.
type BehaviorFlags uint32

const (
	// BFFastAdd may be set to indicate that several checks can be avoided
	// for the block since it is already known to fit into the chain due to
	// already proving it correct links into the chain up to a known
	// checkpoint.  This is primarily used for headers-first mode.
	BFFastAdd BehaviorFlags = 1 << iota

	// BFNoPoWCheck may be set to indicate the proof of work check which
	// ensures a block hashes to a value less than the required target will
	// not be performed.
	BFNoPoWCheck

	// BFDryRun may be set to indicate the block should not modify the chain
	// or memory chain index.  This is useful to test that a block is valid
	// without modifying the current state.
	BFDryRun

	// BFNone is a convenience value to specifically indicate no flags.
	BFNone BehaviorFlags = 0
)

// blockExists determines whether a block with the given hash exists either in
// the main chain or any side chains.
func (b *BlockChain) blockExists(hash *chainhash.Hash) (bool, error) {
	// Check memory chain first (could be main chain or side chain blocks).
	if _, ok := b.index[*hash]; ok {
		return true, nil
	}

	// Check in database (rest of main chain not in memory).
	return b.db.ExistsSha(hash)
}

// processOrphans determines if there are any orphans which depend on the passed
// block hash (they are no longer orphans if true) and potentially accepts them.
// It repeats the process for the newly accepted blocks (to detect further
// orphans which may no longer be orphans) until there are no more.
//
// The flags do not modify the behavior of this function directly, however they
// are needed to pass along to maybeAcceptBlock.
func (b *BlockChain) processOrphans(hash *chainhash.Hash, flags BehaviorFlags) error {
	// Start with processing at least the passed hash.  Leave a little room
	// for additional orphan blocks that need to be processed without
	// needing to grow the array in the common case.
	processHashes := make([]*chainhash.Hash, 0, 10)
	processHashes = append(processHashes, hash)
	for len(processHashes) > 0 {
		// Pop the first hash to process from the slice.
		processHash := processHashes[0]
		processHashes[0] = nil // Prevent GC leak.
		processHashes = processHashes[1:]

		// Look up all orphans that are parented by the block we just
		// accepted.  This will typically only be one, but it could
		// be multiple if multiple blocks are mined and broadcast
		// around the same time.  The one with the most proof of work
		// will eventually win out.  An indexing for loop is
		// intentionally used over a range here as range does not
		// reevaluate the slice on each iteration nor does it adjust the
		// index for the modified slice.
		for i := 0; i < len(b.prevOrphans[*processHash]); i++ {
			orphan := b.prevOrphans[*processHash][i]
			if orphan == nil {
				log.Warnf("Found a nil entry at index %d in the "+
					"orphan dependency list for block %v", i,
					processHash)
				continue
			}

			// Remove the orphan from the orphan pool.
			orphanHash := orphan.block.Sha()
			b.removeOrphanBlock(orphan)
			i--

			// Potentially accept the block into the block chain.
			_, err := b.maybeAcceptBlock(orphan.block, flags)
			if err != nil {
				return err
			}

			// Add this block to the list of blocks to process so
			// any orphan blocks that depend on this block are
			// handled too.
			processHashes = append(processHashes, orphanHash)
		}
	}
	return nil
}

// ProcessBlock is the main workhorse for handling insertion of new blocks into
// the block chain.  It includes functionality such as rejecting duplicate
// blocks, ensuring blocks follow all rules, orphan handling, and insertion into
// the block chain along with best chain selection and reorganization.
//
// It returns a first bool specifying whether or not the block is on on a fork
// or on a side chain. True means it's on the main chain.
//
// It returns a second bool which indicates whether or not the block is an orphan
// and any errors that occurred during processing.  The returned bool is only
// valid when the error is nil.
func (b *BlockChain) ProcessBlock(block *dcrutil.Block,
	timeSource MedianTimeSource, flags BehaviorFlags) (bool, bool, error) {
	fastAdd := flags&BFFastAdd == BFFastAdd
	dryRun := flags&BFDryRun == BFDryRun

	blockHash := block.Sha()
	log.Tracef("Processing block %v", blockHash)
	currentTime := time.Now()
	defer func() {
		elapsedTime := time.Since(currentTime)
		log.Debugf("Block %v (height %v) finished processing in %s",
			blockHash, block.Height(), elapsedTime)
	}()

	// The block must not already exist in the main chain or side chains.
	exists, err := b.blockExists(blockHash)
	if err != nil {
		return false, false, err
	}
	if exists {
		str := fmt.Sprintf("already have block %v", blockHash)
		return false, false, ruleError(ErrDuplicateBlock, str)
	}

	// The block must not already exist as an orphan.
	if _, exists := b.orphans[*blockHash]; exists {
		str := fmt.Sprintf("already have block (orphan) %v", blockHash)
		return false, false, ruleError(ErrDuplicateBlock, str)
	}

	// Perform preliminary sanity checks on the block and its transactions.
	err = checkBlockSanity(block, timeSource, flags, b.chainParams)
	if err != nil {
		return false, false, err
	}

	// Find the previous checkpoint and perform some additional checks based
	// on the checkpoint.  This provides a few nice properties such as
	// preventing old side chain blocks before the last checkpoint,
	// rejecting easy to mine, but otherwise bogus, blocks that could be
	// used to eat memory, and ensuring expected (versus claimed) proof of
	// work requirements since the previous checkpoint are met.
	blockHeader := &block.MsgBlock().Header
	checkpointBlock, err := b.findPreviousCheckpoint()
	if err != nil {
		return false, false, err
	}
	if checkpointBlock != nil {
		// Ensure the block timestamp is after the checkpoint timestamp.
		checkpointHeader := &checkpointBlock.MsgBlock().Header
		checkpointTime := checkpointHeader.Timestamp
		if blockHeader.Timestamp.Before(checkpointTime) {
			str := fmt.Sprintf("block %v has timestamp %v before "+
				"last checkpoint timestamp %v", blockHash,
				blockHeader.Timestamp, checkpointTime)
			return false, false, ruleError(ErrCheckpointTimeTooOld, str)
		}
		if !fastAdd {
			// Even though the checks prior to now have already ensured the
			// proof of work exceeds the claimed amount, the claimed amount
			// is a field in the block header which could be forged.  This
			// check ensures the proof of work is at least the minimum
			// expected based on elapsed time since the last checkpoint and
			// maximum adjustment allowed by the retarget rules.
			duration := blockHeader.Timestamp.Sub(checkpointTime)
			requiredTarget := CompactToBig(b.calcEasiestDifficulty(
				checkpointHeader.Bits, duration))
			currentTarget := CompactToBig(blockHeader.Bits)
			if currentTarget.Cmp(requiredTarget) > 0 {
				str := fmt.Sprintf("block target difficulty of %064x "+
					"is too low when compared to the previous "+
					"checkpoint", currentTarget)
				return false, false, ruleError(ErrDifficultyTooLow, str)
			}
		}
	}

	// Handle orphan blocks.
	prevHash := &blockHeader.PrevBlock
	if !prevHash.IsEqual(zeroHash) {
		prevHashExists, err := b.blockExists(prevHash)
		if err != nil {
			return false, false, err
		}
		if !prevHashExists {
			if !dryRun {
				log.Infof("Adding orphan block %v with parent %v",
					blockHash, prevHash)
				b.addOrphanBlock(block)
			}

			return false, true, err
		}
	}

	// The block has passed all context independent checks and appears sane
	// enough to potentially accept it into the block chain.
	var onMainChain bool
	onMainChain, err = b.maybeAcceptBlock(block, flags)
	if err != nil {
		return false, false, err
	}

	// Don't process any orphans or log when the dry run flag is set.
	if !dryRun {
		// Accept any orphan blocks that depend on this block (they are
		// no longer orphans) and repeat for those accepted blocks until
		// there are no more.
		err := b.processOrphans(blockHash, flags)
		if err != nil {
			return false, false, err
		}

		log.Debugf("Accepted block %v", blockHash)
	}

	return onMainChain, false, err
}
<|endoftext|>


<commit_before>// Package globus implements Globus Online Nexus authentication
package globus

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"github.com/MG-RAST/AWE/lib/auth/basic"
	"github.com/MG-RAST/AWE/lib/conf"
	e "github.com/MG-RAST/AWE/lib/errors"
	"github.com/MG-RAST/AWE/lib/user"
	"io/ioutil"
	"net/http"
	"strings"
)

// Token response struct
type token struct {
	AccessToken     string      `json:"access_token"`
	AccessTokenHash string      `json:"access_token_hash"`
	ClientId        string      `json:"client_id"`
	ExpiresIn       int         `json:"expires_in"`
	Expiry          int         `json:"expiry"`
	IssuedOn        int         `json:"issued_on"`
	Lifetime        int         `json:"lifetime"`
	Scopes          interface{} `json:"scopes"`
	TokenId         string      `json:"token_id"`
	TokeType        string      `json:"token_type"`
	UserName        string      `json:"user_name"`
}

func authHeaderType(header string) string {
	tmp := strings.Split(header, " ")
	if len(tmp) > 1 {
		return strings.ToLower(tmp[0])
	}
	return ""
}

// Auth takes the request authorization header and returns
// user
func Auth(header string) (usr *user.User, err error) {
	bearer := authHeaderType(header)
	if bearer == "" {
		return nil, errors.New("(globus) Invalid authentication header, missing bearer token.")
	}
	if bearer == "basic" {
		if username, password, err := basic.DecodeHeader(header); err == nil {
			if t, err := fetchToken(username, password); err == nil {
				return fetchProfile(t.AccessToken)
			} else {
				return nil, err
			}
		} else {
			return nil, err
		}
	} else if (bearer == "globus-goauthtoken") || (bearer == "globus") || (bearer == "goauth") {
		return fetchProfile(strings.Split(header, " ")[1])
	} else {
		return nil, errors.New("(globus) Invalid authentication header, unknown bearer token: " + bearer)
	}
}

// fetchToken takes username and password and then retrieves user token
func fetchToken(u string, p string) (t *token, err error) {
	client := &http.Client{
		Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
	}
	req, err := http.NewRequest("GET", conf.GLOBUS_TOKEN_URL, nil)
	if err != nil {
		return nil, errors.New("(globus) HTTP GET: " + err.Error())
	}
	req.SetBasicAuth(u, p)
	if resp, err := client.Do(req); err == nil {
		defer resp.Body.Close()
		if resp.StatusCode == http.StatusCreated {
			if body, err := ioutil.ReadAll(resp.Body); err == nil {
				if err = json.Unmarshal(body, &t); err != nil {
					return nil, errors.New("(globus) JSON Unmarshal: " + err.Error())
				}
			}
		} else {
			return nil, errors.New("(globus) Authentication failed: Unexpected response status: " + resp.Status)
		}
	} else {
		return nil, errors.New("(globus) " + err.Error())
	}
	return
}

// fetchProfile validiates token by using it to fetch user profile
func fetchProfile(t string) (u *user.User, err error) {
	client := &http.Client{
		Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
	}
	cid, err := clientId(t)
	if err != nil {
		return nil, errors.New("(globus) " + err.Error())
	}
	req, err := http.NewRequest("GET", conf.GLOBUS_PROFILE_URL+"/"+cid, nil)
	if err != nil {
		return nil, errors.New("(globus) HTTP GET: " + err.Error())
	}
	req.Header.Add("Authorization", "Globus-Goauthtoken "+t)
	if resp, err := client.Do(req); err == nil {
		defer resp.Body.Close()
		if resp.StatusCode == http.StatusOK {
			if body, err := ioutil.ReadAll(resp.Body); err == nil {
				u = &user.User{}
				if err = json.Unmarshal(body, &u); err != nil {
					return nil, errors.New("(globus) JSON Unmarshal: " + err.Error())
				} else {
					if u.Username == "" {
						return nil, errors.New("(globus) " + e.InvalidAuth)
					}
					if err = u.SetMongoInfo(); err != nil {
						return nil, errors.New("(globus) MongoDB: " + err.Error())
					}
				}
			}
		} else if resp.StatusCode == http.StatusForbidden {
			return nil, errors.New("(globus) " + e.InvalidAuth)
		} else {
			return nil, errors.New("(globus) Authentication failed: Unexpected response status: " + resp.Status)
		}
	} else {
		return nil, errors.New("(globus) " + err.Error())
	}
	return
}

func clientId(t string) (c string, err error) {
	// test for old format first
	for _, part := range strings.Split(t, "|") {
		if kv := strings.Split(part, "="); kv[0] == "client_id" {
			return kv[1], nil
		}
	}
	//if we get here then we have a new style token and need to make a call to look up the
	//ID instead of parsing the string
	client := &http.Client{
		Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
	}
	req, err := http.NewRequest("GET", conf.GLOBUS_TOKEN_URL, nil)
	if err != nil {
		return "", errors.New("(globus) HTTP GET: " + err.Error())
	}
	req.Header.Add("X-Globus-Goauthtoken", t)
	if resp, err := client.Do(req); err == nil {
		defer resp.Body.Close()
		if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusOK {
			if body, err := ioutil.ReadAll(resp.Body); err == nil {
				var dat map[string]interface{}
				if err = json.Unmarshal(body, &dat); err != nil {
					return "", errors.New("(globus) JSON Unmarshal: " + err.Error())
				} else {
					return dat["client_id"].(string), nil
				}
			}
		} else if resp.StatusCode == http.StatusForbidden {
			return "", errors.New("(globus) " + e.InvalidAuth)
		} else {
			return "", errors.New("(globus) Authentication failed: Unexpected response status: " + resp.Status)
		}
	} else {
		return "", errors.New("(globus) " + err.Error())
	}
	return
}
<commit_msg>very minor error message<commit_after>// Package globus implements Globus Online Nexus authentication
package globus

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"github.com/MG-RAST/AWE/lib/auth/basic"
	"github.com/MG-RAST/AWE/lib/conf"
	e "github.com/MG-RAST/AWE/lib/errors"
	"github.com/MG-RAST/AWE/lib/user"
	"io/ioutil"
	"net/http"
	"strings"
)

// Token response struct
type token struct {
	AccessToken     string      `json:"access_token"`
	AccessTokenHash string      `json:"access_token_hash"`
	ClientId        string      `json:"client_id"`
	ExpiresIn       int         `json:"expires_in"`
	Expiry          int         `json:"expiry"`
	IssuedOn        int         `json:"issued_on"`
	Lifetime        int         `json:"lifetime"`
	Scopes          interface{} `json:"scopes"`
	TokenId         string      `json:"token_id"`
	TokeType        string      `json:"token_type"`
	UserName        string      `json:"user_name"`
}

func authHeaderType(header string) string {
	tmp := strings.Split(header, " ")
	if len(tmp) > 1 {
		return strings.ToLower(tmp[0])
	}
	return ""
}

// Auth takes the request authorization header and returns
// user
func Auth(header string) (usr *user.User, err error) {
	bearer := authHeaderType(header)
	if bearer == "" {
		return nil, errors.New("(globus) Invalid authentication header, missing bearer token.")
	}
	if bearer == "basic" {
		if username, password, err := basic.DecodeHeader(header); err == nil {
			if t, err := fetchToken(username, password); err == nil {
				return fetchProfile(t.AccessToken)
			} else {
				return nil, err
			}
		} else {
			return nil, errors.New("(basic) " + err.Error())
		}
	} else if (bearer == "globus-goauthtoken") || (bearer == "globus") || (bearer == "goauth") {
		return fetchProfile(strings.Split(header, " ")[1])
	} else {
		return nil, errors.New("(globus) Invalid authentication header, unknown bearer token: " + bearer)
	}
}

// fetchToken takes username and password and then retrieves user token
func fetchToken(u string, p string) (t *token, err error) {
	client := &http.Client{
		Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
	}
	req, err := http.NewRequest("GET", conf.GLOBUS_TOKEN_URL, nil)
	if err != nil {
		return nil, errors.New("(globus) HTTP GET: " + err.Error())
	}
	req.SetBasicAuth(u, p)
	if resp, err := client.Do(req); err == nil {
		defer resp.Body.Close()
		if resp.StatusCode == http.StatusCreated {
			if body, err := ioutil.ReadAll(resp.Body); err == nil {
				if err = json.Unmarshal(body, &t); err != nil {
					return nil, errors.New("(globus) JSON Unmarshal: " + err.Error())
				}
			}
		} else {
			return nil, errors.New("(globus) Authentication failed: Unexpected response status: " + resp.Status)
		}
	} else {
		return nil, errors.New("(globus) " + err.Error())
	}
	return
}

// fetchProfile validiates token by using it to fetch user profile
func fetchProfile(t string) (u *user.User, err error) {
	client := &http.Client{
		Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
	}
	cid, err := clientId(t)
	if err != nil {
		return nil, errors.New("(globus) " + err.Error())
	}
	req, err := http.NewRequest("GET", conf.GLOBUS_PROFILE_URL+"/"+cid, nil)
	if err != nil {
		return nil, errors.New("(globus) HTTP GET: " + err.Error())
	}
	req.Header.Add("Authorization", "Globus-Goauthtoken "+t)
	if resp, err := client.Do(req); err == nil {
		defer resp.Body.Close()
		if resp.StatusCode == http.StatusOK {
			if body, err := ioutil.ReadAll(resp.Body); err == nil {
				u = &user.User{}
				if err = json.Unmarshal(body, &u); err != nil {
					return nil, errors.New("(globus) JSON Unmarshal: " + err.Error())
				} else {
					if u.Username == "" {
						return nil, errors.New("(globus) " + e.InvalidAuth)
					}
					if err = u.SetMongoInfo(); err != nil {
						return nil, errors.New("(globus) MongoDB: " + err.Error())
					}
				}
			}
		} else if resp.StatusCode == http.StatusForbidden {
			return nil, errors.New("(globus) " + e.InvalidAuth)
		} else {
			return nil, errors.New("(globus) Authentication failed: Unexpected response status: " + resp.Status)
		}
	} else {
		return nil, errors.New("(globus) " + err.Error())
	}
	return
}

func clientId(t string) (c string, err error) {
	// test for old format first
	for _, part := range strings.Split(t, "|") {
		if kv := strings.Split(part, "="); kv[0] == "client_id" {
			return kv[1], nil
		}
	}
	//if we get here then we have a new style token and need to make a call to look up the
	//ID instead of parsing the string
	client := &http.Client{
		Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
	}
	req, err := http.NewRequest("GET", conf.GLOBUS_TOKEN_URL, nil)
	if err != nil {
		return "", errors.New("(globus) HTTP GET: " + err.Error())
	}
	req.Header.Add("X-Globus-Goauthtoken", t)
	if resp, err := client.Do(req); err == nil {
		defer resp.Body.Close()
		if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusOK {
			if body, err := ioutil.ReadAll(resp.Body); err == nil {
				var dat map[string]interface{}
				if err = json.Unmarshal(body, &dat); err != nil {
					return "", errors.New("(globus) JSON Unmarshal: " + err.Error())
				} else {
					return dat["client_id"].(string), nil
				}
			}
		} else if resp.StatusCode == http.StatusForbidden {
			return "", errors.New("(globus) " + e.InvalidAuth)
		} else {
			return "", errors.New("(globus) Authentication failed: Unexpected response status: " + resp.Status)
		}
	} else {
		return "", errors.New("(globus) " + err.Error())
	}
	return
}
<|endoftext|>