<commit_before>package sshd

import (
	"context"
	"encoding/base64"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"strconv"
	"time"
	"sync"
	"net/http"

	"github.com/pires/go-proxyproto"
	"golang.org/x/crypto/ssh"

	"gitlab.com/gitlab-org/gitlab-shell/internal/config"
	"gitlab.com/gitlab-org/gitlab-shell/internal/gitlabnet/authorizedkeys"

	"gitlab.com/gitlab-org/labkit/log"
	"gitlab.com/gitlab-org/labkit/correlation"
)

type status int

const(
	StatusStarting status = iota
	StatusReady
	StatusOnShutdown
	StatusClosed
)

type Server struct {
	Config *config.Config

	status status
	statusMu sync.Mutex
	wg sync.WaitGroup
	listener net.Listener
}

func (s *Server) ListenAndServe(ctx context.Context) error {
	if err := s.listen(); err != nil {
		return err
	}
	defer s.listener.Close()

	return s.serve(ctx)
}

func (s *Server) Shutdown() error {
	if s.listener == nil {
		return nil
	}

	s.changeStatus(StatusOnShutdown)

	return s.listener.Close()
}

func (s *Server) MonitoringServeMux() *http.ServeMux {
	mux := http.NewServeMux()

	mux.HandleFunc(s.Config.Server.ReadinessProbe, func(w http.ResponseWriter, r *http.Request) {
		if s.getStatus() == StatusReady {
			w.WriteHeader(http.StatusOK)
		} else {
			w.WriteHeader(http.StatusServiceUnavailable)
		}
	})

	mux.HandleFunc(s.Config.Server.LivenessProbe, func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(http.StatusOK)
	})

	return mux
}

func (s *Server) listen() error {
	sshListener, err := net.Listen("tcp", s.Config.Server.Listen)
	if err != nil {
		return fmt.Errorf("failed to listen for connection: %w", err)
	}

	if s.Config.Server.ProxyProtocol {
		sshListener = &proxyproto.Listener{Listener: sshListener}

		log.Info("Proxy protocol is enabled")
	}

	log.WithFields(log.Fields{"tcp_address": sshListener.Addr().String()}).Info("Listening for SSH connections")

	s.listener = sshListener

	return nil
}

func (s *Server) serve(ctx context.Context) error {
	sshCfg, err := s.initConfig(ctx)
	if err != nil {
		return err
	}

	s.changeStatus(StatusReady)

	for {
		nconn, err := s.listener.Accept()
		if err != nil {
			if s.getStatus() == StatusOnShutdown {
				break
			}

			log.WithError(err).Warn("Failed to accept connection")
			continue
		}

		s.wg.Add(1)
		go s.handleConn(ctx, sshCfg, nconn)
	}

	s.wg.Wait()

	s.changeStatus(StatusClosed)

	return nil
}

func (s *Server) changeStatus(st status) {
	s.statusMu.Lock()
	s.status = st
	s.statusMu.Unlock()
}

func (s *Server) getStatus() status {
	s.statusMu.Lock()
	defer s.statusMu.Unlock()

	return s.status
}

func (s *Server) initConfig(ctx context.Context) (*ssh.ServerConfig, error) {
	authorizedKeysClient, err := authorizedkeys.NewClient(s.Config)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize GitLab client: %w", err)
	}

	sshCfg := &ssh.ServerConfig{
		PublicKeyCallback: func(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
			if conn.User() != s.Config.User {
				return nil, errors.New("unknown user")
			}
			if key.Type() == ssh.KeyAlgoDSA {
				return nil, errors.New("DSA is prohibited")
			}
			ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
			defer cancel()
			res, err := authorizedKeysClient.GetByKey(ctx, base64.RawStdEncoding.EncodeToString(key.Marshal()))
			if err != nil {
				return nil, err
			}

			return &ssh.Permissions{
				// Record the public key used for authentication.
				Extensions: map[string]string{
					"key-id": strconv.FormatInt(res.Id, 10),
				},
			}, nil
		},
	}

	var loadedHostKeys uint
	for _, filename := range s.Config.Server.HostKeyFiles {
		keyRaw, err := ioutil.ReadFile(filename)
		if err != nil {
			log.WithError(err).Warnf("Failed to read host key %v", filename)
			continue
		}
		key, err := ssh.ParsePrivateKey(keyRaw)
		if err != nil {
			log.WithError(err).Warnf("Failed to parse host key %v", filename)
			continue
		}
		loadedHostKeys++
		sshCfg.AddHostKey(key)
	}
	if loadedHostKeys == 0 {
		return nil, fmt.Errorf("No host keys could be loaded, aborting")
	}

	return sshCfg, nil
}

func (s *Server) handleConn(ctx context.Context, sshCfg *ssh.ServerConfig, nconn net.Conn) {
	remoteAddr := nconn.RemoteAddr().String()

	defer s.wg.Done()
	defer nconn.Close()

	// Prevent a panic in a single connection from taking out the whole server
	defer func() {
		if err := recover(); err != nil {
			log.WithFields(log.Fields{"recovered_error": err}).Warnf("panic handling session from %s", remoteAddr)
		}
	}()

	ctx, cancel := context.WithCancel(correlation.ContextWithCorrelation(ctx, correlation.SafeRandomID()))
	defer cancel()

	sconn, chans, reqs, err := ssh.NewServerConn(nconn, sshCfg)
	if err != nil {
		log.WithError(err).Info("Failed to initialize SSH connection")
		return
	}

	go ssh.DiscardRequests(reqs)

	conn := newConnection(s.Config.Server.ConcurrentSessionsLimit, remoteAddr)
	conn.handle(ctx, chans, func(ctx context.Context, channel ssh.Channel, requests <-chan *ssh.Request) {
		session := &session{
			cfg:         s.Config,
			channel:     channel,
			gitlabKeyId: sconn.Permissions.Extensions["key-id"],
			remoteAddr:  remoteAddr,
		}

		session.handle(ctx, requests)
	})
}
<commit_msg>Set a 90-second timeout on proxy headers<commit_after>package sshd

import (
	"context"
	"encoding/base64"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/pires/go-proxyproto"
	"golang.org/x/crypto/ssh"

	"gitlab.com/gitlab-org/gitlab-shell/internal/config"
	"gitlab.com/gitlab-org/gitlab-shell/internal/gitlabnet/authorizedkeys"

	"gitlab.com/gitlab-org/labkit/correlation"
	"gitlab.com/gitlab-org/labkit/log"
)

type status int

const (
	StatusStarting status = iota
	StatusReady
	StatusOnShutdown
	StatusClosed
	ProxyHeaderTimeout = 90 * time.Second
)

type Server struct {
	Config *config.Config

	status   status
	statusMu sync.Mutex
	wg       sync.WaitGroup
	listener net.Listener
}

func (s *Server) ListenAndServe(ctx context.Context) error {
	if err := s.listen(); err != nil {
		return err
	}
	defer s.listener.Close()

	return s.serve(ctx)
}

func (s *Server) Shutdown() error {
	if s.listener == nil {
		return nil
	}

	s.changeStatus(StatusOnShutdown)

	return s.listener.Close()
}

func (s *Server) MonitoringServeMux() *http.ServeMux {
	mux := http.NewServeMux()

	mux.HandleFunc(s.Config.Server.ReadinessProbe, func(w http.ResponseWriter, r *http.Request) {
		if s.getStatus() == StatusReady {
			w.WriteHeader(http.StatusOK)
		} else {
			w.WriteHeader(http.StatusServiceUnavailable)
		}
	})

	mux.HandleFunc(s.Config.Server.LivenessProbe, func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
	})

	return mux
}

func (s *Server) listen() error {
	sshListener, err := net.Listen("tcp", s.Config.Server.Listen)
	if err != nil {
		return fmt.Errorf("failed to listen for connection: %w", err)
	}

	if s.Config.Server.ProxyProtocol {
		sshListener = &proxyproto.Listener{
			Listener:          sshListener,
			ReadHeaderTimeout: ProxyHeaderTimeout,
		}

		log.Info("Proxy protocol is enabled")
	}

	log.WithFields(log.Fields{"tcp_address": sshListener.Addr().String()}).Info("Listening for SSH connections")

	s.listener = sshListener

	return nil
}

func (s *Server) serve(ctx context.Context) error {
	sshCfg, err := s.initConfig(ctx)
	if err != nil {
		return err
	}

	s.changeStatus(StatusReady)

	for {
		nconn, err := s.listener.Accept()
		if err != nil {
			if s.getStatus() == StatusOnShutdown {
				break
			}

			log.WithError(err).Warn("Failed to accept connection")
			continue
		}

		s.wg.Add(1)
		go s.handleConn(ctx, sshCfg, nconn)
	}

	s.wg.Wait()

	s.changeStatus(StatusClosed)

	return nil
}

func (s *Server) changeStatus(st status) {
	s.statusMu.Lock()
	s.status = st
	s.statusMu.Unlock()
}

func (s *Server) getStatus() status {
	s.statusMu.Lock()
	defer s.statusMu.Unlock()

	return s.status
}

func (s *Server) initConfig(ctx context.Context) (*ssh.ServerConfig, error) {
	authorizedKeysClient, err := authorizedkeys.NewClient(s.Config)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize GitLab client: %w", err)
	}

	sshCfg := &ssh.ServerConfig{
		PublicKeyCallback: func(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
			if conn.User() != s.Config.User {
				return nil, errors.New("unknown user")
			}
			if key.Type() == ssh.KeyAlgoDSA {
				return nil, errors.New("DSA is prohibited")
			}
			ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
			defer cancel()
			res, err := authorizedKeysClient.GetByKey(ctx, base64.RawStdEncoding.EncodeToString(key.Marshal()))
			if err != nil {
				return nil, err
			}

			return &ssh.Permissions{
				// Record the public key used for authentication.
				Extensions: map[string]string{
					"key-id": strconv.FormatInt(res.Id, 10),
				},
			}, nil
		},
	}

	var loadedHostKeys uint
	for _, filename := range s.Config.Server.HostKeyFiles {
		keyRaw, err := ioutil.ReadFile(filename)
		if err != nil {
			log.WithError(err).Warnf("Failed to read host key %v", filename)
			continue
		}
		key, err := ssh.ParsePrivateKey(keyRaw)
		if err != nil {
			log.WithError(err).Warnf("Failed to parse host key %v", filename)
			continue
		}
		loadedHostKeys++
		sshCfg.AddHostKey(key)
	}
	if loadedHostKeys == 0 {
		return nil, fmt.Errorf("No host keys could be loaded, aborting")
	}

	return sshCfg, nil
}

func (s *Server) handleConn(ctx context.Context, sshCfg *ssh.ServerConfig, nconn net.Conn) {
	remoteAddr := nconn.RemoteAddr().String()

	defer s.wg.Done()
	defer nconn.Close()

	// Prevent a panic in a single connection from taking out the whole server
	defer func() {
		if err := recover(); err != nil {
			log.WithFields(log.Fields{"recovered_error": err}).Warnf("panic handling session from %s", remoteAddr)
		}
	}()

	ctx, cancel := context.WithCancel(correlation.ContextWithCorrelation(ctx, correlation.SafeRandomID()))
	defer cancel()

	sconn, chans, reqs, err := ssh.NewServerConn(nconn, sshCfg)
	if err != nil {
		log.WithError(err).Info("Failed to initialize SSH connection")
		return
	}

	go ssh.DiscardRequests(reqs)

	conn := newConnection(s.Config.Server.ConcurrentSessionsLimit, remoteAddr)
	conn.handle(ctx, chans, func(ctx context.Context, channel ssh.Channel, requests <-chan *ssh.Request) {
		session := &session{
			cfg:         s.Config,
			channel:     channel,
			gitlabKeyId: sconn.Permissions.Extensions["key-id"],
			remoteAddr:  remoteAddr,
		}

		session.handle(ctx, requests)
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	pathpkg "path"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/charlievieth/pkgs"
)

type mImportPaths struct {
	Fn            string
	Src           string
	Env           map[string]string
	InstallSuffix string
}

type mImportPathsDecl struct {
	Name string `json:"name"`
	Path string `json:"path"`
}

type mImportPathsDeclByName []mImportPathsDecl

func (m mImportPathsDeclByName) Len() int           { return len(m) }
func (m mImportPathsDeclByName) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m mImportPathsDeclByName) Less(i, j int) bool { return m[i].Name < m[j].Name }

type mImportPathsResponse struct {
	Imports []mImportPathsDecl `json:"imports"`
	Paths   []string           `json:"paths"`
}

func (m *mImportPaths) FileImports() ([]mImportPathsDecl, error) {
	fset := token.NewFileSet()
	af, err := parser.ParseFile(fset, m.Fn, m.Src, parser.ImportsOnly)
	if err != nil {
		return nil, err
	}

	imports := make([]mImportPathsDecl, 0, 8)
	for _, decl := range af.Decls {
		d, ok := decl.(*ast.GenDecl)
		if !ok {
			continue
		}
		for _, dspec := range d.Specs {
			spec, ok := dspec.(*ast.ImportSpec)
			if !ok {
				continue
			}
			quoted := spec.Path.Value
			path, err := strconv.Unquote(quoted)
			if err != nil {
				return nil, fmt.Errorf("%s: parser returned invalid quoted string: <%s>", m.Fn, quoted)
			}
			var name string
			if spec.Name != nil {
				name = spec.Name.String()
			} else {
				name = pathpkg.Base(path)
			}
			imports = append(imports, mImportPathsDecl{
				Path: path,
				Name: name,
			})
		}
	}

	return imports, nil
}

func (m *mImportPaths) Call() (interface{}, string) {
	imports, err := m.FileImports()
	if err != nil {
		return M{}, errStr(err)
	}

	names, err := importPaths(m.Env, m.InstallSuffix, filepath.Dir(m.Fn))
	if err != nil && len(names) == 0 {
		return M{}, errStr(err)
	}

	// dedupe since there may be duplicate vendored imports
	// names is sorted
	if len(names) > 0 {
		i := 0
		s := ""
		for _, x := range names {
			if x != s {
				names[i] = x
				s = x
				i++
			}
		}
		names = names[:i]
	}
	sort.Sort(mImportPathsDeclByName(imports))

	return &mImportPathsResponse{Imports: imports, Paths: names}, ""
}

type importsPathCacheEntry struct {
	Created time.Time
	Imports []string
}

// project root => *importsPathCacheEntry
var importsPathCache sync.Map

func init() {
	const TTL = time.Minute * 2

	go func() {
		for {
			time.Sleep(TTL / 4)

			importsPathCache.Range(func(key, value interface{}) bool {
				if e, ok := value.(*importsPathCacheEntry); ok {
					if time.Since(e.Created) > TTL {
						importsPathCache.Delete(key)
					}
				}
				return true
			})
		}
	}()
}

func isRoot(dir string) bool {
	// TODO: add ".svn" ".hg" ???
	for _, name := range []string{"vendor", "go.mod", ".git", "glide.yaml", "Gopkg.toml"} {
		if _, err := os.Lstat(dir + "/" + name); err == nil {
			return true
		}
	}
	return false
}

func projectRoot(dirname string) string {
	const sep = string(os.PathListSeparator)

	// special case for me
	for _, path := range strings.Split(build.Default.GOPATH, sep) {
		pfx := filepath.Join(path, "src", "repl")
		if strings.HasPrefix(dirname, pfx) {
			return pfx
		}
	}

	dir := filepath.ToSlash(dirname)
	for !isRoot(dir) {
		next := filepath.Dir(dir)
		if next == dir {
			break
		}
		dir = next
	}
	return dir
}

func importPaths(environ map[string]string, installSuffix, importDir string) ([]string, error) {
	cacheRoot := projectRoot(importDir)
	if v, ok := importsPathCache.Load(cacheRoot); ok {
		if e, _ := v.(*importsPathCacheEntry); e != nil {
			a := make([]string, len(e.Imports))
			copy(a, e.Imports)
			return a, nil
		}
	}

	// TODO:
	// 	- Consider adding os.GOROOT and os.GOPATH to environ
	// 	- Check for duplicate paths
	var root string
	if s := environ["GOROOT"]; s != "" {
		root = s
	} else {
		root = runtime.GOROOT()
	}
	var path string
	if s := environ["GOPATH"]; s != "" {
		path = s
	} else {
		path = os.Getenv("GOPATH")
	}
	ctxt := build.Default
	if root != "" {
		ctxt.GOROOT = root
	}
	if path != "" {
		ctxt.GOPATH = path
	}
	if installSuffix != "" {
		ctxt.InstallSuffix = installSuffix
	}

	paths, err := pkgs.Walk(&ctxt, importDir)
	if len(paths) != 0 {
		sort.Strings(paths)
	}
	if err != nil {
		return paths, err
	}

	importsPathCache.Store(cacheRoot, &importsPathCacheEntry{
		Created: time.Now(),
		Imports: append([]string(nil), paths...),
	})
	return paths, nil
}

func init() {
	registry.Register("import_paths", func(_ *Broker) Caller {
		return &mImportPaths{
			Env: map[string]string{},
		}
	})
}
<commit_msg>margo: improve logic for finding the project root<commit_after>package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	pathpkg "path"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/charlievieth/pkgs"
)

type mImportPaths struct {
	Fn            string
	Src           string
	Env           map[string]string
	InstallSuffix string
}

type mImportPathsDecl struct {
	Name string `json:"name"`
	Path string `json:"path"`
}

type mImportPathsDeclByName []mImportPathsDecl

func (m mImportPathsDeclByName) Len() int           { return len(m) }
func (m mImportPathsDeclByName) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m mImportPathsDeclByName) Less(i, j int) bool { return m[i].Name < m[j].Name }

type mImportPathsResponse struct {
	Imports []mImportPathsDecl `json:"imports"`
	Paths   []string           `json:"paths"`
}

func (m *mImportPaths) FileImports() ([]mImportPathsDecl, error) {
	fset := token.NewFileSet()
	af, err := parser.ParseFile(fset, m.Fn, m.Src, parser.ImportsOnly)
	if err != nil {
		return nil, err
	}

	imports := make([]mImportPathsDecl, 0, 8)
	for _, decl := range af.Decls {
		d, ok := decl.(*ast.GenDecl)
		if !ok {
			continue
		}
		for _, dspec := range d.Specs {
			spec, ok := dspec.(*ast.ImportSpec)
			if !ok {
				continue
			}
			quoted := spec.Path.Value
			path, err := strconv.Unquote(quoted)
			if err != nil {
				return nil, fmt.Errorf("%s: parser returned invalid quoted string: <%s>", m.Fn, quoted)
			}
			var name string
			if spec.Name != nil {
				name = spec.Name.String()
			} else {
				name = pathpkg.Base(path)
			}
			imports = append(imports, mImportPathsDecl{
				Path: path,
				Name: name,
			})
		}
	}

	return imports, nil
}

func (m *mImportPaths) Call() (interface{}, string) {
	imports, err := m.FileImports()
	if err != nil {
		return M{}, errStr(err)
	}

	names, err := importPaths(m.Env, m.InstallSuffix, filepath.Dir(m.Fn))
	if err != nil && len(names) == 0 {
		return M{}, errStr(err)
	}

	// dedupe since there may be duplicate vendored imports
	// names is sorted
	if len(names) > 0 {
		i := 0
		s := ""
		for _, x := range names {
			if x != s {
				names[i] = x
				s = x
				i++
			}
		}
		names = names[:i]
	}
	sort.Sort(mImportPathsDeclByName(imports))

	return &mImportPathsResponse{Imports: imports, Paths: names}, ""
}

type importsPathCacheEntry struct {
	Created time.Time
	Imports []string
}

// project root => *importsPathCacheEntry
var importsPathCache sync.Map

func init() {
	const TTL = time.Minute * 2

	go func() {
		for {
			time.Sleep(TTL / 4)

			importsPathCache.Range(func(key, value interface{}) bool {
				if e, ok := value.(*importsPathCacheEntry); ok {
					if time.Since(e.Created) > TTL {
						importsPathCache.Delete(key)
					}
				}
				return true
			})
		}
	}()
}

func isRoot(dir string) bool {
	// TODO: add ".svn" ".hg" ???
	for _, name := range []string{"go.mod", ".git", "vendor", "glide.yaml", "Gopkg.toml"} {
		if _, err := os.Lstat(dir + "/" + name); err == nil {
			return true
		}
	}
	return false
}

func hasPathPrefix(s, prefix string) bool {
	rel, err := filepath.Rel(prefix, s)
	if err != nil {
		return false
	}
	return !strings.HasPrefix(rel, "..")
}

// shiftPathElements shifts count path elements from path from to path to.
func shiftPathElements(to, from string, count int) (string, string) {
	a := strings.Split(filepath.ToSlash(from), "/")
	if count > len(a) {
		count = len(a)
	}
	to += "/" + strings.Join(a[:count], "/")
	from = strings.Join(a[count:], "/")

	return filepath.FromSlash(to), filepath.FromSlash(from)
}

func projectRoot(ctxt *build.Context, dirname string) string {
	const sep = string(filepath.Separator)

	// WARN
	if ctxt == nil {
		ctxt = &build.Default
	}

	root := ctxt.GOROOT + sep + "src"
	if hasPathPrefix(dirname, root) {
		return root
	}

	dirname = filepath.Clean(dirname)

	var subpath string
	var rootpath string
	gopaths := strings.Split(ctxt.GOPATH, string(os.PathListSeparator))
	for _, p := range gopaths {
		p += sep + "src"
		s, err := filepath.Rel(p, dirname)
		if err != nil {
			continue
		}
		if s == "." {
			// WARN: you cannot have go files at the root of GOPATH
			// but there's nothing we can do about that here it will
			// simply result in an error from whatever tool we invoke.
			return p
		}
		if s != "" && !strings.HasPrefix(s, "..") {
			subpath = s
			rootpath = p
			break
		}
	}

	// fmt.Printf("subpath: %q\n", subpath)
	// fmt.Printf("rootpath: %q\n", rootpath)
	if subpath == "" {
		// Search for a project dir outside of the GOPATH (basically, search
		// without stopping once we've walked outside the GOPATH)
		//
		d := dirname
		for next := filepath.Dir(d); d != next; d, next = next, filepath.Dir(d) {
			if isRoot(d) {
				return d
			}
		}
		return d
	}

	// CEV: special case for me
	if subpath == "repl" || strings.HasPrefix(subpath, "repl/") ||
		(sep == "\\" && strings.HasPrefix(subpath, "repl\\")) {

		return filepath.Join(rootpath, "repl")
	}

	// CEV: since projects cannot live at $GOPATH and are typically at least two
	// directories removed from the $GOPATH we shift two path elements from the
	// subdir to the rootdir:
	//
	//  ~/go/src ... gh.com/foo/bar
	// 	 =>
	//  ~/src/gh.com/foo ... bar
	//
	rootpath, subpath = shiftPathElements(rootpath, subpath, 2)

	a := strings.Split(subpath, "/")
	for i := len(a); i >= 0; i-- {
		p := rootpath + "/" + strings.Join(a[:i], "/")
		// fmt.Println("p:", p)
		if isRoot(p) {
			return filepath.Clean(p) // WARN: prob don't need clean
		}
	}

	return dirname
}

func importPaths(environ map[string]string, installSuffix, importDir string) ([]string, error) {

	cacheRoot := projectRoot(contextFromEnv(environ), importDir)
	if v, ok := importsPathCache.Load(cacheRoot); ok {
		if e, _ := v.(*importsPathCacheEntry); e != nil {
			a := make([]string, len(e.Imports))
			copy(a, e.Imports)
			return a, nil
		}
	}

	// TODO:
	// 	- Consider adding os.GOROOT and os.GOPATH to environ
	// 	- Check for duplicate paths
	var root string
	if s := environ["GOROOT"]; s != "" {
		root = s
	} else {
		root = runtime.GOROOT()
	}
	var path string
	if s := environ["GOPATH"]; s != "" {
		path = s
	} else {
		path = os.Getenv("GOPATH")
	}
	ctxt := build.Default
	if root != "" {
		ctxt.GOROOT = root
	}
	if path != "" {
		ctxt.GOPATH = path
	}
	if installSuffix != "" {
		ctxt.InstallSuffix = installSuffix
	}

	paths, err := pkgs.Walk(&ctxt, importDir)
	if len(paths) != 0 {
		sort.Strings(paths)
	}
	if err != nil {
		return paths, err
	}

	importsPathCache.Store(cacheRoot, &importsPathCacheEntry{
		Created: time.Now(),
		Imports: append([]string(nil), paths...),
	})
	return paths, nil
}

func init() {
	registry.Register("import_paths", func(_ *Broker) Caller {
		return &mImportPaths{
			Env: map[string]string{},
		}
	})
}
<|endoftext|>


<commit_before>package displayhelpers

import (
	"fmt"
	"os"

	"github.com/concourse/concourse/go-concourse/concourse"

	"github.com/concourse/concourse/fly/ui"
)

func PrintDeprecationWarningHeader() {
	printColorFunc := ui.ErroredColor.SprintFunc()
	fmt.Fprintf(ui.Stderr, "%s\n", printColorFunc("DEPRECATION WARNING:"))
}

func PrintWarningHeader() {
	printColorFunc := ui.BlinkingErrorColor.SprintFunc()
	fmt.Fprintf(ui.Stderr, "%s\n", printColorFunc("WARNING:"))
}

func ShowErrors(errorHeader string, errorMessages []string) {
	fmt.Fprintln(ui.Stderr, "")
	PrintWarningHeader()

	fmt.Fprintln(ui.Stderr, errorHeader+":")
	for _, errorMessage := range errorMessages {
		fmt.Fprintf(ui.Stderr, "  - %s\n", errorMessage)
	}

	fmt.Fprintln(ui.Stderr, "")
}

func ShowWarnings(warnings []concourse.ConfigWarning) {
	fmt.Fprintln(ui.Stderr, "")
	PrintDeprecationWarningHeader()

	for _, warning := range warnings {
		fmt.Fprintf(ui.Stderr, "  - %s\n", warning.Message)
	}

	fmt.Fprintln(ui.Stderr, "")
}

func Failf(message string, args ...interface{}) {
	fmt.Fprintf(ui.Stderr, message+"\n", args...)
	os.Exit(1)
}

func FailWithErrorf(message string, err error, args ...interface{}) {
	templatedMessage := fmt.Sprintf(message, args...)
	Failf("%s: %s", templatedMessage, err.Error())
}
<commit_msg>fly: add reference to documentation on invalid identifier warnings<commit_after>package displayhelpers

import (
	"fmt"
	"os"

	"github.com/concourse/concourse/go-concourse/concourse"

	"github.com/concourse/concourse/fly/ui"
)

func PrintDeprecationWarningHeader() {
	printColorFunc := ui.ErroredColor.SprintFunc()
	fmt.Fprintf(ui.Stderr, "%s\n", printColorFunc("DEPRECATION WARNING:"))
}

func PrintWarningHeader() {
	printColorFunc := ui.BlinkingErrorColor.SprintFunc()
	fmt.Fprintf(ui.Stderr, "%s\n", printColorFunc("WARNING:"))
}

func ShowErrors(errorHeader string, errorMessages []string) {
	fmt.Fprintln(ui.Stderr, "")
	PrintWarningHeader()

	fmt.Fprintln(ui.Stderr, errorHeader+":")
	for _, errorMessage := range errorMessages {
		fmt.Fprintf(ui.Stderr, "  - %s\n", errorMessage)
	}

	fmt.Fprintln(ui.Stderr, "")
}

func ShowWarnings(warnings []concourse.ConfigWarning) {
	fmt.Fprintln(ui.Stderr, "")
	PrintDeprecationWarningHeader()

	warningTypes := make(map[string]bool)
	for _, warning := range warnings {
		warningTypes[warning.Type] = true
		fmt.Fprintf(ui.Stderr, "  - %s\n", warning.Message)
	}

	fmt.Fprintln(ui.Stderr, "")

	if warningTypes["invalid_identifier"] {
		fmt.Fprintln(ui.Stderr, "identifier schema documentation: https://concourse-ci.org/config-basics.html#schema.identifier")
		fmt.Fprintln(ui.Stderr, "")
	}
}

func Failf(message string, args ...interface{}) {
	fmt.Fprintf(ui.Stderr, message+"\n", args...)
	os.Exit(1)
}

func FailWithErrorf(message string, err error, args ...interface{}) {
	templatedMessage := fmt.Sprintf(message, args...)
	Failf("%s: %s", templatedMessage, err.Error())
}
<|endoftext|>


<commit_before>package session // import "github.com/docker/docker/api/server/router/session"

import "github.com/docker/docker/api/server/router"

// sessionRouter is a router to talk with the session controller
type sessionRouter struct {
	backend Backend
	routes  []router.Route
}

// NewRouter initializes a new session router
func NewRouter(b Backend) router.Router {
	r := &sessionRouter{
		backend: b,
	}
	r.initRoutes()
	return r
}

// Routes returns the available routers to the session controller
func (r *sessionRouter) Routes() []router.Route {
	return r.routes
}

func (r *sessionRouter) initRoutes() {
	r.routes = []router.Route{
		router.Experimental(router.NewPostRoute("/session", r.startSession)),
	}
}
<commit_msg>move /session api endpoint out of experimental<commit_after>package session // import "github.com/docker/docker/api/server/router/session"

import "github.com/docker/docker/api/server/router"

// sessionRouter is a router to talk with the session controller
type sessionRouter struct {
	backend Backend
	routes  []router.Route
}

// NewRouter initializes a new session router
func NewRouter(b Backend) router.Router {
	r := &sessionRouter{
		backend: b,
	}
	r.initRoutes()
	return r
}

// Routes returns the available routers to the session controller
func (r *sessionRouter) Routes() []router.Route {
	return r.routes
}

func (r *sessionRouter) initRoutes() {
	r.routes = []router.Route{
		router.NewPostRoute("/session", r.startSession),
	}
}
<|endoftext|>


<commit_before>package schema

const (
	// JSONSchema is the libStorage API JSON schema
	JSONSchema = `{
    "id": "https://github.com/emccode/libstorage",
    "$schema": "http://json-schema.org/draft-04/schema#",
    "title": "libStorage JSON Schema",
    "definitions": {

        "volume": {
            "title": "Volume",
            "description": "Volume provides information about a storage volume.",
            "type": "object",
            "properties": {
                "id": {
                    "type": "string",
                    "description": "ID is a piece of information that uniquely identifies the volume on the storage platform to which the volume belongs. A volume ID is not guaranteed to be unique across multiple, configured services."
                },
                "name": {
                    "type": "string",
                    "description": "The volume name."
                },
                "type": {
                    "type": "string",
                    "description": "The volume type."
                },
                "attachments": {
                    "type": "array",
                    "description": "The volume's attachments.",
                    "items": { "$ref": "#/definitions/volumeAttachment" }
                },
                "availabilityZone": {
                    "type": "string",
                    "description": "The zone for which the volume is available."
                },
                "iops": {
                    "type": "number",
                    "description": "The volume IOPs."
                },
                "networkName": {
                    "type": "string",
                    "description": "The name of the network on which the volume resides."
                },
                "size": {
                    "type": "number",
                    "description": "The volume size (GB)."
                },
                "status": {
                    "type": "string",
                    "description": "The volume status."
                },
                "fields": { "$ref": "#/definitions/fields" }
            },
            "required": [ "id", "name" ],
            "additionalProperties": false
        },

        "volumeAttachment": {
            "title": "VolumeAttachment",
            "description": " VolumeAttachment provides information about an object attached to a storage volume.",
            "type": "object",
            "properties": {
                "instanceID": { "$ref": "#/definitions/instanceID" },
                "deviceName": {
                    "type": "string",
                    "description": "The name of the device on to which the volume is mounted."
                },
                "status": {
                    "type": "string",
                    "description": "The status of the attachment."
                },
                "volumeID": {
                    "type": "string",
                    "description": "The ID of the volume to which the attachment belongs."
                },
                "mountPoint": {
                    "type": "string",
                    "description": "The file system path to which the volume is mounted."
                },
                "fields": { "$ref": "#/definitions/fields" }
            },
            "required": [ "instanceID", "deviceName", "volumeID" ],
            "additionalProperties": false
        },

        "instanceID": {
            "title": "InstanceID",
            "description": "InstanceID identifies a host to a remote storage platform.",
            "type": "object",
            "properties": {
                "id": {
                    "type": "string",
                    "description": "The instance ID."
                },
                "driver": {
                    "type": "string",
                    "description": "The name of the driver that created the instance ID."
                },
                "formatted": {
                    "type": "boolean",
                    "description": "A flag indicating whether or not the instance ID has been formatted by an instance inspection."
                },
                "metadata": {
                    "type": "object",
                    "description": "Extra information about the instance ID."
                }
            },
            "required": [ "id", "driver" ],
            "additionalProperties": false
        },

        "instance": {
            "title": "Instnace",
            "description": "Instance is additional information about a host, generated using the InstanceID.",
            "type": "object",
            "properties": {
                "instanceID": { "$ref": "#/definitions/instanceID" },
                "name": {
                    "type": "string",
                    "description": "The name of the instance."
                },
                "providerName": {
                    "type": "string",
                    "description": "The name of the provider that owns the object."
                },
                "region": {
                    "type": "string",
                    "description": "The region from which the object originates."
                },
                "fields": { "$ref": "#/definitions/fields" }
            },
            "required": [ "id" ],
            "additionalProperties": false
        },

        "snapshot": {
            "title": "Snapshot",
            "description": "Snapshot provides information about a storage volume snapshot.",
            "type": "object",
            "properties": {
                "id": {
                    "type": "string",
                    "description": "The snapshot's ID."
                },
                "name": {
                    "type": "string",
                    "description": "The name of the snapshot."
                },
                "description": {
                    "type": "string",
                    "description": "A description of the snapshot."
                },
                "startTime": {
                    "type": "number",
                    "description": "The time (epoch) at which the request to create the snapshot was submitted."
                },
                "status": {
                    "type": "string",
                    "description": "The status of the snapshot."
                },
                "volumeID": {
                    "type": "string",
                    "description": "The ID of the volume to which the snapshot belongs."
                },
                "volumeSize": {
                    "type": "number",
                    "description": "The size of the volume to which the snapshot belongs."
                },
                "fields": { "$ref": "#/definitions/fields" }
            },
            "required": [ "id" ],
            "additionalProperties": false
        },

        "task": {
            "type": "object",
            "properties": {
                "id": {
                    "type": "number",
                    "description": "The task's unique identifier."
                },
                "name": {
                    "type": "string",
                    "description": "The name of the task."
                },
                "user": {
                    "type": "string",
                    "description": "The name of the user that created the task."
                },
                "completeTime": {
                    "type": "number",
                    "description": "The time stamp (epoch) when the task was completed."
                },
                "queueTime": {
                    "type": "number",
                    "description": "The time stamp (epoch) when the task was created."
                },
                "startTime": {
                    "type": "number",
                    "description": "The time stamp (epoch) when the task started running."
                },
                "result": {
                    "type": "object",
                    "description": "The result of the operation."
                },
                "error": {
                    "type": "object",
                    "description": "If the operation returned an error, this is it."
                },
                "fields": { "$ref": "#/definitions/fields" }
            },
            "required": [ "id", "name",  "user", "queueTime" ],
            "additionalProperties": false
        },

        "serviceInfo": {
            "type": "object",
            "properties": {
                "name": {
                    "type": "string",
                    "description": "Name is the service's name."
                },
                "instance": { "$ref": "#/definitions/instance" },
                "driver": { "$ref": "#/definitions/driverInfo" }
            },
            "required": [ "name", "driver" ],
            "additionalProperties": false
        },

        "driverInfo": {
            "type": "object",
            "properties": {
                "name": {
                    "type": "string",
                    "description": "Ignore is a flag that indicates whether the client logic should invoke the GetNextAvailableDeviceName function prior to submitting an AttachVolume request to the server."
                },
                "type": {
                    "type": "string",
                    "description": "Type is the type of storage the driver provides: block, nas, object."
                },
                "nextDevice": { "$ref": "#/definitions/nextDeviceInfo" }
            },
            "required": [ "name", "type" ],
            "additionalProperties": false
        },

        "executorInfo": {
            "type": "object",
            "properties": {
                "name": {
                    "type": "string",
                    "description": "The name of the executor."
                },
                "md5checksum": {
                    "type": "string",
                    "description": "The file's MD5 checksum. This can be used to determine if a local copy of the executor needs to be updated."
                },
                "size": {
                    "type": "number",
                    "description": "The size of the executor, in bytes."
                },
                "lastModified": {
                    "type": "number",
                    "description": "The time the executor was last modified as an epoch."
                }
            },
            "required": [ "name", "md5checksum", "size", "lastModified" ],
            "additionalProperties": false
        },

        "nextDeviceInfo": {
            "type": "object",
            "properties": {
                "ignore": {
                    "type": "boolean",
                    "description": "Ignore is a flag that indicates whether the client logic should invoke the GetNextAvailableDeviceName function prior to submitting an AttachVolume request to the server."
                },
                "prefix": {
                    "type": "string",
                    "description": "Prefix is the first part of a device path's value after the \"/dev/\" portion. For example, the prefix in \"/dev/xvda\" is \"xvd\"."
                },
                "pattern": {
                    "type": "string",
                    "description": "Pattern is the regex to match the part of a device path after the prefix."
                }
            },
            "additionalProperties": false
        },

        "fields": {
            "type": "object",
            "description": "Fields are additional properties that can be defined for this type.",
            "patternProperties": {
                ".+": { "type": "string" }
            },
            "additionalProperties": true
        },

        "volumeMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/volume" }
            },
            "additionalProperties": false
        },

        "snapshotMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/snapshot" }
            },
            "additionalProperties": false
        },

        "taskMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/task" }
            },
            "additionalProperties": false
        },

        "serviceVolumeMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/volumeMap" }
            },
            "additionalProperties": false
        },

        "serviceSnapshotMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/snapshotMap" }
            },
            "additionalProperties": false
        },

        "serviceTaskMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/taskMap" }
            },
            "additionalProperties": false
        },

        "serviceInfoMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/serviceInfo" }
            },
            "additionalProperties": false
        },

        "executorInfoMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/executorInfo" }
            },
            "additionalProperties": false
        },

        "driverInfoMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/driverInfo" }
            },
            "additionalProperties": false
        },

        "opts": {
            "type": "object",
            "description": "Opts are additional properties that can be defined for POST requests.",
            "patternProperties": {
                "^.+$": {
                    "anyOf": [
                        { "type": "array" },
                        { "type": "boolean" },
                        { "type": "integer" },
                        { "type": "number" },
                        { "type": "null" },
                        { "type": "string" },
                        { "$ref": "#/definitions/opts" }
                    ]
                }
            },
            "additionalProperties": true
        },

        "volumeCreateRequest": {
            "type": "object",
            "properties": {
                "name": {
                    "type": "string"
                },
                "availabilityZone": {
                    "type": "string"
                },
                "iops": {
                    "type": "number"
                },
                "size": {
                    "type": "number"
                },
                "type": {
                    "type": "string"
                },
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "required": [ "name" ],
            "additionalProperties": false
        },

        "volumeCopyRequest": {
            "type": "object",
            "properties": {
                "volumeName": {
                    "type": "string"
                },
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "required": [ "volumeName" ],
            "additionalProperties": false
        },

        "volumeSnapshotRequest": {
            "type": "object",
            "properties": {
                "snapshotName": {
                    "type": "string"
                },
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "required": [ "snapshotName" ],
            "additionalProperties": false
        },

        "volumeAttachRequest": {
            "type": "object",
            "properties": {
                "nextDeviceName": {
                    "type": "string"
                },
                "force": {
                    "type": "boolean"
                },
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "additionalProperties": false
        },

        "volumeAttachResponse": {
            "type": "object",
            "properties": {
                "volume": { "$ref" : "#/definitions/volume" },
                "attachToken" : { "type": "string" }
            },
            "required": [ "volume", "attachToken" ],
            "additionalProperties": false
        },

        "volumeDetachRequest": {
            "type": "object",
            "properties": {
                "force": {
                    "type": "boolean"
                },
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "additionalProperties": false
        },

        "snapshotCopyRequest": {
            "type": "object",
            "properties": {
                "snapshotName": {
                    "type": "string"
                },
                "destinationID": {
                    "type": "string"
                },
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "required": [ "snapshotName", "destinationID" ],
            "additionalProperties": false
        },

        "snapshotRemoveRequest": {
            "type": "object",
            "properties": {
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "additionalProperties": false
        },

        "error": {
            "type": "object",
            "properties": {
                "message": {
                    "type": "string",
                    "pattern": "^.{10,}|.*[Ee]rror$"
                },
                "status": {
                    "type": "number",
                    "minimum": 400,
                    "maximum": 599
                },
                "error": {
                    "type": "object",
                    "additionalProperties": true
                }
            },
            "required": [ "message", "status" ],
            "additionalProperties": false
        }
    }
}
`
)
<commit_msg>Schema Fix<commit_after>package schema

const (
	// JSONSchema is the libStorage API JSON schema
	JSONSchema = `{
    "id": "https://github.com/emccode/libstorage",
    "$schema": "http://json-schema.org/draft-04/schema#",
    "title": "libStorage JSON Schema",
    "definitions": {

        "volume": {
            "title": "Volume",
            "description": "Volume provides information about a storage volume.",
            "type": "object",
            "properties": {
                "id": {
                    "type": "string",
                    "description": "ID is a piece of information that uniquely identifies the volume on the storage platform to which the volume belongs. A volume ID is not guaranteed to be unique across multiple, configured services."
                },
                "name": {
                    "type": "string",
                    "description": "The volume name."
                },
                "type": {
                    "type": "string",
                    "description": "The volume type."
                },
                "attachments": {
                    "type": "array",
                    "description": "The volume's attachments.",
                    "items": { "$ref": "#/definitions/volumeAttachment" }
                },
                "availabilityZone": {
                    "type": "string",
                    "description": "The zone for which the volume is available."
                },
                "iops": {
                    "type": "number",
                    "description": "The volume IOPs."
                },
                "networkName": {
                    "type": "string",
                    "description": "The name of the network on which the volume resides."
                },
                "size": {
                    "type": "number",
                    "description": "The volume size (GB)."
                },
                "status": {
                    "type": "string",
                    "description": "The volume status."
                },
                "fields": { "$ref": "#/definitions/fields" }
            },
            "required": [ "id", "name" ],
            "additionalProperties": false
        },

        "volumeAttachment": {
            "title": "VolumeAttachment",
            "description": " VolumeAttachment provides information about an object attached to a storage volume.",
            "type": "object",
            "properties": {
                "instanceID": { "$ref": "#/definitions/instanceID" },
                "deviceName": {
                    "type": "string",
                    "description": "The name of the device on to which the volume is mounted."
                },
                "status": {
                    "type": "string",
                    "description": "The status of the attachment."
                },
                "volumeID": {
                    "type": "string",
                    "description": "The ID of the volume to which the attachment belongs."
                },
                "mountPoint": {
                    "type": "string",
                    "description": "The file system path to which the volume is mounted."
                },
                "fields": { "$ref": "#/definitions/fields" }
            },
            "required": [ "instanceID", "deviceName", "volumeID" ],
            "additionalProperties": false
        },

        "instanceID": {
            "title": "InstanceID",
            "description": "InstanceID identifies a host to a remote storage platform.",
            "type": "object",
            "properties": {
                "id": {
                    "type": "string",
                    "description": "The instance ID."
                },
                "driver": {
                    "type": "string",
                    "description": "The name of the driver that created the instance ID."
                },
                "formatted": {
                    "type": "boolean",
                    "description": "A flag indicating whether or not the instance ID has been formatted by an instance inspection."
                },
                "metadata": {
                    "type": "object",
                    "description": "Extra information about the instance ID."
                }
            },
            "required": [ "id", "driver" ],
            "additionalProperties": false
        },

        "instance": {
            "title": "Instance",
            "description": "Instance is additional information about a host, generated using the InstanceID.",
            "type": "object",
            "properties": {
                "instanceID": { "$ref": "#/definitions/instanceID" },
                "name": {
                    "type": "string",
                    "description": "The name of the instance."
                },
                "providerName": {
                    "type": "string",
                    "description": "The name of the provider that owns the object."
                },
                "region": {
                    "type": "string",
                    "description": "The region from which the object originates."
                },
                "fields": { "$ref": "#/definitions/fields" }
            },
            "required": [ "id" ],
            "additionalProperties": false
        },

        "snapshot": {
            "title": "Snapshot",
            "description": "Snapshot provides information about a storage volume snapshot.",
            "type": "object",
            "properties": {
                "id": {
                    "type": "string",
                    "description": "The snapshot's ID."
                },
                "name": {
                    "type": "string",
                    "description": "The name of the snapshot."
                },
                "description": {
                    "type": "string",
                    "description": "A description of the snapshot."
                },
                "startTime": {
                    "type": "number",
                    "description": "The time (epoch) at which the request to create the snapshot was submitted."
                },
                "status": {
                    "type": "string",
                    "description": "The status of the snapshot."
                },
                "volumeID": {
                    "type": "string",
                    "description": "The ID of the volume to which the snapshot belongs."
                },
                "volumeSize": {
                    "type": "number",
                    "description": "The size of the volume to which the snapshot belongs."
                },
                "fields": { "$ref": "#/definitions/fields" }
            },
            "required": [ "id" ],
            "additionalProperties": false
        },

        "task": {
            "type": "object",
            "properties": {
                "id": {
                    "type": "number",
                    "description": "The task's unique identifier."
                },
                "name": {
                    "type": "string",
                    "description": "The name of the task."
                },
                "user": {
                    "type": "string",
                    "description": "The name of the user that created the task."
                },
                "completeTime": {
                    "type": "number",
                    "description": "The time stamp (epoch) when the task was completed."
                },
                "queueTime": {
                    "type": "number",
                    "description": "The time stamp (epoch) when the task was created."
                },
                "startTime": {
                    "type": "number",
                    "description": "The time stamp (epoch) when the task started running."
                },
                "result": {
                    "type": "object",
                    "description": "The result of the operation."
                },
                "error": {
                    "type": "object",
                    "description": "If the operation returned an error, this is it."
                },
                "fields": { "$ref": "#/definitions/fields" }
            },
            "required": [ "id", "name",  "user", "queueTime" ],
            "additionalProperties": false
        },

        "serviceInfo": {
            "type": "object",
            "properties": {
                "name": {
                    "type": "string",
                    "description": "Name is the service's name."
                },
                "instance": { "$ref": "#/definitions/instance" },
                "driver": { "$ref": "#/definitions/driverInfo" }
            },
            "required": [ "name", "driver" ],
            "additionalProperties": false
        },

        "driverInfo": {
            "type": "object",
            "properties": {
                "name": {
                    "type": "string",
                    "description": "Ignore is a flag that indicates whether the client logic should invoke the GetNextAvailableDeviceName function prior to submitting an AttachVolume request to the server."
                },
                "type": {
                    "type": "string",
                    "description": "Type is the type of storage the driver provides: block, nas, object."
                },
                "nextDevice": { "$ref": "#/definitions/nextDeviceInfo" }
            },
            "required": [ "name", "type" ],
            "additionalProperties": false
        },

        "executorInfo": {
            "type": "object",
            "properties": {
                "name": {
                    "type": "string",
                    "description": "The name of the executor."
                },
                "md5checksum": {
                    "type": "string",
                    "description": "The file's MD5 checksum. This can be used to determine if a local copy of the executor needs to be updated."
                },
                "size": {
                    "type": "number",
                    "description": "The size of the executor, in bytes."
                },
                "lastModified": {
                    "type": "number",
                    "description": "The time the executor was last modified as an epoch."
                }
            },
            "required": [ "name", "md5checksum", "size", "lastModified" ],
            "additionalProperties": false
        },

        "nextDeviceInfo": {
            "type": "object",
            "properties": {
                "ignore": {
                    "type": "boolean",
                    "description": "Ignore is a flag that indicates whether the client logic should invoke the GetNextAvailableDeviceName function prior to submitting an AttachVolume request to the server."
                },
                "prefix": {
                    "type": "string",
                    "description": "Prefix is the first part of a device path's value after the \"/dev/\" portion. For example, the prefix in \"/dev/xvda\" is \"xvd\"."
                },
                "pattern": {
                    "type": "string",
                    "description": "Pattern is the regex to match the part of a device path after the prefix."
                }
            },
            "additionalProperties": false
        },

        "fields": {
            "type": "object",
            "description": "Fields are additional properties that can be defined for this type.",
            "patternProperties": {
                ".+": { "type": "string" }
            },
            "additionalProperties": true
        },

        "volumeMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/volume" }
            },
            "additionalProperties": false
        },

        "snapshotMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/snapshot" }
            },
            "additionalProperties": false
        },

        "taskMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/task" }
            },
            "additionalProperties": false
        },

        "serviceVolumeMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/volumeMap" }
            },
            "additionalProperties": false
        },

        "serviceSnapshotMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/snapshotMap" }
            },
            "additionalProperties": false
        },

        "serviceTaskMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/taskMap" }
            },
            "additionalProperties": false
        },

        "serviceInfoMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/serviceInfo" }
            },
            "additionalProperties": false
        },

        "executorInfoMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/executorInfo" }
            },
            "additionalProperties": false
        },

        "driverInfoMap": {
            "type": "object",
            "patternProperties": {
                "^.+$": { "$ref": "#/definitions/driverInfo" }
            },
            "additionalProperties": false
        },

        "opts": {
            "type": "object",
            "description": "Opts are additional properties that can be defined for POST requests.",
            "patternProperties": {
                "^.+$": {
                    "anyOf": [
                        { "type": "array" },
                        { "type": "boolean" },
                        { "type": "integer" },
                        { "type": "number" },
                        { "type": "null" },
                        { "type": "string" },
                        { "$ref": "#/definitions/opts" }
                    ]
                }
            },
            "additionalProperties": true
        },

        "volumeCreateRequest": {
            "type": "object",
            "properties": {
                "name": {
                    "type": "string"
                },
                "availabilityZone": {
                    "type": "string"
                },
                "iops": {
                    "type": "number"
                },
                "size": {
                    "type": "number"
                },
                "type": {
                    "type": "string"
                },
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "required": [ "name" ],
            "additionalProperties": false
        },

        "volumeCopyRequest": {
            "type": "object",
            "properties": {
                "volumeName": {
                    "type": "string"
                },
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "required": [ "volumeName" ],
            "additionalProperties": false
        },

        "volumeSnapshotRequest": {
            "type": "object",
            "properties": {
                "snapshotName": {
                    "type": "string"
                },
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "required": [ "snapshotName" ],
            "additionalProperties": false
        },

        "volumeAttachRequest": {
            "type": "object",
            "properties": {
                "nextDeviceName": {
                    "type": "string"
                },
                "force": {
                    "type": "boolean"
                },
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "additionalProperties": false
        },

        "volumeAttachResponse": {
            "type": "object",
            "properties": {
                "volume": { "$ref" : "#/definitions/volume" },
                "attachToken" : { "type": "string" }
            },
            "required": [ "volume", "attachToken" ],
            "additionalProperties": false
        },

        "volumeDetachRequest": {
            "type": "object",
            "properties": {
                "force": {
                    "type": "boolean"
                },
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "additionalProperties": false
        },

        "snapshotCopyRequest": {
            "type": "object",
            "properties": {
                "snapshotName": {
                    "type": "string"
                },
                "destinationID": {
                    "type": "string"
                },
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "required": [ "snapshotName", "destinationID" ],
            "additionalProperties": false
        },

        "snapshotRemoveRequest": {
            "type": "object",
            "properties": {
                "opts": { "$ref" : "#/definitions/opts" }
            },
            "additionalProperties": false
        },

        "error": {
            "type": "object",
            "properties": {
                "message": {
                    "type": "string",
                    "pattern": "^.{10,}|.*[Ee]rror$"
                },
                "status": {
                    "type": "number",
                    "minimum": 400,
                    "maximum": 599
                },
                "error": {
                    "type": "object",
                    "additionalProperties": true
                }
            },
            "required": [ "message", "status" ],
            "additionalProperties": false
        }
    }
}
`
)
<|endoftext|>


<commit_before>package auth

import (
	"errors"
	"net/http"
	"strings"

	log "github.com/sirupsen/logrus"

	"github.com/ivanilves/lstags/api/v1/registry/client/auth/basic"
	"github.com/ivanilves/lstags/api/v1/registry/client/auth/bearer"
	"github.com/ivanilves/lstags/api/v1/registry/client/auth/none"
)

// Token is an abstraction for aggregated token-related information we get from authentication services
type Token interface {
	Method() string
	String() string
	ExpiresIn() int
}

type authHeader string

func extractAuthHeader(hh []string) (authHeader, error) {
	if len(hh) == 0 {
		return "None realm=none", nil
	}

	h := hh[0]

	if len(strings.SplitN(h, " ", 2)) != 2 {
		return "", errors.New("Unexpected 'Www-Authenticate' header: " + h)
	}

	return authHeader(h), nil
}

func getAuthMethod(h authHeader) string {
	return strings.SplitN(string(h), " ", 2)[0]
}

func getAuthParams(h authHeader) map[string]string {
	params := make(map[string]string)

	paramString := strings.SplitN(string(h), " ", 2)[1]

	for _, keyValueString := range strings.Split(paramString, ",") {
		kv := strings.Split(keyValueString, "=")
		if len(kv) == 2 {
			params[kv[0]] = strings.Trim(kv[1], "\"")
		}
	}

	return params
}

// NewToken creates a new instance of Token in two steps:
// * detects authentication type ("Bearer", "Basic" or "None")
// * delegates actual authentication to the type-specific implementation
func NewToken(url, username, password, scope string) (Token, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	authHeader, err := extractAuthHeader(resp.Header["Www-Authenticate"])
	if err != nil {
		return nil, err
	}

	method := getAuthMethod(authHeader)
	params := getAuthParams(authHeader)

	switch method {
	case "None":
		return none.RequestToken()
	case "Basic":
		t, err := basic.RequestToken(url, username, password)
		if err != nil {
			log.Debug(err.Error())

			return none.RequestToken()
		}

		return t, nil
	case "Bearer":
		params["scope"] = scope
		return bearer.RequestToken(username, password, params)
	default:
		return nil, errors.New("Unknown authentication method: " + method)
	}
}
<commit_msg>fix: authentication issue with Nexus by comparing auth methods lower case<commit_after>package auth

import (
	"errors"
	"net/http"
	"strings"

	log "github.com/sirupsen/logrus"

	"github.com/ivanilves/lstags/api/v1/registry/client/auth/basic"
	"github.com/ivanilves/lstags/api/v1/registry/client/auth/bearer"
	"github.com/ivanilves/lstags/api/v1/registry/client/auth/none"
)

// Token is an abstraction for aggregated token-related information we get from authentication services
type Token interface {
	Method() string
	String() string
	ExpiresIn() int
}

type authHeader string

func extractAuthHeader(hh []string) (authHeader, error) {
	if len(hh) == 0 {
		return "None realm=none", nil
	}

	h := hh[0]

	if len(strings.SplitN(h, " ", 2)) != 2 {
		return "", errors.New("Unexpected 'Www-Authenticate' header: " + h)
	}

	return authHeader(h), nil
}

func getAuthMethod(h authHeader) string {
	return strings.SplitN(string(h), " ", 2)[0]
}

func getAuthParams(h authHeader) map[string]string {
	params := make(map[string]string)

	paramString := strings.SplitN(string(h), " ", 2)[1]

	for _, keyValueString := range strings.Split(paramString, ",") {
		kv := strings.Split(keyValueString, "=")
		if len(kv) == 2 {
			params[kv[0]] = strings.Trim(kv[1], "\"")
		}
	}

	return params
}

// NewToken creates a new instance of Token in two steps:
// * detects authentication type ("Bearer", "Basic" or "None")
// * delegates actual authentication to the type-specific implementation
func NewToken(url, username, password, scope string) (Token, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	authHeader, err := extractAuthHeader(resp.Header["Www-Authenticate"])
	if err != nil {
		return nil, err
	}

	method := strings.ToLower(getAuthMethod(authHeader))
	params := getAuthParams(authHeader)

	switch method {
	case "none":
		return none.RequestToken()
	case "basic":
		t, err := basic.RequestToken(url, username, password)
		if err != nil {
			log.Debug(err.Error())

			return none.RequestToken()
		}

		return t, nil
	case "bearer":
		params["scope"] = scope
		return bearer.RequestToken(username, password, params)
	default:
		return nil, errors.New("Unknown authentication method: " + method)
	}
}
<|endoftext|>


<commit_before>package concordances

import (
	"fmt"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	ReadByConceptID(ids []string) (concordances Concordances, found bool, err error)
	ReadByAuthority(authority string, ids []string) (concordances Concordances, found bool, err error)
	CheckConnectivity() error
}

// CypherDriver struct
type CypherDriver struct {
	conn neoutils.NeoConnection
	env  string
}

//NewCypherDriver instantiate driver
func NewCypherDriver(conn neoutils.NeoConnection, env string) CypherDriver {
	return CypherDriver{conn, env}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() error {
	return neoutils.Check(pcw.conn)
}

type neoReadStruct struct {
	UUID          string        `json:"UUID"`
	Types         []string      `json:"TYPES"`
	NeoIdentifier neoIdentifier `json:"IDENTIFIERS"`
	PrefUUID      string        `json:"prefUUID"`
}

type neoIdentifier struct {
	Labels []string `json:"labels"`
	Value  string   `json:"value"`
}

type neoResultStrunct struct {
	Rs []neoReadStruct
}

func (pcw CypherDriver) ReadByConceptID(identifiers []string) (concordances Concordances, found bool, err error) {
	c, f, err := pcw.readByConceptIDNewModel(identifiers)
	if !f {
		c, f, err = pcw.readByConceptIDOldModel(identifiers)
	}
	return c, f, err
}

func (pcw CypherDriver) readByConceptIDNewModel(identifiers []string) (concordances Concordances, found bool, err error) {
	results := []neoReadStruct{}
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Concept)-[:EQUIVALENT_TO]-(cn:Concept)
		WHERE p.uuid in {identifiers}
		MATCH (cn)-[:EQUIVALENT_TO]-(cnn:Concept)
		RETURN cn.prefUUID as prefUUID, cnn.uuid AS UUID, labels(cnn) AS TYPES, {labels:collect(cnn.authority), value:cnn.authorityValue} as IDENTIFIERS
		`,
		Parameters: neoism.Props{"identifiers": identifiers},
		Result:     &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	log.Info(results)

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	for _, neoCon := range results {
		log.Info(neoCon)
		// Each record is now two identifiers, one UPP and one other.
		var con = Concordance{}
		var concept = Concept{}
		concept.ID = mapper.IDURL(neoCon.PrefUUID)
		concept.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.Types, pcw.env)
		con.Concept = concept
		con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.NeoIdentifier.Labels), IdentifierValue: neoCon.NeoIdentifier.Value}
		concordances.Concordance = append(concordances.Concordance, con)

		var con2 = Concordance{}
		var concept2 = Concept{}
		concept2.ID = mapper.IDURL(neoCon.PrefUUID)
		concept2.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.Types, pcw.env)
		con2.Concept = concept
		con2.Identifier = Identifier{Authority: UP_AUTHORITY, IdentifierValue: neoCon.UUID}
		concordances.Concordance = append(concordances.Concordance, con2)
	}

	if (len(concordances.Concordance)) == 0 {
		return Concordances{}, false, nil
	}

	log.Debugf("Returning %v", concordances)
	return concordances, true, nil
}

func (pcw CypherDriver) readByConceptIDOldModel(identifiers []string) (concordances Concordances, found bool, err error) {
	results := []neoReadStruct{}
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Concept)<-[:IDENTIFIES]-(i:UPPIdentifier)
		WHERE i.value in {identifiers}
		MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier)
		RETURN p.uuid as prefUUID, p.uuid AS UUID, labels(p) AS TYPES, {labels:labels(ids), value:ids.value} as IDENTIFIERS
		`,
		Parameters: neoism.Props{"identifiers": identifiers},
		Result:     &results,
	}
	return processCypherQueryToConcordances(pcw, query, &results)
}

func (pcw CypherDriver) ReadByAuthority(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	c, f, err := pcw.readByAuthorityNewModel(authority, identifierValues)
	if !f {
		c, f, err = pcw.readByAuthorityOldModel(authority, identifierValues)
	}
	return c, f, err
}

func (pcw CypherDriver) readByAuthorityNewModel(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	log.Info("readByAuthorityNewModel")
	concordances = Concordances{}
	results := []neoReadStruct{}

	authorityProperty := mapAuthorityToAuthorityProperty(authority)
	log.Info(authorityProperty)
	if authorityProperty == "" {
		return Concordances{}, false, nil
	}

	readByAuthorityQueryStatement := `
		MATCH (p:Concept)-[:EQUIVALENT_TO]-(cn:Concept)
		WHERE p.authorityValue in {identifierValues} AND p.authority = {authority}
		RETURN cn.prefUUID as prefUUID, cn.prefUUID AS UUID, labels(cn) AS TYPES, {labels:collect(p.authority), value:p.authorityValue} as IDENTIFIERS
		`
	query := &neoism.CypherQuery{
		Statement: readByAuthorityQueryStatement,
		Parameters: neoism.Props{
			"identifierValues": identifierValues,
			"authority":        authorityProperty,
		},
		Result: &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	log.Info(results)

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	for _, neoCon := range results {
		log.Info(neoCon)
		// Each record is now two identifiers, one UPP and one other.
		var con = Concordance{}
		var concept = Concept{}
		concept.ID = mapper.IDURL(neoCon.PrefUUID)
		concept.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.Types, pcw.env)
		con.Concept = concept
		con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.NeoIdentifier.Labels), IdentifierValue: neoCon.NeoIdentifier.Value}
		concordances.Concordance = append(concordances.Concordance, con)
	}

	if (len(concordances.Concordance)) == 0 {
		return Concordances{}, false, nil
	}

	log.Debugf("Returning %v", concordances)
	return concordances, true, nil
}

func (pcw CypherDriver) readByAuthorityOldModel(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	log.Info("readByAuthorityOldModel")
	concordances = Concordances{}
	results := []neoReadStruct{}

	identifierLabel := mapAuthorityToIdentifierLabel(authority)

	if identifierLabel == "" {
		return Concordances{}, false, nil
	}

	readByAuthorityQueryStatement := fmt.Sprintf(`
		MATCH (p:Concept)<-[:IDENTIFIES]-(i:%s)
		WHERE i.value in {identifierValues}
		RETURN p.uuid as prefUUID, p.uuid AS UUID, labels(p) AS TYPES, {labels:labels(i), value:i.value} as IDENTIFIERS
		`, identifierLabel)

	query := &neoism.CypherQuery{
		Statement: readByAuthorityQueryStatement,
		Parameters: neoism.Props{
			"identifierValues": identifierValues,
			"authority":        authority,
		},
		Result: &results,
	}
	return processCypherQueryToConcordances(pcw, query, &results)
}

func processCypherQueryToConcordances(pcw CypherDriver, q *neoism.CypherQuery, results *[]neoReadStruct) (concordances Concordances, found bool, err error) {
	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{q})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", q.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	if (len(*results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = neoReadStructToConcordances(results, pcw.env)

	if (len(concordances.Concordance)) == 0 {
		return Concordances{}, false, nil
	}

	log.Debugf("Returning %v", concordances)
	return concordances, true, nil
}

func neoReadStructToConcordances(neo *[]neoReadStruct, env string) (concordances Concordances) {
	log.Debug("Running the old model")
	concordances = Concordances{
		Concordance: []Concordance{},
	}
	for _, neoCon := range *neo {
		var con = Concordance{}
		var concept = Concept{}
		concept.ID = mapper.IDURL(neoCon.UUID)
		concept.APIURL = mapper.APIURL(neoCon.UUID, neoCon.Types, env)
		con.Concept = concept
		con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.NeoIdentifier.Labels), IdentifierValue: neoCon.NeoIdentifier.Value}
		concordances.Concordance = append(concordances.Concordance, con)
	}
	return concordances
}

func mapNeoLabelsToAuthorityValue(labelNames []string) (authority string) {
	for _, label := range labelNames {
		switch label {
		// Old style node label lookup
		case TME_ID_NODE_LABEL:
			return TME_AUTHORITY
		case FS_ID_NODE_LABEL:
			return FS_AUTHORITY
		case UP_ID_NODE_LABEL:
			return UP_AUTHORITY
		case LEI_ID_NODE_LABEL:
			return LEI_AUTHORITY
		case SL_ID_NODE_LABEL:
			return SL_AUTHORITY

		// New style authority properties
		case FS_AUTHORITY_PROPERTY:
			return FS_AUTHORITY
		case UP_AUTHORITY_PROPERTY:
			return UP_AUTHORITY
		case SL_AUTHORITY_PROPERTY:
			return SL_AUTHORITY
		case TME_AUTHORITY_PROPERTY:
			return TME_AUTHORITY
		}
	}
	return ""
}

func mapAuthorityToAuthorityProperty(authority string) string {
	switch authority {
	case TME_AUTHORITY:
		return TME_AUTHORITY_PROPERTY
	case FS_AUTHORITY:
		return FS_AUTHORITY_PROPERTY
	case UP_AUTHORITY:
		return UP_AUTHORITY_PROPERTY
	case SL_AUTHORITY:
		return SL_AUTHORITY_PROPERTY
	default:
		return ""
	}
}

func mapAuthorityToIdentifierLabel(authority string) (label string) {
	switch authority {
	case TME_AUTHORITY:
		return TME_ID_NODE_LABEL
	case FS_AUTHORITY:
		return FS_ID_NODE_LABEL
	case UP_AUTHORITY:
		return UP_ID_NODE_LABEL
	case LEI_AUTHORITY:
		return LEI_ID_NODE_LABEL
	}
	return ""
}

const TME_AUTHORITY = "http://api.ft.com/system/FT-TME"
const FS_AUTHORITY = "http://api.ft.com/system/FACTSET"
const UP_AUTHORITY = "http://api.ft.com/system/UPP"
const LEI_AUTHORITY = "http://api.ft.com/system/LEI"
const SL_AUTHORITY = "http://api.ft.com/system/SMARTLOGIC"

const TME_ID_NODE_LABEL = "TMEIdentifier"
const FS_ID_NODE_LABEL = "FactsetIdentifier"
const UP_ID_NODE_LABEL = "UPPIdentifier"
const LEI_ID_NODE_LABEL = "LegalEntityIdentifier"
const SL_ID_NODE_LABEL = "SmartlogicIdentifier"

const TME_AUTHORITY_PROPERTY = "TME"
const FS_AUTHORITY_PROPERTY = "FACTSET"
const UP_AUTHORITY_PROPERTY = "UPP"
const SL_AUTHORITY_PROPERTY = "Smartlogic"
<commit_msg>Add specific query for UPP authority as the information is stored somewhere different<commit_after>package concordances

import (
	"fmt"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	ReadByConceptID(ids []string) (concordances Concordances, found bool, err error)
	ReadByAuthority(authority string, ids []string) (concordances Concordances, found bool, err error)
	CheckConnectivity() error
}

// CypherDriver struct
type CypherDriver struct {
	conn neoutils.NeoConnection
	env  string
}

//NewCypherDriver instantiate driver
func NewCypherDriver(conn neoutils.NeoConnection, env string) CypherDriver {
	return CypherDriver{conn, env}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() error {
	return neoutils.Check(pcw.conn)
}

type neoReadStruct struct {
	UUID          string        `json:"UUID"`
	Types         []string      `json:"TYPES"`
	NeoIdentifier neoIdentifier `json:"IDENTIFIERS"`
	PrefUUID      string        `json:"prefUUID"`
}

type neoIdentifier struct {
	Labels []string `json:"labels"`
	Value  string   `json:"value"`
}

type neoResultStrunct struct {
	Rs []neoReadStruct
}

func (pcw CypherDriver) ReadByConceptID(identifiers []string) (concordances Concordances, found bool, err error) {
	c, f, err := pcw.readByConceptIDNewModel(identifiers)
	if !f {
		c, f, err = pcw.readByConceptIDOldModel(identifiers)
	}
	return c, f, err
}

func (pcw CypherDriver) readByConceptIDNewModel(identifiers []string) (concordances Concordances, found bool, err error) {
	results := []neoReadStruct{}
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Concept)-[:EQUIVALENT_TO]-(cn:Concept)
		WHERE p.uuid in {identifiers}
		MATCH (cn)-[:EQUIVALENT_TO]-(cnn:Concept)
		RETURN cn.prefUUID as prefUUID, cnn.uuid AS UUID, labels(cnn) AS TYPES, {labels:collect(cnn.authority), value:cnn.authorityValue} as IDENTIFIERS
		`,
		Parameters: neoism.Props{"identifiers": identifiers},
		Result:     &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	log.Info(results)

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	for _, neoCon := range results {
		log.Info(neoCon)
		// Each record is now two identifiers, one UPP and one other.
		var con = Concordance{}
		var concept = Concept{}
		concept.ID = mapper.IDURL(neoCon.PrefUUID)
		concept.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.Types, pcw.env)
		con.Concept = concept
		con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.NeoIdentifier.Labels), IdentifierValue: neoCon.NeoIdentifier.Value}
		concordances.Concordance = append(concordances.Concordance, con)

		var con2 = Concordance{}
		var concept2 = Concept{}
		concept2.ID = mapper.IDURL(neoCon.PrefUUID)
		concept2.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.Types, pcw.env)
		con2.Concept = concept
		con2.Identifier = Identifier{Authority: UP_AUTHORITY, IdentifierValue: neoCon.UUID}
		concordances.Concordance = append(concordances.Concordance, con2)
	}

	if (len(concordances.Concordance)) == 0 {
		return Concordances{}, false, nil
	}

	log.Debugf("Returning %v", concordances)
	return concordances, true, nil
}

func (pcw CypherDriver) readByConceptIDOldModel(identifiers []string) (concordances Concordances, found bool, err error) {
	results := []neoReadStruct{}
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Concept)<-[:IDENTIFIES]-(i:UPPIdentifier)
		WHERE i.value in {identifiers}
		MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier)
		RETURN p.uuid as prefUUID, p.uuid AS UUID, labels(p) AS TYPES, {labels:labels(ids), value:ids.value} as IDENTIFIERS
		`,
		Parameters: neoism.Props{"identifiers": identifiers},
		Result:     &results,
	}
	return processCypherQueryToConcordances(pcw, query, &results)
}

func (pcw CypherDriver) ReadByAuthority(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	c, f, err := pcw.readByAuthorityNewModel(authority, identifierValues)
	if !f {
		c, f, err = pcw.readByAuthorityOldModel(authority, identifierValues)
	}
	return c, f, err
}

func (pcw CypherDriver) readByAuthorityNewModel(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	log.Info("readByAuthorityNewModel")
	concordances = Concordances{}
	results := []neoReadStruct{}

	authorityProperty := mapAuthorityToAuthorityProperty(authority)
	log.Info(authorityProperty)
	if authorityProperty == "" {
		return Concordances{}, false, nil
	}

	readByAuthorityQueryStatement := `
		MATCH (p:Concept)-[:EQUIVALENT_TO]-(cn:Concept)
		WHERE p.authorityValue in {identifierValues} AND p.authority = {authority}
		RETURN cn.prefUUID as prefUUID, cn.prefUUID AS UUID, labels(cn) AS TYPES, {labels:collect(p.authority), value:p.authorityValue} as IDENTIFIERS
		`
	if authorityProperty == "UPP" {
		readByAuthorityQueryStatement = `
			MATCH (p:Concept)-[:EQUIVALENT_TO]-(cn:Concept)
			WHERE p.uuid in {identifierValues}
			RETURN cn.prefUUID as prefUUID, cn.prefUUID AS UUID, labels(cn) AS TYPES, {labels:collect(p.authority), value:p.authorityValue} as IDENTIFIERS
		`
	}

	query := &neoism.CypherQuery{
		Statement: readByAuthorityQueryStatement,
		Parameters: neoism.Props{
			"identifierValues": identifierValues,
			"authority":        authorityProperty,
		},
		Result: &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	log.Info(results)

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	for _, neoCon := range results {
		log.Info(neoCon)
		// Each record is now two identifiers, one UPP and one other.
		var con = Concordance{}
		var concept = Concept{}
		concept.ID = mapper.IDURL(neoCon.PrefUUID)
		concept.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.Types, pcw.env)
		con.Concept = concept
		con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.NeoIdentifier.Labels), IdentifierValue: neoCon.NeoIdentifier.Value}
		concordances.Concordance = append(concordances.Concordance, con)
	}

	if (len(concordances.Concordance)) == 0 {
		return Concordances{}, false, nil
	}

	log.Debugf("Returning %v", concordances)
	return concordances, true, nil
}

func (pcw CypherDriver) readByAuthorityOldModel(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	log.Info("readByAuthorityOldModel")
	concordances = Concordances{}
	results := []neoReadStruct{}

	identifierLabel := mapAuthorityToIdentifierLabel(authority)

	if identifierLabel == "" {
		return Concordances{}, false, nil
	}

	readByAuthorityQueryStatement := fmt.Sprintf(`
		MATCH (p:Concept)<-[:IDENTIFIES]-(i:%s)
		WHERE i.value in {identifierValues}
		RETURN p.uuid as prefUUID, p.uuid AS UUID, labels(p) AS TYPES, {labels:labels(i), value:i.value} as IDENTIFIERS
		`, identifierLabel)

	query := &neoism.CypherQuery{
		Statement: readByAuthorityQueryStatement,
		Parameters: neoism.Props{
			"identifierValues": identifierValues,
			"authority":        authority,
		},
		Result: &results,
	}
	return processCypherQueryToConcordances(pcw, query, &results)
}

func processCypherQueryToConcordances(pcw CypherDriver, q *neoism.CypherQuery, results *[]neoReadStruct) (concordances Concordances, found bool, err error) {
	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{q})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", q.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	if (len(*results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = neoReadStructToConcordances(results, pcw.env)

	if (len(concordances.Concordance)) == 0 {
		return Concordances{}, false, nil
	}

	log.Debugf("Returning %v", concordances)
	return concordances, true, nil
}

func neoReadStructToConcordances(neo *[]neoReadStruct, env string) (concordances Concordances) {
	log.Debug("Running the old model")
	concordances = Concordances{
		Concordance: []Concordance{},
	}
	for _, neoCon := range *neo {
		var con = Concordance{}
		var concept = Concept{}
		concept.ID = mapper.IDURL(neoCon.UUID)
		concept.APIURL = mapper.APIURL(neoCon.UUID, neoCon.Types, env)
		con.Concept = concept
		con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.NeoIdentifier.Labels), IdentifierValue: neoCon.NeoIdentifier.Value}
		concordances.Concordance = append(concordances.Concordance, con)
	}
	return concordances
}

func mapNeoLabelsToAuthorityValue(labelNames []string) (authority string) {
	for _, label := range labelNames {
		switch label {
		// Old style node label lookup
		case TME_ID_NODE_LABEL:
			return TME_AUTHORITY
		case FS_ID_NODE_LABEL:
			return FS_AUTHORITY
		case UP_ID_NODE_LABEL:
			return UP_AUTHORITY
		case LEI_ID_NODE_LABEL:
			return LEI_AUTHORITY
		case SL_ID_NODE_LABEL:
			return SL_AUTHORITY

		// New style authority properties
		case FS_AUTHORITY_PROPERTY:
			return FS_AUTHORITY
		case UP_AUTHORITY_PROPERTY:
			return UP_AUTHORITY
		case SL_AUTHORITY_PROPERTY:
			return SL_AUTHORITY
		case TME_AUTHORITY_PROPERTY:
			return TME_AUTHORITY
		}
	}
	return ""
}

func mapAuthorityToAuthorityProperty(authority string) string {
	switch authority {
	case TME_AUTHORITY:
		return TME_AUTHORITY_PROPERTY
	case FS_AUTHORITY:
		return FS_AUTHORITY_PROPERTY
	case UP_AUTHORITY:
		return UP_AUTHORITY_PROPERTY
	case SL_AUTHORITY:
		return SL_AUTHORITY_PROPERTY
	default:
		return ""
	}
}

func mapAuthorityToIdentifierLabel(authority string) (label string) {
	switch authority {
	case TME_AUTHORITY:
		return TME_ID_NODE_LABEL
	case FS_AUTHORITY:
		return FS_ID_NODE_LABEL
	case UP_AUTHORITY:
		return UP_ID_NODE_LABEL
	case LEI_AUTHORITY:
		return LEI_ID_NODE_LABEL
	}
	return ""
}

const TME_AUTHORITY = "http://api.ft.com/system/FT-TME"
const FS_AUTHORITY = "http://api.ft.com/system/FACTSET"
const UP_AUTHORITY = "http://api.ft.com/system/UPP"
const LEI_AUTHORITY = "http://api.ft.com/system/LEI"
const SL_AUTHORITY = "http://api.ft.com/system/SMARTLOGIC"

const TME_ID_NODE_LABEL = "TMEIdentifier"
const FS_ID_NODE_LABEL = "FactsetIdentifier"
const UP_ID_NODE_LABEL = "UPPIdentifier"
const LEI_ID_NODE_LABEL = "LegalEntityIdentifier"
const SL_ID_NODE_LABEL = "SmartlogicIdentifier"

const TME_AUTHORITY_PROPERTY = "TME"
const FS_AUTHORITY_PROPERTY = "FACTSET"
const UP_AUTHORITY_PROPERTY = "UPP"
const SL_AUTHORITY_PROPERTY = "Smartlogic"
<|endoftext|>


<commit_before>package golog

import (
	"fmt"
	"goprotobuf.googlecode.com/hg/proto"
	"time"
)

const (
	INFO = iota
	WARNING
	ERROR
	FATAL
)

var levelStrings []string = []string{"I", "W", "E", "F"}

type LevelLogger struct {
	// TODO Can we get away with just a Logger?
	FailLogger
}

// Formats the message with metadata. The format is:
// LEVEL HH:MM:SS:NANOSC LOC] MESSAGE
func makeLogClosure(level int, msg func() string) func() *LogMessage {
	// Evaluate this early.
	ns := time.Nanoseconds()

	return func() *LogMessage {
		return &LogMessage{
			Level: proto.Int(level),
			Message: proto.String(msg()),
			Nanoseconds: proto.Int64(ns),
		}
	}
}

func (l *LevelLogger) logCommon(level int, closure func() string) {
	l.Log(level, makeLogClosure(level, closure))
}

func (l *LevelLogger) Info(vals ...interface{}) {
	l.logCommon(INFO, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Infof(f string, args ...interface{}) {
	l.logCommon(INFO, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Infoc(closure func() string) {
	l.logCommon(INFO, closure)
}

func (l *LevelLogger) Warning(vals ...interface{}) {
	l.logCommon(WARNING, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Warningf(f string, args ...interface{}) {
	l.logCommon(WARNING, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Warningc(closure func() string) {
	l.logCommon(WARNING, closure)
}

func (l *LevelLogger) Error(vals ...interface{}) {
	l.logCommon(ERROR, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Errorf(f string, args ...interface{}) {
	l.logCommon(ERROR, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Errorc(closure func() string) {
	l.logCommon(ERROR, closure)
}

func (l *LevelLogger) Fatal(vals ...interface{}) {
	l.logCommon(FATAL, func() string { return fmt.Sprint(vals...) })
	l.FailNow()
}

func (l *LevelLogger) Fatalf(f string, args ...interface{}) {
	l.logCommon(FATAL, func() string { return fmt.Sprintf(f, args...) })
	l.FailNow()
}

func (l *LevelLogger) Fatalc(closure func() string) {
	l.logCommon(FATAL, closure)
	l.FailNow()
}
<commit_msg>Add Constructors for LevelLogger<commit_after>package golog

import (
	"fmt"
	"goprotobuf.googlecode.com/hg/proto"
	"time"
)

const (
	INFO = iota
	WARNING
	ERROR
	FATAL
)

var levelStrings []string = []string{"I", "W", "E", "F"}

type LevelLogger struct {
	// TODO Can we get away with just a Logger?
	FailLogger
}

func NewLevelLogger(f FailLogger) *LevelLogger {
	return &LevelLogger{f}
}

func NewDefaultLevelLogger() *LevelLogger {
	return &LevelLogger{NewDefaultFailLogger()}
}

// Formats the message with metadata. The format is:
// LEVEL HH:MM:SS:NANOSC LOC] MESSAGE
func makeLogClosure(level int, msg func() string) func() *LogMessage {
	// Evaluate this early.
	ns := time.Nanoseconds()

	return func() *LogMessage {
		return &LogMessage{
			Level: proto.Int(level),
			Message: proto.String(msg()),
			Nanoseconds: proto.Int64(ns),
		}
	}
}

func (l *LevelLogger) logCommon(level int, closure func() string) {
	l.Log(level, makeLogClosure(level, closure))
}

func (l *LevelLogger) Info(vals ...interface{}) {
	l.logCommon(INFO, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Infof(f string, args ...interface{}) {
	l.logCommon(INFO, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Infoc(closure func() string) {
	l.logCommon(INFO, closure)
}

func (l *LevelLogger) Warning(vals ...interface{}) {
	l.logCommon(WARNING, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Warningf(f string, args ...interface{}) {
	l.logCommon(WARNING, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Warningc(closure func() string) {
	l.logCommon(WARNING, closure)
}

func (l *LevelLogger) Error(vals ...interface{}) {
	l.logCommon(ERROR, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Errorf(f string, args ...interface{}) {
	l.logCommon(ERROR, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Errorc(closure func() string) {
	l.logCommon(ERROR, closure)
}

func (l *LevelLogger) Fatal(vals ...interface{}) {
	l.logCommon(FATAL, func() string { return fmt.Sprint(vals...) })
	l.FailNow()
}

func (l *LevelLogger) Fatalf(f string, args ...interface{}) {
	l.logCommon(FATAL, func() string { return fmt.Sprintf(f, args...) })
	l.FailNow()
}

func (l *LevelLogger) Fatalc(closure func() string) {
	l.logCommon(FATAL, closure)
	l.FailNow()
}
<|endoftext|>