<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package lsp

import (
	"bytes"
	"context"
	"fmt"

	"golang.org/x/tools/internal/jsonrpc2"
	"golang.org/x/tools/internal/lsp/protocol"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/lsp/telemetry/trace"
	"golang.org/x/tools/internal/lsp/xlog"
	"golang.org/x/tools/internal/span"
)

func (s *Server) didOpen(ctx context.Context, params *protocol.DidOpenTextDocumentParams) error {
	uri := span.NewURI(params.TextDocument.URI)
	text := []byte(params.TextDocument.Text)

	// Confirm that the file's language ID is related to Go.
	fileKind := source.DetectLanguage(params.TextDocument.LanguageID, uri.Filename())

	// Open the file.
	s.session.DidOpen(ctx, uri, fileKind, text)

	// Run diagnostics on the newly-changed file.
	view := s.session.ViewOf(uri)
	go func() {
		ctx := view.BackgroundContext()
		s.Diagnostics(ctx, view, uri)
	}()
	return nil
}

func (s *Server) didChange(ctx context.Context, params *protocol.DidChangeTextDocumentParams) error {
	if len(params.ContentChanges) < 1 {
		return jsonrpc2.NewErrorf(jsonrpc2.CodeInternalError, "no content changes provided")
	}

	uri := span.NewURI(params.TextDocument.URI)

	// Check if the client sent the full content of the file.
	// We accept a full content change even if the server expected incremental changes.
	text, isFullChange := fullChange(params.ContentChanges)

	// We only accept an incremental change if the server expected it.
	if !isFullChange {
		switch s.textDocumentSyncKind {
		case protocol.Full:
			return fmt.Errorf("expected a full content change, received incremental changes for %s", uri)
		case protocol.Incremental:
			// Determine the new file content.
			var err error
			text, err = s.applyChanges(ctx, uri, params.ContentChanges)
			if err != nil {
				return err
			}
		}
	}
	// Cache the new file content and send fresh diagnostics.
	view := s.session.ViewOf(uri)
	if err := view.SetContent(ctx, uri, []byte(text)); err != nil {
		return err
	}
	// Run diagnostics on the newly-changed file.
	go func() {
		ctx := view.BackgroundContext()
		//TODO: connect the remote span?
		ctx, done := trace.StartSpan(ctx, "lsp:background-worker")
		defer done()
		s.Diagnostics(ctx, view, uri)
	}()
	return nil
}

func fullChange(changes []protocol.TextDocumentContentChangeEvent) (string, bool) {
	if len(changes) > 1 {
		return "", false
	}
	// The length of the changes must be 1 at this point.
	if changes[0].Range == nil && changes[0].RangeLength == 0 {
		return changes[0].Text, true
	}
	return "", false
}

func (s *Server) applyChanges(ctx context.Context, uri span.URI, changes []protocol.TextDocumentContentChangeEvent) (string, error) {
	content, _, err := s.session.GetFile(uri).Read(ctx)
	if err != nil {
		return "", jsonrpc2.NewErrorf(jsonrpc2.CodeInternalError, "file not found")
	}
	fset := s.session.Cache().FileSet()
	for _, change := range changes {
		// Update column mapper along with the content.
		m := protocol.NewColumnMapper(uri, uri.Filename(), fset, nil, content)

		spn, err := m.RangeSpan(*change.Range)
		if err != nil {
			return "", err
		}
		if !spn.HasOffset() {
			return "", jsonrpc2.NewErrorf(jsonrpc2.CodeInternalError, "invalid range for content change")
		}
		start, end := spn.Start().Offset(), spn.End().Offset()
		if end < start {
			return "", jsonrpc2.NewErrorf(jsonrpc2.CodeInternalError, "invalid range for content change")
		}
		var buf bytes.Buffer
		buf.Write(content[:start])
		buf.WriteString(change.Text)
		buf.Write(content[end:])
		content = buf.Bytes()
	}
	return string(content), nil
}

func (s *Server) didSave(ctx context.Context, params *protocol.DidSaveTextDocumentParams) error {
	s.session.DidSave(span.NewURI(params.TextDocument.URI))
	return nil
}

func (s *Server) didClose(ctx context.Context, params *protocol.DidCloseTextDocumentParams) error {
	uri := span.NewURI(params.TextDocument.URI)
	s.session.DidClose(uri)
	view := s.session.ViewOf(uri)
	if err := view.SetContent(ctx, uri, nil); err != nil {
		return err
	}
	clear := []span.URI{uri} // by default, clear the closed URI
	defer func() {
		for _, uri := range clear {
			if err := s.publishDiagnostics(ctx, view, uri, []source.Diagnostic{}); err != nil {
				xlog.Errorf(ctx, "failed to clear diagnostics for %s: %v", uri, err)
			}
		}
	}()
	// If the current file was the only open file for its package,
	// clear out all diagnostics for the package.
	f, err := view.GetFile(ctx, uri)
	if err != nil {
		xlog.Errorf(ctx, "no file for %s: %v", uri, err)
		return nil
	}
	// For non-Go files, don't return any diagnostics.
	gof, ok := f.(source.GoFile)
	if !ok {
		xlog.Errorf(ctx, "closing a non-Go file, no diagnostics to clear")
		return nil
	}
	pkg := gof.GetPackage(ctx)
	if pkg == nil {
		xlog.Errorf(ctx, "no package available for %s", uri)
		return nil
	}
	for _, filename := range pkg.GetFilenames() {
		// If other files from this package are open, don't clear.
		if s.session.IsOpen(span.NewURI(filename)) {
			clear = nil
			return nil
		}
		clear = append(clear, span.FileURI(filename))
	}
	return nil
}
<commit_msg>internal/lsp: add tracing to the didOpen call<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package lsp

import (
	"bytes"
	"context"
	"fmt"

	"golang.org/x/tools/internal/jsonrpc2"
	"golang.org/x/tools/internal/lsp/protocol"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/lsp/telemetry/trace"
	"golang.org/x/tools/internal/lsp/xlog"
	"golang.org/x/tools/internal/span"
)

func (s *Server) didOpen(ctx context.Context, params *protocol.DidOpenTextDocumentParams) error {
	uri := span.NewURI(params.TextDocument.URI)
	text := []byte(params.TextDocument.Text)

	// Confirm that the file's language ID is related to Go.
	fileKind := source.DetectLanguage(params.TextDocument.LanguageID, uri.Filename())

	// Open the file.
	s.session.DidOpen(ctx, uri, fileKind, text)

	// Run diagnostics on the newly-changed file.
	view := s.session.ViewOf(uri)
	go func() {
		ctx := view.BackgroundContext()
		ctx, done := trace.StartSpan(ctx, "lsp:background-worker")
		defer done()
		s.Diagnostics(ctx, view, uri)
	}()
	return nil
}

func (s *Server) didChange(ctx context.Context, params *protocol.DidChangeTextDocumentParams) error {
	if len(params.ContentChanges) < 1 {
		return jsonrpc2.NewErrorf(jsonrpc2.CodeInternalError, "no content changes provided")
	}

	uri := span.NewURI(params.TextDocument.URI)

	// Check if the client sent the full content of the file.
	// We accept a full content change even if the server expected incremental changes.
	text, isFullChange := fullChange(params.ContentChanges)

	// We only accept an incremental change if the server expected it.
	if !isFullChange {
		switch s.textDocumentSyncKind {
		case protocol.Full:
			return fmt.Errorf("expected a full content change, received incremental changes for %s", uri)
		case protocol.Incremental:
			// Determine the new file content.
			var err error
			text, err = s.applyChanges(ctx, uri, params.ContentChanges)
			if err != nil {
				return err
			}
		}
	}
	// Cache the new file content and send fresh diagnostics.
	view := s.session.ViewOf(uri)
	if err := view.SetContent(ctx, uri, []byte(text)); err != nil {
		return err
	}
	// Run diagnostics on the newly-changed file.
	go func() {
		ctx := view.BackgroundContext()
		ctx, done := trace.StartSpan(ctx, "lsp:background-worker")
		defer done()
		s.Diagnostics(ctx, view, uri)
	}()
	return nil
}

func fullChange(changes []protocol.TextDocumentContentChangeEvent) (string, bool) {
	if len(changes) > 1 {
		return "", false
	}
	// The length of the changes must be 1 at this point.
	if changes[0].Range == nil && changes[0].RangeLength == 0 {
		return changes[0].Text, true
	}
	return "", false
}

func (s *Server) applyChanges(ctx context.Context, uri span.URI, changes []protocol.TextDocumentContentChangeEvent) (string, error) {
	content, _, err := s.session.GetFile(uri).Read(ctx)
	if err != nil {
		return "", jsonrpc2.NewErrorf(jsonrpc2.CodeInternalError, "file not found")
	}
	fset := s.session.Cache().FileSet()
	for _, change := range changes {
		// Update column mapper along with the content.
		m := protocol.NewColumnMapper(uri, uri.Filename(), fset, nil, content)

		spn, err := m.RangeSpan(*change.Range)
		if err != nil {
			return "", err
		}
		if !spn.HasOffset() {
			return "", jsonrpc2.NewErrorf(jsonrpc2.CodeInternalError, "invalid range for content change")
		}
		start, end := spn.Start().Offset(), spn.End().Offset()
		if end < start {
			return "", jsonrpc2.NewErrorf(jsonrpc2.CodeInternalError, "invalid range for content change")
		}
		var buf bytes.Buffer
		buf.Write(content[:start])
		buf.WriteString(change.Text)
		buf.Write(content[end:])
		content = buf.Bytes()
	}
	return string(content), nil
}

func (s *Server) didSave(ctx context.Context, params *protocol.DidSaveTextDocumentParams) error {
	s.session.DidSave(span.NewURI(params.TextDocument.URI))
	return nil
}

func (s *Server) didClose(ctx context.Context, params *protocol.DidCloseTextDocumentParams) error {
	uri := span.NewURI(params.TextDocument.URI)
	s.session.DidClose(uri)
	view := s.session.ViewOf(uri)
	if err := view.SetContent(ctx, uri, nil); err != nil {
		return err
	}
	clear := []span.URI{uri} // by default, clear the closed URI
	defer func() {
		for _, uri := range clear {
			if err := s.publishDiagnostics(ctx, view, uri, []source.Diagnostic{}); err != nil {
				xlog.Errorf(ctx, "failed to clear diagnostics for %s: %v", uri, err)
			}
		}
	}()
	// If the current file was the only open file for its package,
	// clear out all diagnostics for the package.
	f, err := view.GetFile(ctx, uri)
	if err != nil {
		xlog.Errorf(ctx, "no file for %s: %v", uri, err)
		return nil
	}
	// For non-Go files, don't return any diagnostics.
	gof, ok := f.(source.GoFile)
	if !ok {
		xlog.Errorf(ctx, "closing a non-Go file, no diagnostics to clear")
		return nil
	}
	pkg := gof.GetPackage(ctx)
	if pkg == nil {
		xlog.Errorf(ctx, "no package available for %s", uri)
		return nil
	}
	for _, filename := range pkg.GetFilenames() {
		// If other files from this package are open, don't clear.
		if s.session.IsOpen(span.NewURI(filename)) {
			clear = nil
			return nil
		}
		clear = append(clear, span.FileURI(filename))
	}
	return nil
}
<|endoftext|>


<commit_before>package html

import (
	"encoding/json"
	"strconv"

	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/dop251/goja"
	"github.com/serenize/snaker"

	gohtml "golang.org/x/net/html"
)

func attrToProperty(s string) string {
	idx := strings.Index(s, "-")

	if idx == -1 {
		return s
	}

	return s[0:idx] + snaker.SnakeToCamel(strings.Replace(s[idx+1:], "-", "_", -1))
}

func propertyToAttr(attrName string) string {
	return strings.Replace(snaker.CamelToSnake(attrName), "_", "-", -1)
}

func namespaceURI(prefix string) string {
	switch prefix {
	case "svg":
		return "http://www.w3.org/2000/svg"
	case "math":
		return "http://www.w3.org/1998/Math/MathML"
	default:
		return "http://www.w3.org/1999/xhtml"
	}
}

func valueOrHTML(s *goquery.Selection) string {
	if val, exists := s.Attr("value"); exists {
		return val
	}

	if val, err := s.Html(); err == nil {
		return val
	}

	return ""
}

func getHtmlAttr(node *gohtml.Node, name string) *gohtml.Attribute {
	for i := 0; i < len(node.Attr); i++ {
		if node.Attr[i].Key == name {
			return &node.Attr[i]
		}
	}

	return nil
}

func elemList(s Selection) (items []goja.Value) {
	items = make([]goja.Value, s.Size())
	for i := 0; i < s.Size(); i++ {
		items[i] = selToElement(s.Eq(i))
	}
	return items
}

func nodeToElement(e Element, node *gohtml.Node) goja.Value {
	// Goquery does not expose a way to build a goquery.Selection with an arbitrary html.Node.
	// Workaround by adding a node to an empty Selection
	emptySel := e.sel.emptySelection()
	emptySel.sel.Nodes = append(emptySel.sel.Nodes, node)

	return selToElement(emptySel)
}

func selToElement(sel Selection) goja.Value {
	if sel.sel.Length() == 0 {
		return goja.Undefined()
	}

	elem := Element{sel.sel.Nodes[0], &sel}
	switch elem.node.Data {
	case "a":
		return sel.rt.ToValue(AnchorElement{HrefElement{elem}})

	case "area":
		return sel.rt.ToValue(AreaElement{HrefElement{elem}})

	case "base":
		return sel.rt.ToValue(BaseElement{elem})

	case "button":
		return sel.rt.ToValue(ButtonElement{elem})

	default:
		return sel.rt.ToValue(elem)
	}
}

// Try to read numeric values in data- attributes.
// Return numeric value when the representation is unchanged by conversion to float and back.
// Other potentially numeric values (ie "101.00" "1E02") remain as strings.
func toNumeric(val string) (float64, bool) {
	if fltVal, err := strconv.ParseFloat(val, 64); err != nil {
		return 0, false
	} else if repr := strconv.FormatFloat(fltVal, 'f', -1, 64); repr == val {
		return fltVal, true
	} else {
		return 0, false
	}
}

func convertDataAttrVal(val string) interface{} {
	if len(val) == 0 {
		return goja.Undefined()
	} else if val[0] == '{' || val[0] == '[' {
		var subdata interface{}

		err := json.Unmarshal([]byte(val), &subdata)
		if err == nil {
			return subdata
		} else {
			return val
		}
	} else {
		switch val {
		case "true":
			return true

		case "false":
			return false

		case "null":
			return goja.Undefined()

		case "undefined":
			return goja.Undefined()

		default:
			if fltVal, isOk := toNumeric(val); isOk {
				return fltVal
			} else {
				return val
			}
		}
	}
}

func (e Element) ownerFormSel() (*goquery.Selection, bool) {
	prtForm := e.sel.sel.Closest("form")
	if prtForm.Length() > 0 {
		return prtForm, true
	}

	formId := e.attrAsString("form")
	if formId == "" {
		return nil, false
	}

	findForm := e.sel.sel.Parents().Last().Find("#" + formId)
	if findForm.Length() == 0 {
		return nil, false
	}

	return findForm, true
}
<commit_msg>Remove some code which will be generated<commit_after>package html

import (
	"encoding/json"
	"strconv"

	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/dop251/goja"
	"github.com/serenize/snaker"

	gohtml "golang.org/x/net/html"
)

func attrToProperty(s string) string {
	idx := strings.Index(s, "-")

	if idx == -1 {
		return s
	}

	return s[0:idx] + snaker.SnakeToCamel(strings.Replace(s[idx+1:], "-", "_", -1))
}

func propertyToAttr(attrName string) string {
	return strings.Replace(snaker.CamelToSnake(attrName), "_", "-", -1)
}

func namespaceURI(prefix string) string {
	switch prefix {
	case "svg":
		return "http://www.w3.org/2000/svg"
	case "math":
		return "http://www.w3.org/1998/Math/MathML"
	default:
		return "http://www.w3.org/1999/xhtml"
	}
}

func valueOrHTML(s *goquery.Selection) string {
	if val, exists := s.Attr("value"); exists {
		return val
	}

	if val, err := s.Html(); err == nil {
		return val
	}

	return ""
}

func getHtmlAttr(node *gohtml.Node, name string) *gohtml.Attribute {
	for i := 0; i < len(node.Attr); i++ {
		if node.Attr[i].Key == name {
			return &node.Attr[i]
		}
	}

	return nil
}

func elemList(s Selection) (items []goja.Value) {
	items = make([]goja.Value, s.Size())
	for i := 0; i < s.Size(); i++ {
		items[i] = selToElement(s.Eq(i))
	}
	return items
}

func nodeToElement(e Element, node *gohtml.Node) goja.Value {
	// Goquery does not expose a way to build a goquery.Selection with an arbitrary html.Node.
	// Workaround by adding a node to an empty Selection
	emptySel := e.sel.emptySelection()
	emptySel.sel.Nodes = append(emptySel.sel.Nodes, node)

	return selToElement(emptySel)
}

// Try to read numeric values in data- attributes.
// Return numeric value when the representation is unchanged by conversion to float and back.
// Other potentially numeric values (ie "101.00" "1E02") remain as strings.
func toNumeric(val string) (float64, bool) {
	if fltVal, err := strconv.ParseFloat(val, 64); err != nil {
		return 0, false
	} else if repr := strconv.FormatFloat(fltVal, 'f', -1, 64); repr == val {
		return fltVal, true
	} else {
		return 0, false
	}
}

func convertDataAttrVal(val string) interface{} {
	if len(val) == 0 {
		return goja.Undefined()
	} else if val[0] == '{' || val[0] == '[' {
		var subdata interface{}

		err := json.Unmarshal([]byte(val), &subdata)
		if err == nil {
			return subdata
		} else {
			return val
		}
	} else {
		switch val {
		case "true":
			return true

		case "false":
			return false

		case "null":
			return goja.Undefined()

		case "undefined":
			return goja.Undefined()

		default:
			if fltVal, isOk := toNumeric(val); isOk {
				return fltVal
			} else {
				return val
			}
		}
	}
}

func (e Element) ownerFormSel() (*goquery.Selection, bool) {
	prtForm := e.sel.sel.Closest("form")
	if prtForm.Length() > 0 {
		return prtForm, true
	}

	formId := e.attrAsString("form")
	if formId == "" {
		return nil, false
	}

	findForm := e.sel.sel.Parents().Last().Find("#" + formId)
	if findForm.Length() == 0 {
		return nil, false
	}

	return findForm, true
}
<|endoftext|>


<commit_before><commit_msg>Make 'yo generate overwrite yo_db file (#61)<commit_after><|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.675"
<commit_msg>fnserver: 0.3.676 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.676"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.474"
<commit_msg>fnserver: 0.3.475 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.475"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.580"
<commit_msg>fnserver: 0.3.581 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.581"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.598"
<commit_msg>fnserver: 0.3.599 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.599"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.189"
<commit_msg>fn-server: 0.3.190 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.190"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.167"
<commit_msg>functions: 0.3.168 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.168"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.366"
<commit_msg>fnserver: 0.3.367 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.367"
<|endoftext|>


<commit_before>// Copyright (c) 2018 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tls

import (
	"crypto/tls"
	"crypto/x509"
	"io/ioutil"
)

//------------------------------------------------------------------------------

// Config contains configuration params for TLS.
type Config struct {
	Enabled            bool              `json:"enabled" yaml:"enabled"`
	RootCAsFile        string            `json:"cas_file" yaml:"cas_file"`
	InsecureSkipVerify bool              `json:"skip_cert_verify" yaml:"skip_cert_verify"`
	ClientCertificates []tls.Certificate `json:"client_certs" yaml:"client_certs"`
}

// NewConfig creates a new Config with default values.
func NewConfig() Config {
	return Config{
		Enabled:            false,
		RootCAsFile:        "",
		InsecureSkipVerify: false,
	}
}

//------------------------------------------------------------------------------

// Get returns a valid *tls.Config based on the configuration values of Config.
func (c *Config) Get() (*tls.Config, error) {
	var rootCAs *x509.CertPool
	if len(c.RootCAsFile) > 0 {
		caCert, err := ioutil.ReadFile(c.RootCAsFile)
		if err != nil {
			return nil, err
		}

		rootCAs = x509.NewCertPool()
		rootCAs.AppendCertsFromPEM(caCert)
	}
	return &tls.Config{
		InsecureSkipVerify: c.InsecureSkipVerify,
		RootCAs:            rootCAs,
		Certificates:       c.ClientCertificates,
	}, nil
}

//------------------------------------------------------------------------------
<commit_msg>use cert and key file paths directly for Kafka TLS config<commit_after>// Copyright (c) 2018 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tls

import (
	"crypto/tls"
	"crypto/x509"
	"io/ioutil"
)

//------------------------------------------------------------------------------

// Config contains configuration params for TLS.
type Config struct {
	Enabled            bool   `json:"enabled" yaml:"enabled"`
	RootCAsFile        string `json:"cas_file" yaml:"cas_file"`
	InsecureSkipVerify bool   `json:"skip_cert_verify" yaml:"skip_cert_verify"`
	ClientCertificates []struct {
		CertFile string `json:"cert_file" yaml:"cert_file"`
		KeyFile  string `json:"key_file" yaml:"key_file"`
	} `json:"client_certs" yaml:"client_certs"`
}

// NewConfig creates a new Config with default values.
func NewConfig() Config {
	return Config{
		Enabled:            false,
		RootCAsFile:        "",
		InsecureSkipVerify: false,
	}
}

//------------------------------------------------------------------------------

// Get returns a valid *tls.Config based on the configuration values of Config.
func (c *Config) Get() (*tls.Config, error) {
	var rootCAs *x509.CertPool
	if len(c.RootCAsFile) > 0 {
		caCert, err := ioutil.ReadFile(c.RootCAsFile)
		if err != nil {
			return nil, err
		}

		rootCAs = x509.NewCertPool()
		rootCAs.AppendCertsFromPEM(caCert)
	}

	clientCerts := []tls.Certificate{}

	for _, pair := range c.ClientCertificates {
		keyPair, err := tls.LoadX509KeyPair(pair.CertFile, pair.KeyFile)
		if nil != err {
			return nil, err
		}
		clientCerts = append(clientCerts, keyPair)
	}

	return &tls.Config{
		InsecureSkipVerify: c.InsecureSkipVerify,
		RootCAs:            rootCAs,
		Certificates:       clientCerts,
	}, nil
}

//------------------------------------------------------------------------------
<|endoftext|>


<commit_before>// Paquete encargado de obtener un token de Google en base a un código de autorización
// Documentación: https://developers.google.com/accounts/docs/OAuth2WebServer
package server

import (
    "encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
)

const (
	GOOGLE_API_OAUTH = "https://www.googleapis.com/oauth2/v3/token"
	GRANT_TYPE       = "authorization_code"
)

// Función encargada de solicitar token
func getToken(code, clientId, clientSecret, redirectUri string, oauth *OAuth2WebServer) error {
	resp, err := http.PostForm(
		GOOGLE_API_OAUTH,
		url.Values{
			"code":          {code},
			"client_id":     {clientId},
			"client_secret": {clientSecret},
			"redirect_uri":  {redirectUri},
			"grant_type":    {GRANT_TYPE},
		},
	)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	err = json.Unmarshal(body, &oauth.JsonResponse)
	if err != nil {
		return err
	}
	oauth.Code = code

	return nil
}

// Función encargada de solicitar token
func GetToken(code, clientId, clientSecret, redirectUri string) (*OAuth2WebServer, error) {
	oauth := &OAuth2WebServer{}
	err := getToken(code, clientId, clientSecret, redirectUri, oauth)
	return oauth, err
}
<commit_msg>UPDATE URL API SERVER<commit_after>// Paquete encargado de obtener un token de Google en base a un código de autorización
// Documentación: https://developers.google.com/accounts/docs/OAuth2WebServer
package server

import (
    "encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
)

const (
	GOOGLE_API_OAUTH = "https://www.googleapis.com/oauth2/v4/token"
	GRANT_TYPE       = "authorization_code"
)

// Función encargada de solicitar token
func getToken(code, clientId, clientSecret, redirectUri string, oauth *OAuth2WebServer) error {
	resp, err := http.PostForm(
		GOOGLE_API_OAUTH,
		url.Values{
			"code":          {code},
			"client_id":     {clientId},
			"client_secret": {clientSecret},
			"redirect_uri":  {redirectUri},
			"grant_type":    {GRANT_TYPE},
		},
	)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	err = json.Unmarshal(body, &oauth.JsonResponse)
	if err != nil {
		return err
	}
	oauth.Code = code

	return nil
}

// Función encargada de solicitar token
func GetToken(code, clientId, clientSecret, redirectUri string) (*OAuth2WebServer, error) {
	oauth := &OAuth2WebServer{}
	err := getToken(code, clientId, clientSecret, redirectUri, oauth)
	return oauth, err
}
<|endoftext|>


<commit_before>package objectserver

import (
	"crypto/md5"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log/syslog"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"hummingbird/common"
)

func ReadMetadataFilename(filename string) (map[interface{}]interface{}, error) {
	var pickledMetadata [8192]byte
	offset := 0
	for index := 0; ; index += 1 {
		var metadataName string
		if index == 0 {
			metadataName = "user.swift.metadata"
		} else {
			metadataName = fmt.Sprintf("user.swift.metadata%d", index)
		}
		length, _ := syscall.Getxattr(filename, metadataName, pickledMetadata[offset:])
		if length <= 0 {
			break
		}
		offset += length
	}
	if offset == 0 {
		return nil, errors.New("No metadata data")
	}
	v, err := hummingbird.PickleLoads(pickledMetadata[0:offset])
	if err != nil {
		return nil, err
	}
	return v.(map[interface{}]interface{}), nil
}

func WriteMetadata(fd int, v map[string]interface{}) {
	// TODO: benchmark this with and without chunking up the metadata
	buf := hummingbird.PickleDumps(v)
	for index := 0; len(buf) > 0; index++ {
		var metadataName string
		if index == 0 {
			metadataName = "user.swift.metadata"
		} else {
			metadataName = fmt.Sprintf("user.swift.metadata%d", index)
		}
		writelen := 254
		if len(buf) < writelen {
			writelen = len(buf)
		}
		hummingbird.FSetXattr(fd, metadataName, []byte(buf[0:writelen]))
		buf = buf[writelen:len(buf)]
	}
}

func QuarantineHash(hashDir string) error {
	// FYI- this does not invalidate the hash like swift's version. Please
	// do that yourself
	hash := filepath.Base(hashDir)
	//          drive        objects      partition    suffix       hash
	driveDir := filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(hashDir))))
	// TODO: this will need to be slightly more complicated once policies
	quarantineDir := filepath.Join(driveDir, "quarantined", "objects")
	if err := os.MkdirAll(quarantineDir, 0770); err != nil {
		return err
	}
	destDir := filepath.Join(quarantineDir, fmt.Sprintf("%s-%s", hash, hummingbird.UUID()))
	if err := os.Rename(hashDir, destDir); err != nil {
		return err
	}
	return nil
}

func InvalidateHash(hashDir string, atomic bool) {
	// TODO: return errors
	suffDir := filepath.Dir(hashDir)
	partitionDir := filepath.Dir(suffDir)
	partitionLock, err := hummingbird.LockPath(partitionDir, 10)
	if err != nil {
		return
	}
	defer partitionLock.Close()
	pklFile := fmt.Sprintf("%s/hashes.pkl", partitionDir)
	data, err := ioutil.ReadFile(pklFile)
	if err != nil {
		return
	}
	v, _ := hummingbird.PickleLoads(data)
	v.(map[interface{}]interface{})[suffDir] = nil
	if atomic {
		hummingbird.WriteFileAtomic(pklFile, hummingbird.PickleDumps(v), 0600)
	} else {
		ioutil.WriteFile(pklFile, hummingbird.PickleDumps(v), 0600)
	}
}

func HashCleanupListDir(hashDir string, logger *syslog.Writer) ([]string, *hummingbird.BackendError) {
	fileList, err := ioutil.ReadDir(hashDir)
	returnList := []string{}
	if err != nil {

		if os.IsNotExist(err) {
			return returnList, nil
		}
		if strings.Contains(err.Error(), "not a directory") { // whats the better way to do this?
			return returnList, &hummingbird.BackendError{err, hummingbird.PathNotDirErrorCode}
		}
		return returnList, &hummingbird.BackendError{err, hummingbird.OsErrorCode}
	}
	deleteRest := false
	deleteRestMeta := false
	if len(fileList) == 1 {
		filename := fileList[0].Name()
		if strings.HasSuffix(filename, ".ts") {
			withoutSuffix := strings.Split(filename, ".")[0]
			if strings.Contains(withoutSuffix, "_") {
				withoutSuffix = strings.Split(withoutSuffix, "_")[0]
			}
			timestamp, _ := strconv.ParseFloat(withoutSuffix, 64)
			if time.Now().Unix()-int64(timestamp) > int64(hummingbird.ONE_WEEK) {
				os.RemoveAll(fmt.Sprintf("%s/%s", hashDir, filename))
				return returnList, nil
			}
		}
		returnList = append(returnList, filename)
	} else {
		for index := len(fileList) - 1; index >= 0; index-- {
			filename := fileList[index].Name()
			if deleteRest {
				os.RemoveAll(fmt.Sprintf("%s/%s", hashDir, filename))
			} else {
				if strings.HasSuffix(filename, ".meta") {
					if deleteRestMeta {
						os.RemoveAll(fmt.Sprintf("%s/%s", hashDir, filename))
						continue
					}
					deleteRestMeta = true
				}
				if strings.HasSuffix(filename, ".ts") || strings.HasSuffix(filename, ".data") {
					// TODO: check .ts time for expiration
					deleteRest = true
				}
				returnList = append(returnList, filename)
			}
		}
	}
	return returnList, nil
}

func RecalculateSuffixHash(suffixDir string, logger *syslog.Writer) (string, *hummingbird.BackendError) {
	// the is hash_suffix in swift
	h := md5.New()

	hashList, err := ioutil.ReadDir(suffixDir)
	if err != nil {
		if strings.Contains(err.Error(), "not a directory") { // whats the better way to do this?
			return "", &hummingbird.BackendError{err, hummingbird.PathNotDirErrorCode}
		}
		return "", &hummingbird.BackendError{err, hummingbird.OsErrorCode}
	}
	for index := len(hashList) - 1; index >= 0; index-- {
		hashPath := fmt.Sprintf("%s/%s", suffixDir, hashList[index].Name())
		fileList, err := HashCleanupListDir(hashPath, logger)
		if err != nil {
			if err.Code == hummingbird.PathNotDirErrorCode {
				if QuarantineHash(hashPath) == nil {
					InvalidateHash(hashPath, true)
				}
				continue
			}
			return "", err
		}
		for _, fileName := range fileList {
			io.WriteString(h, fileName)
		}
	}
	return fmt.Sprintf("%x", h.Sum(nil)), nil
}

func GetHashes(driveRoot string, device string, partition string, recalculate []string, logger *syslog.Writer) (map[string]string, *hummingbird.BackendError) {
	/*
			    TODO: this needs to be added later but SAIOs aren't mounted like this
		        devicePath := filepath.Join(driveRoot, device)
				if mounted, err := hummingbird.IsMount(devicePath); err != nil || mounted != true {
					return nil, &hummingbird.BackendError{err, hummingbird.NotMountedErrorCode}
				}
	*/
	partitionDir := filepath.Join(driveRoot, device, "objects", partition)
	pklFile := filepath.Join(partitionDir, "hashes.pkl")

	modified := false
	mtime := int64(-1)
	hashes := make(map[string]string, 4096)
	lsForSuffixes := true
	data, err := ioutil.ReadFile(pklFile)
	// TODO: do I need to defer close this or whatever?
	if err == nil {
		v, err := hummingbird.PickleLoads(data)
		if err == nil {
			pickledHashes, ok := v.(map[string]string)
			if ok {
				fileInfo, err := os.Stat(pklFile)
				if err == nil {
					mtime = fileInfo.ModTime().Unix()
					lsForSuffixes = false
					for suff, hash := range pickledHashes {
						hashes[suff] = hash
					}
				}
			}
		}
	}
	if lsForSuffixes {
		// couldn't load hashes pickle, start building new one
		suffs, _ := ioutil.ReadDir(partitionDir)

		for _, suff := range suffs {
			suffName := suff.Name()
			if len(suffName) == 3 && hashes[suffName] == "" {
				hashes[suffName] = ""
			}
		}
	}
	for _, suffix := range recalculate {
		hashes[suffix] = ""
	}
	for suffix, hash := range hashes {
		if hash == "" {
			modified = true
			suffixDir := fmt.Sprintf("%s/%s/objects/%s/%s", driveRoot, device, partition, suffix)
			recalc_hash, err := RecalculateSuffixHash(suffixDir, logger)
			if err == nil {
				hashes[suffix] = recalc_hash
			} else {
				switch {
				case err.Code == hummingbird.PathNotDirErrorCode:
					delete(hashes, suffix)
				case err.Code == hummingbird.OsErrorCode:
					logger.Err(fmt.Sprintf("Error hashing suffix: %s/%s (%s)", partitionDir, suffix, "asdf"))
				}
			}
		}
	}
	if modified {
		partitionLock, err := hummingbird.LockPath(partitionDir, 10)
		defer partitionLock.Close()
		if err != nil {
			return nil, &hummingbird.BackendError{err, hummingbird.LockPathError}
		} else {
			fileInfo, err := os.Stat(pklFile)
			if lsForSuffixes || os.IsNotExist(err) || mtime != fileInfo.ModTime().Unix() {
				hummingbird.WriteFileAtomic(pklFile, hummingbird.PickleDumps(hashes), 0600)
				return hashes, nil
			}
			logger.Err(fmt.Sprintf("Made recursive call to GetHashes: %s", partitionDir))
			return GetHashes(driveRoot, device, partition, recalculate, logger)
		}
	}
	return hashes, nil
}

func ObjHashDir(vars map[string]string, driveRoot string, hashPathPrefix string, hashPathSuffix string, checkMounts bool) (string, error) {
	h := md5.New()
	io.WriteString(h, fmt.Sprintf("%s/%s/%s/%s%s", hashPathPrefix, vars["account"],
		vars["container"], vars["obj"], hashPathSuffix))
	hexHash := fmt.Sprintf("%x", h.Sum(nil))
	suffix := hexHash[29:32]
	devicePath := fmt.Sprintf("%s/%s", driveRoot, vars["device"])
	if checkMounts {
		if mounted, err := hummingbird.IsMount(devicePath); err != nil || mounted != true {
			return "", errors.New("Not mounted")
		}
	}
	return fmt.Sprintf("%s/%s/%s/%s/%s", devicePath, "objects", vars["partition"], suffix, hexHash), nil
}

func ObjectFiles(directory string) (string, string) {
	fileList, err := ioutil.ReadDir(directory)
	metaFile := ""
	if err != nil {
		return "", ""
	}
	for index := len(fileList) - 1; index >= 0; index-- {
		filename := fileList[index].Name()
		if strings.HasSuffix(filename, ".meta") {
			metaFile = filename
		}
		if strings.HasSuffix(filename, ".ts") || strings.HasSuffix(filename, ".data") {
			if metaFile != "" {
				return filepath.Join(directory, filename), filepath.Join(directory, metaFile)
			} else {
				return filepath.Join(directory, filename), ""
			}
		}
	}
	return "", ""
}

func ObjTempDir(vars map[string]string, driveRoot string) string {
	return fmt.Sprintf("%s/%s/%s", driveRoot, vars["device"], "tmp")
}

func ObjectMetadata(dataFile string, metaFile string) (map[interface{}]interface{}, error) {
	datafileMetadata, err := ReadMetadataFilename(dataFile)
	if err != nil {
		return nil, err
	}

	if metaFile == "" {
		return datafileMetadata, nil
	} else {
		metadata, err := ReadMetadataFilename(metaFile)
		if err != nil {
			return nil, err
		}
		for k, v := range datafileMetadata {
			if k == "Content-Length" || k == "Content-Type" || k == "deleted" || k == "Etag" || strings.HasPrefix(k.(string), "X-Object-Sysmeta-") {
				metadata[k] = v
			}
		}
		return metadata, nil
	}
}
<commit_msg>get rid of hash dir if its empty<commit_after>package objectserver

import (
	"crypto/md5"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log/syslog"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"hummingbird/common"
)

func ReadMetadataFilename(filename string) (map[interface{}]interface{}, error) {
	var pickledMetadata [8192]byte
	offset := 0
	for index := 0; ; index += 1 {
		var metadataName string
		if index == 0 {
			metadataName = "user.swift.metadata"
		} else {
			metadataName = fmt.Sprintf("user.swift.metadata%d", index)
		}
		length, _ := syscall.Getxattr(filename, metadataName, pickledMetadata[offset:])
		if length <= 0 {
			break
		}
		offset += length
	}
	if offset == 0 {
		return nil, errors.New("No metadata data")
	}
	v, err := hummingbird.PickleLoads(pickledMetadata[0:offset])
	if err != nil {
		return nil, err
	}
	return v.(map[interface{}]interface{}), nil
}

func WriteMetadata(fd int, v map[string]interface{}) {
	// TODO: benchmark this with and without chunking up the metadata
	buf := hummingbird.PickleDumps(v)
	for index := 0; len(buf) > 0; index++ {
		var metadataName string
		if index == 0 {
			metadataName = "user.swift.metadata"
		} else {
			metadataName = fmt.Sprintf("user.swift.metadata%d", index)
		}
		writelen := 254
		if len(buf) < writelen {
			writelen = len(buf)
		}
		hummingbird.FSetXattr(fd, metadataName, []byte(buf[0:writelen]))
		buf = buf[writelen:len(buf)]
	}
}

func QuarantineHash(hashDir string) error {
	// FYI- this does not invalidate the hash like swift's version. Please
	// do that yourself
	hash := filepath.Base(hashDir)
	//          drive        objects      partition    suffix       hash
	driveDir := filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(hashDir))))
	// TODO: this will need to be slightly more complicated once policies
	quarantineDir := filepath.Join(driveDir, "quarantined", "objects")
	if err := os.MkdirAll(quarantineDir, 0770); err != nil {
		return err
	}
	destDir := filepath.Join(quarantineDir, fmt.Sprintf("%s-%s", hash, hummingbird.UUID()))
	if err := os.Rename(hashDir, destDir); err != nil {
		return err
	}
	return nil
}

func InvalidateHash(hashDir string, atomic bool) {
	// TODO: return errors
	suffDir := filepath.Dir(hashDir)
	partitionDir := filepath.Dir(suffDir)
	partitionLock, err := hummingbird.LockPath(partitionDir, 10)
	if err != nil {
		return
	}
	defer partitionLock.Close()
	pklFile := fmt.Sprintf("%s/hashes.pkl", partitionDir)
	data, err := ioutil.ReadFile(pklFile)
	if err != nil {
		return
	}
	v, _ := hummingbird.PickleLoads(data)
	v.(map[interface{}]interface{})[suffDir] = nil
	if atomic {
		hummingbird.WriteFileAtomic(pklFile, hummingbird.PickleDumps(v), 0600)
	} else {
		ioutil.WriteFile(pklFile, hummingbird.PickleDumps(v), 0600)
	}
}

func HashCleanupListDir(hashDir string, logger *syslog.Writer) ([]string, *hummingbird.BackendError) {
	fileList, err := ioutil.ReadDir(hashDir)
	returnList := []string{}
	if err != nil {

		if os.IsNotExist(err) {
			return returnList, nil
		}
		if strings.Contains(err.Error(), "not a directory") { // whats the better way to do this?
			return returnList, &hummingbird.BackendError{err, hummingbird.PathNotDirErrorCode}
		}
		return returnList, &hummingbird.BackendError{err, hummingbird.OsErrorCode}
	}
	deleteRest := false
	deleteRestMeta := false
	if len(fileList) == 1 {
		filename := fileList[0].Name()
		if strings.HasSuffix(filename, ".ts") {
			withoutSuffix := strings.Split(filename, ".")[0]
			if strings.Contains(withoutSuffix, "_") {
				withoutSuffix = strings.Split(withoutSuffix, "_")[0]
			}
			timestamp, _ := strconv.ParseFloat(withoutSuffix, 64)
			if time.Now().Unix()-int64(timestamp) > int64(hummingbird.ONE_WEEK) {
				os.RemoveAll(fmt.Sprintf("%s/%s", hashDir, filename))
				return returnList, nil
			}
		}
		returnList = append(returnList, filename)
	} else {
		for index := len(fileList) - 1; index >= 0; index-- {
			filename := fileList[index].Name()
			if deleteRest {
				os.RemoveAll(fmt.Sprintf("%s/%s", hashDir, filename))
			} else {
				if strings.HasSuffix(filename, ".meta") {
					if deleteRestMeta {
						os.RemoveAll(fmt.Sprintf("%s/%s", hashDir, filename))
						continue
					}
					deleteRestMeta = true
				}
				if strings.HasSuffix(filename, ".ts") || strings.HasSuffix(filename, ".data") {
					// TODO: check .ts time for expiration
					deleteRest = true
				}
				returnList = append(returnList, filename)
			}
		}
	}
	return returnList, nil
}

func RecalculateSuffixHash(suffixDir string, logger *syslog.Writer) (string, *hummingbird.BackendError) {
	// the is hash_suffix in swift
	h := md5.New()

	hashList, err := ioutil.ReadDir(suffixDir)
	if err != nil {
		if strings.Contains(err.Error(), "not a directory") { // whats the better way to do this?
			return "", &hummingbird.BackendError{err, hummingbird.PathNotDirErrorCode}
		}
		return "", &hummingbird.BackendError{err, hummingbird.OsErrorCode}
	}
	for index := len(hashList) - 1; index >= 0; index-- {
		hashPath := fmt.Sprintf("%s/%s", suffixDir, hashList[index].Name())
		fileList, err := HashCleanupListDir(hashPath, logger)
		if err != nil {
			if err.Code == hummingbird.PathNotDirErrorCode {
				if QuarantineHash(hashPath) == nil {
					InvalidateHash(hashPath, true)
				}
				continue
			}
			return "", err
		}
		if len(fileList) > 0 {
			for _, fileName := range fileList {
				io.WriteString(h, fileName)
			}
		} else {
			os.Remove(hashPath) // leaves the suffix (swift removes it but who cares)
		}
	}
	return fmt.Sprintf("%x", h.Sum(nil)), nil
}

func GetHashes(driveRoot string, device string, partition string, recalculate []string, logger *syslog.Writer) (map[string]string, *hummingbird.BackendError) {
	/*
			    TODO: this needs to be added later but SAIOs aren't mounted like this
		        devicePath := filepath.Join(driveRoot, device)
				if mounted, err := hummingbird.IsMount(devicePath); err != nil || mounted != true {
					return nil, &hummingbird.BackendError{err, hummingbird.NotMountedErrorCode}
				}
	*/
	partitionDir := filepath.Join(driveRoot, device, "objects", partition)
	pklFile := filepath.Join(partitionDir, "hashes.pkl")

	modified := false
	mtime := int64(-1)
	hashes := make(map[string]string, 4096)
	lsForSuffixes := true
	data, err := ioutil.ReadFile(pklFile)
	if err == nil {
		v, err := hummingbird.PickleLoads(data)
		if err == nil {
			pickledHashes, ok := v.(map[string]string)
			if ok {
				fileInfo, err := os.Stat(pklFile)
				if err == nil {
					mtime = fileInfo.ModTime().Unix()
					lsForSuffixes = false
					for suff, hash := range pickledHashes {
						hashes[suff] = hash
					}
				}
			}
		}
	}
	if lsForSuffixes {
		// couldn't load hashes pickle, start building new one
		suffs, _ := ioutil.ReadDir(partitionDir)

		for _, suff := range suffs {
			suffName := suff.Name()
			if len(suffName) == 3 && hashes[suffName] == "" {
				hashes[suffName] = ""
			}
		}
	}
	for _, suffix := range recalculate {
		hashes[suffix] = ""
	}
	for suffix, hash := range hashes {
		if hash == "" {
			modified = true
			suffixDir := fmt.Sprintf("%s/%s/objects/%s/%s", driveRoot, device, partition, suffix)
			recalc_hash, err := RecalculateSuffixHash(suffixDir, logger)
			if err == nil {
				hashes[suffix] = recalc_hash
			} else {
				switch {
				case err.Code == hummingbird.PathNotDirErrorCode:
					delete(hashes, suffix)
				case err.Code == hummingbird.OsErrorCode:
					logger.Err(fmt.Sprintf("Error hashing suffix: %s/%s (%s)", partitionDir, suffix, "asdf"))
				}
			}
		}
	}
	if modified {
		partitionLock, err := hummingbird.LockPath(partitionDir, 10)
		defer partitionLock.Close()
		if err != nil {
			return nil, &hummingbird.BackendError{err, hummingbird.LockPathError}
		} else {
			fileInfo, err := os.Stat(pklFile)
			if lsForSuffixes || os.IsNotExist(err) || mtime != fileInfo.ModTime().Unix() {
				hummingbird.WriteFileAtomic(pklFile, hummingbird.PickleDumps(hashes), 0600)
				return hashes, nil
			}
			logger.Err(fmt.Sprintf("Made recursive call to GetHashes: %s", partitionDir))
			return GetHashes(driveRoot, device, partition, recalculate, logger)
		}
	}
	return hashes, nil
}

func ObjHashDir(vars map[string]string, driveRoot string, hashPathPrefix string, hashPathSuffix string, checkMounts bool) (string, error) {
	h := md5.New()
	io.WriteString(h, fmt.Sprintf("%s/%s/%s/%s%s", hashPathPrefix, vars["account"],
		vars["container"], vars["obj"], hashPathSuffix))
	hexHash := fmt.Sprintf("%x", h.Sum(nil))
	suffix := hexHash[29:32]
	devicePath := fmt.Sprintf("%s/%s", driveRoot, vars["device"])
	if checkMounts {
		if mounted, err := hummingbird.IsMount(devicePath); err != nil || mounted != true {
			return "", errors.New("Not mounted")
		}
	}
	return fmt.Sprintf("%s/%s/%s/%s/%s", devicePath, "objects", vars["partition"], suffix, hexHash), nil
}

func ObjectFiles(directory string) (string, string) {
	fileList, err := ioutil.ReadDir(directory)
	metaFile := ""
	if err != nil {
		return "", ""
	}
	for index := len(fileList) - 1; index >= 0; index-- {
		filename := fileList[index].Name()
		if strings.HasSuffix(filename, ".meta") {
			metaFile = filename
		}
		if strings.HasSuffix(filename, ".ts") || strings.HasSuffix(filename, ".data") {
			if metaFile != "" {
				return filepath.Join(directory, filename), filepath.Join(directory, metaFile)
			} else {
				return filepath.Join(directory, filename), ""
			}
		}
	}
	return "", ""
}

func ObjTempDir(vars map[string]string, driveRoot string) string {
	return fmt.Sprintf("%s/%s/%s", driveRoot, vars["device"], "tmp")
}

func ObjectMetadata(dataFile string, metaFile string) (map[interface{}]interface{}, error) {
	datafileMetadata, err := ReadMetadataFilename(dataFile)
	if err != nil {
		return nil, err
	}

	if metaFile == "" {
		return datafileMetadata, nil
	} else {
		metadata, err := ReadMetadataFilename(metaFile)
		if err != nil {
			return nil, err
		}
		for k, v := range datafileMetadata {
			if k == "Content-Length" || k == "Content-Type" || k == "deleted" || k == "Etag" || strings.HasPrefix(k.(string), "X-Object-Sysmeta-") {
				metadata[k] = v
			}
		}
		return metadata, nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import "time"

// RPCReconnectInterval specifies the time between reconnect attempts for RPC Connections.
const RPCReconnectInterval = 2 * time.Second

// rekeyRecheckInterval is the time duration to wait for before rechecking for
// rekey for the same TLF. See fbo.Rekey for more details.
const rekeyRecheckInterval = 30 * time.Second

// rekeyInitialTTL is the maximum number rechecks each rekey request can trigger.
const rekeyInitialTTL = 4

// mdserverReconnectBackoffWindow is a backoff window within which we try to
// wait randomly for before reconnecting to MD server.
// TODO: 1 hr.
const mdserverReconnectBackoffWindow = 20 * time.Minute

// registerForUpdatesFireNowThreshold is the maximum length of time that
// KBFS can be idle for, in order to trigger FireNow from RegisterForUpdate.
const registerForUpdatesFireNowThreshold = 10 * time.Minute
<commit_msg>increase the reconnect window to 1hr (#1156)<commit_after>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import "time"

// RPCReconnectInterval specifies the time between reconnect attempts for RPC Connections.
const RPCReconnectInterval = 2 * time.Second

// rekeyRecheckInterval is the time duration to wait for before rechecking for
// rekey for the same TLF. See fbo.Rekey for more details.
const rekeyRecheckInterval = 30 * time.Second

// rekeyInitialTTL is the maximum number rechecks each rekey request can trigger.
const rekeyInitialTTL = 4

// mdserverReconnectBackoffWindow is a backoff window within which we try to
// wait randomly for before reconnecting to MD server.
// TODO: 1 hr.
const mdserverReconnectBackoffWindow = time.Hour

// registerForUpdatesFireNowThreshold is the maximum length of time that
// KBFS can be idle for, in order to trigger FireNow from RegisterForUpdate.
const registerForUpdatesFireNowThreshold = 10 * time.Minute
<|endoftext|>


<commit_before>package main

import (
	"sync"
	"time"

	"github.com/grafana/grafana/pkg/log"
)

type AggMetrics struct {
	store Store
	sync.RWMutex
	Metrics        map[string]*AggMetric
	chunkSpan      uint32
	numChunks      uint32
	aggSettings    []aggSetting // for now we apply the same settings to all AggMetrics. later we may want to have different settings.
	chunkMaxStale  uint32
	metricMaxStale uint32
	ttl            uint32
	gcInterval     time.Duration
}

var totalPoints chan int

func init() {
	// measurements can lag a bit, that's ok
	totalPoints = make(chan int, 1000)
}

func NewAggMetrics(store Store, chunkSpan, numChunks, chunkMaxStale, metricMaxStale uint32, ttl uint32, gcInterval time.Duration, aggSettings []aggSetting) *AggMetrics {
	ms := AggMetrics{
		store:          store,
		Metrics:        make(map[string]*AggMetric),
		chunkSpan:      chunkSpan,
		numChunks:      numChunks,
		aggSettings:    aggSettings,
		chunkMaxStale:  chunkMaxStale,
		metricMaxStale: metricMaxStale,
		ttl:            ttl,
		gcInterval:     gcInterval,
	}

	go ms.stats()
	go ms.GC()
	return &ms
}

// periodically scan chunks and close any that have not received data in a while
// TODO instrument occurences and duration of GC
func (ms *AggMetrics) GC() {
	for {
		unix := time.Duration(time.Now().UnixNano())
		diff := ms.gcInterval - (unix % ms.gcInterval)
		time.Sleep(diff + time.Minute)
		if !clusterStatus.IsPrimary() {
			continue
		}
		log.Info("checking for stale chunks that need persisting.")
		now := uint32(time.Now().Unix())
		chunkMinTs := now - (now % ms.chunkSpan) - uint32(ms.chunkMaxStale)
		metricMinTs := now - (now % ms.chunkSpan) - uint32(ms.metricMaxStale)

		// as this is the only goroutine that can delete from ms.Metrics
		// we only need to lock long enough to get the list of actives metrics.
		// it doesnt matter if new metrics are added while we iterate this list.
		ms.RLock()
		keys := make([]string, 0, len(ms.Metrics))
		for k := range ms.Metrics {
			keys = append(keys, k)
		}
		ms.RUnlock()
		for _, key := range keys {
			gcMetric.Inc(1)
			ms.RLock()
			a := ms.Metrics[key]
			ms.RUnlock()
			if stale := a.GC(chunkMinTs, metricMinTs); stale {
				log.Info("metric %s is stale. Purging data from memory.", key)
				ms.Lock()
				delete(ms.Metrics, key)
				ms.Unlock()
			}
		}

	}
}

func (ms *AggMetrics) stats() {
	for range time.Tick(time.Duration(1) * time.Second) {
		ms.RLock()
		metricsActive.Value(int64(len(ms.Metrics)))
		ms.RUnlock()
	}
}

func (ms *AggMetrics) Get(key string) (Metric, bool) {
	ms.RLock()
	m, ok := ms.Metrics[key]
	ms.RUnlock()
	return m, ok
}

func (ms *AggMetrics) GetOrCreate(key string) Metric {
	ms.Lock()
	m, ok := ms.Metrics[key]
	if !ok {
		m = NewAggMetric(ms.store, key, ms.chunkSpan, ms.numChunks, ms.ttl, ms.aggSettings...)
		ms.Metrics[key] = m
	}
	ms.Unlock()
	return m
}
<commit_msg>remove stale todo<commit_after>package main

import (
	"sync"
	"time"

	"github.com/grafana/grafana/pkg/log"
)

type AggMetrics struct {
	store Store
	sync.RWMutex
	Metrics        map[string]*AggMetric
	chunkSpan      uint32
	numChunks      uint32
	aggSettings    []aggSetting // for now we apply the same settings to all AggMetrics. later we may want to have different settings.
	chunkMaxStale  uint32
	metricMaxStale uint32
	ttl            uint32
	gcInterval     time.Duration
}

var totalPoints chan int

func init() {
	// measurements can lag a bit, that's ok
	totalPoints = make(chan int, 1000)
}

func NewAggMetrics(store Store, chunkSpan, numChunks, chunkMaxStale, metricMaxStale uint32, ttl uint32, gcInterval time.Duration, aggSettings []aggSetting) *AggMetrics {
	ms := AggMetrics{
		store:          store,
		Metrics:        make(map[string]*AggMetric),
		chunkSpan:      chunkSpan,
		numChunks:      numChunks,
		aggSettings:    aggSettings,
		chunkMaxStale:  chunkMaxStale,
		metricMaxStale: metricMaxStale,
		ttl:            ttl,
		gcInterval:     gcInterval,
	}

	go ms.stats()
	go ms.GC()
	return &ms
}

// periodically scan chunks and close any that have not received data in a while
func (ms *AggMetrics) GC() {
	for {
		unix := time.Duration(time.Now().UnixNano())
		diff := ms.gcInterval - (unix % ms.gcInterval)
		time.Sleep(diff + time.Minute)
		if !clusterStatus.IsPrimary() {
			continue
		}
		log.Info("checking for stale chunks that need persisting.")
		now := uint32(time.Now().Unix())
		chunkMinTs := now - (now % ms.chunkSpan) - uint32(ms.chunkMaxStale)
		metricMinTs := now - (now % ms.chunkSpan) - uint32(ms.metricMaxStale)

		// as this is the only goroutine that can delete from ms.Metrics
		// we only need to lock long enough to get the list of actives metrics.
		// it doesnt matter if new metrics are added while we iterate this list.
		ms.RLock()
		keys := make([]string, 0, len(ms.Metrics))
		for k := range ms.Metrics {
			keys = append(keys, k)
		}
		ms.RUnlock()
		for _, key := range keys {
			gcMetric.Inc(1)
			ms.RLock()
			a := ms.Metrics[key]
			ms.RUnlock()
			if stale := a.GC(chunkMinTs, metricMinTs); stale {
				log.Info("metric %s is stale. Purging data from memory.", key)
				ms.Lock()
				delete(ms.Metrics, key)
				ms.Unlock()
			}
		}

	}
}

func (ms *AggMetrics) stats() {
	for range time.Tick(time.Duration(1) * time.Second) {
		ms.RLock()
		metricsActive.Value(int64(len(ms.Metrics)))
		ms.RUnlock()
	}
}

func (ms *AggMetrics) Get(key string) (Metric, bool) {
	ms.RLock()
	m, ok := ms.Metrics[key]
	ms.RUnlock()
	return m, ok
}

func (ms *AggMetrics) GetOrCreate(key string) Metric {
	ms.Lock()
	m, ok := ms.Metrics[key]
	if !ok {
		m = NewAggMetric(ms.store, key, ms.chunkSpan, ms.numChunks, ms.ttl, ms.aggSettings...)
		ms.Metrics[key] = m
	}
	ms.Unlock()
	return m
}
<|endoftext|>


<commit_before><commit_msg>Fix the doc-string for Get*/SetAggregations<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"koding/artifact"
	"koding/db/mongodb/modelhelper"
	"log"
	"math/rand"
	"net"
	"net/http"
	"socialapi/config"
	"sync"
	"time"

	kiteConfig "github.com/koding/kite/config"
	"github.com/koding/logging"
	"github.com/koding/runner"
	"github.com/robfig/cron"

	"github.com/koding/kite"
)

const (
	WorkerName    = "janitor"
	WorkerVersion = "0.0.1"

	// DefaultRangeForQuery defines the range of interval for the queries.
	DefaultRangeForQuery = 3

	// DailyAtTwoPM specifies interval; cron runs at utc, 21 UTC is 2pm PST
	// with daylight savings time.
	DailyAtTwoPM = "0 0 21 * * *"
)

type janitor struct {
	runner     *runner.Runner
	log        logging.Logger
	kiteClient *kite.Client
}

var j = &janitor{}

func main() {
	j.initializeRunner()

	conf := config.MustRead(j.runner.Conf.Path)
	port := conf.Janitor.Port
	konf := conf.Kloud

	kloudSecretKey := conf.Janitor.SecretKey

	go j.runner.Listen()

	err := j.initializeKiteClient(kloudSecretKey, konf.Address)
	if err != nil {
		j.log.Fatal("Error initializing kite: %s", err.Error())
	}

	// warnings contains list of warnings to be iterated upon in a certain
	// interval.
	warnings := []*Warning{
		VMDeletionWarning1,
		VMDeletionWarning2,
		DeleteInactiveUserVM,
		DeleteBlockedUserVM,
		newDeleteInactiveUsersWarning(conf),
	}

	c := cron.New()
	c.AddFunc(DailyAtTwoPM, func() {
		var wg sync.WaitGroup

		for _, w := range warnings {
			wg.Add(1)

			// sleep random time to avoid all workers starting at the same time;
			// random time can be anywhere from 0 seconds to 1.38 hour.
			time.Sleep(time.Second * time.Duration(rand.Intn(5000)))

			go func(warning Warning) {
				defer wg.Done()

				result, err := warning.Run()
				if err != nil {
					j.log.Error(err.Error())
					return
				}

				j.log.Info(result.String())
			}(*w)
		}

		wg.Wait()
	})

	c.Start()

	mux := http.NewServeMux()
	mux.HandleFunc("/version", artifact.VersionHandler())
	mux.HandleFunc("/healthCheck", artifact.HealthCheckHandler(WorkerName))

	listener, err := net.Listen("tcp", ":"+port)
	if err != nil {
		j.log.Fatal("Error opening tcp connection: %s", err.Error())
	}

	j.log.Info("Listening on port: %s", port)

	j.runner.ShutdownHandler = func() {
		listener.Close()
		j.runner.Kite.Close()
		modelhelper.Close()
	}

	if err := http.Serve(listener, mux); err != nil {
		j.log.Fatal("Error starting http server: %s", err.Error())
	}
}

func (j *janitor) initializeRunner() {
	r := runner.New(WorkerName)
	if err := r.Init(); err != nil {
		log.Fatal("Error starting runner: %s", err.Error())
	}

	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)

	j.runner = r
	j.log = r.Log
}

func (j *janitor) initializeKiteClient(kloudKey, kloudAddr string) error {
	config, err := kiteConfig.Get()
	if err != nil {
		return err
	}

	r := j.runner

	// set skeleton config
	r.Kite.Config = config

	// create a new connection to the cloud
	kiteClient := r.Kite.NewClient(kloudAddr)
	kiteClient.Auth = &kite.Auth{Type: WorkerName, Key: kloudKey}
	kiteClient.Reconnect = true

	// dial the kloud address
	if err := kiteClient.DialTimeout(time.Second * 10); err != nil {
		return fmt.Errorf("%s. Is kloud running?", err.Error())
	}

	j.log.Debug("Connected to klient: %s", kloudAddr)

	j.kiteClient = kiteClient

	return nil
}
<commit_msg>go/janitor: remove unnecesary checks<commit_after>package main

import (
	"fmt"
	"koding/artifact"
	"koding/db/mongodb/modelhelper"
	"log"
	"math/rand"
	"net"
	"net/http"
	"socialapi/config"
	"sync"
	"time"

	kiteConfig "github.com/koding/kite/config"
	"github.com/koding/logging"
	"github.com/koding/runner"
	"github.com/robfig/cron"

	"github.com/koding/kite"
)

const (
	WorkerName    = "janitor"
	WorkerVersion = "0.0.1"

	// DefaultRangeForQuery defines the range of interval for the queries.
	DefaultRangeForQuery = 3

	// DailyAtTwoPM specifies interval; cron runs at utc, 21 UTC is 2pm PST
	// with daylight savings time.
	DailyAtTwoPM = "0 0 21 * * *"
)

type janitor struct {
	runner     *runner.Runner
	log        logging.Logger
	kiteClient *kite.Client
}

var j = &janitor{}

func main() {
	j.initializeRunner()

	conf := config.MustRead(j.runner.Conf.Path)
	port := conf.Janitor.Port
	konf := conf.Kloud

	kloudSecretKey := conf.Janitor.SecretKey

	go j.runner.Listen()

	err := j.initializeKiteClient(kloudSecretKey, konf.Address)
	if err != nil {
		j.log.Fatal("Error initializing kite: %s", err.Error())
	}

	// warnings contains list of warnings to be iterated upon in a certain
	// interval.
	warnings := []*Warning{
		newDeleteInactiveUsersWarning(conf),
	}

	c := cron.New()
	c.AddFunc(DailyAtTwoPM, func() {
		var wg sync.WaitGroup

		for _, w := range warnings {
			wg.Add(1)

			// sleep random time to avoid all workers starting at the same time;
			// random time can be anywhere from 0 seconds to 1.38 hour.
			time.Sleep(time.Second * time.Duration(rand.Intn(5000)))

			go func(warning Warning) {
				defer wg.Done()

				result, err := warning.Run()
				if err != nil {
					j.log.Error(err.Error())
					return
				}

				j.log.Info(result.String())
			}(*w)
		}

		wg.Wait()
	})

	c.Start()

	mux := http.NewServeMux()
	mux.HandleFunc("/version", artifact.VersionHandler())
	mux.HandleFunc("/healthCheck", artifact.HealthCheckHandler(WorkerName))

	listener, err := net.Listen("tcp", ":"+port)
	if err != nil {
		j.log.Fatal("Error opening tcp connection: %s", err.Error())
	}

	j.log.Info("Listening on port: %s", port)

	j.runner.ShutdownHandler = func() {
		listener.Close()
		j.runner.Kite.Close()
		modelhelper.Close()
	}

	if err := http.Serve(listener, mux); err != nil {
		j.log.Fatal("Error starting http server: %s", err.Error())
	}
}

func (j *janitor) initializeRunner() {
	r := runner.New(WorkerName)
	if err := r.Init(); err != nil {
		log.Fatal("Error starting runner: %s", err.Error())
	}

	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)

	j.runner = r
	j.log = r.Log
}

func (j *janitor) initializeKiteClient(kloudKey, kloudAddr string) error {
	config, err := kiteConfig.Get()
	if err != nil {
		return err
	}

	r := j.runner

	// set skeleton config
	r.Kite.Config = config

	// create a new connection to the cloud
	kiteClient := r.Kite.NewClient(kloudAddr)
	kiteClient.Auth = &kite.Auth{Type: WorkerName, Key: kloudKey}
	kiteClient.Reconnect = true

	// dial the kloud address
	if err := kiteClient.DialTimeout(time.Second * 10); err != nil {
		return fmt.Errorf("%s. Is kloud running?", err.Error())
	}

	j.log.Debug("Connected to klient: %s", kloudAddr)

	j.kiteClient = kiteClient

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package cmd

import (
	"fmt"
	"go/ast"
	"regexp"
	"sort"
	"strings"
)

// skip adding the const type in the token list
const skip = "Untyped const"

// newContent returns an initialized Content object.
func newContent() content {
	return content{
		Consts:     make(map[string]Const),
		Funcs:      make(map[string]Func),
		Interfaces: make(map[string]Interface),
		Structs:    make(map[string]Struct),
	}
}

// isEmpty returns true if there is no content in any of the fields.
func (c content) isEmpty() bool {
	return len(c.Consts) == 0 && len(c.Funcs) == 0 && len(c.Interfaces) == 0 && len(c.Structs) == 0
}

// adds the specified const declaration to the exports list
func (c *content) addConst(pkg pkg, g *ast.GenDecl) {
	for _, s := range g.Specs {
		co := Const{}
		vs := s.(*ast.ValueSpec)
		v := ""
		// Type is nil for untyped consts
		if vs.Type != nil {
			switch x := vs.Type.(type) {
			case *ast.Ident:
				co.Type = x.Name
				v = vs.Values[0].(*ast.BasicLit).Value
			case *ast.SelectorExpr:
				co.Type = x.Sel.Name
				v = vs.Values[0].(*ast.BasicLit).Value
			default:
				panic(fmt.Sprintf("wrong type %T", vs.Type))
			}

		} else {
			// get the type from the token type
			if bl, ok := vs.Values[0].(*ast.BasicLit); ok {
				co.Type = skip
				v = bl.Value
			} else if ce, ok := vs.Values[0].(*ast.CallExpr); ok {
				// const FooConst = FooType("value")
				co.Type = pkg.getText(ce.Fun.Pos(), ce.Fun.End())
				v = pkg.getText(ce.Args[0].Pos(), ce.Args[0].End())
			} else if ce, ok := vs.Values[0].(*ast.BinaryExpr); ok {
				// const FooConst = "value" + Bar
				co.Type = skip
				v = pkg.getText(ce.X.Pos(), ce.Y.End())
			} else if ce, ok := vs.Values[0].(*ast.UnaryExpr); ok {
				// const FooConst = -1
				co.Type = skip
				v = pkg.getText(ce.Pos(), ce.End())
			} else {
				panic("unhandled case for adding constant")
			}
		}
		co.Value = v
		c.Consts[vs.Names[0].Name] = co
	}
}

func includesType(s []string, t string) bool {
	for _, j := range s {
		if j == t {
			return true
		}
	}
	return false
}

func (c *content) parseConst(tokenList *[]Token) {
	if len(c.Consts) > 0 {
		// create keys slice in order to later sort consts by their types
		keys := []string{}
		// create types slice in order to be able to separate consts by the type they represent
		types := []string{}
		for i, s := range c.Consts {
			keys = append(keys, i)
			if !includesType(types, s.Type) {
				types = append(types, s.Type)
			}
		}
		sort.Strings(keys)
		sort.Strings(types)
		// finalKeys will order const keys by their type
		finalKeys := []string{}
		for _, t := range types {
			for _, k := range keys {
				if t == c.Consts[k].Type {
					finalKeys = append(finalKeys, k)
				}
			}
		}
		for _, t := range types {
			// this token parsing is performed so that const declarations of different types are declared
			// in their own const block to make them easier to click on
			n := t
			makeToken(nil, nil, "", 1, tokenList)
			makeToken(nil, nil, " ", whitespace, tokenList)
			makeToken(nil, nil, "", 1, tokenList)
			makeToken(&n, nil, "const", keyword, tokenList)
			makeToken(nil, nil, " ", whitespace, tokenList)
			makeToken(nil, nil, "(", punctuation, tokenList)
			makeToken(nil, nil, "", 1, tokenList)
			for _, v := range finalKeys {
				if c.Consts[v].Type == t {
					makeConstTokens(&v, c.Consts[v], tokenList)
				}
			}
			makeToken(nil, nil, ")", punctuation, tokenList)
			makeToken(nil, nil, "", 1, tokenList)
		}
	}

}

// adds the specified function declaration to the exports list
func (c *content) addFunc(pkg pkg, f *ast.FuncDecl) {
	// create a method sig, for methods it's a combination of the receiver type
	// with the function name e.g. "FooReceiver.Method", else just the function name.
	sig := ""
	if f.Recv != nil {
		sig = "(c "
		sig += pkg.getText(f.Recv.List[0].Type.Pos(), f.Recv.List[0].Type.End())
		// CP: changed to space, was a period before
		sig += ") "
	}
	sig += f.Name.Name
	c.Funcs[sig] = pkg.buildFunc(f.Type)
}

// adds the specified interface type to the exports list.
func (c *content) addInterface(pkg pkg, name string, i *ast.InterfaceType) {
	in := Interface{Methods: map[string]Func{}}
	if i.Methods != nil {
		for _, m := range i.Methods.List {
			if len(m.Names) > 0 {
				n := m.Names[0].Name
				f := pkg.buildFunc(m.Type.(*ast.FuncType))
				in.Methods[n] = f
			} else {
				n := pkg.getText(m.Type.Pos(), m.Type.End())
				in.EmbeddedInterfaces = append(in.EmbeddedInterfaces, n)
			}
		}
	}
	c.Interfaces[name] = in
}

// adds the specified struct type to the exports list.
func (c *content) parseInterface(tokenList *[]Token) {
	keys := []string{}
	for s := range c.Interfaces {
		keys = append(keys, s)
	}
	sort.Strings(keys)
	for _, k := range keys {
		makeInterfaceTokens(&k, c.Interfaces[k].EmbeddedInterfaces, c.Interfaces[k].Methods, tokenList)
	}
}

// adds the specified struct type to the exports list.
func (c *content) addStruct(pkg pkg, name string, s *ast.StructType) {
	sd := Struct{}
	// assumes all struct types have fields
	pkg.translateFieldList(s.Fields.List, func(n *string, t string) {
		if n == nil {
			sd.AnonymousFields = append(sd.AnonymousFields, t)
		} else {
			if sd.Fields == nil {
				sd.Fields = map[string]string{}
			}
			sd.Fields[*n] = t
		}
	})
	c.Structs[name] = sd
}

// adds the specified struct type to the exports list.
func (c *content) parseStruct(tokenList *[]Token) {
	keys := make([]string, 0, len(c.Structs))
	for k := range c.Structs {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		makeStructTokens(&k, c.Structs[k].AnonymousFields, c.Structs[k].Fields, tokenList)
		c.searchForCtors(k, tokenList)
		c.searchForMethods(k, tokenList)
	}
}

// searchForCtors will search through all of the exported Funcs for a constructor for the name of the
// type that is passed as a param.
func (c *content) searchForCtors(s string, tokenList *[]Token) {
	for i, f := range c.Funcs {
		n := getCtorName(i)
		if s == n {
			makeFuncTokens(&i, f.Params, f.Returns, f.ReturnsNum, tokenList)
			delete(c.Funcs, i)
			return
		}
	}
}

// searchForMethods takes the name of the receiver and looks for Funcs that are methods on that receiver.
func (c *content) searchForMethods(s string, tokenList *[]Token) {
	for i, f := range c.Funcs {
		v, n := getReceiverName(i)
		isPointer := false
		if strings.HasPrefix(n, "*") {
			n = n[1:]
			isPointer = true
		}
		if s == n {
			makeMethodTokens(v, n, isPointer, getMethodName(i), f.Params, f.Returns, f.ReturnsNum, tokenList)
			delete(c.Funcs, i)
		}
		if isOnUnexportedMember(i) || isExampleOrTest(i) {
			delete(c.Funcs, i)
		}
	}
}

// getCtorName returns the name of a constructor without the New prefix.
// TODO improve this to also check the return statement on the constructor to make sure it does in fact only
// return the name of constructors and not other functions that begin with New
func getCtorName(s string) string {
	if strings.HasPrefix(s, "New") {
		ctor := s[3:]
		return ctor
	}
	return ""
}

// getReceiverName returns the components of the receiver on a method signature
// i.e.: (c *Foo) Bar(s string) will return "c" and "Foo".
func getReceiverName(s string) (receiverVar string, receiver string) {
	if strings.HasPrefix(s, "(") {
		parts := strings.Split(s[:strings.Index(s, ")")], " ")
		receiverVar = parts[0][1:]
		receiver = parts[1]
		return
	}
	return "", ""
}

// getMethodName expects a method signature in the param s and removes the receiver portion of the
// method signature before returning the method name.
func getMethodName(s string) string {
	pos := strings.Index(s, ")")
	// return the string after the first ) and add an extra index to omit the space after the receiver
	return s[pos+2:]
}

// isOnUnexportedMember checks for method signatures that are on unexported types,
// it will return true if the method is unexported.
func isOnUnexportedMember(s string) bool {
	r := regexp.MustCompile(`(\(([a-z|A-Z]{1}) (\*){0,1}([a-z]+)([a-z|A-Z]*)\))`)
	return r.MatchString(s)
	// for _, l := range s {
	// 	if unicode.IsLetter(l) {
	// 		return string(l) == strings.ToLower(string(l))
	// 	}
	// }
	// return false
}

// isExampleOrTest returns true if the string passed in begins with "Example" or "Test", this is used
// to help exclude these functions from the API view output
func isExampleOrTest(s string) bool {
	return strings.Contains(s, "Example") || strings.Contains(s, "Test")
}

func (c *content) parseFunc(tokenList *[]Token) {
	keys := make([]string, 0, len(c.Funcs))
	for k := range c.Funcs {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for i, k := range keys {
		if isOnUnexportedMember(k) || isExampleOrTest(k) {
			copy(keys[i:], keys[i+1:])
			keys[len(keys)-1] = ""
			keys = keys[:len(keys)-1]
			delete(c.Funcs, k)
		}
	}
	for _, k := range keys {
		makeToken(nil, nil, "", newline, tokenList)
		makeToken(nil, nil, " ", whitespace, tokenList)
		makeToken(nil, nil, "", newline, tokenList)
		makeFuncTokens(&k, c.Funcs[k].Params, c.Funcs[k].Returns, c.Funcs[k].ReturnsNum, tokenList)
	}
}

// generateNavChildItems will loop through all the consts, interfaces, structs and global functions
// to create the navigation items that will be displayed in the API view.
// For consts, a navigation item will be by const type.
// For interfaces, a navigation item will point to the interface definition.
// For structs, a navigation item will only point to the struct definition and not methods or functions related to the struct.
// For funcs, global funcs that are not constructors for any structs will have a direct navigation item.
func (c *content) generateNavChildItems() []Navigation {
	childItems := []Navigation{}
	types := []string{}
	keys := []string{}
	for _, s := range c.Consts {
		keys = append(keys, s.Type)
	}
	sort.Strings(keys)
	for _, k := range keys {
		if !includesType(types, k) {
			types = append(types, k)
			temp := k
			childItems = append(childItems, Navigation{
				Text:         &temp,
				NavigationId: &temp,
				ChildItems:   []Navigation{},
				Tags: &map[string]string{
					"TypeKind": "enum",
				},
			})
		}
	}
	keys = []string{}
	for i := range c.Interfaces {
		keys = append(keys, i)
	}
	sort.Strings(keys)
	for k := range keys {
		childItems = append(childItems, Navigation{
			Text:         &keys[k],
			NavigationId: &keys[k],
			ChildItems:   []Navigation{},
			Tags: &map[string]string{
				"TypeKind": "interface",
			},
		})
	}
	keys = []string{}
	for i := range c.Structs {
		keys = append(keys, i)
	}
	sort.Strings(keys)
	for k := range keys {
		childItems = append(childItems, Navigation{
			Text:         &keys[k],
			NavigationId: &keys[k],
			ChildItems:   []Navigation{},
			Tags: &map[string]string{
				"TypeKind": "struct",
			},
		})
	}
	keys = []string{}
	for i := range c.Funcs {
		keys = append(keys, i)
	}
	sort.Strings(keys)
	for k := range keys {
		childItems = append(childItems, Navigation{
			Text:         &keys[k],
			NavigationId: &keys[k],
			ChildItems:   []Navigation{},
			Tags: &map[string]string{
				"TypeKind": "unknown",
			},
		})
	}
	return childItems
}
<commit_msg>Move clients up (#1382)<commit_after>// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package cmd

import (
	"fmt"
	"go/ast"
	"regexp"
	"sort"
	"strings"
)

// skip adding the const type in the token list
const skip = "Untyped const"

// newContent returns an initialized Content object.
func newContent() content {
	return content{
		Consts:     make(map[string]Const),
		Funcs:      make(map[string]Func),
		Interfaces: make(map[string]Interface),
		Structs:    make(map[string]Struct),
	}
}

// isEmpty returns true if there is no content in any of the fields.
func (c content) isEmpty() bool {
	return len(c.Consts) == 0 && len(c.Funcs) == 0 && len(c.Interfaces) == 0 && len(c.Structs) == 0
}

// adds the specified const declaration to the exports list
func (c *content) addConst(pkg pkg, g *ast.GenDecl) {
	for _, s := range g.Specs {
		co := Const{}
		vs := s.(*ast.ValueSpec)
		v := ""
		// Type is nil for untyped consts
		if vs.Type != nil {
			switch x := vs.Type.(type) {
			case *ast.Ident:
				co.Type = x.Name
				v = vs.Values[0].(*ast.BasicLit).Value
			case *ast.SelectorExpr:
				co.Type = x.Sel.Name
				v = vs.Values[0].(*ast.BasicLit).Value
			default:
				panic(fmt.Sprintf("wrong type %T", vs.Type))
			}

		} else {
			// get the type from the token type
			if bl, ok := vs.Values[0].(*ast.BasicLit); ok {
				co.Type = skip
				v = bl.Value
			} else if ce, ok := vs.Values[0].(*ast.CallExpr); ok {
				// const FooConst = FooType("value")
				co.Type = pkg.getText(ce.Fun.Pos(), ce.Fun.End())
				v = pkg.getText(ce.Args[0].Pos(), ce.Args[0].End())
			} else if ce, ok := vs.Values[0].(*ast.BinaryExpr); ok {
				// const FooConst = "value" + Bar
				co.Type = skip
				v = pkg.getText(ce.X.Pos(), ce.Y.End())
			} else if ce, ok := vs.Values[0].(*ast.UnaryExpr); ok {
				// const FooConst = -1
				co.Type = skip
				v = pkg.getText(ce.Pos(), ce.End())
			} else {
				panic("unhandled case for adding constant")
			}
		}
		co.Value = v
		c.Consts[vs.Names[0].Name] = co
	}
}

func includesType(s []string, t string) bool {
	for _, j := range s {
		if j == t {
			return true
		}
	}
	return false
}

func (c *content) parseConst(tokenList *[]Token) {
	if len(c.Consts) > 0 {
		// create keys slice in order to later sort consts by their types
		keys := []string{}
		// create types slice in order to be able to separate consts by the type they represent
		types := []string{}
		for i, s := range c.Consts {
			keys = append(keys, i)
			if !includesType(types, s.Type) {
				types = append(types, s.Type)
			}
		}
		sort.Strings(keys)
		sort.Strings(types)
		// finalKeys will order const keys by their type
		finalKeys := []string{}
		for _, t := range types {
			for _, k := range keys {
				if t == c.Consts[k].Type {
					finalKeys = append(finalKeys, k)
				}
			}
		}
		for _, t := range types {
			// this token parsing is performed so that const declarations of different types are declared
			// in their own const block to make them easier to click on
			n := t
			makeToken(nil, nil, "", 1, tokenList)
			makeToken(nil, nil, " ", whitespace, tokenList)
			makeToken(nil, nil, "", 1, tokenList)
			makeToken(&n, nil, "const", keyword, tokenList)
			makeToken(nil, nil, " ", whitespace, tokenList)
			makeToken(nil, nil, "(", punctuation, tokenList)
			makeToken(nil, nil, "", 1, tokenList)
			for _, v := range finalKeys {
				if c.Consts[v].Type == t {
					makeConstTokens(&v, c.Consts[v], tokenList)
				}
			}
			makeToken(nil, nil, ")", punctuation, tokenList)
			makeToken(nil, nil, "", 1, tokenList)
		}
	}

}

// adds the specified function declaration to the exports list
func (c *content) addFunc(pkg pkg, f *ast.FuncDecl) {
	// create a method sig, for methods it's a combination of the receiver type
	// with the function name e.g. "FooReceiver.Method", else just the function name.
	sig := ""
	if f.Recv != nil {
		sig = "(c "
		sig += pkg.getText(f.Recv.List[0].Type.Pos(), f.Recv.List[0].Type.End())
		// CP: changed to space, was a period before
		sig += ") "
	}
	sig += f.Name.Name
	c.Funcs[sig] = pkg.buildFunc(f.Type)
}

// adds the specified interface type to the exports list.
func (c *content) addInterface(pkg pkg, name string, i *ast.InterfaceType) {
	in := Interface{Methods: map[string]Func{}}
	if i.Methods != nil {
		for _, m := range i.Methods.List {
			if len(m.Names) > 0 {
				n := m.Names[0].Name
				f := pkg.buildFunc(m.Type.(*ast.FuncType))
				in.Methods[n] = f
			} else {
				n := pkg.getText(m.Type.Pos(), m.Type.End())
				in.EmbeddedInterfaces = append(in.EmbeddedInterfaces, n)
			}
		}
	}
	c.Interfaces[name] = in
}

// adds the specified struct type to the exports list.
func (c *content) parseInterface(tokenList *[]Token) {
	keys := []string{}
	for s := range c.Interfaces {
		keys = append(keys, s)
	}
	sort.Strings(keys)
	for _, k := range keys {
		makeInterfaceTokens(&k, c.Interfaces[k].EmbeddedInterfaces, c.Interfaces[k].Methods, tokenList)
	}
}

// adds the specified struct type to the exports list.
func (c *content) addStruct(pkg pkg, name string, s *ast.StructType) {
	sd := Struct{}
	// assumes all struct types have fields
	pkg.translateFieldList(s.Fields.List, func(n *string, t string) {
		if n == nil {
			sd.AnonymousFields = append(sd.AnonymousFields, t)
		} else {
			if sd.Fields == nil {
				sd.Fields = map[string]string{}
			}
			sd.Fields[*n] = t
		}
	})
	c.Structs[name] = sd
}

// adds the specified struct type to the exports list.
func (c *content) parseStruct(tokenList *[]Token) {
	keys := make([]string, 0, len(c.Structs))
	for k := range c.Structs {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	clients := []string{}
	for i, k := range keys {
		if strings.HasSuffix(k, "Client") {
			clients = append(clients, k)
			keys = append(keys[:i], keys[i+1:]...)
		}
	}
	clients = append(clients, keys...)
	for _, k := range clients {
		makeStructTokens(&k, c.Structs[k].AnonymousFields, c.Structs[k].Fields, tokenList)
		c.searchForCtors(k, tokenList)
		c.searchForMethods(k, tokenList)
	}
}

// searchForCtors will search through all of the exported Funcs for a constructor for the name of the
// type that is passed as a param.
func (c *content) searchForCtors(s string, tokenList *[]Token) {
	for i, f := range c.Funcs {
		n := getCtorName(i)
		if s == n {
			makeFuncTokens(&i, f.Params, f.Returns, f.ReturnsNum, tokenList)
			delete(c.Funcs, i)
			return
		}
	}
}

// searchForMethods takes the name of the receiver and looks for Funcs that are methods on that receiver.
func (c *content) searchForMethods(s string, tokenList *[]Token) {
	for i, f := range c.Funcs {
		v, n := getReceiverName(i)
		isPointer := false
		if strings.HasPrefix(n, "*") {
			n = n[1:]
			isPointer = true
		}
		if s == n {
			makeMethodTokens(v, n, isPointer, getMethodName(i), f.Params, f.Returns, f.ReturnsNum, tokenList)
			delete(c.Funcs, i)
		}
		if isOnUnexportedMember(i) || isExampleOrTest(i) {
			delete(c.Funcs, i)
		}
	}
}

// getCtorName returns the name of a constructor without the New prefix.
// TODO improve this to also check the return statement on the constructor to make sure it does in fact only
// return the name of constructors and not other functions that begin with New
func getCtorName(s string) string {
	if strings.HasPrefix(s, "New") {
		ctor := s[3:]
		return ctor
	}
	return ""
}

// getReceiverName returns the components of the receiver on a method signature
// i.e.: (c *Foo) Bar(s string) will return "c" and "Foo".
func getReceiverName(s string) (receiverVar string, receiver string) {
	if strings.HasPrefix(s, "(") {
		parts := strings.Split(s[:strings.Index(s, ")")], " ")
		receiverVar = parts[0][1:]
		receiver = parts[1]
		return
	}
	return "", ""
}

// getMethodName expects a method signature in the param s and removes the receiver portion of the
// method signature before returning the method name.
func getMethodName(s string) string {
	pos := strings.Index(s, ")")
	// return the string after the first ) and add an extra index to omit the space after the receiver
	return s[pos+2:]
}

// isOnUnexportedMember checks for method signatures that are on unexported types,
// it will return true if the method is unexported.
func isOnUnexportedMember(s string) bool {
	r := regexp.MustCompile(`(\(([a-z|A-Z]{1}) (\*){0,1}([a-z]+)([a-z|A-Z]*)\))`)
	return r.MatchString(s)
	// for _, l := range s {
	// 	if unicode.IsLetter(l) {
	// 		return string(l) == strings.ToLower(string(l))
	// 	}
	// }
	// return false
}

// isExampleOrTest returns true if the string passed in begins with "Example" or "Test", this is used
// to help exclude these functions from the API view output
func isExampleOrTest(s string) bool {
	return strings.Contains(s, "Example") || strings.Contains(s, "Test")
}

func (c *content) parseFunc(tokenList *[]Token) {
	keys := make([]string, 0, len(c.Funcs))
	for k := range c.Funcs {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for i, k := range keys {
		if isOnUnexportedMember(k) || isExampleOrTest(k) {
			copy(keys[i:], keys[i+1:])
			keys[len(keys)-1] = ""
			keys = keys[:len(keys)-1]
			delete(c.Funcs, k)
		}
	}
	for _, k := range keys {
		makeToken(nil, nil, "", newline, tokenList)
		makeToken(nil, nil, " ", whitespace, tokenList)
		makeToken(nil, nil, "", newline, tokenList)
		makeFuncTokens(&k, c.Funcs[k].Params, c.Funcs[k].Returns, c.Funcs[k].ReturnsNum, tokenList)
	}
}

// generateNavChildItems will loop through all the consts, interfaces, structs and global functions
// to create the navigation items that will be displayed in the API view.
// For consts, a navigation item will be by const type.
// For interfaces, a navigation item will point to the interface definition.
// For structs, a navigation item will only point to the struct definition and not methods or functions related to the struct.
// For funcs, global funcs that are not constructors for any structs will have a direct navigation item.
func (c *content) generateNavChildItems() []Navigation {
	childItems := []Navigation{}
	types := []string{}
	keys := []string{}
	for _, s := range c.Consts {
		keys = append(keys, s.Type)
	}
	sort.Strings(keys)
	for _, k := range keys {
		if !includesType(types, k) {
			types = append(types, k)
			temp := k
			childItems = append(childItems, Navigation{
				Text:         &temp,
				NavigationId: &temp,
				ChildItems:   []Navigation{},
				Tags: &map[string]string{
					"TypeKind": "enum",
				},
			})
		}
	}
	keys = []string{}
	for i := range c.Interfaces {
		keys = append(keys, i)
	}
	sort.Strings(keys)
	for k := range keys {
		childItems = append(childItems, Navigation{
			Text:         &keys[k],
			NavigationId: &keys[k],
			ChildItems:   []Navigation{},
			Tags: &map[string]string{
				"TypeKind": "interface",
			},
		})
	}
	keys = []string{}
	for i := range c.Structs {
		keys = append(keys, i)
	}
	sort.Strings(keys)
	clientsFirst := []string{}
	for i, k := range keys {
		if strings.HasSuffix(k, "Client") {
			clientsFirst = append(clientsFirst, k)
			keys = append(keys[:i], keys[i+1:]...)
		}
	}
	clientsFirst = append(clientsFirst, keys...)
	for k := range clientsFirst {
		childItems = append(childItems, Navigation{
			Text:         &clientsFirst[k],
			NavigationId: &clientsFirst[k],
			ChildItems:   []Navigation{},
			Tags: &map[string]string{
				"TypeKind": "struct",
			},
		})
	}
	keys = []string{}
	for i := range c.Funcs {
		keys = append(keys, i)
	}
	sort.Strings(keys)
	for k := range keys {
		childItems = append(childItems, Navigation{
			Text:         &keys[k],
			NavigationId: &keys[k],
			ChildItems:   []Navigation{},
			Tags: &map[string]string{
				"TypeKind": "unknown",
			},
		})
	}
	return childItems
}
<|endoftext|>