<commit_before>//
// Copyright 2021, Kordian Bruck
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"time"
)

// PackagesService handles communication with the packages related methods
// of the GitLab API.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/packages.html
type PackagesService struct {
	client *Client
}

// Package represents a GitLab single package
//
// GitLab API docs: https://docs.gitlab.com/ee/api/packages.html
type Package struct {
	ID        int        `json:"id"`
	Name      string     `json:"name"`
	Version   string     `json:"version"`
	Type      string     `json:"package_type"`
	CreatedAt *time.Time `json:"created_at"`
}

func (s Package) String() string {
	return Stringify(s)
}

// PackageFile represents one file contained within a package
//
// GitLab API docs: https://docs.gitlab.com/ee/api/packages.html
type PackageFile struct {
	ID        int        `json:"id"`
	PackageId int        `json:"package_id"`
	CreatedAt *time.Time `json:"created_at"`
	FileName  string     `json:"file_name"`
	Size      int        `json:"size"`
	MD5       string     `json:"file_md5"`
	SHA1      string     `json:"file_sha1"`
}

func (s PackageFile) String() string {
	return Stringify(s)
}

// ListProjectPackagesOptions are the parameters available in a ListProjectPackages() Operation
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#within-a-project
type ListProjectPackagesOptions struct {
	ListOptions
	OrderBy            string `url:"order_by,omitempty" json:"order_by,omitempty"`
	Sort               string `url:"sort,omitempty" json:"sort,omitempty"`
	Type               string `url:"package_type,omitempty" json:"package_type,omitempty"`
	Name               string `url:"package_name,omitempty" json:"package_name,omitempty"`
	IncludeVersionless bool   `url:"include_versionless,omitempty" json:"include_versionless,omitempty"`
}

// ListProjectPackages gets a list of packages in a project.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#within-a-project
func (s *PackagesService) ListProjectPackages(pid interface{}, opt *ListProjectPackagesOptions, options ...RequestOptionFunc) ([]*Package, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/packages", pathEscape(project))

	req, err := s.client.NewRequest("GET", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	var packages []*Package
	resp, err := s.client.Do(req, &packages)
	if err != nil {
		return nil, resp, err
	}

	return packages, resp, err
}

// DeleteRegistryRepository deletes a repository in a registry.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#delete-a-project-package
func (s *PackagesService) DeleteProjectPackage(pid interface{}, packageID int, options ...RequestOptionFunc) (*Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("projects/%s/packages/%d", pathEscape(project), packageID)

	req, err := s.client.NewRequest("DELETE", u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}

// ListPackageFilesOptions represents the available
// ListPackageFiles() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#list-package-files
type ListPackageFilesOptions ListOptions

// ListPackageFiles gets a list of files that are within a package
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#list-package-files
func (s *PackagesService) ListPackageFiles(pid interface{}, packageID int, opt *ListPackageFilesOptions, options ...RequestOptionFunc) ([]*PackageFile, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/packages/%d/package_files",
		pathEscape(project),
		packageID,
	)

	req, err := s.client.NewRequest("GET", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	var tags []*PackageFile
	resp, err := s.client.Do(req, &tags)
	if err != nil {
		return nil, resp, err
	}

	return tags, resp, err
}
<commit_msg>lint: One more PackageID.<commit_after>//
// Copyright 2021, Kordian Bruck
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"time"
)

// PackagesService handles communication with the packages related methods
// of the GitLab API.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/packages.html
type PackagesService struct {
	client *Client
}

// Package represents a GitLab single package
//
// GitLab API docs: https://docs.gitlab.com/ee/api/packages.html
type Package struct {
	ID        int        `json:"id"`
	Name      string     `json:"name"`
	Version   string     `json:"version"`
	Type      string     `json:"package_type"`
	CreatedAt *time.Time `json:"created_at"`
}

func (s Package) String() string {
	return Stringify(s)
}

// PackageFile represents one file contained within a package
//
// GitLab API docs: https://docs.gitlab.com/ee/api/packages.html
type PackageFile struct {
	ID        int        `json:"id"`
	PackageID int        `json:"package_id"`
	CreatedAt *time.Time `json:"created_at"`
	FileName  string     `json:"file_name"`
	Size      int        `json:"size"`
	MD5       string     `json:"file_md5"`
	SHA1      string     `json:"file_sha1"`
}

func (s PackageFile) String() string {
	return Stringify(s)
}

// ListProjectPackagesOptions are the parameters available in a ListProjectPackages() Operation
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#within-a-project
type ListProjectPackagesOptions struct {
	ListOptions
	OrderBy            string `url:"order_by,omitempty" json:"order_by,omitempty"`
	Sort               string `url:"sort,omitempty" json:"sort,omitempty"`
	Type               string `url:"package_type,omitempty" json:"package_type,omitempty"`
	Name               string `url:"package_name,omitempty" json:"package_name,omitempty"`
	IncludeVersionless bool   `url:"include_versionless,omitempty" json:"include_versionless,omitempty"`
}

// ListProjectPackages gets a list of packages in a project.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#within-a-project
func (s *PackagesService) ListProjectPackages(pid interface{}, opt *ListProjectPackagesOptions, options ...RequestOptionFunc) ([]*Package, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/packages", pathEscape(project))

	req, err := s.client.NewRequest("GET", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	var packages []*Package
	resp, err := s.client.Do(req, &packages)
	if err != nil {
		return nil, resp, err
	}

	return packages, resp, err
}

// DeleteRegistryRepository deletes a repository in a registry.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#delete-a-project-package
func (s *PackagesService) DeleteProjectPackage(pid interface{}, packageID int, options ...RequestOptionFunc) (*Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("projects/%s/packages/%d", pathEscape(project), packageID)

	req, err := s.client.NewRequest("DELETE", u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}

// ListPackageFilesOptions represents the available
// ListPackageFiles() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#list-package-files
type ListPackageFilesOptions ListOptions

// ListPackageFiles gets a list of files that are within a package
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#list-package-files
func (s *PackagesService) ListPackageFiles(pid interface{}, packageID int, opt *ListPackageFilesOptions, options ...RequestOptionFunc) ([]*PackageFile, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/packages/%d/package_files",
		pathEscape(project),
		packageID,
	)

	req, err := s.client.NewRequest("GET", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	var tags []*PackageFile
	resp, err := s.client.Do(req, &tags)
	if err != nil {
		return nil, resp, err
	}

	return tags, resp, err
}
<|endoftext|>


<commit_before>/*
 * Copyright 2017 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package languageserver provides an implementation of the Language Server
// Protocol v3.0 (https://github.com/Microsoft/language-server-protocol)
// This server implements the following capabilities:
// 		textDocumentSync (full)
//		referenceProvider
package languageserver

import (
	"context"
	"fmt"
	"log"

	"kythe.io/kythe/go/services/xrefs"
	"kythe.io/kythe/go/util/kytheuri"
	"kythe.io/kythe/go/util/markedsource"

	cpb "kythe.io/kythe/proto/common_proto"
	xpb "kythe.io/kythe/proto/xref_proto"

	"github.com/sourcegraph/go-langserver/pkg/lsp"
)

// Server provides a Language Server for interacting with data in a Kythe index
type Server struct {
	docs             map[string]*document
	paths            pathConfig
	XRefs            xrefs.Service
	settingsProvider SettingsProvider
}

// NewServer constructs a Server object
func NewServer(x xrefs.Service, s SettingsProvider) Server {
	var p pathConfig
	return Server{
		docs:             make(map[string]*document),
		paths:            p,
		XRefs:            x,
		settingsProvider: s,
	}
}

// Initialize is invoked before any other methods, and allows the Server to
// receive configuration info (such as the project root) and announce its capabilities.
func (ls *Server) Initialize(params lsp.InitializeParams) (*lsp.InitializeResult, error) {
	log.Println("Server Initializing...")

	local, err := ls.paths.localFromURI(params.Root())
	if err != nil {
		return nil, err
	}

	s, err := ls.settingsProvider(local)
	if err != nil {
		return nil, err
	}

	log.Printf("Server Settings: %#v", s)

	err = ls.paths.loadSettings(*s)
	if err != nil {
		return nil, err
	}

	log.Printf("Root found at %q", ls.paths.root)

	fullSync := lsp.TDSKFull
	return &lsp.InitializeResult{
		Capabilities: lsp.ServerCapabilities{
			TextDocumentSync: lsp.TextDocumentSyncOptionsOrKind{
				Kind:    &fullSync,
				Options: nil,
			},
			ReferencesProvider: true,
			HoverProvider:      true,
		},
	}, nil
}

// TextDocumentDidOpen allows the client to inform the Server that a file has
// been opened. The Kythe Language Server uses this time to fetch file
// decorations.
func (ls *Server) TextDocumentDidOpen(params lsp.DidOpenTextDocumentParams) error {
	local, err := ls.paths.localFromURI(params.TextDocument.URI)
	if err != nil {
		return fmt.Errorf("failed creating local path from URI %q:\n%v", params.TextDocument.URI, err)

	}

	ticket, err := ls.paths.kytheURIFromLocal(local)
	if err != nil {
		return err
	}

	dec, err := ls.XRefs.Decorations(context.TODO(), &xpb.DecorationsRequest{
		Location: &xpb.Location{
			Ticket: ticket.String(),
		},
		References:        true,
		TargetDefinitions: true,
		SourceText:        true,
	})

	if err != nil {
		return fmt.Errorf("failed to find xrefs for %q:\n%v", local, err)
	}

	log.Printf("Server returned %d refs in file %q", len(dec.Reference), local)

	var refs []*RefResolution
	for _, r := range dec.Reference {
		if r.Span == nil {
			continue
		}

		rng := spanToRange(r.Span)
		if rng == nil {
			continue
		}

		refs = append(refs, &RefResolution{
			ticket:   r.TargetTicket,
			def:      r.TargetDefinition,
			oldRange: *rng,
		})
	}

	defLocs := ls.defLocations(dec.DefinitionLocations)

	log.Printf("Found %d refs in file %q", len(refs), ticket.String())
	ls.docs[local] = newDocument(refs, string(dec.SourceText), params.TextDocument.Text, defLocs)
	return nil
}

// TextDocumentDidChange is called when the client edits a file. The Kythe
// Language Server simply stores the new content and marks the file as dirty
func (ls *Server) TextDocumentDidChange(params lsp.DidChangeTextDocumentParams) error {
	local, err := ls.paths.localFromURI(params.TextDocument.URI)
	if err != nil {
		return err
	}

	if doc, ok := ls.docs[local]; ok {
		// Because the sync Kind is full text, each change notification
		// contains exactly 1 change with the full text of the document
		doc.updateSource(params.ContentChanges[0].Text)
	} else {
		return fmt.Errorf("change notification received for file that was never opened %q", local)
	}

	return nil
}

// TextDocumentDidClose removes all cached information about the open document. Because
// all information extracted from documents are stored internally to the document object,
// this removal shouldn't leak memory
func (ls *Server) TextDocumentDidClose(params lsp.DidCloseTextDocumentParams) error {
	log.Printf("Document close notification received: %v", params)
	local, err := ls.paths.localFromURI(params.TextDocument.URI)
	if err != nil {
		return err
	}

	delete(ls.docs, local)
	return nil
}

// TextDocumentReferences uses a position in code to produce a list of
// locations throughout the project that reference the same semantic node. This
// can trigger a diff if the source file is dirty.
//
// NOTE: As per the lsp spec, references must return an error or a valid array.
// Therefore, if no error is returned, a non-nil location slice must be returned
func (ls *Server) TextDocumentReferences(params lsp.ReferenceParams) ([]lsp.Location, error) {
	log.Printf("Searching for references at %v", params.TextDocumentPositionParams)

	local, err := ls.paths.localFromURI(params.TextDocument.URI)
	if err != nil {
		return nil, err
	}

	// If we don't have decorations we can't find references
	doc, exists := ls.docs[local]
	if !exists {
		log.Printf("References requested from unknown file %q", local)
		return []lsp.Location{}, nil
	}

	ref := doc.xrefs(params.Position)
	if ref == nil {
		return []lsp.Location{}, nil
	}

	xrefs, err := ls.XRefs.CrossReferences(context.TODO(), &xpb.CrossReferencesRequest{
		Ticket:          []string{ref.ticket},
		DeclarationKind: xpb.CrossReferencesRequest_ALL_DECLARATIONS,
		DefinitionKind:  xpb.CrossReferencesRequest_BINDING_DEFINITIONS,
		ReferenceKind:   xpb.CrossReferencesRequest_NON_CALL_REFERENCES,
		PageSize:        50, // TODO(djrenren): make this configurable
	})
	if err != nil {
		return nil, fmt.Errorf("failed to find xrefs for ticket %q:\n%v", ref.ticket, err)
	}

	refs := xrefs.CrossReferences[ref.ticket]
	if refs == nil {
		log.Printf("XRef service provided no xrefs for ticket %q", ref.ticket)
		return []lsp.Location{}, nil
	}

	return ls.refLocs(refs), nil
}

// TextDocumentDefinition uses a position in code to produce a list of
// locations throughout the project that define the semantic node at the original position.
// This can trigger a diff if the source file is dirty
//
// NOTE: As per the lsp spec, definition must return an error or a non-null result.
// Therefore, if no error is returned, a non-nil location slice must be returned
func (ls *Server) TextDocumentDefinition(params lsp.TextDocumentPositionParams) ([]lsp.Location, error) {
	log.Printf("Searching for definition at %v", params)
	local, err := ls.paths.localFromURI(params.TextDocument.URI)
	if err != nil {
		return []lsp.Location{}, err
	}

	// If we don't have decorations we can't find definitions
	doc, exists := ls.docs[local]
	if !exists {
		log.Printf("References requested from unknown file %q", local)
		return []lsp.Location{}, nil
	}

	// If there's no ref at the location we don't have definitions
	ref := doc.xrefs(params.Position)
	if ref == nil {
		log.Printf("No ref found at %v", params.Position)
		return []lsp.Location{}, nil
	}

	// If the ref's target definition is in the document's definition locations
	// we can return the loc without a service request
	if l, ok := doc.defLocs[ref.def]; ok {
		log.Printf("Found target definition for %q locally: %q at %v", ref.ticket, ref.def, *l)
		defLocal, err := ls.paths.localFromURI(l.URI)

		if err != nil {
			return []lsp.Location{}, nil
		}

		// If we have the doc containing the reference, map it to its new location
		if defDoc, ok := ls.docs[defLocal]; ok {
			newRange := defDoc.rangeInNewSource(l.Range)
			loc := *l
			if newRange != nil {
				loc.Range = *newRange
				return []lsp.Location{loc}, nil
			}

			// There definition range doesn't exist anymore
			return []lsp.Location{}, nil
		}

		// We don't how to map it so we just return the location from Kythe
		return []lsp.Location{*l}, nil
	}

	xrefs, err := ls.XRefs.CrossReferences(context.TODO(), &xpb.CrossReferencesRequest{
		Ticket:          []string{ref.ticket},
		DeclarationKind: xpb.CrossReferencesRequest_ALL_DECLARATIONS,
		DefinitionKind:  xpb.CrossReferencesRequest_BINDING_DEFINITIONS,
	})

	if err != nil {
		return nil, fmt.Errorf("failed to find xrefs for ticket %q: %v", ref.ticket, err)
	}

	refs := xrefs.CrossReferences[ref.ticket]
	if refs == nil {
		log.Printf("XRef service provided no xrefs for ticket %q", ref.ticket)
		return []lsp.Location{}, nil
	}

	return ls.refLocs(refs), nil
}

// TextDocumentHover produces a documentation string for the entity referenced at a given location
func (ls *Server) TextDocumentHover(params lsp.TextDocumentPositionParams) (lsp.Hover, error) {
	local, err := ls.paths.localFromURI(params.TextDocument.URI)
	if err != nil {
		return lsp.Hover{}, err
	}

	// If we don't have decorations we can't find documentation
	doc, exists := ls.docs[local]
	if !exists {
		log.Printf("References requested from unknown file %q", local)
		return lsp.Hover{}, nil
	}

	// If there's no ref at the location we don't have documentation
	ref := doc.xrefs(params.Position)
	if ref == nil {
		log.Printf("No ref found at %v", params.Position)
		return lsp.Hover{}, nil
	}

	docReply, err := ls.XRefs.Documentation(context.TODO(), &xpb.DocumentationRequest{
		Ticket: []string{ref.ticket},
	})
	if err != nil {
		log.Printf("Error fetching documentation for %q: %v", ref.ticket, err)
		return lsp.Hover{}, nil
	}

	if len(docReply.Document) < 1 || docReply.Document[0].MarkedSource == nil {
		log.Printf("No Documentation found for %q", ref.ticket)
		return lsp.Hover{}, nil
	}

	kuri, err := kytheuri.Parse(docReply.Document[0].Ticket)
	if err != nil {
		log.Printf("Invalid ticket returned from documentation request: %v", err)
		return lsp.Hover{}, nil
	}

	sig := markedsource.Render(docReply.Document[0].MarkedSource)

	return lsp.Hover{
		Contents: []lsp.MarkedString{{
			Language: kuri.Language,
			Value:    sig,
		}},
		Range: ref.newRange,
	}, nil
}

func (ls *Server) anchorToLoc(a *xpb.Anchor) *lsp.Location {
	if a == nil || a.Span == nil {
		return nil
	}

	r := spanToRange(a.Span)
	if r == nil {
		return nil
	}

	ticket, err := kytheuri.Parse(a.Parent)
	if err != nil {
		return nil
	}

	local, err := ls.paths.localFromKytheURI(*ticket)
	if err != nil {
		return nil
	}

	// Map the old location to its new location if we can
	if doc, ok := ls.docs[local]; ok {
		if newRange := doc.rangeInNewSource(*r); newRange != nil {
			r = newRange
		} else {
			return nil
		}
	}
	return &lsp.Location{
		URI:   lsp.DocumentURI(fmt.Sprintf("file://%s", local)),
		Range: *r,
	}

}

func spanToRange(s *cpb.Span) *lsp.Range {
	if s == nil || s.Start == nil || s.End == nil {
		return nil
	}

	// LineNumber is 1 indexed, so 0 indicates unknown which
	// means it can't be used for lookups so we discard
	if s.Start.LineNumber == 0 || s.End.LineNumber == 0 {
		return nil
	}

	return &lsp.Range{
		Start: lsp.Position{
			Line:      int(s.Start.LineNumber - 1),
			Character: int(s.Start.ColumnOffset),
		},
		End: lsp.Position{
			Line:      int(s.End.LineNumber - 1),
			Character: int(s.End.ColumnOffset),
		},
	}
}

func (ls *Server) defLocations(t map[string]*xpb.Anchor) map[string]*lsp.Location {
	m := make(map[string]*lsp.Location)
	for k, v := range t {
		l := ls.anchorToLoc(v)
		if l != nil {
			m[k] = l
		}
	}
	return m
}

// refLocs takes a cross reference set and produces a slice of locations. This slice
// is guaranteed to be non-null even if empty
func (ls *Server) refLocs(r *xpb.CrossReferencesReply_CrossReferenceSet) []lsp.Location {
	locs := []lsp.Location{}
	for _, a := range r.Reference {
		l := ls.anchorToLoc(a.Anchor)
		if l != nil {
			locs = append(locs, *l)
		}
	}

	for _, a := range r.Definition {
		l := ls.anchorToLoc(a.Anchor)
		if l != nil {
			locs = append(locs, *l)
		}
	}

	for _, a := range r.Declaration {
		l := ls.anchorToLoc(a.Anchor)
		if l != nil {
			locs = append(locs, *l)
		}
	}

	return locs
}
<commit_msg>Add definitions as a capability of the server<commit_after>/*
 * Copyright 2017 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package languageserver provides an implementation of the Language Server
// Protocol v3.0 (https://github.com/Microsoft/language-server-protocol)
// This server implements the following capabilities:
// 		textDocumentSync (full)
//		referenceProvider
package languageserver

import (
	"context"
	"fmt"
	"log"

	"kythe.io/kythe/go/services/xrefs"
	"kythe.io/kythe/go/util/kytheuri"
	"kythe.io/kythe/go/util/markedsource"

	cpb "kythe.io/kythe/proto/common_proto"
	xpb "kythe.io/kythe/proto/xref_proto"

	"github.com/sourcegraph/go-langserver/pkg/lsp"
)

// Server provides a Language Server for interacting with data in a Kythe index
type Server struct {
	docs             map[string]*document
	paths            pathConfig
	XRefs            xrefs.Service
	settingsProvider SettingsProvider
}

// NewServer constructs a Server object
func NewServer(x xrefs.Service, s SettingsProvider) Server {
	var p pathConfig
	return Server{
		docs:             make(map[string]*document),
		paths:            p,
		XRefs:            x,
		settingsProvider: s,
	}
}

// Initialize is invoked before any other methods, and allows the Server to
// receive configuration info (such as the project root) and announce its capabilities.
func (ls *Server) Initialize(params lsp.InitializeParams) (*lsp.InitializeResult, error) {
	log.Println("Server Initializing...")

	local, err := ls.paths.localFromURI(params.Root())
	if err != nil {
		return nil, err
	}

	s, err := ls.settingsProvider(local)
	if err != nil {
		return nil, err
	}

	log.Printf("Server Settings: %#v", s)

	err = ls.paths.loadSettings(*s)
	if err != nil {
		return nil, err
	}

	log.Printf("Root found at %q", ls.paths.root)

	fullSync := lsp.TDSKFull
	return &lsp.InitializeResult{
		Capabilities: lsp.ServerCapabilities{
			TextDocumentSync: lsp.TextDocumentSyncOptionsOrKind{
				Kind:    &fullSync,
				Options: nil,
			},
			ReferencesProvider: true,
			HoverProvider:      true,
			DefinitionProvider: true,
		},
	}, nil
}

// TextDocumentDidOpen allows the client to inform the Server that a file has
// been opened. The Kythe Language Server uses this time to fetch file
// decorations.
func (ls *Server) TextDocumentDidOpen(params lsp.DidOpenTextDocumentParams) error {
	local, err := ls.paths.localFromURI(params.TextDocument.URI)
	if err != nil {
		return fmt.Errorf("failed creating local path from URI %q:\n%v", params.TextDocument.URI, err)

	}

	ticket, err := ls.paths.kytheURIFromLocal(local)
	if err != nil {
		return err
	}

	dec, err := ls.XRefs.Decorations(context.TODO(), &xpb.DecorationsRequest{
		Location: &xpb.Location{
			Ticket: ticket.String(),
		},
		References:        true,
		TargetDefinitions: true,
		SourceText:        true,
	})

	if err != nil {
		return fmt.Errorf("failed to find xrefs for %q:\n%v", local, err)
	}

	log.Printf("Server returned %d refs in file %q", len(dec.Reference), local)

	var refs []*RefResolution
	for _, r := range dec.Reference {
		if r.Span == nil {
			continue
		}

		rng := spanToRange(r.Span)
		if rng == nil {
			continue
		}

		refs = append(refs, &RefResolution{
			ticket:   r.TargetTicket,
			def:      r.TargetDefinition,
			oldRange: *rng,
		})
	}

	defLocs := ls.defLocations(dec.DefinitionLocations)

	log.Printf("Found %d refs in file %q", len(refs), ticket.String())
	ls.docs[local] = newDocument(refs, string(dec.SourceText), params.TextDocument.Text, defLocs)
	return nil
}

// TextDocumentDidChange is called when the client edits a file. The Kythe
// Language Server simply stores the new content and marks the file as dirty
func (ls *Server) TextDocumentDidChange(params lsp.DidChangeTextDocumentParams) error {
	local, err := ls.paths.localFromURI(params.TextDocument.URI)
	if err != nil {
		return err
	}

	if doc, ok := ls.docs[local]; ok {
		// Because the sync Kind is full text, each change notification
		// contains exactly 1 change with the full text of the document
		doc.updateSource(params.ContentChanges[0].Text)
	} else {
		return fmt.Errorf("change notification received for file that was never opened %q", local)
	}

	return nil
}

// TextDocumentDidClose removes all cached information about the open document. Because
// all information extracted from documents are stored internally to the document object,
// this removal shouldn't leak memory
func (ls *Server) TextDocumentDidClose(params lsp.DidCloseTextDocumentParams) error {
	log.Printf("Document close notification received: %v", params)
	local, err := ls.paths.localFromURI(params.TextDocument.URI)
	if err != nil {
		return err
	}

	delete(ls.docs, local)
	return nil
}

// TextDocumentReferences uses a position in code to produce a list of
// locations throughout the project that reference the same semantic node. This
// can trigger a diff if the source file is dirty.
//
// NOTE: As per the lsp spec, references must return an error or a valid array.
// Therefore, if no error is returned, a non-nil location slice must be returned
func (ls *Server) TextDocumentReferences(params lsp.ReferenceParams) ([]lsp.Location, error) {
	log.Printf("Searching for references at %v", params.TextDocumentPositionParams)

	local, err := ls.paths.localFromURI(params.TextDocument.URI)
	if err != nil {
		return nil, err
	}

	// If we don't have decorations we can't find references
	doc, exists := ls.docs[local]
	if !exists {
		log.Printf("References requested from unknown file %q", local)
		return []lsp.Location{}, nil
	}

	ref := doc.xrefs(params.Position)
	if ref == nil {
		return []lsp.Location{}, nil
	}

	xrefs, err := ls.XRefs.CrossReferences(context.TODO(), &xpb.CrossReferencesRequest{
		Ticket:          []string{ref.ticket},
		DeclarationKind: xpb.CrossReferencesRequest_ALL_DECLARATIONS,
		DefinitionKind:  xpb.CrossReferencesRequest_BINDING_DEFINITIONS,
		ReferenceKind:   xpb.CrossReferencesRequest_NON_CALL_REFERENCES,
		PageSize:        50, // TODO(djrenren): make this configurable
	})
	if err != nil {
		return nil, fmt.Errorf("failed to find xrefs for ticket %q:\n%v", ref.ticket, err)
	}

	refs := xrefs.CrossReferences[ref.ticket]
	if refs == nil {
		log.Printf("XRef service provided no xrefs for ticket %q", ref.ticket)
		return []lsp.Location{}, nil
	}

	return ls.refLocs(refs), nil
}

// TextDocumentDefinition uses a position in code to produce a list of
// locations throughout the project that define the semantic node at the original position.
// This can trigger a diff if the source file is dirty
//
// NOTE: As per the lsp spec, definition must return an error or a non-null result.
// Therefore, if no error is returned, a non-nil location slice must be returned
func (ls *Server) TextDocumentDefinition(params lsp.TextDocumentPositionParams) ([]lsp.Location, error) {
	log.Printf("Searching for definition at %v", params)
	local, err := ls.paths.localFromURI(params.TextDocument.URI)
	if err != nil {
		return []lsp.Location{}, err
	}

	// If we don't have decorations we can't find definitions
	doc, exists := ls.docs[local]
	if !exists {
		log.Printf("References requested from unknown file %q", local)
		return []lsp.Location{}, nil
	}

	// If there's no ref at the location we don't have definitions
	ref := doc.xrefs(params.Position)
	if ref == nil {
		log.Printf("No ref found at %v", params.Position)
		return []lsp.Location{}, nil
	}

	// If the ref's target definition is in the document's definition locations
	// we can return the loc without a service request
	if l, ok := doc.defLocs[ref.def]; ok {
		log.Printf("Found target definition for %q locally: %q at %v", ref.ticket, ref.def, *l)
		defLocal, err := ls.paths.localFromURI(l.URI)

		if err != nil {
			return []lsp.Location{}, nil
		}

		// If we have the doc containing the reference, map it to its new location
		if defDoc, ok := ls.docs[defLocal]; ok {
			newRange := defDoc.rangeInNewSource(l.Range)
			loc := *l
			if newRange != nil {
				loc.Range = *newRange
				return []lsp.Location{loc}, nil
			}

			// There definition range doesn't exist anymore
			return []lsp.Location{}, nil
		}

		// We don't how to map it so we just return the location from Kythe
		return []lsp.Location{*l}, nil
	}

	xrefs, err := ls.XRefs.CrossReferences(context.TODO(), &xpb.CrossReferencesRequest{
		Ticket:          []string{ref.ticket},
		DeclarationKind: xpb.CrossReferencesRequest_ALL_DECLARATIONS,
		DefinitionKind:  xpb.CrossReferencesRequest_BINDING_DEFINITIONS,
	})

	if err != nil {
		return nil, fmt.Errorf("failed to find xrefs for ticket %q: %v", ref.ticket, err)
	}

	refs := xrefs.CrossReferences[ref.ticket]
	if refs == nil {
		log.Printf("XRef service provided no xrefs for ticket %q", ref.ticket)
		return []lsp.Location{}, nil
	}

	return ls.refLocs(refs), nil
}

// TextDocumentHover produces a documentation string for the entity referenced at a given location
func (ls *Server) TextDocumentHover(params lsp.TextDocumentPositionParams) (lsp.Hover, error) {
	local, err := ls.paths.localFromURI(params.TextDocument.URI)
	if err != nil {
		return lsp.Hover{}, err
	}

	// If we don't have decorations we can't find documentation
	doc, exists := ls.docs[local]
	if !exists {
		log.Printf("References requested from unknown file %q", local)
		return lsp.Hover{}, nil
	}

	// If there's no ref at the location we don't have documentation
	ref := doc.xrefs(params.Position)
	if ref == nil {
		log.Printf("No ref found at %v", params.Position)
		return lsp.Hover{}, nil
	}

	docReply, err := ls.XRefs.Documentation(context.TODO(), &xpb.DocumentationRequest{
		Ticket: []string{ref.ticket},
	})
	if err != nil {
		log.Printf("Error fetching documentation for %q: %v", ref.ticket, err)
		return lsp.Hover{}, nil
	}

	if len(docReply.Document) < 1 || docReply.Document[0].MarkedSource == nil {
		log.Printf("No Documentation found for %q", ref.ticket)
		return lsp.Hover{}, nil
	}

	kuri, err := kytheuri.Parse(docReply.Document[0].Ticket)
	if err != nil {
		log.Printf("Invalid ticket returned from documentation request: %v", err)
		return lsp.Hover{}, nil
	}

	sig := markedsource.Render(docReply.Document[0].MarkedSource)

	return lsp.Hover{
		Contents: []lsp.MarkedString{{
			Language: kuri.Language,
			Value:    sig,
		}},
		Range: ref.newRange,
	}, nil
}

func (ls *Server) anchorToLoc(a *xpb.Anchor) *lsp.Location {
	if a == nil || a.Span == nil {
		return nil
	}

	r := spanToRange(a.Span)
	if r == nil {
		return nil
	}

	ticket, err := kytheuri.Parse(a.Parent)
	if err != nil {
		return nil
	}

	local, err := ls.paths.localFromKytheURI(*ticket)
	if err != nil {
		return nil
	}

	// Map the old location to its new location if we can
	if doc, ok := ls.docs[local]; ok {
		if newRange := doc.rangeInNewSource(*r); newRange != nil {
			r = newRange
		} else {
			return nil
		}
	}
	return &lsp.Location{
		URI:   lsp.DocumentURI(fmt.Sprintf("file://%s", local)),
		Range: *r,
	}

}

func spanToRange(s *cpb.Span) *lsp.Range {
	if s == nil || s.Start == nil || s.End == nil {
		return nil
	}

	// LineNumber is 1 indexed, so 0 indicates unknown which
	// means it can't be used for lookups so we discard
	if s.Start.LineNumber == 0 || s.End.LineNumber == 0 {
		return nil
	}

	return &lsp.Range{
		Start: lsp.Position{
			Line:      int(s.Start.LineNumber - 1),
			Character: int(s.Start.ColumnOffset),
		},
		End: lsp.Position{
			Line:      int(s.End.LineNumber - 1),
			Character: int(s.End.ColumnOffset),
		},
	}
}

func (ls *Server) defLocations(t map[string]*xpb.Anchor) map[string]*lsp.Location {
	m := make(map[string]*lsp.Location)
	for k, v := range t {
		l := ls.anchorToLoc(v)
		if l != nil {
			m[k] = l
		}
	}
	return m
}

// refLocs takes a cross reference set and produces a slice of locations. This slice
// is guaranteed to be non-null even if empty
func (ls *Server) refLocs(r *xpb.CrossReferencesReply_CrossReferenceSet) []lsp.Location {
	locs := []lsp.Location{}
	for _, a := range r.Reference {
		l := ls.anchorToLoc(a.Anchor)
		if l != nil {
			locs = append(locs, *l)
		}
	}

	for _, a := range r.Definition {
		l := ls.anchorToLoc(a.Anchor)
		if l != nil {
			locs = append(locs, *l)
		}
	}

	for _, a := range r.Declaration {
		l := ls.anchorToLoc(a.Anchor)
		if l != nil {
			locs = append(locs, *l)
		}
	}

	return locs
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
  "github.com/aws/aws-sdk-go/aws/session"
  "github.com/aws/aws-sdk-go/aws/credentials"
  "github.com/aws/aws-sdk-go/aws/ec2metadata"
  "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"

	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/s3"
)

type awsHostContext struct {
  Creds   *credentials.Credentials
  Id      string
  Region  string
}

var DEBUG bool
func debug(m ...interface{}) {
	if(DEBUG) {
		fmt.Fprintf(os.Stdout, "%s\n", fmt.Sprint(m...))
	}
}

func main() {
	// currently not using any of the client flags
	_ = flag.String("u", "", "The user account")
	_ = flag.String("k", "", "The public key proffered by the client")
	_ = flag.String("t", "", "The key encryption type")
	_ = flag.String("f", "", "The fingerprint of the public key")
	groupTag := flag.String("group_tag", "access-groups", "The instance tag with csv ssh sccess groups in it")
	s3Bucket := flag.String("s3_bucket", "keys", "The bucket where the access group public keys are stored")
	s3Region := flag.String("s3_region", "eu-west-1", "The region in which the bucket is located")
	debugOutput := flag.Bool("debug", false, "Enable debug output")
	flag.Parse()

	if(*debugOutput) {
		DEBUG = true
		debug("debug output enabled")
		debug("grouptag:  ", *groupTag)
		debug("bucket:    ", *s3Bucket)
		debug("region:    ", *s3Region)
	}

	// get aws host context
	hctx, err := getAwsHostContext()
	if err != nil {
		debug(err.Error())
		os.Exit(1)
	}

	// get the list of access groups the intance belongs to
	accessGroups, err := getInstanceAccessGroups(hctx, *groupTag)
	if err != nil {
		debug(err.Error())
		os.Exit(1)
	}

	// get the authorized_keys files for those instances
	authorizedKeys, err := getAccessKeys(hctx, *s3Bucket, *s3Region, accessGroups)
	if err != nil {
		debug(err.Error())
		os.Exit(1)
	}

	// print all authorized_keys
	debug("print all group keys:")
	for _, k := range authorizedKeys {
		fmt.Print(k)
	}
}

func getAwsHostContext() (*awsHostContext, error) {
  //setup
  hctx := new(awsHostContext)
  cfg := aws.NewConfig()
  svc := ec2metadata.New(session.New(), cfg)
  p := &ec2rolecreds.EC2RoleProvider{
		Client: svc,
	}
  //get creds
  hctx.Creds = credentials.NewCredentials(p)
  //get instance id
  id, err := svc.GetMetadata("instance-id")
  if err != nil {
    return nil, err
  }
  hctx.Id = id
	debug("instance-id: ", id)
  //get region
  region, err := svc.Region()
  if err != nil {
    return nil, err
  }
  hctx.Region = region
	debug("region: ", region)
  return hctx, nil
}

func getInstanceAccessGroups(hctx *awsHostContext, tag string) ([]string, error) {
	// setup
	svc := ec2.New(session.New(aws.NewConfig().WithCredentials(hctx.Creds).WithRegion(hctx.Region)))
	params := &ec2.DescribeTagsInput{
		Filters: []*ec2.Filter{
			{
				Name: aws.String("resource-id"),
				Values: []*string{ aws.String(hctx.Id) },
			},
			{
				Name: aws.String("key"),
				Values: []*string{ aws.String(tag) },
			},
		},
	}
	// fetch tags
	debug("requesting tag: ", tag, ", from resource: hctx.Id", )
	resp, err := svc.DescribeTags(params)
	if err != nil {
		return []string{}, err
	}
	debug("recieved: ", resp.Tags)

	// parse and return response
	var out []string
	for _, s := range strings.Split(*resp.Tags[0].Value, ",") {
		out = append(out, strings.TrimSpace(s))
	}
	debug("access groups: ", out)
	return out, nil
}

func getAccessKeys(hctx *awsHostContext, s3Bucket, s3Region string, accessGroups []string) ([]string, error) {
	// setup
	svc := s3.New(session.New(aws.NewConfig().WithCredentials(hctx.Creds).WithRegion(s3Region)))

	var out []string
	// fetch authorized_keys files
	for _, group := range accessGroups {
		params := &s3.GetObjectInput{
			Bucket:  aws.String(s3Bucket),
			Key:     aws.String(group + "/authorized_keys"),  // Required
		}
		debug("requesting authorized_keys file: s3://", s3Bucket, "/", group, "/authorized_keys")
		resp, err := svc.GetObject(params)
		if err != nil {
			debug(err.Error())
			continue
		}
		debug("retrieved, appending to output")
		if b, err := ioutil.ReadAll(resp.Body); err == nil {
    	out = append(out, string(b))
		}
	}

	return out, nil
}
<commit_msg>handle tag not existing<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
  "github.com/aws/aws-sdk-go/aws/session"
  "github.com/aws/aws-sdk-go/aws/credentials"
  "github.com/aws/aws-sdk-go/aws/ec2metadata"
  "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"

	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/s3"
)

type awsHostContext struct {
  Creds   *credentials.Credentials
  Id      string
  Region  string
}

var DEBUG bool
func debug(m ...interface{}) {
	if(DEBUG) {
		fmt.Fprintf(os.Stdout, "%s\n", fmt.Sprint(m...))
	}
}

func main() {
	// currently not using any of the client flags
	_ = flag.String("u", "", "The user account")
	_ = flag.String("k", "", "The public key proffered by the client")
	_ = flag.String("t", "", "The key encryption type")
	_ = flag.String("f", "", "The fingerprint of the public key")
	groupTag := flag.String("group_tag", "access-groups", "The instance tag with csv ssh sccess groups in it")
	s3Bucket := flag.String("s3_bucket", "keys", "The bucket where the access group public keys are stored")
	s3Region := flag.String("s3_region", "eu-west-1", "The region in which the bucket is located")
	debugOutput := flag.Bool("debug", false, "Enable debug output")
	flag.Parse()

	if(*debugOutput) {
		DEBUG = true
		debug("debug output enabled")
		debug("grouptag:  ", *groupTag)
		debug("bucket:    ", *s3Bucket)
		debug("region:    ", *s3Region)
	}

	// get aws host context
	hctx, err := getAwsHostContext()
	if err != nil {
		debug(err.Error())
		os.Exit(1)
	}

	// get the list of access groups the intance belongs to
	accessGroups, err := getInstanceAccessGroups(hctx, *groupTag)
	if err != nil {
		debug(err.Error())
		os.Exit(1)
	}

	// get the authorized_keys files for those instances
	authorizedKeys, err := getAccessKeys(hctx, *s3Bucket, *s3Region, accessGroups)
	if err != nil {
		debug(err.Error())
		os.Exit(1)
	}

	// print all authorized_keys
	debug("print all group keys:")
	for _, k := range authorizedKeys {
		fmt.Print(k)
	}
}

func getAwsHostContext() (*awsHostContext, error) {
  //setup
  hctx := new(awsHostContext)
  cfg := aws.NewConfig()
  svc := ec2metadata.New(session.New(), cfg)
  p := &ec2rolecreds.EC2RoleProvider{
		Client: svc,
	}
  //get creds
  hctx.Creds = credentials.NewCredentials(p)
  //get instance id
  id, err := svc.GetMetadata("instance-id")
  if err != nil {
    return nil, err
  }
  hctx.Id = id
	debug("instance-id: ", id)
  //get region
  region, err := svc.Region()
  if err != nil {
    return nil, err
  }
  hctx.Region = region
	debug("region: ", region)
  return hctx, nil
}

func getInstanceAccessGroups(hctx *awsHostContext, tag string) ([]string, error) {
	// setup
	svc := ec2.New(session.New(aws.NewConfig().WithCredentials(hctx.Creds).WithRegion(hctx.Region)))
	params := &ec2.DescribeTagsInput{
		Filters: []*ec2.Filter{
			{
				Name: aws.String("resource-id"),
				Values: []*string{ aws.String(hctx.Id) },
			},
			{
				Name: aws.String("key"),
				Values: []*string{ aws.String(tag) },
			},
		},
	}
	// fetch tags
	debug("requesting tag: ", tag, ", from resource: hctx.Id", )
	resp, err := svc.DescribeTags(params)
	if err != nil {
		return []string{}, err
	}
	debug("recieved: ", resp.Tags)

	// valiade response
	if len(resp.Tags) == 0 {
		return []string{}, errors.New("tag " + tag + " does not exist!")
	}
	// parse and return response
	var out []string
	for _, s := range strings.Split(*resp.Tags[0].Value, ",") {
		out = append(out, strings.TrimSpace(s))
	}
	debug("access groups: ", out)
	return out, nil
}

func getAccessKeys(hctx *awsHostContext, s3Bucket, s3Region string, accessGroups []string) ([]string, error) {
	// setup
	svc := s3.New(session.New(aws.NewConfig().WithCredentials(hctx.Creds).WithRegion(s3Region)))

	var out []string
	// fetch authorized_keys files
	for _, group := range accessGroups {
		params := &s3.GetObjectInput{
			Bucket:  aws.String(s3Bucket),
			Key:     aws.String(group + "/authorized_keys"),  // Required
		}
		debug("requesting authorized_keys file: s3://", s3Bucket, "/", group, "/authorized_keys")
		resp, err := svc.GetObject(params)
		if err != nil {
			debug(err.Error())
			continue
		}
		debug("retrieved, appending to output")
		if b, err := ioutil.ReadAll(resp.Body); err == nil {
    	out = append(out, string(b))
		}
	}

	return out, nil
}
<|endoftext|>


<commit_before>package config

import (
    "fmt"
    "encoding/json"
    "log"
    "os"
    "gopkg.in/redis.v3"
    "regexp"
    "time"
)

const SUB_TTL = 10 * time.Second

var subRegexp = regexp.MustCompile(`^(\w+):(\w+)$`)

type SubOptions struct {
    Module  string
    ID      string
}

func (self SubOptions) String() string {
    return fmt.Sprintf("%s:%s", self.Module, self.ID)
}

// Parse the SubOptions.String() format into a SubOptions
func ParseSub(str string) (self SubOptions, err error) {
    if match := subRegexp.FindStringSubmatch(str); match == nil {
        return self, fmt.Errorf("Invalid sub name: %s", str)
    } else {
        self.Module = match[1]
        self.ID = match[2]

        return
    }
}

type Sub struct {
    redis   *Redis
    options SubOptions
    path    string
    log     *log.Logger

    expire      time.Time
    stopChan    chan bool
}

func (self *Sub) init(options SubOptions) error {
    self.options = options
    self.path = self.redis.path(options.Module, options.ID)
    self.log = log.New(os.Stderr, fmt.Sprintf("config.Sub %v: ", self.path), 0)

    return nil
}

func (self *Sub) String() string {
    return self.options.String()
}

// get as generic map[string]interface{}
func (self *Sub) Get() (Config, error) {
    config := make(map[string]interface{})

    if err := self.get(&config); err != nil {
        return nil, err
    } else {
        return config, nil
    }
}

// update config from redis
func (self *Sub) get(config Config) error {
    if jsonBuf, err := self.redis.redisClient.Get(self.path).Bytes(); err != nil {
        return nil
    } else if err := json.Unmarshal(jsonBuf, config); err != nil {
        return err
    } else {
        return nil
    }
}

// update config in redis
func (self *Sub) set(config Config) error {
    if jsonBuf, err := json.Marshal(config); err != nil {
        return err
    } else if res := self.redis.redisClient.Set(self.path, jsonBuf, 0); res.Err() != nil {
        return res.Err()
    } else {
        self.log.Printf("set: %v\n", config)
        return nil
    }
}

// get an existing redis config, or set it
func (self *Sub) sync(config Config) error {
    if exists, err := self.redis.redisClient.Exists(self.path).Result(); err != nil {
        return err
    } else if exists {
        return self.get(config)
    } else {
        return self.set(config)
    }
}

// register config in redis, maintaining both the ZSet and the object expiry keepalive under TTL
func (self *Sub) register() {
    // registration set's path, vs self.path for our object
    path := self.redis.path(self.options.Module, "")

    expire := time.Now().Add(SUB_TTL)
    refreshTimer := time.Tick(SUB_TTL / 2)

    for {
        if res := self.redis.redisClient.ExpireAt(self.path, expire); res.Err() != nil {
            self.log.Printf("refresh ExpireAt %v: %v\n", expire, res.Err())

        } else if res := self.redis.redisClient.ZAdd(path, redis.Z{Score: float64(expire.Unix()), Member: self.path}); res.Err() != nil {
            self.log.Printf("refresh ZAdd %v: %v\n", path, res.Err())
        }

        select {
        case t := <-refreshTimer:
            // update expiry for next iteration
            expire = t.Add(SUB_TTL)

        case <-self.stopChan:
            // unregister
            self.redis.redisClient.ZRem(path, self.path)
            break
        }
    }
}

// Stop refreshing the config in redis, and remove it
func (self *Sub) Stop() error {
    self.stopChan <- true

    if res := self.redis.redisClient.Del(self.path); res.Err() != nil {
        return res.Err()
    }

    return nil
}

func (self *Sub) read(pubsub *redis.PubSub, configChan chan Config, config Config) {
    defer close(configChan)

    for {
        if msg, err := pubsub.ReceiveMessage(); err != nil {
            self.log.Printf("read: %v\n", err)
            break
        } else if err := json.Unmarshal([]byte(msg.Payload), config); err != nil {
            self.log.Printf("read JSON: %v\n", err)
            continue
        } else {
            configChan <- config
        }

        // update stored config
        if err := self.set(config); err != nil {
            self.log.Printf("read -> set: %v\n", err)
            continue
        }
    }
}

// Register ourselves in redis, storing or updating the given Config
// Read updates from redis, storing them into the given Config
// Each updated Config is delivered on the given chan
// Once the config has been sent, it is updated into redis
func (self *Sub) Start(config Config) (chan Config, error) {
    // sync object
    if err := self.sync(config); err != nil {
        return nil, err
    }

    // register top-level module
    if err := self.redis.registerModule(self.options.Module); err != nil {
        return nil, err
    }

    // register the object
    self.stopChan = make(chan bool)

    go self.register()

    // subscribe for updates
    pubsub, err := self.redis.redisClient.Subscribe(self.path)
    if err != nil {
        return nil, err
    }

    configChan := make(chan Config)

    go self.read(pubsub, configChan, config)

    // running
    return configChan, nil
}

// update (partial) params for sub
// return an error if there is no active sub
func (self *Sub) Push(config Config) error {
    if jsonBuf, err := json.Marshal(config); err != nil {
        return err
    } else if count, err := self.redis.redisClient.Publish(self.path, string(jsonBuf)).Result(); err != nil {
        return err
    } else if count == 0 {
        // redis did not have anything SUBSCRIBE'd to this path
        return fmt.Errorf("Publish to empty Sub: %v", self.path)
    } else {
        self.log.Printf("Push: %v\n", config)
        return nil
    }
}
<commit_msg>config: Sub.Get(): include _module= _id= in map<commit_after>package config

import (
    "fmt"
    "encoding/json"
    "log"
    "os"
    "gopkg.in/redis.v3"
    "regexp"
    "time"
)

const SUB_TTL = 10 * time.Second

var subRegexp = regexp.MustCompile(`^(\w+):(\w+)$`)

type SubOptions struct {
    Module  string
    ID      string
}

func (self SubOptions) String() string {
    return fmt.Sprintf("%s:%s", self.Module, self.ID)
}

// Parse the SubOptions.String() format into a SubOptions
func ParseSub(str string) (self SubOptions, err error) {
    if match := subRegexp.FindStringSubmatch(str); match == nil {
        return self, fmt.Errorf("Invalid sub name: %s", str)
    } else {
        self.Module = match[1]
        self.ID = match[2]

        return
    }
}

type Sub struct {
    redis   *Redis
    options SubOptions
    path    string
    log     *log.Logger

    expire      time.Time
    stopChan    chan bool
}

func (self *Sub) init(options SubOptions) error {
    self.options = options
    self.path = self.redis.path(options.Module, options.ID)
    self.log = log.New(os.Stderr, fmt.Sprintf("config.Sub %v: ", self.path), 0)

    return nil
}

func (self *Sub) String() string {
    return self.options.String()
}

// get as generic map[string]interface{}
// includes additional _fields for SubOptions
func (self *Sub) Get() (Config, error) {
    config := map[string]interface{}{
        "_module": self.options.Module,
        "_id": self.options.ID,
    }

    if err := self.get(&config); err != nil {
        return nil, err
    } else {
        return config, nil
    }
}

// update config from redis
func (self *Sub) get(config Config) error {
    if jsonBuf, err := self.redis.redisClient.Get(self.path).Bytes(); err != nil {
        return nil
    } else if err := json.Unmarshal(jsonBuf, config); err != nil {
        return err
    } else {
        return nil
    }
}

// update config in redis
func (self *Sub) set(config Config) error {
    if jsonBuf, err := json.Marshal(config); err != nil {
        return err
    } else if res := self.redis.redisClient.Set(self.path, jsonBuf, 0); res.Err() != nil {
        return res.Err()
    } else {
        self.log.Printf("set: %v\n", config)
        return nil
    }
}

// get an existing redis config, or set it
func (self *Sub) sync(config Config) error {
    if exists, err := self.redis.redisClient.Exists(self.path).Result(); err != nil {
        return err
    } else if exists {
        return self.get(config)
    } else {
        return self.set(config)
    }
}

// register config in redis, maintaining both the ZSet and the object expiry keepalive under TTL
func (self *Sub) register() {
    // registration set's path, vs self.path for our object
    path := self.redis.path(self.options.Module, "")

    expire := time.Now().Add(SUB_TTL)
    refreshTimer := time.Tick(SUB_TTL / 2)

    for {
        if res := self.redis.redisClient.ExpireAt(self.path, expire); res.Err() != nil {
            self.log.Printf("refresh ExpireAt %v: %v\n", expire, res.Err())

        } else if res := self.redis.redisClient.ZAdd(path, redis.Z{Score: float64(expire.Unix()), Member: self.path}); res.Err() != nil {
            self.log.Printf("refresh ZAdd %v: %v\n", path, res.Err())
        }

        select {
        case t := <-refreshTimer:
            // update expiry for next iteration
            expire = t.Add(SUB_TTL)

        case <-self.stopChan:
            // unregister
            self.redis.redisClient.ZRem(path, self.path)
            break
        }
    }
}

// Stop refreshing the config in redis, and remove it
func (self *Sub) Stop() error {
    self.stopChan <- true

    if res := self.redis.redisClient.Del(self.path); res.Err() != nil {
        return res.Err()
    }

    return nil
}

func (self *Sub) read(pubsub *redis.PubSub, configChan chan Config, config Config) {
    defer close(configChan)

    for {
        if msg, err := pubsub.ReceiveMessage(); err != nil {
            self.log.Printf("read: %v\n", err)
            break
        } else if err := json.Unmarshal([]byte(msg.Payload), config); err != nil {
            self.log.Printf("read JSON: %v\n", err)
            continue
        } else {
            configChan <- config
        }

        // update stored config
        if err := self.set(config); err != nil {
            self.log.Printf("read -> set: %v\n", err)
            continue
        }
    }
}

// Register ourselves in redis, storing or updating the given Config
// Read updates from redis, storing them into the given Config
// Each updated Config is delivered on the given chan
// Once the config has been sent, it is updated into redis
func (self *Sub) Start(config Config) (chan Config, error) {
    // sync object
    if err := self.sync(config); err != nil {
        return nil, err
    }

    // register top-level module
    if err := self.redis.registerModule(self.options.Module); err != nil {
        return nil, err
    }

    // register the object
    self.stopChan = make(chan bool)

    go self.register()

    // subscribe for updates
    pubsub, err := self.redis.redisClient.Subscribe(self.path)
    if err != nil {
        return nil, err
    }

    configChan := make(chan Config)

    go self.read(pubsub, configChan, config)

    // running
    return configChan, nil
}

// update (partial) params for sub
// return an error if there is no active sub
func (self *Sub) Push(config Config) error {
    if jsonBuf, err := json.Marshal(config); err != nil {
        return err
    } else if count, err := self.redis.redisClient.Publish(self.path, string(jsonBuf)).Result(); err != nil {
        return err
    } else if count == 0 {
        // redis did not have anything SUBSCRIBE'd to this path
        return fmt.Errorf("Publish to empty Sub: %v", self.path)
    } else {
        self.log.Printf("Push: %v\n", config)
        return nil
    }
}
<|endoftext|>


<commit_before>// Copyright 2016 Stratumn SAS. All rights reserved.
// Use of this source code is governed by an Apache License 2.0
// that can be found in the LICENSE file.

// Package btctimestamper implements a fake Bitcoin timestamper which can be used for testing.
package btctimestamper

import (
	"bytes"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"log"

	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/stratumn/goprivate/blockchain"
	"github.com/stratumn/goprivate/blockchain/btc"
	"github.com/stratumn/goprivate/types"
)

const (
	// DefaultFee is the default transaction fee.
	DefaultFee = int64(15000)
)

// Config contains configuration options for the timestamper.
type Config struct {
	// An unspent transaction finder.
	UnspentFinder btc.UnspentFinder

	// A transaction broadcaster.
	Broadcaster btc.Broadcaster

	// A wallet import format key.
	WIF string

	// Transaction fee
	Fee int64
}

// Timestamper is the type that implements github.com/stratumn/goprivate/blockchain.Timestamper.
type Timestamper struct {
	config    *Config
	net       btc.Network
	netParams *chaincfg.Params
	privKey   *btcec.PrivateKey
	pubKey    *btcec.PublicKey
	address   *btcutil.AddressPubKeyHash
}

// New creates an instance of a Timestamper.
func New(config *Config) (*Timestamper, error) {
	WIF, err := btcutil.DecodeWIF(config.WIF)
	if err != nil {
		return nil, err
	}

	ts := &Timestamper{
		config:  config,
		privKey: WIF.PrivKey,
		pubKey:  WIF.PrivKey.PubKey(),
	}

	if WIF.IsForNet(&chaincfg.TestNet3Params) {
		ts.net = btc.NetworkTest3
		ts.netParams = &chaincfg.TestNet3Params
	} else if WIF.IsForNet(&chaincfg.MainNetParams) {
		ts.net = btc.NetworkMain
		ts.netParams = &chaincfg.MainNetParams
	}

	if ts.netParams == nil {
		return nil, errors.New("unsupported network")
	}

	pubKeyHash := btcutil.Hash160(ts.pubKey.SerializeUncompressed())
	ts.address, err = btcutil.NewAddressPubKeyHash(pubKeyHash, ts.netParams)
	if err != nil {
		return nil, err
	}

	return ts, nil
}

// Network implements fmt.Stringer.
func (ts *Timestamper) Network() blockchain.Network {
	return ts.net
}

// TimestampHash implements github.com/stratumn/goprivate/blockchain.HashTimestamper.
func (ts *Timestamper) TimestampHash(hash *types.Bytes32) (blockchain.TransactionID, error) {
	var prevPKScripts [][]byte

	addr := (*types.ReversedBytes20)(ts.address.Hash160())
	outputs, total, err := ts.config.UnspentFinder.FindUnspent(addr, ts.config.Fee)
	if err != nil {
		return nil, err
	}

	tx := wire.NewMsgTx()
	for _, output := range outputs {
		prevPKScripts = append(prevPKScripts, output.PKScript)
		out := wire.NewOutPoint((*chainhash.Hash)(&output.TXHash), uint32(output.Index))
		tx.AddTxIn(wire.NewTxIn(out, nil))
	}

	payToAddrOut, err := ts.createPayToAddrTxOut(total - ts.config.Fee)
	if err != nil {
		return nil, err
	}
	tx.AddTxOut(payToAddrOut)

	nullDataOut, err := ts.createNullDataTxOut(hash)
	if err != nil {
		return nil, err
	}
	tx.AddTxOut(nullDataOut)

	if err = ts.signTx(tx, prevPKScripts); err != nil {
		return nil, err
	}
	if err = ts.validateTx(tx, prevPKScripts); err != nil {
		return nil, err
	}

	buf := bytes.NewBuffer(nil)
	if err = tx.Serialize(buf); err != nil {
		return nil, err
	}
	raw, err := ioutil.ReadAll(buf)
	if err != nil {
		return nil, err
	}
	err = ts.config.Broadcaster.Broadcast(raw)
	if err != nil {
		return nil, err
	}

	// Reverse the bytes!
	var txHash32 types.Bytes32
	for i, b := range tx.TxHash() {
		txHash32[types.Bytes32Size-i-1] = b
	}

	log.Printf("created transaction %s for hash %s\n", hex.EncodeToString(txHash32[:]), hash)
	return txHash32[:], nil
}

func (ts *Timestamper) createPayToAddrTxOut(amount int64) (*wire.TxOut, error) {
	PKScript, err := txscript.PayToAddrScript(ts.address)
	if err != nil {
		return nil, err
	}

	return wire.NewTxOut(amount, PKScript), nil
}

func (ts *Timestamper) createNullDataTxOut(hash *types.Bytes32) (*wire.TxOut, error) {
	PKScript, err := txscript.NewScriptBuilder().AddOp(txscript.OP_RETURN).AddData(hash[:]).Script()
	if err != nil {
		return nil, err
	}

	return wire.NewTxOut(0, PKScript), nil
}

func (ts *Timestamper) signTx(tx *wire.MsgTx, prevPKScripts [][]byte) error {
	for index, PKScript := range prevPKScripts {
		sig, err := txscript.SignTxOutput(ts.netParams, tx, 0, PKScript,
			txscript.SigHashAll, txscript.KeyClosure(ts.lookupKey), nil, nil)
		if err != nil {
			return err
		}

		tx.TxIn[index].SignatureScript = sig
	}

	return nil
}

const validateTxEngineFlags = txscript.ScriptBip16 | txscript.ScriptVerifyDERSignatures |
	txscript.ScriptStrictMultiSig | txscript.ScriptDiscourageUpgradableNops

func (ts *Timestamper) validateTx(tx *wire.MsgTx, prevPKScripts [][]byte) error {
	for _, PKScript := range prevPKScripts {
		vm, err := txscript.NewEngine(PKScript, tx, 0, validateTxEngineFlags, nil)
		if err != nil {
			return err
		}
		if err := vm.Execute(); err != nil {
			return err
		}
	}

	return nil
}

func (ts *Timestamper) lookupKey(btcutil.Address) (*btcec.PrivateKey, bool, error) {
	// Second values means uncompressed.
	return ts.privKey, false, nil
}
<commit_msg>blockchain: Tweaks<commit_after>// Copyright 2016 Stratumn SAS. All rights reserved.
// Use of this source code is governed by an Apache License 2.0
// that can be found in the LICENSE file.

// Package btctimestamper implements a fake Bitcoin timestamper which can be used for testing.
package btctimestamper

import (
	"bytes"
	"errors"
	"io/ioutil"
	"log"

	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/stratumn/goprivate/blockchain"
	"github.com/stratumn/goprivate/blockchain/btc"
	"github.com/stratumn/goprivate/types"
)

const (
	// DefaultFee is the default transaction fee.
	DefaultFee = int64(15000)
)

// Config contains configuration options for the timestamper.
type Config struct {
	// An unspent transaction finder.
	UnspentFinder btc.UnspentFinder

	// A transaction broadcaster.
	Broadcaster btc.Broadcaster

	// A wallet import format key.
	WIF string

	// Transaction fee
	Fee int64
}

// Timestamper is the type that implements github.com/stratumn/goprivate/blockchain.Timestamper.
type Timestamper struct {
	config    *Config
	net       btc.Network
	netParams *chaincfg.Params
	privKey   *btcec.PrivateKey
	pubKey    *btcec.PublicKey
	address   *btcutil.AddressPubKeyHash
}

// New creates an instance of a Timestamper.
func New(config *Config) (*Timestamper, error) {
	WIF, err := btcutil.DecodeWIF(config.WIF)
	if err != nil {
		return nil, err
	}

	ts := &Timestamper{
		config:  config,
		privKey: WIF.PrivKey,
		pubKey:  WIF.PrivKey.PubKey(),
	}

	if WIF.IsForNet(&chaincfg.TestNet3Params) {
		ts.net = btc.NetworkTest3
		ts.netParams = &chaincfg.TestNet3Params
	} else if WIF.IsForNet(&chaincfg.MainNetParams) {
		ts.net = btc.NetworkMain
		ts.netParams = &chaincfg.MainNetParams
	}

	if ts.netParams == nil {
		return nil, errors.New("unsupported network")
	}

	pubKeyHash := btcutil.Hash160(ts.pubKey.SerializeUncompressed())
	ts.address, err = btcutil.NewAddressPubKeyHash(pubKeyHash, ts.netParams)
	if err != nil {
		return nil, err
	}

	return ts, nil
}

// Network implements fmt.Stringer.
func (ts *Timestamper) Network() blockchain.Network {
	return ts.net
}

// TimestampHash implements github.com/stratumn/goprivate/blockchain.HashTimestamper.
func (ts *Timestamper) TimestampHash(hash *types.Bytes32) (blockchain.TransactionID, error) {
	var prevPKScripts [][]byte

	addr := (*types.ReversedBytes20)(ts.address.Hash160())
	outputs, total, err := ts.config.UnspentFinder.FindUnspent(addr, ts.config.Fee)
	if err != nil {
		return nil, err
	}

	tx := wire.NewMsgTx()
	for _, output := range outputs {
		prevPKScripts = append(prevPKScripts, output.PKScript)
		out := wire.NewOutPoint((*chainhash.Hash)(&output.TXHash), uint32(output.Index))
		tx.AddTxIn(wire.NewTxIn(out, nil))
	}

	payToAddrOut, err := ts.createPayToAddrTxOut(total - ts.config.Fee)
	if err != nil {
		return nil, err
	}
	tx.AddTxOut(payToAddrOut)

	nullDataOut, err := ts.createNullDataTxOut(hash)
	if err != nil {
		return nil, err
	}
	tx.AddTxOut(nullDataOut)

	if err = ts.signTx(tx, prevPKScripts); err != nil {
		return nil, err
	}
	if err = ts.validateTx(tx, prevPKScripts); err != nil {
		return nil, err
	}

	buf := bytes.NewBuffer(nil)
	if err = tx.Serialize(buf); err != nil {
		return nil, err
	}
	raw, err := ioutil.ReadAll(buf)
	if err != nil {
		return nil, err
	}
	err = ts.config.Broadcaster.Broadcast(raw)
	if err != nil {
		return nil, err
	}

	// Reverse the bytes!
	var txHash32 types.Bytes32
	for i, b := range tx.TxHash() {
		txHash32[types.Bytes32Size-i-1] = b
	}

	log.Printf("Created transaction %q for hash %q\n", txHash32.String(), hash)
	return txHash32[:], nil
}

func (ts *Timestamper) createPayToAddrTxOut(amount int64) (*wire.TxOut, error) {
	PKScript, err := txscript.PayToAddrScript(ts.address)
	if err != nil {
		return nil, err
	}

	return wire.NewTxOut(amount, PKScript), nil
}

func (ts *Timestamper) createNullDataTxOut(hash *types.Bytes32) (*wire.TxOut, error) {
	PKScript, err := txscript.NewScriptBuilder().AddOp(txscript.OP_RETURN).AddData(hash[:]).Script()
	if err != nil {
		return nil, err
	}

	return wire.NewTxOut(0, PKScript), nil
}

func (ts *Timestamper) signTx(tx *wire.MsgTx, prevPKScripts [][]byte) error {
	for index, PKScript := range prevPKScripts {
		sig, err := txscript.SignTxOutput(ts.netParams, tx, 0, PKScript,
			txscript.SigHashAll, txscript.KeyClosure(ts.lookupKey), nil, nil)
		if err != nil {
			return err
		}

		tx.TxIn[index].SignatureScript = sig
	}

	return nil
}

const validateTxEngineFlags = txscript.ScriptBip16 | txscript.ScriptVerifyDERSignatures |
	txscript.ScriptStrictMultiSig | txscript.ScriptDiscourageUpgradableNops

func (ts *Timestamper) validateTx(tx *wire.MsgTx, prevPKScripts [][]byte) error {
	for _, PKScript := range prevPKScripts {
		vm, err := txscript.NewEngine(PKScript, tx, 0, validateTxEngineFlags, nil)
		if err != nil {
			return err
		}
		if err := vm.Execute(); err != nil {
			return err
		}
	}

	return nil
}

func (ts *Timestamper) lookupKey(btcutil.Address) (*btcec.PrivateKey, bool, error) {
	// Second values means uncompressed.
	return ts.privKey, false, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"net/http"
	"sort"
	"strings"
	"unicode/utf8"

	log "github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"

	"github.com/camptocamp/conplicity/handler"
	"github.com/camptocamp/conplicity/providers"
	"github.com/camptocamp/conplicity/util"
)

const labelPrefix string = "io.conplicity"

func main() {
	log.Infof("Starting backup...")

	var err error

	c := &handler.Conplicity{}
	err = c.Setup()
	util.CheckErr(err, "Failed to setup Conplicity handler: %v", 1)

	vols, err := c.ListVolumes(docker.ListVolumesOptions{})
	util.CheckErr(err, "Failed to list Docker volumes: %v", 1)

	var metrics []string
	for _, vol := range vols {
		voll, err := c.InspectVolume(vol.Name)
		util.CheckErr(err, "Failed to inspect volume "+vol.Name+": %v", -1)

		_metrics, err := backupVolume(c, voll)
		util.CheckErr(err, "Failed to process volume "+vol.Name+": %v", -1)
		metrics = append(metrics, _metrics...)
	}
	if c.PushgatewayURL != "" {
		url := c.PushgatewayURL + "/metrics/job/conplicity/instance/" + c.Hostname
		data := strings.Join(metrics, "\n") + "\n"
		err = pushToPrometheus(url, data)
		util.CheckErr(err, "Failed post data to Prometheus Pushgateway: %v", 1)
	}

	log.Infof("End backup...")
}

func pushToPrometheus(url, data string) error {
	log.Infof("Sending metrics to Prometheus Pushgateway: %v", data)
	log.Infof("URL=%v", url)

	req, err := http.NewRequest("PUT", url, bytes.NewBufferString(data))
	req.Header.Set("Content-Type", "text/plain; version=0.0.4")

	client := &http.Client{}
	resp, err := client.Do(req)

	log.Infof("resp = %v", resp)

	return err
}

func backupVolume(c *handler.Conplicity, vol *docker.Volume) (metrics []string, err error) {
	if utf8.RuneCountInString(vol.Name) == 64 || vol.Name == "duplicity_cache" {
		log.Infof("Ignoring unnamed volume " + vol.Name)
		return
	}

	list := c.VolumesBlacklist
	i := sort.SearchStrings(list, vol.Name)
	if i < len(list) && list[i] == vol.Name {
		log.Infof("Ignoring blacklisted volume " + vol.Name)
		return
	}

	if getVolumeLabel(vol, ".ignore") == "true" {
		log.Infof("Ignoring blacklisted volume " + vol.Name)
		return
	}

	p := providers.GetProvider(c, vol)
	log.Infof("Using provider %v to backup %v", p.GetName(), vol.Name)
	err = providers.PrepareBackup(p)
	util.CheckErr(err, "Failed to prepare backup for volume "+vol.Name+": %v", -1)
	metrics, err = providers.BackupVolume(p, vol)
	util.CheckErr(err, "Failed to backup volume "+vol.Name+": %v", -1)
	return
}

func getVolumeLabel(vol *docker.Volume, key string) (value string) {
	value = vol.Labels[labelPrefix+key]
	return
}
<commit_msg>Send metrics to pushgateway only if slice is not empty (fixes #47)<commit_after>package main

import (
	"bytes"
	"net/http"
	"sort"
	"strings"
	"unicode/utf8"

	log "github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"

	"github.com/camptocamp/conplicity/handler"
	"github.com/camptocamp/conplicity/providers"
	"github.com/camptocamp/conplicity/util"
)

const labelPrefix string = "io.conplicity"

func main() {
	log.Infof("Starting backup...")

	var err error

	c := &handler.Conplicity{}
	err = c.Setup()
	util.CheckErr(err, "Failed to setup Conplicity handler: %v", 1)

	vols, err := c.ListVolumes(docker.ListVolumesOptions{})
	util.CheckErr(err, "Failed to list Docker volumes: %v", 1)

	var metrics []string
	for _, vol := range vols {
		voll, err := c.InspectVolume(vol.Name)
		util.CheckErr(err, "Failed to inspect volume "+vol.Name+": %v", -1)

		_metrics, err := backupVolume(c, voll)
		util.CheckErr(err, "Failed to process volume "+vol.Name+": %v", -1)
		metrics = append(metrics, _metrics...)
	}
	if len(metrics) > 0 && c.PushgatewayURL != "" {
		url := c.PushgatewayURL + "/metrics/job/conplicity/instance/" + c.Hostname
		data := strings.Join(metrics, "\n") + "\n"
		err = pushToPrometheus(url, data)
		util.CheckErr(err, "Failed post data to Prometheus Pushgateway: %v", 1)
	}

	log.Infof("End backup...")
}

func pushToPrometheus(url, data string) error {
	log.Infof("Sending metrics to Prometheus Pushgateway: %v", data)
	log.Infof("URL=%v", url)

	req, err := http.NewRequest("PUT", url, bytes.NewBufferString(data))
	req.Header.Set("Content-Type", "text/plain; version=0.0.4")

	client := &http.Client{}
	resp, err := client.Do(req)

	log.Infof("resp = %v", resp)

	return err
}

func backupVolume(c *handler.Conplicity, vol *docker.Volume) (metrics []string, err error) {
	if utf8.RuneCountInString(vol.Name) == 64 || vol.Name == "duplicity_cache" {
		log.Infof("Ignoring unnamed volume " + vol.Name)
		return
	}

	list := c.VolumesBlacklist
	i := sort.SearchStrings(list, vol.Name)
	if i < len(list) && list[i] == vol.Name {
		log.Infof("Ignoring blacklisted volume " + vol.Name)
		return
	}

	if getVolumeLabel(vol, ".ignore") == "true" {
		log.Infof("Ignoring blacklisted volume " + vol.Name)
		return
	}

	p := providers.GetProvider(c, vol)
	log.Infof("Using provider %v to backup %v", p.GetName(), vol.Name)
	err = providers.PrepareBackup(p)
	util.CheckErr(err, "Failed to prepare backup for volume "+vol.Name+": %v", -1)
	metrics, err = providers.BackupVolume(p, vol)
	util.CheckErr(err, "Failed to backup volume "+vol.Name+": %v", -1)
	return
}

func getVolumeLabel(vol *docker.Volume, key string) (value string) {
	value = vol.Labels[labelPrefix+key]
	return
}
<|endoftext|>


<commit_before>package tnt

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

const (
	dirSnap = "snap"
	dirWAL  = "wal"
)

// Box is a tarantool instance with specified config and BoxOptions.
type Box struct {
	Root     string
	Port     uint
	cmd      *exec.Cmd
	stopOnce sync.Once
	stopped  chan bool
}

// BoxOptions is the options for the Box instance.
type BoxOptions struct {
	Listen  uint
	PortMin uint
	PortMax uint
}

// NewBox instance.
func NewBox(config string, options ...BoxOptions) (*Box, error) {
	var opts BoxOptions
	if len(options) > 0 {
		opts = options[0]
	} else {
		opts = BoxOptions{}
	}

	if opts.PortMin == 0 {
		opts.PortMin = 8000
	}

	if opts.PortMax == 0 {
		opts.PortMax = 9000
	}

	if opts.Listen != 0 {
		opts.PortMin = opts.Listen
		opts.PortMax = opts.Listen
	}

	var box *Box

START_LOOP:
	for port := opts.PortMin; port <= opts.PortMax; port += 3 {

		tmpDir, err := ioutil.TempDir("", "")
		if err != nil {
			return nil, err
		}

		tarantoolConf := `
		slab_alloc_arena = 1
		slab_alloc_factor = 1.04

		pid_file = {root}/box.pid
		work_dir = {root}
		snap_dir = {snap}
		wal_dir = {wal}

		snap_io_rate_limit = 10
		rows_per_wal = 1000000
		too_long_threshold = 0.025

		primary_port = {port1}
		memcached_expire = false
		memcached_port = {port2}
		admin_port = {port3}
		replication_port = {port4}
        `

		tarantoolConf = strings.Replace(tarantoolConf, "{port1}", fmt.Sprintf("%d", port), -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{port2}", fmt.Sprintf("%d", port+1), -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{port3}", fmt.Sprintf("%d", port+2), -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{port4}", fmt.Sprintf("%d", port+3), -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{root}", tmpDir, -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{snap}", dirSnap, -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{wal}", dirWAL, -1)

		tarantoolConf = fmt.Sprintf("%s\n%s", tarantoolConf, config)

		tarantoolConfFile := path.Join(tmpDir, "tarantool.conf")
		if err = ioutil.WriteFile(tarantoolConfFile, []byte(tarantoolConf), 0644); err != nil {
			return nil, err
		}

		for _, subDir := range []string{"snap", "wal"} {
			if err = os.Mkdir(path.Join(tmpDir, subDir), 0755); err != nil {
				return nil, err
			}
		}

		cmd0 := exec.Command("tarantool_box", "-c", tarantoolConfFile, "--init-storage")
		if err = cmd0.Run(); err != nil {
			return nil, err
		}

		cmd := exec.Command("tarantool_box", "-c", tarantoolConfFile)
		boxStderr, err := cmd.StderrPipe()
		if err != nil {
			return nil, err
		}

		if err = cmd.Start(); err != nil {
			return nil, err
		}

		var boxStderrBuffer bytes.Buffer

		p := make([]byte, 1024)

		box = &Box{
			Root:    tmpDir,
			Port:    port,
			cmd:     cmd,
			stopped: make(chan bool),
		}

	WAIT_LOOP:
		for {
			if strings.Contains(boxStderrBuffer.String(), "entering event loop") {
				break START_LOOP
			}

			if strings.Contains(boxStderrBuffer.String(), "is already in use, will retry binding after") {
				cmd.Process.Kill()
				cmd.Process.Wait()
				break WAIT_LOOP
			}

			n, err := boxStderr.Read(p)
			if n > 0 {
				boxStderrBuffer.Write(p[:n])
			}
			if err != nil {
				fmt.Println(boxStderrBuffer.String())
				return nil, err
			}
		}

		os.RemoveAll(box.Root)
		box = nil
	}

	if box == nil {
		return nil, fmt.Errorf("couldn't bind any port from %d to %d", opts.PortMin, opts.PortMax)
	}

	return box, nil
}

// Listen is the primary addr of the box.
func (box *Box) Listen() string {
	return fmt.Sprintf("127.0.0.1:%v", box.Port)
}

// ListenMemcache is the memcache addr of the box.
func (box *Box) ListenMemcache() string {
	return fmt.Sprintf("127.0.0.1:%v", box.Port+1)
}

// ListenAdmin is the admin addr of the box.
func (box *Box) ListenAdmin() string {
	return fmt.Sprintf("127.0.0.1:%v", box.Port+2)
}

// ListenReplica is the replication addr of the box.
func (box *Box) ListenReplica() string {
	return fmt.Sprintf("127.0.0.1:%v", box.Port+3)
}

// SaveSnapshot and return it's filename (with full path).
func (box *Box) SaveSnapshot() (string, error) {
	const (
		cmdSnapshot   = "save snapshot\n"
		cmdOK         = "ok"
		cmdFileExists = "fail: can't save snapshot, errno 17 (File exists)"
		cmdSeparators = "\n-. "
	)
	conn, err := net.Dial("tcp", box.ListenAdmin())
	if err != nil {
		return "", err
	}
	defer conn.Close()
	if err = conn.SetDeadline(time.Now().Add(5 * time.Second)); err != nil {
		return "", err
	}
	if _, err = conn.Write([]byte(cmdSnapshot)); err != nil {
		return "", err
	}
	respBytes := make([]byte, 256)
	n, err := conn.Read(respBytes)
	if err != nil {
		return "", err
	}
	response := strings.TrimRight(strings.TrimLeft(string(respBytes[:n]), cmdSeparators), cmdSeparators)
	switch response {
	case cmdOK, cmdFileExists:
		return box.Snapshot()
	default:
		return "", errors.New(response)
	}
}

// SnapDir of the box.
func (box *Box) SnapDir() string {
	return filepath.Join(box.Root, dirSnap)
}

// WALDir of the box.
func (box *Box) WALDir() string {
	return filepath.Join(box.Root, dirWAL)
}

var ErrSnapshotNotFound = errors.New("snapshot file hasn't been found")

// Snapshot returns the latest snapshot filename with full path.
func (box *Box) Snapshot() (string, error) {
	const snapExt = ".snap"
	files, err := ioutil.ReadDir(box.SnapDir())
	if err != nil {
		return "", err
	}

	// files are sorted alphabetically
	for i := len(files) - 1; i >= 0; i-- {
		if filepath.Ext(files[i].Name()) != snapExt {
			continue
		}
		return filepath.Join(box.SnapDir(), files[i].Name()), nil
	}
	return "", ErrSnapshotNotFound
}

// Close Box instance.
func (box *Box) Close() {
	box.stopOnce.Do(func() {
		box.cmd.Process.Kill()
		box.cmd.Process.Wait()
		os.RemoveAll(box.Root)
		close(box.stopped)
	})
	<-box.stopped
}
<commit_msg>Added new option for box. InitLua allows you start tarantool with initial lua script<commit_after>package tnt

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

const (
	dirSnap = "snap"
	dirWAL  = "wal"
)

// Box is a tarantool instance with specified config and BoxOptions.
type Box struct {
	Root     string
	Port     uint
	cmd      *exec.Cmd
	stopOnce sync.Once
	stopped  chan bool
}

// BoxOptions is the options for the Box instance.
type BoxOptions struct {
	Listen  uint
	PortMin uint
	PortMax uint
	InitLua string
}

// NewBox instance.
func NewBox(config string, options ...BoxOptions) (*Box, error) {
	var opts BoxOptions
	if len(options) > 0 {
		opts = options[0]
	} else {
		opts = BoxOptions{}
	}

	if opts.PortMin == 0 {
		opts.PortMin = 8000
	}

	if opts.PortMax == 0 {
		opts.PortMax = 9000
	}

	if opts.Listen != 0 {
		opts.PortMin = opts.Listen
		opts.PortMax = opts.Listen
	}

	var box *Box

START_LOOP:
	for port := opts.PortMin; port <= opts.PortMax; port += 3 {

		tmpDir, err := ioutil.TempDir("", "")
		if err != nil {
			return nil, err
		}

		tarantoolConf := `
		slab_alloc_arena = 1
		slab_alloc_factor = 1.04

		pid_file = {root}/box.pid
		work_dir = {root}
		snap_dir = {snap}
		wal_dir = {wal}

		snap_io_rate_limit = 10
		rows_per_wal = 1000000
		too_long_threshold = 0.025

		primary_port = {port1}
		memcached_expire = false
		memcached_port = {port2}
		admin_port = {port3}
		replication_port = {port4}
        `

		tarantoolConf = strings.Replace(tarantoolConf, "{port1}", fmt.Sprintf("%d", port), -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{port2}", fmt.Sprintf("%d", port+1), -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{port3}", fmt.Sprintf("%d", port+2), -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{port4}", fmt.Sprintf("%d", port+3), -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{root}", tmpDir, -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{snap}", dirSnap, -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{wal}", dirWAL, -1)

		tarantoolConf = fmt.Sprintf("%s\n%s", tarantoolConf, config)

		tarantoolConfFile := path.Join(tmpDir, "tarantool.conf")
		if err = ioutil.WriteFile(tarantoolConfFile, []byte(tarantoolConf), 0644); err != nil {
			return nil, err
		}

		initLuaFile := path.Join(tmpDir, "init.lua")
		if err = ioutil.WriteFile(initLuaFile, []byte(opts.InitLua), 0644); err != nil {
			return nil, err
		}

		for _, subDir := range []string{"snap", "wal"} {
			if err = os.Mkdir(path.Join(tmpDir, subDir), 0755); err != nil {
				return nil, err
			}
		}

		cmd0 := exec.Command("tarantool_box", "-c", tarantoolConfFile, "--init-storage")
		if err = cmd0.Run(); err != nil {
			return nil, err
		}

		cmd := exec.Command("tarantool_box", "-c", tarantoolConfFile)
		boxStderr, err := cmd.StderrPipe()
		if err != nil {
			return nil, err
		}

		if err = cmd.Start(); err != nil {
			return nil, err
		}

		var boxStderrBuffer bytes.Buffer

		p := make([]byte, 1024)

		box = &Box{
			Root:    tmpDir,
			Port:    port,
			cmd:     cmd,
			stopped: make(chan bool),
		}

	WAIT_LOOP:
		for {
			if strings.Contains(boxStderrBuffer.String(), "entering event loop") {
				break START_LOOP
			}

			if strings.Contains(boxStderrBuffer.String(), "is already in use, will retry binding after") {
				cmd.Process.Kill()
				cmd.Process.Wait()
				break WAIT_LOOP
			}

			n, err := boxStderr.Read(p)
			if n > 0 {
				boxStderrBuffer.Write(p[:n])
			}
			if err != nil {
				fmt.Println(boxStderrBuffer.String())
				return nil, err
			}
		}

		os.RemoveAll(box.Root)
		box = nil
	}

	if box == nil {
		return nil, fmt.Errorf("couldn't bind any port from %d to %d", opts.PortMin, opts.PortMax)
	}

	return box, nil
}

// Listen is the primary addr of the box.
func (box *Box) Listen() string {
	return fmt.Sprintf("127.0.0.1:%v", box.Port)
}

// ListenMemcache is the memcache addr of the box.
func (box *Box) ListenMemcache() string {
	return fmt.Sprintf("127.0.0.1:%v", box.Port+1)
}

// ListenAdmin is the admin addr of the box.
func (box *Box) ListenAdmin() string {
	return fmt.Sprintf("127.0.0.1:%v", box.Port+2)
}

// ListenReplica is the replication addr of the box.
func (box *Box) ListenReplica() string {
	return fmt.Sprintf("127.0.0.1:%v", box.Port+3)
}

// SaveSnapshot and return it's filename (with full path).
func (box *Box) SaveSnapshot() (string, error) {
	const (
		cmdSnapshot   = "save snapshot\n"
		cmdOK         = "ok"
		cmdFileExists = "fail: can't save snapshot, errno 17 (File exists)"
		cmdSeparators = "\n-. "
	)
	conn, err := net.Dial("tcp", box.ListenAdmin())
	if err != nil {
		return "", err
	}
	defer conn.Close()
	if err = conn.SetDeadline(time.Now().Add(5 * time.Second)); err != nil {
		return "", err
	}
	if _, err = conn.Write([]byte(cmdSnapshot)); err != nil {
		return "", err
	}
	respBytes := make([]byte, 256)
	n, err := conn.Read(respBytes)
	if err != nil {
		return "", err
	}
	response := strings.TrimRight(strings.TrimLeft(string(respBytes[:n]), cmdSeparators), cmdSeparators)
	switch response {
	case cmdOK, cmdFileExists:
		return box.Snapshot()
	default:
		return "", errors.New(response)
	}
}

// SnapDir of the box.
func (box *Box) SnapDir() string {
	return filepath.Join(box.Root, dirSnap)
}

// WALDir of the box.
func (box *Box) WALDir() string {
	return filepath.Join(box.Root, dirWAL)
}

var ErrSnapshotNotFound = errors.New("snapshot file hasn't been found")

// Snapshot returns the latest snapshot filename with full path.
func (box *Box) Snapshot() (string, error) {
	const snapExt = ".snap"
	files, err := ioutil.ReadDir(box.SnapDir())
	if err != nil {
		return "", err
	}

	// files are sorted alphabetically
	for i := len(files) - 1; i >= 0; i-- {
		if filepath.Ext(files[i].Name()) != snapExt {
			continue
		}
		return filepath.Join(box.SnapDir(), files[i].Name()), nil
	}
	return "", ErrSnapshotNotFound
}

// Close Box instance.
func (box *Box) Close() {
	box.stopOnce.Do(func() {
		box.cmd.Process.Kill()
		box.cmd.Process.Wait()
		os.RemoveAll(box.Root)
		close(box.stopped)
	})
	<-box.stopped
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/fatih/color"
	"github.com/hashicorp/consul/api"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

var usage = func() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] [PATH]\n\n", os.Args[0])
	fmt.Fprintln(os.Stderr, "PATH should be the path to a file or directory that contains your data.")
	fmt.Fprintln(os.Stderr, "If no path is provided, stdin is used. In this case, -format must be specified.\n")
	fmt.Fprintln(os.Stderr, "Options:\n")
	flag.PrintDefaults()
}

var (
	debug   = flag.Bool("debug", false, "Show debugging information")
	dump    = flag.Bool("dump", false, "Dump loaded data as JSON, suitable for using in a 'consul kv import'")
	format  = flag.String("format", "", "Specify data format(json or yaml) when reading from stdin.")
	glue    = flag.String("glue", "\n", "Glue to use when joining array values")
	trace   = flag.Bool("trace", false, "Show even more debugging information")
	enc     = json.NewEncoder(os.Stdout)
	path    string
	absPath string
	data    api.KVPairs
	Trace   *log.Logger
	Info    *log.Logger
	Warning *log.Logger
	Error   *log.Logger
)

func logInit() {
	if *trace {
		Trace = log.New(os.Stderr, "TRACE: ", 0)
		Info = log.New(os.Stderr, color.BlueString("INFO: "), 0)
	} else if *debug {
		Trace = log.New(ioutil.Discard, "TRACE: ", 0)
		Info = log.New(os.Stderr, color.BlueString("INFO: "), 0)

	} else {
		Trace = log.New(ioutil.Discard, "TRACE: ", 0)
		Info = log.New(ioutil.Discard, color.BlueString("INFO: "), 0)
	}
	Warning = log.New(os.Stderr, color.RedString("WARNING: "), 0)
	Error = log.New(os.Stderr, color.RedString("ERROR: "), 0)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	logInit()
	switch flag.NArg() {
	case 0:
		// use stdin
		Trace.Println("No arguments, using stdin instead")
		fi, _ := os.Stdin.Stat()
		switch *format {
		case "json":
			fallthrough
		case "yaml":
			Trace.Printf("Stdin format is going to be: %v", *format)
			err := parseConfig(fmt.Sprintf(".%v", *format), fi, nil)
			if err != nil {
				Error.Printf("%v: %v\n", path, err)
			}
		default:
			Error.Fatal("When reading from stdin, the -format option must be provided and must one of: json, yaml")
		}
	case 1:
		path = flag.Arg(0)
		absPath, _ = filepath.Abs(path)
		_, err := os.Stat(absPath)
		if err != nil {
			Error.Fatal(err)
		}
		err = filepath.Walk(absPath, parseConfig)
		if err != nil {
			Error.Fatal(err)
		}
	default:
		Error.Printf("1 argument expected, but found %d\n\n", flag.NArg())
		usage()
		os.Exit(255)
	}
	if *dump {
		exportData()
	}
}

func parseConfig(path string, f os.FileInfo, err error) error {
	var fp *os.File
	Trace.Printf("Traversing %s", path)
	if f.Mode().IsRegular() && flag.NArg() == 1 {
		fp, err = os.Open(path)
	} else if flag.NArg() == 0 {
		fp = os.Stdin
	} else {
		fp, err = os.Open(os.DevNull)
		err = errors.New(fmt.Sprintf("Not a regular file: %s", path))
	}
	if err != nil {
		Warning.Printf("%v: %v\n", path, err)
	}
	keyPrefix := strings.Split(
		// remove leading '/'
		strings.TrimPrefix(
			// remove the file extension
			strings.TrimSuffix(
				// remove the base path that was passed as -path
				strings.TrimPrefix(path, absPath),
				filepath.Ext(path)),
			string(os.PathSeparator)),
		string(os.PathSeparator))
	if keyPrefix[0] == "" {
		// remove the "" value if passed a file directly in -path
		keyPrefix = []string{}
	}
	Info.Printf("keyprefix is %v", keyPrefix)
	switch {
	case strings.HasSuffix(strings.ToLower(path), ".json"):
		Info.Printf("Parsing %s as json", path)
		err := parseJson(fp, keyPrefix)
		if err != nil {
			Warning.Printf("%v: %v\n", path, err)
		}
	case strings.HasSuffix(strings.ToLower(path), ".yml"):
		fallthrough
	case strings.HasSuffix(strings.ToLower(path), ".yaml"):
		Info.Printf("Parsing %s as yaml", path)
		// yaml handling based on https://github.com/bronze1man/yaml2json
		yamlR, yamlW := io.Pipe()
		go func() {
			defer yamlW.Close()
			err := yamlToJson(fp, yamlW)
			if err != nil {
				Warning.Printf("%v: %v\n", path, err)
			}
		}()
		err := parseJson(yamlR, keyPrefix)
		if err != nil {
			Warning.Printf("%v: %v\n", path, err)
		}
	case strings.HasSuffix(strings.ToLower(path), ".properties"):
		Info.Printf("Parsing %s as properties", path)
	case strings.HasSuffix(strings.ToLower(path), ".ini"):
		Info.Printf("Parsing %s as ini", path)
	default:
	}
	return nil
}

func exportData() {
	exported := make([]*kvExportEntry, len(data))
	for i, kv := range data {
		exported[i] = toExportEntry(kv)
	}
	json, err := json.MarshalIndent(exported, "", "\t")
	if err != nil {
		Error.Fatalf("Error exporting data: %s", err)
	}
	os.Stdout.Write(json)
	fmt.Println("")

}
<commit_msg>cleanup<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/fatih/color"
	"github.com/hashicorp/consul/api"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

var usage = func() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] [PATH]\n\n", os.Args[0])
	fmt.Fprintln(os.Stderr, "PATH should be the path to a file or directory that contains your data.")
	fmt.Fprintln(os.Stderr, "If no path is provided, stdin is used. In this case, -format must be specified.\n")
	fmt.Fprintln(os.Stderr, "Options:\n")
	flag.PrintDefaults()
}

var (
	debug   = flag.Bool("debug", false, "Show debugging information")
	dump    = flag.Bool("dump", false, "Dump loaded data as JSON, suitable for using in a 'consul kv import'")
	format  = flag.String("format", "", "Specify data format(json or yaml) when reading from stdin.")
	glue    = flag.String("glue", "\n", "Glue to use when joining array values")
	trace   = flag.Bool("trace", false, "Show even more debugging information")
	enc     = json.NewEncoder(os.Stdout)
	path    string
	absPath string
	data    api.KVPairs
	Trace   *log.Logger
	Info    *log.Logger
	Warning *log.Logger
	Error   *log.Logger
)

func logInit() {
	if *trace {
		Trace = log.New(os.Stderr, "TRACE: ", 0)
		Info = log.New(os.Stderr, color.BlueString("INFO: "), 0)
	} else if *debug {
		Trace = log.New(ioutil.Discard, "TRACE: ", 0)
		Info = log.New(os.Stderr, color.BlueString("INFO: "), 0)

	} else {
		Trace = log.New(ioutil.Discard, "TRACE: ", 0)
		Info = log.New(ioutil.Discard, color.BlueString("INFO: "), 0)
	}
	Warning = log.New(os.Stderr, color.RedString("WARNING: "), 0)
	Error = log.New(os.Stderr, color.RedString("ERROR: "), 0)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	logInit()
	switch flag.NArg() {
	case 0:
		// use stdin
		Trace.Println("No arguments, using stdin instead")
		fi, _ := os.Stdin.Stat()
		switch *format {
		case "json":
			fallthrough
		case "yaml":
			Trace.Printf("Stdin format is going to be: %v", *format)
			err := parseConfig(fmt.Sprintf(".%v", *format), fi, nil)
			if err != nil {
				Error.Printf("%v: %v\n", path, err)
			}
		default:
			Error.Fatal("When reading from stdin, the -format option must be provided and must one of: json, yaml")
		}
	case 1:
		path = flag.Arg(0)
		absPath, _ = filepath.Abs(path)
		_, err := os.Stat(absPath)
		if err != nil {
			Error.Fatal(err)
		}
		err = filepath.Walk(absPath, parseConfig)
		if err != nil {
			Error.Fatal(err)
		}
	default:
		Error.Printf("1 argument expected, but found %d\n\n", flag.NArg())
		usage()
		os.Exit(255)
	}
	if *dump {
		exportData()
	}
}

func parseConfig(path string, f os.FileInfo, err error) error {
	var fp *os.File
	Trace.Printf("Traversing %s", path)
	if f.Mode().IsRegular() && flag.NArg() == 1 {
		fp, err = os.Open(path)
	} else if flag.NArg() == 0 {
		fp = os.Stdin
	} else {
		return nil
	}
	if err != nil {
		Warning.Printf("%v: %v\n", path, err)
	}
	keyPrefix := strings.Split(
		// remove leading '/'
		strings.TrimPrefix(
			// remove the file extension
			strings.TrimSuffix(
				// remove the base path that was passed as -path
				strings.TrimPrefix(path, absPath),
				filepath.Ext(path)),
			string(os.PathSeparator)),
		string(os.PathSeparator))
	if keyPrefix[0] == "" {
		// remove the "" value if passed a file directly in -path
		keyPrefix = []string{}
	}
	Info.Printf("keyprefix is %v", keyPrefix)
	switch {
	case strings.HasSuffix(strings.ToLower(path), ".json"):
		Info.Printf("Parsing %s as json", path)
		err := parseJson(fp, keyPrefix)
		if err != nil {
			Error.Fatalf("%v: %v\n", path, err)
		}
	case strings.HasSuffix(strings.ToLower(path), ".yml"):
		fallthrough
	case strings.HasSuffix(strings.ToLower(path), ".yaml"):
		Info.Printf("Parsing %s as yaml", path)
		// yaml handling based on https://github.com/bronze1man/yaml2json
		yamlR, yamlW := io.Pipe()
		go func() {
			defer yamlW.Close()
			err := yamlToJson(fp, yamlW)
			if err != nil {
				Warning.Printf("%v: %v\n", path, err)
			}
		}()
		err := parseJson(yamlR, keyPrefix)
		if err != nil {
			Warning.Printf("%v: %v\n", path, err)
		}
	case strings.HasSuffix(strings.ToLower(path), ".properties"):
		Info.Printf("Parsing %s as properties", path)
	case strings.HasSuffix(strings.ToLower(path), ".ini"):
		Info.Printf("Parsing %s as ini", path)
	default:
	}
	return nil
}

func exportData() {
	exported := make([]*kvExportEntry, len(data))
	for i, kv := range data {
		exported[i] = toExportEntry(kv)
	}
	json, err := json.MarshalIndent(exported, "", "\t")
	if err != nil {
		Error.Fatalf("Error exporting data: %s", err)
	}
	os.Stdout.Write(json)
	fmt.Println("")

}
<|endoftext|>