<commit_before>// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/m3db/m3ninx/index/segment/fs (interfaces: Writer,Segment)

// Package fs is a generated GoMock package.
package fs

import (
	"io"
	"reflect"
	"regexp"

	"github.com/m3db/m3ninx/doc"
	"github.com/m3db/m3ninx/index"
	"github.com/m3db/m3ninx/index/segment"
	"github.com/m3db/m3ninx/postings"

	"github.com/golang/mock/gomock"
)

// MockWriter is a mock of Writer interface
type MockWriter struct {
	ctrl     *gomock.Controller
	recorder *MockWriterMockRecorder
}

// MockWriterMockRecorder is the mock recorder for MockWriter
type MockWriterMockRecorder struct {
	mock *MockWriter
}

// NewMockWriter creates a new mock instance
func NewMockWriter(ctrl *gomock.Controller) *MockWriter {
	mock := &MockWriter{ctrl: ctrl}
	mock.recorder = &MockWriterMockRecorder{mock}
	return mock
}

// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockWriter) EXPECT() *MockWriterMockRecorder {
	return m.recorder
}

// MajorVersion mocks base method
func (m *MockWriter) MajorVersion() int {
	ret := m.ctrl.Call(m, "MajorVersion")
	ret0, _ := ret[0].(int)
	return ret0
}

// MajorVersion indicates an expected call of MajorVersion
func (mr *MockWriterMockRecorder) MajorVersion() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MajorVersion", reflect.TypeOf((*MockWriter)(nil).MajorVersion))
}

// Metadata mocks base method
func (m *MockWriter) Metadata() []byte {
	ret := m.ctrl.Call(m, "Metadata")
	ret0, _ := ret[0].([]byte)
	return ret0
}

// Metadata indicates an expected call of Metadata
func (mr *MockWriterMockRecorder) Metadata() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Metadata", reflect.TypeOf((*MockWriter)(nil).Metadata))
}

// MinorVersion mocks base method
func (m *MockWriter) MinorVersion() int {
	ret := m.ctrl.Call(m, "MinorVersion")
	ret0, _ := ret[0].(int)
	return ret0
}

// MinorVersion indicates an expected call of MinorVersion
func (mr *MockWriterMockRecorder) MinorVersion() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MinorVersion", reflect.TypeOf((*MockWriter)(nil).MinorVersion))
}

// Reset mocks base method
func (m *MockWriter) Reset(arg0 segment.MutableSegment) error {
	ret := m.ctrl.Call(m, "Reset", arg0)
	ret0, _ := ret[0].(error)
	return ret0
}

// Reset indicates an expected call of Reset
func (mr *MockWriterMockRecorder) Reset(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reset", reflect.TypeOf((*MockWriter)(nil).Reset), arg0)
}

// WriteDocumentsData mocks base method
func (m *MockWriter) WriteDocumentsData(arg0 io.Writer) error {
	ret := m.ctrl.Call(m, "WriteDocumentsData", arg0)
	ret0, _ := ret[0].(error)
	return ret0
}

// WriteDocumentsData indicates an expected call of WriteDocumentsData
func (mr *MockWriterMockRecorder) WriteDocumentsData(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteDocumentsData", reflect.TypeOf((*MockWriter)(nil).WriteDocumentsData), arg0)
}

// WriteDocumentsIndex mocks base method
func (m *MockWriter) WriteDocumentsIndex(arg0 io.Writer) error {
	ret := m.ctrl.Call(m, "WriteDocumentsIndex", arg0)
	ret0, _ := ret[0].(error)
	return ret0
}

// WriteDocumentsIndex indicates an expected call of WriteDocumentsIndex
func (mr *MockWriterMockRecorder) WriteDocumentsIndex(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteDocumentsIndex", reflect.TypeOf((*MockWriter)(nil).WriteDocumentsIndex), arg0)
}

// WriteFSTFields mocks base method
func (m *MockWriter) WriteFSTFields(arg0 io.Writer) error {
	ret := m.ctrl.Call(m, "WriteFSTFields", arg0)
	ret0, _ := ret[0].(error)
	return ret0
}

// WriteFSTFields indicates an expected call of WriteFSTFields
func (mr *MockWriterMockRecorder) WriteFSTFields(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteFSTFields", reflect.TypeOf((*MockWriter)(nil).WriteFSTFields), arg0)
}

// WriteFSTTerms mocks base method
func (m *MockWriter) WriteFSTTerms(arg0 io.Writer) error {
	ret := m.ctrl.Call(m, "WriteFSTTerms", arg0)
	ret0, _ := ret[0].(error)
	return ret0
}

// WriteFSTTerms indicates an expected call of WriteFSTTerms
func (mr *MockWriterMockRecorder) WriteFSTTerms(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteFSTTerms", reflect.TypeOf((*MockWriter)(nil).WriteFSTTerms), arg0)
}

// WritePostingsOffsets mocks base method
func (m *MockWriter) WritePostingsOffsets(arg0 io.Writer) error {
	ret := m.ctrl.Call(m, "WritePostingsOffsets", arg0)
	ret0, _ := ret[0].(error)
	return ret0
}

// WritePostingsOffsets indicates an expected call of WritePostingsOffsets
func (mr *MockWriterMockRecorder) WritePostingsOffsets(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WritePostingsOffsets", reflect.TypeOf((*MockWriter)(nil).WritePostingsOffsets), arg0)
}

// MockSegment is a mock of Segment interface
type MockSegment struct {
	ctrl     *gomock.Controller
	recorder *MockSegmentMockRecorder
}

// MockSegmentMockRecorder is the mock recorder for MockSegment
type MockSegmentMockRecorder struct {
	mock *MockSegment
}

// NewMockSegment creates a new mock instance
func NewMockSegment(ctrl *gomock.Controller) *MockSegment {
	mock := &MockSegment{ctrl: ctrl}
	mock.recorder = &MockSegmentMockRecorder{mock}
	return mock
}

// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockSegment) EXPECT() *MockSegmentMockRecorder {
	return m.recorder
}

// AllDocs mocks base method
func (m *MockSegment) AllDocs() (doc.Iterator, error) {
	ret := m.ctrl.Call(m, "AllDocs")
	ret0, _ := ret[0].(doc.Iterator)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// AllDocs indicates an expected call of AllDocs
func (mr *MockSegmentMockRecorder) AllDocs() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllDocs", reflect.TypeOf((*MockSegment)(nil).AllDocs))
}

// Close mocks base method
func (m *MockSegment) Close() error {
	ret := m.ctrl.Call(m, "Close")
	ret0, _ := ret[0].(error)
	return ret0
}

// Close indicates an expected call of Close
func (mr *MockSegmentMockRecorder) Close() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockSegment)(nil).Close))
}

// ContainsID mocks base method
func (m *MockSegment) ContainsID(arg0 []byte) (bool, error) {
	ret := m.ctrl.Call(m, "ContainsID", arg0)
	ret0, _ := ret[0].(bool)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// ContainsID indicates an expected call of ContainsID
func (mr *MockSegmentMockRecorder) ContainsID(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ContainsID", reflect.TypeOf((*MockSegment)(nil).ContainsID), arg0)
}

// Docs mocks base method
func (m *MockSegment) Docs(arg0 postings.List) (doc.Iterator, error) {
	ret := m.ctrl.Call(m, "Docs", arg0)
	ret0, _ := ret[0].(doc.Iterator)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Docs indicates an expected call of Docs
func (mr *MockSegmentMockRecorder) Docs(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Docs", reflect.TypeOf((*MockSegment)(nil).Docs), arg0)
}

// Fields mocks base method
func (m *MockSegment) Fields() ([][]byte, error) {
	ret := m.ctrl.Call(m, "Fields")
	ret0, _ := ret[0].([][]byte)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Fields indicates an expected call of Fields
func (mr *MockSegmentMockRecorder) Fields() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Fields", reflect.TypeOf((*MockSegment)(nil).Fields))
}

// MatchAll mocks base method
func (m *MockSegment) MatchAll() (postings.MutableList, error) {
	ret := m.ctrl.Call(m, "MatchAll")
	ret0, _ := ret[0].(postings.MutableList)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// MatchAll indicates an expected call of MatchAll
func (mr *MockSegmentMockRecorder) MatchAll() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MatchAll", reflect.TypeOf((*MockSegment)(nil).MatchAll))
}

// MatchRegexp mocks base method
func (m *MockSegment) MatchRegexp(arg0, arg1 []byte, arg2 *regexp.Regexp) (postings.List, error) {
	ret := m.ctrl.Call(m, "MatchRegexp", arg0, arg1, arg2)
	ret0, _ := ret[0].(postings.List)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// MatchRegexp indicates an expected call of MatchRegexp
func (mr *MockSegmentMockRecorder) MatchRegexp(arg0, arg1, arg2 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MatchRegexp", reflect.TypeOf((*MockSegment)(nil).MatchRegexp), arg0, arg1, arg2)
}

// MatchTerm mocks base method
func (m *MockSegment) MatchTerm(arg0, arg1 []byte) (postings.List, error) {
	ret := m.ctrl.Call(m, "MatchTerm", arg0, arg1)
	ret0, _ := ret[0].(postings.List)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// MatchTerm indicates an expected call of MatchTerm
func (mr *MockSegmentMockRecorder) MatchTerm(arg0, arg1 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MatchTerm", reflect.TypeOf((*MockSegment)(nil).MatchTerm), arg0, arg1)
}

// Reader mocks base method
func (m *MockSegment) Reader() (index.Reader, error) {
	ret := m.ctrl.Call(m, "Reader")
	ret0, _ := ret[0].(index.Reader)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Reader indicates an expected call of Reader
func (mr *MockSegmentMockRecorder) Reader() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reader", reflect.TypeOf((*MockSegment)(nil).Reader))
}

// Size mocks base method
func (m *MockSegment) Size() int64 {
	ret := m.ctrl.Call(m, "Size")
	ret0, _ := ret[0].(int64)
	return ret0
}

// Size indicates an expected call of Size
func (mr *MockSegmentMockRecorder) Size() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Size", reflect.TypeOf((*MockSegment)(nil).Size))
}

// Terms mocks base method
func (m *MockSegment) Terms(arg0 []byte) ([][]byte, error) {
	ret := m.ctrl.Call(m, "Terms", arg0)
	ret0, _ := ret[0].([][]byte)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Terms indicates an expected call of Terms
func (mr *MockSegmentMockRecorder) Terms(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Terms", reflect.TypeOf((*MockSegment)(nil).Terms), arg0)
}
<commit_msg>Update mocks (#84)<commit_after>// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/m3db/m3ninx/index/segment/fs (interfaces: Writer,Segment)

// Package fs is a generated GoMock package.
package fs

import (
	"io"
	"reflect"
	"regexp"

	"github.com/m3db/m3ninx/doc"
	"github.com/m3db/m3ninx/index"
	"github.com/m3db/m3ninx/index/segment"
	"github.com/m3db/m3ninx/postings"

	"github.com/golang/mock/gomock"
)

// MockWriter is a mock of Writer interface
type MockWriter struct {
	ctrl     *gomock.Controller
	recorder *MockWriterMockRecorder
}

// MockWriterMockRecorder is the mock recorder for MockWriter
type MockWriterMockRecorder struct {
	mock *MockWriter
}

// NewMockWriter creates a new mock instance
func NewMockWriter(ctrl *gomock.Controller) *MockWriter {
	mock := &MockWriter{ctrl: ctrl}
	mock.recorder = &MockWriterMockRecorder{mock}
	return mock
}

// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockWriter) EXPECT() *MockWriterMockRecorder {
	return m.recorder
}

// MajorVersion mocks base method
func (m *MockWriter) MajorVersion() int {
	ret := m.ctrl.Call(m, "MajorVersion")
	ret0, _ := ret[0].(int)
	return ret0
}

// MajorVersion indicates an expected call of MajorVersion
func (mr *MockWriterMockRecorder) MajorVersion() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MajorVersion", reflect.TypeOf((*MockWriter)(nil).MajorVersion))
}

// Metadata mocks base method
func (m *MockWriter) Metadata() []byte {
	ret := m.ctrl.Call(m, "Metadata")
	ret0, _ := ret[0].([]byte)
	return ret0
}

// Metadata indicates an expected call of Metadata
func (mr *MockWriterMockRecorder) Metadata() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Metadata", reflect.TypeOf((*MockWriter)(nil).Metadata))
}

// MinorVersion mocks base method
func (m *MockWriter) MinorVersion() int {
	ret := m.ctrl.Call(m, "MinorVersion")
	ret0, _ := ret[0].(int)
	return ret0
}

// MinorVersion indicates an expected call of MinorVersion
func (mr *MockWriterMockRecorder) MinorVersion() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MinorVersion", reflect.TypeOf((*MockWriter)(nil).MinorVersion))
}

// Reset mocks base method
func (m *MockWriter) Reset(arg0 segment.MutableSegment) error {
	ret := m.ctrl.Call(m, "Reset", arg0)
	ret0, _ := ret[0].(error)
	return ret0
}

// Reset indicates an expected call of Reset
func (mr *MockWriterMockRecorder) Reset(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reset", reflect.TypeOf((*MockWriter)(nil).Reset), arg0)
}

// WriteDocumentsData mocks base method
func (m *MockWriter) WriteDocumentsData(arg0 io.Writer) error {
	ret := m.ctrl.Call(m, "WriteDocumentsData", arg0)
	ret0, _ := ret[0].(error)
	return ret0
}

// WriteDocumentsData indicates an expected call of WriteDocumentsData
func (mr *MockWriterMockRecorder) WriteDocumentsData(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteDocumentsData", reflect.TypeOf((*MockWriter)(nil).WriteDocumentsData), arg0)
}

// WriteDocumentsIndex mocks base method
func (m *MockWriter) WriteDocumentsIndex(arg0 io.Writer) error {
	ret := m.ctrl.Call(m, "WriteDocumentsIndex", arg0)
	ret0, _ := ret[0].(error)
	return ret0
}

// WriteDocumentsIndex indicates an expected call of WriteDocumentsIndex
func (mr *MockWriterMockRecorder) WriteDocumentsIndex(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteDocumentsIndex", reflect.TypeOf((*MockWriter)(nil).WriteDocumentsIndex), arg0)
}

// WriteFSTFields mocks base method
func (m *MockWriter) WriteFSTFields(arg0 io.Writer) error {
	ret := m.ctrl.Call(m, "WriteFSTFields", arg0)
	ret0, _ := ret[0].(error)
	return ret0
}

// WriteFSTFields indicates an expected call of WriteFSTFields
func (mr *MockWriterMockRecorder) WriteFSTFields(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteFSTFields", reflect.TypeOf((*MockWriter)(nil).WriteFSTFields), arg0)
}

// WriteFSTTerms mocks base method
func (m *MockWriter) WriteFSTTerms(arg0 io.Writer) error {
	ret := m.ctrl.Call(m, "WriteFSTTerms", arg0)
	ret0, _ := ret[0].(error)
	return ret0
}

// WriteFSTTerms indicates an expected call of WriteFSTTerms
func (mr *MockWriterMockRecorder) WriteFSTTerms(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteFSTTerms", reflect.TypeOf((*MockWriter)(nil).WriteFSTTerms), arg0)
}

// WritePostingsOffsets mocks base method
func (m *MockWriter) WritePostingsOffsets(arg0 io.Writer) error {
	ret := m.ctrl.Call(m, "WritePostingsOffsets", arg0)
	ret0, _ := ret[0].(error)
	return ret0
}

// WritePostingsOffsets indicates an expected call of WritePostingsOffsets
func (mr *MockWriterMockRecorder) WritePostingsOffsets(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WritePostingsOffsets", reflect.TypeOf((*MockWriter)(nil).WritePostingsOffsets), arg0)
}

// MockSegment is a mock of Segment interface
type MockSegment struct {
	ctrl     *gomock.Controller
	recorder *MockSegmentMockRecorder
}

// MockSegmentMockRecorder is the mock recorder for MockSegment
type MockSegmentMockRecorder struct {
	mock *MockSegment
}

// NewMockSegment creates a new mock instance
func NewMockSegment(ctrl *gomock.Controller) *MockSegment {
	mock := &MockSegment{ctrl: ctrl}
	mock.recorder = &MockSegmentMockRecorder{mock}
	return mock
}

// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockSegment) EXPECT() *MockSegmentMockRecorder {
	return m.recorder
}

// AllDocs mocks base method
func (m *MockSegment) AllDocs() (index.IDDocIterator, error) {
	ret := m.ctrl.Call(m, "AllDocs")
	ret0, _ := ret[0].(index.IDDocIterator)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// AllDocs indicates an expected call of AllDocs
func (mr *MockSegmentMockRecorder) AllDocs() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllDocs", reflect.TypeOf((*MockSegment)(nil).AllDocs))
}

// Close mocks base method
func (m *MockSegment) Close() error {
	ret := m.ctrl.Call(m, "Close")
	ret0, _ := ret[0].(error)
	return ret0
}

// Close indicates an expected call of Close
func (mr *MockSegmentMockRecorder) Close() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockSegment)(nil).Close))
}

// ContainsID mocks base method
func (m *MockSegment) ContainsID(arg0 []byte) (bool, error) {
	ret := m.ctrl.Call(m, "ContainsID", arg0)
	ret0, _ := ret[0].(bool)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// ContainsID indicates an expected call of ContainsID
func (mr *MockSegmentMockRecorder) ContainsID(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ContainsID", reflect.TypeOf((*MockSegment)(nil).ContainsID), arg0)
}

// Doc mocks base method
func (m *MockSegment) Doc(arg0 postings.ID) (doc.Document, error) {
	ret := m.ctrl.Call(m, "Doc", arg0)
	ret0, _ := ret[0].(doc.Document)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Doc indicates an expected call of Doc
func (mr *MockSegmentMockRecorder) Doc(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Doc", reflect.TypeOf((*MockSegment)(nil).Doc), arg0)
}

// Docs mocks base method
func (m *MockSegment) Docs(arg0 postings.List) (doc.Iterator, error) {
	ret := m.ctrl.Call(m, "Docs", arg0)
	ret0, _ := ret[0].(doc.Iterator)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Docs indicates an expected call of Docs
func (mr *MockSegmentMockRecorder) Docs(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Docs", reflect.TypeOf((*MockSegment)(nil).Docs), arg0)
}

// Fields mocks base method
func (m *MockSegment) Fields() ([][]byte, error) {
	ret := m.ctrl.Call(m, "Fields")
	ret0, _ := ret[0].([][]byte)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Fields indicates an expected call of Fields
func (mr *MockSegmentMockRecorder) Fields() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Fields", reflect.TypeOf((*MockSegment)(nil).Fields))
}

// MatchAll mocks base method
func (m *MockSegment) MatchAll() (postings.MutableList, error) {
	ret := m.ctrl.Call(m, "MatchAll")
	ret0, _ := ret[0].(postings.MutableList)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// MatchAll indicates an expected call of MatchAll
func (mr *MockSegmentMockRecorder) MatchAll() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MatchAll", reflect.TypeOf((*MockSegment)(nil).MatchAll))
}

// MatchRegexp mocks base method
func (m *MockSegment) MatchRegexp(arg0, arg1 []byte, arg2 *regexp.Regexp) (postings.List, error) {
	ret := m.ctrl.Call(m, "MatchRegexp", arg0, arg1, arg2)
	ret0, _ := ret[0].(postings.List)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// MatchRegexp indicates an expected call of MatchRegexp
func (mr *MockSegmentMockRecorder) MatchRegexp(arg0, arg1, arg2 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MatchRegexp", reflect.TypeOf((*MockSegment)(nil).MatchRegexp), arg0, arg1, arg2)
}

// MatchTerm mocks base method
func (m *MockSegment) MatchTerm(arg0, arg1 []byte) (postings.List, error) {
	ret := m.ctrl.Call(m, "MatchTerm", arg0, arg1)
	ret0, _ := ret[0].(postings.List)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// MatchTerm indicates an expected call of MatchTerm
func (mr *MockSegmentMockRecorder) MatchTerm(arg0, arg1 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MatchTerm", reflect.TypeOf((*MockSegment)(nil).MatchTerm), arg0, arg1)
}

// Reader mocks base method
func (m *MockSegment) Reader() (index.Reader, error) {
	ret := m.ctrl.Call(m, "Reader")
	ret0, _ := ret[0].(index.Reader)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Reader indicates an expected call of Reader
func (mr *MockSegmentMockRecorder) Reader() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reader", reflect.TypeOf((*MockSegment)(nil).Reader))
}

// Size mocks base method
func (m *MockSegment) Size() int64 {
	ret := m.ctrl.Call(m, "Size")
	ret0, _ := ret[0].(int64)
	return ret0
}

// Size indicates an expected call of Size
func (mr *MockSegmentMockRecorder) Size() *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Size", reflect.TypeOf((*MockSegment)(nil).Size))
}

// Terms mocks base method
func (m *MockSegment) Terms(arg0 []byte) ([][]byte, error) {
	ret := m.ctrl.Call(m, "Terms", arg0)
	ret0, _ := ret[0].([][]byte)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Terms indicates an expected call of Terms
func (mr *MockSegmentMockRecorder) Terms(arg0 interface{}) *gomock.Call {
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Terms", reflect.TypeOf((*MockSegment)(nil).Terms), arg0)
}
<|endoftext|>


<commit_before>/*
 * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"crypto/tls"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path"
	"sync"
	"time"

	xhttp "github.com/minio/minio/cmd/http"
	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/cmd/rest"
	"github.com/minio/minio/pkg/sync/errgroup"
)

var printEndpointError = func() func(Endpoint, error) {
	var mutex sync.Mutex
	printOnce := make(map[Endpoint]map[string]bool)

	return func(endpoint Endpoint, err error) {
		reqInfo := (&logger.ReqInfo{}).AppendTags("endpoint", endpoint.String())
		ctx := logger.SetReqInfo(context.Background(), reqInfo)
		mutex.Lock()
		defer mutex.Unlock()
		m, ok := printOnce[endpoint]
		if !ok {
			m = make(map[string]bool)
			m[err.Error()] = true
			printOnce[endpoint] = m
			logger.LogAlwaysIf(ctx, err)
			return
		}
		if m[err.Error()] {
			return
		}
		m[err.Error()] = true
		logger.LogAlwaysIf(ctx, err)
	}
}()

// Migrates backend format of local disks.
func formatXLMigrateLocalEndpoints(endpoints Endpoints) error {
	g := errgroup.WithNErrs(len(endpoints))
	for index, endpoint := range endpoints {
		if !endpoint.IsLocal {
			continue
		}
		index := index
		g.Go(func() error {
			epPath := endpoints[index].Path
			formatPath := pathJoin(epPath, minioMetaBucket, formatConfigFile)
			if _, err := os.Stat(formatPath); err != nil {
				if os.IsNotExist(err) {
					return nil
				}
				return fmt.Errorf("unable to access (%s) %w", formatPath, err)
			}
			return formatXLMigrate(epPath)
		}, index)
	}
	for _, err := range g.Wait() {
		if err != nil {
			return err
		}
	}
	return nil
}

// Cleans up tmp directory of local disks.
func formatXLCleanupTmpLocalEndpoints(endpoints Endpoints) error {
	g := errgroup.WithNErrs(len(endpoints))
	for index, endpoint := range endpoints {
		if !endpoint.IsLocal {
			continue
		}
		index := index
		g.Go(func() error {
			epPath := endpoints[index].Path
			// If disk is not formatted there is nothing to be cleaned up.
			formatPath := pathJoin(epPath, minioMetaBucket, formatConfigFile)
			if _, err := os.Stat(formatPath); err != nil {
				if os.IsNotExist(err) {
					return nil
				}
				return fmt.Errorf("unable to access (%s) %w", formatPath, err)
			}
			if _, err := os.Stat(pathJoin(epPath, minioMetaTmpBucket+"-old")); err != nil {
				if !os.IsNotExist(err) {
					return fmt.Errorf("unable to access (%s) %w",
						pathJoin(epPath, minioMetaTmpBucket+"-old"),
						err)
				}
			}

			// Need to move temporary objects left behind from previous run of minio
			// server to a unique directory under `minioMetaTmpBucket-old` to clean
			// up `minioMetaTmpBucket` for the current run.
			//
			// /disk1/.minio.sys/tmp-old/
			//  |__ 33a58b40-aecc-4c9f-a22f-ff17bfa33b62
			//  |__ e870a2c1-d09c-450c-a69c-6eaa54a89b3e
			//
			// In this example, `33a58b40-aecc-4c9f-a22f-ff17bfa33b62` directory contains
			// temporary objects from one of the previous runs of minio server.
			tmpOld := pathJoin(epPath, minioMetaTmpBucket+"-old", mustGetUUID())
			if err := renameAll(pathJoin(epPath, minioMetaTmpBucket),
				tmpOld); err != nil && err != errFileNotFound {
				return fmt.Errorf("unable to rename (%s -> %s) %w",
					pathJoin(epPath, minioMetaTmpBucket),
					tmpOld,
					err)
			}

			// Removal of tmp-old folder is backgrounded completely.
			go removeAll(pathJoin(epPath, minioMetaTmpBucket+"-old"))

			if err := mkdirAll(pathJoin(epPath, minioMetaTmpBucket), 0777); err != nil {
				return fmt.Errorf("unable to create (%s) %w",
					pathJoin(epPath, minioMetaTmpBucket),
					err)
			}
			return nil
		}, index)
	}
	for _, err := range g.Wait() {
		if err != nil {
			return err
		}
	}
	return nil
}

// validate reference format against list of XL formats.
func validateXLFormats(format *formatXLV3, formats []*formatXLV3, endpoints Endpoints, setCount, drivesPerSet int) error {
	for i := range formats {
		if formats[i] == nil {
			continue
		}
		if err := formatXLV3Check(format, formats[i]); err != nil {
			return fmt.Errorf("%s format error: %w", endpoints[i], err)
		}
	}
	if len(format.XL.Sets) != setCount {
		return fmt.Errorf("Current backend format is inconsistent with input args (%s), Expected set count %d, got %d", endpoints, len(format.XL.Sets), setCount)
	}
	if len(format.XL.Sets[0]) != drivesPerSet {
		return fmt.Errorf("Current backend format is inconsistent with input args (%s), Expected drive count per set %d, got %d", endpoints, len(format.XL.Sets[0]), drivesPerSet)
	}

	return nil
}

// Following error message is added to fix a regression in release
// RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3. This
// migration failed to capture '.This' field properly which indicates
// the disk UUID association. Below error message is returned when
// we see this situation in format.json, for more info refer
// https://github.com/minio/minio/issues/5667
var errXLV3ThisEmpty = fmt.Errorf("XL format version 3 has This field empty")

// IsServerResolvable - checks if the endpoint is resolvable
// by sending a naked HTTP request with liveness checks.
func IsServerResolvable(endpoint Endpoint) error {
	serverURL := &url.URL{
		Scheme: endpoint.Scheme,
		Host:   endpoint.Host,
		Path:   path.Join(healthCheckPathPrefix, healthCheckLivenessPath),
	}

	var tlsConfig *tls.Config
	if globalIsSSL {
		tlsConfig = &tls.Config{
			ServerName: endpoint.Hostname(),
			RootCAs:    globalRootCAs,
			NextProtos: []string{"http/1.1"}, // Force http1.1
		}
	}

	req, err := http.NewRequest(http.MethodGet, serverURL.String(), nil)
	if err != nil {
		return err
	}

	httpClient := &http.Client{
		Transport: newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout)(),
	}

	resp, err := httpClient.Do(req)
	if err != nil {
		httpClient.CloseIdleConnections()
		return err
	}
	defer xhttp.DrainBody(resp.Body)

	if resp.StatusCode != http.StatusOK {
		return StorageErr(resp.Status)
	}
	return nil
}

// connect to list of endpoints and load all XL disk formats, validate the formats are correct
// and are in quorum, if no formats are found attempt to initialize all of them for the first
// time. additionally make sure to close all the disks used in this attempt.
func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints, zoneCount, setCount, drivesPerSet int, deploymentID string) (*formatXLV3, error) {
	// Initialize all storage disks
	storageDisks, errs := initStorageDisksWithErrors(endpoints)
	defer closeStorageDisks(storageDisks)

	for i, err := range errs {
		if err != nil {
			if err != errDiskNotFound {
				return nil, fmt.Errorf("Disk %s: %w", endpoints[i], err)
			}
			if retryCount >= 5 {
				logger.Info("Unable to connect to %s: %v\n", endpoints[i], IsServerResolvable(endpoints[i]))
			}
		}
	}

	// Attempt to load all `format.json` from all disks.
	formatConfigs, sErrs := loadFormatXLAll(storageDisks)
	// Check if we have
	for i, sErr := range sErrs {
		if _, ok := formatCriticalErrors[sErr]; ok {
			return nil, fmt.Errorf("Disk %s: %w", endpoints[i], sErr)
		}
	}

	// Pre-emptively check if one of the formatted disks
	// is invalid. This function returns success for the
	// most part unless one of the formats is not consistent
	// with expected XL format. For example if a user is
	// trying to pool FS backend into an XL set.
	if err := checkFormatXLValues(formatConfigs); err != nil {
		return nil, err
	}

	// All disks report unformatted we should initialized everyone.
	if shouldInitXLDisks(sErrs) && firstDisk {
		logger.Info("Formatting %v zone, %v set(s), %v drives per set.",
			zoneCount, setCount, drivesPerSet)

		// Initialize erasure code format on disks
		format, err := initFormatXL(context.Background(), storageDisks, setCount, drivesPerSet, deploymentID)
		if err != nil {
			return nil, err
		}
		// Assign globalDeploymentID on first run for the
		// minio server managing the first disk
		globalDeploymentID = format.ID
		return format, nil
	}

	// Return error when quorum unformatted disks - indicating we are
	// waiting for first server to be online.
	if quorumUnformattedDisks(sErrs) && !firstDisk {
		return nil, errNotFirstDisk
	}

	// Return error when quorum unformatted disks but waiting for rest
	// of the servers to be online.
	if quorumUnformattedDisks(sErrs) && firstDisk {
		return nil, errFirstDiskWait
	}

	// Following function is added to fix a regressions which was introduced
	// in release RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3.
	// This migration failed to capture '.This' field properly which indicates
	// the disk UUID association. Below function is called to handle and fix
	// this regression, for more info refer https://github.com/minio/minio/issues/5667
	if err := fixFormatXLV3(storageDisks, endpoints, formatConfigs); err != nil {
		return nil, err
	}

	// If any of the .This field is still empty, we return error.
	if formatXLV3ThisEmpty(formatConfigs) {
		return nil, errXLV3ThisEmpty
	}

	format, err := getFormatXLInQuorum(formatConfigs)
	if err != nil {
		return nil, err
	}

	if format.ID == "" {
		// Not a first disk, wait until first disk fixes deploymentID
		if !firstDisk {
			return nil, errNotFirstDisk
		}
		if err = formatXLFixDeploymentID(endpoints, storageDisks, format); err != nil {
			return nil, err
		}
	}

	globalDeploymentID = format.ID

	if err = formatXLFixLocalDeploymentID(endpoints, storageDisks, format); err != nil {
		return nil, err
	}

	// The will always recreate some directories inside .minio.sys of
	// the local disk such as tmp, multipart and background-ops
	initXLMetaVolumesInLocalDisks(storageDisks, formatConfigs)

	return format, nil
}

// Format disks before initialization of object layer.
func waitForFormatXL(firstDisk bool, endpoints Endpoints, zoneCount, setCount, drivesPerSet int, deploymentID string) (format *formatXLV3, err error) {
	if len(endpoints) == 0 || setCount == 0 || drivesPerSet == 0 {
		return nil, errInvalidArgument
	}

	if err = formatXLMigrateLocalEndpoints(endpoints); err != nil {
		return nil, err
	}

	if err = formatXLCleanupTmpLocalEndpoints(endpoints); err != nil {
		return nil, err
	}

	// prepare getElapsedTime() to calculate elapsed time since we started trying formatting disks.
	// All times are rounded to avoid showing milli, micro and nano seconds
	formatStartTime := time.Now().Round(time.Second)
	getElapsedTime := func() string {
		return time.Now().Round(time.Second).Sub(formatStartTime).String()
	}

	// Wait on each try for an update.
	ticker := time.NewTicker(500 * time.Millisecond)
	defer ticker.Stop()
	var tries int
	for {
		select {
		case <-ticker.C:
			format, err := connectLoadInitFormats(tries, firstDisk, endpoints, zoneCount, setCount, drivesPerSet, deploymentID)
			if err != nil {
				tries++
				switch err {
				case errNotFirstDisk:
					// Fresh setup, wait for first server to be up.
					logger.Info("Waiting for the first server to format the disks.")
					continue
				case errFirstDiskWait:
					// Fresh setup, wait for other servers to come up.
					logger.Info("Waiting for all other servers to be online to format the disks.")
					continue
				case errXLReadQuorum:
					// no quorum available continue to wait for minimum number of servers.
					logger.Info("Waiting for a minimum of %d disks to come online (elapsed %s)\n", len(endpoints)/2, getElapsedTime())
					continue
				case errXLV3ThisEmpty:
					// need to wait for this error to be healed, so continue.
					continue
				default:
					// For all other unhandled errors we exit and fail.
					return nil, err
				}
			}
			return format, nil
		case <-globalOSSignalCh:
			return nil, fmt.Errorf("Initializing data volumes gracefully stopped")
		}
	}
}
<commit_msg>add additional logging during server formatting (#9102)<commit_after>/*
 * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"crypto/tls"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path"
	"sync"
	"time"

	xhttp "github.com/minio/minio/cmd/http"
	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/cmd/rest"
	"github.com/minio/minio/pkg/sync/errgroup"
)

var printEndpointError = func() func(Endpoint, error) {
	var mutex sync.Mutex
	printOnce := make(map[Endpoint]map[string]bool)

	return func(endpoint Endpoint, err error) {
		reqInfo := (&logger.ReqInfo{}).AppendTags("endpoint", endpoint.String())
		ctx := logger.SetReqInfo(context.Background(), reqInfo)
		mutex.Lock()
		defer mutex.Unlock()
		m, ok := printOnce[endpoint]
		if !ok {
			m = make(map[string]bool)
			m[err.Error()] = true
			printOnce[endpoint] = m
			logger.LogAlwaysIf(ctx, err)
			return
		}
		if m[err.Error()] {
			return
		}
		m[err.Error()] = true
		logger.LogAlwaysIf(ctx, err)
	}
}()

// Migrates backend format of local disks.
func formatXLMigrateLocalEndpoints(endpoints Endpoints) error {
	g := errgroup.WithNErrs(len(endpoints))
	for index, endpoint := range endpoints {
		if !endpoint.IsLocal {
			continue
		}
		index := index
		g.Go(func() error {
			epPath := endpoints[index].Path
			formatPath := pathJoin(epPath, minioMetaBucket, formatConfigFile)
			if _, err := os.Stat(formatPath); err != nil {
				if os.IsNotExist(err) {
					return nil
				}
				return fmt.Errorf("unable to access (%s) %w", formatPath, err)
			}
			return formatXLMigrate(epPath)
		}, index)
	}
	for _, err := range g.Wait() {
		if err != nil {
			return err
		}
	}
	return nil
}

// Cleans up tmp directory of local disks.
func formatXLCleanupTmpLocalEndpoints(endpoints Endpoints) error {
	g := errgroup.WithNErrs(len(endpoints))
	for index, endpoint := range endpoints {
		if !endpoint.IsLocal {
			continue
		}
		index := index
		g.Go(func() error {
			epPath := endpoints[index].Path
			// If disk is not formatted there is nothing to be cleaned up.
			formatPath := pathJoin(epPath, minioMetaBucket, formatConfigFile)
			if _, err := os.Stat(formatPath); err != nil {
				if os.IsNotExist(err) {
					return nil
				}
				return fmt.Errorf("unable to access (%s) %w", formatPath, err)
			}
			if _, err := os.Stat(pathJoin(epPath, minioMetaTmpBucket+"-old")); err != nil {
				if !os.IsNotExist(err) {
					return fmt.Errorf("unable to access (%s) %w",
						pathJoin(epPath, minioMetaTmpBucket+"-old"),
						err)
				}
			}

			// Need to move temporary objects left behind from previous run of minio
			// server to a unique directory under `minioMetaTmpBucket-old` to clean
			// up `minioMetaTmpBucket` for the current run.
			//
			// /disk1/.minio.sys/tmp-old/
			//  |__ 33a58b40-aecc-4c9f-a22f-ff17bfa33b62
			//  |__ e870a2c1-d09c-450c-a69c-6eaa54a89b3e
			//
			// In this example, `33a58b40-aecc-4c9f-a22f-ff17bfa33b62` directory contains
			// temporary objects from one of the previous runs of minio server.
			tmpOld := pathJoin(epPath, minioMetaTmpBucket+"-old", mustGetUUID())
			if err := renameAll(pathJoin(epPath, minioMetaTmpBucket),
				tmpOld); err != nil && err != errFileNotFound {
				return fmt.Errorf("unable to rename (%s -> %s) %w",
					pathJoin(epPath, minioMetaTmpBucket),
					tmpOld,
					err)
			}

			// Removal of tmp-old folder is backgrounded completely.
			go removeAll(pathJoin(epPath, minioMetaTmpBucket+"-old"))

			if err := mkdirAll(pathJoin(epPath, minioMetaTmpBucket), 0777); err != nil {
				return fmt.Errorf("unable to create (%s) %w",
					pathJoin(epPath, minioMetaTmpBucket),
					err)
			}
			return nil
		}, index)
	}
	for _, err := range g.Wait() {
		if err != nil {
			return err
		}
	}
	return nil
}

// validate reference format against list of XL formats.
func validateXLFormats(format *formatXLV3, formats []*formatXLV3, endpoints Endpoints, setCount, drivesPerSet int) error {
	for i := range formats {
		if formats[i] == nil {
			continue
		}
		if err := formatXLV3Check(format, formats[i]); err != nil {
			return fmt.Errorf("%s format error: %w", endpoints[i], err)
		}
	}
	if len(format.XL.Sets) != setCount {
		return fmt.Errorf("Current backend format is inconsistent with input args (%s), Expected set count %d, got %d", endpoints, len(format.XL.Sets), setCount)
	}
	if len(format.XL.Sets[0]) != drivesPerSet {
		return fmt.Errorf("Current backend format is inconsistent with input args (%s), Expected drive count per set %d, got %d", endpoints, len(format.XL.Sets[0]), drivesPerSet)
	}

	return nil
}

// Following error message is added to fix a regression in release
// RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3. This
// migration failed to capture '.This' field properly which indicates
// the disk UUID association. Below error message is returned when
// we see this situation in format.json, for more info refer
// https://github.com/minio/minio/issues/5667
var errXLV3ThisEmpty = fmt.Errorf("XL format version 3 has This field empty")

// IsServerResolvable - checks if the endpoint is resolvable
// by sending a naked HTTP request with liveness checks.
func IsServerResolvable(endpoint Endpoint) error {
	serverURL := &url.URL{
		Scheme: endpoint.Scheme,
		Host:   endpoint.Host,
		Path:   path.Join(healthCheckPathPrefix, healthCheckLivenessPath),
	}

	var tlsConfig *tls.Config
	if globalIsSSL {
		tlsConfig = &tls.Config{
			ServerName: endpoint.Hostname(),
			RootCAs:    globalRootCAs,
			NextProtos: []string{"http/1.1"}, // Force http1.1
		}
	}

	req, err := http.NewRequest(http.MethodGet, serverURL.String(), nil)
	if err != nil {
		return err
	}

	httpClient := &http.Client{
		Transport: newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout)(),
	}

	resp, err := httpClient.Do(req)
	if err != nil {
		httpClient.CloseIdleConnections()
		return err
	}
	defer xhttp.DrainBody(resp.Body)

	if resp.StatusCode != http.StatusOK {
		return StorageErr(resp.Status)
	}
	return nil
}

// connect to list of endpoints and load all XL disk formats, validate the formats are correct
// and are in quorum, if no formats are found attempt to initialize all of them for the first
// time. additionally make sure to close all the disks used in this attempt.
func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints, zoneCount, setCount, drivesPerSet int, deploymentID string) (*formatXLV3, error) {
	// Initialize all storage disks
	storageDisks, errs := initStorageDisksWithErrors(endpoints)
	defer closeStorageDisks(storageDisks)

	for i, err := range errs {
		if err != nil {
			if err != errDiskNotFound {
				return nil, fmt.Errorf("Disk %s: %w", endpoints[i], err)
			}
			if retryCount >= 5 {
				logger.Info("Unable to connect to %s: %v\n", endpoints[i], IsServerResolvable(endpoints[i]))
			}
		}
	}

	// Attempt to load all `format.json` from all disks.
	formatConfigs, sErrs := loadFormatXLAll(storageDisks)
	// Check if we have
	for i, sErr := range sErrs {
		if _, ok := formatCriticalErrors[sErr]; ok {
			return nil, fmt.Errorf("Disk %s: %w", endpoints[i], sErr)
		}
		// not critical error but still print the error, nonetheless, which is perhaps unhandled
		if sErr != errUnformattedDisk && sErr != errDiskNotFound && retryCount >= 5 {
			logger.Info("Unable to read 'format.json' from %s: %v\n", endpoints[i], sErr)
		}
	}

	// Pre-emptively check if one of the formatted disks
	// is invalid. This function returns success for the
	// most part unless one of the formats is not consistent
	// with expected XL format. For example if a user is
	// trying to pool FS backend into an XL set.
	if err := checkFormatXLValues(formatConfigs); err != nil {
		return nil, err
	}

	// All disks report unformatted we should initialized everyone.
	if shouldInitXLDisks(sErrs) && firstDisk {
		logger.Info("Formatting %v zone, %v set(s), %v drives per set.",
			zoneCount, setCount, drivesPerSet)

		// Initialize erasure code format on disks
		format, err := initFormatXL(context.Background(), storageDisks, setCount, drivesPerSet, deploymentID)
		if err != nil {
			return nil, err
		}
		// Assign globalDeploymentID on first run for the
		// minio server managing the first disk
		globalDeploymentID = format.ID
		return format, nil
	}

	// Return error when quorum unformatted disks - indicating we are
	// waiting for first server to be online.
	if quorumUnformattedDisks(sErrs) && !firstDisk {
		return nil, errNotFirstDisk
	}

	// Return error when quorum unformatted disks but waiting for rest
	// of the servers to be online.
	if quorumUnformattedDisks(sErrs) && firstDisk {
		return nil, errFirstDiskWait
	}

	// Following function is added to fix a regressions which was introduced
	// in release RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3.
	// This migration failed to capture '.This' field properly which indicates
	// the disk UUID association. Below function is called to handle and fix
	// this regression, for more info refer https://github.com/minio/minio/issues/5667
	if err := fixFormatXLV3(storageDisks, endpoints, formatConfigs); err != nil {
		return nil, err
	}

	// If any of the .This field is still empty, we return error.
	if formatXLV3ThisEmpty(formatConfigs) {
		return nil, errXLV3ThisEmpty
	}

	format, err := getFormatXLInQuorum(formatConfigs)
	if err != nil {
		return nil, err
	}

	if format.ID == "" {
		// Not a first disk, wait until first disk fixes deploymentID
		if !firstDisk {
			return nil, errNotFirstDisk
		}
		if err = formatXLFixDeploymentID(endpoints, storageDisks, format); err != nil {
			return nil, err
		}
	}

	globalDeploymentID = format.ID

	if err = formatXLFixLocalDeploymentID(endpoints, storageDisks, format); err != nil {
		return nil, err
	}

	// The will always recreate some directories inside .minio.sys of
	// the local disk such as tmp, multipart and background-ops
	initXLMetaVolumesInLocalDisks(storageDisks, formatConfigs)

	return format, nil
}

// Format disks before initialization of object layer.
func waitForFormatXL(firstDisk bool, endpoints Endpoints, zoneCount, setCount, drivesPerSet int, deploymentID string) (format *formatXLV3, err error) {
	if len(endpoints) == 0 || setCount == 0 || drivesPerSet == 0 {
		return nil, errInvalidArgument
	}

	if err = formatXLMigrateLocalEndpoints(endpoints); err != nil {
		return nil, err
	}

	if err = formatXLCleanupTmpLocalEndpoints(endpoints); err != nil {
		return nil, err
	}

	// prepare getElapsedTime() to calculate elapsed time since we started trying formatting disks.
	// All times are rounded to avoid showing milli, micro and nano seconds
	formatStartTime := time.Now().Round(time.Second)
	getElapsedTime := func() string {
		return time.Now().Round(time.Second).Sub(formatStartTime).String()
	}

	// Wait on each try for an update.
	ticker := time.NewTicker(500 * time.Millisecond)
	defer ticker.Stop()
	var tries int
	for {
		select {
		case <-ticker.C:
			format, err := connectLoadInitFormats(tries, firstDisk, endpoints, zoneCount, setCount, drivesPerSet, deploymentID)
			if err != nil {
				tries++
				switch err {
				case errNotFirstDisk:
					// Fresh setup, wait for first server to be up.
					logger.Info("Waiting for the first server to format the disks.")
					continue
				case errFirstDiskWait:
					// Fresh setup, wait for other servers to come up.
					logger.Info("Waiting for all other servers to be online to format the disks.")
					continue
				case errXLReadQuorum:
					// no quorum available continue to wait for minimum number of servers.
					logger.Info("Waiting for a minimum of %d disks to come online (elapsed %s)\n", len(endpoints)/2, getElapsedTime())
					continue
				case errXLV3ThisEmpty:
					// need to wait for this error to be healed, so continue.
					continue
				default:
					// For all other unhandled errors we exit and fail.
					return nil, err
				}
			}
			return format, nil
		case <-globalOSSignalCh:
			return nil, fmt.Errorf("Initializing data volumes gracefully stopped")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"
	"time"

	"github.com/restic/restic/internal/ui/progress"
)

// newProgressMax returns a progress.Counter that prints to stdout.
func newProgressMax(show bool, max uint64, description string) *progress.Counter {
	if !show {
		return nil
	}

	interval := time.Second / 60
	if !stdinIsTerminal() {
		interval = time.Second
	} else {
		fps, err := strconv.ParseInt(os.Getenv("RESTIC_PROGRESS_FPS"), 10, 64)
		if err == nil && fps >= 1 {
			if fps > 60 {
				fps = 60
			}
			interval = time.Second / time.Duration(fps)
		}
	}

	return progress.New(interval, func(v uint64, d time.Duration, final bool) {
		status := fmt.Sprintf("[%s] %s  %d / %d %s",
			formatDuration(d),
			formatPercent(v, max),
			v, max, description)

		if w := stdoutTerminalWidth(); w > 0 {
			status = shortenStatus(w, status)
		}

		PrintProgress("%s", status)
		if final {
			fmt.Print("\n")
		}
	})
}
<commit_msg>Limit progress bar updates to once per second on non-terminal outputs<commit_after>package main

import (
	"fmt"
	"os"
	"strconv"
	"time"

	"github.com/restic/restic/internal/ui/progress"
)

// newProgressMax returns a progress.Counter that prints to stdout.
func newProgressMax(show bool, max uint64, description string) *progress.Counter {
	if !show {
		return nil
	}

	interval := time.Second / 60
	if !stdoutIsTerminal() {
		interval = time.Second
	} else {
		fps, err := strconv.ParseInt(os.Getenv("RESTIC_PROGRESS_FPS"), 10, 64)
		if err == nil && fps >= 1 {
			if fps > 60 {
				fps = 60
			}
			interval = time.Second / time.Duration(fps)
		}
	}

	return progress.New(interval, func(v uint64, d time.Duration, final bool) {
		status := fmt.Sprintf("[%s] %s  %d / %d %s",
			formatDuration(d),
			formatPercent(v, max),
			v, max, description)

		if w := stdoutTerminalWidth(); w > 0 {
			status = shortenStatus(w, status)
		}

		PrintProgress("%s", status)
		if final {
			fmt.Print("\n")
		}
	})
}
<|endoftext|>


<commit_before>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package format

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/OWASP/Amass/net"
	"github.com/OWASP/Amass/requests"
	"github.com/fatih/color"
)

// Banner is the ASCII art logo used within help output.
const Banner = `
        .+++:.            :                             .+++.
      +W@@@@@@8        &+W@#               o8W8:      +W@@@@@@#.   oW@@@W#+
     &@#+   .o@##.    .@@@o@W.o@@o       :@@#&W8o    .@#:  .:oW+  .@#+++&#&
    +@&        &@&     #@8 +@W@&8@+     :@W.   +@8   +@:          .@8
    8@          @@     8@o  8@8  WW    .@W      W@+  .@W.          o@#:
    WW          &@o    &@:  o@+  o@+   #@.      8@o   +W@#+.        +W@8:
    #@          :@W    &@+  &@+   @8  :@o       o@o     oW@@W+        oW@8
    o@+          @@&   &@+  &@+   #@  &@.      .W@W       .+#@&         o@W.
     WW         +@W@8. &@+  :&    o@+ #@      :@W&@&         &@:  ..     :@o
     :@W:      o@# +Wo &@+        :W: +@W&o++o@W. &@&  8@#o+&@W.  #@:    o@+
      :W@@WWWW@@8       +              :&W@@@@&    &W  .o#@@W&.   :W@WWW@@&
        +o&&&&+.                                                    +oooo.
`

const (
	// Version is used to display the current version of Amass.
	Version = "v3.1.5"

	// Author is used to display the Amass Project Team.
	Author = "OWASP Amass Project - @owaspamass"

	// Description is the slogan for the Amass Project.
	Description = "In-depth Attack Surface Mapping and Asset Discovery"
)

var (
	// Colors used to ease the reading of program output
	y      = color.New(color.FgHiYellow)
	g      = color.New(color.FgHiGreen)
	r      = color.New(color.FgHiRed)
	b      = color.New(color.FgHiBlue)
	fgR    = color.New(color.FgRed)
	fgY    = color.New(color.FgYellow)
	yellow = color.New(color.FgHiYellow).SprintFunc()
	green  = color.New(color.FgHiGreen).SprintFunc()
	blue   = color.New(color.FgHiBlue).SprintFunc()
)

// ASNSummaryData stores information related to discovered ASs and netblocks.
type ASNSummaryData struct {
	Name      string
	Netblocks map[string]int
}

// UpdateSummaryData updates the summary maps using the provided requests.Output data.
func UpdateSummaryData(output *requests.Output, tags map[string]int, asns map[int]*ASNSummaryData) {
	tags[output.Tag]++

	for _, addr := range output.Addresses {
		data, found := asns[addr.ASN]
		if !found {
			asns[addr.ASN] = &ASNSummaryData{
				Name:      addr.Description,
				Netblocks: make(map[string]int),
			}
			data = asns[addr.ASN]
		}
		// Increment how many IPs were in this netblock
		data.Netblocks[addr.Netblock.String()]++
	}
}

// PrintEnumerationSummary outputs the summary information utilized by the command-line tools.
func PrintEnumerationSummary(total int, tags map[string]int, asns map[int]*ASNSummaryData, demo bool) {
	pad := func(num int, chr string) {
		for i := 0; i < num; i++ {
			b.Fprint(color.Error, chr)
		}
	}

	fmt.Fprintln(color.Error)
	// Print the header information
	title := "OWASP Amass "
	site := "https://github.com/OWASP/Amass"
	b.Fprint(color.Error, title+Version)
	num := 80 - (len(title) + len(Version) + len(site))
	pad(num, " ")
	b.Fprintf(color.Error, "%s\n", site)
	pad(8, "----------")
	fmt.Fprintf(color.Error, "\n%s%s", yellow(strconv.Itoa(total)), green(" names discovered - "))
	// Print the stats using tag information
	num, length := 1, len(tags)
	for k, v := range tags {
		fmt.Fprintf(color.Error, "%s: %s", green(k), yellow(strconv.Itoa(v)))
		if num < length {
			g.Fprint(color.Error, ", ")
		}
		num++
	}
	fmt.Fprintln(color.Error)

	if len(asns) == 0 {
		return
	}
	// Another line gets printed
	pad(8, "----------")
	fmt.Fprintln(color.Error)
	// Print the ASN and netblock information
	for asn, data := range asns {
		asnstr := strconv.Itoa(asn)
		datastr := data.Name

		if demo && asn > 0 {
			asnstr = censorString(asnstr, 0, len(asnstr))
			datastr = censorString(datastr, 0, len(datastr))
		}

		fmt.Fprintf(color.Error, "%s%s %s %s\n",
			blue("ASN: "), yellow(asnstr), green("-"), green(datastr))

		for cidr, ips := range data.Netblocks {
			countstr := strconv.Itoa(ips)
			cidrstr := cidr

			if demo {
				cidrstr = censorNetBlock(cidrstr)
			}

			countstr = fmt.Sprintf("\t%-4s", countstr)
			cidrstr = fmt.Sprintf("\t%-18s", cidrstr)

			fmt.Fprintf(color.Error, "%s%s %s\n",
				yellow(cidrstr), yellow(countstr), blue("Subdomain Name(s)"))
		}
	}
}

// PrintBanner outputs the Amass banner the same for all tools.
func PrintBanner() {
	y := color.New(color.FgHiYellow)
	r := color.New(color.FgHiRed)
	rightmost := 76

	pad := func(num int) {
		for i := 0; i < num; i++ {
			fmt.Fprint(color.Error, " ")
		}
	}
	r.Fprintln(color.Error, Banner)
	pad(rightmost - len(Version))
	y.Fprintln(color.Error, Version)
	pad(rightmost - len(Author))
	y.Fprintln(color.Error, Author)
	pad(rightmost - len(Description))
	y.Fprintf(color.Error, "%s\n\n\n", Description)
}

func censorDomain(input string) string {
	return censorString(input, strings.Index(input, "."), len(input))
}

func censorIP(input string) string {
	return censorString(input, 0, strings.LastIndex(input, "."))
}

func censorNetBlock(input string) string {
	return censorString(input, 0, strings.Index(input, "/"))
}

func censorString(input string, start, end int) string {
	runes := []rune(input)
	for i := start; i < end; i++ {
		if runes[i] == '.' ||
			runes[i] == '/' ||
			runes[i] == '-' ||
			runes[i] == ' ' {
			continue
		}
		runes[i] = 'x'
	}
	return string(runes)
}

// OutputLineParts returns the parts of a line to be printed for a requests.Output.
func OutputLineParts(out *requests.Output, src, addrs, demo bool) (source, name, ips string) {
	if src {
		source = fmt.Sprintf("%-18s", "["+out.Source+"] ")
	}
	if addrs {
		for i, a := range out.Addresses {
			if i != 0 {
				ips += ","
			}
			if demo {
				ips += censorIP(a.Address.String())
			} else {
				ips += a.Address.String()
			}
		}
		if ips == "" {
			ips = "N/A"
		}
	}
	name = out.Name
	if demo {
		name = censorDomain(name)
	}
	return
}

// DesiredAddrTypes removes undesired address types from the AddressInfo slice.
func DesiredAddrTypes(addrs []requests.AddressInfo, ipv4, ipv6 bool) []requests.AddressInfo {
	if !ipv4 && !ipv6 {
		return addrs
	}

	var keep []requests.AddressInfo
	for _, addr := range addrs {
		if net.IsIPv4(addr.Address) && !ipv4 {
			continue
		} else if net.IsIPv6(addr.Address) && !ipv6 {
			continue
		}
		keep = append(keep, addr)
	}
	return keep
}
<commit_msg>v3.1.6 release<commit_after>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package format

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/OWASP/Amass/net"
	"github.com/OWASP/Amass/requests"
	"github.com/fatih/color"
)

// Banner is the ASCII art logo used within help output.
const Banner = `
        .+++:.            :                             .+++.
      +W@@@@@@8        &+W@#               o8W8:      +W@@@@@@#.   oW@@@W#+
     &@#+   .o@##.    .@@@o@W.o@@o       :@@#&W8o    .@#:  .:oW+  .@#+++&#&
    +@&        &@&     #@8 +@W@&8@+     :@W.   +@8   +@:          .@8
    8@          @@     8@o  8@8  WW    .@W      W@+  .@W.          o@#:
    WW          &@o    &@:  o@+  o@+   #@.      8@o   +W@#+.        +W@8:
    #@          :@W    &@+  &@+   @8  :@o       o@o     oW@@W+        oW@8
    o@+          @@&   &@+  &@+   #@  &@.      .W@W       .+#@&         o@W.
     WW         +@W@8. &@+  :&    o@+ #@      :@W&@&         &@:  ..     :@o
     :@W:      o@# +Wo &@+        :W: +@W&o++o@W. &@&  8@#o+&@W.  #@:    o@+
      :W@@WWWW@@8       +              :&W@@@@&    &W  .o#@@W&.   :W@WWW@@&
        +o&&&&+.                                                    +oooo.
`

const (
	// Version is used to display the current version of Amass.
	Version = "v3.1.6"

	// Author is used to display the Amass Project Team.
	Author = "OWASP Amass Project - @owaspamass"

	// Description is the slogan for the Amass Project.
	Description = "In-depth Attack Surface Mapping and Asset Discovery"
)

var (
	// Colors used to ease the reading of program output
	y      = color.New(color.FgHiYellow)
	g      = color.New(color.FgHiGreen)
	r      = color.New(color.FgHiRed)
	b      = color.New(color.FgHiBlue)
	fgR    = color.New(color.FgRed)
	fgY    = color.New(color.FgYellow)
	yellow = color.New(color.FgHiYellow).SprintFunc()
	green  = color.New(color.FgHiGreen).SprintFunc()
	blue   = color.New(color.FgHiBlue).SprintFunc()
)

// ASNSummaryData stores information related to discovered ASs and netblocks.
type ASNSummaryData struct {
	Name      string
	Netblocks map[string]int
}

// UpdateSummaryData updates the summary maps using the provided requests.Output data.
func UpdateSummaryData(output *requests.Output, tags map[string]int, asns map[int]*ASNSummaryData) {
	tags[output.Tag]++

	for _, addr := range output.Addresses {
		data, found := asns[addr.ASN]
		if !found {
			asns[addr.ASN] = &ASNSummaryData{
				Name:      addr.Description,
				Netblocks: make(map[string]int),
			}
			data = asns[addr.ASN]
		}
		// Increment how many IPs were in this netblock
		data.Netblocks[addr.Netblock.String()]++
	}
}

// PrintEnumerationSummary outputs the summary information utilized by the command-line tools.
func PrintEnumerationSummary(total int, tags map[string]int, asns map[int]*ASNSummaryData, demo bool) {
	pad := func(num int, chr string) {
		for i := 0; i < num; i++ {
			b.Fprint(color.Error, chr)
		}
	}

	fmt.Fprintln(color.Error)
	// Print the header information
	title := "OWASP Amass "
	site := "https://github.com/OWASP/Amass"
	b.Fprint(color.Error, title+Version)
	num := 80 - (len(title) + len(Version) + len(site))
	pad(num, " ")
	b.Fprintf(color.Error, "%s\n", site)
	pad(8, "----------")
	fmt.Fprintf(color.Error, "\n%s%s", yellow(strconv.Itoa(total)), green(" names discovered - "))
	// Print the stats using tag information
	num, length := 1, len(tags)
	for k, v := range tags {
		fmt.Fprintf(color.Error, "%s: %s", green(k), yellow(strconv.Itoa(v)))
		if num < length {
			g.Fprint(color.Error, ", ")
		}
		num++
	}
	fmt.Fprintln(color.Error)

	if len(asns) == 0 {
		return
	}
	// Another line gets printed
	pad(8, "----------")
	fmt.Fprintln(color.Error)
	// Print the ASN and netblock information
	for asn, data := range asns {
		asnstr := strconv.Itoa(asn)
		datastr := data.Name

		if demo && asn > 0 {
			asnstr = censorString(asnstr, 0, len(asnstr))
			datastr = censorString(datastr, 0, len(datastr))
		}

		fmt.Fprintf(color.Error, "%s%s %s %s\n",
			blue("ASN: "), yellow(asnstr), green("-"), green(datastr))

		for cidr, ips := range data.Netblocks {
			countstr := strconv.Itoa(ips)
			cidrstr := cidr

			if demo {
				cidrstr = censorNetBlock(cidrstr)
			}

			countstr = fmt.Sprintf("\t%-4s", countstr)
			cidrstr = fmt.Sprintf("\t%-18s", cidrstr)

			fmt.Fprintf(color.Error, "%s%s %s\n",
				yellow(cidrstr), yellow(countstr), blue("Subdomain Name(s)"))
		}
	}
}

// PrintBanner outputs the Amass banner the same for all tools.
func PrintBanner() {
	y := color.New(color.FgHiYellow)
	r := color.New(color.FgHiRed)
	rightmost := 76

	pad := func(num int) {
		for i := 0; i < num; i++ {
			fmt.Fprint(color.Error, " ")
		}
	}
	r.Fprintln(color.Error, Banner)
	pad(rightmost - len(Version))
	y.Fprintln(color.Error, Version)
	pad(rightmost - len(Author))
	y.Fprintln(color.Error, Author)
	pad(rightmost - len(Description))
	y.Fprintf(color.Error, "%s\n\n\n", Description)
}

func censorDomain(input string) string {
	return censorString(input, strings.Index(input, "."), len(input))
}

func censorIP(input string) string {
	return censorString(input, 0, strings.LastIndex(input, "."))
}

func censorNetBlock(input string) string {
	return censorString(input, 0, strings.Index(input, "/"))
}

func censorString(input string, start, end int) string {
	runes := []rune(input)
	for i := start; i < end; i++ {
		if runes[i] == '.' ||
			runes[i] == '/' ||
			runes[i] == '-' ||
			runes[i] == ' ' {
			continue
		}
		runes[i] = 'x'
	}
	return string(runes)
}

// OutputLineParts returns the parts of a line to be printed for a requests.Output.
func OutputLineParts(out *requests.Output, src, addrs, demo bool) (source, name, ips string) {
	if src {
		source = fmt.Sprintf("%-18s", "["+out.Source+"] ")
	}
	if addrs {
		for i, a := range out.Addresses {
			if i != 0 {
				ips += ","
			}
			if demo {
				ips += censorIP(a.Address.String())
			} else {
				ips += a.Address.String()
			}
		}
		if ips == "" {
			ips = "N/A"
		}
	}
	name = out.Name
	if demo {
		name = censorDomain(name)
	}
	return
}

// DesiredAddrTypes removes undesired address types from the AddressInfo slice.
func DesiredAddrTypes(addrs []requests.AddressInfo, ipv4, ipv6 bool) []requests.AddressInfo {
	if !ipv4 && !ipv6 {
		return addrs
	}

	var keep []requests.AddressInfo
	for _, addr := range addrs {
		if net.IsIPv4(addr.Address) && !ipv4 {
			continue
		} else if net.IsIPv6(addr.Address) && !ipv6 {
			continue
		}
		keep = append(keep, addr)
	}
	return keep
}
<|endoftext|>


<commit_before>// +build !providerless

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gcepd

import (
	"os"
	"path/filepath"
	"testing"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	utiltesting "k8s.io/client-go/util/testing"
	"k8s.io/kubernetes/pkg/volume"
	volumetest "k8s.io/kubernetes/pkg/volume/testing"
)

const (
	testPdName     = "pdVol1"
	testPVName     = "pv1"
	testGlobalPath = "plugins/kubernetes.io/gce-pd/volumeDevices/pdVol1"
	testPodPath    = "pods/poduid/volumeDevices/kubernetes.io~gce-pd"
)

func TestGetVolumeSpecFromGlobalMapPath(t *testing.T) {
	// make our test path for fake GlobalMapPath
	// /tmp symbolized our pluginDir
	// /tmp/testGlobalPathXXXXX/plugins/kubernetes.io/gce-pd/volumeDevices/pdVol1
	tmpVDir, err := utiltesting.MkTmpdir("gceBlockTest")
	if err != nil {
		t.Fatalf("can't make a temp dir: %v", err)
	}
	//deferred clean up
	defer os.RemoveAll(tmpVDir)

	expectedGlobalPath := filepath.Join(tmpVDir, testGlobalPath)

	//Bad Path
	badspec, err := getVolumeSpecFromGlobalMapPath("", "")
	if badspec != nil || err == nil {
		t.Errorf("Expected not to get spec from GlobalMapPath but did")
	}

	// Good Path
	spec, err := getVolumeSpecFromGlobalMapPath("myVolume", expectedGlobalPath)
	if spec == nil || err != nil {
		t.Fatalf("Failed to get spec from GlobalMapPath: %v", err)
	}
	if spec.PersistentVolume.Name != "myVolume" {
		t.Errorf("Invalid PV name from GlobalMapPath spec: %s", spec.PersistentVolume.Name)
	}
	if spec.PersistentVolume.Spec.GCEPersistentDisk.PDName != testPdName {
		t.Errorf("Invalid pdName from GlobalMapPath spec: %s", spec.PersistentVolume.Spec.GCEPersistentDisk.PDName)
	}
	block := v1.PersistentVolumeBlock
	specMode := spec.PersistentVolume.Spec.VolumeMode
	if specMode == nil {
		t.Fatalf("Empty volumeMode from GlobalMapPath spec")
	}
	if *specMode != block {
		t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", *specMode, block)
	}
}

func getTestVolume(readOnly bool, path string, isBlock bool) *volume.Spec {
	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: testPVName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
					PDName: testPdName,
				},
			},
		},
	}

	if isBlock {
		blockMode := v1.PersistentVolumeBlock
		pv.Spec.VolumeMode = &blockMode
	}
	return volume.NewSpecFromPersistentVolume(pv, readOnly)
}

func TestGetPodAndPluginMapPaths(t *testing.T) {
	tmpVDir, err := utiltesting.MkTmpdir("gceBlockTest")
	if err != nil {
		t.Fatalf("can't make a temp dir: %v", err)
	}
	//deferred clean up
	defer os.RemoveAll(tmpVDir)

	expectedGlobalPath := filepath.Join(tmpVDir, testGlobalPath)
	expectedPodPath := filepath.Join(tmpVDir, testPodPath)

	spec := getTestVolume(false, tmpVDir, true /*isBlock*/)
	plugMgr := volume.VolumePluginMgr{}
	plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(t, tmpVDir, nil, nil))
	plug, err := plugMgr.FindMapperPluginByName(gcePersistentDiskPluginName)
	if err != nil {
		os.RemoveAll(tmpVDir)
		t.Fatalf("Can't find the plugin by name: %q", gcePersistentDiskPluginName)
	}
	if plug.GetPluginName() != gcePersistentDiskPluginName {
		t.Fatalf("Wrong name: %s", plug.GetPluginName())
	}
	pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}}
	mapper, err := plug.NewBlockVolumeMapper(spec, pod, volume.VolumeOptions{})
	if err != nil {
		t.Fatalf("Failed to make a new Mounter: %v", err)
	}
	if mapper == nil {
		t.Fatalf("Got a nil Mounter")
	}

	//GetGlobalMapPath
	gMapPath, err := mapper.GetGlobalMapPath(spec)
	if err != nil || len(gMapPath) == 0 {
		t.Fatalf("Invalid GlobalMapPath from spec: %s", spec.PersistentVolume.Spec.GCEPersistentDisk.PDName)
	}
	if gMapPath != expectedGlobalPath {
		t.Errorf("Failed to get GlobalMapPath: %s %s", gMapPath, expectedGlobalPath)
	}

	//GetPodDeviceMapPath
	gDevicePath, gVolName := mapper.GetPodDeviceMapPath()
	if gDevicePath != expectedPodPath {
		t.Errorf("Got unexpected pod path: %s, expected %s", gDevicePath, expectedPodPath)
	}
	if gVolName != testPVName {
		t.Errorf("Got unexpected volNamne: %s, expected %s", gVolName, testPVName)
	}
}
<commit_msg>Correct error log message<commit_after>// +build !providerless

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gcepd

import (
	"os"
	"path/filepath"
	"testing"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	utiltesting "k8s.io/client-go/util/testing"
	"k8s.io/kubernetes/pkg/volume"
	volumetest "k8s.io/kubernetes/pkg/volume/testing"
)

const (
	testPdName     = "pdVol1"
	testPVName     = "pv1"
	testGlobalPath = "plugins/kubernetes.io/gce-pd/volumeDevices/pdVol1"
	testPodPath    = "pods/poduid/volumeDevices/kubernetes.io~gce-pd"
)

func TestGetVolumeSpecFromGlobalMapPath(t *testing.T) {
	// make our test path for fake GlobalMapPath
	// /tmp symbolized our pluginDir
	// /tmp/testGlobalPathXXXXX/plugins/kubernetes.io/gce-pd/volumeDevices/pdVol1
	tmpVDir, err := utiltesting.MkTmpdir("gceBlockTest")
	if err != nil {
		t.Fatalf("can't make a temp dir: %v", err)
	}
	//deferred clean up
	defer os.RemoveAll(tmpVDir)

	expectedGlobalPath := filepath.Join(tmpVDir, testGlobalPath)

	//Bad Path
	badspec, err := getVolumeSpecFromGlobalMapPath("", "")
	if badspec != nil || err == nil {
		t.Errorf("Expected not to get spec from GlobalMapPath but did")
	}

	// Good Path
	spec, err := getVolumeSpecFromGlobalMapPath("myVolume", expectedGlobalPath)
	if spec == nil || err != nil {
		t.Fatalf("Failed to get spec from GlobalMapPath: %v", err)
	}
	if spec.PersistentVolume.Name != "myVolume" {
		t.Errorf("Invalid PV name from GlobalMapPath spec: %s", spec.PersistentVolume.Name)
	}
	if spec.PersistentVolume.Spec.GCEPersistentDisk.PDName != testPdName {
		t.Errorf("Invalid pdName from GlobalMapPath spec: %s", spec.PersistentVolume.Spec.GCEPersistentDisk.PDName)
	}
	block := v1.PersistentVolumeBlock
	specMode := spec.PersistentVolume.Spec.VolumeMode
	if specMode == nil {
		t.Fatalf("Failed to get volumeMode from PersistentVolumeBlock")
	}
	if *specMode != block {
		t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", *specMode, block)
	}
}

func getTestVolume(readOnly bool, path string, isBlock bool) *volume.Spec {
	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: testPVName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
					PDName: testPdName,
				},
			},
		},
	}

	if isBlock {
		blockMode := v1.PersistentVolumeBlock
		pv.Spec.VolumeMode = &blockMode
	}
	return volume.NewSpecFromPersistentVolume(pv, readOnly)
}

func TestGetPodAndPluginMapPaths(t *testing.T) {
	tmpVDir, err := utiltesting.MkTmpdir("gceBlockTest")
	if err != nil {
		t.Fatalf("can't make a temp dir: %v", err)
	}
	//deferred clean up
	defer os.RemoveAll(tmpVDir)

	expectedGlobalPath := filepath.Join(tmpVDir, testGlobalPath)
	expectedPodPath := filepath.Join(tmpVDir, testPodPath)

	spec := getTestVolume(false, tmpVDir, true /*isBlock*/)
	plugMgr := volume.VolumePluginMgr{}
	plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(t, tmpVDir, nil, nil))
	plug, err := plugMgr.FindMapperPluginByName(gcePersistentDiskPluginName)
	if err != nil {
		os.RemoveAll(tmpVDir)
		t.Fatalf("Can't find the plugin by name: %q", gcePersistentDiskPluginName)
	}
	if plug.GetPluginName() != gcePersistentDiskPluginName {
		t.Fatalf("Wrong name: %s", plug.GetPluginName())
	}
	pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}}
	mapper, err := plug.NewBlockVolumeMapper(spec, pod, volume.VolumeOptions{})
	if err != nil {
		t.Fatalf("Failed to make a new Mounter: %v", err)
	}
	if mapper == nil {
		t.Fatalf("Got a nil Mounter")
	}

	//GetGlobalMapPath
	gMapPath, err := mapper.GetGlobalMapPath(spec)
	if err != nil || len(gMapPath) == 0 {
		t.Fatalf("Invalid GlobalMapPath from spec: %s", spec.PersistentVolume.Spec.GCEPersistentDisk.PDName)
	}
	if gMapPath != expectedGlobalPath {
		t.Errorf("Failed to get GlobalMapPath: %s %s", gMapPath, expectedGlobalPath)
	}

	//GetPodDeviceMapPath
	gDevicePath, gVolName := mapper.GetPodDeviceMapPath()
	if gDevicePath != expectedPodPath {
		t.Errorf("Got unexpected pod path: %s, expected %s", gDevicePath, expectedPodPath)
	}
	if gVolName != testPVName {
		t.Errorf("Got unexpected volNamne: %s, expected %s", gVolName, testPVName)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/rverton/webanalyze"
)

var (
	update          bool
	outputMethod    string
	workers         int
	apps            string
	host            string
	hosts           string
	crawlCount      int
	searchSubdomain bool
	silent          bool
	redirect        bool
)

func init() {
	flag.StringVar(&outputMethod, "output", "stdout", "output format (stdout|csv|json)")
	flag.BoolVar(&update, "update", false, "update apps file")
	flag.IntVar(&workers, "worker", 4, "number of worker")
	flag.StringVar(&apps, "apps", "technologies.json", "app definition file.")
	flag.StringVar(&host, "host", "", "single host to test")
	flag.StringVar(&hosts, "hosts", "", "filename with hosts, one host per line.")
	flag.IntVar(&crawlCount, "crawl", 0, "links to follow from the root page (default 0)")
	flag.BoolVar(&searchSubdomain, "search", true, "searches all urls with same base domain (i.e. example.com and sub.example.com)")
	flag.BoolVar(&silent, "silent", false, "avoid printing header (default false)")
	flag.BoolVar(&redirect, "redirect", false, "follow http redirects (default false)")
}

func main() {
	var (
		file io.ReadCloser
		err  error
		wa   *webanalyze.WebAnalyzer

		outWriter *csv.Writer
	)

	flag.Parse()

	if !update && host == "" && hosts == "" {
		flag.Usage()
		return
	}

	if update {
		err = webanalyze.DownloadFile(webanalyze.WappalyzerURL, "technologies.json")
		if err != nil {
			log.Fatalf("error: can not update apps file: %v", err)
		}

    if !silent {
      log.Println("app definition file updated from ", webanalyze.WappalyzerURL)
    }

		if host == "" && hosts == "" {
			return
		}

	}

	// add header if output mode is csv
	if outputMethod == "csv" {
		outWriter = csv.NewWriter(os.Stdout)
		outWriter.Write([]string{"Host", "Category", "App", "Version"})

		defer outWriter.Flush()

	}

	// check single host or hosts file
	if host != "" {
		file = ioutil.NopCloser(strings.NewReader(host))
	} else {
		file, err = os.Open(hosts)
		if err != nil {
			log.Fatalf("error: can not open host file %s: %s", hosts, err)
		}
	}
	defer file.Close()

	var wg sync.WaitGroup
	hosts := make(chan string)

	appsFile, err := os.Open(apps)
	if err != nil {
		log.Fatalf("error: can not open apps file %s: %s", apps, err)
	}
	defer appsFile.Close()
	if wa, err = webanalyze.NewWebAnalyzer(appsFile, nil); err != nil {
		log.Fatalf("initialization failed: %v", err)
	}

	if !silent {
		printHeader()
	}

	for i := 0; i < workers; i++ {
		wg.Add(1)
		go func() {

			for host := range hosts {
				job := webanalyze.NewOnlineJob(host, "", nil, crawlCount, searchSubdomain, redirect)
				result, links := wa.Process(job)

				if searchSubdomain {
					for _, v := range links {
						crawlJob := webanalyze.NewOnlineJob(v, "", nil, 0, false, redirect)
						result, _ := wa.Process(crawlJob)
						output(result, wa, outWriter)
					}
				}

				output(result, wa, outWriter)
			}

			wg.Done()
		}()
	}

	// read hosts from file
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		hosts <- scanner.Text()
	}

	close(hosts)
	wg.Wait()
}

func output(result webanalyze.Result, wa *webanalyze.WebAnalyzer, outWriter *csv.Writer) {
	if result.Error != nil {
		fmt.Fprintf(os.Stderr, "%v error: %v\n", result.Host, result.Error)
		return
	}

	switch outputMethod {
	case "stdout":
		fmt.Printf("%v (%.1fs):\n", result.Host, result.Duration.Seconds())
		for _, a := range result.Matches {

			var categories []string

			for _, cid := range a.App.Cats {
				categories = append(categories, wa.CategoryById(cid))
			}

			fmt.Printf("    %v, %v (%v)\n", a.AppName, a.Version, strings.Join(categories, ", "))
		}
		if len(result.Matches) <= 0 {
			fmt.Printf("    <no results>\n")
		}

	case "csv":
		for _, m := range result.Matches {
			outWriter.Write(
				[]string{
					result.Host,
					strings.Join(m.CatNames, ","),
					m.AppName,
					m.Version,
				},
			)
		}
		outWriter.Flush()
	case "json":

		output := struct {
			Hostname string             `json:"hostname"`
			Matches  []webanalyze.Match `json:"matches"`
		}{
			result.Host,
			result.Matches,
		}

		b, err := json.Marshal(output)
		if err != nil {
			log.Printf("cannot marshal output: %v\n", err)
		}

		b = append(b, '\n')
		os.Stdout.Write(b)
	}
}

func printHeader() {
	printOption("webanalyze", "v"+webanalyze.VERSION)
	printOption("workers", workers)
	printOption("apps", apps)
	printOption("crawl count", crawlCount)
	printOption("search subdomains", searchSubdomain)
	printOption("follow redirects", redirect)
	fmt.Printf("\n")
}

func printOption(name string, value interface{}) {
	fmt.Fprintf(os.Stderr, " :: %-17s : %v\n", name, value)
}
<commit_msg>fixing #52 silent flag not working with update flag<commit_after>package main

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/rverton/webanalyze"
)

var (
	update          bool
	outputMethod    string
	workers         int
	apps            string
	host            string
	hosts           string
	crawlCount      int
	searchSubdomain bool
	silent          bool
	redirect        bool
)

func init() {
	flag.StringVar(&outputMethod, "output", "stdout", "output format (stdout|csv|json)")
	flag.BoolVar(&update, "update", false, "update apps file")
	flag.IntVar(&workers, "worker", 4, "number of worker")
	flag.StringVar(&apps, "apps", "technologies.json", "app definition file.")
	flag.StringVar(&host, "host", "", "single host to test")
	flag.StringVar(&hosts, "hosts", "", "filename with hosts, one host per line.")
	flag.IntVar(&crawlCount, "crawl", 0, "links to follow from the root page (default 0)")
	flag.BoolVar(&searchSubdomain, "search", true, "searches all urls with same base domain (i.e. example.com and sub.example.com)")
	flag.BoolVar(&silent, "silent", false, "avoid printing header (default false)")
	flag.BoolVar(&redirect, "redirect", false, "follow http redirects (default false)")
}

func main() {
	var (
		file io.ReadCloser
		err  error
		wa   *webanalyze.WebAnalyzer

		outWriter *csv.Writer
	)

	flag.Parse()

	if !update && host == "" && hosts == "" {
		flag.Usage()
		return
	}

	if update {
		err = webanalyze.DownloadFile(webanalyze.WappalyzerURL, "technologies.json")
		if err != nil {
			log.Fatalf("error: can not update apps file: %v", err)
		}

		if !silent {
			log.Println("app definition file updated from ", webanalyze.WappalyzerURL)
		}

		if host == "" && hosts == "" {
			return
		}

	}

	// add header if output mode is csv
	if outputMethod == "csv" {
		outWriter = csv.NewWriter(os.Stdout)
		outWriter.Write([]string{"Host", "Category", "App", "Version"})

		defer outWriter.Flush()

	}

	// check single host or hosts file
	if host != "" {
		file = ioutil.NopCloser(strings.NewReader(host))
	} else {
		file, err = os.Open(hosts)
		if err != nil {
			log.Fatalf("error: can not open host file %s: %s", hosts, err)
		}
	}
	defer file.Close()

	var wg sync.WaitGroup
	hosts := make(chan string)

	appsFile, err := os.Open(apps)
	if err != nil {
		log.Fatalf("error: can not open apps file %s: %s", apps, err)
	}
	defer appsFile.Close()
	if wa, err = webanalyze.NewWebAnalyzer(appsFile, nil); err != nil {
		log.Fatalf("initialization failed: %v", err)
	}

	if !silent {
		printHeader()
	}

	for i := 0; i < workers; i++ {
		wg.Add(1)
		go func() {

			for host := range hosts {
				job := webanalyze.NewOnlineJob(host, "", nil, crawlCount, searchSubdomain, redirect)
				result, links := wa.Process(job)

				if searchSubdomain {
					for _, v := range links {
						crawlJob := webanalyze.NewOnlineJob(v, "", nil, 0, false, redirect)
						result, _ := wa.Process(crawlJob)
						output(result, wa, outWriter)
					}
				}

				output(result, wa, outWriter)
			}

			wg.Done()
		}()
	}

	// read hosts from file
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		hosts <- scanner.Text()
	}

	close(hosts)
	wg.Wait()
}

func output(result webanalyze.Result, wa *webanalyze.WebAnalyzer, outWriter *csv.Writer) {
	if result.Error != nil {
		fmt.Fprintf(os.Stderr, "%v error: %v\n", result.Host, result.Error)
		return
	}

	switch outputMethod {
	case "stdout":
		fmt.Printf("%v (%.1fs):\n", result.Host, result.Duration.Seconds())
		for _, a := range result.Matches {

			var categories []string

			for _, cid := range a.App.Cats {
				categories = append(categories, wa.CategoryById(cid))
			}

			fmt.Printf("    %v, %v (%v)\n", a.AppName, a.Version, strings.Join(categories, ", "))
		}
		if len(result.Matches) <= 0 {
			fmt.Printf("    <no results>\n")
		}

	case "csv":
		for _, m := range result.Matches {
			outWriter.Write(
				[]string{
					result.Host,
					strings.Join(m.CatNames, ","),
					m.AppName,
					m.Version,
				},
			)
		}
		outWriter.Flush()
	case "json":

		output := struct {
			Hostname string             `json:"hostname"`
			Matches  []webanalyze.Match `json:"matches"`
		}{
			result.Host,
			result.Matches,
		}

		b, err := json.Marshal(output)
		if err != nil {
			log.Printf("cannot marshal output: %v\n", err)
		}

		b = append(b, '\n')
		os.Stdout.Write(b)
	}
}

func printHeader() {
	printOption("webanalyze", "v"+webanalyze.VERSION)
	printOption("workers", workers)
	printOption("apps", apps)
	printOption("crawl count", crawlCount)
	printOption("search subdomains", searchSubdomain)
	printOption("follow redirects", redirect)
	fmt.Printf("\n")
}

func printOption(name string, value interface{}) {
	fmt.Fprintf(os.Stderr, " :: %-17s : %v\n", name, value)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	mp "github.com/mackerelio/go-mackerel-plugin"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var index map[string](int) = map[string](int){
	"NAME":       0,
	"STATE":      1,
	"CPU_SEC":    2,
	"CPU_PER":    3,
	"MEM_K":      4,
	"MEM_PER":    5,
	"MAXMEM_K":   6,
	"MAXMEM_PER": 7,
	"VCPUS":      8,
	"NETS":       9,
	"NETTX":      10,
	"NETRX":      11,
	"VBDS":       12,
	"VBD_OO":     13,
	"VBD_RD":     14,
	"VBD_WR":     15,
	"VBD_RSECT":  16,
	"VBD_WSECT":  17,
	"SSID":       18,
}

// All metrics are added dinamically at GraphDefinition
var graphdef map[string](mp.Graphs) = map[string](mp.Graphs){}

type XentopMetrics struct {
	HostName string
	Metrics  mp.Metrics
}

type XentopPlugin struct {
	GraphName          string
	GraphUnit          string
	XentopMetricsSlice []XentopMetrics
}

func (m XentopPlugin) FetchMetrics() (map[string]float64, error) {
	stat := make(map[string]float64)
	cmd := exec.Command("/bin/sh", "-c", "sudo xentop --batch -i 1 -f")
	stdout, err := cmd.StdoutPipe()

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	cmd.Start()

	scanner := bufio.NewScanner(stdout)
	scanner.Scan()
	for scanner.Scan() {
		sf := strings.Fields(string(scanner.Text()))
		name := sf[index["NAME"]]

		var err_parse error
		stat[fmt.Sprintf("cpu_%s", name)], err_parse = strconv.ParseFloat(sf[index["CPU_PER"]], 64)
		if err_parse != nil {
			return nil, err_parse
		}
		stat[fmt.Sprintf("memory_%s", name)], err_parse = strconv.ParseFloat(sf[index["MEM_K"]], 64)
		if err_parse != nil {
			return nil, err_parse
		}
		stat[fmt.Sprintf("nettx_%s", name)], err_parse = strconv.ParseFloat(sf[index["NETTX"]], 64)
		if err_parse != nil {
			return nil, err_parse
		}
		stat[fmt.Sprintf("netrx_%s", name)], err_parse = strconv.ParseFloat(sf[index["NETRX"]], 64)
		if err_parse != nil {
			return nil, err_parse
		}
		stat[fmt.Sprintf("vbdrd_%s", name)], err_parse = strconv.ParseFloat(sf[index["VBD_RD"]], 64)
		if err_parse != nil {
			return nil, err_parse
		}
		stat[fmt.Sprintf("vbdwr_%s", name)], err_parse = strconv.ParseFloat(sf[index["VBD_WR"]], 64)
		if err_parse != nil {
			return nil, err_parse
		}
	}
	if err := scanner.Err(); err != nil {
		panic(err)
	}

	return stat, nil
}

func DefineCpuMetrics(names []string) []mp.Metrics {
	cpu_metrics := make([]mp.Metrics, 0)
	for _, name := range names {
		cpu_metrics = append(cpu_metrics, mp.Metrics{Name: fmt.Sprintf("cpu_%s", name), Label: name, Stacked: true})
	}
	return cpu_metrics
}

func DefineMemoryMetrics(names []string) []mp.Metrics {
	memory_metrics := make([]mp.Metrics, 0)
	for _, name := range names {
		memory_metrics = append(memory_metrics, mp.Metrics{Name: fmt.Sprintf("memory_%s", name), Label: name, Stacked: true})
	}
	return memory_metrics
}

func DefineNettxMetrics(names []string) []mp.Metrics {
	nettx_metrics := make([]mp.Metrics, 0)
	for _, name := range names {
		nettx_metrics = append(nettx_metrics, mp.Metrics{Name: fmt.Sprintf("nettx_%s", name), Label: name, Stacked: true})
	}
	return nettx_metrics
}

func DefineNetrxMetrics(names []string) []mp.Metrics {
	netrx_metrics := make([]mp.Metrics, 0)
	for _, name := range names {
		netrx_metrics = append(netrx_metrics, mp.Metrics{Name: fmt.Sprintf("netrx_%s", name), Label: name, Stacked: true})
	}
	return netrx_metrics
}

func DefineVbdrdMetrics(names []string) []mp.Metrics {
	vbdrd_metrics := make([]mp.Metrics, 0)
	for _, name := range names {
		vbdrd_metrics = append(vbdrd_metrics, mp.Metrics{Name: fmt.Sprintf("vbdrd_%s", name), Label: name, Stacked: true})
	}
	return vbdrd_metrics
}

func DefineVbdwrMetrics(names []string) []mp.Metrics {
	vbdwr_metrics := make([]mp.Metrics, 0)
	for _, name := range names {
		vbdwr_metrics = append(vbdwr_metrics, mp.Metrics{Name: fmt.Sprintf("vbdwr_%s", name), Label: name, Stacked: true})
	}
	return vbdwr_metrics
}

func DefineGraphs(names []string) {
	graphdef["xentop.cpu"] = mp.Graphs{
		Label:   "Xentop CPU",
		Unit:    "float",
		Metrics: DefineCpuMetrics(names),
	}
	graphdef["xentop.memory"] = mp.Graphs{
		Label:   "Xentop Memory",
		Unit:    "float",
		Metrics: DefineMemoryMetrics(names),
	}
	graphdef["xentop.nettx"] = mp.Graphs{
		Label:   "Xentop Nettx",
		Unit:    "float",
		Metrics: DefineNettxMetrics(names),
	}
	graphdef["xentop.netrx"] = mp.Graphs{
		Label:   "Xentop Netrx",
		Unit:    "float",
		Metrics: DefineNetrxMetrics(names),
	}
	graphdef["xentop.vbdrd"] = mp.Graphs{
		Label:   "Xentop VBD_RD",
		Unit:    "float",
		Metrics: DefineVbdrdMetrics(names),
	}
	graphdef["xentop.vbdwr"] = mp.Graphs{
		Label:   "Xentop VBD_WR",
		Unit:    "float",
		Metrics: DefineVbdwrMetrics(names),
	}
}

// ここでグラフを定義する
func (m XentopPlugin) GraphDefinition() map[string](mp.Graphs) {
	cmd := exec.Command("/bin/sh", "-c", "sudo xentop --batch -i 1 -f")
	stdout, err := cmd.StdoutPipe()

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	cmd.Start()

	names := make([]string, 0)
	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		sf := strings.Fields(string(scanner.Text()))
		if sf[index["NAME"]] != "NAME" {
			name := sf[index["NAME"]]
			names = append(names, name)
		}
	}
	if err := scanner.Err(); err != nil {
		panic(err)
	}

	DefineGraphs(names)

	return graphdef

}

func main() {
	// TODO: flagの取得

	var xentop XentopPlugin

	helper := mp.NewMackerelPlugin(xentop)
	helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-xentop")

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<commit_msg>Fix units and command<commit_after>package main

import (
	"bufio"
	"fmt"
	mp "github.com/mackerelio/go-mackerel-plugin"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var index map[string](int) = map[string](int){
	"NAME":       0,
	"STATE":      1,
	"CPU_SEC":    2,
	"CPU_PER":    3,
	"MEM_K":      4,
	"MEM_PER":    5,
	"MAXMEM_K":   6,
	"MAXMEM_PER": 7,
	"VCPUS":      8,
	"NETS":       9,
	"NETTX":      10,
	"NETRX":      11,
	"VBDS":       12,
	"VBD_OO":     13,
	"VBD_RD":     14,
	"VBD_WR":     15,
	"VBD_RSECT":  16,
	"VBD_WSECT":  17,
	"SSID":       18,
}

// All metrics are added dinamically at GraphDefinition
var graphdef map[string](mp.Graphs) = map[string](mp.Graphs){}

type XentopMetrics struct {
	HostName string
	Metrics  mp.Metrics
}

type XentopPlugin struct {
	GraphName          string
	GraphUnit          string
	XentopMetricsSlice []XentopMetrics
}

func (m XentopPlugin) FetchMetrics() (map[string]float64, error) {
	stat := make(map[string]float64)
	cmd := exec.Command("/bin/sh", "-c", "sudo xentop --batch -i 1 -f")
	stdout, err := cmd.StdoutPipe()

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	cmd.Start()

	scanner := bufio.NewScanner(stdout)
	scanner.Scan()
	for scanner.Scan() {
		sf := strings.Fields(string(scanner.Text()))
		name := sf[index["NAME"]]

		var err_parse error
		stat[fmt.Sprintf("cpu_%s", name)], err_parse = strconv.ParseFloat(sf[index["CPU_PER"]], 64)
		if err_parse != nil {
			return nil, err_parse
		}
		stat[fmt.Sprintf("memory_%s", name)], err_parse = strconv.ParseFloat(sf[index["MEM_K"]], 64)
		if err_parse != nil {
			return nil, err_parse
		}
		stat[fmt.Sprintf("nettx_%s", name)], err_parse = strconv.ParseFloat(sf[index["NETTX"]], 64)
		if err_parse != nil {
			return nil, err_parse
		}
		stat[fmt.Sprintf("netrx_%s", name)], err_parse = strconv.ParseFloat(sf[index["NETRX"]], 64)
		if err_parse != nil {
			return nil, err_parse
		}
		stat[fmt.Sprintf("vbdrd_%s", name)], err_parse = strconv.ParseFloat(sf[index["VBD_RD"]], 64)
		if err_parse != nil {
			return nil, err_parse
		}
		stat[fmt.Sprintf("vbdwr_%s", name)], err_parse = strconv.ParseFloat(sf[index["VBD_WR"]], 64)
		if err_parse != nil {
			return nil, err_parse
		}
	}
	if err := scanner.Err(); err != nil {
		panic(err)
	}

	return stat, nil
}

func DefineCpuMetrics(names []string) []mp.Metrics {
	cpu_metrics := make([]mp.Metrics, 0)
	for _, name := range names {
		cpu_metrics = append(cpu_metrics, mp.Metrics{Name: fmt.Sprintf("cpu_%s", name), Label: name, Stacked: true})
	}
	return cpu_metrics
}

func DefineMemoryMetrics(names []string) []mp.Metrics {
	memory_metrics := make([]mp.Metrics, 0)
	for _, name := range names {
		memory_metrics = append(memory_metrics, mp.Metrics{Name: fmt.Sprintf("memory_%s", name), Label: name, Stacked: true})
	}
	return memory_metrics
}

func DefineNettxMetrics(names []string) []mp.Metrics {
	nettx_metrics := make([]mp.Metrics, 0)
	for _, name := range names {
		nettx_metrics = append(nettx_metrics, mp.Metrics{Name: fmt.Sprintf("nettx_%s", name), Label: name, Stacked: true})
	}
	return nettx_metrics
}

func DefineNetrxMetrics(names []string) []mp.Metrics {
	netrx_metrics := make([]mp.Metrics, 0)
	for _, name := range names {
		netrx_metrics = append(netrx_metrics, mp.Metrics{Name: fmt.Sprintf("netrx_%s", name), Label: name, Stacked: true})
	}
	return netrx_metrics
}

func DefineVbdrdMetrics(names []string) []mp.Metrics {
	vbdrd_metrics := make([]mp.Metrics, 0)
	for _, name := range names {
		vbdrd_metrics = append(vbdrd_metrics, mp.Metrics{Name: fmt.Sprintf("vbdrd_%s", name), Label: name, Stacked: true})
	}
	return vbdrd_metrics
}

func DefineVbdwrMetrics(names []string) []mp.Metrics {
	vbdwr_metrics := make([]mp.Metrics, 0)
	for _, name := range names {
		vbdwr_metrics = append(vbdwr_metrics, mp.Metrics{Name: fmt.Sprintf("vbdwr_%s", name), Label: name, Stacked: true})
	}
	return vbdwr_metrics
}

func DefineGraphs(names []string) {
	graphdef["xentop.cpu"] = mp.Graphs{
		Label:   "Xentop CPU",
		Unit:    "percentage",
		Metrics: DefineCpuMetrics(names),
	}
	graphdef["xentop.memory"] = mp.Graphs{
		Label:   "Xentop Memory",
		Unit:    "float",
		Metrics: DefineMemoryMetrics(names),
	}
	graphdef["xentop.nettx"] = mp.Graphs{
		Label:   "Xentop Nettx",
		Unit:    "float",
		Metrics: DefineNettxMetrics(names),
	}
	graphdef["xentop.netrx"] = mp.Graphs{
		Label:   "Xentop Netrx",
		Unit:    "float",
		Metrics: DefineNetrxMetrics(names),
	}
	graphdef["xentop.vbdrd"] = mp.Graphs{
		Label:   "Xentop VBD_RD",
		Unit:    "iops",
		Metrics: DefineVbdrdMetrics(names),
	}
	graphdef["xentop.vbdwr"] = mp.Graphs{
		Label:   "Xentop VBD_WR",
		Unit:    "iops",
		Metrics: DefineVbdwrMetrics(names),
	}
}

// ここでグラフを定義する
func (m XentopPlugin) GraphDefinition() map[string](mp.Graphs) {
	cmd := exec.Command("/bin/sh", "-c", "xentop --batch -i 1 -f")
	stdout, err := cmd.StdoutPipe()

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	cmd.Start()

	names := make([]string, 0)
	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		sf := strings.Fields(string(scanner.Text()))
		if sf[index["NAME"]] != "NAME" {
			name := sf[index["NAME"]]
			names = append(names, name)
		}
	}
	if err := scanner.Err(); err != nil {
		panic(err)
	}

	DefineGraphs(names)

	return graphdef

}

func main() {
	// TODO: flagの取得

	var xentop XentopPlugin

	helper := mp.NewMackerelPlugin(xentop)
	helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-xentop")

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<|endoftext|>