<commit_before>// #############################################################################
// This file is part of the "garf" package of the "Eureka" project.
// It is distributed under the MIT License. Refer to the LICENSE file for more
// information.
//
// Website: http://www.github.com/sivachandra/eureka
// #############################################################################

// Package garf provides API to read DWARF debug info from ELF files.
package garf

import (
	"encoding/binary"
	"fmt"
)

import (
	"eureka/golf"
	"eureka/utils/leb128"
)

type DwFormat uint8

const (
	DwFormat32 = DwFormat(0)
	DwFormat64 = DwFormat(1)
)

type AttrForm struct {
	Name DwAt
	Form DwForm
}

type AbbrevEntry struct {
	AbbrevCode  uint64
	Tag         DwTag
	HasChildren bool
	AttrForms   []AttrForm
}

type AbbrevTable map[uint64]AbbrevEntry

type Attribute struct {
	Name  DwAt
	Value interface{}
}

type DIE struct {
	Tag        DwTag
	Attributes []Attribute
	Parent     *DIE
	Children   []*DIE

	debugInfoOffsetStart uint64
	debugInfoOffsetEnd   uint64
}

type DwUnit struct {
	// The parent DwData from which this unit was read from.
	Parent *DwData

	// Type of the unit.
	Type DwUnitType

	// Format of the unit.
	Format DwFormat

	// Size of the unit in the .debug_info section.
	// It is not the same as the initial length feild in the unit's
	// header.
	Size uint64

	// The DWARF version of this unit.
	Version uint16

	// The offset into the .debug_abbrev section where the info for this
	// unit begins.
	DebugAbbrevOffset uint64

	// The size of the DW_AT_addr attributes in this unit.
	AddressSize byte

	// Offset into the .debug_info section at which the data for
	// DIE tree of this unit begins. It is NOT the offset at which the
	// header for this unit begins.
	DebugInfoOffset uint64

	// The complete DIE tree of this unit. Will be nil until a call to the
	// DIETree method.
	dieTree *DIE
}

func (u DwUnit) DIETree() (*DIE, error) {
	if u.dieTree != nil {
		return u.dieTree, nil
	}

	var err error
	u.dieTree, err = u.Parent.readDIETree(&u, u.DebugInfoOffset)
	return u.dieTree, err
}

type DebugStrMap map[uint64]string

type DwData struct {
	fileName    string
	elf         *golf.ELF
	abbrevTable AbbrevTable
	debugStrMap DebugStrMap
	compUnits   []DwUnit
	typeUnits   []DwUnit

	// Mapping from offset into .debug_info section to the DIE at that
	// offset.
	dieMap map[uint64]*DIE
}

func LoadDwData(fileName string) (*DwData, error) {
	dwData := new(DwData)
	var err error

	dwData.fileName = fileName
	dwData.elf, err = golf.Read(fileName)
	if err != nil {
		err = fmt.Errorf("Error loading ELF info from '%s'.\n%s", fileName, err.Error())
		return nil, err
	}

	dwData.dieMap = make(map[uint64]*DIE)

	return dwData, nil
}

func (d *DwData) ELFData() *golf.ELF {
	return d.elf
}

func (d *DwData) FileName() string {
	return d.fileName
}

func (d *DwData) AbbrevTable(offset uint64) (AbbrevTable, error) {
	if d.abbrevTable != nil {
		return d.abbrevTable, nil
	}

	table := make(AbbrevTable)

	sectMap := d.elf.SectMap()
	debugAbbrevSections, exists := sectMap[".debug_abbrev"]
	if !exists {
		return nil, fmt.Errorf(".debug_abbrev section is not present.", nil)
	}

	if len(debugAbbrevSections) > 1 {
		return nil, fmt.Errorf("More than one .debug_abbrev sections.", nil)
	}

	reader, err := debugAbbrevSections[0].NewSectReader()
	if err != nil {
		return nil, fmt.Errorf("Error fetching .debug_abbrev reader.", err)
	}

	_, err = reader.Seek(int64(offset), 0)
	if err != nil {
		return nil, fmt.Errorf("Error seeking to .debug_abbrev offset.")
	}

	for true {
		abbrevCode, err := leb128.ReadUnsigned(reader)
		if err != nil {
			return nil, fmt.Errorf("Error reading abbreviation code.", nil)
		}
		if abbrevCode == NullAbbrevEntry {
			break
		}

		tag, err := leb128.ReadUnsigned(reader)
		if err != nil {
			msg := fmt.Sprintf("Error reading tag for abbrev code %d.", abbrevCode)
			return nil, fmt.Errorf(msg, err)
		}

		hasChildren, err := reader.ReadByte()
		if err != nil {
			msg := fmt.Sprintf(
				"Error reading child determination entry for abbrev code %d.",
				abbrevCode)
			return nil, fmt.Errorf(msg, err)
		}

		var entry AbbrevEntry
		entry.AbbrevCode = abbrevCode
		entry.Tag = DwTag(tag)
		entry.HasChildren = (hasChildren == DW_CHILDREN_yes)
		entry.AttrForms = make([]AttrForm, 0)

		for true {
			attr, err := leb128.ReadUnsigned(reader)
			if err != nil {
				msg := fmt.Sprintf(
					"Error reading an attr name of entry with abbrev code %d.",
					abbrevCode)
				return nil, fmt.Errorf(msg, err)
			}

			form, err := leb128.ReadUnsigned(reader)
			if err != nil {
				msg := fmt.Sprintf(
					"Error reading an attr form of entry with abbrev code %d.",
					abbrevCode)
				return nil, fmt.Errorf(msg, err)
			}

			if form == 0 && attr == 0 {
				break
			}

			var pair AttrForm
			pair.Name = DwAt(attr)
			pair.Form = DwForm(form)
			entry.AttrForms = append(entry.AttrForms, pair)
		}

		table[entry.AbbrevCode] = entry
	}

	d.abbrevTable = table
	return table, nil
}

func (d *DwData) CompUnits() ([]DwUnit, error) {
	if d.compUnits != nil {
		return d.compUnits, nil
	}

	sectMap := d.elf.SectMap()
	debugInfoSections, exists := sectMap[".debug_info"]
	if !exists {
		return nil, fmt.Errorf(".debug_info section is not present.", nil)
	}

	if len(debugInfoSections) > 1 {
		return nil, fmt.Errorf("More than one .debug_info sections.", nil)
	}

	reader, err := debugInfoSections[0].NewSectReader()
	if err != nil {
		return nil, fmt.Errorf("Error fetching .debug_info section reader.", err)
	}
	defer reader.Finish()

	d.compUnits = make([]DwUnit, 0)
	en := d.elf.Endianess()
	for true {
		if reader.Len() == 0 {
			break
		}

		var length uint64
		var format DwFormat
		var size32 uint32

		headerOffset := reader.Size() - reader.Len()

		err := binary.Read(reader, en, &size32)
		if err != nil {
			err = fmt.Errorf(
				"Error reading first 32 bits of length of a unit in .debug_info.",
				err)
			return nil, err
		}

		if size32 == 0xffffffff {
			format = DwFormat64
			var size64 uint64

			err := binary.Read(reader, en, &size64)
			if err != nil {
				err = fmt.Errorf(
					"Error reading 64-bit length of a unit in .debug_info.",
					err)
				return nil, err
			}

			length = size64
		} else {
			format = DwFormat32
			length = uint64(size32)
		}

		var version uint16
		err = binary.Read(reader, en, &version)
		if err != nil {
			err = fmt.Errorf("Error reading version of a unit in .debug_info.", err)
			return nil, err
		}

		unitType := DW_UT_compile
		if version >= 5 {
			err = binary.Read(reader, en, &unitType)
			if err != nil {
				err = fmt.Errorf(
					"Error reading unit type of a unit in .debug_info.", err)
				return nil, err
			}
		}

		var debugAbbrevOffset uint64
		if format == DwFormat32 {
			var offset uint32
			err = binary.Read(reader, en, &offset)
			if err != nil {
				err = fmt.Errorf(
					"Error reading 32-bit debug abbrev offset of a unit.", err)
				return nil, err
			}

			debugAbbrevOffset = uint64(offset)
		} else {
			err = binary.Read(reader, en, &debugAbbrevOffset)
			if err != nil {
				err = fmt.Errorf(
					"Error reading 64-bit debug abbrev offset of a unit.", err)
				return nil, err
			}
		}

		var addrSize byte
		err = binary.Read(reader, en, &addrSize)
		if err != nil {
			err = fmt.Errorf(
				"Error reading address size from a unit header in .debug_info.",
				err)
			return nil, err
		}

		if unitType == DW_UT_type {
		} else {
			var cu DwUnit

			cu.Parent = d
			cu.Type = unitType

			if format == DwFormat64 {
				cu.Size = length + 12
			} else {
				cu.Size = length + 4
			}

			cu.Format = format
			cu.Version = version
			cu.DebugAbbrevOffset = debugAbbrevOffset
			cu.AddressSize = addrSize
			cu.DebugInfoOffset = reader.Size() - reader.Len()
			d.compUnits = append(d.compUnits, cu)
			reader.Seek(int64(cu.Size+headerOffset), 0)
		}
	}

	return d.compUnits, nil
}

func (d *DwData) DebugStr() (DebugStrMap, error) {
	if d.debugStrMap != nil {
		return d.debugStrMap, nil
	}

	sectMap := d.elf.SectMap()
	debugStrSections, exists := sectMap[".debug_str"]
	if !exists {
		return nil, fmt.Errorf(".debug_str section is not present.", nil)
	}

	if len(debugStrSections) > 1 {
		return nil, fmt.Errorf("More than one .debug_str sections.", nil)
	}

	debugStrData, err := debugStrSections[0].RawData()
	if err != nil {
		return nil, fmt.Errorf("Error fetching .debug_str data.", err)
	}

	var str []uint8
	offset := uint64(0)
	debugStrMap := make(DebugStrMap)
	for offset < uint64(len(debugStrData)) {
		c := debugStrData[offset]
		str = append(str, c)
		if c == 0 {
			l := len(str)
			debugStrMap[offset+1-uint64(l)] = string(str[0 : l-1])
			str = nil
		}
		offset++
	}

	d.debugStrMap = debugStrMap
	return d.debugStrMap, nil
}

func (d *DwData) readDIETree(u *DwUnit, offset uint64) (*DIE, error) {
	sectMap := d.elf.SectMap()
	debugInfoSections, exists := sectMap[".debug_info"]
	if !exists {
		return nil, fmt.Errorf(".debug_info section is not present.", nil)
	}

	if len(debugInfoSections) > 1 {
		return nil, fmt.Errorf("More than one .debug_info sections.", nil)
	}

	reader, err := debugInfoSections[0].NewSectReader()
	if err != nil {
		return nil, fmt.Errorf("Error fetching .debug_info section reader.", err)
	}
	defer reader.Finish()

	_, err = reader.Seek(int64(offset), 0)
	if err != nil {
		err = fmt.Errorf(
			"Error seeking to the DIE offset to read the DIE tree.\n%s", err.Error())
		return nil, err
	}

	return d.readDIETreeHelper(u, reader, d.elf.Endianess(), nil)
}

func (d *DwData) readDIETreeHelper(
	u *DwUnit, r *golf.SectReader, en binary.ByteOrder, parent *DIE) (*DIE, error) {
	debugInfoOffset := r.Size() - r.Len()
	die, exists := d.dieMap[debugInfoOffset]
	if exists {
		die.Parent = parent
		r.Seek(int64(die.debugInfoOffsetEnd), 0)

		return die, nil
	}

	abrrevTable, err := d.AbbrevTable(u.DebugAbbrevOffset)
	if err != nil {
		err = fmt.Errorf("Error getting abbrev table while reading a DIE tree.", err)
		return nil, err
	}

	abbrevCode, err := leb128.ReadUnsigned(r)
	if err != nil {
		return nil, fmt.Errorf("Error reading abbrev code of a DIE.", err)
	}

	// Return if its a NULL entry
	if abbrevCode == 0 {
		return nil, nil
	}

	abbrevEntry, exists := abrrevTable[abbrevCode]
	if !exists {
		return nil, fmt.Errorf("Invalid abbrev code for a DIE.", nil)
	}

	attributes := make([]Attribute, 0)
	for _, attrForm := range abbrevEntry.AttrForms {
		attr, err := d.readAttr(u, r, attrForm.Name, attrForm.Form, en)
		if err != nil {
			err = fmt.Errorf("Error reading an attribute value for a DIE.\n%s", err)
			return nil, err
		}
		attributes = append(attributes, attr)
	}

	die = new(DIE)
	die.Tag = abbrevEntry.Tag
	die.Parent = parent
	die.Attributes = attributes
	die.debugInfoOffsetStart = debugInfoOffset

	for abbrevEntry.HasChildren {
		childDie, err := d.readDIETreeHelper(u, r, en, die)
		if err != nil {
			err = fmt.Errorf(
				"Error reading child DIE tree of tag %d.\n%s",
				abbrevEntry.Tag, err.Error())
			return nil, err
		}

		if childDie == nil {
			break
		}

		die.Children = append(die.Children, childDie)
	}
	die.debugInfoOffsetEnd = r.Size() - r.Len()

	d.dieMap[debugInfoOffset] = die
	return die, nil
}
<commit_msg>[garf] Make the Attributes field of DIE a map from name to Attribute.<commit_after>// #############################################################################
// This file is part of the "garf" package of the "Eureka" project.
// It is distributed under the MIT License. Refer to the LICENSE file for more
// information.
//
// Website: http://www.github.com/sivachandra/eureka
// #############################################################################

// Package garf provides API to read DWARF debug info from ELF files.
package garf

import (
	"encoding/binary"
	"fmt"
)

import (
	"eureka/golf"
	"eureka/utils/leb128"
)

type DwFormat uint8

const (
	DwFormat32 = DwFormat(0)
	DwFormat64 = DwFormat(1)
)

type AttrForm struct {
	Name DwAt
	Form DwForm
}

type AbbrevEntry struct {
	AbbrevCode  uint64
	Tag         DwTag
	HasChildren bool
	AttrForms   []AttrForm
}

type AbbrevTable map[uint64]AbbrevEntry

type Attribute struct {
	Name  DwAt
	Value interface{}
}

type DIE struct {
	Tag        DwTag
	Attributes map[DwAt]Attribute
	Parent     *DIE
	Children   []*DIE

	debugInfoOffsetStart uint64
	debugInfoOffsetEnd   uint64
}

type DwUnit struct {
	// The parent DwData from which this unit was read from.
	Parent *DwData

	// Type of the unit.
	Type DwUnitType

	// Format of the unit.
	Format DwFormat

	// Size of the unit in the .debug_info section.
	// It is not the same as the initial length feild in the unit's
	// header.
	Size uint64

	// The DWARF version of this unit.
	Version uint16

	// The offset into the .debug_abbrev section where the info for this
	// unit begins.
	DebugAbbrevOffset uint64

	// The size of the DW_AT_addr attributes in this unit.
	AddressSize byte

	// Offset into the .debug_info section at which the data for
	// DIE tree of this unit begins. It is NOT the offset at which the
	// header for this unit begins.
	DebugInfoOffset uint64

	// The complete DIE tree of this unit. Will be nil until a call to the
	// DIETree method.
	dieTree *DIE
}

func (u DwUnit) DIETree() (*DIE, error) {
	if u.dieTree != nil {
		return u.dieTree, nil
	}

	var err error
	u.dieTree, err = u.Parent.readDIETree(&u, u.DebugInfoOffset)
	return u.dieTree, err
}

type DebugStrMap map[uint64]string

type DwData struct {
	fileName    string
	elf         *golf.ELF
	abbrevTable AbbrevTable
	debugStrMap DebugStrMap
	compUnits   []DwUnit
	typeUnits   []DwUnit

	// Mapping from offset into .debug_info section to the DIE at that
	// offset.
	dieMap map[uint64]*DIE
}

func LoadDwData(fileName string) (*DwData, error) {
	dwData := new(DwData)
	var err error

	dwData.fileName = fileName
	dwData.elf, err = golf.Read(fileName)
	if err != nil {
		err = fmt.Errorf("Error loading ELF info from '%s'.\n%s", fileName, err.Error())
		return nil, err
	}

	dwData.dieMap = make(map[uint64]*DIE)

	return dwData, nil
}

func (d *DwData) ELFData() *golf.ELF {
	return d.elf
}

func (d *DwData) FileName() string {
	return d.fileName
}

func (d *DwData) AbbrevTable(offset uint64) (AbbrevTable, error) {
	if d.abbrevTable != nil {
		return d.abbrevTable, nil
	}

	table := make(AbbrevTable)

	sectMap := d.elf.SectMap()
	debugAbbrevSections, exists := sectMap[".debug_abbrev"]
	if !exists {
		return nil, fmt.Errorf(".debug_abbrev section is not present.", nil)
	}

	if len(debugAbbrevSections) > 1 {
		return nil, fmt.Errorf("More than one .debug_abbrev sections.", nil)
	}

	reader, err := debugAbbrevSections[0].NewSectReader()
	if err != nil {
		return nil, fmt.Errorf("Error fetching .debug_abbrev reader.", err)
	}

	_, err = reader.Seek(int64(offset), 0)
	if err != nil {
		return nil, fmt.Errorf("Error seeking to .debug_abbrev offset.")
	}

	for true {
		abbrevCode, err := leb128.ReadUnsigned(reader)
		if err != nil {
			return nil, fmt.Errorf("Error reading abbreviation code.", nil)
		}
		if abbrevCode == NullAbbrevEntry {
			break
		}

		tag, err := leb128.ReadUnsigned(reader)
		if err != nil {
			msg := fmt.Sprintf("Error reading tag for abbrev code %d.", abbrevCode)
			return nil, fmt.Errorf(msg, err)
		}

		hasChildren, err := reader.ReadByte()
		if err != nil {
			msg := fmt.Sprintf(
				"Error reading child determination entry for abbrev code %d.",
				abbrevCode)
			return nil, fmt.Errorf(msg, err)
		}

		var entry AbbrevEntry
		entry.AbbrevCode = abbrevCode
		entry.Tag = DwTag(tag)
		entry.HasChildren = (hasChildren == DW_CHILDREN_yes)
		entry.AttrForms = make([]AttrForm, 0)

		for true {
			attr, err := leb128.ReadUnsigned(reader)
			if err != nil {
				msg := fmt.Sprintf(
					"Error reading an attr name of entry with abbrev code %d.",
					abbrevCode)
				return nil, fmt.Errorf(msg, err)
			}

			form, err := leb128.ReadUnsigned(reader)
			if err != nil {
				msg := fmt.Sprintf(
					"Error reading an attr form of entry with abbrev code %d.",
					abbrevCode)
				return nil, fmt.Errorf(msg, err)
			}

			if form == 0 && attr == 0 {
				break
			}

			var pair AttrForm
			pair.Name = DwAt(attr)
			pair.Form = DwForm(form)
			entry.AttrForms = append(entry.AttrForms, pair)
		}

		table[entry.AbbrevCode] = entry
	}

	d.abbrevTable = table
	return table, nil
}

func (d *DwData) CompUnits() ([]DwUnit, error) {
	if d.compUnits != nil {
		return d.compUnits, nil
	}

	sectMap := d.elf.SectMap()
	debugInfoSections, exists := sectMap[".debug_info"]
	if !exists {
		return nil, fmt.Errorf(".debug_info section is not present.", nil)
	}

	if len(debugInfoSections) > 1 {
		return nil, fmt.Errorf("More than one .debug_info sections.", nil)
	}

	reader, err := debugInfoSections[0].NewSectReader()
	if err != nil {
		return nil, fmt.Errorf("Error fetching .debug_info section reader.", err)
	}
	defer reader.Finish()

	d.compUnits = make([]DwUnit, 0)
	en := d.elf.Endianess()
	for true {
		if reader.Len() == 0 {
			break
		}

		var length uint64
		var format DwFormat
		var size32 uint32

		headerOffset := reader.Size() - reader.Len()

		err := binary.Read(reader, en, &size32)
		if err != nil {
			err = fmt.Errorf(
				"Error reading first 32 bits of length of a unit in .debug_info.",
				err)
			return nil, err
		}

		if size32 == 0xffffffff {
			format = DwFormat64
			var size64 uint64

			err := binary.Read(reader, en, &size64)
			if err != nil {
				err = fmt.Errorf(
					"Error reading 64-bit length of a unit in .debug_info.",
					err)
				return nil, err
			}

			length = size64
		} else {
			format = DwFormat32
			length = uint64(size32)
		}

		var version uint16
		err = binary.Read(reader, en, &version)
		if err != nil {
			err = fmt.Errorf("Error reading version of a unit in .debug_info.", err)
			return nil, err
		}

		unitType := DW_UT_compile
		if version >= 5 {
			err = binary.Read(reader, en, &unitType)
			if err != nil {
				err = fmt.Errorf(
					"Error reading unit type of a unit in .debug_info.", err)
				return nil, err
			}
		}

		var debugAbbrevOffset uint64
		if format == DwFormat32 {
			var offset uint32
			err = binary.Read(reader, en, &offset)
			if err != nil {
				err = fmt.Errorf(
					"Error reading 32-bit debug abbrev offset of a unit.", err)
				return nil, err
			}

			debugAbbrevOffset = uint64(offset)
		} else {
			err = binary.Read(reader, en, &debugAbbrevOffset)
			if err != nil {
				err = fmt.Errorf(
					"Error reading 64-bit debug abbrev offset of a unit.", err)
				return nil, err
			}
		}

		var addrSize byte
		err = binary.Read(reader, en, &addrSize)
		if err != nil {
			err = fmt.Errorf(
				"Error reading address size from a unit header in .debug_info.",
				err)
			return nil, err
		}

		if unitType == DW_UT_type {
		} else {
			var cu DwUnit

			cu.Parent = d
			cu.Type = unitType

			if format == DwFormat64 {
				cu.Size = length + 12
			} else {
				cu.Size = length + 4
			}

			cu.Format = format
			cu.Version = version
			cu.DebugAbbrevOffset = debugAbbrevOffset
			cu.AddressSize = addrSize
			cu.DebugInfoOffset = reader.Size() - reader.Len()
			d.compUnits = append(d.compUnits, cu)
			reader.Seek(int64(cu.Size+headerOffset), 0)
		}
	}

	return d.compUnits, nil
}

func (d *DwData) DebugStr() (DebugStrMap, error) {
	if d.debugStrMap != nil {
		return d.debugStrMap, nil
	}

	sectMap := d.elf.SectMap()
	debugStrSections, exists := sectMap[".debug_str"]
	if !exists {
		return nil, fmt.Errorf(".debug_str section is not present.", nil)
	}

	if len(debugStrSections) > 1 {
		return nil, fmt.Errorf("More than one .debug_str sections.", nil)
	}

	debugStrData, err := debugStrSections[0].RawData()
	if err != nil {
		return nil, fmt.Errorf("Error fetching .debug_str data.", err)
	}

	var str []uint8
	offset := uint64(0)
	debugStrMap := make(DebugStrMap)
	for offset < uint64(len(debugStrData)) {
		c := debugStrData[offset]
		str = append(str, c)
		if c == 0 {
			l := len(str)
			debugStrMap[offset+1-uint64(l)] = string(str[0 : l-1])
			str = nil
		}
		offset++
	}

	d.debugStrMap = debugStrMap
	return d.debugStrMap, nil
}

func (d *DwData) readDIETree(u *DwUnit, offset uint64) (*DIE, error) {
	sectMap := d.elf.SectMap()
	debugInfoSections, exists := sectMap[".debug_info"]
	if !exists {
		return nil, fmt.Errorf(".debug_info section is not present.", nil)
	}

	if len(debugInfoSections) > 1 {
		return nil, fmt.Errorf("More than one .debug_info sections.", nil)
	}

	reader, err := debugInfoSections[0].NewSectReader()
	if err != nil {
		return nil, fmt.Errorf("Error fetching .debug_info section reader.", err)
	}
	defer reader.Finish()

	_, err = reader.Seek(int64(offset), 0)
	if err != nil {
		err = fmt.Errorf(
			"Error seeking to the DIE offset to read the DIE tree.\n%s", err.Error())
		return nil, err
	}

	return d.readDIETreeHelper(u, reader, d.elf.Endianess(), nil)
}

func (d *DwData) readDIETreeHelper(
	u *DwUnit, r *golf.SectReader, en binary.ByteOrder, parent *DIE) (*DIE, error) {
	debugInfoOffset := r.Size() - r.Len()
	die, exists := d.dieMap[debugInfoOffset]
	if exists {
		die.Parent = parent
		r.Seek(int64(die.debugInfoOffsetEnd), 0)

		return die, nil
	}

	abrrevTable, err := d.AbbrevTable(u.DebugAbbrevOffset)
	if err != nil {
		err = fmt.Errorf("Error getting abbrev table while reading a DIE tree.", err)
		return nil, err
	}

	abbrevCode, err := leb128.ReadUnsigned(r)
	if err != nil {
		return nil, fmt.Errorf("Error reading abbrev code of a DIE.", err)
	}

	// Return if its a NULL entry
	if abbrevCode == 0 {
		return nil, nil
	}

	abbrevEntry, exists := abrrevTable[abbrevCode]
	if !exists {
		return nil, fmt.Errorf("Invalid abbrev code for a DIE.", nil)
	}

	attributes := make(map[DwAt]Attribute)
	for _, attrForm := range abbrevEntry.AttrForms {
		attr, err := d.readAttr(u, r, attrForm.Name, attrForm.Form, en)
		if err != nil {
			err = fmt.Errorf("Error reading an attribute value for a DIE.\n%s", err)
			return nil, err
		}
		attributes[attr.Name] = attr
	}

	die = new(DIE)
	die.Tag = abbrevEntry.Tag
	die.Parent = parent
	die.Attributes = attributes
	die.debugInfoOffsetStart = debugInfoOffset

	for abbrevEntry.HasChildren {
		childDie, err := d.readDIETreeHelper(u, r, en, die)
		if err != nil {
			err = fmt.Errorf(
				"Error reading child DIE tree of tag %d.\n%s",
				abbrevEntry.Tag, err.Error())
			return nil, err
		}

		if childDie == nil {
			break
		}

		die.Children = append(die.Children, childDie)
	}
	die.debugInfoOffsetEnd = r.Size() - r.Len()

	d.dieMap[debugInfoOffset] = die
	return die, nil
}
<|endoftext|>


<commit_before>package appui

import (
	"fmt"
	"github.com/moncho/dry/docker"
	"strings"

	gizaktermui "github.com/gizak/termui"
	"github.com/moncho/dry/ui/termui"
)

type sortMode = docker.SortMode

//SortableColumnHeader is a column header associated to a  sort mode
type SortableColumnHeader struct {
	Title string // Title to display in the tableHeader.
	Mode  sortMode
}

type WidgetHeader struct {
	elements map[string]string
	keys     []string
	Y        int
}

func NewWidgetHeader() *WidgetHeader {
	return &WidgetHeader{
		elements: make(map[string]string),
	}
}
func (header *WidgetHeader) HeaderEntry(key, value string) {
	header.keys = append(header.keys, key)
	header.elements[key] = value
}
func (header *WidgetHeader) GetHeight() int {
	return 1
}

// Buffer return this paragraph content as a termui.Buffer
func (header *WidgetHeader) Buffer() gizaktermui.Buffer {
	var entries []string
	width := 0
	for _, k := range header.keys {
		entries = append(entries,
			fmt.Sprintf("<b><blue>%s: </><yellow>%s</></>", k, header.elements[k]))
		width += len(k) + len(header.elements[k])
	}
	s := strings.Join(entries, " <blue>|</> ")
	par := termui.NewParFromMarkupText(DryTheme, s)

	par.SetX(0)
	par.SetY(header.Y)
	par.Border = false
	par.Width = len([]rune(s))
	par.TextBgColor = gizaktermui.Attribute(DryTheme.Bg)
	par.Bg = gizaktermui.Attribute(DryTheme.Bg)
	return par.Buffer()
}
<commit_msg>Add missing docs<commit_after>package appui

import (
	"fmt"
	"github.com/moncho/dry/docker"
	"strings"

	gizaktermui "github.com/gizak/termui"
	"github.com/moncho/dry/ui/termui"
)

type sortMode = docker.SortMode

//SortableColumnHeader is a column header associated to a sort mode
type SortableColumnHeader struct {
	Title string // Title to display in the tableHeader.
	Mode  sortMode
}

//WidgetHeader is a widget for the header of a widget
type WidgetHeader struct {
	elements map[string]string
	keys     []string
	Y        int
}

//NewWidgetHeader creates WidgetHeader
func NewWidgetHeader() *WidgetHeader {
	return &WidgetHeader{
		elements: make(map[string]string),
	}
}

//HeaderEntry adds a new key-value entry to this header
func (header *WidgetHeader) HeaderEntry(key, value string) {
	header.keys = append(header.keys, key)
	header.elements[key] = value
}

//GetHeight returns the widget height
func (header *WidgetHeader) GetHeight() int {
	return 1
}

// Buffer return this widget content as a termui.Buffer
func (header *WidgetHeader) Buffer() gizaktermui.Buffer {
	var entries []string
	width := 0
	for _, k := range header.keys {
		entries = append(entries,
			fmt.Sprintf("<b><blue>%s: </><yellow>%s</></>", k, header.elements[k]))
		width += len(k) + len(header.elements[k])
	}
	s := strings.Join(entries, " <blue>|</> ")
	par := termui.NewParFromMarkupText(DryTheme, s)

	par.SetX(0)
	par.SetY(header.Y)
	par.Border = false
	par.Width = len([]rune(s))
	par.TextBgColor = gizaktermui.Attribute(DryTheme.Bg)
	par.Bg = gizaktermui.Attribute(DryTheme.Bg)
	return par.Buffer()
}
<|endoftext|>


<commit_before>// +build go1.4

// Copyright 2015 Robert S. Gerus. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package bot

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sync"
	"testing"
	"time"

	cfg "github.com/arachnist/gorepost/config"
	"github.com/arachnist/gorepost/irc"
)

var eventTests = []struct {
	desc        string
	in          irc.Message
	expectedOut []irc.Message
}{
	{
		desc: "ping",
		in: irc.Message{
			Command:  "PING",
			Trailing: "foobar",
		},
		expectedOut: []irc.Message{
			{
				Command:  "PONG",
				Trailing: "foobar",
			},
		},
	},
	{
		desc: "invitki",
		in: irc.Message{
			Command:  "INVITE",
			Trailing: "#test-channel",
		},
		expectedOut: []irc.Message{
			{
				Command: "JOIN",
				Params:  []string{"#test-channel"},
			},
		},
	},
	{
		desc: "channel join",
		in: irc.Message{
			Command: "001",
			Context: map[string]string{
				"Network": "TestNetwork",
			},
		},
		expectedOut: []irc.Message{
			{
				Command: "JOIN",
				Params:  []string{"#testchan-1"},
			},
			{
				Command: "JOIN",
				Params:  []string{"#testchan-2"},
			},
		},
	},
	{
		desc: "msgping",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: ":ping",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "pingity pong",
			},
		},
	},
	{
		desc: "nickserv",
		in: irc.Message{
			Command:  "NOTICE",
			Params:   []string{"gorepost"},
			Trailing: "This nickname is registered. Please choose a different nickname, or identify via …",
			Prefix: &irc.Prefix{
				Name: "NickServ",
				User: "NickServ",
				Host: "services.",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"NickServ"},
				Trailing: "IDENTIFY test_password",
			},
		},
	},
	{
		desc: "pick",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: ":pick test",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "test",
			},
		},
	},
	{
		desc: "google",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: ":g google.com",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "Google https://www.google.com/",
			},
		},
	},
	{
		desc: "linktitle",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "https://www.google.com/",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "↳ title: Google",
			},
		},
	},
	{
		desc: "nickserv channeljoin",
		in: irc.Message{
			Command:  "NOTICE",
			Params:   []string{"gorepost"},
			Trailing: "You are now identified",
			Prefix: &irc.Prefix{
				Name: "NickServ",
				User: "NickServ",
				Host: "services.",
			},
		},
		expectedOut: []irc.Message{
			{
				Command: "JOIN",
				Params:  []string{"#securechan-1"},
			},
			{
				Command: "JOIN",
				Params:  []string{"#securechan-2"},
			},
		},
	},
	{
		desc: "linktitle connection refused",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "http://127.0.0.1:333/conn-refused",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "↳ title: error:Get http://127.0.0.1:333/conn-refused: dial tcp 127.0.0.1:333: getsockopt: connection refused",
			},
		},
	},
	{
		desc: "linktitle iso8859-2",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "http://arachnist.is-a.cat/test-iso8859-2.html",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "↳ title: Tytuł używający przestarzałego kodowania znaków",
			},
		},
	},
	{
		desc: "linktitle common exploit",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "http://arachnist.is-a.cat/test.html",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "↳ title: Tak Aż zbyt dobrze. Naprawdę QUIT dupa",
			},
		},
	},
	{
		desc: "seen noone",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: ":seen noone",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "nope, never",
			},
		},
	},
	{
		desc: "linktitle missing title",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "http://arachnist.is-a.cat/test-no-title.html",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{},
	},
	{
		desc: "linktitle notitle",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "https://www.google.com/ notitle",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{},
	},
	{
		desc: "nickserv spoof",
		in: irc.Message{
			Command:  "NOTICE",
			Params:   []string{"gorepost"},
			Trailing: "This nickname is registered. Please choose a different nickname, or identify via …",
			Prefix: &irc.Prefix{
				Name: "NickServ",
				User: "NickServ",
				Host: "fake.",
			},
		},
		expectedOut: []irc.Message{},
	},
	{
		desc: "nickserv other message",
		in: irc.Message{
			Command:  "NOTICE",
			Params:   []string{"gorepost"},
			Trailing: "Some other random message…",
			Prefix: &irc.Prefix{
				Name: "NickServ",
				User: "NickServ",
				Host: "services.",
			},
		},
		expectedOut: []irc.Message{},
	},
	{
		desc: "non-matching",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "foo bar baz",
		},
		expectedOut: []irc.Message{},
	},
}

func TestPlugins(t *testing.T) {
	var r []irc.Message
	var wg sync.WaitGroup

	// fake irc.Conn Sender replacement
	output := func(msg irc.Message) {
		wg.Done()
		r = append(r, msg)
	}

	for _, e := range eventTests {
		t.Log("running test", e.desc)
		r = r[:0]

		wg.Add(len(e.expectedOut))

		Dispatcher(output, e.in)

		time.Sleep(3000000 * time.Nanosecond)

		wg.Wait()

		if fmt.Sprintf("%+v", r) != fmt.Sprintf("%+v", e.expectedOut) {
			t.Logf("expected: %+v\n", e.expectedOut)
			t.Logf("result: %+v\n", r)
			t.Fail()
		}
	}
}

func configLookupHelper(map[string]string) []string {
	return []string{".testconfig.json"}
}

func TestMain(m *testing.M) {
	log.SetOutput(ioutil.Discard)
	cfg.SetFileListBuilder(configLookupHelper)
	os.Exit(m.Run())
}
<commit_msg>Add a better test for seen.<commit_after>// +build go1.4

// Copyright 2015 Robert S. Gerus. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package bot

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sync"
	"testing"
	"time"

	cfg "github.com/arachnist/gorepost/config"
	"github.com/arachnist/gorepost/irc"
)

var eventTests = []struct {
	desc        string
	in          irc.Message
	expectedOut []irc.Message
}{
	{
		desc: "seen noone",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: ":seen noone",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "nope, never",
			},
		},
	},
	{
		desc: "seen myself",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: ":seen idontexist",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: fmt.Sprintf("Last seen idontexist on /#testchan-1 at %v saying: :seen idontexist", time.Now().Round(time.Second)),
			},
		},
	},
	{
		desc: "ping",
		in: irc.Message{
			Command:  "PING",
			Trailing: "foobar",
		},
		expectedOut: []irc.Message{
			{
				Command:  "PONG",
				Trailing: "foobar",
			},
		},
	},
	{
		desc: "invitki",
		in: irc.Message{
			Command:  "INVITE",
			Trailing: "#test-channel",
		},
		expectedOut: []irc.Message{
			{
				Command: "JOIN",
				Params:  []string{"#test-channel"},
			},
		},
	},
	{
		desc: "channel join",
		in: irc.Message{
			Command: "001",
			Context: map[string]string{
				"Network": "TestNetwork",
			},
		},
		expectedOut: []irc.Message{
			{
				Command: "JOIN",
				Params:  []string{"#testchan-1"},
			},
			{
				Command: "JOIN",
				Params:  []string{"#testchan-2"},
			},
		},
	},
	{
		desc: "msgping",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: ":ping",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "pingity pong",
			},
		},
	},
	{
		desc: "nickserv",
		in: irc.Message{
			Command:  "NOTICE",
			Params:   []string{"gorepost"},
			Trailing: "This nickname is registered. Please choose a different nickname, or identify via …",
			Prefix: &irc.Prefix{
				Name: "NickServ",
				User: "NickServ",
				Host: "services.",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"NickServ"},
				Trailing: "IDENTIFY test_password",
			},
		},
	},
	{
		desc: "pick",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: ":pick test",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "test",
			},
		},
	},
	{
		desc: "google",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: ":g google.com",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "Google https://www.google.com/",
			},
		},
	},
	{
		desc: "linktitle",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "https://www.google.com/",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "↳ title: Google",
			},
		},
	},
	{
		desc: "nickserv channeljoin",
		in: irc.Message{
			Command:  "NOTICE",
			Params:   []string{"gorepost"},
			Trailing: "You are now identified",
			Prefix: &irc.Prefix{
				Name: "NickServ",
				User: "NickServ",
				Host: "services.",
			},
		},
		expectedOut: []irc.Message{
			{
				Command: "JOIN",
				Params:  []string{"#securechan-1"},
			},
			{
				Command: "JOIN",
				Params:  []string{"#securechan-2"},
			},
		},
	},
	{
		desc: "linktitle connection refused",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "http://127.0.0.1:333/conn-refused",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "↳ title: error:Get http://127.0.0.1:333/conn-refused: dial tcp 127.0.0.1:333: getsockopt: connection refused",
			},
		},
	},
	{
		desc: "linktitle iso8859-2",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "http://arachnist.is-a.cat/test-iso8859-2.html",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "↳ title: Tytuł używający przestarzałego kodowania znaków",
			},
		},
	},
	{
		desc: "linktitle common exploit",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "http://arachnist.is-a.cat/test.html",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"#testchan-1"},
				Trailing: "↳ title: Tak Aż zbyt dobrze. Naprawdę QUIT dupa",
			},
		},
	},
	{
		desc: "linktitle missing title",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "http://arachnist.is-a.cat/test-no-title.html",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{},
	},
	{
		desc: "linktitle notitle",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "https://www.google.com/ notitle",
			Params:   []string{"#testchan-1"},
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{},
	},
	{
		desc: "nickserv spoof",
		in: irc.Message{
			Command:  "NOTICE",
			Params:   []string{"gorepost"},
			Trailing: "This nickname is registered. Please choose a different nickname, or identify via …",
			Prefix: &irc.Prefix{
				Name: "NickServ",
				User: "NickServ",
				Host: "fake.",
			},
		},
		expectedOut: []irc.Message{},
	},
	{
		desc: "nickserv other message",
		in: irc.Message{
			Command:  "NOTICE",
			Params:   []string{"gorepost"},
			Trailing: "Some other random message…",
			Prefix: &irc.Prefix{
				Name: "NickServ",
				User: "NickServ",
				Host: "services.",
			},
		},
		expectedOut: []irc.Message{},
	},
	{
		desc: "non-matching",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "foo bar baz",
		},
		expectedOut: []irc.Message{},
	},
}

func TestPlugins(t *testing.T) {
	var r []irc.Message
	var wg sync.WaitGroup

	// fake irc.Conn Sender replacement
	output := func(msg irc.Message) {
		wg.Done()
		r = append(r, msg)
	}

	for _, e := range eventTests {
		t.Log("running test", e.desc)
		r = r[:0]

		wg.Add(len(e.expectedOut))

		Dispatcher(output, e.in)

		time.Sleep(3000000 * time.Nanosecond)

		wg.Wait()

		if fmt.Sprintf("%+v", r) != fmt.Sprintf("%+v", e.expectedOut) {
			t.Logf("expected: %+v\n", e.expectedOut)
			t.Logf("result: %+v\n", r)
			t.Fail()
		}
	}
}

func configLookupHelper(map[string]string) []string {
	return []string{".testconfig.json"}
}

func TestMain(m *testing.M) {
	log.SetOutput(ioutil.Discard)
	cfg.SetFileListBuilder(configLookupHelper)
	os.Exit(m.Run())
}
<|endoftext|>


<commit_before>// Package profile provides a simple way to manage runtime/pprof
// profiling of your Go application.
package profile

import (
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"runtime/pprof"
)

// Config controls the operation of the profile package.
type Config struct {
	// Verbose controls the output of informational messages
	// during profiling. It defaults to true. If set to false
	// only error messages will be output.
	Verbose bool

	// CPUProfile controls if cpu profiling will be enabled.
	// It defaults to false.
	CPUProfile bool

	// MemProfile controls if memory profiling will be enabled.
	// It defaults to false.
	MemProfile bool

	// ProfilePath controls the base path where various profiling
	// files are written. It defaults to the output of
	// ioutil.TempDir.
	ProfilePath string
}

var CPUProfile = &Config{
	Verbose:    true,
	CPUProfile: true,
}

var MemProfile = &Config{
	Verbose:    true,
	MemProfile: true,
}

func (c *Config) getVerbose() bool {
	if c == nil {
		return true
	}
	return c.Verbose
}

func (c *Config) getCPUProfile() bool {
	if c == nil {
		return false
	}
	return c.CPUProfile
}

func (c *Config) getMemProfile() bool {
	if c == nil {
		return false
	}
	return c.MemProfile
}

func (c *Config) getProfilePath() string {
	if c == nil {
		return ""
	}
	return c.ProfilePath
}

// P represents a set of running profiles.
type P struct {
	path string
	*Config
	closers []func()
}

func (p *P) Stop() {
	for _, c := range p.closers {
		c()
	}
}

// Start starts a new profiling session configured using *Config.
// The caller should call the Stop method on the value returned
// to cleanly stop profiling.
// Passing a nil *Config is the same as passing a *Config with
// defaults chosen.
func Start(cfg *Config) *P {
	path := cfg.getProfilePath()
	var err error
	if path == "" {
		path, err = ioutil.TempDir("", "profile")
	} else {
		err = os.MkdirAll(path, 0777)
	}
	if err != nil {
		log.Fatalf("profile: could not create initial output directory: %v", err)
	}
	p := &P{
		path:   path,
		Config: cfg,
	}

	if p.getCPUProfile() {
		fn := filepath.Join(p.path, "cpu.pprof")
		f, err := os.Create(fn)
		if err != nil {
			log.Fatal("profile: could not create cpu profile file %q: %v", fn, err)
		}
		if p.getVerbose() {
			log.Printf("profile: cpu profiling enabled, %s", fn)
		}
		pprof.StartCPUProfile(f)
		p.closers = append(p.closers, func() {
			pprof.StopCPUProfile()
			f.Close()
		})
	}

	if p.getMemProfile() {
		fn := filepath.Join(p.path, "mem.pprof")
		f, err := os.Create(fn)
		if err != nil {
			log.Fatal("profile: could not create memory profile file %q: %v", fn, err)
		}
		if p.getVerbose() {
			log.Printf("profile: memory profiling enabled, %s", fn)
		}
		p.closers = append(p.closers, func() {
			pprof.Lookup("heap").WriteTo(f, 0)
			f.Close()
		})
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		<-c

		log.Println("profile: caught interrupt, stopping profiles")
		p.Stop()

		os.Exit(0)
	}()

	return p
}
<commit_msg>Added the ability to control hooking ^C<commit_after>// Package profile provides a simple way to manage runtime/pprof
// profiling of your Go application.
package profile

import (
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"runtime/pprof"
)

// Config controls the operation of the profile package.
type Config struct {
	// Verbose controls the output of informational messages
	// during profiling. It defaults to true. If set to false
	// only error messages will be output.
	Verbose bool

	// CPUProfile controls if cpu profiling will be enabled.
	// It defaults to false.
	CPUProfile bool

	// MemProfile controls if memory profiling will be enabled.
	// It defaults to false.
	MemProfile bool

	// ProfilePath controls the base path where various profiling
	// files are written. It defaults to the output of
	// ioutil.TempDir.
	ProfilePath string

	// HandleInterrupt controls whether the profiling package should
	// hook SIGINT to write profiles cleanly.
	// It defaults to true, programs with more sophisticated signal
	// handling should set this to false and ensure the Stop() function
	// returned from Start() is called during shutdown.
	HandleInterrupt bool
}

var CPUProfile = &Config{
	Verbose:         true,
	CPUProfile:      true,
	HandleInterrupt: true,
}

var MemProfile = &Config{
	Verbose:         true,
	MemProfile:      true,
	HandleInterrupt: true,
}

func (c *Config) getVerbose() bool {
	if c == nil {
		return true
	}
	return c.Verbose
}

func (c *Config) getCPUProfile() bool {
	if c == nil {
		return false
	}
	return c.CPUProfile
}

func (c *Config) getMemProfile() bool {
	if c == nil {
		return false
	}
	return c.MemProfile
}

func (c *Config) getProfilePath() string {
	if c == nil {
		return ""
	}
	return c.ProfilePath
}

func (c *Config) getHandleInterrupt() bool {
	if c == nil {
		return true
	}
	return c.HandleInterrupt
}

type profile struct {
	path string
	*Config
	closers []func()
}

func (p *profile) Stop() {
	for _, c := range p.closers {
		c()
	}
}

// Start starts a new profiling session configured using *Config.
// The caller should call the Stop method on the value returned
// to cleanly stop profiling.
// Passing a nil *Config is the same as passing a *Config with
// defaults chosen.
func Start(cfg *Config) interface {
	Stop()
} {
	path := cfg.getProfilePath()
	var err error
	if path == "" {
		path, err = ioutil.TempDir("", "profile")
	} else {
		err = os.MkdirAll(path, 0777)
	}
	if err != nil {
		log.Fatalf("profile: could not create initial output directory: %v", err)
	}
	prof := &profile{
		path:   path,
		Config: cfg,
	}

	if prof.getCPUProfile() {
		fn := filepath.Join(prof.path, "cpu.pprof")
		f, err := os.Create(fn)
		if err != nil {
			log.Fatal("profile: could not create cpu profile file %q: %v", fn, err)
		}
		if prof.getVerbose() {
			log.Printf("profile: cpu profiling enabled, %s", fn)
		}
		pprof.StartCPUProfile(f)
		prof.closers = append(prof.closers, func() {
			pprof.StopCPUProfile()
			f.Close()
		})
	}

	if prof.getMemProfile() {
		fn := filepath.Join(prof.path, "mem.pprof")
		f, err := os.Create(fn)
		if err != nil {
			log.Fatal("profile: could not create memory profile file %q: %v", fn, err)
		}
		if prof.getVerbose() {
			log.Printf("profile: memory profiling enabled, %s", fn)
		}
		prof.closers = append(prof.closers, func() {
			pprof.Lookup("heap").WriteTo(f, 0)
			f.Close()
		})
	}

	if prof.getHandleInterrupt() {
		go func() {
			c := make(chan os.Signal, 1)
			signal.Notify(c, os.Interrupt)
			<-c

			log.Println("profile: caught interrupt, stopping profiles")
			prof.Stop()

			os.Exit(0)
		}()
	}

	return prof
}
<|endoftext|>


<commit_before>package lapi

import (
	"fmt"
	"net/http"

	"github.com/goline/errors"
)

// Rescuer handles error
type Rescuer interface {
	// Rescue handles error, it returns nil if error is handled,
	// and error itself if could not be handled properly
	// Server should panic if an error is returned
	Rescue(connection Connection, v interface{}) error
}

func NewRescuer() Rescuer {
	return &FactoryRescuer{}
}

type ErrorResponse struct {
	// The error code
	// Required: true
	Code string `json:"code"`

	// The error message
	// Required: true
	Message string `json:"message"`
}

type FactoryRescuer struct {
	parser Parser
}

func (r *FactoryRescuer) Rescue(c Connection, v interface{}) error {
	if c == nil {
		return errors.New(ERR_INVALID_ARGUMENT, "Connection must be not nil")
	}
	if r.parser == nil {
		r.parser = new(JsonParser)
	}
	c.Response().
		WithContentType(CONTENT_TYPE_JSON).
		WithParser(r.parser)

	var code, message string
	code = ERR_HTTP_UNKNOWN_ERROR
	if e, ok := v.(errors.Error); ok == true {
		code = e.Code()
		switch code {
		case ERR_HTTP_NOT_FOUND:
			c.Response().WithStatus(http.StatusNotFound)
		case ERR_HTTP_BAD_REQUEST:
			c.Response().WithStatus(http.StatusBadRequest)
		case ERR_HTTP_INTERNAL_SERVER_ERROR:
			c.Response().WithStatus(http.StatusInternalServerError)
		default:
			if e.Status() == http.StatusOK {
				c.Response().WithStatus(http.StatusInternalServerError)
			} else {
				c.Response().WithStatus(e.Status())
			}
		}
		message = e.Message()
	} else if e, ok := v.(error); ok == true {
		message = e.Error()
		c.Response().WithStatus(http.StatusInternalServerError)
	} else {
		message = fmt.Sprintf("%s", v)
		c.Response().WithStatus(http.StatusInternalServerError)
	}
	c.Response().WithContent(&ErrorResponse{code, message})

	return nil
}
<commit_msg>Fix status of rescuer<commit_after>package lapi

import (
	"fmt"
	"net/http"

	"github.com/goline/errors"
)

// Rescuer handles error
type Rescuer interface {
	// Rescue handles error, it returns nil if error is handled,
	// and error itself if could not be handled properly
	// Server should panic if an error is returned
	Rescue(connection Connection, v interface{}) error
}

func NewRescuer() Rescuer {
	return &FactoryRescuer{}
}

type ErrorResponse struct {
	// The error code
	// Required: true
	Code string `json:"code"`

	// The error message
	// Required: true
	Message string `json:"message"`
}

type FactoryRescuer struct {
	parser Parser
}

func (r *FactoryRescuer) Rescue(c Connection, v interface{}) error {
	if c == nil {
		return errors.New(ERR_INVALID_ARGUMENT, "Connection must be not nil")
	}
	if r.parser == nil {
		r.parser = new(JsonParser)
	}
	c.Response().
		WithContentType(CONTENT_TYPE_JSON).
		WithParser(r.parser)

	var code, message string
	code = ERR_HTTP_UNKNOWN_ERROR
	if e, ok := v.(errors.Error); ok == true {
		code = e.Code()
		switch code {
		case ERR_HTTP_NOT_FOUND:
			c.Response().WithStatus(http.StatusNotFound)
		case ERR_HTTP_BAD_REQUEST:
			c.Response().WithStatus(http.StatusBadRequest)
		case ERR_HTTP_INTERNAL_SERVER_ERROR:
			c.Response().WithStatus(http.StatusInternalServerError)
		default:
			if e.Status() == http.StatusOK {
				c.Response().WithStatus(http.StatusInternalServerError)
			} else if c.Response().Status() == http.StatusOK {
				c.Response().WithStatus(e.Status())
			}
		}
		message = e.Message()
	} else if e, ok := v.(error); ok == true {
		message = e.Error()
		c.Response().WithStatus(http.StatusInternalServerError)
	} else {
		message = fmt.Sprintf("%s", v)
		c.Response().WithStatus(http.StatusInternalServerError)
	}
	c.Response().WithContent(&ErrorResponse{code, message})

	return nil
}
<|endoftext|>


<commit_before>package response

import (
	"errors"
	"net/http"
)

var (
	genericError = errors.New("an error occurred")
)

type BadRequest struct {
	error
}

func (err BadRequest) Name() string { return "koding.BadRequest" }

func (err BadRequest) StatusCode() int { return http.StatusBadRequest }

type NotFoundError struct {
	error
}

func (err NotFoundError) Name() string { return "koding.NotFoundError" }

func (err NotFoundError) StatusCode() int { return http.StatusNotFound }

type LimitRateExceededError struct{ Err error }

func (e LimitRateExceededError) Error() string {
	return e.Name()
}

func (err LimitRateExceededError) Name() string {
	return "koding.LimitRateExceededError"
}

func (err LimitRateExceededError) StatusCode() int {
	// to many requests, it is not in standarts yet but browsers already
	// implemented it, Go has builtin status code but not exported yet.
	return 429
}

type SuccessResponse struct {
	Data    interface{}
	Error   error
	Success bool
}

func NewSuccessResponse(data interface{}) *SuccessResponse {
	return &SuccessResponse{
		Success: true,
		Data:    data,
	}
}
<commit_msg>social: change Success field of SuccessResponse to Status<commit_after>package response

import (
	"errors"
	"net/http"
)

var (
	genericError = errors.New("an error occurred")
)

type BadRequest struct {
	error
}

func (err BadRequest) Name() string { return "koding.BadRequest" }

func (err BadRequest) StatusCode() int { return http.StatusBadRequest }

type NotFoundError struct {
	error
}

func (err NotFoundError) Name() string { return "koding.NotFoundError" }

func (err NotFoundError) StatusCode() int { return http.StatusNotFound }

type LimitRateExceededError struct{ Err error }

func (e LimitRateExceededError) Error() string {
	return e.Name()
}

func (err LimitRateExceededError) Name() string {
	return "koding.LimitRateExceededError"
}

func (err LimitRateExceededError) StatusCode() int {
	// to many requests, it is not in standarts yet but browsers already
	// implemented it, Go has builtin status code but not exported yet.
	return 429
}

type SuccessResponse struct {
	Data   interface{}
	Error  error
	Status bool
}

func NewSuccessResponse(data interface{}) *SuccessResponse {
	return &SuccessResponse{
		Status: true,
		Data:   data,
	}
}
<|endoftext|>


<commit_before>package packet

// Length of the Variable header of the CONNECT Packet.
const lenCONNECTVariableHeader = 10

// CONNECT represents the CONNECT Packet.
type CONNECT struct {
	Base
	// clientID is the Client Identifier (ClientId) identifies the Client to the Server.
	ClientID string
	// cleanSession is the Clean Session of the connect flags.
	CleanSession bool
	// willTopic is the Will Topic of the payload.
	WillTopic string
	// willMessage is the Will Message of the payload.
	WillMessage string
	// willQoS is the Will QoS of the connect flags.
	WillQoS uint
	// willRetain is the Will Retain of the connect flags.
	WillRetain bool
	// userName is the user name used by the server for authentication and authorization.
	UserName string
	// password is the password used by the server for authentication and authorization.
	Password string
	// keepAlive is the Keep Alive in the variable header.
	KeepAlive uint
}

// setFixedHeader sets the Fixed header to the Packet.
func (p *CONNECT) setFixedHeader() {
	// Create a byte slice holding the Fixed header.
	b := []byte{TypeCONNECT << 4}

	// Calculate the Remaining Length.
	rl := encodeLength(uint(lenCONNECTVariableHeader + len(p.Payload)))

	if rl&0xFF000000 > 0 {
		b = append(b, byte((rl&0xFF000000)>>24))
		b = append(b, byte((rl&0x00FF0000)>>16))
		b = append(b, byte((rl&0x0000FF00)>>8))
		b = append(b, byte(rl&0x000000FF))
	} else if rl&0x00FF0000 > 0 {
		b = append(b, byte((rl&0x00FF0000)>>16))
		b = append(b, byte((rl&0x0000FF00)>>8))
		b = append(b, byte(rl&0x000000FF))
	} else if rl&0x0000FF00 > 0 {
		b = append(b, byte((rl&0x0000FF00)>>8))
		b = append(b, byte(rl&0x000000FF))
	} else {
		b = append(b, byte(rl&0x000000FF))
	}

	p.FixedHeader = b
}

// setVariableHeader sets the Variable header.
func (p *CONNECT) setVariableHeader() {
	// Create a byte slice holding the Variable header.
	b := make([]byte, lenCONNECTVariableHeader)

	// Set bytes.
	b[0] = 0                // Length MSB (0)
	b[1] = 4                // Length LSB (4)
	b[2] = 77               // 'M'
	b[3] = 81               // 'Q'
	b[4] = 84               // 'T'
	b[5] = 84               // 'T'
	b[6] = 4                // Level (4)
	b[7] = p.connectFlags() // Connect Flags

	// Set the Keep Alive.
	keepAlive := encodeUint16(uint16(p.KeepAlive))
	b[8] = keepAlive[0]
	b[9] = keepAlive[1]

	// Set the byte slice to the Variable header.
	p.VariableHeader = b
}

// setPayload sets the Payload to the Packet.
func (p *CONNECT) setPayload() {
	// Create a byte slice holding the Payload.
	var b []byte

	// Append the Client Identifier.
	appendCONNECTPayload(&b, p.ClientID)

	// Append the Will Topic and Will Message
	if p.will() {
		appendCONNECTPayload(&b, p.WillTopic)
		appendCONNECTPayload(&b, p.WillMessage)
	}

	if p.UserName != "" {
		appendCONNECTPayload(&b, p.UserName)
	}

	if p.Password != "" {
		appendCONNECTPayload(&b, p.Password)
	}

	p.Payload = b
}

// connectFlags creates and returns the bytes representing the Connect Flags.
func (p *CONNECT) connectFlags() byte {
	var b byte

	if p.UserName != "" {
		b |= 128
	}

	if p.Password != "" {
		b |= 64
	}

	if p.WillRetain {
		b |= 32
	}

	b |= byte(p.WillQoS) << 3

	if p.will() {
		b |= 4
	}

	if p.CleanSession {
		b |= 2
	}

	return b
}

// will returns if the Packet has both the Will Topic and Will Message.
func (p *CONNECT) will() bool {
	return p.WillTopic != "" && p.WillMessage != ""
}

func appendCONNECTPayload(b *[]byte, s string) {
	*b = append(*b, encodeUint16(uint16(len(s)))...)
	*b = append(*b, []byte(s)...)
}

// NewCONNECT creates and returns the CONNECT Packet.
func NewCONNECT(opts *CONNECTOptions) Packet {
	// Initialize the options.
	if opts == nil {
		opts = &CONNECTOptions{}
	}
	opts.Init()

	// Create the CONNECT Packet.
	p := &CONNECT{
		ClientID:     opts.ClientID,
		CleanSession: *opts.CleanSession,
		WillTopic:    opts.WillTopic,
		WillMessage:  opts.WillMessage,
		WillQoS:      opts.WillQoS,
		WillRetain:   opts.WillRetain,
		UserName:     opts.UserName,
		Password:     opts.Password,
		KeepAlive:    *opts.KeepAlive,
	}

	// Set the Variable header.
	p.setVariableHeader()

	// Set the Payload.
	p.setPayload()

	// Set the Fixed header.
	p.setFixedHeader()

	// Return the CONNECT Packet.
	return p
}
<commit_msg>Update common/packet/connect.go<commit_after>package packet

// Length of the Variable header of the CONNECT Packet.
const lenCONNECTVariableHeader = 10

// Protocol levels
const protocolLevelVersion3_1_1 = 0x04

// CONNECT represents the CONNECT Packet.
type CONNECT struct {
	Base
	// clientID is the Client Identifier (ClientId) identifies the Client to the Server.
	ClientID string
	// cleanSession is the Clean Session of the connect flags.
	CleanSession bool
	// willTopic is the Will Topic of the payload.
	WillTopic string
	// willMessage is the Will Message of the payload.
	WillMessage string
	// willQoS is the Will QoS of the connect flags.
	WillQoS uint
	// willRetain is the Will Retain of the connect flags.
	WillRetain bool
	// userName is the user name used by the server for authentication and authorization.
	UserName string
	// password is the password used by the server for authentication and authorization.
	Password string
	// keepAlive is the Keep Alive in the variable header.
	KeepAlive uint
}

// setFixedHeader sets the Fixed header to the Packet.
func (p *CONNECT) setFixedHeader() {
	// Create a byte slice holding the Fixed header.
	b := []byte{TypeCONNECT << 4}

	// Calculate the Remaining Length.
	rl := encodeLength(uint(lenCONNECTVariableHeader + len(p.Payload)))

	if rl&0xFF000000 > 0 {
		b = append(b, byte((rl&0xFF000000)>>24))
		b = append(b, byte((rl&0x00FF0000)>>16))
		b = append(b, byte((rl&0x0000FF00)>>8))
		b = append(b, byte(rl&0x000000FF))
	} else if rl&0x00FF0000 > 0 {
		b = append(b, byte((rl&0x00FF0000)>>16))
		b = append(b, byte((rl&0x0000FF00)>>8))
		b = append(b, byte(rl&0x000000FF))
	} else if rl&0x0000FF00 > 0 {
		b = append(b, byte((rl&0x0000FF00)>>8))
		b = append(b, byte(rl&0x000000FF))
	} else {
		b = append(b, byte(rl&0x000000FF))
	}

	p.FixedHeader = b
}

// setVariableHeader sets the Variable header.
func (p *CONNECT) setVariableHeader() {
	// Create a byte slice holding the Variable header.
	b := make([]byte, lenCONNECTVariableHeader)

	// Set bytes.
	b[0] = 0                         // Length MSB (0)
	b[1] = 4                         // Length LSB (4)
	b[2] = 77                        // 'M'
	b[3] = 81                        // 'Q'
	b[4] = 84                        // 'T'
	b[5] = 84                        // 'T'
	b[6] = protocolLevelVersion3_1_1 // Protocol Level
	b[7] = p.connectFlags()          // Connect Flags

	// Set the Keep Alive.
	keepAlive := encodeUint16(uint16(p.KeepAlive))
	b[8] = keepAlive[0]
	b[9] = keepAlive[1]

	// Set the byte slice to the Variable header.
	p.VariableHeader = b
}

// setPayload sets the Payload to the Packet.
func (p *CONNECT) setPayload() {
	// Create a byte slice holding the Payload.
	var b []byte

	// Append the Client Identifier.
	appendCONNECTPayload(&b, p.ClientID)

	// Append the Will Topic and Will Message
	if p.will() {
		appendCONNECTPayload(&b, p.WillTopic)
		appendCONNECTPayload(&b, p.WillMessage)
	}

	if p.UserName != "" {
		appendCONNECTPayload(&b, p.UserName)
	}

	if p.Password != "" {
		appendCONNECTPayload(&b, p.Password)
	}

	p.Payload = b
}

// connectFlags creates and returns the bytes representing the Connect Flags.
func (p *CONNECT) connectFlags() byte {
	var b byte

	if p.UserName != "" {
		b |= 128
	}

	if p.Password != "" {
		b |= 64
	}

	if p.WillRetain {
		b |= 32
	}

	b |= byte(p.WillQoS) << 3

	if p.will() {
		b |= 4
	}

	if p.CleanSession {
		b |= 2
	}

	return b
}

// will returns if the Packet has both the Will Topic and Will Message.
func (p *CONNECT) will() bool {
	return p.WillTopic != "" && p.WillMessage != ""
}

func appendCONNECTPayload(b *[]byte, s string) {
	*b = append(*b, encodeUint16(uint16(len(s)))...)
	*b = append(*b, []byte(s)...)
}

// NewCONNECT creates and returns the CONNECT Packet.
func NewCONNECT(opts *CONNECTOptions) Packet {
	// Initialize the options.
	if opts == nil {
		opts = &CONNECTOptions{}
	}
	opts.Init()

	// Create the CONNECT Packet.
	p := &CONNECT{
		ClientID:     opts.ClientID,
		CleanSession: *opts.CleanSession,
		WillTopic:    opts.WillTopic,
		WillMessage:  opts.WillMessage,
		WillQoS:      opts.WillQoS,
		WillRetain:   opts.WillRetain,
		UserName:     opts.UserName,
		Password:     opts.Password,
		KeepAlive:    *opts.KeepAlive,
	}

	// Set the Variable header.
	p.setVariableHeader()

	// Set the Payload.
	p.setPayload()

	// Set the Fixed header.
	p.setFixedHeader()

	// Return the CONNECT Packet.
	return p
}
<|endoftext|>


<commit_before>package weed_server

import (
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/chrislusf/seaweedfs/weed/cluster"
	"github.com/chrislusf/seaweedfs/weed/pb"

	"github.com/chrislusf/raft"
	"github.com/gorilla/mux"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/sequence"
	"github.com/chrislusf/seaweedfs/weed/shell"
	"github.com/chrislusf/seaweedfs/weed/topology"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
)

const (
	SequencerType        = "master.sequencer.type"
	SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id"
)

type MasterOption struct {
	Master            pb.ServerAddress
	MetaFolder        string
	VolumeSizeLimitMB uint32
	VolumePreallocate bool
	// PulseSeconds            int
	DefaultReplicaPlacement string
	GarbageThreshold        float64
	WhiteList               []string
	DisableHttp             bool
	MetricsAddress          string
	MetricsIntervalSec      int
	IsFollower              bool
}

type MasterServer struct {
	master_pb.UnimplementedSeaweedServer
	option *MasterOption
	guard  *security.Guard

	preallocateSize int64

	Topo *topology.Topology
	vg   *topology.VolumeGrowth
	vgCh chan *topology.VolumeGrowRequest

	boundedLeaderChan chan int

	// notifying clients
	clientChansLock sync.RWMutex
	clientChans     map[string]chan *master_pb.KeepConnectedResponse

	grpcDialOption grpc.DialOption

	MasterClient *wdclient.MasterClient

	adminLocks *AdminLocks

	Cluster *cluster.Cluster
}

func NewMasterServer(r *mux.Router, option *MasterOption, peers []pb.ServerAddress) *MasterServer {

	v := util.GetViper()
	signingKey := v.GetString("jwt.signing.key")
	v.SetDefault("jwt.signing.expires_after_seconds", 10)
	expiresAfterSec := v.GetInt("jwt.signing.expires_after_seconds")

	readSigningKey := v.GetString("jwt.signing.read.key")
	v.SetDefault("jwt.signing.read.expires_after_seconds", 60)
	readExpiresAfterSec := v.GetInt("jwt.signing.read.expires_after_seconds")

	v.SetDefault("master.replication.treat_replication_as_minimums", false)
	replicationAsMin := v.GetBool("master.replication.treat_replication_as_minimums")

	v.SetDefault("master.volume_growth.copy_1", 7)
	v.SetDefault("master.volume_growth.copy_2", 6)
	v.SetDefault("master.volume_growth.copy_3", 3)
	v.SetDefault("master.volume_growth.copy_other", 1)
	v.SetDefault("master.volume_growth.threshold", 0.9)

	var preallocateSize int64
	if option.VolumePreallocate {
		preallocateSize = int64(option.VolumeSizeLimitMB) * (1 << 20)
	}

	grpcDialOption := security.LoadClientTLS(v, "grpc.master")
	ms := &MasterServer{
		option:          option,
		preallocateSize: preallocateSize,
		vgCh:            make(chan *topology.VolumeGrowRequest, 1<<6),
		clientChans:     make(map[string]chan *master_pb.KeepConnectedResponse),
		grpcDialOption:  grpcDialOption,
		MasterClient:    wdclient.NewMasterClient(grpcDialOption, cluster.MasterType, option.Master, "", peers),
		adminLocks:      NewAdminLocks(),
		Cluster:         cluster.NewCluster(),
	}
	ms.boundedLeaderChan = make(chan int, 16)

	seq := ms.createSequencer(option)
	if nil == seq {
		glog.Fatalf("create sequencer failed.")
	}
	ms.Topo = topology.NewTopology("topo", seq, uint64(ms.option.VolumeSizeLimitMB)*1024*1024, 5, replicationAsMin)
	ms.vg = topology.NewDefaultVolumeGrowth()
	glog.V(0).Infoln("Volume Size Limit is", ms.option.VolumeSizeLimitMB, "MB")

	ms.guard = security.NewGuard(ms.option.WhiteList, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec)

	handleStaticResources2(r)
	r.HandleFunc("/", ms.proxyToLeader(ms.uiStatusHandler))
	r.HandleFunc("/ui/index.html", ms.uiStatusHandler)
	if !ms.option.DisableHttp {
		r.HandleFunc("/dir/assign", ms.proxyToLeader(ms.guard.WhiteList(ms.dirAssignHandler)))
		r.HandleFunc("/dir/lookup", ms.guard.WhiteList(ms.dirLookupHandler))
		r.HandleFunc("/dir/status", ms.proxyToLeader(ms.guard.WhiteList(ms.dirStatusHandler)))
		r.HandleFunc("/col/delete", ms.proxyToLeader(ms.guard.WhiteList(ms.collectionDeleteHandler)))
		r.HandleFunc("/vol/grow", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeGrowHandler)))
		r.HandleFunc("/vol/status", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeStatusHandler)))
		r.HandleFunc("/vol/vacuum", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeVacuumHandler)))
		r.HandleFunc("/submit", ms.guard.WhiteList(ms.submitFromMasterServerHandler))
		/*
			r.HandleFunc("/stats/health", ms.guard.WhiteList(statsHealthHandler))
			r.HandleFunc("/stats/counter", ms.guard.WhiteList(statsCounterHandler))
			r.HandleFunc("/stats/memory", ms.guard.WhiteList(statsMemoryHandler))
		*/
		r.HandleFunc("/{fileId}", ms.redirectHandler)
	}

	ms.Topo.StartRefreshWritableVolumes(
		ms.grpcDialOption,
		ms.option.GarbageThreshold,
		v.GetFloat64("master.volume_growth.threshold"),
		ms.preallocateSize,
	)

	ms.ProcessGrowRequest()

	if !option.IsFollower {
		ms.startAdminScripts()
	}

	return ms
}

func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
	ms.Topo.RaftServer = raftServer.raftServer
	ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) {
		glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
		if ms.Topo.RaftServer.Leader() != "" {
			glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
		}
	})
	if ms.Topo.IsLeader() {
		glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", "I am the leader!")
	} else {
		if ms.Topo.RaftServer.Leader() != "" {
			glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "is the leader.")
		}
	}
}

func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if ms.Topo.IsLeader() {
			f(w, r)
		} else if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
			ms.boundedLeaderChan <- 1
			defer func() { <-ms.boundedLeaderChan }()
			targetUrl, err := url.Parse("http://" + ms.Topo.RaftServer.Leader())
			if err != nil {
				writeJsonError(w, r, http.StatusInternalServerError,
					fmt.Errorf("Leader URL http://%s Parse Error: %v", ms.Topo.RaftServer.Leader(), err))
				return
			}
			glog.V(4).Infoln("proxying to leader", ms.Topo.RaftServer.Leader())
			proxy := httputil.NewSingleHostReverseProxy(targetUrl)
			director := proxy.Director
			proxy.Director = func(req *http.Request) {
				actualHost, err := security.GetActualRemoteHost(req)
				if err == nil {
					req.Header.Set("HTTP_X_FORWARDED_FOR", actualHost)
				}
				director(req)
			}
			proxy.Transport = util.Transport
			proxy.ServeHTTP(w, r)
		} else {
			// handle requests locally
			f(w, r)
		}
	}
}

func (ms *MasterServer) startAdminScripts() {
	var err error

	v := util.GetViper()
	adminScripts := v.GetString("master.maintenance.scripts")
	if adminScripts == "" {
		adminScripts = `
		lock
		ec.encode -fullPercent=95 -quietFor=1h
		ec.rebuild -force
		ec.balance -force
		volume.deleteEmpty -quietFor=24h -force
		volume.balance -force
		volume.fix.replication
		s3.clean.uploads -timeAgo=24h
		unlock
	`
	}
	glog.V(0).Infof("adminScripts: %v", adminScripts)

	v.SetDefault("master.maintenance.sleep_minutes", 17)
	sleepMinutes := v.GetInt("master.maintenance.sleep_minutes")

	v.SetDefault("master.filer.default", "localhost:8888")
	filerHostPort := v.GetString("master.filer.default")

	scriptLines := strings.Split(adminScripts, "\n")
	if !strings.Contains(adminScripts, "lock") {
		scriptLines = append(append([]string{}, "lock"), scriptLines...)
		scriptLines = append(scriptLines, "unlock")
	}

	masterAddress := string(ms.option.Master)

	var shellOptions shell.ShellOptions
	shellOptions.GrpcDialOption = security.LoadClientTLS(v, "grpc.master")
	shellOptions.Masters = &masterAddress

	shellOptions.FilerAddress = pb.ServerAddress(filerHostPort)
	shellOptions.Directory = "/"
	if err != nil {
		glog.V(0).Infof("failed to parse master.filer.default = %s : %v\n", filerHostPort, err)
		return
	}

	commandEnv := shell.NewCommandEnv(shellOptions)

	reg, _ := regexp.Compile(`'.*?'|".*?"|\S+`)

	go commandEnv.MasterClient.KeepConnectedToMaster()

	go func() {
		commandEnv.MasterClient.WaitUntilConnected()

		c := time.Tick(time.Duration(sleepMinutes) * time.Minute)
		for range c {
			if ms.Topo.IsLeader() {
				for _, line := range scriptLines {
					for _, c := range strings.Split(line, ";") {
						processEachCmd(reg, c, commandEnv)
					}
				}
			}
		}
	}()
}

func processEachCmd(reg *regexp.Regexp, line string, commandEnv *shell.CommandEnv) {
	cmds := reg.FindAllString(line, -1)
	if len(cmds) == 0 {
		return
	}
	args := make([]string, len(cmds[1:]))
	for i := range args {
		args[i] = strings.Trim(string(cmds[1+i]), "\"'")
	}
	cmd := strings.ToLower(cmds[0])

	for _, c := range shell.Commands {
		if c.Name() == cmd {
			glog.V(0).Infof("executing: %s %v", cmd, args)
			if err := c.Do(args, commandEnv, os.Stdout); err != nil {
				glog.V(0).Infof("error: %v", err)
			}
		}
	}
}

func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer {
	var seq sequence.Sequencer
	v := util.GetViper()
	seqType := strings.ToLower(v.GetString(SequencerType))
	glog.V(1).Infof("[%s] : [%s]", SequencerType, seqType)
	switch strings.ToLower(seqType) {
	case "snowflake":
		var err error
		snowflakeId := v.GetInt(SequencerSnowflakeId)
		seq, err = sequence.NewSnowflakeSequencer(string(option.Master), snowflakeId)
		if err != nil {
			glog.Error(err)
			seq = nil
		}
	default:
		seq = sequence.NewMemorySequencer()
	}
	return seq
}
<commit_msg>sleep between the gap of each execution<commit_after>package weed_server

import (
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/chrislusf/seaweedfs/weed/cluster"
	"github.com/chrislusf/seaweedfs/weed/pb"

	"github.com/chrislusf/raft"
	"github.com/gorilla/mux"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/sequence"
	"github.com/chrislusf/seaweedfs/weed/shell"
	"github.com/chrislusf/seaweedfs/weed/topology"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
)

const (
	SequencerType        = "master.sequencer.type"
	SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id"
)

type MasterOption struct {
	Master            pb.ServerAddress
	MetaFolder        string
	VolumeSizeLimitMB uint32
	VolumePreallocate bool
	// PulseSeconds            int
	DefaultReplicaPlacement string
	GarbageThreshold        float64
	WhiteList               []string
	DisableHttp             bool
	MetricsAddress          string
	MetricsIntervalSec      int
	IsFollower              bool
}

type MasterServer struct {
	master_pb.UnimplementedSeaweedServer
	option *MasterOption
	guard  *security.Guard

	preallocateSize int64

	Topo *topology.Topology
	vg   *topology.VolumeGrowth
	vgCh chan *topology.VolumeGrowRequest

	boundedLeaderChan chan int

	// notifying clients
	clientChansLock sync.RWMutex
	clientChans     map[string]chan *master_pb.KeepConnectedResponse

	grpcDialOption grpc.DialOption

	MasterClient *wdclient.MasterClient

	adminLocks *AdminLocks

	Cluster *cluster.Cluster
}

func NewMasterServer(r *mux.Router, option *MasterOption, peers []pb.ServerAddress) *MasterServer {

	v := util.GetViper()
	signingKey := v.GetString("jwt.signing.key")
	v.SetDefault("jwt.signing.expires_after_seconds", 10)
	expiresAfterSec := v.GetInt("jwt.signing.expires_after_seconds")

	readSigningKey := v.GetString("jwt.signing.read.key")
	v.SetDefault("jwt.signing.read.expires_after_seconds", 60)
	readExpiresAfterSec := v.GetInt("jwt.signing.read.expires_after_seconds")

	v.SetDefault("master.replication.treat_replication_as_minimums", false)
	replicationAsMin := v.GetBool("master.replication.treat_replication_as_minimums")

	v.SetDefault("master.volume_growth.copy_1", 7)
	v.SetDefault("master.volume_growth.copy_2", 6)
	v.SetDefault("master.volume_growth.copy_3", 3)
	v.SetDefault("master.volume_growth.copy_other", 1)
	v.SetDefault("master.volume_growth.threshold", 0.9)

	var preallocateSize int64
	if option.VolumePreallocate {
		preallocateSize = int64(option.VolumeSizeLimitMB) * (1 << 20)
	}

	grpcDialOption := security.LoadClientTLS(v, "grpc.master")
	ms := &MasterServer{
		option:          option,
		preallocateSize: preallocateSize,
		vgCh:            make(chan *topology.VolumeGrowRequest, 1<<6),
		clientChans:     make(map[string]chan *master_pb.KeepConnectedResponse),
		grpcDialOption:  grpcDialOption,
		MasterClient:    wdclient.NewMasterClient(grpcDialOption, cluster.MasterType, option.Master, "", peers),
		adminLocks:      NewAdminLocks(),
		Cluster:         cluster.NewCluster(),
	}
	ms.boundedLeaderChan = make(chan int, 16)

	seq := ms.createSequencer(option)
	if nil == seq {
		glog.Fatalf("create sequencer failed.")
	}
	ms.Topo = topology.NewTopology("topo", seq, uint64(ms.option.VolumeSizeLimitMB)*1024*1024, 5, replicationAsMin)
	ms.vg = topology.NewDefaultVolumeGrowth()
	glog.V(0).Infoln("Volume Size Limit is", ms.option.VolumeSizeLimitMB, "MB")

	ms.guard = security.NewGuard(ms.option.WhiteList, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec)

	handleStaticResources2(r)
	r.HandleFunc("/", ms.proxyToLeader(ms.uiStatusHandler))
	r.HandleFunc("/ui/index.html", ms.uiStatusHandler)
	if !ms.option.DisableHttp {
		r.HandleFunc("/dir/assign", ms.proxyToLeader(ms.guard.WhiteList(ms.dirAssignHandler)))
		r.HandleFunc("/dir/lookup", ms.guard.WhiteList(ms.dirLookupHandler))
		r.HandleFunc("/dir/status", ms.proxyToLeader(ms.guard.WhiteList(ms.dirStatusHandler)))
		r.HandleFunc("/col/delete", ms.proxyToLeader(ms.guard.WhiteList(ms.collectionDeleteHandler)))
		r.HandleFunc("/vol/grow", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeGrowHandler)))
		r.HandleFunc("/vol/status", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeStatusHandler)))
		r.HandleFunc("/vol/vacuum", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeVacuumHandler)))
		r.HandleFunc("/submit", ms.guard.WhiteList(ms.submitFromMasterServerHandler))
		/*
			r.HandleFunc("/stats/health", ms.guard.WhiteList(statsHealthHandler))
			r.HandleFunc("/stats/counter", ms.guard.WhiteList(statsCounterHandler))
			r.HandleFunc("/stats/memory", ms.guard.WhiteList(statsMemoryHandler))
		*/
		r.HandleFunc("/{fileId}", ms.redirectHandler)
	}

	ms.Topo.StartRefreshWritableVolumes(
		ms.grpcDialOption,
		ms.option.GarbageThreshold,
		v.GetFloat64("master.volume_growth.threshold"),
		ms.preallocateSize,
	)

	ms.ProcessGrowRequest()

	if !option.IsFollower {
		ms.startAdminScripts()
	}

	return ms
}

func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
	ms.Topo.RaftServer = raftServer.raftServer
	ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) {
		glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
		if ms.Topo.RaftServer.Leader() != "" {
			glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
		}
	})
	if ms.Topo.IsLeader() {
		glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", "I am the leader!")
	} else {
		if ms.Topo.RaftServer.Leader() != "" {
			glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "is the leader.")
		}
	}
}

func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if ms.Topo.IsLeader() {
			f(w, r)
		} else if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
			ms.boundedLeaderChan <- 1
			defer func() { <-ms.boundedLeaderChan }()
			targetUrl, err := url.Parse("http://" + ms.Topo.RaftServer.Leader())
			if err != nil {
				writeJsonError(w, r, http.StatusInternalServerError,
					fmt.Errorf("Leader URL http://%s Parse Error: %v", ms.Topo.RaftServer.Leader(), err))
				return
			}
			glog.V(4).Infoln("proxying to leader", ms.Topo.RaftServer.Leader())
			proxy := httputil.NewSingleHostReverseProxy(targetUrl)
			director := proxy.Director
			proxy.Director = func(req *http.Request) {
				actualHost, err := security.GetActualRemoteHost(req)
				if err == nil {
					req.Header.Set("HTTP_X_FORWARDED_FOR", actualHost)
				}
				director(req)
			}
			proxy.Transport = util.Transport
			proxy.ServeHTTP(w, r)
		} else {
			// handle requests locally
			f(w, r)
		}
	}
}

func (ms *MasterServer) startAdminScripts() {
	var err error

	v := util.GetViper()
	adminScripts := v.GetString("master.maintenance.scripts")
	if adminScripts == "" {
		adminScripts = `
		lock
		ec.encode -fullPercent=95 -quietFor=1h
		ec.rebuild -force
		ec.balance -force
		volume.deleteEmpty -quietFor=24h -force
		volume.balance -force
		volume.fix.replication
		s3.clean.uploads -timeAgo=24h
		unlock
	`
	}
	glog.V(0).Infof("adminScripts: %v", adminScripts)

	v.SetDefault("master.maintenance.sleep_minutes", 17)
	sleepMinutes := v.GetInt("master.maintenance.sleep_minutes")

	v.SetDefault("master.filer.default", "localhost:8888")
	filerHostPort := v.GetString("master.filer.default")

	scriptLines := strings.Split(adminScripts, "\n")
	if !strings.Contains(adminScripts, "lock") {
		scriptLines = append(append([]string{}, "lock"), scriptLines...)
		scriptLines = append(scriptLines, "unlock")
	}

	masterAddress := string(ms.option.Master)

	var shellOptions shell.ShellOptions
	shellOptions.GrpcDialOption = security.LoadClientTLS(v, "grpc.master")
	shellOptions.Masters = &masterAddress

	shellOptions.FilerAddress = pb.ServerAddress(filerHostPort)
	shellOptions.Directory = "/"
	if err != nil {
		glog.V(0).Infof("failed to parse master.filer.default = %s : %v\n", filerHostPort, err)
		return
	}

	commandEnv := shell.NewCommandEnv(shellOptions)

	reg, _ := regexp.Compile(`'.*?'|".*?"|\S+`)

	go commandEnv.MasterClient.KeepConnectedToMaster()

	go func() {
		commandEnv.MasterClient.WaitUntilConnected()

		for {
			time.Sleep(time.Duration(sleepMinutes) * time.Minute)
			if ms.Topo.IsLeader() {
				for _, line := range scriptLines {
					for _, c := range strings.Split(line, ";") {
						processEachCmd(reg, c, commandEnv)
					}
				}
			}
		}
	}()
}

func processEachCmd(reg *regexp.Regexp, line string, commandEnv *shell.CommandEnv) {
	cmds := reg.FindAllString(line, -1)
	if len(cmds) == 0 {
		return
	}
	args := make([]string, len(cmds[1:]))
	for i := range args {
		args[i] = strings.Trim(string(cmds[1+i]), "\"'")
	}
	cmd := strings.ToLower(cmds[0])

	for _, c := range shell.Commands {
		if c.Name() == cmd {
			glog.V(0).Infof("executing: %s %v", cmd, args)
			if err := c.Do(args, commandEnv, os.Stdout); err != nil {
				glog.V(0).Infof("error: %v", err)
			}
		}
	}
}

func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer {
	var seq sequence.Sequencer
	v := util.GetViper()
	seqType := strings.ToLower(v.GetString(SequencerType))
	glog.V(1).Infof("[%s] : [%s]", SequencerType, seqType)
	switch strings.ToLower(seqType) {
	case "snowflake":
		var err error
		snowflakeId := v.GetInt(SequencerSnowflakeId)
		seq, err = sequence.NewSnowflakeSequencer(string(option.Master), snowflakeId)
		if err != nil {
			glog.Error(err)
			seq = nil
		}
	default:
		seq = sequence.NewMemorySequencer()
	}
	return seq
}
<|endoftext|>