<commit_before>package irc

import (
	"log"
	"strings"
	"sync"
)

// The CmdResponseWriter interface sends an IRC message for a Cmd.
type CmdResponseWriter interface {
	Write(p []byte) (int, error)
}

// cmdResponseWriter is a simple writer that abstracts away the Msg struct.
type cmdResponseWriter struct {
	send     chan<- *Msg
	receiver string
}

// Compose a message to send back to the receiver (channel).
func (w cmdResponseWriter) Write(p []byte) (int, error) {
	w.send <- &Msg{Cmd: "PRIVMSG", Params: []string{w.receiver, string(p)}}
	return len(p), nil
}

// The Cmd interface responds to incoming chat commands.
type Cmd interface {
	Respond(body, source string, w CmdResponseWriter)
}

// A CmdFunc responds to incoming chat commands.
type CmdFunc func(body, source string, w CmdResponseWriter)

// Shim struct to allow users who don't need state to more easily register a
// CmdFunc while not modifying our handling code.
type cmd struct {
	cmdFunc CmdFunc
}

// Respond on our shim just passes through to the user func.
func (c cmd) Respond(body, source string, w CmdResponseWriter) {
	c.cmdFunc(body, source, w)
}

// A CmdHandler dispatches for a group of commands with a common prefix.
type CmdHandler struct {
	prefix  string
	cmdsMtx sync.Mutex
	cmds    map[string]Cmd
}

// NewCmdHandler creates a new CmdHandler with the given command prefix.
func NewCmdHandler(prefix string) *CmdHandler {
	return &CmdHandler{prefix: prefix, cmds: make(map[string]Cmd)}
}

// Accepts for a CmdHandler ensures the msg contains a chat command.
func (cmdHandler *CmdHandler) Accepts(msg *Msg) bool {
	isPrivmsg := msg.Cmd == "PRIVMSG"
	hasCmdPrefix := len(msg.Params) == 2 &&
		strings.HasPrefix(msg.Params[1], cmdHandler.prefix)
	return isPrivmsg && hasCmdPrefix
}

// Handle for a CmdHandler extracts the relevant parts of a command msg and
// dispatches to a Cmd, if one is found with the given name.
func (cmdHandler *CmdHandler) Handle(msg *Msg, send chan<- *Msg) {
	receiver, body, err := msg.ExtractPrivmsg()
	if err != nil {
		log.Println(err)
		return
	}
	nameAndBody := strings.SplitN(body, " ", 2)
	name := strings.TrimPrefix(nameAndBody[0], cmdHandler.prefix)
	if len(nameAndBody) > 1 {
		body = nameAndBody[1]
	} else {
		body = ""
	}
	source, err := msg.ExtractNick()
	if err != nil {
		log.Println(err)
		return
	}
	cmdHandler.cmdsMtx.Lock()
	ccmd, ok := cmdHandler.cmds[name]
	cmdHandler.cmdsMtx.Unlock()
	if ok {
		go ccmd.Respond(body, source,
			cmdResponseWriter{receiver: receiver, send: send})
	}
}

// Register adds a Cmd to be executed when the given name is matched.
func (cmdHandler *CmdHandler) Register(name string, cmd Cmd) {
	cmdHandler.cmdsMtx.Lock()
	defer cmdHandler.cmdsMtx.Unlock()
	cmdHandler.cmds[name] = cmd
}

// RegisterFunc adds a CmdFunc to be executed when the given name is matched.
func (cmdHandler *CmdHandler) RegisterFunc(name string, cmdFunc CmdFunc) {
	cmdHandler.Register(name, cmd{cmdFunc: cmdFunc})
}
<commit_msg>Added CmdHandler.RegisteredNames()<commit_after>package irc

import (
	"log"
	"strings"
	"sync"
)

// The CmdResponseWriter interface sends an IRC message for a Cmd.
type CmdResponseWriter interface {
	Write(p []byte) (int, error)
}

// cmdResponseWriter is a simple writer that abstracts away the Msg struct.
type cmdResponseWriter struct {
	send     chan<- *Msg
	receiver string
}

// Compose a message to send back to the receiver (channel).
func (w cmdResponseWriter) Write(p []byte) (int, error) {
	w.send <- &Msg{Cmd: "PRIVMSG", Params: []string{w.receiver, string(p)}}
	return len(p), nil
}

// The Cmd interface responds to incoming chat commands.
type Cmd interface {
	Respond(body, source string, w CmdResponseWriter)
}

// A CmdFunc responds to incoming chat commands.
type CmdFunc func(body, source string, w CmdResponseWriter)

// Shim struct to allow users who don't need state to more easily register a
// CmdFunc while not modifying our handling code.
type cmd struct {
	cmdFunc CmdFunc
}

// Respond on our shim just passes through to the user func.
func (c cmd) Respond(body, source string, w CmdResponseWriter) {
	c.cmdFunc(body, source, w)
}

// A CmdHandler dispatches for a group of commands with a common prefix.
type CmdHandler struct {
	prefix  string
	cmdsMtx sync.Mutex
	cmds    map[string]Cmd
}

// NewCmdHandler creates a new CmdHandler with the given command prefix.
func NewCmdHandler(prefix string) *CmdHandler {
	return &CmdHandler{prefix: prefix, cmds: make(map[string]Cmd)}
}

// Accepts for a CmdHandler ensures the msg contains a chat command.
func (cmdHandler *CmdHandler) Accepts(msg *Msg) bool {
	isPrivmsg := msg.Cmd == "PRIVMSG"
	hasCmdPrefix := len(msg.Params) == 2 &&
		strings.HasPrefix(msg.Params[1], cmdHandler.prefix)
	return isPrivmsg && hasCmdPrefix
}

// Handle for a CmdHandler extracts the relevant parts of a command msg and
// dispatches to a Cmd, if one is found with the given name.
func (cmdHandler *CmdHandler) Handle(msg *Msg, send chan<- *Msg) {
	receiver, body, err := msg.ExtractPrivmsg()
	if err != nil {
		log.Println(err)
		return
	}
	nameAndBody := strings.SplitN(body, " ", 2)
	name := strings.TrimPrefix(nameAndBody[0], cmdHandler.prefix)
	if len(nameAndBody) > 1 {
		body = nameAndBody[1]
	} else {
		body = ""
	}
	source, err := msg.ExtractNick()
	if err != nil {
		log.Println(err)
		return
	}
	cmdHandler.cmdsMtx.Lock()
	ccmd, ok := cmdHandler.cmds[name]
	cmdHandler.cmdsMtx.Unlock()
	if ok {
		go ccmd.Respond(body, source,
			cmdResponseWriter{receiver: receiver, send: send})
	}
}

func (cmdHandler *CmdHandler) RegisteredNames() (names []string) {
	cmdHandler.cmdsMtx.Lock()
	defer cmdHandler.cmdsMtx.Unlock()
	for name := range cmdHandler.cmds {
		names = append(names, name)
	}
	return
}

// Register adds a Cmd to be executed when the given name is matched.
func (cmdHandler *CmdHandler) Register(name string, cmd Cmd) {
	cmdHandler.cmdsMtx.Lock()
	defer cmdHandler.cmdsMtx.Unlock()
	cmdHandler.cmds[name] = cmd
}

// RegisterFunc adds a CmdFunc to be executed when the given name is matched.
func (cmdHandler *CmdHandler) RegisterFunc(name string, cmdFunc CmdFunc) {
	cmdHandler.Register(name, cmd{cmdFunc: cmdFunc})
}
<|endoftext|>


<commit_before>// Copyright © 2014, All rights reserved
// Joel Scoble, https://github.com/mohae/tomd
//
// This is licensed under The MIT License. Please refer to the included
// LICENSE file for more information. If the LICENSE file has not been
// included, please refer to the url above.
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
//
// tomd: to markdown, takes input and converts it to markdown
//
// Notes:
//	* This is not a general markdown processor. It is a package to provide
//      functions that allow things to be converted to their representation
//      in markdown.
//      Currently that means taking a .csv file and converting it to a table.
//	* Uses seelog for 'library logging', to enable logging see:
//        http://github.com/cihub/seelog/wiki/Writing-libraries-with-Seelog
package tomd

import (
	"encoding/csv"
	"fmt"
	"io"
	"os"
	"path/filepath"
	_ "strconv"
	"strings"
)

// CSV is a struct for representing and working with csv data.
type CSV struct {
	// Source is the source of the CSV data. It is currently assumed to be
	// a path location
	source string

	// destination is where the generated markdown should be put, if it is
	// to be put anywhere. When used, this setting is used in conjunction
	// with destinationType. Not all destinationTypes need to specify a
	// destinatin, bytes, for example.
	destination string

	// destinationType is the type of destination for the md, e.g. file.
	// If the destinationType requires specification of the destination,
	// the Destination variable should be set to that value.
	destinationType string

	// hasHeaderRows: whether the csv data includes a header row as its
	// first row. If the csv data does not include header data, the header
	// data must be provided via template, e.g. false implies
	// 'useFormat' == true. True does not have any implications on using
	// the format file.
	hasHeaderRow bool

	// headerRow contains the header row information. This is when a format
	// has been supplied, the header row information is set.
	headerRow []string

	// columnAlignment contains the alignment information for each column
	// in the table. This is supplied by the format
	columnAlignment []string

	// columnEmphasis contains the emphasis information, if any. for each
	// column. This is supplied by the format.
	columnEmphasis []string

	// formatSource: the location and name of the source file to use. It
	// can either be explicitely set, or TOMD will look for it as
	// `source.fmt`, for `source.csv`.
	formatSource string

	// useFormat: whether there's a format to use with the CSV or not. For
	// files, this is usually a file, with the same name and path as the
	// source, using the 'fmt' extension. This can also be set explicitely.
	// 'useFormat' == false implies 'hasHeaderRow' == true.
	useFormat bool

	// useFormatFile: whether the Format to use is a format file or not.
	// When true, the format will be loaded from the file. When false, the
	// format information must be set using their setters: headerRow,
	// columAlignment, and columnEmphasis.
	useFormatFile bool

	// table is the parsed csv data
	table [][]string

	// md holds the md representation of the csv data
	md []byte
}

// NewCSV returns an initialize CSV object. It still needs to be configured
// for use.
func NewCSV() *CSV {
	C := &CSV{destinationType: "bytes", table: [][]string{}}
	return C
}

// ToMDTable takes a reader for csv and converts the read csv to a markdown
// table. To get the md, call CSV.md()
func (c *CSV) ToMDTable(r io.Reader) error {
	var err error
	c.table, err = ReadCSV(r)
	if err != nil {
		logger.Error(err)
		return err
	}

	//Now convert the data to md
	c.toMD()
	return nil
}

// FileToMDTable takes a file and marshals it to a md table.
func (c *CSV) FileToMDTable(source string) error{
	var err error
	// Try to read the source
	c.table, err = ReadCSVFile(source)
	if err != nil {
		logger.Error(err)
		return err
	}

	var formatName string
	// otherwise see if  HasFormat
	if c.useFormat {
		//derive the format filename
		filename := filepath.Base(source)
		if filename == "." {
			err = fmt.Errorf("unable to determine format filename")
			logger.Error(err)
			return err
		}

		dir := filepath.Dir(source)
		parts := strings.Split(filename, ".")
		formatName = parts[0] + ".fmt"
		if dir != "." {
			formatName = dir + formatName
		}
	}

	if c.useFormat {
		err := c.formatFromFile(formatName)
		if err != nil {
			logger.Error(err)
			return err
		}
	}

	// Now convert the data to md
	c.toMD()
	return nil
}

// md() returns the markdown as []byte
func (c *CSV) MD() []byte {
	return c.md
}

// ReadCSV takes a reader, and reads the data connected with it as CSV data.
// A slice of slice of type string, or an error, are returned. This reads the
// entire file, so if the file is very large and you don't have sufficent RAM
// you will not like the results. There may be a row oriented implementation
// in the future.
func ReadCSV(r io.Reader ) ([][]string, error) {
	cr := csv.NewReader(r)
	rows, err := cr.ReadAll()
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	return rows, nil
}

// ReadCSVFile takes a path, reads the contents of the file and returns int.
func ReadCSVFile(f string) ([][]string, error) {
	file, err := os.Open(f)
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	// because we don't want to forget or worry about hanldling close prior
	// to every return.
	defer file.Close()

	//
	data, err := ReadCSV(file)
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	return data, nil
}

// tomd does table header processing then converts its table data to md,
func (c *CSV) toMD() ()  {
	// Process the header first
	c.addHeader()

	// for each row of table data, process it.
	for _, row := range c.table {
		c.rowToMD(row)
	}

	return
}

// rowTomd takes a csv table row and returns the md version of it consistent
// with its configuration.
func (c *CSV) rowToMD(cols []string) {
	c.appendColumnSeparator()

	for _, col := range cols {
		// TODO this is where column data decoration would occur
		// with templates
		bcol := []byte(col)
		c.md = append(c.md, bcol...)
		c.appendColumnSeparator()
	}

}

// addHeader adds the table header row and the separator row that goes between
// the header row and the data.
func (c *CSV) addHeader() () {
	if c.hasHeaderRow {
		c.rowToMD(c.table[0])
		//remove the first row
		c.table = append(c.table[1:])
	} else {
		if c.useFormat {
			c.rowToMD(c.headerRow)
		}
	}

	c.appendHeaderSeparatorRow(len(c.table[0]))
	return
}

// appendHeaderSeparator adds the configured column  separator
func (c *CSV) appendHeaderSeparatorRow(cols int) {
	c.appendColumnSeparator()

	for i := 0; i < cols; i++ {
		var separator []byte

		if c.useFormat {
			switch c.columnAlignment[i] {
			case "left", "l":
				separator = mdLeftJustify
			case "center", "c":
				separator = mdCentered
			case "right", "r":
				separator = mdRightJustify
			default:
				separator = mdDontJustify
			}
		} else {
			separator = mdDontJustify
		}

		separator = append(separator, mdPipe...)

		c.md = append(c.md, separator...)
	}

	return

}

// appendColumnSeparator appends a pip to the md array
func (c *CSV) appendColumnSeparator() {
	c.md = append(c.md, mdPipe...)
}

// FormatFromFile loads the format file specified.
func (c *CSV) formatFromFile(s string) error {
	table, err := ReadCSVFile(s)
	if err != nil {
		logger.Error(err)
		return err
	}

	//Row 0 is the header information
	c.headerRow = table[0]
	c.columnAlignment = table[1]
	c.columnEmphasis = table[2]

	return nil
}

// HasHeaderRow returns whether, or not, this csv file has a format file to
// use.
func (c *CSV) HasHeaderRow() bool {
	return c.hasHeaderRow
}

// SetHasHeaderRow sets whether, or not, the source has a header row.
func (c *CSV) SetHasHeaderRow(b bool) {
	c.hasHeaderRow = b
}

// HeaderRow returns the column headers; i.e., the header row.
func (c *CSV) HeaderRow() []string {
	return c.headerRow
}

// SetHeaderRow sets the headerRow information.
func (c *CSV) SetHeaderRow(s []string) {
	c.headerRow = s
}

// ColumnAlignment returns the columnAlignment information. This can be set
// either explicitely or using a format file.
func (c *CSV) ColumnAlignment() []string {
	return c.columnAlignment
}

// SetColumnAlignment sets the columnAlignment informatin.
func (c *CSV) SetColumnAlignment(s []string) {
	c.columnAlignment = s
}

// ColumnEmphasis returns the columnEmphasis information. This can be set
// either explicitly or with a format file.
func (c *CSV) ColumnEmphasis() []string {
	return c.columnEmphasis
}

// SetColumnEmphasis sets columnEmphasis information.
func (c *CSV) SetColumnEmphasis(s []string) {
	c.columnEmphasis = s
}

// FormatSource returns the formatSource information.
func (c *CSV) FormatSource() string {
	return c.formatSource
}

// SetFormatSource sets formatSource information. A side-affect of this is that
// setting the format file will automatically set `useFormat` and
// `useFormatFile`.
func (c *CSV) SetFormatSource(s string) {
	c.formatSource = s
}

// UseFormat returns whether this csv file has a format file to use.
func (c *CSV) UseFormat() bool {
	return c.useFormat
}

// SetUseFormat sets whether a format should be used.
func (c *CSV) SetUseFormat(b bool) {
	c.useFormat = b
}

// UseFormat returns whether this csv file has a format file to use.
func (c *CSV) UseFormatFile() bool {
	return c.useFormat
}

// SetUseFormatFile sets whether a format file should be used.
func (c *CSV) SetUseFormatFile(b bool) {
	c.useFormatFile = b
}

<commit_msg>added some more setup code<commit_after>// Copyright © 2014, All rights reserved
// Joel Scoble, https://github.com/mohae/tomd
//
// This is licensed under The MIT License. Please refer to the included
// LICENSE file for more information. If the LICENSE file has not been
// included, please refer to the url above.
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
//
// tomd: to markdown, takes input and converts it to markdown
//
// Notes:
//	* This is not a general markdown processor. It is a package to provide
//      functions that allow things to be converted to their representation
//      in markdown.
//      Currently that means taking a .csv file and converting it to a table.
//	* Uses seelog for 'library logging', to enable logging see:
//        http://github.com/cihub/seelog/wiki/Writing-libraries-with-Seelog
package tomd

import (
	"encoding/csv"
	"fmt"
	"io"
	"os"
	"path/filepath"
	_ "strconv"
	"strings"
)

// CSV is a struct for representing and working with csv data.
type CSV struct {
	// Source is the source of the CSV data. It is currently assumed to be
	// a path location
	source string

	// destination is where the generated markdown should be put, if it is
	// to be put anywhere. When used, this setting is used in conjunction
	// with destinationType. Not all destinationTypes need to specify a
	// destinatin, bytes, for example.
	destination string

	// destinationType is the type of destination for the md, e.g. file.
	// If the destinationType requires specification of the destination,
	// the Destination variable should be set to that value.
	// Supported:
	//	[]byte	no destination needed
	//	file	destination optional, if not set the output will be
	//		`sourceFilename.md` instead of `sourceFilename.csv`.
	destinationType string

	// hasHeaderRows: whether the csv data includes a header row as its
	// first row. If the csv data does not include header data, the header
	// data must be provided via template, e.g. false implies
	// 'useFormat' == true. True does not have any implications on using
	// the format file.
	hasHeaderRow bool

	// headerRow contains the header row information. This is when a format
	// has been supplied, the header row information is set.
	headerRow []string

	// columnAlignment contains the alignment information for each column
	// in the table. This is supplied by the format
	columnAlignment []string

	// columnEmphasis contains the emphasis information, if any. for each
	// column. This is supplied by the format.
	columnEmphasis []string

	// formatSource: the location and name of the source file to use. It
	// can either be explicitely set, or TOMD will look for it as
	// `source.fmt`, for `source.csv`.
	formatSource string

	// useFormat: whether there's a format to use with the CSV or not. For
	// files, this is usually a file, with the same name and path as the
	// source, using the 'fmt' extension. This can also be set explicitely.
	// 'useFormat' == false implies 'hasHeaderRow' == true.
	useFormat bool

	// useFormatFile: whether the Format to use is a format file or not.
	// When true, the format will be loaded from the file. When false, the
	// format information must be set using their setters: headerRow,
	// columAlignment, and columnEmphasis.
	useFormatFile bool

	// table is the parsed csv data
	table [][]string

	// md holds the md representation of the csv data
	md []byte
}

// NewCSV returns an initialize CSV object. It still needs to be configured
// for use.
func NewCSV() *CSV {
	C := &CSV{
		hasHeader: true,
		destinationType: "bytes",
		table: [][]string{}
	}
	return C
}

// ToMDTable takes a reader for csv and converts the read csv to a markdown
// table. To get the md, call CSV.md()
func (c *CSV) ToMDTable(r io.Reader) error {
	var err error
	c.table, err = ReadCSV(r)
	if err != nil {
		logger.Error(err)
		return err
	}

	//Now convert the data to md
	c.toMD()
	return nil
}

// FileToMDTable takes a file and marshals it to a md table.
func (c *CSV) FileToMDTable(source string) error{
	var err error
	// Try to read the source
	c.table, err = ReadCSVFile(source)
	if err != nil {
		logger.Error(err)
		return err
	}

	var formatName string
	// otherwise see if  HasFormat
	if c.useFormat {
		//derive the format filename
		filename := filepath.Base(source)
		if filename == "." {
			err = fmt.Errorf("unable to determine format filename")
			logger.Error(err)
			return err
		}

		dir := filepath.Dir(source)
		parts := strings.Split(filename, ".")
		formatName = parts[0] + ".fmt"
		if dir != "." {
			formatName = dir + formatName
		}
	}

	if c.useFormat {
		err := c.formatFromFile(formatName)
		if err != nil {
			logger.Error(err)
			return err
		}
	}

	// Now convert the data to md
	c.toMD()
	return nil
}

// md() returns the markdown as []byte
func (c *CSV) MD() []byte {
	return c.md
}

// ReadCSV takes a reader, and reads the data connected with it as CSV data.
// A slice of slice of type string, or an error, are returned. This reads the
// entire file, so if the file is very large and you don't have sufficent RAM
// you will not like the results. There may be a row oriented implementation
// in the future.
func ReadCSV(r io.Reader ) ([][]string, error) {
	cr := csv.NewReader(r)
	rows, err := cr.ReadAll()
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	return rows, nil
}

// ReadCSVFile takes a path, reads the contents of the file and returns int.
func ReadCSVFile(f string) ([][]string, error) {
	file, err := os.Open(f)
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	// because we don't want to forget or worry about hanldling close prior
	// to every return.
	defer file.Close()

	//
	data, err := ReadCSV(file)
	if err != nil {
		logger.Error(err)
		return nil, err
	}

	return data, nil
}

// tomd does table header processing then converts its table data to md,
func (c *CSV) toMD() ()  {
	// Process the header first
	c.addHeader()

	// for each row of table data, process it.
	for _, row := range c.table {
		c.rowToMD(row)
	}

	return
}

// rowTomd takes a csv table row and returns the md version of it consistent
// with its configuration.
func (c *CSV) rowToMD(cols []string) {
	c.appendColumnSeparator()

	for _, col := range cols {
		// TODO this is where column data decoration would occur
		// with templates
		bcol := []byte(col)
		c.md = append(c.md, bcol...)
		c.appendColumnSeparator()
	}

}

// addHeader adds the table header row and the separator row that goes between
// the header row and the data.
func (c *CSV) addHeader() () {
	if c.hasHeaderRow {
		c.rowToMD(c.table[0])
		//remove the first row
		c.table = append(c.table[1:])
	} else {
		if c.useFormat {
			c.rowToMD(c.headerRow)
		}
	}

	c.appendHeaderSeparatorRow(len(c.table[0]))
	return
}

// appendHeaderSeparator adds the configured column  separator
func (c *CSV) appendHeaderSeparatorRow(cols int) {
	c.appendColumnSeparator()

	for i := 0; i < cols; i++ {
		var separator []byte

		if c.useFormat {
			switch c.columnAlignment[i] {
			case "left", "l":
				separator = mdLeftJustify
			case "center", "c":
				separator = mdCentered
			case "right", "r":
				separator = mdRightJustify
			default:
				separator = mdDontJustify
			}
		} else {
			separator = mdDontJustify
		}

		separator = append(separator, mdPipe...)

		c.md = append(c.md, separator...)
	}

	return

}

// appendColumnSeparator appends a pip to the md array
func (c *CSV) appendColumnSeparator() {
	c.md = append(c.md, mdPipe...)
}

// FormatFromFile loads the format file specified.
func (c *CSV) formatFromFile(s string) error {
	table, err := ReadCSVFile(s)
	if err != nil {
		logger.Error(err)
		return err
	}

	//Row 0 is the header information
	c.headerRow = table[0]
	c.columnAlignment = table[1]
	c.columnEmphasis = table[2]

	return nil
}

// Destination is the of destination for the output, if applicable.
func (c *CSV) DestinationType() string {
	return c.destinationType
}

// SetDestination sets the destination of the output, if applicable.
func (c *CSV) SetDestinationType(string) {
	return c.destinationType
}

// DestinationType is the type of destination for the output.
func (c *CSV) DestinationType() string {
	return c.destinationType
}

// SetDestinationType sets the destinationType.
func (c *CSV) SetDestinationType(string) {
	return c.destinationType
}

// HasHeaderRow returns whether, or not, this csv file has a format file to
// use.
func (c *CSV) HasHeaderRow() bool {
	return c.hasHeaderRow
}

// SetHasHeaderRow sets whether, or not, the source has a header row.
func (c *CSV) SetHasHeaderRow(b bool) {
	c.hasHeaderRow = b
}

// HeaderRow returns the column headers; i.e., the header row.
func (c *CSV) HeaderRow() []string {
	return c.headerRow
}

// SetHeaderRow sets the headerRow information.
func (c *CSV) SetHeaderRow(s []string) {
	c.headerRow = s
}

// ColumnAlignment returns the columnAlignment information. This can be set
// either explicitely or using a format file.
func (c *CSV) ColumnAlignment() []string {
	return c.columnAlignment
}

// SetColumnAlignment sets the columnAlignment informatin.
func (c *CSV) SetColumnAlignment(s []string) {
	c.columnAlignment = s
}

// ColumnEmphasis returns the columnEmphasis information. This can be set
// either explicitly or with a format file.
func (c *CSV) ColumnEmphasis() []string {
	return c.columnEmphasis
}

// SetColumnEmphasis sets columnEmphasis information.
func (c *CSV) SetColumnEmphasis(s []string) {
	c.columnEmphasis = s
}

// FormatSource returns the formatSource information.
func (c *CSV) FormatSource() string {
	return c.formatSource
}

// SetFormatSource sets formatSource information. A side-affect of this is that
// setting the format file will automatically set `useFormat` and
// `useFormatFile`.
func (c *CSV) SetFormatSource(s string) {
	c.formatSource = s
}

// UseFormat returns whether this csv file has a format file to use.
func (c *CSV) UseFormat() bool {
	return c.useFormat
}

// SetUseFormat sets whether a format should be used.
func (c *CSV) SetUseFormat(b bool) {
	c.useFormat = b
}

// UseFormat returns whether this csv file has a format file to use.
func (c *CSV) UseFormatFile() bool {
	return c.useFormat
}

// SetUseFormatFile sets whether a format file should be used.
func (c *CSV) SetUseFormatFile(b bool) {
	c.useFormatFile = b
}

<|endoftext|>


<commit_before>package gweb

import (
	"encoding/json"
	"log"
	"net/http"
	"os"
	"strings"

	. "github.com/eynstudio/gobreak"
	"github.com/eynstudio/gobreak/di"
)

type Ctx struct {
	*App
	di.Container
	*Req
	*Resp
	Scope   M
	isErr   bool
	afters  []Handler
	Handled bool
}

func (p *Ctx) Error(code int) *Ctx {
	p.WriteHeader(code)
	p.isErr = true
	return p
}

func (p *Ctx) Set(k string, v T)   { p.Scope[k] = v }
func (p *Ctx) IsErr() bool         { return p.isErr }
func (p *Ctx) Get(k string) string { return p.Scope.GetStr(k) }

func (p *Ctx) NotFound()           { p.Error(http.StatusNotFound) }
func (p *Ctx) Forbidden()          { p.Error(http.StatusForbidden) }
func (p *Ctx) Redirect(url string) { http.Redirect(p.Resp, p.Request, url, http.StatusFound) }

func (p *Ctx) Json(m T) {
	if p.IsErr() {
		return
	}
	if b, err := json.Marshal(m); err != nil {
		p.Error(http.StatusInternalServerError)
	} else {
		p.Resp.Header().Set("Content-Type", "application/json; charset=utf-8")
		p.Resp.Write(b)
	}
}

func (p *Ctx) Tmpl(tpl string, o T) {
	p.Resp.Header().Set("Content-Type", "text/html; charset=utf-8")
	if err := p.App.Tmpl.Execute(p.Resp, tpl, o); err != nil {
		log.Println(err)
		p.Error(http.StatusInternalServerError)
	}
}

func (p *Ctx) ServeFile() bool {
	url := p.Url()
	for _, path := range p.Cfg.ServeFiles {
		if strings.HasPrefix(url, path) {
			if fi, err := os.Stat(url[1:]); err != nil || fi.IsDir() {
				p.NotFound()
			} else {
				http.ServeFile(p.Resp, p.Request, url[1:])
			}
			return true
		}
	}
	return false
}
<commit_msg>add ctx.OK<commit_after>package gweb

import (
	"encoding/json"
	"log"
	"net/http"
	"os"
	"strings"

	. "github.com/eynstudio/gobreak"
	"github.com/eynstudio/gobreak/di"
)

type Ctx struct {
	*App
	di.Container
	*Req
	*Resp
	Scope   M
	isErr   bool
	afters  []Handler
	Handled bool
}

func (p *Ctx) Error(code int) *Ctx {
	p.WriteHeader(code)
	p.isErr = true
	return p
}

func (p *Ctx) Set(k string, v T)   { p.Scope[k] = v }
func (p *Ctx) IsErr() bool         { return p.isErr }
func (p *Ctx) Get(k string) string { return p.Scope.GetStr(k) }

func (p *Ctx) OK()                 { p.WriteHeader(http.StatusOK) }
func (p *Ctx) NotFound()           { p.Error(http.StatusNotFound) }
func (p *Ctx) Forbidden()          { p.Error(http.StatusForbidden) }
func (p *Ctx) Redirect(url string) { http.Redirect(p.Resp, p.Request, url, http.StatusFound) }

func (p *Ctx) Json(m T) {
	if p.IsErr() {
		return
	}
	if b, err := json.Marshal(m); err != nil {
		p.Error(http.StatusInternalServerError)
	} else {
		p.Resp.Header().Set("Content-Type", "application/json; charset=utf-8")
		p.Resp.Write(b)
	}
}

func (p *Ctx) Tmpl(tpl string, o T) {
	p.Resp.Header().Set("Content-Type", "text/html; charset=utf-8")
	if err := p.App.Tmpl.Execute(p.Resp, tpl, o); err != nil {
		log.Println(err)
		p.Error(http.StatusInternalServerError)
	}
}

func (p *Ctx) ServeFile() bool {
	url := p.Url()
	for _, path := range p.Cfg.ServeFiles {
		if strings.HasPrefix(url, path) {
			if fi, err := os.Stat(url[1:]); err != nil || fi.IsDir() {
				p.NotFound()
			} else {
				http.ServeFile(p.Resp, p.Request, url[1:])
			}
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package scraper

import (
	"bufio"
	"github.com/ProfessorBeekums/PbStockResearcher/log"
	"io"
	"net/http"
	"strconv"
	"strings"
)

// create a function to scrape an index given a year and quarter
// the function will also take in a delay between loading each file
// save results to a data store
// do not re-download things already in the data store!

const EDGAR_FULL_INDEX_URL_PREFIX = "http://www.sec.gov/Archives/edgar/full-index/"
const INDEX_FILE_NAME = "/xbrl.idx"

type EdgarFullIndexScraper struct {
	year, quarter int
}

func NewEdgarFullIndexScraper(year, quarter int) *EdgarFullIndexScraper {
	return &EdgarFullIndexScraper{year: year, quarter: quarter}
}

func (efis *EdgarFullIndexScraper) ScrapeEdgarQuarterlyIndex() {
	log.Println("Starting to scrape the full index for year <", efis.year,
		"> and quarter:", efis.quarter)

	indexUrl := EDGAR_FULL_INDEX_URL_PREFIX +
		strconv.FormatInt(int64(efis.year), 10) +
		"/QTR" + strconv.FormatInt(int64(efis.quarter),10) + INDEX_FILE_NAME

	getResp, getErr := http.Get(indexUrl)

	if getErr != nil {
		log.Error("Failed to retrieve index for url <", indexUrl,
		"> with error: ", getErr)
	} else if getResp.StatusCode != 200 {
		log.Error("Received status code <", getResp.Status, "> for url: ", indexUrl)
	} else {
		log.Println("@@@ Success!", indexUrl, getResp)
		defer getResp.Body.Close()

		efis.ParseIndexFile(getResp.Body)
	}
}

// Parses a ReadCloser that contains a Full Index file. The caller is
// responsible for closing the ReadCloser.
func (efis *EdgarFullIndexScraper) ParseIndexFile(fileReader io.ReadCloser) {
	listBegun := false // we need to parse the header before we get the list
	var line []byte = nil
	var readErr error = nil
	var isPrefix bool = false

	reader := bufio.NewReader(fileReader)
	for readErr == nil {
		// none of these lines should be bigger than the buffer
		line, isPrefix, readErr = reader.ReadLine()
		if isPrefix {
			// don't bother parsing here, just log that we had an error
			log.Error("This index file has a line that's too long!")
			continue
		}

		if line != nil {
			lineStr := string(line)
			if !listBegun && strings.Contains(lineStr, "-------") {
				listBegun = true
				continue
			}

			// headers done, now we can start parsing
			if listBegun {
				elements := strings.Split(lineStr, "|")
				cik := elements[0]
				companyName := elements[1]
				formType := elements[2]
				dateFiled := elements[3]
				filename := elements[4]

				log.Println("CIK: ", cik, " Company Name: ", companyName, " Form type: ", formType, "  Date Filed: ", dateFiled, "  FileName: ", filename)
			}
		}
	}
}

/*

func getXbrl(edgarFilename string) {
    // TODO log if not a txt file
    parts := strings.Split(edgarFilename, "/")
    baseName := strings.Trim(parts[3], ".txt")
    preBase := strings.Replace(baseName, "-", "", -1)
    parts[3] = preBase + "/" + baseName + XBRL_ZIP_SUFFIX

    fullUrl := SEC_EDGAR_BASE_URL + strings.Join(parts, "/")

    logger.Println("getting xbrl from ", fullUrl)

    getResp, getErr := http.Get(fullUrl)

    if getErr != nil {
        logger.Println("Failed get to: ", fullUrl)
    } else {
        defer getResp.Body.Close()

        data, readErr := ioutil.ReadAll(getResp.Body)

        if readErr != nil {
            logger.Println("Failed to read")
        } else {
            outputFileName := time.Now().String() + baseName + XBRL_ZIP_SUFFIX
            ioutil.WriteFile(outputFileName, data, 0777)

            zipReader, zipErr := zip.OpenReader(outputFileName)
            if zipErr != nil {
                logger.Println("Failed to open zip: ", outputFileName)
            } else {
                defer zipReader.Close()

                for _, zippedFile := range zipReader.File {
                    zippedFileName := zippedFile.Name
                    isMatch,_ := regexp.MatchString("[a-z]+-[0-9]{8}.xml", zippedFileName)
                    if isMatch {
                        logger.Println("Found zipped file: ", zippedFileName )
                    }
                }
            }
        }
    }
}
*/
<commit_msg>extract xbrl file and save to directory<commit_after>package scraper

import (
	"archive/zip"
	"bufio"
	"github.com/ProfessorBeekums/PbStockResearcher/log"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// create a function to scrape an index given a year and quarter
// the function will also take in a delay between loading each file
// save results to a data store
// do not re-download things already in the data store!

const EDGAR_FULL_INDEX_URL_PREFIX = "http://www.sec.gov/Archives/edgar/full-index/"
const INDEX_FILE_NAME = "/xbrl.idx"

const SEC_EDGAR_BASE_URL = "http://www.sec.gov/Archives/"
const XBRL_ZIP_SUFFIX = "-xbrl.zip"

type EdgarFullIndexScraper struct {
	year, quarter int
}

func NewEdgarFullIndexScraper(year, quarter int) *EdgarFullIndexScraper {
	return &EdgarFullIndexScraper{year: year, quarter: quarter}
}

func (efis *EdgarFullIndexScraper) ScrapeEdgarQuarterlyIndex() {
	log.Println("Starting to scrape the full index for year <", efis.year,
		"> and quarter:", efis.quarter)

	indexUrl := EDGAR_FULL_INDEX_URL_PREFIX +
		strconv.FormatInt(int64(efis.year), 10) +
		"/QTR" + strconv.FormatInt(int64(efis.quarter),10) + INDEX_FILE_NAME

	getResp, getErr := http.Get(indexUrl)

	if getErr != nil {
		log.Error("Failed to retrieve index for url <", indexUrl,
		"> with error: ", getErr)
	} else if getResp.StatusCode != 200 {
		log.Error("Received status code <", getResp.Status, "> for url: ", indexUrl)
	} else {
		defer getResp.Body.Close()

		efis.ParseIndexFile(getResp.Body)
	}
}

// Parses a ReadCloser that contains a Full Index file. The caller is
// responsible for closing the ReadCloser.
func (efis *EdgarFullIndexScraper) ParseIndexFile(fileReader io.ReadCloser) {
	listBegun := false // we need to parse the header before we get the list
	var line []byte = nil
	var readErr error = nil
	var isPrefix bool = false

	reader := bufio.NewReader(fileReader)
	for readErr == nil {
		// none of these lines should be bigger than the buffer
		line, isPrefix, readErr = reader.ReadLine()
		if isPrefix {
			// don't bother parsing here, just log that we had an error
			log.Error("This index file has a line that's too long!")
			continue
		}

		if line != nil {
			lineStr := string(line)
			if !listBegun && strings.Contains(lineStr, "-------") {
				listBegun = true
				continue
			}

			// headers done, now we can start parsing
			if listBegun {
				elements := strings.Split(lineStr, "|")
				cik := elements[0]
				companyName := elements[1]
				formType := elements[2]
				dateFiled := elements[3]
				filename := elements[4]

				log.Println("CIK: ", cik, " Company Name: ", companyName, " Form type: ", formType, "  Date Filed: ", dateFiled, "  FileName: ", filename)

				efis.GetXbrl(filename)
				// TODO - temporary hack for testing
				break
			}
		}
	}
}

// The full index provides links to txt files. We want to convert these to retrieve the corresponding zip of xbrl files
// and extract the main xbrl file.
func (efis *EdgarFullIndexScraper) GetXbrl(edgarFilename string) {
	if !strings.Contains(edgarFilename, ".txt") {
		log.Error("Unexpected file type: ", edgarFilename)
		return
	}

    parts := strings.Split(edgarFilename, "/")
    baseName := strings.Trim(parts[3], ".txt")
    preBase := strings.Replace(baseName, "-", "", -1)
    parts[3] = preBase + "/" + baseName + XBRL_ZIP_SUFFIX

    fullUrl := SEC_EDGAR_BASE_URL + strings.Join(parts, "/")

    log.Println("Getting xbrl zip from ", fullUrl)

    getResp, getErr := http.Get(fullUrl)

    if getErr != nil {
        log.Error("Failed get to: ", fullUrl)
    } else {
        defer getResp.Body.Close()

        data, readErr := ioutil.ReadAll(getResp.Body)

        if readErr != nil {
            log.Error("Failed to read")
        } else {
            outputFileName := strconv.Itoa(int(time.Now().Unix()) )+ baseName + XBRL_ZIP_SUFFIX
            // TODO configure a data directory
            writeErr := ioutil.WriteFile(outputFileName, data, 0777)

            if writeErr != nil {
            	log.Error("Failed to write file: ", writeErr)
            } else {
            	efis.getXbrlFromZip(outputFileName)
            }
        }
    }
}

func (efis *EdgarFullIndexScraper) getXbrlFromZip(zipFileName string) {
	zipReader, zipErr := zip.OpenReader(zipFileName)

	if zipErr != nil {
        log.Error("Failed to open zip: ", zipFileName, " with error: ", zipErr)
    } else {
        defer zipReader.Close()

        for _, zippedFile := range zipReader.File {
            zippedFileName := zippedFile.Name
            isMatch,_ := regexp.MatchString("[a-z]+-[0-9]{8}.xml", zippedFileName)
            if isMatch {
                log.Println("Found zipped file: ", zippedFileName )

                xbrlFile, xbrlErr := zippedFile.Open()

                if xbrlErr != nil {
                	log.Error("Failed to open zip file")
                } else {
                	data, readErr := ioutil.ReadAll(xbrlFile)
                	if readErr != nil {
			            log.Error("Failed to read")
			        } else {
			        	writeErr := ioutil.WriteFile(zippedFileName, data, 0777)

			        	if writeErr != nil {
			            	log.Error("Failed to write file: ", writeErr)
			            }
			        }
                }

                // we don't care about the other stuff
                break
            }
        }
    }
}
<|endoftext|>


<commit_before>package hbase

import (
	"bytes"
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/tsuna/gohbase"
	"github.com/tsuna/gohbase/hrpc"
	"io"
)

func init() {
	filer.Stores = append(filer.Stores, &HbaseStore{})
}

type HbaseStore struct {
	Client    gohbase.Client
	table     []byte
	cfKv      string
	cfMetaDir string
	column    string
}

func (store *HbaseStore) GetName() string {
	return "hbase"
}

func (store *HbaseStore) Initialize(configuration util.Configuration, prefix string) (err error) {
	return store.initialize(
		configuration.GetString(prefix+"zkquorum"),
		configuration.GetString(prefix+"table"),
	)
}

func (store *HbaseStore) initialize(zkquorum, table string) (err error) {
	store.Client = gohbase.NewClient(zkquorum)
	store.table = []byte(table)
	store.cfKv = "kv"
	store.cfMetaDir = "meta"
	store.column = "a"

	// check table exists
	key := "whatever"
	headers := map[string][]string{store.cfMetaDir: nil}
	get, err := hrpc.NewGet(context.Background(), store.table, []byte(key), hrpc.Families(headers))
	if err != nil {
		return fmt.Errorf("NewGet returned an error: %v", err)
	}
	_, err = store.Client.Get(get)
	if err != gohbase.TableNotFound {
		return nil
	}

	// create table
	adminClient := gohbase.NewAdminClient(zkquorum)
	cFamilies := []string{store.cfKv, store.cfMetaDir}
	cf := make(map[string]map[string]string, len(cFamilies))
	for _, f := range cFamilies {
		cf[f] = nil
	}
	ct := hrpc.NewCreateTable(context.Background(), []byte(table), cf)
	if err := adminClient.CreateTable(ct); err != nil {
		return err
	}

	return nil
}

func (store *HbaseStore) InsertEntry(ctx context.Context, entry *filer.Entry) error {
	value, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
	}
	if len(entry.Chunks) > 50 {
		value = util.MaybeGzipData(value)
	}

	return store.doPut(ctx, store.cfMetaDir, []byte(entry.FullPath), value)
}

func (store *HbaseStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
	return store.InsertEntry(ctx, entry)
}

func (store *HbaseStore) FindEntry(ctx context.Context, path util.FullPath) (entry *filer.Entry, err error) {
	value, err := store.doGet(ctx, store.cfMetaDir, []byte(path))
	if err != nil {
		if err == filer.ErrKvNotFound {
			return nil, filer_pb.ErrNotFound
		}
		return nil, err
	}

	entry = &filer.Entry{
		FullPath: path,
	}
	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(value))
	if err != nil {
		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
	}
	return entry, nil
}

func (store *HbaseStore) DeleteEntry(ctx context.Context, path util.FullPath) (err error) {
	return store.doDelete(ctx, store.cfMetaDir, []byte(path))
}

func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.FullPath) (err error) {

	family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}}
	expectedPrefix := []byte(path+"/")
	scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family))
	if err != nil {
		return err
	}

	scanner := store.Client.Scan(scan)
	defer scanner.Close()
	for {
		res, err := scanner.Next()
		if err != nil {
			break
		}
		if len(res.Cells) == 0 {
			continue
		}
		cell := res.Cells[0]

		if !bytes.HasPrefix(cell.Row, expectedPrefix) {
			break
		}

		err = store.doDelete(ctx, store.cfMetaDir, cell.Row)
		if err != nil {
			break
		}

	}
	return
}

func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*filer.Entry, error) {
	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "")
}

func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*filer.Entry, error) {
	family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}}
	expectedPrefix := []byte(dirPath.Child(prefix))
	scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family))
	if err != nil {
		return nil, err
	}

	var entries []*filer.Entry
	scanner := store.Client.Scan(scan)
	defer scanner.Close()
	for {
		res, err := scanner.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return entries, err
		}
		if len(res.Cells) == 0 {
			continue
		}
		cell := res.Cells[0]

		if !bytes.HasPrefix(cell.Row, expectedPrefix) {
			break
		}

		fullpath := util.FullPath(cell.Row)

		value := cell.Value

		_, fileName := fullpath.DirAndName()

		if fileName == startFileName && !includeStartFile {
			continue
		}

		limit--
		if limit < 0 {
			break
		}
		entry := &filer.Entry{
			FullPath: fullpath,
		}
		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(value)); decodeErr != nil {
			err = decodeErr
			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
			break
		}
		entries = append(entries, entry)
	}

	return entries, nil
}

func (store *HbaseStore) BeginTransaction(ctx context.Context) (context.Context, error) {
	return ctx, nil
}

func (store *HbaseStore) CommitTransaction(ctx context.Context) error {
	return nil
}

func (store *HbaseStore) RollbackTransaction(ctx context.Context) error {
	return nil
}

func (store *HbaseStore) Shutdown() {
	store.Client.Close()
}
<commit_msg>adjust for directory listing<commit_after>package hbase

import (
	"bytes"
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/tsuna/gohbase"
	"github.com/tsuna/gohbase/hrpc"
	"io"
)

func init() {
	filer.Stores = append(filer.Stores, &HbaseStore{})
}

type HbaseStore struct {
	Client    gohbase.Client
	table     []byte
	cfKv      string
	cfMetaDir string
	column    string
}

func (store *HbaseStore) GetName() string {
	return "hbase"
}

func (store *HbaseStore) Initialize(configuration util.Configuration, prefix string) (err error) {
	return store.initialize(
		configuration.GetString(prefix+"zkquorum"),
		configuration.GetString(prefix+"table"),
	)
}

func (store *HbaseStore) initialize(zkquorum, table string) (err error) {
	store.Client = gohbase.NewClient(zkquorum)
	store.table = []byte(table)
	store.cfKv = "kv"
	store.cfMetaDir = "meta"
	store.column = "a"

	// check table exists
	key := "whatever"
	headers := map[string][]string{store.cfMetaDir: nil}
	get, err := hrpc.NewGet(context.Background(), store.table, []byte(key), hrpc.Families(headers))
	if err != nil {
		return fmt.Errorf("NewGet returned an error: %v", err)
	}
	_, err = store.Client.Get(get)
	if err != gohbase.TableNotFound {
		return nil
	}

	// create table
	adminClient := gohbase.NewAdminClient(zkquorum)
	cFamilies := []string{store.cfKv, store.cfMetaDir}
	cf := make(map[string]map[string]string, len(cFamilies))
	for _, f := range cFamilies {
		cf[f] = nil
	}
	ct := hrpc.NewCreateTable(context.Background(), []byte(table), cf)
	if err := adminClient.CreateTable(ct); err != nil {
		return err
	}

	return nil
}

func (store *HbaseStore) InsertEntry(ctx context.Context, entry *filer.Entry) error {
	value, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
	}
	if len(entry.Chunks) > 50 {
		value = util.MaybeGzipData(value)
	}

	return store.doPut(ctx, store.cfMetaDir, []byte(entry.FullPath), value)
}

func (store *HbaseStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
	return store.InsertEntry(ctx, entry)
}

func (store *HbaseStore) FindEntry(ctx context.Context, path util.FullPath) (entry *filer.Entry, err error) {
	value, err := store.doGet(ctx, store.cfMetaDir, []byte(path))
	if err != nil {
		if err == filer.ErrKvNotFound {
			return nil, filer_pb.ErrNotFound
		}
		return nil, err
	}

	entry = &filer.Entry{
		FullPath: path,
	}
	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(value))
	if err != nil {
		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
	}
	return entry, nil
}

func (store *HbaseStore) DeleteEntry(ctx context.Context, path util.FullPath) (err error) {
	return store.doDelete(ctx, store.cfMetaDir, []byte(path))
}

func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.FullPath) (err error) {

	family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}}
	expectedPrefix := []byte(path + "/")
	scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family))
	if err != nil {
		return err
	}

	scanner := store.Client.Scan(scan)
	defer scanner.Close()
	for {
		res, err := scanner.Next()
		if err != nil {
			break
		}
		if len(res.Cells) == 0 {
			continue
		}
		cell := res.Cells[0]

		if !bytes.HasPrefix(cell.Row, expectedPrefix) {
			break
		}
		fullpath := util.FullPath(cell.Row)
		dir, _ := fullpath.DirAndName()
		if dir != string(dirPath) {
			continue
		}

		err = store.doDelete(ctx, store.cfMetaDir, cell.Row)
		if err != nil {
			break
		}

	}
	return
}

func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*filer.Entry, error) {
	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "")
}

func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*filer.Entry, error) {
	family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}}
	expectedPrefix := []byte(string(dirPath) + "/" + prefix)
	scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family))
	if err != nil {
		return nil, err
	}

	var entries []*filer.Entry
	scanner := store.Client.Scan(scan)
	defer scanner.Close()
	for {
		res, err := scanner.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return entries, err
		}
		if len(res.Cells) == 0 {
			continue
		}
		cell := res.Cells[0]

		if !bytes.HasPrefix(cell.Row, expectedPrefix) {
			break
		}

		fullpath := util.FullPath(cell.Row)
		dir, fileName := fullpath.DirAndName()
		if dir != string(dirPath) {
			continue
		}

		value := cell.Value

		if fileName == startFileName && !includeStartFile {
			continue
		}

		limit--
		if limit < 0 {
			break
		}
		entry := &filer.Entry{
			FullPath: fullpath,
		}
		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(value)); decodeErr != nil {
			err = decodeErr
			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
			break
		}
		entries = append(entries, entry)
	}

	return entries, nil
}

func (store *HbaseStore) BeginTransaction(ctx context.Context) (context.Context, error) {
	return ctx, nil
}

func (store *HbaseStore) CommitTransaction(ctx context.Context) error {
	return nil
}

func (store *HbaseStore) RollbackTransaction(ctx context.Context) error {
	return nil
}

func (store *HbaseStore) Shutdown() {
	store.Client.Close()
}
<|endoftext|>


<commit_before>package backend

import (
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"os"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
)

type BackendStorageFile interface {
	io.ReaderAt
	io.WriterAt
	Truncate(off int64) error
	io.Closer
	GetStat() (datSize int64, modTime time.Time, err error)
	Name() string
	Sync() error
}

type BackendStorage interface {
	ToProperties() map[string]string
	NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) BackendStorageFile
	CopyFile(f *os.File, attributes map[string]string, fn func(progressed int64, percentage float32) error) (key string, size int64, err error)
	DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error) (size int64, err error)
	DeleteFile(key string) (err error)
}

type StringProperties interface {
	GetString(key string) string
}
type StorageType string
type BackendStorageFactory interface {
	StorageType() StorageType
	BuildStorage(configuration StringProperties, configPrefix string, id string) (BackendStorage, error)
}

var (
	BackendStorageFactories = make(map[StorageType]BackendStorageFactory)
	BackendStorages         = make(map[string]BackendStorage)
)

// used by master to load remote storage configurations
func LoadConfiguration(config *util.ViperProxy) {

	StorageBackendPrefix := "storage.backend"

	for backendTypeName := range config.GetStringMap(StorageBackendPrefix) {
		backendStorageFactory, found := BackendStorageFactories[StorageType(backendTypeName)]
		if !found {
			glog.Fatalf("backend storage type %s not found", backendTypeName)
		}
		for backendStorageId := range config.GetStringMap(StorageBackendPrefix + "." + backendTypeName) {
			if !config.GetBool(StorageBackendPrefix + "." + backendTypeName + "." + backendStorageId + ".enabled") {
				continue
			}
			backendStorage, buildErr := backendStorageFactory.BuildStorage(config,
				StorageBackendPrefix+"."+backendTypeName+"."+backendStorageId+".", backendStorageId)
			if buildErr != nil {
				glog.Fatalf("fail to create backend storage %s.%s", backendTypeName, backendStorageId)
			}
			BackendStorages[backendTypeName+"."+backendStorageId] = backendStorage
			if backendStorageId == "default" {
				BackendStorages[backendTypeName] = backendStorage
			}
		}
	}

}

// used by volume server to receive remote storage configurations from master
func LoadFromPbStorageBackends(storageBackends []*master_pb.StorageBackend) {

	for _, storageBackend := range storageBackends {
		backendStorageFactory, found := BackendStorageFactories[StorageType(storageBackend.Type)]
		if !found {
			glog.Warningf("storage type %s not found", storageBackend.Type)
			continue
		}
		backendStorage, buildErr := backendStorageFactory.BuildStorage(newProperties(storageBackend.Properties), "", storageBackend.Id)
		if buildErr != nil {
			glog.Fatalf("fail to create backend storage %s.%s", storageBackend.Type, storageBackend.Id)
		}
		BackendStorages[storageBackend.Type+"."+storageBackend.Id] = backendStorage
		if storageBackend.Id == "default" {
			BackendStorages[storageBackend.Type] = backendStorage
		}
	}
}

type Properties struct {
	m map[string]string
}

func newProperties(m map[string]string) *Properties {
	return &Properties{m: m}
}

func (p *Properties) GetString(key string) string {
	if v, found := p.m[key]; found {
		return v
	}
	return ""
}

func ToPbStorageBackends() (backends []*master_pb.StorageBackend) {
	for sName, s := range BackendStorages {
		sType, sId := BackendNameToTypeId(sName)
		if sType == "" {
			continue
		}
		backends = append(backends, &master_pb.StorageBackend{
			Type:       sType,
			Id:         sId,
			Properties: s.ToProperties(),
		})
	}
	return
}

func BackendNameToTypeId(backendName string) (backendType, backendId string) {
	parts := strings.Split(backendName, ".")
	if len(parts) == 1 {
		return backendName, "default"
	}
	if len(parts) != 2 {
		return
	}

	backendType, backendId = parts[0], parts[1]
	return
}
<commit_msg>skip already loaded backends<commit_after>package backend

import (
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"os"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
)

type BackendStorageFile interface {
	io.ReaderAt
	io.WriterAt
	Truncate(off int64) error
	io.Closer
	GetStat() (datSize int64, modTime time.Time, err error)
	Name() string
	Sync() error
}

type BackendStorage interface {
	ToProperties() map[string]string
	NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) BackendStorageFile
	CopyFile(f *os.File, attributes map[string]string, fn func(progressed int64, percentage float32) error) (key string, size int64, err error)
	DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error) (size int64, err error)
	DeleteFile(key string) (err error)
}

type StringProperties interface {
	GetString(key string) string
}
type StorageType string
type BackendStorageFactory interface {
	StorageType() StorageType
	BuildStorage(configuration StringProperties, configPrefix string, id string) (BackendStorage, error)
}

var (
	BackendStorageFactories = make(map[StorageType]BackendStorageFactory)
	BackendStorages         = make(map[string]BackendStorage)
)

// used by master to load remote storage configurations
func LoadConfiguration(config *util.ViperProxy) {

	StorageBackendPrefix := "storage.backend"

	for backendTypeName := range config.GetStringMap(StorageBackendPrefix) {
		backendStorageFactory, found := BackendStorageFactories[StorageType(backendTypeName)]
		if !found {
			glog.Fatalf("backend storage type %s not found", backendTypeName)
		}
		for backendStorageId := range config.GetStringMap(StorageBackendPrefix + "." + backendTypeName) {
			if !config.GetBool(StorageBackendPrefix + "." + backendTypeName + "." + backendStorageId + ".enabled") {
				continue
			}
			if _, found := BackendStorages[backendTypeName+"."+backendStorageId]; found {
				continue
			}
			backendStorage, buildErr := backendStorageFactory.BuildStorage(config,
				StorageBackendPrefix+"."+backendTypeName+"."+backendStorageId+".", backendStorageId)
			if buildErr != nil {
				glog.Fatalf("fail to create backend storage %s.%s", backendTypeName, backendStorageId)
			}
			BackendStorages[backendTypeName+"."+backendStorageId] = backendStorage
			if backendStorageId == "default" {
				BackendStorages[backendTypeName] = backendStorage
			}
		}
	}

}

// used by volume server to receive remote storage configurations from master
func LoadFromPbStorageBackends(storageBackends []*master_pb.StorageBackend) {

	for _, storageBackend := range storageBackends {
		backendStorageFactory, found := BackendStorageFactories[StorageType(storageBackend.Type)]
		if !found {
			glog.Warningf("storage type %s not found", storageBackend.Type)
			continue
		}
		if _, found := BackendStorages[storageBackend.Type+"."+storageBackend.Id]; found {
			continue
		}
		backendStorage, buildErr := backendStorageFactory.BuildStorage(newProperties(storageBackend.Properties), "", storageBackend.Id)
		if buildErr != nil {
			glog.Fatalf("fail to create backend storage %s.%s", storageBackend.Type, storageBackend.Id)
		}
		BackendStorages[storageBackend.Type+"."+storageBackend.Id] = backendStorage
		if storageBackend.Id == "default" {
			BackendStorages[storageBackend.Type] = backendStorage
		}
	}
}

type Properties struct {
	m map[string]string
}

func newProperties(m map[string]string) *Properties {
	return &Properties{m: m}
}

func (p *Properties) GetString(key string) string {
	if v, found := p.m[key]; found {
		return v
	}
	return ""
}

func ToPbStorageBackends() (backends []*master_pb.StorageBackend) {
	for sName, s := range BackendStorages {
		sType, sId := BackendNameToTypeId(sName)
		if sType == "" {
			continue
		}
		backends = append(backends, &master_pb.StorageBackend{
			Type:       sType,
			Id:         sId,
			Properties: s.ToProperties(),
		})
	}
	return
}

func BackendNameToTypeId(backendName string) (backendType, backendId string) {
	parts := strings.Split(backendName, ".")
	if len(parts) == 1 {
		return backendName, "default"
	}
	if len(parts) != 2 {
		return
	}

	backendType, backendId = parts[0], parts[1]
	return
}
<|endoftext|>


<commit_before>package firewaller

import (
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/watcher"
	"launchpad.net/tomb"
)

// Firewaller manages the opening and closing of ports.
type Firewaller struct {
	st       *state.State
	environ  environs.Environ
	tomb     tomb.Tomb
	machines map[int]*machine
}

// NewFirewaller returns a new Firewaller.
func NewFirewaller(environ environs.Environ) (*Firewaller, error) {
	info, err := environ.StateInfo()
	if err != nil {
		return nil, err
	}
	st, err := state.Open(info)
	if err != nil {
		return nil, err
	}
	fw := &Firewaller{
		st:       st,
		environ:  environ,
		machines: make(map[int]*machine),
	}
	go fw.loop()
	return fw, nil
}

func (fw *Firewaller) loop() {
	defer fw.finish()
	// Set up channels and watchers.
	machinesWatcher := fw.st.WatchMachines()
	defer watcher.Stop(machinesWatcher, &fw.tomb)
	machineUnitsChanges := make(chan *machineUnitsChange)
	defer close(machineUnitsChanges)
	for {
		select {
		case <-fw.tomb.Dying():
			return
		case change, ok := <-machinesWatcher.Changes():
			if !ok {
				err := machinesWatcher.Stop()
				if err != nil {
					fw.tomb.Kill(watcher.MustErr(machinesWatcher))
				}
				return
			}
			for _, removedMachine := range change.Removed {
				m, ok := fw.machines[removedMachine.Id()]
				if !ok {
					panic("trying to remove unmanaged machine")
				}
				if err := m.stop(); err != nil {
					panic("can't stop machine tracker")
				}
				delete(fw.machines, removedMachine.Id())
			}
			for _, addedMachine := range change.Added {
				m := newMachine(addedMachine, fw, machineUnitsChanges)
				fw.machines[addedMachine.Id()] = m
				log.Debugf("Added machine %v", m.id)
			}
		case <-machineUnitsChanges:
			// TODO(mue) fill with life.
		}
	}
}

// finishes cleans up when the firewaller is stopping.
func (fw *Firewaller) finish() {
	for _, m := range fw.machines {
		fw.tomb.Kill(m.stop())
	}
	fw.st.Close()
	fw.tomb.Done()
}

// Wait waits for the Firewaller to exit.
func (fw *Firewaller) Wait() error {
	return fw.tomb.Wait()
}

// Stop stops the Firewaller and returns any error encountered while stopping.
func (fw *Firewaller) Stop() error {
	fw.tomb.Kill(nil)
	return fw.tomb.Wait()
}

// machineUnitsChange contains the changed units for one specific machine.
type machineUnitsChange struct {
	machine *machine
	change  *state.MachineUnitsChange
}

// machine keeps track of the unit changes of a machine.
type machine struct {
	firewaller *Firewaller
	changes    chan *machineUnitsChange
	tomb       tomb.Tomb
	id         int
	watcher    *state.MachineUnitsWatcher
	ports      map[state.Port]*unit
}

// newMachine creates a new machine to be watched for units changes.
func newMachine(mst *state.Machine, fw *Firewaller, changes chan *machineUnitsChange) *machine {
	m := &machine{
		firewaller: fw,
		changes:    changes,
		id:         mst.Id(),
		watcher:    mst.WatchUnits(),
		ports:      make(map[state.Port]*unit),
	}
	go m.loop()
	return m
}

// loop is the backend watching for machine units changes.
func (m *machine) loop() {
	defer m.tomb.Done()
	defer m.watcher.Stop()
	for {
		select {
		case <-m.firewaller.tomb.Dying():
			return
		case <-m.tomb.Dying():
			return
		case change, ok := <-m.watcher.Changes():
			select {
			case m.changes <- &machineUnitsChange{m, change}:
			case <-m.firewaller.tomb.Dying():
				return
			}
			if !ok {
				m.firewaller.tomb.Kill(watcher.MustErr(m.watcher))
				return
			}
		}
	}
}

// stop lets the machine tracker stop working.
func (m *machine) stop() error {
	m.tomb.Kill(nil)
	return m.tomb.Wait()
}

type service struct {
	exposed bool
}

type unit struct {
	svc   *service
	id    string
	ports []state.Port
}
<commit_msg>firewaller: changes after reviews and discussions on irc<commit_after>package firewaller

import (
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/watcher"
	"launchpad.net/tomb"
)

// Firewaller watches the state for ports open or closed
// and reflects those changes onto the backing environment.
type Firewaller struct {
	st       *state.State
	environ  environs.Environ
	tomb     tomb.Tomb
	machines map[int]*machineTracker
}

// NewFirewaller returns a new Firewaller.
func NewFirewaller(environ environs.Environ) (*Firewaller, error) {
	info, err := environ.StateInfo()
	if err != nil {
		return nil, err
	}
	st, err := state.Open(info)
	if err != nil {
		return nil, err
	}
	fw := &Firewaller{
		st:       st,
		environ:  environ,
		machines: make(map[int]*machineTracker),
	}
	go fw.loop()
	return fw, nil
}

func (fw *Firewaller) loop() {
	defer fw.finish()
	// Set up channels and watchers.
	machineUnitsChanges := make(chan *machineUnitsChange)
	defer close(machineUnitsChanges)
	machinesWatcher := fw.st.WatchMachines()
	defer watcher.Stop(machinesWatcher, &fw.tomb)
	for {
		select {
		case <-fw.tomb.Dying():
			return
		case change, ok := <-machinesWatcher.Changes():
			if !ok {
				return
			}
			for _, removedMachine := range change.Removed {
				m, ok := fw.machines[removedMachine.Id()]
				if !ok {
					panic("trying to remove machine that wasn't added")
				}
				if err := m.stop(); err != nil {
					log.Printf("can't stop machine tracker: %v", err)
					continue
				}
				delete(fw.machines, removedMachine.Id())
			}
			for _, addedMachine := range change.Added {
				m := newMachineTracker(addedMachine, fw, machineUnitsChanges)
				fw.machines[addedMachine.Id()] = m
				log.Debugf("Added machine %v", m.id)
			}
		case <-machineUnitsChanges:
			// TODO(mue) fill with life.
		}
	}
}

// finishes cleans up when the firewaller is stopping.
func (fw *Firewaller) finish() {
	for _, m := range fw.machines {
		fw.tomb.Kill(m.stop())
	}
	fw.st.Close()
	fw.tomb.Done()
}

// Wait waits for the Firewaller to exit.
func (fw *Firewaller) Wait() error {
	return fw.tomb.Wait()
}

// Stop stops the Firewaller and returns any error encountered while stopping.
func (fw *Firewaller) Stop() error {
	fw.tomb.Kill(nil)
	return fw.tomb.Wait()
}

// machineUnitsChange contains the changed units for one specific machine.
type machineUnitsChange struct {
	machine *machineTracker
	change  *state.MachineUnitsChange
}

// machineTracker keeps track of the unit changes of a machine.
type machineTracker struct {
	firewaller *Firewaller
	changes    chan<- *machineUnitsChange
	tomb       tomb.Tomb
	id         int
	watcher    *state.MachineUnitsWatcher
	ports      map[state.Port]*unitTracker
}

// newMachineTracker creates a new machine tracker keeping track of
// unit changes of the passed machine.
func newMachineTracker(mst *state.Machine, fw *Firewaller, changes chan<- *machineUnitsChange) *machineTracker {
	mt := &machineTracker{
		firewaller: fw,
		changes:    changes,
		id:         mst.Id(),
		watcher:    mst.WatchUnits(),
		ports:      make(map[state.Port]*unitTracker),
	}
	go mt.loop()
	return mt
}

// loop is the backend watching for machine units changes.
func (mt *machineTracker) loop() {
	defer mt.tomb.Done()
	defer mt.watcher.Stop()
	for {
		select {
		case <-mt.firewaller.tomb.Dying():
			return
		case <-mt.tomb.Dying():
			return
		case change, ok := <-mt.watcher.Changes():
			// Send change or nil.
			select {
			case mt.changes <- &machineUnitsChange{mt, change}:
			case <-mt.firewaller.tomb.Dying():
				return
			case <-mt.tomb.Dying():
				return
			}
			// Had been an error, so end the loop.
			if !ok {
				mt.firewaller.tomb.Kill(watcher.MustErr(mt.watcher))
				return
			}
		}
	}
}

// stop stops the machine tracker.
func (mt *machineTracker) stop() error {
	mt.tomb.Kill(nil)
	return mt.tomb.Wait()
}

type serviceTracker struct {
	exposed bool
}

type unitTracker struct {
	service *serviceTracker
	id      string
	ports   []state.Port
}
<|endoftext|>


<commit_before>// Package demux provides interface to Linux DVB demux device
package demux

import (
	"os"
	"syscall"
	"unsafe"
)

// Filter implements common functionality for all specific filters
type Filter struct {
	file *os.File
}

func (f Filter) Close() error {
	return f.file.Close()
}

func (f Filter) Read(buf []byte) (int, error) {
	return f.file.Read(buf)
}

func (f Filter) Start() error {
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL, uintptr(f.file.Fd()), _DMX_START, 0,
	)
	if e != 0 {
		return e
	}
	return nil
}

func (f Filter) Stop() error {
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL, uintptr(f.file.Fd()), _DMX_STOP, 0,
	)
	if e != 0 {
		return e
	}
	return nil
}

func (f Filter) SetBufferLen(n uint32) error {
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL, uintptr(f.file.Fd()),
		_DMX_SET_BUFFER_SIZE, uintptr(n),
	)
	if e != 0 {
		return e
	}
	return nil
}

// Parameters for StreamFilter

type Input uint32

const (
	InFrontend Input = iota
	InDvr
)

type Output uint32

const (
	OutDecoder Output = iota
	OutTap
	OutTSTap
	OutTSDemuxTap
)

type StreamType uint32

const (
	Audi StreamType = iota
	Video
	Teletext
	Subtitle
	PCR
)

const (
	Audio0 StreamType = iota
	Video0
	Teletext0
	Subtitle0
	PCR0

	Audio1
	Video1
	Teletext1
	Subtitle1
	PCR1

	Audio2
	Video2
	Teletext2
	Subtitle2
	PCR2

	Audio3
	Video3
	Teletext3
	Subtitle3
	PCR3

	Other
)

type Flags uint32

const (
	CheckCRC Flags = 1 << iota
	Oneshot
	ImmediateStart

	KernelClient Flags = 0x8000
)

type StreamFilterParam struct {
	Pid   int16
	In    Input
	Out   Output
	Type  StreamType
	Flags Flags
}

// StreamFilter represents PES filter configured in Linux kernel
type StreamFilter struct {
	Filter
}

func (f StreamFilter) AddPid(pid int16) error {
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL,
		uintptr(f.file.Fd()),
		_DMX_ADD_PID,
		uintptr(unsafe.Pointer(&pid)),
	)
	if e != 0 {
		return e
	}
	return nil
}

func (f StreamFilter) RemovePid(pid int16) error {
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL,
		uintptr(f.file.Fd()),
		_DMX_REMOVE_PID,
		uintptr(unsafe.Pointer(&pid)),
	)
	if e != 0 {
		return e
	}
	return nil
}

// Parameters for SectionFilter

type Pattern struct {
	Bits [16]byte
	Mask [16]byte
	Mode [16]byte
}

type SectionFilterParam struct {
	Pid     int16
	Pattern Pattern
	Timeout uint32
	Flags   Flags
}

// SectionFilter represents filter configured in Linux kernel
type SectionFilter struct {
	Filter
}

// Dev represents Linux DVB demux device
type Device string

// Returns a handler to elementary stream filter.
func (d Device) NewStreamFilter(p *StreamFilterParam) (f StreamFilter, err error) {
	f.file, err = os.Open(string(d))
	if err != nil {
		return
	}
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL,
		uintptr(f.file.Fd()),
		_DMX_SET_PES_FILTER,
		uintptr(unsafe.Pointer(p)),
	)
	if e != 0 {
		err = e
	}
	return
}

// Returns a handler to section filter.
func (d Device) NewSectionFilter(p *SectionFilterParam) (f SectionFilter, err error) {
	f.file, err = os.Open(string(d))
	if err != nil {
		return
	}
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL,
		uintptr(f.file.Fd()),
		_DMX_SET_FILTER,
		uintptr(unsafe.Pointer(p)),
	)
	if e != 0 {
		err = e
	}
	return
}
<commit_msg>demux: Change name of method RemovePid to DelPid<commit_after>// Package demux provides interface to Linux DVB demux device
package demux

import (
	"os"
	"syscall"
	"unsafe"
)

// Filter implements common functionality for all specific filters
type Filter struct {
	file *os.File
}

func (f Filter) Close() error {
	err := f.file.Close()
	f.file = nil
	return err
}

func (f Filter) Closed() bool {
	return f.file == nil
}

func (f Filter) Read(buf []byte) (int, error) {
	return f.file.Read(buf)
}

func (f Filter) Start() error {
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL, uintptr(f.file.Fd()), _DMX_START, 0,
	)
	if e != 0 {
		return e
	}
	return nil
}

func (f Filter) Stop() error {
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL, uintptr(f.file.Fd()), _DMX_STOP, 0,
	)
	if e != 0 {
		return e
	}
	return nil
}

func (f Filter) SetBufferLen(n uint32) error {
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL, uintptr(f.file.Fd()),
		_DMX_SET_BUFFER_SIZE, uintptr(n),
	)
	if e != 0 {
		return e
	}
	return nil
}

// Parameters for StreamFilter

type Input uint32

const (
	InFrontend Input = iota
	InDvr
)

type Output uint32

const (
	OutDecoder Output = iota
	OutTap
	OutTSTap
	OutTSDemuxTap
)

type StreamType uint32

const (
	Audi StreamType = iota
	Video
	Teletext
	Subtitle
	PCR
)

const (
	Audio0 StreamType = iota
	Video0
	Teletext0
	Subtitle0
	PCR0

	Audio1
	Video1
	Teletext1
	Subtitle1
	PCR1

	Audio2
	Video2
	Teletext2
	Subtitle2
	PCR2

	Audio3
	Video3
	Teletext3
	Subtitle3
	PCR3

	Other
)

type Flags uint32

const (
	CheckCRC Flags = 1 << iota
	Oneshot
	ImmediateStart

	KernelClient Flags = 0x8000
)

type StreamFilterParam struct {
	Pid   int16
	In    Input
	Out   Output
	Type  StreamType
	Flags Flags
}

// StreamFilter represents PES filter configured in Linux kernel
type StreamFilter struct {
	Filter
}

func (f StreamFilter) AddPid(pid int16) error {
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL,
		uintptr(f.file.Fd()),
		_DMX_ADD_PID,
		uintptr(unsafe.Pointer(&pid)),
	)
	if e != 0 {
		return e
	}
	return nil
}

func (f StreamFilter) DelPid(pid int16) error {
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL,
		uintptr(f.file.Fd()),
		_DMX_REMOVE_PID,
		uintptr(unsafe.Pointer(&pid)),
	)
	if e != 0 {
		return e
	}
	return nil
}

// Parameters for SectionFilter

type Pattern struct {
	Bits [16]byte
	Mask [16]byte
	Mode [16]byte
}

type SectionFilterParam struct {
	Pid     int16
	Pattern Pattern
	Timeout uint32
	Flags   Flags
}

// SectionFilter represents filter configured in Linux kernel
type SectionFilter struct {
	Filter
}

// Dev represents Linux DVB demux device
type Device string

// Returns a handler to elementary stream filter.
func (d Device) NewStreamFilter(p *StreamFilterParam) (f StreamFilter, err error) {
	f.file, err = os.Open(string(d))
	if err != nil {
		return
	}
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL,
		uintptr(f.file.Fd()),
		_DMX_SET_PES_FILTER,
		uintptr(unsafe.Pointer(p)),
	)
	if e != 0 {
		err = e
	}
	return
}

// Returns a handler to section filter.
func (d Device) NewSectionFilter(p *SectionFilterParam) (f SectionFilter, err error) {
	f.file, err = os.Open(string(d))
	if err != nil {
		return
	}
	_, _, e := syscall.Syscall(
		syscall.SYS_IOCTL,
		uintptr(f.file.Fd()),
		_DMX_SET_FILTER,
		uintptr(unsafe.Pointer(p)),
	)
	if e != 0 {
		err = e
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// +build windows

package libdokan

import (
	"time"

	"github.com/keybase/kbfs/dokan"
)

type emptyFile struct{}

func (t emptyFile) Cleanup(fi *dokan.FileInfo) {
}
func (t emptyFile) CloseFile(*dokan.FileInfo) {
}
func (t emptyFile) SetEndOfFile(fi *dokan.FileInfo, length int64) error {
	return dokan.ErrNotSupported
}
func (t emptyFile) ReadFile(fi *dokan.FileInfo, bs []byte, offset int64) (int, error) {
	return 0, dokan.ErrNotSupported
}
func (t emptyFile) WriteFile(fi *dokan.FileInfo, bs []byte, offset int64) (int, error) {
	return 0, dokan.ErrNotSupported
}
func (t emptyFile) FlushFileBuffers(*dokan.FileInfo) error {
	return dokan.ErrNotSupported
}
func (t emptyFile) FindFiles(*dokan.FileInfo, func(*dokan.NamedStat) error) error {
	return dokan.ErrNotSupported
}
func (t emptyFile) SetFileTime(*dokan.FileInfo, time.Time, time.Time, time.Time) error {
	return dokan.ErrNotSupported
}
func (t emptyFile) SetFileAttributes(fi *dokan.FileInfo, fileAttributes uint32) error {
	return dokan.ErrNotSupported
}
func (t emptyFile) LockFile(fi *dokan.FileInfo, offset int64, length int64) error {
	return dokan.ErrNotSupported
}
func (t emptyFile) UnlockFile(fi *dokan.FileInfo, offset int64, length int64) error {
	return dokan.ErrNotSupported
}
func (t emptyFile) CanDeleteFile(*dokan.FileInfo) error {
	return nil
}
func (t emptyFile) CanDeleteDirectory(*dokan.FileInfo) error {
	return nil
}
<commit_msg>libdokan: return ErrAccessDenied by default<commit_after>// Copyright 2015 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// +build windows

package libdokan

import (
	"time"

	"github.com/keybase/kbfs/dokan"
)

type emptyFile struct{}

func (t emptyFile) Cleanup(fi *dokan.FileInfo) {
}
func (t emptyFile) CloseFile(*dokan.FileInfo) {
}
func (t emptyFile) SetEndOfFile(fi *dokan.FileInfo, length int64) error {
	return dokan.ErrAccessDenied

}
func (t emptyFile) ReadFile(fi *dokan.FileInfo, bs []byte, offset int64) (int, error) {
	return 0, dokan.ErrAccessDenied
}
func (t emptyFile) WriteFile(fi *dokan.FileInfo, bs []byte, offset int64) (int, error) {
	return 0, dokan.ErrAccessDenied
}
func (t emptyFile) FlushFileBuffers(*dokan.FileInfo) error {
	return dokan.ErrAccessDenied
}
func (t emptyFile) FindFiles(*dokan.FileInfo, func(*dokan.NamedStat) error) error {
	return dokan.ErrAccessDenied
}
func (t emptyFile) SetFileTime(*dokan.FileInfo, time.Time, time.Time, time.Time) error {
	return dokan.ErrAccessDenied
}
func (t emptyFile) SetFileAttributes(fi *dokan.FileInfo, fileAttributes uint32) error {
	return dokan.ErrAccessDenied
}
func (t emptyFile) LockFile(fi *dokan.FileInfo, offset int64, length int64) error {
	return dokan.ErrAccessDenied
}
func (t emptyFile) UnlockFile(fi *dokan.FileInfo, offset int64, length int64) error {
	return dokan.ErrAccessDenied
}
func (t emptyFile) CanDeleteFile(*dokan.FileInfo) error {
	return dokan.ErrAccessDenied
}
func (t emptyFile) CanDeleteDirectory(*dokan.FileInfo) error {
	return dokan.ErrAccessDenied
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	_ "github.com/eaciit/dbox/dbc/mongo"
	_ "github.com/eaciit/dbox/dbc/mssql"
	"github.com/eaciit/orm"
	. "github.com/eaciit/powerplant/sec/consoleapp/controllers"
	// . "github.com/eaciit/powerplant/sec/consoleapp/models"
	tk "github.com/eaciit/toolkit"
)

var (
	wd = func() string {
		d, _ := os.Getwd()
		return d + "/"
	}()
)

func main() {
	tk.Println("Starting the app..")
	mongo, e := PrepareConnection("mongo")
	if e != nil {
		tk.Println(e)
	}

	sql, e := PrepareConnection("mssql")
	if e != nil {
		tk.Println(e)
	}

	base := new(BaseController)
	base.MongoCtx = orm.New(mongo)
	base.SqlCtx = orm.New(sql)

	defer base.MongoCtx.Close()
	defer base.SqlCtx.Close()

	// convert(new(MasterUnitNoTurbineParent), base) //		done
	// convert(new(MasterFailureCode), base) //				done
	// convert(new(WOList), base) // 						done
	// convert(new(AnomaliesWOList), base) //				done
	// convert(new(Availability), base) //					done
	// convert(new(Consolidated), base) // 					done
	// convert(new(FuelCost), base) //						done
	// convert(new(FuelTransport), base) // 				done
	// convert(new(FunctionalLocation), base) // 			done
	// convert(new(AnomaliesFunctionalLocation), base) //	done
	// convert(new(GenerationAppendix), base) //			done
	// convert(new(MaintenanceCost), base) //				done
	// convert(new(MaintenanceCostFL), base) //				done
	// convert(new(MaintenanceCostByHour), base) //			done
	// convert(new(MaintenancePlan), base) // 				done
	// convert(new(MaintenanceWorkOrder), base) // 			done
	// convert(new(MappedEquipmentType), base) // 			done
	// convert(new(MasterEquipmentType), base) // 			done
	// convert(new(MasterMROElement), base) // 				done
	// convert(new(MasterOrderType), base) // 				done
	// convert(new(MasterPlant), base) // 					done
	// convert(new(NewEquipmentType), base) // 				done
	// convert(new(NotificationFailure), base) // 			done
	// convert(new(NotificationFailureNoYear), base) // 	done
	// convert(new(OperationalData), base) // 				done
	// convert(new(PerformanceFactors), base) // 			done
	// convert(new(PlannedMaintenance), base) // 			done
	// convert(new(PowerPlantCoordinates), base) // 		done
	// convert(new(PowerPlantInfo),base) // 				done
	// convert(new(PrevMaintenanceValueEquation), base) // 	done
	// convert(new(RPPCloseWO), base) // 					done
	// convert(new(StartupPaymentAndPenalty), base) // 		done
	// convert(new(SyntheticPM), base) // 					done
	// convert(new(UnitCost), base) // 						done
	// convert(new(Vibration), base) // 					done

	// convert(new(MORSummary), base) // 					done
	// convert(new(MORCalculationFlatSummary), base)   // 	done
	// convert(new(PreventiveCorrectiveSummary), base) //	done
	// convert(new(WODurationSummary), base) // 			done
	// convert(new(WOListSummary), base) // 				done

	// convert(new(FailureAfterPreventiveSummary), base)// 	done
	// convert(new(RegenMasterPlant), base) // 				done
	// convert(new(MasterFailureCode), base) // 			done
	// convert(new(MasterUnitNoTurbineParent), base) // 	done
	// convert(new(DataTempMaintenance), base) // 			done

	// convert(new(SummaryData), base) // 					done
	// convert(new(DataBrowser), base)

<<<<<<< HEAD
=======
	migrate := new(MigrateData)
	migrate.BaseController = base

	// migrate.DoCostSheet() // 							done
>>>>>>> refs/remotes/origin/master
}

func convert(m orm.IModel, base *BaseController) {
	e := base.ConvertMGOToSQLServer(m)
	if e != nil {
		tk.Printf("\nERROR: %v \n", e.Error())
	}
}
<commit_msg>Update main.go<commit_after>package main

import (
	"os"

	_ "github.com/eaciit/dbox/dbc/mongo"
	_ "github.com/eaciit/dbox/dbc/mssql"
	"github.com/eaciit/orm"
	. "github.com/eaciit/powerplant/sec/consoleapp/controllers"
	// . "github.com/eaciit/powerplant/sec/consoleapp/models"
	tk "github.com/eaciit/toolkit"
)

var (
	wd = func() string {
		d, _ := os.Getwd()
		return d + "/"
	}()
)

func main() {
	tk.Println("Starting the app..")
	mongo, e := PrepareConnection("mongo")
	if e != nil {
		tk.Println(e)
	}

	sql, e := PrepareConnection("mssql")
	if e != nil {
		tk.Println(e)
	}

	base := new(BaseController)
	base.MongoCtx = orm.New(mongo)
	base.SqlCtx = orm.New(sql)

	defer base.MongoCtx.Close()
	defer base.SqlCtx.Close()

	// convert(new(MasterUnitNoTurbineParent), base) //		done
	// convert(new(MasterFailureCode), base) //				done
	// convert(new(WOList), base) // 						done
	// convert(new(AnomaliesWOList), base) //				done
	// convert(new(Availability), base) //					done
	// convert(new(Consolidated), base) // 					done
	// convert(new(FuelCost), base) //						done
	// convert(new(FuelTransport), base) // 				done
	// convert(new(FunctionalLocation), base) // 			done
	// convert(new(AnomaliesFunctionalLocation), base) //	done
	// convert(new(GenerationAppendix), base) //			done
	// convert(new(MaintenanceCost), base) //				done
	// convert(new(MaintenanceCostFL), base) //				done
	// convert(new(MaintenanceCostByHour), base) //			done
	// convert(new(MaintenancePlan), base) // 				done
	// convert(new(MaintenanceWorkOrder), base) // 			done
	// convert(new(MappedEquipmentType), base) // 			done
	// convert(new(MasterEquipmentType), base) // 			done
	// convert(new(MasterMROElement), base) // 				done
	// convert(new(MasterOrderType), base) // 				done
	// convert(new(MasterPlant), base) // 					done
	// convert(new(NewEquipmentType), base) // 				done
	// convert(new(NotificationFailure), base) // 			done
	// convert(new(NotificationFailureNoYear), base) // 	done
	// convert(new(OperationalData), base) // 				done
	// convert(new(PerformanceFactors), base) // 			done
	// convert(new(PlannedMaintenance), base) // 			done
	// convert(new(PowerPlantCoordinates), base) // 		done
	// convert(new(PowerPlantInfo),base) // 				done
	// convert(new(PrevMaintenanceValueEquation), base) // 	done
	// convert(new(RPPCloseWO), base) // 					done
	// convert(new(StartupPaymentAndPenalty), base) // 		done
	// convert(new(SyntheticPM), base) // 					done
	// convert(new(UnitCost), base) // 						done
	// convert(new(Vibration), base) // 					done

	// convert(new(MORSummary), base) // 					done
	// convert(new(MORCalculationFlatSummary), base)   // 	done
	// convert(new(PreventiveCorrectiveSummary), base) //	done
	// convert(new(WODurationSummary), base) // 			done
	// convert(new(WOListSummary), base) // 				done

	// convert(new(FailureAfterPreventiveSummary), base)// 	done
	// convert(new(RegenMasterPlant), base) // 				done
	// convert(new(MasterFailureCode), base) // 			done
	// convert(new(MasterUnitNoTurbineParent), base) // 	done
	// convert(new(DataTempMaintenance), base) // 			done

	// convert(new(SummaryData), base) // 					done
	// convert(new(DataBrowser), base)

	migrate := new(MigrateData)
	migrate.BaseController = base

	// migrate.DoCostSheet() // 							done
}

func convert(m orm.IModel, base *BaseController) {
	e := base.ConvertMGOToSQLServer(m)
	if e != nil {
		tk.Printf("\nERROR: %v \n", e.Error())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"encoding/json"
	"log"
	"net/http"
	"os"
	"os/exec"
	"io/ioutil"
	"bufio"
	"runtime"
	"fmt"
	"sync"
	"path"
	"strconv"
	"strings"
)

var META_VERSION = 1
var BASE_PORT = 10000
var globalWorldBaseDir string
var globalServerMap = NewServerMap()
var globalIdSequence = NewIdSequence(0)
var globalPortMapper = NewPortMapper()

type PortMapper struct {
	portSequence int
	freePorts []int
	mutex sync.Mutex
}

func (pm *PortMapper) getPort() int {
	pm.mutex.Lock()
	defer pm.mutex.Unlock()
	if len(pm.freePorts) == 0 {
		result := pm.portSequence
		pm.portSequence++
		return result
	}
	result := pm.freePorts[0]
	pm.freePorts = pm.freePorts[1:]
	return result
}

func (pm *PortMapper) freePort(port int) {
	pm.mutex.Lock()
	defer pm.mutex.Unlock()
	pm.freePorts = append(pm.freePorts, port)
}

func NewPortMapper() *PortMapper {
	return &PortMapper{
		freePorts: make([]int, 0, 10),
	}
}

type IdSequence struct {
	nextValue int
	mutex sync.Mutex
}

func (is *IdSequence) getId() int {
	is.mutex.Lock()
	defer is.mutex.Unlock()
	result := is.nextValue
	is.nextValue++
	return result
}

func NewIdSequence(start int) *IdSequence {
	return &IdSequence{
		nextValue: start,
	}
}

type Server struct {
	Id int
	CreatorId int
	Name string
	PortOffset int
	Handle *exec.Cmd `json:"-"`
}

func NewServer(id int, creatorId int, name string) *Server {
	if id < 0 {
		id = globalIdSequence.getId()
	}
	return &Server{
		Id: id,
		CreatorId: creatorId,
		Name: strings.TrimSpace(name),
		PortOffset: globalPortMapper.getPort(),
	}
}

type ServerMap struct {
	servers map[string]*Server
	mutex   sync.Mutex
}

func (sm *ServerMap) Put(server *Server) {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	sm.servers[str(server.Id)] = server
}

func (sm *ServerMap) Get(id int) *Server {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	return sm.servers[str(id)]
}

func (sm *ServerMap) Remove(id int) {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	server := sm.servers[str(id)]
	delete(sm.servers, str(id))
	err := server.Handle.Process.Signal(os.Interrupt)
	if err != nil {
		log.Println("Error while sending SIGINT to running server", err)
	}
	globalPortMapper.freePort(server.PortOffset)
}

func (sm *ServerMap) Encode(w http.ResponseWriter) error {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	err := json.NewEncoder(w).Encode(sm.servers)
	return err
}

func NewServerMap() *ServerMap {
	return &ServerMap{
		servers: make(map[string]*Server),
	}
}

func runServer(server *Server) {
	app := "./server"

	arg0 := "-client"
	arg1 := "client"
	arg2 := "-world"
	arg3 := worldDir(server.Id)
	arg4 := "-host"
	arg5 := ":" + str(BASE_PORT+server.PortOffset)

	cmd := exec.Command(app, arg0, arg1, arg2, arg3, arg4, arg5)
	server.Handle = cmd
	globalServerMap.Put(server)
	err := cmd.Run()

	if err != nil {
		log.Println(err.Error())
	}
}

type ApiGeneric struct {
	ServerId int
}

type ApiCreate struct {
	CreatorId int
	ServerName string
}

type ServerListResponse struct {
	List map[string]Server
}

func parseGenericRequest(r *http.Request) (ApiGeneric, error) {
	var result ApiGeneric
	err := json.NewDecoder(r.Body).Decode(&result)
	if err != nil {
		log.Println("Error while parsing generic request")
		return result, err
	}
	return result, nil
}

func getHandler(w http.ResponseWriter, r *http.Request) {
	request, err := parseGenericRequest(r)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	server := globalServerMap.Get(request.ServerId)
	if server == nil {
		http.NotFound(w, r)
		return
	}

	err = json.NewEncoder(w).Encode(server)
	if err != nil {
		log.Println("Error marshalling server")
		http.Error(w, err.Error(), 500)
	}
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	err := globalServerMap.Encode(w)
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
}

func createHandler(w http.ResponseWriter, r *http.Request) {
	var request ApiCreate
	err := json.NewDecoder(r.Body).Decode(&request)
	if err != nil {
		log.Println("Error parsing create request", err)
		http.Error(w, err.Error(), 500)
	}

	server := NewServer(-1, request.CreatorId, request.ServerName)
	saveServer(server)
	go runServer(server)
}

func deleteHandler(w http.ResponseWriter, r *http.Request) {
	request, err := parseGenericRequest(r)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	globalServerMap.Remove(request.ServerId)
}

func saveServer(server *Server) {
	file, err := os.Create(path.Join(worldDir(server.Id), "meta.server"))
	if err != nil {
		log.Println("Error creating file to save server", err)
	}
	defer file.Close()

	meta := "version:" + str(META_VERSION) + "\n" +
			"id:" + str(server.Id) + "\n" +
			"creatorId:" + str(server.CreatorId) + "\n" +
			"name:" + server.Name + "\n"

	writer := bufio.NewWriter(file)
	_, err = writer.WriteString(meta)
	if err != nil {
		log.Println("Error writing meta data.", err)
	}
}

func loadServers() {
	files, err := ioutil.ReadDir(globalWorldBaseDir)
	if err != nil {
		log.Println("Error reading base world directory")
		return
	}

	for _, fileInfo := range files {
		loadServer(fileInfo)
	}
}

func loadServer(fileInfo os.FileInfo) {
	if !fileInfo.IsDir() {
		return
	}

	file, err := os.Open(path.Join(globalWorldBaseDir, fileInfo.Name(), "meta.server"))
	if err != nil {
		log.Println("Error opening server meta data", err)
		return
	}
	defer file.Close()

	var version int
	id := -1
	creatorId := -1
	var name string

	versionOkay := false

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		option := strings.SplitN(line, ":", 1)
		if len(option) != 2 {
			log.Println("Malformed server meta data", line)
			continue
		}

		name := strings.TrimSpace(option[0])
		val := strings.TrimSpace(option[1])

		switch name {
		case "version":
			version, err = strconv.Atoi(val)
			if err != nil {
				log.Println("Error, invalid version format:", line)
				return
			}
			versionOkay = version == META_VERSION
			if !versionOkay {
				log.Println("Error, wrong meta version. Expected", META_VERSION, "got", version)
				return
			}
		case "id":
			if !versionOkay {
				log.Println("Version must be the first line in the meta data")
			}
			id, err = strconv.Atoi(val)
			if err != nil {
				log.Println("Error, invalid id format:", line)
			}
		case "creatorId":
            if !versionOkay {
                log.Println("Version must be the first line in the meta data")
            }
            creatorId, err = strconv.Atoi(val)
            if err != nil {
                log.Println("Error, invalid creatorId format:", line)
            }
		case "name":
            if !versionOkay {
                log.Println("Version must be the first line in the meta data")
            }
			name = val
		default:
			log.Println("Unknown option. You sure your version info is correct?", line)
		}
	}

	if id < 0 || creatorId < 0 || len(name) == 0 {
		log.Println("Invalid server meta data:", fileInfo.Name())
	}

	server := NewServer(id, creatorId, name)
	go runServer(server)
}

func main() {
	worldFolder := flag.String("worlds", "worlds/", "Sets the base folder used to store the worlds.")
	flag.Parse()
	globalWorldBaseDir = *worldFolder

	loadServers()

	runtime.GOMAXPROCS(runtime.NumCPU())

	http.HandleFunc("/get", getHandler)
	http.HandleFunc("/list", listHandler)
	http.HandleFunc("/create", createHandler)
	http.HandleFunc("/delete", deleteHandler)
	err := http.ListenAndServe(":3001", nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}

// Utility function
func str(i int) string {
	return fmt.Sprintf("%d", i)
}

func worldDir(serverId int) string {
	return path.Join(globalWorldBaseDir, "world" + str(serverId))
}
<commit_msg>Test and fix the bugs for tracking server metadata<commit_after>package main

import (
	"flag"
	"encoding/json"
	"log"
	"net/http"
	"os"
	"os/exec"
	"io/ioutil"
	"bufio"
	"runtime"
	"fmt"
	"sync"
	"path"
	"strconv"
	"strings"
)

var META_VERSION = 1
var BASE_PORT = 10000
var globalWorldBaseDir string
var globalServerMap = NewServerMap()
var globalIdSequence = NewIdSequence(0)
var globalPortMapper = NewPortMapper()

type PortMapper struct {
	portSequence int
	freePorts []int
	mutex sync.Mutex
}

func (pm *PortMapper) getPort() int {
	pm.mutex.Lock()
	defer pm.mutex.Unlock()
	if len(pm.freePorts) == 0 {
		result := pm.portSequence
		pm.portSequence++
		return result
	}
	result := pm.freePorts[len(pm.freePorts)-1]
	pm.freePorts = pm.freePorts[:len(pm.freePorts)-1]
	return result
}

func (pm *PortMapper) freePort(port int) {
	pm.mutex.Lock()
	defer pm.mutex.Unlock()
	pm.freePorts = append(pm.freePorts, port)
}

func NewPortMapper() *PortMapper {
	return &PortMapper{
		freePorts: make([]int, 0, 10),
	}
}

type IdSequence struct {
	nextValue int
	mutex sync.Mutex
}

func (is *IdSequence) getId() int {
	is.mutex.Lock()
	defer is.mutex.Unlock()
	result := is.nextValue
	is.nextValue++
	return result
}

func NewIdSequence(start int) *IdSequence {
	return &IdSequence{
		nextValue: start,
	}
}

type Server struct {
	Id int
	CreatorId int
	Name string
	PortOffset int
	Handle *exec.Cmd `json:"-"`
}

func NewServer(id int, creatorId int, name string) *Server {
	if id < 0 {
		id = globalIdSequence.getId()
	}
	return &Server{
		Id: id,
		CreatorId: creatorId,
		Name: strings.TrimSpace(name),
		PortOffset: globalPortMapper.getPort(),
	}
}

type ServerMap struct {
	servers map[string]*Server
	mutex   sync.Mutex
}

func (sm *ServerMap) Put(server *Server) {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	sm.servers[str(server.Id)] = server
}

func (sm *ServerMap) Get(id int) *Server {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	return sm.servers[str(id)]
}

func (sm *ServerMap) Remove(id int) {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	server := sm.servers[str(id)]
	delete(sm.servers, str(id))
	err := server.Handle.Process.Signal(os.Interrupt)
	if err != nil {
		log.Println("Error while sending SIGINT to running server", err)
	}
	globalPortMapper.freePort(server.PortOffset)
}

func (sm *ServerMap) Encode(w http.ResponseWriter) error {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	err := json.NewEncoder(w).Encode(sm.servers)
	return err
}

func NewServerMap() *ServerMap {
	return &ServerMap{
		servers: make(map[string]*Server),
	}
}

func runServer(server *Server) {
	app := "./server"

	arg0 := "-client"
	arg1 := "client"
	arg2 := "-world"
	arg3 := worldDir(server.Id)
	arg4 := "-host"
	arg5 := ":" + str(BASE_PORT+server.PortOffset)

	cmd := exec.Command(app, arg0, arg1, arg2, arg3, arg4, arg5)
	server.Handle = cmd
	globalServerMap.Put(server)
	err := cmd.Run()

	if err != nil {
		log.Println(err.Error())
	}
}

type ApiGeneric struct {
	ServerId int
}

type ApiCreate struct {
	CreatorId int
	ServerName string
}

type ServerListResponse struct {
	List map[string]Server
}

func parseGenericRequest(r *http.Request) (ApiGeneric, error) {
	var result ApiGeneric
	err := json.NewDecoder(r.Body).Decode(&result)
	if err != nil {
		log.Println("Error while parsing generic request")
		return result, err
	}
	return result, nil
}

func getHandler(w http.ResponseWriter, r *http.Request) {
	request, err := parseGenericRequest(r)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	server := globalServerMap.Get(request.ServerId)
	if server == nil {
		http.NotFound(w, r)
		return
	}

	err = json.NewEncoder(w).Encode(server)
	if err != nil {
		log.Println("Error marshalling server")
		http.Error(w, err.Error(), 500)
	}
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	err := globalServerMap.Encode(w)
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
}

func createHandler(w http.ResponseWriter, r *http.Request) {
	var request ApiCreate
	err := json.NewDecoder(r.Body).Decode(&request)
	if err != nil {
		log.Println("Error parsing create request", err)
		http.Error(w, err.Error(), 500)
	}

	server := NewServer(-1, request.CreatorId, request.ServerName)
	saveServer(server)
	go runServer(server)
}

func deleteHandler(w http.ResponseWriter, r *http.Request) {
	request, err := parseGenericRequest(r)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	globalServerMap.Remove(request.ServerId)
}

func saveServer(server *Server) {
	err := os.MkdirAll(worldDir(server.Id), 0755)
	if err != nil {
		log.Println("Error creating required directories:", err)
		return
	}

	file, err := os.Create(path.Join(worldDir(server.Id), "meta.server"))
	if err != nil {
		log.Println("Error creating file to save server:", err)
		return
	}
	defer file.Close()

	meta := "version:" + str(META_VERSION) + "\n" +
			"id:" + str(server.Id) + "\n" +
			"creatorId:" + str(server.CreatorId) + "\n" +
			"name:" + server.Name + "\n"

	writer := bufio.NewWriter(file)
	_, err = writer.WriteString(meta)
	if err != nil {
		log.Println("Error writing meta data.", err)
	}
	writer.Flush()
}

func loadServers() {
	files, err := ioutil.ReadDir(globalWorldBaseDir)
	if err != nil {
		log.Println("Error reading base world directory")
		return
	}

	for _, fileInfo := range files {
		loadServer(fileInfo)
	}
}

func loadServer(fileInfo os.FileInfo) {
	if !fileInfo.IsDir() {
		return
	}

	file, err := os.Open(path.Join(globalWorldBaseDir, fileInfo.Name(), "meta.server"))
	if err != nil {
		log.Println("Error opening server meta data:", err)
		return
	}
	defer file.Close()

	var version int
	id := -1
	creatorId := -1
	var name string

	versionOkay := false

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		components := strings.SplitN(line, ":", 2)
		if len(components) != 2 {
			log.Println("Malformed server meta data(", len(components), "):", line)
			continue
		}

		option := strings.TrimSpace(components[0])
		val := strings.TrimSpace(components[1])

		switch option {
		case "version":
			version, err = strconv.Atoi(val)
			if err != nil {
				log.Println("Error, invalid version format:", line)
				return
			}
			versionOkay = version == META_VERSION
			if !versionOkay {
				log.Println("Error, wrong meta version. Expected", META_VERSION, "got", version)
				return
			}
		case "id":
			if !versionOkay {
				log.Println("Version must be the first line in the meta data")
			}
			id, err = strconv.Atoi(val)
			if err != nil {
				log.Println("Error, invalid id format:", line)
			}
		case "creatorId":
            if !versionOkay {
                log.Println("Version must be the first line in the meta data")
            }
            creatorId, err = strconv.Atoi(val)
            if err != nil {
                log.Println("Error, invalid creatorId format:", line)
            }
		case "name":
            if !versionOkay {
                log.Println("Version must be the first line in the meta data")
            }
			name = val
		default:
			log.Println("Unknown option. You sure your version info is correct?", line)
		}
	}

	if id < 0 || creatorId < 0 || len(name) == 0 {
		log.Println("Invalid server meta data:", "id(", id, ") creatorId(", creatorId, ") name(", name, ")", fileInfo.Name())
	}

	server := NewServer(id, creatorId, name)
	go runServer(server)
}

func main() {
	worldFolder := flag.String("worlds", "worlds/", "Sets the base folder used to store the worlds.")
	flag.Parse()
	globalWorldBaseDir = *worldFolder

	loadServers()

	runtime.GOMAXPROCS(runtime.NumCPU())

	http.HandleFunc("/get", getHandler)
	http.HandleFunc("/list", listHandler)
	http.HandleFunc("/create", createHandler)
	http.HandleFunc("/delete", deleteHandler)
	err := http.ListenAndServe(":3001", nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}

// Utility function
func str(i int) string {
	return fmt.Sprintf("%d", i)
}

func worldDir(serverId int) string {
	return path.Join(globalWorldBaseDir, "world" + str(serverId))
}
<|endoftext|>