<commit_before>// Copyright 2013 Alexandre Fiori
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package diam

import (
	"bytes"
	"encoding/binary"
	"errors"
)

// pad4 returns n padded to 4 bytes
func pad4(n uint32) uint32 {
	return n + ((4 - n) & 3)
}

// uint24To32 converts b from [3]uint8 to uint32 in network byte order.
func uint24To32(b [3]uint8) uint32 {
	return uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2])
}

// uint32To24 converts b from uint32 to [3]uint8 in network byte order.
func uint32To24(b uint32) [3]uint8 {
	var r [3]uint8
	r[0] = uint8(b >> 16)
	r[1] = uint8(b >> 8)
	r[2] = uint8(b)
	return r
}

var invalid = errors.New("Invalid type for conversion")

// nToNetBytes converts any numeric type to byte array in network byte order.
func nToNetBytes(v interface{}) ([]byte, error) {
	buf := bytes.NewBuffer(nil)
	switch v.(type) {
	case int32:
		binary.Write(buf, binary.BigEndian, v.(int32))
	case int64:
		binary.Write(buf, binary.BigEndian, v.(int64))
	case uint32:
		binary.Write(buf, binary.BigEndian, v.(uint32))
	case uint64:
		binary.Write(buf, binary.BigEndian, v.(uint64))
	case float32:
		binary.Write(buf, binary.BigEndian, v.(float32))
	case float64:
		binary.Write(buf, binary.BigEndian, v.(float64))
	default:
		return nil, invalid
	}
	return buf.Bytes(), nil
}

// netBytesToN converts byte array in network byte order to any numeric type.
func netBytesToN(b []byte, v interface{}) error {
	switch v.(type) {
	case *int32:
		binary.Read(bytes.NewBuffer(b), binary.BigEndian, v.(*int32))
	case *int64:
		binary.Read(bytes.NewBuffer(b), binary.BigEndian, v.(*int64))
	case *uint32:
		binary.Read(bytes.NewBuffer(b), binary.BigEndian, v.(*uint32))
	case *uint64:
		binary.Read(bytes.NewBuffer(b), binary.BigEndian, v.(*uint64))
	case *float32:
		binary.Read(bytes.NewBuffer(b), binary.BigEndian, v.(*float32))
	case *float64:
		binary.Read(bytes.NewBuffer(b), binary.BigEndian, v.(*float64))
	default:
		return invalid
	}
	return nil
}
<commit_msg>Review<commit_after>// Copyright 2013 Alexandre Fiori
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Helper functions for type conversion between diameter data types.

package diam

import (
	"bytes"
	"encoding/binary"
	"errors"
)

// pad4 returns n padded to 4 bytes
func pad4(n uint32) uint32 {
	return n + ((4 - n) & 3)
}

// uint24To32 converts b from [3]uint8 to uint32 in network byte order.
func uint24To32(b [3]uint8) uint32 {
	return uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2])
}

// uint32To24 converts b from uint32 to [3]uint8 in network byte order.
func uint32To24(b uint32) [3]uint8 {
	var r [3]uint8
	r[0] = uint8(b >> 16)
	r[1] = uint8(b >> 8)
	r[2] = uint8(b)
	return r
}

var invalidType = errors.New("invalidType type for conversion")

// nToNetBytes converts any numeric type to byte array in network byte order.
func nToNetBytes(v interface{}) ([]byte, error) {
	buf := bytes.NewBuffer(nil)
	switch v.(type) {
	case int32:
		binary.Write(buf, binary.BigEndian, v.(int32))
	case int64:
		binary.Write(buf, binary.BigEndian, v.(int64))
	case uint32:
		binary.Write(buf, binary.BigEndian, v.(uint32))
	case uint64:
		binary.Write(buf, binary.BigEndian, v.(uint64))
	case float32:
		binary.Write(buf, binary.BigEndian, v.(float32))
	case float64:
		binary.Write(buf, binary.BigEndian, v.(float64))
	default:
		return nil, invalidType
	}
	return buf.Bytes(), nil
}

// netBytesToN converts byte array in network byte order to any numeric type.
func netBytesToN(b []byte, v interface{}) error {
	switch v.(type) {
	case *int32:
		binary.Read(bytes.NewBuffer(b), binary.BigEndian, v.(*int32))
	case *int64:
		binary.Read(bytes.NewBuffer(b), binary.BigEndian, v.(*int64))
	case *uint32:
		binary.Read(bytes.NewBuffer(b), binary.BigEndian, v.(*uint32))
	case *uint64:
		binary.Read(bytes.NewBuffer(b), binary.BigEndian, v.(*uint64))
	case *float32:
		binary.Read(bytes.NewBuffer(b), binary.BigEndian, v.(*float32))
	case *float64:
		binary.Read(bytes.NewBuffer(b), binary.BigEndian, v.(*float64))
	default:
		return invalidType
	}
	return nil
}
<|endoftext|>


<commit_before>package evedata

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/jmcvetta/napping"
)

var stations map[int64]int64

func goEMDRCrestBridge(c *AppContext) {

	type marketRegions struct {
		RegionID   int64  `db:"regionID"`
		RegionName string `db:"regionName"`
	}

	type marketTypes struct {
		TypeID   int64  `db:"typeID"`
		TypeName string `db:"typeName"`
	}

	regions := []marketRegion{}
	err := c.Db.Select(&regions, `
		SELECT 	regionID, regionName
		FROM 	mapRegions
		WHERE 	regionID < 11000000
			AND regionID NOT IN(10000001, 10000017, 10000019, 10000004);
	`)
	if err != nil {
		log.Fatal("EMDRCrestBridge:", err)
	}
	log.Printf("EMDRCrestBridge: Loaded %d Regions", len(regions))

	types := []marketTypes{}
	err = c.Db.Select(&types, `
		SELECT 	typeID, typeName
		FROM 	invTypes
		WHERE 	marketGroupID IS NOT NULL
			AND typeID < 250000;
	`)
	if err != nil {
		log.Fatal("EMDRCrestBridge:", err)
	}
	log.Printf("EMDRCrestBridge: Loaded %d items", len(types))

	stations = make(map[int64]int64)
	rows, err := c.Db.Query(`
		SELECT stationID, solarSystemID
		FROM staStations;
	`)
	for rows.Next() {

		var stationID int64
		var systemID int64

		if err := rows.Scan(&stationID, &systemID); err != nil {
			log.Fatal("EMDRCrestBridge: ", err)
		}
		stations[stationID] = systemID
	}
	rows.Close()

	if err != nil {
		log.Fatal("EMDRCrestBridge: ", err)
	}
	log.Printf("EMDRCrestBridge: Loaded %d stations", len(stations))

	// Throttle Crest Requests
	rate := time.Second / 8
	throttle := time.Tick(rate)

	// semaphore to prevent runaways
	sem := make(chan bool, c.Conf.EMDRCrestBridge.MaxGoRoutines)

	// CREST Session
	crest := napping.Session{}

	for {
		// loop through all regions
		for _, r := range regions {
			// and each item per region
			for _, t := range types {
				<-throttle // impliment throttle

				sem <- true
				go func() {
					defer func() { <-sem }()
					// Process Market History
					h := marketHistory{}
					url := fmt.Sprintf("https://public-crest.eveonline.com/market/%d/types/%d/history/", r.RegionID, t.TypeID)
					response, err := crest.Get(url, nil, &h, nil)
					if err != nil {
						log.Printf("EMDRCrestBridge: %s %+v", err, response.RawText)
						return
					}
					if response.Status() == 200 {
						if len(h.Items) > 0 {
							go postHistory(h, c, t.TypeID, r.RegionID)
						}
					}
				}()

				sem <- true
				go func() {
					defer func() { <-sem }()
					// Process Market Buy Orders
					b := marketOrders{}
					url := fmt.Sprintf("https://public-crest.eveonline.com/market/%d/orders/buy/?type=https://public-crest.eveonline.com/types/%d/", r.RegionID, t.TypeID)
					response, err := crest.Get(url, nil, &b, nil)
					if err != nil {
						log.Printf("EMDRCrestBridge: %s %+v", err, response.RawText)
						return
					}
					if response.Status() == 200 {
						if len(b.Items) > 0 {
							go postOrders(b, c, 1, t.TypeID, r.RegionID)
						}
					}
				}()

				sem <- true
				go func() {
					defer func() { <-sem }()
					// Process Market Sell Orders
					s := marketOrders{}
					url := fmt.Sprintf("https://public-crest.eveonline.com/market/%d/orders/sell/?type=https://public-crest.eveonline.com/types/%d/", r.RegionID, t.TypeID)
					response, err := crest.Get(url, nil, &s, nil)
					if err != nil {
						log.Printf("EMDRCrestBridge: %s %+v", err, response.RawText)
						return
					}
					if response.Status() == 200 {
						if len(s.Items) > 0 {

							go postOrders(s, c, 0, t.TypeID, r.RegionID)
						}
					}
				}()
			}
		}
	}
}

func postHistory(h marketHistory, c *AppContext, typeID int64, regionID int64) {
	if c.Conf.EMDRCrestBridge.Import {
		historyUpdate, err := c.Db.Prepare(`
			INSERT IGNORE INTO market_history
				(date, low, high, mean, quantity, orders, itemID, regionID)
				VALUES(?,?,?,?,?,?,?,?);
		`)
		defer historyUpdate.Close()
		if err != nil {
			log.Printf("EMDRCrestBridge: %s", err)
		} else {
			tx, _ := c.Db.Begin()
			for _, e := range h.Items {
				tx.Stmt(historyUpdate).Exec(e.Date, e.LowPrice, e.HighPrice, e.AvgPrice, typeID, regionID)
			}
			tx.Commit()
		}
	}

	if c.Conf.EMDRCrestBridge.Upload {
		u := newUUDIFHeader()
		u.ResultType = "history"
		u.Columns = []string{"date", "orders", "quantity", "low", "high", "average"}

		u.Rowsets = make([]rowsetsUUDIF, 1)

		u.Rowsets[0].RegionID = regionID
		u.Rowsets[0].TypeID = typeID
		u.Rowsets[0].GeneratedAt = time.Now()

		u.Rowsets[0].Rows = make([][]interface{}, len(h.Items))

		for i, e := range h.Items {
			u.Rowsets[0].Rows[i] = make([]interface{}, 6)
			u.Rowsets[0].Rows[i][0] = e.Date + "+00:00"
			u.Rowsets[0].Rows[i][1] = e.OrderCount
			u.Rowsets[0].Rows[i][2] = e.Volume
			u.Rowsets[0].Rows[i][3] = e.LowPrice
			u.Rowsets[0].Rows[i][4] = e.HighPrice
			u.Rowsets[0].Rows[i][5] = e.AvgPrice
		}

		enc, err := json.Marshal(u)
		if err != nil {
			log.Println("EMDRCrestBridge:", err)
		} else {
			postUUDIF(c.Conf.EMDRCrestBridge.URL, enc)
		}
	}
}

func postOrders(o marketOrders, c *AppContext, buy int, typeID int64, regionID int64) {

	if c.Conf.EMDRCrestBridge.Import {
		orderUpdate, err := c.Db.Prepare(`
					INSERT INTO market
						(orderID, price, remainingVolume, typeID, enteredVolume, minVolume, bid, issued, duration, stationID, regionID, systemID, reported)
						VALUES(?,?,?,?,?,?,?,?,?,?,?,?,NOW())
						ON DUPLICATE KEY UPDATE price=VALUES(price),
												remainingVolume=VALUES(remainingVolume),
												issued=VALUES(issued),
												duration=VALUES(duration),
												reported=VALUES(reported),
												done=0;
				`)
		defer orderUpdate.Close()

		orderMark, err := c.Db.Prepare(`
					UPDATE market SET done = 1 WHERE regionID = ? AND typeID =?
				`)
		defer orderMark.Close()

		if err != nil {
			log.Fatalf("EMDRCrestBridge: %s", err)
		} else {

			// Mark orders complete
			tx, err := c.Db.Begin()
			tx.Stmt(orderMark).Exec(regionID, typeID)

			// Add or update orders
			for _, e := range o.Items {
				tx.Stmt(orderUpdate).Exec(e.ID, e.Price, e.Volume, typeID, e.VolumeEntered, e.MinVolume, buy, e.Issued, e.Duration, e.Location.ID, regionID, stations[e.Location.ID])
			}

			err = tx.Commit()

			if err != nil {
				log.Println("EMDRCrestBridge:", err)
			}
		}
	}

	if c.Conf.EMDRCrestBridge.Upload {
		u := newUUDIFHeader()
		u.ResultType = "orders"
		u.Columns = []string{"price", "volRemaining", "range", "orderID", "volEntered", "minVolume", "bid", "issueDate", "duration", "stationID", "solarSystemID"}

		u.Rowsets = make([]rowsetsUUDIF, 1)

		u.Rowsets[0].RegionID = regionID
		u.Rowsets[0].TypeID = typeID
		u.Rowsets[0].GeneratedAt = time.Now()

		u.Rowsets[0].Rows = make([][]interface{}, len(o.Items))

		for i, e := range o.Items {

			var r int
			switch {
			case e.Range == "station":
				r = -1
			case e.Range == "solarsystem":
				r = 0
			case e.Range == "region":
				r = 32767
			default:
				r, _ = strconv.Atoi(e.Range)
			}

			u.Rowsets[0].Rows[i] = make([]interface{}, 11)
			u.Rowsets[0].Rows[i][0] = e.Price
			u.Rowsets[0].Rows[i][1] = e.Volume
			u.Rowsets[0].Rows[i][2] = r
			u.Rowsets[0].Rows[i][3] = e.ID
			u.Rowsets[0].Rows[i][4] = e.VolumeEntered
			u.Rowsets[0].Rows[i][5] = e.MinVolume
			u.Rowsets[0].Rows[i][6] = e.Buy
			u.Rowsets[0].Rows[i][7] = e.Issued + "+00:00"
			u.Rowsets[0].Rows[i][8] = e.Duration
			u.Rowsets[0].Rows[i][9] = e.Location.ID
			u.Rowsets[0].Rows[i][10] = stations[e.Location.ID]
		}

		enc, err := json.Marshal(u)
		if err != nil {
			log.Println("EMDRCrestBridge:", err)
		} else {
			postUUDIF(c.Conf.EMDRCrestBridge.URL, enc)
		}
	}
}

func postUUDIF(url string, j []byte) {
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(j))
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Println("EMDRCrestBridge:", err)
	} else {
		defer resp.Body.Close()

		if resp.Status != "200 OK" {
			body, _ := ioutil.ReadAll(resp.Body)
			log.Println("EMDRCrestBridge:", string(body))
			log.Println("EMDRCrestBridge:", string(resp.Status))
		}
	}
}

func newUUDIFHeader() marketUUDIF {
	n := marketUUDIF{}

	n.Version = "0.1"

	n.Generator.Name = "EveData.Org"
	n.Generator.Version = "0.025a"

	n.UploadKeys = make([]uploadKeysUUDIF, 1)
	n.UploadKeys[0] = uploadKeysUUDIF{"EveData.Org", "TheCheeseIsBree"}

	n.CurrentTime = time.Now()

	return n
}

type rowsetsUUDIF struct {
	GeneratedAt time.Time       `json:"generatedAt"`
	RegionID    int64           `json:"regionID"`
	TypeID      int64           `json:"typeID"`
	Rows        [][]interface{} `json:"rows"`
}

type uploadKeysUUDIF struct {
	Name string `json:"name"`
	Key  string `json:"key"`
}

type marketUUDIF struct {
	ResultType string            `json:"resultType"`
	Version    string            `json:"version"`
	UploadKeys []uploadKeysUUDIF `json:"uploadKeys"`
	Generator  struct {
		Name    string `json:"name"`
		Version string `json:"version"`
	} `json:"generator"`
	Columns     []string       `json:"columns"`
	CurrentTime time.Time      `json:"currentTime"`
	Rowsets     []rowsetsUUDIF `json:"rowsets"`
}

type marketHistory struct {
	TotalCount_Str string
	Items          []struct {
		OrderCount int64
		LowPrice   float64
		HighPrice  float64
		AvgPrice   float64
		Volume     int64
		Date       string
	}
	PageCount  int64
	TotalCount int64
}

type marketOrders struct {
	Items []struct {
		Buy           bool
		Issued        string
		Price         float64
		VolumeEntered int64
		MinVolume     int64
		Volume        int64
		Range         string
		Duration      int64
		ID            int64
		Location      struct {
			ID   int64
			Name string
		}
		Type struct {
			ID   int64
			Name string
		}
	}
	PageCount  int64
	TotalCount int64
}
<commit_msg>.... we can do this...<commit_after>package evedata

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/jmcvetta/napping"
)

var stations map[int64]int64

func goEMDRCrestBridge(c *AppContext) {

	type marketRegions struct {
		RegionID   int64  `db:"regionID"`
		RegionName string `db:"regionName"`
	}

	type marketTypes struct {
		TypeID   int64  `db:"typeID"`
		TypeName string `db:"typeName"`
	}

	regions := []marketRegion{}
	err := c.Db.Select(&regions, `
		SELECT 	regionID, regionName
		FROM 	mapRegions
		WHERE 	regionID < 11000000
			AND regionID NOT IN(10000001, 10000017, 10000019, 10000004);
	`)
	if err != nil {
		log.Fatal("EMDRCrestBridge:", err)
	}
	log.Printf("EMDRCrestBridge: Loaded %d Regions", len(regions))

	types := []marketTypes{}
	err = c.Db.Select(&types, `
		SELECT 	typeID, typeName
		FROM 	invTypes
		WHERE 	marketGroupID IS NOT NULL
			AND typeID < 250000;
	`)
	if err != nil {
		log.Fatal("EMDRCrestBridge:", err)
	}
	log.Printf("EMDRCrestBridge: Loaded %d items", len(types))

	stations = make(map[int64]int64)
	rows, err := c.Db.Query(`
		SELECT stationID, solarSystemID
		FROM staStations;
	`)
	for rows.Next() {

		var stationID int64
		var systemID int64

		if err := rows.Scan(&stationID, &systemID); err != nil {
			log.Fatal("EMDRCrestBridge: ", err)
		}
		stations[stationID] = systemID
	}
	rows.Close()

	if err != nil {
		log.Fatal("EMDRCrestBridge: ", err)
	}
	log.Printf("EMDRCrestBridge: Loaded %d stations", len(stations))

	// Throttle Crest Requests
	rate := time.Second / 8
	throttle := time.Tick(rate)

	// semaphore to prevent runaways
	sem := make(chan bool, c.Conf.EMDRCrestBridge.MaxGoRoutines)

	// CREST Session
	crest := napping.Session{}

	for {
		// loop through all regions
		for _, r := range regions {
			// and each item per region
			for _, t := range types {
				<-throttle // impliment throttle

				sem <- true
				go func() {
					defer func() { <-sem }()
					// Process Market History
					h := marketHistory{}
					url := fmt.Sprintf("https://public-crest.eveonline.com/market/%d/types/%d/history/", r.RegionID, t.TypeID)
					response, err := crest.Get(url, nil, &h, nil)
					if err != nil {
						log.Printf("EMDRCrestBridge: %s %s", err, response.RawText())
						return
					}
					if response.Status() == 200 {
						if len(h.Items) > 0 {
							go postHistory(h, c, t.TypeID, r.RegionID)
						}
					}
				}()

				sem <- true
				go func() {
					defer func() { <-sem }()
					// Process Market Buy Orders
					b := marketOrders{}
					url := fmt.Sprintf("https://public-crest.eveonline.com/market/%d/orders/buy/?type=https://public-crest.eveonline.com/types/%d/", r.RegionID, t.TypeID)
					response, err := crest.Get(url, nil, &b, nil)
					if err != nil {
						log.Printf("EMDRCrestBridge: %s %s", err, response.RawText())
						return
					}
					if response.Status() == 200 {
						if len(b.Items) > 0 {
							go postOrders(b, c, 1, t.TypeID, r.RegionID)
						}
					}
				}()

				sem <- true
				go func() {
					defer func() { <-sem }()
					// Process Market Sell Orders
					s := marketOrders{}
					url := fmt.Sprintf("https://public-crest.eveonline.com/market/%d/orders/sell/?type=https://public-crest.eveonline.com/types/%d/", r.RegionID, t.TypeID)
					response, err := crest.Get(url, nil, &s, nil)
					if err != nil {
						log.Printf("EMDRCrestBridge: %s %s", err, response.RawText())
						return
					}
					if response.Status() == 200 {
						if len(s.Items) > 0 {

							go postOrders(s, c, 0, t.TypeID, r.RegionID)
						}
					}
				}()
			}
		}
	}
}

func postHistory(h marketHistory, c *AppContext, typeID int64, regionID int64) {
	if c.Conf.EMDRCrestBridge.Import {
		historyUpdate, err := c.Db.Prepare(`
			INSERT IGNORE INTO market_history
				(date, low, high, mean, quantity, orders, itemID, regionID)
				VALUES(?,?,?,?,?,?,?,?);
		`)
		defer historyUpdate.Close()
		if err != nil {
			log.Printf("EMDRCrestBridge: %s", err)
		} else {
			tx, _ := c.Db.Begin()
			for _, e := range h.Items {
				tx.Stmt(historyUpdate).Exec(e.Date, e.LowPrice, e.HighPrice, e.AvgPrice, typeID, regionID)
			}
			tx.Commit()
		}
	}

	if c.Conf.EMDRCrestBridge.Upload {
		u := newUUDIFHeader()
		u.ResultType = "history"
		u.Columns = []string{"date", "orders", "quantity", "low", "high", "average"}

		u.Rowsets = make([]rowsetsUUDIF, 1)

		u.Rowsets[0].RegionID = regionID
		u.Rowsets[0].TypeID = typeID
		u.Rowsets[0].GeneratedAt = time.Now()

		u.Rowsets[0].Rows = make([][]interface{}, len(h.Items))

		for i, e := range h.Items {
			u.Rowsets[0].Rows[i] = make([]interface{}, 6)
			u.Rowsets[0].Rows[i][0] = e.Date + "+00:00"
			u.Rowsets[0].Rows[i][1] = e.OrderCount
			u.Rowsets[0].Rows[i][2] = e.Volume
			u.Rowsets[0].Rows[i][3] = e.LowPrice
			u.Rowsets[0].Rows[i][4] = e.HighPrice
			u.Rowsets[0].Rows[i][5] = e.AvgPrice
		}

		enc, err := json.Marshal(u)
		if err != nil {
			log.Println("EMDRCrestBridge:", err)
		} else {
			postUUDIF(c.Conf.EMDRCrestBridge.URL, enc)
		}
	}
}

func postOrders(o marketOrders, c *AppContext, buy int, typeID int64, regionID int64) {

	if c.Conf.EMDRCrestBridge.Import {
		orderUpdate, err := c.Db.Prepare(`
					INSERT INTO market
						(orderID, price, remainingVolume, typeID, enteredVolume, minVolume, bid, issued, duration, stationID, regionID, systemID, reported)
						VALUES(?,?,?,?,?,?,?,?,?,?,?,?,NOW())
						ON DUPLICATE KEY UPDATE price=VALUES(price),
												remainingVolume=VALUES(remainingVolume),
												issued=VALUES(issued),
												duration=VALUES(duration),
												reported=VALUES(reported),
												done=0;
				`)
		defer orderUpdate.Close()

		orderMark, err := c.Db.Prepare(`
					UPDATE market SET done = 1 WHERE regionID = ? AND typeID =?
				`)
		defer orderMark.Close()

		if err != nil {
			log.Fatalf("EMDRCrestBridge: %s", err)
		} else {

			// Mark orders complete
			tx, err := c.Db.Begin()
			tx.Stmt(orderMark).Exec(regionID, typeID)

			// Add or update orders
			for _, e := range o.Items {
				tx.Stmt(orderUpdate).Exec(e.ID, e.Price, e.Volume, typeID, e.VolumeEntered, e.MinVolume, buy, e.Issued, e.Duration, e.Location.ID, regionID, stations[e.Location.ID])
			}

			err = tx.Commit()

			if err != nil {
				log.Println("EMDRCrestBridge:", err)
			}
		}
	}

	if c.Conf.EMDRCrestBridge.Upload {
		u := newUUDIFHeader()
		u.ResultType = "orders"
		u.Columns = []string{"price", "volRemaining", "range", "orderID", "volEntered", "minVolume", "bid", "issueDate", "duration", "stationID", "solarSystemID"}

		u.Rowsets = make([]rowsetsUUDIF, 1)

		u.Rowsets[0].RegionID = regionID
		u.Rowsets[0].TypeID = typeID
		u.Rowsets[0].GeneratedAt = time.Now()

		u.Rowsets[0].Rows = make([][]interface{}, len(o.Items))

		for i, e := range o.Items {

			var r int
			switch {
			case e.Range == "station":
				r = -1
			case e.Range == "solarsystem":
				r = 0
			case e.Range == "region":
				r = 32767
			default:
				r, _ = strconv.Atoi(e.Range)
			}

			u.Rowsets[0].Rows[i] = make([]interface{}, 11)
			u.Rowsets[0].Rows[i][0] = e.Price
			u.Rowsets[0].Rows[i][1] = e.Volume
			u.Rowsets[0].Rows[i][2] = r
			u.Rowsets[0].Rows[i][3] = e.ID
			u.Rowsets[0].Rows[i][4] = e.VolumeEntered
			u.Rowsets[0].Rows[i][5] = e.MinVolume
			u.Rowsets[0].Rows[i][6] = e.Buy
			u.Rowsets[0].Rows[i][7] = e.Issued + "+00:00"
			u.Rowsets[0].Rows[i][8] = e.Duration
			u.Rowsets[0].Rows[i][9] = e.Location.ID
			u.Rowsets[0].Rows[i][10] = stations[e.Location.ID]
		}

		enc, err := json.Marshal(u)
		if err != nil {
			log.Println("EMDRCrestBridge:", err)
		} else {
			postUUDIF(c.Conf.EMDRCrestBridge.URL, enc)
		}
	}
}

func postUUDIF(url string, j []byte) {
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(j))
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Println("EMDRCrestBridge:", err)
	} else {
		defer resp.Body.Close()

		if resp.Status != "200 OK" {
			body, _ := ioutil.ReadAll(resp.Body)
			log.Println("EMDRCrestBridge:", string(body))
			log.Println("EMDRCrestBridge:", string(resp.Status))
		}
	}
}

func newUUDIFHeader() marketUUDIF {
	n := marketUUDIF{}

	n.Version = "0.1"

	n.Generator.Name = "EveData.Org"
	n.Generator.Version = "0.025a"

	n.UploadKeys = make([]uploadKeysUUDIF, 1)
	n.UploadKeys[0] = uploadKeysUUDIF{"EveData.Org", "TheCheeseIsBree"}

	n.CurrentTime = time.Now()

	return n
}

type rowsetsUUDIF struct {
	GeneratedAt time.Time       `json:"generatedAt"`
	RegionID    int64           `json:"regionID"`
	TypeID      int64           `json:"typeID"`
	Rows        [][]interface{} `json:"rows"`
}

type uploadKeysUUDIF struct {
	Name string `json:"name"`
	Key  string `json:"key"`
}

type marketUUDIF struct {
	ResultType string            `json:"resultType"`
	Version    string            `json:"version"`
	UploadKeys []uploadKeysUUDIF `json:"uploadKeys"`
	Generator  struct {
		Name    string `json:"name"`
		Version string `json:"version"`
	} `json:"generator"`
	Columns     []string       `json:"columns"`
	CurrentTime time.Time      `json:"currentTime"`
	Rowsets     []rowsetsUUDIF `json:"rowsets"`
}

type marketHistory struct {
	TotalCount_Str string
	Items          []struct {
		OrderCount int64
		LowPrice   float64
		HighPrice  float64
		AvgPrice   float64
		Volume     int64
		Date       string
	}
	PageCount  int64
	TotalCount int64
}

type marketOrders struct {
	Items []struct {
		Buy           bool
		Issued        string
		Price         float64
		VolumeEntered int64
		MinVolume     int64
		Volume        int64
		Range         string
		Duration      int64
		ID            int64
		Location      struct {
			ID   int64
			Name string
		}
		Type struct {
			ID   int64
			Name string
		}
	}
	PageCount  int64
	TotalCount int64
}
<|endoftext|>


<commit_before>// Copyright © 2014 Steve Francia <spf@spf13.com>.
//
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.

package commands

import (
	"fmt"
	"os"
	"os/signal"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var CfgFile string

var RootCmd = &cobra.Command{
	Use:   "dagobah",
	Short: "Dagobah is an awesome planet style RSS aggregator",
	Long: `Dagobah provides planet style RSS aggregation. It
is inspired by python planet. It has a simple YAML configuration
and provides it's own webserver.`,
	Run: rootRun,
}

func rootRun(cmd *cobra.Command, args []string) {
	fmt.Println(viper.Get("feeds"))
	fmt.Println(viper.GetString("appname"))
}

func init() {
	cobra.OnInitialize(initConfig)

	viper.SetDefault("feeds", []map[string]string{{"Name": "Hacking Management", "Url": "http://spf13.com/index.xml"}})

	RootCmd.PersistentFlags().StringVar(&CfgFile, "config", "", "config file (default is $HOME/dagobah/config.yaml)")
	RootCmd.PersistentFlags().StringP("dbname", "d", "dagobah", "name of the database")
	RootCmd.PersistentFlags().Int("dbport", 27017, "port to access mongoDB")
	RootCmd.PersistentFlags().String("dbhost", "localhost", "host where mongoDB is")
	RootCmd.PersistentFlags().String("dbusername", "", "username to connect to mongoDB with")
	RootCmd.PersistentFlags().String("dbpassword", "", "password to connect to mongoDB with")

	viper.BindPFlag("dbusername", RootCmd.PersistentFlags().Lookup("dbusername"))
	viper.BindPFlag("dbpassword", RootCmd.PersistentFlags().Lookup("dbpassword"))
	viper.BindPFlag("dbhost", RootCmd.PersistentFlags().Lookup("dbhost"))
	viper.BindPFlag("dbport", RootCmd.PersistentFlags().Lookup("dbport"))
	viper.BindPFlag("dbname", RootCmd.PersistentFlags().Lookup("dbname"))
}

func initConfig() {
	if CfgFile != "" {
		viper.SetConfigFile(CfgFile)
	}
	viper.SetConfigName("config")          // name of config file (without extension)
	viper.AddConfigPath("/etc/dagobah/")   // path to look for the config file in
	viper.AddConfigPath("$HOME/.dagobah/") // call multiple times to add many search paths
	viper.ReadInConfig()
}

func addCommands() {
	RootCmd.AddCommand(fetchCmd)
	RootCmd.AddCommand(serverCmd)
}

func Execute() {
	addCommands()
	err := RootCmd.Execute()
	if err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt)
	<-sigChan
}
<commit_msg>More idiomatic if statement<commit_after>// Copyright © 2014 Steve Francia <spf@spf13.com>.
//
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.

package commands

import (
	"fmt"
	"os"
	"os/signal"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var CfgFile string

var RootCmd = &cobra.Command{
	Use:   "dagobah",
	Short: "Dagobah is an awesome planet style RSS aggregator",
	Long: `Dagobah provides planet style RSS aggregation. It
is inspired by python planet. It has a simple YAML configuration
and provides it's own webserver.`,
	Run: rootRun,
}

func rootRun(cmd *cobra.Command, args []string) {
	fmt.Println(viper.Get("feeds"))
	fmt.Println(viper.GetString("appname"))
}

func init() {
	cobra.OnInitialize(initConfig)

	viper.SetDefault("feeds", []map[string]string{{"Name": "Hacking Management", "Url": "http://spf13.com/index.xml"}})

	RootCmd.PersistentFlags().StringVar(&CfgFile, "config", "", "config file (default is $HOME/dagobah/config.yaml)")
	RootCmd.PersistentFlags().StringP("dbname", "d", "dagobah", "name of the database")
	RootCmd.PersistentFlags().Int("dbport", 27017, "port to access mongoDB")
	RootCmd.PersistentFlags().String("dbhost", "localhost", "host where mongoDB is")
	RootCmd.PersistentFlags().String("dbusername", "", "username to connect to mongoDB with")
	RootCmd.PersistentFlags().String("dbpassword", "", "password to connect to mongoDB with")

	viper.BindPFlag("dbusername", RootCmd.PersistentFlags().Lookup("dbusername"))
	viper.BindPFlag("dbpassword", RootCmd.PersistentFlags().Lookup("dbpassword"))
	viper.BindPFlag("dbhost", RootCmd.PersistentFlags().Lookup("dbhost"))
	viper.BindPFlag("dbport", RootCmd.PersistentFlags().Lookup("dbport"))
	viper.BindPFlag("dbname", RootCmd.PersistentFlags().Lookup("dbname"))
}

func initConfig() {
	if CfgFile != "" {
		viper.SetConfigFile(CfgFile)
	}
	viper.SetConfigName("config")          // name of config file (without extension)
	viper.AddConfigPath("/etc/dagobah/")   // path to look for the config file in
	viper.AddConfigPath("$HOME/.dagobah/") // call multiple times to add many search paths
	viper.ReadInConfig()
}

func addCommands() {
	RootCmd.AddCommand(fetchCmd)
	RootCmd.AddCommand(serverCmd)
}

func Execute() {
	addCommands()

	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt)
	<-sigChan
}
<|endoftext|>


<commit_before>package ratchet

import (
	"container/list"
	"context"
	"fmt"
	"sync"

	"github.com/rhansen2/ratchet/data"
)

// DataProcessor is the interface that should be implemented to perform data-related
// tasks within a Pipeline. DataProcessors are responsible for receiving, processing,
// and then sending data on to the next stage of processing.
type DataProcessor interface {
	// ProcessData will be called for each data sent from the previous stage.
	// ProcessData is called with a data.JSON instance, which is the data being received,
	// an outputChan, which is the channel to send data to, and a killChan,
	// which is a channel to send unexpected errors to (halting execution of the Pipeline).
	ProcessData(d data.JSON, outputChan chan data.JSON, killChan chan error, ctx context.Context)

	// Finish will be called after the previous stage has finished sending data,
	// and no more data will be received by this DataProcessor. Often times
	// Finish can be an empty function implementation, but sometimes it is
	// necessary to perform final data processing.
	Finish(outputChan chan data.JSON, killChan chan error, ctx context.Context)
}

// dataProcessor is a type used internally to the Pipeline management
// code, and wraps a DataProcessor instance. DataProcessor is the main
// interface that should be implemented to perform work within the data
// pipeline, and this dataProcessor type simply embeds it and adds some
// helpful channel management and other attributes.
type dataProcessor struct {
	DataProcessor
	executionStat
	concurrentDataProcessor
	chanBrancher
	chanMerger
	outputs    []DataProcessor
	inputChan  chan data.JSON
	outputChan chan data.JSON
	ctx        context.Context
}

type chanBrancher struct {
	branchOutChans []chan data.JSON
}

func (dp *dataProcessor) branchOut() {
	go func() {
		for d := range dp.outputChan {
			for _, out := range dp.branchOutChans {
				// Make a copy to ensure concurrent stages
				// can alter data as needed.
				dc := make(data.JSON, len(d))
				copy(dc, d)
				select {
				case out <- dc:
				case <-dp.ctx.Done():
					return
				}
			}
			dp.recordDataSent(d)
		}
		// Once all data is received, also close all the outputs
		for _, out := range dp.branchOutChans {
			close(out)
		}
	}()
}

type chanMerger struct {
	mergeInChans []chan data.JSON
	mergeWait    sync.WaitGroup
}

func (dp *dataProcessor) mergeIn() {
	// Start a merge goroutine for each input channel.
	mergeData := func(c chan data.JSON) {
		defer dp.mergeWait.Done()
		for {
			select {
			case d, ok := <-c:
				if !ok {
					return
				}
				select {
				case dp.inputChan <- d:
				case <-dp.ctx.Done():
					return
				}
			case <-dp.ctx.Done():
				return
			}
		}
	}
	dp.mergeWait.Add(len(dp.mergeInChans))
	for _, in := range dp.mergeInChans {
		go mergeData(in)
	}

	go func() {
		dp.mergeWait.Wait()
		close(dp.inputChan)
	}()
}

// Do takes a DataProcessor instance and returns the dataProcessor
// type that will wrap it for internal ratchet processing. The details
// of the dataProcessor wrapper type are abstracted away from the
// implementing end-user code. The "Do" function is named
// succinctly to provide a nicer syntax when creating a PipelineLayout.
// See the ratchet package documentation for code examples of creating
// a new branching pipeline layout.
func Do(processor DataProcessor) *dataProcessor {
	dp := dataProcessor{DataProcessor: processor}
	dp.outputChan = make(chan data.JSON)
	dp.inputChan = make(chan data.JSON)

	if isConcurrent(processor) {
		dp.concurrency = processor.(ConcurrentDataProcessor).Concurrency()
		dp.workThrottle = make(chan workSignal, dp.concurrency)
		dp.workList = list.New()
		dp.doneChan = make(chan bool)
		dp.inputClosed = false
	}

	return &dp
}

// Outputs should be called to specify which DataProcessor instances the current
// processor should send it's output to. See the ratchet package
// documentation for code examples and diagrams.
func (dp *dataProcessor) Outputs(processors ...DataProcessor) *dataProcessor {
	dp.outputs = processors
	return dp
}

// pass through String output to the DataProcessor
func (dp *dataProcessor) String() string {
	return fmt.Sprintf("%v", dp.DataProcessor)
}
<commit_msg>Kill leaking goroutine<commit_after>package ratchet

import (
	"container/list"
	"context"
	"fmt"
	"sync"

	"github.com/rhansen2/ratchet/data"
)

// DataProcessor is the interface that should be implemented to perform data-related
// tasks within a Pipeline. DataProcessors are responsible for receiving, processing,
// and then sending data on to the next stage of processing.
type DataProcessor interface {
	// ProcessData will be called for each data sent from the previous stage.
	// ProcessData is called with a data.JSON instance, which is the data being received,
	// an outputChan, which is the channel to send data to, and a killChan,
	// which is a channel to send unexpected errors to (halting execution of the Pipeline).
	ProcessData(d data.JSON, outputChan chan data.JSON, killChan chan error, ctx context.Context)

	// Finish will be called after the previous stage has finished sending data,
	// and no more data will be received by this DataProcessor. Often times
	// Finish can be an empty function implementation, but sometimes it is
	// necessary to perform final data processing.
	Finish(outputChan chan data.JSON, killChan chan error, ctx context.Context)
}

// dataProcessor is a type used internally to the Pipeline management
// code, and wraps a DataProcessor instance. DataProcessor is the main
// interface that should be implemented to perform work within the data
// pipeline, and this dataProcessor type simply embeds it and adds some
// helpful channel management and other attributes.
type dataProcessor struct {
	DataProcessor
	executionStat
	concurrentDataProcessor
	chanBrancher
	chanMerger
	outputs    []DataProcessor
	inputChan  chan data.JSON
	outputChan chan data.JSON
	ctx        context.Context
}

type chanBrancher struct {
	branchOutChans []chan data.JSON
}

func (dp *dataProcessor) branchOut() {
	go func() {
	processLoop:
		for {
			select {
			case d, ok := <-dp.outputChan:
				if !ok {
					break processLoop
				}
				for _, out := range dp.branchOutChans {
					// Make a copy to ensure concurrent stages
					// can alter data as needed.
					dc := make(data.JSON, len(d))
					copy(dc, d)
					select {
					case out <- dc:
					case <-dp.ctx.Done():
						return
					}
				}
				dp.recordDataSent(d)
			case <-dp.ctx.Done():
				return
			}
		}
		// Once all data is received, also close all the outputs
		for _, out := range dp.branchOutChans {
			close(out)
		}
	}()
}

type chanMerger struct {
	mergeInChans []chan data.JSON
	mergeWait    sync.WaitGroup
}

func (dp *dataProcessor) mergeIn() {
	// Start a merge goroutine for each input channel.
	mergeData := func(c chan data.JSON) {
		defer dp.mergeWait.Done()
		for {
			select {
			case d, ok := <-c:
				if !ok {
					return
				}
				select {
				case dp.inputChan <- d:
				case <-dp.ctx.Done():
					return
				}
			case <-dp.ctx.Done():
				return
			}
		}
	}
	dp.mergeWait.Add(len(dp.mergeInChans))
	for _, in := range dp.mergeInChans {
		go mergeData(in)
	}

	go func() {
		dp.mergeWait.Wait()
		close(dp.inputChan)
	}()
}

// Do takes a DataProcessor instance and returns the dataProcessor
// type that will wrap it for internal ratchet processing. The details
// of the dataProcessor wrapper type are abstracted away from the
// implementing end-user code. The "Do" function is named
// succinctly to provide a nicer syntax when creating a PipelineLayout.
// See the ratchet package documentation for code examples of creating
// a new branching pipeline layout.
func Do(processor DataProcessor) *dataProcessor {
	dp := dataProcessor{DataProcessor: processor}
	dp.outputChan = make(chan data.JSON)
	dp.inputChan = make(chan data.JSON)

	if isConcurrent(processor) {
		dp.concurrency = processor.(ConcurrentDataProcessor).Concurrency()
		dp.workThrottle = make(chan workSignal, dp.concurrency)
		dp.workList = list.New()
		dp.doneChan = make(chan bool)
		dp.inputClosed = false
	}

	return &dp
}

// Outputs should be called to specify which DataProcessor instances the current
// processor should send it's output to. See the ratchet package
// documentation for code examples and diagrams.
func (dp *dataProcessor) Outputs(processors ...DataProcessor) *dataProcessor {
	dp.outputs = processors
	return dp
}

// pass through String output to the DataProcessor
func (dp *dataProcessor) String() string {
	return fmt.Sprintf("%v", dp.DataProcessor)
}
<|endoftext|>


<commit_before>// Package sigmon simplifies os.Signal handling.
package sigmon

import (
	"os"
	"os/signal"
	"sync"
	"syscall"
)

// {Signal} constants are text representations of the handled os.Signals.
const (
	SIGHUP  Signal = "HUP"
	SIGINT  Signal = "INT"
	SIGTERM Signal = "TERM"
	SIGUSR1 Signal = "USR1"
	SIGUSR2 Signal = "USR2"
)

// SignalMonitor helps manage signal handling.
type SignalMonitor struct {
	handler func(*SignalMonitor)
	sig     Signal
	isOn    bool
	off     chan bool
	set     chan func(*SignalMonitor)
}

// Signal wraps the string type to reduce confusion when checking Sig.
type Signal string

// New takes a function and returns a SignalMonitor.  When a nil arg is
// provided, no action will be taken during signal handling.  Run must be
// called in order to begin monitoring.
func New(handler func(*SignalMonitor)) (s *SignalMonitor) {
	s = &SignalMonitor{handler: handler, off: make(chan bool),
		set: make(chan func(*SignalMonitor), 1),
	}
	return s
}

// Set allows the handler function to be added or removed.  Only the most
// recently passed function will have any relevance.
func (s *SignalMonitor) Set(handler func(*SignalMonitor)) {
	select {
	case <-s.set:
	default:
	}
	s.set <- handler
}

// Run starts signal monitoring.  If no function has been provided, no action
// will be taken during signal handling.  The os.Signal which was called will
// be stored as a string within the SignalMonitor for retrieval using GetLast.
// Stop should be called within the provided functions and is not a default
// behavior.
func (s *SignalMonitor) Run() {
	if !s.isOn {
		s.isOn = true
		wg := &sync.WaitGroup{}
		wg.Add(1)
		go func(sm *SignalMonitor) {
			h := make(chan os.Signal, 1)
			i := make(chan os.Signal, 1)
			t := make(chan os.Signal, 1)
			u1 := make(chan os.Signal, 1)
			u2 := make(chan os.Signal, 1)
			signal.Notify(h, syscall.SIGHUP)
			signal.Notify(i, syscall.SIGINT)
			signal.Notify(t, syscall.SIGTERM)
			signal.Notify(u1, syscall.SIGUSR1)
			signal.Notify(u2, syscall.SIGUSR2)
			wg.Done()

			for {
				select {
				case <-sm.off:
					return
				case f := <-sm.set:
					sm.handler = f
				case <-h:
					sm.sig = SIGHUP
					if sm.handler != nil {
						sm.handler(sm)
					}
				case <-i:
					sm.sig = SIGINT
					if sm.handler != nil {
						sm.handler(sm)
					}
				case <-t:
					sm.sig = SIGTERM
					if sm.handler != nil {
						sm.handler(sm)
					}
				case <-u1:
					sm.sig = SIGUSR1
					if sm.handler != nil {
						sm.handler(sm)
					}
				case <-u2:
					sm.sig = SIGUSR2
					if sm.handler != nil {
						sm.handler(sm)
					}
				}
			}
		}(s)
		wg.Wait()
	}
}

// Stop kills the goroutine which is monitoring signals.
func (s *SignalMonitor) Stop() {
	if s.isOn {
		s.off <- true
		s.isOn = false
	}
}

// Sig returns a string of the most recently called os.Signal.
func (s *SignalMonitor) Sig() Signal {
	return s.sig
}
<commit_msg>Moved type def and fixed comment.<commit_after>// Package sigmon simplifies os.Signal handling.
package sigmon

import (
	"os"
	"os/signal"
	"sync"
	"syscall"
)

// Signal wraps the string type to reduce confusion when checking Sig.
type Signal string

// {Signal} constants are text representations of the handled os.Signals.
const (
	SIGHUP  Signal = "HUP"
	SIGINT  Signal = "INT"
	SIGTERM Signal = "TERM"
	SIGUSR1 Signal = "USR1"
	SIGUSR2 Signal = "USR2"
)

// SignalMonitor helps manage signal handling.
type SignalMonitor struct {
	handler func(*SignalMonitor)
	sig     Signal
	isOn    bool
	off     chan bool
	set     chan func(*SignalMonitor)
}

// New takes a function and returns a SignalMonitor.  When a nil arg is
// provided, no action will be taken during signal handling.  Run must be
// called in order to begin monitoring.
func New(handler func(*SignalMonitor)) (s *SignalMonitor) {
	s = &SignalMonitor{handler: handler, off: make(chan bool),
		set: make(chan func(*SignalMonitor), 1),
	}
	return s
}

// Set allows the handler function to be added or removed.  Only the most
// recently passed function will have any relevance.
func (s *SignalMonitor) Set(handler func(*SignalMonitor)) {
	select {
	case <-s.set:
	default:
	}
	s.set <- handler
}

// Run starts signal monitoring.  If no function has been provided, no action
// will be taken during signal handling.  The os.Signal which was called will
// be stored as a string within the SignalMonitor for retrieval using Sig.
// Stop should be called within the provided functions and is not a default
// behavior.
func (s *SignalMonitor) Run() {
	if !s.isOn {
		s.isOn = true
		wg := &sync.WaitGroup{}
		wg.Add(1)
		go func(sm *SignalMonitor) {
			h := make(chan os.Signal, 1)
			i := make(chan os.Signal, 1)
			t := make(chan os.Signal, 1)
			u1 := make(chan os.Signal, 1)
			u2 := make(chan os.Signal, 1)
			signal.Notify(h, syscall.SIGHUP)
			signal.Notify(i, syscall.SIGINT)
			signal.Notify(t, syscall.SIGTERM)
			signal.Notify(u1, syscall.SIGUSR1)
			signal.Notify(u2, syscall.SIGUSR2)
			wg.Done()

			for {
				select {
				case <-sm.off:
					return
				case f := <-sm.set:
					sm.handler = f
				case <-h:
					sm.sig = SIGHUP
					if sm.handler != nil {
						sm.handler(sm)
					}
				case <-i:
					sm.sig = SIGINT
					if sm.handler != nil {
						sm.handler(sm)
					}
				case <-t:
					sm.sig = SIGTERM
					if sm.handler != nil {
						sm.handler(sm)
					}
				case <-u1:
					sm.sig = SIGUSR1
					if sm.handler != nil {
						sm.handler(sm)
					}
				case <-u2:
					sm.sig = SIGUSR2
					if sm.handler != nil {
						sm.handler(sm)
					}
				}
			}
		}(s)
		wg.Wait()
	}
}

// Stop kills the goroutine which is monitoring signals.
func (s *SignalMonitor) Stop() {
	if s.isOn {
		s.off <- true
		s.isOn = false
	}
}

// Sig returns a string of the most recently called os.Signal.
func (s *SignalMonitor) Sig() Signal {
	return s.sig
}
<|endoftext|>


<commit_before>package database

import (
	"github.com/jmoiron/sqlx"
	"strconv"
	"strings"
)

type Event struct {
	Id            string  `db:"id"`
	Kind          string  `db:"kind"`
	Status        string  `db:"status"`
	TriggerTag    string  `db:"trigger_tag"`
	TriggerEvent  string  `db:"trigger_event"`
	TransactionId string  `db:"transaction_id"`
	Data          *string `db:"data"`
}

type Events []*Event

func NewEvent(eventData string) *Event {
	separator := strings.Split(eventData, ",")

	return &Event{
		Id:            separator[0],
		Kind:          separator[1],
		Status:        "",
		TriggerTag:    separator[2],
		TriggerEvent:  separator[3],
		TransactionId: separator[4],
		Data:          &separator[5],
	}
}

func (db *Database) GetEvents(status string, limit int) ([]*Event, error) {
	var err error
	var events []*Event

	if limit == -1 {
		err = db.selectObjs(&events, "SELECT * FROM teleport.event WHERE status = $1 ORDER BY id ASC;", status)
	} else {
		err = db.selectObjs(&events, "SELECT * FROM teleport.event WHERE status = $1 ORDER BY id ASC LIMIT $2;", status, limit)
	}

	return events, err
}

func (db *Database) GetEvent(id string) (*Event, error) {
	var events []*Event
	err := db.selectObjs(&events, "SELECT * FROM teleport.event WHERE id = $1;", id)
	return events[0], err
}

func (e *Event) InsertQuery(tx *sqlx.Tx) error {
	args := make([]interface{}, 0)
	var query string

	// If there's no id, insert without id
	if e.Id == "" {
		query = "INSERT INTO teleport.event (kind, status, trigger_tag, trigger_event, transaction_id, data) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id;"
	} else {
		query = "INSERT INTO teleport.event (id, kind, status, trigger_tag, trigger_event, transaction_id, data) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id;"
		args = append(args, e.Id)
	}

	args = append(args,
		e.Kind,
		e.Status,
		e.TriggerTag,
		e.TriggerEvent,
		e.TransactionId,
		e.Data,
	)

	return tx.Get(&e.Id, query, args...)
}

func (e *Event) UpdateQuery(tx *sqlx.Tx) error {
	_, err := tx.Exec(
		"UPDATE teleport.event SET status = $1 WHERE id = $2;",
		e.Status,
		e.Id,
	)

	return err
}

// Implement Interface
func (slice Events) Len() int {
	return len(slice)
}

func (slice Events) Less(i, j int) bool {
	iInt, _ := strconv.Atoi(slice[i].Id)
	jInt, _ := strconv.Atoi(slice[j].Id)
	return iInt < jInt
}

func (slice Events) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}
<commit_msg>Fix batcher when no maxEventsPerBatch is provided.<commit_after>package database

import (
	"github.com/jmoiron/sqlx"
	"strconv"
	"strings"
)

type Event struct {
	Id            string  `db:"id"`
	Kind          string  `db:"kind"`
	Status        string  `db:"status"`
	TriggerTag    string  `db:"trigger_tag"`
	TriggerEvent  string  `db:"trigger_event"`
	TransactionId string  `db:"transaction_id"`
	Data          *string `db:"data"`
}

type Events []*Event

func NewEvent(eventData string) *Event {
	separator := strings.Split(eventData, ",")

	return &Event{
		Id:            separator[0],
		Kind:          separator[1],
		Status:        "",
		TriggerTag:    separator[2],
		TriggerEvent:  separator[3],
		TransactionId: separator[4],
		Data:          &separator[5],
	}
}

func (db *Database) GetEvents(status string, limit int) ([]*Event, error) {
	var err error
	var events []*Event

	if limit <= 0 {
		err = db.selectObjs(&events, "SELECT * FROM teleport.event WHERE status = $1 ORDER BY id ASC;", status)
	} else {
		err = db.selectObjs(&events, "SELECT * FROM teleport.event WHERE status = $1 ORDER BY id ASC LIMIT $2;", status, limit)
	}

	return events, err
}

func (db *Database) GetEvent(id string) (*Event, error) {
	var events []*Event
	err := db.selectObjs(&events, "SELECT * FROM teleport.event WHERE id = $1;", id)
	return events[0], err
}

func (e *Event) InsertQuery(tx *sqlx.Tx) error {
	args := make([]interface{}, 0)
	var query string

	// If there's no id, insert without id
	if e.Id == "" {
		query = "INSERT INTO teleport.event (kind, status, trigger_tag, trigger_event, transaction_id, data) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id;"
	} else {
		query = "INSERT INTO teleport.event (id, kind, status, trigger_tag, trigger_event, transaction_id, data) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id;"
		args = append(args, e.Id)
	}

	args = append(args,
		e.Kind,
		e.Status,
		e.TriggerTag,
		e.TriggerEvent,
		e.TransactionId,
		e.Data,
	)

	return tx.Get(&e.Id, query, args...)
}

func (e *Event) UpdateQuery(tx *sqlx.Tx) error {
	_, err := tx.Exec(
		"UPDATE teleport.event SET status = $1 WHERE id = $2;",
		e.Status,
		e.Id,
	)

	return err
}

// Implement Interface
func (slice Events) Len() int {
	return len(slice)
}

func (slice Events) Less(i, j int) bool {
	iInt, _ := strconv.Atoi(slice[i].Id)
	jInt, _ := strconv.Atoi(slice[j].Id)
	return iInt < jInt
}

func (slice Events) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}
<|endoftext|>


<commit_before>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     October 24, 2018
;       related version of root zone:     2018102401
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<commit_msg>auto-update<commit_after>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     November 14, 2018
;       related version of root zone:     2018111402
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<|endoftext|>


<commit_before>/*
Logos

This program analyzes articles, identifying important elements in highly ranking articles.
*/

import (
  "bufio"
  "io"
  "ioutil"
  "strings"
  "unicode"
  "strconv"
  "errors"
)

type Publication struct {
  Score float64
  Author string
  Text PublicationBody
}

type PublicationBody interface {
  NextLine() (string)
  HasNextLine() (bool)
  NextWord() (string)
  HasNextWord() (bool)
  ResetSeeker()
}

type StringPublicationBody struct {
  Lines []string
  Words [][]string
  CurrentLine int
  CurrentWord int
}

func (s StringPublicationBody) HasNextLine() (bool) {
  return len(s.Lines) > s.CurrentLine
}

func (s StringPublicationBody) NextLine() (string) {
  res := s.Lines[s.CurrentLine]
  s.CurrentLine++
  return res
}

func (s StringPublicationBody) HasNextWord() (bool) {
  if (len(s.Lines) <= s.CurrentLine) {
    return false
  }

  return (len(s.Words[s.CurrentLine]) > s.CurrentWord) || (len(s.Lines) > s.CurrentLine + 1)
}

func (s StringPublicationBody) NextWord() (bool) {
  word := s.Words[s.CurrentLine][s.CurrentWord]
  if (len(s.Words[s.CurrentLine]) < s.CurrentWord + 1) {
    s.CurrentLine++
    s.CurrentWord = 0
  }
  return word
}

func (s StringPublicationBody) ResetSeeker() {
  s.CurrentLine = 0
  s.CurrentWord = 0
}

func CreateStringPubBody(input io.Reader) (StringPublicationBody, error) {
  scanner := bufio.NewScanner(input)
  lines := make([]string, 0)
  words := make([][]string, 0)

  for scanner.Scan() {
    line := scanner.Text()
    lines = append(lines, line)
    words = append(words, splitWords(line))
  }

  return StringPublicationBody{
    Lines: lines,
    Words: words,
    CurrentLine: 0,
    Currentword: 0,
  }
}

/*
---------------------------------------------------------------
-------------- Measures of Publication Quality ----------------
---------------------------------------------------------------
*/

func (p PublicationBody) WordCount() (int) {
  count := 0

  for (p.HasNextWord()) {
    l := p.NextWord()
    count++
  }

  return count
}

func (p PublicationBody) AverageWordsPerLine() (int) {
  sum := 0
  count := 0

  for (p.HasNextLine()) {
    l := p.NextLine()
    words := splitWords(l)
    sum += len(words)
    count++
  }

  return float64(sum) / count
}

func (p PublicationBody) AverageWordLength() (int) {
  sum := 0
  count := 0

  for (p.HasNextWord()) {
    w := p.NextWord()
    sum += len(w)
    count++
  }

  return float64(sum) / count
}

func (p PublicationBody) WordsLongerThan(x int) (int) {
  count := 0

  for (p.HasNextWord()) {
    w := p.NextWord()
    if (len(w) > x) {
      count++
    }
  }

  return count
}

func (p PublicationBody) WordsIn(list WordList) (int) {
  count := 0

  for (p.HasNextWord()) {
    w := p.NextWord()
    if (list.Contains(w)) {
      count++
    }
  }

  return count
}

func (p PublicationBody) ConstructMarkovMatrix(ngramSize int) (MarkovMatrix) {
  prevNGram := NGram{ngramSize, []string{}}
  matrix := CreateMarkovMatrix()

  for (p.HasNextWord()) {
    w := p.NextWord()
    newWords = append(prevNGram.Words, w)

    if (len(newWords) == ngramSize + 1) {
      ngram := NGram{ngramSize, newWords[1:]}
      curCount := matrix.GetProbability(prevNGram, ngram)

      matrix.SetProbability(prevNGram, ngram, curCount + 1.0)
    }
  }

  res := CreateMarkovMatrix()
  for i := range matrix.Matrix {
    total := 0.0

    for j := range matrix.Matrix[i] {
      total += matrix.Matrix[i][j]
    }

    for j := range matrix.Matrix[i] {
      res.SetProbability(i, j, matrix.GetProbability(i, j) / total)
    }
  }

  return res
}

type NGram struct {
  Size int
  Words []string
}

func (n NGram) Hash() (string) {
  s := []string{strconv.Itoa(n.Size)}
  s := append(s, n.Words...)
  return strings.Join(s, ".")
}

type MarkovMatrix struct {
  Matrix map[string]map[string]float64
}

func CreateMarkovMatrix() (MarkovMatrix) {
  m := make(map[string]map[string]float64)
  return MarkovMatrix{m}
}

func (m MarkovMatrix) SetProbability(i, j NGram, prob float64) {
  entry := m.Matrx[i.Hash()]
  if (entry == nil) {
    entry = make(map[string]float64)
  }
  entry[j.Hash()] = prob
  m.Matrix[i.Hash()] = entry
}

func (m MarkovMatrix) GetProbability(i, j NGram) (float64) {
  entry := m.Matrx[i.Hash()]
  if (entry == nil) {
    return 0.0
  } else {
    return entry[j.Hash()]
  }
}

type WordList struct {
  Words map[string]bool
}

func ConstructWordList(words []string) (WordList) {
  list := make(map[string]bool)

  for _, w := range words {
    list[w] = true
  }

  return list
}

func (w WordList) Contains(word string) (bool) {
  return w.Words[word]
}

/*
splitWords returns the separate words that make up a particular string, making
sure to remove punctuation and spaces.
*/
func splitWords(line string) ([]string) {
  f := func(c rune) bool {
    return unicode.IsPunct(c) || unicode.IsSpace(c)
  }
  return strings.FieldsFunc(sentence, f)
}
<commit_msg>Bugfixes for compiling.<commit_after>package logos

/*
Logos

This program analyzes articles, identifying important elements in highly ranking articles.
*/

import (
  "bufio"
  "io"
  "strings"
  "unicode"
  "strconv"
)

type Publication struct {
  Score float64
  Author string
  Text PublicationBody
}

type PublicationBody interface {
  NextLine() (string)
  HasNextLine() (bool)
  NextWord() (string)
  HasNextWord() (bool)
  ResetSeeker()
}

type StringPublicationBody struct {
  Lines []string
  Words [][]string
  CurrentLine int
  CurrentWord int
}

func (s StringPublicationBody) HasNextLine() (bool) {
  return len(s.Lines) > s.CurrentLine
}

func (s StringPublicationBody) NextLine() (string) {
  res := s.Lines[s.CurrentLine]
  s.CurrentLine++
  return res
}

func (s StringPublicationBody) HasNextWord() (bool) {
  if (len(s.Lines) <= s.CurrentLine) {
    return false
  }

  return (len(s.Words[s.CurrentLine]) > s.CurrentWord) || (len(s.Lines) > s.CurrentLine + 1)
}

func (s StringPublicationBody) NextWord() (string) {
  word := s.Words[s.CurrentLine][s.CurrentWord]
  if (len(s.Words[s.CurrentLine]) < s.CurrentWord + 1) {
    s.CurrentLine++
    s.CurrentWord = 0
  }
  return word
}

func (s StringPublicationBody) ResetSeeker() {
  s.CurrentLine = 0
  s.CurrentWord = 0
}

func CreateStringPubBody(input io.Reader) (StringPublicationBody, error) {
  scanner := bufio.NewScanner(input)
  lines := make([]string, 0)
  words := make([][]string, 0)

  for scanner.Scan() {
    line := scanner.Text()
    lineWords := splitWords(line)

    if (len(lineWords) > 0) {
      lines = append(lines, line)
      words = append(words, lineWords)
    }
  }

  body := StringPublicationBody{
    Lines: lines,
    Words: words,
    CurrentLine: 0,
    CurrentWord: 0,
  }
  return body, nil
}

/*
---------------------------------------------------------------
-------------- Measures of Publication Quality ----------------
---------------------------------------------------------------
*/

func (p PublicationBody) WordCount() (int) {
  count := 0

  for (p.HasNextWord()) {
    l := p.NextWord()
    count++
  }

  return count
}

func (p PublicationBody) AverageWordsPerLine() (float64) {
  sum := 0
  count := 0

  for (p.HasNextLine()) {
    l := p.NextLine()
    words := splitWords(l)
    sum += len(words)
    count++
  }

  return float64(sum) / float64(count)
}

func (p PublicationBody) AverageWordLength() (float64) {
  sum := 0
  count := 0

  for (p.HasNextWord()) {
    w := p.NextWord()
    sum += len(w)
    count++
  }

  return float64(sum) / float64(count)
}

func (p PublicationBody) WordsLongerThan(x int) (int) {
  count := 0

  for (p.HasNextWord()) {
    w := p.NextWord()
    if (len(w) > x) {
      count++
    }
  }

  return count
}

func (p PublicationBody) WordsIn(list WordList) (int) {
  count := 0

  for (p.HasNextWord()) {
    w := p.NextWord()
    if (list.Contains(w)) {
      count++
    }
  }

  return count
}

func (p PublicationBody) ConstructMarkovMatrix(ngramSize int) (MarkovMatrix) {
  prevNGram := NGram{ngramSize, []string{}}
  matrix := CreateMarkovMatrix()

  for (p.HasNextWord()) {
    w := p.NextWord()
    newWords := append(prevNGram.Words, w)

    if (len(newWords) == ngramSize + 1) {
      ngram := NGram{ngramSize, newWords[1:]}
      curCount := matrix.GetProbability(prevNGram, ngram)

      matrix.SetProbability(prevNGram, ngram, curCount + 1.0)
    }
  }

  res := CreateMarkovMatrix()
  for i := range matrix.Matrix {
    total := 0.0

    for j := range matrix.Matrix[i] {
      total += matrix.Matrix[i][j]
    }

    for j := range matrix.Matrix[i] {
      ig := HashToNGram(i)
      jg := HashToNGram(j)
      res.SetProbability(ig, jg, matrix.GetProbability(ig, jg) / total)
    }
  }

  return res
}

type NGram struct {
  Size int
  Words []string
}

func (n NGram) Hash() (string) {
  s := []string{strconv.Itoa(n.Size)}
  s = append(s, n.Words...)
  return strings.Join(s, ".")
}

func HashToNGram(hash string) (NGram) {
  s := strings.Split(hash, ".")
  size, _ := strconv.Atoi(s[0])
  return NGram{size, s[1:]}
}

type MarkovMatrix struct {
  Matrix map[string]map[string]float64
}

func CreateMarkovMatrix() (MarkovMatrix) {
  m := make(map[string]map[string]float64)
  return MarkovMatrix{m}
}

func (m MarkovMatrix) SetProbability(i, j NGram, prob float64) {
  entry := m.Matrix[i.Hash()]
  if (entry == nil) {
    entry = make(map[string]float64)
  }
  entry[j.Hash()] = prob
  m.Matrix[i.Hash()] = entry
}

func (m MarkovMatrix) GetProbability(i, j NGram) (float64) {
  entry := m.Matrix[i.Hash()]
  if (entry == nil) {
    return 0.0
  } else {
    return entry[j.Hash()]
  }
}

type WordList struct {
  Words map[string]bool
}

func ConstructWordList(words []string) (WordList) {
  list := make(map[string]bool)

  for _, w := range words {
    list[w] = true
  }

  return WordList{list}
}

func (w WordList) Contains(word string) (bool) {
  return w.Words[word]
}

/*
splitWords returns the separate words that make up a particular string, making
sure to remove punctuation and spaces.
*/
func splitWords(line string) ([]string) {
  f := func(c rune) bool {
    return unicode.IsPunct(c) || unicode.IsSpace(c)
  }
  return strings.FieldsFunc(line, f)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"
	"time"

	"github.com/flynn/flynn-host/sampi"
	"github.com/flynn/flynn-host/types"
	"github.com/flynn/go-discoverd"
	"github.com/flynn/go-flynn/attempt"
	"github.com/flynn/go-flynn/cluster"
	rpc "github.com/flynn/rpcplus/comborpc"
	"github.com/technoweenie/grohl"
)

// Attempts is the attempt strategy that is used to connect to discoverd.
var Attempts = attempt.Strategy{
	Min:   5,
	Total: 5 * time.Second,
	Delay: 200 * time.Millisecond,
}

// A command line flag to accumulate multiple key-value pairs into Attributes,
// e.g. flynn-host -attribute foo=bar -attribute bar=foo
type AttributeFlag map[string]string

func (a AttributeFlag) Set(val string) error {
	kv := strings.SplitN(val, "=", 2)
	a[kv[0]] = kv[1]
	return nil
}

func (a AttributeFlag) String() string {
	res := make([]string, 0, len(a))
	for k, v := range a {
		res = append(res, k+"="+v)
	}
	return strings.Join(res, ", ")
}

func main() {
	hostname, _ := os.Hostname()
	externalAddr := flag.String("external", "", "external IP of host")
	bindAddr := flag.String("bind", "", "bind containers to this IP")
	configFile := flag.String("config", "", "configuration file")
	manifestFile := flag.String("manifest", "/etc/flynn-host.json", "manifest file")
	hostID := flag.String("id", hostname, "host id")
	force := flag.Bool("force", false, "kill all containers booted by flynn-host before starting")
	attributes := make(AttributeFlag)
	flag.Var(&attributes, "attribute", "key=value pair to add as an attribute")
	flag.Parse()
	grohl.AddContext("app", "lorne")
	grohl.Log(grohl.Data{"at": "start"})
	g := grohl.NewContext(grohl.Data{"fn": "main"})

	state := NewState()
	backend, err := NewDockerBackend(state, *bindAddr)
	if err != nil {
		log.Fatal(err)
	}

	go serveHTTP(&Host{state: state}, &attachHandler{state: state, backend: backend})

	if *force {
		if err := backend.Cleanup(); err != nil {
			log.Fatal(err)
		}
	}

	runner := &manifestRunner{
		env:          parseEnviron(),
		externalAddr: *externalAddr,
		bindAddr:     *bindAddr,
		backend:      backend,
	}

	discAddr := os.Getenv("DISCOVERD")
	var disc *discoverd.Client
	if *manifestFile != "" {
		var r io.Reader
		var f *os.File
		if *manifestFile == "-" {
			r = os.Stdin
		} else {
			f, err = os.Open(*manifestFile)
			if err != nil {
				log.Fatal(err)
			}
			r = f
		}
		services, err := runner.runManifest(r)
		if err != nil {
			log.Fatal(err)
		}
		if f != nil {
			f.Close()
		}

		if d, ok := services["discoverd"]; ok {
			discAddr = fmt.Sprintf("%s:%d", d.InternalIP, d.TCPPorts[0])
			var disc *discoverd.Client
			err = Attempts.Run(func() (err error) {
				disc, err = discoverd.NewClientWithAddr(discAddr)
				return
			})
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	if discAddr == "" && *externalAddr != "" {
		discAddr = *externalAddr + ":1111"
	}
	// HACK: use env as global for discoverd connection in sampic
	os.Setenv("DISCOVERD", discAddr)
	if disc == nil {
		disc, err = discoverd.NewClientWithAddr(discAddr)
		if err != nil {
			log.Fatal(err)
		}
	}
	sampiStandby, err := disc.RegisterAndStandby("flynn-host", *externalAddr+":1113", map[string]string{"id": *hostID})
	if err != nil {
		log.Fatal(err)
	}

	// Check if we are the leader so that we can use the cluster functions directly
	sampiCluster := sampi.NewCluster(sampi.NewState())
	select {
	case <-sampiStandby:
		g.Log(grohl.Data{"at": "sampi_leader"})
		rpc.Register(sampiCluster)
	case <-time.After(5 * time.Millisecond):
		go func() {
			<-sampiStandby
			g.Log(grohl.Data{"at": "sampi_leader"})
			rpc.Register(sampiCluster)
		}()
	}
	cluster, err := cluster.NewClientWithSelf(*hostID, NewLocalClient(*hostID, sampiCluster))
	if err != nil {
		log.Fatal(err)
	}

	g.Log(grohl.Data{"at": "sampi_connected"})

	events := make(chan host.Event)
	state.AddListener("all", events)
	go syncScheduler(cluster, events)

	h := &host.Host{}
	if *configFile != "" {
		h, err = openConfig(*configFile)
		if err != nil {
			log.Fatal(err)
		}
	}
	if h.Attributes == nil {
		h.Attributes = make(map[string]string)
	}
	for k, v := range attributes {
		h.Attributes[k] = v
	}
	h.ID = *hostID

	for {
		newLeader := cluster.NewLeaderSignal()

		h.Jobs = state.ClusterJobs()
		jobs := make(chan *host.Job)
		hostErr := cluster.RegisterHost(h, jobs)
		g.Log(grohl.Data{"at": "host_registered"})
		for job := range jobs {
			if *externalAddr != "" {
				job.Config.Env = appendUnique(job.Config.Env, "EXTERNAL_IP="+*externalAddr, "DISCOVERD="+discAddr)
			}
			backend.Run(job)
		}
		g.Log(grohl.Data{"at": "sampi_disconnected", "err": *hostErr})

		<-newLeader
	}
}

type sampiClient interface {
	ConnectHost(*host.Host, chan *host.Job) *error
	RemoveJobs([]string) error
}

type sampiSyncClient interface {
	RemoveJobs([]string) error
}

func syncScheduler(scheduler sampiSyncClient, events <-chan host.Event) {
	for event := range events {
		if event.Event != "stop" {
			continue
		}
		grohl.Log(grohl.Data{"fn": "scheduler_event", "at": "remove_job", "job.id": event.JobID})
		if err := scheduler.RemoveJobs([]string{event.JobID}); err != nil {
			grohl.Log(grohl.Data{"fn": "scheduler_event", "at": "remove_job", "status": "error", "err": err, "job.id": event.JobID})
		}
	}
}
<commit_msg>host: Ensure the RPC Host has a backend<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"
	"time"

	"github.com/flynn/flynn-host/sampi"
	"github.com/flynn/flynn-host/types"
	"github.com/flynn/go-discoverd"
	"github.com/flynn/go-flynn/attempt"
	"github.com/flynn/go-flynn/cluster"
	rpc "github.com/flynn/rpcplus/comborpc"
	"github.com/technoweenie/grohl"
)

// Attempts is the attempt strategy that is used to connect to discoverd.
var Attempts = attempt.Strategy{
	Min:   5,
	Total: 5 * time.Second,
	Delay: 200 * time.Millisecond,
}

// A command line flag to accumulate multiple key-value pairs into Attributes,
// e.g. flynn-host -attribute foo=bar -attribute bar=foo
type AttributeFlag map[string]string

func (a AttributeFlag) Set(val string) error {
	kv := strings.SplitN(val, "=", 2)
	a[kv[0]] = kv[1]
	return nil
}

func (a AttributeFlag) String() string {
	res := make([]string, 0, len(a))
	for k, v := range a {
		res = append(res, k+"="+v)
	}
	return strings.Join(res, ", ")
}

func main() {
	hostname, _ := os.Hostname()
	externalAddr := flag.String("external", "", "external IP of host")
	bindAddr := flag.String("bind", "", "bind containers to this IP")
	configFile := flag.String("config", "", "configuration file")
	manifestFile := flag.String("manifest", "/etc/flynn-host.json", "manifest file")
	hostID := flag.String("id", hostname, "host id")
	force := flag.Bool("force", false, "kill all containers booted by flynn-host before starting")
	attributes := make(AttributeFlag)
	flag.Var(&attributes, "attribute", "key=value pair to add as an attribute")
	flag.Parse()
	grohl.AddContext("app", "lorne")
	grohl.Log(grohl.Data{"at": "start"})
	g := grohl.NewContext(grohl.Data{"fn": "main"})

	state := NewState()
	backend, err := NewDockerBackend(state, *bindAddr)
	if err != nil {
		log.Fatal(err)
	}

	go serveHTTP(&Host{state: state, backend: backend}, &attachHandler{state: state, backend: backend})

	if *force {
		if err := backend.Cleanup(); err != nil {
			log.Fatal(err)
		}
	}

	runner := &manifestRunner{
		env:          parseEnviron(),
		externalAddr: *externalAddr,
		bindAddr:     *bindAddr,
		backend:      backend,
	}

	discAddr := os.Getenv("DISCOVERD")
	var disc *discoverd.Client
	if *manifestFile != "" {
		var r io.Reader
		var f *os.File
		if *manifestFile == "-" {
			r = os.Stdin
		} else {
			f, err = os.Open(*manifestFile)
			if err != nil {
				log.Fatal(err)
			}
			r = f
		}
		services, err := runner.runManifest(r)
		if err != nil {
			log.Fatal(err)
		}
		if f != nil {
			f.Close()
		}

		if d, ok := services["discoverd"]; ok {
			discAddr = fmt.Sprintf("%s:%d", d.InternalIP, d.TCPPorts[0])
			var disc *discoverd.Client
			err = Attempts.Run(func() (err error) {
				disc, err = discoverd.NewClientWithAddr(discAddr)
				return
			})
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	if discAddr == "" && *externalAddr != "" {
		discAddr = *externalAddr + ":1111"
	}
	// HACK: use env as global for discoverd connection in sampic
	os.Setenv("DISCOVERD", discAddr)
	if disc == nil {
		disc, err = discoverd.NewClientWithAddr(discAddr)
		if err != nil {
			log.Fatal(err)
		}
	}
	sampiStandby, err := disc.RegisterAndStandby("flynn-host", *externalAddr+":1113", map[string]string{"id": *hostID})
	if err != nil {
		log.Fatal(err)
	}

	// Check if we are the leader so that we can use the cluster functions directly
	sampiCluster := sampi.NewCluster(sampi.NewState())
	select {
	case <-sampiStandby:
		g.Log(grohl.Data{"at": "sampi_leader"})
		rpc.Register(sampiCluster)
	case <-time.After(5 * time.Millisecond):
		go func() {
			<-sampiStandby
			g.Log(grohl.Data{"at": "sampi_leader"})
			rpc.Register(sampiCluster)
		}()
	}
	cluster, err := cluster.NewClientWithSelf(*hostID, NewLocalClient(*hostID, sampiCluster))
	if err != nil {
		log.Fatal(err)
	}

	g.Log(grohl.Data{"at": "sampi_connected"})

	events := make(chan host.Event)
	state.AddListener("all", events)
	go syncScheduler(cluster, events)

	h := &host.Host{}
	if *configFile != "" {
		h, err = openConfig(*configFile)
		if err != nil {
			log.Fatal(err)
		}
	}
	if h.Attributes == nil {
		h.Attributes = make(map[string]string)
	}
	for k, v := range attributes {
		h.Attributes[k] = v
	}
	h.ID = *hostID

	for {
		newLeader := cluster.NewLeaderSignal()

		h.Jobs = state.ClusterJobs()
		jobs := make(chan *host.Job)
		hostErr := cluster.RegisterHost(h, jobs)
		g.Log(grohl.Data{"at": "host_registered"})
		for job := range jobs {
			if *externalAddr != "" {
				job.Config.Env = appendUnique(job.Config.Env, "EXTERNAL_IP="+*externalAddr, "DISCOVERD="+discAddr)
			}
			backend.Run(job)
		}
		g.Log(grohl.Data{"at": "sampi_disconnected", "err": *hostErr})

		<-newLeader
	}
}

type sampiClient interface {
	ConnectHost(*host.Host, chan *host.Job) *error
	RemoveJobs([]string) error
}

type sampiSyncClient interface {
	RemoveJobs([]string) error
}

func syncScheduler(scheduler sampiSyncClient, events <-chan host.Event) {
	for event := range events {
		if event.Event != "stop" {
			continue
		}
		grohl.Log(grohl.Data{"fn": "scheduler_event", "at": "remove_job", "job.id": event.JobID})
		if err := scheduler.RemoveJobs([]string{event.JobID}); err != nil {
			grohl.Log(grohl.Data{"fn": "scheduler_event", "at": "remove_job", "status": "error", "err": err, "job.id": event.JobID})
		}
	}
}
<|endoftext|>


<commit_before>package ls

import "fmt"
import "io"
import "os"
import "path"
import "strings"
import "time"

import "../box"

var exeSuffixes = map[string]bool{
	".bat": true,
	".cmd": true,
	".com": true,
	".exe": true,
}

const (
	O_STRIP_DIR = 1
	O_LONG      = 2
	O_INDICATOR = 4
	O_COLOR     = 8
)

type fileInfoT struct {
	name string
	info os.FileInfo
}

const (
	ANSI_EXEC     = "\x1B[1;35m"
	ANSI_DIR      = "\x1B[1;32m"
	ANSI_NORM     = "\x1B[1;37m"
	ANSI_READONLY = "\x1B[1;33m"
	ANSI_END      = "\x1B[0;39m"
)

func (this fileInfoT) Name() string       { return this.name }
func (this fileInfoT) Size() int64        { return this.info.Size() }
func (this fileInfoT) Mode() os.FileMode  { return this.info.Mode() }
func (this fileInfoT) ModTime() time.Time { return this.info.ModTime() }
func (this fileInfoT) IsDir() bool        { return this.info.IsDir() }
func (this fileInfoT) Sys() interface{}   { return this.info.Sys() }

func newMyFileInfoT(name string, info os.FileInfo) *fileInfoT {
	this := new(fileInfoT)
	this.name = name
	this.info = info
	return this
}

func lsOneLong(status os.FileInfo, flag int, out io.Writer) {
	indicator := " "
	prefix := ""
	postfix := ""
	if status.IsDir() {
		io.WriteString(out, "d")
		indicator = "/"
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_DIR
			postfix = ANSI_END
		}
	} else {
		io.WriteString(out, "-")
	}
	mode := status.Mode()
	perm := mode.Perm()
	name := status.Name()
	if (perm & 4) > 0 {
		io.WriteString(out, "r")
	} else {
		io.WriteString(out, "-")
	}
	if (perm & 2) > 0 {
		io.WriteString(out, "w")
	} else {
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_READONLY
			postfix = ANSI_END
		}
		io.WriteString(out, "-")
	}
	if (perm & 1) > 0 {
		io.WriteString(out, "x")
	} else if exeSuffixes[strings.ToLower(path.Ext(name))] {
		io.WriteString(out, "x")
		indicator = "*"
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_EXEC
			postfix = ANSI_END
		}
	} else {
		io.WriteString(out, "-")
	}
	if (flag & O_STRIP_DIR) > 0 {
		name = path.Base(name)
	}
	io.WriteString(out, fmt.Sprintf("%7d %s%s%s",
		status.Size(),
		prefix,
		name,
		postfix))
	if (flag & O_INDICATOR) > 0 {
		io.WriteString(out, indicator)
	}
	io.WriteString(out, "\n")
}

func lsBox(nodes []os.FileInfo, flag int, out io.Writer) {
	nodes_ := make([]string, len(nodes))
	for key, val := range nodes {
		prefix := ""
		postfix := ""
		if val.IsDir() {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_DIR
				postfix = ANSI_END
			}
			if (flag & O_INDICATOR) != 0 {
				postfix += "/"
			}
		} else if (val.Mode().Perm() & 2) == 0 {
			prefix = ANSI_READONLY
			postfix = ANSI_END
		} else if exeSuffixes[strings.ToLower(path.Ext(val.Name()))] {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_DIR
				postfix = ANSI_END
			}
			if (flag & O_INDICATOR) != 0 {
				postfix += "*"
			}
		}
		nodes_[key] = prefix + val.Name() + postfix
	}
	box.Print(nodes_, 80, out)
}

func lsLong(nodes []os.FileInfo, flag int, out io.Writer) {
	for _, finfo := range nodes {
		lsOneLong(finfo, flag, out)
	}
}

type fileInfoCollection struct {
	nodes []os.FileInfo
}

func (this *fileInfoCollection) Len() int {
	return len(this.nodes)
}
func (this *fileInfoCollection) Less(i, j int) bool {
	return this.nodes[i].Name() < this.nodes[j].Name()
}
func (this *fileInfoCollection) Swap(i, j int) {
	tmp := this.nodes[i]
	this.nodes[i] = this.nodes[j]
	this.nodes[j] = tmp
}

func lsFolder(folder string, flag int, out io.Writer) error {
	fd, err := os.Open(folder)
	if err != nil {
		return err
	}
	defer fd.Close()
	var nodesArray fileInfoCollection
	nodesArray.nodes, err = fd.Readdir(-1)
	if err != nil {
		return err
	}
	if (flag & O_LONG) > 0 {
		lsLong(nodesArray.nodes, O_STRIP_DIR|flag, out)
	} else {
		lsBox(nodesArray.nodes, O_STRIP_DIR|flag, out)
	}
	return nil
}

func lsCore(paths []string, flag int, out io.Writer) error {
	if len(paths) <= 0 {
		return lsFolder(".", flag, out)
	}
	dirs := make([]string, 0)
	printCount := 0
	files := make([]os.FileInfo, 0)
	for _, name := range paths {
		status, err := os.Stat(name)
		if err != nil {
			return err
		}
		if status.IsDir() {
			dirs = append(dirs, name)
		} else if (flag & O_LONG) != 0 {
			lsOneLong(newMyFileInfoT(name, status), flag, out)
			printCount += 1
		} else {
			files = append(files, newMyFileInfoT(name, status))
		}
	}
	if len(files) > 0 {
		lsBox(files, flag, out)
		printCount = len(files)
	}
	for _, name := range dirs {
		if len(paths) > 1 {
			if printCount > 0 {
				io.WriteString(out, "\n")
			}
			io.WriteString(out, name)
			io.WriteString(out, ":\n")
		}
		err := lsFolder(name, flag, out)
		if err != nil {
			return err
		}
		printCount++
	}
	return nil
}

var option = map[rune](func(*int) error){
	'l': func(flag *int) error {
		*flag |= O_LONG
		return nil
	},
	'F': func(flag *int) error {
		*flag |= O_INDICATOR
		return nil
	},
	'o': func(flag *int) error {
		*flag |= O_COLOR
		return nil
	},
}

// 存在しないオプションに関するエラー
type OptionError struct {
	Option rune
}

func (this OptionError) Error() string {
	return fmt.Sprintf("-%c: No such option", this.Option)
}

// ls 機能のエントリ:引数をオプションとパスに分離する
func Main(args []string, out io.Writer) error {
	flag := 0
	paths := make([]string, 0)
	for _, arg := range args {
		if strings.HasPrefix(arg, "-") {
			for _, o := range arg[1:] {
				setter, ok := option[o]
				if !ok {
					var err OptionError
					err.Option = o
					return err
				}
				err := setter(&flag)
				if err != nil {
					return err
				}
			}
		} else {
			paths = append(paths, arg)
		}
	}
	return lsCore(paths, flag, out)
}

// vim:set fenc=utf8 ts=4 sw=4 noet:
<commit_msg>ls - READONLY と色のリセットがおかしくなっていた不具合を修正<commit_after>package ls

import "fmt"
import "io"
import "os"
import "path"
import "strings"
import "time"

import "../box"

var exeSuffixes = map[string]bool{
	".bat": true,
	".cmd": true,
	".com": true,
	".exe": true,
}

const (
	O_STRIP_DIR = 1
	O_LONG      = 2
	O_INDICATOR = 4
	O_COLOR     = 8
)

type fileInfoT struct {
	name string
	info os.FileInfo
}

const (
	ANSI_EXEC     = "\x1B[1;35m"
	ANSI_DIR      = "\x1B[1;32m"
	ANSI_NORM     = "\x1B[1;37m"
	ANSI_READONLY = "\x1B[1;33m"
	ANSI_END      = "\x1B[39m"
)

func (this fileInfoT) Name() string       { return this.name }
func (this fileInfoT) Size() int64        { return this.info.Size() }
func (this fileInfoT) Mode() os.FileMode  { return this.info.Mode() }
func (this fileInfoT) ModTime() time.Time { return this.info.ModTime() }
func (this fileInfoT) IsDir() bool        { return this.info.IsDir() }
func (this fileInfoT) Sys() interface{}   { return this.info.Sys() }

func newMyFileInfoT(name string, info os.FileInfo) *fileInfoT {
	this := new(fileInfoT)
	this.name = name
	this.info = info
	return this
}

func lsOneLong(status os.FileInfo, flag int, out io.Writer) {
	indicator := " "
	prefix := ""
	postfix := ""
	if status.IsDir() {
		io.WriteString(out, "d")
		indicator = "/"
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_DIR
			postfix = ANSI_END
		}
	} else {
		io.WriteString(out, "-")
	}
	mode := status.Mode()
	perm := mode.Perm()
	name := status.Name()
	if (perm & 4) > 0 {
		io.WriteString(out, "r")
	} else {
		io.WriteString(out, "-")
	}
	if (perm & 2) > 0 {
		io.WriteString(out, "w")
	} else {
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_READONLY
			postfix = ANSI_END
		}
		io.WriteString(out, "-")
	}
	if (perm & 1) > 0 {
		io.WriteString(out, "x")
	} else if exeSuffixes[strings.ToLower(path.Ext(name))] {
		io.WriteString(out, "x")
		indicator = "*"
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_EXEC
			postfix = ANSI_END
		}
	} else {
		io.WriteString(out, "-")
	}
	if (flag & O_STRIP_DIR) > 0 {
		name = path.Base(name)
	}
	io.WriteString(out, fmt.Sprintf("%7d %s%s%s",
		status.Size(),
		prefix,
		name,
		postfix))
	if (flag & O_INDICATOR) > 0 {
		io.WriteString(out, indicator)
	}
	io.WriteString(out, "\n")
}

func lsBox(nodes []os.FileInfo, flag int, out io.Writer) {
	nodes_ := make([]string, len(nodes))
	for key, val := range nodes {
		prefix := ""
		postfix := ""
		if val.IsDir() {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_DIR
				postfix = ANSI_END
			}
			if (flag & O_INDICATOR) != 0 {
				postfix += "/"
			}
		} else if (val.Mode().Perm() & 2) == 0 {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_READONLY
				postfix = ANSI_END
			}
		} else if exeSuffixes[strings.ToLower(path.Ext(val.Name()))] {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_DIR
				postfix = ANSI_END
			}
			if (flag & O_INDICATOR) != 0 {
				postfix += "*"
			}
		}
		nodes_[key] = prefix + val.Name() + postfix
	}
	box.Print(nodes_, 80, out)
}

func lsLong(nodes []os.FileInfo, flag int, out io.Writer) {
	for _, finfo := range nodes {
		lsOneLong(finfo, flag, out)
	}
}

type fileInfoCollection struct {
	nodes []os.FileInfo
}

func (this *fileInfoCollection) Len() int {
	return len(this.nodes)
}
func (this *fileInfoCollection) Less(i, j int) bool {
	return this.nodes[i].Name() < this.nodes[j].Name()
}
func (this *fileInfoCollection) Swap(i, j int) {
	tmp := this.nodes[i]
	this.nodes[i] = this.nodes[j]
	this.nodes[j] = tmp
}

func lsFolder(folder string, flag int, out io.Writer) error {
	fd, err := os.Open(folder)
	if err != nil {
		return err
	}
	defer fd.Close()
	var nodesArray fileInfoCollection
	nodesArray.nodes, err = fd.Readdir(-1)
	if err != nil {
		return err
	}
	if (flag & O_LONG) > 0 {
		lsLong(nodesArray.nodes, O_STRIP_DIR|flag, out)
	} else {
		lsBox(nodesArray.nodes, O_STRIP_DIR|flag, out)
	}
	return nil
}

func lsCore(paths []string, flag int, out io.Writer) error {
	if len(paths) <= 0 {
		return lsFolder(".", flag, out)
	}
	dirs := make([]string, 0)
	printCount := 0
	files := make([]os.FileInfo, 0)
	for _, name := range paths {
		status, err := os.Stat(name)
		if err != nil {
			return err
		}
		if status.IsDir() {
			dirs = append(dirs, name)
		} else if (flag & O_LONG) != 0 {
			lsOneLong(newMyFileInfoT(name, status), flag, out)
			printCount += 1
		} else {
			files = append(files, newMyFileInfoT(name, status))
		}
	}
	if len(files) > 0 {
		lsBox(files, flag, out)
		printCount = len(files)
	}
	for _, name := range dirs {
		if len(paths) > 1 {
			if printCount > 0 {
				io.WriteString(out, "\n")
			}
			io.WriteString(out, name)
			io.WriteString(out, ":\n")
		}
		err := lsFolder(name, flag, out)
		if err != nil {
			return err
		}
		printCount++
	}
	return nil
}

var option = map[rune](func(*int) error){
	'l': func(flag *int) error {
		*flag |= O_LONG
		return nil
	},
	'F': func(flag *int) error {
		*flag |= O_INDICATOR
		return nil
	},
	'o': func(flag *int) error {
		*flag |= O_COLOR
		return nil
	},
}

// 存在しないオプションに関するエラー
type OptionError struct {
	Option rune
}

func (this OptionError) Error() string {
	return fmt.Sprintf("-%c: No such option", this.Option)
}

// ls 機能のエントリ:引数をオプションとパスに分離する
func Main(args []string, out io.Writer) error {
	flag := 0
	paths := make([]string, 0)
	for _, arg := range args {
		if strings.HasPrefix(arg, "-") {
			for _, o := range arg[1:] {
				setter, ok := option[o]
				if !ok {
					var err OptionError
					err.Option = o
					return err
				}
				err := setter(&flag)
				if err != nil {
					return err
				}
			}
		} else {
			paths = append(paths, arg)
		}
	}
	return lsCore(paths, flag, out)
}

// vim:set fenc=utf8 ts=4 sw=4 noet:
<|endoftext|>