<commit_before>package repository

import (
	"fmt"
	"log"

	"database/sql"

	"github.com/mleonard87/merknera/rpchelper"
)

type BotStatus string

const (
	BOT_STATUS_ONLINE  BotStatus = "ONLINE"
	BOT_STATUS_OFFLINE BotStatus = "OFFLINE"
	BOT_STATUS_ERROR   BotStatus = "ERROR"
)

type Bot struct {
	Id                  int
	Name                string
	Version             string
	gameTypeId          int
	gameType            GameType
	userId              int
	user                User
	RPCEndpoint         string
	ProgrammingLanguage string
	Website             string
	Status              BotStatus
}

func (b *Bot) GameType() (GameType, error) {
	if b.gameType == (GameType{}) {
		gt, err := GetGameTypeById(b.gameTypeId)
		if err != nil {
			log.Printf("An error occurred in bot.GameType():\n%s\n", err)
			return GameType{}, err
		}
		b.gameType = gt
	}

	return b.gameType, nil
}

func (b *Bot) User() (User, error) {
	if b.user == (User{}) {
		u, err := GetUserById(b.userId)
		if err != nil {
			log.Printf("An error occurred in bot.User():\n%s\n", err)
			return User{}, err
		}
		b.user = u
	}

	return b.user, nil
}

// Ping will make an RPC call to the Status.Ping method. If this does not return
// then mark te bot as offline and will not participate in any further games until
// it is found to be online again.
func (b *Bot) Ping() (bool, error) {
	fmt.Printf("Pinging %s\n", b.Name)
	err := rpchelper.Ping(b.RPCEndpoint)
	if err != nil {
		err2 := b.MarkOffline()
		if err2 != nil {
			log.Printf("An error occurred in bot.Ping():1:\n%s\n", err2)
			return false, err2
		}
		log.Printf("An error occurred in bot.Ping():2:\n%s\n", err)
		return false, err
	}

	err = b.MarkOnline()
	if err != nil {
		log.Printf("An error occurred in bot.Ping():3:\n%s\n", err)
		return false, err
	}

	return true, nil
}

func (b *Bot) setStatus(status BotStatus) error {
	db := GetDB()
	_, err := db.Exec(`
	UPDATE bot
	SET status = $1
	WHERE id = $2
	`, string(status), b.Id)
	if err != nil {
		log.Printf("An error occurred in bot.setStatus():\n%s\n", err)
		return err
	}

	return nil
}

func (b *Bot) MarkOffline() error {
	b.Status = BOT_STATUS_OFFLINE
	return b.setStatus(BOT_STATUS_OFFLINE)
}

func (b *Bot) MarkOnline() error {
	b.Status = BOT_STATUS_ONLINE
	return b.setStatus(BOT_STATUS_ONLINE)
}

func (b *Bot) MarkError() error {
	b.Status = BOT_STATUS_ERROR
	return b.setStatus(BOT_STATUS_ERROR)
}

func (b *Bot) DoesVersionExist(version string) (bool, error) {
	var botId int
	db := GetDB()
	err := db.QueryRow(`
	SELECT
	  id
	FROM bot
	WHERE name = $1
	AND version = $2
	`, b.Name, version).Scan(&botId)
	if err != nil {
		if err == sql.ErrNoRows {
			return false, nil
		}
		log.Printf("An error occurred in bot.DoesVersionExist():\n%s\n", err)
		return true, err
	}

	return true, nil
}

func RegisterBot(name string, version string, gameType GameType, user User, rpcEndpoint string, programmingLanguage string, website string) (Bot, error) {
	var botId int
	db := GetDB()
	err := db.QueryRow(`
	INSERT INTO bot (
	  name
	, version
	, game_type_id
	, user_id
	, rpc_endpoint
	, programming_language
	, website
	, status
	) VALUES (
	  $1
	, $2
	, $3
	, $4
	, $5
	, $6
	, $7
	, $8
	) RETURNING id
	`, name, version, gameType.Id, user.Id, rpcEndpoint, programmingLanguage, website, string(BOT_STATUS_ONLINE)).Scan(&botId)
	if err != nil {
		log.Printf("An error occurred in bot.RegisterBot():1:\n%s\n", err)
		return Bot{}, err
	}

	bot, err := GetBotById(botId)
	if err != nil {
		log.Printf("An error occurred in bot.RegisterBot():2:\n%s\n", err)
		return Bot{}, err
	}
	return bot, nil
}

func GetBotById(id int) (Bot, error) {
	var bot Bot
	var status string
	db := GetDB()
	err := db.QueryRow(`
	SELECT
	  id
	, name
	, version
	, game_type_id
	, user_id
	, rpc_endpoint
	, programming_language
	, website
	, status
	FROM bot
	WHERE id = $1
	`, id).Scan(&bot.Id, &bot.Name, &bot.Version, &bot.gameTypeId, &bot.userId, &bot.RPCEndpoint, &bot.ProgrammingLanguage, &bot.Website, &status)
	if err != nil {
		log.Printf("An error occurred in bot.GetBotById():\n%s\n", err)
		return Bot{}, err
	}
	bot.Status = BotStatus(status)

	return bot, nil
}

func GetBotByName(name string) (Bot, error) {
	var bot Bot
	var status string
	db := GetDB()
	err := db.QueryRow(`
	SELECT
	  id
	, name
	, version
	, game_type_id
	, user_id
	, rpc_endpoint
	, programming_language
	, website
	, status
	FROM bot
	WHERE name = $1
	`, name).Scan(&bot.Id, &bot.Name, &bot.Version, &bot.gameTypeId, &bot.userId, &bot.RPCEndpoint, &bot.ProgrammingLanguage, &bot.Website, &status)
	if err != nil {
		if err != sql.ErrNoRows {
			log.Printf("An error occurred in bot.GetBotByName():\n%s\n", err)
		}
		return Bot{}, err
	}
	bot.Status = BotStatus(status)

	return bot, nil
}

func ListBotsForGameType(gameType GameType) ([]Bot, error) {
	db := GetDB()
	rows, err := db.Query(`
	SELECT
	  b.id
	, b.name
	, b.version
	, b.game_type_id
	, b.user_id
	, b.rpc_endpoint
	, b.programming_language
	, b.website
	, b.status
	FROM bot b
	WHERE b.game_type_id = $1
	`, gameType.Id)
	if err != nil {
		return []Bot{}, err
	}

	var botList []Bot
	for rows.Next() {
		var bot Bot
		var status string
		err := rows.Scan(&bot.Id, &bot.Name, &bot.Version, &bot.gameTypeId, &bot.userId, &bot.RPCEndpoint, &bot.ProgrammingLanguage, &bot.Website, &status)
		if err != nil {
			log.Printf("An error occurred in bot.ListBotsForGameType():\n%s\n", err)
			return botList, err
		}
		bot.Status = BotStatus(status)
		botList = append(botList, bot)
	}

	return botList, nil
}

func ListBots() ([]Bot, error) {
	db := GetDB()
	rows, err := db.Query(`
	SELECT
	  b.id
	, b.name
	, b.version
	, b.game_type_id
	, b.user_id
	, b.rpc_endpoint
	, b.programming_language
	, b.website
	, b.status
	FROM bot b
	`)
	if err != nil {
		log.Printf("An error occurred in bot.ListBots():1:\n%s\n", err)
		return []Bot{}, err
	}

	var botList []Bot
	for rows.Next() {
		var bot Bot
		var status string
		err := rows.Scan(&bot.Id, &bot.Name, &bot.Version, &bot.gameTypeId, &bot.userId, &bot.RPCEndpoint, &bot.ProgrammingLanguage, &bot.Website, &status)
		if err != nil {
			log.Printf("An error occurred in bot.ListBots():2:\n%s\n", err)
			return botList, err
		}
		bot.Status = BotStatus(status)
		botList = append(botList, bot)
	}

	return botList, nil
}
<commit_msg>Remove ping debug.<commit_after>package repository

import (
	"log"

	"database/sql"

	"github.com/mleonard87/merknera/rpchelper"
)

type BotStatus string

const (
	BOT_STATUS_ONLINE  BotStatus = "ONLINE"
	BOT_STATUS_OFFLINE BotStatus = "OFFLINE"
	BOT_STATUS_ERROR   BotStatus = "ERROR"
)

type Bot struct {
	Id                  int
	Name                string
	Version             string
	gameTypeId          int
	gameType            GameType
	userId              int
	user                User
	RPCEndpoint         string
	ProgrammingLanguage string
	Website             string
	Status              BotStatus
}

func (b *Bot) GameType() (GameType, error) {
	if b.gameType == (GameType{}) {
		gt, err := GetGameTypeById(b.gameTypeId)
		if err != nil {
			log.Printf("An error occurred in bot.GameType():\n%s\n", err)
			return GameType{}, err
		}
		b.gameType = gt
	}

	return b.gameType, nil
}

func (b *Bot) User() (User, error) {
	if b.user == (User{}) {
		u, err := GetUserById(b.userId)
		if err != nil {
			log.Printf("An error occurred in bot.User():\n%s\n", err)
			return User{}, err
		}
		b.user = u
	}

	return b.user, nil
}

// Ping will make an RPC call to the Status.Ping method. If this does not return
// then mark te bot as offline and will not participate in any further games until
// it is found to be online again.
func (b *Bot) Ping() (bool, error) {
	err := rpchelper.Ping(b.RPCEndpoint)
	if err != nil {
		err2 := b.MarkOffline()
		if err2 != nil {
			log.Printf("An error occurred in bot.Ping():1:\n%s\n", err2)
			return false, err2
		}
		log.Printf("An error occurred in bot.Ping():2:\n%s\n", err)
		return false, err
	}

	err = b.MarkOnline()
	if err != nil {
		log.Printf("An error occurred in bot.Ping():3:\n%s\n", err)
		return false, err
	}

	return true, nil
}

func (b *Bot) setStatus(status BotStatus) error {
	db := GetDB()
	_, err := db.Exec(`
	UPDATE bot
	SET status = $1
	WHERE id = $2
	`, string(status), b.Id)
	if err != nil {
		log.Printf("An error occurred in bot.setStatus():\n%s\n", err)
		return err
	}

	return nil
}

func (b *Bot) MarkOffline() error {
	b.Status = BOT_STATUS_OFFLINE
	return b.setStatus(BOT_STATUS_OFFLINE)
}

func (b *Bot) MarkOnline() error {
	b.Status = BOT_STATUS_ONLINE
	return b.setStatus(BOT_STATUS_ONLINE)
}

func (b *Bot) MarkError() error {
	b.Status = BOT_STATUS_ERROR
	return b.setStatus(BOT_STATUS_ERROR)
}

func (b *Bot) DoesVersionExist(version string) (bool, error) {
	var botId int
	db := GetDB()
	err := db.QueryRow(`
	SELECT
	  id
	FROM bot
	WHERE name = $1
	AND version = $2
	`, b.Name, version).Scan(&botId)
	if err != nil {
		if err == sql.ErrNoRows {
			return false, nil
		}
		log.Printf("An error occurred in bot.DoesVersionExist():\n%s\n", err)
		return true, err
	}

	return true, nil
}

func RegisterBot(name string, version string, gameType GameType, user User, rpcEndpoint string, programmingLanguage string, website string) (Bot, error) {
	var botId int
	db := GetDB()
	err := db.QueryRow(`
	INSERT INTO bot (
	  name
	, version
	, game_type_id
	, user_id
	, rpc_endpoint
	, programming_language
	, website
	, status
	) VALUES (
	  $1
	, $2
	, $3
	, $4
	, $5
	, $6
	, $7
	, $8
	) RETURNING id
	`, name, version, gameType.Id, user.Id, rpcEndpoint, programmingLanguage, website, string(BOT_STATUS_ONLINE)).Scan(&botId)
	if err != nil {
		log.Printf("An error occurred in bot.RegisterBot():1:\n%s\n", err)
		return Bot{}, err
	}

	bot, err := GetBotById(botId)
	if err != nil {
		log.Printf("An error occurred in bot.RegisterBot():2:\n%s\n", err)
		return Bot{}, err
	}
	return bot, nil
}

func GetBotById(id int) (Bot, error) {
	var bot Bot
	var status string
	db := GetDB()
	err := db.QueryRow(`
	SELECT
	  id
	, name
	, version
	, game_type_id
	, user_id
	, rpc_endpoint
	, programming_language
	, website
	, status
	FROM bot
	WHERE id = $1
	`, id).Scan(&bot.Id, &bot.Name, &bot.Version, &bot.gameTypeId, &bot.userId, &bot.RPCEndpoint, &bot.ProgrammingLanguage, &bot.Website, &status)
	if err != nil {
		log.Printf("An error occurred in bot.GetBotById():\n%s\n", err)
		return Bot{}, err
	}
	bot.Status = BotStatus(status)

	return bot, nil
}

func GetBotByName(name string) (Bot, error) {
	var bot Bot
	var status string
	db := GetDB()
	err := db.QueryRow(`
	SELECT
	  id
	, name
	, version
	, game_type_id
	, user_id
	, rpc_endpoint
	, programming_language
	, website
	, status
	FROM bot
	WHERE name = $1
	`, name).Scan(&bot.Id, &bot.Name, &bot.Version, &bot.gameTypeId, &bot.userId, &bot.RPCEndpoint, &bot.ProgrammingLanguage, &bot.Website, &status)
	if err != nil {
		if err != sql.ErrNoRows {
			log.Printf("An error occurred in bot.GetBotByName():\n%s\n", err)
		}
		return Bot{}, err
	}
	bot.Status = BotStatus(status)

	return bot, nil
}

func ListBotsForGameType(gameType GameType) ([]Bot, error) {
	db := GetDB()
	rows, err := db.Query(`
	SELECT
	  b.id
	, b.name
	, b.version
	, b.game_type_id
	, b.user_id
	, b.rpc_endpoint
	, b.programming_language
	, b.website
	, b.status
	FROM bot b
	WHERE b.game_type_id = $1
	`, gameType.Id)
	if err != nil {
		return []Bot{}, err
	}

	var botList []Bot
	for rows.Next() {
		var bot Bot
		var status string
		err := rows.Scan(&bot.Id, &bot.Name, &bot.Version, &bot.gameTypeId, &bot.userId, &bot.RPCEndpoint, &bot.ProgrammingLanguage, &bot.Website, &status)
		if err != nil {
			log.Printf("An error occurred in bot.ListBotsForGameType():\n%s\n", err)
			return botList, err
		}
		bot.Status = BotStatus(status)
		botList = append(botList, bot)
	}

	return botList, nil
}

func ListBots() ([]Bot, error) {
	db := GetDB()
	rows, err := db.Query(`
	SELECT
	  b.id
	, b.name
	, b.version
	, b.game_type_id
	, b.user_id
	, b.rpc_endpoint
	, b.programming_language
	, b.website
	, b.status
	FROM bot b
	`)
	if err != nil {
		log.Printf("An error occurred in bot.ListBots():1:\n%s\n", err)
		return []Bot{}, err
	}

	var botList []Bot
	for rows.Next() {
		var bot Bot
		var status string
		err := rows.Scan(&bot.Id, &bot.Name, &bot.Version, &bot.gameTypeId, &bot.userId, &bot.RPCEndpoint, &bot.ProgrammingLanguage, &bot.Website, &status)
		if err != nil {
			log.Printf("An error occurred in bot.ListBots():2:\n%s\n", err)
			return botList, err
		}
		bot.Status = BotStatus(status)
		botList = append(botList, bot)
	}

	return botList, nil
}
<|endoftext|>


<commit_before>package syslog

import (
	"errors"
	"github.com/vaughan0/go-logging"
	"log/syslog"
	"strings"
)

// SyslogOutputter implements Outputter by logging to the system log daemon.
type SyslogOutputter struct {
	Writer    *syslog.Writer
	Formatter logging.Formatter
}

// Creates a new SyslogOutputter with a custom facility (see syslog.Priority).
func NewSyslogFacility(format logging.Formatter, tag string, facility syslog.Priority) (*SyslogOutputter, error) {
	writer, err := syslog.New(facility, tag)
	if err != nil {
		return nil, err
	}
	return &SyslogOutputter{
		Writer:    writer,
		Formatter: format,
	}, nil
}

// Creates a new SyslogOutputter with a name (tag) and the USER facility.
func NewSyslog(format logging.Formatter, tag string) (*SyslogOutputter, error) {
	return NewSyslogFacility(format, tag, syslog.LOG_USER)
}

// Implements Outputter.
func (s SyslogOutputter) Output(msg *logging.Message) {
	str := s.Formatter.Format(msg)
	switch msg.Level {
	case logging.Fatal:
		s.Writer.Crit(str)
	case logging.Error:
		s.Writer.Err(str)
	case logging.Warn:
		s.Writer.Warning(str)
	case logging.Notice:
		s.Writer.Notice(str)
	case logging.Info:
		s.Writer.Info(str)
	case logging.Debug, logging.Trace:
		s.Writer.Debug(str)
	default:
		s.Writer.Notice(str)
	}
}

var facilityMap = map[string]syslog.Priority{
	"kern":     syslog.LOG_KERN,
	"user":     syslog.LOG_USER,
	"mail":     syslog.LOG_MAIL,
	"daemon":   syslog.LOG_DAEMON,
	"auth":     syslog.LOG_AUTH,
	"syslog":   syslog.LOG_SYSLOG,
	"lpr":      syslog.LOG_LPR,
	"news":     syslog.LOG_NEWS,
	"uucp":     syslog.LOG_UUCP,
	"cron":     syslog.LOG_CRON,
	"authpriv": syslog.LOG_AUTHPRIV,
	"ftp":      syslog.LOG_FTP,
	"local0":   syslog.LOG_LOCAL0,
	"local1":   syslog.LOG_LOCAL1,
	"local2":   syslog.LOG_LOCAL2,
	"local3":   syslog.LOG_LOCAL3,
	"local4":   syslog.LOG_LOCAL4,
	"local5":   syslog.LOG_LOCAL5,
	"local6":   syslog.LOG_LOCAL6,
	"local7":   syslog.LOG_LOCAL7,
}

var syslogPlugin = logging.OutputPluginFunc(func(options map[string]string) (result logging.Outputter, err error) {

	// Setup formatter
	format := options["format"]
	if format == "" {
		return nil, errors.New("syslog formatting string not specified")
	}

	tag := options["tag"]
	if tag == "" {
		return nil, errors.New("syslog tag not specified")
	}

	facility := syslog.LOG_USER
	if facilityName, ok := options["facility"]; ok {
		if facility, ok = facilityMap[strings.ToLower(facilityName)]; !ok {
			return nil, errors.New("invalid syslog facility: " + facilityName)
		}
	}

	return NewSyslogFacility(logging.NewBasicFormatter(format), tag, facility)
})

func init() {
	logging.RegisterOutputPlugin("syslog", syslogPlugin)
}
<commit_msg>Added a package comment to syslog.go<commit_after>// Package syslog provides a syslog plugin for go-logging.
package syslog

import (
	"errors"
	"github.com/vaughan0/go-logging"
	"log/syslog"
	"strings"
)

// SyslogOutputter implements Outputter by logging to the system log daemon.
type SyslogOutputter struct {
	Writer    *syslog.Writer
	Formatter logging.Formatter
}

// Creates a new SyslogOutputter with a custom facility (see syslog.Priority).
func NewSyslogFacility(format logging.Formatter, tag string, facility syslog.Priority) (*SyslogOutputter, error) {
	writer, err := syslog.New(facility, tag)
	if err != nil {
		return nil, err
	}
	return &SyslogOutputter{
		Writer:    writer,
		Formatter: format,
	}, nil
}

// Creates a new SyslogOutputter with a name (tag) and the USER facility.
func NewSyslog(format logging.Formatter, tag string) (*SyslogOutputter, error) {
	return NewSyslogFacility(format, tag, syslog.LOG_USER)
}

// Implements Outputter.
func (s SyslogOutputter) Output(msg *logging.Message) {
	str := s.Formatter.Format(msg)
	switch msg.Level {
	case logging.Fatal:
		s.Writer.Crit(str)
	case logging.Error:
		s.Writer.Err(str)
	case logging.Warn:
		s.Writer.Warning(str)
	case logging.Notice:
		s.Writer.Notice(str)
	case logging.Info:
		s.Writer.Info(str)
	case logging.Debug, logging.Trace:
		s.Writer.Debug(str)
	default:
		s.Writer.Notice(str)
	}
}

var facilityMap = map[string]syslog.Priority{
	"kern":     syslog.LOG_KERN,
	"user":     syslog.LOG_USER,
	"mail":     syslog.LOG_MAIL,
	"daemon":   syslog.LOG_DAEMON,
	"auth":     syslog.LOG_AUTH,
	"syslog":   syslog.LOG_SYSLOG,
	"lpr":      syslog.LOG_LPR,
	"news":     syslog.LOG_NEWS,
	"uucp":     syslog.LOG_UUCP,
	"cron":     syslog.LOG_CRON,
	"authpriv": syslog.LOG_AUTHPRIV,
	"ftp":      syslog.LOG_FTP,
	"local0":   syslog.LOG_LOCAL0,
	"local1":   syslog.LOG_LOCAL1,
	"local2":   syslog.LOG_LOCAL2,
	"local3":   syslog.LOG_LOCAL3,
	"local4":   syslog.LOG_LOCAL4,
	"local5":   syslog.LOG_LOCAL5,
	"local6":   syslog.LOG_LOCAL6,
	"local7":   syslog.LOG_LOCAL7,
}

var syslogPlugin = logging.OutputPluginFunc(func(options map[string]string) (result logging.Outputter, err error) {

	// Setup formatter
	format := options["format"]
	if format == "" {
		return nil, errors.New("syslog formatting string not specified")
	}

	tag := options["tag"]
	if tag == "" {
		return nil, errors.New("syslog tag not specified")
	}

	facility := syslog.LOG_USER
	if facilityName, ok := options["facility"]; ok {
		if facility, ok = facilityMap[strings.ToLower(facilityName)]; !ok {
			return nil, errors.New("invalid syslog facility: " + facilityName)
		}
	}

	return NewSyslogFacility(logging.NewBasicFormatter(format), tag, facility)
})

func init() {
	logging.RegisterOutputPlugin("syslog", syslogPlugin)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/nilslice/cms/content"
	"github.com/nilslice/cms/management/editor"
	"github.com/nilslice/cms/management/manager"
	"github.com/nilslice/cms/system/db"
)

func main() {
	http.HandleFunc("/admin/edit", func(res http.ResponseWriter, req *http.Request) {
		switch req.Method {
		case http.MethodGet:
			q := req.URL.Query()
			i := q.Get("id")
			t := q.Get("type")
			contentType, ok := content.Types[t]
			if !ok {
				fmt.Fprintf(res, content.ErrTypeNotRegistered, t)
				return
			}
			post := contentType()

			if i != "" {
				fmt.Println("Need to show post id:", i, "(", t, ")")

				data, err := db.Get(t + ":" + i)
				if err != nil {
					fmt.Println(err)
					res.WriteHeader(http.StatusInternalServerError)
					return
				}

				err = json.Unmarshal(data, post)
				if err != nil {
					fmt.Println(err)
					res.WriteHeader(http.StatusInternalServerError)
					return
				}
			}

			view, err := manager.Manage(post.(editor.Editable), t)
			if err != nil {
				fmt.Println(err)
				res.WriteHeader(http.StatusInternalServerError)
				return
			}
			res.Header().Set("Content-Type", "text/html")
			res.Write(view)

		case http.MethodPost:
			err := req.ParseForm()
			if err != nil {
				fmt.Println(err)
				res.WriteHeader(http.StatusBadRequest)
				return
			}

			cid := req.FormValue("id")
			t := req.FormValue("type")
			fmt.Println("query data: t=", t, "id=", cid)

			id, err := db.Set(t+":"+cid, req.PostForm)
			if err != nil {
				fmt.Println(err)
				res.WriteHeader(http.StatusInternalServerError)
				return
			}

			fmt.Println(t, "post created:", id)
			scheme := req.URL.Scheme
			host := req.URL.Host
			path := req.URL.Path
			desURL := scheme + host + path + "?type=" + t + "&id=" + fmt.Sprintf("%d", id)
			http.Redirect(res, req, desURL, http.StatusFound)
		}
	})

	http.ListenAndServe(":8080", nil)

}
<commit_msg>removing fmt printlns from debugging<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/nilslice/cms/content"
	"github.com/nilslice/cms/management/editor"
	"github.com/nilslice/cms/management/manager"
	"github.com/nilslice/cms/system/db"
)

func main() {
	http.HandleFunc("/admin/edit", func(res http.ResponseWriter, req *http.Request) {
		switch req.Method {
		case http.MethodGet:
			q := req.URL.Query()
			i := q.Get("id")
			t := q.Get("type")
			contentType, ok := content.Types[t]
			if !ok {
				fmt.Fprintf(res, content.ErrTypeNotRegistered, t)
				return
			}
			post := contentType()

			if i != "" {
				data, err := db.Get(t + ":" + i)
				if err != nil {
					fmt.Println(err)
					res.WriteHeader(http.StatusInternalServerError)
					return
				}

				err = json.Unmarshal(data, post)
				if err != nil {
					fmt.Println(err)
					res.WriteHeader(http.StatusInternalServerError)
					return
				}
			}

			view, err := manager.Manage(post.(editor.Editable), t)
			if err != nil {
				fmt.Println(err)
				res.WriteHeader(http.StatusInternalServerError)
				return
			}
			res.Header().Set("Content-Type", "text/html")
			res.Write(view)

		case http.MethodPost:
			err := req.ParseForm()
			if err != nil {
				fmt.Println(err)
				res.WriteHeader(http.StatusBadRequest)
				return
			}

			cid := req.FormValue("id")
			t := req.FormValue("type")
			id, err := db.Set(t+":"+cid, req.PostForm)
			if err != nil {
				fmt.Println(err)
				res.WriteHeader(http.StatusInternalServerError)
				return
			}

			scheme := req.URL.Scheme
			host := req.URL.Host
			path := req.URL.Path
			id = fmt.Sprintf("%d", id)
			desURL := scheme + host + path + "?type=" + t + "&id=" + id
			http.Redirect(res, req, desURL, http.StatusFound)
		}
	})

	http.ListenAndServe(":8080", nil)

}
<|endoftext|>


<commit_before>package crowler

import (
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/celrenheit/spider"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type originType struct {
	City    string `bson:"city"`
	State   string `bson:"state"`
	Country string `bson:"country"`
}

type brotherType struct {
	ID         bson.ObjectId `bson:"_id"`
	Name       string        `bson:"name"`
	Birthdate  time.Time     `bson:"birthdate"`
	Origin     originType    `bson:"origin"`
	Occupation string        `bson:"occupation"`
	Edition    int           `bson:"edition"`
}

type brothersList []interface{}

var newBrothers brothersList

var bbbSpider = spider.Get("https://pt.wikipedia.org/wiki/Lista_de_participantes_do_Big_Brother_Brasil", func(ctx *spider.Context) error {
	fmt.Println(time.Now())
	// Execute the request
	if _, err := ctx.DoRequest(); err != nil {
		return err
	}

	// Get goquery's html parser
	htmlparser, err := ctx.HTMLParser()
	if err != nil {
		return err
	}

	summary := htmlparser.Find(".wikitable > tbody")

	for i := 0; i < summary.Length(); i++ {
		brothers := summary.Eq(i).Find(".wikitable > tbody > tr")

		for j := 0; j < brothers.Length(); j++ {
			brother := brothers.Eq(j).Find(".wikitable > tbody > tr > td")

			if brother.Length() > 0 {
				dateRE := regexp.MustCompile(`\(\d{4}\)`)
				birthdateString := dateRE.FindString(brother.Eq(1).Text())
				if birthdateString != "" {
					birthdateString = strings.Trim(birthdateString, "()")
					birthdateString = fmt.Sprintf("01/01/%s", birthdateString)
				} else {
					birthdateString = strings.Split(brother.Eq(1).Text(), "-")[0]
					birthdateString = strings.Trim(birthdateString, " ")
				}

				birthdate, err := time.Parse("02/01/2006", birthdateString)
				if err != nil {
					return err
				}

				newBrother := brotherType{
					ID:        bson.NewObjectId(),
					Name:      brother.Eq(0).Text(),
					Birthdate: birthdate,
					Edition:   i + 1,
				}

				var origin []string

				if i < 12 {
					newBrother.Occupation = brother.Eq(2).Text()

					origin = strings.Split(brother.Eq(3).Text(), ",")
				} else {
					newBrother.Occupation = brother.Eq(3).Text()

					origin = strings.Split(brother.Eq(2).Text(), ",")
				}

				if len(origin) > 1 {
					newBrother.Origin.City = strings.Trim(origin[0], " ")
					newBrother.Origin.State = strings.Trim(origin[1], " ")
					newBrother.Origin.Country = "Brasil"
				} else {
					newBrother.Origin.Country = origin[0]
				}

				newBrothers = append(newBrothers, newBrother)
			}
		}
	}

	s, err := mgo.Dial("mongodb://localhost")
	if err != nil {
		return err
	}

	defer s.Close()

	err = s.DB("bbb").C("brothers").Insert(newBrothers...)

	return err
})

func SaveBrothersData() (err error) {
	ctx, err := bbbSpider.Setup(nil)
	if err != nil {
		return
	}

	err = bbbSpider.Spin(ctx)

	return

}
<commit_msg>brothers crowler refacs<commit_after>package crowler

import (
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/celrenheit/spider"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type originType struct {
	City    string `bson:"city"`
	State   string `bson:"state"`
	Country string `bson:"country"`
}

type brotherType struct {
	ID         bson.ObjectId `bson:"_id"`
	Name       string        `bson:"name"`
	Birthdate  time.Time     `bson:"birthdate"`
	Origin     originType    `bson:"origin"`
	Occupation string        `bson:"occupation"`
	Edition    int           `bson:"edition"`
}

type brothersList []interface{}

var newBrothers brothersList

func formatBirthdate(birthdate string) (formatedBirthdate time.Time, err error) {
	dateRE := regexp.MustCompile(`\(\d{4}\)`)
	birthdateString := dateRE.FindString(birthdate)
	if birthdateString != "" {
		birthdateString = strings.Trim(birthdateString, "()")
		birthdateString = fmt.Sprintf("01/01/%s", birthdateString)
	} else {
		birthdateString = strings.Split(birthdate, "-")[0]
		birthdateString = strings.Trim(birthdateString, " ")
	}

	formatedBirthdate, err = time.Parse("02/01/2006", birthdateString)

	return
}

var bbbSpider = spider.Get("https://pt.wikipedia.org/wiki/Lista_de_participantes_do_Big_Brother_Brasil", func(ctx *spider.Context) error {
	// Execute the request
	if _, err := ctx.DoRequest(); err != nil {
		return err
	}

	// Get goquery's html parser
	htmlparser, err := ctx.HTMLParser()
	if err != nil {
		return err
	}

	summary := htmlparser.Find(".wikitable > tbody")

	for i := 0; i < summary.Length(); i++ {
		brothers := summary.Eq(i).Find(".wikitable > tbody > tr")

		for j := 0; j < brothers.Length(); j++ {
			brother := brothers.Eq(j).Find(".wikitable > tbody > tr > td")

			if brother.Length() > 0 {
				birthdate, err := formatBirthdate(brother.Eq(1).Text())
				if err != nil {
					return err
				}

				newBrother := brotherType{
					ID:        bson.NewObjectId(),
					Name:      brother.Eq(0).Text(),
					Birthdate: birthdate,
					Edition:   i + 1,
				}

				var origin []string

				if i < 12 {
					newBrother.Occupation = brother.Eq(2).Text()

					origin = strings.Split(brother.Eq(3).Text(), ",")
				} else {
					newBrother.Occupation = brother.Eq(3).Text()

					origin = strings.Split(brother.Eq(2).Text(), ",")
				}

				if len(origin) > 1 {
					newBrother.Origin.City = strings.Trim(origin[0], " ")
					newBrother.Origin.State = strings.Trim(origin[1], " ")
					newBrother.Origin.Country = "Brasil"
				} else {
					newBrother.Origin.Country = origin[0]
				}

				newBrothers = append(newBrothers, newBrother)
			}
		}
	}

	s, err := mgo.Dial("mongodb://localhost")
	if err != nil {
		return err
	}

	defer s.Close()

	err = s.DB("bbb").C("brothers").Insert(newBrothers...)

	return err
})

// SaveBrothersData get the brothers data and save in the database
func SaveBrothersData() (err error) {
	ctx, err := bbbSpider.Setup(nil)
	if err != nil {
		return
	}

	err = bbbSpider.Spin(ctx)

	return

}
<|endoftext|>


<commit_before>package appStats

import (
	"fmt"
	//"github.com/Sirupsen/logrus"
	//"os"
  "sort"
	"sync"
	//"time"
  "github.com/jroimartin/gocui"
  "github.com/cloudfoundry/cli/plugin"
  //cfclient "github.com/cloudfoundry-community/go-cfclient"
  //"github.com/kkellner/cloudfoundry-top-plugin/debug"
  "github.com/kkellner/cloudfoundry-top-plugin/metadata"
  "github.com/mohae/deepcopy"
)

type AppStatsUI struct {
  processor     *AppStatsEventProcessor
  cliConnection   plugin.CliConnection
  mu  sync.Mutex // protects ctr

  filterAppName string

  lastRefreshAppMap      map[string]*AppStats
}

func NewAppStatsUI(cliConnection plugin.CliConnection ) *AppStatsUI {
  processor := NewAppStatsEventProcessor()
  return &AppStatsUI {
    processor:  processor,
    cliConnection: cliConnection,
  }
}

func (asUI *AppStatsUI) Start() {
  go asUI.loadMetadata()
}

func (asUI *AppStatsUI) GetProcessor() *AppStatsEventProcessor {
    return asUI.processor
}

func (asUI *AppStatsUI) ClearStats(g *gocui.Gui, v *gocui.View) error {
  asUI.processor.Clear()
	return nil
}

func (asUI *AppStatsUI) UpdateDisplay(g *gocui.Gui) error {
	asUI.mu.Lock()
	orgAppMap := asUI.processor.GetAppMap()
  m := deepcopy.Copy(orgAppMap).(map[string]*AppStats)
	asUI.mu.Unlock()

  asUI.updateHeader(g, m)

  v, err := g.View("detailView")
  if err != nil {
		return err
	}

  //maxX, maxY := v.Size()
  _, maxY := v.Size()
	if len(m) > 0 {
		v.Clear()
    row := 1
		fmt.Fprintf(v, "%-50v %-10v %-10v %6v %6v %6v %6v %6v %6v %6v\n",
      "APPLICATION","SPACE","ORG", "2XX","3XX","4XX","5XX","TOTAL", "INTRVL", "CPU%")

    sortedStatList := asUI.getStats2(m)

    for _, appStats := range sortedStatList {

      appId := formatUUID(appStats.AppUUID)
      lastEventCount := uint64(0)
      if asUI.lastRefreshAppMap[appId] != nil {
        lastEventCount = asUI.lastRefreshAppMap[appId].EventCount
      }
      eventsPerRefresh := appStats.EventCount - lastEventCount

      maxCpuPercentage := -1.0
      for _, cm := range appStats.ContainerMetric {
        if (cm != nil) {
          cpuPercentage := *cm.CpuPercentage
          if (cpuPercentage > maxCpuPercentage) {
            maxCpuPercentage = cpuPercentage
          }
        }
      }

      row++
      fmt.Fprintf(v, "%-50.50v %-10.10v %-10.10v %6d %6d %6d %6d %6d %6d %6.2f\n",
          appStats.AppName,
          appStats.SpaceName,
          appStats.OrgName,
          appStats.Event2xxCount,
          appStats.Event3xxCount,
          appStats.Event4xxCount,
          appStats.Event5xxCount,
          appStats.EventCount, eventsPerRefresh,
          maxCpuPercentage)
      if row == maxY {
        break
      }
		}
    asUI.lastRefreshAppMap = m

	} else {
		v.Clear()
		fmt.Fprintln(v, "No data yet...")
	}
	return nil

}

func (asUI *AppStatsUI) getStats2(statsMap map[string]*AppStats) []*AppStats {
  s := make(dataSlice, 0, len(statsMap))
  for _, d := range statsMap {

      appMetadata := metadata.FindAppMetadata(d.AppId)
      appName := appMetadata.Name
      if appName == "" {
        appName = d.AppId
        //appName = appStats.AppUUID.String()
      }
      d.AppName = appName

      spaceMetadata := metadata.FindSpaceMetadata(appMetadata.SpaceGuid)
      spaceName := spaceMetadata.Name
      if spaceName == "" {
        spaceName = "unknown"
      }
      d.SpaceName = spaceName

      orgMetadata := metadata.FindOrgMetadata(spaceMetadata.OrgGuid)
      orgName := orgMetadata.Name
      if orgName == "" {
        orgName = "unknown"
      }
      d.OrgName = orgName

      s = append(s, d)
  }
  sort.Sort(sort.Reverse(s))
  return s
}

func (asUI *AppStatsUI) updateHeader(g *gocui.Gui, appStatsMap map[string]*AppStats) error {
  v, err := g.View("summaryView")
  if err != nil {
    return err
  }
  fmt.Fprintf(v, "Total Apps: %-11v", metadata.AppMetadataSize())
  fmt.Fprintf(v, "Reporting Apps: %-11v", len(appStatsMap))
  return nil
}

func (asUI *AppStatsUI) loadMetadata() {
  metadata.LoadAppCache(asUI.cliConnection)
  metadata.LoadSpaceCache(asUI.cliConnection)
  metadata.LoadOrgCache(asUI.cliConnection)
}
<commit_msg>Max cpu per instance<commit_after>package appStats

import (
	"fmt"
	//"github.com/Sirupsen/logrus"
	//"os"
  "sort"
	"sync"
	//"time"
  "github.com/jroimartin/gocui"
  "github.com/cloudfoundry/cli/plugin"
  //cfclient "github.com/cloudfoundry-community/go-cfclient"
  //"github.com/kkellner/cloudfoundry-top-plugin/debug"
  "github.com/kkellner/cloudfoundry-top-plugin/metadata"
  "github.com/mohae/deepcopy"
)

type AppStatsUI struct {
  processor     *AppStatsEventProcessor
  cliConnection   plugin.CliConnection
  mu  sync.Mutex // protects ctr

  filterAppName string

  lastRefreshAppMap      map[string]*AppStats
}

func NewAppStatsUI(cliConnection plugin.CliConnection ) *AppStatsUI {
  processor := NewAppStatsEventProcessor()
  return &AppStatsUI {
    processor:  processor,
    cliConnection: cliConnection,
  }
}

func (asUI *AppStatsUI) Start() {
  go asUI.loadMetadata()
}

func (asUI *AppStatsUI) GetProcessor() *AppStatsEventProcessor {
    return asUI.processor
}

func (asUI *AppStatsUI) ClearStats(g *gocui.Gui, v *gocui.View) error {
  asUI.processor.Clear()
	return nil
}

func (asUI *AppStatsUI) UpdateDisplay(g *gocui.Gui) error {
	asUI.mu.Lock()
	orgAppMap := asUI.processor.GetAppMap()
  m := deepcopy.Copy(orgAppMap).(map[string]*AppStats)
	asUI.mu.Unlock()

  asUI.updateHeader(g, m)

  v, err := g.View("detailView")
  if err != nil {
		return err
	}

  //maxX, maxY := v.Size()
  _, maxY := v.Size()
	if len(m) > 0 {
		v.Clear()
    row := 1
		fmt.Fprintf(v, "%-50v %-10v %-10v %6v %6v %6v %6v %6v %6v %9v\n",
      "APPLICATION","SPACE","ORG", "2XX","3XX","4XX","5XX","TOTAL", "INTR", "CPU%/I ")

    sortedStatList := asUI.getStats2(m)

    for _, appStats := range sortedStatList {

      appId := formatUUID(appStats.AppUUID)
      lastEventCount := uint64(0)
      if asUI.lastRefreshAppMap[appId] != nil {
        lastEventCount = asUI.lastRefreshAppMap[appId].EventCount
      }
      eventsPerRefresh := appStats.EventCount - lastEventCount

      maxCpuPercentage := -1.0
      maxCpuAppInstance := -1
      for i, cm := range appStats.ContainerMetric {
        if (cm != nil) {
          cpuPercentage := *cm.CpuPercentage
          if (cpuPercentage > maxCpuPercentage) {
            maxCpuPercentage = cpuPercentage
            maxCpuAppInstance = i
          }
        }
      }

      maxCpuInfo := ""
      if maxCpuPercentage==-1 {
        maxCpuInfo = fmt.Sprintf("%8v", "--")
      } else {
        maxCpuInfo = fmt.Sprintf("%6.2f/%-2v", maxCpuPercentage, maxCpuAppInstance)
      }

      row++
      fmt.Fprintf(v, "%-50.50v %-10.10v %-10.10v %6d %6d %6d %6d %6d %6d %8v\n",
          appStats.AppName,
          appStats.SpaceName,
          appStats.OrgName,
          appStats.Event2xxCount,
          appStats.Event3xxCount,
          appStats.Event4xxCount,
          appStats.Event5xxCount,
          appStats.EventCount, eventsPerRefresh,
          maxCpuInfo)
      if row == maxY {
        break
      }
		}
    asUI.lastRefreshAppMap = m

	} else {
		v.Clear()
		fmt.Fprintln(v, "No data yet...")
	}
	return nil

}

func (asUI *AppStatsUI) getStats2(statsMap map[string]*AppStats) []*AppStats {
  s := make(dataSlice, 0, len(statsMap))
  for _, d := range statsMap {

      appMetadata := metadata.FindAppMetadata(d.AppId)
      appName := appMetadata.Name
      if appName == "" {
        appName = d.AppId
        //appName = appStats.AppUUID.String()
      }
      d.AppName = appName

      spaceMetadata := metadata.FindSpaceMetadata(appMetadata.SpaceGuid)
      spaceName := spaceMetadata.Name
      if spaceName == "" {
        spaceName = "unknown"
      }
      d.SpaceName = spaceName

      orgMetadata := metadata.FindOrgMetadata(spaceMetadata.OrgGuid)
      orgName := orgMetadata.Name
      if orgName == "" {
        orgName = "unknown"
      }
      d.OrgName = orgName

      s = append(s, d)
  }
  sort.Sort(sort.Reverse(s))
  return s
}

func (asUI *AppStatsUI) updateHeader(g *gocui.Gui, appStatsMap map[string]*AppStats) error {
  v, err := g.View("summaryView")
  if err != nil {
    return err
  }
  fmt.Fprintf(v, "Total Apps: %-11v", metadata.AppMetadataSize())
  fmt.Fprintf(v, "Reporting Apps: %-11v", len(appStatsMap))
  return nil
}

func (asUI *AppStatsUI) loadMetadata() {
  metadata.LoadAppCache(asUI.cliConnection)
  metadata.LoadSpaceCache(asUI.cliConnection)
  metadata.LoadOrgCache(asUI.cliConnection)
}
<|endoftext|>


<commit_before>package crypto

import (
	"bytes"
	"compress/zlib"
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"crypto/x509"
	"io/ioutil"

	"github.com/lucas-clemente/quic-go/utils"
)

// KeyData stores a key and a certificate for the server proof
type KeyData struct {
	key  *rsa.PrivateKey
	cert *x509.Certificate
}

// LoadKeyData loads the key and cert from files
func LoadKeyData(certFileName string, keyFileName string) (*KeyData, error) {
	keyDER, err := ioutil.ReadFile(keyFileName)
	if err != nil {
		return nil, err
	}
	key, err := x509.ParsePKCS1PrivateKey(keyDER)
	if err != nil {
		return nil, err
	}
	certDER, err := ioutil.ReadFile(certFileName)
	if err != nil {
		return nil, err
	}
	cert, err := x509.ParseCertificate(certDER)
	if err != nil {
		return nil, err
	}

	return &KeyData{key: key, cert: cert}, nil
}

// SignServerProof signs CHLO and server config for use in the server proof
func (kd *KeyData) SignServerProof(chlo []byte, serverConfigData []byte) ([]byte, error) {
	hash := sha256.New()
	hash.Write([]byte("QUIC CHLO and server config signature\x00"))
	chloHash := sha256.Sum256(chlo)
	hash.Write([]byte{32, 0, 0, 0})
	hash.Write(chloHash[:])
	hash.Write(serverConfigData)
	return rsa.SignPSS(rand.Reader, kd.key, crypto.SHA256, hash.Sum(nil), &rsa.PSSOptions{SaltLength: 32})
}
<commit_msg>implement basic certificate compression<commit_after>package crypto

import (
	"bytes"
	"compress/zlib"
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"crypto/x509"
	"io/ioutil"

	"github.com/lucas-clemente/quic-go/utils"
)

// KeyData stores a key and a certificate for the server proof
type KeyData struct {
	key  *rsa.PrivateKey
	cert *x509.Certificate
}

// LoadKeyData loads the key and cert from files
func LoadKeyData(certFileName string, keyFileName string) (*KeyData, error) {
	keyDER, err := ioutil.ReadFile(keyFileName)
	if err != nil {
		return nil, err
	}
	key, err := x509.ParsePKCS1PrivateKey(keyDER)
	if err != nil {
		return nil, err
	}
	certDER, err := ioutil.ReadFile(certFileName)
	if err != nil {
		return nil, err
	}
	cert, err := x509.ParseCertificate(certDER)
	if err != nil {
		return nil, err
	}

	return &KeyData{key: key, cert: cert}, nil
}

// SignServerProof signs CHLO and server config for use in the server proof
func (kd *KeyData) SignServerProof(chlo []byte, serverConfigData []byte) ([]byte, error) {
	hash := sha256.New()
	hash.Write([]byte("QUIC CHLO and server config signature\x00"))
	chloHash := sha256.Sum256(chlo)
	hash.Write([]byte{32, 0, 0, 0})
	hash.Write(chloHash[:])
	hash.Write(serverConfigData)
	return rsa.SignPSS(rand.Reader, kd.key, crypto.SHA256, hash.Sum(nil), &rsa.PSSOptions{SaltLength: 32})
}

// GetCERTdata gets the certificate in the format described by the QUIC crypto doc
func (kd *KeyData) GetCERTdata() []byte {
	b := &bytes.Buffer{}
	b.WriteByte(1) // Entry type compressed
	b.WriteByte(0) // Entry type end_of_list
	utils.WriteUint32(b, uint32(len(kd.cert.Raw)+4))
	gz := zlib.NewWriter(b)
	lenCert := len(kd.cert.Raw)
	gz.Write([]byte{
		byte(lenCert & 0xff),
		byte((lenCert >> 8) & 0xff),
		byte((lenCert >> 16) & 0xff),
		byte((lenCert >> 24) & 0xff),
	})
	gz.Write(kd.cert.Raw)
	gz.Close()
	return b.Bytes()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/url"

	"github.com/golang/protobuf/proto"
	pb "github.com/googleapis/gnostic/OpenAPIv2"
	"github.com/googleapis/gnostic/apps/discovery/disco"
	"github.com/googleapis/gnostic/compiler"
)

// Select an API.
const apiName = "people"
const apiVersion = "v1"

func main() {
	// Read the list of APIs from the apis/list service.
	apiListServiceURL := "https://www.googleapis.com/discovery/v1/apis"
	bytes, err := compiler.FetchFile(apiListServiceURL)
	if err != nil {
		log.Fatal("%+v", err)
	}
	// Unpack the apis/list response.
	listResponse, err := disco.NewList(bytes)
	// List the APIs.
	for _, api := range listResponse.APIs {
		fmt.Printf("%s\n", api.ID)
	}
	// Get the description of an API
	api := listResponse.APIWithID(apiName + ":" + apiVersion)
	if api == nil {
		log.Fatal("Error: API not found")
	}
	fmt.Printf("API: %+v\n", api)
	// Fetch the discovery description of the API.
	bytes, err = compiler.FetchFile(api.DiscoveryRestURL)
	if err != nil {
		log.Fatal("%+v", err)
	}
	// Unpack the discovery response.
	discoveryDocument, err := disco.NewDocument(bytes)
	if err != nil {
		log.Fatal("%+v", err)
	}
	fmt.Printf("DISCOVERY: %+v\n", discoveryDocument)
	// Generate the OpenAPI equivalent
	openAPIDocument := buildOpenAPI2Document(discoveryDocument)
	bytes, err = proto.Marshal(openAPIDocument)
	if err != nil {
		panic(err)
	}
	err = ioutil.WriteFile(apiName+"-"+apiVersion+".pb", bytes, 0644)
	if err != nil {
		panic(err)
	}
}

func addOpenAPI2SchemaForSchema(d *pb.Document, name string, schema *disco.Schema) {
	fmt.Printf("SCHEMA %s\n", name)
	d.Definitions.AdditionalProperties = append(d.Definitions.AdditionalProperties,
		&pb.NamedSchema{
			Name:  schema.Name,
			Value: buildOpenAPI2SchemaForSchema(schema),
		})
}

func buildOpenAPI2SchemaForSchema(schema *disco.Schema) *pb.Schema {
	s := &pb.Schema{}

	if description := schema.Description; description != "" {
		s.Description = description
	}
	if typeName := schema.Type; typeName != "" {
		s.Type = &pb.TypeItem{[]string{typeName}}
	}
	if ref := schema.Ref; ref != "" {
		s.XRef = "#/definitions/" + ref
	}
	if len(schema.Enums) > 0 {
		for _, e := range schema.Enums {
			s.Enum = append(s.Enum, &pb.Any{Yaml: e})
		}
	}
	if schema.ItemSchema != nil {
		s2 := buildOpenAPI2SchemaForSchema(schema.ItemSchema)
		s.Items = &pb.ItemsItem{}
		s.Items.Schema = append(s.Items.Schema, s2)
	}
	if len(schema.Properties) > 0 {
		s.Properties = &pb.Properties{}
		for _, property := range schema.Properties {
			s.Properties.AdditionalProperties = append(s.Properties.AdditionalProperties,
				&pb.NamedSchema{
					Name:  property.Name,
					Value: buildOpenAPI2SchemaForSchema(property.Schema),
				},
			)
		}
	}
	return s
}

func buildOpenAPI2OperationForMethod(method *disco.Method) *pb.Operation {
	fmt.Printf("METHOD %s %s %s %s\n", method.Name, method.FlatPath, method.HTTPMethod, method.ID)
	//fmt.Printf("MAP %+v\n", method.JSONMap)

	parameters := make([]*pb.ParametersItem, 0)
	for _, p := range method.Parameters {
		fmt.Printf("- PARAMETER %+v\n", p)
		parameters = append(parameters,
			&pb.ParametersItem{
				Oneof: &pb.ParametersItem_Parameter{
					Parameter: &pb.Parameter{
						Oneof: &pb.Parameter_NonBodyParameter{
							NonBodyParameter: &pb.NonBodyParameter{
								Oneof: &pb.NonBodyParameter_QueryParameterSubSchema{
									QueryParameterSubSchema: &pb.QueryParameterSubSchema{
										Name:        p.Name,
										In:          "XXX",
										Description: p.Description,
										Required:    false,
										Type:        "XXX",
										Format:      "XXX",
									},
								},
							},
						},
					},
				},
			})
	}

	fmt.Printf("- RESPONSE %+v\n", method.Response)

	responses := &pb.Responses{
		ResponseCode: []*pb.NamedResponseValue{
			&pb.NamedResponseValue{
				Name: "201",
				Value: &pb.ResponseValue{
					Oneof: &pb.ResponseValue_Response{
						Response: &pb.Response{
							Description: "Null response",
						},
					},
				},
			},
			&pb.NamedResponseValue{
				Name: "default",
				Value: &pb.ResponseValue{
					Oneof: &pb.ResponseValue_Response{
						Response: &pb.Response{
							Description: "unexpected error",
							Schema: &pb.SchemaItem{
								Oneof: &pb.SchemaItem_Schema{
									Schema: &pb.Schema{
										XRef: "#/definitions/Error",
									},
								},
							},
						},
					},
				},
			},
		},
	}

	return &pb.Operation{
		Summary:     method.Description,
		OperationId: method.ID,
		Responses:   responses,
		Parameters:  parameters,
	}
}

func getOpenAPI2PathItemForPath(d *pb.Document, path string) *pb.PathItem {
	// First, try to find a path item with the specified path. If it exists, return it.
	for _, item := range d.Paths.Path {
		if item.Name == path {
			return item.Value
		}
	}
	// Otherwise, create and return a new path item.
	pathItem := &pb.PathItem{}
	d.Paths.Path = append(d.Paths.Path,
		&pb.NamedPathItem{
			Name:  path,
			Value: pathItem,
		},
	)
	return pathItem
}

func addOpenAPI2PathsForMethod(d *pb.Document, method *disco.Method) {
	operation := buildOpenAPI2OperationForMethod(method)
	pathItem := getOpenAPI2PathItemForPath(d, "/"+method.FlatPath)
	switch method.HTTPMethod {
	case "GET":
		pathItem.Get = operation
	case "POST":
		pathItem.Post = operation
	case "PUT":
		pathItem.Put = operation
	case "DELETE":
		pathItem.Delete = operation
	default:
		log.Printf("UNKNOWN HTTP METHOD %s", method.HTTPMethod)
	}
}

func addOpenAPI2PathsForResource(d *pb.Document, resource *disco.Resource) {
	fmt.Printf("RESOURCE %s (%s)\n", resource.Name, resource.FullName)
	for _, method := range resource.Methods {
		addOpenAPI2PathsForMethod(d, method)
	}
	for _, resource2 := range resource.Resources {
		addOpenAPI2PathsForResource(d, resource2)
	}
}

func buildOpenAPI2Document(api *disco.Document) *pb.Document {
	d := &pb.Document{}
	d.Swagger = "2.0"
	d.Info = &pb.Info{
		Title:       api.Title,
		Version:     api.Version,
		Description: api.Description,
	}

	url, _ := url.Parse(api.RootURL)
	d.Host = url.Host
	d.BasePath = url.Path
	d.Schemes = []string{url.Scheme}
	d.Consumes = []string{"application/json"}
	d.Produces = []string{"application/json"}
	d.Paths = &pb.Paths{}
	d.Definitions = &pb.Definitions{}

	for name, schema := range api.Schemas {
		addOpenAPI2SchemaForSchema(d, name, schema)
	}

	for _, method := range api.Methods {
		addOpenAPI2PathsForMethod(d, method)
	}

	for _, resource := range api.Resources {
		addOpenAPI2PathsForResource(d, resource)
	}

	/*
		d.Paths.Path = append(d.Paths.Path,
			&pb.NamedPathItem{
				Name: "/pets",
				Value: &pb.PathItem{
					Get: &pb.Operation{
						Summary:     "List all pets",
						OperationId: "listPets",
						Tags:        []string{"pets"},
						Parameters: []*pb.ParametersItem{
							&pb.ParametersItem{
								Oneof: &pb.ParametersItem_Parameter{
									Parameter: &pb.Parameter{
										Oneof: &pb.Parameter_NonBodyParameter{
											NonBodyParameter: &pb.NonBodyParameter{
												Oneof: &pb.NonBodyParameter_QueryParameterSubSchema{
													QueryParameterSubSchema: &pb.QueryParameterSubSchema{
														Name:        "limit",
														In:          "query",
														Description: "How many items to return at one time (max 100)",
														Required:    false,
														Type:        "integer",
														Format:      "int32",
													},
												},
											},
										},
									},
								},
							},
						},
						Responses: &pb.Responses{
							ResponseCode: []*pb.NamedResponseValue{
								&pb.NamedResponseValue{
									Name: "200",
									Value: &pb.ResponseValue{
										Oneof: &pb.ResponseValue_Response{
											Response: &pb.Response{
												Description: "An paged array of pets", // [sic] match other examples
												Schema: &pb.SchemaItem{
													Oneof: &pb.SchemaItem_Schema{
														Schema: &pb.Schema{
															XRef: "#/definitions/Pets",
														},
													},
												},
												Headers: &pb.Headers{
													AdditionalProperties: []*pb.NamedHeader{
														&pb.NamedHeader{
															Name: "x-next",
															Value: &pb.Header{
																Type:        "string",
																Description: "A link to the next page of responses",
															},
														},
													},
												},
											},
										},
									},
								},
								&pb.NamedResponseValue{
									Name: "default",
									Value: &pb.ResponseValue{
										Oneof: &pb.ResponseValue_Response{
											Response: &pb.Response{
												Description: "unexpected error",
												Schema: &pb.SchemaItem{
													Oneof: &pb.SchemaItem_Schema{
														Schema: &pb.Schema{
															XRef: "#/definitions/Error",
														},
													},
												},
											},
										},
									},
								},
							},
						},
					},
					Post: &pb.Operation{
						Summary:     "Create a pet",
						OperationId: "createPets",
						Tags:        []string{"pets"},
						Parameters:  []*pb.ParametersItem{},
						Responses: &pb.Responses{
							ResponseCode: []*pb.NamedResponseValue{
								&pb.NamedResponseValue{
									Name: "201",
									Value: &pb.ResponseValue{
										Oneof: &pb.ResponseValue_Response{
											Response: &pb.Response{
												Description: "Null response",
											},
										},
									},
								},
								&pb.NamedResponseValue{
									Name: "default",
									Value: &pb.ResponseValue{
										Oneof: &pb.ResponseValue_Response{
											Response: &pb.Response{
												Description: "unexpected error",
												Schema: &pb.SchemaItem{
													Oneof: &pb.SchemaItem_Schema{
														Schema: &pb.Schema{
															XRef: "#/definitions/Error",
														},
													},
												},
											},
										},
									},
								},
							},
						},
					},
				}})
		d.Paths.Path = append(d.Paths.Path,
			&pb.NamedPathItem{
				Name: "/pets/{petId}",
				Value: &pb.PathItem{
					Get: &pb.Operation{
						Summary:     "Info for a specific pet",
						OperationId: "showPetById",
						Tags:        []string{"pets"},
						Parameters: []*pb.ParametersItem{
							&pb.ParametersItem{
								Oneof: &pb.ParametersItem_Parameter{
									Parameter: &pb.Parameter{
										Oneof: &pb.Parameter_NonBodyParameter{
											NonBodyParameter: &pb.NonBodyParameter{
												Oneof: &pb.NonBodyParameter_PathParameterSubSchema{
													PathParameterSubSchema: &pb.PathParameterSubSchema{
														Name:        "petId",
														In:          "path",
														Description: "The id of the pet to retrieve",
														Required:    true,
														Type:        "string",
													},
												},
											},
										},
									},
								},
							},
						},
						Responses: &pb.Responses{
							ResponseCode: []*pb.NamedResponseValue{
								&pb.NamedResponseValue{
									Name: "200",
									Value: &pb.ResponseValue{
										Oneof: &pb.ResponseValue_Response{
											Response: &pb.Response{
												Description: "Expected response to a valid request",
												Schema: &pb.SchemaItem{
													Oneof: &pb.SchemaItem_Schema{
														Schema: &pb.Schema{
															XRef: "#/definitions/Pets",
														},
													},
												},
											},
										},
									},
								},
								&pb.NamedResponseValue{
									Name: "default",
									Value: &pb.ResponseValue{
										Oneof: &pb.ResponseValue_Response{
											Response: &pb.Response{
												Description: "unexpected error",
												Schema: &pb.SchemaItem{
													Oneof: &pb.SchemaItem_Schema{
														Schema: &pb.Schema{
															XRef: "#/definitions/Error",
														},
													},
												},
											},
										},
									},
								},
							},
						},
					},
				}})
	*/
	/*
		d.Definitions.AdditionalProperties = append(d.Definitions.AdditionalProperties,
			&pb.NamedSchema{
				Name: "Pet",
				Value: &pb.Schema{
					Required: []string{"id", "name"},
					Properties: &pb.Properties{
						AdditionalProperties: []*pb.NamedSchema{
							&pb.NamedSchema{Name: "id", Value: &pb.Schema{
								Type:   &pb.TypeItem{[]string{"integer"}},
								Format: "int64"}},
							&pb.NamedSchema{Name: "name", Value: &pb.Schema{Type: &pb.TypeItem{[]string{"string"}}}},
							&pb.NamedSchema{Name: "tag", Value: &pb.Schema{Type: &pb.TypeItem{[]string{"string"}}}},
						},
					},
				}})
		d.Definitions.AdditionalProperties = append(d.Definitions.AdditionalProperties,
			&pb.NamedSchema{
				Name: "Pets",
				Value: &pb.Schema{
					Type:  &pb.TypeItem{[]string{"array"}},
					Items: &pb.ItemsItem{[]*pb.Schema{&pb.Schema{XRef: "#/definitions/Pet"}}},
				}})
		d.Definitions.AdditionalProperties = append(d.Definitions.AdditionalProperties,
			&pb.NamedSchema{
				Name: "Error",
				Value: &pb.Schema{
					Required: []string{"code", "message"},
					Properties: &pb.Properties{
						AdditionalProperties: []*pb.NamedSchema{
							&pb.NamedSchema{Name: "code", Value: &pb.Schema{
								Type:   &pb.TypeItem{[]string{"integer"}},
								Format: "int32"}},
							&pb.NamedSchema{Name: "message", Value: &pb.Schema{Type: &pb.TypeItem{[]string{"string"}}}},
						},
					},
				}})
	*/
	return d
}
<commit_msg>First pseudo-complete conversion from Discovery Format to OpenAPIv2.<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/url"

	"github.com/golang/protobuf/proto"
	pb "github.com/googleapis/gnostic/OpenAPIv2"
	"github.com/googleapis/gnostic/apps/discovery/disco"
	"github.com/googleapis/gnostic/compiler"
)

// Select an API.
const apiName = "people"
const apiVersion = "v1"

func main() {
	// Read the list of APIs from the apis/list service.
	apiListServiceURL := "https://www.googleapis.com/discovery/v1/apis"
	bytes, err := compiler.FetchFile(apiListServiceURL)
	if err != nil {
		log.Fatalf("%+v", err)
	}
	// Unpack the apis/list response.
	listResponse, err := disco.NewList(bytes)
	// List the APIs.
	for _, api := range listResponse.APIs {
		fmt.Printf("%s\n", api.ID)
	}
	// Get the description of an API
	api := listResponse.APIWithID(apiName + ":" + apiVersion)
	if api == nil {
		log.Fatalf("Error: API not found")
	}
	//fmt.Printf("API: %+v\n", api)
	// Fetch the discovery description of the API.
	bytes, err = compiler.FetchFile(api.DiscoveryRestURL)
	if err != nil {
		log.Fatalf("%+v", err)
	}
	// Unpack the discovery response.
	discoveryDocument, err := disco.NewDocument(bytes)
	if err != nil {
		log.Fatalf("%+v", err)
	}
	//fmt.Printf("DISCOVERY: %+v\n", discoveryDocument)
	// Generate the OpenAPI equivalent
	openAPIDocument := buildOpenAPI2DocumentForDocument(discoveryDocument)
	bytes, err = proto.Marshal(openAPIDocument)
	if err != nil {
		panic(err)
	}
	err = ioutil.WriteFile(apiName+"-"+apiVersion+".pb", bytes, 0644)
	if err != nil {
		panic(err)
	}
}

func addOpenAPI2SchemaForSchema(d *pb.Document, name string, schema *disco.Schema) {
	//log.Printf("SCHEMA %s\n", name)
	d.Definitions.AdditionalProperties = append(d.Definitions.AdditionalProperties,
		&pb.NamedSchema{
			Name:  schema.Name,
			Value: buildOpenAPI2SchemaForSchema(schema),
		})
}

func buildOpenAPI2SchemaForSchema(schema *disco.Schema) *pb.Schema {
	s := &pb.Schema{}

	if description := schema.Description; description != "" {
		s.Description = description
	}
	if typeName := schema.Type; typeName != "" {
		s.Type = &pb.TypeItem{[]string{typeName}}
	}
	if ref := schema.Ref; ref != "" {
		s.XRef = "#/definitions/" + ref
	}
	if len(schema.Enums) > 0 {
		for _, e := range schema.Enums {
			s.Enum = append(s.Enum, &pb.Any{Yaml: e})
		}
	}
	if schema.ItemSchema != nil {
		s2 := buildOpenAPI2SchemaForSchema(schema.ItemSchema)
		s.Items = &pb.ItemsItem{}
		s.Items.Schema = append(s.Items.Schema, s2)
	}
	if len(schema.Properties) > 0 {
		s.Properties = &pb.Properties{}
		for _, property := range schema.Properties {
			s.Properties.AdditionalProperties = append(s.Properties.AdditionalProperties,
				&pb.NamedSchema{
					Name:  property.Name,
					Value: buildOpenAPI2SchemaForSchema(property.Schema),
				},
			)
		}
	}
	return s
}

func buildOpenAPI2ParameterForParameter(p *disco.Parameter) *pb.Parameter {
	//log.Printf("- PARAMETER %+v\n", p.Name)
	typeName := p.Schema.Type
	format := p.Schema.Format
	location := p.Location
	switch location {
	case "query":
		return &pb.Parameter{
			Oneof: &pb.Parameter_NonBodyParameter{
				NonBodyParameter: &pb.NonBodyParameter{
					Oneof: &pb.NonBodyParameter_QueryParameterSubSchema{
						QueryParameterSubSchema: &pb.QueryParameterSubSchema{
							Name:        p.Name,
							In:          "query",
							Description: p.Description,
							Required:    false,
							Type:        typeName,
							Format:      format,
						},
					},
				},
			},
		}
	case "path":
		return &pb.Parameter{
			Oneof: &pb.Parameter_NonBodyParameter{
				NonBodyParameter: &pb.NonBodyParameter{
					Oneof: &pb.NonBodyParameter_PathParameterSubSchema{
						PathParameterSubSchema: &pb.PathParameterSubSchema{
							Name:        p.Name,
							In:          "path",
							Description: p.Description,
							Required:    false,
							Type:        typeName,
							Format:      format,
						},
					},
				},
			},
		}
	default:
		return nil
	}
}

func buildOpenAPI2ResponseForSchema(schema *disco.Schema) *pb.Response {
	//log.Printf("- RESPONSE %+v\n", schema)
	ref := schema.Ref
	if ref == "" {
		log.Printf("ERROR: UNHANDLED RESPONSE SCHEMA %+v", schema)
	}
	return &pb.Response{
		Description: "Successful operation",
		Schema: &pb.SchemaItem{
			Oneof: &pb.SchemaItem_Schema{
				Schema: &pb.Schema{
					XRef: "#/definitions/" + ref,
				},
			},
		},
	}
}

func buildOpenAPI2OperationForMethod(method *disco.Method) *pb.Operation {
	//log.Printf("METHOD %s %s %s %s\n", method.Name, method.FlatPath, method.HTTPMethod, method.ID)
	//log.Printf("MAP %+v\n", method.JSONMap)
	parameters := make([]*pb.ParametersItem, 0)
	for _, p := range method.Parameters {
		parameters = append(parameters, &pb.ParametersItem{
			Oneof: &pb.ParametersItem_Parameter{
				Parameter: buildOpenAPI2ParameterForParameter(p),
			},
		})
	}
	responses := &pb.Responses{
		ResponseCode: []*pb.NamedResponseValue{
			&pb.NamedResponseValue{
				Name: "default",
				Value: &pb.ResponseValue{
					Oneof: &pb.ResponseValue_Response{
						Response: buildOpenAPI2ResponseForSchema(method.Response),
					},
				},
			},
		},
	}
	return &pb.Operation{
		Summary:     method.Description,
		OperationId: method.ID,
		Parameters:  parameters,
		Responses:   responses,
	}
}

func getOpenAPI2PathItemForPath(d *pb.Document, path string) *pb.PathItem {
	// First, try to find a path item with the specified path. If it exists, return it.
	for _, item := range d.Paths.Path {
		if item.Name == path {
			return item.Value
		}
	}
	// Otherwise, create and return a new path item.
	pathItem := &pb.PathItem{}
	d.Paths.Path = append(d.Paths.Path,
		&pb.NamedPathItem{
			Name:  path,
			Value: pathItem,
		},
	)
	return pathItem
}

func addOpenAPI2PathsForMethod(d *pb.Document, method *disco.Method) {
	operation := buildOpenAPI2OperationForMethod(method)
	pathItem := getOpenAPI2PathItemForPath(d, "/"+method.FlatPath)
	switch method.HTTPMethod {
	case "GET":
		pathItem.Get = operation
	case "POST":
		pathItem.Post = operation
	case "PUT":
		pathItem.Put = operation
	case "DELETE":
		pathItem.Delete = operation
	default:
		log.Printf("ERROR: UNKNOWN HTTP METHOD %s", method.HTTPMethod)
	}
}

func addOpenAPI2PathsForResource(d *pb.Document, resource *disco.Resource) {
	//log.Printf("RESOURCE %s (%s)\n", resource.Name, resource.FullName)
	for _, method := range resource.Methods {
		addOpenAPI2PathsForMethod(d, method)
	}
	for _, resource2 := range resource.Resources {
		addOpenAPI2PathsForResource(d, resource2)
	}
}

func buildOpenAPI2DocumentForDocument(api *disco.Document) *pb.Document {
	d := &pb.Document{}
	d.Swagger = "2.0"
	d.Info = &pb.Info{
		Title:       api.Title,
		Version:     api.Version,
		Description: api.Description,
	}
	url, _ := url.Parse(api.RootURL)
	d.Host = url.Host
	d.BasePath = url.Path
	d.Schemes = []string{url.Scheme}
	d.Consumes = []string{"application/json"}
	d.Produces = []string{"application/json"}
	d.Paths = &pb.Paths{}
	d.Definitions = &pb.Definitions{}
	for name, schema := range api.Schemas {
		addOpenAPI2SchemaForSchema(d, name, schema)
	}
	for _, method := range api.Methods {
		addOpenAPI2PathsForMethod(d, method)
	}
	for _, resource := range api.Resources {
		addOpenAPI2PathsForResource(d, resource)
	}
	return d
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/reducedb/bloom"
	"github.com/reducedb/bloom/scalable"
)

type (
	Bloomer struct {
		bloom.Bloom
		filters []bloom.Bloom
	}
)

var (
	intersection = flag.Bool("i", false, "calculate the intersection")
	diff         = flag.Bool("d", false, "calculate the difference")
	union        = flag.Bool("u", false, "calculate the union")

	// activate lossy processing
	blooms = flag.Uint("blooms", 0, "number of bloom filters to use (lossy)")

	// buffered io
	stdout = bufio.NewWriterSize(os.Stdout, 4096)

	// total tokens in output
	total uint64
)

func main() {

	start := time.Now()

	defer func() {
		stdout.Flush()
		fmt.Fprintln(os.Stderr, "** Token Report **")
		fmt.Fprintln(os.Stderr, "Tokens output: ", total)
		fmt.Fprintln(os.Stderr, "Total time: ", time.Since(start))
	}()

	flag.Parse()

	if !*intersection && !*diff && !*union {
		fmt.Println("Usage: tt -[i,d,u] [-blooms N [-mash]] file1 file2[ file3..]")
		flag.PrintDefaults()
		os.Exit(1)
	}

	file_paths := flag.Args()

	switch {

	// activate bloom filter solution
	case *blooms > 0:

		if *union {

			unique_set := NewScalableBloom(*blooms)

			for _, file_path := range file_paths {

				file, err := os.Open(file_path)
				if err != nil {
					log.Fatal(err)
				}

				scanner := bufio.NewScanner(file)

				for scanner.Scan() {
					token := scanner.Bytes()
					if !unique_set.Check(token) {
						stdout.Write(token)
						stdout.WriteByte('\n')
						total++
						unique_set.Add(token)
					}
				}

				file.Close()

			}

			return
		}

		// multi file handling below
		sets := make([]bloom.Bloom, len(file_paths))

		// may require throttling due to disk thrashing
		// initial scan to fill the bloom filters
		for i, file_path := range file_paths {

			set := NewScalableBloom(*blooms)

			file, err := os.Open(file_path)
			if err != nil {
				log.Fatal(err)
			}

			scanner := bufio.NewScanner(file)

			for scanner.Scan() {
				set.Add(scanner.Bytes())
			}

			file.Close()

			sets[i] = set

		}

		// do the work
		switch {

		// unique set of tokens that exist in all files
		case *intersection:

			echoed_set := NewScalableBloom(*blooms)

			for _, file_path := range file_paths {

				file, err := os.Open(file_path)
				if err != nil {
					log.Fatal(err)
				}

				scanner := bufio.NewScanner(file)

			NEXT_TOKEN:
				for scanner.Scan() {

					token := scanner.Bytes()

					if echoed_set.Check(token) {
						goto NEXT_TOKEN
					}

					for _, set := range sets {
						if !set.Check(token) {
							goto NEXT_TOKEN
						}
					}

					stdout.Write(token)
					stdout.WriteByte('\n')
					total++
					echoed_set.Add(token)

				}

				file.Close()

			}

		// unique set of tokens not in the intersection
		case *diff:

			echoed_set := NewScalableBloom(*blooms)

			for _, file_path := range file_paths {

				file, err := os.Open(file_path)
				if err != nil {
					log.Fatal(err)
				}

				scanner := bufio.NewScanner(file)

				for scanner.Scan() {

					token := scanner.Bytes()

					if echoed_set.Check(token) {
						continue
					}

					for _, set := range sets {
						if !set.Check(token) {
							stdout.Write(token)
							stdout.WriteByte('\n')
							total++
							echoed_set.Add(token)
						}
					}

				}

				file.Close()

			}
		}

	// defaults to map solution
	default:

		if *union {

			unique_set := make(map[string]bool)

			for _, file_path := range file_paths {

				file, err := os.Open(file_path)
				if err != nil {
					log.Fatal(err)
				}

				scanner := bufio.NewScanner(file)

				for scanner.Scan() {
					token := scanner.Text()
					if _, exists := unique_set[token]; !exists {
						stdout.WriteString(token)
						stdout.WriteByte('\n')
						total++
						unique_set[token] = true
					}
				}

				file.Close()

			}

			return
		}

		// multi file handling below
		sets := make([]map[string]bool, len(file_paths))

		// may require throttling due to disk thrashing
		// initial scan to fill the bloom filters
		for i, file_path := range file_paths {

			set := make(map[string]bool)

			file, err := os.Open(file_path)
			if err != nil {
				log.Fatal(err)
			}

			scanner := bufio.NewScanner(file)

			for scanner.Scan() {
				set[scanner.Text()] = true
			}

			file.Close()

			sets[i] = set

		}

		// do the work
		switch {

		// unique set of tokens that exist in all files
		case *intersection:

			echoed_set := make(map[string]bool)

			for _, file_path := range file_paths {

				file, err := os.Open(file_path)
				if err != nil {
					log.Fatal(err)
				}

				scanner := bufio.NewScanner(file)

			NEXT_TOKEN2:
				for scanner.Scan() {

					token := scanner.Text()

					if _, echoed := echoed_set[token]; echoed {
						goto NEXT_TOKEN2
					}

					for _, set := range sets {
						if _, in_this_set := set[token]; !in_this_set {
							goto NEXT_TOKEN2
						}
					}

					stdout.WriteString(token)
					stdout.WriteByte('\n')
					total++
					echoed_set[token] = true

				}

				file.Close()

			}

		// unique set of tokens not in the intersection
		case *diff:

			echoed_set := make(map[string]bool)

			for _, file_path := range file_paths {

				file, err := os.Open(file_path)
				if err != nil {
					log.Fatal(err)
				}

				scanner := bufio.NewScanner(file)

				for scanner.Scan() {

					token := scanner.Text()

					if _, echoed := echoed_set[token]; echoed {
						continue
					}

					for _, set := range sets {
						if _, in_this_set := set[token]; !in_this_set {
							stdout.WriteString(token)
							stdout.WriteByte('\n')
							total++
							echoed_set[token] = true
							break
						}
					}

				}

				file.Close()

			}
		}

	}

}

func NewScalableBloom(size uint) bloom.Bloom {

	filters := make([]bloom.Bloom, size)

	for i, _ := range filters {
		filter := scalable.New(4096)
		// filter.SetHasher(adler32.New())
		filter.Reset()
		filters[i] = filter
	}

	return &Bloomer{
		filters: filters,
	}

}

func (b *Bloomer) Add(token []byte) bloom.Bloom {

	if *blooms > 1 {
		token = append(make([]byte, len(token)), token...)
	}

	for _, filter := range b.filters {
		filter.Add(token)
		if *blooms > 1 {
			mash(token)
		}
	}

	return b
}

func (b *Bloomer) Check(token []byte) bool {
	if *blooms > 1 {
		token = append(make([]byte, len(token)), token...)
	}
	for _, filter := range b.filters {
		if !filter.Check(token) {
			return false
		}
		if *blooms > 1 {
			mash(token)
		}
	}
	return true
}

// modifies the underlying structure
func mash(token []byte) {
	for i, c := range token {
		token[i] ^= (20 * c)
	}
}
<commit_msg>cleanup<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/reducedb/bloom"
	"github.com/reducedb/bloom/scalable"
)

type (
	Bloomer struct {
		bloom.Bloom
		filters []bloom.Bloom
	}
)

var (
	intersection = flag.Bool("i", false, "calculate the intersection")
	diff         = flag.Bool("d", false, "calculate the difference")
	union        = flag.Bool("u", false, "calculate the union")

	// activate lossy processing
	blooms = flag.Uint("blooms", 0, "number of bloom filters to use (lossy)")

	// buffered io
	stdout = bufio.NewWriterSize(os.Stdout, 4096)

	// total tokens in output
	total uint64
)

func main() {

	start := time.Now()

	defer func() {
		stdout.Flush()
		fmt.Fprintln(os.Stderr, "** Token Report **")
		fmt.Fprintln(os.Stderr, "Tokens output: ", total)
		fmt.Fprintln(os.Stderr, "Total time: ", time.Since(start))
	}()

	flag.Parse()

	if !*intersection && !*diff && !*union {
		fmt.Println("Usage: tt -[i,d,u] [-blooms N] file1 file2[ file3..]")
		flag.PrintDefaults()
		os.Exit(1)
	}

	file_paths := flag.Args()

	switch {

	// activate bloom filter solution
	case *blooms > 0:

		if *union {

			unique_set := NewScalableBloom(*blooms)

			for _, file_path := range file_paths {

				file, err := os.Open(file_path)
				if err != nil {
					log.Fatal(err)
				}

				scanner := bufio.NewScanner(file)

				for scanner.Scan() {
					token := scanner.Bytes()
					if !unique_set.Check(token) {
						stdout.Write(token)
						stdout.WriteByte('\n')
						total++
						unique_set.Add(token)
					}
				}

				file.Close()

			}

			return
		}

		// multi file handling below
		sets := make([]bloom.Bloom, len(file_paths))

		// may require throttling due to disk thrashing
		// initial scan to fill the bloom filters
		for i, file_path := range file_paths {

			set := NewScalableBloom(*blooms)

			file, err := os.Open(file_path)
			if err != nil {
				log.Fatal(err)
			}

			scanner := bufio.NewScanner(file)

			for scanner.Scan() {
				set.Add(scanner.Bytes())
			}

			file.Close()

			sets[i] = set

		}

		// do the work
		switch {

		// unique set of tokens that exist in all files
		case *intersection:

			echoed_set := NewScalableBloom(*blooms)

			for _, file_path := range file_paths {

				file, err := os.Open(file_path)
				if err != nil {
					log.Fatal(err)
				}

				scanner := bufio.NewScanner(file)

			NEXT_TOKEN:
				for scanner.Scan() {

					token := scanner.Bytes()

					if echoed_set.Check(token) {
						goto NEXT_TOKEN
					}

					for _, set := range sets {
						if !set.Check(token) {
							goto NEXT_TOKEN
						}
					}

					stdout.Write(token)
					stdout.WriteByte('\n')
					total++
					echoed_set.Add(token)

				}

				file.Close()

			}

		// unique set of tokens not in the intersection
		case *diff:

			echoed_set := NewScalableBloom(*blooms)

			for _, file_path := range file_paths {

				file, err := os.Open(file_path)
				if err != nil {
					log.Fatal(err)
				}

				scanner := bufio.NewScanner(file)

				for scanner.Scan() {

					token := scanner.Bytes()

					if echoed_set.Check(token) {
						continue
					}

					for _, set := range sets {
						if !set.Check(token) {
							stdout.Write(token)
							stdout.WriteByte('\n')
							total++
							echoed_set.Add(token)
						}
					}

				}

				file.Close()

			}
		}

	// defaults to map solution
	default:

		if *union {

			unique_set := make(map[string]bool)

			for _, file_path := range file_paths {

				file, err := os.Open(file_path)
				if err != nil {
					log.Fatal(err)
				}

				scanner := bufio.NewScanner(file)

				for scanner.Scan() {
					token := scanner.Text()
					if _, exists := unique_set[token]; !exists {
						stdout.WriteString(token)
						stdout.WriteByte('\n')
						total++
						unique_set[token] = true
					}
				}

				file.Close()

			}

			return
		}

		// multi file handling below
		sets := make([]map[string]bool, len(file_paths))

		// may require throttling due to disk thrashing
		// initial scan to fill the bloom filters
		for i, file_path := range file_paths {

			set := make(map[string]bool)

			file, err := os.Open(file_path)
			if err != nil {
				log.Fatal(err)
			}

			scanner := bufio.NewScanner(file)

			for scanner.Scan() {
				set[scanner.Text()] = true
			}

			file.Close()

			sets[i] = set

		}

		// do the work
		switch {

		// unique set of tokens that exist in all files
		case *intersection:

			echoed_set := make(map[string]bool)

			for _, file_path := range file_paths {

				file, err := os.Open(file_path)
				if err != nil {
					log.Fatal(err)
				}

				scanner := bufio.NewScanner(file)

			NEXT_TOKEN2:
				for scanner.Scan() {

					token := scanner.Text()

					if _, echoed := echoed_set[token]; echoed {
						goto NEXT_TOKEN2
					}

					for _, set := range sets {
						if _, in_this_set := set[token]; !in_this_set {
							goto NEXT_TOKEN2
						}
					}

					stdout.WriteString(token)
					stdout.WriteByte('\n')
					total++
					echoed_set[token] = true

				}

				file.Close()

			}

		// unique set of tokens not in the intersection
		case *diff:

			echoed_set := make(map[string]bool)

			for _, file_path := range file_paths {

				file, err := os.Open(file_path)
				if err != nil {
					log.Fatal(err)
				}

				scanner := bufio.NewScanner(file)

				for scanner.Scan() {

					token := scanner.Text()

					if _, echoed := echoed_set[token]; echoed {
						continue
					}

					for _, set := range sets {
						if _, in_this_set := set[token]; !in_this_set {
							stdout.WriteString(token)
							stdout.WriteByte('\n')
							total++
							echoed_set[token] = true
							break
						}
					}

				}

				file.Close()

			}
		}

	}

}

func NewScalableBloom(size uint) bloom.Bloom {

	filters := make([]bloom.Bloom, size)

	for i, _ := range filters {
		filter := scalable.New(4096)
		// filter.SetHasher(adler32.New())
		filter.Reset()
		filters[i] = filter
	}

	return &Bloomer{
		filters: filters,
	}

}

func (b *Bloomer) Add(token []byte) bloom.Bloom {

	if *blooms > 1 {
		token = append(make([]byte, len(token)), token...)
	}

	for _, filter := range b.filters {
		filter.Add(token)
		if *blooms > 1 {
			mash(token)
		}
	}

	return b
}

func (b *Bloomer) Check(token []byte) bool {
	if *blooms > 1 {
		token = append(make([]byte, len(token)), token...)
	}
	for _, filter := range b.filters {
		if !filter.Check(token) {
			return false
		}
		if *blooms > 1 {
			mash(token)
		}
	}
	return true
}

// modifies the underlying structure
func mash(token []byte) {
	for i, c := range token {
		token[i] ^= (20 * c)
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio";
	"flag";
	"fmt";
	"http";
	"io";
	"net";
	"os";
)

// hello world, the web server
func HelloServer(c *http.Conn, req *http.Request) {
	io.WriteString(c, "hello, world!\n");
}

// simple counter server
type Counter struct {
	n int;
}

func (ctr *Counter) ServeHTTP(c *http.Conn, req *http.Request) {
	fmt.Fprintf(c, "counter = %d\n", ctr.n);
	ctr.n++;
}

// simple file server
var webroot = flag.String("root", "/home/rsc", "web root directory")
func FileServer(c *http.Conn, req *http.Request) {
	c.SetHeader("content-type", "text/plain; charset=utf-8");
	path := *webroot + req.Url.Path;	// TODO: insecure: use os.CleanName
	fd, err := os.Open(path, os.O_RDONLY, 0);
	if err != nil {
		c.WriteHeader(http.StatusNotFound);
		fmt.Fprintf(c, "open %s: %v\n", path, err);
		return;
	}
	n, err1 := io.Copy(fd, c);
	fmt.Fprintf(c, "[%d bytes]\n", n);
}

// a channel (just for the fun of it)
type Chan chan int

func ChanCreate() Chan {
	c := make(Chan);
	go func(c Chan) {
		for x := 0;; x++ {
			c <- x
		}
	}(c);
	return c;
}

func (ch Chan) ServeHTTP(c *http.Conn, req *http.Request) {
	io.WriteString(c, fmt.Sprintf("channel send #%d\n", <-ch));
}

func main() {
	flag.Parse();
	http.Handle("/counter", new(Counter));
	http.Handle("/go/", http.HandlerFunc(FileServer));
	http.Handle("/go/hello", http.HandlerFunc(HelloServer));
	http.Handle("/chan", ChanCreate());
	err := http.ListenAndServe(":12345", nil);
	if err != nil {
		panic("ListenAndServe: ", err.String())
	}
}

<commit_msg>more fun with triv.go: flags and arguments<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio";
	"flag";
	"fmt";
	"http";
	"io";
	"net";
	"os";
)

// hello world, the web server
func HelloServer(c *http.Conn, req *http.Request) {
	io.WriteString(c, "hello, world!\n");
}

// simple counter server
type Counter struct {
	n int;
}

func (ctr *Counter) ServeHTTP(c *http.Conn, req *http.Request) {
	fmt.Fprintf(c, "counter = %d\n", ctr.n);
	ctr.n++;
}

// simple file server
var webroot = flag.String("root", "/home/rsc", "web root directory")
func FileServer(c *http.Conn, req *http.Request) {
	c.SetHeader("content-type", "text/plain; charset=utf-8");
	path := *webroot + req.Url.Path;	// TODO: insecure: use os.CleanName
	fd, err := os.Open(path, os.O_RDONLY, 0);
	if err != nil {
		c.WriteHeader(http.StatusNotFound);
		fmt.Fprintf(c, "open %s: %v\n", path, err);
		return;
	}
	n, err1 := io.Copy(fd, c);
	fmt.Fprintf(c, "[%d bytes]\n", n);
}

// simple flag server
var booleanflag = flag.Bool("boolean", true, "another flag for testing")
func FlagServer(c *http.Conn, req *http.Request) {
	c.SetHeader("content-type", "text/plain; charset=utf-8");
	fmt.Fprint(c, "Flags:\n");
	flag.VisitAll(func (f *flag.Flag) {
		if f.Value.String() != f.DefValue {
			fmt.Fprintf(c, "%s = %s [default = %s]\n", f.Name, f.Value.String(), f.DefValue);
		} else {
			fmt.Fprintf(c, "%s = %s\n", f.Name, f.Value.String());
		}
	});
}

// simple argument server
func ArgServer(c *http.Conn, req *http.Request) {
	for i, s := range sys.Args {
		fmt.Fprint(c, s, " ");
	}
}

// a channel (just for the fun of it)
type Chan chan int

func ChanCreate() Chan {
	c := make(Chan);
	go func(c Chan) {
		for x := 0;; x++ {
			c <- x
		}
	}(c);
	return c;
}

func (ch Chan) ServeHTTP(c *http.Conn, req *http.Request) {
	io.WriteString(c, fmt.Sprintf("channel send #%d\n", <-ch));
}

func main() {
	flag.Parse();
	http.Handle("/counter", new(Counter));
	http.Handle("/go/", http.HandlerFunc(FileServer));
	http.Handle("/flags/", http.HandlerFunc(FlagServer));
	http.Handle("/args/", http.HandlerFunc(ArgServer));
	http.Handle("/go/hello", http.HandlerFunc(HelloServer));
	http.Handle("/chan", ChanCreate());
	err := http.ListenAndServe(":12345", nil);
	if err != nil {
		panic("ListenAndServe: ", err.String())
	}
}

<|endoftext|>


<commit_before>package device

import (
	"fmt"
	"time"

	"github.com/TheThingsNetwork/ttn/core/storage"
	"github.com/TheThingsNetwork/ttn/core/types"
)

// Options for the specified device
type Options struct {
	DisableFCntCheck bool // Disable Frame counter check (insecure)
	Uses32BitFCnt    bool // Use 32-bit Frame counters
}

// Device contains the state of a device
type Device struct {
	DevEUI      types.DevEUI
	AppEUI      types.AppEUI
	DevAddr     types.DevAddr
	NwkSKey     types.NwkSKey
	FCntUp      uint32
	FCntDown    uint32
	LastSeen    time.Time
	Options     Options
	Utilization Utilization
}

// DeviceProperties contains all properties of a Device that can be stored in Redis.
var DeviceProperties = []string{
	"dev_eui",
	"app_eui",
	"dev_addr",
	"nwk_s_key",
	"f_cnt_up",
	"f_cnt_down",
	"last_seen",
	"options",
	"utilization",
}

// ToStringStringMap converts the given properties of Device to a
// map[string]string for storage in Redis.
func (device *Device) ToStringStringMap(properties ...string) (map[string]string, error) {
	output := make(map[string]string)
	for _, p := range properties {
		property, err := device.formatProperty(p)
		if err != nil {
			return output, err
		}
		if property != "" {
			output[p] = property
		}
	}
	return output, nil
}

// FromStringStringMap imports known values from the input to a Device.
func (device *Device) FromStringStringMap(input map[string]string) error {
	for k, v := range input {
		device.parseProperty(k, v)
	}
	return nil
}

func (device *Device) formatProperty(property string) (formatted string, err error) {
	switch property {
	case "dev_eui":
		formatted = device.DevEUI.String()
	case "app_eui":
		formatted = device.AppEUI.String()
	case "dev_addr":
		formatted = device.DevAddr.String()
	case "nwk_s_key":
		formatted = device.NwkSKey.String()
	case "f_cnt_up":
		formatted = storage.FormatUint32(device.FCntUp)
	case "f_cnt_down":
		formatted = storage.FormatUint32(device.FCntDown)
	case "last_seen":
		formatted = device.LastSeen.Format(time.RFC3339Nano)
	case "options":
		// TODO
	case "utilization":
		// TODO
	default:
		err = fmt.Errorf("Property %s does not exist in Status", property)
	}
	return
}

func (device *Device) parseProperty(property string, value string) error {
	if value == "" {
		return nil
	}
	switch property {
	case "dev_eui":
		val, err := types.ParseDevEUI(value)
		if err != nil {
			return err
		}
		device.DevEUI = val
	case "app_eui":
		val, err := types.ParseAppEUI(value)
		if err != nil {
			return err
		}
		device.AppEUI = val
	case "dev_addr":
		val, err := types.ParseDevAddr(value)
		if err != nil {
			return err
		}
		device.DevAddr = val
	case "nwk_s_key":
		val, err := types.ParseNwkSKey(value)
		if err != nil {
			return err
		}
		device.NwkSKey = val
	case "f_cnt_up":
		val, err := storage.ParseUint32(value)
		if err != nil {
			return err
		}
		device.FCntUp = val
	case "f_cnt_down":
		val, err := storage.ParseUint32(value)
		if err != nil {
			return err
		}
		device.FCntDown = val
	case "last_seen":
		val, err := time.Parse(time.RFC3339Nano, value)
		if err != nil {
			return err
		}
		device.LastSeen = val
	case "options":
		// TODO
	case "utilization":
		// TODO
	}
	return nil
}
<commit_msg>Always use UTC time<commit_after>package device

import (
	"fmt"
	"time"

	"github.com/TheThingsNetwork/ttn/core/storage"
	"github.com/TheThingsNetwork/ttn/core/types"
)

// Options for the specified device
type Options struct {
	DisableFCntCheck bool // Disable Frame counter check (insecure)
	Uses32BitFCnt    bool // Use 32-bit Frame counters
}

// Device contains the state of a device
type Device struct {
	DevEUI      types.DevEUI
	AppEUI      types.AppEUI
	DevAddr     types.DevAddr
	NwkSKey     types.NwkSKey
	FCntUp      uint32
	FCntDown    uint32
	LastSeen    time.Time
	Options     Options
	Utilization Utilization
}

// DeviceProperties contains all properties of a Device that can be stored in Redis.
var DeviceProperties = []string{
	"dev_eui",
	"app_eui",
	"dev_addr",
	"nwk_s_key",
	"f_cnt_up",
	"f_cnt_down",
	"last_seen",
	"options",
	"utilization",
}

// ToStringStringMap converts the given properties of Device to a
// map[string]string for storage in Redis.
func (device *Device) ToStringStringMap(properties ...string) (map[string]string, error) {
	output := make(map[string]string)
	for _, p := range properties {
		property, err := device.formatProperty(p)
		if err != nil {
			return output, err
		}
		if property != "" {
			output[p] = property
		}
	}
	return output, nil
}

// FromStringStringMap imports known values from the input to a Device.
func (device *Device) FromStringStringMap(input map[string]string) error {
	for k, v := range input {
		device.parseProperty(k, v)
	}
	return nil
}

func (device *Device) formatProperty(property string) (formatted string, err error) {
	switch property {
	case "dev_eui":
		formatted = device.DevEUI.String()
	case "app_eui":
		formatted = device.AppEUI.String()
	case "dev_addr":
		formatted = device.DevAddr.String()
	case "nwk_s_key":
		formatted = device.NwkSKey.String()
	case "f_cnt_up":
		formatted = storage.FormatUint32(device.FCntUp)
	case "f_cnt_down":
		formatted = storage.FormatUint32(device.FCntDown)
	case "last_seen":
		formatted = device.LastSeen.UTC().Format(time.RFC3339Nano)
	case "options":
		// TODO
	case "utilization":
		// TODO
	default:
		err = fmt.Errorf("Property %s does not exist in Status", property)
	}
	return
}

func (device *Device) parseProperty(property string, value string) error {
	if value == "" {
		return nil
	}
	switch property {
	case "dev_eui":
		val, err := types.ParseDevEUI(value)
		if err != nil {
			return err
		}
		device.DevEUI = val
	case "app_eui":
		val, err := types.ParseAppEUI(value)
		if err != nil {
			return err
		}
		device.AppEUI = val
	case "dev_addr":
		val, err := types.ParseDevAddr(value)
		if err != nil {
			return err
		}
		device.DevAddr = val
	case "nwk_s_key":
		val, err := types.ParseNwkSKey(value)
		if err != nil {
			return err
		}
		device.NwkSKey = val
	case "f_cnt_up":
		val, err := storage.ParseUint32(value)
		if err != nil {
			return err
		}
		device.FCntUp = val
	case "f_cnt_down":
		val, err := storage.ParseUint32(value)
		if err != nil {
			return err
		}
		device.FCntDown = val
	case "last_seen":
		val, err := time.Parse(time.RFC3339Nano, value)
		if err != nil {
			return err
		}
		device.LastSeen = val
	case "options":
		// TODO
	case "utilization":
		// TODO
	}
	return nil
}
<|endoftext|>


<commit_before>package task

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"neon/build"
	"neon/util"
	"os"
	"strings"
)

func init() {
	build.TaskMap["tar"] = build.TaskDescriptor{
		Constructor: Tar,
		Help: `Create a tar archive.

Arguments:
- tar: the list of globs of files to tar (as a string or list of strings).
- dir: the root directory for glob (as a string, optional).
- exclude: globs of files to exclude (as a string or list of strings,
  optional).
- to: the name of the tar file to create as a string.
- prefix: prefix directory in the archive.

Examples:
# tar files in build directory in file named build.tar.gz
- tar: "build/**/*"
  to: "build.tar.gz"

Notes:
- If archive filename ends with gz (with a name such as foo.tar.gz or foo.tgz)
  the tar archive is compressed with gzip.`,
	}
}

func Tar(target *build.Target, args util.Object) (build.Task, error) {
	fields := []string{"tar", "to", "dir", "exclude", "prefix"}
	if err := CheckFields(args, fields, fields[:2]); err != nil {
		return nil, err
	}
	includes, err := args.GetListStringsOrString("tar")
	if err != nil {
		return nil, fmt.Errorf("argument tar must be a string or list of strings")
	}
	var to string
	if args.HasField("to") {
		to, err = args.GetString("to")
		if err != nil {
			return nil, fmt.Errorf("argument to of task tar must be a string")
		}
	}
	var dir string
	if args.HasField("dir") {
		dir, err = args.GetString("dir")
		if err != nil {
			return nil, fmt.Errorf("argument dir of task tar must be a string")
		}
	}
	var excludes []string
	if args.HasField("exclude") {
		excludes, err = args.GetListStringsOrString("exclude")
		if err != nil {
			return nil, fmt.Errorf("argument exclude of task tar must be string or list of strings")
		}
	}
	var prefix string
	if args.HasField("prefix") {
		prefix, err = args.GetString("prefix")
		if err != nil {
			return nil, fmt.Errorf("argument prefix of task tar must be a string")
		}
	}
	return func() error {
		// evaluate arguments
		for index, pattern := range includes {
			eval, err := target.Build.Context.ReplaceProperties(pattern)
			if err != nil {
				return fmt.Errorf("evaluating pattern: %v", err)
			}
			includes[index] = eval
		}
		eval, err := target.Build.Context.ReplaceProperties(to)
		if err != nil {
			return fmt.Errorf("evaluating destination file: %v", err)
		}
		to = eval
		eval, err = target.Build.Context.ReplaceProperties(dir)
		if err != nil {
			return fmt.Errorf("evaluating source directory: %v", err)
		}
		dir = eval
		eval, err = target.Build.Context.ReplaceProperties(prefix)
		if err != nil {
			return fmt.Errorf("evaluating destination file: %v", err)
		}
		prefix = eval
		// find source files
		files, err := target.Build.Context.FindFiles(dir, includes, excludes)
		if err != nil {
			return fmt.Errorf("getting source files for tar task: %v", err)
		}
		if len(files) > 0 {
			target.Build.Info("Tarring %d file(s)", len(files))
			// tar files
			err = Writetar(files, prefix, to)
			if err != nil {
				return fmt.Errorf("tarring files: %v", err)
			}
		}
		return nil
	}, nil
}

func Writetar(files []string, prefix, to string) error {
	file, err := os.Create(to)
	if err != nil {
		return fmt.Errorf("creating tar archive: %v", err)
	}
	defer file.Close()
	var fileWriter io.WriteCloser = file
	if strings.HasSuffix(to, "gz") {
		fileWriter = gzip.NewWriter(file)
		defer fileWriter.Close()
	}
	writer := tar.NewWriter(fileWriter)
	defer writer.Close()
	for _, name := range files {
		err := writeFileToTar(writer, name, prefix)
		if err != nil {
			return fmt.Errorf("writing file to tar archive: %v", err)
		}
	}
	return nil
}

func writeFileToTar(writer *tar.Writer, filename, prefix string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()
	stat, err := file.Stat()
	if err != nil {
		return err
	}
	name := sanitizedName(filename)
	if prefix != "" {
		name = prefix + "/" + name
	}
	header := &tar.Header{
		Name:    name,
		Mode:    int64(stat.Mode()),
		Uid:     os.Getuid(),
		Gid:     os.Getgid(),
		Size:    stat.Size(),
		ModTime: stat.ModTime(),
	}
	if err = writer.WriteHeader(header); err != nil {
		return err
	}
	_, err = io.Copy(writer, file)
	return err
}
<commit_msg>Fixed tar task<commit_after>package task

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"neon/build"
	"neon/util"
	"os"
	"path/filepath"
	"strings"
)

func init() {
	build.TaskMap["tar"] = build.TaskDescriptor{
		Constructor: Tar,
		Help: `Create a tar archive.

Arguments:
- tar: the list of globs of files to tar (as a string or list of strings).
- dir: the root directory for glob (as a string, optional).
- exclude: globs of files to exclude (as a string or list of strings,
  optional).
- to: the name of the tar file to create as a string.
- prefix: prefix directory in the archive.

Examples:
# tar files in build directory in file named build.tar.gz
- tar: "build/**/*"
  to: "build.tar.gz"

Notes:
- If archive filename ends with gz (with a name such as foo.tar.gz or foo.tgz)
  the tar archive is compressed with gzip.`,
	}
}

func Tar(target *build.Target, args util.Object) (build.Task, error) {
	fields := []string{"tar", "to", "dir", "exclude", "prefix"}
	if err := CheckFields(args, fields, fields[:2]); err != nil {
		return nil, err
	}
	includes, err := args.GetListStringsOrString("tar")
	if err != nil {
		return nil, fmt.Errorf("argument tar must be a string or list of strings")
	}
	var to string
	if args.HasField("to") {
		to, err = args.GetString("to")
		if err != nil {
			return nil, fmt.Errorf("argument to of task tar must be a string")
		}
	}
	var dir string
	if args.HasField("dir") {
		dir, err = args.GetString("dir")
		if err != nil {
			return nil, fmt.Errorf("argument dir of task tar must be a string")
		}
	}
	var excludes []string
	if args.HasField("exclude") {
		excludes, err = args.GetListStringsOrString("exclude")
		if err != nil {
			return nil, fmt.Errorf("argument exclude of task tar must be string or list of strings")
		}
	}
	var prefix string
	if args.HasField("prefix") {
		prefix, err = args.GetString("prefix")
		if err != nil {
			return nil, fmt.Errorf("argument prefix of task tar must be a string")
		}
	}
	return func() error {
		// evaluate arguments
		for index, pattern := range includes {
			eval, err := target.Build.Context.ReplaceProperties(pattern)
			if err != nil {
				return fmt.Errorf("evaluating pattern: %v", err)
			}
			includes[index] = eval
		}
		eval, err := target.Build.Context.ReplaceProperties(to)
		if err != nil {
			return fmt.Errorf("evaluating destination file: %v", err)
		}
		to = eval
		eval, err = target.Build.Context.ReplaceProperties(dir)
		if err != nil {
			return fmt.Errorf("evaluating source directory: %v", err)
		}
		dir = eval
		eval, err = target.Build.Context.ReplaceProperties(prefix)
		if err != nil {
			return fmt.Errorf("evaluating destination file: %v", err)
		}
		prefix = eval
		// find source files
		files, err := target.Build.Context.FindFiles(dir, includes, excludes)
		if err != nil {
			return fmt.Errorf("getting source files for tar task: %v", err)
		}
		if len(files) > 0 {
			target.Build.Info("Tarring %d file(s)", len(files))
			// tar files
			err = Writetar(dir, files, prefix, to)
			if err != nil {
				return fmt.Errorf("tarring files: %v", err)
			}
		}
		return nil
	}, nil
}

func Writetar(dir string, files []string, prefix, to string) error {
	file, err := os.Create(to)
	if err != nil {
		return fmt.Errorf("creating tar archive: %v", err)
	}
	defer file.Close()
	var fileWriter io.WriteCloser = file
	if strings.HasSuffix(to, "gz") {
		fileWriter = gzip.NewWriter(file)
		defer fileWriter.Close()
	}
	writer := tar.NewWriter(fileWriter)
	defer writer.Close()
	for _, name := range files {
		var path string
		if dir != "" {
			path = filepath.Join(dir, name)
		} else {
			path = name
		}
		err := writeFileToTar(writer, path, name, prefix)
		if err != nil {
			return fmt.Errorf("writing file to tar archive: %v", err)
		}
	}
	return nil
}

func writeFileToTar(writer *tar.Writer, path, name, prefix string) error {
	file, err := os.Open(path)
	if err != nil {
		return err
	}
	defer file.Close()
	stat, err := file.Stat()
	if err != nil {
		return err
	}
	name = sanitizedName(name)
	if prefix != "" {
		name = prefix + "/" + name
	}
	header := &tar.Header{
		Name:    name,
		Mode:    int64(stat.Mode()),
		Uid:     os.Getuid(),
		Gid:     os.Getgid(),
		Size:    stat.Size(),
		ModTime: stat.ModTime(),
	}
	if err = writer.WriteHeader(header); err != nil {
		return err
	}
	_, err = io.Copy(writer, file)
	return err
}
<|endoftext|>