<commit_before>package main

import (
	"flag"
	"fmt"
	"math"
	"strings"
	"sync"
	"time"

	"github.com/admpub/confl"
	"github.com/admpub/log"
	"github.com/webx-top/db"
	"github.com/webx-top/db/lib/factory"
	"github.com/webx-top/db/mongo"
	"github.com/webx-top/db/mysql"
	"gopkg.in/mgo.v2/bson"
)

type CateItem struct {
	ID   string `bson:"id" db:"id"`
	Name string `bson:"name" db:"name"`
}

type CateModel struct {
	First  *CateItem `bson:"first" db:"first"`
	Second *CateItem `bson:"second" db:"second"`
	Third  *CateItem `bson:"third" db:"third"`
}

type ContentModel struct {
	ID    interface{} `bson:"id" db:"id"`
	Title string      `bson:"title" db:"title"`
	Cate  *CateModel  `bson:"cate" db:"cate"`
}

type EventModel struct {
	ID        bson.ObjectId `bson:"_id" db:"_id"`
	Event     string        `bson:"event" db:"event"`
	Timestamp uint          `bson:"timestamp" db:"timestamp"`
	Content   *ContentModel `bson:"content" db:"content"`
	Udid      string        `bson:"udid" db:"udid"`
	Platform  string        `bson:"platform" db:"platform"`
	OS        string        `bson:"os" db:"os"`
	OsType    string        `bson:"osType" db:"osType"`
	Version   string        `bson:"version" db:"version"`
	BundleId  string        `bson:"bundleId" db:"bundleId"`
	IP        string        `bson:"ip" db:"ip"`
	Account   struct {
		ID string `bson:"accountId" db:"accountId"`
	} `bson:"account" db:"account"`
}

var config = struct {
	ConfigFile *string
	Operation  *string
}{}

func main() {
	config.ConfigFile = flag.String(`c`, `dbconfig.yml`, `database setting`)
	config.Operation = flag.String(`t`, `insertBrandId`, `operation type: removeDuplicates / updateEvent / updateOsType)`)
	flag.Parse()

	log.Sync()
	log.DefaultLog.AddSpace = true
	log.SetFatalAction(log.ActionExit)

	dbConfig := struct {
		Mongo mongo.ConnectionURL
		MySQL mysql.ConnectionURL
	}{}

	_, err := confl.DecodeFile(*config.ConfigFile, &dbConfig)
	if err != nil {
		log.Fatal(err)
	}
	mongo.ConnTimeout = time.Second * 30
	dbMongo, err := mongo.Open(dbConfig.Mongo)
	if err != nil {
		log.Fatal(err)
	}

	dbMySQL, err := mysql.Open(dbConfig.MySQL)
	if err != nil {
		log.Fatal(err)
	}

	cluster := factory.NewCluster().AddW(dbMongo)
	factory.AddCluster(cluster) //第一次添加。索引编号为0
	clusterMySQL := factory.NewCluster().AddW(dbMySQL)
	factory.AddCluster(clusterMySQL) //第二次添加。索引编号为1，以此类推。
	factory.SetDebug(true)           //调试时可以打开Debug模式来查看sql语句
	defer factory.CloseAll()

	detail := map[string]string{}
	detail["appid"] = "11244bf15870d8567b41d99b908544ed"

	wg := &sync.WaitGroup{}
	if _, ok := detail["appid"]; ok {
		wg.Add(1)
		go checkAppID(detail, wg)
	} else {
		//使用Link(1)来选择索引编号为1的数据库连接(默认使用编号为0的连接)
		result := factory.NewParam().Setter().Link(1).C(`libuser_detail`).Result()
		total, err := factory.NewParam().Setter().Link(1).C(`libuser_detail`).Count()
		if err != nil {
			log.Fatal(err)
		}
		wg.Add(int(total))
		for result.Next(&detail) {
			switch *config.Operation { //修改event值infoXXX为downloadXXX
			case `updateEvent`:
				go checkEvent(detail, wg)
			default:
				go checkAppID(detail, wg)
			}
		}
		result.Close()
	}
	wg.Wait()
}

type Executor struct {
	Cond db.Cond
	Func func(EventModel, map[string]string) error
}

var executors = map[string]*Executor{
	"updateOsType": &Executor{ //更新osType
		Cond: db.Cond{
			"udid":   "00old00analysis00",
			"osType": "windows",
		},
		Func: updateOsType,
	},
	"removeDuplicates": &Executor{ //删除重复数据
		Cond: db.Cond{"udid": "00old00analysis00"},
		Func: removeDuplicates,
	},
	"insertBrandId": &Executor{
		Cond: db.Cond{
			"timestamp >=":         strToTime(`2016-11-01 00:00:00`).Unix(),
			"content.bid $exists":  false,
			"content.cate $exists": false,
			"event IN": []string{
				"downloadMag",
				"infoMag",
			},
		},
		Func: insertBrandId,
	},
}

func strToTime(value string) time.Time {
	t, _ := time.Parse(`2006-01-02 15:04:05`, value)
	return t
}

func checkAppID(detail map[string]string, wg *sync.WaitGroup) {
	defer wg.Done()
	if len(detail["appid"]) == 0 {
		return
	}
	log.Info(`AppID`, detail["appid"])

	mdt := new([]EventModel)
	cond := db.Cond{}
	executor, ok := executors[*config.Operation]
	if !ok {
		return
	}
	for k, v := range executor.Cond {
		cond[k] = v
	}
	size := 1000
	page := 1

	//这里没有使用Link()函数，默认选择索引编号为0的数据库连接
	cnt, err := factory.NewParam().Setter().C(`event` + detail["appid"]).Args(cond).Page(page).Size(size).Recv(mdt).List()
	if err != nil {
		if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
			log.Error(err)
			return
		}
		log.Fatal(err)
	}
	tot := cnt()
	pages := int(math.Ceil(float64(tot) / float64(size)))
	for ; page <= pages; page++ {
		if page > 1 {
			_, err = factory.NewParam().Setter().C(`event` + detail["appid"]).Args(cond).Page(page).Size(size).Recv(mdt).List()
			if err != nil {
				if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
					log.Error(err)
					break
				}
				log.Fatal(err)
			}
		}
		for _, row := range *mdt {
			err := executor.Func(row, detail)
			if err != nil {
				if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
					log.Error(err)
					break
				}
				log.Fatal(err)
			}
		}
	}
}

//删除重复数据
func removeDuplicates(row EventModel, detail map[string]string) error {
	n, err := factory.NewParam().Setter().C(`event` + detail["appid"]).Args(db.Cond{
		"_id <>":            row.ID,
		"timestamp":         row.Timestamp,
		"account.accountId": row.Account.ID,
	}).Count()
	if err == nil && n > 0 {
		log.Infof(`Found %d duplicate(s) => %s`, n, row.ID)
		err = factory.NewParam().Setter().C(`event` + detail["appid"]).Args(db.Cond{"_id": row.ID}).Delete()
		if err == nil {
			log.Info(`Remove success.`)
		}
	}
	return err
}

//更新osType
func updateOsType(row EventModel, detail map[string]string) error {
	var osType, bundleId string
	switch row.Platform {
	case `pc`, `pc_down`:
		osType = `Windows`
		bundleId = `com.dooland.pc`
	case `ipad`:
		osType = `iOS`
		bundleId = `com.dooland.padforiosfromweb.reader`
	case `iphone`:
		osType = `iOS`
		bundleId = `com.dooland.mobileforiosfromweb.reader`
	case `android`:
		osType = `Android`
		bundleId = `com.dooland.padforandroidfromweb.reader`
	case `androidmobile`:
		osType = `Android`
		bundleId = `com.dooland.mobileforandroidfromweb.reader`
	case `waparticle`:
		osType = `Wap`
		bundleId = `com.dooland.wapforweb.reader`
	case `article`:
		osType = `Windows`
		bundleId = `com.dooland.pc`
	case `dudubao`:
		osType = `Dudubao`
		bundleId = `com.dooland.dudubao`
	case `dudubao_down`:
		osType = `Dudubao`
		bundleId = `com.dooland.dudubao`
	default:
		return nil
	}
	log.Infof(`Update [%s] %s => %s, %s => %s`, row.ID, row.OsType, osType, row.BundleId, bundleId)
	err := factory.NewParam().Setter().C(`event` + detail["appid"]).Args(db.Cond{"_id": row.ID}).Send(map[string]string{
		"osType":   osType,
		"bundleId": bundleId,
	}).Update()
	return err
}

//修改infoXXX为downloadXXX
func updateEvent(row EventModel, detail map[string]string) error {
	if strings.HasPrefix(row.Event, `info`) == false {
		return nil
	}
	event := `download` + strings.TrimPrefix(row.Event, `info`)
	log.Infof(`Update [%s] %s => %s, %s => %s`, row.ID, row.Event, event)
	err := factory.NewParam().Setter().C(`event` + detail["appid"]).Args(db.Cond{"_id": row.ID}).Send(map[string]string{
		"event": event,
	}).Update()
	return err
}

//修改infoXXX为downloadXXX
func checkEvent(detail map[string]string, wg *sync.WaitGroup) {
	defer wg.Done()
	if len(detail["appid"]) == 0 {
		return
	}
	log.Info(`AppID`, detail["appid"])

	size := 1000
	page := 1
	r := []map[string]string{}
	cnt, err := factory.NewParam().Setter().Link(1).C(`user_down_mag`).Args(db.Cond{"lib_id": detail["id"]}).Recv(&r).Page(page).Size(size).List()
	if err != nil {
		if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
			log.Error(err)
			return
		}
		log.Fatal(err)
	}
	tot := cnt()
	pages := int(math.Ceil(float64(tot) / float64(size)))
	for ; page <= pages; page++ {
		if page > 1 {
			_, err = factory.NewParam().Setter().Link(1).C(`user_down_mag`).Args(db.Cond{"lib_id": detail["id"]}).Recv(&r).Page(page).Size(size).List()
			if err != nil {
				if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
					log.Error(err)
					break
				}
				log.Fatal(err)
			}
		}
		for _, row := range r {
			t, err := time.Parse(`2006-01-02 15:04:05`, row["add_time"])
			if err != nil {
				log.Error(err)
				continue
			}
			mdt := new(EventModel)
			cond := db.Cond{
				"udid":              "00old00analysis00",
				"event IN":          []string{"infoMag", "infoBook"},
				"account.accountId": row["user_id"],
				"timestamp":         t.Unix(),
			}
			err = factory.NewParam().Setter().C(`event` + detail["appid"]).Args(cond).Page(page).Size(size).Recv(mdt).One()
			if err != nil {
				if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
					log.Error(err)
					continue
				}
				log.Fatal(err)
			}
			err = updateEvent(*mdt, detail)
			if err != nil {
				if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
					log.Error(err)
					continue
				}
				log.Fatal(err)
			}
		}
	}
}

//插入品牌ID
func insertBrandId(row EventModel, detail map[string]string) error {
	if row.Content.ID == nil {
		log.Warn(`content.id is empty: `, row.ID)
		return nil
	}
	id := fmt.Sprint(row.Content.ID)
	var brandId string
	recv := map[string]string{}
	err := factory.NewParam().Setter().Link(1).C(`dudubao.mag_list`).Args(db.Cond{"id": id}).Recv(&recv).One()
	if err != nil {
		if err == db.ErrNoMoreRows {
			err = factory.NewParam().Setter().Link(1).C(`dudubao_bak.mag_list_bak`).Args(db.Cond{"id": id}).Recv(&recv).One()
		}
	}
	if err != nil {
		return err
	}
	log.Infof(`Update [%s] %s => %s`, row.ID, id, recv["sort_id"])
	brandId = recv["sort_id"]
	if len(brandId) == 0 || brandId == `0` {
		log.Warn(` -> Skiped.`)
		return nil
	}
	err = factory.NewParam().Setter().C(`event` + detail["appid"]).Args(db.Cond{"_id": row.ID}).Send(map[string]string{
		"content.bid": brandId,
	}).Update()
	return err
}
<commit_msg>update<commit_after>package main

import (
	"flag"
	"fmt"
	"math"
	"strings"
	"sync"
	"time"

	"github.com/admpub/confl"
	"github.com/admpub/log"
	"github.com/webx-top/db"
	"github.com/webx-top/db/lib/factory"
	"github.com/webx-top/db/mongo"
	"github.com/webx-top/db/mysql"
	"gopkg.in/mgo.v2/bson"
)

type CateItem struct {
	ID   string `bson:"id" db:"id"`
	Name string `bson:"name" db:"name"`
}

type CateModel struct {
	First  *CateItem `bson:"first" db:"first"`
	Second *CateItem `bson:"second" db:"second"`
	Third  *CateItem `bson:"third" db:"third"`
}

type ContentModel struct {
	ID    interface{} `bson:"id" db:"id"`
	Title string      `bson:"title" db:"title"`
	Cate  *CateModel  `bson:"cate" db:"cate"`
}

type EventModel struct {
	ID        bson.ObjectId `bson:"_id" db:"_id"`
	Event     string        `bson:"event" db:"event"`
	Timestamp uint          `bson:"timestamp" db:"timestamp"`
	Content   *ContentModel `bson:"content" db:"content"`
	Udid      string        `bson:"udid" db:"udid"`
	Platform  string        `bson:"platform" db:"platform"`
	OS        string        `bson:"os" db:"os"`
	OsType    string        `bson:"osType" db:"osType"`
	Version   string        `bson:"version" db:"version"`
	BundleId  string        `bson:"bundleId" db:"bundleId"`
	IP        string        `bson:"ip" db:"ip"`
	Account   struct {
		ID string `bson:"accountId" db:"accountId"`
	} `bson:"account" db:"account"`
}

var config = struct {
	ConfigFile *string
	Operation  *string
}{}

func main() {
	config.ConfigFile = flag.String(`c`, `dbconfig.yml`, `database setting`)
	config.Operation = flag.String(`t`, `insertBrandId`, `operation type: removeDuplicates / updateEvent / updateOsType)`)
	flag.Parse()

	log.Sync()
	log.DefaultLog.AddSpace = true
	log.SetFatalAction(log.ActionExit)

	dbConfig := struct {
		Mongo mongo.ConnectionURL
		MySQL mysql.ConnectionURL
	}{}

	_, err := confl.DecodeFile(*config.ConfigFile, &dbConfig)
	if err != nil {
		log.Fatal(err)
	}
	mongo.ConnTimeout = time.Second * 30
	dbMongo, err := mongo.Open(dbConfig.Mongo)
	if err != nil {
		log.Fatal(err)
	}

	dbMySQL, err := mysql.Open(dbConfig.MySQL)
	if err != nil {
		log.Fatal(err)
	}

	cluster := factory.NewCluster().AddW(dbMongo)
	factory.AddCluster(cluster) //第一次添加。索引编号为0
	clusterMySQL := factory.NewCluster().AddW(dbMySQL)
	factory.AddCluster(clusterMySQL) //第二次添加。索引编号为1，以此类推。
	//factory.SetDebug(true)           //调试时可以打开Debug模式来查看sql语句
	defer factory.CloseAll()

	detail := map[string]string{}
	detail["appid"] = "11244bf15870d8567b41d99b908544ed"

	wg := &sync.WaitGroup{}
	if _, ok := detail["appid"]; ok {
		wg.Add(1)
		checkAppID(detail, wg)
	} else {
		//使用Link(1)来选择索引编号为1的数据库连接(默认使用编号为0的连接)
		result := factory.NewParam().Setter().Link(1).C(`libuser_detail`).Result()
		total, err := factory.NewParam().Setter().Link(1).C(`libuser_detail`).Count()
		if err != nil {
			log.Fatal(err)
		}
		wg.Add(int(total))
		for result.Next(&detail) {
			switch *config.Operation { //修改event值infoXXX为downloadXXX
			case `updateEvent`:
				go checkEvent(detail, wg)
			default:
				go checkAppID(detail, wg)
			}
		}
		result.Close()
	}
	wg.Wait()
	log.Info(`任务完成`)
}

type Executor struct {
	Cond db.Cond
	Func func(EventModel, map[string]string) error
}

var executors = map[string]*Executor{
	"updateOsType": &Executor{ //更新osType
		Cond: db.Cond{
			"udid":   "00old00analysis00",
			"osType": "windows",
		},
		Func: updateOsType,
	},
	"removeDuplicates": &Executor{ //删除重复数据
		Cond: db.Cond{"udid": "00old00analysis00"},
		Func: removeDuplicates,
	},
	"insertBrandId": &Executor{
		Cond: db.Cond{
			"timestamp >=":         strToTime(`2016-11-01 00:00:00`).Unix(),
			"content.bid $exists":  false,
			"content.cate $exists": false,
			"event IN": []string{
				"downloadMag",
				"infoMag",
			},
		},
		Func: insertBrandId,
	},
}

func strToTime(value string) time.Time {
	t, _ := time.Parse(`2006-01-02 15:04:05`, value)
	return t
}

func checkAppID(detail map[string]string, wg *sync.WaitGroup) {
	defer wg.Done()
	if len(detail["appid"]) == 0 {
		return
	}
	log.Info(`AppID`, detail["appid"])

	mdt := new([]EventModel)
	cond := db.Cond{}
	executor, ok := executors[*config.Operation]
	if !ok {
		return
	}
	for k, v := range executor.Cond {
		cond[k] = v
	}
	size := 1000
	page := 1

	log.Info(`开始查询第1页`)

	//这里没有使用Link()函数，默认选择索引编号为0的数据库连接
	cnt, err := factory.NewParam().Setter().C(`event` + detail["appid"]).Args(cond).Page(page).Size(size).Recv(mdt).List()
	if err != nil {
		if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
			log.Error(err)
			return
		}
		log.Fatal(err)
	}
	tot := cnt()
	pages := int(math.Ceil(float64(tot) / float64(size)))
	for ; page <= pages; page++ {
		if page > 1 {
			log.Infof(`开始查询第%d页，共%d页`, page, pages)
			_, err = factory.NewParam().Setter().C(`event` + detail["appid"]).Args(cond).Page(page).Size(size).Recv(mdt).List()
			if err != nil {
				if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
					log.Error(err)
					break
				}
				log.Fatal(err)
			}
		}
		for _, row := range *mdt {
			err := executor.Func(row, detail)
			if err != nil {
				if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
					log.Error(err)
					break
				}
				log.Fatal(err)
			}
		}
	}
}

//删除重复数据
func removeDuplicates(row EventModel, detail map[string]string) error {
	n, err := factory.NewParam().Setter().C(`event` + detail["appid"]).Args(db.Cond{
		"_id <>":            row.ID,
		"timestamp":         row.Timestamp,
		"account.accountId": row.Account.ID,
	}).Count()
	if err == nil && n > 0 {
		log.Infof(`Found %d duplicate(s) => %s`, n, row.ID)
		err = factory.NewParam().Setter().C(`event` + detail["appid"]).Args(db.Cond{"_id": row.ID}).Delete()
		if err == nil {
			log.Info(`Remove success.`)
		}
	}
	return err
}

//更新osType
func updateOsType(row EventModel, detail map[string]string) error {
	var osType, bundleId string
	switch row.Platform {
	case `pc`, `pc_down`:
		osType = `Windows`
		bundleId = `com.dooland.pc`
	case `ipad`:
		osType = `iOS`
		bundleId = `com.dooland.padforiosfromweb.reader`
	case `iphone`:
		osType = `iOS`
		bundleId = `com.dooland.mobileforiosfromweb.reader`
	case `android`:
		osType = `Android`
		bundleId = `com.dooland.padforandroidfromweb.reader`
	case `androidmobile`:
		osType = `Android`
		bundleId = `com.dooland.mobileforandroidfromweb.reader`
	case `waparticle`:
		osType = `Wap`
		bundleId = `com.dooland.wapforweb.reader`
	case `article`:
		osType = `Windows`
		bundleId = `com.dooland.pc`
	case `dudubao`:
		osType = `Dudubao`
		bundleId = `com.dooland.dudubao`
	case `dudubao_down`:
		osType = `Dudubao`
		bundleId = `com.dooland.dudubao`
	default:
		return nil
	}
	log.Infof(`Update [%s] %s => %s, %s => %s`, row.ID, row.OsType, osType, row.BundleId, bundleId)
	err := factory.NewParam().Setter().C(`event` + detail["appid"]).Args(db.Cond{"_id": row.ID}).Send(map[string]string{
		"osType":   osType,
		"bundleId": bundleId,
	}).Update()
	return err
}

//修改infoXXX为downloadXXX
func updateEvent(row EventModel, detail map[string]string) error {
	if strings.HasPrefix(row.Event, `info`) == false {
		return nil
	}
	event := `download` + strings.TrimPrefix(row.Event, `info`)
	log.Infof(`Update [%s] %s => %s, %s => %s`, row.ID, row.Event, event)
	err := factory.NewParam().Setter().C(`event` + detail["appid"]).Args(db.Cond{"_id": row.ID}).Send(map[string]string{
		"event": event,
	}).Update()
	return err
}

//修改infoXXX为downloadXXX
func checkEvent(detail map[string]string, wg *sync.WaitGroup) {
	defer wg.Done()
	if len(detail["appid"]) == 0 {
		return
	}
	log.Info(`AppID`, detail["appid"])

	size := 1000
	page := 1
	r := []map[string]string{}
	cnt, err := factory.NewParam().Setter().Link(1).C(`user_down_mag`).Args(db.Cond{"lib_id": detail["id"]}).Recv(&r).Page(page).Size(size).List()
	if err != nil {
		if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
			log.Error(err)
			return
		}
		log.Fatal(err)
	}
	tot := cnt()
	pages := int(math.Ceil(float64(tot) / float64(size)))
	for ; page <= pages; page++ {
		if page > 1 {
			_, err = factory.NewParam().Setter().Link(1).C(`user_down_mag`).Args(db.Cond{"lib_id": detail["id"]}).Recv(&r).Page(page).Size(size).List()
			if err != nil {
				if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
					log.Error(err)
					break
				}
				log.Fatal(err)
			}
		}
		for _, row := range r {
			t, err := time.Parse(`2006-01-02 15:04:05`, row["add_time"])
			if err != nil {
				log.Error(err)
				continue
			}
			mdt := new(EventModel)
			cond := db.Cond{
				"udid":              "00old00analysis00",
				"event IN":          []string{"infoMag", "infoBook"},
				"account.accountId": row["user_id"],
				"timestamp":         t.Unix(),
			}
			err = factory.NewParam().Setter().C(`event` + detail["appid"]).Args(cond).Page(page).Size(size).Recv(mdt).One()
			if err != nil {
				if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
					log.Error(err)
					continue
				}
				log.Fatal(err)
			}
			err = updateEvent(*mdt, detail)
			if err != nil {
				if err == db.ErrNoMoreRows || factory.IsTimeoutError(err) {
					log.Error(err)
					continue
				}
				log.Fatal(err)
			}
		}
	}
}

//插入品牌ID
func insertBrandId(row EventModel, detail map[string]string) error {
	if row.Content.ID == nil {
		log.Warn(`content.id is empty: `, row.ID)
		return nil
	}
	id := fmt.Sprint(row.Content.ID)
	var brandId string
	recv := map[string]string{}
	err := factory.NewParam().Setter().Link(1).C(`dudubao.mag_list`).Args(db.Cond{"id": id}).Recv(&recv).One()
	if err != nil {
		if err == db.ErrNoMoreRows {
			err = factory.NewParam().Setter().Link(1).C(`dudubao_bak.mag_list_bak`).Args(db.Cond{"id": id}).Recv(&recv).One()
		}
	}
	if err != nil {
		return err
	}
	log.Infof(`Update [%s] %s => %s`, row.ID, id, recv["sort_id"])
	brandId = recv["sort_id"]
	if len(brandId) == 0 || brandId == `0` {
		log.Warn(` -> Skiped.`)
		return nil
	}
	err = factory.NewParam().Setter().C(`event` + detail["appid"]).Args(db.Cond{"_id": row.ID}).Send(map[string]string{
		"content.bid": brandId,
	}).Update()
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"bufio"
	"unicode"
	"fmt"
	"strconv"
)

type TokenType int

const (
	ErrorToken TokenType = iota
	KeywordToken
	IdentToken
	IntegerToken
)

type Keyword int

const (
	VarKeyword Keyword = iota // var
	AssignKeyword // =
	AddKeyword // +
	SubKeyword // -
	MulKeyword // *
	ModKeyword // %
	SemiColonKeyword // ;
)

type Token struct {
	Type TokenType
	Payload interface{}
	Line, Column int
}

type StateFunc func (l *Lexer) StateFunc

type Lexer struct {
	Input *bufio.Reader
	buffer string
	lastRuneWidth int
	Line, Column int
	State StateFunc
	tokens []*Token
}

func (l *Lexer) Emit() string {
	var n string
	b := l.buffer
	l.buffer = n
	return b
}

func (l *Lexer) Next() (rune, error) {
	r, w, err := l.Input.ReadRune()
	if err != nil {
		return r, err
	}
	l.lastRuneWidth = w
	l.buffer += string(r)
	if r == '\n' {
		l.Line++
		l.Column = 0
	} else {
		l.Column++
	}
	return r, nil
}

type RunePredicate func (rune) bool

// Reads up to unaccepted rune
func (l *Lexer) NextUpTo(pred RunePredicate) (r rune, err error) {
	for {
		r, err = l.Peek()
		if err != nil || !pred(r) {
			return r, err
		}
		l.Next()
	}
}

// Ignores up to unaccepted rune
func (l *Lexer) IgnoreUpTo(pred RunePredicate) (r rune, err error) {
	for {
		r, err = l.Peek()
		if err != nil || !pred(r) {
			return r, err
		}
		l.Ignore()
	}
}

func (l *Lexer) Ignore() (rune, error) {
	r, err := l.Next()
	if err != nil {
		return r, err
	}
	l.buffer = l.buffer[:len(l.buffer)-l.lastRuneWidth]
	return r, nil
}

func (l *Lexer) Back() error {
	err := l.Input.UnreadRune()
	if err != nil {
		return err
	}
	l.buffer = l.buffer[:len(l.buffer)-l.lastRuneWidth]
	return nil
}

func (l *Lexer) Peek() (rune, error) {
	r, err := l.Next()
	if err != nil {
		return r, err
	}
	err = l.Back()
	if err != nil {
		return r, err
	}
	return r, nil
}

func (l *Lexer) emitToken(t *Token) {
	l.tokens = append(l.tokens, t)
}

// utility whitespace lex function
func (l *Lexer) whitespace() error {
	r, err := l.Peek()
	if err != nil {
		return err
	}
	for unicode.Is(unicode.White_Space, r) {
		l.Next()
		r, err = l.Peek()
		if err != nil {
			return err
		}
	}
	l.Emit() // dump
	return nil
}

func floatState(l *Lexer) StateFunc {
	return nil
}

func numberState(l *Lexer) StateFunc {
	base := 10

	// possible prefix
	r, err := l.Peek()
	if err != nil {
		return nil
	}
	if r == '0' {
		l.Next()
		r, err := l.Next()
		if err != nil {
			return nil
		}
		switch r {
		case 'x':
			l.Emit()
			base = 16 // hexadecimal
		case 'c':
			l.Emit()
			base = 8 // octal
		case 'b':
			l.Emit()
			base = 2 // binary
		default:
			l.Back()
		}
	}

	r, _ = l.NextUpTo(func(r rune) bool {
		// allow '_' separator
		if base == 16 {
			return unicode.Is(unicode.Hex_Digit, r) || r == '_'
		} else {
			return unicode.IsDigit(r) || r == '_'
		}
	})

	if r == '.' {
		// floating point number
		return floatState
	} else {
		i, err := strconv.ParseInt(l.Emit(), base, 64)
		if err != nil {
			l.emitToken(&Token {
				Type: ErrorToken,
				Payload: fmt.Sprint(err),
				Line: l.Line,
				Column: l.Column,
			})
			return startState
		}
		l.emitToken(&Token {
			Type: IntegerToken,
			Payload: i,
			Line: l.Line,
			Column: l.Column,
		})
		return expressionState
	}
}

func expressionState(l *Lexer) StateFunc {
	if l.whitespace() != nil {
		return nil
	}
	r, err := l.Peek()
	if err != nil {
		return nil
	}
	switch {
	case unicode.IsDigit(r):
		return numberState
	case r == '+' || r == '-' || r == '*' || r == '%':
		return nil
	case r == ';':
		l.Next()
		l.emitToken(&Token {
			Type: KeywordToken,
			Payload: SemiColonKeyword,
			Line: l.Line,
			Column: l.Column,
		})
		return startState
	default:
		l.Next()
		l.emitToken(&Token {
			Type: ErrorToken,
			Payload: fmt.Sprintf("Unexpected rune '%c' in expression", r),
			Line: l.Line,
			Column: l.Column,
		})
		return startState
	}
}

// = 1+3*a
func assignExpressionState(l *Lexer) StateFunc {
	if l.whitespace() != nil {
		return nil
	}
	r, err := l.Ignore()
	if err != nil {
		return nil
	}
	if r != '=' {
		// error, missing equals sign
		l.emitToken(&Token {
			Type: ErrorToken,
			Payload: fmt.Sprintf("Expected '=', found %c\n", r),
			Line: l.Line,
			Column: l.Column,
		})
		return startState
	} else {
		l.emitToken(&Token {
			Type: KeywordToken,
			Payload: AssignKeyword,
			Line: l.Line,
			Column: l.Column,
		})
		return expressionState
	}
}

func varIdentState(l *Lexer) StateFunc {
	l.NextUpTo(func (r rune) bool {
		return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_'
	})
	l.emitToken(&Token {
		Type: IdentToken,
		Payload: l.Emit(),
		Line: l.Line,
		Column: l.Column,
	})
	return assignExpressionState
}

func varState(l *Lexer) StateFunc {
	if l.whitespace() != nil {
		return nil
	}
	r, err := l.Peek()
	if err != nil {
		return nil
	}
	if unicode.IsLetter(r) || r == '_' {
		return varIdentState
	}
	return nil
}

func keywordState(l *Lexer) StateFunc {
	l.NextUpTo(unicode.IsLetter)
	l.emitToken(&Token {
		Type: KeywordToken,
		Payload: VarKeyword,
		Line: l.Line,
		Column: l.Column,
	})
	if l.Emit() == "var" {
		return varState
	}
	return nil
}

func startState(l *Lexer) StateFunc {
	if l.whitespace() != nil {
		return nil
	}
	r, err := l.Peek()
	if err != nil {
		return nil
	}
	if unicode.IsLetter(r) {
		return keywordState
	}
	return nil
}

func (l *Lexer) NextToken() *Token {
	for l.State != nil {
		l.State = l.State(l)
		if len(l.tokens) > 0 {
			t := l.tokens[len(l.tokens)-1]
			l.tokens = l.tokens[:len(l.tokens)-1]
			return t
		}
	}
	return nil
}

// Prompting Reader
type Repl struct {
	Input *bufio.Reader
	Output *bufio.Writer
	Prompt string
	lexer *Lexer
}

func (r *Repl) prompt() {
	defer r.Output.Flush()
	fmt.Fprint(r.Output, r.Prompt)
}

func (r *Repl) Init() {
	r.lexer = &Lexer {
		State: startState,
		Input: bufio.NewReader(r),
	}
	r.prompt()
}

func (r *Repl) Read(p []byte) (n int, err error) {
	for i := 0; i < len(p); i++ {
		b, err := r.Input.ReadByte()
		if err != nil {
			return i, err
		}
		p[i] = b
		if b == '\n' {
			r.prompt()
			return i, nil
		}
	}
	return len(p), nil
}

func (r *Repl) NextToken() *Token {
	return r.lexer.NextToken()
}

func main() {
	r := Repl {
		Input: bufio.NewReader(os.Stdin),
		Output: bufio.NewWriter(os.Stdout),
		Prompt: ">>> ",
	}
	r.Init()
	for t := r.NextToken(); t != nil; t = r.NextToken() {
		if t.Type == ErrorToken {
			fmt.Println(t.Payload)
		}
	}
	fmt.Printf("\n")
}
<commit_msg>screw prompts for now<commit_after>package main

import (
	"os"
	"bufio"
	"unicode"
	"fmt"
	"strconv"
)

type TokenType int

const (
	ErrorToken TokenType = iota
	KeywordToken
	IdentToken
	IntegerToken
)

type Keyword int

const (
	VarKeyword Keyword = iota // var
	AssignKeyword // =
	AddKeyword // +
	SubKeyword // -
	MulKeyword // *
	ModKeyword // %
	SemiColonKeyword // ;
)

type Token struct {
	Type TokenType
	Payload interface{}
	Line, Column int
}

type StateFunc func (l *Lexer) StateFunc

type Lexer struct {
	Input *bufio.Reader
	buffer string
	lastRuneWidth int
	Line, Column int
	State StateFunc
	tokens []*Token
}

func (l *Lexer) Emit() string {
	var n string
	b := l.buffer
	l.buffer = n
	return b
}

func (l *Lexer) Next() (rune, error) {
	r, w, err := l.Input.ReadRune()
	if err != nil {
		return r, err
	}
	l.lastRuneWidth = w
	l.buffer += string(r)
	if r == '\n' {
		l.Line++
		l.Column = 0
	} else {
		l.Column++
	}
	return r, nil
}

type RunePredicate func (rune) bool

// Reads up to unaccepted rune
func (l *Lexer) NextUpTo(pred RunePredicate) (r rune, err error) {
	for {
		r, err = l.Peek()
		if err != nil || !pred(r) {
			return r, err
		}
		l.Next()
	}
}

// Ignores up to unaccepted rune
func (l *Lexer) IgnoreUpTo(pred RunePredicate) (r rune, err error) {
	for {
		r, err = l.Peek()
		if err != nil || !pred(r) {
			return r, err
		}
		l.Ignore()
	}
}

func (l *Lexer) Ignore() (rune, error) {
	r, err := l.Next()
	if err != nil {
		return r, err
	}
	l.buffer = l.buffer[:len(l.buffer)-l.lastRuneWidth]
	return r, nil
}

func (l *Lexer) Back() error {
	err := l.Input.UnreadRune()
	if err != nil {
		return err
	}
	l.buffer = l.buffer[:len(l.buffer)-l.lastRuneWidth]
	return nil
}

func (l *Lexer) Peek() (rune, error) {
	r, err := l.Next()
	if err != nil {
		return r, err
	}
	err = l.Back()
	if err != nil {
		return r, err
	}
	return r, nil
}

func (l *Lexer) emitToken(t *Token) {
	l.tokens = append(l.tokens, t)
}

// utility whitespace lex function
func (l *Lexer) whitespace() error {
	r, err := l.Peek()
	if err != nil {
		return err
	}
	for unicode.Is(unicode.White_Space, r) {
		l.Next()
		r, err = l.Peek()
		if err != nil {
			return err
		}
	}
	l.Emit() // dump
	return nil
}

func floatState(l *Lexer) StateFunc {
	return nil
}

func numberState(l *Lexer) StateFunc {
	base := 10

	// possible prefix
	r, err := l.Peek()
	if err != nil {
		return nil
	}
	if r == '0' {
		l.Next()
		r, err := l.Next()
		if err != nil {
			return nil
		}
		switch r {
		case 'x':
			l.Emit()
			base = 16 // hexadecimal
		case 'c':
			l.Emit()
			base = 8 // octal
		case 'b':
			l.Emit()
			base = 2 // binary
		default:
			l.Back()
		}
	}

	r, _ = l.NextUpTo(func(r rune) bool {
		// allow '_' separator
		if base == 16 {
			return unicode.Is(unicode.Hex_Digit, r) || r == '_'
		} else {
			return unicode.IsDigit(r) || r == '_'
		}
	})

	if r == '.' {
		// floating point number
		return floatState
	} else {
		i, err := strconv.ParseInt(l.Emit(), base, 64)
		if err != nil {
			l.emitToken(&Token {
				Type: ErrorToken,
				Payload: fmt.Sprint(err),
				Line: l.Line,
				Column: l.Column,
			})
			return startState
		}
		l.emitToken(&Token {
			Type: IntegerToken,
			Payload: i,
			Line: l.Line,
			Column: l.Column,
		})
		return expressionState
	}
}

func expressionState(l *Lexer) StateFunc {
	if l.whitespace() != nil {
		return nil
	}
	r, err := l.Peek()
	if err != nil {
		return nil
	}
	switch {
	case unicode.IsDigit(r):
		return numberState
	case r == '+' || r == '-' || r == '*' || r == '%':
		return nil
	case r == ';':
		l.Next()
		l.emitToken(&Token {
			Type: KeywordToken,
			Payload: SemiColonKeyword,
			Line: l.Line,
			Column: l.Column,
		})
		return startState
	default:
		l.Next()
		l.emitToken(&Token {
			Type: ErrorToken,
			Payload: fmt.Sprintf("Unexpected rune '%c' in expression", r),
			Line: l.Line,
			Column: l.Column,
		})
		return startState
	}
}

// = 1+3*a
func assignExpressionState(l *Lexer) StateFunc {
	if l.whitespace() != nil {
		return nil
	}
	r, err := l.Ignore()
	if err != nil {
		return nil
	}
	if r != '=' {
		// error, missing equals sign
		l.emitToken(&Token {
			Type: ErrorToken,
			Payload: fmt.Sprintf("Expected '=', found %c\n", r),
			Line: l.Line,
			Column: l.Column,
		})
		return startState
	} else {
		l.emitToken(&Token {
			Type: KeywordToken,
			Payload: AssignKeyword,
			Line: l.Line,
			Column: l.Column,
		})
		return expressionState
	}
}

func varIdentState(l *Lexer) StateFunc {
	l.NextUpTo(func (r rune) bool {
		return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_'
	})
	l.emitToken(&Token {
		Type: IdentToken,
		Payload: l.Emit(),
		Line: l.Line,
		Column: l.Column,
	})
	return assignExpressionState
}

func varState(l *Lexer) StateFunc {
	if l.whitespace() != nil {
		return nil
	}
	r, err := l.Peek()
	if err != nil {
		return nil
	}
	if unicode.IsLetter(r) || r == '_' {
		return varIdentState
	}
	return nil
}

func keywordState(l *Lexer) StateFunc {
	l.NextUpTo(unicode.IsLetter)
	l.emitToken(&Token {
		Type: KeywordToken,
		Payload: VarKeyword,
		Line: l.Line,
		Column: l.Column,
	})
	if l.Emit() == "var" {
		return varState
	}
	return nil
}

func startState(l *Lexer) StateFunc {
	if l.whitespace() != nil {
		return nil
	}
	r, err := l.Peek()
	if err != nil {
		return nil
	}
	if unicode.IsLetter(r) {
		return keywordState
	}
	return nil
}

func (l *Lexer) NextToken() *Token {
	for l.State != nil {
		l.State = l.State(l)
		if len(l.tokens) > 0 {
			t := l.tokens[len(l.tokens)-1]
			l.tokens = l.tokens[:len(l.tokens)-1]
			return t
		}
	}
	return nil
}

// Prompting Reader
type Repl struct {
	Input *bufio.Reader
	Output *bufio.Writer
	lexer *Lexer
}

func (r *Repl) Init() {
	r.lexer = &Lexer {
		State: startState,
		Input: bufio.NewReader(r),
	}
}

func (r *Repl) Read(p []byte) (n int, err error) {
	for i := 0; i < len(p); i++ {
		b, err := r.Input.ReadByte()
		if err != nil {
			return i, err
		}
		p[i] = b
		if b == '\n' {
			return i, nil
		}
	}
	return len(p), nil
}

func (r *Repl) NextToken() *Token {
	return r.lexer.NextToken()
}

func main() {
	r := Repl {
		Input: bufio.NewReader(os.Stdin),
		Output: bufio.NewWriter(os.Stdout),
	}
	r.Init()
	for t := r.NextToken(); t != nil; t = r.NextToken() {
		if t.Type == ErrorToken {
			fmt.Println(t.Payload)
		}
	}
	fmt.Printf("\n")
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"github.com/prodhe/slides/parse"
)

func main() {
	flag.Parse()

	var f *os.File
	var err error
	var path string

	if flag.NArg() < 1 || flag.Arg(0) == "-" {
		f = os.Stdin
	} else {
		path = flag.Arg(0)
		f, err = os.Open(path)
		if err != nil {
			fmt.Printf("%s: %v\n", path, err)
			os.Exit(1)
		}
	}
	reader := bufio.NewReader(f)
	input, err := ioutil.ReadAll(reader)
	if err != nil {
		fmt.Printf("ReadAll error: %v\n", err)
		os.Exit(1)
	}
	f.Close()

	p := parse.NewParser(path, string(input))

	data, err := p.Parse()
	if err != nil {
		fmt.Printf("parse error: %s", err)
		os.Exit(1)
	}

	http.Handle("/", slides(toHtml(data)))
	fmt.Println("Slides are available at http://localhost:3001/")
	err = http.ListenAndServe(":3001", nil)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func slides(data string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, data)
	})
}

func toHtml(data string) string {
	result := HTML_TMPL

	result = strings.Replace(result, "{{style}}", STYLESHEET, -1)
	result = strings.Replace(result, "{{javascript}}", JAVASCRIPT, -1)

	result = strings.Replace(result, "{{data}}", data, -1)

	return result
}
<commit_msg>Add support for static file serving.<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"github.com/prodhe/slides/parse"
)

func main() {
	flag.Parse()

	var f *os.File
	var err error
	var path string

	if flag.NArg() < 1 || flag.Arg(0) == "-" {
		f = os.Stdin
	} else {
		path = flag.Arg(0)
		f, err = os.Open(path)
		if err != nil {
			fmt.Printf("%s: %v\n", path, err)
			os.Exit(1)
		}
	}
	reader := bufio.NewReader(f)
	input, err := ioutil.ReadAll(reader)
	if err != nil {
		fmt.Printf("ReadAll error: %v\n", err)
		os.Exit(1)
	}
	f.Close()

	p := parse.NewParser(path, string(input))

	data, err := p.Parse()
	if err != nil {
		fmt.Printf("parse error: %s", err)
		os.Exit(1)
	}

	http.Handle("/f/", http.StripPrefix("/f/", http.FileServer(http.Dir("./"))))
	http.Handle("/", slides(toHtml(data)))
	fmt.Println("Slides are available at http://localhost:3001/")
	err = http.ListenAndServe(":3001", nil)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func slides(data string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, data)
	})
}

func toHtml(data string) string {
	result := HTML_TMPL

	result = strings.Replace(result, "{{style}}", STYLESHEET, -1)
	result = strings.Replace(result, "{{javascript}}", JAVASCRIPT, -1)

	result = strings.Replace(result, "{{data}}", data, -1)

	return result
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"
)

// config vars, to be manipulated via command line flags
var (
	carbon    string
	prefix    string
	flush     int
	gen       int
	agentSize int
	jitter    int
	maxAgents int
)

type Agent struct {
	ID            int
	FlushInterval time.Duration
	Addr          string
	MetricNames   []string
}

func (a *Agent) Loop() {
	for {
		select {
		case <-time.NewTicker(a.FlushInterval).C:
			err := a.Flush()
			if err != nil {
				log.Printf("agent %d: %s\n", a.ID, err)
			}
		}
	}
}

func (a *Agent) Flush() error {
	conn, err := net.Dial("tcp", a.Addr)
	if err != nil {
		return err
	}
	defer conn.Close()

	epoch := time.Now().Unix()
	for _, name := range a.MetricNames {
		err := carbonate(conn, name, rand.Intn(1000), epoch)
		if err != nil {
			return err
		}
	}

	log.Printf("agent %d: flushed %d metrics\n", a.ID, len(a.MetricNames))
	return nil
}

// generate N metric names, prefixed with batchID for tracking
func genMetricNames(prefix string, id, n int) []string {
	names := make([]string, n)
	for i := 0; i < n; i++ {
		names[i] = fmt.Sprintf("%s.agent.%d.metrics.%d", prefix, id, i)
	}

	return names
}

// actually write the data in carbon line format
func carbonate(w io.ReadWriteCloser, name string, value int, epoch int64) error {
	_, err := fmt.Fprintf(w, "%s %d %d\n", name, value, epoch)
	if err != nil {
		return err
	}
	return nil
}

func launchAgent(id, n int, flush time.Duration, addr, prefix string) {
	metricNames := genMetricNames(prefix, id, n)

	a := &Agent{ID: id, FlushInterval: time.Duration(flush), Addr: addr, MetricNames: metricNames}
	a.Loop()
}

func init() {
	flag.StringVar(&carbon, "carbon", "localhost:2003", "address of carbon host")
	flag.StringVar(&prefix, "prefix", "bench", "prefix for metrics")
	flag.IntVar(&flush, "flush", 10000, "how often to flush metrics, in millis")
	flag.IntVar(&gen, "gen", 10000, "how often to gen new agents, in millis")
	flag.IntVar(&agentSize, "agent-size", 10000, "number of metrics for each agent to hold")
	flag.IntVar(&jitter, "jitter", 10000, "max amount of jitter to introduce in between agent launches")
	flag.IntVar(&maxAgents, "max-agents", 100, "max number of agents to run concurrently")
}

func main() {
	flag.Parse()

	spawnAgents := true
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGUSR1)

	curID := 0

	log.Printf("master: pid %d\n", os.Getpid())

	go launchAgent(curID, agentSize, time.Duration(flush)*time.Millisecond, carbon, prefix)
	log.Printf("agent %d: launched\n", curID)
	curID++

	for {
		select {
		case <-sigChan:
			spawnAgents = !spawnAgents
			log.Printf("master: spawn_agents=%t\n", spawnAgents)
		case <-time.NewTicker(time.Duration(gen) * time.Millisecond).C:
			if curID < maxAgents {
				if spawnAgents {
					// sleep for some jitter
					time.Sleep(time.Duration(rand.Intn(jitter)) * time.Millisecond)

					go launchAgent(curID, agentSize, time.Duration(flush)*time.Millisecond, carbon, prefix)
					log.Printf("agent %d: launched\n", curID)
					curID++
				}
			}
		}
	}
}
<commit_msg>Use a timer instead of a ticker to manage spawn rate and jitter<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"
)

// config vars, to be manipulated via command line flags
var (
	carbon    string
	prefix    string
	flush     int
	gen       int
	agentSize int
	jitter    int
	maxAgents int
)

type Agent struct {
	ID            int
	FlushInterval time.Duration
	Addr          string
	MetricNames   []string
}

func (a *Agent) Loop() {
	for {
		select {
		case <-time.NewTicker(a.FlushInterval).C:
			err := a.Flush()
			if err != nil {
				log.Printf("agent %d: %s\n", a.ID, err)
			}
		}
	}
}

func (a *Agent) Flush() error {
	conn, err := net.Dial("tcp", a.Addr)
	if err != nil {
		return err
	}
	defer conn.Close()

	epoch := time.Now().Unix()
	for _, name := range a.MetricNames {
		err := carbonate(conn, name, rand.Intn(1000), epoch)
		if err != nil {
			return err
		}
	}

	log.Printf("agent %d: flushed %d metrics\n", a.ID, len(a.MetricNames))
	return nil
}

// generate N metric names, prefixed with batchID for tracking
func genMetricNames(prefix string, id, n int) []string {
	names := make([]string, n)
	for i := 0; i < n; i++ {
		names[i] = fmt.Sprintf("%s.agent.%d.metrics.%d", prefix, id, i)
	}

	return names
}

// actually write the data in carbon line format
func carbonate(w io.ReadWriteCloser, name string, value int, epoch int64) error {
	_, err := fmt.Fprintf(w, "%s %d %d\n", name, value, epoch)
	if err != nil {
		return err
	}
	return nil
}

func launchAgent(id, n int, flush time.Duration, addr, prefix string) {
	metricNames := genMetricNames(prefix, id, n)

	a := &Agent{ID: id, FlushInterval: time.Duration(flush), Addr: addr, MetricNames: metricNames}
	a.Loop()
}

func init() {
	flag.StringVar(&carbon, "carbon", "localhost:2003", "address of carbon host")
	flag.StringVar(&prefix, "prefix", "bench", "prefix for metrics")
	flag.IntVar(&flush, "flush", 10000, "how often to flush metrics, in millis")
	flag.IntVar(&gen, "gen", 10000, "how often to gen new agents, in millis")
	flag.IntVar(&agentSize, "agent-size", 10000, "number of metrics for each agent to hold")
	flag.IntVar(&jitter, "jitter", 10000, "max amount of jitter to introduce in between agent launches")
	flag.IntVar(&maxAgents, "max-agents", 100, "max number of agents to run concurrently")
}

func main() {
	flag.Parse()

	spawnAgents := true
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGUSR1)

	// start timer at 1 milli so we launch an agent right away
	timer := time.NewTimer(1 * time.Millisecond)

	curID := 0

	log.Printf("master: pid %d\n", os.Getpid())

	go launchAgent(curID, agentSize, time.Duration(flush)*time.Millisecond, carbon, prefix)
	log.Printf("agent %d: launched\n", curID)
	curID++

	for {
		select {
		case <-sigChan:
			spawnAgents = !spawnAgents
			log.Printf("master: spawn_agents=%t\n", spawnAgents)
		case <-timer.C:
			if curID < maxAgents {
				if spawnAgents {
					go launchAgent(curID, agentSize, time.Duration(flush)*time.Millisecond, carbon, prefix)
					log.Printf("agent %d: launched\n", curID)
					curID++

					timer = time.NewTimer(time.Duration(flush+rand.Intn(jitter)) * time.Millisecond)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>// +build ignore

// This program builds Drone.
// $ go run make.go build test
//
// The output binaries go into the ./bin/ directory (under the
// project root, where make.go is)
package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/jteeuwen/go-bindata"
)

var (
	version = "0.4"
	sha     = rev()
)

// list of all posible steps that can be executed
// as part of the build process.
var steps = map[string]step{
	"scripts": scripts,
	"styles":  styles,
	"json":    json,
	"embed":   embed,
	"vet":     vet,
	"bindata": bindat,
	"build":   build,
	"test":    test,
	"image":   image,
	"clean":   clean,
}

func main() {
	for _, arg := range os.Args[1:] {
		step, ok := steps[arg]
		if !ok {
			fmt.Println("error: invalid step", arg)
			os.Exit(1)
		}
		err := step()
		if err != nil {
			fmt.Println("error: failed step", arg)
			os.Exit(1)
		}
	}
}

type step func() error

// embed step embeds static files in .go files.
func embed() error {
	// embed drone.{revision}.css
	// embed drone.{revision}.js
	return nil
}

// scripts step concatinates all javascript files.
func scripts() error {
	files := []string{
		"cmd/drone-server/static/scripts/term.js",
		"cmd/drone-server/static/scripts/drone.js",
		"cmd/drone-server/static/scripts/controllers/repos.js",
		"cmd/drone-server/static/scripts/controllers/builds.js",
		"cmd/drone-server/static/scripts/controllers/users.js",
		"cmd/drone-server/static/scripts/services/repos.js",
		"cmd/drone-server/static/scripts/services/builds.js",
		"cmd/drone-server/static/scripts/services/users.js",
		"cmd/drone-server/static/scripts/services/logs.js",
		"cmd/drone-server/static/scripts/services/tokens.js",
		"cmd/drone-server/static/scripts/services/feed.js",
		"cmd/drone-server/static/scripts/filters/filter.js",
		"cmd/drone-server/static/scripts/filters/gravatar.js",
		"cmd/drone-server/static/scripts/filters/time.js",
	}

	f, err := os.OpenFile(
		"cmd/drone-server/static/scripts/drone.min.js",
		os.O_CREATE|os.O_RDWR|os.O_TRUNC,
		0660)

	defer f.Close()

	if err != nil {
		fmt.Println("Failed to open output file")
		return err
	}

	for _, input := range files {
		content, err := ioutil.ReadFile(input)

		if err != nil {
			return err
		}

		f.Write(content)
	}

	return nil
}

// styles step concatinates the stylesheet files.
func styles() error {
	files := []string{
		"cmd/drone-server/static/styles/reset.css",
		"cmd/drone-server/static/styles/fonts.css",
		"cmd/drone-server/static/styles/alert.css",
		"cmd/drone-server/static/styles/blankslate.css",
		"cmd/drone-server/static/styles/list.css",
		"cmd/drone-server/static/styles/label.css",
		"cmd/drone-server/static/styles/range.css",
		"cmd/drone-server/static/styles/switch.css",
		"cmd/drone-server/static/styles/main.css",
	}

	f, err := os.OpenFile(
		"cmd/drone-server/static/styles/drone.min.css",
		os.O_CREATE|os.O_RDWR|os.O_TRUNC,
		0660)

	defer f.Close()

	if err != nil {
		fmt.Println("Failed to open output file")
		return err
	}

	for _, input := range files {
		content, err := ioutil.ReadFile(input)

		if err != nil {
			return err
		}

		f.Write(content)
	}

	return nil
}

// json step generates optimized json marshal and
// unmarshal functions to override defaults.
func json() error {
	return nil
}

// bindata step generates go-bindata package.
func bindat() error {
	var paths = []struct {
		input     string
		recursive bool
	}{
		{"cmd/drone-server/static", true},
	}

	c := bindata.NewConfig()
	c.Output = "cmd/drone-server/drone_bindata.go"
	c.Input = make([]bindata.InputConfig, len(paths))

	for i, path := range paths {
		c.Input[i] = bindata.InputConfig{
			Path:      path.input,
			Recursive: path.recursive,
		}
	}

	return bindata.Translate(c)
}

// build step creates the application binaries.
func build() error {
	var bins = []struct {
		input  string
		output string
	}{
		{"github.com/drone/drone/cmd/drone-server", "bin/drone"},
	}
	for _, bin := range bins {
		ldf := fmt.Sprintf("-X main.revision=%s -X main.version=%s", sha, version)
		cmd := exec.Command("go", "build", "-o", bin.output, "-ldflags", ldf, bin.input)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		trace(cmd.Args)
		err := cmd.Run()
		if err != nil {
			return err
		}
	}
	return nil
}

// vet step executes the `go vet` command
func vet() error {
	cmd := exec.Command("go", "vet",
		"github.com/drone/drone/pkg/...",
		"github.com/drone/drone/cmd/...")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	trace(cmd.Args)
	return cmd.Run()
}

// test step executes unit tests and coverage.
func test() error {
	cmd := exec.Command("go", "test", "-cover", "./pkg/...")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	trace(cmd.Args)
	return cmd.Run()
}

// image step builds Docker images.
func image() error {
	var images = []struct {
		dir  string
		name string
	}{
		{"./bin/drone-server", "drone/drone"},
	}
	for _, image := range images {
		path := filepath.Join(image.dir, "Dockerfile")
		name := image.name + ":" + version
		cmd := exec.Command("docker", "build", "-rm", path, name)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		trace(cmd.Args)
		err := cmd.Run()
		if err != nil {
			return err
		}
	}
	return nil
}

func clean() error {
	err := filepath.Walk(".", func(path string, f os.FileInfo, err error) error {
		suffixes := []string{
			".out",
			"_bindata.go",
		}

		for _, suffix := range suffixes {
			if strings.HasSuffix(path, suffix) {
				if err := os.Remove(path); err != nil {
					return err
				}
			}
		}

		return nil
	})

	if err != nil {
		return err
	}

	files := []string{
		"bin/drone",
	}

	for _, file := range files {
		if _, err := os.Stat(file); err != nil {
			continue
		}

		if err := os.Remove(file); err != nil {
			return err
		}
	}

	return nil
}

// trace is a helper fucntion that writes a command
// to stdout similar to bash +x
func trace(args []string) {
	print("+ ")
	println(strings.Join(args, " "))
}

// helper function to parse the git revision
func rev() string {
	cmd := exec.Command("git", "rev-parse", "--short", "HEAD")
	raw, err := cmd.CombinedOutput()
	if err != nil {
		return "HEAD"
	}
	return strings.Trim(string(raw), "\n")
}
<commit_msg>Updated header comment<commit_after>// +build ignore

// This program builds Drone.
// $ go run make.go deps bindata build test

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/jteeuwen/go-bindata"
)

var (
	version = "0.4"
	sha     = rev()
)

// list of all posible steps that can be executed
// as part of the build process.
var steps = map[string]step{
	"scripts": scripts,
	"styles":  styles,
	"json":    json,
	"embed":   embed,
	"vet":     vet,
	"bindata": bindat,
	"build":   build,
	"test":    test,
	"image":   image,
	"clean":   clean,
}

func main() {
	for _, arg := range os.Args[1:] {
		step, ok := steps[arg]
		if !ok {
			fmt.Println("error: invalid step", arg)
			os.Exit(1)
		}
		err := step()
		if err != nil {
			fmt.Println("error: failed step", arg)
			os.Exit(1)
		}
	}
}

type step func() error

// embed step embeds static files in .go files.
func embed() error {
	// embed drone.{revision}.css
	// embed drone.{revision}.js
	return nil
}

// scripts step concatinates all javascript files.
func scripts() error {
	files := []string{
		"cmd/drone-server/static/scripts/term.js",
		"cmd/drone-server/static/scripts/drone.js",
		"cmd/drone-server/static/scripts/controllers/repos.js",
		"cmd/drone-server/static/scripts/controllers/builds.js",
		"cmd/drone-server/static/scripts/controllers/users.js",
		"cmd/drone-server/static/scripts/services/repos.js",
		"cmd/drone-server/static/scripts/services/builds.js",
		"cmd/drone-server/static/scripts/services/users.js",
		"cmd/drone-server/static/scripts/services/logs.js",
		"cmd/drone-server/static/scripts/services/tokens.js",
		"cmd/drone-server/static/scripts/services/feed.js",
		"cmd/drone-server/static/scripts/filters/filter.js",
		"cmd/drone-server/static/scripts/filters/gravatar.js",
		"cmd/drone-server/static/scripts/filters/time.js",
	}

	f, err := os.OpenFile(
		"cmd/drone-server/static/scripts/drone.min.js",
		os.O_CREATE|os.O_RDWR|os.O_TRUNC,
		0660)

	defer f.Close()

	if err != nil {
		fmt.Println("Failed to open output file")
		return err
	}

	for _, input := range files {
		content, err := ioutil.ReadFile(input)

		if err != nil {
			return err
		}

		f.Write(content)
	}

	return nil
}

// styles step concatinates the stylesheet files.
func styles() error {
	files := []string{
		"cmd/drone-server/static/styles/reset.css",
		"cmd/drone-server/static/styles/fonts.css",
		"cmd/drone-server/static/styles/alert.css",
		"cmd/drone-server/static/styles/blankslate.css",
		"cmd/drone-server/static/styles/list.css",
		"cmd/drone-server/static/styles/label.css",
		"cmd/drone-server/static/styles/range.css",
		"cmd/drone-server/static/styles/switch.css",
		"cmd/drone-server/static/styles/main.css",
	}

	f, err := os.OpenFile(
		"cmd/drone-server/static/styles/drone.min.css",
		os.O_CREATE|os.O_RDWR|os.O_TRUNC,
		0660)

	defer f.Close()

	if err != nil {
		fmt.Println("Failed to open output file")
		return err
	}

	for _, input := range files {
		content, err := ioutil.ReadFile(input)

		if err != nil {
			return err
		}

		f.Write(content)
	}

	return nil
}

// json step generates optimized json marshal and
// unmarshal functions to override defaults.
func json() error {
	return nil
}

// bindata step generates go-bindata package.
func bindat() error {
	var paths = []struct {
		input     string
		recursive bool
	}{
		{"cmd/drone-server/static", true},
	}

	c := bindata.NewConfig()
	c.Output = "cmd/drone-server/drone_bindata.go"
	c.Input = make([]bindata.InputConfig, len(paths))

	for i, path := range paths {
		c.Input[i] = bindata.InputConfig{
			Path:      path.input,
			Recursive: path.recursive,
		}
	}

	return bindata.Translate(c)
}

// build step creates the application binaries.
func build() error {
	var bins = []struct {
		input  string
		output string
	}{
		{"github.com/drone/drone/cmd/drone-server", "bin/drone"},
	}
	for _, bin := range bins {
		ldf := fmt.Sprintf("-X main.revision=%s -X main.version=%s", sha, version)
		cmd := exec.Command("go", "build", "-o", bin.output, "-ldflags", ldf, bin.input)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		trace(cmd.Args)
		err := cmd.Run()
		if err != nil {
			return err
		}
	}
	return nil
}

// vet step executes the `go vet` command
func vet() error {
	cmd := exec.Command("go", "vet",
		"github.com/drone/drone/pkg/...",
		"github.com/drone/drone/cmd/...")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	trace(cmd.Args)
	return cmd.Run()
}

// test step executes unit tests and coverage.
func test() error {
	cmd := exec.Command("go", "test", "-cover", "./pkg/...")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	trace(cmd.Args)
	return cmd.Run()
}

// image step builds Docker images.
func image() error {
	var images = []struct {
		dir  string
		name string
	}{
		{"./bin/drone-server", "drone/drone"},
	}
	for _, image := range images {
		path := filepath.Join(image.dir, "Dockerfile")
		name := image.name + ":" + version
		cmd := exec.Command("docker", "build", "-rm", path, name)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		trace(cmd.Args)
		err := cmd.Run()
		if err != nil {
			return err
		}
	}
	return nil
}

func clean() error {
	err := filepath.Walk(".", func(path string, f os.FileInfo, err error) error {
		suffixes := []string{
			".out",
			"_bindata.go",
		}

		for _, suffix := range suffixes {
			if strings.HasSuffix(path, suffix) {
				if err := os.Remove(path); err != nil {
					return err
				}
			}
		}

		return nil
	})

	if err != nil {
		return err
	}

	files := []string{
		"bin/drone",
	}

	for _, file := range files {
		if _, err := os.Stat(file); err != nil {
			continue
		}

		if err := os.Remove(file); err != nil {
			return err
		}
	}

	return nil
}

// trace is a helper fucntion that writes a command
// to stdout similar to bash +x
func trace(args []string) {
	print("+ ")
	println(strings.Join(args, " "))
}

// helper function to parse the git revision
func rev() string {
	cmd := exec.Command("git", "rev-parse", "--short", "HEAD")
	raw, err := cmd.CombinedOutput()
	if err != nil {
		return "HEAD"
	}
	return strings.Trim(string(raw), "\n")
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Laurent Moussault. All rights reserved.
// Licensed under a simplified BSD license (see LICENSE file).

package glam

import "github.com/drakmaniso/glam/math"

//------------------------------------------------------------------------------

// `Mat4` is a single-precision matrix with 4 columns and 4 rows.
//
// Note: matrix are stored in column-major order, so when writing literals
// remember to use the transpose.
type Mat4 [4][4]float32

//------------------------------------------------------------------------------

// `NewMat4` allocates and returns a new matrix. The elements are stored in
// alphabetical order (column-major).
func NewMat4(
	a, e, i, m,
	b, f, j, n,
	c, g, k, o,
	d, h, l, p float32,
) *Mat4 {
	return &Mat4{
		{a, b, c, d},
		{e, f, g, h},
		{i, j, k, l},
		{m, n, o, p},
	}
}

// `MakeMat4` returns (by value) a matrix. The elements are stored in
// alphabetical order (column-major).
func MakeMat4(
	a, e, i, m,
	b, f, j, n,
	c, g, k, o,
	d, h, l, p float32,
) Mat4 {
	return Mat4{
		{a, b, c, d},
		{e, f, g, h},
		{i, j, k, l},
		{m, n, o, p},
	}
}

// `SetTo` initializes `matrix`. The elements are stored in
// alphabetical order (column-major).
func (matrix *Mat4) SetTo(
	a, e, i, m,
	b, f, j, n,
	c, g, k, o,
	d, h, l, p float32,
) {
	matrix[0][0] = a
	matrix[0][1] = b
	matrix[0][2] = c
	matrix[0][3] = d

	matrix[1][0] = e
	matrix[1][1] = f
	matrix[1][2] = g
	matrix[1][3] = h

	matrix[2][0] = i
	matrix[2][1] = j
	matrix[2][2] = k
	matrix[2][3] = l

	matrix[3][0] = m
	matrix[3][1] = n
	matrix[3][2] = o
	matrix[3][3] = p
}

//------------------------------------------------------------------------------

// `At` returns the element at '(column, row)`.
func (m Mat4) At(column, row int) float32 {
	return m[column][row]
}

// `Set` sets the element at `(column, row)` to `value`.
func (m *Mat4) Set(column, row int, value float32) {
	m[column][row] = value
}

//------------------------------------------------------------------------------

// `Perspective` returns (by value) a perspective projection matrix.
func Perspective(fieldOfView float32, aspectRatio float32, near float32, far float32) Mat4 {
	f := float32(1.0) / math.Tan(fieldOfView/float32(2.0))

	return Mat4{
		{f / aspectRatio, 0, 0, 0},
		{0, f, 0, 0},
		{0, 0, (far + near) / (near - far), -1},
		{0, 0, (2 * far * near) / (near - far), 0},
	}
}

// `SetToPerspective` sets `m` to a perspective projection matrix.
func (m *Mat4) SetToPerspective(fieldOfView float32, aspectRatio float32, near float32, far float32) {
	f := float32(1.0) / math.Tan(fieldOfView/float32(2.0))

	m[0][0] = f / aspectRatio
	m[0][1] = 0
	m[0][2] = 0
	m[0][3] = 0

	m[0][0] = 0
	m[0][1] = f
	m[0][2] = 0
	m[0][3] = 0

	m[0][0] = 0
	m[0][1] = 0
	m[0][2] = (far + near) / (near - far)
	m[0][3] = -1

	m[0][0] = 0
	m[0][1] = 0
	m[0][2] = (2 * far * near) / (near - far)
	m[0][3] = 0
}

//------------------------------------------------------------------------------
<commit_msg>Typo.<commit_after>// Copyright (c) 2013 Laurent Moussault. All rights reserved.
// Licensed under a simplified BSD license (see LICENSE file).

package glam

import "github.com/drakmaniso/glam/math"

//------------------------------------------------------------------------------

// `Mat4` is a single-precision matrix with 4 columns and 4 rows.
//
// Note: matrices are stored in column-major order, so when writing literals
// remember to use the transpose.
type Mat4 [4][4]float32

//------------------------------------------------------------------------------

// `NewMat4` allocates and returns a new matrix. The elements are stored in
// alphabetical order (column-major order).
func NewMat4(
	a, e, i, m,
	b, f, j, n,
	c, g, k, o,
	d, h, l, p float32,
) *Mat4 {
	return &Mat4{
		{a, b, c, d},
		{e, f, g, h},
		{i, j, k, l},
		{m, n, o, p},
	}
}

// `MakeMat4` returns (by value) a matrix. The elements are stored in
// alphabetical order (column-major order).
func MakeMat4(
	a, e, i, m,
	b, f, j, n,
	c, g, k, o,
	d, h, l, p float32,
) Mat4 {
	return Mat4{
		{a, b, c, d},
		{e, f, g, h},
		{i, j, k, l},
		{m, n, o, p},
	}
}

// `SetTo` initializes `matrix`. The elements are stored in
// alphabetical order (column-major order).
func (matrix *Mat4) SetTo(
	a, e, i, m,
	b, f, j, n,
	c, g, k, o,
	d, h, l, p float32,
) {
	matrix[0][0] = a
	matrix[0][1] = b
	matrix[0][2] = c
	matrix[0][3] = d

	matrix[1][0] = e
	matrix[1][1] = f
	matrix[1][2] = g
	matrix[1][3] = h

	matrix[2][0] = i
	matrix[2][1] = j
	matrix[2][2] = k
	matrix[2][3] = l

	matrix[3][0] = m
	matrix[3][1] = n
	matrix[3][2] = o
	matrix[3][3] = p
}

//------------------------------------------------------------------------------

// `At` returns the element at '(column, row)`.
func (m Mat4) At(column, row int) float32 {
	return m[column][row]
}

// `Set` sets the element at `(column, row)` to `value`.
func (m *Mat4) Set(column, row int, value float32) {
	m[column][row] = value
}

//------------------------------------------------------------------------------

// `Perspective` returns (by value) a perspective projection matrix.
func Perspective(fieldOfView float32, aspectRatio float32, near float32, far float32) Mat4 {
	f := float32(1.0) / math.Tan(fieldOfView/float32(2.0))

	return Mat4{
		{f / aspectRatio, 0, 0, 0},
		{0, f, 0, 0},
		{0, 0, (far + near) / (near - far), -1},
		{0, 0, (2 * far * near) / (near - far), 0},
	}
}

// `SetToPerspective` sets `m` to a perspective projection matrix.
func (m *Mat4) SetToPerspective(fieldOfView float32, aspectRatio float32, near float32, far float32) {
	f := float32(1.0) / math.Tan(fieldOfView/float32(2.0))

	m[0][0] = f / aspectRatio
	m[0][1] = 0
	m[0][2] = 0
	m[0][3] = 0

	m[0][0] = 0
	m[0][1] = f
	m[0][2] = 0
	m[0][3] = 0

	m[0][0] = 0
	m[0][1] = 0
	m[0][2] = (far + near) / (near - far)
	m[0][3] = -1

	m[0][0] = 0
	m[0][1] = 0
	m[0][2] = (2 * far * near) / (near - far)
	m[0][3] = 0
}

//------------------------------------------------------------------------------
<|endoftext|>


<commit_before>// menu.go
package main

import (
	"fmt"
	"os"

	"github.com/daviddengcn/go-colortext"
	"github.com/kylelemons/goat/term"
)

func main() {
	list := os.Args[1:]
	if len(list) == 0 {
		return
	}
	current := 0

	const (
		UP    = "\x1b\x5b\x41"
		DOWN  = "\x1b\x5b\x42"
		ENTER = "\r"
	)

	for i, line := range list {
		if i == current {
			ct.ChangeColor(ct.Black, false, ct.White, false)
		} else {
			ct.ResetColor()
		}
		fmt.Print(line)
		ct.ResetColor()
		if i < len(list)-1 {
			fmt.Print("\r\n")
		}
	}

	for i := range list {
		if i > 0 {
			fmt.Print(UP)
		}
	}

	raw := make([]byte, 10)
	tty := term.NewRawTTY(os.Stdin)
	tty.SetEcho(nil)
	for {
		n, err := tty.Read(raw)
		if err != nil {
			fmt.Printf("read: %s\n", err)
			return
		}
		str := string(raw[:n])
		switch str {
		case term.Interrupt:
			for i := current; i < len(list); i++ {
				fmt.Print("\n")
			}
			fmt.Print("\r")
			os.Exit(-1)
		case UP:
			if current > 0 {
				fmt.Print("\r")
				fmt.Print(list[current])

				current--
				fmt.Print(str)

				fmt.Print("\r")
				ct.ChangeColor(ct.Black, false, ct.White, false)
				fmt.Print(list[current])
				ct.ResetColor()
			}
		case DOWN:
			if current < len(list)-1 {
				fmt.Print("\r")
				fmt.Print(list[current])

				current++
				fmt.Print(str)

				fmt.Print("\r")
				ct.ChangeColor(ct.Black, false, ct.White, false)
				fmt.Print(list[current])
				ct.ResetColor()
			}
		case ENTER:
			for i := current; i < len(list); i++ {
				fmt.Print("\n")
			}
			fmt.Print("\r")
			os.Exit(current)
		}
	}
}
<commit_msg>Create Select function for importing<commit_after>// menu.go
package main

import (
	"fmt"
	"os"

	"github.com/daviddengcn/go-colortext"
	"github.com/kylelemons/goat/term"
)

func Select(list ...string) int {
	current := 0

	const (
		UP    = "\x1b\x5b\x41"
		DOWN  = "\x1b\x5b\x42"
		ENTER = "\r"
	)

	for i, line := range list {
		if i == current {
			ct.ChangeColor(ct.Black, false, ct.White, false)
		} else {
			ct.ResetColor()
		}
		fmt.Print(line)
		ct.ResetColor()
		if i < len(list)-1 {
			fmt.Print("\r\n")
		}
	}

	for i := range list {
		if i > 0 {
			fmt.Print(UP)
		}
	}

	raw := make([]byte, 10)
	tty := term.NewRawTTY(os.Stdin)
	tty.SetEcho(nil)
	for {
		n, err := tty.Read(raw)
		if err != nil {
			fmt.Printf("read: %s\n", err)
			return -1
		}
		str := string(raw[:n])
		switch str {
		case term.Interrupt:
			for i := current; i < len(list); i++ {
				fmt.Print("\n")
			}
			fmt.Print("\r")
			return -1
		case UP:
			if current > 0 {
				fmt.Print("\r")
				fmt.Print(list[current])

				current--
				fmt.Print(str)

				fmt.Print("\r")
				ct.ChangeColor(ct.Black, false, ct.White, false)
				fmt.Print(list[current])
				ct.ResetColor()
			}
		case DOWN:
			if current < len(list)-1 {
				fmt.Print("\r")
				fmt.Print(list[current])

				current++
				fmt.Print(str)

				fmt.Print("\r")
				ct.ChangeColor(ct.Black, false, ct.White, false)
				fmt.Print(list[current])
				ct.ResetColor()
			}
		case ENTER:
			for i := current; i < len(list); i++ {
				fmt.Print("\n")
			}
			fmt.Print("\r")
			return current
		}
	}
}

func main() {
	list := os.Args[1:]
	if len(list) == 0 {
		return
	}
	os.Exit(Select(list...))
}
<|endoftext|>


<commit_before>package gwr

import "text/template"

const metaNounName = "/meta/nouns"

var nounsTextTemplate = template.Must(template.New("meta_nouns_text").Parse(`
{{- define "get" -}}
{{ range $name, $info := . -}}
- {{ $name }} formats: {{ $info.Formats }}
{{ end -}}
{{- end -}}
`))

type dataSourceUpdate struct {
	Type string         `json:"type"`
	Info DataSourceInfo `json:"info"`
}

type metaNounDataSource struct {
	sources *DataSources
	watcher GenericDataWatcher
}

func (nds *metaNounDataSource) Name() string {
	return metaNounName
}

func (nds *metaNounDataSource) Attrs() map[string]interface{} {
	return nil
}

func (nds *metaNounDataSource) TextTemplate() *template.Template {
	return nounsTextTemplate
}

func (nds *metaNounDataSource) Get() interface{} {
	sources := nds.sources.sources
	info := make(map[string]DataSourceInfo, len(sources))
	for name, ds := range sources {
		info[name] = dsInfo(ds)
	}
	return info
}

func (nds *metaNounDataSource) GetInit() interface{} {
	return nds.Get()
}

func (nds *metaNounDataSource) Watch(watcher GenericDataWatcher) {
	nds.watcher = watcher
}

func (nds *metaNounDataSource) dataSourceAdded(ds DataSource) {
	if nds.watcher != nil {
		nds.watcher(dataSourceUpdate{"add", dsInfo(ds)})
	}
}

func dsInfo(ds DataSource) DataSourceInfo {
	return ds.Info()
}
<commit_msg>metaNounDataSource: stop dealing in DataSourceInfo structs<commit_after>package gwr

import "text/template"

const metaNounName = "/meta/nouns"

var nounsTextTemplate = template.Must(template.New("meta_nouns_text").Parse(`
{{- define "get" -}}
{{ range $name, $info := . -}}
- {{ $name }} formats: {{ $info.Formats }}
{{ end -}}
{{- end -}}
`))

type dataSourceUpdate struct {
	Type string                 `json:"type"`
	Info map[string]interface{} `json:"info"`
}

type metaNounDataSource struct {
	sources *DataSources
	watcher GenericDataWatcher
}

func (nds *metaNounDataSource) Name() string {
	return metaNounName
}

func (nds *metaNounDataSource) Attrs() map[string]interface{} {
	return nil
}

func (nds *metaNounDataSource) TextTemplate() *template.Template {
	return nounsTextTemplate
}

func (nds *metaNounDataSource) Get() interface{} {
	sources := nds.sources.sources
	info := make(map[string]interface{}, len(sources))
	for name, ds := range sources {
		info[name] = dsInfo(ds)
	}
	return info
}

func (nds *metaNounDataSource) GetInit() interface{} {
	return nds.Get()
}

func (nds *metaNounDataSource) Watch(watcher GenericDataWatcher) {
	nds.watcher = watcher
}

func (nds *metaNounDataSource) dataSourceAdded(ds DataSource) {
	if nds.watcher != nil {
		nds.watcher(dataSourceUpdate{"add", dsInfo(ds)})
	}
}

func dsInfo(ds DataSource) map[string]interface{} {
	info := ds.Info()
	return map[string]interface{}{
		"formats": info.Formats,
		"attrs": info.Attrs,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"io"
	"log"
	"mime"
	"net/http"
	"strings"
)

// Filtering by MIME type.

// The action to take for files of each type.
// If it is not specified here, types beginning with "text" will be
// content-filtered, and others will be allowed without scanning the content.
var mimeActions = map[string]action{}

// We use the action type from categories.go, but instead of BLOCK, IGNORE, and
// ALLOW, here it is BLOCK, FILTER, and ALLOW.
const FILTER = IGNORE

var mimeAllow = newActiveFlag("mime-allow", "", "content type to allow without phrase scan",
	func(t string) error {
		mimeActions[t] = ALLOW
		return nil
	})

var mimeFilter = newActiveFlag("mime-filter", "", "content type to filter",
	func(t string) error {
		mimeActions[t] = FILTER
		return nil
	})

var mimeBlock = newActiveFlag("mime-block", "", "content type to block",
	func(t string) error {
		mimeActions[t] = BLOCK
		return nil
	})

// checkContentType examines the request's Content-Type header, and potentially
// its content as well, to determine the content's MIME type.
// Then it decides, based on the MIME type, whether it should be allowed,
// filtered, or blocked.
func checkContentType(resp *http.Response) (contentType string, a action) {
	ct, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
	if err != nil || !strings.Contains(ct, "/") {
		ct = ""
	}

	if ce := resp.Header.Get("Content-Encoding"); ce != "" && ce != "gzip" {
		// If the server is using a Content-Encoding that we don't understand,
		// we can't decode the content to filter it.
		log.Println("unknown Content-Encoding", ce, "for", resp.Request.URL)
		return ct, ALLOW
	}

	switch ct {
	case "text/plain", "text/html", "unknown/unknown", "application/unknown", "*/*", "", "application/octet-stream":
		if resp.Header.Get("Content-Encoding") == "" {
			// These types tend to be used for content whose type is unknown,
			// so we should try to second-guess them.
			// But we don't bother if the content is gzipped; then we'd get application/gzip.
			// We can hope (probably in vain) that a server smart enough to compress the
			// content is smart enough to give us a correct media type.
			preview := make([]byte, 512)
			n, _ := resp.Body.Read(preview)
			preview = preview[:n]

			if n > 0 {
				ct, _, _ = mime.ParseMediaType(http.DetectContentType(preview))

				// Make the preview data available for re-reading.
				var rc struct {
					io.Reader
					io.Closer
				}
				rc.Reader = io.MultiReader(bytes.NewBuffer(preview), resp.Body)
				rc.Closer = resp.Body
				resp.Body = rc
			}
		}
	}

	if a, ok := mimeActions[ct]; ok {
		return ct, a
	}
	if strings.HasPrefix(ct, "text/") {
		return ct, FILTER
	}
	return ct, ALLOW
}

// baseType strips off any modifiers (such as charset) and returns the simple
// MIME type.
func baseType(t string) string {
	if semicolon := strings.Index(t, ";"); semicolon != -1 {
		t = t[:semicolon]
	}
	return strings.TrimSpace(t)
}
<commit_msg>Handle (broken) Content-Encoding: utf-8 header.<commit_after>package main

import (
	"bytes"
	"io"
	"log"
	"mime"
	"net/http"
	"strings"
)

// Filtering by MIME type.

// The action to take for files of each type.
// If it is not specified here, types beginning with "text" will be
// content-filtered, and others will be allowed without scanning the content.
var mimeActions = map[string]action{}

// We use the action type from categories.go, but instead of BLOCK, IGNORE, and
// ALLOW, here it is BLOCK, FILTER, and ALLOW.
const FILTER = IGNORE

var mimeAllow = newActiveFlag("mime-allow", "", "content type to allow without phrase scan",
	func(t string) error {
		mimeActions[t] = ALLOW
		return nil
	})

var mimeFilter = newActiveFlag("mime-filter", "", "content type to filter",
	func(t string) error {
		mimeActions[t] = FILTER
		return nil
	})

var mimeBlock = newActiveFlag("mime-block", "", "content type to block",
	func(t string) error {
		mimeActions[t] = BLOCK
		return nil
	})

// checkContentType examines the request's Content-Type header, and potentially
// its content as well, to determine the content's MIME type.
// Then it decides, based on the MIME type, whether it should be allowed,
// filtered, or blocked.
func checkContentType(resp *http.Response) (contentType string, a action) {
	ct, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
	if err != nil || !strings.Contains(ct, "/") {
		ct = ""
	}

	switch resp.Header.Get("Content-Encoding") {
	case "", "gzip":
		// This is an encoding we can understand.

	case "utf-8":
		// This is an error.
		resp.Header.Set("Content-Encoding", "")

	default:
		// If the server is using a Content-Encoding that we don't understand,
		// we can't decode the content to filter it.
		log.Println("unknown Content-Encoding", resp.Header.Get("Content-Encoding"), "for", resp.Request.URL)
		return ct, ALLOW
	}

	switch ct {
	case "text/plain", "text/html", "unknown/unknown", "application/unknown", "*/*", "", "application/octet-stream":
		if resp.Header.Get("Content-Encoding") == "" {
			// These types tend to be used for content whose type is unknown,
			// so we should try to second-guess them.
			// But we don't bother if the content is gzipped; then we'd get application/gzip.
			// We can hope (probably in vain) that a server smart enough to compress the
			// content is smart enough to give us a correct media type.
			preview := make([]byte, 512)
			n, _ := resp.Body.Read(preview)
			preview = preview[:n]

			if n > 0 {
				ct, _, _ = mime.ParseMediaType(http.DetectContentType(preview))

				// Make the preview data available for re-reading.
				var rc struct {
					io.Reader
					io.Closer
				}
				rc.Reader = io.MultiReader(bytes.NewBuffer(preview), resp.Body)
				rc.Closer = resp.Body
				resp.Body = rc
			}
		}
	}

	if a, ok := mimeActions[ct]; ok {
		return ct, a
	}
	if strings.HasPrefix(ct, "text/") {
		return ct, FILTER
	}
	return ct, ALLOW
}

// baseType strips off any modifiers (such as charset) and returns the simple
// MIME type.
func baseType(t string) string {
	if semicolon := strings.Index(t, ";"); semicolon != -1 {
		t = t[:semicolon]
	}
	return strings.TrimSpace(t)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/flate"
	"encoding/binary"
	"fmt"
	"os"
)

type PDFormat struct {
	Name             [32]byte //start at byte 0
	Attributes       uint16   //32
	Version          uint16   //34
	CreationDate     uint32   //36
	ModifyDate       uint32   //40
	BackupDate       uint32   //44
	ModifyNumber     uint32   //48
	AppInfoID        uint32   //52
	SortInfoID       uint32   //56
	Type             [4]byte  //60
	Creator          [4]byte  //64
	UniqueIDSeed     uint32   //68
	NextRecordListID uint32   //72
	SectionCount     uint16   //76-78
}

type PDRecordInfoSection struct {
	DataOffset uint32  //starts at byte 0
	Attributes byte    //4
	UniqueID   [3]byte //5-8
}

type PDHeader struct {
	CompressionType uint16 //starts at byte 0
	_               uint16 //2 (always zero?)
	TextLength      uint32 //4
	RecordCount     uint16 //8
	RecordSize      uint16 //10
	CurrentPosition uint32 //12-16
}

type Mobi8Header struct {
	//note that since the values are being read
	//by bytes.Read, we can't use "_" as a name,
	//or have any unexported fields.
	//This is stupid, but whatever.
	//We use the name "SkipX" instead.
	CompressionType     uint16   //start at byte 0
	Skip1               uint16   //2 (always zero?)
	TextLength          uint32   //4
	RecordCount         uint16   //8
	RecordSize          uint16   //10
	CryptoType          uint16   //12
	Skip2               uint16   //14 filler
	Identifier          [4]byte  //16
	HeaderLength        uint32   //20
	Type                uint32   //24
	TextEncoding        uint32   //28
	UniqueID            uint32   //32
	Version             uint32   //36
	OrtographicIndex    uint32   //40
	IncflectionIndex    uint32   //44
	IndexNames          uint32   //48
	IndexKeys           uint32   //52
	Extra               [24]byte //56
	FirstNontext        uint32   //80
	TitleOffset         uint32   //84
	TitleLength         uint32   //88
	Locale              uint32   //92
	InputLanguage       uint32   //96
	OutputLanguage      uint32   //100
	MinVersion          uint32   //104
	FirstImageOffset    uint32   //108
	HuffmanRecordOffset uint32   //112
	HuffmanRecordCount  uint32   //116
	HuffmanTableOffset  uint32   //120
	HuffTableLength     uint32   //124
	ExthFlags           uint32   //128
	Skip3               [32]byte //132
	Unknown0            uint32   //164
	DrmOffset           uint32   //168
	DrmCount            uint32   //172
	DrmSize             uint32   //176
	DrmFlags            uint32   //180
	Skip4               [8]byte  //184
	FirstContentNumber  uint32   //192
	FdstFlowCount       uint32   //196
	FcisOffset          uint32   //200
	FcisCount           uint32   //204
	FlisOffset          uint32   //208
	FlisCount           uint32   //212
	Skip5               [8]byte  //216
	SrcsOffset          uint32   //224
	SrcsCount           uint32   //228
	Skip6               [8]byte  //232
	TrailDataFlags      uint16   //240
	NcxIndex            uint32   //244
	FragmentIndex       uint32   //248
	SkeletonIndex       uint32   //252
	DatpOffset          uint32   //256
	GuideIndex          uint32   //260
}

func GetStruct(file *os.File, hd interface{}, length int, offset int64) (rd int, err error) {
	b := make([]byte, length)
	rd, err = file.ReadAt(b, offset)
	if err != nil {
		return
	}
	buf := bytes.NewBuffer(b)
	err = binary.Read(buf, binary.BigEndian, hd)
	return
}

type ExthHeader struct {
	Identifier   uint32 //starts at byte 0
	HeaderLength uint32 //4
	RecordCount  uint32 //8-12
}

type ExthRecordInfo struct {
	RecordType   uint32 //starts at 0
	RecordLength uint32 //4-8
}

type ExthRecordData []byte

type FileHeader struct {
	Format   PDFormat
	Sections []PDRecordInfoSection
}

func main() {
	hd, err := GetFileHeader("file.mobi")
	check(err)
	fmt.Printf("%#v\n", hd.Format)
	fmt.Printf("%v %v\n", hd.Sections[0], hd.Sections[181])
	var pd Mobi8Header
	file, err := os.Open("file.mobi")
	_, err = GetStruct(file, &pd, 300, 100)
	fmt.Println(err, pd)
}

//GetPDRecordInfoSectionList reads `count` items from `file`,
//starting at byte `offset` and placing the result in in `ris`.
//Returns the number of records read, and any error.
func GetPDRecordInfoSectionList(file *os.File, ris *[]PDRecordInfoSection, count int, start int) (ii int, err error) {
	for ii = 0; ii < count; ii++ {
		var section PDRecordInfoSection
		_, err = GetStruct(file, &section, 8, int64(start+ii*8))
		*ris = append(*ris, section)
	}
	return
}

//GetFileHeader reads the header information from the file at path `path`.
//Returns the FileHeader as read, and any error.
func GetFileHeader(path string) (hd FileHeader, err error) {
	file, err := os.Open(path)
	defer file.Close()
	start := 0
	if err != nil {
		return
	}

	start, err = GetStruct(file, &hd.Format, 78, 0)
	if err != nil {
		return
	}

	rdr := flate.NewReader(file)
	defer rdr.Close()

	_, err = GetPDRecordInfoSectionList(file, &hd.Sections, int(hd.Format.SectionCount), start)
	fmt.Println(start)
	return
}

//check helps panic when there's an error.
func check(err error) {
	if err != nil {
		panic(err)
	}
}
<commit_msg>reading a single mobiheader seems to be working.<commit_after>package main

import (
	"bytes"
	"compress/flate"
	"encoding/binary"
	"fmt"
	"os"
)

type PDFormat struct {
	Name             [32]byte //start at byte 0
	Attributes       uint16   //32
	Version          uint16   //34
	CreationDate     uint32   //36
	ModifyDate       uint32   //40
	BackupDate       uint32   //44
	ModifyNumber     uint32   //48
	AppInfoID        uint32   //52
	SortInfoID       uint32   //56
	Type             [4]byte  //60
	Creator          [4]byte  //64
	UniqueIDSeed     uint32   //68
	NextRecordListID uint32   //72
	SectionCount     uint16   //76-78
}

type PDRecordInfoSection struct {
	DataOffset uint32  //starts at byte 0
	Attributes byte    //4
	UniqueID   [3]byte //5-8
}

type PDHeader struct {
	CompressionType uint16 //starts at byte 0
	_               uint16 //2 (always zero?)
	TextLength      uint32 //4
	RecordCount     uint16 //8
	RecordSize      uint16 //10
	CurrentPosition uint32 //12-16
}

type Mobi8Header struct {
	//note that since the values are being read
	//by bytes.Read, we can't use "_" as a name,
	//or have any unexported fields.
	//This is stupid, but whatever.
	//We use the name "SkipX" instead.
	CompressionType     uint16   //start at byte 0
	Skip1               uint16   //2 (always zero?)
	TextLength          uint32   //4
	RecordCount         uint16   //8
	RecordSize          uint16   //10
	CryptoType          uint16   //12
	Skip2               uint16   //14 filler
	Identifier          [4]byte  //16
	HeaderLength        uint32   //20
	Type                uint32   //24
	TextEncoding        uint32   //28
	UniqueID            uint32   //32
	Version             uint32   //36
	OrtographicIndex    uint32   //40
	IncflectionIndex    uint32   //44
	IndexNames          uint32   //48
	IndexKeys           uint32   //52
	Extra               [24]byte //56
	FirstNontext        uint32   //80
	TitleOffset         uint32   //84
	TitleLength         uint32   //88
	Locale              uint32   //92
	InputLanguage       uint32   //96
	OutputLanguage      uint32   //100
	MinVersion          uint32   //104
	FirstImageOffset    uint32   //108
	HuffmanRecordOffset uint32   //112
	HuffmanRecordCount  uint32   //116
	HuffmanTableOffset  uint32   //120
	HuffTableLength     uint32   //124
	ExthFlags           uint32   //128
	Skip3               [32]byte //132
	Unknown0            uint32   //164
	DrmOffset           uint32   //168
	DrmCount            uint32   //172
	DrmSize             uint32   //176
	DrmFlags            uint32   //180
	Skip4               [8]byte  //184
	FirstContentNumber  uint32   //192
	FdstFlowCount       uint32   //196
	FcisOffset          uint32   //200
	FcisCount           uint32   //204
	FlisOffset          uint32   //208
	FlisCount           uint32   //212
	Skip5               [8]byte  //216
	SrcsOffset          uint32   //224
	SrcsCount           uint32   //228
	Skip6               [8]byte  //232
	TrailDataFlags      uint16   //240
	NcxIndex            uint32   //244
	FragmentIndex       uint32   //248
	SkeletonIndex       uint32   //252
	DatpOffset          uint32   //256
	GuideIndex          uint32   //260
}

func GetStruct(file *os.File, hd interface{}, length int, offset int64) (rd int, err error) {
	b := make([]byte, length)
	rd, err = file.ReadAt(b, offset)
	if err != nil {
		return
	}
	buf := bytes.NewBuffer(b)
	err = binary.Read(buf, binary.BigEndian, hd)
	return
}

type ExthHeader struct {
	Identifier   uint32 //starts at byte 0
	HeaderLength uint32 //4
	RecordCount  uint32 //8-12
}

type ExthRecordInfo struct {
	RecordType   uint32 //starts at 0
	RecordLength uint32 //4-8
}

type ExthRecordData []byte

type FileHeader struct {
	Format   PDFormat
	Sections []PDRecordInfoSection
}

func main() {
	hd, err := GetFileHeader("file.mobi")
	check(err)
	fmt.Printf("%#v\n", hd.Format)
	fmt.Printf("%v %v\n", hd.Sections[0], hd.Sections[181])
	var pd Mobi8Header
	file, err := os.Open("file.mobi")
	rd, err := GetStruct(file, &pd, 300, int64(hd.Sections[0].DataOffset))
	fmt.Println(err, rd, pd)
	fmt.Println(pd.TextEncoding, pd.UniqueID)
}

//GetPDRecordInfoSectionList reads `count` items from `file`,
//starting at byte `offset` and placing the result in in `ris`.
//Returns the number of records read, and any error.
func GetPDRecordInfoSectionList(file *os.File, ris *[]PDRecordInfoSection, count int, start int) (ii int, err error) {
	for ii = 0; ii < count; ii++ {
		var section PDRecordInfoSection
		_, err = GetStruct(file, &section, 8, int64(start+ii*8))
		*ris = append(*ris, section)
	}
	return
}

//GetFileHeader reads the header information from the file at path `path`.
//Returns the FileHeader as read, and any error.
func GetFileHeader(path string) (hd FileHeader, err error) {
	file, err := os.Open(path)
	defer file.Close()
	start := 0
	if err != nil {
		return
	}

	start, err = GetStruct(file, &hd.Format, 78, 0)
	if err != nil {
		return
	}

	rdr := flate.NewReader(file)
	defer rdr.Close()

	_, err = GetPDRecordInfoSectionList(file, &hd.Sections, int(hd.Format.SectionCount), start)
	fmt.Println(start)
	return
}

//check helps panic when there's an error.
func check(err error) {
	if err != nil {
		panic(err)
	}
}
<|endoftext|>