<commit_before>package main

import (
	"fmt"
    "strings"
    "time"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type Tokens struct {
	ID        bson.ObjectId `bson:"_id,omitempty"`
	Tokens      []string
	//count     int
	Timestamp time.Time
}

type Ngrams struct {
	ID        bson.ObjectId `bson:"_id,omitempty"`
	Ngram      string
	Count     int
	Timestamp time.Time
}

type PosWordGram struct {
	ID        bson.ObjectId `bson:"_id,omitempty"`
	PosTag      string		"posTag"
	Word		string
	Count     int
}

type PosUniGram struct {
	ID        bson.ObjectId `bson:"_id,omitempty"`
	PosTag      string		"posTag"
	Count     int
}

func InsertTokens(tokens []string) {
    maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    c := session.DB("nlprokz").C("brownCorpus")
    err = c.Insert(&Tokens{Tokens: tokens, Timestamp: time.Now()})

	if err != nil {
		panic(err)
	}
}

func InsertWordPosgram(tokens []string) {
    maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("wordPosgram")
    //pos := session.DB("nlprokz").C("posUnigram")
	for j := 0; j < len(tokens); j++ {

        var temp = strings.Split(tokens[j],"/")
        if(len(temp) > 1) {
            posTag := temp[len(temp)-1]
            if(strings.Index(posTag,"+") != -1 && strings.Index(posTag,"+") != (len(posTag)-1)  ) {
                temp = strings.Split(posTag,"+")
                if(len(temp[1]) > 1 || temp[1] != "*") {
                    pos.Upsert(bson.M{"posTag":temp[1]},bson.M{"$inc": bson.M{"count": 1}})
                }
                posTag = temp[0]
            }
            if(strings.Contains(posTag,"-")) {
                posTag = strings.Split(posTag,"-")[0]
            }
            word := temp[0]
            if(len(temp) > 2) {
                word = strings.Join(temp[0:len(temp)-1],"/")
                fmt.Println("%s------------%s\n",tokens,word)
            }
            // if(strings.Contains(posTag,"-") ||strings.Contains(posTag,"+") || len(posTag) < 1 || posTag == "*") {
            //     fmt.Printf("%#v\n",tokens)
            // }

            pos.Upsert(bson.M{"posTag":posTag,"word":word},bson.M{"$inc": bson.M{"count": 1}})
        }
	}
}

func InsertPOSNgram(tokens []string, n int) {
    maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("posTags")
	for j := 0; j < (len(tokens) - n); j++ {
        posSeq := make([]string,0)
        otherSeq := make([]string,0)
        for a := 0; a < n; a++ {
            var temp = strings.Split(tokens[j+a],"/")
            if(len(temp) > 1) {
                posTag := temp[len(temp)-1]
                if(strings.Index(posTag,"+") != -1 && strings.Index(posTag,"+") != (len(posTag)-1)  ) {
                    temp = strings.Split(posTag,"+")
                    if(len(temp[1]) > 1 || temp[1] != "*") {
                        //pos.Upsert(bson.M{"posTag":temp[1]},bson.M{"$inc": bson.M{"count": 1}})
                        if (len(posSeq) == 1) {
                            pos.Upsert(&Ngrams{Ngram:posSeq[0]+" "+temp[1]},bson.M{"$inc": bson.M{"count": 1}})
                            if(len(otherSeq) == 1) {
                                fmt.Printf("%s----------",otherSeq[0]+otherSeq[1])
                                pos.Upsert(&Ngrams{Ngram:otherSeq[0]+" "+otherSeq[1]},bson.M{"$inc": bson.M{"count": 1}})
                            }
                        } else {
                            otherSeq = append(otherSeq,temp[1])
                        }
                    }
                    posTag = temp[0]
                }
                if(strings.Contains(posTag,"-")) {
                    posTag = strings.Split(posTag,"-")[0]
                }
                posSeq = append(posSeq,posTag)
            }
        }
        if (len(otherSeq) == 1 && len(posSeq) > 1) {
            // fmt.Printf("%v\n",tokens)
            // fmt.Printf("%s++++++++++\n",otherSeq[0]+posSeq[1])
            pos.Upsert(&Ngrams{Ngram:otherSeq[0]+" "+posSeq[1]},bson.M{"$inc": bson.M{"count": 1}})
        }
        pos.Upsert(&Ngrams{Ngram:strings.Join(posSeq," ")},bson.M{"$inc": bson.M{"count": 1}})
	}
}

func getPosgram(posGram string) int {
    session, err := mgo.Dial("127.0.0.1")
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("posTags")
	results := Ngrams{}
	pos.Find(bson.M{"ngram": posGram}).One(&results)
	//fmt.Printf("%#v",results.Count)
	return results.Count
}

func getAllPosgram(posGram []string) []Ngrams {
    session, err := mgo.Dial("127.0.0.1")
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("posTags")
	var results []Ngrams
	pos.Find(bson.M{"ngram": bson.M{"$in":posGram}}).All(&results)
	// fmt.Printf("%#v",posGram)
	// fmt.Printf("%v==========\n",len(results))
	return results
}

func getWordPosgram(word string) []PosWordGram {
	maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("wordPosgram")
	var results []PosWordGram
	pos.Find(bson.M{"word":word}).All(&results)
	// fmt.Printf("%#v",results)
	return results
}

func getWordPosgramCount(word string) []PosWordGram {
	maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("wordPosgram")
	var results []PosWordGram
	pos.Find(bson.M{"word":word}).All(&results)
	// fmt.Printf("%#v",results)
	return results
}

func getAllPosUnigrams() []PosUniGram {
	maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("posUnigram")
	var results []PosUniGram
	pos.Find(bson.M{}).All(&results)
	// fmt.Printf("%#v",results)
	return results
}

func getFrequencyAggregation() []bson.M {
	maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("posTags")
	result := []bson.M{}
	pos.Pipe([]bson.M{bson.M{"$match": bson.M{}},bson.M{"$group":bson.M{"_id":"$count","count":bson.M{"$sum":1}}},bson.M{"$sort":bson.M{"count":-1}}}).All(&result)
	//fmt.Printf("%v",len(result))
	return result
}
// func main() {
//
//
//
//
// 	// // Drop Database
// 	// if IsDrop {
// 	// 	err = session.DB("nlprokz").DropDatabase()
// 	// 	if err != nil {
// 	// 		panic(err)
// 	// 	}
// 	// }
//
// 	// Collection People
//
//
// 	// Index
// 	// index := mgo.Index{
// 	// 	Key:        []string{"name", "phone"},
// 	// 	Unique:     true,
// 	// 	DropDups:   true,
// 	// 	Background: true,
// 	// 	Sparse:     true,
// 	// }
//
// 	// err = c.EnsureIndex(index)
// 	// if err != nil {
// 	// 	panic(err)
// 	// }
//
// 	// Insert Datas
//
//
// 	// Query One
// 	result := Person{}
// 	err = c.Find(bson.M{"name": "Ale"}).Select(bson.M{"phone": 0}).One(&result)
// 	if err != nil {
// 		panic(err)
// 	}
// 	fmt.Println("Phone", result)
//
// 	// Query All
// 	var results []Person
// 	err = c.Find(bson.M{"name": "Ale"}).Sort("-timestamp").All(&results)
//
// 	if err != nil {
// 		panic(err)
// 	}
// 	fmt.Println("Results All: ", results)
//
// 	// Update
// 	colQuerier := bson.M{"name": "Ale"}
// 	change := bson.M{"$set": bson.M{"phone": "+86 99 8888 7777", "timestamp": time.Now()}}
// 	err = c.Update(colQuerier, change)
// 	if err != nil {
// 		panic(err)
// 	}
//
// 	// Query All
// 	err = c.Find(bson.M{"name": "Ale"}).Sort("-timestamp").All(&results)
//
// 	if err != nil {
// 		panic(err)
// 	}
// 	fmt.Println("Results All: ", results)
// }
<commit_msg>added pre-processing for the aggregation result<commit_after>package main

import (
	"fmt"
    "strings"
    "time"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type Tokens struct {
	ID        bson.ObjectId `bson:"_id,omitempty"`
	Tokens      []string
	//count     int
	Timestamp time.Time
}

type Ngrams struct {
	ID        bson.ObjectId `bson:"_id,omitempty"`
	Ngram      string
	Count     int
	Timestamp time.Time
}

type PosWordGram struct {
	ID        bson.ObjectId `bson:"_id,omitempty"`
	PosTag      string		"posTag"
	Word		string
	Count     int
}

type PosUniGram struct {
	ID        bson.ObjectId `bson:"_id,omitempty"`
	PosTag      string		"posTag"
	Count     int
}

func InsertTokens(tokens []string) {
    maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    c := session.DB("nlprokz").C("brownCorpus")
    err = c.Insert(&Tokens{Tokens: tokens, Timestamp: time.Now()})

	if err != nil {
		panic(err)
	}
}

func InsertWordPosgram(tokens []string) {
    maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("wordPosgram")
    //pos := session.DB("nlprokz").C("posUnigram")
	for j := 0; j < len(tokens); j++ {

        var temp = strings.Split(tokens[j],"/")
        if(len(temp) > 1) {
            posTag := temp[len(temp)-1]
            if(strings.Index(posTag,"+") != -1 && strings.Index(posTag,"+") != (len(posTag)-1)  ) {
                temp = strings.Split(posTag,"+")
                if(len(temp[1]) > 1 || temp[1] != "*") {
                    pos.Upsert(bson.M{"posTag":temp[1]},bson.M{"$inc": bson.M{"count": 1}})
                }
                posTag = temp[0]
            }
            if(strings.Contains(posTag,"-")) {
                posTag = strings.Split(posTag,"-")[0]
            }
            word := temp[0]
            if(len(temp) > 2) {
                word = strings.Join(temp[0:len(temp)-1],"/")
                fmt.Println("%s------------%s\n",tokens,word)
            }
            // if(strings.Contains(posTag,"-") ||strings.Contains(posTag,"+") || len(posTag) < 1 || posTag == "*") {
            //     fmt.Printf("%#v\n",tokens)
            // }

            pos.Upsert(bson.M{"posTag":posTag,"word":word},bson.M{"$inc": bson.M{"count": 1}})
        }
	}
}

func InsertPOSNgram(tokens []string, n int) {
    maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("posTags")
	for j := 0; j < (len(tokens) - n); j++ {
        posSeq := make([]string,0)
        otherSeq := make([]string,0)
        for a := 0; a < n; a++ {
            var temp = strings.Split(tokens[j+a],"/")
            if(len(temp) > 1) {
                posTag := temp[len(temp)-1]
                if(strings.Index(posTag,"+") != -1 && strings.Index(posTag,"+") != (len(posTag)-1)  ) {
                    temp = strings.Split(posTag,"+")
                    if(len(temp[1]) > 1 || temp[1] != "*") {
                        //pos.Upsert(bson.M{"posTag":temp[1]},bson.M{"$inc": bson.M{"count": 1}})
                        if (len(posSeq) == 1) {
                            pos.Upsert(&Ngrams{Ngram:posSeq[0]+" "+temp[1]},bson.M{"$inc": bson.M{"count": 1}})
                            if(len(otherSeq) == 1) {
                                fmt.Printf("%s----------",otherSeq[0]+otherSeq[1])
                                pos.Upsert(&Ngrams{Ngram:otherSeq[0]+" "+otherSeq[1]},bson.M{"$inc": bson.M{"count": 1}})
                            }
                        } else {
                            otherSeq = append(otherSeq,temp[1])
                        }
                    }
                    posTag = temp[0]
                }
                if(strings.Contains(posTag,"-")) {
                    posTag = strings.Split(posTag,"-")[0]
                }
                posSeq = append(posSeq,posTag)
            }
        }
        if (len(otherSeq) == 1 && len(posSeq) > 1) {
            // fmt.Printf("%v\n",tokens)
            // fmt.Printf("%s++++++++++\n",otherSeq[0]+posSeq[1])
            pos.Upsert(&Ngrams{Ngram:otherSeq[0]+" "+posSeq[1]},bson.M{"$inc": bson.M{"count": 1}})
        }
        pos.Upsert(&Ngrams{Ngram:strings.Join(posSeq," ")},bson.M{"$inc": bson.M{"count": 1}})
	}
}

func getPosgram(posGram string) int {
    session, err := mgo.Dial("127.0.0.1")
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("posTags")
	results := Ngrams{}
	pos.Find(bson.M{"ngram": posGram}).One(&results)
	//fmt.Printf("%#v",results.Count)
	return results.Count
}

func getAllPosgram(posGram []string) []Ngrams {
    session, err := mgo.Dial("127.0.0.1")
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("posTags")
	var results []Ngrams
	pos.Find(bson.M{"ngram": bson.M{"$in":posGram}}).All(&results)
	// fmt.Printf("%#v",posGram)
	// fmt.Printf("%v==========\n",len(results))
	return results
}

func getWordPosgram(word string) []PosWordGram {
	maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("wordPosgram")
	var results []PosWordGram
	pos.Find(bson.M{"word":word}).All(&results)
	// fmt.Printf("%#v",results)
	return results
}

func getWordPosgramCount(word string) []PosWordGram {
	maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("wordPosgram")
	var results []PosWordGram
	pos.Find(bson.M{"word":word}).All(&results)
	// fmt.Printf("%#v",results)
	return results
}

func getAllPosUnigrams() []PosUniGram {
	maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("posUnigram")
	var results []PosUniGram
	pos.Find(bson.M{}).All(&results)
	// fmt.Printf("%#v",results)
	return results
}

func getFrequencyAggregation() []bson.M {
	maxWait := time.Duration(5 * time.Second)
    session, err := mgo.DialWithTimeout("127.0.0.1",maxWait)
	if err != nil {
		panic(err)
	}
    defer session.Close()
	session.SetMode(mgo.Monotonic, true)
    pos := session.DB("nlprokz").C("posTags")
	result := []bson.M{}
	pos.Pipe([]bson.M{bson.M{"$match": bson.M{}},bson.M{"$group":bson.M{"_id":"$count","count":bson.M{"$sum":1}}},bson.M{"$sort":bson.M{"count":-1}}}).All(&result)
	//fmt.Printf("%v",len(result))
	results := make(map[int] int,len(result))
	for _,i := range result {
		//fmt.Printf("%v\n",i["_id"])
		results[i["_id"].(int)] = i["count"].(int)
	}
	fmt.Printf("%v",len(results))
	return result
}
// func main() {
//
//
//
//
// 	// // Drop Database
// 	// if IsDrop {
// 	// 	err = session.DB("nlprokz").DropDatabase()
// 	// 	if err != nil {
// 	// 		panic(err)
// 	// 	}
// 	// }
//
// 	// Collection People
//
//
// 	// Index
// 	// index := mgo.Index{
// 	// 	Key:        []string{"name", "phone"},
// 	// 	Unique:     true,
// 	// 	DropDups:   true,
// 	// 	Background: true,
// 	// 	Sparse:     true,
// 	// }
//
// 	// err = c.EnsureIndex(index)
// 	// if err != nil {
// 	// 	panic(err)
// 	// }
//
// 	// Insert Datas
//
//
// 	// Query One
// 	result := Person{}
// 	err = c.Find(bson.M{"name": "Ale"}).Select(bson.M{"phone": 0}).One(&result)
// 	if err != nil {
// 		panic(err)
// 	}
// 	fmt.Println("Phone", result)
//
// 	// Query All
// 	var results []Person
// 	err = c.Find(bson.M{"name": "Ale"}).Sort("-timestamp").All(&results)
//
// 	if err != nil {
// 		panic(err)
// 	}
// 	fmt.Println("Results All: ", results)
//
// 	// Update
// 	colQuerier := bson.M{"name": "Ale"}
// 	change := bson.M{"$set": bson.M{"phone": "+86 99 8888 7777", "timestamp": time.Now()}}
// 	err = c.Update(colQuerier, change)
// 	if err != nil {
// 		panic(err)
// 	}
//
// 	// Query All
// 	err = c.Find(bson.M{"name": "Ale"}).Sort("-timestamp").All(&results)
//
// 	if err != nil {
// 		panic(err)
// 	}
// 	fmt.Println("Results All: ", results)
// }
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	maxminddb "github.com/oschwald/maxminddb-golang"
)

type DB struct {
	path string
}

// Note that cache may not always be filled.
type MetaCache struct {
	sync.RWMutex
	cache *maxminddb.Metadata
}

var mcache = &MetaCache{}

func (d *DB) checkForUpdates() (needsUpdate bool, err error) {
	curSeconds := time.Now().UnixNano() / int64(time.Second)
	stat, err := os.Stat(d.path)
	if err != nil {
		if os.IsNotExist(err) {
			return true, nil
		}

		return true, err
	}

	var db *maxminddb.Reader

	db, err = maxminddb.Open(d.path)
	if err != nil {
		return true, err
	}

	mcache.Lock()
	mcache.cache = &db.Metadata
	mcache.Unlock()

	if curSeconds-(stat.ModTime().UnixNano()/int64(time.Second)) < 604800 {
		return false, nil
	}

	return true, nil
}

func (d *DB) update(url string) error {
	started := time.Now()
	debug.Printf("fetching new geoip data from: %s", url)

	// Create or truncate if already exists.
	tmpfile, err := ioutil.TempFile("", "geoipdb-")
	if err != nil {
		return fmt.Errorf("unable to create temp file: %s", err)
	}
	defer func() {
		if err = tmpfile.Close(); err != nil {
			debug.Printf("error while closing %q: %s", tmpfile.Name(), err)
		}
		debug.Printf("deleting: %q", tmpfile.Name())
		if err = os.Remove(tmpfile.Name()); err != nil {
			debug.Printf("error while removing %q: %s", tmpfile.Name(), err)
		}
	}()

	debug.Printf("streaming new database to: %q", tmpfile.Name())
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	if resp.Body != nil {
		defer resp.Body.Close()
	}

	gz, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}

	if _, err = io.Copy(tmpfile, gz); err != nil {
		return err
	}

	if err = gz.Close(); err != nil {
		return err
	}

	debug.Printf("successfully downloaded and decompressed new database to %q, verifying now", tmpfile.Name())
	if _, err = tmpfile.Seek(0, 0); err != nil {
		return err
	}

	db, err := maxminddb.Open(tmpfile.Name())
	if err != nil {
		return fmt.Errorf("error while attempting to verify geoip data: %s", err)
	}

	if err = db.Verify(); err != nil {
		db.Close()
		return fmt.Errorf("error while attempting to verify geoip data: %s", err)
	}

	mcache.Lock()
	mcache.cache = &db.Metadata
	mcache.Unlock()
	db.Close()

	debug.Println("verification complete, updating active database")

	file, err := os.Create(d.path)
	if err != nil {
		return err
	}

	var written int64
	written, err = io.Copy(file, tmpfile)
	if err != nil {
		return err
	}

	debug.Printf("successfully wrote %d bytes to %q (took %s)", written, file.Name(), time.Since(started))

	return nil
}

// IPSearch is the struct->tag search query to search through the Maxmind DB.
type IPSearch struct {
	City struct {
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"city"`
	Country struct {
		Code  string            `maxminddb:"iso_code"`
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"country"`
	Continent struct {
		Code  string            `maxminddb:"code"`
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"continent"`
	Location struct {
		Lat       float64 `maxminddb:"latitude"`
		Long      float64 `maxminddb:"longitude"`
		MetroCode int     `maxminddb:"metro_code"`
		TimeZone  string  `maxminddb:"time_zone"`
	} `maxminddb:"location"`
	Postal struct {
		Code string `maxminddb:"code"`
	} `maxminddb:"postal"`
	Subdivisions []struct {
		Code  string            `maxminddb:"iso_code"`
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"subdivisions"`
	Traits struct {
		Proxy bool `maxminddb:"is_anonymous_proxy"`
	} `maxminddb:"traits"`

	// RegisteredCountry struct {
	// 	Code  string            `maxminddb:"iso_code"`
	// 	Names map[string]string `maxminddb:"names"`
	// } `maxminddb:"registered_country"`
	// RepresentedCountry struct {
	// 	Code  string            `maxminddb:"iso_code"`
	// 	Names map[string]string `maxminddb:"names"`
	// 	Type  string            `maxminddb:"type"`
	// } `maxminddb:"represented_country"`
}

// AddrResult contains the geolocation and host information for an IP/host.
type AddrResult struct {
	IP            net.IP   `json:"ip"`
	City          string   `json:"city"`
	Subdivision   string   `json:"subdivision"`
	Country       string   `json:"country"`
	CountryCode   string   `json:"country_abbr"`
	Continent     string   `json:"continent"`
	ContinentCode string   `json:"continent_abbr"`
	Lat           float64  `json:"latitude"`
	Long          float64  `json:"longitude"`
	Timezone      string   `json:"timezone"`
	PostalCode    string   `json:"postal_code"`
	Proxy         bool     `json:"proxy"`
	Hosts         []string `json:"hosts"`
	MapURL        string   `json:"map_url"`
	MapEmbedURL   string   `json:"map_embed_url"`
}

const mapURI = "https://maps.google.com/maps?f=q&ie=UTF8&iwloc=A&z=0&q=%s"
const mapEmbedURI = "https://maps.google.com/maps?f=q&ie=UTF8&iwloc=A&output=embed&z=%d&q=%s"

// addrLookup does a geoip lookup of an IP address
func addrLookup(path string, addr net.IP) (*AddrResult, error) {
	db, err := maxminddb.Open(path)
	if err != nil {
		return nil, err
	}

	var query IPSearch

	err = db.Lookup(addr, &query)
	db.Close()

	if err != nil {
		return nil, err
	}

	result := &AddrResult{
		IP:            addr,
		City:          query.City.Names["en"],
		Country:       query.Country.Names["en"],
		CountryCode:   query.Country.Code,
		Continent:     query.Continent.Names["en"],
		ContinentCode: query.Continent.Code,
		Lat:           query.Location.Lat,
		Long:          query.Location.Long,
		Timezone:      query.Location.TimeZone,
		PostalCode:    query.Postal.Code,
		Proxy:         query.Traits.Proxy,
	}

	var subdiv []string
	for i := 0; i < len(query.Subdivisions); i++ {
		subdiv = append(subdiv, query.Subdivisions[i].Names["en"])
	}
	result.Subdivision = strings.Join(subdiv, ", ")

	mapZoom := 2
	var mapQuery string
	switch {
	case result.City != "":
		mapZoom = 6
	case result.Subdivision != "":
		mapZoom = 4
	case result.Country != "":
		mapZoom = 3
	}

	switch {
	case result.Lat != 0 || result.Long != 0:
		mapQuery = fmt.Sprintf("%f,%f", result.Lat, result.Long)
	case result.City != "" && result.Subdivision != "" && result.CountryCode != "":
		mapQuery = fmt.Sprintf("%s, %s, %s", result.City, result.Subdivision, result.CountryCode)
	case result.Subdivision != "" && result.CountryCode != "":
		mapQuery = fmt.Sprintf("%s, %s", result.Subdivision, result.CountryCode)
	default:
		mapQuery = result.Country
	}
	result.MapURL = fmt.Sprintf(mapURI, url.QueryEscape(mapQuery))
	result.MapEmbedURL = fmt.Sprintf(mapEmbedURI, mapZoom, url.QueryEscape(mapQuery))

	if names, err := net.LookupAddr(addr.String()); err == nil {
		for i := 0; i < len(names); i++ {
			// These are FQDN's where absolute hosts contain a suffixed ".".
			result.Hosts = append(result.Hosts, strings.TrimSuffix(names[i], "."))
		}
	}

	return result, nil
}
<commit_msg>remove unused fields<commit_after>package main

import (
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	maxminddb "github.com/oschwald/maxminddb-golang"
)

type DB struct {
	path string
}

// Note that cache may not always be filled.
type MetaCache struct {
	sync.RWMutex
	cache *maxminddb.Metadata
}

var mcache = &MetaCache{}

func (d *DB) checkForUpdates() (needsUpdate bool, err error) {
	curSeconds := time.Now().UnixNano() / int64(time.Second)
	stat, err := os.Stat(d.path)
	if err != nil {
		if os.IsNotExist(err) {
			return true, nil
		}

		return true, err
	}

	var db *maxminddb.Reader

	db, err = maxminddb.Open(d.path)
	if err != nil {
		return true, err
	}

	mcache.Lock()
	mcache.cache = &db.Metadata
	mcache.Unlock()

	if curSeconds-(stat.ModTime().UnixNano()/int64(time.Second)) < 604800 {
		return false, nil
	}

	return true, nil
}

func (d *DB) update(url string) error {
	started := time.Now()
	debug.Printf("fetching new geoip data from: %s", url)

	// Create or truncate if already exists.
	tmpfile, err := ioutil.TempFile("", "geoipdb-")
	if err != nil {
		return fmt.Errorf("unable to create temp file: %s", err)
	}
	defer func() {
		if err = tmpfile.Close(); err != nil {
			debug.Printf("error while closing %q: %s", tmpfile.Name(), err)
		}
		debug.Printf("deleting: %q", tmpfile.Name())
		if err = os.Remove(tmpfile.Name()); err != nil {
			debug.Printf("error while removing %q: %s", tmpfile.Name(), err)
		}
	}()

	debug.Printf("streaming new database to: %q", tmpfile.Name())
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	if resp.Body != nil {
		defer resp.Body.Close()
	}

	gz, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}

	if _, err = io.Copy(tmpfile, gz); err != nil {
		return err
	}

	if err = gz.Close(); err != nil {
		return err
	}

	debug.Printf("successfully downloaded and decompressed new database to %q, verifying now", tmpfile.Name())
	if _, err = tmpfile.Seek(0, 0); err != nil {
		return err
	}

	db, err := maxminddb.Open(tmpfile.Name())
	if err != nil {
		return fmt.Errorf("error while attempting to verify geoip data: %s", err)
	}

	if err = db.Verify(); err != nil {
		db.Close()
		return fmt.Errorf("error while attempting to verify geoip data: %s", err)
	}

	mcache.Lock()
	mcache.cache = &db.Metadata
	mcache.Unlock()
	db.Close()

	debug.Println("verification complete, updating active database")

	file, err := os.Create(d.path)
	if err != nil {
		return err
	}

	var written int64
	written, err = io.Copy(file, tmpfile)
	if err != nil {
		return err
	}

	debug.Printf("successfully wrote %d bytes to %q (took %s)", written, file.Name(), time.Since(started))

	return nil
}

// IPSearch is the struct->tag search query to search through the Maxmind DB.
type IPSearch struct {
	City struct {
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"city"`
	Country struct {
		Code  string            `maxminddb:"iso_code"`
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"country"`
	Continent struct {
		Code  string            `maxminddb:"code"`
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"continent"`
	Location struct {
		Lat       float64 `maxminddb:"latitude"`
		Long      float64 `maxminddb:"longitude"`
		MetroCode int     `maxminddb:"metro_code"`
		TimeZone  string  `maxminddb:"time_zone"`
	} `maxminddb:"location"`
	Postal struct {
		Code string `maxminddb:"code"`
	} `maxminddb:"postal"`
	Subdivisions []struct {
		Code  string            `maxminddb:"iso_code"`
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"subdivisions"`
	Traits struct {
		Proxy bool `maxminddb:"is_anonymous_proxy"`
	} `maxminddb:"traits"`
}

// AddrResult contains the geolocation and host information for an IP/host.
type AddrResult struct {
	IP            net.IP   `json:"ip"`
	City          string   `json:"city"`
	Subdivision   string   `json:"subdivision"`
	Country       string   `json:"country"`
	CountryCode   string   `json:"country_abbr"`
	Continent     string   `json:"continent"`
	ContinentCode string   `json:"continent_abbr"`
	Lat           float64  `json:"latitude"`
	Long          float64  `json:"longitude"`
	Timezone      string   `json:"timezone"`
	PostalCode    string   `json:"postal_code"`
	Proxy         bool     `json:"proxy"`
	Hosts         []string `json:"hosts"`
	MapURL        string   `json:"map_url"`
	MapEmbedURL   string   `json:"map_embed_url"`
}

const mapURI = "https://maps.google.com/maps?f=q&ie=UTF8&iwloc=A&z=0&q=%s"
const mapEmbedURI = "https://maps.google.com/maps?f=q&ie=UTF8&iwloc=A&output=embed&z=%d&q=%s"

// addrLookup does a geoip lookup of an IP address
func addrLookup(path string, addr net.IP) (*AddrResult, error) {
	db, err := maxminddb.Open(path)
	if err != nil {
		return nil, err
	}

	var query IPSearch

	err = db.Lookup(addr, &query)
	db.Close()

	if err != nil {
		return nil, err
	}

	result := &AddrResult{
		IP:            addr,
		City:          query.City.Names["en"],
		Country:       query.Country.Names["en"],
		CountryCode:   query.Country.Code,
		Continent:     query.Continent.Names["en"],
		ContinentCode: query.Continent.Code,
		Lat:           query.Location.Lat,
		Long:          query.Location.Long,
		Timezone:      query.Location.TimeZone,
		PostalCode:    query.Postal.Code,
		Proxy:         query.Traits.Proxy,
	}

	var subdiv []string
	for i := 0; i < len(query.Subdivisions); i++ {
		subdiv = append(subdiv, query.Subdivisions[i].Names["en"])
	}
	result.Subdivision = strings.Join(subdiv, ", ")

	mapZoom := 2
	var mapQuery string
	switch {
	case result.City != "":
		mapZoom = 6
	case result.Subdivision != "":
		mapZoom = 4
	case result.Country != "":
		mapZoom = 3
	}

	switch {
	case result.Lat != 0 || result.Long != 0:
		mapQuery = fmt.Sprintf("%f,%f", result.Lat, result.Long)
	case result.City != "" && result.Subdivision != "" && result.CountryCode != "":
		mapQuery = fmt.Sprintf("%s, %s, %s", result.City, result.Subdivision, result.CountryCode)
	case result.Subdivision != "" && result.CountryCode != "":
		mapQuery = fmt.Sprintf("%s, %s", result.Subdivision, result.CountryCode)
	default:
		mapQuery = result.Country
	}
	result.MapURL = fmt.Sprintf(mapURI, url.QueryEscape(mapQuery))
	result.MapEmbedURL = fmt.Sprintf(mapEmbedURI, mapZoom, url.QueryEscape(mapQuery))

	if names, err := net.LookupAddr(addr.String()); err == nil {
		for i := 0; i < len(names); i++ {
			// These are FQDN's where absolute hosts contain a suffixed ".".
			result.Hosts = append(result.Hosts, strings.TrimSuffix(names[i], "."))
		}
	}

	return result, nil
}
<|endoftext|>


<commit_before>/*
Package clerk provides file persistence for your Go structures.

It can replace external databases for small projects that can keep data in memory and don't make a lot of writes.
*/
package clerk

import (
	"encoding/gob"
	"errors"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"
)

var (
	errDBInited    = errors.New("clerk: database inited multiple times")
	errDBNotInited = errors.New("clerk: database not inited")
)

// DB provides methods to persist your data.
type DB struct {
	UpdatedAt   time.Time // UpdatedAt is the timestamp of the last data modification (by calling DB.Lock).
	filename    string
	tmpFilename string
	parent      interface{}
	mu          sync.RWMutex
}

// DBInterface represents an underlying database.
type DBInterface interface {
	setFilename(string)
	setParent(interface{})
	inited() bool
	Save() error
	Rebase() error
	Remove() error
	Lock()
	RLock()
	Unlock()
	RUnlock()
}

// Init sets the filename for source saving.
func Init(filename string, db DBInterface) {
	if db.inited() {
		panic(errDBInited)
	}
	db.setFilename(filename)
	db.setParent(db)
}

func (d *DB) setFilename(name string) {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.filename = name
	// Set d.tmpFilename
	dir := filepath.Dir(name)
	base := filepath.Base(name)
	d.tmpFilename = filepath.Join(dir, "~"+base)
}

func (d *DB) setParent(p interface{}) {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.parent = p
}

func (d *DB) inited() bool {
	return d.filename != "" && d.tmpFilename != "" && d.parent != nil
}

// Save persists the database in the file set on init.
// Be sure the database is locked for reading.
func (d *DB) Save() error {
	if !d.inited() {
		return errDBNotInited
	}

	tmpFile, err := os.OpenFile(d.tmpFilename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return err
	}
	if err = gob.NewEncoder(tmpFile).Encode(d.parent); err != nil && err != io.EOF {
		tmpFile.Close()
		return err
	}
	if err = tmpFile.Close(); err != nil {
		return err
	}
	if err = os.Rename(d.tmpFilename, d.filename); err != nil {
		return err
	}
	return nil
}

// Rebase replaces the data with the content of the file set on init.
// Be sure the database is locked for writing.
func (d *DB) Rebase() error {
	if !d.inited() {
		return errDBNotInited
	}

	file, err := os.OpenFile(d.filename, os.O_RDONLY|os.O_CREATE, 0666)
	if err != nil {
		return err
	}
	defer file.Close()

	if err = gob.NewDecoder(file).Decode(d.parent); err != nil && err != io.EOF {
		return err
	}
	return nil
}

// Remove deletes the database file.
func (d *DB) Remove() error {
	return os.Remove(d.filename)
}

// Lock locks database for reading and writing.
func (d *DB) Lock() {
	d.UpdatedAt = time.Now()
	d.mu.Lock()
}

// RLock locks database for reading.
func (d *DB) RLock() {
	d.mu.RLock()
}

// Unlock unlocks database for reading and writing.
func (d *DB) Unlock() {
	d.mu.Unlock()
}

// RUnlock unlocks database for reading.
func (d *DB) RUnlock() {
	d.mu.RUnlock()
}
<commit_msg>Add DB.Migrate<commit_after>/*
Package clerk provides file persistence for your Go structures.

It can replace external databases for small projects that can keep data in memory and don't make a lot of writes.
*/
package clerk

import (
	"encoding/gob"
	"errors"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"
)

// Errors.
var (
	ErrDBInited     = errors.New("clerk: database inited multiple times")
	ErrDBNotInited  = errors.New("clerk: database not inited")
	ErrOldMigration = errors.New("clerk: database in file is newer than migration timestamp")
)

// DB provides methods to persist your data.
type DB struct {
	UpdatedAt   time.Time // UpdatedAt is the timestamp of the last data modification (by calling DB.Lock).
	filename    string
	tmpFilename string
	parent      interface{}
	mu          sync.RWMutex
}

// DBInterface represents an underlying database.
type DBInterface interface {
	updatedAt() time.Time
	setFilename(string)
	setParent(interface{})
	inited() bool
	Touch()
	Save() error
	Migrate(string, DBInterface, func() error) error
	MigrateOrRebase(string, DBInterface, func() error)
	Rebase() error
	Remove() error
	Lock()
	RLock()
	Unlock()
	RUnlock()
}

// Init sets the filename for source saving.
func Init(filename string, db DBInterface) {
	if db.inited() {
		panic(ErrDBInited)
	}
	db.setFilename(filename)
	db.setParent(db)
}

func (db *DB) updatedAt() time.Time {
	return db.UpdatedAt
}

func (db *DB) setFilename(name string) {
	db.filename = name
	// Set d.tmpFilename
	dir := filepath.Dir(name)
	base := filepath.Base(name)
	db.tmpFilename = filepath.Join(dir, "~"+base)
}

func (db *DB) setParent(p interface{}) {
	db.parent = p
}

func (db *DB) inited() bool {
	db.mu.RLock()
	defer db.mu.RUnlock()
	return db.filename != "" && db.tmpFilename != "" && db.parent != nil
}

// Touch updates the UpdatedAt field.
func (db *DB) Touch() {
	db.UpdatedAt = time.Now()
}

// Save persists the database in the file set on init.
// Be sure the database is locked for reading.
func (db *DB) Save() error {
	if !db.inited() {
		return ErrDBNotInited
	}
	db.Touch()
	tmpFile, err := os.OpenFile(db.tmpFilename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return err
	}
	if err = gob.NewEncoder(tmpFile).Encode(db.parent); err != nil && err != io.EOF {
		tmpFile.Close()
		return err
	}
	if err = tmpFile.Close(); err != nil {
		return err
	}
	if err = os.Rename(db.tmpFilename, db.filename); err != nil {
		return err
	}
	return nil
}

// Rebase replaces the data with the content of the file set on init.
// Be sure the database is locked for writing.
func (db *DB) Rebase() error {
	if !db.inited() {
		return ErrDBNotInited
	}

	file, err := os.OpenFile(db.filename, os.O_RDONLY|os.O_CREATE, 0666)
	if err != nil {
		return err
	}
	defer file.Close()

	if err = gob.NewDecoder(file).Decode(db.parent); err != nil && err != io.EOF {
		return err
	}
	return nil
}

// Migrate runs a migration if provided date is after the one registered in the data file.
// It rebases the database file in oldDB, runs function do and saves database.
// timestamp format is:
//	2006-01-02 15:04:05 -07
func (db *DB) Migrate(timestamp string, oldDB DBInterface, do func() error) error {
	if !db.inited() {
		return ErrDBNotInited
	}
	ts, err := time.Parse("2006-01-02 15:04:05 -07", timestamp)
	if err != nil {
		return err
	}
	Init(db.filename, oldDB)
	if err = oldDB.Rebase(); err != nil {
		return err
	}
	if !ts.After(oldDB.updatedAt()) {
		return ErrOldMigration
	}
	if err = do(); err != nil {
		return err
	}
	return db.Save()
}

// MigrateOrRebase runs Migrate.
// If the migration has not been used, database is rebased.
// It panics on error.
func (db *DB) MigrateOrRebase(timestamp string, oldDB DBInterface, do func() error) {
	if err := db.Migrate(timestamp, oldDB, do); err != nil {
		if err != ErrOldMigration {
			panic(err)
		}
		if err = db.Rebase(); err != nil {
			panic(err)
		}
	}
}

// Remove deletes the database file.
func (db *DB) Remove() error {
	return os.Remove(db.filename)
}

// Lock locks database for reading and writing.
func (db *DB) Lock() {
	db.Touch()
	db.mu.Lock()
}

// RLock locks database for reading.
func (db *DB) RLock() {
	db.mu.RLock()
}

// Unlock unlocks database for reading and writing.
func (db *DB) Unlock() {
	db.mu.Unlock()
}

// RUnlock unlocks database for reading.
func (db *DB) RUnlock() {
	db.mu.RUnlock()
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"time"

	"github.com/umisama/go-sqlbuilder"
)

var (
	peopleTable = sqlbuilder.NewTable(
		"people",
		&sqlbuilder.TableOption{Unique: [][]string{{"hub", "topic"}}},
		sqlbuilder.StringColumn("feed_url", &sqlbuilder.ColumnOption{NotNull: true, PrimaryKey: true}),
		sqlbuilder.DateColumn("first_seen", &sqlbuilder.ColumnOption{NotNull: true}),
		sqlbuilder.StringColumn("name", nil),
		sqlbuilder.StringColumn("display_name", nil),
		sqlbuilder.StringColumn("email", nil),
		sqlbuilder.StringColumn("summary", nil),
		sqlbuilder.StringColumn("note", nil),
	)

	postsTable = sqlbuilder.NewTable(
		"posts",
		&sqlbuilder.TableOption{Unique: [][]string{{"feed_url", "id"}}},
		sqlbuilder.IntColumn("ROWID", nil),
		sqlbuilder.StringColumn("feed_url", &sqlbuilder.ColumnOption{NotNull: true}),
		sqlbuilder.StringColumn("id", &sqlbuilder.ColumnOption{NotNull: true}),
		sqlbuilder.StringColumn("created_at", &sqlbuilder.ColumnOption{NotNull: true}),
		sqlbuilder.StringColumn("raw_entry", &sqlbuilder.ColumnOption{NotNull: true}),
	)
)

func savePerson(db *sql.DB, feedURL string, author *AtomAuthor) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	var name, displayName, email, summary, note string
	if err := tx.QueryRow("select name, display_name, email, summary, note from people where feed_url = $1", feedURL).Scan(&name, &displayName, &email, &summary, &note); err != nil {
		if err != sql.ErrNoRows {
			return err
		}

		if _, err := tx.Exec("insert into people (feed_url, first_seen, name, display_name, email, summary, note) values ($1, $2, $3, $4, $5, $6, $7)", feedURL, time.Now(), author.Name, author.DisplayName, author.Email, author.Summary, author.Note); err != nil {
			return err
		}
	} else {
		if name == author.Name && displayName == author.DisplayName && email == author.Email && summary == author.Summary && note == author.Note {
			return nil
		}

		if _, err := tx.Exec("update people set name = $1, display_name = $2, email = $3, summary = $4, note = $5 where feed_url = $6", author.Name, author.DisplayName, author.Email, author.Summary, author.Note, feedURL); err != nil {
			return err
		}
	}

	return tx.Commit()
}

func saveEntry(db *sql.DB, feedURL string, entry *AtomEntry) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	var exists int
	if err := tx.QueryRow("select count(1) from posts where feed_url = $1 and id = $2", feedURL, entry.ID).Scan(&exists); err != nil {
		return err
	}

	if exists > 0 {
		return nil
	}

	d, err := json.Marshal(entry)
	if err != nil {
		return err
	}

	if _, err := tx.Exec("insert into posts (feed_url, id, created_at, raw_entry) values ($1, $2, $3, $4)", feedURL, entry.ID, entry.Published, string(d)); err != nil {
		return err
	}

	return tx.Commit()
}

type getPublicTimelineArgs struct {
	AfterID int `qstring:"after_id"`
	Offset  int `qstring:"offset,omitempty"`
	Limit   int `qstring:"limit,omitempty"`
}

func getPublicTimeline(db *sql.DB, args getPublicTimelineArgs) ([]UIStatus, error) {
	qb := sqlbuilder.
		Select(postsTable.LeftOuterJoin(peopleTable, peopleTable.C("feed_url").Eq(postsTable.C("feed_url")))).
		Columns(
			postsTable.C("ROWID"),
			postsTable.C("feed_url"),
			postsTable.C("raw_entry"),
			peopleTable.C("name"),
			peopleTable.C("display_name"),
			peopleTable.C("email"),
		).
		OrderBy(true, postsTable.C("created_at"))

	if args.Offset > 0 && args.Offset < 225 {
		qb = qb.Offset(args.Offset)
	}

	if args.Limit > 0 && args.Limit <= 25 {
		qb = qb.Limit(args.Limit)
	} else {
		qb = qb.Limit(25)
	}

	if args.AfterID != 0 {
		qb = qb.Where(postsTable.C("ROWID").Gt(args.AfterID))
	}

	q, vars, err := qb.ToSql()
	if err != nil {
		return nil, err
	}

	rows, err := db.Query(q, vars...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var posts []UIStatus
	for rows.Next() {
		var id int
		var feedURL, rawEntry string
		var name, displayName, email sql.NullString
		if err := rows.Scan(&id, &feedURL, &rawEntry, &name, &displayName, &email); err != nil {
			return nil, err
		}

		var entry AtomEntry
		if err := json.Unmarshal([]byte(rawEntry), &entry); err != nil {
			return nil, err
		}

		post := UIStatus{ID: id}

		if name.Valid {
			post.AuthorAcct = email.String
			post.AuthorName = name.String
		}

		if t, err := time.Parse(time.RFC3339, entry.Published); err == nil {
			post.Time = t
		}

		if entry.Content != nil {
			post.ContentHTML = entry.Content.HTML()
			post.ContentText = entry.Content.Text()
		}

		posts = append(posts, post)
	}

	return posts, nil
}
<commit_msg>add ROWID to people table object for query builder<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"time"

	"github.com/umisama/go-sqlbuilder"
)

var (
	peopleTable = sqlbuilder.NewTable(
		"people",
		&sqlbuilder.TableOption{Unique: [][]string{{"hub", "topic"}}},
		sqlbuilder.IntColumn("ROWID", nil),
		sqlbuilder.StringColumn("feed_url", &sqlbuilder.ColumnOption{NotNull: true, PrimaryKey: true}),
		sqlbuilder.DateColumn("first_seen", &sqlbuilder.ColumnOption{NotNull: true}),
		sqlbuilder.StringColumn("name", nil),
		sqlbuilder.StringColumn("display_name", nil),
		sqlbuilder.StringColumn("email", nil),
		sqlbuilder.StringColumn("summary", nil),
		sqlbuilder.StringColumn("note", nil),
	)

	postsTable = sqlbuilder.NewTable(
		"posts",
		&sqlbuilder.TableOption{Unique: [][]string{{"feed_url", "id"}}},
		sqlbuilder.IntColumn("ROWID", nil),
		sqlbuilder.StringColumn("feed_url", &sqlbuilder.ColumnOption{NotNull: true}),
		sqlbuilder.StringColumn("id", &sqlbuilder.ColumnOption{NotNull: true}),
		sqlbuilder.StringColumn("created_at", &sqlbuilder.ColumnOption{NotNull: true}),
		sqlbuilder.StringColumn("raw_entry", &sqlbuilder.ColumnOption{NotNull: true}),
	)
)

func savePerson(db *sql.DB, feedURL string, author *AtomAuthor) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	var name, displayName, email, summary, note string
	if err := tx.QueryRow("select name, display_name, email, summary, note from people where feed_url = $1", feedURL).Scan(&name, &displayName, &email, &summary, &note); err != nil {
		if err != sql.ErrNoRows {
			return err
		}

		if _, err := tx.Exec("insert into people (feed_url, first_seen, name, display_name, email, summary, note) values ($1, $2, $3, $4, $5, $6, $7)", feedURL, time.Now(), author.Name, author.DisplayName, author.Email, author.Summary, author.Note); err != nil {
			return err
		}
	} else {
		if name == author.Name && displayName == author.DisplayName && email == author.Email && summary == author.Summary && note == author.Note {
			return nil
		}

		if _, err := tx.Exec("update people set name = $1, display_name = $2, email = $3, summary = $4, note = $5 where feed_url = $6", author.Name, author.DisplayName, author.Email, author.Summary, author.Note, feedURL); err != nil {
			return err
		}
	}

	return tx.Commit()
}

func saveEntry(db *sql.DB, feedURL string, entry *AtomEntry) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	var exists int
	if err := tx.QueryRow("select count(1) from posts where feed_url = $1 and id = $2", feedURL, entry.ID).Scan(&exists); err != nil {
		return err
	}

	if exists > 0 {
		return nil
	}

	d, err := json.Marshal(entry)
	if err != nil {
		return err
	}

	if _, err := tx.Exec("insert into posts (feed_url, id, created_at, raw_entry) values ($1, $2, $3, $4)", feedURL, entry.ID, entry.Published, string(d)); err != nil {
		return err
	}

	return tx.Commit()
}

type getPublicTimelineArgs struct {
	AfterID int `qstring:"after_id"`
	Offset  int `qstring:"offset,omitempty"`
	Limit   int `qstring:"limit,omitempty"`
}

func getPublicTimeline(db *sql.DB, args getPublicTimelineArgs) ([]UIStatus, error) {
	qb := sqlbuilder.
		Select(postsTable.LeftOuterJoin(peopleTable, peopleTable.C("feed_url").Eq(postsTable.C("feed_url")))).
		Columns(
			postsTable.C("ROWID"),
			postsTable.C("feed_url"),
			postsTable.C("raw_entry"),
			peopleTable.C("name"),
			peopleTable.C("display_name"),
			peopleTable.C("email"),
		).
		OrderBy(true, postsTable.C("created_at"))

	if args.Offset > 0 && args.Offset < 225 {
		qb = qb.Offset(args.Offset)
	}

	if args.Limit > 0 && args.Limit <= 25 {
		qb = qb.Limit(args.Limit)
	} else {
		qb = qb.Limit(25)
	}

	if args.AfterID != 0 {
		qb = qb.Where(postsTable.C("ROWID").Gt(args.AfterID))
	}

	q, vars, err := qb.ToSql()
	if err != nil {
		return nil, err
	}

	rows, err := db.Query(q, vars...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var posts []UIStatus
	for rows.Next() {
		var id int
		var feedURL, rawEntry string
		var name, displayName, email sql.NullString
		if err := rows.Scan(&id, &feedURL, &rawEntry, &name, &displayName, &email); err != nil {
			return nil, err
		}

		var entry AtomEntry
		if err := json.Unmarshal([]byte(rawEntry), &entry); err != nil {
			return nil, err
		}

		post := UIStatus{ID: id}

		if name.Valid {
			post.AuthorAcct = email.String
			post.AuthorName = name.String
		}

		if t, err := time.Parse(time.RFC3339, entry.Published); err == nil {
			post.Time = t
		}

		if entry.Content != nil {
			post.ContentHTML = entry.Content.HTML()
			post.ContentText = entry.Content.Text()
		}

		posts = append(posts, post)
	}

	return posts, nil
}
<|endoftext|>


<commit_before>// Copyright ©2016 The go-lsst Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"log"
	"time"

	"github.com/boltdb/bolt"
)

var (
	bucketMon = []byte("data-mon")
)

func openDB(name string) *bolt.DB {
	db, err := bolt.Open(name, 0644, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		log.Fatal(err)
	}

	db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(bucketMon)
		return err
	})

	return db
}

func (srv *server) monitor() {
	hfreq := time.NewTicker(1 * time.Second)
	defer hfreq.Stop()
	lfreq := time.NewTicker(5 * time.Minute)
	defer lfreq.Stop()

	srv.publishData()
	srv.storeMonData()
	for {
		select {
		case <-hfreq.C:
			srv.publishData()
		case <-lfreq.C:
			srv.publishData()
			srv.storeMonData()
		}
	}
}

func (srv *server) storeMonData() {
	db := openDB("mon.db")
	defer db.Close()
	db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists(bucketMon)
		if err != nil {
			return err
		}
		for _, motor := range srv.motors() {
			var buf [monDataLen]byte
			mon := motor.histos.rows[len(motor.histos.rows)-1]
			mon.write(buf[:])
			err = bucket.Put(buf[:8], buf[8:])
			if err != nil {
				return err
			}
		}
		return err
	})
}
<commit_msg>db: reduce monitor frequency (1Hz -> 1/5Hz)<commit_after>// Copyright ©2016 The go-lsst Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"log"
	"time"

	"github.com/boltdb/bolt"
)

var (
	bucketMon = []byte("data-mon")
)

func openDB(name string) *bolt.DB {
	db, err := bolt.Open(name, 0644, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		log.Fatal(err)
	}

	db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(bucketMon)
		return err
	})

	return db
}

func (srv *server) monitor() {
	hfreq := time.NewTicker(5 * time.Second)
	defer hfreq.Stop()
	lfreq := time.NewTicker(5 * time.Minute)
	defer lfreq.Stop()

	srv.publishData()
	srv.storeMonData()
	for {
		select {
		case <-hfreq.C:
			srv.publishData()
		case <-lfreq.C:
			srv.publishData()
			srv.storeMonData()
		}
	}
}

func (srv *server) storeMonData() {
	db := openDB("mon.db")
	defer db.Close()
	db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists(bucketMon)
		if err != nil {
			return err
		}
		for _, motor := range srv.motors() {
			var buf [monDataLen]byte
			mon := motor.histos.rows[len(motor.histos.rows)-1]
			mon.write(buf[:])
			err = bucket.Put(buf[:8], buf[8:])
			if err != nil {
				return err
			}
		}
		return err
	})
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"

	_ "github.com/go-sql-driver/mysql"
)

var db *sql.DB

var insTopicStmt, upLastIdStmt, upModTimeStmt, getTopicListStmt *sql.Stmt

func initStmt() (err error) {
	insTopicStmt, err = db.Prepare(`
		INSERT INTO topics (id, title, author, time, modified)
		VALUES             (?,  ?,     ?,      ?,    ?       )`)
	if err != nil {
		return err
	}

	upLastIdStmt, err = db.Prepare(`UPDATE lastID SET id = ? WHERE id = ?`)
	if err != nil {
		return err
	}

	upModTimeStmt, err = db.Prepare(`UPDATE topics SET modified = ? WHERE id = ?`)
	if err != nil {
		return err
	}

	getTopicListStmt, err = db.Prepare(`SELECT id, title, author, time FROM topics ORDER BY modified DESC`)
	if err != nil {
		return err
	}

	return nil
}

func createTopic(name string) error {
	_, err := db.Exec(`
		CREATE TABLE IF NOT EXISTS t_` + name + `(
			id    char(16)      NOT NULL,
			user  varchar(16)   NOT NULL,
			value varchar(2048) NOT NULL,
			time  datetime      NOT NULL)`)
	if err != nil {
		return err
	}
	return nil
}

func initTable() error {
	err := createTopic("hall")
	if err != nil {
		return err
	}

	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS lastID (id char(8))`)
	if err != nil {
		return err
	}

	var l int
	err = db.QueryRow(`SELECT COUNT(*) FROM lastID`).Scan(&l)
	if err != nil {
		return err
	}
	if l == 0 {
		_, err = db.Exec(`INSERT INTO lastID VALUES ('00000000')`)
		if err != nil {
			return err
		}
	}

	_, err = db.Exec(`
		CREATE TABLE IF NOT EXISTS topics (
			id       char(8)     NOT NULL,
			title    varchar(50) NOT NULL,
			author   varchar(16) NOT NULL,
			time     datetime    NOT NULL,
			modified datetime    NOT NULL)`)
	if err != nil {
		return err
	}

	return nil
}

func initDB() error {
	var err error
	db, err = sql.Open("mysql", dsn)
	if err != nil {
		return err
	}

	err = initTable()
	if err != nil {
		return err
	}

	err = initStmt()
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>修改单条消息长度限制<commit_after>package main

import (
	"database/sql"

	_ "github.com/go-sql-driver/mysql"
)

var db *sql.DB

var insTopicStmt, upLastIdStmt, upModTimeStmt, getTopicListStmt *sql.Stmt

func initStmt() (err error) {
	insTopicStmt, err = db.Prepare(`
		INSERT INTO topics (id, title, author, time, modified)
		VALUES             (?,  ?,     ?,      ?,    ?       )`)
	if err != nil {
		return err
	}

	upLastIdStmt, err = db.Prepare(`UPDATE lastID SET id = ? WHERE id = ?`)
	if err != nil {
		return err
	}

	upModTimeStmt, err = db.Prepare(`UPDATE topics SET modified = ? WHERE id = ?`)
	if err != nil {
		return err
	}

	getTopicListStmt, err = db.Prepare(`SELECT id, title, author, time FROM topics ORDER BY modified DESC`)
	if err != nil {
		return err
	}

	return nil
}

func createTopic(name string) error {
	_, err := db.Exec(`
		CREATE TABLE IF NOT EXISTS t_` + name + `(
			id    char(16)      NOT NULL,
			user  varchar(16)   NOT NULL,
			value varchar(4096) NOT NULL,
			time  datetime      NOT NULL)`)
	if err != nil {
		return err
	}
	return nil
}

func initTable() error {
	err := createTopic("hall")
	if err != nil {
		return err
	}

	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS lastID (id char(8))`)
	if err != nil {
		return err
	}

	var l int
	err = db.QueryRow(`SELECT COUNT(*) FROM lastID`).Scan(&l)
	if err != nil {
		return err
	}
	if l == 0 {
		_, err = db.Exec(`INSERT INTO lastID VALUES ('00000000')`)
		if err != nil {
			return err
		}
	}

	_, err = db.Exec(`
		CREATE TABLE IF NOT EXISTS topics (
			id       char(8)     NOT NULL,
			title    varchar(50) NOT NULL,
			author   varchar(16) NOT NULL,
			time     datetime    NOT NULL,
			modified datetime    NOT NULL)`)
	if err != nil {
		return err
	}

	return nil
}

func initDB() error {
	var err error
	db, err = sql.Open("mysql", dsn)
	if err != nil {
		return err
	}

	err = initTable()
	if err != nil {
		return err
	}

	err = initStmt()
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package dry

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"os"
)

type CountingReader struct {
	Reader    io.Reader
	BytesRead int
}

func (r *CountingReader) Read(p []byte) (n int, err error) {
	n, err = r.Reader.Read(p)
	r.BytesRead += n
	return n, err
}

type CountingWriter struct {
	Writer       io.Writer
	BytesWritten int
}

func (r *CountingWriter) Write(p []byte) (n int, err error) {
	n, err = r.Writer.Write(p)
	r.BytesWritten += n
	return n, err
}

type CountingReadWriter struct {
	ReadWriter   io.ReadWriter
	BytesRead    int
	BytesWritten int
}

func (rw *CountingReadWriter) Read(p []byte) (n int, err error) {
	n, err = rw.ReadWriter.Read(p)
	rw.BytesRead += n
	return n, err
}

func (rw *CountingReadWriter) Write(p []byte) (n int, err error) {
	n, err = rw.ReadWriter.Write(p)
	rw.BytesWritten += n
	return n, err
}

// ReadBinary wraps binary.Read with a CountingReader and returns
// the acutal bytes read by it.
func ReadBinary(r io.Reader, order binary.ByteOrder, data interface{}) (n int, err error) {
	countingReader := CountingReader{Reader: r}
	err = binary.Read(&countingReader, order, data)
	return countingReader.BytesRead, err
}

// WriteFull calls writer.Write until all of data is written,
// or an is error returned.
func WriteFull(data []byte, writer io.Writer) (n int, err error) {
	dataSize := len(data)
	for n = 0; n < dataSize; {
		m, err := writer.Write(data[n:])
		n += m
		if err != nil {
			return n, err
		}
	}
	return dataSize, nil
}

// ReadLine reads unbuffered until a newline '\n' byte and removes
// an optional carriege return '\r' at the end of the line.
// In case of an error, the string up to the error is returned.
func ReadLine(reader io.Reader) (line string, err error) {
	buffer := bytes.NewBuffer(make([]byte, 0, 4096))
	p := make([]byte, 1)
	for {
		var n int
		n, err = reader.Read(p)
		if err != nil || p[0] == '\n' {
			break
		}
		if n > 0 {
			buffer.WriteByte(p[0])
		}
	}
	data := buffer.Bytes()
	if len(data) > 0 && data[len(data)-1] == '\r' {
		data = data[:len(data)-1]
	}
	return string(data), err
}

// WaitForStdin blocks until input is available from os.Stdin.
// The first byte from os.Stdin is returned as result.
// If there are println arguments, then fmt.Println will be
// called with those before reading from os.Stdin.
func WaitForStdin(println ...interface{}) byte {
	if len(println) > 0 {
		fmt.Println(println...)
	}
	buffer := make([]byte, 1)
	os.Stdin.Read(buffer)
	return buffer[0]
}
<commit_msg>added ReaderFunc and WriterFunc<commit_after>package dry

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"os"
)

type CountingReader struct {
	Reader    io.Reader
	BytesRead int
}

func (r *CountingReader) Read(p []byte) (n int, err error) {
	n, err = r.Reader.Read(p)
	r.BytesRead += n
	return n, err
}

type CountingWriter struct {
	Writer       io.Writer
	BytesWritten int
}

func (r *CountingWriter) Write(p []byte) (n int, err error) {
	n, err = r.Writer.Write(p)
	r.BytesWritten += n
	return n, err
}

type CountingReadWriter struct {
	ReadWriter   io.ReadWriter
	BytesRead    int
	BytesWritten int
}

func (rw *CountingReadWriter) Read(p []byte) (n int, err error) {
	n, err = rw.ReadWriter.Read(p)
	rw.BytesRead += n
	return n, err
}

func (rw *CountingReadWriter) Write(p []byte) (n int, err error) {
	n, err = rw.ReadWriter.Write(p)
	rw.BytesWritten += n
	return n, err
}

// ReadBinary wraps binary.Read with a CountingReader and returns
// the acutal bytes read by it.
func ReadBinary(r io.Reader, order binary.ByteOrder, data interface{}) (n int, err error) {
	countingReader := CountingReader{Reader: r}
	err = binary.Read(&countingReader, order, data)
	return countingReader.BytesRead, err
}

// WriteFull calls writer.Write until all of data is written,
// or an is error returned.
func WriteFull(data []byte, writer io.Writer) (n int, err error) {
	dataSize := len(data)
	for n = 0; n < dataSize; {
		m, err := writer.Write(data[n:])
		n += m
		if err != nil {
			return n, err
		}
	}
	return dataSize, nil
}

// ReadLine reads unbuffered until a newline '\n' byte and removes
// an optional carriege return '\r' at the end of the line.
// In case of an error, the string up to the error is returned.
func ReadLine(reader io.Reader) (line string, err error) {
	buffer := bytes.NewBuffer(make([]byte, 0, 4096))
	p := make([]byte, 1)
	for {
		var n int
		n, err = reader.Read(p)
		if err != nil || p[0] == '\n' {
			break
		}
		if n > 0 {
			buffer.WriteByte(p[0])
		}
	}
	data := buffer.Bytes()
	if len(data) > 0 && data[len(data)-1] == '\r' {
		data = data[:len(data)-1]
	}
	return string(data), err
}

// WaitForStdin blocks until input is available from os.Stdin.
// The first byte from os.Stdin is returned as result.
// If there are println arguments, then fmt.Println will be
// called with those before reading from os.Stdin.
func WaitForStdin(println ...interface{}) byte {
	if len(println) > 0 {
		fmt.Println(println...)
	}
	buffer := make([]byte, 1)
	os.Stdin.Read(buffer)
	return buffer[0]
}

// ReaderFunc implements io.Reader as function type with a Read method.
type ReaderFunc func(p []byte) (int, error)

func (f ReaderFunc) Read(p []byte) (int, error) {
	return f(p)
}

// WriterFunc implements io.Writer as function type with a Write method.
type WriterFunc func(p []byte) (int, error)

func (f WriterFunc) Write(p []byte) (int, error) {
	return f(p)
}
<|endoftext|>


<commit_before>// +build js

// TODO: adapt this from ansi.go for js

package main

import (
	"bytes"
	"errors"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/gopherjs/gopherjs/js"
)

var Version string = "v0.4"

func main() {
	tui := &termui{}
	err := tui.Init()
	if err != nil {
		log.Fatal("boohu: %v\n", err)
	}
	defer tui.Close()

	tui.PostInit()

	tui.DrawWelcome()
	g := &game{}
	//load, err := g.Load()
	//if !load {
	//g.InitLevel()
	//} else if err != nil {
	//g.InitLevel()
	//g.Print("Error loading saved game… starting new game.")
	//}
	g.InitLevel()
	g.ui = tui
	g.EventLoop()
}

// io compatibility functions

func (g *game) DataDir() (string, error) {
	return "", nil
}

func (g *game) Save() error {
	return nil
}

func (g *game) RemoveSaveFile() error {
	return nil
}

func (g *game) Load() (bool, error) {
	return false, nil
}

func (g *game) WriteDump() error {
	return nil
}

// End of io compatibility functions

type UICell struct {
	fg uicolor
	bg uicolor
	r  rune
}

func (c uicolor) String() string {
	color := "#002b36"
	switch c {
	case 0:
		color = "#073642"
	case 1:
		color = "#dc322f"
	case 2:
		color = "#859900"
	case 3:
		color = "#b58900"
	case 4:
		color = "#268bd2"
	case 5:
		color = "#d33682"
	case 6:
		color = "#2aa198"
	case 7:
		color = "#eee8d5"
	case 8:
		color = "#002b36"
	case 9:
		color = "#cb4b16"
	case 10:
		color = "#586e75"
	case 11:
		color = "#657b83"
	case 12:
		color = "#839496"
	case 13:
		color = "#6c71c4"
	case 14:
		color = "#93a1a1"
	case 15:
		color = "#fdf6e3"
	}
	return color
}

type termui struct {
	cells      []UICell
	backBuffer []UICell
	cursor     position
	display    *js.Object
}

const (
	UIWidth = 103
	//UIWidth = 10
	UIHeight = 27
	//UIHeight = 5
)

func (ui *termui) GetIndex(x, y int) int {
	return y*UIWidth + x
}

func (ui *termui) GetPos(i int) (int, int) {
	return i - (i/UIWidth)*UIWidth, i / UIWidth
}

func (ui *termui) ResetCells() {
	for i := 0; i < len(ui.cells); i++ {
		ui.cells[i].r = ' '
		ui.cells[i].bg = ColorBg
	}
}

var ch chan string
var wants chan bool

func init() {
	ch = make(chan string)
	wants = make(chan bool)
}

func (ui *termui) Init() error {
	ui.cells = make([]UICell, UIWidth*UIHeight)
	js.Global.Get("document").Call("addEventListener", "keypress", func(e *js.Object) {
		select {
		case <-wants:
			s := e.Get("key").String()
			ch <- s
		default:
		}
	})
	ui.ResetCells()
	ui.backBuffer = make([]UICell, UIWidth*UIHeight)
	//ui.display = js.Global.Get("ROT").Call("Display", map[string]int{"width": UIWidth, "height": UIHeight})
	//ui.display = js.Global.Get("ROT").Call("Display")
	//container := ui.display.Call("getContainer")
	//js.Global.Get("document").Get("body").Call("appendChild", container)
	//ui.display.Call("draw", 5, 4, "@")
	// TODO: init pre?
	return nil
}

func (ui *termui) Close() {
	// TODO
}

func (ui *termui) PostInit() {
	SolarizedPalette()
	ui.HideCursor()
}

func (ui *termui) Clear() {
	ui.ResetCells()
}

func (ui *termui) Flush() {
	buf := &bytes.Buffer{}
	canvas := js.Global.Get("document").Call("getElementById", "gamecanvas")
	ctx := canvas.Call("getContext", "2d")
	ctx.Set("font", "12px monospace")
	mesure := ctx.Call("measureText", "W")
	width := mesure.Get("width").Int() + 1
	//height := mesure.Get("height")
	//ctx.Set("fillStyle", "#002b36")
	//ctx.Call("fillRect", 0, 0, UIWidth*10, UIHeight*10)
	for i := 0; i < len(ui.cells); i++ {
		if ui.cells[i] == ui.backBuffer[i] {
			continue
		}
		cell := ui.cells[i]
		// TODO: print newlines and html markup
		if i%UIWidth == 0 {
			fmt.Fprintf(buf, "\n")
		}
		if cell.r == ' ' {
			cell.r = ' '
		}
		x, y := ui.GetPos(i)
		ctx.Set("fillStyle", cell.bg.String())
		ctx.Call("fillRect", width*x, 17*y, width, 17)
		ctx.Set("fillStyle", cell.fg.String())
		ctx.Call("fillText", string(cell.r), width*x, 17*y+12)
		//ui.display.Call("draw", x, y, string(cell.r))
		ui.backBuffer[i] = cell
	}
	//ui.MoveTo(ui.cursor.X, ui.cursor.Y)
	//if ui.cursor.X >= 0 && ui.cursor.Y >= 0 {
	//fmt.Fprintf(buf, "\x1b[?25h")
	//} else {
	//fmt.Fprintf(buf, "\x1b[?25l")
	//}
	//ctx.Call("fillText", buf.String(), 10, 10)
	//js.Global.Get("document").Call("getElementById", "game").Set("innerHTML", buf.String())
}

func (ui *termui) HideCursor() {
	ui.cursor = position{-1, -1}
}

func (ui *termui) SetCursor(pos position) {
	ui.cursor = pos
}

func (ui *termui) SetCell(x, y int, r rune, fg, bg uicolor) {
	i := ui.GetIndex(x, y)
	if i >= len(ui.cells) {
		return
	}
	ui.cells[ui.GetIndex(x, y)] = UICell{fg: fg, bg: bg, r: r}
}

func (ui *termui) ReadChar() rune {
	wants <- true
	s := <-ch
	bs := strings.NewReader(s)
	r, _, _ := bs.ReadRune()
	return r
}

func (ui *termui) ExploreStep(g *game) bool {
	time.Sleep(10 * time.Millisecond)
	ui.DrawDungeonView(g, false)
	return false
}

func (ui *termui) WaitForContinue(g *game) {
loop:
	for {
		r := ui.ReadChar()
		switch r {
		case '\x1b', ' ':
			break loop
		}
	}
}

func (ui *termui) PromptConfirmation(g *game) bool {
	for {
		r := ui.ReadChar()
		switch r {
		case 'Y', 'y':
			return true
		default:
			return false
		}
	}
}

func (ui *termui) PressAnyKey() error {
	for {
		ui.ReadChar()
		return nil
	}
}

func (ui *termui) PlayerTurnEvent(g *game, ev event) (err error, again, quit bool) {
	again = true
	r := ui.ReadChar()
	switch r {
	case 'S', 'Q', '#':
		err = errors.New("Command not available (still) for the js version.")
		return nil, true, false
	case 'W':
		ui.EnterWizard(g)
		return nil, true, false
		//case 'Q':
		//if ui.Quit(g) {
		//return nil, false, true
		//}
		//return nil, true, false
	}
	err, again, quit = ui.HandleCharacter(g, ev, r)
	if err != nil {
		again = true
	}
	return err, again, quit
}

func (ui *termui) Scroll(n int) (m int, quit bool) {
	r := ui.ReadChar()
	switch r {
	case '\x1b', ' ':
		quit = true
	case 'u':
		n -= 12
	case 'd':
		n += 12
	case 'j':
		n++
	case 'k':
		n--
	}
	return n, quit
}

func (ui *termui) TargetModeEvent(g *game, targ Targetter, pos position, data *examineData) bool {
	r := ui.ReadChar()
	if r == '\x1b' || r == ' ' {
		return true
	}
	return ui.CursorCharAction(g, targ, r, pos, data)
}

func (ui *termui) Select(g *game, ev event, l int) (index int, alternate bool, err error) {
	for {
		r := ui.ReadChar()
		switch {
		case r == '\x1b' || r == ' ':
			return -1, false, errors.New("Ok, then.")
		case 97 <= r && int(r) < 97+l:
			return int(r - 97), false, nil
		case r == '?':
			return -1, true, nil
		case r == ' ':
			return -1, false, errors.New("Ok, then.")
		}
	}
}
<commit_msg>some little performance improvements<commit_after>// +build js

// TODO: adapt this from ansi.go for js

package main

import (
	"errors"
	"log"
	"strings"
	"time"

	"github.com/gopherjs/gopherjs/js"
)

var Version string = "v0.4"

func main() {
	tui := &termui{}
	err := tui.Init()
	if err != nil {
		log.Fatal("boohu: %v\n", err)
	}
	defer tui.Close()

	tui.PostInit()

	tui.DrawWelcome()
	g := &game{}
	//load, err := g.Load()
	//if !load {
	//g.InitLevel()
	//} else if err != nil {
	//g.InitLevel()
	//g.Print("Error loading saved game… starting new game.")
	//}
	g.InitLevel()
	g.ui = tui
	g.EventLoop()
}

// io compatibility functions

func (g *game) DataDir() (string, error) {
	return "", nil
}

func (g *game) Save() error {
	return nil
}

func (g *game) RemoveSaveFile() error {
	return nil
}

func (g *game) Load() (bool, error) {
	return false, nil
}

func (g *game) WriteDump() error {
	return nil
}

// End of io compatibility functions

type UICell struct {
	fg uicolor
	bg uicolor
	r  rune
}

func (c uicolor) String() string {
	color := "#002b36"
	switch c {
	case 0:
		color = "#073642"
	case 1:
		color = "#dc322f"
	case 2:
		color = "#859900"
	case 3:
		color = "#b58900"
	case 4:
		color = "#268bd2"
	case 5:
		color = "#d33682"
	case 6:
		color = "#2aa198"
	case 7:
		color = "#eee8d5"
	case 8:
		color = "#002b36"
	case 9:
		color = "#cb4b16"
	case 10:
		color = "#586e75"
	case 11:
		color = "#657b83"
	case 12:
		color = "#839496"
	case 13:
		color = "#6c71c4"
	case 14:
		color = "#93a1a1"
	case 15:
		color = "#fdf6e3"
	}
	return color
}

type termui struct {
	cells      []UICell
	backBuffer []UICell
	cursor     position
	display    *js.Object
	cache      map[UICell]*js.Object
	ctx        *js.Object
	width      int
}

const (
	UIWidth = 103
	//UIWidth = 10
	UIHeight = 27
	//UIHeight = 5
)

func (ui *termui) GetIndex(x, y int) int {
	return y*UIWidth + x
}

func (ui *termui) GetPos(i int) (int, int) {
	return i - (i/UIWidth)*UIWidth, i / UIWidth
}

func (ui *termui) ResetCells() {
	for i := 0; i < len(ui.cells); i++ {
		ui.cells[i].r = ' '
		ui.cells[i].bg = ColorBg
	}
}

var ch chan string
var wants chan bool

func init() {
	ch = make(chan string)
	wants = make(chan bool)
}

func (ui *termui) Init() error {
	ui.cells = make([]UICell, UIWidth*UIHeight)
	js.Global.Get("document").Call("addEventListener", "keypress", func(e *js.Object) {
		select {
		case <-wants:
			s := e.Get("key").String()
			ch <- s
		default:
		}
	})
	ui.ResetCells()
	ui.backBuffer = make([]UICell, UIWidth*UIHeight)
	canvas := js.Global.Get("document").Call("getElementById", "gamecanvas")
	ui.ctx = canvas.Call("getContext", "2d")
	ui.ctx.Set("font", "12px monospace")
	mesure := ui.ctx.Call("measureText", "W")
	ui.width = mesure.Get("width").Int() + 1
	ui.cache = make(map[UICell]*js.Object)
	return nil
}

func (ui *termui) Close() {
	// TODO
}

func (ui *termui) PostInit() {
	SolarizedPalette()
	ui.HideCursor()
}

func (ui *termui) Clear() {
	ui.ResetCells()
}

func (ui *termui) Draw(cell UICell, x, y int) {
	var canvas *js.Object
	if cv, ok := ui.cache[cell]; ok {
		canvas = cv
	} else {
		canvas = js.Global.Get("document").Call("createElement", "canvas")
		ctx := canvas.Call("getContext", "2d")
		canvas.Set("width", ui.width)
		canvas.Set("height", 17)
		ctx.Set("font", ui.ctx.Get("font"))
		ctx.Set("fillStyle", cell.bg.String())
		ctx.Call("fillRect", 0, 0, ui.width, 17)
		ctx.Set("fillStyle", cell.fg.String())
		ctx.Call("fillText", string(cell.r), 0, 12)
		ui.cache[cell] = canvas
	}
	ui.ctx.Call("drawImage", canvas, x*ui.width, 17*y)
}

func (ui *termui) Flush() {
	for i := 0; i < len(ui.cells); i++ {
		if ui.cells[i] == ui.backBuffer[i] {
			continue
		}
		cell := ui.cells[i]
		if cell.r == ' ' {
			cell.r = ' '
		}
		x, y := ui.GetPos(i)
		ui.Draw(cell, x, y)
		ui.backBuffer[i] = cell
	}
}

func (ui *termui) HideCursor() {
	ui.cursor = position{-1, -1}
}

func (ui *termui) SetCursor(pos position) {
	ui.cursor = pos
}

func (ui *termui) SetCell(x, y int, r rune, fg, bg uicolor) {
	i := ui.GetIndex(x, y)
	if i >= len(ui.cells) {
		return
	}
	ui.cells[ui.GetIndex(x, y)] = UICell{fg: fg, bg: bg, r: r}
}

func (ui *termui) ReadChar() rune {
	wants <- true
	s := <-ch
	bs := strings.NewReader(s)
	r, _, _ := bs.ReadRune()
	return r
}

func (ui *termui) ExploreStep(g *game) bool {
	time.Sleep(5 * time.Millisecond)
	ui.DrawDungeonView(g, false)
	return false
}

func (ui *termui) WaitForContinue(g *game) {
loop:
	for {
		r := ui.ReadChar()
		switch r {
		case '\x1b', ' ':
			break loop
		}
	}
}

func (ui *termui) PromptConfirmation(g *game) bool {
	for {
		r := ui.ReadChar()
		switch r {
		case 'Y', 'y':
			return true
		default:
			return false
		}
	}
}

func (ui *termui) PressAnyKey() error {
	for {
		ui.ReadChar()
		return nil
	}
}

func (ui *termui) PlayerTurnEvent(g *game, ev event) (err error, again, quit bool) {
	again = true
	r := ui.ReadChar()
	switch r {
	case 'S', 'Q', '#':
		err = errors.New("Command not available (still) for the js version.")
		return nil, true, false
	case 'W':
		ui.EnterWizard(g)
		return nil, true, false
		//case 'Q':
		//if ui.Quit(g) {
		//return nil, false, true
		//}
		//return nil, true, false
	}
	err, again, quit = ui.HandleCharacter(g, ev, r)
	if err != nil {
		again = true
	}
	return err, again, quit
}

func (ui *termui) Scroll(n int) (m int, quit bool) {
	r := ui.ReadChar()
	switch r {
	case '\x1b', ' ':
		quit = true
	case 'u':
		n -= 12
	case 'd':
		n += 12
	case 'j':
		n++
	case 'k':
		n--
	}
	return n, quit
}

func (ui *termui) TargetModeEvent(g *game, targ Targetter, pos position, data *examineData) bool {
	r := ui.ReadChar()
	if r == '\x1b' || r == ' ' {
		return true
	}
	return ui.CursorCharAction(g, targ, r, pos, data)
}

func (ui *termui) Select(g *game, ev event, l int) (index int, alternate bool, err error) {
	for {
		r := ui.ReadChar()
		switch {
		case r == '\x1b' || r == ' ':
			return -1, false, errors.New("Ok, then.")
		case 97 <= r && int(r) < 97+l:
			return int(r - 97), false, nil
		case r == '?':
			return -1, true, nil
		case r == ' ':
			return -1, false, errors.New("Ok, then.")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"strings"

	"github.com/dylanmei/winrmfs/winrmfs"
	"github.com/masterzen/winrm/winrm"
	"github.com/mitchellh/cli"
)

type lsCommand struct {
	shutdown <-chan struct{}
	ui       cli.Ui
}

func (c *lsCommand) Help() string {
	text := `
Usage: winrmfs ls [options] directory

  List files of a remote directory.

Options:

  -addr=127.0.0.1:5985    Host and port of the remote machine
  -user=""                Name of the user to authenticate as
  -pass=""                Password to authenticate with
`
	return strings.TrimSpace(text)
}

func (c *lsCommand) Synopsis() string {
	return "List files of a remote directory"
}

func (c *lsCommand) Run(args []string) int {
	var user string
	var pass string

	flags := flag.NewFlagSet("ls", flag.ContinueOnError)
	flags.Usage = func() { c.ui.Output(c.Help()) }
	flags.StringVar(&user, "user", "", "auth name")
	flags.StringVar(&pass, "pass", "", "auth password")
	addr := addrFlag(flags)

	if err := flags.Parse(args); err != nil {
		return 1
	}

	args = flags.Args()
	if len(args) < 1 {
		c.ui.Error("A directory is required.\n")
		c.ui.Error(c.Help())
		return 1
	}
	if len(args) > 1 {
		c.ui.Error("Too many arguments. Only a directory is required.\n")
		c.ui.Error(c.Help())
		return 1
	}

	dir := args[0]
	endpoint, err := parseEndpoint(*addr)
	if err != nil {
		c.ui.Error(err.Error())
		return 1
	}

	client := winrm.NewClient(endpoint, user, pass)
	fs := winrmfs.New(client)

	list, err := fs.List(dir)
	if err != nil {
		c.ui.Error(err.Error())
		return 1
	}

	for _, fi := range list {
		fmt.Printf("Mode: %s, ", fi.Mode)
		fmt.Printf("Name: %s, ", fi.Name)
		fmt.Printf("LastWriteTime: %s, ", fi.LastWriteTime)
		fmt.Printf("Length: %d\n", fi.Length)
	}

	return 0
}
<commit_msg>Use tabwriter to display ls contents<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
	"text/tabwriter"

	"github.com/dylanmei/winrmfs/winrmfs"
	"github.com/masterzen/winrm/winrm"
	"github.com/mitchellh/cli"
)

type lsCommand struct {
	shutdown <-chan struct{}
	ui       cli.Ui
}

func (c *lsCommand) Help() string {
	text := `
Usage: winrmfs ls [options] directory

  List files of a remote directory.

Options:

  -addr=127.0.0.1:5985    Host and port of the remote machine
  -user=""                Name of the user to authenticate as
  -pass=""                Password to authenticate with
`
	return strings.TrimSpace(text)
}

func (c *lsCommand) Synopsis() string {
	return "List files of a remote directory"
}

func (c *lsCommand) Run(args []string) int {
	var user string
	var pass string

	flags := flag.NewFlagSet("ls", flag.ContinueOnError)
	flags.Usage = func() { c.ui.Output(c.Help()) }
	flags.StringVar(&user, "user", "", "auth name")
	flags.StringVar(&pass, "pass", "", "auth password")
	addr := addrFlag(flags)

	if err := flags.Parse(args); err != nil {
		return 1
	}

	args = flags.Args()
	if len(args) < 1 {
		c.ui.Error("A directory is required.\n")
		c.ui.Error(c.Help())
		return 1
	}
	if len(args) > 1 {
		c.ui.Error("Too many arguments. Only a directory is required.\n")
		c.ui.Error(c.Help())
		return 1
	}

	dir := args[0]
	endpoint, err := parseEndpoint(*addr)
	if err != nil {
		c.ui.Error(err.Error())
		return 1
	}

	client := winrm.NewClient(endpoint, user, pass)
	fs := winrmfs.New(client)

	list, err := fs.List(dir)
	if err != nil {
		c.ui.Error(err.Error())
		return 1
	}

	tw := new(tabwriter.Writer)
	tw.Init(os.Stdout, 0, 8, 0, '\t', 0)
	fmt.Fprintf(tw, "\tName\tLastWriteTime\tLength\n")
	fmt.Fprintf(tw, "\t----\t-------------\t------\n")
	for _, fi := range list {
		fmt.Fprintf(tw, "%s\t%s\t%s\t%d\n", fi.Mode, fi.Name, fi.LastWriteTime, fi.Length)
	}
	tw.Flush()

	return 0
}
<|endoftext|>


<commit_before>// vim: tabstop=4 softtabstop=4 shiftwidth=4 noexpandtab tw=72
package main

import (
	"bufio"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/atotto/clipboard"
	"os"
	"strconv"
	"strings"
)

type existsFunc func(string) bool

func osStatExists(file string) bool {
	_, err := os.Stat(file)
	return err == nil
}

var ignoreList = [...]string{"/", ".", "./", "..", "../"}

//var rootListing, _ = ioutil.ReadDir("/")
//var pwdListing, _ = ioutil.ReadDir(".")

func ignored(file string) bool {
	for _, val := range ignoreList {
		if file == val {
			return true
		}
	}
	return false
}

func longestFileEndIndex(line []rune, exists existsFunc) int {
	// Possible optimisations:
	// 1. this should start at the end - the longest substring first
	// 2. it could be a good strategy to list files and try to
	//    find a common prefix - if not just stop right there and then
	//    from / if it starts with / and if not from `pwd`
	//    do file listing from / and `pwd` only once
	//    need to consider relative dirs though which is annoying

	maxIndex := 0
	for i, _ := range line {
		slice := line[0 : i+1]
		file := string(slice)
		if !ignored(file) {
			if exists(file) {
				// TODO if this is not a dir, stop here
				maxIndex = i
			}
		}
	}
	return maxIndex
}

func longestFileInLine(line string, exists existsFunc) (firstCharIndex int, lastCharIndex int) {
	for searchStartIndex, _ := range line {
		searchSpace := []rune(line[searchStartIndex:len(line)])
		lastCharIndexInSlice := longestFileEndIndex(searchSpace, exists)
		lastCharIndexInLine := lastCharIndexInSlice + searchStartIndex
		if lastCharIndexInSlice > 0 && lastCharIndexInLine > lastCharIndex {
			lastCharIndex = lastCharIndexInLine
			firstCharIndex = searchStartIndex
		}
	}

	return firstCharIndex, lastCharIndex
}

func askUser() (requestedNumbers []string, err error) {
	fmt.Println()
	fmt.Print("to clipboard: ")
	ttyFile, err := os.Open("/dev/tty")
	if err != nil {
		return nil, err
	}
	defer ttyFile.Close()
	ttyReader := bufio.NewReader(ttyFile)
	s, err := ttyReader.ReadString('\n')
	if err != nil {
		return nil, err
	}
	return strings.Fields(s), nil
}

type PrintFunction func(string, string, int, int)

type Processor interface {
	processFile(file string) error
	processEnd() error
}

type NumbersGiven struct {
	clip      *bytes.Buffer
	fileCount *int
	numbers   []string
}

type AskForNumbers struct {
	clip  *bytes.Buffer
	files []string
}

func printShortFormat(fileCount *int) PrintFunction {
	return func(file string, line string, firstCharIndex int, lastCharIndex int) {
		fmt.Println(strconv.Itoa(*fileCount), file)
	}
}

func printLongFormat(fileCount *int) PrintFunction {
	return func(file string, line string, firstCharIndex int, lastCharIndex int) {
		fmt.Print("{")
		fmt.Print(strconv.Itoa(*fileCount))
		fmt.Print("} ")
		fmt.Print(line[:firstCharIndex])
		fmt.Print("{")
		fmt.Print(file)
		fmt.Print("}")
		fmt.Print(line[lastCharIndex+1:])
	}
}

func (ng *NumbersGiven) processEnd() error {
	writeToClipboard(ng.clip)
	return nil
}

func (ng *NumbersGiven) processFile(file string) error {
	// collect any file position arguments to copy to the
	// clipboard later
	for _, v := range ng.numbers {
		n, err := strconv.Atoi(v)
		if err != nil {
			fmt.Fprintf(os.Stderr, "nw: %s is not a number\n", v)
			return err
		}
		if n == *ng.fileCount {
			ng.clip.WriteString(file)
			ng.clip.WriteString(" ")
		}
	}
	return nil
}

func (afn *AskForNumbers) processEnd() error {
	if len(afn.files) == 0 {
		fmt.Println("nw: no files names found, NUMBERWANG!")
		return nil
	}
	requestedNumbers, err := askUser()
	if err != nil {
		fmt.Fprintf(os.Stderr, "nw: failed to read input: %s\n", err)
		return err
	}

	for _, n := range requestedNumbers {
		i, err := strconv.Atoi(n)
		if err != nil {
			fmt.Fprintf(os.Stderr, "nw: %s is not a number\n", n)
			return err
		}
		if i <= 0 || i > len(afn.files) {
			fmt.Fprintf(os.Stderr, "nw: %s is not a valid choice\n", n)
			return errors.New("invalid choice")
		}
		afn.clip.WriteString(afn.files[i-1])
		afn.clip.WriteString(" ")
	}

	writeToClipboard(afn.clip)
	return nil
}

func (afn *AskForNumbers) processFile(file string) error {
	afn.files = append(afn.files, file)
	return nil
}

func writeToClipboard(buffer *bytes.Buffer) {
	clipboardOutput := buffer.String()
	if clipboardOutput != "" {
		clipboard.WriteAll(clipboardOutput)
		fmt.Printf("nw: wrote \"%s\" to clipboard\n", clipboardOutput)
	}
}

func main() {
	var fileCount int
	var clip bytes.Buffer

	short := flag.Bool("s", false, "short format, only display file names")
	flag.Parse()

	extraArgs := flag.Args()

	var processor Processor
	if len(extraArgs) == 0 {
		processor = &AskForNumbers{clip: &clip}
	} else {
		processor = &NumbersGiven{
			clip:      &clip,
			fileCount: &fileCount,
			numbers:   extraArgs,
		}
	}
	var printer PrintFunction
	if *short {
		printer = printShortFormat(&fileCount)
	} else {
		printer = printLongFormat(&fileCount)
	}

	reader := bufio.NewReader(os.Stdin)

	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			break
		}

		firstCharIndex, lastCharIndex := longestFileInLine(line, osStatExists)

		if lastCharIndex > 0 {
			fileCount++
			file := line[firstCharIndex : lastCharIndex+1]

			printer(file, line, firstCharIndex, lastCharIndex)
			err := processor.processFile(file)
			if err != nil {
				os.Exit(1)
			}
		} else if !*short {
			fmt.Print(line)
		}
	}

	err := processor.processEnd()
	if err != nil {
		os.Exit(1)
	}
}
<commit_msg>fix typo<commit_after>// vim: tabstop=4 softtabstop=4 shiftwidth=4 noexpandtab tw=72
package main

import (
	"bufio"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/atotto/clipboard"
	"os"
	"strconv"
	"strings"
)

type existsFunc func(string) bool

func osStatExists(file string) bool {
	_, err := os.Stat(file)
	return err == nil
}

var ignoreList = [...]string{"/", ".", "./", "..", "../"}

//var rootListing, _ = ioutil.ReadDir("/")
//var pwdListing, _ = ioutil.ReadDir(".")

func ignored(file string) bool {
	for _, val := range ignoreList {
		if file == val {
			return true
		}
	}
	return false
}

func longestFileEndIndex(line []rune, exists existsFunc) int {
	// Possible optimisations:
	// 1. this should start at the end - the longest substring first
	// 2. it could be a good strategy to list files and try to
	//    find a common prefix - if not just stop right there and then
	//    from / if it starts with / and if not from `pwd`
	//    do file listing from / and `pwd` only once
	//    need to consider relative dirs though which is annoying

	maxIndex := 0
	for i, _ := range line {
		slice := line[0 : i+1]
		file := string(slice)
		if !ignored(file) {
			if exists(file) {
				// TODO if this is not a dir, stop here
				maxIndex = i
			}
		}
	}
	return maxIndex
}

func longestFileInLine(line string, exists existsFunc) (firstCharIndex int, lastCharIndex int) {
	for searchStartIndex, _ := range line {
		searchSpace := []rune(line[searchStartIndex:len(line)])
		lastCharIndexInSlice := longestFileEndIndex(searchSpace, exists)
		lastCharIndexInLine := lastCharIndexInSlice + searchStartIndex
		if lastCharIndexInSlice > 0 && lastCharIndexInLine > lastCharIndex {
			lastCharIndex = lastCharIndexInLine
			firstCharIndex = searchStartIndex
		}
	}

	return firstCharIndex, lastCharIndex
}

func askUser() (requestedNumbers []string, err error) {
	fmt.Println()
	fmt.Print("to clipboard: ")
	ttyFile, err := os.Open("/dev/tty")
	if err != nil {
		return nil, err
	}
	defer ttyFile.Close()
	ttyReader := bufio.NewReader(ttyFile)
	s, err := ttyReader.ReadString('\n')
	if err != nil {
		return nil, err
	}
	return strings.Fields(s), nil
}

type PrintFunction func(string, string, int, int)

type Processor interface {
	processFile(file string) error
	processEnd() error
}

type NumbersGiven struct {
	clip      *bytes.Buffer
	fileCount *int
	numbers   []string
}

type AskForNumbers struct {
	clip  *bytes.Buffer
	files []string
}

func printShortFormat(fileCount *int) PrintFunction {
	return func(file string, line string, firstCharIndex int, lastCharIndex int) {
		fmt.Println(strconv.Itoa(*fileCount), file)
	}
}

func printLongFormat(fileCount *int) PrintFunction {
	return func(file string, line string, firstCharIndex int, lastCharIndex int) {
		fmt.Print("{")
		fmt.Print(strconv.Itoa(*fileCount))
		fmt.Print("} ")
		fmt.Print(line[:firstCharIndex])
		fmt.Print("{")
		fmt.Print(file)
		fmt.Print("}")
		fmt.Print(line[lastCharIndex+1:])
	}
}

func (ng *NumbersGiven) processEnd() error {
	writeToClipboard(ng.clip)
	return nil
}

func (ng *NumbersGiven) processFile(file string) error {
	// collect any file position arguments to copy to the
	// clipboard later
	for _, v := range ng.numbers {
		n, err := strconv.Atoi(v)
		if err != nil {
			fmt.Fprintf(os.Stderr, "nw: %s is not a number\n", v)
			return err
		}
		if n == *ng.fileCount {
			ng.clip.WriteString(file)
			ng.clip.WriteString(" ")
		}
	}
	return nil
}

func (afn *AskForNumbers) processEnd() error {
	if len(afn.files) == 0 {
		fmt.Println("nw: no file names found, NUMBERWANG!")
		return nil
	}
	requestedNumbers, err := askUser()
	if err != nil {
		fmt.Fprintf(os.Stderr, "nw: failed to read input: %s\n", err)
		return err
	}

	for _, n := range requestedNumbers {
		i, err := strconv.Atoi(n)
		if err != nil {
			fmt.Fprintf(os.Stderr, "nw: %s is not a number\n", n)
			return err
		}
		if i <= 0 || i > len(afn.files) {
			fmt.Fprintf(os.Stderr, "nw: %s is not a valid choice\n", n)
			return errors.New("invalid choice")
		}
		afn.clip.WriteString(afn.files[i-1])
		afn.clip.WriteString(" ")
	}

	writeToClipboard(afn.clip)
	return nil
}

func (afn *AskForNumbers) processFile(file string) error {
	afn.files = append(afn.files, file)
	return nil
}

func writeToClipboard(buffer *bytes.Buffer) {
	clipboardOutput := buffer.String()
	if clipboardOutput != "" {
		clipboard.WriteAll(clipboardOutput)
		fmt.Printf("nw: wrote \"%s\" to clipboard\n", clipboardOutput)
	}
}

func main() {
	var fileCount int
	var clip bytes.Buffer

	short := flag.Bool("s", false, "short format, only display file names")
	flag.Parse()

	extraArgs := flag.Args()

	var processor Processor
	if len(extraArgs) == 0 {
		processor = &AskForNumbers{clip: &clip}
	} else {
		processor = &NumbersGiven{
			clip:      &clip,
			fileCount: &fileCount,
			numbers:   extraArgs,
		}
	}
	var printer PrintFunction
	if *short {
		printer = printShortFormat(&fileCount)
	} else {
		printer = printLongFormat(&fileCount)
	}

	reader := bufio.NewReader(os.Stdin)

	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			break
		}

		firstCharIndex, lastCharIndex := longestFileInLine(line, osStatExists)

		if lastCharIndex > 0 {
			fileCount++
			file := line[firstCharIndex : lastCharIndex+1]

			printer(file, line, firstCharIndex, lastCharIndex)
			err := processor.processFile(file)
			if err != nil {
				os.Exit(1)
			}
		} else if !*short {
			fmt.Print(line)
		}
	}

	err := processor.processEnd()
	if err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package cc1100

func (dev *Device) InitRF() error {
	err := dev.WriteEach([]byte{
		// Carrier sense: high if RSSI level is above threshold
		IOCFG2, 0x0E,
		IOCFG1, 0x00,
		// Assert when sync word has been sent/received
		IOCFG0, 0x06,

		// Sync word
		SYNC1, 0xFF,
		SYNC0, 0x00,

		// Packet length
		PKTLEN, 0xFF,

		// Always accept sync word
		// Do not append status
		// No address check
		PKTCTRL1, 0x00,

		// No whitening mode
		// Normal format
		// Disable CRC calculation and check
		// Fixed packet length mode
		PKTCTRL0, 0x00,

		// channel number
		CHANNR, 0x00,

		// Intermediate frequency
		// 0x06 * 26 MHz / 2^10 == 152 kHz
		FSCTRL1, 0x06,

		// Frequency offset
		FSCTRL0, 0x00,

		// 24-bit base frequency
		// 0x2340FC * 26 MHz / 2^16 == 916.6 MHz (916599975 Hz)
		FREQ2, 0x23,
		FREQ1, 0x40,
		FREQ0, 0xFC,

		// CHANBW_E = 2, CHANBW_M = 1, DRATE_E = 9
		// Channel BW = 26 MHz / (8 * (4 + CHANBW_M) * 2^CHANBW_E) == 162.5 kHz
		MDMCFG4, ((2 << MDMCFG4_CHANBW_E_SHIFT) |
			(1 << MDMCFG4_CHANBW_M_SHIFT) |
			(9 << MDMCFG4_DRATE_E_SHIFT)),

		// DRATE_M = 74 (0x4A)
		// Data rate = (256 + DRATE_M) * 2^DRATE_E * 26 MHz / 2^28 == 16365 Baud
		MDMCFG3, 0x4A,

		MDMCFG2, (MDMCFG2_DEM_DCFILT_ON |
			MDMCFG2_MOD_FORMAT_ASK_OOK |
			MDMCFG2_SYNC_MODE_30_32),

		// CHANSPC_E = 1
		MDMCFG1, (MDMCFG1_FEC_DIS |
			MDMCFG1_NUM_PREAMBLE_16 |
			(1 << MDMCFG1_CHANSPC_E_SHIFT)),

		// CHANSPC_M = 248 (0xF8)
		// cHannel spacing
		// (256 + CHANSPC_M) * 2^CHANSPC_E * 26 MHz / 2^18 == 99975 Hz
		MDMCFG0, 0xF8,

		MCSM2, MCSM2_RX_TIME_END_OF_PACKET,

		MCSM1, (MCSM1_CCA_MODE_RSSI_BELOW_UNLESS_RECEIVING |
			MCSM1_RXOFF_MODE_IDLE |
			MCSM1_TXOFF_MODE_IDLE),

		MCSM0, (MCSM0_FS_AUTOCAL_FROM_IDLE |
			MCSM0_MAGIC_3 |
			MCSM0_CLOSE_IN_RX_0DB),

		FOCCFG, (FOCCFG_FOC_PRE_K_3K |
			FOCCFG_FOC_POST_K_PRE_K_OVER_2 |
			FOCCFG_FOC_LIMIT_BW_OVER_2),

		BSCFG, (BSCFG_BS_PRE_K_2K |
			BSCFG_BS_PRE_KP_3KP |
			BSCFG_BS_POST_KI_PRE_KI_OVER_2 |
			BSCFG_BS_LIMIT_0),

		AGCCTRL2, (AGCCTRL2_MAX_DVGA_GAIN_ALL |
			AGCCTRL2_MAX_LNA_GAIN_0 |
			AGCCTRL2_MAGN_TARGET_33dB),

		AGCCTRL1, (AGCCTRL1_AGC_LNA_PRIORITY_1 |
			AGCCTRL1_CARRIER_SENSE_REL_THR_DISABLE |
			AGCCTRL1_CARRIER_SENSE_ABS_THR_0DB),

		AGCCTRL0, (AGCCTRL0_HYST_LEVEL_MEDIUM |
			AGCCTRL0_WAIT_TIME_16 |
			AGCCTRL0_AGC_FREEZE_NORMAL |
			AGCCTRL0_FILTER_LENGTH_16),

		FREND1, ((1 << FREND1_LNA_CURRENT_SHIFT) |
			(1 << FREND1_LNA2MIX_CURRENT_SHIFT) |
			(1 << FREND1_LODIV_BUF_CURRENT_RX_SHIFT) |
			(2 << FREND1_MIX_CURRENT_SHIFT)),

		// Use PA_TABLE 1 for transmitting '1' in ASK
		// (PA_TABLE 0 is always used for '0')
		FREND0, ((1 << FREND0_LODIV_BUF_CURRENT_TX_SHIFT) |
			(1 << FREND0_PA_POWER_SHIFT)),

		FSCAL3, (3 << 6) | (2 << 4) | 0x09,
		FSCAL2, (1 << 5) | 0x0A, // VCO high
		FSCAL1, 0x00,
		FSCAL0, 0x1F,

		TEST2, TEST2_RX_LOW_DATA_RATE_MAGIC,
		TEST1, TEST1_RX_LOW_DATA_RATE_MAGIC,
		TEST0, (2 << 2) | 1, // disable VCO selection calibration
	})
	if err != nil {
		return err
	}

	// Power amplifier output settings
	// (see Table 72 on page 207 of the datasheet)
	err = dev.spiDev.Transfer([]byte{
		BURST_MODE | PATABLE,
		0x00,
		0xC0, // 10dBm, 36mA
	})
	if err != nil {
		return err
	}

	return nil
}

func (dev *Device) ReadFrequency() (uint32, error) {
	freq2, err := dev.ReadRegister(FREQ2)
	if err != nil {
		return 0, err
	}
	freq1, err := dev.ReadRegister(FREQ1)
	if err != nil {
		return 0, err
	}
	freq0, err := dev.ReadRegister(FREQ0)
	if err != nil {
		return 0, err
	}
	f := uint32(freq2)<<16 + uint32(freq1)<<8 + uint32(freq0)
	return uint32(uint64(f) * FXOSC >> 16), nil
}

func (dev *Device) ReadIF() (uint32, error) {
	f, err := dev.ReadRegister(FSCTRL1)
	if err != nil {
		return 0, err
	}
	return uint32(uint64(f) * FXOSC >> 10), nil
}

func (dev *Device) ReadChannelParams() (chanbw uint32, drate uint32, err error) {
	var m4 byte
	m4, err = dev.ReadRegister(MDMCFG4)
	if err != nil {
		return
	}
	chanbw_E := (m4 >> MDMCFG4_CHANBW_E_SHIFT) & 0x3
	chanbw_M := (m4 >> MDMCFG4_CHANBW_M_SHIFT) & 0x3
	drate_E := (m4 >> MDMCFG4_DRATE_E_SHIFT) & 0xF

	var drate_M byte
	drate_M, err = dev.ReadRegister(MDMCFG3)
	if err != nil {
		return
	}

	chanbw = uint32(FXOSC / ((4 + uint64(chanbw_M)) << (chanbw_E + 3)))
	drate = uint32((((256 + uint64(drate_M)) << drate_E) * FXOSC) >> 28)
	return
}

func (dev *Device) ReadModemConfig() (fec bool, minPreamble byte, chanspc uint32, err error) {
	var m1 byte
	m1, err = dev.ReadRegister(MDMCFG1)
	if err != nil {
		return
	}
	fec = m1&MDMCFG1_FEC_EN != 0
	minPreamble = numPreamble[(m1&MDMCFG1_NUM_PREAMBLE_MASK)>>4]
	chanspc_E := m1 & MDMCFG1_CHANSPC_E_MASK
	var chanspc_M byte
	chanspc_M, err = dev.ReadRegister(MDMCFG0)
	if err != nil {
		return
	}
	chanspc = uint32((((256 + uint64(chanspc_M)) << chanspc_E) * FXOSC) >> 18)
	return
}

const (
	rssi_offset = 74 // see data sheet section 17.3
)

func (dev *Device) ReadRSSI() (int, error) {
	r, err := dev.ReadRegister(RSSI)
	if err != nil {
		return 0, err
	}
	d := int(r)
	if d >= 128 {
		d -= 256
	}
	return d/2 - rssi_offset, nil
}
<commit_msg>Tweak RF parameters per TI Design Note DN022 (swra215e)<commit_after>package cc1100

func (dev *Device) InitRF() error {
	err := dev.WriteEach([]byte{
		// Carrier sense: high if RSSI level is above threshold
		IOCFG2, 0x0E,
		IOCFG1, 0x00,
		// Assert when sync word has been sent/received
		IOCFG0, 0x06,

		// Sync word
		SYNC1, 0xFF,
		SYNC0, 0x00,

		// Packet length
		PKTLEN, 0xFF,

		// Always accept sync word
		// Do not append status
		// No address check
		PKTCTRL1, 2 << PKTCTRL1_PQT_SHIFT,

		// No whitening mode
		// Normal format
		// Disable CRC calculation and check
		// Fixed packet length mode
		PKTCTRL0, 0x00,

		// channel number
		CHANNR, 0x00,

		// Intermediate frequency
		// 0x06 * 26 MHz / 2^10 == 152 kHz
		FSCTRL1, 0x06,

		// Frequency offset
		FSCTRL0, 0x00,

		// 24-bit base frequency
		// 0x2340FC * 26 MHz / 2^16 == 916.6 MHz (916599975 Hz)
		FREQ2, 0x23,
		FREQ1, 0x40,
		FREQ0, 0xFC,

		// CHANBW_E = 2, CHANBW_M = 1, DRATE_E = 9
		// Channel BW = 26 MHz / (8 * (4 + CHANBW_M) * 2^CHANBW_E) == 162.5 kHz
		MDMCFG4, ((2 << MDMCFG4_CHANBW_E_SHIFT) |
			(1 << MDMCFG4_CHANBW_M_SHIFT) |
			(9 << MDMCFG4_DRATE_E_SHIFT)),

		// DRATE_M = 74 (0x4A)
		// Data rate = (256 + DRATE_M) * 2^DRATE_E * 26 MHz / 2^28 == 16365 Baud
		MDMCFG3, 0x4A,

		MDMCFG2, (MDMCFG2_DEM_DCFILT_ON |
			MDMCFG2_MOD_FORMAT_ASK_OOK |
			MDMCFG2_SYNC_MODE_30_32_THRES),

		// CHANSPC_E = 1
		MDMCFG1, (MDMCFG1_FEC_DIS |
			MDMCFG1_NUM_PREAMBLE_16 |
			(1 << MDMCFG1_CHANSPC_E_SHIFT)),

		// CHANSPC_M = 248 (0xF8)
		// cHannel spacing
		// (256 + CHANSPC_M) * 2^CHANSPC_E * 26 MHz / 2^18 == 99975 Hz
		MDMCFG0, 0xF8,

		MCSM2, MCSM2_RX_TIME_END_OF_PACKET,

		MCSM1, (MCSM1_CCA_MODE_RSSI_BELOW_UNLESS_RECEIVING |
			MCSM1_RXOFF_MODE_IDLE |
			MCSM1_TXOFF_MODE_IDLE),

		MCSM0, (MCSM0_FS_AUTOCAL_FROM_IDLE |
			MCSM0_MAGIC_3 |
			MCSM0_CLOSE_IN_RX_0DB),

		FOCCFG, (FOCCFG_FOC_PRE_K_3K |
			FOCCFG_FOC_POST_K_PRE_K_OVER_2 |
			FOCCFG_FOC_LIMIT_BW_OVER_2),

		BSCFG, (BSCFG_BS_PRE_K_2K |
			BSCFG_BS_PRE_KP_3KP |
			BSCFG_BS_POST_KI_PRE_KI_OVER_2 |
			BSCFG_BS_LIMIT_0),

		AGCCTRL2, (AGCCTRL2_MAX_DVGA_GAIN_ALL |
			AGCCTRL2_MAX_LNA_GAIN_0 |
			AGCCTRL2_MAGN_TARGET_38dB),

		AGCCTRL1, (AGCCTRL1_AGC_LNA_PRIORITY_0 |
			AGCCTRL1_CARRIER_SENSE_REL_THR_DISABLE |
			AGCCTRL1_CARRIER_SENSE_ABS_THR_0DB),

		AGCCTRL0, (AGCCTRL0_HYST_LEVEL_MEDIUM |
			AGCCTRL0_WAIT_TIME_16 |
			AGCCTRL0_AGC_FREEZE_NORMAL |
			AGCCTRL0_FILTER_LENGTH_32),

		FREND1, ((1 << FREND1_LNA_CURRENT_SHIFT) |
			(1 << FREND1_LNA2MIX_CURRENT_SHIFT) |
			(1 << FREND1_LODIV_BUF_CURRENT_RX_SHIFT) |
			(2 << FREND1_MIX_CURRENT_SHIFT)),

		// Use PA_TABLE 1 for transmitting '1' in ASK
		// (PA_TABLE 0 is always used for '0')
		FREND0, ((1 << FREND0_LODIV_BUF_CURRENT_TX_SHIFT) |
			(1 << FREND0_PA_POWER_SHIFT)),

		FSCAL3, (3 << 6) | (2 << 4) | 0x09,
		FSCAL2, (1 << 5) | 0x0A, // VCO high
		FSCAL1, 0x00,
		FSCAL0, 0x1F,

		TEST2, TEST2_RX_LOW_DATA_RATE_MAGIC,
		TEST1, TEST1_RX_LOW_DATA_RATE_MAGIC,
		TEST0, (2 << 2) | 1, // disable VCO selection calibration
	})
	if err != nil {
		return err
	}

	// Power amplifier output settings
	// (see Table 72 on page 207 of the datasheet)
	err = dev.spiDev.Transfer([]byte{
		BURST_MODE | PATABLE,
		0x00,
		0xC0, // 10dBm, 36mA
	})
	if err != nil {
		return err
	}

	return nil
}

func (dev *Device) ReadFrequency() (uint32, error) {
	freq2, err := dev.ReadRegister(FREQ2)
	if err != nil {
		return 0, err
	}
	freq1, err := dev.ReadRegister(FREQ1)
	if err != nil {
		return 0, err
	}
	freq0, err := dev.ReadRegister(FREQ0)
	if err != nil {
		return 0, err
	}
	f := uint32(freq2)<<16 + uint32(freq1)<<8 + uint32(freq0)
	return uint32(uint64(f) * FXOSC >> 16), nil
}

func (dev *Device) ReadIF() (uint32, error) {
	f, err := dev.ReadRegister(FSCTRL1)
	if err != nil {
		return 0, err
	}
	return uint32(uint64(f) * FXOSC >> 10), nil
}

func (dev *Device) ReadChannelParams() (chanbw uint32, drate uint32, err error) {
	var m4 byte
	m4, err = dev.ReadRegister(MDMCFG4)
	if err != nil {
		return
	}
	chanbw_E := (m4 >> MDMCFG4_CHANBW_E_SHIFT) & 0x3
	chanbw_M := (m4 >> MDMCFG4_CHANBW_M_SHIFT) & 0x3
	drate_E := (m4 >> MDMCFG4_DRATE_E_SHIFT) & 0xF

	var drate_M byte
	drate_M, err = dev.ReadRegister(MDMCFG3)
	if err != nil {
		return
	}

	chanbw = uint32(FXOSC / ((4 + uint64(chanbw_M)) << (chanbw_E + 3)))
	drate = uint32((((256 + uint64(drate_M)) << drate_E) * FXOSC) >> 28)
	return
}

func (dev *Device) ReadModemConfig() (fec bool, minPreamble byte, chanspc uint32, err error) {
	var m1 byte
	m1, err = dev.ReadRegister(MDMCFG1)
	if err != nil {
		return
	}
	fec = m1&MDMCFG1_FEC_EN != 0
	minPreamble = numPreamble[(m1&MDMCFG1_NUM_PREAMBLE_MASK)>>4]
	chanspc_E := m1 & MDMCFG1_CHANSPC_E_MASK
	var chanspc_M byte
	chanspc_M, err = dev.ReadRegister(MDMCFG0)
	if err != nil {
		return
	}
	chanspc = uint32((((256 + uint64(chanspc_M)) << chanspc_E) * FXOSC) >> 18)
	return
}

func (dev *Device) ReadRSSI() (int, error) {
	const rssi_offset = 74 // see data sheet section 17.3
	r, err := dev.ReadRegister(RSSI)
	if err != nil {
		return 0, err
	}
	d := int(r)
	if d >= 128 {
		d -= 256
	}
	return d/2 - rssi_offset, nil
}
<|endoftext|>