<commit_before>// The idx package provides a metadata index for metrics
// it's based on the index from https://github.com/dgryski/carbonmem/
// it seems promising and performant for our current needs, but also experimental
// We don't necessarily want to start maintaining our own little search engine.
// Future needs (scale, features) may deprecate this package in favor of something else
// like ES, bleve, etc.  But for now it seems to work well, so let's see where it takes us

// callers are responsible for thread safety, period pruning if desired
package idx

import (
	"path/filepath"
	"sort"
	"strings"
	//	"time"

	"github.com/armon/go-radix"
	"github.com/dgryski/go-trigram"
)

type MatchType uint8

const (
	MatchLiteral MatchType = iota
	MatchPrefix
	MatchTrigram
)

type MetricID uint32 // same thing as trigram.DocID. just syntactic sugar

type Glob struct {
	Id     MetricID
	Metric string
	IsLeaf bool
}

const OTTBase = uint32(2147483648)             // 1 << 31
const PublicOrgTrigram = trigram.T(2147483648) // 1 << 31. eg base + 0

// orgToTrigram takes in an orgid between -1 (special) and 214748345 and converts it to a special trigram
func orgToTrigram(org int) trigram.T {
	if org < -1 {
		panic("orgid must be >= -1")
	}
	// org+1 cannot be 2147483647 because we would end up at MaxUint32 == 4294967295 == 0xffffffff which is reserved
	if org > 2147483645 {
		panic("org too high")
	}
	ret := OTTBase + uint32(org) + 1
	return trigram.T(ret)
}

type globByName []Glob

func (g globByName) Len() int           { return len(g) }
func (g globByName) Swap(i, j int)      { g[i], g[j] = g[j], g[i] }
func (g globByName) Less(i, j int) bool { return g[i].Metric < g[j].Metric }

type Idx struct {
	// all metrics
	keys  []map[string]MetricID // slice is per org
	revs  []string              // MetricID to string key. doesn't have to be isolated per org
	count int                   // global across all orgs

	// currently 'active'
	active map[MetricID]int // doesn't have to be isolated per org

	prefix []*radix.Tree // per org. each org is stored at pos orgid+1 because we have a special org -1

	// stores all orgs in one index, but with special org-id trigrams
	Pathidx trigram.Index
}

func New() *Idx {
	return &Idx{
		keys: make([]map[string]MetricID, 0),

		active: make(map[MetricID]int),
		prefix: make([]*radix.Tree, 0),

		Pathidx: trigram.NewIndex(nil),
	}

}

func (l *Idx) Len() int {
	return len(l.keys)
}

// Get retrieves the metricId and whether it was found
// it first checks the given org, but also falls back to org -1
func (l *Idx) Get(org int, key string) (MetricID, bool) {
	pos := org + 1 // org -1 is pos 0, etc

	for len(l.keys) < pos+1 {
		l.keys = append(l.keys, make(map[string]MetricID))
	}

	id, ok := l.keys[pos][key]
	if !ok {
		id, ok = l.keys[0][key]
	}

	return id, ok
}

// GetsOrAdd retrieves the key if it exists for the org (but doesn't do additional check for org -1)
// and adds it for the org if non-existant
func (l *Idx) GetOrAdd(org int, key string) MetricID {
	pos := org + 1 // org -1 is pos 0, etc

	for len(l.keys) < pos+1 {
		l.keys = append(l.keys, make(map[string]MetricID))
	}

	id, ok := l.keys[pos][key]

	if ok {
		return id
	}

	id = MetricID(l.count)
	l.count++
	l.revs = append(l.revs, key)

	l.keys[pos][key] = id

	// like l.Pathidx.Insert(key, trigram.DocID(id)) but with added org trigram
	ts := trigram.ExtractAll(key, nil)
	ts = append(ts, orgToTrigram(org))
	l.Pathidx.InsertTrigrams(ts, trigram.DocID(id))

	return id
}

func (l *Idx) Key(id MetricID) string {
	return l.revs[id]
}

func (l *Idx) AddRef(org int, id MetricID) {
	v, ok := l.active[id]
	if !ok {
		pos := org + 1
		for len(l.prefix) < pos+1 {
			l.prefix = append(l.prefix, radix.New())
		}
		l.prefix[pos].Insert(l.revs[id], id)
	}

	l.active[id] = v + 1
}

// only call this after having made sure the org entries exists! e.g. after Get/GetOrAdd and AddRef
func (l *Idx) DelRef(org int, id MetricID) {
	l.active[id]--
	if l.active[id] == 0 {
		delete(l.active, id)
		delete(l.keys[org], l.revs[id])
		l.prefix[org+1].Delete(l.revs[id])
		if l.Pathidx != nil {
			l.Pathidx.Delete(l.revs[id], trigram.DocID(id))
		}
		l.revs[id] = ""
	}
}

func (l *Idx) Active(id MetricID) bool {
	return l.active[id] != 0
}

func (l *Idx) Prefix(org int, query string, fn radix.WalkFn) {
	pos := org + 1
	for len(l.prefix) < pos+1 {
		l.prefix = append(l.prefix, radix.New())
	}
	l.prefix[pos].WalkPrefix(query, fn)
}

// for a "filesystem glob" query (e.g. supports * and [] but not {})
// looks in the trigrams index
func (l *Idx) QueryPath(org int, query string) []Glob {

	if l.Pathidx == nil {
		return nil
	}

	ts := extractTrigrams(query)
	ids := l.Pathidx.QueryTrigrams(ts)
	ids = l.Pathidx.FilterOr(ids, [][]trigram.T{{PublicOrgTrigram}, {orgToTrigram(org)}})

	seen := make(map[string]bool)
	out := make([]Glob, 0)

	for _, id := range ids {

		p := l.revs[MetricID(id)]

		dir := filepath.Dir(p)

		if seen[dir] {
			continue
		}

		if matched, err := filepath.Match(query, dir); err == nil && matched {
			seen[dir] = true
			out = append(out, Glob{0, dir, false}) // for directories we leave id=0
			continue
		}

		if matched, err := filepath.Match(query, p); err == nil && matched {
			seen[p] = true
			out = append(out, Glob{MetricID(id), p, true})
		}
	}

	return out
}

func (l *Idx) List(org int) []MetricID {
	var response []MetricID
	fn := func(k string, v interface{}) bool {
		response = append(response, v.(MetricID))
		return false
	}
	if org == -1 {
		for i := 0; i < len(l.prefix); i++ {
			l.prefix[i].Walk(fn)
		}
	} else {
		l.Prefix(-1, "", fn)
		l.Prefix(org, "", fn)
	}
	return response
}
func (i *Idx) QueryRadix(org int, query string) []Glob {
	var response []Glob
	l := len(query)
	seen := make(map[string]bool)
	fn := func(k string, v interface{}) bool {
		// figure out if we're a leaf or not
		dot := strings.IndexByte(k[l:], '.')
		var leaf bool
		m := k
		if dot == -1 {
			leaf = true
		} else {
			m = k[:dot+l]
		}
		if !seen[m] {
			seen[m] = true
			response = append(response, Glob{Id: v.(MetricID), Metric: m, IsLeaf: leaf})
		}
		// false == "don't terminate iteration"
		return false
	}
	i.Prefix(-1, query, fn)
	i.Prefix(org, query, fn)
	return response
}

// TODO(dgryski): this needs most of the logic in grobian/carbsonerver:findHandler()
// Dieter: this doesn't support {, }, [, ]

func (i *Idx) Match(org int, query string) (MatchType, []Glob) {

	// no wildcard == exact match only
	var star int
	if star = strings.Index(query, "*"); star == -1 {
		var id MetricID
		var ok bool
		if id, ok = i.Get(org, query); !ok {
			return MatchLiteral, nil
		}
		return MatchLiteral, []Glob{{Id: id, Metric: query, IsLeaf: true}}
	}

	var response []Glob
	var matchType MatchType

	if star == len(query)-1 {
		// only one trailing star
		query = query[:len(query)-1]
		response = i.QueryRadix(org, query)
		matchType = MatchPrefix
	} else {
		// at least one interior star
		response = i.QueryPath(org, query)
		matchType = MatchTrigram
	}

	sort.Sort(globByName(response))

	return matchType, response
}

func extractTrigrams(query string) []trigram.T {

	if len(query) < 3 {
		return nil
	}

	var start int
	var i int

	var trigrams []trigram.T

	for i < len(query) {
		if query[i] == '[' || query[i] == '*' || query[i] == '?' {
			trigrams = trigram.Extract(query[start:i], trigrams)

			if query[i] == '[' {
				for i < len(query) && query[i] != ']' {
					i++
				}
			}

			start = i + 1
		}
		i++
	}

	trigrams = trigram.Extract(query[start:i], trigrams)

	return trigrams
}

func (i *Idx) Prune(pct float64) int {
	return i.Pathidx.Prune(pct)
}
<commit_msg>implement custom pruning ourself instead of in forked go-trigram<commit_after>// The idx package provides a metadata index for metrics
// it's based on the index from https://github.com/dgryski/carbonmem/
// it seems promising and performant for our current needs, but also experimental
// We don't necessarily want to start maintaining our own little search engine.
// Future needs (scale, features) may deprecate this package in favor of something else
// like ES, bleve, etc.  But for now it seems to work well, so let's see where it takes us

// callers are responsible for thread safety, period pruning if desired
package idx

import (
	"path/filepath"
	"sort"
	"strings"
	//	"time"

	"github.com/armon/go-radix"
	"github.com/dgryski/go-trigram"
)

type MatchType uint8

const (
	MatchLiteral MatchType = iota
	MatchPrefix
	MatchTrigram
)

type MetricID uint32 // same thing as trigram.DocID. just syntactic sugar

type Glob struct {
	Id     MetricID
	Metric string
	IsLeaf bool
}

const OTTBase = uint32(2147483648)             // 1 << 31
const PublicOrgTrigram = trigram.T(2147483648) // 1 << 31. eg base + 0

// orgToTrigram takes in an orgid between -1 (special) and 214748345 and converts it to a special trigram
func orgToTrigram(org int) trigram.T {
	if org < -1 {
		panic("orgid must be >= -1")
	}
	// org+1 cannot be 2147483647 because we would end up at MaxUint32 == 4294967295 == 0xffffffff which is reserved
	if org > 2147483645 {
		panic("org too high")
	}
	ret := OTTBase + uint32(org) + 1
	return trigram.T(ret)
}

type globByName []Glob

func (g globByName) Len() int           { return len(g) }
func (g globByName) Swap(i, j int)      { g[i], g[j] = g[j], g[i] }
func (g globByName) Less(i, j int) bool { return g[i].Metric < g[j].Metric }

type Idx struct {
	// all metrics
	keys  []map[string]MetricID // slice is per org
	revs  []string              // MetricID to string key. doesn't have to be isolated per org
	count int                   // global across all orgs

	// currently 'active'
	active map[MetricID]int // doesn't have to be isolated per org

	prefix []*radix.Tree // per org. each org is stored at pos orgid+1 because we have a special org -1

	// stores all orgs in one index, but with special org-id trigrams
	Pathidx trigram.Index
}

func New() *Idx {
	return &Idx{
		keys: make([]map[string]MetricID, 0),

		active: make(map[MetricID]int),
		prefix: make([]*radix.Tree, 0),

		Pathidx: trigram.NewIndex(nil),
	}

}

func (l *Idx) Len() int {
	return len(l.keys)
}

// Get retrieves the metricId and whether it was found
// it first checks the given org, but also falls back to org -1
func (l *Idx) Get(org int, key string) (MetricID, bool) {
	pos := org + 1 // org -1 is pos 0, etc

	for len(l.keys) < pos+1 {
		l.keys = append(l.keys, make(map[string]MetricID))
	}

	id, ok := l.keys[pos][key]
	if !ok {
		id, ok = l.keys[0][key]
	}

	return id, ok
}

// GetsOrAdd retrieves the key if it exists for the org (but doesn't do additional check for org -1)
// and adds it for the org if non-existant
func (l *Idx) GetOrAdd(org int, key string) MetricID {
	pos := org + 1 // org -1 is pos 0, etc

	for len(l.keys) < pos+1 {
		l.keys = append(l.keys, make(map[string]MetricID))
	}

	id, ok := l.keys[pos][key]

	if ok {
		return id
	}

	id = MetricID(l.count)
	l.count++
	l.revs = append(l.revs, key)

	l.keys[pos][key] = id

	// like l.Pathidx.Insert(key, trigram.DocID(id)) but with added org trigram
	ts := trigram.ExtractAll(key, nil)
	ts = append(ts, orgToTrigram(org))
	l.Pathidx.InsertTrigrams(ts, trigram.DocID(id))

	return id
}

func (l *Idx) Key(id MetricID) string {
	return l.revs[id]
}

func (l *Idx) AddRef(org int, id MetricID) {
	v, ok := l.active[id]
	if !ok {
		pos := org + 1
		for len(l.prefix) < pos+1 {
			l.prefix = append(l.prefix, radix.New())
		}
		l.prefix[pos].Insert(l.revs[id], id)
	}

	l.active[id] = v + 1
}

// only call this after having made sure the org entries exists! e.g. after Get/GetOrAdd and AddRef
func (l *Idx) DelRef(org int, id MetricID) {
	l.active[id]--
	if l.active[id] == 0 {
		delete(l.active, id)
		delete(l.keys[org], l.revs[id])
		l.prefix[org+1].Delete(l.revs[id])
		if l.Pathidx != nil {
			l.Pathidx.Delete(l.revs[id], trigram.DocID(id))
		}
		l.revs[id] = ""
	}
}

func (l *Idx) Active(id MetricID) bool {
	return l.active[id] != 0
}

func (l *Idx) Prefix(org int, query string, fn radix.WalkFn) {
	pos := org + 1
	for len(l.prefix) < pos+1 {
		l.prefix = append(l.prefix, radix.New())
	}
	l.prefix[pos].WalkPrefix(query, fn)
}

// for a "filesystem glob" query (e.g. supports * and [] but not {})
// looks in the trigrams index
func (l *Idx) QueryPath(org int, query string) []Glob {

	if l.Pathidx == nil {
		return nil
	}

	ts := extractTrigrams(query)
	ids := l.Pathidx.QueryTrigrams(ts)
	ids = l.Pathidx.FilterOr(ids, [][]trigram.T{{PublicOrgTrigram}, {orgToTrigram(org)}})

	seen := make(map[string]bool)
	out := make([]Glob, 0)

	for _, id := range ids {

		p := l.revs[MetricID(id)]

		dir := filepath.Dir(p)

		if seen[dir] {
			continue
		}

		if matched, err := filepath.Match(query, dir); err == nil && matched {
			seen[dir] = true
			out = append(out, Glob{0, dir, false}) // for directories we leave id=0
			continue
		}

		if matched, err := filepath.Match(query, p); err == nil && matched {
			seen[p] = true
			out = append(out, Glob{MetricID(id), p, true})
		}
	}

	return out
}

func (l *Idx) List(org int) []MetricID {
	var response []MetricID
	fn := func(k string, v interface{}) bool {
		response = append(response, v.(MetricID))
		return false
	}
	if org == -1 {
		for i := 0; i < len(l.prefix); i++ {
			l.prefix[i].Walk(fn)
		}
	} else {
		l.Prefix(-1, "", fn)
		l.Prefix(org, "", fn)
	}
	return response
}
func (i *Idx) QueryRadix(org int, query string) []Glob {
	var response []Glob
	l := len(query)
	seen := make(map[string]bool)
	fn := func(k string, v interface{}) bool {
		// figure out if we're a leaf or not
		dot := strings.IndexByte(k[l:], '.')
		var leaf bool
		m := k
		if dot == -1 {
			leaf = true
		} else {
			m = k[:dot+l]
		}
		if !seen[m] {
			seen[m] = true
			response = append(response, Glob{Id: v.(MetricID), Metric: m, IsLeaf: leaf})
		}
		// false == "don't terminate iteration"
		return false
	}
	i.Prefix(-1, query, fn)
	i.Prefix(org, query, fn)
	return response
}

// TODO(dgryski): this needs most of the logic in grobian/carbsonerver:findHandler()
// Dieter: this doesn't support {, }, [, ]

func (i *Idx) Match(org int, query string) (MatchType, []Glob) {

	// no wildcard == exact match only
	var star int
	if star = strings.Index(query, "*"); star == -1 {
		var id MetricID
		var ok bool
		if id, ok = i.Get(org, query); !ok {
			return MatchLiteral, nil
		}
		return MatchLiteral, []Glob{{Id: id, Metric: query, IsLeaf: true}}
	}

	var response []Glob
	var matchType MatchType

	if star == len(query)-1 {
		// only one trailing star
		query = query[:len(query)-1]
		response = i.QueryRadix(org, query)
		matchType = MatchPrefix
	} else {
		// at least one interior star
		response = i.QueryPath(org, query)
		matchType = MatchTrigram
	}

	sort.Sort(globByName(response))

	return matchType, response
}

func extractTrigrams(query string) []trigram.T {

	if len(query) < 3 {
		return nil
	}

	var start int
	var i int

	var trigrams []trigram.T

	for i < len(query) {
		if query[i] == '[' || query[i] == '*' || query[i] == '?' {
			trigrams = trigram.Extract(query[start:i], trigrams)

			if query[i] == '[' {
				for i < len(query) && query[i] != ']' {
					i++
				}
			}

			start = i + 1
		}
		i++
	}

	trigrams = trigram.Extract(query[start:i], trigrams)

	return trigrams
}

// Prune is a custom pruning function for the trigram index
// it removes all trigrams that are present in more than the specified percentage of the documents.
// However, it makes sure that neither tAllDocIDs,
// nor any of the org specific trigrams used for access control are ever pruned.
func (i *Idx) Prune(pct float64) {
	// 0xFFFFFFFF is just tAllDocIDs in the trigram package but it's not yet exported
	// https://github.com/dgryski/go-trigram/pull/18
	maxDocs := int(pct * float64(len(i.Pathidx[0xFFFFFFFF])))
	for k, v := range i.Pathidx {
		// all values >= 1<<31 are special trigrams that should be kept
		if uint32(k) < uint32(2147483648) && len(v) > maxDocs {
			i.Pathidx[k] = nil
		}
	}
}
<|endoftext|>


<commit_before>package lscolors

import (
	"os"
	"runtime"
	"testing"

	"github.com/elves/elvish/util"
)

func TestDetermineFeature(t *testing.T) {
	test := func(fname string, mh bool, wantedFeature feature) {
		feature, err := determineFeature(fname, mh)
		if err != nil {
			t.Errorf("determineFeature(%q, %v) returns error %v, want no error",
				fname, mh, err)
		}
		if feature != wantedFeature {
			t.Errorf("determineFeature(%q, %v) returns feature %v, want %v",
				fname, mh, feature, wantedFeature)
		}
	}

	_, cleanup := util.InTestDir()
	defer cleanup()

	create("a", 0600)
	// Regular file.
	test("a", true, featureRegular)

	// Symlink.
	err := os.Symlink("a", "symlink")
	if err != nil {
		t.Logf("Failed to create symlink: %v; skipping symlink test", err)
	} else {
		test("symlink", true, featureSymlink)
	}

	// Broken symlink.
	err = os.Symlink("aaaa", "bad-symlink")
	if err != nil {
		t.Logf("Failed to create bad symlink: %v; skipping bad symlink test", err)
	} else {
		test("bad-symlink", true, featureOrphanedSymlink)
	}

	if runtime.GOOS != "windows" {
		// Multiple hard links.
		os.Link("a", "a2")
		test("a", true, featureMultiHardLink)
	}

	// Don't test for multiple hard links.
	test("a", false, featureRegular)

	// Setuid and Setgid.
	// XXX(xiaq): Fails.
	/*
		create("su", os.ModeSetuid)
		test("su", true, featureSetuid)
		create("sg", os.ModeSetgid)
		test("sg", true, featureSetgid)
	*/

	if runtime.GOOS != "windows" {
		// Executable.
		create("xu", 0100)
		create("xg", 0010)
		create("xo", 0001)
		test("xu", true, featureExecutable)
		test("xg", true, featureExecutable)
		test("xo", true, featureExecutable)
	}
}

func create(fname string, perm os.FileMode) {
	f, err := os.OpenFile(fname, os.O_CREATE, perm)
	if err != nil {
		panic(err)
	}
	f.Close()
}
<commit_msg>edit/lscolors: Fix test for termux.<commit_after>package lscolors

import (
	"os"
	"runtime"
	"testing"

	"github.com/elves/elvish/util"
)

func TestDetermineFeature(t *testing.T) {
	test := func(fname string, mh bool, wantedFeature feature) {
		t.Helper()
		feature, err := determineFeature(fname, mh)
		if err != nil {
			t.Errorf("determineFeature(%q, %v) returns error %v, want no error",
				fname, mh, err)
		}
		if feature != wantedFeature {
			t.Errorf("determineFeature(%q, %v) returns feature %v, want %v",
				fname, mh, feature, wantedFeature)
		}
	}

	_, cleanup := util.InTestDir()
	defer cleanup()

	create("a", 0600)
	// Regular file.
	test("a", true, featureRegular)

	// Symlink.
	err := os.Symlink("a", "symlink")
	if err != nil {
		t.Logf("Failed to create symlink: %v; skipping symlink test", err)
	} else {
		test("symlink", true, featureSymlink)
	}

	// Broken symlink.
	err = os.Symlink("aaaa", "bad-symlink")
	if err != nil {
		t.Logf("Failed to create bad symlink: %v; skipping bad symlink test", err)
	} else {
		test("bad-symlink", true, featureOrphanedSymlink)
	}

	if runtime.GOOS != "windows" {
		// Multiple hard links.
		err := os.Link("a", "a2")
		if err != nil {
			t.Logf("Failed to create hard link: %v; skipping hard link test", err)
		} else {
			test("a", true, featureMultiHardLink)
		}
	}

	// Don't test for multiple hard links.
	test("a", false, featureRegular)

	// Setuid and Setgid.
	// XXX(xiaq): Fails.
	/*
		create("su", os.ModeSetuid)
		test("su", true, featureSetuid)
		create("sg", os.ModeSetgid)
		test("sg", true, featureSetgid)
	*/

	if runtime.GOOS != "windows" {
		// Executable.
		create("xu", 0100)
		create("xg", 0010)
		create("xo", 0001)
		test("xu", true, featureExecutable)
		test("xg", true, featureExecutable)
		test("xo", true, featureExecutable)
	}
}

func create(fname string, perm os.FileMode) {
	f, err := os.OpenFile(fname, os.O_CREATE, perm)
	if err != nil {
		panic(err)
	}
	// The following call works around a termux bug where os.OpenFile does not
	// set the permission bits for group and other correctly.
	f.Chmod(perm)
	f.Close()
}
<|endoftext|>


<commit_before>package middleware

import (
	"bufio"
	"compress/gzip"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"strings"

	"github.com/labstack/echo"
)

type (
	// GzipConfig defines the config for Gzip middleware.
	GzipConfig struct {
		// Skipper defines a function to skip middleware.
		Skipper Skipper

		// Gzip compression level.
		// Optional. Default value -1.
		Level int `json:"level"`
	}

	gzipResponseWriter struct {
		io.Writer
		http.ResponseWriter
	}
)

const (
	gzipScheme = "gzip"
)

var (
	// DefaultGzipConfig is the default Gzip middleware config.
	DefaultGzipConfig = GzipConfig{
		Skipper: DefaultSkipper,
		Level:   -1,
	}
)

// Gzip returns a middleware which compresses HTTP response using gzip compression
// scheme.
func Gzip() echo.MiddlewareFunc {
	return GzipWithConfig(DefaultGzipConfig)
}

// GzipWithConfig return Gzip middleware with config.
// See: `Gzip()`.
func GzipWithConfig(config GzipConfig) echo.MiddlewareFunc {
	// Defaults
	if config.Skipper == nil {
		config.Skipper = DefaultGzipConfig.Skipper
	}
	if config.Level == 0 {
		config.Level = DefaultGzipConfig.Level
	}

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			if config.Skipper(c) {
				return next(c)
			}

			res := c.Response()
			res.Header().Add(echo.HeaderVary, echo.HeaderAcceptEncoding)
			res.Header().Add(echo.HeaderContentEncoding, gzipScheme) // issue #806
			if strings.Contains(c.Request().Header.Get(echo.HeaderAcceptEncoding), gzipScheme) {
				rw := res.Writer
				w, err := gzip.NewWriterLevel(rw, config.Level)
				if err != nil {
					return err
				}
				defer func() {

					if res.Size == 0 {
						// We have to reset response to it's pristine state when
						// nothing is written to body or error is returned.
						// See issue #424, #407.
						res.Writer = rw
						w.Reset(ioutil.Discard)
					}
					w.Close()
				}()
				grw := &gzipResponseWriter{Writer: w, ResponseWriter: rw}
				res.Writer = grw
			}
			return next(c)
		}
	}
}

func (w *gzipResponseWriter) WriteHeader(code int) {
	if code == http.StatusNoContent { // Issue #489
		w.ResponseWriter.Header().Del(echo.HeaderContentEncoding)
	}
	w.ResponseWriter.WriteHeader(code)
}

func (w *gzipResponseWriter) Write(b []byte) (int, error) {
	if w.Header().Get(echo.HeaderContentType) == "" {
		w.Header().Set(echo.HeaderContentType, http.DetectContentType(b))
	}
	return w.Writer.Write(b)
}

func (w *gzipResponseWriter) Flush() error {
	return w.Writer.(*gzip.Writer).Flush()
}

func (w *gzipResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	return w.ResponseWriter.(http.Hijacker).Hijack()
}

func (w *gzipResponseWriter) CloseNotify() <-chan bool {
	return w.ResponseWriter.(http.CloseNotifier).CloseNotify()
}
<commit_msg>Set the Content-Encoding header only when requested<commit_after>package middleware

import (
	"bufio"
	"compress/gzip"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"strings"

	"github.com/labstack/echo"
)

type (
	// GzipConfig defines the config for Gzip middleware.
	GzipConfig struct {
		// Skipper defines a function to skip middleware.
		Skipper Skipper

		// Gzip compression level.
		// Optional. Default value -1.
		Level int `json:"level"`
	}

	gzipResponseWriter struct {
		io.Writer
		http.ResponseWriter
	}
)

const (
	gzipScheme = "gzip"
)

var (
	// DefaultGzipConfig is the default Gzip middleware config.
	DefaultGzipConfig = GzipConfig{
		Skipper: DefaultSkipper,
		Level:   -1,
	}
)

// Gzip returns a middleware which compresses HTTP response using gzip compression
// scheme.
func Gzip() echo.MiddlewareFunc {
	return GzipWithConfig(DefaultGzipConfig)
}

// GzipWithConfig return Gzip middleware with config.
// See: `Gzip()`.
func GzipWithConfig(config GzipConfig) echo.MiddlewareFunc {
	// Defaults
	if config.Skipper == nil {
		config.Skipper = DefaultGzipConfig.Skipper
	}
	if config.Level == 0 {
		config.Level = DefaultGzipConfig.Level
	}

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			if config.Skipper(c) {
				return next(c)
			}

			res := c.Response()
			res.Header().Add(echo.HeaderVary, echo.HeaderAcceptEncoding)
			if strings.Contains(c.Request().Header.Get(echo.HeaderAcceptEncoding), gzipScheme) {
				res.Header().Add(echo.HeaderContentEncoding, gzipScheme) // issue #806
				rw := res.Writer
				w, err := gzip.NewWriterLevel(rw, config.Level)
				if err != nil {
					return err
				}
				defer func() {
					if res.Size == 0 {
						if res.Header().Get(echo.HeaderContentEncoding) == gzipScheme {
							res.Header().Del(echo.HeaderContentEncoding)
						}
						// We have to reset response to it's pristine state when
						// nothing is written to body or error is returned.
						// See issue #424, #407.
						res.Writer = rw
						w.Reset(ioutil.Discard)
					}
					w.Close()
				}()
				grw := &gzipResponseWriter{Writer: w, ResponseWriter: rw}
				res.Writer = grw
			}
			return next(c)
		}
	}
}

func (w *gzipResponseWriter) WriteHeader(code int) {
	if code == http.StatusNoContent { // Issue #489
		w.ResponseWriter.Header().Del(echo.HeaderContentEncoding)
	}
	w.ResponseWriter.WriteHeader(code)
}

func (w *gzipResponseWriter) Write(b []byte) (int, error) {
	if w.Header().Get(echo.HeaderContentType) == "" {
		w.Header().Set(echo.HeaderContentType, http.DetectContentType(b))
	}
	return w.Writer.Write(b)
}

func (w *gzipResponseWriter) Flush() error {
	return w.Writer.(*gzip.Writer).Flush()
}

func (w *gzipResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	return w.ResponseWriter.(http.Hijacker).Hijack()
}

func (w *gzipResponseWriter) CloseNotify() <-chan bool {
	return w.ResponseWriter.(http.CloseNotifier).CloseNotify()
}
<|endoftext|>


<commit_before>package middleware

import (
	"errors"
	"time"

	"github.com/achilleasa/usrv"
	"golang.org/x/net/context"
)

// Apply a throttling middleware to the input handler that throttles
// request handling to maxConcurrent requests.
//
// If the pending requests exceed the specified timeout, they will be aborted
// with ErrTimeout.
func Throttle(maxConcurrent int, timeout time.Duration) usrv.EndpointOption {

	return func(ep *usrv.Endpoint) error {

		if maxConcurrent <= 0 {
			return errors.New("maxConcurrent should be > 0")
		}

		// Allocate a buffered channel and pre-fill it with tokens
		tokens := make(chan struct{}, maxConcurrent)
		for i := 0; i < maxConcurrent; i++ {
			tokens <- struct{}{}
		}

		// Wrap original method
		originalHandler := ep.Handler
		ep.Handler = usrv.HandlerFunc(func(ctx context.Context, responseWriter usrv.ResponseWriter, request *usrv.Message) {
			fctx, cancel := context.WithTimeout(ctx, timeout)
			defer cancel()

			select {
			case <-tokens:
				// We got a token, execute request
				originalHandler.Serve(fctx, responseWriter, request)

				// Return back token
				tokens <- struct{}{}
			case <-fctx.Done():
				responseWriter.WriteError(usrv.ErrTimeout)
			}

		})

		return nil
	}
}
<commit_msg>Throttle distinguishes between context cancellation and timeout errors<commit_after>package middleware

import (
	"errors"
	"time"

	"github.com/achilleasa/usrv"
	"golang.org/x/net/context"
)

// Apply a throttling middleware to the input handler that throttles
// request handling to maxConcurrent requests.
//
// If the pending requests exceed the specified timeout, they will be aborted
// with ErrTimeout.
func Throttle(maxConcurrent int, timeout time.Duration) usrv.EndpointOption {

	return func(ep *usrv.Endpoint) error {

		if maxConcurrent <= 0 {
			return errors.New("maxConcurrent should be > 0")
		}

		// Allocate a buffered channel and pre-fill it with tokens
		tokens := make(chan struct{}, maxConcurrent)
		for i := 0; i < maxConcurrent; i++ {
			tokens <- struct{}{}
		}

		// Wrap original method
		originalHandler := ep.Handler
		ep.Handler = usrv.HandlerFunc(func(ctx context.Context, responseWriter usrv.ResponseWriter, request *usrv.Message) {
			fctx, cancel := context.WithTimeout(ctx, timeout)
			defer cancel()

			select {
			case <-tokens:
				// We got a token, execute request
				originalHandler.Serve(fctx, responseWriter, request)

				// Return back token
				tokens <- struct{}{}
			case <-fctx.Done():
				if fctx.Err() == context.Canceled {
					responseWriter.WriteError(usrv.ErrCancelled)
				} else {
					responseWriter.WriteError(usrv.ErrTimeout)
				}
			}

		})

		return nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Brian J. Downs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package formatifier is a library to easily format strings in a user defined
// and predefined manner.
package formatifier

import (
	"crypto/rand"
	"regexp"
	"strings"
)

const (
	lengthError = "ERROR: String not long enough to convert."
	pirateLink  = "http://www.isithackday.com/arrpi.php?text=%s"
)

// Type to hold user input string.
type formatifier struct {
	theString string
}

// New will create an instance of the String object.
func New(s string) *formatifier { return &formatifier{theString: s} }

// makeLower will turn the user entered string to lower case
func (f *formatifier) makeLower() { f.theString = strings.ToLower(f.theString) }

// removeNonDigits removes any non digit characters from the string.
func (f *formatifier) removeNonDigits() {
	rp := regexp.MustCompile(`\D`)
	f.theString = rp.ReplaceAllString(f.theString, "")
}

// removeNonWordChars removes all non word characters.
func (f *formatifier) removeNonWordChars() {
	if len(f.theString) > 0 {
		rp := regexp.MustCompile(`\W|\s|_`)
		f.theString = rp.ReplaceAllString(f.theString, "")
	}
}

// urlEncodeSpaces will replace spaces with "%20"'s
func (f *formatifier) urlEncodeSpaces() {
	rp := regexp.MustCompile(`\s`)
	f.theString = rp.ReplaceAllString(f.theString, "%20")
}

// random select will return a random selection from an int slice
func randomSelect(anySlice []int) int {
	var tmpIndex int
	length := len(anySlice)
	randBytes := make([]byte, length)
	if _, err := rand.Read(randBytes); err == nil {
		tmpIndex = int(randBytes[0]) % length
	}
	return anySlice[tmpIndex]
}

// leet speak map of string slices
var leet = map[string][]string{
	"leet":     []string{"1337"},
	"the":      []string{"teh"},
	"cool":     []string{"kewl"},
	"dude":     []string{"d00d"},
	"you":      []string{"u"},
	"noob":     []string{"n00b"},
	"noobs":    []string{"n00bs"},
	"own":      []string{"pwn"},
	"owned":    []string{"pwned"},
	"rocks":    []string{"roxx0rs"},
	"exploits": []string{"sploitz"},
	"woot":     []string{"w00t"},
	"hacker":   []string{"hax0r"},
	"hackers":  []string{"hax0rz"},
	"a":        []string{"4", "@"},
	"b":        []string{"8", "]3", "]8", "|3", "|8", "13"},
	"c":        []string{"(", "{"},
	"d":        []string{")", "[}", "|)", "|}", "|>"},
	"e":        []string{"3"},
	"f":        []string{"|=", "ph"},
	"g":        []string{"6", "9", "&"},
	"h":        []string{"#", "|-|"},
	"i":        []string{"1", "!", "|"},
	"j":        []string{"_|", "u|"},
	"k":        []string{"|<", "|{"},
	"l":        []string{"|", "1", "|_"},
	"m":        []string{"/\\/\\", "|\\/|"},
	"n":        []string{"/\\/", "|\\|"},
	"o":        []string{"0", "()"},
	"p":        []string{"|D", "|*"},
	"q":        []string{"(,)", "O\\", "[]\\"},
	"r":        []string{"|2", "|?", "][2"},
	"s":        []string{"5", "$"},
	"t":        []string{"7", "+"},
	"u":        []string{"(_)", "|_|"},
	"v":        []string{"\\/", "\\\\//"},
	"w":        []string{"\\/\\/", "|/\\|", "VV"},
	"x":        []string{"><", "}{"},
	"y":        []string{"'/", "%"},
	"z":        []string{"2", "7_"},
}

// irsa conversion map
var irsa = map[string]string{
	" ": " | ",
	"a": "alfa",
	"b": "bravo",
	"c": "charlie",
	"d": "delta",
	"e": "echo",
	"f": "foxtrot",
	"g": "golf",
	"h": "hotel",
	"i": "india",
	"j": "juliet",
	"k": "kilo",
	"l": "lima",
	"m": "mike",
	"n": "november",
	"o": "oscar",
	"p": "papa",
	"q": "quebec",
	"r": "romeo",
	"s": "sierra",
	"t": "tango",
	"u": "uniform",
	"v": "victor",
	"w": "whiskey",
	"x": "x-ray",
	"y": "yankee",
	"z": "zulu",
}

// morese holds conversion chars for Morse Code
var morse = map[string]string{
	"a":  ". _",
	"b":  "_ . . .",
	"c":  "_ . _ .",
	"d":  "_ . .",
	"e":  ".",
	"f":  ". . _ .",
	"g":  "_ _ .",
	"h":  ". . . .",
	"i":  ". .",
	"j":  ". _ _ _",
	"k":  "_ . _",
	"l":  ". _ . .",
	"m":  "_ _",
	"n":  "_ .",
	"o":  "_ _ _",
	"p":  ". _ _ .",
	"q":  "_ _ . _",
	"r":  ". _ .",
	"s":  ". . .",
	"t":  "_",
	"u":  ". . _",
	"v":  ". . . _",
	"w":  ". _ _",
	"x":  "_ . . _",
	"y":  "_ . _ _",
	"z":  "_ _ . .",
	"0":  "_ _ _ _ _",
	"1":  ". _ _ _ _",
	"2":  ". . _ _ _",
	"3":  ". . . _ _",
	"4":  ". . . . _",
	"5":  ". . . . .",
	"6":  "_ . . . .",
	"7":  "_ _ . . .",
	"8":  "_ _ _ . .",
	"9":  "_ _ _ _ .",
	".":  "·–·–·– ",
	",":  "––··–– ",
	"?":  "··––·· ",
	"'":  "·––––· ",
	"!":  "–·–·–– ",
	"/":  "–··–· ",
	"(":  "–·––· ",
	")":  "–·––·– ",
	"&":  "·–··· ",
	":":  "–––··· ",
	";":  "–·–·–· ",
	"=":  "–···– ",
	"+":  "·–·–· ",
	"–":  "–····– ",
	"_":  "··––·– ",
	"\"": "·–··–· ",
	"$":  "···–··– ",
	"@":  "·––·–· ",
}
<commit_msg>adjust random junction<commit_after>// Copyright 2014 Brian J. Downs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package formatifier is a library to easily format strings in a user defined
// and predefined manner.
package formatifier

import (
	"crypto/rand"
	"regexp"
	"strings"
)

const (
	lengthError = "ERROR: String not long enough to convert."
	pirateLink  = "http://www.isithackday.com/arrpi.php?text=%s"
)

// Type to hold user input string.
type formatifier struct {
	theString string
}

// New will create an instance of the String object.
func New(s string) *formatifier { return &formatifier{theString: s} }

// makeLower will turn the user entered string to lower case
func (f *formatifier) makeLower() { f.theString = strings.ToLower(f.theString) }

// removeNonDigits removes any non digit characters from the string.
func (f *formatifier) removeNonDigits() {
	rp := regexp.MustCompile(`\D`)
	f.theString = rp.ReplaceAllString(f.theString, "")
}

// removeNonWordChars removes all non word characters.
func (f *formatifier) removeNonWordChars() {
	if len(f.theString) > 0 {
		rp := regexp.MustCompile(`\W|\s|_`)
		f.theString = rp.ReplaceAllString(f.theString, "")
	}
}

// urlEncodeSpaces will replace spaces with "%20"'s
func (f *formatifier) urlEncodeSpaces() {
	rp := regexp.MustCompile(`\s`)
	f.theString = rp.ReplaceAllString(f.theString, "%20")
}

// random select will return a random selection from an int slice
func randomSelect(a []int) int {
	var tmpIndex int
	length := len(a)
	randBytes := make([]byte, length)
	if _, err := rand.Read(randBytes); err == nil {
		tmpIndex = int(randBytes[0]) % length
	}
	return a[tmpIndex]
}

// leet speak map of string slices
var leet = map[string][]string{
	"leet":     []string{"1337"},
	"the":      []string{"teh"},
	"cool":     []string{"kewl"},
	"dude":     []string{"d00d"},
	"you":      []string{"u"},
	"noob":     []string{"n00b"},
	"noobs":    []string{"n00bs"},
	"own":      []string{"pwn"},
	"owned":    []string{"pwned"},
	"rocks":    []string{"roxx0rs"},
	"exploits": []string{"sploitz"},
	"woot":     []string{"w00t"},
	"hacker":   []string{"hax0r"},
	"hackers":  []string{"hax0rz"},
	"a":        []string{"4", "@"},
	"b":        []string{"8", "]3", "]8", "|3", "|8", "13"},
	"c":        []string{"(", "{"},
	"d":        []string{")", "[}", "|)", "|}", "|>"},
	"e":        []string{"3"},
	"f":        []string{"|=", "ph"},
	"g":        []string{"6", "9", "&"},
	"h":        []string{"#", "|-|"},
	"i":        []string{"1", "!", "|"},
	"j":        []string{"_|", "u|"},
	"k":        []string{"|<", "|{"},
	"l":        []string{"|", "1", "|_"},
	"m":        []string{"/\\/\\", "|\\/|"},
	"n":        []string{"/\\/", "|\\|"},
	"o":        []string{"0", "()"},
	"p":        []string{"|D", "|*"},
	"q":        []string{"(,)", "O\\", "[]\\"},
	"r":        []string{"|2", "|?", "][2"},
	"s":        []string{"5", "$"},
	"t":        []string{"7", "+"},
	"u":        []string{"(_)", "|_|"},
	"v":        []string{"\\/", "\\\\//"},
	"w":        []string{"\\/\\/", "|/\\|", "VV"},
	"x":        []string{"><", "}{"},
	"y":        []string{"'/", "%"},
	"z":        []string{"2", "7_"},
}

// irsa conversion map
var irsa = map[string]string{
	" ": " | ",
	"a": "alfa",
	"b": "bravo",
	"c": "charlie",
	"d": "delta",
	"e": "echo",
	"f": "foxtrot",
	"g": "golf",
	"h": "hotel",
	"i": "india",
	"j": "juliet",
	"k": "kilo",
	"l": "lima",
	"m": "mike",
	"n": "november",
	"o": "oscar",
	"p": "papa",
	"q": "quebec",
	"r": "romeo",
	"s": "sierra",
	"t": "tango",
	"u": "uniform",
	"v": "victor",
	"w": "whiskey",
	"x": "x-ray",
	"y": "yankee",
	"z": "zulu",
}

// morese holds conversion chars for Morse Code
var morse = map[string]string{
	"a":  ". _",
	"b":  "_ . . .",
	"c":  "_ . _ .",
	"d":  "_ . .",
	"e":  ".",
	"f":  ". . _ .",
	"g":  "_ _ .",
	"h":  ". . . .",
	"i":  ". .",
	"j":  ". _ _ _",
	"k":  "_ . _",
	"l":  ". _ . .",
	"m":  "_ _",
	"n":  "_ .",
	"o":  "_ _ _",
	"p":  ". _ _ .",
	"q":  "_ _ . _",
	"r":  ". _ .",
	"s":  ". . .",
	"t":  "_",
	"u":  ". . _",
	"v":  ". . . _",
	"w":  ". _ _",
	"x":  "_ . . _",
	"y":  "_ . _ _",
	"z":  "_ _ . .",
	"0":  "_ _ _ _ _",
	"1":  ". _ _ _ _",
	"2":  ". . _ _ _",
	"3":  ". . . _ _",
	"4":  ". . . . _",
	"5":  ". . . . .",
	"6":  "_ . . . .",
	"7":  "_ _ . . .",
	"8":  "_ _ _ . .",
	"9":  "_ _ _ _ .",
	".":  "·–·–·– ",
	",":  "––··–– ",
	"?":  "··––·· ",
	"'":  "·––––· ",
	"!":  "–·–·–– ",
	"/":  "–··–· ",
	"(":  "–·––· ",
	")":  "–·––·– ",
	"&":  "·–··· ",
	":":  "–––··· ",
	";":  "–·–·–· ",
	"=":  "–···– ",
	"+":  "·–·–· ",
	"–":  "–····– ",
	"_":  "··––·– ",
	"\"": "·–··–· ",
	"$":  "···–··– ",
	"@":  "·––·–· ",
}
<|endoftext|>


<commit_before>package support

// Monolithic Message-Oriented Application (MMOA)
// Trash
// Copyright © 2016 Eduard Sesigin. All rights reserved. Contacts: <claygod@yandex.ru>

//import "fmt"
import "github.com/claygod/mmoa/tools"
import "github.com/claygod/mmoa/service"

// NewTrash - create a new Trash
func NewTrash(chIn chan *tools.Message, chBus chan *tools.Message) *Trash {
	The := tools.NewThemes()
	s := &Trash{
		service.NewService(The.Service.Trash, chIn, chBus),
		service.NewLogger(),
	}
	s.MethodWork = s.toLog
	s.Start()
	return s
}

// Trash structure
type Trash struct {
	service.Service
	logger *service.Logger
}

func (s *Trash) toLog(msg7 *tools.Message) {
	if msg7.AddsFrom == s.The.Service.Aggregator {
		errAgg := msg7.MsgCtx[s.The.Attach.Aggregate].(*service.Aggregate)
		s.logger.Message().
			Field("service", s.Name).
			Field("resurce", s.The.Attach.Aggregate).
			Field("event", msg7.MsgTheme).
			Field("cid", msg7.MsgCid).
			Field("reporter", msg7.AddsFrom).
			Field("attach", errAgg.Messages).
			Send()
	} else {
		errMsg := msg7.MsgCtx[s.The.Attach.Message].(*tools.Message)
		s.logger.Message().
			Field("service", s.Name).
			Field("resurce", s.The.Attach.Message).
			Field("event", msg7.MsgTheme).
			Field("cid", msg7.MsgCid).
			Field("reporter", msg7.AddsFrom).
			Field("attach", errMsg).
			Send()

	}
}
<commit_msg>Delete trash.go<commit_after><|endoftext|>


<commit_before>package qesygo

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type RandWeiht struct {
	Name   string
	Weight int
}

type RandWeihtArr []RandWeiht

func Substr(str string, start int, end int) string {
	var endNum int
	s := []byte(str)
	if end > 0 {
		endNum = start + end
	} else {
		endNum = len(str) + end
	}
	return string(s[start:endNum])
}

func Rand(Min int, Max int) int {
	tempNum := Max - Min
	if tempNum <= 0 {
		return Min
	}
	randSeek := time.Now().UnixNano()
	rand.Seed(randSeek)
	time.Sleep(time.Nanosecond * 1)
	return Min + rand.Intn(tempNum)
}

func Rate(num int) bool {
	rand := Rand(1, 100)
	if rand <= num {
		return true
	} else {
		return false
	}
}

/*
* RandWeihtArr := &lib.RandWeihtArr{{"user1",8}, {"user2",1},{"user3",1}}
* who := RandWeihtArr.RandWeight()
 */
func (arr *RandWeihtArr) RandWeight() string {
	var all int
	for _, v := range *arr {
		all += v.Weight
	}
	plusNum := 0
	tempArr := make(map[string][2]int)
	for _, v := range *arr {
		plusNum += v.Weight
		tempArr[v.Name] = [2]int{plusNum - v.Weight, plusNum}
	}
	randNum := Rand(0, all) + 1
	var ret string
	for k, v := range tempArr {
		if randNum > v[0] && randNum <= v[1] {
			ret = k
			break
		}
	}
	return ret
}

func ReadFile(str string) ([]byte, error) {
	return ioutil.ReadFile(str)
}

func JsonEncode(arr interface{}) ([]byte, error) {
	return json.Marshal(arr)
}

func JsonDecode(str []byte, jsonArr interface{}) error {
	strNew := string(str)
	if strNew == "null" || strNew == "" {
		return nil
	}
	err := json.Unmarshal(str, jsonArr)
	return err

}

func Printf(format string, a ...interface{}) {
	fmt.Printf(format, a)
}

func Fprintf(w http.ResponseWriter, str string) {
	fmt.Fprintf(w, str)
}

func Die(v interface{}) {
	log.Fatal(v)
}

func Implode(arr []string, sep string) string {
	return strings.Join(arr, sep)
}

func Explode(str string, sep string) []string {
	if str == "" {
		return []string{}
	}
	return strings.Split(str, sep)
}

func Err(str string) error {
	return errors.New(str)
}

func Println(str ...interface{}) {
	fmt.Println(str)
}

func Time(str string) int64 {
	now := time.Now()
	t := now.UnixNano()
	switch str {
	case "Microsecond":
		t = now.UnixNano() / 1000
	case "Millisecond":
		t = now.UnixNano() / 1000000
	case "Second":
		t = now.UnixNano() / 1000000000
	}
	return t
}

func TimeStr(str string) string {
	t := Time(str)
	return strconv.FormatInt(t, 10)
}

func TimeInt(str string) int {
	t := Time(str)
	ret, _ := Int64ToInt(t)
	return ret
}

//-- format : "2006-01-02 03:04:05 PM" --
/*
月份 1,01,Jan,January
日　 2,02,_2
时　 3,03,15,PM,pm,AM,am
分　 4,04
秒　 5,05
年　 06,2006
周几 Mon,Monday
时区时差表示 -07,-0700,Z0700,Z07:00,-07:00,MST
时区字母缩写 MST
*/
func Date(timestamp int64, format string) string {
	tm := time.Unix(timestamp, 0)
	return tm.Format(format)
}

func DateTimeGet() int { //获取当天0点时间戳
	t := time.Now()
	timeStr := t.Format("2006-01-02")
	t, _ = time.ParseInLocation("2006-01-02", timeStr, time.Now().Location())
	return int(t.Unix())
}

//-- "01/02/2006", "02/08/2015" --
func StrToTimeByDate(format string, input string) int64 {
	tm2, _ := time.ParseInLocation(format, input, time.Now().Location())
	return tm2.Unix()
}

func StrToTime(format string, input string) int64 {
	loc, _ := time.LoadLocation("Local")
	theTime, _ := time.ParseInLocation(format, input, loc)
	return theTime.Unix()
}

func Int64ToInt(num int64) (int, error) {
	str := strconv.FormatInt(num, 10)
	return strconv.Atoi(str)
}

func IntToStr(a interface{}) string {
	return fmt.Sprintf("%d", a)
}

func StrToInt(str string) int {
	ret, _ := strconv.Atoi(str)
	return ret
}

func StrToInt32(str string) int32 {
	return int32(StrToInt(str))
}

func StrToInt64(str string) int64 {
	ret, _ := strconv.ParseInt(str, 10, 64)
	return ret
}

func Unset(arr []string, str string) []string {
	newArr := []string{}
	for _, v := range arr {
		if v != str && v != "" {
			newArr = append(newArr, v)
		}
	}
	return newArr
}

//-- kind 0:纯数字，1：小写，2：大写，3：数字+大小写字幕 --
func Krand(size int, kind int) []byte {
	ikind, kinds, result := kind, [][]int{[]int{10, 48}, []int{26, 97}, []int{26, 65}}, make([]byte, size)
	is_all := kind > 2 || kind < 0
	for i := 0; i < size; i++ {
		if is_all { // random ikind
			ikind = Rand(1, 3)
		}
		scope, base := kinds[ikind][0], kinds[ikind][1]
		result[i] = uint8(base + Rand(1, scope))
	}
	return result
}

func Wordwrap(Str string, width int, breakStr string) string {
	breakStrByte := []byte(breakStr)
	strByte := []byte(Str)
	var tempByte []byte
	for k, v := range strByte {
		if (k+1)%width == 0 && k != len(strByte)-1 {
			tempByte = append(tempByte, v)
			tempByte = append(tempByte, breakStrByte...)
		} else {
			tempByte = append(tempByte, v)
		}
	}
	return string(tempByte)
}

func Base64Encode(json string) string {
	b64 := base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/")
	return b64.EncodeToString([]byte(json))
}

func Base64Decode(json string) ([]byte, error) {
	b64 := base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/")
	return b64.DecodeString(json)
}

func Abs(a int) (ret int) {
	ret = (a ^ a>>31) - a>>31
	return
}

func Md5(str string) string {
	h := md5.New()
	h.Write([]byte(str))
	return hex.EncodeToString(h.Sum(nil))
}

func IntToBytes(x int32) []byte {
	b_buf := bytes.NewBuffer([]byte{})
	binary.Write(b_buf, binary.LittleEndian, x)
	return b_buf.Bytes()
}

func BytesToInt(b []byte) int32 {
	var x int32
	b_buf := bytes.NewBuffer(b)
	binary.Read(b_buf, binary.LittleEndian, &x)
	return x
}
<commit_msg>add veri<commit_after>package qesygo

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type RandWeiht struct {
	Name   string
	Weight int
}

type RandWeihtArr []RandWeiht

func Substr(str string, start int, end int) string {
	var endNum int
	s := []byte(str)
	if end > 0 {
		endNum = start + end
	} else {
		endNum = len(str) + end
	}
	return string(s[start:endNum])
}

func Rand(Min int, Max int) int {
	tempNum := Max - Min
	if tempNum <= 0 {
		return Min
	}
	randSeek := time.Now().UnixNano()
	rand.Seed(randSeek)
	time.Sleep(time.Nanosecond * 1)
	return Min + rand.Intn(tempNum)
}

func Rate(num int) bool {
	rand := Rand(1, 100)
	if rand <= num {
		return true
	} else {
		return false
	}
}

/*
* RandWeihtArr := &lib.RandWeihtArr{{"user1",8}, {"user2",1},{"user3",1}}
* who := RandWeihtArr.RandWeight()
 */
func (arr *RandWeihtArr) RandWeight() string {
	var all int
	for _, v := range *arr {
		all += v.Weight
	}
	plusNum := 0
	tempArr := make(map[string][2]int)
	for _, v := range *arr {
		plusNum += v.Weight
		tempArr[v.Name] = [2]int{plusNum - v.Weight, plusNum}
	}
	randNum := Rand(0, all) + 1
	var ret string
	for k, v := range tempArr {
		if randNum > v[0] && randNum <= v[1] {
			ret = k
			break
		}
	}
	return ret
}

func VeriPara(Req map[string]string, Para []string) bool {
	for _, val := range Para {
		if Str, ok := Req[val]; !ok || Str == "" {
			return false
		}
	}
	return true
}

func ReadFile(str string) ([]byte, error) {
	return ioutil.ReadFile(str)
}

func JsonEncode(arr interface{}) ([]byte, error) {
	return json.Marshal(arr)
}

func JsonDecode(str []byte, jsonArr interface{}) error {
	strNew := string(str)
	if strNew == "null" || strNew == "" {
		return nil
	}
	err := json.Unmarshal(str, jsonArr)
	return err

}

func Printf(format string, a ...interface{}) {
	fmt.Printf(format, a)
}

func Fprintf(w http.ResponseWriter, str string) {
	fmt.Fprintf(w, str)
}

func Die(v interface{}) {
	log.Fatal(v)
}

func Implode(arr []string, sep string) string {
	return strings.Join(arr, sep)
}

func Explode(str string, sep string) []string {
	if str == "" {
		return []string{}
	}
	return strings.Split(str, sep)
}

func Err(str string) error {
	return errors.New(str)
}

func Println(str ...interface{}) {
	fmt.Println(str)
}

func Time(str string) int64 {
	now := time.Now()
	t := now.UnixNano()
	switch str {
	case "Microsecond":
		t = now.UnixNano() / 1000
	case "Millisecond":
		t = now.UnixNano() / 1000000
	case "Second":
		t = now.UnixNano() / 1000000000
	}
	return t
}

func TimeStr(str string) string {
	t := Time(str)
	return strconv.FormatInt(t, 10)
}

func TimeInt(str string) int {
	t := Time(str)
	ret, _ := Int64ToInt(t)
	return ret
}

//-- format : "2006-01-02 03:04:05 PM" --
/*
月份 1,01,Jan,January
日　 2,02,_2
时　 3,03,15,PM,pm,AM,am
分　 4,04
秒　 5,05
年　 06,2006
周几 Mon,Monday
时区时差表示 -07,-0700,Z0700,Z07:00,-07:00,MST
时区字母缩写 MST
*/
func Date(timestamp int64, format string) string {
	tm := time.Unix(timestamp, 0)
	return tm.Format(format)
}

func DateTimeGet() int { //获取当天0点时间戳
	t := time.Now()
	timeStr := t.Format("2006-01-02")
	t, _ = time.ParseInLocation("2006-01-02", timeStr, time.Now().Location())
	return int(t.Unix())
}

//-- "01/02/2006", "02/08/2015" --
func StrToTimeByDate(format string, input string) int64 {
	tm2, _ := time.ParseInLocation(format, input, time.Now().Location())
	return tm2.Unix()
}

func StrToTime(format string, input string) int64 {
	loc, _ := time.LoadLocation("Local")
	theTime, _ := time.ParseInLocation(format, input, loc)
	return theTime.Unix()
}

func Int64ToInt(num int64) (int, error) {
	str := strconv.FormatInt(num, 10)
	return strconv.Atoi(str)
}

func IntToStr(a interface{}) string {
	return fmt.Sprintf("%d", a)
}

func StrToInt(str string) int {
	ret, _ := strconv.Atoi(str)
	return ret
}

func StrToInt32(str string) int32 {
	return int32(StrToInt(str))
}

func StrToInt64(str string) int64 {
	ret, _ := strconv.ParseInt(str, 10, 64)
	return ret
}

func Unset(arr []string, str string) []string {
	newArr := []string{}
	for _, v := range arr {
		if v != str && v != "" {
			newArr = append(newArr, v)
		}
	}
	return newArr
}

//-- kind 0:纯数字，1：小写，2：大写，3：数字+大小写字幕 --
func Krand(size int, kind int) []byte {
	ikind, kinds, result := kind, [][]int{[]int{10, 48}, []int{26, 97}, []int{26, 65}}, make([]byte, size)
	is_all := kind > 2 || kind < 0
	for i := 0; i < size; i++ {
		if is_all { // random ikind
			ikind = Rand(1, 3)
		}
		scope, base := kinds[ikind][0], kinds[ikind][1]
		result[i] = uint8(base + Rand(1, scope))
	}
	return result
}

func Wordwrap(Str string, width int, breakStr string) string {
	breakStrByte := []byte(breakStr)
	strByte := []byte(Str)
	var tempByte []byte
	for k, v := range strByte {
		if (k+1)%width == 0 && k != len(strByte)-1 {
			tempByte = append(tempByte, v)
			tempByte = append(tempByte, breakStrByte...)
		} else {
			tempByte = append(tempByte, v)
		}
	}
	return string(tempByte)
}

func Base64Encode(json string) string {
	b64 := base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/")
	return b64.EncodeToString([]byte(json))
}

func Base64Decode(json string) ([]byte, error) {
	b64 := base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/")
	return b64.DecodeString(json)
}

func Abs(a int) (ret int) {
	ret = (a ^ a>>31) - a>>31
	return
}

func Md5(str string) string {
	h := md5.New()
	h.Write([]byte(str))
	return hex.EncodeToString(h.Sum(nil))
}

func IntToBytes(x int32) []byte {
	b_buf := bytes.NewBuffer([]byte{})
	binary.Write(b_buf, binary.LittleEndian, x)
	return b_buf.Bytes()
}

func BytesToInt(b []byte) int32 {
	var x int32
	b_buf := bytes.NewBuffer(b)
	binary.Read(b_buf, binary.LittleEndian, &x)
	return x
}
<|endoftext|>


<commit_before>package ts

import "io"
import "log"
import "errors"
import "github.com/32bitkid/bitreader"

func NewReader(reader io.Reader) TransportStreamReader {
	br := bitreader.NewReader32(reader)
	return &tsReader{br}
}

type TransportStreamReader interface {
	Next() (*TsPacket, error)
}

type tsReader struct {
	bitreader.Reader32
}

const SyncByte = 0x47

type TsPacket struct {
	TransportErrorIndicator    bool
	PayloadUnitStartIndicator  bool
	TransportPrriority         bool
	PID                        uint32
	TransportScramblingControl uint32
	AdaptationFieldControl     uint32
	ContinuityCounter          uint32
	Payload                    []byte
}

func isFatalErr(err error) bool {
	return err != nil && err != io.EOF
}

func (tsr *tsReader) Next() (*TsPacket, error) {

	var err error

	aligned, err := tsr.isAligned()
	if isFatalErr(err) {
		return nil, err
	}

	if !aligned {
		err = tsr.realign()
		if isFatalErr(err) {
			return nil, err
		}
	}

	if err = tsr.Trash(8); isFatalErr(err) {
		return nil, err
	}

	packet := TsPacket{}

	packet.TransportErrorIndicator, err = tsr.ReadBit()
	if isFatalErr(err) {
		return nil, err
	}

	packet.PayloadUnitStartIndicator, err = tsr.ReadBit()
	if isFatalErr(err) {
		return nil, err
	}

	packet.TransportPrriority, err = tsr.ReadBit()
	if isFatalErr(err) {
		return nil, err
	}

	packet.PID, err = tsr.Read32(13)
	if isFatalErr(err) {
		return nil, err
	}

	packet.TransportScramblingControl, err = tsr.Read32(2)
	if isFatalErr(err) {
		return nil, err
	}

	packet.AdaptationFieldControl, err = tsr.Read32(2)
	if isFatalErr(err) {
		return nil, err
	}

	packet.ContinuityCounter, err = tsr.Read32(4)
	if isFatalErr(err) {
		return nil, err
	}

	// TODO handle adaptation field

	packet.Payload = make([]byte, 184)

	var val uint32
	for i := 0; i < 184; i++ {
		val, err = tsr.Read32(8)
		if isFatalErr(err) {
			return nil, err
		}
		packet.Payload[i] = byte(val)
	}

	return &packet, nil
}

func (tsr *tsReader) isAligned() (bool, error) {
	val, err := tsr.Peek32(8)
	return val == SyncByte, err
}

func (tsr *tsReader) realign() error {
	log.Printf("Attempting to realign")
	for i := 0; i < 188; i++ {
		tsr.Trash(8)
		isAligned, err := tsr.isAligned()
		if err != nil {
			return err
		}
		if isAligned {
			log.Printf("Realigned after %d bytes.\n", i)
			return nil
		}
	}
	return errors.New("Unable to find SyncByte")
}
<commit_msg>Better reasoning about missing sync bytes<commit_after>package ts

import "io"
import "log"
import "errors"
import "github.com/32bitkid/bitreader"

func NewReader(reader io.Reader) TransportStreamReader {
	br := bitreader.NewReader32(reader)
	return &tsReader{br}
}

type TransportStreamReader interface {
	Next() (*TsPacket, error)
}

type tsReader struct {
	bitreader.Reader32
}

const SyncByte = 0x47

var ErrNoSyncByte = errors.New("no sync byte")

type TsPacket struct {
	TransportErrorIndicator    bool
	PayloadUnitStartIndicator  bool
	TransportPrriority         bool
	PID                        uint32
	TransportScramblingControl uint32
	AdaptationFieldControl     uint32
	ContinuityCounter          uint32
	Payload                    []byte
}

func isFatalErr(err error) bool {
	return err != nil && err != io.EOF
}

func (tsr *tsReader) Next() (*TsPacket, error) {

	var err error

	aligned, err := tsr.isAligned()
	if isFatalErr(err) {
		return nil, err
	}

	if !aligned {
		err = tsr.realign()
		if err != nil {
			return nil, ErrNoSyncByte
		}
	}

	err = tsr.Trash(8)
	if isFatalErr(err) {
		return nil, err
	}

	packet := TsPacket{}

	packet.TransportErrorIndicator, err = tsr.ReadBit()
	if isFatalErr(err) {
		return nil, err
	}

	packet.PayloadUnitStartIndicator, err = tsr.ReadBit()
	if isFatalErr(err) {
		return nil, err
	}

	packet.TransportPrriority, err = tsr.ReadBit()
	if isFatalErr(err) {
		return nil, err
	}

	packet.PID, err = tsr.Read32(13)
	if isFatalErr(err) {
		return nil, err
	}

	packet.TransportScramblingControl, err = tsr.Read32(2)
	if isFatalErr(err) {
		return nil, err
	}

	packet.AdaptationFieldControl, err = tsr.Read32(2)
	if isFatalErr(err) {
		return nil, err
	}

	packet.ContinuityCounter, err = tsr.Read32(4)
	if isFatalErr(err) {
		return nil, err
	}

	// TODO handle adaptation field

	packet.Payload = make([]byte, 184)

	var val uint32
	for i := 0; i < 184; i++ {
		val, err = tsr.Read32(8)
		if isFatalErr(err) {
			return nil, err
		}
		packet.Payload[i] = byte(val)
	}

	return &packet, nil
}

func (tsr *tsReader) isAligned() (bool, error) {
	val, err := tsr.Peek32(8)
	return val == SyncByte, err
}

func (tsr *tsReader) realign() error {
	log.Printf("Attempting to realign")
	for i := 0; i < 188; i++ {
		if err := tsr.Trash(8); isFatalErr(err) {
			return err
		}
		isAligned, err := tsr.isAligned()
		if isFatalErr(err) {
			return err
		}
		if isAligned {
			log.Printf("Realigned after %d bytes.\n", i)
			return nil
		}
	}
	return ErrNoSyncByte
}
<|endoftext|>


<commit_before>package goose

import (
	"regexp"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

var normalizeWhitespaceRegexp = regexp.MustCompile(`[ \r\f\v\t]+`)
var normalizeNl = regexp.MustCompile(`[\n]+`)

type outputFormatter struct {
	topNode  *goquery.Selection
	config   Configuration
	language string
}

func (formatter *outputFormatter) getLanguage(lang string) string {
	if formatter.config.useMetaLanguage && "" != lang {
		return lang
	}
	return formatter.config.targetLanguage
}

func (formatter *outputFormatter) getTopNode() *goquery.Selection {
	return formatter.topNode
}

func (formatter *outputFormatter) getFormattedText(topNode *goquery.Selection, lang string) (output string, links []string) {
	formatter.topNode = topNode
	formatter.language = formatter.getLanguage(lang)
	if formatter.language == "" {
		formatter.language = formatter.config.targetLanguage
	}
	formatter.removeNegativescoresNodes()
	links = formatter.linksToText()
	formatter.replaceTagsWithText()
	formatter.removeParagraphsWithFewWords()
	output = formatter.getOutputText()
	return output, links
}

func (formatter *outputFormatter) convertToText() string {
	var txts []string
	selections := formatter.topNode
	selections.Each(func(i int, s *goquery.Selection) {
		txt := s.Text()
		if txt != "" {
			// txt = txt //unescape
			txtLis := strings.Trim(txt, "\n")
			txts = append(txts, txtLis)
		}
	})
	return strings.Join(txts, "\n\n")
}

func (formatter *outputFormatter) linksToText() []string {
	var urlList []string
	links := formatter.topNode.Find("a")
	links.Each(func(i int, a *goquery.Selection) {
		imgs := a.Find("img")
		if imgs.Length() == 0 {
			node := a.Get(0)
			node.Data = a.Text()
			node.Type = html.TextNode
			// save a list of URLs
			url, _ := a.Attr("href")
			isValidURL, _ := regexp.MatchString("^http[s]?://", url)
			if isValidURL {
				urlList = append(urlList, url)
			}
		}
	})
	return urlList
}

func (formatter *outputFormatter) getOutputText() string {

	out := formatter.topNode.Text()
	out = normalizeWhitespaceRegexp.ReplaceAllString(out, " ")

	strArr := strings.Split(out, "\n")
	resArr := []string{}

	for i, v := range strArr {
		v = strings.TrimSpace(v)
		if v != "" {
			resArr = append(resArr, v)
		} else if i > 2 && strArr[i-2] != "" {
			resArr = append(resArr, "")
		}
	}

	out = strings.Join(resArr, "\n")
	out = normalizeNl.ReplaceAllString(out, "\n\n")

	out = strings.TrimSpace(out)
	return out
}

func (formatter *outputFormatter) removeNegativescoresNodes() {
	gravityItems := formatter.topNode.Find("*[gravityScore]")
	gravityItems.Each(func(i int, s *goquery.Selection) {
		score := 0
		sscore, exists := s.Attr("gravityScore")
		if exists {
			score, _ = strconv.Atoi(sscore)
			if score < 1 {
				sNode := s.Get(0)
				sNode.Parent.RemoveChild(sNode)
			}
		}

	})
}

func (formatter *outputFormatter) replaceTagsWithText() {
	strongs := formatter.topNode.Find("strong")
	strongs.Each(func(i int, strong *goquery.Selection) {
		text := strong.Text()
		node := strong.Get(0)
		node.Type = html.TextNode
		node.Data = text
	})

	bolds := formatter.topNode.Find("b")
	bolds.Each(func(i int, bold *goquery.Selection) {
		text := bold.Text()
		node := bold.Get(0)
		node.Type = html.TextNode
		node.Data = text
	})

	italics := formatter.topNode.Find("i")
	italics.Each(func(i int, italic *goquery.Selection) {
		text := italic.Text()
		node := italic.Get(0)
		node.Type = html.TextNode
		node.Data = text
	})
}

func (formatter *outputFormatter) removeParagraphsWithFewWords() {
	language := formatter.language
	if language == "" {
		language = "en"
	}
	allNodes := formatter.topNode.Children()
	allNodes.Each(func(i int, s *goquery.Selection) {
		sw := formatter.config.stopWords.stopWordsCount(language, s.Text())
		if sw.wordCount < 5 && s.Find("object").Length() == 0 && s.Find("em").Length() == 0 {
			node := s.Get(0)
			node.Parent.RemoveChild(node)
		}
	})

}
<commit_msg>pre-compile regexp<commit_after>package goose

import (
	"regexp"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

var normalizeWhitespaceRegexp = regexp.MustCompile(`[ \r\f\v\t]+`)
var normalizeNl = regexp.MustCompile(`[\n]+`)
var validURLRegex = regexp.MustCompile("^http[s]?://")

type outputFormatter struct {
	topNode  *goquery.Selection
	config   Configuration
	language string
}

func (formatter *outputFormatter) getLanguage(lang string) string {
	if formatter.config.useMetaLanguage && "" != lang {
		return lang
	}
	return formatter.config.targetLanguage
}

func (formatter *outputFormatter) getTopNode() *goquery.Selection {
	return formatter.topNode
}

func (formatter *outputFormatter) getFormattedText(topNode *goquery.Selection, lang string) (output string, links []string) {
	formatter.topNode = topNode
	formatter.language = formatter.getLanguage(lang)
	if formatter.language == "" {
		formatter.language = formatter.config.targetLanguage
	}
	formatter.removeNegativescoresNodes()
	links = formatter.linksToText()
	formatter.replaceTagsWithText()
	formatter.removeParagraphsWithFewWords()
	output = formatter.getOutputText()
	return output, links
}

func (formatter *outputFormatter) convertToText() string {
	var txts []string
	selections := formatter.topNode
	selections.Each(func(i int, s *goquery.Selection) {
		txt := s.Text()
		if txt != "" {
			// txt = txt //unescape
			txtLis := strings.Trim(txt, "\n")
			txts = append(txts, txtLis)
		}
	})
	return strings.Join(txts, "\n\n")
}

// check if this is a valid URL
func isValidURL(u string) bool {
	return validURLRegex.MatchString(u)
}

func (formatter *outputFormatter) linksToText() []string {
	var urlList []string
	links := formatter.topNode.Find("a")
	links.Each(func(i int, a *goquery.Selection) {
		imgs := a.Find("img")
		if imgs.Length() == 0 {
			node := a.Get(0)
			node.Data = a.Text()
			node.Type = html.TextNode
			// save a list of URLs
			url, _ := a.Attr("href")
			if isValidURL(url) {
				urlList = append(urlList, url)
			}
		}
	})
	return urlList
}

func (formatter *outputFormatter) getOutputText() string {

	out := formatter.topNode.Text()
	out = normalizeWhitespaceRegexp.ReplaceAllString(out, " ")

	strArr := strings.Split(out, "\n")
	resArr := []string{}

	for i, v := range strArr {
		v = strings.TrimSpace(v)
		if v != "" {
			resArr = append(resArr, v)
		} else if i > 2 && strArr[i-2] != "" {
			resArr = append(resArr, "")
		}
	}

	out = strings.Join(resArr, "\n")
	out = normalizeNl.ReplaceAllString(out, "\n\n")

	out = strings.TrimSpace(out)
	return out
}

func (formatter *outputFormatter) removeNegativescoresNodes() {
	gravityItems := formatter.topNode.Find("*[gravityScore]")
	gravityItems.Each(func(i int, s *goquery.Selection) {
		score := 0
		sscore, exists := s.Attr("gravityScore")
		if exists {
			score, _ = strconv.Atoi(sscore)
			if score < 1 {
				sNode := s.Get(0)
				sNode.Parent.RemoveChild(sNode)
			}
		}

	})
}

func (formatter *outputFormatter) replaceTagsWithText() {
	strongs := formatter.topNode.Find("strong")
	strongs.Each(func(i int, strong *goquery.Selection) {
		text := strong.Text()
		node := strong.Get(0)
		node.Type = html.TextNode
		node.Data = text
	})

	bolds := formatter.topNode.Find("b")
	bolds.Each(func(i int, bold *goquery.Selection) {
		text := bold.Text()
		node := bold.Get(0)
		node.Type = html.TextNode
		node.Data = text
	})

	italics := formatter.topNode.Find("i")
	italics.Each(func(i int, italic *goquery.Selection) {
		text := italic.Text()
		node := italic.Get(0)
		node.Type = html.TextNode
		node.Data = text
	})
}

func (formatter *outputFormatter) removeParagraphsWithFewWords() {
	language := formatter.language
	if language == "" {
		language = "en"
	}
	allNodes := formatter.topNode.Children()
	allNodes.Each(func(i int, s *goquery.Selection) {
		sw := formatter.config.stopWords.stopWordsCount(language, s.Text())
		if sw.wordCount < 5 && s.Find("object").Length() == 0 && s.Find("em").Length() == 0 {
			node := s.Get(0)
			node.Parent.RemoveChild(node)
		}
	})

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/url"
	"strconv"
	"strings"

	"github.com/agl/xmpp-client/xmpp"
	"golang.org/x/crypto/otr"
	"golang.org/x/crypto/ssh/terminal"
	"golang.org/x/net/proxy"
)

type Config struct {
	filename                      string `json:"-"`
	Account                       string
	Server                        string   `json:",omitempty"`
	Proxies                       []string `json:",omitempty"`
	Password                      string   `json:",omitempty"`
	Port                          int      `json:",omitempty"`
	PrivateKey                    []byte
	KnownFingerprints             []KnownFingerprint
	RawLogFile                    string   `json:",omitempty"`
	NotifyCommand                 []string `json:",omitempty"`
	IdleSecondsBeforeNotification int      `json:",omitempty"`
	Bell                          bool
	HideStatusUpdates             bool
	UseTor                        bool
	OTRAutoTearDown               bool
	OTRAutoAppendTag              bool
	OTRAutoStartSession           bool
	ServerCertificateSHA256       string   `json:",omitempty"`
	AlwaysEncrypt                 bool     `json:",omitempty"`
	AlwaysEncryptWith             []string `json:",omitempty"`
}

type KnownFingerprint struct {
	UserId         string
	FingerprintHex string
	fingerprint    []byte `json:"-"`
}

func ParseConfig(filename string) (c *Config, err error) {
	contents, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	c = new(Config)
	if err = json.Unmarshal(contents, &c); err != nil {
		return
	}

	c.filename = filename

	for i, known := range c.KnownFingerprints {
		c.KnownFingerprints[i].fingerprint, err = hex.DecodeString(known.FingerprintHex)
		if err != nil {
			err = errors.New("xmpp: failed to parse hex fingerprint for " + known.UserId + ": " + err.Error())
			return
		}
	}

	return
}

func (c *Config) Save() error {
	for i, known := range c.KnownFingerprints {
		c.KnownFingerprints[i].FingerprintHex = hex.EncodeToString(known.fingerprint)
	}

	contents, err := json.MarshalIndent(c, "", "\t")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(c.filename, contents, 0600)
}

func (c *Config) UserIdForFingerprint(fpr []byte) string {
	for _, known := range c.KnownFingerprints {
		if bytes.Equal(fpr, known.fingerprint) {
			return known.UserId
		}
	}

	return ""
}

func (c *Config) HasFingerprint(uid string) bool {
	for _, known := range c.KnownFingerprints {
		if uid == known.UserId {
			return true
		}
	}

	return false
}

func (c *Config) ShouldEncryptTo(uid string) bool {
	if c.AlwaysEncrypt {
		return true
	}

	for _, contact := range c.AlwaysEncryptWith {
		if contact == uid {
			return true
		}
	}
	return false
}

func enroll(config *Config, term *terminal.Terminal) bool {
	var err error
	warn(term, "Enrolling new config file")

	var domain string
	for {
		term.SetPrompt("Account (i.e. user@example.com, enter to quit): ")
		if config.Account, err = term.ReadLine(); err != nil || len(config.Account) == 0 {
			return false
		}

		parts := strings.SplitN(config.Account, "@", 2)
		if len(parts) != 2 {
			alert(term, "invalid username (want user@domain): "+config.Account)
			continue
		}
		domain = parts[1]
		break
	}

	term.SetPrompt("Enable debug logging to /tmp/xmpp-client-debug.log? ")
	if debugLog, err := term.ReadLine(); err != nil || debugLog != "yes" {
		info(term, "Not enabling debug logging...")
	} else {
		info(term, "Debug logging enabled...")
		config.RawLogFile = "/tmp/xmpp-client-debug.log"
	}

	term.SetPrompt("Use Tor?: ")
	if useTorQuery, err := term.ReadLine(); err != nil || len(useTorQuery) == 0 || useTorQuery[0] != 'y' && useTorQuery[0] != 'Y' {
		info(term, "Not using Tor...")
		config.UseTor = false
	} else {
		info(term, "Using Tor...")
		config.UseTor = true
	}

	term.SetPrompt("File to import libotr private key from (enter to generate): ")

	var priv otr.PrivateKey
	for {
		importFile, err := term.ReadLine()
		if err != nil {
			return false
		}
		if len(importFile) > 0 {
			privKeyBytes, err := ioutil.ReadFile(importFile)
			if err != nil {
				alert(term, "Failed to open private key file: "+err.Error())
				continue
			}

			if !priv.Import(privKeyBytes) {
				alert(term, "Failed to parse libotr private key file (the parser is pretty simple I'm afraid)")
				continue
			}
			break
		} else {
			info(term, "Generating private key...")
			priv.Generate(rand.Reader)
			break
		}
	}
	config.PrivateKey = priv.Serialize(nil)

	config.OTRAutoAppendTag = true
	config.OTRAutoStartSession = true
	config.OTRAutoTearDown = false

	// List well known Tor hidden services.
	knownTorDomain := map[string]string{
		"jabber.ccc.de":             "okj7xc6j2szr2y75.onion",
		"riseup.net":                "4cjw6cwpeaeppfqz.onion",
		"jabber.calyxinstitute.org": "ijeeynrc6x2uy5ob.onion",
		"jabber.otr.im":             "5rgdtlawqkcplz75.onion",
		"wtfismyip.com":             "ofkztxcohimx34la.onion",
		"rows.io":                   "yz6yiv2hxyagvwy6.onion",
	}

	// Autoconfigure well known Tor hidden services.
	if hiddenService, ok := knownTorDomain[domain]; ok && config.UseTor {
		const torProxyURL = "socks5://127.0.0.1:9050"
		info(term, "It appears that you are using a well known server and we will use its Tor hidden service to connect.")
		config.Server = hiddenService
		config.Port = 5222
		config.Proxies = []string{torProxyURL}
		term.SetPrompt("> ")
		return true
	}

	var proxyStr string
	proxyDefaultPrompt := ", enter for none"
	if config.UseTor {
		proxyDefaultPrompt = ", which is the default"
	}
	term.SetPrompt("Proxy (i.e socks5://127.0.0.1:9050" + proxyDefaultPrompt + "): ")

	for {
		if proxyStr, err = term.ReadLine(); err != nil {
			return false
		}
		if len(proxyStr) == 0 {
			if !config.UseTor {
				break
			} else {
				proxyStr = "socks5://127.0.0.1:9050"
			}
		}
		u, err := url.Parse(proxyStr)
		if err != nil {
			alert(term, "Failed to parse "+proxyStr+" as a URL: "+err.Error())
			continue
		}
		if _, err = proxy.FromURL(u, proxy.Direct); err != nil {
			alert(term, "Failed to parse "+proxyStr+" as a proxy: "+err.Error())
			continue
		}
		break
	}

	if len(proxyStr) > 0 {
		config.Proxies = []string{proxyStr}

		info(term, "Since you selected a proxy, we need to know the server and port to connect to as a SRV lookup would leak information every time.")
		term.SetPrompt("Server (i.e. xmpp.example.com, enter to lookup using unproxied DNS): ")
		if config.Server, err = term.ReadLine(); err != nil {
			return false
		}
		if len(config.Server) == 0 {
			var port uint16
			info(term, "Performing SRV lookup")
			if config.Server, port, err = xmpp.Resolve(domain); err != nil {
				alert(term, "SRV lookup failed: "+err.Error())
				return false
			}
			config.Port = int(port)
			info(term, "Resolved "+config.Server+":"+strconv.Itoa(config.Port))
		} else {
			for {
				term.SetPrompt("Port (enter for 5222): ")
				portStr, err := term.ReadLine()
				if err != nil {
					return false
				}
				if len(portStr) == 0 {
					portStr = "5222"
				}
				if config.Port, err = strconv.Atoi(portStr); err != nil || config.Port <= 0 || config.Port > 65535 {
					info(term, "Port numbers must be 0 < port <= 65535")
					continue
				}
				break
			}
		}
	}

	term.SetPrompt("> ")

	return true
}
<commit_msg>Accept 'y', 'yes', 'Yes', etc for enrollment answers.<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/url"
	"strconv"
	"strings"

	"github.com/agl/xmpp-client/xmpp"
	"golang.org/x/crypto/otr"
	"golang.org/x/crypto/ssh/terminal"
	"golang.org/x/net/proxy"
)

type Config struct {
	filename                      string `json:"-"`
	Account                       string
	Server                        string   `json:",omitempty"`
	Proxies                       []string `json:",omitempty"`
	Password                      string   `json:",omitempty"`
	Port                          int      `json:",omitempty"`
	PrivateKey                    []byte
	KnownFingerprints             []KnownFingerprint
	RawLogFile                    string   `json:",omitempty"`
	NotifyCommand                 []string `json:",omitempty"`
	IdleSecondsBeforeNotification int      `json:",omitempty"`
	Bell                          bool
	HideStatusUpdates             bool
	UseTor                        bool
	OTRAutoTearDown               bool
	OTRAutoAppendTag              bool
	OTRAutoStartSession           bool
	ServerCertificateSHA256       string   `json:",omitempty"`
	AlwaysEncrypt                 bool     `json:",omitempty"`
	AlwaysEncryptWith             []string `json:",omitempty"`
}

type KnownFingerprint struct {
	UserId         string
	FingerprintHex string
	fingerprint    []byte `json:"-"`
}

func ParseConfig(filename string) (c *Config, err error) {
	contents, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	c = new(Config)
	if err = json.Unmarshal(contents, &c); err != nil {
		return
	}

	c.filename = filename

	for i, known := range c.KnownFingerprints {
		c.KnownFingerprints[i].fingerprint, err = hex.DecodeString(known.FingerprintHex)
		if err != nil {
			err = errors.New("xmpp: failed to parse hex fingerprint for " + known.UserId + ": " + err.Error())
			return
		}
	}

	return
}

func (c *Config) Save() error {
	for i, known := range c.KnownFingerprints {
		c.KnownFingerprints[i].FingerprintHex = hex.EncodeToString(known.fingerprint)
	}

	contents, err := json.MarshalIndent(c, "", "\t")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(c.filename, contents, 0600)
}

func (c *Config) UserIdForFingerprint(fpr []byte) string {
	for _, known := range c.KnownFingerprints {
		if bytes.Equal(fpr, known.fingerprint) {
			return known.UserId
		}
	}

	return ""
}

func (c *Config) HasFingerprint(uid string) bool {
	for _, known := range c.KnownFingerprints {
		if uid == known.UserId {
			return true
		}
	}

	return false
}

func (c *Config) ShouldEncryptTo(uid string) bool {
	if c.AlwaysEncrypt {
		return true
	}

	for _, contact := range c.AlwaysEncryptWith {
		if contact == uid {
			return true
		}
	}
	return false
}

func isYes(s string) bool {
	lower := strings.ToLower(s)
	return lower == "yes" || lower == "y"
}

func enroll(config *Config, term *terminal.Terminal) bool {
	var err error
	warn(term, "Enrolling new config file")

	var domain string
	for {
		term.SetPrompt("Account (i.e. user@example.com, enter to quit): ")
		if config.Account, err = term.ReadLine(); err != nil || len(config.Account) == 0 {
			return false
		}

		parts := strings.SplitN(config.Account, "@", 2)
		if len(parts) != 2 {
			alert(term, "invalid username (want user@domain): "+config.Account)
			continue
		}
		domain = parts[1]
		break
	}

	const debugLogFile = "/tmp/xmpp-client-debug.log"
	term.SetPrompt("Enable debug logging to " + debugLogFile + " (y/n)?: ")
	if debugLog, err := term.ReadLine(); err != nil || !isYes(debugLog) {
		info(term, "Not enabling debug logging...")
	} else {
		config.RawLogFile = debugLogFile
		info(term, "Debug logging enabled.")
	}

	term.SetPrompt("Use Tor (y/n)?: ")
	if useTorQuery, err := term.ReadLine(); err != nil || !isYes(useTorQuery) {
		info(term, "Not using Tor...")
		config.UseTor = false
	} else {
		info(term, "Using Tor...")
		config.UseTor = true
	}

	term.SetPrompt("File to import libotr private key from (enter to generate): ")

	var priv otr.PrivateKey
	for {
		importFile, err := term.ReadLine()
		if err != nil {
			return false
		}
		if len(importFile) > 0 {
			privKeyBytes, err := ioutil.ReadFile(importFile)
			if err != nil {
				alert(term, "Failed to open private key file: "+err.Error())
				continue
			}

			if !priv.Import(privKeyBytes) {
				alert(term, "Failed to parse libotr private key file (the parser is pretty simple I'm afraid)")
				continue
			}
			break
		} else {
			info(term, "Generating private key...")
			priv.Generate(rand.Reader)
			break
		}
	}
	config.PrivateKey = priv.Serialize(nil)

	config.OTRAutoAppendTag = true
	config.OTRAutoStartSession = true
	config.OTRAutoTearDown = false

	// List well known Tor hidden services.
	knownTorDomain := map[string]string{
		"jabber.ccc.de":             "okj7xc6j2szr2y75.onion",
		"riseup.net":                "4cjw6cwpeaeppfqz.onion",
		"jabber.calyxinstitute.org": "ijeeynrc6x2uy5ob.onion",
		"jabber.otr.im":             "5rgdtlawqkcplz75.onion",
		"wtfismyip.com":             "ofkztxcohimx34la.onion",
		"rows.io":                   "yz6yiv2hxyagvwy6.onion",
	}

	// Autoconfigure well known Tor hidden services.
	if hiddenService, ok := knownTorDomain[domain]; ok && config.UseTor {
		const torProxyURL = "socks5://127.0.0.1:9050"
		info(term, "It appears that you are using a well known server and we will use its Tor hidden service to connect.")
		config.Server = hiddenService
		config.Port = 5222
		config.Proxies = []string{torProxyURL}
		term.SetPrompt("> ")
		return true
	}

	var proxyStr string
	proxyDefaultPrompt := ", enter for none"
	if config.UseTor {
		proxyDefaultPrompt = ", which is the default"
	}
	term.SetPrompt("Proxy (i.e socks5://127.0.0.1:9050" + proxyDefaultPrompt + "): ")

	for {
		if proxyStr, err = term.ReadLine(); err != nil {
			return false
		}
		if len(proxyStr) == 0 {
			if !config.UseTor {
				break
			} else {
				proxyStr = "socks5://127.0.0.1:9050"
			}
		}
		u, err := url.Parse(proxyStr)
		if err != nil {
			alert(term, "Failed to parse "+proxyStr+" as a URL: "+err.Error())
			continue
		}
		if _, err = proxy.FromURL(u, proxy.Direct); err != nil {
			alert(term, "Failed to parse "+proxyStr+" as a proxy: "+err.Error())
			continue
		}
		break
	}

	if len(proxyStr) > 0 {
		config.Proxies = []string{proxyStr}

		info(term, "Since you selected a proxy, we need to know the server and port to connect to as a SRV lookup would leak information every time.")
		term.SetPrompt("Server (i.e. xmpp.example.com, enter to lookup using unproxied DNS): ")
		if config.Server, err = term.ReadLine(); err != nil {
			return false
		}
		if len(config.Server) == 0 {
			var port uint16
			info(term, "Performing SRV lookup")
			if config.Server, port, err = xmpp.Resolve(domain); err != nil {
				alert(term, "SRV lookup failed: "+err.Error())
				return false
			}
			config.Port = int(port)
			info(term, "Resolved "+config.Server+":"+strconv.Itoa(config.Port))
		} else {
			for {
				term.SetPrompt("Port (enter for 5222): ")
				portStr, err := term.ReadLine()
				if err != nil {
					return false
				}
				if len(portStr) == 0 {
					portStr = "5222"
				}
				if config.Port, err = strconv.Atoi(portStr); err != nil || config.Port <= 0 || config.Port > 65535 {
					info(term, "Port numbers must be 0 < port <= 65535")
					continue
				}
				break
			}
		}
	}

	term.SetPrompt("> ")

	return true
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"log"
	"os"
	"path/filepath"
)

type Config struct {
	Uuid     string `json:"uuid"`
	CpuCount int    `json:"cpu_count"`
	Memory   int    `json:"memory"`
}

func saveConfig(uuid string, cpus, mem int) {
	output, err := os.Create(filepath.Join(PREFIX, "/etc/dlite.conf"))
	if err != nil {
		log.Fatalln(err)
	}

	defer output.Close()
	config := Config{
		Uuid:     uuid,
		CpuCount: cpus,
		Memory:   mem,
	}

	b, err := json.Marshal(config)
	if err != nil {
		log.Fatalln(err)
	}

	output.Write(b)
}

func readConfig() Config {
	var config Config
	file, err := os.Open(filepath.Join(PREFIX, "/etc/dlite.conf"))
	if err != nil {
		log.Fatalln(err)
	}

	defer file.Close()
	decoder := json.NewDecoder(file)
	err = decoder.Decode(&config)
	if err != nil {
		log.Fatalln(err)
	}

	return config
}
<commit_msg>remove another unneeded file<commit_after><|endoftext|>


<commit_before>package gitmedia

import (
	"fmt"
	"os"
	"path"
	"regexp"
	"strings"
)

type Configuration struct {
	gitConfig map[string]string
	remotes   []string
}

var (
	Config       = &Configuration{}
	httpPrefixRe = regexp.MustCompile("\\Ahttps?://")
)

func (c *Configuration) Endpoint() string {
	if url, ok := c.GitConfig("media.url"); ok {
		return url
	}

	return c.RemoteEndpoint("origin")
}

func (c *Configuration) RemoteEndpoint(remote string) string {
	if url, ok := c.GitConfig("remote." + remote + ".media"); ok {
		return url
	}

	if url, ok := c.GitConfig("remote." + remote + ".url"); ok {
		if !httpPrefixRe.MatchString(url) {
			pieces := strings.SplitN(url, ":", 2)
			hostPieces := strings.SplitN(pieces[0], "@", 2)
			url = fmt.Sprintf("https://%s/%s", hostPieces[1], pieces[1])
		}

		if path.Ext(url) == ".git" {
			return url + "/info/media"
		}
		return url + ".git/info/media"
	}

	return ""
}

func (c *Configuration) Remotes() []string {
	if c.remotes == nil {
		c.loadGitConfig()
	}
	return c.remotes
}

func (c *Configuration) GitConfig(key string) (string, bool) {
	if c.gitConfig == nil {
		c.loadGitConfig()
	}
	value, ok := c.gitConfig[key]
	return value, ok
}

func (c *Configuration) SetConfig(key, value string) {
	if c.gitConfig == nil {
		c.loadGitConfig()
	}
	c.gitConfig[key] = value
}

type AltConfig struct {
	Remote map[string]*struct {
		Media string
	}

	Media struct {
		Url string
	}
}

func (c *Configuration) loadGitConfig() {
	uniqRemotes := make(map[string]bool)

	c.gitConfig = make(map[string]string)

	var output string
	output = SimpleExec("git", "config", "-l")
	output += "\n"
	output += SimpleExec("git", "config", "-l", "-f", ".gitconfig")

	lines := strings.Split(output, "\n")
	for _, line := range lines {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 {
			continue
		}
		key := pieces[0]
		c.gitConfig[key] = pieces[1]

		keyParts := strings.Split(key, ".")
		if len(keyParts) > 1 && keyParts[0] == "remote" {
			remote := keyParts[1]
			uniqRemotes[remote] = remote == "origin"
		}
	}

	c.remotes = make([]string, 0, len(uniqRemotes))
	for remote, isOrigin := range uniqRemotes {
		if isOrigin {
			continue
		}
		c.remotes = append(c.remotes, remote)
	}
}

func configFileExists(filename string) bool {
	if _, err := os.Stat(filename); err == nil {
		return true
	}
	return false
}
<commit_msg>ンンンンン ンンンン<commit_after>package gitmedia

import (
	"fmt"
	"os"
	"path"
	"regexp"
	"strings"
)

type Configuration struct {
	gitConfig map[string]string
	remotes   []string
}

var (
	Config       = &Configuration{}
	httpPrefixRe = regexp.MustCompile("\\Ahttps?://")
)

func (c *Configuration) Endpoint() string {
	if url, ok := c.GitConfig("media.url"); ok {
		return url
	}

	return c.RemoteEndpoint("origin")
}

func (c *Configuration) RemoteEndpoint(remote string) string {
	if url, ok := c.GitConfig("remote." + remote + ".media"); ok {
		return url
	}

	if url, ok := c.GitConfig("remote." + remote + ".url"); ok {
		if !httpPrefixRe.MatchString(url) {
			pieces := strings.SplitN(url, ":", 2)
			hostPieces := strings.SplitN(pieces[0], "@", 2)
			if len(hostPieces) < 2 {
				return "unknown"
			}
			url = fmt.Sprintf("https://%s/%s", hostPieces[1], pieces[1])
		}

		if path.Ext(url) == ".git" {
			return url + "/info/media"
		}
		return url + ".git/info/media"
	}

	return ""
}

func (c *Configuration) Remotes() []string {
	if c.remotes == nil {
		c.loadGitConfig()
	}
	return c.remotes
}

func (c *Configuration) GitConfig(key string) (string, bool) {
	if c.gitConfig == nil {
		c.loadGitConfig()
	}
	value, ok := c.gitConfig[key]
	return value, ok
}

func (c *Configuration) SetConfig(key, value string) {
	if c.gitConfig == nil {
		c.loadGitConfig()
	}
	c.gitConfig[key] = value
}

type AltConfig struct {
	Remote map[string]*struct {
		Media string
	}

	Media struct {
		Url string
	}
}

func (c *Configuration) loadGitConfig() {
	uniqRemotes := make(map[string]bool)

	c.gitConfig = make(map[string]string)

	var output string
	output = SimpleExec("git", "config", "-l")
	output += "\n"
	output += SimpleExec("git", "config", "-l", "-f", ".gitconfig")

	lines := strings.Split(output, "\n")
	for _, line := range lines {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 {
			continue
		}
		key := pieces[0]
		c.gitConfig[key] = pieces[1]

		keyParts := strings.Split(key, ".")
		if len(keyParts) > 1 && keyParts[0] == "remote" {
			remote := keyParts[1]
			uniqRemotes[remote] = remote == "origin"
		}
	}

	c.remotes = make([]string, 0, len(uniqRemotes))
	for remote, isOrigin := range uniqRemotes {
		if isOrigin {
			continue
		}
		c.remotes = append(c.remotes, remote)
	}
}

func configFileExists(filename string) bool {
	if _, err := os.Stat(filename); err == nil {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package bla

type Config struct {
	BaseURL  string
	HostName string

	RootPath string
	LinkPath []string

	HomeDocCount int
	Title        string
	UserName     string
	Password     string
}

func DefaultConfig() *Config {

	defaultLinks := []string{
		"libs", "asset",
	}

	return &Config{
		BaseURL:  "",
		HostName: "meng.zhuo.blog",

		RootPath: "root",
		LinkPath: defaultLinks,

		HomeDocCount: 5,
		Title:        "笔记本",
		UserName:     "admin",
		Password:     "ha?",
	}
}
<commit_msg>update default config<commit_after>package bla

import (
	"fmt"
	"os"
	"os/user"
)

type Config struct {
	BaseURL  string
	HostName string

	RootPath string
	LinkPath []string

	HomeDocCount int
	Title        string
	UserName     string
	Password     string
}

func DefaultConfig() *Config {

	defaultLinks := []string{
		"libs", "asset",
	}
	name := user.Current().Name
	return &Config{
		BaseURL:  "",
		HostName: os.Hostname(),

		RootPath: "root",
		LinkPath: defaultLinks,

		HomeDocCount: 5,
		Title:        fmt.Sprintf("%s's blog", name),
		UserName:     name,
		Password:     "PLEASE_UPDATE_PASSWORD!",
	}
}
<|endoftext|>