<commit_before>package cache

import (
	"github.com/LapisBlue/Lapitar/lapitar/mc"
	"log"
	"sync"
	"time"
)

type memorySkinCache struct {
	uuidLoader     map[string]chan SkinMeta
	uuidLoaderLock sync.RWMutex

	skins          map[string]SkinMeta
	skinsLock      sync.RWMutex
	skinLoader     map[string]chan SkinMeta
	skinLoaderLock sync.RWMutex
}

func Memory() SkinCache {
	result := &memorySkinCache{
		uuidLoader: make(map[string]chan SkinMeta),
		skins:      make(map[string]SkinMeta),
		skinLoader: make(map[string]chan SkinMeta),
	}

	result.skins["steve"] = Steve()
	result.skins["char"] = Steve()
	result.skins["alex"] = Alex()
	return result
}

type memorySkinMeta struct {
	mc.SkinMeta
	fallback  SkinMeta
	profile   mc.Profile
	timestamp time.Time
	skin      mc.Skin
	lock      sync.RWMutex
}

func (cache *memorySkinCache) FetchByName(realName string) (skin SkinMeta) {
	name := mc.ToLower(realName)
	skin = cache.pullSkin(name)
	if skin != nil {
		return
	}

	loader := cache.pullUUIDLoader(name)
	if loader != nil {
		skin = <-loader
		loader <- skin
		return skin
	}

	loader = make(chan SkinMeta, 1)
	go cache.pushUUIDLoader(name, loader)
	defer func() {
		go cache.pushUUIDLoader(name, nil)
	}()

	profile, err := mc.FetchProfile(name)
	if profile == nil || err != nil {
		log.Println("Failed to fetch UUID for", realName, err)
		skin = FallbackByName(realName)
		loader <- skin
		go cache.pushFallbackSkin(name, skin)
		return
	}

	skin = Fetch(profile.UUID())
	loader <- skin
	return
}

func (cache *memorySkinCache) Fetch(uuid string) (skin SkinMeta) {
	uuid = mc.ParseUUID(uuid)

	skin = cache.pullSkin(uuid)
	if skin != nil {
		return
	}

	loader := cache.pullSkinLoader(uuid)

	if loader != nil {
		skin = <-loader
		loader <- skin
		return
	}

	loader = make(chan SkinMeta, 1)
	go cache.pushSkinLoader(uuid, loader)
	defer func() {
		go cache.pushSkinLoader(uuid, nil)
	}()

	sk, err := mc.FetchSkin(uuid)
	if sk == nil || err != nil {
		log.Println("Failed to skin profile for", uuid, err)
		skin = FallbackByUUID(uuid)
		loader <- skin
		go cache.pushFallbackSkin(uuid, skin)
		return
	}

	skin = &memorySkinMeta{
		SkinMeta:  sk.Skin(),
		profile:   sk.Profile(),
		timestamp: time.Now(),
	}
	loader <- skin
	go cache.pushSkin(sk.Profile(), skin)
	return
}

func (cache *memorySkinCache) pullUUIDLoader(name string) chan SkinMeta {
	cache.uuidLoaderLock.RLock()
	defer cache.uuidLoaderLock.RUnlock()
	return cache.uuidLoader[name]
}

func (cache *memorySkinCache) pushUUIDLoader(name string, loader chan SkinMeta) {
	cache.uuidLoaderLock.Lock()
	defer cache.uuidLoaderLock.Unlock()
	cache.uuidLoader[name] = loader
}

func (cache *memorySkinCache) pullSkin(uuid string) SkinMeta {
	cache.skinsLock.RLock()
	defer cache.skinsLock.RUnlock()
	return cache.skins[uuid]
}

func (cache *memorySkinCache) pushFallbackSkin(id string, meta SkinMeta) {
	cache.skinsLock.Lock()
	defer cache.skinsLock.Unlock()
	cache.skins[id] = meta
}

func (cache *memorySkinCache) pushSkin(profile mc.Profile, meta SkinMeta) {
	cache.skinsLock.Lock()
	defer cache.skinsLock.Unlock()
	cache.skins[mc.ToLower(profile.UUID())] = meta
	cache.skins[mc.ToLower(profile.Name())] = meta
}

func (cache *memorySkinCache) pullSkinLoader(uuid string) chan SkinMeta {
	cache.skinLoaderLock.RLock()
	defer cache.skinLoaderLock.RUnlock()
	return cache.skinLoader[uuid]
}

func (cache *memorySkinCache) pushSkinLoader(uuid string, loader chan SkinMeta) {
	cache.skinLoaderLock.Lock()
	defer cache.skinLoaderLock.Unlock()
	cache.skinLoader[uuid] = loader
}

func (meta *memorySkinMeta) Profile() mc.Profile {
	return meta.profile
}

func (meta *memorySkinMeta) Fetch() (m SkinMeta, sk mc.Skin) {
	m = meta

	meta.lock.RLock()
	if meta.fallback == nil {
		sk = meta.skin
	} else {
		m, sk = meta.fallback.Fetch()
	}
	meta.lock.RUnlock()

	if sk != nil {
		return
	}

	// We need to download the skin first
	meta.lock.Lock()
	defer meta.lock.Unlock()

	if meta.fallback == nil {
		sk = meta.skin
	} else {
		m, sk = meta.fallback.Fetch()
	}

	if sk != nil {
		return
	}

	sk, err := meta.SkinMeta.Download()
	if err != nil {
		// Meh, we can't download the skin right now
		log.Println("Failed to fetch skin from", meta.URL(), err)
		m, sk = Fallback(meta.Profile()).Fetch()
		meta.fallback = m
		return
	}

	meta.skin = sk
	return
}

func (meta *memorySkinMeta) Download() (mc.Skin, error) {
	_, sk := meta.Fetch()
	return sk, nil
}

func (meta *memorySkinMeta) Timestamp() time.Time {
	return meta.timestamp
}
<commit_msg>char != Steve<commit_after>package cache

import (
	"github.com/LapisBlue/Lapitar/lapitar/mc"
	"log"
	"sync"
	"time"
)

type memorySkinCache struct {
	uuidLoader     map[string]chan SkinMeta
	uuidLoaderLock sync.RWMutex

	skins          map[string]SkinMeta
	skinsLock      sync.RWMutex
	skinLoader     map[string]chan SkinMeta
	skinLoaderLock sync.RWMutex
}

func Memory() SkinCache {
	result := &memorySkinCache{
		uuidLoader: make(map[string]chan SkinMeta),
		skins:      make(map[string]SkinMeta),
		skinLoader: make(map[string]chan SkinMeta),
	}

	result.skins["steve"] = Steve()
	result.skins["alex"] = Alex()
	return result
}

type memorySkinMeta struct {
	mc.SkinMeta
	fallback  SkinMeta
	profile   mc.Profile
	timestamp time.Time
	skin      mc.Skin
	lock      sync.RWMutex
}

func (cache *memorySkinCache) FetchByName(realName string) (skin SkinMeta) {
	name := mc.ToLower(realName)
	skin = cache.pullSkin(name)
	if skin != nil {
		return
	}

	loader := cache.pullUUIDLoader(name)
	if loader != nil {
		skin = <-loader
		loader <- skin
		return skin
	}

	loader = make(chan SkinMeta, 1)
	go cache.pushUUIDLoader(name, loader)
	defer func() {
		go cache.pushUUIDLoader(name, nil)
	}()

	profile, err := mc.FetchProfile(name)
	if profile == nil || err != nil {
		log.Println("Failed to fetch UUID for", realName, err)
		skin = FallbackByName(realName)
		loader <- skin
		go cache.pushFallbackSkin(name, skin)
		return
	}

	skin = Fetch(profile.UUID())
	loader <- skin
	return
}

func (cache *memorySkinCache) Fetch(uuid string) (skin SkinMeta) {
	uuid = mc.ParseUUID(uuid)

	skin = cache.pullSkin(uuid)
	if skin != nil {
		return
	}

	loader := cache.pullSkinLoader(uuid)

	if loader != nil {
		skin = <-loader
		loader <- skin
		return
	}

	loader = make(chan SkinMeta, 1)
	go cache.pushSkinLoader(uuid, loader)
	defer func() {
		go cache.pushSkinLoader(uuid, nil)
	}()

	sk, err := mc.FetchSkin(uuid)
	if sk == nil || err != nil {
		log.Println("Failed to skin profile for", uuid, err)
		skin = FallbackByUUID(uuid)
		loader <- skin
		go cache.pushFallbackSkin(uuid, skin)
		return
	}

	skin = &memorySkinMeta{
		SkinMeta:  sk.Skin(),
		profile:   sk.Profile(),
		timestamp: time.Now(),
	}
	loader <- skin
	go cache.pushSkin(sk.Profile(), skin)
	return
}

func (cache *memorySkinCache) pullUUIDLoader(name string) chan SkinMeta {
	cache.uuidLoaderLock.RLock()
	defer cache.uuidLoaderLock.RUnlock()
	return cache.uuidLoader[name]
}

func (cache *memorySkinCache) pushUUIDLoader(name string, loader chan SkinMeta) {
	cache.uuidLoaderLock.Lock()
	defer cache.uuidLoaderLock.Unlock()
	cache.uuidLoader[name] = loader
}

func (cache *memorySkinCache) pullSkin(uuid string) SkinMeta {
	cache.skinsLock.RLock()
	defer cache.skinsLock.RUnlock()
	return cache.skins[uuid]
}

func (cache *memorySkinCache) pushFallbackSkin(id string, meta SkinMeta) {
	cache.skinsLock.Lock()
	defer cache.skinsLock.Unlock()
	cache.skins[id] = meta
}

func (cache *memorySkinCache) pushSkin(profile mc.Profile, meta SkinMeta) {
	cache.skinsLock.Lock()
	defer cache.skinsLock.Unlock()
	cache.skins[mc.ToLower(profile.UUID())] = meta
	cache.skins[mc.ToLower(profile.Name())] = meta
}

func (cache *memorySkinCache) pullSkinLoader(uuid string) chan SkinMeta {
	cache.skinLoaderLock.RLock()
	defer cache.skinLoaderLock.RUnlock()
	return cache.skinLoader[uuid]
}

func (cache *memorySkinCache) pushSkinLoader(uuid string, loader chan SkinMeta) {
	cache.skinLoaderLock.Lock()
	defer cache.skinLoaderLock.Unlock()
	cache.skinLoader[uuid] = loader
}

func (meta *memorySkinMeta) Profile() mc.Profile {
	return meta.profile
}

func (meta *memorySkinMeta) Fetch() (m SkinMeta, sk mc.Skin) {
	m = meta

	meta.lock.RLock()
	if meta.fallback == nil {
		sk = meta.skin
	} else {
		m, sk = meta.fallback.Fetch()
	}
	meta.lock.RUnlock()

	if sk != nil {
		return
	}

	// We need to download the skin first
	meta.lock.Lock()
	defer meta.lock.Unlock()

	if meta.fallback == nil {
		sk = meta.skin
	} else {
		m, sk = meta.fallback.Fetch()
	}

	if sk != nil {
		return
	}

	sk, err := meta.SkinMeta.Download()
	if err != nil {
		// Meh, we can't download the skin right now
		log.Println("Failed to fetch skin from", meta.URL(), err)
		m, sk = Fallback(meta.Profile()).Fetch()
		meta.fallback = m
		return
	}

	meta.skin = sk
	return
}

func (meta *memorySkinMeta) Download() (mc.Skin, error) {
	_, sk := meta.Fetch()
	return sk, nil
}

func (meta *memorySkinMeta) Timestamp() time.Time {
	return meta.timestamp
}
<|endoftext|>


<commit_before>package flux

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strings"
	"time"
)

//ToDuration returns the duration in millisecond of a given time from the current time
func ToDuration(ms time.Time) time.Duration {
	return Elapsed(ms, time.Now())
}

//Elapsed takes a time and delta it from the current time to return a duration in milliseconds
func Elapsed(ms time.Time, diff time.Time) time.Duration {
	return time.Duration(diff.UTC().Sub(ms).Nanoseconds() / 1e6)
}

//FileCloser provides a means of closing a file
type FileCloser struct {
	*os.File
	path string
}

//Close ends and deletes the file
func (f *FileCloser) Close() error {
	ec := f.File.Close()
	log.Printf("Will Remove %s", f.path)
	ex := os.Remove(f.path)

	if ex == nil {
		return ec
	}

	return ex
}

//NewFileCloser returns a new file closer
func NewFileCloser(path string) (*FileCloser, error) {
	ff, err := os.Open(path)

	if err != nil {
		return nil, err
	}

	return &FileCloser{ff, path}, nil
}

//BufferCloser closes a byte.Buffer
type BufferCloser struct {
	*bytes.Buffer
}

//NewBufferCloser returns a new closer for a bytes.Buffer
func NewBufferCloser(bu *bytes.Buffer) *BufferCloser {
	return &BufferCloser{bu}
}

//Close resets the internal buffer
func (b *BufferCloser) Close() error {
	b.Buffer.Reset()
	return nil
}

//GzipWalker walks a path and turns it into a tar written into a bytes.Buffer
func GzipWalker(file string, tmp io.Writer) error {
	f, err := os.Open(file)

	if err != nil {
		return err
	}

	defer f.Close()

	//gzipper
	gz := gzip.NewWriter(tmp)
	defer gz.Close()

	_, err = io.Copy(gz, f)

	return err
}

//TarWalker walks a path and turns it into a tar written into a bytes.Buffer
func TarWalker(rootpath string, w io.Writer) error {
	tz := tar.NewWriter(w)
	defer tz.Close()

	walkFn := func(path string, info os.FileInfo, err error) error {
		if !info.Mode().IsRegular() {
			return nil
		}

		np, err := filepath.Rel(rootpath, path)
		if err != nil {
			return err
		}

		fl, err := os.Open(path)
		if err != nil {
			return err
		}

		defer fl.Close()

		var h *tar.Header
		if h, err = tar.FileInfoHeader(info, ""); err != nil {
			return err
		}

		h.Name = np

		if err := tz.WriteHeader(h); err != nil {
			return err
		}

		if _, err := io.Copy(tz, fl); err != nil {
			return err
		}

		return nil
	}

	err := filepath.Walk(rootpath, walkFn)
	if err != nil {
		return err
	}

	return nil
}

//Backwards takes a value and walks Backward till 0
func Backwards(to int, fx func(int)) {
	for i := to; i > 0; i-- {
		fx(i)
	}
}

//Forwards takes a value and walks Backward till 0
func Forwards(to int, fx func(int)) {
	for i := 1; i <= to; i++ {
		fx(i)
	}
}

//BackwardsIf takes a value and walks Backward till 0 unless the stop function is called
func BackwardsIf(to int, fx func(int, func())) {
	state := true
	for i := to; i > 0; i-- {
		if !state {
			break
		}
		fx(i, func() { state = false })
	}
}

//ForwardsIf takes a value and walks Backward till 0 unless the stop func is called
func ForwardsIf(to int, fx func(int, func())) {
	state := true
	for i := 1; i <= to; i++ {
		if !state {
			break
		}
		fx(i, func() { state = false })
	}
}

//BackwardsSkip takes a value and walks Backward till 0 unless the skip function is called it will go through all sequence
func BackwardsSkip(to int, fx func(int, func())) {
	for i := to; i > 0; i-- {
		fx(i, func() { i-- })
	}
}

//ForwardsSkip takes a value and walks Backward till 0 unless the skip func is called it will go throuh all sequence
func ForwardsSkip(to int, fx func(int, func())) {
	for i := 1; i <= to; i++ {
		fx(i, func() { i++ })
	}
}

//Report provides a nice abstaction for doing basic report
func Report(e error, msg string) {
	if e != nil {
		log.Fatalf("Message: (%s) with Error: (%s)", msg, e.Error())
	} else {
		log.Printf("Message: (%s) with NoError", msg)
	}
}

//GoDefer letsw you run a function inside a goroutine that gets a defer recovery
func GoDefer(title string, fx func()) {
	go func() {
		defer func() {
			if err := recover(); err != nil {
				var stacks []byte
				runtime.Stack(stacks, true)
				log.Printf("---------%s-Panic----------------:", strings.ToUpper(title))
				log.Printf("Stack Error: %+s", err)
				log.Printf("Debug Stack: %+s", debug.Stack())
				log.Printf("Stack List: %+s", stacks)
				log.Printf("---------%s--END-----------------:", strings.ToUpper(title))
			}
		}()
		fx()
	}()
}

//Close provides a basic io.WriteCloser write method
func (w *FuncWriter) Close() error {
	w.fx = nil
	return nil
}

//Write provides a basic io.Writer write method
func (w *FuncWriter) Write(b []byte) (int, error) {
	w.fx(b)
	return len(b), nil
}

//NewFuncWriter returns a new function writer instance
func NewFuncWriter(fx func([]byte)) *FuncWriter {
	return &FuncWriter{fx}
}

type (
	//FuncWriter provides a means of creation io.Writer on functions
	FuncWriter struct {
		fx func([]byte)
	}
)
<commit_msg>fixing utc call to target time<commit_after>package flux

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strings"
	"time"
)

//ToDuration returns the duration in millisecond of a given time from the current time
func ToDuration(ms time.Time) time.Duration {
	return Elapsed(ms, time.Now())
}

//Elapsed takes a time and delta it from the current time to return a duration in milliseconds
func Elapsed(ms time.Time, diff time.Time) time.Duration {
	return time.Duration(diff.UTC().Sub(ms.UTC()).Nanoseconds() / 1e6)
}

//FileCloser provides a means of closing a file
type FileCloser struct {
	*os.File
	path string
}

//Close ends and deletes the file
func (f *FileCloser) Close() error {
	ec := f.File.Close()
	log.Printf("Will Remove %s", f.path)
	ex := os.Remove(f.path)

	if ex == nil {
		return ec
	}

	return ex
}

//NewFileCloser returns a new file closer
func NewFileCloser(path string) (*FileCloser, error) {
	ff, err := os.Open(path)

	if err != nil {
		return nil, err
	}

	return &FileCloser{ff, path}, nil
}

//BufferCloser closes a byte.Buffer
type BufferCloser struct {
	*bytes.Buffer
}

//NewBufferCloser returns a new closer for a bytes.Buffer
func NewBufferCloser(bu *bytes.Buffer) *BufferCloser {
	return &BufferCloser{bu}
}

//Close resets the internal buffer
func (b *BufferCloser) Close() error {
	b.Buffer.Reset()
	return nil
}

//GzipWalker walks a path and turns it into a tar written into a bytes.Buffer
func GzipWalker(file string, tmp io.Writer) error {
	f, err := os.Open(file)

	if err != nil {
		return err
	}

	defer f.Close()

	//gzipper
	gz := gzip.NewWriter(tmp)
	defer gz.Close()

	_, err = io.Copy(gz, f)

	return err
}

//TarWalker walks a path and turns it into a tar written into a bytes.Buffer
func TarWalker(rootpath string, w io.Writer) error {
	tz := tar.NewWriter(w)
	defer tz.Close()

	walkFn := func(path string, info os.FileInfo, err error) error {
		if !info.Mode().IsRegular() {
			return nil
		}

		np, err := filepath.Rel(rootpath, path)
		if err != nil {
			return err
		}

		fl, err := os.Open(path)
		if err != nil {
			return err
		}

		defer fl.Close()

		var h *tar.Header
		if h, err = tar.FileInfoHeader(info, ""); err != nil {
			return err
		}

		h.Name = np

		if err := tz.WriteHeader(h); err != nil {
			return err
		}

		if _, err := io.Copy(tz, fl); err != nil {
			return err
		}

		return nil
	}

	err := filepath.Walk(rootpath, walkFn)
	if err != nil {
		return err
	}

	return nil
}

//Backwards takes a value and walks Backward till 0
func Backwards(to int, fx func(int)) {
	for i := to; i > 0; i-- {
		fx(i)
	}
}

//Forwards takes a value and walks Backward till 0
func Forwards(to int, fx func(int)) {
	for i := 1; i <= to; i++ {
		fx(i)
	}
}

//BackwardsIf takes a value and walks Backward till 0 unless the stop function is called
func BackwardsIf(to int, fx func(int, func())) {
	state := true
	for i := to; i > 0; i-- {
		if !state {
			break
		}
		fx(i, func() { state = false })
	}
}

//ForwardsIf takes a value and walks Backward till 0 unless the stop func is called
func ForwardsIf(to int, fx func(int, func())) {
	state := true
	for i := 1; i <= to; i++ {
		if !state {
			break
		}
		fx(i, func() { state = false })
	}
}

//BackwardsSkip takes a value and walks Backward till 0 unless the skip function is called it will go through all sequence
func BackwardsSkip(to int, fx func(int, func())) {
	for i := to; i > 0; i-- {
		fx(i, func() { i-- })
	}
}

//ForwardsSkip takes a value and walks Backward till 0 unless the skip func is called it will go throuh all sequence
func ForwardsSkip(to int, fx func(int, func())) {
	for i := 1; i <= to; i++ {
		fx(i, func() { i++ })
	}
}

//Report provides a nice abstaction for doing basic report
func Report(e error, msg string) {
	if e != nil {
		log.Fatalf("Message: (%s) with Error: (%s)", msg, e.Error())
	} else {
		log.Printf("Message: (%s) with NoError", msg)
	}
}

//GoDefer letsw you run a function inside a goroutine that gets a defer recovery
func GoDefer(title string, fx func()) {
	go func() {
		defer func() {
			if err := recover(); err != nil {
				var stacks []byte
				runtime.Stack(stacks, true)
				log.Printf("---------%s-Panic----------------:", strings.ToUpper(title))
				log.Printf("Stack Error: %+s", err)
				log.Printf("Debug Stack: %+s", debug.Stack())
				log.Printf("Stack List: %+s", stacks)
				log.Printf("---------%s--END-----------------:", strings.ToUpper(title))
			}
		}()
		fx()
	}()
}

//Close provides a basic io.WriteCloser write method
func (w *FuncWriter) Close() error {
	w.fx = nil
	return nil
}

//Write provides a basic io.Writer write method
func (w *FuncWriter) Write(b []byte) (int, error) {
	w.fx(b)
	return len(b), nil
}

//NewFuncWriter returns a new function writer instance
func NewFuncWriter(fx func([]byte)) *FuncWriter {
	return &FuncWriter{fx}
}

type (
	//FuncWriter provides a means of creation io.Writer on functions
	FuncWriter struct {
		fx func([]byte)
	}
)
<|endoftext|>


<commit_before>package gocql

func ptrOfType(t string) interface{} {
	switch t {
	case "ascii", "varchar", "inet":
		return new(string)
	case "bigint", "counter", "timestamp":
		return new(int64)
	case "blob":
		return new([]byte)
	case "boolean":
		return new(bool)
	case "decimal", "float":
		return new(float32)
	case "double":
		return new(float64)
	case "int":
		return new(int32)
	case "uuid", "timeuuid":
		return new(UUID)
	case "list", "set":
		return new([]interface{})
	case "map":
		return new(map[interface{}]interface{})
	default:
		return new([]byte)
	}
}

func dereference(i interface{}) interface{} {
	switch i.(type) {
	case *string:
		return *i.(*string)
	case *int64:
		return *i.(*int64)
	case *[]byte:
		return *i.(*[]byte)
	case *bool:
		return *i.(*bool)
	case *float32:
		return *i.(*float32)
	case *float64:
		return *i.(*float64)
	case *int32:
		return *i.(*int32)
	case *UUID:
		return *i.(*UUID)
	case *[]interface{}:
		return *i.(*[]interface{})
	case *map[interface{}]interface{}:
		return *i.(*map[interface{}]interface{})
	default:
		return i
	}
}

// SliceMap is a helper function to make the API easier to use
// returns the data from the query in the form of []map[string]interface{}
func (iter *Iter) SliceMap() ([]map[string]interface{}, error) {
	if iter.err != nil {
		return nil, iter.err
	}
	interfacesToScan := make([]interface{}, 0)
	for _, column := range iter.Columns() {
		i := ptrOfType(column.TypeInfo.String())
		interfacesToScan = append(interfacesToScan, i)
	}
	dataToReturn := make([]map[string]interface{}, 0)
	for iter.Scan(interfacesToScan...) {
		m := make(map[string]interface{})
		for i, column := range iter.Columns() {
			m[column.Name] = dereference(interfacesToScan[i])
		}
		dataToReturn = append(dataToReturn, m)
	}
	return dataToReturn, nil
}
<commit_msg>added all types possible to ptrOfType and dereference functions<commit_after>package gocql

func ptrOfType(t *TypeInfo) interface{} {
	switch t.Type {
	case TypeVarchar, TypeAscii:
		return new(string)
	case TypeBigInt, TypeCounter, TypeTimestamp:
		return new(int64)
	case TypeBlob:
		return new([]byte)
	case TypeBoolean:
		return new(bool)
	case TypeFloat:
		return new(float32)
	case TypeDouble:
		return new(float64)
	case TypeInt:
		return new(int32)
	case TypeUUID, TypeTimeUUID:
		return new(UUID)
	case TypeList, TypeSet:
		switch t.Elem.Type {
		case TypeVarchar, TypeAscii:
			return new([]string)
		case TypeBigInt, TypeCounter, TypeTimestamp:
			return new([]int64)
		case TypeBlob:
			return new([][]byte)
		case TypeBoolean:
			return new([]bool)
		case TypeFloat:
			return new([]float32)
		case TypeDouble:
			return new([]float64)
		case TypeInt:
			return new([]int32)
		case TypeUUID, TypeTimeUUID:
			return new([]UUID)
		}
		return new([][]byte)

	case TypeMap:
		switch t.Key.Type {
		case TypeVarchar, TypeAscii:
			switch t.Elem.Type {
			case TypeVarchar, TypeAscii:
				return new(map[string]string)
			case TypeBigInt, TypeCounter, TypeTimestamp:
				return new(map[string]int64)
			case TypeBlob:
				return new(map[string][]byte)
			case TypeBoolean:
				return new(map[string]bool)
			case TypeFloat:
				return new(map[string]float32)
			case TypeDouble:
				return new(map[string]float64)
			case TypeInt:
				return new(map[string]int32)
			case TypeUUID, TypeTimeUUID:
				return new(map[string]UUID)
			}
		case TypeBigInt, TypeCounter, TypeTimestamp:
			switch t.Elem.Type {
			case TypeVarchar, TypeAscii:
				return new(map[int64]string)
			case TypeBigInt, TypeCounter, TypeTimestamp:
				return new(map[int64]int64)
			case TypeBlob:
				return new(map[int64][]byte)
			case TypeBoolean:
				return new(map[int64]bool)
			case TypeFloat:
				return new(map[int64]float32)
			case TypeDouble:
				return new(map[int64]float64)
			case TypeInt:
				return new(map[int64]int32)
			case TypeUUID, TypeTimeUUID:
				return new(map[int64]UUID)
			}
		case TypeBlob:
			switch t.Elem.Type {
			case TypeVarchar, TypeAscii:
				return new(map[string]string)
			case TypeBigInt, TypeCounter, TypeTimestamp:
				return new(map[string]int64)
			case TypeBlob:
				return new(map[string][]byte)
			case TypeBoolean:
				return new(map[string]bool)
			case TypeFloat:
				return new(map[string]float32)
			case TypeDouble:
				return new(map[string]float64)
			case TypeInt:
				return new(map[string]int32)
			case TypeUUID, TypeTimeUUID:
				return new(map[string]UUID)
			}
		case TypeBoolean:
			switch t.Elem.Type {
			case TypeVarchar, TypeAscii:
				return new(map[bool]string)
			case TypeBigInt, TypeCounter, TypeTimestamp:
				return new(map[bool]int64)
			case TypeBlob:
				return new(map[bool][]byte)
			case TypeBoolean:
				return new(map[bool]bool)
			case TypeFloat:
				return new(map[bool]float32)
			case TypeDouble:
				return new(map[bool]float64)
			case TypeInt:
				return new(map[bool]int32)
			case TypeUUID, TypeTimeUUID:
				return new(map[bool]UUID)
			}
		case TypeFloat:
			switch t.Elem.Type {
			case TypeVarchar, TypeAscii:
				return new(map[float32]string)
			case TypeBigInt, TypeCounter, TypeTimestamp:
				return new(map[float32]int64)
			case TypeBlob:
				return new(map[float32][]byte)
			case TypeBoolean:
				return new(map[float32]bool)
			case TypeFloat:
				return new(map[float32]float32)
			case TypeDouble:
				return new(map[float32]float64)
			case TypeInt:
				return new(map[float32]int32)
			case TypeUUID, TypeTimeUUID:
				return new(map[float32]UUID)
			}
		case TypeDouble:
			switch t.Elem.Type {
			case TypeVarchar, TypeAscii:
				return new(map[float64]string)
			case TypeBigInt, TypeCounter, TypeTimestamp:
				return new(map[float64]int64)
			case TypeBlob:
				return new(map[float64][]byte)
			case TypeBoolean:
				return new(map[float64]bool)
			case TypeFloat:
				return new(map[float64]float32)
			case TypeDouble:
				return new(map[float64]float64)
			case TypeInt:
				return new(map[float64]int32)
			case TypeUUID, TypeTimeUUID:
				return new(map[float64]UUID)
			}
		case TypeInt:
			switch t.Elem.Type {
			case TypeVarchar, TypeAscii:
				return new(map[int32]string)
			case TypeBigInt, TypeCounter, TypeTimestamp:
				return new(map[int32]int64)
			case TypeBlob:
				return new(map[int32][]byte)
			case TypeBoolean:
				return new(map[int32]bool)
			case TypeFloat:
				return new(map[int32]float32)
			case TypeDouble:
				return new(map[int32]float64)
			case TypeInt:
				return new(map[int32]int32)
			case TypeUUID, TypeTimeUUID:
				return new(map[int32]UUID)
			}
		case TypeUUID, TypeTimeUUID:
			switch t.Elem.Type {
			case TypeVarchar, TypeAscii:
				return new(map[UUID]string)
			case TypeBigInt, TypeCounter, TypeTimestamp:
				return new(map[UUID]int64)
			case TypeBlob:
				return new(map[UUID][]byte)
			case TypeBoolean:
				return new(map[UUID]bool)
			case TypeFloat:
				return new(map[UUID]float32)
			case TypeDouble:
				return new(map[UUID]float64)
			case TypeInt:
				return new(map[UUID]int32)
			case TypeUUID, TypeTimeUUID:
				return new(map[UUID]UUID)
			}
		}
	default:
		return new([]byte)
	}
	return new(map[string]string)
}

func dereference(i interface{}) interface{} {
	switch i.(type) {
	case *string:
		return *i.(*string)
	case *int64:
		return *i.(*int64)
	case *[]byte:
		return *i.(*[]byte)
	case *bool:
		return *i.(*bool)
	case *float32:
		return *i.(*float32)
	case *float64:
		return *i.(*float64)
	case *int32:
		return *i.(*int32)
	case *UUID:
		return *i.(*UUID)
	case *[]string:
		return *i.(*[]string)
	case *[]int64:
		return *i.(*[]int64)
	case *[][]byte:
		return *i.(*[][]byte)
	case *[]bool:
		return *i.(*[]bool)
	case *[]float32:
		return *i.(*[]float32)
	case *[]float64:
		return *i.(*[]float64)
	case *[]int32:
		return *i.(*[]int32)
	case *[]UUID:
		return *i.(*[]UUID)
	case *map[string]string:
		return *i.(*map[string]string)
	case *map[string]int64:
		return *i.(*map[string]int64)
	case *map[string][]byte:
		return *i.(*map[string][]byte)
	case *map[string]bool:
		return *i.(*map[string]bool)
	case *map[string]float32:
		return *i.(*map[string]float32)
	case *map[string]float64:
		return *i.(*map[string]float64)
	case *map[string]int32:
		return *i.(*map[string]int32)
	case *map[string]UUID:
		return *i.(*map[string]UUID)
	case *map[int64]string:
		return *i.(*map[int64]string)
	case *map[int64]int64:
		return *i.(*map[int64]int64)
	case *map[int64][]byte:
		return *i.(*map[int64][]byte)
	case *map[int64]bool:
		return *i.(*map[int64]bool)
	case *map[int64]float32:
		return *i.(*map[int64]float32)
	case *map[int64]float64:
		return *i.(*map[int64]float64)
	case *map[int64]int32:
		return *i.(*map[int64]int32)
	case *map[int64]UUID:
		return *i.(*map[int64]UUID)
	case *map[bool]string:
		return *i.(*map[bool]string)
	case *map[bool]int64:
		return *i.(*map[bool]int64)
	case *map[bool][]byte:
		return *i.(*map[bool][]byte)
	case *map[bool]bool:
		return *i.(*map[bool]bool)
	case *map[bool]float32:
		return *i.(*map[bool]float32)
	case *map[bool]float64:
		return *i.(*map[bool]float64)
	case *map[bool]int32:
		return *i.(*map[bool]int32)
	case *map[bool]UUID:
		return *i.(*map[bool]UUID)
	case *map[float32]string:
		return *i.(*map[float32]string)
	case *map[float32]int64:
		return *i.(*map[float32]int64)
	case *map[float32][]byte:
		return *i.(*map[float32][]byte)
	case *map[float32]bool:
		return *i.(*map[float32]bool)
	case *map[float32]float32:
		return *i.(*map[float32]float32)
	case *map[float32]float64:
		return *i.(*map[float32]float64)
	case *map[float32]int32:
		return *i.(*map[float32]int32)
	case *map[float32]UUID:
		return *i.(*map[float32]UUID)
	case *map[float64]string:
		return *i.(*map[float64]string)
	case *map[float64]int64:
		return *i.(*map[float64]int64)
	case *map[float64][]byte:
		return *i.(*map[float64][]byte)
	case *map[float64]bool:
		return *i.(*map[float64]bool)
	case *map[float64]float32:
		return *i.(*map[float64]float32)
	case *map[float64]float64:
		return *i.(*map[float64]float64)
	case *map[float64]int32:
		return *i.(*map[float64]int32)
	case *map[float64]UUID:
		return *i.(*map[float64]UUID)
	case *map[int32]string:
		return *i.(*map[int32]string)
	case *map[int32]int64:
		return *i.(*map[int32]int64)
	case *map[int32][]byte:
		return *i.(*map[int32][]byte)
	case *map[int32]bool:
		return *i.(*map[int32]bool)
	case *map[int32]float32:
		return *i.(*map[int32]float32)
	case *map[int32]float64:
		return *i.(*map[int32]float64)
	case *map[int32]int32:
		return *i.(*map[int32]int32)
	case *map[int32]UUID:
		return *i.(*map[int32]UUID)
	case *map[UUID]string:
		return *i.(*map[UUID]string)
	case *map[UUID]int64:
		return *i.(*map[UUID]int64)
	case *map[UUID][]byte:
		return *i.(*map[UUID][]byte)
	case *map[UUID]bool:
		return *i.(*map[UUID]bool)
	case *map[UUID]float32:
		return *i.(*map[UUID]float32)
	case *map[UUID]float64:
		return *i.(*map[UUID]float64)
	case *map[UUID]int32:
		return *i.(*map[UUID]int32)
	case *map[UUID]UUID:
		return *i.(*map[UUID]UUID)
	default:
		return i
	}
}

// SliceMap is a helper function to make the API easier to use
// returns the data from the query in the form of []map[string]interface{}
func (iter *Iter) SliceMap() ([]map[string]interface{}, error) {
	if iter.err != nil {
		return nil, iter.err
	}
	interfacesToScan := make([]interface{}, 0)
	for _, column := range iter.Columns() {
		i := ptrOfType(column.TypeInfo)
		interfacesToScan = append(interfacesToScan, i)
	}
	dataToReturn := make([]map[string]interface{}, 0)
	for iter.Scan(interfacesToScan...) {
		m := make(map[string]interface{})
		for i, column := range iter.Columns() {
			m[column.Name] = dereference(interfacesToScan[i])
		}
		dataToReturn = append(dataToReturn, m)
	}
	if iter.err != nil {
		return nil, iter.err
	}
	return dataToReturn, nil
}
<|endoftext|>


<commit_before>package qtypes

import (
	"log"
	"fmt"
	"strings"
	"github.com/zpatrick/go-config"
	"github.com/qnib/qframe-utils"
	"time"
	"github.com/pkg/errors"
	"strconv"
)

const (
	FILTER = "filter"
	COLLECTOR = "collector"
	HANDLER = "handler"
)

type Plugin struct {
	QChan 			QChan
	Cfg 			*config.Config
	MyID			int
	Typ				string
	Pkg				string
	Version 		string
	Name 			string
	LogOnlyPlugs 	[]string
	MsgCount		map[string]float64
	LocalCfg 		map[string]string

}

func NewPlugin(qChan QChan, cfg *config.Config) Plugin {
	return Plugin{
		QChan: qChan,
		Cfg: cfg,
	}
}

func NewNamedPlugin(qChan QChan, cfg *config.Config, typ, pkg, name, version string) Plugin {
	p := Plugin{
		QChan: 			qChan,
		Cfg:   			cfg,
		Typ:   			typ,
		Pkg:  			pkg,
		Version:		version,
		Name: 			name,
		LogOnlyPlugs:   []string{},
		MsgCount:       map[string]float64{
			"received": 0.0,
			"loopDrop": 0.0,
			"inputDrop": 0.0,
			"successDrop": 0.0,
		},
	}
	p.LocalCfg, _  = cfg.Settings()
	logPlugs, err := cfg.String("log.only-plugins")
	if err == nil {
		p.LogOnlyPlugs = strings.Split(logPlugs, ",")
	}
	return p
}

func LogStrToInt(level string) int {
	def := 6
	switch level {
	case "panic":
		return 0
	case "error":
		return 3
	case "warn":
		return 4
	case "notice":
		return 5
	case "info":
		return 6
	case "debug":
		return 7
	case "trace":
		return 8
	default:
		return def
	}
}

func (p *Plugin) CfgString(path string) (string, error) {
	key := fmt.Sprintf("%s.%s.%s", p.Typ, p.Name, path)
	if res, ok := p.LocalCfg[key]; ok {
		return res, nil
	}
	return "", errors.New("Could not find "+key)
}

func (p *Plugin) CfgStringOr(path, alt string) string {
	res, err := p.CfgString(path)
	if err != nil {
		return alt
	}
	return res
}

func (p *Plugin) CfgInt(path string) (int, error) {
	key := fmt.Sprintf("%s.%s.%s", p.Typ, p.Name, path)
	if res, ok := p.LocalCfg[key]; ok {
		return strconv.Atoi(res)
	}
	return 0, errors.New("Could not find "+key)
}

func (p *Plugin) CfgIntOr(path string, alt int) int {
	res, err := p.CfgInt(path)
	if err != nil {
		return alt
	}
	return res
}

func (p *Plugin) CfgBool(path string) (bool, error) {
	key := fmt.Sprintf("%s.%s.%s", p.Typ, p.Name, path)
	if res, ok := p.LocalCfg[key]; ok {
		switch res {
		case "true":
			return true, nil
		case "false":
			return false, nil
		default:
			return false, errors.New(fmt.Sprintf("Key '%s' neither false not true, but %s: ", key, res))

		}
	}
	return false, errors.New("Could not find "+key)
}

func (p *Plugin) CfgBoolOr(path string, alt bool) bool {
	res, err := p.CfgBool(path)
	if err != nil {
		return alt
	}
	return res
}

func (p *Plugin) GetInputs() []string {
	inStr, err := p.CfgString("inputs")
	if err != nil {
		inStr = ""
	}
	return strings.Split(inStr, ",")
}

func (p *Plugin) GetCfgItems(key string) []string {
	inStr, err := p.CfgString(key)
	if err != nil {
		inStr = ""
	}
	return strings.Split(inStr, ",")
}

func (p *Plugin) Log(logLevel, msg string) {
	if len(p.LogOnlyPlugs) != 0 && ! qutils.IsItem(p.LogOnlyPlugs, p.Name) {
		return
	}
	// TODO: Setup in each Log() invocation seems rude
	log.SetFlags(log.LstdFlags | log.Lmicroseconds)
	dL, _ := p.Cfg.StringOr("log.level", "info")
	dI := LogStrToInt(dL)
	lI := LogStrToInt(logLevel)
	lMsg := fmt.Sprintf("[%+6s] %15s Name:%-10s >> %s", strings.ToUpper(logLevel), p.Pkg, p.Name, msg)
	if lI == 0 {
		log.Panic(lMsg)
	} else if dI >= lI {
		log.Println(lMsg)
	}
}

func (p *Plugin) StartTicker(name string, durMs int) Ticker {
	p.Log("info", fmt.Sprintf("Start ticker '%s' with duration of %dms", name, durMs))
	ticker := NewTicker(name, durMs)
	go ticker.DispatchTicker(p.QChan)
	return ticker
}

func (p *Plugin) StopProcessingMessage(qm Message, allowEmptyInput bool) bool {
	p.MsgCount["received"]++
	if p.MyID == qm.SourceID {
		p.Log("debug", "Msg came from the same GID")
		p.MsgCount["loopDrop"]++
		return true
	}
	// TODO: Most likely invoked often, so check if performant enough
	inputs := p.GetInputs()
	if ! allowEmptyInput && len(inputs) == 0 {
		msg := fmt.Sprintf("Plugin '%s' does not allow empty imputs, please set '%s.%s.inputs'", p.Name, p.Typ, p.Name)
		log.Fatal(msg)
	}
	srcSuccess := p.CfgBoolOr("source-success", true)
	if ! qm.InputsMatch(inputs) {
		p.Log("debug", fmt.Sprintf("InputsMatch(%v) != %s | Msg: %s", inputs, qm.GetLastSource(), qm.Message))
		p.MsgCount["inputDrop"]++
		return true
	}
	if qm.SourceSuccess != srcSuccess {
		p.Log("debug", fmt.Sprintf("qm.SourceSuccess (%v) != (%v) srcSuccess", qm.SourceSuccess, srcSuccess))
		p.MsgCount["successDrop"]++
		return true
	}
	return false
}

func (p *Plugin) StopProcessingMetric(qm Metric, allowEmptyInput bool) bool {
	p.MsgCount["received"]++
	// TODO: Most likely invoked often, so check if performant enough
	inputs := p.GetInputs()
	if ! allowEmptyInput && len(inputs) == 0 {
		msg := fmt.Sprintf("Plugin '%s' does not allow empty imputs, please set '%s.%s.inputs'", p.Name, p.Typ, p.Name)
		log.Fatal(msg)
	}
	srcSuccess := p.CfgBoolOr("source-success", true)
	if ! qm.InputsMatch(inputs) {
		p.Log("debug", fmt.Sprintf("InputsMatch(%v) = false", inputs))
		p.MsgCount["inputDrop"]++
		return true
	}
	if qm.SourceSuccess != srcSuccess {
		p.Log("debug", "qcs.SourceSuccess != srcSuccess")
		p.MsgCount["successDrop"]++
		return true
	}
	return false
}

func (p *Plugin) StopProcessingCntEvent(ce ContainerEvent, allowEmptyInput bool) bool {
	p.MsgCount["received"]++
	// TODO: Most likely invoked often, so check if performant enough
	inputs := p.GetInputs()
	if ! allowEmptyInput && len(inputs) == 0 {
		msg := fmt.Sprintf("Plugin '%s' does not allow empty imputs, please set '%s.%s.inputs'", p.Name, p.Typ, p.Name)
		log.Fatal(msg)
	}
	if ! ce.InputsMatch(inputs) {
		p.Log("debug", fmt.Sprintf("InputsMatch(%v) = false", inputs))
		p.MsgCount["inputDrop"]++
		return true
	}
	srcSuccess := p.CfgBoolOr("source-success", true)
	if ce.SourceSuccess != srcSuccess {
		p.Log("debug", "ce.SourceSuccess != srcSuccess")
		p.MsgCount["successDrop"]++
		return true
	}
	return false
}

func (p *Plugin) StopProcessingCntStats(qcs ContainerStats, allowEmptyInput bool) bool {
	p.MsgCount["received"]++
	inputs := p.GetInputs()
	if ! allowEmptyInput && len(inputs) == 0 {
		msg := fmt.Sprintf("Plugin '%s' does not allow empty imputs, please set '%s.%s.inputs'", p.Name, p.Typ, p.Name)
		log.Fatal(msg)
	}
	if qcs.IsLastSource(p.Name) {
		p.Log("debug", "IsLastSource() = true")
		return true

	}
	if ! qcs.InputsMatch(inputs) {
		p.Log("debug", fmt.Sprintf("InputsMatch(%v) = false", inputs))
		return true

	}
	srcSuccess := p.CfgBoolOr("source-success", true)
	if qcs.SourceSuccess != srcSuccess {
		p.Log("debug", "qcs.SourceSuccess != srcSuccess")
		return true
	}
	return false
}

func (p *Plugin) DispatchMsgCount() {
	tickMs := p.CfgIntOr("count-ticker-ms", 5000)
	p.Log("info", fmt.Sprintf("Dispatch goroutine to send MsgCount every %dms", tickMs))
	ticker := time.NewTicker(time.Duration(tickMs)*time.Millisecond).C
	pre := map[string]float64{}
	for {
		tick := <-ticker
		pre = p.SendMsgCount(tick, pre)
	}
}

func (p *Plugin) SendMsgCount(tick time.Time, pre map[string]float64) map[string]float64 {
	dims := map[string]string{
		"plugin_name": p.Name,
		"plugin_version": p.Version,
		"plugin_type": p.Typ,
	}
	qm := NewExt(p.Name, "none", Counter, 0.0, dims, tick, false)
	for k,v := range p.MsgCount {
		if _, ok := pre[k]; !ok {
			pre[k] = v
		} else if pre[k] == v {
			continue
		}
		qm.Name = fmt.Sprintf("msg.%s", k)
		p.Log("debug", fmt.Sprintf("Send MsgCount %s=%f", qm.Name,v))
		qm.Value = float64(v)
		p.QChan.SendData(qm)
	}
	return pre
}
<commit_msg>make StopProcessingMetrics more verbose<commit_after>package qtypes

import (
	"log"
	"fmt"
	"strings"
	"github.com/zpatrick/go-config"
	"github.com/qnib/qframe-utils"
	"time"
	"github.com/pkg/errors"
	"strconv"
)

const (
	FILTER = "filter"
	COLLECTOR = "collector"
	HANDLER = "handler"
)

type Plugin struct {
	QChan 			QChan
	Cfg 			*config.Config
	MyID			int
	Typ				string
	Pkg				string
	Version 		string
	Name 			string
	LogOnlyPlugs 	[]string
	MsgCount		map[string]float64
	LocalCfg 		map[string]string

}

func NewPlugin(qChan QChan, cfg *config.Config) Plugin {
	return Plugin{
		QChan: qChan,
		Cfg: cfg,
	}
}

func NewNamedPlugin(qChan QChan, cfg *config.Config, typ, pkg, name, version string) Plugin {
	p := Plugin{
		QChan: 			qChan,
		Cfg:   			cfg,
		Typ:   			typ,
		Pkg:  			pkg,
		Version:		version,
		Name: 			name,
		LogOnlyPlugs:   []string{},
		MsgCount:       map[string]float64{
			"received": 0.0,
			"loopDrop": 0.0,
			"inputDrop": 0.0,
			"successDrop": 0.0,
		},
	}
	p.LocalCfg, _  = cfg.Settings()
	logPlugs, err := cfg.String("log.only-plugins")
	if err == nil {
		p.LogOnlyPlugs = strings.Split(logPlugs, ",")
	}
	return p
}

func LogStrToInt(level string) int {
	def := 6
	switch level {
	case "panic":
		return 0
	case "error":
		return 3
	case "warn":
		return 4
	case "notice":
		return 5
	case "info":
		return 6
	case "debug":
		return 7
	case "trace":
		return 8
	default:
		return def
	}
}

func (p *Plugin) CfgString(path string) (string, error) {
	key := fmt.Sprintf("%s.%s.%s", p.Typ, p.Name, path)
	if res, ok := p.LocalCfg[key]; ok {
		return res, nil
	}
	return "", errors.New("Could not find "+key)
}

func (p *Plugin) CfgStringOr(path, alt string) string {
	res, err := p.CfgString(path)
	if err != nil {
		return alt
	}
	return res
}

func (p *Plugin) CfgInt(path string) (int, error) {
	key := fmt.Sprintf("%s.%s.%s", p.Typ, p.Name, path)
	if res, ok := p.LocalCfg[key]; ok {
		return strconv.Atoi(res)
	}
	return 0, errors.New("Could not find "+key)
}

func (p *Plugin) CfgIntOr(path string, alt int) int {
	res, err := p.CfgInt(path)
	if err != nil {
		return alt
	}
	return res
}

func (p *Plugin) CfgBool(path string) (bool, error) {
	key := fmt.Sprintf("%s.%s.%s", p.Typ, p.Name, path)
	if res, ok := p.LocalCfg[key]; ok {
		switch res {
		case "true":
			return true, nil
		case "false":
			return false, nil
		default:
			return false, errors.New(fmt.Sprintf("Key '%s' neither false not true, but %s: ", key, res))

		}
	}
	return false, errors.New("Could not find "+key)
}

func (p *Plugin) CfgBoolOr(path string, alt bool) bool {
	res, err := p.CfgBool(path)
	if err != nil {
		return alt
	}
	return res
}

func (p *Plugin) GetInputs() []string {
	inStr, err := p.CfgString("inputs")
	if err != nil {
		inStr = ""
	}
	return strings.Split(inStr, ",")
}

func (p *Plugin) GetCfgItems(key string) []string {
	inStr, err := p.CfgString(key)
	if err != nil {
		inStr = ""
	}
	return strings.Split(inStr, ",")
}

func (p *Plugin) Log(logLevel, msg string) {
	if len(p.LogOnlyPlugs) != 0 && ! qutils.IsItem(p.LogOnlyPlugs, p.Name) {
		return
	}
	// TODO: Setup in each Log() invocation seems rude
	log.SetFlags(log.LstdFlags | log.Lmicroseconds)
	dL, _ := p.Cfg.StringOr("log.level", "info")
	dI := LogStrToInt(dL)
	lI := LogStrToInt(logLevel)
	lMsg := fmt.Sprintf("[%+6s] %15s Name:%-10s >> %s", strings.ToUpper(logLevel), p.Pkg, p.Name, msg)
	if lI == 0 {
		log.Panic(lMsg)
	} else if dI >= lI {
		log.Println(lMsg)
	}
}

func (p *Plugin) StartTicker(name string, durMs int) Ticker {
	p.Log("info", fmt.Sprintf("Start ticker '%s' with duration of %dms", name, durMs))
	ticker := NewTicker(name, durMs)
	go ticker.DispatchTicker(p.QChan)
	return ticker
}

func (p *Plugin) StopProcessingMessage(qm Message, allowEmptyInput bool) bool {
	p.MsgCount["received"]++
	if p.MyID == qm.SourceID {
		p.Log("debug", "Msg came from the same GID")
		p.MsgCount["loopDrop"]++
		return true
	}
	// TODO: Most likely invoked often, so check if performant enough
	inputs := p.GetInputs()
	if ! allowEmptyInput && len(inputs) == 0 {
		msg := fmt.Sprintf("Plugin '%s' does not allow empty imputs, please set '%s.%s.inputs'", p.Name, p.Typ, p.Name)
		log.Fatal(msg)
	}
	srcSuccess := p.CfgBoolOr("source-success", true)
	if ! qm.InputsMatch(inputs) {
		p.Log("debug", fmt.Sprintf("InputsMatch(%v) != %s | Msg: %s", inputs, qm.GetLastSource(), qm.Message))
		p.MsgCount["inputDrop"]++
		return true
	}
	if qm.SourceSuccess != srcSuccess {
		p.Log("debug", fmt.Sprintf("qm.SourceSuccess (%v) != (%v) srcSuccess", qm.SourceSuccess, srcSuccess))
		p.MsgCount["successDrop"]++
		return true
	}
	return false
}

func (p *Plugin) StopProcessingMetric(qm Metric, allowEmptyInput bool) bool {
	p.MsgCount["received"]++
	// TODO: Most likely invoked often, so check if performant enough
	inputs := p.GetInputs()
	if ! allowEmptyInput && len(inputs) == 0 {
		msg := fmt.Sprintf("Plugin '%s' does not allow empty imputs, please set '%s.%s.inputs'", p.Name, p.Typ, p.Name)
		log.Fatal(msg)
	}
	srcSuccess := p.CfgBoolOr("source-success", true)
	if ! qm.InputsMatch(inputs) {
		p.Log("debug", fmt.Sprintf("InputsMatch(%v) != %s", inputs, qm.GetLastSource()))
		p.MsgCount["inputDrop"]++
		return true
	}
	if qm.SourceSuccess != srcSuccess {
		p.Log("debug", fmt.Sprintf("qcs.SourceSuccess != srcSuccess (%v)", srcSuccess))
		p.MsgCount["successDrop"]++
		return true
	}
	return false
}

func (p *Plugin) StopProcessingCntEvent(ce ContainerEvent, allowEmptyInput bool) bool {
	p.MsgCount["received"]++
	// TODO: Most likely invoked often, so check if performant enough
	inputs := p.GetInputs()
	if ! allowEmptyInput && len(inputs) == 0 {
		msg := fmt.Sprintf("Plugin '%s' does not allow empty imputs, please set '%s.%s.inputs'", p.Name, p.Typ, p.Name)
		log.Fatal(msg)
	}
	if ! ce.InputsMatch(inputs) {
		p.Log("debug", fmt.Sprintf("InputsMatch(%v) = false", inputs))
		p.MsgCount["inputDrop"]++
		return true
	}
	srcSuccess := p.CfgBoolOr("source-success", true)
	if ce.SourceSuccess != srcSuccess {
		p.Log("debug", "ce.SourceSuccess != srcSuccess")
		p.MsgCount["successDrop"]++
		return true
	}
	return false
}

func (p *Plugin) StopProcessingCntStats(qcs ContainerStats, allowEmptyInput bool) bool {
	p.MsgCount["received"]++
	inputs := p.GetInputs()
	if ! allowEmptyInput && len(inputs) == 0 {
		msg := fmt.Sprintf("Plugin '%s' does not allow empty imputs, please set '%s.%s.inputs'", p.Name, p.Typ, p.Name)
		log.Fatal(msg)
	}
	if qcs.IsLastSource(p.Name) {
		p.Log("debug", "IsLastSource() = true")
		return true

	}
	if ! qcs.InputsMatch(inputs) {
		p.Log("debug", fmt.Sprintf("InputsMatch(%v) = false", inputs))
		return true

	}
	srcSuccess := p.CfgBoolOr("source-success", true)
	if qcs.SourceSuccess != srcSuccess {
		p.Log("debug", "qcs.SourceSuccess != srcSuccess")
		return true
	}
	return false
}

func (p *Plugin) DispatchMsgCount() {
	tickMs := p.CfgIntOr("count-ticker-ms", 5000)
	p.Log("info", fmt.Sprintf("Dispatch goroutine to send MsgCount every %dms", tickMs))
	ticker := time.NewTicker(time.Duration(tickMs)*time.Millisecond).C
	pre := map[string]float64{}
	for {
		tick := <-ticker
		pre = p.SendMsgCount(tick, pre)
	}
}

func (p *Plugin) SendMsgCount(tick time.Time, pre map[string]float64) map[string]float64 {
	dims := map[string]string{
		"plugin_name": p.Name,
		"plugin_version": p.Version,
		"plugin_type": p.Typ,
	}
	qm := NewExt(p.Name, "none", Counter, 0.0, dims, tick, false)
	for k,v := range p.MsgCount {
		if _, ok := pre[k]; !ok {
			pre[k] = v
		} else if pre[k] == v {
			continue
		}
		qm.Name = fmt.Sprintf("msg.%s", k)
		p.Log("debug", fmt.Sprintf("Send MsgCount %s=%f", qm.Name,v))
		qm.Value = float64(v)
		p.QChan.SendData(qm)
	}
	return pre
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"syscall"

	"github.com/dickeyxxx/golock"
	"github.com/heroku/heroku-cli/gode"
)

// Plugin represents a javascript plugin
type Plugin struct {
	Name     string     `json:"name"`
	Version  string     `json:"version"`
	Topics   TopicSet   `json:"topics"`
	Topic    *Topic     `json:"topic"`
	Commands CommandSet `json:"commands"`
}

// SetupNode sets up node and npm in ~/.heroku
func SetupNode() {
	gode.SetRootPath(AppDir())
	setup, err := gode.IsSetup()
	WarnIfError(err)
	if !setup {
		if err := gode.Setup(); err != nil {
			panic(err)
		}
	}
}

// LoadPlugins loads the topics and commands from the JavaScript plugins into the CLI
func (cli *Cli) LoadPlugins(plugins map[string]*Plugin) {
	for _, plugin := range plugins {
		for _, topic := range plugin.Topics {
			cli.AddTopic(topic)
		}
		if plugin.Topic != nil {
			cli.AddTopic(plugin.Topic)
		}
		for _, command := range plugin.Commands {
			if !cli.AddCommand(command) {
				Errf("WARNING: command %s has already been defined\n", command)
			}
		}
	}
	sort.Sort(cli.Topics)
	sort.Sort(cli.Commands)
}

var pluginsTopic = &Topic{
	Name:        "plugins",
	Description: "manage plugins",
}

var pluginsInstallCmd = &Command{
	Topic:       "plugins",
	Command:     "install",
	Hidden:      true,
	Args:        []Arg{{Name: "name"}},
	Description: "Installs a plugin into the CLI",
	Help: `Install a Heroku plugin

  Example:
  $ heroku plugins:install dickeyxxx/heroku-production-status`,

	Run: func(ctx *Context) {
		name := ctx.Args.(map[string]string)["name"]
		if len(name) == 0 {
			Errln("Must specify a plugin name")
			return
		}
		action("Installing plugin "+name, "done", func() {
			ExitIfError(installPlugins(name))
		})
	},
}

var pluginsLinkCmd = &Command{
	Topic:       "plugins",
	Command:     "link",
	Description: "Links a local plugin into CLI",
	Args:        []Arg{{Name: "path", Optional: true}},
	Help: `Links a local plugin into CLI.
	This is useful when developing plugins locally.
	It simply symlinks the specified path into ~/.heroku/node_modules

  Example:
	$ heroku plugins:link .`,

	Run: func(ctx *Context) {
		path := ctx.Args.(map[string]string)["path"]
		if path == "" {
			path = "."
		}
		path, err := filepath.Abs(path)
		if err != nil {
			panic(err)
		}
		if _, err := os.Stat(path); err != nil {
			panic(err)
		}
		name := filepath.Base(path)
		newPath := pluginPath(name)
		os.Remove(newPath)
		os.RemoveAll(newPath)
		err = os.Symlink(path, newPath)
		if err != nil {
			panic(err)
		}
		plugin, err := ParsePlugin(name)
		ExitIfError(err)
		if name != plugin.Name {
			path = newPath
			newPath = pluginPath(plugin.Name)
			os.Remove(newPath)
			os.RemoveAll(newPath)
			os.Rename(path, newPath)
		}
		Println("Symlinked", plugin.Name)
		AddPluginsToCache(plugin)
	},
}

var pluginsUninstallCmd = &Command{
	Topic:       "plugins",
	Command:     "uninstall",
	Hidden:      true,
	Args:        []Arg{{Name: "name"}},
	Description: "Uninstalls a plugin from the CLI",
	Help: `Uninstalls a Heroku plugin

  Example:
  $ heroku plugins:uninstall heroku-production-status`,

	Run: func(ctx *Context) {
		name := ctx.Args.(map[string]string)["name"]
		if !contains(PluginNames(), name) {
			ExitIfError(errors.New(name + " is not installed"))
		}
		Errf("Uninstalling plugin %s...", name)
		ExitIfError(gode.RemovePackages(name))
		RemovePluginFromCache(name)
		Errln(" done")
	},
}

var pluginsListCmd = &Command{
	Topic:            "plugins",
	Hidden:           true,
	Description:      "Lists installed plugins",
	DisableAnalytics: true,
	Help: `
Example:
  $ heroku plugins`,

	Run: func(ctx *Context) {
		SetupBuiltinPlugins()
		var plugins []string
		for _, plugin := range GetPlugins() {
			symlinked := ""
			if isPluginSymlinked(plugin.Name) {
				symlinked = " (symlinked)"
			}
			plugins = append(plugins, fmt.Sprintf("%s %s %s", plugin.Name, plugin.Version, symlinked))
		}
		sort.Strings(plugins)
		for _, plugin := range plugins {
			Println(plugin)
		}
	},
}

func runFn(plugin *Plugin, topic, command string) func(ctx *Context) {
	return func(ctx *Context) {
		readLockPlugin(plugin.Name)
		ctx.Dev = isPluginSymlinked(plugin.Name)
		ctxJSON, err := json.Marshal(ctx)
		if err != nil {
			panic(err)
		}
		title, _ := json.Marshal(processTitle(ctx))
		script := fmt.Sprintf(`'use strict';
var moduleName = '%s';
var moduleVersion = '%s';
var topic = '%s';
var command = '%s';
process.title = %s;
var ctx = %s;
ctx.version = ctx.version + ' ' + moduleName + '/' + moduleVersion + ' node-' + process.version;
var logPath = %s;
process.chdir(ctx.cwd);
if (!ctx.dev) {
	process.on('uncaughtException', function (err) {
		// ignore EPIPE errors (usually from piping to head)
		if (err.code === "EPIPE") return;
		console.error(' !   Error in ' + moduleName + ':')
		console.error(' !   ' + err.message || err);
		if (err.stack) {
			var fs = require('fs');
			var log = function (line) {
				var d = new Date().toISOString()
				.replace(/T/, ' ')
				.replace(/-/g, '/')
				.replace(/\..+/, '');
				fs.appendFileSync(logPath, d + ' ' + line + '\n');
			}
			log('Error during ' + topic + ':' + command);
			log(err.stack);
			console.error(' !   See ' + logPath + ' for more info.');
		}
		process.exit(1);
	});
}
if (command === '') { command = null }
var module = require(moduleName);
var cmd = module.commands.filter(function (c) {
	return c.topic === topic && c.command == command;
})[0];
cmd.run(ctx);
`, plugin.Name, plugin.Version, topic, command, string(title), ctxJSON, strconv.Quote(ErrLogPath))

		// swallow sigint since the plugin will handle it
		swallowSigint = true

		currentAnalyticsCommand.Plugin = plugin.Name
		currentAnalyticsCommand.Version = plugin.Version
		currentAnalyticsCommand.Language = fmt.Sprintf("node/%s", gode.NodeVersion)

		cmd, done := gode.RunScript(script)
		defer done()
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if ctx.Flags["debugger"] == true {
			cmd = gode.DebugScript(script)
		}
		err = cmd.Run()
		Exit(getExitCode(err))
	}
}

func getExitCode(err error) int {
	switch e := err.(type) {
	case nil:
		return 0
	case *exec.ExitError:
		status, ok := e.Sys().(syscall.WaitStatus)
		if !ok {
			panic(err)
		}
		return status.ExitStatus()
	default:
		panic(err)
	}
}

var pluginInstallRetrying = false

// ParsePlugin requires the plugin's node module
// to get the commands and metadata
func ParsePlugin(name string) (*Plugin, error) {
	script := `
	var plugin = require('` + name + `');
	if (!plugin.commands) throw new Error('Contains no commands. Is this a real plugin?');
	var pjson  = require('` + name + `/package.json');

	plugin.name    = pjson.name;
	plugin.version = pjson.version;

	console.log(JSON.stringify(plugin))`
	cmd, done := gode.RunScript(script)
	cmd.Stderr = Stderr
	output, err := cmd.Output()
	done()

	if err != nil {
		// try again but this time grab stdout and stderr
		cmd, done := gode.RunScript(script)
		output, err = cmd.CombinedOutput() // sometimes this actually works the second time
		if err != nil {
			done()
			if !pluginInstallRetrying && strings.Contains(string(output), "Error: Cannot find module") {
				pluginInstallRetrying = true
				Warn("Failed to install " + name + ". Retrying...")
				WarnIfError(gode.RemovePackages(name))
				WarnIfError(gode.ClearCache())
				WarnIfError(gode.InstallPackages(name))
				return ParsePlugin(name)
			}
			return nil, fmt.Errorf("Error reading plugin: %s\n%s\n%s", name, err, output)
		}
	}
	var plugin Plugin
	err = json.Unmarshal([]byte(output), &plugin)
	if err != nil {
		return nil, fmt.Errorf("Error parsing plugin: %s\n%s\n%s", name, err, string(output))
	}
	for _, command := range plugin.Commands {
		if command == nil {
			continue
		}
		command.Plugin = plugin.Name
		command.Help = strings.TrimSpace(command.Help)
	}
	return &plugin, nil
}

// GetPlugins goes through all the node plugins and returns them in Go stucts
func GetPlugins() map[string]*Plugin {
	plugins := FetchPluginCache()
	for name, plugin := range plugins {
		if plugin == nil || !pluginExists(name) || plugin.Commands.Len() == 0 {
			delete(plugins, name)
		} else {
			for _, command := range plugin.Commands {
				command.Run = runFn(plugin, command.Topic, command.Command)
			}
		}
	}
	return plugins
}

// PluginNames lists all the plugin names
func PluginNames() []string {
	plugins := FetchPluginCache()
	names := make([]string, 0, len(plugins))
	for _, plugin := range plugins {
		if plugin != nil && pluginExists(plugin.Name) && len(plugin.Commands) > 0 {
			names = append(names, plugin.Name)
		}
	}
	return names
}

// PluginNamesNotSymlinked returns all the plugins that are not symlinked
func PluginNamesNotSymlinked() []string {
	a := PluginNames()
	b := make([]string, 0, len(a))
	for _, plugin := range a {
		if !isPluginSymlinked(plugin) {
			b = append(b, plugin)
		}
	}
	return b
}

func isPluginSymlinked(plugin string) bool {
	path := filepath.Join(AppDir(), "node_modules", plugin)
	fi, err := os.Lstat(path)
	if err != nil {
		return false
	}
	return fi.Mode()&os.ModeSymlink != 0
}

// SetupBuiltinPlugins ensures all the builtinPlugins are installed
func SetupBuiltinPlugins() {
	pluginNames := difference(BuiltinPlugins, PluginNames())
	if len(pluginNames) == 0 {
		return
	}
	action("heroku-cli: Installing core plugins", "done", func() {
		if err := installPlugins(pluginNames...); err != nil {
			// retry once
			WarnIfError(gode.RemovePackages(pluginNames...))
			WarnIfError(gode.ClearCache())
			Err("\rheroku-cli: Installing core plugins (retrying)...")
			ExitIfError(installPlugins(pluginNames...))
		}
	})
}

func difference(a, b []string) []string {
	res := make([]string, 0, len(a))
	for _, aa := range a {
		if !contains(b, aa) {
			res = append(res, aa)
		}
	}
	return res
}

func contains(arr []string, s string) bool {
	for _, a := range arr {
		if a == s {
			return true
		}
	}
	return false
}

func installPlugins(names ...string) error {
	for _, name := range names {
		lockPlugin(name)
	}
	defer func() {
		for _, name := range names {
			unlockPlugin(name)
		}
	}()
	err := gode.InstallPackages(names...)
	if err != nil {
		return err
	}
	plugins := make([]*Plugin, len(names))
	for i, name := range names {
		plugin, err := ParsePlugin(name)
		if err != nil {
			return err
		}
		plugins[i] = plugin
	}
	AddPluginsToCache(plugins...)
	return nil
}

func pluginExists(plugin string) bool {
	exists, _ := fileExists(pluginPath(plugin))
	return exists
}

// directory location of plugin
func pluginPath(plugin string) string {
	return filepath.Join(AppDir(), "node_modules", plugin)
}

// lock a plugin for reading
func readLockPlugin(name string) {
	lockfile := updateLockPath + "." + name
	locked, err := golock.IsLocked(lockfile)
	LogIfError(err)
	if locked {
		lockPlugin(name)
		unlockPlugin(name)
	}
}

// lock a plugin for writing
func lockPlugin(name string) {
	LogIfError(golock.Lock(updateLockPath + "." + name))
}

// unlock a plugin
func unlockPlugin(name string) {
	LogIfError(golock.Unlock(updateLockPath + "." + name))
}
<commit_msg>error handling of package install failure<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"syscall"

	"github.com/dickeyxxx/golock"
	"github.com/heroku/heroku-cli/gode"
)

// Plugin represents a javascript plugin
type Plugin struct {
	Name     string     `json:"name"`
	Version  string     `json:"version"`
	Topics   TopicSet   `json:"topics"`
	Topic    *Topic     `json:"topic"`
	Commands CommandSet `json:"commands"`
}

// SetupNode sets up node and npm in ~/.heroku
func SetupNode() {
	gode.SetRootPath(AppDir())
	setup, err := gode.IsSetup()
	WarnIfError(err)
	if !setup {
		if err := gode.Setup(); err != nil {
			panic(err)
		}
	}
}

// LoadPlugins loads the topics and commands from the JavaScript plugins into the CLI
func (cli *Cli) LoadPlugins(plugins map[string]*Plugin) {
	for _, plugin := range plugins {
		for _, topic := range plugin.Topics {
			cli.AddTopic(topic)
		}
		if plugin.Topic != nil {
			cli.AddTopic(plugin.Topic)
		}
		for _, command := range plugin.Commands {
			if !cli.AddCommand(command) {
				Errf("WARNING: command %s has already been defined\n", command)
			}
		}
	}
	sort.Sort(cli.Topics)
	sort.Sort(cli.Commands)
}

var pluginsTopic = &Topic{
	Name:        "plugins",
	Description: "manage plugins",
}

var pluginsInstallCmd = &Command{
	Topic:       "plugins",
	Command:     "install",
	Hidden:      true,
	Args:        []Arg{{Name: "name"}},
	Description: "Installs a plugin into the CLI",
	Help: `Install a Heroku plugin

  Example:
  $ heroku plugins:install dickeyxxx/heroku-production-status`,

	Run: func(ctx *Context) {
		name := ctx.Args.(map[string]string)["name"]
		if len(name) == 0 {
			Errln("Must specify a plugin name")
			return
		}
		action("Installing plugin "+name, "done", func() {
			ExitIfError(installPlugins(name))
		})
	},
}

var pluginsLinkCmd = &Command{
	Topic:       "plugins",
	Command:     "link",
	Description: "Links a local plugin into CLI",
	Args:        []Arg{{Name: "path", Optional: true}},
	Help: `Links a local plugin into CLI.
	This is useful when developing plugins locally.
	It simply symlinks the specified path into ~/.heroku/node_modules

  Example:
	$ heroku plugins:link .`,

	Run: func(ctx *Context) {
		path := ctx.Args.(map[string]string)["path"]
		if path == "" {
			path = "."
		}
		path, err := filepath.Abs(path)
		if err != nil {
			panic(err)
		}
		if _, err := os.Stat(path); err != nil {
			panic(err)
		}
		name := filepath.Base(path)
		newPath := pluginPath(name)
		os.Remove(newPath)
		os.RemoveAll(newPath)
		err = os.Symlink(path, newPath)
		if err != nil {
			panic(err)
		}
		plugin, err := ParsePlugin(name)
		ExitIfError(err)
		if name != plugin.Name {
			path = newPath
			newPath = pluginPath(plugin.Name)
			os.Remove(newPath)
			os.RemoveAll(newPath)
			os.Rename(path, newPath)
		}
		Println("Symlinked", plugin.Name)
		AddPluginsToCache(plugin)
	},
}

var pluginsUninstallCmd = &Command{
	Topic:       "plugins",
	Command:     "uninstall",
	Hidden:      true,
	Args:        []Arg{{Name: "name"}},
	Description: "Uninstalls a plugin from the CLI",
	Help: `Uninstalls a Heroku plugin

  Example:
  $ heroku plugins:uninstall heroku-production-status`,

	Run: func(ctx *Context) {
		name := ctx.Args.(map[string]string)["name"]
		if !contains(PluginNames(), name) {
			ExitIfError(errors.New(name + " is not installed"))
		}
		Errf("Uninstalling plugin %s...", name)
		ExitIfError(gode.RemovePackages(name))
		RemovePluginFromCache(name)
		Errln(" done")
	},
}

var pluginsListCmd = &Command{
	Topic:            "plugins",
	Hidden:           true,
	Description:      "Lists installed plugins",
	DisableAnalytics: true,
	Help: `
Example:
  $ heroku plugins`,

	Run: func(ctx *Context) {
		SetupBuiltinPlugins()
		var plugins []string
		for _, plugin := range GetPlugins() {
			symlinked := ""
			if isPluginSymlinked(plugin.Name) {
				symlinked = " (symlinked)"
			}
			plugins = append(plugins, fmt.Sprintf("%s %s %s", plugin.Name, plugin.Version, symlinked))
		}
		sort.Strings(plugins)
		for _, plugin := range plugins {
			Println(plugin)
		}
	},
}

func runFn(plugin *Plugin, topic, command string) func(ctx *Context) {
	return func(ctx *Context) {
		readLockPlugin(plugin.Name)
		ctx.Dev = isPluginSymlinked(plugin.Name)
		ctxJSON, err := json.Marshal(ctx)
		if err != nil {
			panic(err)
		}
		title, _ := json.Marshal(processTitle(ctx))
		script := fmt.Sprintf(`'use strict';
var moduleName = '%s';
var moduleVersion = '%s';
var topic = '%s';
var command = '%s';
process.title = %s;
var ctx = %s;
ctx.version = ctx.version + ' ' + moduleName + '/' + moduleVersion + ' node-' + process.version;
var logPath = %s;
process.chdir(ctx.cwd);
if (!ctx.dev) {
	process.on('uncaughtException', function (err) {
		// ignore EPIPE errors (usually from piping to head)
		if (err.code === "EPIPE") return;
		console.error(' !   Error in ' + moduleName + ':')
		console.error(' !   ' + err.message || err);
		if (err.stack) {
			var fs = require('fs');
			var log = function (line) {
				var d = new Date().toISOString()
				.replace(/T/, ' ')
				.replace(/-/g, '/')
				.replace(/\..+/, '');
				fs.appendFileSync(logPath, d + ' ' + line + '\n');
			}
			log('Error during ' + topic + ':' + command);
			log(err.stack);
			console.error(' !   See ' + logPath + ' for more info.');
		}
		process.exit(1);
	});
}
if (command === '') { command = null }
var module = require(moduleName);
var cmd = module.commands.filter(function (c) {
	return c.topic === topic && c.command == command;
})[0];
cmd.run(ctx);
`, plugin.Name, plugin.Version, topic, command, string(title), ctxJSON, strconv.Quote(ErrLogPath))

		// swallow sigint since the plugin will handle it
		swallowSigint = true

		currentAnalyticsCommand.Plugin = plugin.Name
		currentAnalyticsCommand.Version = plugin.Version
		currentAnalyticsCommand.Language = fmt.Sprintf("node/%s", gode.NodeVersion)

		cmd, done := gode.RunScript(script)
		defer done()
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if ctx.Flags["debugger"] == true {
			cmd = gode.DebugScript(script)
		}
		err = cmd.Run()
		Exit(getExitCode(err))
	}
}

func getExitCode(err error) int {
	switch e := err.(type) {
	case nil:
		return 0
	case *exec.ExitError:
		status, ok := e.Sys().(syscall.WaitStatus)
		if !ok {
			panic(err)
		}
		return status.ExitStatus()
	default:
		panic(err)
	}
}

var pluginInstallRetrying = false

// ParsePlugin requires the plugin's node module
// to get the commands and metadata
func ParsePlugin(name string) (*Plugin, error) {
	script := `
	var plugin = require('` + name + `');
	if (!plugin.commands) throw new Error('Contains no commands. Is this a real plugin?');
	var pjson  = require('` + name + `/package.json');

	plugin.name    = pjson.name;
	plugin.version = pjson.version;

	console.log(JSON.stringify(plugin))`
	cmd, done := gode.RunScript(script)
	cmd.Stderr = Stderr
	output, err := cmd.Output()
	done()

	if err != nil {
		// try again but this time grab stdout and stderr
		cmd, done := gode.RunScript(script)
		output, err = cmd.CombinedOutput() // sometimes this actually works the second time
		if err != nil {
			done()
			if !pluginInstallRetrying && strings.Contains(string(output), "Error: Cannot find module") {
				pluginInstallRetrying = true
				Warn("Failed to install " + name + ". Retrying...")
				WarnIfError(gode.RemovePackages(name))
				WarnIfError(gode.ClearCache())
				if err := gode.InstallPackages(name); err != nil {
					return nil, err
				}
				return ParsePlugin(name)
			}
			return nil, fmt.Errorf("Error reading plugin: %s\n%s\n%s", name, err, output)
		}
	}
	var plugin Plugin
	err = json.Unmarshal([]byte(output), &plugin)
	if err != nil {
		return nil, fmt.Errorf("Error parsing plugin: %s\n%s\n%s", name, err, string(output))
	}
	for _, command := range plugin.Commands {
		if command == nil {
			continue
		}
		command.Plugin = plugin.Name
		command.Help = strings.TrimSpace(command.Help)
	}
	return &plugin, nil
}

// GetPlugins goes through all the node plugins and returns them in Go stucts
func GetPlugins() map[string]*Plugin {
	plugins := FetchPluginCache()
	for name, plugin := range plugins {
		if plugin == nil || !pluginExists(name) || plugin.Commands.Len() == 0 {
			delete(plugins, name)
		} else {
			for _, command := range plugin.Commands {
				command.Run = runFn(plugin, command.Topic, command.Command)
			}
		}
	}
	return plugins
}

// PluginNames lists all the plugin names
func PluginNames() []string {
	plugins := FetchPluginCache()
	names := make([]string, 0, len(plugins))
	for _, plugin := range plugins {
		if plugin != nil && pluginExists(plugin.Name) && len(plugin.Commands) > 0 {
			names = append(names, plugin.Name)
		}
	}
	return names
}

// PluginNamesNotSymlinked returns all the plugins that are not symlinked
func PluginNamesNotSymlinked() []string {
	a := PluginNames()
	b := make([]string, 0, len(a))
	for _, plugin := range a {
		if !isPluginSymlinked(plugin) {
			b = append(b, plugin)
		}
	}
	return b
}

func isPluginSymlinked(plugin string) bool {
	path := filepath.Join(AppDir(), "node_modules", plugin)
	fi, err := os.Lstat(path)
	if err != nil {
		return false
	}
	return fi.Mode()&os.ModeSymlink != 0
}

// SetupBuiltinPlugins ensures all the builtinPlugins are installed
func SetupBuiltinPlugins() {
	pluginNames := difference(BuiltinPlugins, PluginNames())
	if len(pluginNames) == 0 {
		return
	}
	action("heroku-cli: Installing core plugins", "done", func() {
		if err := installPlugins(pluginNames...); err != nil {
			// retry once
			WarnIfError(gode.RemovePackages(pluginNames...))
			WarnIfError(gode.ClearCache())
			Err("\rheroku-cli: Installing core plugins (retrying)...")
			ExitIfError(installPlugins(pluginNames...))
		}
	})
}

func difference(a, b []string) []string {
	res := make([]string, 0, len(a))
	for _, aa := range a {
		if !contains(b, aa) {
			res = append(res, aa)
		}
	}
	return res
}

func contains(arr []string, s string) bool {
	for _, a := range arr {
		if a == s {
			return true
		}
	}
	return false
}

func installPlugins(names ...string) error {
	for _, name := range names {
		lockPlugin(name)
	}
	defer func() {
		for _, name := range names {
			unlockPlugin(name)
		}
	}()
	err := gode.InstallPackages(names...)
	if err != nil {
		return err
	}
	plugins := make([]*Plugin, len(names))
	for i, name := range names {
		plugin, err := ParsePlugin(name)
		if err != nil {
			return err
		}
		plugins[i] = plugin
	}
	AddPluginsToCache(plugins...)
	return nil
}

func pluginExists(plugin string) bool {
	exists, _ := fileExists(pluginPath(plugin))
	return exists
}

// directory location of plugin
func pluginPath(plugin string) string {
	return filepath.Join(AppDir(), "node_modules", plugin)
}

// lock a plugin for reading
func readLockPlugin(name string) {
	lockfile := updateLockPath + "." + name
	locked, err := golock.IsLocked(lockfile)
	LogIfError(err)
	if locked {
		lockPlugin(name)
		unlockPlugin(name)
	}
}

// lock a plugin for writing
func lockPlugin(name string) {
	LogIfError(golock.Lock(updateLockPath + "." + name))
}

// unlock a plugin
func unlockPlugin(name string) {
	LogIfError(golock.Unlock(updateLockPath + "." + name))
}
<|endoftext|>


<commit_before>package readline

import (
	"bufio"
	"container/list"
	"os"
	"strings"

	"github.com/chzyer/readline/runes"
)

type hisItem struct {
	Source  []rune
	Version int64
	Tmp     []rune
}

func (h *hisItem) Clean() {
	h.Source = nil
	h.Tmp = nil
}

type opHistory struct {
	cfg        *Config
	history    *list.List
	historyVer int64
	current    *list.Element
	fd         *os.File
}

func newOpHistory(cfg *Config) (o *opHistory) {
	o = &opHistory{
		cfg:     cfg,
		history: list.New(),
	}
	if o.cfg.HistoryFile != "" {
		o.historyUpdatePath(o.cfg.HistoryFile)
	}
	return
}

// only called by newOpHistory
func (o *opHistory) historyUpdatePath(path string) {
	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		return
	}
	o.fd = f
	r := bufio.NewReader(o.fd)
	total := 0
	for ; ; total++ {
		line, err := r.ReadSlice('\n')
		if err != nil {
			break
		}
		o.PushHistory([]rune(strings.TrimSpace(string(line))))
		o.CompactHistory()
	}
	if total > o.cfg.HistoryLimit {
		o.HistoryRewrite()
	}
	o.historyVer++
	o.PushHistory(nil)
	return
}

func (o *opHistory) CompactHistory() {
	for o.history.Len() > o.cfg.HistoryLimit {
		o.history.Remove(o.history.Front())
	}
}

func (o *opHistory) HistoryRewrite() {
	if o.cfg.HistoryFile == "" {
		return
	}

	tmpFile := o.cfg.HistoryFile + ".tmp"
	fd, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND, 0666)
	if err != nil {
		return
	}
	defer fd.Close()

	buf := bufio.NewWriter(fd)
	for elem := o.history.Front(); elem != nil; elem = elem.Next() {
		buf.WriteString(string(elem.Value.(*hisItem).Source))
	}
	buf.Flush()

	// replace history file
	if err = os.Rename(tmpFile, o.cfg.HistoryFile); err != nil {
		return
	}

	if o.fd != nil {
		o.fd.Close()
	}
	// fd is write only, just satisfy what we need.
	o.fd = fd
}

func (o *opHistory) CloseHistory() {
	if o.fd != nil {
		o.fd.Close()
	}
}

func (o *opHistory) FindHistoryBck(isNewSearch bool, rs []rune, start int) (int, *list.Element) {
	for elem := o.current; elem != nil; elem = elem.Prev() {
		item := o.showItem(elem.Value)
		if isNewSearch {
			start += len(rs)
		}
		if elem == o.current {
			if len(item) >= start {
				item = item[:start]
			}
		}
		idx := runes.IndexAllBck(item, rs)
		if idx < 0 {
			continue
		}
		return idx, elem
	}
	return -1, nil
}

func (o *opHistory) FindHistoryFwd(isNewSearch bool, rs []rune, start int) (int, *list.Element) {
	for elem := o.current; elem != nil; elem = elem.Next() {
		item := o.showItem(elem.Value)
		if isNewSearch {
			start -= len(rs)
			if start < 0 {
				start = 0
			}
		}
		if elem == o.current {
			if len(item)-1 >= start {
				item = item[start:]
			} else {
				continue
			}
		}
		idx := runes.IndexAll(item, rs)
		if idx < 0 {
			continue
		}
		if elem == o.current {
			idx += start
		}
		return idx, elem
	}
	return -1, nil
}

func (o *opHistory) showItem(obj interface{}) []rune {
	item := obj.(*hisItem)
	if item.Version == o.historyVer {
		return item.Tmp
	}
	return item.Source
}

func (o *opHistory) PrevHistory() []rune {
	if o.current == nil {
		return nil
	}
	current := o.current.Prev()
	if current == nil {
		return nil
	}
	o.current = current
	return o.showItem(current.Value)
}

func (o *opHistory) NextHistory() ([]rune, bool) {
	if o.current == nil {
		return nil, false
	}
	current := o.current.Next()
	if current == nil {
		return nil, false
	}

	o.current = current
	return o.showItem(current.Value), true
}

func (o *opHistory) NewHistory(current []rune) {
	// if just use last command without modify
	// just clean lastest history
	if back := o.history.Back(); back != nil {
		prev := back.Prev()
		if prev != nil {
			use := o.showItem(o.current.Value.(*hisItem))
			if runes.Equal(use, prev.Value.(*hisItem).Source) {
				o.current = o.history.Back()
				o.current.Value.(*hisItem).Clean()
				o.historyVer++
				return
			}
		}
	}
	if len(current) == 0 {
		o.current = o.history.Back()
		if o.current != nil {
			o.current.Value.(*hisItem).Clean()
			o.historyVer++
			return
		}
	}

	if o.current != o.history.Back() {
		// move history item to current command
		use := o.current.Value.(*hisItem)
		o.current = o.history.Back()
		current = use.Tmp
	}

	o.UpdateHistory(current, true)

	// push a new one to commit current command
	o.historyVer++
	o.PushHistory(nil)
}

func (o *opHistory) UpdateHistory(s []rune, commit bool) {
	if o.current == nil {
		o.PushHistory(s)
		return
	}
	r := o.current.Value.(*hisItem)
	r.Version = o.historyVer
	if commit {
		r.Source = make([]rune, len(s))
		copy(r.Source, s)
		if o.fd != nil {
			o.fd.Write([]byte(string(r.Source) + "\n"))
		}
	} else {
		r.Tmp = append(r.Tmp[:0], s...)
	}
	o.current.Value = r
}

func (o *opHistory) PushHistory(s []rune) {
	newCopy := make([]rune, len(s))
	copy(newCopy, s)
	elem := o.history.PushBack(&hisItem{Source: newCopy})
	o.current = elem
}
<commit_msg>fix history rewrite fd close<commit_after>package readline

import (
	"bufio"
	"container/list"
	"os"
	"strings"

	"github.com/chzyer/readline/runes"
)

type hisItem struct {
	Source  []rune
	Version int64
	Tmp     []rune
}

func (h *hisItem) Clean() {
	h.Source = nil
	h.Tmp = nil
}

type opHistory struct {
	cfg        *Config
	history    *list.List
	historyVer int64
	current    *list.Element
	fd         *os.File
}

func newOpHistory(cfg *Config) (o *opHistory) {
	o = &opHistory{
		cfg:     cfg,
		history: list.New(),
	}
	if o.cfg.HistoryFile != "" {
		o.historyUpdatePath(o.cfg.HistoryFile)
	}
	return
}

// only called by newOpHistory
func (o *opHistory) historyUpdatePath(path string) {
	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		return
	}
	o.fd = f
	r := bufio.NewReader(o.fd)
	total := 0
	for ; ; total++ {
		line, err := r.ReadSlice('\n')
		if err != nil {
			break
		}
		o.PushHistory([]rune(strings.TrimSpace(string(line))))
		o.CompactHistory()
	}
	if total > o.cfg.HistoryLimit {
		o.HistoryRewrite()
	}
	o.historyVer++
	o.PushHistory(nil)
	return
}

func (o *opHistory) CompactHistory() {
	for o.history.Len() > o.cfg.HistoryLimit {
		o.history.Remove(o.history.Front())
	}
}

func (o *opHistory) HistoryRewrite() {
	if o.cfg.HistoryFile == "" {
		return
	}

	tmpFile := o.cfg.HistoryFile + ".tmp"
	fd, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND, 0666)
	if err != nil {
		return
	}

	buf := bufio.NewWriter(fd)
	for elem := o.history.Front(); elem != nil; elem = elem.Next() {
		buf.WriteString(string(elem.Value.(*hisItem).Source))
	}
	buf.Flush()

	// replace history file
	if err = os.Rename(tmpFile, o.cfg.HistoryFile); err != nil {
		fd.Close()
		return
	}

	if o.fd != nil {
		o.fd.Close()
	}
	// fd is write only, just satisfy what we need.
	o.fd = fd
}

func (o *opHistory) CloseHistory() {
	if o.fd != nil {
		o.fd.Close()
	}
}

func (o *opHistory) FindHistoryBck(isNewSearch bool, rs []rune, start int) (int, *list.Element) {
	for elem := o.current; elem != nil; elem = elem.Prev() {
		item := o.showItem(elem.Value)
		if isNewSearch {
			start += len(rs)
		}
		if elem == o.current {
			if len(item) >= start {
				item = item[:start]
			}
		}
		idx := runes.IndexAllBck(item, rs)
		if idx < 0 {
			continue
		}
		return idx, elem
	}
	return -1, nil
}

func (o *opHistory) FindHistoryFwd(isNewSearch bool, rs []rune, start int) (int, *list.Element) {
	for elem := o.current; elem != nil; elem = elem.Next() {
		item := o.showItem(elem.Value)
		if isNewSearch {
			start -= len(rs)
			if start < 0 {
				start = 0
			}
		}
		if elem == o.current {
			if len(item)-1 >= start {
				item = item[start:]
			} else {
				continue
			}
		}
		idx := runes.IndexAll(item, rs)
		if idx < 0 {
			continue
		}
		if elem == o.current {
			idx += start
		}
		return idx, elem
	}
	return -1, nil
}

func (o *opHistory) showItem(obj interface{}) []rune {
	item := obj.(*hisItem)
	if item.Version == o.historyVer {
		return item.Tmp
	}
	return item.Source
}

func (o *opHistory) PrevHistory() []rune {
	if o.current == nil {
		return nil
	}
	current := o.current.Prev()
	if current == nil {
		return nil
	}
	o.current = current
	return o.showItem(current.Value)
}

func (o *opHistory) NextHistory() ([]rune, bool) {
	if o.current == nil {
		return nil, false
	}
	current := o.current.Next()
	if current == nil {
		return nil, false
	}

	o.current = current
	return o.showItem(current.Value), true
}

func (o *opHistory) NewHistory(current []rune) {
	// if just use last command without modify
	// just clean lastest history
	if back := o.history.Back(); back != nil {
		prev := back.Prev()
		if prev != nil {
			use := o.showItem(o.current.Value.(*hisItem))
			if runes.Equal(use, prev.Value.(*hisItem).Source) {
				o.current = o.history.Back()
				o.current.Value.(*hisItem).Clean()
				o.historyVer++
				return
			}
		}
	}
	if len(current) == 0 {
		o.current = o.history.Back()
		if o.current != nil {
			o.current.Value.(*hisItem).Clean()
			o.historyVer++
			return
		}
	}

	if o.current != o.history.Back() {
		// move history item to current command
		use := o.current.Value.(*hisItem)
		o.current = o.history.Back()
		current = use.Tmp
	}

	o.UpdateHistory(current, true)

	// push a new one to commit current command
	o.historyVer++
	o.PushHistory(nil)
}

func (o *opHistory) UpdateHistory(s []rune, commit bool) {
	if o.current == nil {
		o.PushHistory(s)
		return
	}
	r := o.current.Value.(*hisItem)
	r.Version = o.historyVer
	if commit {
		r.Source = make([]rune, len(s))
		copy(r.Source, s)
		if o.fd != nil {
			o.fd.Write([]byte(string(r.Source) + "\n"))
		}
	} else {
		r.Tmp = append(r.Tmp[:0], s...)
	}
	o.current.Value = r
}

func (o *opHistory) PushHistory(s []rune) {
	newCopy := make([]rune, len(s))
	copy(newCopy, s)
	elem := o.history.PushBack(&hisItem{Source: newCopy})
	o.current = elem
}
<|endoftext|>


<commit_before>package lint

import (
	"bytes"
	"os/exec"
	"strings"
	"unicode/utf8"

	"github.com/ValeLint/vale/core"
	"github.com/russross/blackfriday"
	"golang.org/x/net/html"
	"matloob.io/regexp"
)

// reStructuredText configuration.
//
// reCodeBlock is used to convert Sphinx-style code directives to the regular
// `::` for rst2html.
var reCodeBlock = regexp.MustCompile(`.. (?:raw|code(?:-block)?):: (\w+)`)
var rstArgs = []string{
	"--quiet",
	"--halt=5",
	"--report=5",
	"--link-stylesheet",
	"--no-file-insertion",
	"--no-toc-backlinks",
	"--no-footnote-backlinks",
	"--no-section-numbering",
}

// AsciiDoc configuration.
var adocArgs = []string{
	"-s",
	"--quiet",
	"--safe-mode",
	"secure",
	"-",
}

// Blackfriday configuration.
var commonHTMLFlags = 0 | blackfriday.HTML_USE_XHTML
var commonExtensions = 0 |
	blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
	blackfriday.EXTENSION_TABLES |
	blackfriday.EXTENSION_FENCED_CODE
var renderer = blackfriday.HtmlRenderer(commonHTMLFlags, "", "")
var options = blackfriday.Options{Extensions: commonExtensions}

// HTML configuration.
var heading = regexp.MustCompile(`^h\d$`)

// skipTags are tags that we don't want to lint.
var skipTags = []string{"script", "style", "pre", "figure"}
var inlineTags = []string{
	"b", "big", "i", "small", "abbr", "acronym", "cite", "dfn", "em", "kbd",
	"strong", "a", "br", "img", "span", "sub", "sup", "code", "tt"}
var tagToScope = map[string]string{
	"th": "text.table.header",
	"td": "text.table.cell",
	"li": "text.list",
}

func (l Linter) lintHTMLTokens(f *core.File, ctx string, fsrc []byte, offset int) {
	var txt string
	var tokt html.TokenType
	var tok html.Token
	var inBlock, inline, skip bool

	punct := []string{".", "?", "!", ",", ":", ";"}
	lines := len(f.Lines) + offset
	buf := bytes.NewBufferString("")
	queue := []string{}
	tagHistory := []string{}

	tokens := html.NewTokenizer(bytes.NewReader(fsrc))
	for {
		tokt = tokens.Next()
		tok = tokens.Token()
		txt = html.UnescapeString(strings.TrimSpace(tok.Data))

		if tokt == html.ErrorToken {
			break
		} else if tokt == html.StartTagToken && core.StringInSlice(txt, skipTags) {
			inBlock = true
		} else if inBlock && core.StringInSlice(txt, skipTags) {
			inBlock = false
		} else if tokt == html.StartTagToken {
			inline = core.StringInSlice(txt, inlineTags)
			skip = (txt == "tt" || txt == "code")
			tagHistory = append(tagHistory, txt)
		} else if tokt == html.CommentToken {
			f.UpdateComments(txt)
		} else if tokt == html.TextToken {
			queue = append(queue, txt)
			if !inBlock {
				first, _ := utf8.DecodeRuneInString(txt)
				if skip {
					txt, _ = core.Substitute(txt, txt, '*')
					txt = "`" + txt + "`"
					skip = false
				}
				if inline && !core.StringInSlice(string(first), punct) {
					txt = " " + txt
				}
				buf.WriteString(txt)
			}
		}

		if tokt == html.EndTagToken && !core.StringInSlice(txt, inlineTags) {
			content := buf.String()
			if content != "" {
				l.lintScope(f, ctx, content, tagHistory, lines)
			}
			for _, s := range queue {
				ctx = updateCtx(ctx, s, html.TextToken)
			}
			queue = []string{}
			tagHistory = []string{}
			buf.Reset()
		}

		ctx = clearElements(ctx, tok)
	}
}

func (l Linter) lintScope(f *core.File, ctx, txt string, tags []string, lines int) {
	for _, tag := range tags {
		scope, match := tagToScope[tag]
		if match || heading.MatchString(tag) {
			if match {
				scope = scope + f.RealExt
			} else {
				scope = "text.heading." + tag + f.RealExt
			}
			txt = strings.TrimLeft(txt, " ")
			l.lintText(f, NewBlock(ctx, txt, scope), lines, 0)
			return
		}
	}
	l.lintProse(f, ctx, txt, lines, 0)
}

func clearElements(ctx string, tok html.Token) string {
	if tok.Data == "img" || tok.Data == "a" {
		for _, a := range tok.Attr {
			if a.Key == "alt" || a.Key == "href" {
				ctx = updateCtx(ctx, a.Val, html.TextToken)
			}
		}
	}
	return ctx
}

func updateCtx(ctx string, txt string, tokt html.TokenType) string {
	var found bool
	if (tokt == html.TextToken || tokt == html.CommentToken) && txt != "" {
		for _, s := range strings.Split(txt, "\n") {
			ctx, found = core.Substitute(ctx, s, '@')
			if !found {
				for _, w := range strings.Fields(s) {
					ctx, _ = core.Substitute(ctx, w, '@')
				}
			}
		}
	}
	return ctx
}

func (l Linter) lintHTML(f *core.File) {
	l.lintHTMLTokens(f, f.Content, []byte(f.Content), 0)
}

func (l Linter) lintMarkdown(f *core.File) {
	html := blackfriday.MarkdownOptions([]byte(f.Content), renderer, options)
	l.lintHTMLTokens(f, f.Content, html, 0)
}

func (l Linter) lintRST(f *core.File, python string, rst2html string) {
	var out bytes.Buffer
	cmd := exec.Command(python, append([]string{rst2html}, rstArgs...)...)
	cmd.Stdin = strings.NewReader(reCodeBlock.ReplaceAllString(f.Content, "::"))
	cmd.Stdout = &out
	if core.CheckError(cmd.Run()) {
		html := bytes.Replace(out.Bytes(), []byte("\r"), []byte(""), -1)
		bodyStart := bytes.Index(html, []byte("\n"))
		if bodyStart < 0 {
			bodyStart = -7
		}
		bodyEnd := bytes.Index(html, []byte("\n"))
		if bodyEnd < 0 || bodyEnd >= len(html) {
			bodyEnd = len(html) - 1
			if bodyEnd < 0 {
				bodyEnd = 0
			}
		}
		l.lintHTMLTokens(f, f.Content, html[bodyStart+7:bodyEnd], 0)
	}
}

func (l Linter) lintADoc(f *core.File, asciidoctor string) {
	var out bytes.Buffer
	cmd := exec.Command(asciidoctor, adocArgs...)
	cmd.Stdin = strings.NewReader(f.Content)
	cmd.Stdout = &out
	if core.CheckError(cmd.Run()) {
		l.lintHTMLTokens(f, f.Content, out.Bytes(), 0)
	}
}
<commit_msg>fix: set `skip` to false at the end of a block<commit_after>package lint

import (
	"bytes"
	"os/exec"
	"strings"
	"unicode/utf8"

	"github.com/ValeLint/vale/core"
	"github.com/russross/blackfriday"
	"golang.org/x/net/html"
	"matloob.io/regexp"
)

// reStructuredText configuration.
//
// reCodeBlock is used to convert Sphinx-style code directives to the regular
// `::` for rst2html.
var reCodeBlock = regexp.MustCompile(`.. (?:raw|code(?:-block)?):: (\w+)`)
var rstArgs = []string{
	"--quiet",
	"--halt=5",
	"--report=5",
	"--link-stylesheet",
	"--no-file-insertion",
	"--no-toc-backlinks",
	"--no-footnote-backlinks",
	"--no-section-numbering",
}

// AsciiDoc configuration.
var adocArgs = []string{
	"-s",
	"--quiet",
	"--safe-mode",
	"secure",
	"-",
}

// Blackfriday configuration.
var commonHTMLFlags = 0 | blackfriday.HTML_USE_XHTML
var commonExtensions = 0 |
	blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
	blackfriday.EXTENSION_TABLES |
	blackfriday.EXTENSION_FENCED_CODE
var renderer = blackfriday.HtmlRenderer(commonHTMLFlags, "", "")
var options = blackfriday.Options{Extensions: commonExtensions}

// HTML configuration.
var heading = regexp.MustCompile(`^h\d$`)

// skipTags are tags that we don't want to lint.
var skipTags = []string{"script", "style", "pre", "figure"}
var inlineTags = []string{
	"b", "big", "i", "small", "abbr", "acronym", "cite", "dfn", "em", "kbd",
	"strong", "a", "br", "img", "span", "sub", "sup", "code", "tt"}
var tagToScope = map[string]string{
	"th": "text.table.header",
	"td": "text.table.cell",
	"li": "text.list",
}

func (l Linter) lintHTMLTokens(f *core.File, ctx string, fsrc []byte, offset int) {
	var txt string
	var tokt html.TokenType
	var tok html.Token
	var inBlock, inline, skip bool

	punct := []string{".", "?", "!", ",", ":", ";"}
	lines := len(f.Lines) + offset
	buf := bytes.NewBufferString("")
	queue := []string{}
	tagHistory := []string{}

	tokens := html.NewTokenizer(bytes.NewReader(fsrc))
	for {
		tokt = tokens.Next()
		tok = tokens.Token()
		txt = html.UnescapeString(strings.TrimSpace(tok.Data))

		if tokt == html.ErrorToken {
			break
		} else if tokt == html.StartTagToken && core.StringInSlice(txt, skipTags) {
			inBlock = true
		} else if inBlock && core.StringInSlice(txt, skipTags) {
			skip, inBlock = false, false
		} else if tokt == html.StartTagToken {
			inline = core.StringInSlice(txt, inlineTags)
			skip = (txt == "tt" || txt == "code")
			tagHistory = append(tagHistory, txt)
		} else if tokt == html.CommentToken {
			f.UpdateComments(txt)
		} else if tokt == html.TextToken {
			queue = append(queue, txt)
			if !inBlock {
				first, _ := utf8.DecodeRuneInString(txt)
				if skip {
					txt, _ = core.Substitute(txt, txt, '*')
					txt = "`" + txt + "`"
					skip = false
				}
				if inline && !core.StringInSlice(string(first), punct) {
					txt = " " + txt
				}
				buf.WriteString(txt)
			}
		}

		if tokt == html.EndTagToken && !core.StringInSlice(txt, inlineTags) {
			content := buf.String()
			if content != "" {
				l.lintScope(f, ctx, content, tagHistory, lines)
			}
			for _, s := range queue {
				ctx = updateCtx(ctx, s, html.TextToken)
			}
			queue = []string{}
			tagHistory = []string{}
			buf.Reset()
		}

		ctx = clearElements(ctx, tok)
	}
}

func (l Linter) lintScope(f *core.File, ctx, txt string, tags []string, lines int) {
	for _, tag := range tags {
		scope, match := tagToScope[tag]
		if match || heading.MatchString(tag) {
			if match {
				scope = scope + f.RealExt
			} else {
				scope = "text.heading." + tag + f.RealExt
			}
			txt = strings.TrimLeft(txt, " ")
			l.lintText(f, NewBlock(ctx, txt, scope), lines, 0)
			return
		}
	}
	l.lintProse(f, ctx, txt, lines, 0)
}

func clearElements(ctx string, tok html.Token) string {
	if tok.Data == "img" || tok.Data == "a" {
		for _, a := range tok.Attr {
			if a.Key == "alt" || a.Key == "href" {
				ctx = updateCtx(ctx, a.Val, html.TextToken)
			}
		}
	}
	return ctx
}

func updateCtx(ctx string, txt string, tokt html.TokenType) string {
	var found bool
	if (tokt == html.TextToken || tokt == html.CommentToken) && txt != "" {
		for _, s := range strings.Split(txt, "\n") {
			ctx, found = core.Substitute(ctx, s, '@')
			if !found {
				for _, w := range strings.Fields(s) {
					ctx, _ = core.Substitute(ctx, w, '@')
				}
			}
		}
	}
	return ctx
}

func (l Linter) lintHTML(f *core.File) {
	l.lintHTMLTokens(f, f.Content, []byte(f.Content), 0)
}

func (l Linter) lintMarkdown(f *core.File) {
	html := blackfriday.MarkdownOptions([]byte(f.Content), renderer, options)
	l.lintHTMLTokens(f, f.Content, html, 0)
}

func (l Linter) lintRST(f *core.File, python string, rst2html string) {
	var out bytes.Buffer
	cmd := exec.Command(python, append([]string{rst2html}, rstArgs...)...)
	cmd.Stdin = strings.NewReader(reCodeBlock.ReplaceAllString(f.Content, "::"))
	cmd.Stdout = &out
	if core.CheckError(cmd.Run()) {
		html := bytes.Replace(out.Bytes(), []byte("\r"), []byte(""), -1)
		bodyStart := bytes.Index(html, []byte("\n"))
		if bodyStart < 0 {
			bodyStart = -7
		}
		bodyEnd := bytes.Index(html, []byte("\n"))
		if bodyEnd < 0 || bodyEnd >= len(html) {
			bodyEnd = len(html) - 1
			if bodyEnd < 0 {
				bodyEnd = 0
			}
		}
		l.lintHTMLTokens(f, f.Content, html[bodyStart+7:bodyEnd], 0)
	}
}

func (l Linter) lintADoc(f *core.File, asciidoctor string) {
	var out bytes.Buffer
	cmd := exec.Command(asciidoctor, adocArgs...)
	cmd.Stdin = strings.NewReader(f.Content)
	cmd.Stdout = &out
	if core.CheckError(cmd.Run()) {
		l.lintHTMLTokens(f, f.Content, out.Bytes(), 0)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"strconv"
	"syscall"

    "github.com/dchest/uniuri"
	"github.com/gin-gonic/gin"
	"github.com/jinzhu/gorm"
	_ "github.com/mattn/go-sqlite3"
)

type LMVFile struct {
	Id        int64      `json:"-"`
	Size      int64      `bson:"size"         json:"size"         binding:"required"`
	Name      string     `bson:"name"         json:"name"         binding:"required"`
	Algorithm string     `bson:"algorithm"    json:"algorithm"    binding:"required"`
	Chunks    []LMVChunk `bson:"chunks"       json:"chunks"       binding:"required"`
	Tar       bool       `bson:"tar"          json:"tar"`
	Token     string     `bson:"token"        json:"token"`
}

type LMVChunk struct {
	Id        int64  `json:"-"`
	LMVFileId int64  `json:"-"`
	Hash      string `bson:"hash"         json:"hash"         binding:"required"`
	Size      int64  `bson:"size"         json:"size"         binding:"required"`
	Index     int    `bson:"index"        json:"index"        binding:"required"`
}

const (
	token_length = 10
)

func GenerateToken() string {

    pool := []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
    return uniuri.NewLenChars(token_length, pool)

}

func main() {

	pid := flag.Bool("pid", false, "Save the PID to lmv-server.pid")

	flag.Parse()

	if *pid {
		ioutil.WriteFile("lmv-tracker.pid", []byte(strconv.Itoa(syscall.Getpid())), 0644)
	}

	db, err := gorm.Open("sqlite3", "lmv-tracker.db")

	if err != nil {
		log.Fatal(err)
	}

	db.DB()
	db.DB().Ping()

	db.CreateTable(LMVFile{})
	db.CreateTable(LMVChunk{})

	r := gin.Default()

	r.GET("/files/", func(gc *gin.Context) {
		var lmvfiles []LMVFile
		var response []LMVFile

		db.Find(&lmvfiles)

		for _, file := range lmvfiles {

			var chunks []LMVChunk

			db.Where(&LMVChunk{LMVFileId: file.Id}).Find(&chunks)

			file.Chunks = chunks

			response = append(response, file)
		}

		gc.JSON(200, response)
	})

	r.GET("/files/:token/", func(gc *gin.Context) {

		token := gc.Params.ByName("token")
		var lmv_file LMVFile
		var response LMVFile

		db.Where(&LMVFile{Token: token}).First(&lmv_file)

		if lmv_file.Name != "" {

			var chunks []LMVChunk

			db.Where(&LMVChunk{LMVFileId: lmv_file.Id}).Find(&chunks)

			response = lmv_file
			response.Chunks = chunks

			gc.JSON(200, response)

		} else {

			gc.JSON(404, nil)

		}

	})

	r.POST("/files/", func(gc *gin.Context) {

		token := GenerateToken()
		var lmv_file LMVFile

		gc.Bind(&lmv_file)

		for {

			var testFile = LMVFile{}
			db.Where(&LMVFile{Token: token}).First(&testFile)

			if testFile.Name != "" {

				token = GenerateToken()

			} else {

				break

			}

		}

		lmv_file.Token = token

		db.Create(&lmv_file)

		gc.JSON(200, map[string]interface{}{"token": token})

	})

	r.GET("/ping/", func(c *gin.Context) {
		c.String(200, "pong")
	})

	r.Run(":8080")

}
<commit_msg>Use tsuru's config package to process a config file<commit_after>package main

import (
	"io/ioutil"
	"log"
	"strconv"
	"syscall"

    "github.com/tsuru/config"
    "github.com/dchest/uniuri"
	"github.com/gin-gonic/gin"
	"github.com/jinzhu/gorm"
	_ "github.com/mattn/go-sqlite3"
)

type LMVFile struct {
	Id        int64      `json:"-"`
	Size      int64      `bson:"size"         json:"size"         binding:"required"`
	Name      string     `bson:"name"         json:"name"         binding:"required"`
	Algorithm string     `bson:"algorithm"    json:"algorithm"    binding:"required"`
	Chunks    []LMVChunk `bson:"chunks"       json:"chunks"       binding:"required"`
	Tar       bool       `bson:"tar"          json:"tar"`
	Token     string     `bson:"token"        json:"token"`
}

type LMVChunk struct {
	Id        int64  `json:"-"`
	LMVFileId int64  `json:"-"`
	Hash      string `bson:"hash"         json:"hash"         binding:"required"`
	Size      int64  `bson:"size"         json:"size"         binding:"required"`
	Index     int    `bson:"index"        json:"index"        binding:"required"`
}

type Configuration struct {
    Web     struct {
        Address     string
    }
    System  struct {
        Pid         bool
    }
}

const (
	token_length = 10
)

func GenerateToken() string {

    pool := []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
    return uniuri.NewLenChars(token_length, pool)

}

func processConfig() Configuration {

    conf := Configuration{}

    config.ReadConfigFile("lmv-tracker.yml")

    address, _ := config.Get("web:address")
    conf.Web.Address = address.(string)

    pid, _ := config.GetBool("system:pid")
    conf.System.Pid = pid

    return conf
}

func main() {

    conf := processConfig()

	if conf.System.Pid {
		ioutil.WriteFile("lmv-tracker.pid", []byte(strconv.Itoa(syscall.Getpid())), 0644)
	}

	db, err := gorm.Open("sqlite3", "lmv-tracker.db")

	if err != nil {
		log.Fatal(err)
	}

	db.DB()
	db.DB().Ping()

	db.CreateTable(LMVFile{})
	db.CreateTable(LMVChunk{})

	r := gin.Default()

	r.GET("/files/", func(gc *gin.Context) {
		var lmvfiles []LMVFile
		var response []LMVFile

		db.Find(&lmvfiles)

		for _, file := range lmvfiles {

			var chunks []LMVChunk

			db.Where(&LMVChunk{LMVFileId: file.Id}).Find(&chunks)

			file.Chunks = chunks

			response = append(response, file)
		}

		gc.JSON(200, response)
	})

	r.GET("/files/:token/", func(gc *gin.Context) {

		token := gc.Params.ByName("token")
		var lmv_file LMVFile
		var response LMVFile

		db.Where(&LMVFile{Token: token}).First(&lmv_file)

		if lmv_file.Name != "" {

			var chunks []LMVChunk

			db.Where(&LMVChunk{LMVFileId: lmv_file.Id}).Find(&chunks)

			response = lmv_file
			response.Chunks = chunks

			gc.JSON(200, response)

		} else {

			gc.JSON(404, nil)

		}

	})

	r.POST("/files/", func(gc *gin.Context) {

		token := GenerateToken()
		var lmv_file LMVFile

		gc.Bind(&lmv_file)

		for {

			var testFile = LMVFile{}
			db.Where(&LMVFile{Token: token}).First(&testFile)

			if testFile.Name != "" {

				token = GenerateToken()

			} else {

				break

			}

		}

		lmv_file.Token = token

		db.Create(&lmv_file)

		gc.JSON(200, map[string]interface{}{"token": token})

	})

	r.GET("/ping/", func(c *gin.Context) {
		c.String(200, "pong")
	})

	r.Run(conf.Web.Address)

}
<|endoftext|>