<commit_before>package registry

import (
	"bytes"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/ivanilves/lstags/tag"
)

// TraceRequests defines if we should print out HTTP request URLs and response headers/bodies
var TraceRequests = false

func getAuthorizationType(authorization string) string {
	return strings.Split(authorization, " ")[0]
}

func getRequestID() string {
	data := make([]byte, 10)

	for i := range data {
		data[i] = byte(rand.Intn(256))
	}

	return fmt.Sprintf("%x", sha256.Sum256(data))[0:7]
}

func httpResponseBody(resp *http.Response) string {
	b, _ := ioutil.ReadAll(resp.Body)
	resp.Body = ioutil.NopCloser(bytes.NewBuffer(b))

	return string(b)
}

func httpRequest(url, authorization, mode string) (*http.Response, error) {
	hc := &http.Client{}
	rid := getRequestID()

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Authorization", authorization)
	req.Header.Set("Accept", "application/json")

	switch mode {
	case "v1":
		req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v1+json")
	case "v2":
		req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v2+json")
	default:
		return nil, errors.New("Unknown request mode: " + mode)
	}

	resp, err := hc.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return nil, errors.New("Bad response status: " + resp.Status + " >> " + url)
	}

	if TraceRequests {
		fmt.Printf("%s|@URL: %s\n", rid, url)
		for k, v := range resp.Header {
			fmt.Printf("%s|@HEADER: %-40s = %s\n", rid, k, v)
		}
		fmt.Printf("%s|--- BODY BEGIN ---\n", rid)
		for _, line := range strings.Split(httpResponseBody(resp), "\n") {
			fmt.Printf("%s|%s\n", rid, line)
		}
		fmt.Printf("%s|--- BODY END ---\n", rid)
	}

	return resp, nil
}

type tagNameInfo struct {
	TagNames []string `json:"tags"`
}

func parseTagNamesJSON(data io.ReadCloser) ([]string, error) {
	tn := tagNameInfo{}

	err := json.NewDecoder(data).Decode(&tn)
	if err != nil {
		return nil, err
	}

	return tn.TagNames, nil
}

func fetchTagNames(registry, repo, authorization string) ([]string, error) {
	url := "https://" + registry + "/v2/" + repo + "/tags/list"

	resp, err := httpRequest(url, authorization, "v2")
	if err != nil {
		return nil, err
	}

	tagNames, err := parseTagNamesJSON(resp.Body)
	if err != nil {
		return make([]string, 0), nil
	}

	return tagNames, nil
}

func extractCreatedFromHistory(s string) (int64, error) {
	type v1History struct {
		Created string `json:"created"`
	}

	history := v1History{}

	err := json.Unmarshal([]byte(s), &history)
	if err != nil {
		return 0, err
	}

	t, err := time.Parse(time.RFC3339, history.Created)

	return t.Unix(), nil
}

func fetchCreated(url, authorization string) (int64, error) {
	resp, err := httpRequest(url, authorization, "v1")
	if err != nil {
		return -1, nil
	}

	type manifestResponse struct {
		History []map[string]string `json:"history"`
	}

	mr := manifestResponse{}

	decodingError := json.NewDecoder(resp.Body).Decode(&mr)
	if decodingError != nil {
		return -1, decodingError
	}

	if len(mr.History) > 0 {
		created, err := extractCreatedFromHistory(mr.History[0]["v1Compatibility"])
		if err != nil {
			return -1, err
		}

		return created, nil
	}

	return -1, errors.New("no source to fetch image creation date/time from")
}

func fetchDigest(url, authorization string) (string, error) {
	resp, err := httpRequest(url, authorization, "v2")
	if err != nil {
		return "", err
	}

	digests, defined := resp.Header["Docker-Content-Digest"]
	if !defined {
		return "", errors.New("header 'Docker-Content-Digest' not found in HTTP response")
	}

	return digests[0], nil
}

func fetchDetails(registry, repo, tagName, authorization string) (string, int64, error) {
	url := "https://" + registry + "/v2/" + repo + "/manifests/" + tagName

	dc := make(chan string, 0)
	cc := make(chan int64, 0)
	ec := make(chan error, 0)

	go func(url, authorization string, dc chan string, ec chan error) {
		digest, err := fetchDigest(url, authorization)
		if err != nil {
			ec <- err
		}

		dc <- digest
	}(url, authorization, dc, ec)

	go func(url, authorization string, cc chan int64, ec chan error) {
		created, err := fetchCreated(url, authorization)
		if err != nil {
			ec <- err
		}

		cc <- created
	}(url, authorization, cc, ec)

	var digest string
	var created int64

	waitForDigest := true
	waitForCreated := true
	for waitForDigest || waitForCreated {
		select {
		case digest = <-dc:
			waitForDigest = false
		case created = <-cc:
			waitForCreated = false
		case err := <-ec:
			if err != nil {
				return "", 0, err
			}
		}
	}

	return digest, created, nil
}

type detailResponse struct {
	TagName string
	Digest  string
	Created int64
	Error   error
}

func validateConcurrency(concurrency int) (int, error) {
	const min = 1
	const max = 128

	if concurrency < min {
		return 0, errors.New("Concurrency could not be lower than " + strconv.Itoa(min))
	}

	if concurrency > max {
		return 0, errors.New("Concurrency could not be higher than " + strconv.Itoa(max))
	}

	return concurrency, nil
}

func calculateBatchSteps(count, limit int) (int, int) {
	total := count / limit
	remain := count % limit

	if remain == 0 {
		return total, 0
	}

	return total + 1, remain
}

func calculateBatchStepSize(stepNumber, stepsTotal, remain, limit int) int {
	if remain != 0 && stepNumber == stepsTotal {
		return remain
	}

	return limit
}

// FetchTags looks up Docker repo tags present on remote Docker registry
func FetchTags(registry, repo, authorization string, concurrency int) (map[string]*tag.Tag, error) {
	batchLimit, err := validateConcurrency(concurrency)
	if err != nil {
		return nil, err
	}

	tagNames, err := fetchTagNames(registry, repo, authorization)
	if err != nil {
		return nil, err
	}

	tags := make(map[string]*tag.Tag)

	batchSteps, batchRemain := calculateBatchSteps(len(tagNames), batchLimit)

	var stepSize int
	var tagIndex = 0
	for b := 1; b <= batchSteps; b++ {
		stepSize = calculateBatchStepSize(b, batchSteps, batchRemain, batchLimit)

		ch := make(chan detailResponse, stepSize)

		for s := 1; s <= stepSize; s++ {
			go func(registry, repo, tagName, authorization string, ch chan detailResponse) {
				digest, created, err := fetchDetails(registry, repo, tagName, authorization)

				ch <- detailResponse{TagName: tagName, Digest: digest, Created: created, Error: err}
			}(registry, repo, tagNames[tagIndex], authorization, ch)

			tagIndex++
		}

		for s := 1; s <= stepSize; s++ {
			dr := <-ch

			if dr.Error != nil {
				return nil, dr.Error
			}

			tt, err := tag.New(dr.TagName, dr.Digest)
			if err != nil {
				return nil, err
			}

			tt.SetCreated(dr.Created)

			tags[tt.GetName()] = tt
		}
	}

	return tags, nil
}

// FormatRepoName formats repository name for use with Docker registry
func FormatRepoName(repository, registry string) string {
	if !strings.Contains(repository, "/") {
		return "library/" + repository
	}

	if strings.HasPrefix(repository, registry) {
		return strings.Replace(repository, registry+"/", "", 1)
	}

	return repository
}
<commit_msg>Scout rule: Introduce anonymous structures<commit_after>package registry

import (
	"bytes"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/ivanilves/lstags/tag"
)

// TraceRequests defines if we should print out HTTP request URLs and response headers/bodies
var TraceRequests = false

func getAuthorizationType(authorization string) string {
	return strings.Split(authorization, " ")[0]
}

func getRequestID() string {
	data := make([]byte, 10)

	for i := range data {
		data[i] = byte(rand.Intn(256))
	}

	return fmt.Sprintf("%x", sha256.Sum256(data))[0:7]
}

func httpResponseBody(resp *http.Response) string {
	b, _ := ioutil.ReadAll(resp.Body)
	resp.Body = ioutil.NopCloser(bytes.NewBuffer(b))

	return string(b)
}

func httpRequest(url, authorization, mode string) (*http.Response, error) {
	hc := &http.Client{}
	rid := getRequestID()

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Authorization", authorization)
	req.Header.Set("Accept", "application/json")

	switch mode {
	case "v1":
		req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v1+json")
	case "v2":
		req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v2+json")
	default:
		return nil, errors.New("Unknown request mode: " + mode)
	}

	resp, err := hc.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return nil, errors.New("Bad response status: " + resp.Status + " >> " + url)
	}

	if TraceRequests {
		fmt.Printf("%s|@URL: %s\n", rid, url)
		for k, v := range resp.Header {
			fmt.Printf("%s|@HEADER: %-40s = %s\n", rid, k, v)
		}
		fmt.Printf("%s|--- BODY BEGIN ---\n", rid)
		for _, line := range strings.Split(httpResponseBody(resp), "\n") {
			fmt.Printf("%s|%s\n", rid, line)
		}
		fmt.Printf("%s|--- BODY END ---\n", rid)
	}

	return resp, nil
}

type tagNameInfo struct {
	TagNames []string `json:"tags"`
}

func parseTagNamesJSON(data io.ReadCloser) ([]string, error) {
	tn := tagNameInfo{}

	err := json.NewDecoder(data).Decode(&tn)
	if err != nil {
		return nil, err
	}

	return tn.TagNames, nil
}

func fetchTagNames(registry, repo, authorization string) ([]string, error) {
	url := "https://" + registry + "/v2/" + repo + "/tags/list"

	resp, err := httpRequest(url, authorization, "v2")
	if err != nil {
		return nil, err
	}

	tagNames, err := parseTagNamesJSON(resp.Body)
	if err != nil {
		return make([]string, 0), nil
	}

	return tagNames, nil
}

func extractCreatedFromHistory(s string) (int64, error) {
	var history struct {
		Created string `json:"created"`
	}

	err := json.Unmarshal([]byte(s), &history)
	if err != nil {
		return 0, err
	}

	t, err := time.Parse(time.RFC3339, history.Created)

	return t.Unix(), nil
}

func fetchCreated(url, authorization string) (int64, error) {
	resp, err := httpRequest(url, authorization, "v1")
	if err != nil {
		return -1, nil
	}

	var v1manifest struct {
		History []map[string]string `json:"history"`
	}

	decodingError := json.NewDecoder(resp.Body).Decode(&v1manifest)
	if decodingError != nil {
		return -1, decodingError
	}

	if len(v1manifest.History) > 0 {
		created, err := extractCreatedFromHistory(v1manifest.History[0]["v1Compatibility"])
		if err != nil {
			return -1, err
		}

		return created, nil
	}

	return -1, errors.New("no source to fetch image creation date/time from")
}

func fetchDigest(url, authorization string) (string, error) {
	resp, err := httpRequest(url, authorization, "v2")
	if err != nil {
		return "", err
	}

	digests, defined := resp.Header["Docker-Content-Digest"]
	if !defined {
		return "", errors.New("header 'Docker-Content-Digest' not found in HTTP response")
	}

	return digests[0], nil
}

func fetchDetails(registry, repo, tagName, authorization string) (string, int64, error) {
	url := "https://" + registry + "/v2/" + repo + "/manifests/" + tagName

	dc := make(chan string, 0)
	cc := make(chan int64, 0)
	ec := make(chan error, 0)

	go func(url, authorization string, dc chan string, ec chan error) {
		digest, err := fetchDigest(url, authorization)
		if err != nil {
			ec <- err
		}

		dc <- digest
	}(url, authorization, dc, ec)

	go func(url, authorization string, cc chan int64, ec chan error) {
		created, err := fetchCreated(url, authorization)
		if err != nil {
			ec <- err
		}

		cc <- created
	}(url, authorization, cc, ec)

	var digest string
	var created int64

	waitForDigest := true
	waitForCreated := true
	for waitForDigest || waitForCreated {
		select {
		case digest = <-dc:
			waitForDigest = false
		case created = <-cc:
			waitForCreated = false
		case err := <-ec:
			if err != nil {
				return "", 0, err
			}
		}
	}

	return digest, created, nil
}

type detailResponse struct {
	TagName string
	Digest  string
	Created int64
	Error   error
}

func validateConcurrency(concurrency int) (int, error) {
	const min = 1
	const max = 128

	if concurrency < min {
		return 0, errors.New("Concurrency could not be lower than " + strconv.Itoa(min))
	}

	if concurrency > max {
		return 0, errors.New("Concurrency could not be higher than " + strconv.Itoa(max))
	}

	return concurrency, nil
}

func calculateBatchSteps(count, limit int) (int, int) {
	total := count / limit
	remain := count % limit

	if remain == 0 {
		return total, 0
	}

	return total + 1, remain
}

func calculateBatchStepSize(stepNumber, stepsTotal, remain, limit int) int {
	if remain != 0 && stepNumber == stepsTotal {
		return remain
	}

	return limit
}

// FetchTags looks up Docker repo tags present on remote Docker registry
func FetchTags(registry, repo, authorization string, concurrency int) (map[string]*tag.Tag, error) {
	batchLimit, err := validateConcurrency(concurrency)
	if err != nil {
		return nil, err
	}

	tagNames, err := fetchTagNames(registry, repo, authorization)
	if err != nil {
		return nil, err
	}

	tags := make(map[string]*tag.Tag)

	batchSteps, batchRemain := calculateBatchSteps(len(tagNames), batchLimit)

	var stepSize int
	var tagIndex = 0
	for b := 1; b <= batchSteps; b++ {
		stepSize = calculateBatchStepSize(b, batchSteps, batchRemain, batchLimit)

		ch := make(chan detailResponse, stepSize)

		for s := 1; s <= stepSize; s++ {
			go func(registry, repo, tagName, authorization string, ch chan detailResponse) {
				digest, created, err := fetchDetails(registry, repo, tagName, authorization)

				ch <- detailResponse{TagName: tagName, Digest: digest, Created: created, Error: err}
			}(registry, repo, tagNames[tagIndex], authorization, ch)

			tagIndex++
		}

		for s := 1; s <= stepSize; s++ {
			dr := <-ch

			if dr.Error != nil {
				return nil, dr.Error
			}

			tt, err := tag.New(dr.TagName, dr.Digest)
			if err != nil {
				return nil, err
			}

			tt.SetCreated(dr.Created)

			tags[tt.GetName()] = tt
		}
	}

	return tags, nil
}

// FormatRepoName formats repository name for use with Docker registry
func FormatRepoName(repository, registry string) string {
	if !strings.Contains(repository, "/") {
		return "library/" + repository
	}

	if strings.HasPrefix(repository, registry) {
		return strings.Replace(repository, registry+"/", "", 1)
	}

	return repository
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"fmt"
	"time"

	"github.com/bradfitz/gomemcache/memcache"

	ecache "github.com/dgryski/go-expirecache"
)

type bytesCache interface {
	get(k string) ([]byte, bool)
	set(k string, v []byte, expire int32)
}

type nullCache struct{}

func (_ nullCache) get(string) ([]byte, bool) { return nil, false }
func (_ nullCache) set(string, []byte, int32) {}

type expireCache struct {
	ec *ecache.Cache
}

func (ec expireCache) get(k string) ([]byte, bool) {
	v, ok := ec.ec.Get(k)

	if !ok {
		return nil, false
	}

	return v.([]byte), true
}

func (ec expireCache) set(k string, v []byte, expire int32) {
	ec.ec.Set(k, v, uint64(len(v)), expire)
}

type memcachedCache struct {
	client *memcache.Client
}

func (m *memcachedCache) get(k string) ([]byte, bool) {
	hk := fmt.Sprintf("%x", md5.Sum([]byte(k)))
	done := make(chan bool, 1)

	var err error
	var item *memcache.Item

	go func() {
		item, err = m.client.Get(hk)
		done <- true
	}()

	timeout := time.After(50 * time.Millisecond)

	select {
	case <-timeout:
		Metrics.MemcacheTimeouts.Add(1)
		return nil, false
	case <-done:
	}

	if err != nil {
		return nil, false
	}

	return item.Value, true
}

func (m *memcachedCache) set(k string, v []byte, expire int32) {
	hk := fmt.Sprintf("%x", md5.Sum([]byte(k)))
	go m.client.Set(&memcache.Item{Key: hk, Value: v, Expiration: expire})
}
<commit_msg>remove unnamed receiver<commit_after>package main

import (
	"crypto/md5"
	"fmt"
	"time"

	"github.com/bradfitz/gomemcache/memcache"

	ecache "github.com/dgryski/go-expirecache"
)

type bytesCache interface {
	get(k string) ([]byte, bool)
	set(k string, v []byte, expire int32)
}

type nullCache struct{}

func (nullCache) get(string) ([]byte, bool) { return nil, false }
func (nullCache) set(string, []byte, int32) {}

type expireCache struct {
	ec *ecache.Cache
}

func (ec expireCache) get(k string) ([]byte, bool) {
	v, ok := ec.ec.Get(k)

	if !ok {
		return nil, false
	}

	return v.([]byte), true
}

func (ec expireCache) set(k string, v []byte, expire int32) {
	ec.ec.Set(k, v, uint64(len(v)), expire)
}

type memcachedCache struct {
	client *memcache.Client
}

func (m *memcachedCache) get(k string) ([]byte, bool) {
	hk := fmt.Sprintf("%x", md5.Sum([]byte(k)))
	done := make(chan bool, 1)

	var err error
	var item *memcache.Item

	go func() {
		item, err = m.client.Get(hk)
		done <- true
	}()

	timeout := time.After(50 * time.Millisecond)

	select {
	case <-timeout:
		Metrics.MemcacheTimeouts.Add(1)
		return nil, false
	case <-done:
	}

	if err != nil {
		return nil, false
	}

	return item.Value, true
}

func (m *memcachedCache) set(k string, v []byte, expire int32) {
	hk := fmt.Sprintf("%x", md5.Sum([]byte(k)))
	go m.client.Set(&memcache.Item{Key: hk, Value: v, Expiration: expire})
}
<|endoftext|>


<commit_before>package cache

import (
	"context"
	"encoding/binary"
	"errors"
	"fmt"
	"log"
	"sync/atomic"
	"time"

	"github.com/VictoriaMetrics/fastcache"
	"github.com/go-redis/redis/v8"
	"github.com/klauspost/compress/s2"
	"github.com/vmihailenco/bufpool"
	"github.com/vmihailenco/msgpack/v5"
	"golang.org/x/sync/singleflight"
)

const compressionThreshold = 64
const timeLen = 4

const (
	noCompression = 0x0
	s2Compression = 0x1
)

var ErrCacheMiss = errors.New("cache: key is missing")
var errRedisLocalCacheNil = errors.New("cache: both Redis and LocalCache are nil")

type rediser interface {
	Set(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	SetXX(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.BoolCmd
	SetNX(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.BoolCmd

	Get(ctx context.Context, key string) *redis.StringCmd
	Del(ctx context.Context, keys ...string) *redis.IntCmd
}

type Item struct {
	Ctx context.Context

	Key   string
	Value interface{}

	// TTL is the cache expiration time.
	// Default TTL is 1 hour.
	TTL time.Duration

	// Do returns value to be cached.
	Do func(*Item) (interface{}, error)

	// IfExists only sets the key if it already exist.
	IfExists bool

	// IfNotExists only sets the key if it does not already exist.
	IfNotExists bool

	// SkipLocalCache skips local cache as if it is not set.
	SkipLocalCache bool
}

func (item *Item) Context() context.Context {
	if item.Ctx == nil {
		return context.Background()
	}
	return item.Ctx
}

func (item *Item) value() (interface{}, error) {
	if item.Do != nil {
		return item.Do(item)
	}
	if item.Value != nil {
		return item.Value, nil
	}
	return nil, nil
}

func (item *Item) ttl() time.Duration {
	if item.TTL < 0 {
		return 0
	}
	if item.TTL != 0 && item.TTL < time.Second {
		log.Printf("too short TTL for key=%q: %s", item.Key, item.TTL)
		return time.Hour
	}
	return item.TTL
}

//------------------------------------------------------------------------------

type Options struct {
	Redis rediser

	LocalCache    *fastcache.Cache
	LocalCacheTTL time.Duration

	StatsEnabled bool
}

func (opt *Options) init() {
	switch opt.LocalCacheTTL {
	case -1:
		opt.LocalCacheTTL = 0
	case 0:
		opt.LocalCacheTTL = time.Minute
	}
}

type Cache struct {
	opt *Options

	group   singleflight.Group
	bufpool bufpool.Pool

	hits   uint64
	misses uint64
}

func New(opt *Options) *Cache {
	opt.init()
	return &Cache{
		opt: opt,
	}
}

// Set caches the item.
func (cd *Cache) Set(item *Item) error {
	_, _, err := cd.set(item)
	return err
}

func (cd *Cache) set(item *Item) ([]byte, bool, error) {
	value, err := item.value()
	if err != nil {
		return nil, false, err
	}

	b, err := cd.Marshal(value)
	if err != nil {
		return nil, false, err
	}

	if cd.opt.LocalCache != nil {
		cd.localSet(item.Key, b)
	}

	if cd.opt.Redis == nil {
		if cd.opt.LocalCache == nil {
			return b, true, errRedisLocalCacheNil
		}
		return b, true, nil
	}

	if item.IfExists {
		return b, true, cd.opt.Redis.SetXX(item.Context(), item.Key, b, item.ttl()).Err()
	}

	if item.IfNotExists {
		return b, true, cd.opt.Redis.SetNX(item.Context(), item.Key, b, item.ttl()).Err()
	}

	return b, true, cd.opt.Redis.Set(item.Context(), item.Key, b, item.ttl()).Err()
}

// Exists reports whether value for the given key exists.
func (cd *Cache) Exists(ctx context.Context, key string) bool {
	return cd.Get(ctx, key, nil) == nil
}

// Get gets the value for the given key.
func (cd *Cache) Get(ctx context.Context, key string, value interface{}) error {
	return cd.get(ctx, key, value, false)
}

// Get gets the value for the given key skipping local cache.
func (cd *Cache) GetSkippingLocalCache(
	ctx context.Context, key string, value interface{},
) error {
	return cd.get(ctx, key, value, true)
}

func (cd *Cache) get(
	ctx context.Context,
	key string,
	value interface{},
	skipLocalCache bool,
) error {
	b, err := cd.getBytes(ctx, key, skipLocalCache)
	if err != nil {
		return err
	}
	return cd.Unmarshal(b, value)
}

func (cd *Cache) getBytes(ctx context.Context, key string, skipLocalCache bool) ([]byte, error) {
	if !skipLocalCache && cd.opt.LocalCache != nil {
		b, ok := cd.localGet(key)
		if ok {
			return b, nil
		}
	}

	if cd.opt.Redis == nil {
		if cd.opt.LocalCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return nil, ErrCacheMiss
	}

	b, err := cd.opt.Redis.Get(ctx, key).Bytes()
	if err != nil {
		if cd.opt.StatsEnabled {
			atomic.AddUint64(&cd.misses, 1)
		}
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		return nil, err
	}

	if cd.opt.StatsEnabled {
		atomic.AddUint64(&cd.hits, 1)
	}

	if !skipLocalCache && cd.opt.LocalCache != nil {
		cd.localSet(key, b)
	}
	return b, nil
}

// Once gets the item.Value for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Cache) Once(item *Item) error {
	b, cached, err := cd.getSetItemBytesOnce(item)
	if err != nil {
		return err
	}

	if item.Value == nil || len(b) == 0 {
		return nil
	}

	if err := cd.Unmarshal(b, item.Value); err != nil {
		if cached {
			_ = cd.Delete(item.Context(), item.Key)
			return cd.Once(item)
		}
		return err
	}

	return nil
}

func (cd *Cache) getSetItemBytesOnce(item *Item) (b []byte, cached bool, err error) {
	if cd.opt.LocalCache != nil {
		b, ok := cd.localGet(item.Key)
		if ok {
			return b, true, nil
		}
	}

	v, err, _ := cd.group.Do(item.Key, func() (interface{}, error) {
		b, err := cd.getBytes(item.Context(), item.Key, item.SkipLocalCache)
		if err == nil {
			cached = true
			return b, nil
		}

		b, ok, err := cd.set(item)
		if ok {
			return b, nil
		}
		return nil, err
	})
	if err != nil {
		return nil, false, err
	}
	return v.([]byte), cached, nil
}

func (cd *Cache) Delete(ctx context.Context, key string) error {
	if cd.opt.LocalCache != nil {
		cd.opt.LocalCache.Del([]byte(key))
	}

	if cd.opt.Redis == nil {
		if cd.opt.LocalCache == nil {
			return errRedisLocalCacheNil
		}
		return nil
	}

	_, err := cd.opt.Redis.Del(ctx, key).Result()
	return err
}

func (cd *Cache) localSet(key string, b []byte) {
	if cd.opt.LocalCacheTTL > 0 {
		pos := len(b)
		b = append(b, make([]byte, 4)...)
		encodeTime(b[pos:], time.Now())
	}

	cd.opt.LocalCache.Set([]byte(key), b)
}

func (cd *Cache) localGet(key string) ([]byte, bool) {
	b, ok := cd.opt.LocalCache.HasGet(nil, []byte(key))
	if !ok {
		return b, false
	}

	if len(b) == 0 || cd.opt.LocalCacheTTL == 0 {
		return b, true
	}
	if len(b) < timeLen {
		panic("not reached")
	}

	tm := decodeTime(b[len(b)-timeLen:])
	if time.Since(tm) > cd.opt.LocalCacheTTL {
		cd.opt.LocalCache.Del([]byte(key))
		return nil, false
	}

	return b[:len(b)-timeLen], true
}

func (cd *Cache) Marshal(value interface{}) ([]byte, error) {
	switch value := value.(type) {
	case nil:
		return nil, nil
	case []byte:
		return value, nil
	case string:
		return []byte(value), nil
	}

	buf := cd.bufpool.Get()
	defer cd.bufpool.Put(buf)

	enc := msgpack.GetEncoder()
	enc.Reset(buf)
	enc.UseCompactInts(true)

	err := enc.Encode(value)

	msgpack.PutEncoder(enc)

	if err != nil {
		return nil, err
	}

	return compress(buf.Bytes()), nil
}

func compress(data []byte) []byte {
	if len(data) < compressionThreshold {
		n := len(data) + 1
		b := make([]byte, n, n+timeLen)
		copy(b, data)
		b[len(b)-1] = noCompression
		return b
	}

	n := s2.MaxEncodedLen(len(data)) + 1
	b := make([]byte, n, n+timeLen)
	b = s2.Encode(b, data)
	b = append(b, s2Compression)
	return b
}

func (cd *Cache) Unmarshal(b []byte, value interface{}) error {
	if len(b) == 0 {
		return nil
	}

	switch value := value.(type) {
	case nil:
		return nil
	case *[]byte:
		clone := make([]byte, len(b))
		copy(clone, b)
		*value = clone
		return nil
	case *string:
		*value = string(b)
		return nil
	}

	switch c := b[len(b)-1]; c {
	case noCompression:
		b = b[:len(b)-1]
	case s2Compression:
		b = b[:len(b)-1]

		n, err := s2.DecodedLen(b)
		if err != nil {
			return err
		}

		buf := bufpool.Get(n)
		defer bufpool.Put(buf)

		b, err = s2.Decode(buf.Bytes(), b)
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown compression method: %x", c)
	}

	return msgpack.Unmarshal(b, value)
}

//------------------------------------------------------------------------------

type Stats struct {
	Hits   uint64
	Misses uint64
}

// Stats returns cache statistics.
func (cd *Cache) Stats() *Stats {
	if !cd.opt.StatsEnabled {
		return nil
	}
	return &Stats{
		Hits:   atomic.LoadUint64(&cd.hits),
		Misses: atomic.LoadUint64(&cd.misses),
	}
}

//------------------------------------------------------------------------------

var epoch = time.Date(2020, time.January, 01, 00, 0, 0, 0, time.UTC).Unix()

func encodeTime(b []byte, tm time.Time) {
	secs := tm.Unix() - epoch
	binary.LittleEndian.PutUint32(b, uint32(secs))
}

func decodeTime(b []byte) time.Time {
	secs := binary.LittleEndian.Uint32(b)
	return time.Unix(int64(secs)+epoch, 0)
}
<commit_msg>feat(serialization): use json tag when (de)serialization<commit_after>package cache

import (
	"context"
	"encoding/binary"
	"errors"
	"fmt"
	"log"
	"sync/atomic"
	"time"

	"github.com/VictoriaMetrics/fastcache"
	"github.com/go-redis/redis/v8"
	"github.com/klauspost/compress/s2"
	"github.com/vmihailenco/bufpool"
	"github.com/vmihailenco/msgpack/v5"
	"golang.org/x/sync/singleflight"
)

const compressionThreshold = 64
const timeLen = 4

const (
	noCompression = 0x0
	s2Compression = 0x1
)

var ErrCacheMiss = errors.New("cache: key is missing")
var errRedisLocalCacheNil = errors.New("cache: both Redis and LocalCache are nil")

type rediser interface {
	Set(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	SetXX(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.BoolCmd
	SetNX(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.BoolCmd

	Get(ctx context.Context, key string) *redis.StringCmd
	Del(ctx context.Context, keys ...string) *redis.IntCmd
}

type Item struct {
	Ctx context.Context

	Key   string
	Value interface{}

	// TTL is the cache expiration time.
	// Default TTL is 1 hour.
	TTL time.Duration

	// Do returns value to be cached.
	Do func(*Item) (interface{}, error)

	// IfExists only sets the key if it already exist.
	IfExists bool

	// IfNotExists only sets the key if it does not already exist.
	IfNotExists bool

	// SkipLocalCache skips local cache as if it is not set.
	SkipLocalCache bool
}

func (item *Item) Context() context.Context {
	if item.Ctx == nil {
		return context.Background()
	}
	return item.Ctx
}

func (item *Item) value() (interface{}, error) {
	if item.Do != nil {
		return item.Do(item)
	}
	if item.Value != nil {
		return item.Value, nil
	}
	return nil, nil
}

func (item *Item) ttl() time.Duration {
	if item.TTL < 0 {
		return 0
	}
	if item.TTL != 0 && item.TTL < time.Second {
		log.Printf("too short TTL for key=%q: %s", item.Key, item.TTL)
		return time.Hour
	}
	return item.TTL
}

//------------------------------------------------------------------------------

type Options struct {
	Redis rediser

	LocalCache    *fastcache.Cache
	LocalCacheTTL time.Duration

	StatsEnabled bool
}

func (opt *Options) init() {
	switch opt.LocalCacheTTL {
	case -1:
		opt.LocalCacheTTL = 0
	case 0:
		opt.LocalCacheTTL = time.Minute
	}
}

type Cache struct {
	opt *Options

	group   singleflight.Group
	bufpool bufpool.Pool

	hits   uint64
	misses uint64
}

func New(opt *Options) *Cache {
	opt.init()
	return &Cache{
		opt: opt,
	}
}

// Set caches the item.
func (cd *Cache) Set(item *Item) error {
	_, _, err := cd.set(item)
	return err
}

func (cd *Cache) set(item *Item) ([]byte, bool, error) {
	value, err := item.value()
	if err != nil {
		return nil, false, err
	}

	b, err := cd.Marshal(value)
	if err != nil {
		return nil, false, err
	}

	if cd.opt.LocalCache != nil {
		cd.localSet(item.Key, b)
	}

	if cd.opt.Redis == nil {
		if cd.opt.LocalCache == nil {
			return b, true, errRedisLocalCacheNil
		}
		return b, true, nil
	}

	if item.IfExists {
		return b, true, cd.opt.Redis.SetXX(item.Context(), item.Key, b, item.ttl()).Err()
	}

	if item.IfNotExists {
		return b, true, cd.opt.Redis.SetNX(item.Context(), item.Key, b, item.ttl()).Err()
	}

	return b, true, cd.opt.Redis.Set(item.Context(), item.Key, b, item.ttl()).Err()
}

// Exists reports whether value for the given key exists.
func (cd *Cache) Exists(ctx context.Context, key string) bool {
	return cd.Get(ctx, key, nil) == nil
}

// Get gets the value for the given key.
func (cd *Cache) Get(ctx context.Context, key string, value interface{}) error {
	return cd.get(ctx, key, value, false)
}

// Get gets the value for the given key skipping local cache.
func (cd *Cache) GetSkippingLocalCache(
	ctx context.Context, key string, value interface{},
) error {
	return cd.get(ctx, key, value, true)
}

func (cd *Cache) get(
	ctx context.Context,
	key string,
	value interface{},
	skipLocalCache bool,
) error {
	b, err := cd.getBytes(ctx, key, skipLocalCache)
	if err != nil {
		return err
	}
	return cd.Unmarshal(b, value)
}

func (cd *Cache) getBytes(ctx context.Context, key string, skipLocalCache bool) ([]byte, error) {
	if !skipLocalCache && cd.opt.LocalCache != nil {
		b, ok := cd.localGet(key)
		if ok {
			return b, nil
		}
	}

	if cd.opt.Redis == nil {
		if cd.opt.LocalCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return nil, ErrCacheMiss
	}

	b, err := cd.opt.Redis.Get(ctx, key).Bytes()
	if err != nil {
		if cd.opt.StatsEnabled {
			atomic.AddUint64(&cd.misses, 1)
		}
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		return nil, err
	}

	if cd.opt.StatsEnabled {
		atomic.AddUint64(&cd.hits, 1)
	}

	if !skipLocalCache && cd.opt.LocalCache != nil {
		cd.localSet(key, b)
	}
	return b, nil
}

// Once gets the item.Value for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Cache) Once(item *Item) error {
	b, cached, err := cd.getSetItemBytesOnce(item)
	if err != nil {
		return err
	}

	if item.Value == nil || len(b) == 0 {
		return nil
	}

	if err := cd.Unmarshal(b, item.Value); err != nil {
		if cached {
			_ = cd.Delete(item.Context(), item.Key)
			return cd.Once(item)
		}
		return err
	}

	return nil
}

func (cd *Cache) getSetItemBytesOnce(item *Item) (b []byte, cached bool, err error) {
	if cd.opt.LocalCache != nil {
		b, ok := cd.localGet(item.Key)
		if ok {
			return b, true, nil
		}
	}

	v, err, _ := cd.group.Do(item.Key, func() (interface{}, error) {
		b, err := cd.getBytes(item.Context(), item.Key, item.SkipLocalCache)
		if err == nil {
			cached = true
			return b, nil
		}

		b, ok, err := cd.set(item)
		if ok {
			return b, nil
		}
		return nil, err
	})
	if err != nil {
		return nil, false, err
	}
	return v.([]byte), cached, nil
}

func (cd *Cache) Delete(ctx context.Context, key string) error {
	if cd.opt.LocalCache != nil {
		cd.opt.LocalCache.Del([]byte(key))
	}

	if cd.opt.Redis == nil {
		if cd.opt.LocalCache == nil {
			return errRedisLocalCacheNil
		}
		return nil
	}

	_, err := cd.opt.Redis.Del(ctx, key).Result()
	return err
}

func (cd *Cache) localSet(key string, b []byte) {
	if cd.opt.LocalCacheTTL > 0 {
		pos := len(b)
		b = append(b, make([]byte, 4)...)
		encodeTime(b[pos:], time.Now())
	}

	cd.opt.LocalCache.Set([]byte(key), b)
}

func (cd *Cache) localGet(key string) ([]byte, bool) {
	b, ok := cd.opt.LocalCache.HasGet(nil, []byte(key))
	if !ok {
		return b, false
	}

	if len(b) == 0 || cd.opt.LocalCacheTTL == 0 {
		return b, true
	}
	if len(b) < timeLen {
		panic("not reached")
	}

	tm := decodeTime(b[len(b)-timeLen:])
	if time.Since(tm) > cd.opt.LocalCacheTTL {
		cd.opt.LocalCache.Del([]byte(key))
		return nil, false
	}

	return b[:len(b)-timeLen], true
}

func (cd *Cache) Marshal(value interface{}) ([]byte, error) {
	switch value := value.(type) {
	case nil:
		return nil, nil
	case []byte:
		return value, nil
	case string:
		return []byte(value), nil
	}

	buf := cd.bufpool.Get()
	defer cd.bufpool.Put(buf)

	enc := msgpack.GetEncoder()
	enc.Reset(buf)
	enc.UseCompactInts(true)
	enc.UseJSONTag(true)

	err := enc.Encode(value)

	msgpack.PutEncoder(enc)

	if err != nil {
		return nil, err
	}

	return compress(buf.Bytes()), nil
}

func compress(data []byte) []byte {
	if len(data) < compressionThreshold {
		n := len(data) + 1
		b := make([]byte, n, n+timeLen)
		copy(b, data)
		b[len(b)-1] = noCompression
		return b
	}

	n := s2.MaxEncodedLen(len(data)) + 1
	b := make([]byte, n, n+timeLen)
	b = s2.Encode(b, data)
	b = append(b, s2Compression)
	return b
}

func (cd *Cache) Unmarshal(b []byte, value interface{}) error {
	if len(b) == 0 {
		return nil
	}

	switch value := value.(type) {
	case nil:
		return nil
	case *[]byte:
		clone := make([]byte, len(b))
		copy(clone, b)
		*value = clone
		return nil
	case *string:
		*value = string(b)
		return nil
	}

	switch c := b[len(b)-1]; c {
	case noCompression:
		b = b[:len(b)-1]
	case s2Compression:
		b = b[:len(b)-1]

		n, err := s2.DecodedLen(b)
		if err != nil {
			return err
		}

		buf := bufpool.Get(n)
		defer bufpool.Put(buf)

		b, err = s2.Decode(buf.Bytes(), b)
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown compression method: %x", c)
	}

	return unmarshal(b, value)
}

// modified from github.com/vmihailenco/msgpack/v5@v5.0.0-alpha.2/decode.go:Unmarshal
//
// add dec.UseJSONTag(true)
func unmarshal(data []byte, v interface{}) error {
	dec := msgpack.GetDecoder()

	dec.ResetBytes(data)
	dec.UseJSONTag(true)
	err := dec.Decode(v)

	msgpack.PutDecoder(dec)

	return err
}

//------------------------------------------------------------------------------

type Stats struct {
	Hits   uint64
	Misses uint64
}

// Stats returns cache statistics.
func (cd *Cache) Stats() *Stats {
	if !cd.opt.StatsEnabled {
		return nil
	}
	return &Stats{
		Hits:   atomic.LoadUint64(&cd.hits),
		Misses: atomic.LoadUint64(&cd.misses),
	}
}

//------------------------------------------------------------------------------

var epoch = time.Date(2020, time.January, 01, 00, 0, 0, 0, time.UTC).Unix()

func encodeTime(b []byte, tm time.Time) {
	secs := tm.Unix() - epoch
	binary.LittleEndian.PutUint32(b, uint32(secs))
}

func decodeTime(b []byte) time.Time {
	secs := binary.LittleEndian.Uint32(b)
	return time.Unix(int64(secs)+epoch, 0)
}
<|endoftext|>


<commit_before>package targets

import "fmt"
import "errors"
import "strings"
import "github.com/cnf/go-claw/clog"

type TargetManager struct {
    targets map[string]Target
    target_cmds map[string]map[string]*Command
}

func NewTargetManager() *TargetManager {
    ret := &TargetManager{
            targets    : make(map[string]Target),
            target_cmds: make(map[string]map[string]*Command),
        }
    return ret
}

// Adds a new target using the specified module, given the specified name and parameters
func (t *TargetManager) Add(module, name string, params map[string]string) error {
    // Validate name
    var err error
    if err := validateTargetName(name); err != nil {
        return err
    }
    // check if target already exists
    if _, ok := t.targets[name]; ok {
        clog.Warn("TargetManager::Add(): Target name already existed - removing first")
        t.Remove(name)
    }
    // Check if the requested module exists
    if _, ok := targetlist[module]; !ok {
        return errors.New("could not create target '" + name + "': module '" + module + "' is not registered")
    }
    // Create the target instance
    var tgt Target
    tgt, err = targetlist[module](name, params)
    if err != nil {
        clog.Warn("Could not create %s::%s: %s", module, name, err.Error())
        return err
    }
    t.targets[name] = tgt

    // Fetch the command list
    tcmdlist := tgt.Commands()
    if tcmdlist == nil {
        clog.Warn("warning: %s::%s returned an empty command list!", module, name)
    } else {
        t.target_cmds[name] = tcmdlist
    }

    return nil
}

func (t *TargetManager) Remove(name string) error {
    if _, ok := t.targets[name]; !ok {
        return errors.New("cannot remove " + name + ": does not exist")
    }
    if err := t.targets[name].Stop(); err != nil {
        return err
    }
    delete(t.targets, name)
    if _, ok := t.target_cmds[name]; ok {
        delete(t.target_cmds, name)
    }
    return nil
}

func (t *TargetManager) Stop() error {
    for k := range t.targets {
        if err := t.Remove(k); err != nil {
            return err
        }
    }
    t.targets    = make(map[string]Target)
    t.target_cmds = make(map[string]map[string]*Command)
    return nil
}

func (t *TargetManager) RunCommand(cmdstring string) error {
    splitstr := strings.SplitN(cmdstring, "::", 2)
    if len(splitstr) != 2 {
        return fmt.Errorf("invalid command string '%s', expected it to contain '::'", cmdstring)
    }
    // Validate the name of the target we just parsed out
    if err := validateTargetName(splitstr[0]); err != nil {
        return err
    }
    tgtname := splitstr[0]

    if _, ok := t.targets[tgtname]; !ok {
        return fmt.Errorf("command '%s' uses a target '%s' that does not exist", cmdstring, tgtname)
    }

    // Split the command
    splitcmd := splitQuoted(splitstr[1])
    if len(splitcmd)  == 0 {
        return fmt.Errorf("empty target command in '%s'", cmdstring)
    }
    tcommand := splitcmd[0]
    tparams := splitcmd[1:]

    // Check if the instance provided a commands list to check
    if _, ok := t.target_cmds[tgtname]; ok && t.target_cmds[tgtname] != nil {
        // Check if the command exists for this target
        if _, ok := t.target_cmds[tgtname][tcommand]; !ok {
            return fmt.Errorf("command '%s' not recognized by target '%s'", tcommand, tgtname)
        }
        // Validate all parameters
        pc := 0
        tparams_n := make([]string, 0)
        for prm := range t.target_cmds[tgtname][tcommand].Parameters {
            if pc < len(tparams) {
                // Parameter not present, check if required
                if !t.target_cmds[tgtname][tcommand].Parameters[prm].Optional {
                    return fmt.Errorf("non-optional parameter '%s' missing for command '%s', target '%s'",
                            t.target_cmds[tgtname][tcommand].Parameters[prm].Name,
                            tcommand,
                            tgtname,
                        )
                }
            } else {
                pval, err := t.target_cmds[tgtname][tcommand].Parameters[prm].Validate(tparams[pc])
                if (err != nil) {
                    // validation returned an error
                    return err
                }
                tparams_n = append(tparams_n, pval)
            }
            pc++
        }
        // replace the original parameters with the validated parameters
        tparams = tparams_n
    }
    // Run the command
    return t.targets[tgtname].SendCommand(tcommand, tparams...)
}

func splitQuoted(s string) []string {
    var ret []string
    var curr = make([]rune, len(s))
    var cpos = 0
    var quoted = ' '
    var escaped = false
    sr := strings.NewReader(s)

    for {
        r, _, err := sr.ReadRune()
        if err != nil {
            // Append last
            if cpos != 0 {
                ret = append(ret, string(curr[0:cpos]))
                cpos = 0
            }
            break
        }
        switch r {
        case ' ':
            if quoted != ' ' {
                if escaped {
                    curr[cpos] = '\\'
                    cpos++
                    escaped = false
                }
                curr[cpos] = ' '
                cpos++
            } else if escaped {
                curr[cpos] = ' '
                cpos++
                escaped = false
            } else if cpos != 0 {
                ret = append(ret, string(curr[0:cpos]))
                cpos = 0
            }
        case '\t', '\n', '\r':
            // the other white space - cannot escape this!
            if quoted != ' ' {
                curr[cpos] = r
                cpos++
            } else if cpos != 0 {
                // Add to lst
                ret = append(ret, string(curr[0:cpos]))
                cpos = 0
            }
        case '"', '\'':
            if escaped {
                curr[cpos] = r
                cpos++
                escaped = false
            } else if quoted == r {
                // Quoted string closed - add to list
                ret = append(ret, string(curr[0:cpos]))
                cpos = 0
                quoted = ' '
            } else if quoted == ' ' {
                // New quote, start new entry
                quoted = r
            } else {
                curr[cpos] = r
                cpos++
            }
        case '\\':
            if escaped {
                curr[cpos] = '\\'
                cpos++
                escaped = false
            } else {
                escaped = true
            }
        default:
            if escaped {
                curr[cpos] = '\\'
                cpos++
                escaped = false
            }
            curr[cpos] = r
            cpos++
        }
    }
    return ret
}

func validateTargetName(name string) error {
    if name == "" {
        return errors.New("a target name cannot be empty")
    }
    if strings.ContainsAny(name, "\t\n\r :@!+=*") {
        return fmt.Errorf("target name '%s' cannot contain whitespace, ':', '@', '!', '+', '=' or '*' characters")
    }
    return nil
}

<commit_msg>Updated command splitting function<commit_after>package targets

import "fmt"
import "errors"
import "strings"
import "github.com/cnf/go-claw/clog"

type TargetManager struct {
    targets map[string]Target
    target_cmds map[string]map[string]*Command
}

// Create and initialize a new TargetManager object
func NewTargetManager() *TargetManager {
    ret := &TargetManager{
            targets    : make(map[string]Target),
            target_cmds: make(map[string]map[string]*Command),
        }
    return ret
}

// Adds a new target using the specified module, given the specified name and parameters
func (t *TargetManager) Add(module, name string, params map[string]string) error {
    // Validate name
    var err error
    if err := validateTargetName(name); err != nil {
        return err
    }
    // check if target already exists
    if _, ok := t.targets[name]; ok {
        clog.Warn("TargetManager::Add(): Target name already existed - removing first")
        t.Remove(name)
    }
    // Check if the requested module exists
    if _, ok := targetlist[module]; !ok {
        return errors.New("could not create target '" + name + "': module '" + module + "' is not registered")
    }
    // Create the target instance
    var tgt Target
    tgt, err = targetlist[module](name, params)
    if err != nil {
        clog.Warn("Could not create %s::%s: %s", module, name, err.Error())
        return err
    }
    t.targets[name] = tgt

    // Fetch the command list
    tcmdlist := tgt.Commands()
    if tcmdlist == nil {
        clog.Warn("warning: %s::%s returned an empty command list!", module, name)
    } else {
        t.target_cmds[name] = tcmdlist
    }

    return nil
}

// Remove a target instance from the list
func (t *TargetManager) Remove(name string) error {
    if _, ok := t.targets[name]; !ok {
        return errors.New("cannot remove " + name + ": does not exist")
    }
    if err := t.targets[name].Stop(); err != nil {
        return err
    }
    delete(t.targets, name)
    if _, ok := t.target_cmds[name]; ok {
        delete(t.target_cmds, name)
    }
    return nil
}

// Stops all target instances and removes them
func (t *TargetManager) Stop() error {
    for k := range t.targets {
        if err := t.Remove(k); err != nil {
            return err
        }
    }
    t.targets    = make(map[string]Target)
    t.target_cmds = make(map[string]map[string]*Command)
    return nil
}

// Parses command, determines which target should run it, checks the provided parameters,
// and if all is good - run the command.
func (t *TargetManager) RunCommand(cmdstring string) error {
    splitstr := strings.SplitN(cmdstring, "::", 2)
    if len(splitstr) != 2 {
        return fmt.Errorf("invalid command string '%s', expected it to contain '::'", cmdstring)
    }
    // Validate the name of the target we just parsed out
    if err := validateTargetName(splitstr[0]); err != nil {
        return err
    }
    tgtname := splitstr[0]

    if _, ok := t.targets[tgtname]; !ok {
        return fmt.Errorf("command '%s' uses a target '%s' that does not exist", cmdstring, tgtname)
    }

    // Split the command
    splitcmd := splitQuoted(splitstr[1])
    if len(splitcmd)  == 0 {
        return fmt.Errorf("empty target command in '%s'", cmdstring)
    }
    tcommand := splitcmd[0]
    tparams := splitcmd[1:]

    // Check if the instance provided a commands list to check
    if _, ok := t.target_cmds[tgtname]; ok && t.target_cmds[tgtname] != nil {
        // Check if the command exists for this target
        if _, ok := t.target_cmds[tgtname][tcommand]; !ok {
            return fmt.Errorf("command '%s' not recognized by target '%s'", tcommand, tgtname)
        }
        // Validate all parameters
        pc := 0
        tparams_n := make([]string, 0)
        for prm := range t.target_cmds[tgtname][tcommand].Parameters {
            if pc < len(tparams) {
                // Parameter not present, check if required
                if !t.target_cmds[tgtname][tcommand].Parameters[prm].Optional {
                    return fmt.Errorf("non-optional parameter '%s' missing for command '%s', target '%s'",
                            t.target_cmds[tgtname][tcommand].Parameters[prm].Name,
                            tcommand,
                            tgtname,
                        )
                }
            } else {
                pval, err := t.target_cmds[tgtname][tcommand].Parameters[prm].Validate(tparams[pc])
                if (err != nil) {
                    // validation returned an error
                    return err
                }
                tparams_n = append(tparams_n, pval)
            }
            pc++
        }
        // replace the original parameters with the validated parameters
        tparams = tparams_n
    }
    // Run the command
    return t.targets[tgtname].SendCommand(tcommand, tparams...)
}

// Split a string containing quoted strings on newlines, quotes, ...
// Supports escaping of space, newline, ...
func splitQuoted(s string) []string {
    var ret []string
    var curr = make([]rune, len(s))
    var cpos = 0
    var quoted = ' '
    var escaped = false
    sr := strings.NewReader(s)

    for {
        r, _, err := sr.ReadRune()
        if err != nil {
            // Append last
            if cpos != 0 {
                ret = append(ret, string(curr[0:cpos]))
                cpos = 0
            }
            break
        }
        switch r {
        case ' ':
            if quoted != ' ' {
                if escaped {
                    curr[cpos] = '\\'
                    cpos++
                    escaped = false
                }
                curr[cpos] = ' '
                cpos++
            } else if escaped {
                curr[cpos] = ' '
                cpos++
                escaped = false
            } else if cpos != 0 {
                ret = append(ret, string(curr[0:cpos]))
                cpos = 0
            }
        case '\t', '\n', '\r':
            // the other white space - cannot escape this!
            if quoted != ' ' {
                curr[cpos] = r
                cpos++
            } else if cpos != 0 {
                // Add to lst
                ret = append(ret, string(curr[0:cpos]))
                cpos = 0
            }
        case '"', '\'':
            if escaped {
                curr[cpos] = r
                cpos++
                escaped = false
            } else if quoted == r {
                // Quoted string closed
                // Don't add to list yet, whitespace should follow if
                // it's a new string/parameter, otherwise treat it as
                // the same
                //ret = append(ret, string(curr[0:cpos]))
                //cpos = 0
                quoted = ' '
            } else if quoted == ' ' {
                // New quote, start new entry
                quoted = r
            } else {
                curr[cpos] = r
                cpos++
            }
        case '\\':
            if escaped {
                curr[cpos] = '\\'
                cpos++
                escaped = false
            } else {
                escaped = true
            }
        default:
            if escaped {
                curr[cpos] = '\\'
                cpos++
                escaped = false
            }
            curr[cpos] = r
            cpos++
        }
    }
    return ret
}

func validateTargetName(name string) error {
    if name == "" {
        return errors.New("a target name cannot be empty")
    }
    if strings.ContainsAny(name, "\t\n\r :@!+=*") {
        return fmt.Errorf("target name '%s' cannot contain whitespace, ':', '@', '!', '+', '=' or '*' characters")
    }
    return nil
}

<|endoftext|>


<commit_before>package functions_test

import (
	"context"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/andreyvit/diff"
	"github.com/influxdata/platform/query"
	_ "github.com/influxdata/platform/query/builtin"
	"github.com/influxdata/platform/query/csv"
	"github.com/influxdata/platform/query/influxql"
	"github.com/influxdata/platform/query/querytest"
)

var skipTests = map[string]string{
	"derivative":                "derivative not supported by influxql (https://github.com/influxdata/platform/issues/93)",
	"window_group_mean_ungroup": "error in influxql: failed to run query: timeValue column \"_start\" does not exist (https://github.com/influxdata/platform/issues/97)",
	"string_max":                "error: unimplemented row selector type",
	"null_as_value":             "null not supported as value in influxql (https://github.com/influxdata/platform/issues/353)",
}

// Change as needed

func Test_QueryEndToEnd(t *testing.T) {
	qs := querytest.GetQueryServiceBridge()

	influxqlTranspiler := influxql.NewTranspiler()

	dir, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	path := filepath.Join(dir, "testdata")

	fluxFiles, err := filepath.Glob(filepath.Join(path, "*.flux"))
	if err != nil {
		t.Fatalf("error searching for Flux files: %s", err)
	}

	for _, fluxFile := range fluxFiles {
		ext := filepath.Ext(fluxFile)
		prefix := fluxFile[0 : len(fluxFile)-len(ext)]

		_, caseName := filepath.Split(prefix)
		if reason, ok := skipTests[caseName]; ok {
			t.Run(caseName, func(t *testing.T) {
				t.Skip(reason)
			})
			continue
		}

		fluxName := caseName + ".flux"
		influxqlName := caseName + ".influxql"
		t.Run(fluxName, func(t *testing.T) {
			queryTester(t, qs, prefix, ".flux")
		})
		t.Run(influxqlName, func(t *testing.T) {
			queryTranspileTester(t, influxqlTranspiler, qs, prefix, ".influxql")
		})
	}
}

func queryTester(t *testing.T, qs query.QueryService, prefix, queryExt string) {
	q, err := querytest.GetTestData(prefix, queryExt)
	if err != nil {
		t.Fatal(err)
	}

	csvOut, err := querytest.GetTestData(prefix, ".out.csv")
	if err != nil {
		t.Fatal(err)
	}

	spec, err := query.Compile(context.Background(), q)
	if err != nil {
		t.Fatalf("failed to compile: %v", err)
	}

	csvIn := prefix + ".in.csv"
	enc := csv.NewMultiResultEncoder(csv.DefaultEncoderConfig())
	QueryTestCheckSpec(t, qs, spec, csvIn, csvOut, enc)
}

func queryTranspileTester(t *testing.T, transpiler query.Transpiler, qs query.QueryService, prefix, queryExt string) {
	q, err := querytest.GetTestData(prefix, queryExt)
	if err != nil {
		if os.IsNotExist(err) {
			t.Skip("query missing")
		} else {
			t.Fatal(err)
		}
	}

	csvOut, err := querytest.GetTestData(prefix, ".out.csv")
	if err != nil {
		t.Fatal(err)
	}

	spec, err := transpiler.Transpile(context.Background(), q)
	if err != nil {
		t.Fatalf("failed to transpile: %v", err)
	}

	csvIn := prefix + ".in.csv"
	enc := csv.NewMultiResultEncoder(csv.DefaultEncoderConfig())
	QueryTestCheckSpec(t, qs, spec, csvIn, csvOut, enc)

	enc = influxql.NewMultiResultEncoder()
	jsonOut, err := querytest.GetTestData(prefix, ".out.json")
	if err != nil {
		t.Logf("skipping json evaluation: %s", err)
		return
	}
	QueryTestCheckSpec(t, qs, spec, csvIn, jsonOut, enc)
}

func QueryTestCheckSpec(t *testing.T, qs query.QueryService, spec *query.Spec, inputFile, want string, enc query.MultiResultEncoder) {
	t.Helper()
	querytest.ReplaceFromSpec(spec, inputFile)

	got, err := querytest.GetQueryEncodedResults(qs, spec, inputFile, enc)
	if err != nil {
		t.Errorf("failed to run query: %v", err)
	}
	if g, w := strings.TrimSpace(got), strings.TrimSpace(want); g != w {
		t.Errorf("result not as expected want(-) got (+):\n%v", diff.LineDiff(w, g))
	}
}
<commit_msg>Handle query execution errors in query_test to avoid infinite blocking issues after fail<commit_after>package functions_test

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/influxdata/platform/query"
	_ "github.com/influxdata/platform/query/builtin"
	"github.com/influxdata/platform/query/csv"
	"github.com/influxdata/platform/query/influxql"
	"github.com/influxdata/platform/query/querytest"

	"github.com/andreyvit/diff"
)

var skipTests = map[string]string{
	"derivative":                "derivative not supported by influxql (https://github.com/influxdata/platform/issues/93)",
	"filter_by_tags":            "arbitrary filtering not supported by influxql (https://github.com/influxdata/platform/issues/94)",
	"window_group_mean_ungroup": "error in influxql: failed to run query: timeValue column \"_start\" does not exist (https://github.com/influxdata/platform/issues/97)",
	"string_max":                "error: invalid use of function: *functions.MaxSelector has no implementation for type string",
	"null_as_value":             "null not supported as value in influxql (https://github.com/influxdata/platform/issues/353)",
}

func Test_QueryEndToEnd(t *testing.T) {
	qs := querytest.GetQueryServiceBridge()

	influxqlTranspiler := influxql.NewTranspiler()

	dir, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	path := filepath.Join(dir, "testdata")

	fluxFiles, err := filepath.Glob(filepath.Join(path, "*.flux"))
	if err != nil {
		t.Fatalf("error searching for Flux files: %s", err)
	}

	for _, fluxFile := range fluxFiles {
		ext := filepath.Ext(fluxFile)
		prefix := fluxFile[0 : len(fluxFile)-len(ext)]

		_, caseName := filepath.Split(prefix)
		if reason, ok := skipTests[caseName]; ok {
			t.Run(caseName, func(t *testing.T) {
				t.Skip(reason)
			})
			continue
		}

		fluxName := caseName + ".flux"
		influxqlName := caseName + ".influxql"
		t.Run(fluxName, func(t *testing.T) {
			err := queryTester(t, qs, prefix, ".flux")
			if err != nil {
				qs = querytest.GetQueryServiceBridge()
			}
		})
		t.Run(influxqlName, func(t *testing.T) {
			err := queryTranspileTester(t, influxqlTranspiler, qs, prefix, ".influxql")
			if err != nil {
				qs = querytest.GetQueryServiceBridge()
			}
		})
	}
}

func queryTester(t *testing.T, qs query.QueryService, prefix, queryExt string) error {
	q, err := querytest.GetTestData(prefix, queryExt)
	if err != nil {
		t.Fatal(err)
	}

	csvOut, err := querytest.GetTestData(prefix, ".out.csv")
	if err != nil {
		t.Fatal(err)
	}

	spec, err := query.Compile(context.Background(), q)
	if err != nil {
		t.Fatalf("failed to compile: %v", err)
	}

	csvIn := prefix + ".in.csv"
	enc := csv.NewMultiResultEncoder(csv.DefaultEncoderConfig())

	return QueryTestCheckSpec(t, qs, spec, csvIn, csvOut, enc)
}

func queryTranspileTester(t *testing.T, transpiler query.Transpiler, qs query.QueryService, prefix, queryExt string) error {
	q, err := querytest.GetTestData(prefix, queryExt)
	if err != nil {
		if os.IsNotExist(err) {
			t.Skip("query missing")
		} else {
			t.Fatal(err)
		}
	}

	csvOut, err := querytest.GetTestData(prefix, ".out.csv")
	if err != nil {
		t.Fatal(err)
	}

	spec, err := transpiler.Transpile(context.Background(), q)
	if err != nil {
		t.Fatalf("failed to transpile: %v", err)
	}

	csvIn := prefix + ".in.csv"
	enc := csv.NewMultiResultEncoder(csv.DefaultEncoderConfig())
	QueryTestCheckSpec(t, qs, spec, csvIn, csvOut, enc)

	enc = influxql.NewMultiResultEncoder()
	jsonOut, err := querytest.GetTestData(prefix, ".out.json")
	if err != nil {
		t.Logf("skipping json evaluation: %s", err)
		return nil
	}
	return QueryTestCheckSpec(t, qs, spec, csvIn, jsonOut, enc)
}

func QueryTestCheckSpec(t *testing.T, qs query.QueryService, spec *query.Spec, inputFile, want string, enc query.MultiResultEncoder) error {
	t.Helper()

	querytest.ReplaceFromSpec(spec, inputFile)

	got, err := querytest.GetQueryEncodedResults(qs, spec, inputFile, enc)
	if err != nil {
		t.Errorf("failed to run query: %v", err)
		return fmt.Errorf("Error running query")
	}

	if g, w := strings.TrimSpace(got), strings.TrimSpace(want); g != w {
		t.Errorf("result not as expected want(-) got (+):\n%v", diff.LineDiff(w, g))
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) Mainflux
// SPDX-License-Identifier: Apache-2.0

package cassandra

import (
	"encoding/json"
	"fmt"
	"strconv"

	"github.com/gocql/gocql"
	"github.com/mainflux/mainflux/pkg/errors"
	jsont "github.com/mainflux/mainflux/pkg/transformers/json"
	"github.com/mainflux/mainflux/pkg/transformers/senml"
	"github.com/mainflux/mainflux/readers"
)

var errReadMessages = errors.New("failed to read messages from cassandra database")

const (
	format = "format"
	// Table for SenML messages
	defTable = "messages"
)

var _ readers.MessageRepository = (*cassandraRepository)(nil)

type cassandraRepository struct {
	session *gocql.Session
}

// New instantiates Cassandra message repository.
func New(session *gocql.Session) readers.MessageRepository {
	return cassandraRepository{
		session: session,
	}
}

func (cr cassandraRepository) ReadAll(chanID string, offset, limit uint64, query map[string]string) (readers.MessagesPage, error) {
	table, ok := query[format]
	if !ok {
		table = defTable
	}
	// Remove format filter and format the rest properly.
	delete(query, format)

	q, vals := buildQuery(chanID, offset, limit, query)

	selectCQL := fmt.Sprintf(`SELECT channel, subtopic, publisher, protocol, name, unit,
		value, string_value, bool_value, data_value, sum, time,
		update_time FROM messages WHERE channel = ? %s LIMIT ?
		ALLOW FILTERING`, q)
	if table != defTable {
		selectCQL = fmt.Sprintf(`SELECT channel, subtopic, publisher, protocol, created, payload FROM %s WHERE channel = ? %s LIMIT ?
			ALLOW FILTERING`, table, q)
	}

	countCQL := fmt.Sprintf(`SELECT COUNT(*) FROM %s WHERE channel = ? %s ALLOW FILTERING`, defTable, q)

	iter := cr.session.Query(selectCQL, vals...).Iter()
	defer iter.Close()
	scanner := iter.Scanner()

	// skip first OFFSET rows
	for i := uint64(0); i < offset; i++ {
		if !scanner.Next() {
			break
		}
	}

	page := readers.MessagesPage{
		Offset:   offset,
		Limit:    limit,
		Messages: []readers.Message{},
	}

	switch table {
	case defTable:
		for scanner.Next() {
			var msg senml.Message
			err := scanner.Scan(&msg.Channel, &msg.Subtopic, &msg.Publisher, &msg.Protocol,
				&msg.Name, &msg.Unit, &msg.Value, &msg.StringValue, &msg.BoolValue,
				&msg.DataValue, &msg.Sum, &msg.Time, &msg.UpdateTime)
			if err != nil {
				return readers.MessagesPage{}, errors.Wrap(errReadMessages, err)
			}
			page.Messages = append(page.Messages, msg)
		}
	default:
		for scanner.Next() {
			var msg jsonMessage
			err := scanner.Scan(&msg.Channel, &msg.Subtopic, &msg.Publisher, &msg.Protocol, &msg.Created, &msg.Payload)
			if err != nil {
				return readers.MessagesPage{}, errors.Wrap(errReadMessages, err)
			}
			m, err := msg.toMap()
			if err != nil {
				return readers.MessagesPage{}, errors.Wrap(errReadMessages, err)
			}
			m["payload"] = jsont.ParseFlat(m["payload"])
			page.Messages = append(page.Messages, m)
		}
	}

	if err := cr.session.Query(countCQL, vals[:len(vals)-1]...).Scan(&page.Total); err != nil {
		return readers.MessagesPage{}, errors.Wrap(errReadMessages, err)
	}

	return page, nil
}

func buildQuery(chanID string, offset, limit uint64, query map[string]string) (string, []interface{}) {
	var condCQL string
	vals := []interface{}{chanID}

	for name, val := range query {
		switch name {
		case
			"channel",
			"subtopic",
			"publisher",
			"name",
			"protocol":
			vals = append(vals, val)
			condCQL = fmt.Sprintf(`%s AND %s = ?`, condCQL, name)
		case "v":
			fVal, err := strconv.ParseFloat(val, 64)
			if err != nil {
				continue
			}
			vals = append(vals, fVal)
			condCQL = fmt.Sprintf(`%s AND value = ?`, condCQL)
		case "vb":
			bVal, err := strconv.ParseBool(val)
			if err != nil {
				continue
			}
			vals = append(vals, bVal)
			condCQL = fmt.Sprintf(`%s AND bool_value = ?`, condCQL)
		case "vs":
			vals = append(vals, val)
			condCQL = fmt.Sprintf(`%s AND string_value = ?`, condCQL)
		case "vd":
			vals = append(vals, val)
			condCQL = fmt.Sprintf(`%s AND data_value = ?`, condCQL)
		case "from":
			fVal, err := strconv.ParseFloat(val, 64)
			if err != nil {
				continue
			}
			vals = append(vals, fVal)
			condCQL = fmt.Sprintf(`%s AND time >= ?`, condCQL)
		case "to":
			fVal, err := strconv.ParseFloat(val, 64)
			if err != nil {
				continue
			}
			vals = append(vals, fVal)
			condCQL = fmt.Sprintf(`%s AND time < ?`, condCQL)
		}
	}
	vals = append(vals, offset+limit)

	return condCQL, vals
}

type jsonMessage struct {
	ID        string
	Channel   string
	Created   int64
	Subtopic  string
	Publisher string
	Protocol  string
	Payload   string
}

func (msg jsonMessage) toMap() (map[string]interface{}, error) {
	ret := map[string]interface{}{
		"id":        msg.ID,
		"channel":   msg.Channel,
		"created":   msg.Created,
		"subtopic":  msg.Subtopic,
		"publisher": msg.Publisher,
		"protocol":  msg.Protocol,
		"payload":   map[string]interface{}{},
	}
	pld := make(map[string]interface{})
	if err := json.Unmarshal([]byte(msg.Payload), &pld); err != nil {
		return nil, err
	}
	ret["payload"] = pld
	return ret, nil
}
<commit_msg>MF-1061 - Fix cassandra-reader count for json format (#1327)<commit_after>// Copyright (c) Mainflux
// SPDX-License-Identifier: Apache-2.0

package cassandra

import (
	"encoding/json"
	"fmt"
	"strconv"

	"github.com/gocql/gocql"
	"github.com/mainflux/mainflux/pkg/errors"
	jsont "github.com/mainflux/mainflux/pkg/transformers/json"
	"github.com/mainflux/mainflux/pkg/transformers/senml"
	"github.com/mainflux/mainflux/readers"
)

var errReadMessages = errors.New("failed to read messages from cassandra database")

const (
	format = "format"
	// Table for SenML messages
	defTable = "messages"
)

var _ readers.MessageRepository = (*cassandraRepository)(nil)

type cassandraRepository struct {
	session *gocql.Session
}

// New instantiates Cassandra message repository.
func New(session *gocql.Session) readers.MessageRepository {
	return cassandraRepository{
		session: session,
	}
}

func (cr cassandraRepository) ReadAll(chanID string, offset, limit uint64, query map[string]string) (readers.MessagesPage, error) {
	table, ok := query[format]
	if !ok {
		table = defTable
	}
	// Remove format filter and format the rest properly.
	delete(query, format)

	q, vals := buildQuery(chanID, offset, limit, query)

	selectCQL := fmt.Sprintf(`SELECT channel, subtopic, publisher, protocol, name, unit,
		value, string_value, bool_value, data_value, sum, time,
		update_time FROM messages WHERE channel = ? %s LIMIT ?
		ALLOW FILTERING`, q)
	countCQL := fmt.Sprintf(`SELECT COUNT(*) FROM %s WHERE channel = ? %s ALLOW FILTERING`, defTable, q)

	if table != defTable {
		selectCQL = fmt.Sprintf(`SELECT channel, subtopic, publisher, protocol, created, payload FROM %s WHERE channel = ? %s LIMIT ?
			ALLOW FILTERING`, table, q)
		countCQL = fmt.Sprintf(`SELECT COUNT(*) FROM %s WHERE channel = ? %s ALLOW FILTERING`, table, q)
	}

	iter := cr.session.Query(selectCQL, vals...).Iter()
	defer iter.Close()
	scanner := iter.Scanner()

	// skip first OFFSET rows
	for i := uint64(0); i < offset; i++ {
		if !scanner.Next() {
			break
		}
	}

	page := readers.MessagesPage{
		Offset:   offset,
		Limit:    limit,
		Messages: []readers.Message{},
	}

	switch table {
	case defTable:
		for scanner.Next() {
			var msg senml.Message
			err := scanner.Scan(&msg.Channel, &msg.Subtopic, &msg.Publisher, &msg.Protocol,
				&msg.Name, &msg.Unit, &msg.Value, &msg.StringValue, &msg.BoolValue,
				&msg.DataValue, &msg.Sum, &msg.Time, &msg.UpdateTime)
			if err != nil {
				return readers.MessagesPage{}, errors.Wrap(errReadMessages, err)
			}
			page.Messages = append(page.Messages, msg)
		}
	default:
		for scanner.Next() {
			var msg jsonMessage
			err := scanner.Scan(&msg.Channel, &msg.Subtopic, &msg.Publisher, &msg.Protocol, &msg.Created, &msg.Payload)
			if err != nil {
				return readers.MessagesPage{}, errors.Wrap(errReadMessages, err)
			}
			m, err := msg.toMap()
			if err != nil {
				return readers.MessagesPage{}, errors.Wrap(errReadMessages, err)
			}
			m["payload"] = jsont.ParseFlat(m["payload"])
			page.Messages = append(page.Messages, m)
		}
	}

	if err := cr.session.Query(countCQL, vals[:len(vals)-1]...).Scan(&page.Total); err != nil {
		return readers.MessagesPage{}, errors.Wrap(errReadMessages, err)
	}

	return page, nil
}

func buildQuery(chanID string, offset, limit uint64, query map[string]string) (string, []interface{}) {
	var condCQL string
	vals := []interface{}{chanID}

	for name, val := range query {
		switch name {
		case
			"channel",
			"subtopic",
			"publisher",
			"name",
			"protocol":
			vals = append(vals, val)
			condCQL = fmt.Sprintf(`%s AND %s = ?`, condCQL, name)
		case "v":
			fVal, err := strconv.ParseFloat(val, 64)
			if err != nil {
				continue
			}
			vals = append(vals, fVal)
			condCQL = fmt.Sprintf(`%s AND value = ?`, condCQL)
		case "vb":
			bVal, err := strconv.ParseBool(val)
			if err != nil {
				continue
			}
			vals = append(vals, bVal)
			condCQL = fmt.Sprintf(`%s AND bool_value = ?`, condCQL)
		case "vs":
			vals = append(vals, val)
			condCQL = fmt.Sprintf(`%s AND string_value = ?`, condCQL)
		case "vd":
			vals = append(vals, val)
			condCQL = fmt.Sprintf(`%s AND data_value = ?`, condCQL)
		case "from":
			fVal, err := strconv.ParseFloat(val, 64)
			if err != nil {
				continue
			}
			vals = append(vals, fVal)
			condCQL = fmt.Sprintf(`%s AND time >= ?`, condCQL)
		case "to":
			fVal, err := strconv.ParseFloat(val, 64)
			if err != nil {
				continue
			}
			vals = append(vals, fVal)
			condCQL = fmt.Sprintf(`%s AND time < ?`, condCQL)
		}
	}
	vals = append(vals, offset+limit)

	return condCQL, vals
}

type jsonMessage struct {
	ID        string
	Channel   string
	Created   int64
	Subtopic  string
	Publisher string
	Protocol  string
	Payload   string
}

func (msg jsonMessage) toMap() (map[string]interface{}, error) {
	ret := map[string]interface{}{
		"id":        msg.ID,
		"channel":   msg.Channel,
		"created":   msg.Created,
		"subtopic":  msg.Subtopic,
		"publisher": msg.Publisher,
		"protocol":  msg.Protocol,
		"payload":   map[string]interface{}{},
	}
	pld := make(map[string]interface{})
	if err := json.Unmarshal([]byte(msg.Payload), &pld); err != nil {
		return nil, err
	}
	ret["payload"] = pld
	return ret, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interfacer

import (
	"fmt"
	"go/ast"
	"go/token"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

func toDiscard(vu *varUsage) bool {
	if vu.discard {
		return true
	}
	for to := range vu.assigned {
		if toDiscard(to) {
			return true
		}
	}
	return false
}

func (v *visitor) interfaceMatching(vr *types.Var, vu *varUsage) (string, string) {
	if toDiscard(vu) {
		return "", ""
	}
	allFuncs := typeFuncMap(vr.Type())
	called := make(map[string]string, len(vu.calls))
	for fname := range vu.calls {
		called[fname] = allFuncs[fname]
	}
	s := funcMapString(called)
	name := v.ifaceOf(s)
	if name == "" {
		return "", ""
	}
	for t := range vu.usedAs {
		switch x := t.(type) {
		case *types.Interface:
			asMethods := methoderFuncMap(x, false)
			as := funcMapString(asMethods)
			if !assignable(s, as, called, asMethods) {
				return "", ""
			}
		case *types.Basic:
			if x.Kind() != types.UntypedNil {
				return "", ""
			}
		default:
			return "", ""
		}
	}
	return name, s
}

func orderedPkgs(prog *loader.Program) ([]*types.Package, error) {
	// InitialPackages() is not in the order that we passed to it
	// via Import() calls.
	// For now, make it deterministic by sorting import paths
	// alphabetically.
	unordered := prog.InitialPackages()
	paths := make([]string, 0, len(unordered))
	byPath := make(map[string]*types.Package, len(unordered))
	for _, info := range unordered {
		if info.Errors != nil {
			return nil, info.Errors[0]
		}
		path := info.Pkg.Path()
		paths = append(paths, path)
		byPath[path] = info.Pkg
	}
	sort.Sort(ByAlph(paths))
	pkgs := make([]*types.Package, 0, len(unordered))
	for _, path := range paths {
		pkgs = append(pkgs, byPath[path])
	}
	return pkgs, nil
}

// relPathErr makes converts errors by go/types and go/loader that use
// absolute paths into errors with relative paths
func relPathErr(err error) error {
	errStr := fmt.Sprintf("%v", err)
	if !strings.HasPrefix(errStr, "/") {
		return err
	}
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	if strings.HasPrefix(errStr, wd) {
		return fmt.Errorf(errStr[len(wd)+1:])
	}
	return err
}

func CheckArgs(args []string, w io.Writer, verbose bool) error {
	paths, err := recurse(args)
	if err != nil {
		return err
	}
	c := newCache()
	if _, err := c.FromArgs(paths, false); err != nil {
		return err
	}
	prog, err := c.Load()
	if err != nil {
		return err
	}
	pkgs, err := orderedPkgs(prog)
	if err != nil {
		return relPathErr(err)
	}
	c.typesGet(pkgs)
	for _, pkg := range pkgs {
		info := prog.AllPackages[pkg]
		if verbose {
			fmt.Fprintln(w, info.Pkg.Path())
		}
		v := &visitor{
			cache:       c,
			PackageInfo: info,
			w:           w,
			fset:        prog.Fset,
			vars:        make(map[*types.Var]*varUsage),
		}
		for _, f := range info.Files {
			ast.Walk(v, f)
		}
	}
	return nil
}

type varUsage struct {
	calls   map[string]struct{}
	usedAs  map[types.Type]struct{}
	discard bool

	assigned map[*varUsage]struct{}
}

type visitor struct {
	*cache
	*loader.PackageInfo

	w     io.Writer
	fset  *token.FileSet
	signs []*types.Signature
	warns [][]string
	level int

	vars map[*types.Var]*varUsage

	skipNext bool
}

func paramType(sign *types.Signature, i int) types.Type {
	params := sign.Params()
	extra := sign.Variadic() && i >= params.Len()-1
	if !extra {
		if i >= params.Len() {
			// builtins with multiple signatures
			return nil
		}
		return params.At(i).Type()
	}
	last := params.At(params.Len() - 1)
	switch x := last.Type().(type) {
	case *types.Slice:
		return x.Elem()
	default:
		return x
	}
}

func (v *visitor) varUsage(id *ast.Ident) *varUsage {
	vr, ok := v.ObjectOf(id).(*types.Var)
	if !ok {
		return nil
	}
	if vu, e := v.vars[vr]; e {
		return vu
	}
	vu := &varUsage{
		calls:    make(map[string]struct{}),
		usedAs:   make(map[types.Type]struct{}),
		assigned: make(map[*varUsage]struct{}),
	}
	v.vars[vr] = vu
	return vu
}

func (v *visitor) addUsed(id *ast.Ident, as types.Type) {
	if as == nil {
		return
	}
	vu := v.varUsage(id)
	if vu == nil {
		// not a variable
		return
	}
	vu.usedAs[as.Underlying()] = struct{}{}
}

func (v *visitor) addAssign(to, from *ast.Ident) {
	pto := v.varUsage(to)
	pfrom := v.varUsage(from)
	if pto == nil || pfrom == nil {
		// either isn't a variable
		return
	}
	pfrom.assigned[pto] = struct{}{}
}

func (v *visitor) discard(e ast.Expr) {
	id, ok := e.(*ast.Ident)
	if !ok {
		return
	}
	vu := v.varUsage(id)
	if vu == nil {
		// not a variable
		return
	}
	vu.discard = true
}

func (v *visitor) implementsIface(sign *types.Signature) bool {
	s := signString(sign)
	return v.funcOf(s) != ""
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	if v.skipNext {
		v.skipNext = false
		return nil
	}
	var sign *types.Signature
	switch x := node.(type) {
	case *ast.FuncLit:
		sign = v.Types[x].Type.(*types.Signature)
		if v.implementsIface(sign) {
			return nil
		}
	case *ast.FuncDecl:
		sign = v.Defs[x.Name].Type().(*types.Signature)
		if v.implementsIface(sign) {
			return nil
		}
	case *ast.SelectorExpr:
		v.discard(x.X)
	case *ast.UnaryExpr:
		v.discard(x.X)
	case *ast.IndexExpr:
		v.discard(x.X)
	case *ast.IncDecStmt:
		v.discard(x.X)
	case *ast.BinaryExpr:
		if id, ok := x.X.(*ast.Ident); ok {
			v.addUsed(id, v.TypeOf(x.Y))
		}
		if id, ok := x.Y.(*ast.Ident); ok {
			v.addUsed(id, v.TypeOf(x.X))
		}
	case *ast.AssignStmt:
		v.onAssign(x)
	case *ast.CallExpr:
		v.onCall(x)
	case nil:
		if top := v.signs[len(v.signs)-1]; top != nil {
			v.funcEnded(top)
		}
		v.signs = v.signs[:len(v.signs)-1]
	}
	if node != nil {
		v.signs = append(v.signs, sign)
		if sign != nil {
			v.level++
		}
	}
	return v
}

func (v *visitor) onAssign(as *ast.AssignStmt) {
	for i, e := range as.Rhs {
		id, ok := e.(*ast.Ident)
		if !ok {
			continue
		}
		left := as.Lhs[i]
		v.addUsed(id, v.Types[left].Type)
		if lid, ok := left.(*ast.Ident); ok {
			v.addAssign(lid, id)
		}
	}
}

func (v *visitor) onCall(ce *ast.CallExpr) {
	sign, ok := v.Types[ce.Fun].Type.(*types.Signature)
	if !ok {
		return
	}
	switch y := ce.Fun.(type) {
	case *ast.Ident:
		v.skipNext = true
	case *ast.SelectorExpr:
		if _, ok := y.X.(*ast.Ident); ok {
			v.skipNext = true
		}
	}
	for i, e := range ce.Args {
		if id, ok := e.(*ast.Ident); ok {
			v.addUsed(id, paramType(sign, i))
		}
	}
	sel, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return
	}
	left, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	vu := v.varUsage(left)
	if vu == nil {
		// not a variable
		return
	}
	vu.calls[sel.Sel.Name] = struct{}{}
}

func (v *visitor) funcEnded(sign *types.Signature) {
	v.level--
	v.warns = append(v.warns, v.funcWarns(sign))
	if v.level > 0 {
		return
	}
	for i := len(v.warns) - 1; i >= 0; i-- {
		warns := v.warns[i]
		for _, warn := range warns {
			fmt.Fprintln(v.w, warn)
		}
	}
	v.warns = nil
	v.vars = make(map[*types.Var]*varUsage)
}

func (v *visitor) funcWarns(sign *types.Signature) []string {
	var warns []string
	params := sign.Params()
	for i := 0; i < params.Len(); i++ {
		vr := params.At(i)
		vu := v.vars[vr]
		if vu == nil {
			continue
		}
		if warn := v.paramWarn(vr, vu); warn != "" {
			warns = append(warns, warn)
		}
	}
	return warns
}

func (v *visitor) paramWarn(vr *types.Var, vu *varUsage) string {
	ifname, iftype := v.interfaceMatching(vr, vu)
	if ifname == "" {
		return ""
	}
	t := vr.Type()
	if _, haveIface := t.Underlying().(*types.Interface); haveIface {
		if ifname == t.String() {
			return ""
		}
		have := funcMapString(typeFuncMap(t))
		if have == iftype {
			return ""
		}
	}
	pos := v.fset.Position(vr.Pos())
	fname := pos.Filename
	if fname[0] == '/' {
		fname = filepath.Join(v.Pkg.Path(), filepath.Base(fname))
	}
	pname := v.Pkg.Name()
	if strings.HasPrefix(ifname, pname+".") {
		ifname = ifname[len(pname)+1:]
	}
	return fmt.Sprintf("%s:%d:%d: %s can be %s",
		fname, pos.Line, pos.Column, vr.Name(), ifname)
}
<commit_msg>Ignore leading ./ from suggested types<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interfacer

import (
	"fmt"
	"go/ast"
	"go/token"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

func toDiscard(vu *varUsage) bool {
	if vu.discard {
		return true
	}
	for to := range vu.assigned {
		if toDiscard(to) {
			return true
		}
	}
	return false
}

func (v *visitor) interfaceMatching(vr *types.Var, vu *varUsage) (string, string) {
	if toDiscard(vu) {
		return "", ""
	}
	allFuncs := typeFuncMap(vr.Type())
	called := make(map[string]string, len(vu.calls))
	for fname := range vu.calls {
		called[fname] = allFuncs[fname]
	}
	s := funcMapString(called)
	name := v.ifaceOf(s)
	if name == "" {
		return "", ""
	}
	for t := range vu.usedAs {
		switch x := t.(type) {
		case *types.Interface:
			asMethods := methoderFuncMap(x, false)
			as := funcMapString(asMethods)
			if !assignable(s, as, called, asMethods) {
				return "", ""
			}
		case *types.Basic:
			if x.Kind() != types.UntypedNil {
				return "", ""
			}
		default:
			return "", ""
		}
	}
	return name, s
}

func orderedPkgs(prog *loader.Program) ([]*types.Package, error) {
	// InitialPackages() is not in the order that we passed to it
	// via Import() calls.
	// For now, make it deterministic by sorting import paths
	// alphabetically.
	unordered := prog.InitialPackages()
	paths := make([]string, 0, len(unordered))
	byPath := make(map[string]*types.Package, len(unordered))
	for _, info := range unordered {
		if info.Errors != nil {
			return nil, info.Errors[0]
		}
		path := info.Pkg.Path()
		paths = append(paths, path)
		byPath[path] = info.Pkg
	}
	sort.Sort(ByAlph(paths))
	pkgs := make([]*types.Package, 0, len(unordered))
	for _, path := range paths {
		pkgs = append(pkgs, byPath[path])
	}
	return pkgs, nil
}

// relPathErr makes converts errors by go/types and go/loader that use
// absolute paths into errors with relative paths
func relPathErr(err error) error {
	errStr := fmt.Sprintf("%v", err)
	if !strings.HasPrefix(errStr, "/") {
		return err
	}
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	if strings.HasPrefix(errStr, wd) {
		return fmt.Errorf(errStr[len(wd)+1:])
	}
	return err
}

func CheckArgs(args []string, w io.Writer, verbose bool) error {
	paths, err := recurse(args)
	if err != nil {
		return err
	}
	c := newCache()
	if _, err := c.FromArgs(paths, false); err != nil {
		return err
	}
	prog, err := c.Load()
	if err != nil {
		return err
	}
	pkgs, err := orderedPkgs(prog)
	if err != nil {
		return relPathErr(err)
	}
	c.typesGet(pkgs)
	for _, pkg := range pkgs {
		info := prog.AllPackages[pkg]
		if verbose {
			fmt.Fprintln(w, info.Pkg.Path())
		}
		v := &visitor{
			cache:       c,
			PackageInfo: info,
			w:           w,
			fset:        prog.Fset,
			vars:        make(map[*types.Var]*varUsage),
		}
		for _, f := range info.Files {
			ast.Walk(v, f)
		}
	}
	return nil
}

type varUsage struct {
	calls   map[string]struct{}
	usedAs  map[types.Type]struct{}
	discard bool

	assigned map[*varUsage]struct{}
}

type visitor struct {
	*cache
	*loader.PackageInfo

	w     io.Writer
	fset  *token.FileSet
	signs []*types.Signature
	warns [][]string
	level int

	vars map[*types.Var]*varUsage

	skipNext bool
}

func paramType(sign *types.Signature, i int) types.Type {
	params := sign.Params()
	extra := sign.Variadic() && i >= params.Len()-1
	if !extra {
		if i >= params.Len() {
			// builtins with multiple signatures
			return nil
		}
		return params.At(i).Type()
	}
	last := params.At(params.Len() - 1)
	switch x := last.Type().(type) {
	case *types.Slice:
		return x.Elem()
	default:
		return x
	}
}

func (v *visitor) varUsage(id *ast.Ident) *varUsage {
	vr, ok := v.ObjectOf(id).(*types.Var)
	if !ok {
		return nil
	}
	if vu, e := v.vars[vr]; e {
		return vu
	}
	vu := &varUsage{
		calls:    make(map[string]struct{}),
		usedAs:   make(map[types.Type]struct{}),
		assigned: make(map[*varUsage]struct{}),
	}
	v.vars[vr] = vu
	return vu
}

func (v *visitor) addUsed(id *ast.Ident, as types.Type) {
	if as == nil {
		return
	}
	vu := v.varUsage(id)
	if vu == nil {
		// not a variable
		return
	}
	vu.usedAs[as.Underlying()] = struct{}{}
}

func (v *visitor) addAssign(to, from *ast.Ident) {
	pto := v.varUsage(to)
	pfrom := v.varUsage(from)
	if pto == nil || pfrom == nil {
		// either isn't a variable
		return
	}
	pfrom.assigned[pto] = struct{}{}
}

func (v *visitor) discard(e ast.Expr) {
	id, ok := e.(*ast.Ident)
	if !ok {
		return
	}
	vu := v.varUsage(id)
	if vu == nil {
		// not a variable
		return
	}
	vu.discard = true
}

func (v *visitor) implementsIface(sign *types.Signature) bool {
	s := signString(sign)
	return v.funcOf(s) != ""
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	if v.skipNext {
		v.skipNext = false
		return nil
	}
	var sign *types.Signature
	switch x := node.(type) {
	case *ast.FuncLit:
		sign = v.Types[x].Type.(*types.Signature)
		if v.implementsIface(sign) {
			return nil
		}
	case *ast.FuncDecl:
		sign = v.Defs[x.Name].Type().(*types.Signature)
		if v.implementsIface(sign) {
			return nil
		}
	case *ast.SelectorExpr:
		v.discard(x.X)
	case *ast.UnaryExpr:
		v.discard(x.X)
	case *ast.IndexExpr:
		v.discard(x.X)
	case *ast.IncDecStmt:
		v.discard(x.X)
	case *ast.BinaryExpr:
		if id, ok := x.X.(*ast.Ident); ok {
			v.addUsed(id, v.TypeOf(x.Y))
		}
		if id, ok := x.Y.(*ast.Ident); ok {
			v.addUsed(id, v.TypeOf(x.X))
		}
	case *ast.AssignStmt:
		v.onAssign(x)
	case *ast.CallExpr:
		v.onCall(x)
	case nil:
		if top := v.signs[len(v.signs)-1]; top != nil {
			v.funcEnded(top)
		}
		v.signs = v.signs[:len(v.signs)-1]
	}
	if node != nil {
		v.signs = append(v.signs, sign)
		if sign != nil {
			v.level++
		}
	}
	return v
}

func (v *visitor) onAssign(as *ast.AssignStmt) {
	for i, e := range as.Rhs {
		id, ok := e.(*ast.Ident)
		if !ok {
			continue
		}
		left := as.Lhs[i]
		v.addUsed(id, v.Types[left].Type)
		if lid, ok := left.(*ast.Ident); ok {
			v.addAssign(lid, id)
		}
	}
}

func (v *visitor) onCall(ce *ast.CallExpr) {
	sign, ok := v.Types[ce.Fun].Type.(*types.Signature)
	if !ok {
		return
	}
	switch y := ce.Fun.(type) {
	case *ast.Ident:
		v.skipNext = true
	case *ast.SelectorExpr:
		if _, ok := y.X.(*ast.Ident); ok {
			v.skipNext = true
		}
	}
	for i, e := range ce.Args {
		if id, ok := e.(*ast.Ident); ok {
			v.addUsed(id, paramType(sign, i))
		}
	}
	sel, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return
	}
	left, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	vu := v.varUsage(left)
	if vu == nil {
		// not a variable
		return
	}
	vu.calls[sel.Sel.Name] = struct{}{}
}

func (v *visitor) funcEnded(sign *types.Signature) {
	v.level--
	v.warns = append(v.warns, v.funcWarns(sign))
	if v.level > 0 {
		return
	}
	for i := len(v.warns) - 1; i >= 0; i-- {
		warns := v.warns[i]
		for _, warn := range warns {
			fmt.Fprintln(v.w, warn)
		}
	}
	v.warns = nil
	v.vars = make(map[*types.Var]*varUsage)
}

func (v *visitor) funcWarns(sign *types.Signature) []string {
	var warns []string
	params := sign.Params()
	for i := 0; i < params.Len(); i++ {
		vr := params.At(i)
		vu := v.vars[vr]
		if vu == nil {
			continue
		}
		if warn := v.paramWarn(vr, vu); warn != "" {
			warns = append(warns, warn)
		}
	}
	return warns
}

func (v *visitor) paramWarn(vr *types.Var, vu *varUsage) string {
	ifname, iftype := v.interfaceMatching(vr, vu)
	if ifname == "" {
		return ""
	}
	t := vr.Type()
	if _, haveIface := t.Underlying().(*types.Interface); haveIface {
		if ifname == t.String() {
			return ""
		}
		have := funcMapString(typeFuncMap(t))
		if have == iftype {
			return ""
		}
	}
	pos := v.fset.Position(vr.Pos())
	fname := pos.Filename
	if fname[0] == '/' {
		fname = filepath.Join(v.Pkg.Path(), filepath.Base(fname))
	}
	pname := v.Pkg.Name()
	if strings.HasPrefix(ifname, "./") {
		ifname = ifname[2:]
	}
	if strings.HasPrefix(ifname, pname+".") {
		ifname = ifname[len(pname)+1:]
	}
	return fmt.Sprintf("%s:%d:%d: %s can be %s",
		fname, pos.Line, pos.Column, vr.Name(), ifname)
}
<|endoftext|>