<commit_before>package main

import (
	"time"

	"github.com/jasonsoft/napnap"
	"github.com/satori/go.uuid"
)

func upateOrCreateConsumerEndpoint(c *napnap.Context) {
	var target Consumer
	err := c.BindJSON(&target)
	if err != nil {
		panic(AppError{ErrorCode: "INVALID_DATA", Message: err.Error()})
	}

	if len(target.Username) == 0 {
		panic(AppError{ErrorCode: "INVALID_DATA", Message: "username is invalid."})
	}

	if len(target.App) == 0 {
		panic(AppError{ErrorCode: "INVALID_DATA", Message: "app is invalid."})
	}

	consumer := _consumerRepo.GetByUsername(target.App, target.Username)

	if consumer == nil {
		// create consumer
		target.ID = uuid.NewV4().String()
		err = _consumerRepo.Insert(&target)
		if err != nil {
			panic(err)
		}
		c.JSON(201, target)
		return
	}

	// update consumer
	target.ID = consumer.ID
	target.CreatedAt = consumer.CreatedAt
	err = _consumerRepo.Update(&target)
	if err != nil {
		panic(err)
	}
	c.JSON(200, target)
}

func getConsumerEndpoint(c *napnap.Context) {
	consumerID := c.Param("consumer_id")

	if len(consumerID) == 0 {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "consumer was not found"})
	}

	consumer := _consumerRepo.Get(consumerID)
	if consumer == nil {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "consumer was not found"})
	}

	c.JSON(200, consumer)
}

func getConsumerCountEndpoint(c *napnap.Context) {
	result := ApiCount{
		Count: _consumerRepo.Count(),
	}
	c.JSON(200, result)
}

func deletedConsumerEndpoint(c *napnap.Context) {
	consumerID := c.Param("consumer_id")

	if len(consumerID) == 0 {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "consumer was not found"})
	}

	consumer := _consumerRepo.Get(consumerID)
	if consumer == nil {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "consumer was not found"})
	}

	err := _consumerRepo.Delete(consumerID)
	if err != nil {
		panic(err)
	}

	c.JSON(204, nil)
}

func getTokenEndpoint(c *napnap.Context) {
	key := c.Param("key")

	if len(key) == 0 {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "key was not found"})
	}

	token := _tokenRepo.Get(key)
	if token == nil {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "token was not found"})
	}

	c.JSON(200, token)
}

func getTokensEndpoint(c *napnap.Context) {
	consumerId := c.Query("consumer_id")

	if len(consumerId) > 0 {
		result := _tokenRepo.GetByConsumerID(consumerId)
		c.JSON(200, result)
		return
	}

	//TODO: find all tokens and pagination
}

func createTokenEndpoint(c *napnap.Context) {
	var target Token
	err := c.BindJSON(&target)
	if err != nil {
		panic(AppError{ErrorCode: "INVALID_DATA", Message: err.Error()})
	}

	if len(target.ConsumerID) == 0 {
		panic(AppError{ErrorCode: "INVALID_DATA", Message: "consumer_id field was invalid."})
	}

	consumer := _consumerRepo.Get(target.ConsumerID)
	if consumer == nil {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "consumer was not found."})
	}

	if len(target.Key) == 0 {
		target.Key = uuid.NewV4().String()
	}

	now := time.Now().UTC()
	if target.Expiration.IsZero() {
		target.Expiration = now.Add(time.Duration(_config.Token.Timeout) * time.Minute)
	} else {
		if now.After(target.Expiration) {
			panic(AppError{ErrorCode: "INVALID_DATA", Message: "expiration field was invalid."})
		}
	}

	err = _tokenRepo.Insert(&target)
	if err != nil {
		panic(err)
	}
	c.JSON(201, target)
}

func deleteTokenEndpoint(c *napnap.Context) {
	key := c.Param("key")

	if len(key) == 0 {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "token was not found"})
	}

	token := _tokenRepo.Get(key)
	if token == nil {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "token was not found"})
	}

	err := _tokenRepo.Delete(key)
	if err != nil {
		panic(err)
	}

	c.Status(204)
}

func deleteTokensEndpoint(c *napnap.Context) {
	consumerId := c.Query("consumer_id")
	var tokens []*Token

	if len(consumerId) > 0 {
		// get all tokens by consumer id
		tokens = _tokenRepo.GetByConsumerID(consumerId)
	}

	if len(tokens) == 0 {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "tokens were not found"})
	}

	// delete all token by consumer id
	_tokenRepo.DeleteByConsumerID(consumerId)
	c.Status(204)
}

func getStatus(c *napnap.Context) {
	c.JSON(200, _status)
}
<commit_msg>return empty array when tokens were empty<commit_after>package main

import (
	"time"

	"github.com/jasonsoft/napnap"
	"github.com/satori/go.uuid"
)

func upateOrCreateConsumerEndpoint(c *napnap.Context) {
	var target Consumer
	err := c.BindJSON(&target)
	if err != nil {
		panic(AppError{ErrorCode: "INVALID_DATA", Message: err.Error()})
	}

	if len(target.Username) == 0 {
		panic(AppError{ErrorCode: "INVALID_DATA", Message: "username is invalid."})
	}

	if len(target.App) == 0 {
		panic(AppError{ErrorCode: "INVALID_DATA", Message: "app is invalid."})
	}

	consumer := _consumerRepo.GetByUsername(target.App, target.Username)

	if consumer == nil {
		// create consumer
		target.ID = uuid.NewV4().String()
		err = _consumerRepo.Insert(&target)
		if err != nil {
			panic(err)
		}
		c.JSON(201, target)
		return
	}

	// update consumer
	target.ID = consumer.ID
	target.CreatedAt = consumer.CreatedAt
	err = _consumerRepo.Update(&target)
	if err != nil {
		panic(err)
	}
	c.JSON(200, target)
}

func getConsumerEndpoint(c *napnap.Context) {
	consumerID := c.Param("consumer_id")

	if len(consumerID) == 0 {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "consumer was not found"})
	}

	consumer := _consumerRepo.Get(consumerID)
	if consumer == nil {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "consumer was not found"})
	}

	c.JSON(200, consumer)
}

func getConsumerCountEndpoint(c *napnap.Context) {
	result := ApiCount{
		Count: _consumerRepo.Count(),
	}
	c.JSON(200, result)
}

func deletedConsumerEndpoint(c *napnap.Context) {
	consumerID := c.Param("consumer_id")

	if len(consumerID) == 0 {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "consumer was not found"})
	}

	consumer := _consumerRepo.Get(consumerID)
	if consumer == nil {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "consumer was not found"})
	}

	err := _consumerRepo.Delete(consumerID)
	if err != nil {
		panic(err)
	}

	c.JSON(204, nil)
}

func getTokenEndpoint(c *napnap.Context) {
	key := c.Param("key")

	if len(key) == 0 {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "key was not found"})
	}

	token := _tokenRepo.Get(key)
	if token == nil {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "token was not found"})
	}

	c.JSON(200, token)
}

func getTokensEndpoint(c *napnap.Context) {
	consumerId := c.Query("consumer_id")
	if len(consumerId) > 0 {
		result := _tokenRepo.GetByConsumerID(consumerId)
		if result == nil {
			c.JSON(200, []Token{})
			return
		}
		c.JSON(200, result)
		return
	}

	//TODO: find all tokens and pagination
}

func createTokenEndpoint(c *napnap.Context) {
	var target Token
	err := c.BindJSON(&target)
	if err != nil {
		panic(AppError{ErrorCode: "INVALID_DATA", Message: err.Error()})
	}

	if len(target.ConsumerID) == 0 {
		panic(AppError{ErrorCode: "INVALID_DATA", Message: "consumer_id field was invalid."})
	}

	consumer := _consumerRepo.Get(target.ConsumerID)
	if consumer == nil {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "consumer was not found."})
	}

	if len(target.Key) == 0 {
		target.Key = uuid.NewV4().String()
	}

	now := time.Now().UTC()
	if target.Expiration.IsZero() {
		target.Expiration = now.Add(time.Duration(_config.Token.Timeout) * time.Minute)
	} else {
		if now.After(target.Expiration) {
			panic(AppError{ErrorCode: "INVALID_DATA", Message: "expiration field was invalid."})
		}
	}

	err = _tokenRepo.Insert(&target)
	if err != nil {
		panic(err)
	}
	c.JSON(201, target)
}

func deleteTokenEndpoint(c *napnap.Context) {
	key := c.Param("key")

	if len(key) == 0 {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "token was not found"})
	}

	token := _tokenRepo.Get(key)
	if token == nil {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "token was not found"})
	}

	err := _tokenRepo.Delete(key)
	if err != nil {
		panic(err)
	}

	c.Status(204)
}

func deleteTokensEndpoint(c *napnap.Context) {
	consumerId := c.Query("consumer_id")
	var tokens []*Token

	if len(consumerId) > 0 {
		// get all tokens by consumer id
		tokens = _tokenRepo.GetByConsumerID(consumerId)
	}

	if len(tokens) == 0 {
		panic(AppError{ErrorCode: "NOT_FOUND", Message: "tokens were not found"})
	}

	// delete all token by consumer id
	_tokenRepo.DeleteByConsumerID(consumerId)
	c.Status(204)
}

func getStatus(c *napnap.Context) {
	c.JSON(200, _status)
}
<|endoftext|>


<commit_before>// gochan package is a wrapper around io.ReadWriter interface
// to create channels for communication. gochan does this by
// encapsulating io.ReadWriter interface into two separate channels
// for reading and writing.
//
// Data read from the reading channel is  wrapped in the ChanData struct
// which also contains a Err field if there is any error during reading.
// Data can be feed into the writing channel as a slice of bytes directly
// without any wrapping.
//
// When done with gochan, simply close the writing channel through CloseWriteChan()
// method or by closing  the io.ReadWriter interface.
// You should note that closing the writing channel won't affect the reading channel,
// but closing the io.ReadWriter interface will causes both reading and writing channels to be closed.
// Note:
//      1. If you closed io.ReadWriter interface and the writing channel
//         still has buffered data at the point, these data will be lost.To prevent this,
//         use CloseWriteChan() method. CloseWriteChan() will block until the
//         writing channel has finished writing buffered data or encountered errors.
//         After then, you can safely close the io.ReadWriter interface.
//      2. The reading channel won't be closed since it blocks at reading the data
//         from the io.ReadWriter interface. So the cleanest way to cleanup both
//         reading and writing channels is using CloseWriteChan() method followed by
//         a Close() method of the underlying io.ReadWriter interface if any.
//
// Internally, gochan uses two goroutines for reading and writing.
// The reading goroutine will exit when io.EOF or io.ErrClosedPipe is encountered.
// The writing goroutine will exit when there is any error during the writing.
package gochan

import (
	"io"
	"log"
	"sync"
	"sync/atomic"
)

// ChanData contains data and error(if any) during transmission
type ChanData struct {
	Data []byte
	Err  error
}

type empty struct{}

// goChan is a internal struct to represent a proxy between clients and io(network, filesystem etc...)
type goChan struct {
	rc       chan ChanData
	wc       chan []byte
	done     chan empty
	rw       io.ReadWriter
	readSize uint32
}

var (
	mtx sync.Mutex
	rcm map[<-chan ChanData]*goChan
	wcm map[chan<- []byte]*goChan
)

func init() {
	log.SetFlags(log.Flags() | log.Lmicroseconds)
	rcm = make(map[<-chan ChanData]*goChan)
	wcm = make(map[chan<- []byte]*goChan)
}

func readChan(gc *goChan) {
	defer func() {
		_ = recover()
	}()
	defer CloseReadChan(gc.rc)

	for {
		readsize := atomic.LoadUint32(&gc.readSize)
		log.Printf("about to read %d bytes, %p\n", readsize, &gc.readSize)
		b := make([]byte, readsize)
		n, err := gc.rw.Read(b)
		b = b[0:n]
		if err == io.EOF || err == io.ErrClosedPipe {
			gc.rc <- ChanData{b, err}
			break
		} else {
			gc.rc <- ChanData{b, err}
		}
	}
}

func writeChan(gc *goChan) {
	defer CloseWriteChan(gc.wc)
	for {
		s, ok := <-gc.wc
		if !ok { // client closed the channel
			gc.done <- empty{} // work is done
			break
		}
		_, err := gc.rw.Write(s)
		if err != nil {
			log.Printf("write error %v\n", err)
			gc.done <- empty{} // work is done
			break
		}
	}
}

// NewChan creates two channels for reading and writing.
// The reading channel is typed by ChanData struct, while the writing channel is typed by []byte.
// @rw: anything that implements io.ReadWriter interface.
// @rcBufsize: the buffer size of the reading channel, a unbuffered channel will be created if 0 provided.
// @wcBufsize: the buffer size of the writing channel, a unbuffered channel will be created if 0 provided.
// @readSize: the read size for io.Read() method, it might return less bytes.
// Every read operation on the channel will return a ChanData containing data and error if any,
// the actual bytes read can be obtained from len(ChanData.Data).
// Every write operation should provided a ChanData containing a slice of bytes and
// the Err field in the ChanData struct will be ignored.
func NewChan(rw io.ReadWriter, rcBufsize uint, wcBufsize, readSize uint32) (<-chan ChanData, chan<- []byte) {
	mtx.Lock()
	defer mtx.Unlock()
	var gc = goChan{make(chan ChanData, rcBufsize), make(chan []byte, wcBufsize), make(chan empty, 1), rw, readSize}
	rcm[gc.rc] = &gc
	wcm[gc.wc] = &gc
	go readChan(&gc)
	go writeChan(&gc)
	return gc.rc, gc.wc
}

// NewReadonlyChan creates a readonly channel typed by ChanData struct.
// see comments on NewChan() method for details on parameters.
func NewReadonlyChan(rw io.ReadWriter, rcBufsize uint, readSize uint32) <-chan ChanData {
	mtx.Lock()
	defer mtx.Unlock()
	var gc = goChan{rc: make(chan ChanData, rcBufsize), done: make(chan empty, 1), rw: rw, readSize: readSize}
	rcm[gc.rc] = &gc
	go readChan(&gc)
	return gc.rc
}

// NewWriteonlyChan creates a writeonly channel typed by []byte.
// see comments on NewChan() method for details on parameters.
func NewWriteonlyChan(rw io.ReadWriter, wcBufsize uint) chan<- []byte {
	mtx.Lock()
	defer mtx.Unlock()
	var gc = goChan{wc: make(chan []byte, wcBufsize), done: make(chan empty, 1), rw: rw}
	go writeChan(&gc)
	wcm[gc.wc] = &gc
	return gc.wc
}

// CloseWriteChan closes the writing channel.
// it blocks until the writing channel finished writing its
// buffered data or encountered errors.
// @wc: the writing side channel
func CloseWriteChan(wc chan<- []byte) error {
	mtx.Lock()
	defer mtx.Unlock()
	if gc, ok := wcm[wc]; ok {
		delete(wcm, wc)
		close(wc)
		_ = <-gc.done // wait for works to be done
	}

	return nil
}

// CloseReadChan closes the reading channel.
// @wc: the reading side channel
func CloseReadChan(rc <-chan ChanData) error {
	mtx.Lock()
	defer mtx.Unlock()
	if gc, ok := rcm[rc]; ok {
		delete(rcm, rc)
		close(gc.rc)
	}
	return nil
}
<commit_msg>sperate interfaces<commit_after>// gochan package is a wrapper around io.ReadWriter interface
// to create channels for communication. gochan does this by
// encapsulating io.ReadWriter interface into two separate channels
// for reading and writing.
//
// Data read from the reading channel is  wrapped in the ChanData struct
// which also contains a Err field if there is any error during reading.
// Data can be feed into the writing channel as a slice of bytes directly
// without any wrapping.
//
// When done with gochan, simply close the writing channel through CloseWriteChan()
// method or by closing  the io.ReadWriter interface.
// You should note that closing the writing channel won't affect the reading channel,
// but closing the io.ReadWriter interface will causes both reading and writing channels to be closed.
// Note:
//      1. If you closed io.ReadWriter interface and the writing channel
//         still has buffered data at the point, these data will be lost.To prevent this,
//         use CloseWriteChan() method. CloseWriteChan() will block until the
//         writing channel has finished writing buffered data or encountered errors.
//         After then, you can safely close the io.ReadWriter interface.
//      2. The reading channel won't be closed since it blocks at reading the data
//         from the io.ReadWriter interface. So the cleanest way to cleanup both
//         reading and writing channels is using CloseWriteChan() method followed by
//         a Close() method of the underlying io.ReadWriter interface if any.
//
// Internally, gochan uses two goroutines for reading and writing.
// The reading goroutine will exit when io.EOF or io.ErrClosedPipe is encountered.
// The writing goroutine will exit when there is any error during the writing.
package gochan

import (
	"io"
	"log"
	"sync"
	"sync/atomic"
)

// ChanData contains data and error(if any) during transmission
type ChanData struct {
	Data []byte
	Err  error
}

type empty struct{}

// goChan is a internal struct to represent a proxy between clients and io(network, filesystem etc...)
type goChan struct {
	rc       chan ChanData
	wc       chan []byte
	done     chan empty
	r        io.Reader
	w        io.Writer
	readSize uint32
}

var (
	mtx sync.Mutex
	rcm map[<-chan ChanData]*goChan
	wcm map[chan<- []byte]*goChan
)

func init() {
	log.SetFlags(log.Flags() | log.Lmicroseconds)
	rcm = make(map[<-chan ChanData]*goChan)
	wcm = make(map[chan<- []byte]*goChan)
}

func readChan(gc *goChan) {
	defer func() {
		_ = recover()
	}()
	defer CloseReadChan(gc.rc)

	for {
		readsize := atomic.LoadUint32(&gc.readSize)
		b := make([]byte, readsize)
		n, err := gc.r.Read(b)
		b = b[0:n]
		if err == io.EOF || err == io.ErrClosedPipe {
			gc.rc <- ChanData{b, err}
			break
		} else {
			gc.rc <- ChanData{b, err}
		}
	}
}

func writeChan(gc *goChan) {
	defer CloseWriteChan(gc.wc)
	for {
		s, ok := <-gc.wc
		if !ok { // client closed the channel
			gc.done <- empty{} // work is done
			break
		}
		_, err := gc.w.Write(s)
		if err != nil {
			log.Printf("write error %v\n", err)
			gc.done <- empty{} // work is done
			break
		}
	}
}

// NewChan creates two channels for reading and writing.
// The reading channel is typed by ChanData struct, while the writing channel is typed by []byte.
// @rw: anything that implements io.ReadWriter interface.
// @rcBufsize: the buffer size of the reading channel, a unbuffered channel will be created if 0 provided.
// @wcBufsize: the buffer size of the writing channel, a unbuffered channel will be created if 0 provided.
// @readSize: the read size for io.Read() method, it might return less bytes.
// Every read operation on the channel will return a ChanData containing data and error if any,
// the actual bytes read can be obtained from len(ChanData.Data).
// Every write operation should provided a ChanData containing a slice of bytes and
// the Err field in the ChanData struct will be ignored.
func NewChan(rw io.ReadWriter, rcBufsize uint, wcBufsize, readSize uint32) (<-chan ChanData, chan<- []byte) {
	mtx.Lock()
	defer mtx.Unlock()
	var gc = goChan{make(chan ChanData, rcBufsize), make(chan []byte, wcBufsize), make(chan empty, 1), rw, rw, readSize}
	rcm[gc.rc] = &gc
	wcm[gc.wc] = &gc
	go readChan(&gc)
	go writeChan(&gc)
	return gc.rc, gc.wc
}

// NewReadonlyChan creates a readonly channel typed by ChanData struct.
// see comments on NewChan() method for details on parameters.
func NewReadonlyChan(r io.Reader, rcBufsize uint, readSize uint32) <-chan ChanData {
	mtx.Lock()
	defer mtx.Unlock()
	var gc = goChan{rc: make(chan ChanData, rcBufsize), done: make(chan empty, 1), r: r, readSize: readSize}
	rcm[gc.rc] = &gc
	go readChan(&gc)
	return gc.rc
}

// NewWriteonlyChan creates a writeonly channel typed by []byte.
// see comments on NewChan() method for details on parameters.
func NewWriteonlyChan(w io.Writer, wcBufsize uint) chan<- []byte {
	mtx.Lock()
	defer mtx.Unlock()
	var gc = goChan{wc: make(chan []byte, wcBufsize), done: make(chan empty, 1), w: w}
	go writeChan(&gc)
	wcm[gc.wc] = &gc
	return gc.wc
}

// CloseWriteChan closes the writing channel.
// it blocks until the writing channel finished writing its
// buffered data or encountered errors.
// @wc: the writing side channel
func CloseWriteChan(wc chan<- []byte) error {
	mtx.Lock()
	defer mtx.Unlock()
	if gc, ok := wcm[wc]; ok {
		delete(wcm, wc)
		close(wc)
		_ = <-gc.done // wait for works to be done
	}

	return nil
}

// CloseReadChan closes the reading channel.
// @wc: the reading side channel
func CloseReadChan(rc <-chan ChanData) error {
	mtx.Lock()
	defer mtx.Unlock()
	if gc, ok := rcm[rc]; ok {
		delete(rcm, rc)
		close(gc.rc)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/gophergala/goffee/tor"

	"flag"

	"github.com/gophergala/goffee/web"

	"github.com/gophergala/goffee/queue"

	"sync"
	"time"
)

var webMode bool
var torFetch bool
var redisAddress string

func init() {
	flag.BoolVar(&webMode, "webmode", false, "Run goffee in webmode")
	flag.BoolVar(&torFetch, "torfetch", false, "Fetch something via Tor")
	flag.StringVar(&redisAddress, "redisaddress", "", "Address of redis including port")
	flag.Parse()
}

func main() {
	if webMode {
		web.StartServer()
	}

	if torFetch {
		queue.RedisAddressWithPort = redisAddress

		var wg sync.WaitGroup

		for { // ever
			fmt.Println("Redis fetch")
			batch := queue.FetchBatch()
			fmt.Println(batch)
			for _, item := range batch {
				if item == "newip" {
					ip, _ := tor.NewIP()
					fmt.Println(ip) // may not print anything, info from tor is not reliable
				} else {
					wg.Add(1)
					go check(item, &wg)
				}
			}

			wg.Wait()
		}
	}
}

func check(address string, wg *sync.WaitGroup) {
	defer wg.Done()

	status, err := tor.TorGetStatus(address)
	var result string
	if err != nil {
		result = err.Error()
	} else {
		result = status
	}

	queue.WriteResult(time.Now().Format(time.RFC3339) + " " + address + " " + result)
}
<commit_msg>Adds ip check from ip reflector<commit_after>package main

import (
	"fmt"
	"github.com/gophergala/goffee/tor"

	"flag"

	"github.com/gophergala/goffee/web"

	"github.com/gophergala/goffee/queue"

	"sync"
	"time"
)

var webMode bool
var torFetch bool
var redisAddress string
const ipReflector = "http://stephensykes.com/ip_reflection.html"

func init() {
	flag.BoolVar(&webMode, "webmode", false, "Run goffee in webmode")
	flag.BoolVar(&torFetch, "torfetch", false, "Fetch something via Tor")
	flag.StringVar(&redisAddress, "redisaddress", "", "Address of redis including port")
	flag.Parse()
}

func main() {
	if webMode {
		web.StartServer()
	}

	if torFetch {
		queue.RedisAddressWithPort = redisAddress

		var wg sync.WaitGroup

		for { // ever
			fmt.Println("Redis fetch")
			batch := queue.FetchBatch()
			fmt.Println(batch)
			for _, item := range batch {
				if item == "newip" {
					newip()
				} else {
					wg.Add(1)
					go check(item, &wg)
				}
			}

			wg.Wait()
		}
	}
}

func newip() {
	tor.NewIP()
	body, err := tor.TorGet(ipReflector)
	var result string
	if err != nil {
		result = err.Error()
	} else {
		result = body
	}
	queue.WriteResult(time.Now().Format(time.RFC3339) + " newip " + result)
}

func check(address string, wg *sync.WaitGroup) {
	defer wg.Done()

	status, err := tor.TorGetStatus(address)
	var result string
	if err != nil {
		result = err.Error()
	} else {
		result = status
	}

	queue.WriteResult(time.Now().Format(time.RFC3339) + " " + address + " " + result)
}
<|endoftext|>


<commit_before>package gomapr

import (
	"errors"
	"log"
	"math/rand"
	"sync"
)

var (
	EndOfEmit = errors.New("Nothing left to emit")
)

type Event interface{}
type Partial interface{}
type ReduceKey interface{}

type MapReduce interface {
	Emit() (Event, error)
	Map(Event) (ReduceKey, Partial)
	Reduce(ReduceKey, []Partial) (ReduceKey, Partial)
}

// Corresponds to a set of values that a reducer can join.
type partialGroup struct {
	values []Partial
	l      *sync.RWMutex
}

func newPartialGroup() *partialGroup {
	return &partialGroup{
		values: make([]Partial, 0),
		l:      &sync.RWMutex{},
	}
}

// Adds a value to the group.
func (p *partialGroup) add(v interface{}) {
	p.values = append(p.values, v)
}

// Replaces the contents of the partial group.
func (p *partialGroup) replace(v interface{}) {
	p.values = []Partial{v}
}

// Contains all partial groups.
type reduceWorkspace struct {
	groups map[ReduceKey]*partialGroup
	l      *sync.RWMutex
}

func newReduceWorkspace() *reduceWorkspace {
	return &reduceWorkspace{
		make(map[ReduceKey]*partialGroup),
		&sync.RWMutex{},
	}
}

// Returns a partial group by its key.
func (r *reduceWorkspace) getPartialGroup(key ReduceKey) *partialGroup {
	r.l.Lock()
	defer r.l.Unlock()

	partialGroup, ok := r.groups[key]
	if !ok {
		partialGroup = newPartialGroup()
		r.groups[key] = partialGroup
	}

	return partialGroup
}

// Adds a key-value pair to its appropriate partial group.
func (r *reduceWorkspace) add(key ReduceKey, value Partial) {
	partialGroup := r.getPartialGroup(key)

	partialGroup.l.Lock()
	partialGroup.add(value)
	partialGroup.l.Unlock()
}

// Replaces an existing partial group with the input arguments.
func (r *reduceWorkspace) replace(key ReduceKey, value Partial) {
	partialGroup := r.getPartialGroup(key)

	partialGroup.l.Lock()
	partialGroup.replace(value)
	partialGroup.l.Unlock()
}

// Contains configuration for a MapReduce task.
type Runner struct {
	reduceWorkspace *reduceWorkspace
	mr              MapReduce
	mapWg           *sync.WaitGroup
	reduceWg        *sync.WaitGroup
	mappers         int
	reduceFactor    float64
	unreduced       map[ReduceKey]struct{}
	unreducedL      *sync.Mutex
}

func NewRunner(mr MapReduce, mappers int, reduceFactor float64) *Runner {
	if reduceFactor < 0 || reduceFactor > 1 {
		panic("Invalid reduce factor")
	}

	return &Runner{
		reduceWorkspace: newReduceWorkspace(),
		mr:              mr,
		mapWg:           &sync.WaitGroup{},
		reduceWg:        &sync.WaitGroup{},
		mappers:         mappers,
		reduceFactor:    reduceFactor,
		unreduced:       make(map[ReduceKey]struct{}),
		unreducedL:      &sync.Mutex{},
	}
}

// Returns the map containing all groups. Only safe to call after
// the task has completed.
func (r *Runner) Groups() map[ReduceKey]Partial {
	r.reduceWorkspace.l.Lock()
	defer r.reduceWorkspace.l.Unlock()

	groups := make(map[ReduceKey]Partial)

	for k, v := range r.reduceWorkspace.groups {
		groups[k] = v.values[0]
	}

	return groups
}

// Maps the input it receives on its emitted channel, spawning
// a reduce task when appropriate.
func (r *Runner) mapWorker(emitted chan Event) {
	for val := range emitted {
		key, mapped := r.mr.Map(val)
		r.reduceWorkspace.add(key, mapped)

		// Launch reduce step probabilistically.
		if rand.Float64() < r.reduceFactor {
			r.unreducedL.Lock()
			delete(r.unreduced, key)
			r.unreducedL.Unlock()

			r.reduceWg.Add(1)
			go r.reduce(key)
		} else {
			r.unreducedL.Lock()
			r.unreduced[key] = struct{}{}
			r.unreducedL.Unlock()
		}
	}

	r.mapWg.Done()
}

// Reduces the partial group with the matching input key.
func (r *Runner) reduce(key ReduceKey) {
	partialGroup := r.reduceWorkspace.getPartialGroup(key)

	partialGroup.l.Lock()
	defer partialGroup.l.Unlock()

	if len(partialGroup.values) > 1 {
		newKey, partial := r.mr.Reduce(key, partialGroup.values)

		if key == newKey {
			partialGroup.replace(partial)
		} else {
			r.reduceWorkspace.replace(key, partial)
		}
	}

	r.reduceWg.Done()
}

// Starts the MapReduce task.
func (r *Runner) Run() {
	events := make(chan Event, r.mappers)

	// Create background mapping workers.
	for i := 0; i < r.mappers; i++ {
		r.mapWg.Add(1)
		go r.mapWorker(events)
	}

	// Emit all events.
	go func() {
		for {
			event, err := r.mr.Emit()
			if err != EndOfEmit && err != nil {
				log.Printf("Error emitting: %v", err)
				break
			}

			events <- event

			if err == EndOfEmit {
				break
			}

		}
		close(events)
	}()

	r.mapWg.Wait()

	// Reduce unreduced keys.
	for key, _ := range r.unreduced {
		r.reduceWg.Add(1)
		go r.reduce(key)
	}

	r.reduceWg.Wait()
}

// Runs MapReduce synchronously.
func (r *Runner) RunSynchronous() {
	for {
		event, err := r.mr.Emit()
		if err != EndOfEmit && err != nil {
			log.Printf("Error emitting: %v", err)
			break
		}

		key, mapped := r.mr.Map(event)
		r.reduceWorkspace.add(key, mapped)

		if err == EndOfEmit {
			break
		}

	}

	for key, _ := range r.reduceWorkspace.groups {
		r.reduceWg.Add(1)
		r.reduce(key)
	}
}
<commit_msg>Use a regular mutex<commit_after>package gomapr

import (
	"errors"
	"log"
	"math/rand"
	"sync"
)

var (
	EndOfEmit = errors.New("Nothing left to emit")
)

type Event interface{}
type Partial interface{}
type ReduceKey interface{}

type MapReduce interface {
	Emit() (Event, error)
	Map(Event) (ReduceKey, Partial)
	Reduce(ReduceKey, []Partial) (ReduceKey, Partial)
}

// Corresponds to a set of values that a reducer can join.
type partialGroup struct {
	values []Partial
	l      *sync.Mutex
}

func newPartialGroup() *partialGroup {
	return &partialGroup{
		values: make([]Partial, 0),
		l:      &sync.Mutex{},
	}
}

// Adds a value to the group.
func (p *partialGroup) add(v interface{}) {
	p.values = append(p.values, v)
}

// Replaces the contents of the partial group.
func (p *partialGroup) replace(v interface{}) {
	p.values = []Partial{v}
}

// Contains all partial groups.
type reduceWorkspace struct {
	groups map[ReduceKey]*partialGroup
	l      *sync.Mutex
}

func newReduceWorkspace() *reduceWorkspace {
	return &reduceWorkspace{
		make(map[ReduceKey]*partialGroup),
		&sync.Mutex{},
	}
}

// Returns a partial group by its key.
func (r *reduceWorkspace) getPartialGroup(key ReduceKey) *partialGroup {
	r.l.Lock()
	defer r.l.Unlock()

	partialGroup, ok := r.groups[key]
	if !ok {
		partialGroup = newPartialGroup()
		r.groups[key] = partialGroup
	}

	return partialGroup
}

// Adds a key-value pair to its appropriate partial group.
func (r *reduceWorkspace) add(key ReduceKey, value Partial) {
	partialGroup := r.getPartialGroup(key)

	partialGroup.l.Lock()
	partialGroup.add(value)
	partialGroup.l.Unlock()
}

// Replaces an existing partial group with the input arguments.
func (r *reduceWorkspace) replace(key ReduceKey, value Partial) {
	partialGroup := r.getPartialGroup(key)

	partialGroup.l.Lock()
	partialGroup.replace(value)
	partialGroup.l.Unlock()
}

// Contains configuration for a MapReduce task.
type Runner struct {
	reduceWorkspace *reduceWorkspace
	mr              MapReduce
	mapWg           *sync.WaitGroup
	reduceWg        *sync.WaitGroup
	mappers         int
	reduceFactor    float64
	unreduced       map[ReduceKey]struct{}
	unreducedL      *sync.Mutex
}

func NewRunner(mr MapReduce, mappers int, reduceFactor float64) *Runner {
	if reduceFactor < 0 || reduceFactor > 1 {
		panic("Invalid reduce factor")
	}

	return &Runner{
		reduceWorkspace: newReduceWorkspace(),
		mr:              mr,
		mapWg:           &sync.WaitGroup{},
		reduceWg:        &sync.WaitGroup{},
		mappers:         mappers,
		reduceFactor:    reduceFactor,
		unreduced:       make(map[ReduceKey]struct{}),
		unreducedL:      &sync.Mutex{},
	}
}

// Returns the map containing all groups. Only safe to call after
// the task has completed.
func (r *Runner) Groups() map[ReduceKey]Partial {
	r.reduceWorkspace.l.Lock()
	defer r.reduceWorkspace.l.Unlock()

	groups := make(map[ReduceKey]Partial)

	for k, v := range r.reduceWorkspace.groups {
		groups[k] = v.values[0]
	}

	return groups
}

// Maps the input it receives on its emitted channel, spawning
// a reduce task when appropriate.
func (r *Runner) mapWorker(emitted chan Event) {
	for val := range emitted {
		key, mapped := r.mr.Map(val)
		r.reduceWorkspace.add(key, mapped)

		// Launch reduce step probabilistically.
		if rand.Float64() < r.reduceFactor {
			r.unreducedL.Lock()
			delete(r.unreduced, key)
			r.unreducedL.Unlock()

			r.reduceWg.Add(1)
			go r.reduce(key)
		} else {
			r.unreducedL.Lock()
			r.unreduced[key] = struct{}{}
			r.unreducedL.Unlock()
		}
	}

	r.mapWg.Done()
}

// Reduces the partial group with the matching input key.
func (r *Runner) reduce(key ReduceKey) {
	partialGroup := r.reduceWorkspace.getPartialGroup(key)

	partialGroup.l.Lock()
	defer partialGroup.l.Unlock()

	if len(partialGroup.values) > 1 {
		newKey, partial := r.mr.Reduce(key, partialGroup.values)

		if key == newKey {
			partialGroup.replace(partial)
		} else {
			r.reduceWorkspace.replace(key, partial)
		}
	}

	r.reduceWg.Done()
}

// Starts the MapReduce task.
func (r *Runner) Run() {
	events := make(chan Event, r.mappers)

	// Create background mapping workers.
	for i := 0; i < r.mappers; i++ {
		r.mapWg.Add(1)
		go r.mapWorker(events)
	}

	// Emit all events.
	go func() {
		for {
			event, err := r.mr.Emit()
			if err != EndOfEmit && err != nil {
				log.Printf("Error emitting: %v", err)
				break
			}

			events <- event

			if err == EndOfEmit {
				break
			}

		}
		close(events)
	}()

	r.mapWg.Wait()

	// Reduce unreduced keys.
	for key, _ := range r.unreduced {
		r.reduceWg.Add(1)
		go r.reduce(key)
	}

	r.reduceWg.Wait()
}

// Runs MapReduce synchronously.
func (r *Runner) RunSynchronous() {
	for {
		event, err := r.mr.Emit()
		if err != EndOfEmit && err != nil {
			log.Printf("Error emitting: %v", err)
			break
		}

		key, mapped := r.mr.Map(event)
		r.reduceWorkspace.add(key, mapped)

		if err == EndOfEmit {
			break
		}

	}

	for key, _ := range r.reduceWorkspace.groups {
		r.reduceWg.Add(1)
		r.reduce(key)
	}
}
<|endoftext|>


<commit_before>package lnwire

import (
	"encoding/binary"
	"io"
	"math"

	"github.com/go-errors/errors"
)

// featureFlag represent the status of the feature optional/required and needed
// to allow future incompatible changes, or backward compatible changes.
type featureFlag uint8

// String returns the string representation for the featureFlag.
func (f featureFlag) String() string {
	switch f {
	case OptionalFlag:
		return "optional"
	case RequiredFlag:
		return "required"
	default:
		return "<unknown>"
	}
}

// featureName represent the name of the feature and needed in order to have the
// compile errors if we specify wrong feature name.
type featureName string

const (
	// OptionalFlag represent the feature which we already have but it isn't
	// required yet, and if remote peer doesn't have this feature we may
	// turn it off without disconnecting with peer.
	OptionalFlag featureFlag = 2 // 0b10

	// RequiredFlag represent the features which is required for proper
	// peer interaction, we disconnect with peer if it doesn't have this
	// particular feature.
	RequiredFlag featureFlag = 1 // 0b01

	// flagMask is a mask which is needed to extract feature flag value.
	flagMask = 3 // 0b11

	// flagBitsSize represent the size of the feature flag in bits. For
	// more information read the init message specification.
	flagBitsSize = 2

	// maxAllowedSize is a maximum allowed size of feature vector.
	// NOTE: Within the protocol, the maximum allowed message size is 65535
	// bytes. Adding the overhead from the crypto protocol (the 2-byte packet
	// length and 16-byte MAC), we arrive at 65569 bytes. Accounting for the
	// overhead within the feature message to signal the type of the message,
	// that leaves 65567 bytes for the init message itself. Next, we reserve
	// 4-bytes to encode the lengths of both the local and global feature
	// vectors, so 65563 for the global and local features. Knocking off one
	// byte for the sake of the calculation, that leads to a max allowed
	// size of 32781 bytes for each feature vector, or 131124 different
	// features.
	maxAllowedSize = 32781
)

// Feature represent the feature which is used on stage of initialization of
// feature vector. Initial feature flags might be changed dynamically later.
type Feature struct {
	Name featureName
	Flag featureFlag
}

// FeatureVector represents the global/local feature vector. With this structure
// you may set/get the feature by name and compare feature vector with remote
// one.
type FeatureVector struct {
	// featuresMap is the map which stores the correspondence between
	// feature name and its index within feature vector. Index within
	// feature vector and actual binary position of feature
	// are different things)
	featuresMap map[featureName]int // name -> index

	// flags is the map which stores the correspondence between feature
	// index and its flag.
	flags map[int]featureFlag // index -> flag
}

// NewFeatureVector creates new instance of feature vector.
func NewFeatureVector(features []Feature) *FeatureVector {
	featuresMap := make(map[featureName]int)
	flags := make(map[int]featureFlag)

	for index, feature := range features {
		featuresMap[feature.Name] = index
		flags[index] = feature.Flag
	}

	return &FeatureVector{
		featuresMap: featuresMap,
		flags:       flags,
	}
}

// SetFeatureFlag assign flag to the feature.
func (f *FeatureVector) SetFeatureFlag(name featureName, flag featureFlag) error {
	position, ok := f.featuresMap[name]
	if !ok {
		return errors.Errorf("can't find feature with name: %v", name)
	}

	f.flags[position] = flag
	return nil
}

// serializedSize returns the number of bytes which is needed to represent
// feature vector in byte format.
func (f *FeatureVector) serializedSize() uint16 {
	return uint16(math.Ceil(float64(flagBitsSize*len(f.flags)) / 8))
}

// NewFeatureVectorFromReader decodes the feature vector from binary
// representation and creates the instance of it.
// Every feature decoded as 2 bits where odd bit determine whether the feature
// is "optional" and even bit told us whether the feature is "required". The
// even/odd semantic allows future incompatible changes, or backward compatible
// changes. Bits generally assigned in pairs, so that optional features can
// later become compulsory.
func NewFeatureVectorFromReader(r io.Reader) (*FeatureVector, error) {
	f := &FeatureVector{
		flags: make(map[int]featureFlag),
	}

	getFlag := func(data []byte, position int) featureFlag {
		byteNumber := uint(position / 8)
		bitNumber := uint(position % 8)

		return featureFlag((data[byteNumber] >> bitNumber) & flagMask)
	}

	// Read the length of the feature vector.
	var l [2]byte
	if _, err := r.Read(l[:]); err != nil {
		return nil, err
	}
	length := binary.BigEndian.Uint16(l[:])

	// Read the feature vector data.
	data := make([]byte, length)
	if _, err := r.Read(data); err != nil {
		return nil, err
	}

	// Initialize feature vector.
	bitsNumber := len(data) * 8
	for position := 0; position <= bitsNumber-flagBitsSize; position += flagBitsSize {
		flag := getFlag(data, position)
		switch flag {
		case OptionalFlag, RequiredFlag:
			// Every feature/flag takes 2 bits, so in order to get
			// the feature/flag index we should divide position
			// on 2.
			index := position / flagBitsSize
			f.flags[index] = flag
		default:
			continue
		}
	}

	return f, nil
}

// Encode encodes the features vector into bytes representation, every
// feature encoded as 2 bits where odd bit determine whether the feature is
// "optional" and even bit told us whether the feature is "required". The
// even/odd semantic allows future incompatible changes, or backward compatible
// changes. Bits generally assigned in pairs, so that optional features can
// later become compulsory.
func (f *FeatureVector) Encode(w io.Writer) error {
	setFlag := func(data []byte, position int, flag featureFlag) {
		byteNumber := uint(position / 8)
		bitNumber := uint(position % 8)

		data[byteNumber] |= (byte(flag) << bitNumber)
	}

	// Write length of feature vector.
	var l [2]byte
	length := f.serializedSize()
	binary.BigEndian.PutUint16(l[:], length)
	if _, err := w.Write(l[:]); err != nil {
		return err
	}

	// Generate the data and write it.
	data := make([]byte, length)
	for index, flag := range f.flags {
		// Every feature takes 2 bits, so in order to get the
		// feature bits position we should multiply index by 2.
		position := index * flagBitsSize
		setFlag(data, position, flag)
	}

	_, err := w.Write(data)
	return err
}

// Compare checks that features are compatible and returns the features which
// were present in both remote and local feature vectors. If remote/local node
// doesn't have the feature and local/remote node require it than such vectors
// are incompatible.
func (f *FeatureVector) Compare(f2 *FeatureVector) (*SharedFeatures, error) {
	shared := newSharedFeatures(f.Copy())

	for index, flag := range f.flags {
		if _, exist := f2.flags[index]; !exist {
			switch flag {
			case RequiredFlag:
				return nil, errors.New("Remote node hasn't " +
					"locally required feature")
			case OptionalFlag:
				// If feature is optional and remote side
				// haven't it than it might be safely disabled.
				delete(shared.flags, index)
				continue
			}
		}

		// If feature exists on both sides than such feature might be
		// considered as active.
		shared.flags[index] = flag
	}

	for index, flag := range f2.flags {
		if _, exist := f.flags[index]; !exist {
			switch flag {
			case RequiredFlag:
				return nil, errors.New("Local node hasn't " +
					"locally required feature")
			case OptionalFlag:
				// If feature is optional and local side
				// haven't it than it might be safely disabled.
				delete(shared.flags, index)
				continue
			}
		}

		// If feature exists on both sides than such feature might be
		// considered as active.
		shared.flags[index] = flag
	}

	return shared, nil
}

// Copy generate new distinct instance of the feature vector.
func (f *FeatureVector) Copy() *FeatureVector {
	features := make([]Feature, len(f.featuresMap))

	for name, index := range f.featuresMap {
		features[index] = Feature{
			Name: name,
			Flag: f.flags[index],
		}
	}

	return NewFeatureVector(features)
}

// SharedFeatures is a product of comparison of two features vector
// which consist of features which are present in both local and remote
// features vectors.
type SharedFeatures struct {
	*FeatureVector
}

// newSharedFeatures creates new shared features instance.
func newSharedFeatures(f *FeatureVector) *SharedFeatures {
	return &SharedFeatures{f}
}

// IsActive checks is feature active or not, it might be disabled during
// comparision with remote feature vector if it was optional and
// remote peer doesn't support it.
func (f *SharedFeatures) IsActive(name featureName) bool {
	index, ok := f.featuresMap[name]
	if !ok {
		// If we even have no such feature in feature map, than it
		// can't be active in any circumstances.
		return false
	}

	_, exist := f.flags[index]
	return exist
}
<commit_msg>lnwire: properly fold at 80-columns within features.go<commit_after>package lnwire

import (
	"encoding/binary"
	"io"
	"math"

	"github.com/go-errors/errors"
)

// featureFlag represent the status of the feature optional/required and needed
// to allow future incompatible changes, or backward compatible changes.
type featureFlag uint8

// String returns the string representation for the featureFlag.
func (f featureFlag) String() string {
	switch f {
	case OptionalFlag:
		return "optional"
	case RequiredFlag:
		return "required"
	default:
		return "<unknown>"
	}
}

// featureName represent the name of the feature and needed in order to have
// the compile errors if we specify wrong feature name.
type featureName string

const (
	// OptionalFlag represent the feature which we already have but it
	// isn't required yet, and if remote peer doesn't have this feature we
	// may turn it off without disconnecting with peer.
	OptionalFlag featureFlag = 2 // 0b10

	// RequiredFlag represent the features which is required for proper
	// peer interaction, we disconnect with peer if it doesn't have this
	// particular feature.
	RequiredFlag featureFlag = 1 // 0b01

	// flagMask is a mask which is needed to extract feature flag value.
	flagMask = 3 // 0b11

	// flagBitsSize represent the size of the feature flag in bits. For
	// more information read the init message specification.
	flagBitsSize = 2

	// maxAllowedSize is a maximum allowed size of feature vector.
	//
	// NOTE: Within the protocol, the maximum allowed message size is 65535
	// bytes. Adding the overhead from the crypto protocol (the 2-byte
	// packet length and 16-byte MAC), we arrive at 65569 bytes. Accounting
	// for the overhead within the feature message to signal the type of
	// the message, that leaves 65567 bytes for the init message itself.
	// Next, we reserve 4-bytes to encode the lengths of both the local and
	// global feature vectors, so 65563 for the global and local features.
	// Knocking off one byte for the sake of the calculation, that leads to
	// a max allowed size of 32781 bytes for each feature vector, or 131124
	// different features.
	maxAllowedSize = 32781
)

// Feature represent the feature which is used on stage of initialization of
// feature vector. Initial feature flags might be changed dynamically later.
type Feature struct {
	Name featureName
	Flag featureFlag
}

// FeatureVector represents the global/local feature vector. With this
// structure you may set/get the feature by name and compare feature vector
// with remote one.
type FeatureVector struct {
	// featuresMap is the map which stores the correspondence between
	// feature name and its index within feature vector. Index within
	// feature vector and actual binary position of feature are different
	// things)
	featuresMap map[featureName]int // name -> index

	// flags is the map which stores the correspondence between feature
	// index and its flag.
	flags map[int]featureFlag // index -> flag
}

// NewFeatureVector creates new instance of feature vector.
func NewFeatureVector(features []Feature) *FeatureVector {
	featuresMap := make(map[featureName]int)
	flags := make(map[int]featureFlag)

	for index, feature := range features {
		featuresMap[feature.Name] = index
		flags[index] = feature.Flag
	}

	return &FeatureVector{
		featuresMap: featuresMap,
		flags:       flags,
	}
}

// SetFeatureFlag assign flag to the feature.
func (f *FeatureVector) SetFeatureFlag(name featureName, flag featureFlag) error {
	position, ok := f.featuresMap[name]
	if !ok {
		return errors.Errorf("can't find feature with name: %v", name)
	}

	f.flags[position] = flag
	return nil
}

// serializedSize returns the number of bytes which is needed to represent
// feature vector in byte format.
func (f *FeatureVector) serializedSize() uint16 {
	return uint16(math.Ceil(float64(flagBitsSize*len(f.flags)) / 8))
}

// NewFeatureVectorFromReader decodes the feature vector from binary
// representation and creates the instance of it.  Every feature decoded as 2
// bits where odd bit determine whether the feature is "optional" and even bit
// told us whether the feature is "required". The even/odd semantic allows
// future incompatible changes, or backward compatible changes. Bits generally
// assigned in pairs, so that optional features can later become compulsory.
func NewFeatureVectorFromReader(r io.Reader) (*FeatureVector, error) {
	f := &FeatureVector{
		flags: make(map[int]featureFlag),
	}

	getFlag := func(data []byte, position int) featureFlag {
		byteNumber := uint(position / 8)
		bitNumber := uint(position % 8)

		return featureFlag((data[byteNumber] >> bitNumber) & flagMask)
	}

	// Read the length of the feature vector.
	var l [2]byte
	if _, err := r.Read(l[:]); err != nil {
		return nil, err
	}
	length := binary.BigEndian.Uint16(l[:])

	// Read the feature vector data.
	data := make([]byte, length)
	if _, err := r.Read(data); err != nil {
		return nil, err
	}

	// Initialize feature vector.
	bitsNumber := len(data) * 8
	for position := 0; position <= bitsNumber-flagBitsSize; position += flagBitsSize {
		flag := getFlag(data, position)
		switch flag {
		case OptionalFlag, RequiredFlag:
			// Every feature/flag takes 2 bits, so in order to get
			// the feature/flag index we should divide position
			// on 2.
			index := position / flagBitsSize
			f.flags[index] = flag
		default:
			continue
		}
	}

	return f, nil
}

// Encode encodes the features vector into bytes representation, every feature
// encoded as 2 bits where odd bit determine whether the feature is "optional"
// and even bit told us whether the feature is "required". The even/odd
// semantic allows future incompatible changes, or backward compatible changes.
// Bits generally assigned in pairs, so that optional features can later become
// compulsory.
func (f *FeatureVector) Encode(w io.Writer) error {
	setFlag := func(data []byte, position int, flag featureFlag) {
		byteNumber := uint(position / 8)
		bitNumber := uint(position % 8)

		data[byteNumber] |= (byte(flag) << bitNumber)
	}

	// Write length of feature vector.
	var l [2]byte
	length := f.serializedSize()
	binary.BigEndian.PutUint16(l[:], length)
	if _, err := w.Write(l[:]); err != nil {
		return err
	}

	// Generate the data and write it.
	data := make([]byte, length)
	for index, flag := range f.flags {
		// Every feature takes 2 bits, so in order to get the feature
		// bits position we should multiply index by 2.
		position := index * flagBitsSize
		setFlag(data, position, flag)
	}

	_, err := w.Write(data)
	return err
}

// Compare checks that features are compatible and returns the features which
// were present in both remote and local feature vectors. If remote/local node
// doesn't have the feature and local/remote node require it than such vectors
// are incompatible.
func (f *FeatureVector) Compare(f2 *FeatureVector) (*SharedFeatures, error) {
	shared := newSharedFeatures(f.Copy())

	for index, flag := range f.flags {
		if _, exist := f2.flags[index]; !exist {
			switch flag {
			case RequiredFlag:
				return nil, errors.New("Remote node hasn't " +
					"locally required feature")
			case OptionalFlag:
				// If feature is optional and remote side
				// haven't it than it might be safely disabled.
				delete(shared.flags, index)
				continue
			}
		}

		// If feature exists on both sides than such feature might be
		// considered as active.
		shared.flags[index] = flag
	}

	for index, flag := range f2.flags {
		if _, exist := f.flags[index]; !exist {
			switch flag {
			case RequiredFlag:
				return nil, errors.New("Local node hasn't " +
					"locally required feature")
			case OptionalFlag:
				// If feature is optional and local side
				// haven't it than it might be safely disabled.
				delete(shared.flags, index)
				continue
			}
		}

		// If feature exists on both sides than such feature might be
		// considered as active.
		shared.flags[index] = flag
	}

	return shared, nil
}

// Copy generate new distinct instance of the feature vector.
func (f *FeatureVector) Copy() *FeatureVector {
	features := make([]Feature, len(f.featuresMap))

	for name, index := range f.featuresMap {
		features[index] = Feature{
			Name: name,
			Flag: f.flags[index],
		}
	}

	return NewFeatureVector(features)
}

// SharedFeatures is a product of comparison of two features vector which
// consist of features which are present in both local and remote features
// vectors.
type SharedFeatures struct {
	*FeatureVector
}

// newSharedFeatures creates new shared features instance.
func newSharedFeatures(f *FeatureVector) *SharedFeatures {
	return &SharedFeatures{f}
}

// IsActive checks is feature active or not, it might be disabled during
// comparision with remote feature vector if it was optional and remote peer
// doesn't support it.
func (f *SharedFeatures) IsActive(name featureName) bool {
	index, ok := f.featuresMap[name]
	if !ok {
		// If we even have no such feature in feature map, than it
		// can't be active in any circumstances.
		return false
	}

	_, exist := f.flags[index]
	return exist
}
<|endoftext|>


<commit_before>package channels

import (
	"fmt"
	"io"
	"log"
	"os"

	"github.com/mackerelio/mackerel-client-go"
	"github.com/mackerelio/mkr/format"
	"github.com/mackerelio/mkr/logger"
	"github.com/mackerelio/mkr/mackerelclient"
)

type channelsApp struct {
	client    mackerelclient.Client
	outStream io.Writer
}

func (app *channelsApp) run() error {
	channels, err := app.client.FindChannels()
	if err != nil {
		return err
	}

	format.PrettyPrintJSON(app.outStream, channels)
	return nil
}

func (app *channelsApp) pullChannels(isVerbose bool, optFilePath string) error {
	channels, err := mackerelclient.NewFromContext(c).FindChannels()
	logger.DieIf(err)

	channelSaveRules(channels, optFilePath)

	if isVerbose {
		format.PrettyPrintJSON(os.Stdout, channels)
	}

	if optFilePath == "" {
		optFilePath = "channels.json"
	}
	logger.Log("info", fmt.Sprintf("Channels are saved to '%s' (%d rules).", optFilePath, len(channels)))
	return nil
}

func channelSaveRules(rules []*mackerel.Channel, optFilePath string) error {
	filePath := "channels.json"
	if optFilePath != "" {
		filePath = optFilePath
	}
	file, err := os.Create(filePath)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	channels := map[string]interface{}{"channels": rules}
	data := format.JSONMarshalIndent(channels, "", "    ") + "\n"

	_, err = file.WriteString(data)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>avoid assigning to file path twice<commit_after>package channels

import (
	"fmt"
	"io"
	"log"
	"os"

	"github.com/mackerelio/mackerel-client-go"
	"github.com/mackerelio/mkr/format"
	"github.com/mackerelio/mkr/logger"
	"github.com/mackerelio/mkr/mackerelclient"
)

type channelsApp struct {
	client    mackerelclient.Client
	outStream io.Writer
}

func (app *channelsApp) run() error {
	channels, err := app.client.FindChannels()
	if err != nil {
		return err
	}

	format.PrettyPrintJSON(app.outStream, channels)
	return nil
}

func (app *channelsApp) pullChannels(isVerbose bool, optFilePath string) error {
	channels, err := mackerelclient.NewFromContext(c).FindChannels()
	logger.DieIf(err)

	filePath := "channels.json"
	if optFilePath != "" {
		filePath = optFilePath
	}

	channelSaveRules(channels, filePath)

	if isVerbose {
		format.PrettyPrintJSON(os.Stdout, channels)
	}

	logger.Log("info", fmt.Sprintf("Channels are saved to '%s' (%d rules).", filePath, len(channels)))
	return nil
}

func channelSaveRules(rules []*mackerel.Channel, filePath string) error {
	file, err := os.Create(filePath)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	channels := map[string]interface{}{"channels": rules}
	data := format.JSONMarshalIndent(channels, "", "    ") + "\n"

	_, err = file.WriteString(data)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	. "eaciit/wfdemo-git/processapp/eventConverter/conversion"
	"log"
	"os"
	"runtime"
	"strings"

	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	"github.com/eaciit/orm"
	tk "github.com/eaciit/toolkit"
)

var (
	wd = func() string {
		d, _ := os.Getwd()
		return d + separator
	}()
	separator = string(os.PathSeparator)
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	log.Println("Starting convert event data to down...")
	conn, err := PrepareConnection()
	if err != nil {
		log.Println(err)
	}
	ctx := orm.New(conn)

	config := ReadConfig()
	dir := config["FileProcess"]

	/*
		// under development, not done yet
		event := NewCombineRaw(ctx)
		event.Run()*/

	/*event := NewEventRawConversion(ctx, dir)
	event.Run()*/

	down := NewDownConversion(ctx, dir)
	down.Run()

	log.Println("End processing event data to down...")
}

func PrepareConnection() (dbox.IConnection, error) {
	config := ReadConfig()
	ci := &dbox.ConnectionInfo{config["host"], config["database"], config["username"], config["password"], tk.M{}.Set("timeout", 3000)}

	c, e := dbox.NewConnection("mongo", ci)

	if e != nil {
		return nil, e
	}

	e = c.Connect()
	if e != nil {
		return nil, e
	}

	log.Println("DB Connect...")

	return c, nil
}

func ReadConfig() map[string]string {
	ret := make(map[string]string)
	file, err := os.Open(wd + "conf" + separator + "app.conf")
	if err == nil {
		defer file.Close()

		reader := bufio.NewReader(file)
		for {
			line, _, e := reader.ReadLine()
			if e != nil {
				break
			}

			sval := strings.Split(string(line), "=")
			ret[sval[0]] = sval[1]
		}
	} else {
		log.Println(err.Error())
	}

	return ret
}
<commit_msg>bug fixing on generate down<commit_after>package main

import (
	"bufio"
	. "eaciit/wfdemo-git/processapp/eventConverter/conversion"
	"log"
	"os"
	"runtime"
	"strings"

	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	"github.com/eaciit/orm"
	tk "github.com/eaciit/toolkit"
)

var (
	wd = func() string {
		d, _ := os.Getwd()
		return d + separator
	}()
	separator = string(os.PathSeparator)
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	log.Println("Starting convert event data to down...")
	conn, err := PrepareConnection()
	if err != nil {
		log.Println(err)
	}
	ctx := orm.New(conn)

	config := ReadConfig()
	dir := config["FileProcess"]

	/*
		// under development, not done yet
		event := NewCombineRaw(ctx)
		event.Run()*/

	event := NewEventRawConversion(ctx, dir)
	event.Run()

	down := NewDownConversion(ctx, dir)
	down.Run()

	log.Println("End processing event data to down...")
}

func PrepareConnection() (dbox.IConnection, error) {
	config := ReadConfig()
	ci := &dbox.ConnectionInfo{config["host"], config["database"], config["username"], config["password"], tk.M{}.Set("timeout", 3000)}

	c, e := dbox.NewConnection("mongo", ci)

	if e != nil {
		return nil, e
	}

	e = c.Connect()
	if e != nil {
		return nil, e
	}

	log.Println("DB Connect...")

	return c, nil
}

func ReadConfig() map[string]string {
	ret := make(map[string]string)
	file, err := os.Open(wd + "conf" + separator + "app.conf")
	if err == nil {
		defer file.Close()

		reader := bufio.NewReader(file)
		for {
			line, _, e := reader.ReadLine()
			if e != nil {
				break
			}

			sval := strings.Split(string(line), "=")
			ret[sval[0]] = sval[1]
		}
	} else {
		log.Println(err.Error())
	}

	return ret
}
<|endoftext|>


<commit_before>// Licensed to Elasticsearch B.V. under one or more agreements.
// Elasticsearch B.V. licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information.

package gentests

import (
  "fmt"
  "strings"

  "gopkg.in/yaml.v2"
)

var skipTests map[string][]string

func init() {
  err := yaml.NewDecoder(strings.NewReader(skipTestsYAML)).Decode(&skipTests)
  if err != nil {
    panic(fmt.Sprintf("ERROR: %v", err))
  }
}

var skipFiles = []string{
  "update/85_fields_meta.yml",            // Uses non-existing API property
  "update/86_fields_meta_with_types.yml", // --||--

  "ml/jobs_get_result_buckets.yml",    // Passes string value to int variable
  "ml/jobs_get_result_categories.yml", // --||--
  "ml/set_upgrade_mode.yml",           // --||--

  "ml/evaluate_data_frame.yml", // Floats as map keys

  "search/320_disallow_queries.yml", // Tries to match key in an empty map (`transient:{}`)

  "watcher/stats/10_basic.yml", // Sets "emit_stacktraces" as string ("true"), not bool
}

// TODO: Comments into descriptions for `Skip()`
//
var skipTestsYAML = `
---
# Cannot distinguish between missing value for refresh and an empty string
bulk/50_refresh.yml:
  - refresh=empty string immediately makes changes are visible in search
bulk/51_refresh_with_types.yml:
  - refresh=empty string immediately makes changes are visible in search
create/60_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
create/61_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
delete/50_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
delete/51_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
index/60_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
index/61_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
update/60_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
update/61_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"

# catch: bad_request, Expected [status] to not be nil
indices.data_stream/10_basic.yml:
  - Create data stream with invalid name

# Stash in value
cluster.reroute/11_explain.yml:
nodes.info/30_settings.yml:
nodes.stats/20_response_filtering.yml:
nodes.stats/30_discovery.yml:
  - Discovery stats
nodes.discovery/30_discovery.yml:
  - Discovery stats

# Arbitrary key
indices.shrink/10_basic.yml:
indices.shrink/20_source_mapping.yml:
indices.shrink/30_copy_settings.yml:
indices.split/30_copy_settings.yml:
nodes.info/10_basic.yml:
nodes.reload_secure_settings/10_basic.yml:

# Parsed response is YAML: value is map[interface {}]interface {}, not map[string]interface {}
cat.aliases/20_headers.yml:
  - Simple alias with yaml body through Accept header

# Incorrect int instead of float in match (aggregations.date_range.buckets.0.from: 1000000); TODO: PR
search.aggregation/40_range.yml:
  - Date range

# Mismatch in number parsing, 8623000 != 8.623e+06
search.aggregation/340_geo_distance.yml:
  - avg_bucket

# No support for headers per request yet
tasks.list/10_basic.yml:
  - tasks_list headers

# Node Selector feature not implemented
cat.aliases/10_basic.yml:
  - "Help (pre 7.4.0)"
  - "Simple alias (pre 7.4.0)"
  - "Complex alias (pre 7.4.0)"
  - "Column headers (pre 7.4.0)"
  - "Alias against closed index (pre 7.4.0)"

indices.put_mapping/10_basic.yml:
  - "Put mappings with explicit _doc type bwc"

# Test fails with: [400 Bad Request] illegal_argument_exception, "template [test] has index patterns [test-*] matching patterns from existing index templates [test2,test] with patterns (test2 => [test-*],test => [test-*, test2-*]), use index templates (/_index_template) instead"
test/indices.put_template/10_basic.yml:

# Incompatible regex
cat.templates/10_basic.yml:
  - "Sort templates"
  - "Multiple template"

# Missing test setup
cluster.voting_config_exclusions/10_basic.yml:
  - "Add voting config exclusion by unknown node name"

# Not relevant
search/issue4895.yml:
search/issue9606.yml:

# FIXME
bulk/80_cas.yml:
bulk/81_cas_with_types.yml:

# ----- X-Pack ----------------------------------------------------------------

# Float "3.0" handled as "3" in fmt.Sprintf()
analytics/top_metrics.yml:

# Stash in body
api_key/10_basic.yml:
  - Test invalidate api key
api_key/11_invalidation.yml:
  - Test invalidate api key by username
rollup/put_job.yml:
  - Test put job with templates

# Changing password locks out tests
change_password/10_basic.yml:
  - Test user changing their own password

# Missing refreshes in the test
data_frame/transforms_start_stop.yml:
ml/index_layout.yml:
transform/transforms_start_stop.yml:
  - Verify start transform reuses destination index
transform/transforms_start_stop.yml:
  - Test get multiple transform stats
transform/transforms_stats.yml:
  - Test get multiple transform stats
  - Test get multiple transform stats where one does not have a task

# More QA tests than API tests
data_frame/transforms_stats.yml:
  - Test get multiple transform stats
  - Test get transform stats on missing transform
  - Test get multiple transform stats where one does not have a task
ml/jobs_crud.yml:
  - Test reopen job resets the finished time

# Invalid license makes subsequent tests fail
license/20_put_license.yml:

# Test tries to match on map from body, but Go keys are not sorted
ml/jobs_crud.yml:
  - Test job with rules
  - Test put job with model_memory_limit as number
  - Test put job with model_memory_limit as string and lazy open

# Test gets stuck every time
ml/jobs_get_stats.yml:

# status_exception, Cannot process data because job [post-data-job] does not have a corresponding autodetect process
# resource_already_exists_exception, task with id {job-post-data-job} already exist
# status_exception, Cannot open job [start-stop-datafeed-job-foo-1] because it has already been opened
ml/post_data.yml:
  - Test flush with skip_time
  - Test POST data job api, flush, close and verify DataCounts doc
  - Test flush and close job WITHOUT sending any data
ml/start_stop_datafeed.yml:
  - Test stop given expression
transform/transforms_start_stop.yml:
  - Test start transform
  - Verify start transform reuses destination index

# Possible bad test setup, Cannot open job [start-stop-datafeed-job] because it has already been opened
# resource_already_exists_exception, task with id {job-start-stop-datafeed-job-foo-2} already exist
ml/start_stop_datafeed.yml:
  - Test start datafeed when persistent task allocation disabled

# Indexing step doesn't appear to work (getting total.hits=0)
monitoring/bulk/10_basic.yml:
  - Bulk indexing of monitoring data on closed indices should throw an export exception
# Indexing step doesn't appear to work (getting total.hits=0)
monitoring/bulk/20_privileges.yml:
  - Monitoring Bulk API

# Test tries to match on whole body, but map keys are unstable in Go
rollup/security_tests.yml:

# Test tries to match on map key, but map keys are unstable in Go
ml/data_frame_analytics_crud.yml:
  - Test put with description
  - Test put valid config with custom outlier detection

# Unsupported feature: allowed_warnings
ml/data_frame_analytics_crud.yml:
  - Test put classification given deprecated maximum_number_trees

# This test suite keeps failing too often: disable it altogether
ml/data_frame_analytics_crud.yml:

# 404s, panics, ... possible bad setup/teardown
ml/delete_model_snapshot.yml:
ml/get_datafeed_stats.yml:
ml/get_model_snapshots.yml:

# TEMPORARY: Missing 'body: { indices: "test_index" }' payload, TODO: PR
snapshot/10_basic.yml:
  - Create a source only snapshot and then restore it

# illegal_argument_exception: Provided password hash uses [NOOP] but the configured hashing algorithm is [BCRYPT]
users/10_basic.yml:
  - Test put user with password hash

# Slash in index name is not escaped (BUG)
security/authz/13_index_datemath.yml:
  - Test indexing documents with datemath, when permitted

# Possibly a cluster health color mismatch...
security/authz/14_cat_indices.yml:

# Test looks for "testnode.crt", but "ca.crt" is returned first
ssl/10_basic.yml:
  - Test get SSL certificates

# class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$DenseVectorScriptDocValues cannot be cast to class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$SparseVectorScriptDocValues ...
vectors/30_sparse_vector_basic.yml:
  - Dot Product
# java.lang.IllegalArgumentException: No field found for [my_dense_vector] in mapping
vectors/40_sparse_vector_special_cases.yml:
  - Vectors of different dimensions and data types
  - Dimensions can be sorted differently
  - Distance functions for documents missing vector field should return 0

# Cannot connect to Docker IP
watcher/execute_watch/60_http_input.yml:

# Test tries to match on "tagline", which requires "human=false", which doesn't work in the Go API.
# Also test does too much within a single test, so has to be disabled as whole, unfortunately.
xpack/15_basic.yml:

# Test uses "y" as a property name, which is parsed as 'true' in the Go YAML library;
# see https://yaml.org/type/bool.html
ml/explain_data_frame_analytics.yml:
  - Test empty data frame given body
  - Test non-empty data frame given body

# Test uses "n" as a property name, which is parsed as 'false' in the Go YAML library;
search.aggregation/10_histogram.yml:
`
<commit_msg>Generator: Tests: Update the list of skipped tests<commit_after>// Licensed to Elasticsearch B.V. under one or more agreements.
// Elasticsearch B.V. licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information.

package gentests

import (
  "fmt"
  "strings"

  "gopkg.in/yaml.v2"
)

var skipTests map[string][]string

func init() {
  err := yaml.NewDecoder(strings.NewReader(skipTestsYAML)).Decode(&skipTests)
  if err != nil {
    panic(fmt.Sprintf("ERROR: %v", err))
  }
}

var skipFiles = []string{
  "update/85_fields_meta.yml",            // Uses non-existing API property
  "update/86_fields_meta_with_types.yml", // --||--

  "ml/jobs_get_result_buckets.yml",    // Passes string value to int variable
  "ml/jobs_get_result_categories.yml", // --||--
  "ml/set_upgrade_mode.yml",           // --||--

  "ml/evaluate_data_frame.yml", // Floats as map keys

  "search/320_disallow_queries.yml", // Tries to match key in an empty map (`transient:{}`)

  "watcher/stats/10_basic.yml", // Sets "emit_stacktraces" as string ("true"), not bool
}

// TODO: Comments into descriptions for `Skip()`
//
var skipTestsYAML = `
---
# Cannot distinguish between missing value for refresh and an empty string
bulk/50_refresh.yml:
  - refresh=empty string immediately makes changes are visible in search
bulk/51_refresh_with_types.yml:
  - refresh=empty string immediately makes changes are visible in search
create/60_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
create/61_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
delete/50_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
delete/51_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
index/60_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
index/61_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
update/60_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
update/61_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"

# catch: bad_request, Expected [status] to not be nil
indices.data_stream/10_basic.yml:
  - Create data stream with invalid name

# Stash in value
cluster.reroute/11_explain.yml:
nodes.info/30_settings.yml:
nodes.stats/20_response_filtering.yml:
nodes.stats/30_discovery.yml:
  - Discovery stats
nodes.discovery/30_discovery.yml:
  - Discovery stats

# Arbitrary key
indices.shrink/10_basic.yml:
indices.shrink/20_source_mapping.yml:
indices.shrink/30_copy_settings.yml:
indices.split/30_copy_settings.yml:
nodes.info/10_basic.yml:
nodes.reload_secure_settings/10_basic.yml:
nodes.stats/50_indexing_pressure.yml:

# Parsed response is YAML: value is map[interface {}]interface {}, not map[string]interface {}
cat.aliases/20_headers.yml:
  - Simple alias with yaml body through Accept header

# Incorrect int instead of float in match (aggregations.date_range.buckets.0.from: 1000000); TODO: PR
search.aggregation/40_range.yml:
  - Date range

# Mismatch in number parsing, 8623000 != 8.623e+06
search.aggregation/340_geo_distance.yml:
  - avg_bucket

# No support for headers per request yet
tasks.list/10_basic.yml:
  - tasks_list headers

# Node Selector feature not implemented
cat.aliases/10_basic.yml:
  - "Help (pre 7.4.0)"
  - "Simple alias (pre 7.4.0)"
  - "Complex alias (pre 7.4.0)"
  - "Column headers (pre 7.4.0)"
  - "Alias against closed index (pre 7.4.0)"

indices.put_mapping/10_basic.yml:
  - "Put mappings with explicit _doc type bwc"

# Test fails with: [400 Bad Request] illegal_argument_exception, "template [test] has index patterns [test-*] matching patterns from existing index templates [test2,test] with patterns (test2 => [test-*],test => [test-*, test2-*]), use index templates (/_index_template) instead"
test/indices.put_template/10_basic.yml:

# Incompatible regex
cat.templates/10_basic.yml:
  - "Sort templates"
  - "Multiple template"

# Missing test setup
cluster.voting_config_exclusions/10_basic.yml:
  - "Add voting config exclusion by unknown node name"

# Not relevant
search/issue4895.yml:
search/issue9606.yml:

# FIXME
bulk/80_cas.yml:
bulk/81_cas_with_types.yml:

# ----- X-Pack ----------------------------------------------------------------

# Float "3.0" handled as "3" in fmt.Sprintf()
analytics/top_metrics.yml:

# Stash in body
api_key/10_basic.yml:
  - Test invalidate api key
api_key/11_invalidation.yml:
  - Test invalidate api key by username
rollup/put_job.yml:
  - Test put job with templates

# Changing password locks out tests
change_password/10_basic.yml:
  - Test user changing their own password

# Missing refreshes in the test
data_frame/transforms_start_stop.yml:
ml/index_layout.yml:
transform/transforms_start_stop.yml:
  - Verify start transform reuses destination index
transform/transforms_start_stop.yml:
  - Test get multiple transform stats
transform/transforms_stats.yml:
  - Test get multiple transform stats
  - Test get multiple transform stats where one does not have a task

# More QA tests than API tests
data_frame/transforms_stats.yml:
  - Test get multiple transform stats
  - Test get transform stats on missing transform
  - Test get multiple transform stats where one does not have a task
ml/jobs_crud.yml:
  - Test reopen job resets the finished time

# Invalid license makes subsequent tests fail
license/20_put_license.yml:

# Test tries to match on map from body, but Go keys are not sorted
ml/jobs_crud.yml:
  - Test job with rules
  - Test put job with model_memory_limit as number
  - Test put job with model_memory_limit as string and lazy open

# Test gets stuck every time
ml/jobs_get_stats.yml:

# status_exception, Cannot process data because job [post-data-job] does not have a corresponding autodetect process
# resource_already_exists_exception, task with id {job-post-data-job} already exist
# status_exception, Cannot open job [start-stop-datafeed-job-foo-1] because it has already been opened
ml/post_data.yml:
  - Test flush with skip_time
  - Test POST data job api, flush, close and verify DataCounts doc
  - Test flush and close job WITHOUT sending any data
ml/start_stop_datafeed.yml:
  - Test stop given expression
transform/transforms_start_stop.yml:
  - Test start transform
  - Verify start transform reuses destination index

# Possible bad test setup, Cannot open job [start-stop-datafeed-job] because it has already been opened
# resource_already_exists_exception, task with id {job-start-stop-datafeed-job-foo-2} already exist
ml/start_stop_datafeed.yml:
  - Test start datafeed when persistent task allocation disabled

# Indexing step doesn't appear to work (getting total.hits=0)
monitoring/bulk/10_basic.yml:
  - Bulk indexing of monitoring data on closed indices should throw an export exception
# Indexing step doesn't appear to work (getting total.hits=0)
monitoring/bulk/20_privileges.yml:
  - Monitoring Bulk API

# Test tries to match on whole body, but map keys are unstable in Go
rollup/security_tests.yml:

# Test tries to match on map key, but map keys are unstable in Go
ml/data_frame_analytics_crud.yml:
  - Test put with description
  - Test put valid config with custom outlier detection

# Unsupported feature: allowed_warnings
ml/data_frame_analytics_crud.yml:
  - Test put classification given deprecated maximum_number_trees

# This test suite keeps failing too often: disable it altogether
ml/data_frame_analytics_crud.yml:

# 404s, panics, ... possible bad setup/teardown
ml/delete_model_snapshot.yml:
ml/get_datafeed_stats.yml:
ml/get_model_snapshots.yml:

# TEMPORARY: Missing 'body: { indices: "test_index" }' payload, TODO: PR
snapshot/10_basic.yml:
  - Create a source only snapshot and then restore it

# illegal_argument_exception: Provided password hash uses [NOOP] but the configured hashing algorithm is [BCRYPT]
users/10_basic.yml:
  - Test put user with password hash

# Slash in index name is not escaped (BUG)
security/authz/13_index_datemath.yml:
  - Test indexing documents with datemath, when permitted

# Possibly a cluster health color mismatch...
security/authz/14_cat_indices.yml:

# Test looks for "testnode.crt", but "ca.crt" is returned first
ssl/10_basic.yml:
  - Test get SSL certificates

# class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$DenseVectorScriptDocValues cannot be cast to class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$SparseVectorScriptDocValues ...
vectors/30_sparse_vector_basic.yml:
  - Dot Product
# java.lang.IllegalArgumentException: No field found for [my_dense_vector] in mapping
vectors/40_sparse_vector_special_cases.yml:
  - Vectors of different dimensions and data types
  - Dimensions can be sorted differently
  - Distance functions for documents missing vector field should return 0

# Cannot connect to Docker IP
watcher/execute_watch/60_http_input.yml:

# Test tries to match on "tagline", which requires "human=false", which doesn't work in the Go API.
# Also test does too much within a single test, so has to be disabled as whole, unfortunately.
xpack/15_basic.yml:

# Test uses "y" as a property name, which is parsed as 'true' in the Go YAML library;
# see https://yaml.org/type/bool.html
ml/explain_data_frame_analytics.yml:
  - Test empty data frame given body
  - Test non-empty data frame given body

# Test uses "n" as a property name, which is parsed as 'false' in the Go YAML library;
search.aggregation/10_histogram.yml:
`
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package render formats Go documentation as HTML.
// It is an internal component that powers dochtml.
package render

import (
	"context"
	"go/ast"
	"go/token"
	"regexp"
	"strings"

	"github.com/google/safehtml"
	"github.com/google/safehtml/template"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/experiment"
	"golang.org/x/pkgsite/internal/godoc/internal/doc"
)

var (
	// Regexp for headings.
	headingHead = `^[\p{Lu}]`                                  // any uppercase letter
	headingBody = `([^,.;:!?+*/=()\[\]{}_^°&§~%#@<">\\]|'s )*` // any non-illegal character
	headingTail = `([\p{L}\p{Nd}]|'s)?$`                       // any letter or digit

	headingRx = regexp.MustCompile(headingHead + headingBody + headingTail)

	// Regexp for example outputs.
	exampleOutputRx = regexp.MustCompile(`(?i)//[[:space:]]*(unordered )?output:`)
)

type Renderer struct {
	fset              *token.FileSet
	pids              *packageIDs
	packageURL        func(string) string
	disableHotlinking bool
	disablePermalinks bool
	ctx               context.Context
	docTmpl           *template.Template
	exampleTmpl       *template.Template
}

type Options struct {
	// RelatedPackages is a list of related packages to use for hotlinking.
	// A recommended heuristic is to include all packages imported by the
	// given package, its tests, and its example tests.
	//
	// Only relevant for HTML formatting.
	RelatedPackages []*doc.Package

	// PackageURL is a function that given a package path,
	// returns a URL for navigating to the godoc for that package.
	//
	// Only relevant for HTML formatting.
	PackageURL func(pkgPath string) (url string)

	// DisableHotlinking turns off hotlinking behavior.
	//
	// Only relevant for HTML formatting.
	DisableHotlinking bool

	// DisablePermalinks turns off inserting of '¶' permalinks in headings.
	//
	// Only relevant for HTML formatting.
	DisablePermalinks bool
}

// docDataTmpl renders documentation. It expects a docData.
var docDataTmpl = template.Must(template.New("").Parse(`
{{- range .Elements -}}
  {{- if .IsHeading -}}

{{.Title}}
    {{- if not $.DisablePermalinks}}¶{{end -}}

  {{else if .IsPreformat -}}

{{.Body}}

  {{- else -}}

{{.Body}}

  {{- end -}}
{{end}}`))

// exampleTmpl renders code for an example. It expect an Example.
var exampleTmpl = template.Must(template.New("").Parse(`

{{range .}}
  {{- if .Comment -}}
    {{.Text}}
  {{- else -}}
    {{.Text}}
  {{- end -}}
{{end}}

`))

// legacyDocDataTmpl renders documentation. It expects a docData.
var legacyDocDataTmpl = template.Must(template.New("").Parse(`
{{- range .Elements -}}
  {{- if .IsHeading -}}

{{.Title}}
    {{- if not $.DisablePermalinks}}¶{{end -}}

  {{else if .IsPreformat -}}

{{.Body}}

  {{- else -}}

{{.Body}}

  {{- end -}}
{{end}}`))

// legacyExampleTmpl renders code for an example. It expect an Example.
var legacyExampleTmpl = template.Must(template.New("").Parse(`

{{range .}}
  {{- if .Comment -}}
    {{.Text}}
  {{- else -}}
    {{.Text}}
  {{- end -}}
{{end}}

`))

func New(ctx context.Context, fset *token.FileSet, pkg *doc.Package, opts *Options) *Renderer {
	var others []*doc.Package
	var packageURL func(string) string
	var disableHotlinking bool
	var disablePermalinks bool
	if opts != nil {
		if len(opts.RelatedPackages) > 0 {
			others = opts.RelatedPackages
		}
		if opts.PackageURL != nil {
			packageURL = opts.PackageURL
		}
		disableHotlinking = opts.DisableHotlinking
		disablePermalinks = opts.DisablePermalinks
	}
	pids := newPackageIDs(pkg, others...)

	d := legacyDocDataTmpl
	e := legacyExampleTmpl
	if experiment.IsActive(ctx, internal.ExperimentUnitPage) {
		d = docDataTmpl
		e = exampleTmpl
	}
	return &Renderer{
		fset:              fset,
		pids:              pids,
		packageURL:        packageURL,
		disableHotlinking: disableHotlinking,
		disablePermalinks: disablePermalinks,
		docTmpl:           d,
		exampleTmpl:       e,
	}
}

const maxSynopsisNodeDepth = 10

// ShortSynopsis returns a very short, one-line summary of the given input node.
// It currently only supports *ast.FuncDecl nodes and will return a non-nil
// error otherwise.
func (r *Renderer) ShortSynopsis(n ast.Node) (string, error) {
	return shortOneLineNodeDepth(r.fset, n, 0)
}

// Synopsis returns a one-line summary of the given input node.
func (r *Renderer) Synopsis(n ast.Node) string {
	return oneLineNodeDepth(r.fset, n, 0)
}

// DocHTML formats documentation text as HTML.
//
// Each span of unindented non-blank lines is converted into a single paragraph.
// There is one exception to the rule: a span that consists of a
// single line, is followed by another paragraph span, begins with a capital
// letter, and contains no punctuation is formatted as a heading.
//
// A span of indented lines is converted into a
 block, with the common
// indent prefix removed.
//
// URLs in the comment text are converted into links. Any word that matches
// an exported top-level identifier in the package is automatically converted
// into a hyperlink to the declaration of that identifier.
//
// This returns formatted HTML with:
//
                elements for plain documentation text
//
              elements for preformatted text
//
  elements for headings with the "id" attribute
//	     elements for URL hyperlinks
//
// DocHTML is intended for documentation for the package and examples.
func (r *Renderer) DocHTML(doc string) safehtml.HTML {
	return r.declHTML(doc, nil).Doc
}

// DeclHTML formats the doc and decl and returns a tuple of
// strings corresponding to each input argument.
//
// This formats documentation HTML according to the same rules as DocHTML.
//
// This formats declaration HTML with:
//
                       element wrapping the entire declaration
//	 elements for many top-level declarations
//	      elements for every Go comment
//	              elements for URL hyperlinks
//
// DeclHTML is intended for top-level package declarations.
func (r *Renderer) DeclHTML(doc string, decl ast.Decl) (out struct{ Doc, Decl safehtml.HTML }) {
	// This returns an anonymous struct instead of multiple return values since
	// the template package only allows single return values.
	return r.declHTML(doc, decl)
}

// CodeHTML formats example code. If the code is a single block statement,
// the outer braces are stripped and the code unindented. If the example code
// contains an output comment, that will stripped as well.
//
// The code type must be *ast.File, *CommentedNode, []ast.Decl, []ast.Stmt
// or assignment-compatible to ast.Expr, ast.Decl, ast.Spec, or ast.Stmt.
//
// This returns formatted HTML with:
//
                   element wrapping entire block
//	  elements for every Go comment
//
// CodeHTML is intended for use with example code snippets.
func (r *Renderer) CodeHTML(ex *doc.Example) safehtml.HTML {
	return r.codeHTML(ex)
}

// block is (*heading | *paragraph | *preformat).
type block interface{}

type (
	lines   []string
	heading struct {
		title string
	}
	paragraph struct {
		lines lines
	}
	preformat struct {
		lines lines
	}
)

// docToBlocks converts doc string into list of blocks.
//
// Heading block is a non-blank line, surrounded by blank lines
// and the next non-blank line is not indented.
//
// Preformat block contains single line or consecutive lines which have indent greater than 0.
//
// Paragraph block is a default block type if a block does not fall into heading and preformat.
func docToBlocks(doc string) []block {
	docLines := unindent(strings.Split(strings.Trim(doc, "\n"), "\n"))

	// Group the lines based on indentation and blank lines.
	var groups [][]string
	var lastGroup []string
	var wasInCode bool
	for _, line := range docLines {
		hasIndent := indentLength(line) > 0
		nowInCode := hasIndent || (wasInCode && line == "")
		newGroup := wasInCode != nowInCode || (!nowInCode && line == "")
		wasInCode = nowInCode
		if newGroup && len(lastGroup) > 0 {
			groups = append(groups, lastGroup)
			lastGroup = nil
		}
		if line != "" || nowInCode {
			lastGroup = append(lastGroup, line)
		}
	}
	if len(lastGroup) > 0 {
		groups = append(groups, lastGroup)
	}

	// Classify groups of lines as individual blocks.
	var blks []block
	var lastBlk block
	for i, group := range groups {
		willParagraph := i+1 < len(groups) && indentLength(groups[i+1][0]) == 0
		for len(group) > 0 && group[len(group)-1] == "" {
			group = group[:len(group)-1] // remove trailing empty lines
		}
		_, wasHeading := lastBlk.(*heading)
		switch {
		case indentLength(group[0]) > 0:
			blks = append(blks, &preformat{unindent(group)})
		case i != 0 && !wasHeading && len(group) == 1 && headingRx.MatchString(group[0]) && willParagraph:
			blks = append(blks, &heading{group[0]})
		default:
			blks = append(blks, &paragraph{group})
		}
		lastBlk = blks[len(blks)-1]
	}
	return blks
}

func indentLength(s string) int {
	return len(s) - len(trimIndent(s))
}

func trimIndent(s string) string {
	return strings.TrimLeft(s, " \t")
}

func commonPrefixLength(a, b string) (n int) {
	for n < len(a) && n < len(b) && a[n] == b[n] {
		n++
	}
	return n
}

func unindent(lines []string) []string {
	if len(lines) > 0 {
		npre := indentLength(lines[0])
		for _, line := range lines {
			if line != "" {
				npre = commonPrefixLength(lines[0][:npre], line)
			}
		}
		for i, line := range lines {
			if line != "" {
				lines[i] = line[npre:]
			}
		}
	}
	return lines
}
<commit_msg>internal/godoc/dochtml/internal/render: change docDataHTML h3 to h4<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package render formats Go documentation as HTML.
// It is an internal component that powers dochtml.
package render

import (
	"context"
	"go/ast"
	"go/token"
	"regexp"
	"strings"

	"github.com/google/safehtml"
	"github.com/google/safehtml/template"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/experiment"
	"golang.org/x/pkgsite/internal/godoc/internal/doc"
)

var (
	// Regexp for headings.
	headingHead = `^[\p{Lu}]`                                  // any uppercase letter
	headingBody = `([^,.;:!?+*/=()\[\]{}_^°&§~%#@<">\\]|'s )*` // any non-illegal character
	headingTail = `([\p{L}\p{Nd}]|'s)?$`                       // any letter or digit

	headingRx = regexp.MustCompile(headingHead + headingBody + headingTail)

	// Regexp for example outputs.
	exampleOutputRx = regexp.MustCompile(`(?i)//[[:space:]]*(unordered )?output:`)
)

type Renderer struct {
	fset              *token.FileSet
	pids              *packageIDs
	packageURL        func(string) string
	disableHotlinking bool
	disablePermalinks bool
	ctx               context.Context
	docTmpl           *template.Template
	exampleTmpl       *template.Template
}

type Options struct {
	// RelatedPackages is a list of related packages to use for hotlinking.
	// A recommended heuristic is to include all packages imported by the
	// given package, its tests, and its example tests.
	//
	// Only relevant for HTML formatting.
	RelatedPackages []*doc.Package

	// PackageURL is a function that given a package path,
	// returns a URL for navigating to the godoc for that package.
	//
	// Only relevant for HTML formatting.
	PackageURL func(pkgPath string) (url string)

	// DisableHotlinking turns off hotlinking behavior.
	//
	// Only relevant for HTML formatting.
	DisableHotlinking bool

	// DisablePermalinks turns off inserting of '¶' permalinks in headings.
	//
	// Only relevant for HTML formatting.
	DisablePermalinks bool
}

// docDataTmpl renders documentation. It expects a docData.
var docDataTmpl = template.Must(template.New("").Parse(`
{{- range .Elements -}}
  {{- if .IsHeading -}}

{{.Title}}
    {{- if not $.DisablePermalinks}}¶{{end -}}

  {{else if .IsPreformat -}}

{{.Body}}

  {{- else -}}

{{.Body}}

  {{- end -}}
{{end}}`))

// exampleTmpl renders code for an example. It expect an Example.
var exampleTmpl = template.Must(template.New("").Parse(`

{{range .}}
  {{- if .Comment -}}
    {{.Text}}
  {{- else -}}
    {{.Text}}
  {{- end -}}
{{end}}

`))

// legacyDocDataTmpl renders documentation. It expects a docData.
var legacyDocDataTmpl = template.Must(template.New("").Parse(`
{{- range .Elements -}}
  {{- if .IsHeading -}}

{{.Title}}
    {{- if not $.DisablePermalinks}}¶{{end -}}

  {{else if .IsPreformat -}}

{{.Body}}

  {{- else -}}

{{.Body}}

  {{- end -}}
{{end}}`))

// legacyExampleTmpl renders code for an example. It expect an Example.
var legacyExampleTmpl = template.Must(template.New("").Parse(`

{{range .}}
  {{- if .Comment -}}
    {{.Text}}
  {{- else -}}
    {{.Text}}
  {{- end -}}
{{end}}

`))

func New(ctx context.Context, fset *token.FileSet, pkg *doc.Package, opts *Options) *Renderer {
	var others []*doc.Package
	var packageURL func(string) string
	var disableHotlinking bool
	var disablePermalinks bool
	if opts != nil {
		if len(opts.RelatedPackages) > 0 {
			others = opts.RelatedPackages
		}
		if opts.PackageURL != nil {
			packageURL = opts.PackageURL
		}
		disableHotlinking = opts.DisableHotlinking
		disablePermalinks = opts.DisablePermalinks
	}
	pids := newPackageIDs(pkg, others...)

	d := legacyDocDataTmpl
	e := legacyExampleTmpl
	if experiment.IsActive(ctx, internal.ExperimentUnitPage) {
		d = docDataTmpl
		e = exampleTmpl
	}
	return &Renderer{
		fset:              fset,
		pids:              pids,
		packageURL:        packageURL,
		disableHotlinking: disableHotlinking,
		disablePermalinks: disablePermalinks,
		docTmpl:           d,
		exampleTmpl:       e,
	}
}

const maxSynopsisNodeDepth = 10

// ShortSynopsis returns a very short, one-line summary of the given input node.
// It currently only supports *ast.FuncDecl nodes and will return a non-nil
// error otherwise.
func (r *Renderer) ShortSynopsis(n ast.Node) (string, error) {
	return shortOneLineNodeDepth(r.fset, n, 0)
}

// Synopsis returns a one-line summary of the given input node.
func (r *Renderer) Synopsis(n ast.Node) string {
	return oneLineNodeDepth(r.fset, n, 0)
}

// DocHTML formats documentation text as HTML.
//
// Each span of unindented non-blank lines is converted into a single paragraph.
// There is one exception to the rule: a span that consists of a
// single line, is followed by another paragraph span, begins with a capital
// letter, and contains no punctuation is formatted as a heading.
//
// A span of indented lines is converted into a
 block, with the common
// indent prefix removed.
//
// URLs in the comment text are converted into links. Any word that matches
// an exported top-level identifier in the package is automatically converted
// into a hyperlink to the declaration of that identifier.
//
// This returns formatted HTML with:
//
                elements for plain documentation text
//
              elements for preformatted text
//
  elements for headings with the "id" attribute
//	     elements for URL hyperlinks
//
// DocHTML is intended for documentation for the package and examples.
func (r *Renderer) DocHTML(doc string) safehtml.HTML {
	return r.declHTML(doc, nil).Doc
}

// DeclHTML formats the doc and decl and returns a tuple of
// strings corresponding to each input argument.
//
// This formats documentation HTML according to the same rules as DocHTML.
//
// This formats declaration HTML with:
//
                       element wrapping the entire declaration
//	 elements for many top-level declarations
//	      elements for every Go comment
//	              elements for URL hyperlinks
//
// DeclHTML is intended for top-level package declarations.
func (r *Renderer) DeclHTML(doc string, decl ast.Decl) (out struct{ Doc, Decl safehtml.HTML }) {
	// This returns an anonymous struct instead of multiple return values since
	// the template package only allows single return values.
	return r.declHTML(doc, decl)
}

// CodeHTML formats example code. If the code is a single block statement,
// the outer braces are stripped and the code unindented. If the example code
// contains an output comment, that will stripped as well.
//
// The code type must be *ast.File, *CommentedNode, []ast.Decl, []ast.Stmt
// or assignment-compatible to ast.Expr, ast.Decl, ast.Spec, or ast.Stmt.
//
// This returns formatted HTML with:
//
                   element wrapping entire block
//	  elements for every Go comment
//
// CodeHTML is intended for use with example code snippets.
func (r *Renderer) CodeHTML(ex *doc.Example) safehtml.HTML {
	return r.codeHTML(ex)
}

// block is (*heading | *paragraph | *preformat).
type block interface{}

type (
	lines   []string
	heading struct {
		title string
	}
	paragraph struct {
		lines lines
	}
	preformat struct {
		lines lines
	}
)

// docToBlocks converts doc string into list of blocks.
//
// Heading block is a non-blank line, surrounded by blank lines
// and the next non-blank line is not indented.
//
// Preformat block contains single line or consecutive lines which have indent greater than 0.
//
// Paragraph block is a default block type if a block does not fall into heading and preformat.
func docToBlocks(doc string) []block {
	docLines := unindent(strings.Split(strings.Trim(doc, "\n"), "\n"))

	// Group the lines based on indentation and blank lines.
	var groups [][]string
	var lastGroup []string
	var wasInCode bool
	for _, line := range docLines {
		hasIndent := indentLength(line) > 0
		nowInCode := hasIndent || (wasInCode && line == "")
		newGroup := wasInCode != nowInCode || (!nowInCode && line == "")
		wasInCode = nowInCode
		if newGroup && len(lastGroup) > 0 {
			groups = append(groups, lastGroup)
			lastGroup = nil
		}
		if line != "" || nowInCode {
			lastGroup = append(lastGroup, line)
		}
	}
	if len(lastGroup) > 0 {
		groups = append(groups, lastGroup)
	}

	// Classify groups of lines as individual blocks.
	var blks []block
	var lastBlk block
	for i, group := range groups {
		willParagraph := i+1 < len(groups) && indentLength(groups[i+1][0]) == 0
		for len(group) > 0 && group[len(group)-1] == "" {
			group = group[:len(group)-1] // remove trailing empty lines
		}
		_, wasHeading := lastBlk.(*heading)
		switch {
		case indentLength(group[0]) > 0:
			blks = append(blks, &preformat{unindent(group)})
		case i != 0 && !wasHeading && len(group) == 1 && headingRx.MatchString(group[0]) && willParagraph:
			blks = append(blks, &heading{group[0]})
		default:
			blks = append(blks, &paragraph{group})
		}
		lastBlk = blks[len(blks)-1]
	}
	return blks
}

func indentLength(s string) int {
	return len(s) - len(trimIndent(s))
}

func trimIndent(s string) string {
	return strings.TrimLeft(s, " \t")
}

func commonPrefixLength(a, b string) (n int) {
	for n < len(a) && n < len(b) && a[n] == b[n] {
		n++
	}
	return n
}

func unindent(lines []string) []string {
	if len(lines) > 0 {
		npre := indentLength(lines[0])
		for _, line := range lines {
			if line != "" {
				npre = commonPrefixLength(lines[0][:npre], line)
			}
		}
		for i, line := range lines {
			if line != "" {
				lines[i] = line[npre:]
			}
		}
	}
	return lines
}
<|endoftext|>