<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/coreos/go-oidc"
	"github.com/gin-gonic/gin"
	"github.com/googleinterns/recomator/pkg/automation"
	"golang.org/x/oauth2"
	"google.golang.org/api/googleapi"
)

const tokenExpiry = 24 * time.Hour

// User contains fields describing authorized user:
// Email and GoogleService with required access to Google APIs.
type User struct {
	service automation.GoogleService
	email   string
}

// AuthorizationService creates the user using authentication code and returns idToken.
// Returns authorized user for this idToken.
type AuthorizationService interface {
	// Returns idToken that should be used to authorize
	CreateUser(authCode string) (string, error)
	// Returns GoogleService that should be used to make requests to Google APIs
	Authorize(token string) (User, error)
}

type authorizationService struct {
	verifier            *oidc.IDTokenVerifier
	tokenExpirationTime time.Duration
	mutex               sync.Mutex
	services            map[string]automation.GoogleService // key is email of the user
}

// NewAuthorizationService creates new AuthorizationService to access GoogleAPIs
func NewAuthorizationService() (AuthorizationService, error) {
	provider, err := oidc.NewProvider(oauth2.NoContext, "https://accounts.google.com")
	if err != nil {
		return nil, err
	}
	authService := &authorizationService{tokenExpirationTime: tokenExpiry, services: make(map[string]automation.GoogleService)}
	authService.verifier = provider.Verifier(&oidc.Config{ClientID: config.ClientID, SkipExpiryCheck: true})
	return authService, nil
}

func (s *authorizationService) Authorize(idToken string) (User, error) {
	email, err := s.verify(idToken)
	if err != nil {
		return User{}, err
	}

	s.mutex.Lock()
	service, ok := s.services[email]
	s.mutex.Unlock()

	if !ok {
		return User{}, fmt.Errorf("User with %s email not found", email)
	}

	return User{service: service, email: email}, nil
}

// Returns idToken that should be used for authorization later.
func (s *authorizationService) CreateUser(authCode string) (string, error) {
	token, err := config.Exchange(oauth2.NoContext, authCode)
	if err != nil {
		return "", err
	}
	idToken, ok := token.Extra("id_token").(string)
	if !ok {
		return "", fmt.Errorf("No valid id token where given. Casting to string failed")
	}

	service, err := automation.NewGoogleService(oauth2.NoContext, &config, token)
	if err != nil {
		return "", err
	}

	email, err := s.verify(idToken)
	if err != nil {
		return "", err
	}

	s.mutex.Lock()
	s.services[email] = service
	s.mutex.Unlock()
	return idToken, nil
}

// Verifies idToken and returns email if everything suceeded.
func (s *authorizationService) verify(rawToken string) (string, error) {
	idToken, err := s.verifier.Verify(oauth2.NoContext, rawToken)
	if err != nil {
		return "", err
	}
	if time.Since(time.Time(idToken.IssuedAt)) > s.tokenExpirationTime {
		return "", &googleapi.Error{Code: http.StatusUnauthorized, Message: "Token expired"}
	}
	var claims struct {
		Email string `json:"email"`
	}
	if err := idToken.Claims(&claims); err != nil {
		return "", fmt.Errorf("Extracting email failed:" + err.Error())
	}
	return claims.Email, nil
}

func getAuthHandler(service *sharedService) func(c *gin.Context) {
	return func(c *gin.Context) {
		authCode := c.Query("code")
		token, err := service.auth.CreateUser(authCode)
		if err != nil {
			sendError(c, err, http.StatusBadRequest)
			return
		}
		c.JSON(http.StatusOK, gin.H{"token": token})
	}
}

// authorizeRequest extracts the token from Authorization header in request
// and uses it to return authorized user using authService.
func authorizeRequest(authService AuthorizationService, request *http.Request) (User, error) {
	bearToken := request.Header["Authorization"]
	log.Println(len(bearToken))
	if len(bearToken) != 0 {
		strArr := strings.Split(bearToken[0], " ")
		if len(strArr) == 2 && strArr[0] == "Bearer" {
			return authService.Authorize(strArr[1])
		}
	}
	return User{}, &googleapi.Error{Code: http.StatusBadRequest, Message: "Authorization header not in the form 'Bearer <token>'"}

}

// redirects to google for login, login_hint query parameter(user's email) might be specified for faster login.
func redirectHandler(c *gin.Context) {
	email := c.Query("login_hint")
	authOptions := []oauth2.AuthCodeOption{oauth2.AccessTypeOffline, oauth2.ApprovalForce}

	if len(email) != 0 {
		authOptions = append(authOptions, oauth2.SetAuthURLParam("login_hint", email))
	}

	url := config.AuthCodeURL(config.RedirectURL, authOptions...)
	c.Redirect(http.StatusSeeOther, url)
	return
}
<commit_msg>add redirect parameter<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/coreos/go-oidc"
	"github.com/gin-gonic/gin"
	"github.com/googleinterns/recomator/pkg/automation"
	"golang.org/x/oauth2"
	"google.golang.org/api/googleapi"
)

const tokenExpiry = 24 * time.Hour

// User contains fields describing authorized user:
// Email and GoogleService with required access to Google APIs.
type User struct {
	service automation.GoogleService
	email   string
}

// AuthorizationService creates the user using authentication code and returns idToken.
// Returns authorized user for this idToken.
type AuthorizationService interface {
	// Returns idToken that should be used to authorize
	CreateUser(authCode string) (string, error)
	// Returns GoogleService that should be used to make requests to Google APIs
	Authorize(token string) (User, error)
}

type authorizationService struct {
	verifier            *oidc.IDTokenVerifier
	tokenExpirationTime time.Duration
	mutex               sync.Mutex
	services            map[string]automation.GoogleService // key is email of the user
}

// NewAuthorizationService creates new AuthorizationService to access GoogleAPIs
func NewAuthorizationService() (AuthorizationService, error) {
	provider, err := oidc.NewProvider(oauth2.NoContext, "https://accounts.google.com")
	if err != nil {
		return nil, err
	}
	authService := &authorizationService{tokenExpirationTime: tokenExpiry, services: make(map[string]automation.GoogleService)}
	authService.verifier = provider.Verifier(&oidc.Config{ClientID: config.ClientID, SkipExpiryCheck: true})
	return authService, nil
}

func (s *authorizationService) Authorize(idToken string) (User, error) {
	email, err := s.verify(idToken)
	if err != nil {
		return User{}, err
	}

	s.mutex.Lock()
	service, ok := s.services[email]
	s.mutex.Unlock()

	if !ok {
		return User{}, fmt.Errorf("User with %s email not found", email)
	}

	return User{service: service, email: email}, nil
}

// Returns idToken that should be used for authorization later.
func (s *authorizationService) CreateUser(authCode string) (string, error) {
	token, err := config.Exchange(oauth2.NoContext, authCode)
	if err != nil {
		return "", err
	}
	idToken, ok := token.Extra("id_token").(string)
	if !ok {
		return "", fmt.Errorf("No valid id token where given. Casting to string failed")
	}

	service, err := automation.NewGoogleService(oauth2.NoContext, &config, token)
	if err != nil {
		return "", err
	}

	email, err := s.verify(idToken)
	if err != nil {
		return "", err
	}

	s.mutex.Lock()
	s.services[email] = service
	s.mutex.Unlock()
	return idToken, nil
}

// Verifies idToken and returns email if everything suceeded.
func (s *authorizationService) verify(rawToken string) (string, error) {
	idToken, err := s.verifier.Verify(oauth2.NoContext, rawToken)
	if err != nil {
		return "", err
	}
	if time.Since(time.Time(idToken.IssuedAt)) > s.tokenExpirationTime {
		return "", &googleapi.Error{Code: http.StatusUnauthorized, Message: "Token expired"}
	}
	var claims struct {
		Email string `json:"email"`
	}
	if err := idToken.Claims(&claims); err != nil {
		return "", fmt.Errorf("Extracting email failed:" + err.Error())
	}
	return claims.Email, nil
}

func getAuthHandler(service *sharedService) func(c *gin.Context) {
	return func(c *gin.Context) {
		authCode := c.Query("code")
		token, err := service.auth.CreateUser(authCode)
		if err != nil {
			sendError(c, err, http.StatusBadRequest)
			return
		}
		c.JSON(http.StatusOK, gin.H{"token": token})
	}
}

// authorizeRequest extracts the token from Authorization header in request
// and uses it to return authorized user using authService.
func authorizeRequest(authService AuthorizationService, request *http.Request) (User, error) {
	bearToken := request.Header["Authorization"]
	log.Println(len(bearToken))
	if len(bearToken) != 0 {
		strArr := strings.Split(bearToken[0], " ")
		if len(strArr) == 2 && strArr[0] == "Bearer" {
			return authService.Authorize(strArr[1])
		}
	}
	return User{}, &googleapi.Error{Code: http.StatusBadRequest, Message: "Authorization header not in the form 'Bearer <token>'"}

}

// redirects to google for login, login_hint query parameter(user's email) might be specified for faster login.
func redirectHandler(c *gin.Context) {
	email := c.Query("login_hint")
	redirectURL := c.Query("redirect_uri")
	authOptions := []oauth2.AuthCodeOption{oauth2.AccessTypeOffline, oauth2.ApprovalForce}

	if len(email) != 0 {
		authOptions = append(authOptions, oauth2.SetAuthURLParam("login_hint", email))
	}

	url := config.AuthCodeURL(redirectURL, authOptions...)
	c.Redirect(http.StatusSeeOther, url)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"

	"github.com/thomersch/grandine/lib/mvt"
	"github.com/thomersch/grandine/lib/progressbar"
	"github.com/thomersch/grandine/lib/spaten"
	"github.com/thomersch/grandine/lib/spatial"
	"github.com/thomersch/grandine/lib/tile"
)

type zmLvl []int

func (zm *zmLvl) String() string {
	return fmt.Sprintf("%d", *zm)
}

func (zm *zmLvl) Set(value string) error {
	for _, s := range strings.Split(value, ",") {
		v, err := strconv.Atoi(strings.TrimSpace(s))
		if err != nil {
			return fmt.Errorf("%s (only integer values are allowed)", value)
		}
		*zm = append(*zm, v)
	}
	return nil
}

type bbox spatial.BBox

func (b *bbox) String() string {
	return fmt.Sprintf("%v", *b)
}

func (b *bbox) Set(value string) error {
	var (
		fl    [4]float64
		parts = strings.Split(value, ",")
		err   error
	)
	if len(parts) != 4 {
		return errors.New("bbox takes exactly 4 parameters: SW Lon, SW Lat, NE Lon, NE Lat")
	}
	for i, s := range parts {
		fl[i], err = strconv.ParseFloat(s, 64)
		if err != nil {
			return fmt.Errorf("could not parse bbox expression: %v", err)
		}
	}
	b.SW = spatial.Point{fl[0], fl[1]}
	b.NE = spatial.Point{fl[2], fl[3]}
	return nil
}

var (
	zoomlevels zmLvl
	quiet      *bool
)

func main() {
	var (
		sourceStdIn bool
		tileCodec   tile.Codec
	)
	source := flag.String("in", "", "file to read from, supported format: spaten")
	target := flag.String("out", "tiles", "path where the tiles will be written")
	defaultLayer := flag.Bool("default-layer", true, "if no layer name is specified in the feature, whether it will be put into a default layer")
	workersNumber := flag.Int("workers", runtime.GOMAXPROCS(0), "number of workers")
	cpuProfile := flag.String("cpuprof", "", "writes CPU profiling data into a file")
	geojsonCodec := flag.Bool("geojson", false, "encode tiles into geojson instead of MVT, for debugging purposes")
	compressTiles := flag.Bool("compress", false, "compress tiles with gzip")
	quiet = flag.Bool("q", false, "argument to use if program should be run in quiet mode with reduced logging")

	flag.Var(&zoomlevels, "zoom", "one or more zoom level of which the tiles will be rendered")
	flag.Parse()

	if len(*source) == 0 {
		log.Print("grandine-tiler: Reading from stdin.")
		sourceStdIn = true
	}

	if len(zoomlevels) == 0 {
		log.Fatal("no zoom levels specified")
	}

	if len(*cpuProfile) != 0 {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *geojsonCodec {
		tileCodec = &tile.GeoJSONCodec{}
	} else {
		tileCodec = &mvt.Codec{}
	}

	var (
		f   io.Reader
		err error
	)

	if !sourceStdIn {
		f, err = os.Open(*source)
		if err != nil {
			log.Fatal(err)
		}
		defer f.(io.Closer).Close()
	} else {
		f = os.Stdin
	}

	err = os.MkdirAll(*target, 0777)
	if err != nil {
		log.Fatal(err)
	}

	log.Println("parsing input...")
	fc := spatial.FeatureCollection{}
	codec := spaten.Codec{}
	err = codec.Decode(f, &fc)
	if err != nil {
		log.Fatalf("could not read incoming file: %v", err)
	}

	if len(fc.Features) == 0 {
		log.Fatal("no features in input file")
	}

	log.Printf("read %d features", len(fc.Features))

	var (
		bboxPts []spatial.Point
		ft      = featureTable(zoomlevels)
	)
	for featID, feat := range fc.Features {
		for _, zl := range zoomlevels {
			if !renderable(feat.Props, zl) {
				continue
			}
			for _, tid := range tile.Coverage(feat.Geometry.BBox(), zl) {
				ft[zl][tid.X][tid.Y] = append(ft[zl][tid.X][tid.Y], &fc.Features[featID])
			}
		}
		bb := feat.Geometry.BBox()
		bboxPts = append(bboxPts, bb.SW, bb.NE)
	}

	log.Println("determining which tiles need to be generated")
	var tc []tile.ID
	for _, zoomlevel := range zoomlevels {
		tc = append(tc, tile.Coverage(spatial.Line(bboxPts).BBox(), zoomlevel)...)
	}

	log.Printf("starting to generate %d tiles...", len(tc))

	dtw := diskTileWriter{basedir: *target, compressTiles: *compressTiles}
	dlm := defaultLayerMapper{defaultLayer: *defaultLayer}

	shuffleWork(tc) // randomize order for better worker saturation
	var (
		wg       sync.WaitGroup
		ws       = workerSlices(tc, *workersNumber)
		pb, done = progressbar.NewBar(len(tc), len(ws))
	)
	for wrk := 0; wrk < len(ws); wrk++ {
		wg.Add(1)
		go func(i int) {
			generateTiles(ws[i], ft, &dtw, tileCodec, &dlm, pb)
			wg.Done()
		}(wrk)
	}
	wg.Wait()
	done()
}

func renderable(props map[string]interface{}, zl int) bool {
	var (
		zmin = 0
		zmax = 99
	)
	v, ok := props["@zoom:min"]
	if ok {
		zmin, _ = v.(int)
	}
	v, ok = props["@zoom:max"]
	if ok {
		zmax, _ = v.(int)
	}
	return (zmin <= zl) && (zl <= zmax)
}

func featureTable(zls []int) map[int][][][]*spatial.Feature {
	r := map[int][][][]*spatial.Feature{}
	for _, zl := range zls {
		l := pow(2, zl)
		r[zl] = make([][][]*spatial.Feature, l)
		for x := range r[zl] {
			r[zl][x] = make([][]*spatial.Feature, l)
		}
	}
	return r
}

func workerSlices(tiles []tile.ID, wrkNum int) [][]tile.ID {
	var r [][]tile.ID
	if len(tiles) <= wrkNum {
		for t := 0; t < len(tiles); t++ {
			r = append(r, []tile.ID{tiles[t]})
		}
		return r
	}
	for wrkr := 0; wrkr < wrkNum; wrkr++ {
		start := (len(tiles) / wrkNum) * wrkr
		end := (len(tiles) / wrkNum) * (wrkr + 1)
		if wrkr == wrkNum-1 {
			end = len(tiles)
		}
		r = append(r, tiles[start:end])
	}
	return r
}

type diskTileWriter struct {
	basedir       string
	compressTiles bool
}

func (tw *diskTileWriter) WriteTile(tID tile.ID, buf []byte) error {
	err := os.MkdirAll(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X)), 0777)
	if err != nil {
		return err
	}
	tf, err := os.Create(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X), strconv.Itoa(tID.Y)+".mvt")) // TODO: file extension based on codec
	if err != nil {
		return err
	}
	defer tf.Close()

	if tw.compressTiles {
		_, err = gzip.NewWriter(tf).Write(buf)
	} else {
		_, err = tf.Write(buf)
	}
	if err != nil {
		return err
	}
	return nil
}

type defaultLayerMapper struct {
	defaultLayer bool
}

func (dlm *defaultLayerMapper) LayerName(props map[string]interface{}) string {
	if layerName, ok := props["@layer"]; ok {
		return layerName.(string)
	}
	if dlm.defaultLayer {
		return "default"
	}
	return ""
}

type layerMapper interface {
	LayerName(map[string]interface{}) string
}

type tileWriter interface {
	WriteTile(tile.ID, []byte) error
}

func generateTiles(tIDs []tile.ID, features map[int][][][]*spatial.Feature, tw tileWriter, encoder tile.Codec, lm layerMapper, pb chan<- struct{}) {
	for _, tID := range tIDs {
		var (
			layers = map[string][]spatial.Feature{}
			ln     string
		)
		defer func() { pb <- struct{}{} }()
		for _, ft := range features[tID.Z][tID.X][tID.Y] {
			feat := *ft
			ln = lm.LayerName(feat.Props)
			if len(ln) != 0 {
				if _, ok := layers[ln]; !ok {
					layers[ln] = []spatial.Feature{feat}
				} else {
					layers[ln] = append(layers[ln], feat)
				}
			}
		}
		if !anyFeatures(layers) {
			continue
		}
		buf, err := encoder.EncodeTile(layers, tID)
		if err != nil {
			log.Fatal(err)
		}

		err = tw.WriteTile(tID, buf)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func anyFeatures(layers map[string][]spatial.Feature) bool {
	for i := range layers {
		if len(layers[i]) > 0 {
			return true
		}
	}
	return false
}

func pow(x, y int) int {
	var res = 1
	for i := 1; i <= y; i++ {
		res *= x
	}
	return res
}
<commit_msg>cmd/tiler: zoom levels might be floats when converting from json<commit_after>package main

import (
	"compress/gzip"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"

	"github.com/thomersch/grandine/lib/mvt"
	"github.com/thomersch/grandine/lib/progressbar"
	"github.com/thomersch/grandine/lib/spaten"
	"github.com/thomersch/grandine/lib/spatial"
	"github.com/thomersch/grandine/lib/tile"
)

type zmLvl []int

func (zm *zmLvl) String() string {
	return fmt.Sprintf("%d", *zm)
}

func (zm *zmLvl) Set(value string) error {
	for _, s := range strings.Split(value, ",") {
		v, err := strconv.Atoi(strings.TrimSpace(s))
		if err != nil {
			return fmt.Errorf("%s (only integer values are allowed)", value)
		}
		*zm = append(*zm, v)
	}
	return nil
}

type bbox spatial.BBox

func (b *bbox) String() string {
	return fmt.Sprintf("%v", *b)
}

func (b *bbox) Set(value string) error {
	var (
		fl    [4]float64
		parts = strings.Split(value, ",")
		err   error
	)
	if len(parts) != 4 {
		return errors.New("bbox takes exactly 4 parameters: SW Lon, SW Lat, NE Lon, NE Lat")
	}
	for i, s := range parts {
		fl[i], err = strconv.ParseFloat(s, 64)
		if err != nil {
			return fmt.Errorf("could not parse bbox expression: %v", err)
		}
	}
	b.SW = spatial.Point{fl[0], fl[1]}
	b.NE = spatial.Point{fl[2], fl[3]}
	return nil
}

var (
	zoomlevels zmLvl
	quiet      *bool
)

func main() {
	var (
		sourceStdIn bool
		tileCodec   tile.Codec
	)
	source := flag.String("in", "", "file to read from, supported format: spaten")
	target := flag.String("out", "tiles", "path where the tiles will be written")
	defaultLayer := flag.Bool("default-layer", true, "if no layer name is specified in the feature, whether it will be put into a default layer")
	workersNumber := flag.Int("workers", runtime.GOMAXPROCS(0), "number of workers")
	cpuProfile := flag.String("cpuprof", "", "writes CPU profiling data into a file")
	geojsonCodec := flag.Bool("geojson", false, "encode tiles into geojson instead of MVT, for debugging purposes")
	compressTiles := flag.Bool("compress", false, "compress tiles with gzip")
	quiet = flag.Bool("q", false, "argument to use if program should be run in quiet mode with reduced logging")

	flag.Var(&zoomlevels, "zoom", "one or more zoom level of which the tiles will be rendered")
	flag.Parse()

	if len(*source) == 0 {
		log.Print("grandine-tiler: Reading from stdin.")
		sourceStdIn = true
	}

	if len(zoomlevels) == 0 {
		log.Fatal("no zoom levels specified")
	}

	if len(*cpuProfile) != 0 {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *geojsonCodec {
		tileCodec = &tile.GeoJSONCodec{}
	} else {
		tileCodec = &mvt.Codec{}
	}

	var (
		f   io.Reader
		err error
	)

	if !sourceStdIn {
		f, err = os.Open(*source)
		if err != nil {
			log.Fatal(err)
		}
		defer f.(io.Closer).Close()
	} else {
		f = os.Stdin
	}

	err = os.MkdirAll(*target, 0777)
	if err != nil {
		log.Fatal(err)
	}

	log.Println("parsing input...")
	fc := spatial.FeatureCollection{}
	codec := spaten.Codec{}
	err = codec.Decode(f, &fc)
	if err != nil {
		log.Fatalf("could not read incoming file: %v", err)
	}

	if len(fc.Features) == 0 {
		log.Fatal("no features in input file")
	}

	log.Printf("read %d features", len(fc.Features))

	var (
		bboxPts []spatial.Point
		ft      = featureTable(zoomlevels)
	)
	for featID, feat := range fc.Features {
		for _, zl := range zoomlevels {
			if !renderable(feat.Props, zl) {
				continue
			}
			for _, tid := range tile.Coverage(feat.Geometry.BBox(), zl) {
				ft[zl][tid.X][tid.Y] = append(ft[zl][tid.X][tid.Y], &fc.Features[featID])
			}
		}
		bb := feat.Geometry.BBox()
		bboxPts = append(bboxPts, bb.SW, bb.NE)
	}

	log.Println("determining which tiles need to be generated")
	var tc []tile.ID
	for _, zoomlevel := range zoomlevels {
		tc = append(tc, tile.Coverage(spatial.Line(bboxPts).BBox(), zoomlevel)...)
	}

	log.Printf("starting to generate %d tiles...", len(tc))

	dtw := diskTileWriter{basedir: *target, compressTiles: *compressTiles}
	dlm := defaultLayerMapper{defaultLayer: *defaultLayer}

	shuffleWork(tc) // randomize order for better worker saturation
	var (
		wg       sync.WaitGroup
		ws       = workerSlices(tc, *workersNumber)
		pb, done = progressbar.NewBar(len(tc), len(ws))
	)
	for wrk := 0; wrk < len(ws); wrk++ {
		wg.Add(1)
		go func(i int) {
			generateTiles(ws[i], ft, &dtw, tileCodec, &dlm, pb)
			wg.Done()
		}(wrk)
	}
	wg.Wait()
	done()
}

func renderable(props map[string]interface{}, zl int) bool {
	var propInt = func(props map[string]interface{}, name string, defaultVal int) int {
		v, ok := props[name]
		if !ok {
			return defaultVal
		}
		i, ok := v.(int)
		if ok {
			return i
		}
		f, ok := v.(float64)
		if ok {
			return int(f)
		}
		log.Printf("%v is neither int nor float: %v", props, v)
		return defaultVal
	}
	return (zl >= propInt(props, "@zoom:min", 0)) && (zl <= propInt(props, "@zoom:max", 99))
}

func featureTable(zls []int) map[int][][][]*spatial.Feature {
	r := map[int][][][]*spatial.Feature{}
	for _, zl := range zls {
		l := pow(2, zl)
		r[zl] = make([][][]*spatial.Feature, l)
		for x := range r[zl] {
			r[zl][x] = make([][]*spatial.Feature, l)
		}
	}
	return r
}

func workerSlices(tiles []tile.ID, wrkNum int) [][]tile.ID {
	var r [][]tile.ID
	if len(tiles) <= wrkNum {
		for t := 0; t < len(tiles); t++ {
			r = append(r, []tile.ID{tiles[t]})
		}
		return r
	}
	for wrkr := 0; wrkr < wrkNum; wrkr++ {
		start := (len(tiles) / wrkNum) * wrkr
		end := (len(tiles) / wrkNum) * (wrkr + 1)
		if wrkr == wrkNum-1 {
			end = len(tiles)
		}
		r = append(r, tiles[start:end])
	}
	return r
}

type diskTileWriter struct {
	basedir       string
	compressTiles bool
}

func (tw *diskTileWriter) WriteTile(tID tile.ID, buf []byte) error {
	err := os.MkdirAll(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X)), 0777)
	if err != nil {
		return err
	}
	tf, err := os.Create(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X), strconv.Itoa(tID.Y)+".mvt")) // TODO: file extension based on codec
	if err != nil {
		return err
	}
	defer tf.Close()

	if tw.compressTiles {
		_, err = gzip.NewWriter(tf).Write(buf)
	} else {
		_, err = tf.Write(buf)
	}
	if err != nil {
		return err
	}
	return nil
}

type defaultLayerMapper struct {
	defaultLayer bool
}

func (dlm *defaultLayerMapper) LayerName(props map[string]interface{}) string {
	if layerName, ok := props["@layer"]; ok {
		return layerName.(string)
	}
	if dlm.defaultLayer {
		return "default"
	}
	return ""
}

type layerMapper interface {
	LayerName(map[string]interface{}) string
}

type tileWriter interface {
	WriteTile(tile.ID, []byte) error
}

func generateTiles(tIDs []tile.ID, features map[int][][][]*spatial.Feature, tw tileWriter, encoder tile.Codec, lm layerMapper, pb chan<- struct{}) {
	for _, tID := range tIDs {
		var (
			layers = map[string][]spatial.Feature{}
			ln     string
		)
		defer func() { pb <- struct{}{} }()
		for _, ft := range features[tID.Z][tID.X][tID.Y] {
			feat := *ft
			ln = lm.LayerName(feat.Props)
			if len(ln) != 0 {
				if _, ok := layers[ln]; !ok {
					layers[ln] = []spatial.Feature{feat}
				} else {
					layers[ln] = append(layers[ln], feat)
				}
			}
		}
		if !anyFeatures(layers) {
			continue
		}
		buf, err := encoder.EncodeTile(layers, tID)
		if err != nil {
			log.Fatal(err)
		}

		err = tw.WriteTile(tID, buf)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func anyFeatures(layers map[string][]spatial.Feature) bool {
	for i := range layers {
		if len(layers[i]) > 0 {
			return true
		}
	}
	return false
}

func pow(x, y int) int {
	var res = 1
	for i := 1; i <= y; i++ {
		res *= x
	}
	return res
}
<|endoftext|>


<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"

	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/app"
	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/migration"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/provision/docker"
	"gopkg.in/mgo.v2/bson"
	"launchpad.net/gnuflag"
)

func getProvisioner() (string, error) {
	provisioner, err := config.GetString("provisioner")
	if provisioner == "" {
		provisioner = "docker"
	}
	return provisioner, err
}

type migrateCmd struct {
	fs  *gnuflag.FlagSet
	dry bool
}

func (*migrateCmd) Info() *cmd.Info {
	return &cmd.Info{
		Name:  "migrate",
		Usage: "migrate",
		Desc:  "Runs migrations from previous versions of tsr",
	}
}

func (c *migrateCmd) Run(context *cmd.Context, client *cmd.Client) error {
	err := migration.Register("migrate-docker-images", c.migrateImages)
	if err != nil {
		return err
	}
	err = migration.Register("migrate-pool", c.migratePool)
	if err != nil {
		return err
	}
	return migration.Run(context.Stdout, c.dry)
}

func (c *migrateCmd) migrateImages() error {
	provisioner, _ := getProvisioner()
	if provisioner == "docker" {
		p, err := provision.Get(provisioner)
		if err != nil {
			return err
		}
		err = p.(provision.InitializableProvisioner).Initialize()
		if err != nil {
			return err
		}
		return docker.MigrateImages()
	}
	return nil
}

func (c *migrateCmd) migratePool() error {
	db, err := db.Conn()
	if err != nil {
		return err
	}
	defer db.Close()
	dbName, _ := config.GetString("database:name")
	fromColl := fmt.Sprintf("%s.docker_scheduler", dbName)
	toColl := fmt.Sprintf("%s.pool", dbName)
	session := db.Collection("docker_scheduler").Database.Session
	err = session.Run(bson.D{{"renameCollection", fromColl}, {"to", toColl}}, &bson.M{})
	if err != nil {
		return err
	}
	return nil
}

func (c *migrateCmd) setPoolToApps() error {
	db, err := db.Conn()
	if err != nil {
		return err
	}
	defer db.Close()
	var apps []app.App
	var tooManyPoolsApps []app.App
	err = db.Apps().Find(nil).All(&apps)
	if err != nil {
		return err
	}
	for _, a := range apps {
		err = a.SetPool()
		if err != nil {
			tooManyPoolsApps = append(tooManyPoolsApps, a)
			continue
		}
		err = db.Apps().Update(bson.M{"name": a.Name}, bson.M{"$set": bson.M{"pool": a.Pool}})
		if err != nil {
			return err
		}
	}
	if len(tooManyPoolsApps) > 0 {
		fmt.Println("Apps bellow couldn't be migrated because they are in more than one pool.")
		fmr.Println("To fix this, please run `tsuru app-change-pool <pool_name> -a app` for each app.")
		fmt.Println("*****************************************")
		for _, a := range tooManyPoolsApps {
			fmt.Println(a.Name)
		}
	}
	return nil
}

func (c *migrateCmd) Flags() *gnuflag.FlagSet {
	if c.fs == nil {
		c.fs = gnuflag.NewFlagSet("migrate", gnuflag.ExitOnError)
		c.fs.BoolVar(&c.dry, "dry", false, "Do not run migrations, just print what would run")
		c.fs.BoolVar(&c.dry, "n", false, "Do not run migrations, just print what would run")
	}
	return c.fs
}
<commit_msg>cmd/tsr/migrate: fix typo.<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"

	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/app"
	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/migration"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/provision/docker"
	"gopkg.in/mgo.v2/bson"
	"launchpad.net/gnuflag"
)

func getProvisioner() (string, error) {
	provisioner, err := config.GetString("provisioner")
	if provisioner == "" {
		provisioner = "docker"
	}
	return provisioner, err
}

type migrateCmd struct {
	fs  *gnuflag.FlagSet
	dry bool
}

func (*migrateCmd) Info() *cmd.Info {
	return &cmd.Info{
		Name:  "migrate",
		Usage: "migrate",
		Desc:  "Runs migrations from previous versions of tsr",
	}
}

func (c *migrateCmd) Run(context *cmd.Context, client *cmd.Client) error {
	err := migration.Register("migrate-docker-images", c.migrateImages)
	if err != nil {
		return err
	}
	err = migration.Register("migrate-pool", c.migratePool)
	if err != nil {
		return err
	}
	err = migration.Register("migrate-set-pool-to-app", c.setPoolToApps)
	if err != nil {
		return err
	}
	return migration.Run(context.Stdout, c.dry)
}

func (c *migrateCmd) migrateImages() error {
	provisioner, _ := getProvisioner()
	if provisioner == "docker" {
		p, err := provision.Get(provisioner)
		if err != nil {
			return err
		}
		err = p.(provision.InitializableProvisioner).Initialize()
		if err != nil {
			return err
		}
		return docker.MigrateImages()
	}
	return nil
}

func (c *migrateCmd) migratePool() error {
	db, err := db.Conn()
	if err != nil {
		return err
	}
	defer db.Close()
	dbName, _ := config.GetString("database:name")
	fromColl := fmt.Sprintf("%s.docker_scheduler", dbName)
	toColl := fmt.Sprintf("%s.pool", dbName)
	session := db.Collection("docker_scheduler").Database.Session
	err = session.Run(bson.D{{"renameCollection", fromColl}, {"to", toColl}}, &bson.M{})
	if err != nil {
		return err
	}
	return nil
}

func (c *migrateCmd) setPoolToApps() error {
	db, err := db.Conn()
	if err != nil {
		return err
	}
	defer db.Close()
	var apps []app.App
	var tooManyPoolsApps []app.App
	err = db.Apps().Find(nil).All(&apps)
	if err != nil {
		return err
	}
	for _, a := range apps {
		err = a.SetPool()
		if err != nil {
			tooManyPoolsApps = append(tooManyPoolsApps, a)
			continue
		}
		err = db.Apps().Update(bson.M{"name": a.Name}, bson.M{"$set": bson.M{"pool": a.Pool}})
		if err != nil {
			return err
		}
	}
	if len(tooManyPoolsApps) > 0 {
		fmt.Println("Apps bellow couldn't be migrated because they are in more than one pool.")
		fmt.Println("To fix this, please run `tsuru app-change-pool <pool_name> -a app` for each app.")
		fmt.Println("*****************************************")
		for _, a := range tooManyPoolsApps {
			fmt.Println(a.Name)
		}
	}
	return nil
}

func (c *migrateCmd) Flags() *gnuflag.FlagSet {
	if c.fs == nil {
		c.fs = gnuflag.NewFlagSet("migrate", gnuflag.ExitOnError)
		c.fs.BoolVar(&c.dry, "dry", false, "Do not run migrations, just print what would run")
		c.fs.BoolVar(&c.dry, "n", false, "Do not run migrations, just print what would run")
	}
	return c.fs
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 ScyllaDB
// Use of this source code is governed by a ALv2-style
// license that can be found in the LICENSE file.

// +build all integration

package gocqlx_test

import (
	"math/big"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/gocql/gocql"
	"github.com/scylladb/gocqlx"
	. "github.com/scylladb/gocqlx/gocqlxtest"
	"github.com/scylladb/gocqlx/qb"
	"gopkg.in/inf.v0"
)

type FullName struct {
	FirstName string
	LastName  string
}

func (n FullName) MarshalCQL(info gocql.TypeInfo) ([]byte, error) {
	return []byte(n.FirstName + " " + n.LastName), nil
}

func (n *FullName) UnmarshalCQL(info gocql.TypeInfo, data []byte) error {
	t := strings.SplitN(string(data), " ", 2)
	n.FirstName, n.LastName = t[0], t[1]
	return nil
}

func TestStruct(t *testing.T) {
	session := CreateSession(t)
	defer session.Close()
	if err := ExecStmt(session, `CREATE TABLE gocqlx_test.struct_table (
			testuuid       timeuuid PRIMARY KEY,
			testtimestamp  timestamp,
			testvarchar    varchar,
			testbigint     bigint,
			testblob       blob,
			testbool       boolean,
			testfloat      float,
			testdouble     double,
			testint        int,
			testdecimal    decimal,
			testlist       list<text>,
			testset        set<int>,
			testmap        map<varchar, varchar>,
			testvarint     varint,
			testinet       inet,
			testcustom     text

		)`); err != nil {
		t.Fatal("create table:", err)
	}

	type StructTable struct {
		Testuuid      gocql.UUID
		Testvarchar   string
		Testbigint    int64
		Testtimestamp time.Time
		Testblob      []byte
		Testbool      bool
		Testfloat     float32
		Testdouble    float64
		Testint       int
		Testdecimal   *inf.Dec
		Testlist      []string
		Testset       []int
		Testmap       map[string]string
		Testvarint    *big.Int
		Testinet      string
		Testcustom    FullName
	}

	bigInt := new(big.Int)
	if _, ok := bigInt.SetString("830169365738487321165427203929228", 10); !ok {
		t.Fatal("failed setting bigint by string")
	}

	m := StructTable{
		Testuuid:      gocql.TimeUUID(),
		Testvarchar:   "Test VarChar",
		Testbigint:    time.Now().Unix(),
		Testtimestamp: time.Now().Truncate(time.Millisecond).UTC(),
		Testblob:      []byte("test blob"),
		Testbool:      true,
		Testfloat:     float32(4.564),
		Testdouble:    float64(4.815162342),
		Testint:       2343,
		Testdecimal:   inf.NewDec(100, 0),
		Testlist:      []string{"quux", "foo", "bar", "baz", "quux"},
		Testset:       []int{1, 2, 3, 4, 5, 6, 7, 8, 9},
		Testmap:       map[string]string{"field1": "val1", "field2": "val2", "field3": "val3"},
		Testvarint:    bigInt,
		Testinet:      "213.212.2.19",
		Testcustom:    FullName{FirstName: "John", LastName: "Doe"},
	}

	if err := session.Query(`INSERT INTO struct_table (testuuid, testtimestamp, testvarchar, testbigint, testblob, testbool, testfloat,testdouble, testint, testdecimal, testlist, testset, testmap, testvarint, testinet, testcustom) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
		m.Testuuid,
		m.Testtimestamp,
		m.Testvarchar,
		m.Testbigint,
		m.Testblob,
		m.Testbool,
		m.Testfloat,
		m.Testdouble,
		m.Testint,
		m.Testdecimal,
		m.Testlist,
		m.Testset,
		m.Testmap,
		m.Testvarint,
		m.Testinet,
		m.Testcustom).Exec(); err != nil {
		t.Fatal("insert:", err)
	}

	t.Run("get", func(t *testing.T) {
		var v StructTable
		if err := gocqlx.Get(&v, session.Query(`SELECT * FROM struct_table`)); err != nil {
			t.Fatal("get failed", err)
		}

		if !reflect.DeepEqual(m, v) {
			t.Fatal("not equals")
		}
	})

	t.Run("select", func(t *testing.T) {
		var v []StructTable
		if err := gocqlx.Select(&v, session.Query(`SELECT * FROM struct_table`)); err != nil {
			t.Fatal("select failed", err)
		}

		if len(v) != 1 {
			t.Fatal("select unexpecrted number of rows", len(v))
		}

		if !reflect.DeepEqual(m, v[0]) {
			t.Fatal("not equals")
		}
	})

	t.Run("select ptr", func(t *testing.T) {
		var v []*StructTable
		if err := gocqlx.Select(&v, session.Query(`SELECT * FROM struct_table`)); err != nil {
			t.Fatal("select failed", err)
		}

		if len(v) != 1 {
			t.Fatal("select unexpecrted number of rows", len(v))
		}

		if !reflect.DeepEqual(&m, v[0]) {
			t.Fatal("not equals")
		}
	})
}

func TestScannable(t *testing.T) {
	session := CreateSession(t)
	defer session.Close()
	if err := ExecStmt(session, `CREATE TABLE gocqlx_test.scannable_table (testfullname text PRIMARY KEY)`); err != nil {
		t.Fatal("create table:", err)
	}
	m := FullName{"John", "Doe"}

	if err := session.Query(`INSERT INTO scannable_table (testfullname) values (?)`, m).Exec(); err != nil {
		t.Fatal("insert:", err)
	}

	t.Run("get", func(t *testing.T) {
		var v FullName
		if err := gocqlx.Get(&v, session.Query(`SELECT testfullname FROM scannable_table`)); err != nil {
			t.Fatal("get failed", err)
		}

		if !reflect.DeepEqual(m, v) {
			t.Fatal("not equals")
		}
	})

	t.Run("select", func(t *testing.T) {
		var v []FullName
		if err := gocqlx.Select(&v, session.Query(`SELECT testfullname FROM scannable_table`)); err != nil {
			t.Fatal("get failed", err)
		}

		if len(v) != 1 {
			t.Fatal("select unexpecrted number of rows", len(v))
		}

		if !reflect.DeepEqual(m, v[0]) {
			t.Fatal("not equals")
		}
	})

	t.Run("select ptr", func(t *testing.T) {
		var v []*FullName
		if err := gocqlx.Select(&v, session.Query(`SELECT testfullname FROM scannable_table`)); err != nil {
			t.Fatal("get failed", err)
		}

		if len(v) != 1 {
			t.Fatal("select unexpecrted number of rows", len(v))
		}

		if !reflect.DeepEqual(&m, v[0]) {
			t.Fatal("not equals")
		}
	})
}

func TestUnsafe(t *testing.T) {
	session := CreateSession(t)
	defer session.Close()
	if err := ExecStmt(session, `CREATE TABLE gocqlx_test.unsafe_table (testtext text PRIMARY KEY, testtextunbound text)`); err != nil {
		t.Fatal("create table:", err)
	}
	if err := session.Query(`INSERT INTO unsafe_table (testtext, testtextunbound) values (?, ?)`, "test", "test").Exec(); err != nil {
		t.Fatal("insert:", err)
	}

	type UnsafeTable struct {
		Testtext string
	}

	t.Run("safe get", func(t *testing.T) {
		var v UnsafeTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM unsafe_table`))
		if err := i.Get(&v); err == nil || err.Error() != "missing destination name \"testtextunbound\" in *gocqlx_test.UnsafeTable" {
			t.Fatal("expected ErrNotFound", "got", err)
		}
	})

	t.Run("safe select", func(t *testing.T) {
		var v []UnsafeTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM unsafe_table`))
		if err := i.Select(&v); err == nil || err.Error() != "missing destination name \"testtextunbound\" in *gocqlx_test.UnsafeTable" {
			t.Fatal("expected ErrNotFound", "got", err)
		}
		if cap(v) > 0 {
			t.Fatal("side effect alloc")
		}
	})

	t.Run("unsafe get", func(t *testing.T) {
		var v UnsafeTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM unsafe_table`))
		if err := i.Unsafe().Get(&v); err != nil {
			t.Fatal(err)
		}
		if v.Testtext != "test" {
			t.Fatal("get failed")
		}
	})

	t.Run("unsafe select", func(t *testing.T) {
		var v []UnsafeTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM unsafe_table`))
		if err := i.Unsafe().Select(&v); err != nil {
			t.Fatal(err)
		}
		if len(v) != 1 {
			t.Fatal("select failed")
		}
		if v[0].Testtext != "test" {
			t.Fatal("select failed")
		}
	})

	t.Run("DefaultUnsafe select", func(t *testing.T) {
		gocqlx.DefaultUnsafe = true
		defer func() { gocqlx.DefaultUnsafe = true }()
		var v []UnsafeTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM unsafe_table`))
		if err := i.Select(&v); err != nil {
			t.Fatal(err)
		}
		if len(v) != 1 {
			t.Fatal("select failed")
		}
		if v[0].Testtext != "test" {
			t.Fatal("select failed")
		}
	})
}

func TestNotFound(t *testing.T) {
	session := CreateSession(t)
	defer session.Close()
	if err := ExecStmt(session, `CREATE TABLE gocqlx_test.not_found_table (testtext text PRIMARY KEY)`); err != nil {
		t.Fatal("create table:", err)
	}

	type NotFoundTable struct {
		Testtext string
	}

	t.Run("get cql error", func(t *testing.T) {
		var v NotFoundTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM not_found_table WRONG`).RetryPolicy(nil))

		err := i.Get(&v)
		if err == nil || !strings.Contains(err.Error(), "WRONG") {
			t.Fatal(err)
		}
	})

	t.Run("get", func(t *testing.T) {
		var v NotFoundTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM not_found_table`))
		if err := i.Get(&v); err != gocql.ErrNotFound {
			t.Fatal("expected ErrNotFound", "got", err)
		}
	})

	t.Run("select cql error", func(t *testing.T) {
		var v []NotFoundTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM not_found_table WRONG`).RetryPolicy(nil))

		err := i.Select(&v)
		if err == nil || !strings.Contains(err.Error(), "WRONG") {
			t.Fatal(err)
		}
	})

	t.Run("select", func(t *testing.T) {
		var v []NotFoundTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM not_found_table`))
		if err := i.Select(&v); err != nil {
			t.Fatal(err)
		}
		if cap(v) > 0 {
			t.Fatal("side effect alloc")
		}
	})
}

func TestPaging(t *testing.T) {
	session := CreateSession(t)
	defer session.Close()
	if err := ExecStmt(session, `CREATE TABLE gocqlx_test.paging_table (id int PRIMARY KEY, val int)`); err != nil {
		t.Fatal("create table:", err)
	}
	if err := ExecStmt(session, `CREATE INDEX id_val_index ON gocqlx_test.paging_table (val)`); err != nil {
		t.Fatal("create index:", err)
	}
	stmt, names := qb.Insert("gocqlx_test.paging_table").Columns("id", "val").ToCql()
	q := gocqlx.Query(session.Query(stmt), names)
	for i := 0; i < 5000; i++ {
		if err := q.Bind(i, i).Exec(); err != nil {
			t.Fatal(err)
		}
	}

	type Paging struct {
		ID  int
		Val int
	}

	t.Run("iter", func(t *testing.T) {
		stmt, names := qb.Select("gocqlx_test.paging_table").
			Where(qb.Lt("val")).
			AllowFiltering().
			Columns("id", "val").ToCql()
		it := gocqlx.Query(session.Query(stmt, 100).PageSize(10), names).Iter()
		defer it.Close()
		var cnt int
		for {
			p := &Paging{}
			if !it.StructScan(p) {
				break
			}
			cnt++
		}
		if cnt != 100 {
			t.Fatal("expected 100", "got", cnt)
		}
	})
}
<commit_msg>Fix resetting DefaultUnsafe option after the DefaultUnsafe select test<commit_after>// Copyright (C) 2017 ScyllaDB
// Use of this source code is governed by a ALv2-style
// license that can be found in the LICENSE file.

// +build all integration

package gocqlx_test

import (
	"math/big"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/gocql/gocql"
	"github.com/scylladb/gocqlx"
	. "github.com/scylladb/gocqlx/gocqlxtest"
	"github.com/scylladb/gocqlx/qb"
	"gopkg.in/inf.v0"
)

type FullName struct {
	FirstName string
	LastName  string
}

func (n FullName) MarshalCQL(info gocql.TypeInfo) ([]byte, error) {
	return []byte(n.FirstName + " " + n.LastName), nil
}

func (n *FullName) UnmarshalCQL(info gocql.TypeInfo, data []byte) error {
	t := strings.SplitN(string(data), " ", 2)
	n.FirstName, n.LastName = t[0], t[1]
	return nil
}

func TestStruct(t *testing.T) {
	session := CreateSession(t)
	defer session.Close()
	if err := ExecStmt(session, `CREATE TABLE gocqlx_test.struct_table (
			testuuid       timeuuid PRIMARY KEY,
			testtimestamp  timestamp,
			testvarchar    varchar,
			testbigint     bigint,
			testblob       blob,
			testbool       boolean,
			testfloat      float,
			testdouble     double,
			testint        int,
			testdecimal    decimal,
			testlist       list<text>,
			testset        set<int>,
			testmap        map<varchar, varchar>,
			testvarint     varint,
			testinet       inet,
			testcustom     text

		)`); err != nil {
		t.Fatal("create table:", err)
	}

	type StructTable struct {
		Testuuid      gocql.UUID
		Testvarchar   string
		Testbigint    int64
		Testtimestamp time.Time
		Testblob      []byte
		Testbool      bool
		Testfloat     float32
		Testdouble    float64
		Testint       int
		Testdecimal   *inf.Dec
		Testlist      []string
		Testset       []int
		Testmap       map[string]string
		Testvarint    *big.Int
		Testinet      string
		Testcustom    FullName
	}

	bigInt := new(big.Int)
	if _, ok := bigInt.SetString("830169365738487321165427203929228", 10); !ok {
		t.Fatal("failed setting bigint by string")
	}

	m := StructTable{
		Testuuid:      gocql.TimeUUID(),
		Testvarchar:   "Test VarChar",
		Testbigint:    time.Now().Unix(),
		Testtimestamp: time.Now().Truncate(time.Millisecond).UTC(),
		Testblob:      []byte("test blob"),
		Testbool:      true,
		Testfloat:     float32(4.564),
		Testdouble:    float64(4.815162342),
		Testint:       2343,
		Testdecimal:   inf.NewDec(100, 0),
		Testlist:      []string{"quux", "foo", "bar", "baz", "quux"},
		Testset:       []int{1, 2, 3, 4, 5, 6, 7, 8, 9},
		Testmap:       map[string]string{"field1": "val1", "field2": "val2", "field3": "val3"},
		Testvarint:    bigInt,
		Testinet:      "213.212.2.19",
		Testcustom:    FullName{FirstName: "John", LastName: "Doe"},
	}

	if err := session.Query(`INSERT INTO struct_table (testuuid, testtimestamp, testvarchar, testbigint, testblob, testbool, testfloat,testdouble, testint, testdecimal, testlist, testset, testmap, testvarint, testinet, testcustom) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
		m.Testuuid,
		m.Testtimestamp,
		m.Testvarchar,
		m.Testbigint,
		m.Testblob,
		m.Testbool,
		m.Testfloat,
		m.Testdouble,
		m.Testint,
		m.Testdecimal,
		m.Testlist,
		m.Testset,
		m.Testmap,
		m.Testvarint,
		m.Testinet,
		m.Testcustom).Exec(); err != nil {
		t.Fatal("insert:", err)
	}

	t.Run("get", func(t *testing.T) {
		var v StructTable
		if err := gocqlx.Get(&v, session.Query(`SELECT * FROM struct_table`)); err != nil {
			t.Fatal("get failed", err)
		}

		if !reflect.DeepEqual(m, v) {
			t.Fatal("not equals")
		}
	})

	t.Run("select", func(t *testing.T) {
		var v []StructTable
		if err := gocqlx.Select(&v, session.Query(`SELECT * FROM struct_table`)); err != nil {
			t.Fatal("select failed", err)
		}

		if len(v) != 1 {
			t.Fatal("select unexpecrted number of rows", len(v))
		}

		if !reflect.DeepEqual(m, v[0]) {
			t.Fatal("not equals")
		}
	})

	t.Run("select ptr", func(t *testing.T) {
		var v []*StructTable
		if err := gocqlx.Select(&v, session.Query(`SELECT * FROM struct_table`)); err != nil {
			t.Fatal("select failed", err)
		}

		if len(v) != 1 {
			t.Fatal("select unexpecrted number of rows", len(v))
		}

		if !reflect.DeepEqual(&m, v[0]) {
			t.Fatal("not equals")
		}
	})
}

func TestScannable(t *testing.T) {
	session := CreateSession(t)
	defer session.Close()
	if err := ExecStmt(session, `CREATE TABLE gocqlx_test.scannable_table (testfullname text PRIMARY KEY)`); err != nil {
		t.Fatal("create table:", err)
	}
	m := FullName{"John", "Doe"}

	if err := session.Query(`INSERT INTO scannable_table (testfullname) values (?)`, m).Exec(); err != nil {
		t.Fatal("insert:", err)
	}

	t.Run("get", func(t *testing.T) {
		var v FullName
		if err := gocqlx.Get(&v, session.Query(`SELECT testfullname FROM scannable_table`)); err != nil {
			t.Fatal("get failed", err)
		}

		if !reflect.DeepEqual(m, v) {
			t.Fatal("not equals")
		}
	})

	t.Run("select", func(t *testing.T) {
		var v []FullName
		if err := gocqlx.Select(&v, session.Query(`SELECT testfullname FROM scannable_table`)); err != nil {
			t.Fatal("get failed", err)
		}

		if len(v) != 1 {
			t.Fatal("select unexpecrted number of rows", len(v))
		}

		if !reflect.DeepEqual(m, v[0]) {
			t.Fatal("not equals")
		}
	})

	t.Run("select ptr", func(t *testing.T) {
		var v []*FullName
		if err := gocqlx.Select(&v, session.Query(`SELECT testfullname FROM scannable_table`)); err != nil {
			t.Fatal("get failed", err)
		}

		if len(v) != 1 {
			t.Fatal("select unexpecrted number of rows", len(v))
		}

		if !reflect.DeepEqual(&m, v[0]) {
			t.Fatal("not equals")
		}
	})
}

func TestUnsafe(t *testing.T) {
	session := CreateSession(t)
	defer session.Close()
	if err := ExecStmt(session, `CREATE TABLE gocqlx_test.unsafe_table (testtext text PRIMARY KEY, testtextunbound text)`); err != nil {
		t.Fatal("create table:", err)
	}
	if err := session.Query(`INSERT INTO unsafe_table (testtext, testtextunbound) values (?, ?)`, "test", "test").Exec(); err != nil {
		t.Fatal("insert:", err)
	}

	type UnsafeTable struct {
		Testtext string
	}

	t.Run("safe get", func(t *testing.T) {
		var v UnsafeTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM unsafe_table`))
		if err := i.Get(&v); err == nil || err.Error() != "missing destination name \"testtextunbound\" in *gocqlx_test.UnsafeTable" {
			t.Fatal("expected ErrNotFound", "got", err)
		}
	})

	t.Run("safe select", func(t *testing.T) {
		var v []UnsafeTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM unsafe_table`))
		if err := i.Select(&v); err == nil || err.Error() != "missing destination name \"testtextunbound\" in *gocqlx_test.UnsafeTable" {
			t.Fatal("expected ErrNotFound", "got", err)
		}
		if cap(v) > 0 {
			t.Fatal("side effect alloc")
		}
	})

	t.Run("unsafe get", func(t *testing.T) {
		var v UnsafeTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM unsafe_table`))
		if err := i.Unsafe().Get(&v); err != nil {
			t.Fatal(err)
		}
		if v.Testtext != "test" {
			t.Fatal("get failed")
		}
	})

	t.Run("unsafe select", func(t *testing.T) {
		var v []UnsafeTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM unsafe_table`))
		if err := i.Unsafe().Select(&v); err != nil {
			t.Fatal(err)
		}
		if len(v) != 1 {
			t.Fatal("select failed")
		}
		if v[0].Testtext != "test" {
			t.Fatal("select failed")
		}
	})

	t.Run("DefaultUnsafe select", func(t *testing.T) {
		gocqlx.DefaultUnsafe = true
		defer func() { gocqlx.DefaultUnsafe = false }()
		var v []UnsafeTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM unsafe_table`))
		if err := i.Select(&v); err != nil {
			t.Fatal(err)
		}
		if len(v) != 1 {
			t.Fatal("select failed")
		}
		if v[0].Testtext != "test" {
			t.Fatal("select failed")
		}
	})
}

func TestNotFound(t *testing.T) {
	session := CreateSession(t)
	defer session.Close()
	if err := ExecStmt(session, `CREATE TABLE gocqlx_test.not_found_table (testtext text PRIMARY KEY)`); err != nil {
		t.Fatal("create table:", err)
	}

	type NotFoundTable struct {
		Testtext string
	}

	t.Run("get cql error", func(t *testing.T) {
		var v NotFoundTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM not_found_table WRONG`).RetryPolicy(nil))

		err := i.Get(&v)
		if err == nil || !strings.Contains(err.Error(), "WRONG") {
			t.Fatal(err)
		}
	})

	t.Run("get", func(t *testing.T) {
		var v NotFoundTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM not_found_table`))
		if err := i.Get(&v); err != gocql.ErrNotFound {
			t.Fatal("expected ErrNotFound", "got", err)
		}
	})

	t.Run("select cql error", func(t *testing.T) {
		var v []NotFoundTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM not_found_table WRONG`).RetryPolicy(nil))

		err := i.Select(&v)
		if err == nil || !strings.Contains(err.Error(), "WRONG") {
			t.Fatal(err)
		}
	})

	t.Run("select", func(t *testing.T) {
		var v []NotFoundTable
		i := gocqlx.Iter(session.Query(`SELECT * FROM not_found_table`))
		if err := i.Select(&v); err != nil {
			t.Fatal(err)
		}
		if cap(v) > 0 {
			t.Fatal("side effect alloc")
		}
	})
}

func TestPaging(t *testing.T) {
	session := CreateSession(t)
	defer session.Close()
	if err := ExecStmt(session, `CREATE TABLE gocqlx_test.paging_table (id int PRIMARY KEY, val int)`); err != nil {
		t.Fatal("create table:", err)
	}
	if err := ExecStmt(session, `CREATE INDEX id_val_index ON gocqlx_test.paging_table (val)`); err != nil {
		t.Fatal("create index:", err)
	}
	stmt, names := qb.Insert("gocqlx_test.paging_table").Columns("id", "val").ToCql()
	q := gocqlx.Query(session.Query(stmt), names)
	for i := 0; i < 5000; i++ {
		if err := q.Bind(i, i).Exec(); err != nil {
			t.Fatal(err)
		}
	}

	type Paging struct {
		ID  int
		Val int
	}

	t.Run("iter", func(t *testing.T) {
		stmt, names := qb.Select("gocqlx_test.paging_table").
			Where(qb.Lt("val")).
			AllowFiltering().
			Columns("id", "val").ToCql()
		it := gocqlx.Query(session.Query(stmt, 100).PageSize(10), names).Iter()
		defer it.Close()
		var cnt int
		for {
			p := &Paging{}
			if !it.StructScan(p) {
				break
			}
			cnt++
		}
		if cnt != 100 {
			t.Fatal("expected 100", "got", cnt)
		}
	})
}
<|endoftext|>


<commit_before>package jade

import (
	"log"
	"os"
	"path/filepath"
	"strings"
)

func (t *Tree) topParse() {
	t.Root = t.newList(t.peek().pos)
	var (
		ext   bool
		token = t.nextNonSpace()
	)
	if token.typ == itemExtends {
		ext = true
		t.Root.append(t.parseSubFile(token.val))
		token = t.nextNonSpace()
	}
	for {
		switch token.typ {
		case itemInclude:
			t.Root.append(t.parseInclude(token))
		case itemBlock, itemBlockPrepend, itemBlockAppend:
			if ext {
				t.parseBlock(token)
			} else {
				t.Root.append(t.parseBlock(token))
			}
		case itemMixin:
			t.mixin[token.val] = t.parseMixin(token)
		case itemEOF:
			return
		case itemExtends:
			t.errorf(`Declaration of template inheritance ("extends") should be the first thing in the file. There can only be one extends statement per file.`)
		case itemError:
			t.errorf("%s line: %d\n", token.val, token.line)
		default:
			if ext {
				t.errorf(`Only import, named blocks and mixins can appear at the top level of an extending template`)
			}
			t.Root.append(t.hub(token))
		}
		token = t.nextNonSpace()
	}
}

func (t *Tree) hub(token item) (n Node) {
	for {
		switch token.typ {
		case itemDiv:
			token.val = "div"
			fallthrough
		case itemTag, itemTagInline, itemTagVoid, itemTagVoidInline:
			return t.parseTag(token)
		case itemText, itemComment, itemHTMLTag:
			return t.newText(token.pos, []byte(token.val), token.typ)
		case itemCode, itemCodeBuffered, itemCodeUnescaped, itemMixinBlock:
			return t.newCode(token.pos, token.val, token.typ)
		case itemIf, itemUnless:
			return t.parseIf(token)
		case itemFor, itemEach, itemWhile:
			return t.parseFor(token)
		case itemCase:
			return t.parseCase(token)
		case itemBlock, itemBlockPrepend, itemBlockAppend:
			return t.parseBlock(token)
		case itemMixinCall:
			return t.parseMixinUse(token)
		case itemInclude:
			return t.parseInclude(token)
		case itemDoctype:
			return t.newDoctype(token.pos, token.val)
		case itemFilter:
			return t.parseFilter(token)
		case itemError:
			t.errorf("Error lex: %s line: %d\n", token.val, token.line)
		default:
			t.errorf(`Error hub(): unexpected token  "%s"  type  "%s"`, token.val, token.typ)
		}
	}
}

func (t *Tree) parseFilter(tk item) Node {
	var subf, args, text string
Loop:
	for {
		switch token := t.nextNonSpace(); token.typ {
		case itemFilterSubf:
			subf = token.val
		case itemFilterArgs:
			args = strings.Trim(token.val, " \t\r\n")
		case itemFilterText:
			text = strings.Trim(token.val, " \t\r\n")
		default:
			break Loop
		}
	}
	t.backup()
	switch tk.val {
	case "go":
		filterGo(subf, args, text)
	case "markdown", "markdown-it":
		// TODO: filterMarkdown(subf, args, text)
	}
	return t.newList(tk.pos) // for return nothing
}

func filterGo(subf, args, text string) {
	switch subf {
	case "func":
		Go.Name = ""
		switch args {
		case "name":
			Go.Name = text
		case "arg", "args":
			if Go.Args != "" {
				Go.Args += ", " + strings.Trim(text, "()")
			} else {
				Go.Args = strings.Trim(text, "()")
			}
		default:
			fn := strings.Split(text, "(")
			if len(fn) == 2 {
				Go.Name = strings.Trim(fn[0], " \t\n)")
				Go.Args = strings.Trim(fn[1], " \t\n)")
			} else {
				log.Fatal(":go:func filter error in " + text)
			}
		}
	case "import":
		Go.Import = text
	}
}

func (t *Tree) parseTag(tk item) Node {
	var (
		deep = tk.depth
		tag  = t.newTag(tk.pos, tk.val, tk.typ)
	)
Loop:
	for {
		switch token := t.nextNonSpace(); {
		case token.depth > deep:
			if tag.tagType == itemTagVoid || tag.tagType == itemTagVoidInline {
				break Loop
			}
			tag.append(t.hub(token))
		case token.depth == deep:
			switch token.typ {
			case itemClass:
				tag.attr("class", `"`+token.val+`"`, false)
			case itemID:
				tag.attr("id", `"`+token.val+`"`, false)
			case itemAttrStart:
				t.parseAttributes(tag, `"`)
			case itemTagEnd:
				tag.tagType = itemTagVoid
				return tag
			default:
				break Loop
			}
		default:
			break Loop
		}
	}
	t.backup()
	return tag
}

type pAttr interface {
	attr(string, string, bool)
}

func (t *Tree) parseAttributes(tag pAttr, qw string) {
	var (
		aname string
		equal bool
		unesc bool
		stack = make([]string, 0, 4)
	)
	for {
		switch token := t.next(); token.typ {
		case itemAttrSpace:
			// skip
		case itemAttr:
			switch {
			case aname == "":
				aname = token.val
			case aname != "" && !equal:
				tag.attr(aname, qw+aname+qw, unesc)
				aname = token.val
			case aname != "" && equal:
				stack = append(stack, token.val)
			}
		case itemAttrEqual, itemAttrEqualUn:
			if token.typ == itemAttrEqual {
				unesc = false
			} else {
				unesc = true
			}
			equal = true
			switch len_stack := len(stack); {
			case len_stack == 0 && aname != "":
				// skip
			case len_stack > 1 && aname != "":
				tag.attr(aname, strings.Join(stack[:len(stack)-1], " "), unesc)

				aname = stack[len(stack)-1]
				stack = stack[:0]
			case len_stack == 1 && aname == "":
				aname = stack[0]
				stack = stack[:0]
			default:
				t.errorf("unexpected '='")
			}
		case itemAttrComma:
			equal = false
			switch len_stack := len(stack); {
			case len_stack > 0 && aname != "":
				tag.attr(aname, strings.Join(stack, " "), unesc)
				aname = ""
				stack = stack[:0]
			case len_stack == 0 && aname != "":
				tag.attr(aname, qw+aname+qw, unesc)
				aname = ""
			}
		case itemAttrEnd:
			switch len_stack := len(stack); {
			case len_stack > 0 && aname != "":
				tag.attr(aname, strings.Join(stack, " "), unesc)
			case len_stack > 0 && aname == "":
				for _, a := range stack {
					tag.attr(a, a, unesc)
				}
			case len_stack == 0 && aname != "":
				tag.attr(aname, qw+aname+qw, unesc)
			}
			return
		default:
			t.errorf("unexpected %s", token.val)
		}
	}
}

func (t *Tree) parseIf(tk item) Node {
	var (
		deep = tk.depth
		cond = t.newCond(tk.pos, tk.val, tk.typ)
	)
Loop:
	for {
		switch token := t.nextNonSpace(); {
		case token.depth > deep:
			cond.append(t.hub(token))
		case token.depth == deep:
			switch token.typ {
			case itemElse:
				ni := t.peek()
				if ni.typ == itemIf {
					token = t.next()
					cond.append(t.newCode(token.pos, token.val, itemElseIf))
				} else {
					cond.append(t.newCode(token.pos, token.val, token.typ))
				}
			default:
				break Loop
			}
		default:
			break Loop
		}
	}
	t.backup()
	return cond
}

func (t *Tree) parseFor(tk item) Node {
	var (
		deep = tk.depth
		cond = t.newCond(tk.pos, tk.val, tk.typ)
	)
Loop:
	for {
		switch token := t.nextNonSpace(); {
		case token.depth > deep:
			cond.append(t.hub(token))
		case token.depth == deep:
			if token.typ == itemElse {
				cond.condType = itemForIfNotContain
				cond.append(t.newCode(token.pos, token.val, itemForElse))
			} else {
				break Loop
			}
		default:
			break Loop
		}
	}
	t.backup()
	return cond
}

func (t *Tree) parseCase(tk item) Node {
	var (
		deep  = tk.depth
		iCase = t.newCond(tk.pos, tk.val, tk.typ)
	)
	for {
		if token := t.nextNonSpace(); token.depth > deep {
			switch token.typ {
			case itemCaseWhen, itemCaseDefault:
				iCase.append(t.newCode(token.pos, token.val, token.typ))
			default:
				iCase.append(t.hub(token))
			}
		} else {
			break
		}
	}
	t.backup()
	return iCase
}

func (t *Tree) parseMixin(tk item) *MixinNode {
	var (
		deep  = tk.depth
		mixin = t.newMixin(tk.pos)
	)
Loop:
	for {
		switch token := t.nextNonSpace(); {
		case token.depth > deep:
			mixin.append(t.hub(token))
		case token.depth == deep:
			if token.typ == itemAttrStart {
				t.parseAttributes(mixin, "")
			} else {
				break Loop
			}
		default:
			break Loop
		}
	}
	t.backup()
	return mixin
}

func (t *Tree) parseMixinUse(tk item) Node {
	tMix, ok := t.mixin[tk.val]
	if !ok {
		t.errorf(`Mixin "%s" must be declared before use.`, tk.val)
	}
	var (
		deep  = tk.depth
		mixin = tMix.CopyMixin()
	)
Loop:
	for {
		switch token := t.nextNonSpace(); {
		case token.depth > deep:
			mixin.appendToBlock(t.hub(token))
		case token.depth == deep:
			if token.typ == itemAttrStart {
				t.parseAttributes(mixin, "")
			} else {
				break Loop
			}
		default:
			break Loop
		}
	}
	t.backup()

	use := len(mixin.AttrName)
	tpl := len(tMix.AttrName)
	switch {
	case use < tpl:
		i := 0
		diff := tpl - use
		mixin.AttrCode = append(mixin.AttrCode, make([]string, diff)...) // Extend slice
		for index := 0; index < diff; index++ {
			i = tpl - index - 1
			if tMix.AttrName[i] != tMix.AttrCode[i] {
				mixin.AttrCode[i] = tMix.AttrCode[i]
			} else {
				mixin.AttrCode[i] = `""`
			}
		}
		mixin.AttrName = tMix.AttrName
	case use > tpl:
		if tpl <= 0 {
			break
		}
		if strings.HasPrefix(tMix.AttrName[tpl-1], "...") {
			mixin.AttrRest = mixin.AttrCode[tpl-1:]
		}
		mixin.AttrCode = mixin.AttrCode[:tpl]
		mixin.AttrName = tMix.AttrName
	case use == tpl:
		mixin.AttrName = tMix.AttrName
	}
	return mixin
}

func (t *Tree) parseBlock(tk item) *BlockNode {
	block := t.newList(tk.pos)
	for {
		token := t.nextNonSpace()
		if token.depth > tk.depth {
			block.append(t.hub(token))
		} else {
			break
		}
	}
	t.backup()
	var suf string
	switch tk.typ {
	case itemBlockPrepend:
		suf = "_prepend"
	case itemBlockAppend:
		suf = "_append"
	}
	t.block[tk.val+suf] = block
	return t.newBlock(tk.pos, tk.val, tk.typ)
}

func (t *Tree) parseInclude(tk item) *ListNode {
	switch ext := filepath.Ext(tk.val); ext {
	case ".jade", ".pug", "":
		return t.parseSubFile(tk.val)
	case ".js", ".css", ".tpl", ".md":
		ln := t.newList(tk.pos)
		ln.append(t.newText(tk.pos, t.read(tk.val), itemText))
		return ln
	default:
		t.errorf(`file extension is not supported`)
		return nil
	}
}

func (t *Tree) parseSubFile(path string) *ListNode {
	// log.Println("subtemplate: " + path)
	var incTree = New(path)
	incTree.block = t.block
	incTree.mixin = t.mixin

	_, err := incTree.Parse(t.read(path))
	if err != nil {
		d, _ := os.Getwd()
		t.errorf(`in '%s' subtemplate '%s': parseSubFile() error: %s`, d, path, err)
	}

	return incTree.Root
}

var extensions = [...]string{".jade", ".pug", ".js", ".css", ".tpl", ".md"}

func (t *Tree) read(path string) []byte {
	currentTmplDir, _ := filepath.Split(t.Name)
	path = currentTmplDir + path

	var (
		bb  []byte
		err error
	)

	ext := filepath.Ext(path)

	for _, s := range extensions {
		if s == ext {
			bb, err = t.ReadFunc(path)
			break
		}

		if bb, err = t.ReadFunc(path + s); err == nil {
			break
		}
	}

	if err != nil {
		wd, _ := os.Getwd()
		t.errorf(`%s  work dir: %s `, err, wd)
	}

	return bb
}
<commit_msg>final touch<commit_after>package jade

import (
	"log"
	"os"
	"path/filepath"
	"strings"
)

func (t *Tree) topParse() {
	t.Root = t.newList(t.peek().pos)
	var (
		ext   bool
		token = t.nextNonSpace()
	)
	if token.typ == itemExtends {
		ext = true
		t.Root.append(t.parseSubFile(token.val))
		token = t.nextNonSpace()
	}
	for {
		switch token.typ {
		case itemInclude:
			t.Root.append(t.parseInclude(token))
		case itemBlock, itemBlockPrepend, itemBlockAppend:
			if ext {
				t.parseBlock(token)
			} else {
				t.Root.append(t.parseBlock(token))
			}
		case itemMixin:
			t.mixin[token.val] = t.parseMixin(token)
		case itemEOF:
			return
		case itemExtends:
			t.errorf(`Declaration of template inheritance ("extends") should be the first thing in the file. There can only be one extends statement per file.`)
		case itemError:
			t.errorf("%s line: %d\n", token.val, token.line)
		default:
			if ext {
				t.errorf(`Only import, named blocks and mixins can appear at the top level of an extending template`)
			}
			t.Root.append(t.hub(token))
		}
		token = t.nextNonSpace()
	}
}

func (t *Tree) hub(token item) (n Node) {
	for {
		switch token.typ {
		case itemDiv:
			token.val = "div"
			fallthrough
		case itemTag, itemTagInline, itemTagVoid, itemTagVoidInline:
			return t.parseTag(token)
		case itemText, itemComment, itemHTMLTag:
			return t.newText(token.pos, []byte(token.val), token.typ)
		case itemCode, itemCodeBuffered, itemCodeUnescaped, itemMixinBlock:
			return t.newCode(token.pos, token.val, token.typ)
		case itemIf, itemUnless:
			return t.parseIf(token)
		case itemFor, itemEach, itemWhile:
			return t.parseFor(token)
		case itemCase:
			return t.parseCase(token)
		case itemBlock, itemBlockPrepend, itemBlockAppend:
			return t.parseBlock(token)
		case itemMixinCall:
			return t.parseMixinUse(token)
		case itemInclude:
			return t.parseInclude(token)
		case itemDoctype:
			return t.newDoctype(token.pos, token.val)
		case itemFilter:
			return t.parseFilter(token)
		case itemError:
			t.errorf("Error lex: %s line: %d\n", token.val, token.line)
		default:
			t.errorf(`Error hub(): unexpected token  "%s"  type  "%s"`, token.val, token.typ)
		}
	}
}

func (t *Tree) parseFilter(tk item) Node {
	var subf, args, text string
Loop:
	for {
		switch token := t.nextNonSpace(); token.typ {
		case itemFilterSubf:
			subf = token.val
		case itemFilterArgs:
			args = strings.Trim(token.val, " \t\r\n")
		case itemFilterText:
			text = strings.Trim(token.val, " \t\r\n")
		default:
			break Loop
		}
	}
	t.backup()
	switch tk.val {
	case "go":
		filterGo(subf, args, text)
	case "markdown", "markdown-it":
		// TODO: filterMarkdown(subf, args, text)
	}
	return t.newList(tk.pos) // for return nothing
}

func filterGo(subf, args, text string) {
	switch subf {
	case "func":
		Go.Name = ""
		switch args {
		case "name":
			Go.Name = text
		case "arg", "args":
			if Go.Args != "" {
				Go.Args += ", " + strings.Trim(text, "()")
			} else {
				Go.Args = strings.Trim(text, "()")
			}
		default:
			fn := strings.Split(text, "(")
			if len(fn) == 2 {
				Go.Name = strings.Trim(fn[0], " \t\n)")
				Go.Args = strings.Trim(fn[1], " \t\n)")
			} else {
				log.Fatal(":go:func filter error in " + text)
			}
		}
	case "import":
		Go.Import = text
	}
}

func (t *Tree) parseTag(tk item) Node {
	var (
		deep = tk.depth
		tag  = t.newTag(tk.pos, tk.val, tk.typ)
	)
Loop:
	for {
		switch token := t.nextNonSpace(); {
		case token.depth > deep:
			if tag.tagType == itemTagVoid || tag.tagType == itemTagVoidInline {
				break Loop
			}
			tag.append(t.hub(token))
		case token.depth == deep:
			switch token.typ {
			case itemClass:
				tag.attr("class", `"`+token.val+`"`, false)
			case itemID:
				tag.attr("id", `"`+token.val+`"`, false)
			case itemAttrStart:
				t.parseAttributes(tag, `"`)
			case itemTagEnd:
				tag.tagType = itemTagVoid
				return tag
			default:
				break Loop
			}
		default:
			break Loop
		}
	}
	t.backup()
	return tag
}

type pAttr interface {
	attr(string, string, bool)
}

func (t *Tree) parseAttributes(tag pAttr, qw string) {
	var (
		aname string
		equal bool
		unesc bool
		stack = make([]string, 0, 4)
	)
	for {
		switch token := t.next(); token.typ {
		case itemAttrSpace:
			// skip
		case itemAttr:
			switch {
			case aname == "":
				aname = token.val
			case aname != "" && !equal:
				tag.attr(aname, qw+aname+qw, unesc)
				aname = token.val
			case aname != "" && equal:
				stack = append(stack, token.val)
			}
		case itemAttrEqual, itemAttrEqualUn:
			if token.typ == itemAttrEqual {
				unesc = false
			} else {
				unesc = true
			}
			equal = true
			switch len_stack := len(stack); {
			case len_stack == 0 && aname != "":
				// skip
			case len_stack > 1 && aname != "":
				tag.attr(aname, strings.Join(stack[:len(stack)-1], " "), unesc)

				aname = stack[len(stack)-1]
				stack = stack[:0]
			case len_stack == 1 && aname == "":
				aname = stack[0]
				stack = stack[:0]
			default:
				t.errorf("unexpected '='")
			}
		case itemAttrComma:
			equal = false
			switch len_stack := len(stack); {
			case len_stack > 0 && aname != "":
				tag.attr(aname, strings.Join(stack, " "), unesc)
				aname = ""
				stack = stack[:0]
			case len_stack == 0 && aname != "":
				tag.attr(aname, qw+aname+qw, unesc)
				aname = ""
			}
		case itemAttrEnd:
			switch len_stack := len(stack); {
			case len_stack > 0 && aname != "":
				tag.attr(aname, strings.Join(stack, " "), unesc)
			case len_stack > 0 && aname == "":
				for _, a := range stack {
					tag.attr(a, a, unesc)
				}
			case len_stack == 0 && aname != "":
				tag.attr(aname, qw+aname+qw, unesc)
			}
			return
		default:
			t.errorf("unexpected %s", token.val)
		}
	}
}

func (t *Tree) parseIf(tk item) Node {
	var (
		deep = tk.depth
		cond = t.newCond(tk.pos, tk.val, tk.typ)
	)
Loop:
	for {
		switch token := t.nextNonSpace(); {
		case token.depth > deep:
			cond.append(t.hub(token))
		case token.depth == deep:
			switch token.typ {
			case itemElse:
				ni := t.peek()
				if ni.typ == itemIf {
					token = t.next()
					cond.append(t.newCode(token.pos, token.val, itemElseIf))
				} else {
					cond.append(t.newCode(token.pos, token.val, token.typ))
				}
			default:
				break Loop
			}
		default:
			break Loop
		}
	}
	t.backup()
	return cond
}

func (t *Tree) parseFor(tk item) Node {
	var (
		deep = tk.depth
		cond = t.newCond(tk.pos, tk.val, tk.typ)
	)
Loop:
	for {
		switch token := t.nextNonSpace(); {
		case token.depth > deep:
			cond.append(t.hub(token))
		case token.depth == deep:
			if token.typ == itemElse {
				cond.condType = itemForIfNotContain
				cond.append(t.newCode(token.pos, token.val, itemForElse))
			} else {
				break Loop
			}
		default:
			break Loop
		}
	}
	t.backup()
	return cond
}

func (t *Tree) parseCase(tk item) Node {
	var (
		deep  = tk.depth
		iCase = t.newCond(tk.pos, tk.val, tk.typ)
	)
	for {
		if token := t.nextNonSpace(); token.depth > deep {
			switch token.typ {
			case itemCaseWhen, itemCaseDefault:
				iCase.append(t.newCode(token.pos, token.val, token.typ))
			default:
				iCase.append(t.hub(token))
			}
		} else {
			break
		}
	}
	t.backup()
	return iCase
}

func (t *Tree) parseMixin(tk item) *MixinNode {
	var (
		deep  = tk.depth
		mixin = t.newMixin(tk.pos)
	)
Loop:
	for {
		switch token := t.nextNonSpace(); {
		case token.depth > deep:
			mixin.append(t.hub(token))
		case token.depth == deep:
			if token.typ == itemAttrStart {
				t.parseAttributes(mixin, "")
			} else {
				break Loop
			}
		default:
			break Loop
		}
	}
	t.backup()
	return mixin
}

func (t *Tree) parseMixinUse(tk item) Node {
	tMix, ok := t.mixin[tk.val]
	if !ok {
		t.errorf(`Mixin "%s" must be declared before use.`, tk.val)
	}
	var (
		deep  = tk.depth
		mixin = tMix.CopyMixin()
	)
Loop:
	for {
		switch token := t.nextNonSpace(); {
		case token.depth > deep:
			mixin.appendToBlock(t.hub(token))
		case token.depth == deep:
			if token.typ == itemAttrStart {
				t.parseAttributes(mixin, "")
			} else {
				break Loop
			}
		default:
			break Loop
		}
	}
	t.backup()

	use := len(mixin.AttrName)
	tpl := len(tMix.AttrName)
	switch {
	case use < tpl:
		i := 0
		diff := tpl - use
		mixin.AttrCode = append(mixin.AttrCode, make([]string, diff)...) // Extend slice
		for index := 0; index < diff; index++ {
			i = tpl - index - 1
			if tMix.AttrName[i] != tMix.AttrCode[i] {
				mixin.AttrCode[i] = tMix.AttrCode[i]
			} else {
				mixin.AttrCode[i] = `""`
			}
		}
		mixin.AttrName = tMix.AttrName
	case use > tpl:
		if tpl <= 0 {
			break
		}
		if strings.HasPrefix(tMix.AttrName[tpl-1], "...") {
			mixin.AttrRest = mixin.AttrCode[tpl-1:]
		}
		mixin.AttrCode = mixin.AttrCode[:tpl]
		mixin.AttrName = tMix.AttrName
	case use == tpl:
		mixin.AttrName = tMix.AttrName
	}
	return mixin
}

func (t *Tree) parseBlock(tk item) *BlockNode {
	block := t.newList(tk.pos)
	for {
		token := t.nextNonSpace()
		if token.depth > tk.depth {
			block.append(t.hub(token))
		} else {
			break
		}
	}
	t.backup()
	var suf string
	switch tk.typ {
	case itemBlockPrepend:
		suf = "_prepend"
	case itemBlockAppend:
		suf = "_append"
	}
	t.block[tk.val+suf] = block
	return t.newBlock(tk.pos, tk.val, tk.typ)
}

func (t *Tree) parseInclude(tk item) *ListNode {
	switch ext := filepath.Ext(tk.val); ext {
	case ".jade", ".pug", "":
		return t.parseSubFile(tk.val)
	case ".js", ".css", ".tpl", ".md":
		ln := t.newList(tk.pos)
		ln.append(t.newText(tk.pos, t.read(tk.val), itemText))
		return ln
	default:
		t.errorf(`file extension is not supported`)
		return nil
	}
}

func (t *Tree) parseSubFile(path string) *ListNode {
	// log.Println("subtemplate: " + path)
	currentTmplDir, _ := filepath.Split(t.Name)
	var incTree = New(currentTmplDir + path)
	incTree.block = t.block
	incTree.mixin = t.mixin

	_, err := incTree.Parse(t.read(path))
	if err != nil {
		d, _ := os.Getwd()
		t.errorf(`in '%s' subtemplate '%s': parseSubFile() error: %s`, d, path, err)
	}

	return incTree.Root
}

var extensions = [...]string{".jade", ".pug", ".js", ".css", ".tpl", ".md"}

func (t *Tree) read(path string) []byte {
	currentTmplDir, _ := filepath.Split(t.Name)
	path = currentTmplDir + path

	var (
		bb  []byte
		err error
	)

	ext := filepath.Ext(path)

	for _, s := range extensions {
		if s == ext {
			bb, err = t.ReadFunc(path)
			break
		}

		if bb, err = t.ReadFunc(path + s); err == nil {
			break
		}
	}

	if err != nil {
		wd, _ := os.Getwd()
		t.errorf(`%s  work dir: %s `, err, wd)
	}

	return bb
}
<|endoftext|>


<commit_before>package job

import (
	"fmt"
	"log"
	"github.com/PeopleAdmin/pdfcombiner/testmode"
	"io/ioutil"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
)

var (
	awsRegion              = aws.USEast
	uploadedFilePermission = s3.Private
)

// Get retrieves the requested document, either from S3 or by decoding the
// embedded `Data` attribute of the Document.
func (doc *Document) Get() (docContent []byte, err error) {
	log.Printf("%s GET s3://%s/%s\n", doc.Id(), doc.parent.BucketName, doc.s3Path())
	if testmode.IsEnabled() {
		return
	}
	switch doc.Data {
	case "":
		return doc.parent.download(doc)
	default:
		docContent, err = decodeEmbeddedData(doc.Data)
		doc.Data = ""
		doc.FileSize = len(docContent)
		return
	}
}

// UploadCombinedFile sends a file to the job's CombinedKey on S3.
func (j *Job) UploadCombinedFile() (err error) {
	log.Printf("%s PUT s3://%s/%s\n", j.Id(), j.BucketName, j.CombinedKey)
	if testmode.IsEnabled() {
		return
	}
	content, err := ioutil.ReadFile(j.LocalPath())
	if err != nil || len(content) == 0 {
		j.AddError(fmt.Errorf("Reading file '%v' for upload: %v", j.LocalPath(), err))
		return
	}
	err = j.bucket.Put(j.CombinedKey, content, "application/pdf", uploadedFilePermission)
	if err != nil {
		j.AddError(fmt.Errorf("Uploading to S3 key '%s': %v", j.CombinedKey, err))
		return
	}
	j.uploadComplete = true
	return
}

// Connect to AWS and add the handle to the Job object.
func (j *Job) connect() (err error) {
	auth, err := aws.EnvAuth()
	if err != nil {
		return
	}
	s := s3.New(auth, awsRegion)
	j.bucket = s.Bucket(j.BucketName)
	return
}

func (j *Job) download(doc *Document) (content []byte, err error) {
	remotePath := doc.s3Path()
	content, err = j.bucket.Get(remotePath)
	doc.FileSize = len(content)
	return
}
<commit_msg>Abort everything if s3 credentials wrong<commit_after>package job

import (
	"fmt"
	"github.com/PeopleAdmin/pdfcombiner/testmode"
	"io/ioutil"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
	"log"
	"os"
)

var (
	awsRegion              = aws.USEast
	uploadedFilePermission = s3.Private
)

// Get retrieves the requested document, either from S3 or by decoding the
// embedded `Data` attribute of the Document.
func (doc *Document) Get() (docContent []byte, err error) {
	log.Printf("%s GET s3://%s/%s\n", doc.Id(), doc.parent.BucketName, doc.s3Path())
	if testmode.IsEnabled() {
		return
	}
	switch doc.Data {
	case "":
		return doc.parent.download(doc)
	default:
		docContent, err = decodeEmbeddedData(doc.Data)
		doc.Data = ""
		doc.FileSize = len(docContent)
		return
	}
}

// UploadCombinedFile sends a file to the job's CombinedKey on S3.
func (j *Job) UploadCombinedFile() (err error) {
	log.Printf("%s PUT s3://%s/%s\n", j.Id(), j.BucketName, j.CombinedKey)
	if testmode.IsEnabled() {
		return
	}
	content, err := ioutil.ReadFile(j.LocalPath())
	if err != nil || len(content) == 0 {
		j.AddError(fmt.Errorf("Reading file '%v' for upload: %v", j.LocalPath(), err))
		return
	}
	err = j.bucket.Put(j.CombinedKey, content, "application/pdf", uploadedFilePermission)
	if err != nil {
		j.AddError(fmt.Errorf("Uploading to S3 key '%s': %v", j.CombinedKey, err))
		return
	}
	j.uploadComplete = true
	return
}

// Connect to AWS and add the handle to the Job object.
func (j *Job) connect() (err error) {
	auth, err := aws.EnvAuth()
	if err != nil {
		println("S3 connect failed due to auth issues, exiting!")
		os.Exit(1)
	}
	s := s3.New(auth, awsRegion)
	j.bucket = s.Bucket(j.BucketName)
	return
}

func (j *Job) download(doc *Document) (content []byte, err error) {
	remotePath := doc.s3Path()
	content, err = j.bucket.Get(remotePath)
	doc.FileSize = len(content)
	return
}
<|endoftext|>