<commit_before>package data

import (
	"database/sql"

	"github.com/UHERO/rest-api/models"
)

type GeographyRepository struct {
	DB *sql.DB
}

func (r *GeographyRepository) GetAllGeographies() (geographies []models.DataPortalGeography, err error) {
	rows, err := r.DB.Query(`SELECT fips, display_name, handle FROM geographies;`)
	if err != nil {
		return
	}
	for rows.Next() {
		geography := models.Geography{}
		err = rows.Scan(
			&geography.FIPS,
			&geography.Name,
			&geography.Handle,
		)
		if err != nil {
			continue
		}
		dataPortalGeography := models.DataPortalGeography{Handle: geography.Handle}
		if geography.FIPS.Valid {
			dataPortalGeography.FIPS = geography.FIPS.String
		}
		if geography.Name.Valid {
			dataPortalGeography.Name = geography.Name.String
		}
		geographies = append(geographies, dataPortalGeography)
	}
	return
}

func (r *GeographyRepository) GetGeographiesByCategory(categoryId int64) (geographies []models.DataPortalGeography, err error) {
	rows, err := r.DB.Query(
		`SELECT DISTINCT geographies.fips, geographies.display_name_short, geographies.handle
		FROM categories
		LEFT JOIN data_list_measurements ON data_list_measurements.data_list_id = categories.data_list_id
		LEFT JOIN measurement_series ON measurement_series.measurement_id = data_list_measurements.measurement_id
		LEFT JOIN series ON series.id = measurement_series.series_id
		LEFT JOIN geographies ON geographies.id = series.geography_id
		LEFT JOIN feature_toggles ON feature_toggles.universe = series.universe AND feature_toggles.name = 'filter_by_quarantine'
		WHERE (categories.id = ? OR categories.ancestry REGEXP CONCAT('[[:<:]]', ?, '[[:>:]]'))
		AND NOT categories.hidden
		AND NOT series.restricted
		AND (feature_toggles.status IS NULL OR NOT feature_toggles.status OR NOT series.quarantined);`,
		categoryId,
		categoryId,
	)
	if err != nil {
		return
	}
	for rows.Next() {
		geography := models.Geography{}
		err = rows.Scan(
			&geography.FIPS,
			&geography.Name,
			&geography.Handle,
		)
		if err != nil {
			continue
		}
		dataPortalGeography := models.DataPortalGeography{Handle: geography.Handle}
		if geography.FIPS.Valid {
			dataPortalGeography.FIPS = geography.FIPS.String
		}
		if geography.Name.Valid {
			dataPortalGeography.Name = geography.Name.String
		}
		geographies = append(geographies, dataPortalGeography)
	}
	return
}

func (r *GeographyRepository) GetSeriesSiblingsGeoById(seriesId int64) (geographies []models.DataPortalGeography, err error) {
	rows, err := r.DB.Query(
		`SELECT DISTINCT geographies.fips, geographies.display_name_short, geographies.handle
		FROM series
		JOIN (SELECT name, universe FROM series where id = ?) AS original_series
		LEFT JOIN geographies ON geographies.id = series.geography_id
		LEFT JOIN feature_toggles ON feature_toggles.universe = series.universe AND feature_toggles.name = 'filter_by_quarantine'
		WHERE series.universe = original_series.universe
		AND substring_index(series.name, '@', 1) = substring_index(original_series.name, '@', 1)
		AND NOT series.restricted
		AND (feature_toggles.status IS NULL OR NOT feature_toggles.status OR NOT series.quarantined);`, seriesId)
	if err != nil {
		return
	}
	for rows.Next() {
		geography := models.Geography{}
		err = rows.Scan(
			&geography.FIPS,
			&geography.Name,
			&geography.Handle,
		)
		if err != nil {
			continue
		}
		dataPortalGeography := models.DataPortalGeography{Handle: geography.Handle}
		if geography.FIPS.Valid {
			dataPortalGeography.FIPS = geography.FIPS.String
		}
		if geography.Name.Valid {
			dataPortalGeography.Name = geography.Name.String
		}
		geographies = append(geographies, dataPortalGeography)
	}
	return
}
<commit_msg>Add ShortName to Geo endpoints<commit_after>package data

import (
	"database/sql"

	"github.com/UHERO/rest-api/models"
)

type GeographyRepository struct {
	DB *sql.DB
}

func (r *GeographyRepository) GetAllGeographies() (geographies []models.DataPortalGeography, err error) {
	rows, err := r.DB.Query(`SELECT fips, display_name, display_name_short, handle FROM geographies;`)
	if err != nil {
		return
	}
	for rows.Next() {
		geography := models.Geography{}
		err = rows.Scan(
			&geography.FIPS,
			&geography.Name,
			&geography.ShortName,
			&geography.Handle,
		)
		if err != nil {
			continue
		}
		dataPortalGeography := models.DataPortalGeography{Handle: geography.Handle}
		if geography.FIPS.Valid {
			dataPortalGeography.FIPS = geography.FIPS.String
		}
		if geography.Name.Valid {
			dataPortalGeography.Name = geography.Name.String
		}
		if geography.ShortName.Valid {
			dataPortalGeography.ShortName = geography.ShortName.String
		}
		geographies = append(geographies, dataPortalGeography)
	}
	return
}

func (r *GeographyRepository) GetGeographiesByCategory(categoryId int64) (geographies []models.DataPortalGeography, err error) {
	rows, err := r.DB.Query(
		`SELECT DISTINCT geographies.fips, geographies.display_name, geographies.display_name_short, geographies.handle
		FROM categories
		LEFT JOIN data_list_measurements ON data_list_measurements.data_list_id = categories.data_list_id
		LEFT JOIN measurement_series ON measurement_series.measurement_id = data_list_measurements.measurement_id
		LEFT JOIN series ON series.id = measurement_series.series_id
		LEFT JOIN geographies ON geographies.id = series.geography_id
		LEFT JOIN feature_toggles ON feature_toggles.universe = series.universe AND feature_toggles.name = 'filter_by_quarantine'
		WHERE (categories.id = ? OR categories.ancestry REGEXP CONCAT('[[:<:]]', ?, '[[:>:]]'))
		AND NOT categories.hidden
		AND NOT series.restricted
		AND (feature_toggles.status IS NULL OR NOT feature_toggles.status OR NOT series.quarantined);`,
		categoryId,
		categoryId,
	)
	if err != nil {
		return
	}
	for rows.Next() {
		geography := models.Geography{}
		err = rows.Scan(
			&geography.FIPS,
			&geography.Name,
			&geography.ShortName,
			&geography.Handle,
		)
		if err != nil {
			continue
		}
		dataPortalGeography := models.DataPortalGeography{Handle: geography.Handle}
		if geography.FIPS.Valid {
			dataPortalGeography.FIPS = geography.FIPS.String
		}
		if geography.Name.Valid {
			dataPortalGeography.Name = geography.Name.String
		}
		if geography.ShortName.Valid {
			dataPortalGeography.ShortName = geography.ShortName.String
		}
		geographies = append(geographies, dataPortalGeography)
	}
	return
}

func (r *GeographyRepository) GetSeriesSiblingsGeoById(seriesId int64) (geographies []models.DataPortalGeography, err error) {
	rows, err := r.DB.Query(
		`SELECT DISTINCT geographies.fips, geographies.display_name, geographies.display_name_short, geographies.handle
		FROM series
		JOIN (SELECT name, universe FROM series where id = ?) AS original_series
		LEFT JOIN geographies ON geographies.id = series.geography_id
		LEFT JOIN feature_toggles ON feature_toggles.universe = series.universe AND feature_toggles.name = 'filter_by_quarantine'
		WHERE series.universe = original_series.universe
		AND substring_index(series.name, '@', 1) = substring_index(original_series.name, '@', 1)
		AND NOT series.restricted
		AND (feature_toggles.status IS NULL OR NOT feature_toggles.status OR NOT series.quarantined);`, seriesId)
	if err != nil {
		return
	}
	for rows.Next() {
		geography := models.Geography{}
		err = rows.Scan(
			&geography.FIPS,
			&geography.Name,
			&geography.ShortName,
			&geography.Handle,
		)
		if err != nil {
			continue
		}
		dataPortalGeography := models.DataPortalGeography{Handle: geography.Handle}
		if geography.FIPS.Valid {
			dataPortalGeography.FIPS = geography.FIPS.String
		}
		if geography.Name.Valid {
			dataPortalGeography.Name = geography.Name.String
		}
		if geography.ShortName.Valid {
			dataPortalGeography.ShortName = geography.ShortName.String
		}
		geographies = append(geographies, dataPortalGeography)
	}
	return
}
<|endoftext|>


<commit_before>package ec2

import (
	"io"
	"launchpad.net/goamz/ec2"
	"launchpad.net/goamz/s3"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/trivial"
	"net/http"
)

type BootstrapState struct {
	StateInstances []state.InstanceId
}

func LoadState(e environs.Environ) (*BootstrapState, error) {
	s, err := e.(*environ).loadState()
	if err != nil {
		return nil, err
	}
	return &BootstrapState{s.StateInstances}, nil
}

func JujuGroupName(e environs.Environ) string {
	return e.(*environ).jujuGroupName()
}

func MachineGroupName(e environs.Environ, machineId string) string {
	return e.(*environ).machineGroupName(machineId)
}

func EnvironEC2(e environs.Environ) *ec2.EC2 {
	return e.(*environ).ec2()
}

func EnvironS3(e environs.Environ) *s3.S3 {
	return e.(*environ).s3()
}

func DeleteStorageContent(s environs.Storage) error {
	return s.(*storage).deleteAll()
}

func InstanceEC2(inst environs.Instance) *ec2.Instance {
	return inst.(*instance).Instance
}

// BucketStorage returns a storage instance addressing
// an arbitrary s3 bucket.
func BucketStorage(b *s3.Bucket) environs.Storage {
	return &storage{
		bucket: b,
	}
}

var origImagesHost = imagesHost

func init() {
	// Make the images data accessible through the "file" protocol.
	http.DefaultTransport.(*http.Transport).RegisterProtocol("file", http.NewFileTransport(http.Dir("testdata")))
}

func UseTestImageData(local bool) {
	if local {
		imagesHost = "file:"
	} else {
		imagesHost = origImagesHost
	}
}

var origMetadataHost = metadataHost

func UseTestMetadata(local bool) {
	if local {
		metadataHost = "file:"
	} else {
		metadataHost = origMetadataHost
	}
}

var originalShortAttempt = shortAttempt
var originalLongAttempt = longAttempt

// ShortTimeouts sets the timeouts to a short period as we
// know that the ec2test server doesn't get better with time,
// and this reduces the test time from 30s to 3s.
func ShortTimeouts(short bool) {
	if short {
		shortAttempt = trivial.AttemptStrategy{
			Total: 0.25e9,
			Delay: 0.01e9,
		}
		longAttempt = shortAttempt
	} else {
		shortAttempt = originalShortAttempt
		longAttempt = originalLongAttempt
	}
}

var ShortAttempt = &shortAttempt

func EC2ErrCode(err error) string {
	return ec2ErrCode(err)
}

// FabricateInstance creates a new fictitious instance
// given an existing instance and a new id.
func FabricateInstance(inst environs.Instance, newId string) environs.Instance {
	oldi := inst.(*instance)
	newi := &instance{oldi.e, &ec2.Instance{}}
	*newi.Instance = *oldi.Instance
	newi.InstanceId = newId
	return newi
}

// Access non exported methods on ec2.storage
type Storage interface {
	Put(file string, r io.Reader, length int64) error
	ResetMadeBucket()
}

func (s *storage) ResetMadeBucket() {
	s.bucketMutex.Lock()
	defer s.bucketMutex.Unlock()
	s.madeBucket = false
}
<commit_msg>Fix test failures due to rename of bucketMutex in environs/ec2<commit_after>package ec2

import (
	"io"
	"launchpad.net/goamz/ec2"
	"launchpad.net/goamz/s3"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/trivial"
	"net/http"
)

type BootstrapState struct {
	StateInstances []state.InstanceId
}

func LoadState(e environs.Environ) (*BootstrapState, error) {
	s, err := e.(*environ).loadState()
	if err != nil {
		return nil, err
	}
	return &BootstrapState{s.StateInstances}, nil
}

func JujuGroupName(e environs.Environ) string {
	return e.(*environ).jujuGroupName()
}

func MachineGroupName(e environs.Environ, machineId string) string {
	return e.(*environ).machineGroupName(machineId)
}

func EnvironEC2(e environs.Environ) *ec2.EC2 {
	return e.(*environ).ec2()
}

func EnvironS3(e environs.Environ) *s3.S3 {
	return e.(*environ).s3()
}

func DeleteStorageContent(s environs.Storage) error {
	return s.(*storage).deleteAll()
}

func InstanceEC2(inst environs.Instance) *ec2.Instance {
	return inst.(*instance).Instance
}

// BucketStorage returns a storage instance addressing
// an arbitrary s3 bucket.
func BucketStorage(b *s3.Bucket) environs.Storage {
	return &storage{
		bucket: b,
	}
}

var origImagesHost = imagesHost

func init() {
	// Make the images data accessible through the "file" protocol.
	http.DefaultTransport.(*http.Transport).RegisterProtocol("file", http.NewFileTransport(http.Dir("testdata")))
}

func UseTestImageData(local bool) {
	if local {
		imagesHost = "file:"
	} else {
		imagesHost = origImagesHost
	}
}

var origMetadataHost = metadataHost

func UseTestMetadata(local bool) {
	if local {
		metadataHost = "file:"
	} else {
		metadataHost = origMetadataHost
	}
}

var originalShortAttempt = shortAttempt
var originalLongAttempt = longAttempt

// ShortTimeouts sets the timeouts to a short period as we
// know that the ec2test server doesn't get better with time,
// and this reduces the test time from 30s to 3s.
func ShortTimeouts(short bool) {
	if short {
		shortAttempt = trivial.AttemptStrategy{
			Total: 0.25e9,
			Delay: 0.01e9,
		}
		longAttempt = shortAttempt
	} else {
		shortAttempt = originalShortAttempt
		longAttempt = originalLongAttempt
	}
}

var ShortAttempt = &shortAttempt

func EC2ErrCode(err error) string {
	return ec2ErrCode(err)
}

// FabricateInstance creates a new fictitious instance
// given an existing instance and a new id.
func FabricateInstance(inst environs.Instance, newId string) environs.Instance {
	oldi := inst.(*instance)
	newi := &instance{oldi.e, &ec2.Instance{}}
	*newi.Instance = *oldi.Instance
	newi.InstanceId = newId
	return newi
}

// Access non exported methods on ec2.storage
type Storage interface {
	Put(file string, r io.Reader, length int64) error
	ResetMadeBucket()
}

func (s *storage) ResetMadeBucket() {
	s.Lock()
	defer s.Unlock()
	s.madeBucket = false
}
<|endoftext|>


<commit_before>// Package xattr provides a simple interface to user extended attributes on Linux and OSX.
// Support for xattrs is filesystem dependant, so not a given even if you are running one of those operating systems.
//
// On Linux you have to edit /etc/fstab to include "user_xattr". Also, Linux extended attributes have a manditory
// prefix of "user.". This is prepended transparently for Get/Set/Remove and hidden in List.
package xattr

// XAttrError records an error and the operation, file path and attribute that caused it.
type XAttrError struct {
	Op   string
	Path string
	Attr string
	Err  error
}

func (e *XAttrError) Error() string {
	return e.Op + " " + e.Path + " " + e.Attr + ": " + e.Err.Error()
}

// Returns whether the error is known to report that a extended attribute does not exist.
func IsNotExist(err error) bool {
	e, ok := err.(*XAttrError)
	if ok {
		return isNotExist(e)
	}
	return false
}

// Converts an array of NUL terminated UTF-8 strings
// to a []string.
func nullTermToStrings(buf []byte) (result []string) {
	offset := 0
	for index, b := range buf {
		if b == 0 {
			result = append(result, string(buf[offset:index]))
			offset = index + 1
		}
	}
	return
}

// Retrieves extended attribute data associated with path.
func Get(path, attr string) ([]byte, error) {
	attr = prefix + attr

	// find size
	size, err := get(path, attr, nil)
	if err != nil {
		return nil, &XAttrError{"getxattr", path, attr, err}
	}
	if size == 0 {
		return []byte{}, nil
	}

	// read into buffer of that size
	buf := make([]byte, size)
	size, err = get(path, attr, buf)
	if err != nil {
		return nil, &XAttrError{"getxattr", path, attr, err}
	}
	return buf[:size], nil
}

// GetTo retrivies extended attribute data associated with path into dest. It
// returns number of bytes retrived or non nil error.
//
// If attribute size is unknown caller should call GetTo with empty buffer or
// guess it. If buffer is too short for value, GetTo returns error.
//
// GetTo is similar to Get but more efficient, because it issues one
// getxattr(2) syscall per call, doesn't allocate memory for attribute data and
// allows reuse buffer.
func GetTo(path, attr string, dest []byte) (n int, err error) {
	attr = prefix + attr
	return get(path, attr, dest)
}

// Retrieves a list of names of extended attributes associated with path.
func List(path string) ([]string, error) {
	// find size
	size, err := list(path, nil)
	if err != nil {
		return nil, &XAttrError{"listxattr", path, "", err}
	}
	if size == 0 {
		return []string{}, nil
	}

	// read into buffer of that size
	buf := make([]byte, size)
	size, err = list(path, buf)
	if err != nil {
		return nil, &XAttrError{"listxattr", path, "", err}
	}
	return stripPrefix(nullTermToStrings(buf[:size])), nil
}

// Associates data as an extended attribute of path.
func Set(path, attr string, data []byte) error {
	attr = prefix + attr

	if err := set(path, attr, data, 0); err != nil {
		return &XAttrError{"setxattr", path, attr, err}
	}
	return nil
}

// Removes the extended attribute.
func Remove(path, attr string) error {
	attr = prefix + attr
	if err := remove(path, attr); err != nil {
		return &XAttrError{"removexattr", path, attr, err}
	}
	return nil
}
<commit_msg>expose syscalls wrappers to public interface<commit_after>// Package xattr provides a simple interface to user extended attributes on Linux and OSX.
// Support for xattrs is filesystem dependant, so not a given even if you are running one of those operating systems.
//
// On Linux you have to edit /etc/fstab to include "user_xattr". Also, Linux extended attributes have a manditory
// prefix of "user.". This is prepended transparently for Get/Set/Remove and hidden in List.
package xattr

// XAttrError records an error and the operation, file path and attribute that caused it.
type XAttrError struct {
	Op   string
	Path string
	Attr string
	Err  error
}

func (e *XAttrError) Error() string {
	return e.Op + " " + e.Path + " " + e.Attr + ": " + e.Err.Error()
}

// Returns whether the error is known to report that a extended attribute does not exist.
func IsNotExist(err error) bool {
	e, ok := err.(*XAttrError)
	if ok {
		return isNotExist(e)
	}
	return false
}

// Converts an array of NUL terminated UTF-8 strings
// to a []string.
func nullTermToStrings(buf []byte) (result []string) {
	offset := 0
	for index, b := range buf {
		if b == 0 {
			result = append(result, string(buf[offset:index]))
			offset = index + 1
		}
	}
	return
}

// Getxattr retrieves value of the extended attribute identified by attr
// associated with given path in filesystem into buffer dest.
//
// On success, dest contains data associated with attr, retrieved value size sz
// and nil error returned.
//
// On error, non-nil error returned. Getxattr returns error if dest was too
// small for attribute value.
//
// A nil slice can be passed as dest to get current size of attribute value,
// which can be used to estimate dest length for value associated with attr.
//
// Get is high-level function on top of Getxattr. Getxattr more efficient,
// because it issues one syscall per call, doesn't allocate memory for
// attribute data (caller can reuse buffer).
func Getxattr(path, attr string, dest []byte) (sz int, err error) {
	return get(path, attr, dest)
}

// Retrieves extended attribute data associated with path.
func Get(path, attr string) ([]byte, error) {
	attr = prefix + attr

	// find size
	size, err := Getxattr(path, attr, nil)
	if err != nil {
		return nil, &XAttrError{"getxattr", path, attr, err}
	}
	if size == 0 {
		return []byte{}, nil
	}

	// read into buffer of that size
	buf := make([]byte, size)
	size, err = Getxattr(path, attr, buf)
	if err != nil {
		return nil, &XAttrError{"getxattr", path, attr, err}
	}
	return buf[:size], nil
}

func Listxattr(path string, dest []byte) (sz int, err error) {
	return list(path, dest)
}

// Retrieves a list of names of extended attributes associated with path.
func List(path string) ([]string, error) {
	// find size
	size, err := Listxattr(path, nil)
	if err != nil {
		return nil, &XAttrError{"listxattr", path, "", err}
	}
	if size == 0 {
		return []string{}, nil
	}

	// read into buffer of that size
	buf := make([]byte, size)
	size, err = Listxattr(path, buf)
	if err != nil {
		return nil, &XAttrError{"listxattr", path, "", err}
	}
	return stripPrefix(nullTermToStrings(buf[:size])), nil
}

func Setxattr(path, attr string, data []byte, flags int) error {
	return set(path, attr, data, flags)
}

// Associates data as an extended attribute of path.
func Set(path, attr string, data []byte) error {
	attr = prefix + attr

	if err := Setxattr(path, attr, data, 0); err != nil {
		return &XAttrError{"setxattr", path, attr, err}
	}
	return nil
}

func Removexattr(path, attr string) error {
	return remove(path, attr)
}

// Removes the extended attribute.
func Remove(path, attr string) error {
	attr = prefix + attr
	if err := Removexattr(path, attr); err != nil {
		return &XAttrError{"removexattr", path, attr, err}
	}
	return nil
}
<|endoftext|>


<commit_before>// This is Free Software covered by the terms of the MIT license.
// See LICENSE file for details.
// Copyright 2017 by Intevation GmbH
package main

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
)

type xhtml struct {
	out bytes.Buffer
}

func (x *xhtml) tag(name string) func(*node) error {
	tag := "<" + name + "/>"
	return func(*node) error {
		x.out.WriteString(tag)
		return nil
	}
}

func (x *xhtml) open(name string) func(*node) error {
	tag := "<" + name + ">"
	return func(*node) error {
		x.out.WriteString(tag)
		return nil
	}
}

func (x *xhtml) close(name string) func(*node) error {
	tag := "</" + name + ">"
	return func(*node) error {
		x.out.WriteString(tag)
		return nil
	}
}

func (x *xhtml) element(name string) *visitor {
	return &visitor{x.open(name), x.close(name)}
}

func (x *xhtml) heading(level int) *visitor {
	tag := fmt.Sprintf("h%d", level)
	return &visitor{x.open(tag), x.close(tag)}
}

func (x *xhtml) text(n *node) error {
	enc := xml.NewEncoder(&x.out)
	txt := n.value.(string)
	enc.EncodeToken(xml.CharData(txt))
	enc.Flush()
	return nil
}

func (x *xhtml) noWikiInline(n *node) error {
	x.out.WriteString("<tt>")
	x.text(n)
	x.out.WriteString("</tt>")
	return nil
}

func (x *xhtml) noWiki(n *node) error {
	x.out.WriteString("
")
	x.noWikiInline(n)
	x.out.WriteString("
\n")
	return nil
}

func (x *xhtml) link(n *node) error {
	href := n.value.(string)
	enc := xml.NewEncoder(&x.out)
	enc.EncodeToken(xml.StartElement{
		Name: xml.Name{Local: "a"},
		Attr: []xml.Attr{{xml.Name{Local: "href"}, href}},
	})
	enc.Flush()
	return nil
}

func exportXHTML(doc *document, out io.Writer) error {
	// TODO: Implement me!

	var x xhtml

	x.out.WriteString("\n\n")

	err := doc.traverse(map[nodeType]*visitor{
		orderedListNode:     x.element("ol"),
		unorderedListNode:   x.element("ul"),
		listItemNode:        x.element("li"),
		textNode:            &visitor{enter: x.text},
		boldNode:            x.element("strong"),
		italicsNode:         x.element("i"),
		underlinedNode:      x.element("em"),
		strikeNode:          x.element("del"),
		superscriptNode:     x.element("sup"),
		subscriptNode:       x.element("sub"),
		tableNode:           x.element("table"),
		tableRowNode:        x.element("tr"),
		tableCellNode:       x.element("td"),
		tableHeaderRowNode:  x.element("th"),
		tableHeaderCellNode: x.element("td"),
		heading1Node:        x.heading(1),
		heading2Node:        x.heading(2),
		heading3Node:        x.heading(3),
		heading4Node:        x.heading(4),
		heading5Node:        x.heading(5),
		heading6Node:        x.heading(6),
		paragraphNode:       x.element("p"),
		lineBreakNode:       &visitor{enter: x.tag("br")},
		escapeNode:          &visitor{enter: x.text},
		noWikiNode:          &visitor{enter: x.noWiki},
		noWikiInlineNode:    &visitor{enter: x.noWikiInline},
		// placeholderNode not supported, yet.
		// TODO: Implement image node.
		linkNode:           &visitor{enter: x.link, leave: x.close("a")},
		horizontalLineNode: &visitor{enter: x.tag("hr")},
	})
	if err != nil {
		x.out.WriteString("\n\n\n")
		_, err = x.out.WriteTo(out)
	}
	return err
}
<commit_msg>Write XML header to XHTML output.<commit_after>// This is Free Software covered by the terms of the MIT license.
// See LICENSE file for details.
// Copyright 2017 by Intevation GmbH
package main

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
)

type xhtml struct {
	out bytes.Buffer
}

func (x *xhtml) tag(name string) func(*node) error {
	tag := "<" + name + "/>"
	return func(*node) error {
		x.out.WriteString(tag)
		return nil
	}
}

func (x *xhtml) open(name string) func(*node) error {
	tag := "<" + name + ">"
	return func(*node) error {
		x.out.WriteString(tag)
		return nil
	}
}

func (x *xhtml) close(name string) func(*node) error {
	tag := "</" + name + ">"
	return func(*node) error {
		x.out.WriteString(tag)
		return nil
	}
}

func (x *xhtml) element(name string) *visitor {
	return &visitor{x.open(name), x.close(name)}
}

func (x *xhtml) heading(level int) *visitor {
	tag := fmt.Sprintf("h%d", level)
	return &visitor{x.open(tag), x.close(tag)}
}

func (x *xhtml) text(n *node) error {
	enc := xml.NewEncoder(&x.out)
	txt := n.value.(string)
	enc.EncodeToken(xml.CharData(txt))
	enc.Flush()
	return nil
}

func (x *xhtml) noWikiInline(n *node) error {
	x.out.WriteString("<tt>")
	x.text(n)
	x.out.WriteString("</tt>")
	return nil
}

func (x *xhtml) noWiki(n *node) error {
	x.out.WriteString("
")
	x.noWikiInline(n)
	x.out.WriteString("
\n")
	return nil
}

func (x *xhtml) link(n *node) error {
	href := n.value.(string)
	enc := xml.NewEncoder(&x.out)
	enc.EncodeToken(xml.StartElement{
		Name: xml.Name{Local: "a"},
		Attr: []xml.Attr{{xml.Name{Local: "href"}, href}},
	})
	enc.Flush()
	return nil
}

func exportXHTML(doc *document, out io.Writer) error {
	// TODO: Implement me!

	var x xhtml

	x.out.WriteString(xml.Header)
	x.out.WriteString("\n\n")

	err := doc.traverse(map[nodeType]*visitor{
		orderedListNode:     x.element("ol"),
		unorderedListNode:   x.element("ul"),
		listItemNode:        x.element("li"),
		textNode:            &visitor{enter: x.text},
		boldNode:            x.element("strong"),
		italicsNode:         x.element("i"),
		underlinedNode:      x.element("em"),
		strikeNode:          x.element("del"),
		superscriptNode:     x.element("sup"),
		subscriptNode:       x.element("sub"),
		tableNode:           x.element("table"),
		tableRowNode:        x.element("tr"),
		tableCellNode:       x.element("td"),
		tableHeaderRowNode:  x.element("th"),
		tableHeaderCellNode: x.element("td"),
		heading1Node:        x.heading(1),
		heading2Node:        x.heading(2),
		heading3Node:        x.heading(3),
		heading4Node:        x.heading(4),
		heading5Node:        x.heading(5),
		heading6Node:        x.heading(6),
		paragraphNode:       x.element("p"),
		lineBreakNode:       &visitor{enter: x.tag("br")},
		escapeNode:          &visitor{enter: x.text},
		noWikiNode:          &visitor{enter: x.noWiki},
		noWikiInlineNode:    &visitor{enter: x.noWikiInline},
		// placeholderNode not supported, yet.
		// TODO: Implement image node.
		linkNode:           &visitor{enter: x.link, leave: x.close("a")},
		horizontalLineNode: &visitor{enter: x.tag("hr")},
	})
	if err != nil {
		x.out.WriteString("\n\n\n")
		_, err = x.out.WriteTo(out)
	}
	return err
}
<|endoftext|>


<commit_before>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package xurls

import (
	"regexp"
)

//go:generate go run tools/tldsgen/main.go
//go:generate go run tools/regexgen/main.go

const (
	letters    = "a-zA-Z\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF"
	iriChar    = letters + `0-9`
	ipv4Addr   = `(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[0-9])`
	ipv6Addr   = `([0-9A-Fa-f]{1,4}:([0-9A-Fa-f]{1,4}:([0-9A-Fa-f]{1,4}:([0-9A-Fa-f]{1,4}:([0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{0,4}|:[0-9A-Fa-f]{1,4})?|(:[0-9A-Fa-f]{1,4}){0,2})|(:[0-9A-Fa-f]{1,4}){0,3})|(:[0-9A-Fa-f]{1,4}){0,4})|:(:[0-9A-Fa-f]{1,4}){0,5})((:[0-9A-Fa-f]{1,4}){2}|:(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])){3})|(([0-9A-Fa-f]{1,4}:){1,6}|:):[0-9A-Fa-f]{0,4}|([0-9A-Fa-f]{1,4}:){7}:`
	ipAddr     = `(` + ipv4Addr + `|` + ipv6Addr + `)`
	iri        = `[` + iriChar + `]([` + iriChar + `\-]{0,61}[` + iriChar + `])?`
	hostName   = `(` + iri + `\.)+` + gtld
	domainName = `(` + hostName + `|` + ipAddr + `|localhost)`
	webURL     = `((https?://([a-zA-Z0-9$-_.+!*'(),;?&=]{1,64}(:[a-zA-Z0-9$\-_.+!*'(),;?&=]{1,25})?\@)?)?(` + domainName + `)(:\d{1,5})?)(/([` + iriChar + `;/?:@&=#~\-.+!*'(),_])*)?`
	email      = `[a-zA-Z0-9._%\-+]{1,256}@` + domainName
	all        = `(` + webURL + `|` + email + `)`
)

// Regex expressions that match various kinds of urls and addresses
var (
	WebURL = regexp.MustCompile(webURL)
	Email  = regexp.MustCompile(email)
	All    = regexp.MustCompile(all)
)
<commit_msg>Replace \d by [0-9] for consistency<commit_after>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package xurls

import (
	"regexp"
)

//go:generate go run tools/tldsgen/main.go
//go:generate go run tools/regexgen/main.go

const (
	letters    = "a-zA-Z\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF"
	iriChar    = letters + `0-9`
	ipv4Addr   = `(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[0-9])`
	ipv6Addr   = `([0-9A-Fa-f]{1,4}:([0-9A-Fa-f]{1,4}:([0-9A-Fa-f]{1,4}:([0-9A-Fa-f]{1,4}:([0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{0,4}|:[0-9A-Fa-f]{1,4})?|(:[0-9A-Fa-f]{1,4}){0,2})|(:[0-9A-Fa-f]{1,4}){0,3})|(:[0-9A-Fa-f]{1,4}){0,4})|:(:[0-9A-Fa-f]{1,4}){0,5})((:[0-9A-Fa-f]{1,4}){2}|:(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])){3})|(([0-9A-Fa-f]{1,4}:){1,6}|:):[0-9A-Fa-f]{0,4}|([0-9A-Fa-f]{1,4}:){7}:`
	ipAddr     = `(` + ipv4Addr + `|` + ipv6Addr + `)`
	iri        = `[` + iriChar + `]([` + iriChar + `\-]{0,61}[` + iriChar + `])?`
	hostName   = `(` + iri + `\.)+` + gtld
	domainName = `(` + hostName + `|` + ipAddr + `|localhost)`
	webURL     = `((https?://([a-zA-Z0-9$-_.+!*'(),;?&=]{1,64}(:[a-zA-Z0-9$\-_.+!*'(),;?&=]{1,25})?\@)?)?(` + domainName + `)(:[0-9]{1,5})?)(/([` + iriChar + `;/?:@&=#~\-.+!*'(),_])*)?`
	email      = `[a-zA-Z0-9._%\-+]{1,256}@` + domainName
	all        = `(` + webURL + `|` + email + `)`
)

// Regex expressions that match various kinds of urls and addresses
var (
	WebURL = regexp.MustCompile(webURL)
	Email  = regexp.MustCompile(email)
	All    = regexp.MustCompile(all)
)
<|endoftext|>


<commit_before>package pig

import (
	"fmt"
	"regexp"
	"strings"
	"time"
)

// Note : with the GAE datastore it is *not* possible
// to have a slice of slices inside a struct.

// Idiom is the main entity of programming-idioms.org .
// An Idiom contains its Implementations.
// It is in theory independent from any framework, but has been used only in
// Google App Engine so far.
type Idiom struct {
	// Id is auto-incremented 1, 2, 3...
	// TODO name ID instead
	Id int

	// Reserved in case one idiom derived from another
	// TODO name OrigID instead
	OrigId int

	// Title is like the "idiom name"
	Title string

	// LeadParagraph is the idiom Description : 1 to 3 lines are fine
	LeadParagraph string

	// Author is the name of the original creator of this idiom on this website
	Author string

	// CreationDate is the date of creation of this idiom on this site
	CreationDate time.Time

	// LastEditor is the name of the last person who modified this idiom
	LastEditor string

	// EditSummary is the comment explaining why LastEditor made the edit.
	// It is not displayed except in history views.
	EditSummary string

	// LastEditedImplID is the ID of the only impl modified by
	// last edit. LastEditedImplID should be 0 if last
	// edit was on the idiom statement, not on an impl.
	LastEditedImplID int

	// Please acknowledge sources (idiom statement, not snippet).
	OriginalAttributionURL string

	// Picture representing the concept, if necessary
	Picture string

	ImageURL string

	// Autoincremented at each update 1, 2, 3...
	Version int

	// Date of last update
	VersionDate time.Time

	// List of implementations of this idiom in specific languages
	Implementations []Impl

	// (Denormalized) number of contained implementation, for datastore querying
	ImplCount int

	// How many votes for the idiom itself  (votes up - votes down)
	Rating int

	// Index-like array of important words : those from the title
	// DEPRECATED: use the new Text Search API instead.
	WordsTitle []string

	// Index-like array of words from title, description and implementation contents
	// DEPRECATED: use the new Text Search API instead.
	Words []string

	// Did the admin validate this idiom statement ?
	Checked bool

	// Extra calculated data like "Has this idiom been upvoted by this user?"
	// Ignored by the datastore.
	Deco IdiomRenderingDecoration `datastore:"-" json:"-"`

	// Related idioms ids "See also..."
	RelatedIdiomIds []int

	// NoSQL-style : store directly some data from other objects
	RelatedIdiomTitles []string
}

// Impl is a specific implementation of one Idiom in one programming language.
// It is in theory independent from any framework, but has been used only in
// Google App Engine so far.
type Impl struct {
	// Id is Internal. Not displayed on screen (but present in URL).
	// TODO name ID instead
	Id int

	// OrigId is reserved in case one impl derived from another
	// TODO name OrigID instead
	OrigId int

	// Author is the name of the original creator of this implementation on this site.
	Author string

	// CreationDate of this implementation on this website
	CreationDate time.Time

	// LastEditor is the name of the last person who modified this impl.
	LastEditor string

	// LanguageName is the programming language of this impl.
	// It is used to visualy identify the impl inside the idiom.
	// But note that an idiom may have several implementations for same language.
	LanguageName string

	// CodeBlock contains the snippet.
	// It should contain only instructions code, not comments.
	CodeBlock string

	// OriginalAttributionURL: please acknowledge sources.
	OriginalAttributionURL string

	// DemoURL is an optional link to an online demo
	DemoURL string

	// DocumentationURL is an optional link to official doc
	DocumentationURL string

	// AuthorComment comments about the CodeBlock.
	// This comment is always displayed on the right of te code.
	// TODO rename this to CodeBlockComment.
	AuthorComment string

	// Version is incremented at each update 1, 2, 3...
	Version int

	// VersionDate of last update
	VersionDate time.Time

	// Rating is the votes count for this specific impl  (votes up - votes down)
	Rating int

	// Checked is true if an admin has validated this implementation.
	Checked bool

	// ImplRenderingDecoration is some extra calculated data like "Has this implementation been upvoted by this user?"
	// Ignored by the datastore.
	Deco ImplRenderingDecoration `datastore:"-" json:"-"`

	// ImportsBlock contains the import directives, appart from main code section.
	ImportsBlock string
}

// IdiomRenderingDecoration is the "current user" vote on this Idiom, if any.
// This struct does not contain the Idiom ID, so it must be part of a larger struct.
type IdiomRenderingDecoration struct {
	UpVoted   bool
	DownVoted bool
}

// ImplRenderingDecoration is the "current user" vote on this Impl, if any.
// This struct does not contain Impl ID nor Idiom ID, so it must be part of a larger struct.
type ImplRenderingDecoration struct {
	UpVoted   bool
	DownVoted bool
	// Matching is set to true if current impl matches user text search query.
	Matching bool
	// SearchedLang is set to true if current impl lang is the user typed lang.
	SearchedLang bool
}

// IdiomVoteLog is a history trace of an Idiom vote, from a specific user.
// This struct does not contain the nickname of the voter.
// However it does contain the Idiom ID.
// Each vote will have a voting booth as ancestor, specific for the nickname.
type IdiomVoteLog struct {
	IdiomId int
	// Typicaly +1 or -1
	Value int
	// IpHash stored only to prevent abusive multiple votes
	IpHash string
	Date   time.Time
}

// ImplVoteLog is a history trace of an Impl vote, from a specific user.
// This structure does not contain the nickname of the voter.
// However it does contain the Idiom ID and Impl ID.
// Each vote will have a voting booth as ancestor, specific for the nickname.
type ImplVoteLog struct {
	IdiomId int
	ImplId  int
	// Typicaly +1 or -1
	Value int
	// IpHash stored only to prevent abusive multiple votes
	IpHash string
	Date   time.Time
}

// IdiomHistory stores all the history: old versions of Idioms.
type IdiomHistory struct {
	// Just embeds Idiom
	Idiom
	// If needed, add specific history fields
	UpdatedImplId int
	// TODO: how to get rid properly?
	// Got `datastore: cannot load field "EditorSummary" into a "pig.IdiomHistory": no such struct field`
	EditorSummary string `deprecated`
}

func (ih *IdiomHistory) AsIdiomPtr() *Idiom {
	return &(ih.Idiom)
}

type MessageForUser struct {
	CreationDate,
	FirstViewDate,
	LastViewDate,
	DismissalDate,
	ExpirationDate time.Time
	Message  string
	Username string
}

/* ---- */

// FindImplInIdiom is a (unoptimized) iteration to retrieve an Impl by its ID,
// inside an Idiom.
//
// It returns a pointer to the Impl, not a copy.
func (idiom *Idiom) FindImplInIdiom(implId int) (int, *Impl, bool) {
	for i := range idiom.Implementations {
		impl := &idiom.Implementations[i]
		if impl.Id == implId {
			return i, impl, true
		}
	}
	return -1, nil, false
}

// ExtractIndexableWords compute the list of words contained in an Idiom.
// First return value is the list of all matchable words.
// Second return value is the list of matchable words from title only.
func (idiom *Idiom) ExtractIndexableWords() (w []string, wTitle []string, wLead []string) {
	w = SplitForIndexing(idiom.Title, true)
	w = append(w, fmt.Sprintf("%d", idiom.Id))
	wTitle = w
	wLead = SplitForIndexing(idiom.LeadParagraph, true)
	w = append(w, wLead...)
	for i := range idiom.Implementations {
		impl := &idiom.Implementations[i]
		wImpl := impl.ExtractIndexableWords()
		w = append(w, wImpl...)
	}
	return w, wTitle, wLead
}

// ExtractIndexableWords compute the list of words contained in an Impl.
func (impl *Impl) ExtractIndexableWords() []string {
	w := make([]string, 0, 20)
	w = append(w, fmt.Sprintf("%d", impl.Id))
	w = append(w, strings.ToLower(impl.LanguageName))
	w = append(w, SplitForIndexing(impl.CodeBlock, true)...)
	if len(impl.AuthorComment) >= 3 {
		w = append(w, SplitForIndexing(impl.AuthorComment, true)...)
	}
	return w
}

var regexpWhiteSpace = regexp.MustCompile("[ \\t\\n]")
var regexpWhiteSpaceDash = regexp.MustCompile("[ \\t\\n-]")
var regexpDigitsOnly = regexp.MustCompile("^\\d+$")

// SplitForIndexing cuts sentences or paragrahs into words.
// Words of 2 letters of less are discarded.
func SplitForIndexing(s string, normalize bool) []string {
	if normalize {
		s = NormalizeRunes(s)
	}
	chunks := regexpWhiteSpace.Split(s, -1)
	realChunks := make([]string, 0, len(chunks))

	for _, chunk := range chunks {
		// Accepted :
		// All words having at least 3 characters
		// All 1-digits words and 2-digits words
		if len(chunk) >= 3 || regexpDigitsOnly.MatchString(chunk) {
			realChunks = append(realChunks, NormalizeRunes(chunk))
		}
	}

	// Stategy for dash-compound words: all bits get indexed (in addition to the full compound)
	for _, chunk := range chunks {
		if strings.Contains(chunk, "-") {
			for _, bit := range strings.Split(chunk, "-") {
				if bit != "" {
					realChunks = append(realChunks, bit)
				}
			}
		}
	}

	return realChunks
}

// SplitForSearching cuts an input search string into a slice of search terms.
func SplitForSearching(s string, normalize bool) []string {
	if normalize {
		s = NormalizeRunes(s)
	}
	chunks := regexpWhiteSpaceDash.Split(s, -1)
	chunks = FilterOut(chunks, []string{""})
	// All typed chunk are considered acceptable search terms
	return chunks
}

// NormalizeRunes discard special characters from a string, for indexing and for searching.
// Some letters with diacritics are replaced by the same letter without diacritics.
func NormalizeRunes(str string) string {
	str = strings.ToLower(str)
	norm := func(r rune) rune {
		switch r {
		// TODO find a standard golang normalization ?
		case ' ', '\t', '(', ')', '"', '\'', ',', ';', ':', '?', '.', '/', '+':
			return ' '
		case '%', '^', '=', '`', '*', '&', '!', '°', '_':
			return ' '
		case 'à', 'ä':
			return 'a'
		case 'ç':
			return 'c'
		case 'é', 'è', 'ê', 'ë':
			return 'e'
		case 'ï', 'î':
			return 'i'
		case 'ô', 'ö':
			return 'o'
		case 'û', 'ü':
			return 'u'
		}
		switch {
		case r >= 'a' && r <= 'z':
			return r
		case r >= 'A' && r <= 'Z':
			return r
		case r >= '0' && r <= '9':
			return r
		case r == '-':
			return r
		}
		// Unknown characters should not be allowed in
		return -1
	}
	return strings.Map(norm, str)
}

func containsInt(a []int, x int) bool {
	for _, i := range a {
		if i == x {
			return true
		}
	}
	return false
}

// AddRelation creates a bidirectional link between 2 related Idioms.
func (idiom *Idiom) AddRelation(other *Idiom) {
	if !containsInt(idiom.RelatedIdiomIds, other.Id) {
		idiom.RelatedIdiomIds = append(idiom.RelatedIdiomIds, other.Id)
		idiom.RelatedIdiomTitles = append(idiom.RelatedIdiomTitles, other.Title)
	}
	if !containsInt(other.RelatedIdiomIds, idiom.Id) {
		other.RelatedIdiomIds = append(other.RelatedIdiomIds, idiom.Id)
		other.RelatedIdiomTitles = append(other.RelatedIdiomTitles, idiom.Title)
	}
}
<commit_msg>Search: discard small terms.<commit_after>package pig

import (
	"fmt"
	"regexp"
	"strings"
	"time"
)

// Note : with the GAE datastore it is *not* possible
// to have a slice of slices inside a struct.

// Idiom is the main entity of programming-idioms.org .
// An Idiom contains its Implementations.
// It is in theory independent from any framework, but has been used only in
// Google App Engine so far.
type Idiom struct {
	// Id is auto-incremented 1, 2, 3...
	// TODO name ID instead
	Id int

	// Reserved in case one idiom derived from another
	// TODO name OrigID instead
	OrigId int

	// Title is like the "idiom name"
	Title string

	// LeadParagraph is the idiom Description : 1 to 3 lines are fine
	LeadParagraph string

	// Author is the name of the original creator of this idiom on this website
	Author string

	// CreationDate is the date of creation of this idiom on this site
	CreationDate time.Time

	// LastEditor is the name of the last person who modified this idiom
	LastEditor string

	// EditSummary is the comment explaining why LastEditor made the edit.
	// It is not displayed except in history views.
	EditSummary string

	// LastEditedImplID is the ID of the only impl modified by
	// last edit. LastEditedImplID should be 0 if last
	// edit was on the idiom statement, not on an impl.
	LastEditedImplID int

	// Please acknowledge sources (idiom statement, not snippet).
	OriginalAttributionURL string

	// Picture representing the concept, if necessary
	Picture string

	ImageURL string

	// Autoincremented at each update 1, 2, 3...
	Version int

	// Date of last update
	VersionDate time.Time

	// List of implementations of this idiom in specific languages
	Implementations []Impl

	// (Denormalized) number of contained implementation, for datastore querying
	ImplCount int

	// How many votes for the idiom itself  (votes up - votes down)
	Rating int

	// Index-like array of important words : those from the title
	// DEPRECATED: use the new Text Search API instead.
	WordsTitle []string

	// Index-like array of words from title, description and implementation contents
	// DEPRECATED: use the new Text Search API instead.
	Words []string

	// Did the admin validate this idiom statement ?
	Checked bool

	// Extra calculated data like "Has this idiom been upvoted by this user?"
	// Ignored by the datastore.
	Deco IdiomRenderingDecoration `datastore:"-" json:"-"`

	// Related idioms ids "See also..."
	RelatedIdiomIds []int

	// NoSQL-style : store directly some data from other objects
	RelatedIdiomTitles []string
}

// Impl is a specific implementation of one Idiom in one programming language.
// It is in theory independent from any framework, but has been used only in
// Google App Engine so far.
type Impl struct {
	// Id is Internal. Not displayed on screen (but present in URL).
	// TODO name ID instead
	Id int

	// OrigId is reserved in case one impl derived from another
	// TODO name OrigID instead
	OrigId int

	// Author is the name of the original creator of this implementation on this site.
	Author string

	// CreationDate of this implementation on this website
	CreationDate time.Time

	// LastEditor is the name of the last person who modified this impl.
	LastEditor string

	// LanguageName is the programming language of this impl.
	// It is used to visualy identify the impl inside the idiom.
	// But note that an idiom may have several implementations for same language.
	LanguageName string

	// CodeBlock contains the snippet.
	// It should contain only instructions code, not comments.
	CodeBlock string

	// OriginalAttributionURL: please acknowledge sources.
	OriginalAttributionURL string

	// DemoURL is an optional link to an online demo
	DemoURL string

	// DocumentationURL is an optional link to official doc
	DocumentationURL string

	// AuthorComment comments about the CodeBlock.
	// This comment is always displayed on the right of te code.
	// TODO rename this to CodeBlockComment.
	AuthorComment string

	// Version is incremented at each update 1, 2, 3...
	Version int

	// VersionDate of last update
	VersionDate time.Time

	// Rating is the votes count for this specific impl  (votes up - votes down)
	Rating int

	// Checked is true if an admin has validated this implementation.
	Checked bool

	// ImplRenderingDecoration is some extra calculated data like "Has this implementation been upvoted by this user?"
	// Ignored by the datastore.
	Deco ImplRenderingDecoration `datastore:"-" json:"-"`

	// ImportsBlock contains the import directives, appart from main code section.
	ImportsBlock string
}

// IdiomRenderingDecoration is the "current user" vote on this Idiom, if any.
// This struct does not contain the Idiom ID, so it must be part of a larger struct.
type IdiomRenderingDecoration struct {
	UpVoted   bool
	DownVoted bool
}

// ImplRenderingDecoration is the "current user" vote on this Impl, if any.
// This struct does not contain Impl ID nor Idiom ID, so it must be part of a larger struct.
type ImplRenderingDecoration struct {
	UpVoted   bool
	DownVoted bool
	// Matching is set to true if current impl matches user text search query.
	Matching bool
	// SearchedLang is set to true if current impl lang is the user typed lang.
	SearchedLang bool
}

// IdiomVoteLog is a history trace of an Idiom vote, from a specific user.
// This struct does not contain the nickname of the voter.
// However it does contain the Idiom ID.
// Each vote will have a voting booth as ancestor, specific for the nickname.
type IdiomVoteLog struct {
	IdiomId int
	// Typicaly +1 or -1
	Value int
	// IpHash stored only to prevent abusive multiple votes
	IpHash string
	Date   time.Time
}

// ImplVoteLog is a history trace of an Impl vote, from a specific user.
// This structure does not contain the nickname of the voter.
// However it does contain the Idiom ID and Impl ID.
// Each vote will have a voting booth as ancestor, specific for the nickname.
type ImplVoteLog struct {
	IdiomId int
	ImplId  int
	// Typicaly +1 or -1
	Value int
	// IpHash stored only to prevent abusive multiple votes
	IpHash string
	Date   time.Time
}

// IdiomHistory stores all the history: old versions of Idioms.
type IdiomHistory struct {
	// Just embeds Idiom
	Idiom
	// If needed, add specific history fields
	UpdatedImplId int
	// TODO: how to get rid properly?
	// Got `datastore: cannot load field "EditorSummary" into a "pig.IdiomHistory": no such struct field`
	EditorSummary string `deprecated`
}

func (ih *IdiomHistory) AsIdiomPtr() *Idiom {
	return &(ih.Idiom)
}

type MessageForUser struct {
	CreationDate,
	FirstViewDate,
	LastViewDate,
	DismissalDate,
	ExpirationDate time.Time
	Message  string
	Username string
}

/* ---- */

// FindImplInIdiom is a (unoptimized) iteration to retrieve an Impl by its ID,
// inside an Idiom.
//
// It returns a pointer to the Impl, not a copy.
func (idiom *Idiom) FindImplInIdiom(implId int) (int, *Impl, bool) {
	for i := range idiom.Implementations {
		impl := &idiom.Implementations[i]
		if impl.Id == implId {
			return i, impl, true
		}
	}
	return -1, nil, false
}

// ExtractIndexableWords compute the list of words contained in an Idiom.
// First return value is the list of all matchable words.
// Second return value is the list of matchable words from title only.
func (idiom *Idiom) ExtractIndexableWords() (w []string, wTitle []string, wLead []string) {
	w = SplitForIndexing(idiom.Title, true)
	w = append(w, fmt.Sprintf("%d", idiom.Id))
	wTitle = w
	wLead = SplitForIndexing(idiom.LeadParagraph, true)
	w = append(w, wLead...)
	for i := range idiom.Implementations {
		impl := &idiom.Implementations[i]
		wImpl := impl.ExtractIndexableWords()
		w = append(w, wImpl...)
	}
	return w, wTitle, wLead
}

// ExtractIndexableWords compute the list of words contained in an Impl.
func (impl *Impl) ExtractIndexableWords() []string {
	w := make([]string, 0, 20)
	w = append(w, fmt.Sprintf("%d", impl.Id))
	w = append(w, strings.ToLower(impl.LanguageName))
	w = append(w, SplitForIndexing(impl.CodeBlock, true)...)
	if len(impl.AuthorComment) >= 3 {
		w = append(w, SplitForIndexing(impl.AuthorComment, true)...)
	}
	return w
}

var regexpWhiteSpace = regexp.MustCompile("[ \\t\\n]")
var regexpWhiteSpaceDash = regexp.MustCompile("[ \\t\\n-]")
var regexpDigitsOnly = regexp.MustCompile("^\\d+$")

// SplitForIndexing cuts sentences or paragrahs into words.
// Words of 2 letters of less are discarded.
func SplitForIndexing(s string, normalize bool) []string {
	if normalize {
		s = NormalizeRunes(s)
	}
	chunks := regexpWhiteSpace.Split(s, -1)
	realChunks := make([]string, 0, len(chunks))

	for _, chunk := range chunks {
		// Accepted :
		// All words having at least 3 characters
		// All 1-digits words and 2-digits words
		if len(chunk) >= 3 || regexpDigitsOnly.MatchString(chunk) {
			realChunks = append(realChunks, NormalizeRunes(chunk))
		}
	}

	// Stategy for dash-compound words: all bits get indexed (in addition to the full compound)
	for _, chunk := range chunks {
		if strings.Contains(chunk, "-") {
			for _, bit := range strings.Split(chunk, "-") {
				if bit != "" {
					realChunks = append(realChunks, bit)
				}
			}
		}
	}

	return realChunks
}

// SplitForSearching cuts an input search string into a slice of search terms.
func SplitForSearching(s string, normalize bool) []string {
	if normalize {
		s = NormalizeRunes(s)
	}
	chunks := regexpWhiteSpaceDash.Split(s, -1)
	realChunks := make([]string, 0, len(chunks))
	for _, chunk := range chunks {
		// Small terms (1 or 2 chars) must be discarded,
		// because they weren't indexed in the first place.
		if len(chunk) >= 3 || regexpDigitsOnly.MatchString(chunk) {
			realChunks = append(realChunks, NormalizeRunes(chunk))
		}
	}
	return realChunks
}

// NormalizeRunes discard special characters from a string, for indexing and for searching.
// Some letters with diacritics are replaced by the same letter without diacritics.
func NormalizeRunes(str string) string {
	str = strings.ToLower(str)
	norm := func(r rune) rune {
		switch r {
		// TODO find a standard golang normalization ?
		case ' ', '\t', '(', ')', '"', '\'', ',', ';', ':', '?', '.', '/', '+':
			return ' '
		case '%', '^', '=', '`', '*', '&', '!', '°', '_':
			return ' '
		case 'à', 'ä':
			return 'a'
		case 'ç':
			return 'c'
		case 'é', 'è', 'ê', 'ë':
			return 'e'
		case 'ï', 'î':
			return 'i'
		case 'ô', 'ö':
			return 'o'
		case 'û', 'ü':
			return 'u'
		}
		switch {
		case r >= 'a' && r <= 'z':
			return r
		case r >= 'A' && r <= 'Z':
			return r
		case r >= '0' && r <= '9':
			return r
		case r == '-':
			return r
		}
		// Unknown characters should not be allowed in
		return -1
	}
	return strings.Map(norm, str)
}

func containsInt(a []int, x int) bool {
	for _, i := range a {
		if i == x {
			return true
		}
	}
	return false
}

// AddRelation creates a bidirectional link between 2 related Idioms.
func (idiom *Idiom) AddRelation(other *Idiom) {
	if !containsInt(idiom.RelatedIdiomIds, other.Id) {
		idiom.RelatedIdiomIds = append(idiom.RelatedIdiomIds, other.Id)
		idiom.RelatedIdiomTitles = append(idiom.RelatedIdiomTitles, other.Title)
	}
	if !containsInt(other.RelatedIdiomIds, idiom.Id) {
		other.RelatedIdiomIds = append(other.RelatedIdiomIds, idiom.Id)
		other.RelatedIdiomTitles = append(other.RelatedIdiomTitles, idiom.Title)
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"bufio"
	"fmt"
	"strings"

	"github.com/containerum/chkit/pkg/chkitErrors"
	"github.com/containerum/chkit/pkg/cli/prerun"
	"github.com/containerum/chkit/pkg/client"
	"github.com/containerum/chkit/pkg/context"
	"github.com/containerum/chkit/pkg/util/activekit"
	"github.com/containerum/chkit/pkg/util/angel"
	"github.com/containerum/chkit/pkg/util/ferr"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

const (
	ErrUnableToReadLogs chkitErrors.Err = "unable to read logs"
)

var logsCommandAliases = []string{"log"}

func Logs(ctx *context.Context) *cobra.Command {
	var logsConfig = struct {
		Quiet  bool
		Follow bool
		Prev   bool
		Tail   uint
	}{}
	command := &cobra.Command{
		Use:     "logs",
		Aliases: logsCommandAliases,
		Short:   "View pod logs",
		Long:    `view pod logs. Aliases: ` + strings.Join(logsCommandAliases, ", "),
		Example: `logs pod_label [container] [--follow] [--prev] [--tail n] [--quiet]`,
		PreRun: func(cmd *cobra.Command, args []string) {
			if err := prerun.PreRun(ctx); err != nil {
				angel.Angel(ctx, err)
				ctx.Exit(1)
			}
			if err := prerun.GetNamespaceByUserfriendlyID(ctx, cmd.Flags()); err != nil {
				ferr.Println(err)
				ctx.Exit(1)
			}
		},
		Run: func(cmd *cobra.Command, args []string) {
			var podName string
			var containerName string
			client := ctx.GetClient()
			switch len(args) {
			case 2:
				containerName = args[1]
				fallthrough
			case 1:
				podName = args[0]
			default:
				cmd.Help()
				return
			}

			params := chClient.GetPodLogsParams{
				Namespace: ctx.GetNamespace().ID,
				Pod:       podName,
				Container: containerName,
				Follow:    logsConfig.Follow,
				Previous:  logsConfig.Prev,
				Tail:      int(logsConfig.Tail),
			}
			rc, err := client.GetPodLogs(params)
			if err != nil {
				logrus.WithError(err).Errorf("error while getting logs")
				activekit.Attention(err.Error())
			}
			defer rc.Close()

			scanner := bufio.NewScanner(rc)
			var nLines uint64
			for scanner.Scan() {
				if err := scanner.Err(); err != nil {
					err = ErrUnableToReadLogs.Wrap(err)
					logrus.WithError(err).Errorf("unable to scan logs byte stream")
					activekit.Attention(err.Error())
					ctx.Exit(1)
				}
				fmt.Println(scanner.Text())
				nLines++
			}
			if err != nil {
				activekit.Attention(err.Error())
				ctx.Exit(1)
			}
		},
	}
	command.PersistentFlags().
		BoolVarP(&logsConfig.Quiet, "quiet", "q", false, "print only logs and errors")
	command.PersistentFlags().
		BoolVarP(&logsConfig.Follow, "follow", "f", false, `follow pod logs`)
	command.PersistentFlags().
		UintVarP(&logsConfig.Tail, "tail", "t", 100, `print last <value> log lines`)
	return command
}
<commit_msg>add pod selector<commit_after>package cli

import (
	"bufio"
	"fmt"

	"github.com/containerum/chkit/pkg/chkitErrors"
	"github.com/containerum/chkit/pkg/cli/prerun"
	"github.com/containerum/chkit/pkg/client"
	"github.com/containerum/chkit/pkg/context"
	"github.com/containerum/chkit/pkg/util/activekit"
	"github.com/containerum/chkit/pkg/util/angel"
	"github.com/containerum/chkit/pkg/util/ferr"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

const (
	ErrUnableToReadLogs chkitErrors.Err = "unable to read logs"
)

var logsCommandAliases = []string{"log"}

func Logs(ctx *context.Context) *cobra.Command {
	var logsConfig = struct {
		Quiet  bool
		Follow bool
		Prev   bool
		Tail   uint
	}{}
	command := &cobra.Command{
		Use:     "logs",
		Aliases: logsCommandAliases,
		Short:   "View pod logs",
		Example: `logs pod_label [container] [--follow] [--prev] [--tail n] [--quiet]`,
		PreRun: func(cmd *cobra.Command, args []string) {
			if err := prerun.PreRun(ctx); err != nil {
				angel.Angel(ctx, err)
				ctx.Exit(1)
			}
			if err := prerun.GetNamespaceByUserfriendlyID(ctx, cmd.Flags()); err != nil {
				ferr.Println(err)
				ctx.Exit(1)
			}
		},
		Run: func(cmd *cobra.Command, args []string) {
			var podName string
			var containerName string
			client := ctx.GetClient()
			switch len(args) {
			case 2:
				containerName = args[1]
				fallthrough
			case 1:
				podName = args[0]
			default:
				var pods, err = client.GetPodList(ctx.GetNamespace().ID)
				if err != nil {
					ferr.Println(err)
					ctx.Exit(1)
				}
				(&activekit.Menu{
					Title: "Select pod",
					Items: activekit.ItemsFromIter(uint(pods.Len()), func(index uint) *activekit.MenuItem {
						var po = pods[index]
						return &activekit.MenuItem{
							Label: po.Name,
							Action: func() error {
								podName = po.Name
								return nil
							},
						}
					}),
				}).Run()
			}

			params := chClient.GetPodLogsParams{
				Namespace: ctx.GetNamespace().ID,
				Pod:       podName,
				Container: containerName,
				Follow:    logsConfig.Follow,
				Previous:  logsConfig.Prev,
				Tail:      int(logsConfig.Tail),
			}
			rc, err := client.GetPodLogs(params)
			if err != nil {
				logrus.WithError(err).Errorf("error while getting logs")
				activekit.Attention(err.Error())
			}
			defer rc.Close()
			scanner := bufio.NewScanner(rc)
			var nLines uint64
			for scanner.Scan() {
				if err := scanner.Err(); err != nil {
					err = ErrUnableToReadLogs.Wrap(err)
					logrus.WithError(err).Errorf("unable to scan logs byte stream")
					activekit.Attention(err.Error())
					ctx.Exit(1)
				}
				fmt.Println(scanner.Text())
				nLines++
			}
			if err != nil {
				activekit.Attention(err.Error())
				ctx.Exit(1)
			}
		},
	}
	command.PersistentFlags().
		BoolVarP(&logsConfig.Quiet, "quiet", "q", false, "print only logs and errors")
	command.PersistentFlags().
		BoolVarP(&logsConfig.Follow, "follow", "f", false, `follow pod logs`)
	command.PersistentFlags().
		UintVarP(&logsConfig.Tail, "tail", "t", 100, `print last <value> log lines`)
	return command
}
<|endoftext|>


<commit_before>package cli

import (
	"context"
	"log"
	"os"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/taku-k/polymerase/pkg/allocator"
	"github.com/taku-k/polymerase/pkg/storage/storagepb"
	"github.com/taku-k/polymerase/pkg/tempbackup/tempbackuppb"
)

func cleanupTempDirRunE(wrapped func(*cobra.Command, []string) error) func(*cobra.Command, []string) error {
	return func(cmd *cobra.Command, args []string) error {
		err := wrapped(cmd, args)
		os.RemoveAll(xtrabackupCfg.LsnTempDir)
		return err
	}
}

func getAppropriateStorageClient(ctx context.Context, db string) (storagepb.StorageServiceClient, error) {
	cli, err := clientv3.New(clientv3.Config{
		Endpoints:   []string{baseCfg.Addr},
		Context:     ctx,
		DialTimeout: 5 * time.Second,
	})
	if err != nil {
		return nil, err
	}
	defer cli.Close()

	addr, err := allocator.SearchStoredAddr(cli, db)
	if err != nil {
		return nil, err
	}

	log.Printf("Located Address is %s", addr)

	c, err := connectGRPC(ctx, addr)
	if err != nil {
		return nil, err
	}

	return storagepb.NewStorageServiceClient(c), nil
}

func getStorageClient(ctx context.Context, addr string) (storagepb.StorageServiceClient, error) {
	c, err := connectGRPC(ctx, addr)
	if err != nil {
		return nil, err
	}
	return storagepb.NewStorageServiceClient(c), nil
}

func getTempBackupClient(ctx context.Context, db string) (tempbackuppb.BackupTransferServiceClient, error) {
	cli, err := clientv3.New(clientv3.Config{
		Endpoints:   []string{baseCfg.Addr},
		Context:     ctx,
		DialTimeout: 5 * time.Second,
	})
	if err != nil {
		return nil, err
	}
	defer cli.Close()

	node, addr, err := allocator.SelectAppropriateHost(cli, db)
	if err != nil {
		return nil, err
	}

	c, err := connectGRPC(ctx, addr)
	if err != nil {
		return nil, err
	}

	log.Printf("Select node as backup: %s\n", node)
	return tempbackuppb.NewBackupTransferServiceClient(c), nil
}

func usageAndError(cmd *cobra.Command) error {
	if err := cmd.Usage(); err != nil {
		return err
	}
	return errors.New("invalid arguments")
}
<commit_msg>Disallow using tempbackup client<commit_after>package cli

import (
	"context"
	"log"
	"os"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/taku-k/polymerase/pkg/allocator"
	"github.com/taku-k/polymerase/pkg/storage/storagepb"
)

func cleanupTempDirRunE(wrapped func(*cobra.Command, []string) error) func(*cobra.Command, []string) error {
	return func(cmd *cobra.Command, args []string) error {
		err := wrapped(cmd, args)
		os.RemoveAll(xtrabackupCfg.LsnTempDir)
		return err
	}
}

func getAppropriateStorageClient(ctx context.Context, db string) (storagepb.StorageServiceClient, error) {
	cli, err := clientv3.New(clientv3.Config{
		Endpoints:   []string{baseCfg.Addr},
		Context:     ctx,
		DialTimeout: 5 * time.Second,
	})
	if err != nil {
		return nil, err
	}
	defer cli.Close()

	addr, err := allocator.SearchStoredAddr(cli, db)
	if err != nil {
		return nil, err
	}

	log.Printf("Located Address is %s", addr)

	c, err := connectGRPC(ctx, addr)
	if err != nil {
		return nil, err
	}

	return storagepb.NewStorageServiceClient(c), nil
}

func getStorageClient(ctx context.Context, addr string) (storagepb.StorageServiceClient, error) {
	c, err := connectGRPC(ctx, addr)
	if err != nil {
		return nil, err
	}
	return storagepb.NewStorageServiceClient(c), nil
}

func getTempBackupClient(ctx context.Context, db string) (storagepb.StorageServiceClient, error) {
	cli, err := clientv3.New(clientv3.Config{
		Endpoints:   []string{baseCfg.Addr},
		Context:     ctx,
		DialTimeout: 5 * time.Second,
	})
	if err != nil {
		return nil, err
	}
	defer cli.Close()

	node, addr, err := allocator.SelectAppropriateHost(cli, db)
	if err != nil {
		return nil, err
	}

	c, err := connectGRPC(ctx, addr)
	if err != nil {
		return nil, err
	}

	log.Printf("Select node as backup: %s\n", node)
	return storagepb.NewStorageServiceClient(c), nil
}

func usageAndError(cmd *cobra.Command) error {
	if err := cmd.Usage(); err != nil {
		return err
	}
	return errors.New("invalid arguments")
}
<|endoftext|>


<commit_before>package lang

import "fmt"

type Parser struct {
	l *Lexer

	buf []buf
	n   int
}

func NewParser(l *Lexer) *Parser {
	return &Parser{
		l: l,
	}
}

func (p *Parser) Parse() *Program {
	prog := &Program{
		scope: &Scope{
			make(map[string]string),
		},
	}

	for {
		tok, lit := p.scanSkipWhitespace()
		if tok == EOF {
			fmt.Println("REACHED EOF!!!")
			break
		}

		p.unscan()

		if p.is(MatchAssignment...) {
			tok, lit = p.scanSkipWhitespace()
			// Got name
			assign := &AssignmentStatement{
				Name: lit,
			}

			tok, lit = p.scanSkipWhitespace()
			if tok != String {
				fmt.Println("NOT TYPE")
				fmt.Println(tok, lit)
				break
			}

			tok, lit = p.scanSkipWhitespace()
			if tok != Assign {
				fmt.Println("NOT ASSIGN. TIME TO DIE!")
				break
			}

			tok, lit = p.scanSkipWhitespace()
			if tok != Quotes {
				fmt.Println("NOT quotes. TIME TO DIE")
				fmt.Println(tok, lit)
				break
			}

			var buf string

			for {
				tok, lit := p.scan()

				if tok == EOF {
					p.unscan()
					break
				}

				if tok == Quotes {
					break
				}

				buf += lit
			}

			fmt.Println("[DONE] got value", buf)

			assign.Value = buf

			prog.statements = append(prog.statements, assign)

			continue
		}

		fmt.Println("didnt match")
	}

	fmt.Println(prog)

	return prog
}

var MatchAssignment = []Token{Identifier, Whitespace, String}

func (p *Parser) is(ts ...Token) bool {
	for _, t := range ts {
		tok, _ := p.scan()

		defer func() { p.unscan() }()

		if tok != t {
			fmt.Println("Got", t, "expected", tok)
			return false
		}
	}

	return true
}

// If it can pull n from tokens, do that... else scan new tok
// and add it to the buf
func (p *Parser) scan() (Token, string) {
	defer func() {
		p.n++
	}()

	if p.n >= len(p.buf) {
		fmt.Println("Scanning new token", p.n)
		tok, lit := p.l.Scan()
		p.buf = append(p.buf, buf{
			tok: tok,
			lit: lit,
		})

		return tok, lit
	}

	b := p.buf[p.n]

	fmt.Println("Retrieving old token", p.n)
	return b.tok, b.lit
}

func (p *Parser) unscan() {
	p.n--
}

func (p *Parser) scanSkipWhitespace() (Token, string) {
	for {
		tok, lit := p.scan()

		if tok != Whitespace {
			return tok, lit
		}
	}
}

type buf struct {
	tok Token
	lit string
}
<commit_msg>Move assignment scanning to different function<commit_after>package lang

import "fmt"

type Parser struct {
	l *Lexer

	buf []buf
	n   int
}

func NewParser(l *Lexer) *Parser {
	return &Parser{
		l: l,
	}
}

func (p *Parser) Parse() *Program {
	prog := &Program{
		scope: &Scope{
			make(map[string]string),
		},
	}

	for {
		tok, _ := p.scanSkipWhitespace()
		if tok == EOF {
			fmt.Println("REACHED EOF!!!")
			break
		}

		p.unscan()

		if p.is(MatchAssignment...) {
			as, _ := p.parseAssignment()

			prog.statements = append(prog.statements, as)
			continue
		}

		fmt.Println("didnt match")
	}

	fmt.Println(prog)

	return prog
}

var MatchAssignment = []Token{Identifier, Whitespace, String}

func (p *Parser) parseAssignment() (*AssignmentStatement, error) {
	tok, lit := p.scanSkipWhitespace()
	// Got name
	assign := &AssignmentStatement{
		Name: lit,
	}

	tok, lit = p.scanSkipWhitespace()
	if tok != String {
		fmt.Println("NOT TYPE")
		return nil, fmt.Errorf("found %v expected String")
	}

	tok, lit = p.scanSkipWhitespace()
	if tok != Assign {
		fmt.Println("NOT ASSIGN. TIME TO DIE!")
		return nil, fmt.Errorf("found %v expected String")
	}

	tok, lit = p.scanSkipWhitespace()
	if tok != Quotes {
		fmt.Println("NOT quotes. TIME TO DIE")
		fmt.Println(tok, lit)

		return nil, fmt.Errorf("found %v expected quotes")
	}

	var buf string

	for {
		tok, lit := p.scan()

		if tok == EOF {
			p.unscan()
			break
		}

		if tok == Quotes {
			break
		}

		buf += lit
	}

	fmt.Println("[DONE] got value", buf)

	assign.Value = buf

	return assign, nil
}

func (p *Parser) is(ts ...Token) bool {
	for _, t := range ts {
		tok, _ := p.scan()

		defer func() { p.unscan() }()

		if tok != t {
			fmt.Println("Got", t, "expected", tok)
			return false
		}
	}

	return true
}

// If it can pull n from tokens, do that... else scan new tok
// and add it to the buf
func (p *Parser) scan() (Token, string) {
	defer func() {
		p.n++
	}()

	if p.n >= len(p.buf) {
		tok, lit := p.l.Scan()
		fmt.Println("Scanning new token", p.n, tok)
		p.buf = append(p.buf, buf{
			tok: tok,
			lit: lit,
		})

		return tok, lit
	}

	b := p.buf[p.n]

	fmt.Println("Retrieving old token", p.n, b.tok)
	return b.tok, b.lit
}

func (p *Parser) unscan() {
	p.n--
}

func (p *Parser) scanSkipWhitespace() (Token, string) {
	for {
		tok, lit := p.scan()

		if tok != Whitespace {
			return tok, lit
		}
	}
}

type buf struct {
	tok Token
	lit string
}
<|endoftext|>


<commit_before>package sprite_sass

import (
	"bytes"
	"fmt"
	"io"
	"log"
)

func init() {
	log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
}

type Replace struct {
	Start, End int
	Value      []byte
}

type Parser struct {
	Idx, shift                    int
	Chop                          []Replace
	Pwd, Input                    string
	BuildDir, ImageDir, GenImgDir string
	Includes                      []string
	Items                         []Item
	Output                        []byte
	InlineImgs, Sprites           map[string]ImageList
	Vars                          map[string]string
}

func NewParser() *Parser {
	return &Parser{}
}

// Parser reads the tokens from the lexer and performs
// conversions and/or substitutions for sprite*() calls.
//
// Parser creates a map of all variables and sprites
// (created via sprite-map calls).
func (p *Parser) Start(in io.Reader, pkgdir string) ([]byte, error) {
	p.Vars = make(map[string]string)
	p.Sprites = make(map[string]ImageList)
	p.InlineImgs = make(map[string]ImageList)
	if p.ImageDir == "" {
		p.ImageDir = pkgdir
	}
	buf := bytes.NewBuffer(make([]byte, 0, bytes.MinRead))
	buf.ReadFrom(in)

	// This pass resolves all the imports, but positions will
	// be off due to @import calls
	items, input, err := p.GetItems(pkgdir, string(buf.Bytes()))
	if err != nil {
		return []byte(""), err
	}
	// This call will have valid token positions
	items, input, err = p.GetItems(pkgdir, input)

	p.Input = input
	p.Items = items
	if err != nil {
		panic(err)
	}
	// DEBUG
	// for _, item := range p.Items {
	// 	fmt.Printf("%s %s\n", item.Type, item)
	// }
	p.Parse(p.Items)

	p.Output = []byte(p.Input)
	p.Replace()
	// fmt.Printf("out: % #v\n", p.Sprites)
	return p.Output, nil
}

// Find Paren that matches the current (
func RParen(items []Item) (int, int) {
	if len(items) == 0 {
		return 0, 0
	}
	if items[0].Type != LPAREN {
		panic("Expected: ( was: " + items[0].Value)
	}
	pos := 1
	match := 1
	nest := false
	nestPos := 0

	for match != 0 && pos < len(items) {
		switch items[pos].Type {
		case LPAREN:
			match++
		case RPAREN:
			match--
		}
		if match > 1 {
			if !nest {
				nestPos = pos
			}
			// Nested command must be resolved
			nest = true
		}
		pos++
	}

	return pos, nestPos
}

func RBracket(items []Item, pos int) (int, int) {
	if items[pos].Type != LBRACKET && items[pos].Type != INTP {
		panic("Expected: { was: " + items[0].Value)
	}

	// Move to next item and set match to 1
	pos++
	match := 1
	nest := false
	nestPos := 0
	for match != 0 && pos < len(items) {
		switch items[pos].Type {
		case LBRACKET, INTP:
			match++
		case RBRACKET:
			match--
		}
		if match > 1 {
			if !nest {
				nestPos = pos
			}
			// Nested command must be resolved
			nest = true
		}
		pos++
	}
	return pos, nestPos
}

func (p *Parser) Parse(items []Item) []byte {
	var (
		out []byte
		eoc int
	)
	_ = eoc
	if len(items) == 0 {
		return []byte("")
	}
	j := 1
	item := items[0]
	switch item.Type {
	case VAR:
		if items[1].Value != ":" {
			log.Fatal(": expected after variable declaration")
		}
		for j < len(items) && items[j].Type != SEMIC {
			j++
		}
		// Eliminate variables for known commands
		switch items[2].Value {
		case "sprite-file":
			p.Mark(item.Pos, items[j].Pos+len(items[j].Value), "")
		}
		if items[2].Type != CMDVAR {
			// Hackery for empty sass maps
			val := string(p.Parse(items[2:j]))
			// TODO: $var: $anothervar doesnt work
			// setting other things like $var: darken(#123, 10%)
			if val != "()" && val != "" {
				// fmt.Println("SETTING", item, val)
				p.Vars[item.String()] = val
			}
		} else if items[2].Value == "sprite-map" {
			// Special parsing of sprite-maps
			p.Mark(items[0].Pos,
				items[j].Pos+len(items[j].Value), "")
			imgs := ImageList{
				ImageDir:  p.ImageDir,
				BuildDir:  p.BuildDir,
				GenImgDir: p.GenImgDir,
			}
			name := fmt.Sprintf("%s", items[0])
			glob := fmt.Sprintf("%s", items[4])
			imgs.Decode(glob)
			imgs.Vertical = true
			imgs.Combine()
			p.Sprites[name] = imgs
			//TODO: Generate filename
			_, err := imgs.Export()
			if err != nil {
				log.Printf("Failed to save sprite: %s", name)
				log.Println(err)
			}
		}
	case INTP:
		pos, _ := RBracket(items, 0)
		j = pos
		val := items[1].Value
		if val, ok := p.Vars[val]; ok {
			p.Mark(item.Pos, items[2].Pos+len(items[2].Value), val)
		}
	case SUB:
		break
		val, ok := p.Vars[item.Value]
		// Do not replace if nothing was found
		if !ok {
			val = item.Value
		}
		_ = val
		//p.Mark(item.Pos, item.Pos+len(item.Value), val)
	case CMD:
		for j < len(items) && items[j].Type != SEMIC {
			j++
		}
		out, eoc = p.Command(items[0:j])
	case TEXT:
		out = append(out, item.Value...)
	case MIXIN, FUNC, IF, ELSE, EACH:
		// Ignore the entire mixin and move to the next line
		lpos := 0
		for {
			if items[lpos].Type == LBRACKET {
				break
			}
			lpos++
		}
		pos, _ := RBracket(items, lpos)
		for i := 0; i < pos; i++ {
			out = append(out, items[i].Value...)
		}
		// fmt.Println(">>", item.Type, items[lpos:pos], "<<")
		j = pos
	default:
		if item.Type == INCLUDE {
			// Eat @include if command after is understood
			if Lookup(items[1].Value) > -1 {
				p.Mark(item.Pos, items[1].Pos, "")
			}
		}
		out = append(out, item.Value...)
	}

	return append(out, p.Parse(items[j:])...)
}

// Passed sass-command( args...)
func (p *Parser) Command(items []Item) ([]byte, int) {

	i := 0
	_ = i
	cmd := items[0]
	repl := ""
	if len(items) == 0 {
		panic(items)
	}
	eoc, nPos := RParen(items[1:])
	// Determine our offset from the source items
	if false && nPos != 0 {
		rightPos, _ := RParen(items[nPos:])
		p.Command(items[nPos:rightPos])
	}

	switch cmd.Value {
	case "sprite":
		//Capture sprite
		sprite := p.Sprites[fmt.Sprintf("%s", items[2])]
		pos, _ := RParen(items[1:])
		//Capture filename
		name := fmt.Sprintf("%s", items[3])
		repl = sprite.CSS(name)
		p.Mark(items[0].Pos, items[pos].Pos+len(items[pos].Value), repl)
	case "sprite-height":
		sprite := p.Sprites[fmt.Sprintf("%s", items[2])]
		repl = fmt.Sprintf("%dpx", sprite.SImageHeight(items[3].String()))
		p.Mark(cmd.Pos, items[eoc].Pos+len(items[eoc].Value), repl)
	case "sprite-width":
		sprite := p.Sprites[fmt.Sprintf("%s", items[2])]
		repl = fmt.Sprintf("%dpx",
			sprite.SImageWidth(items[3].String()))
		p.Mark(cmd.Pos, items[eoc].Pos+len(items[eoc].Value), repl)
	case "sprite-dimensions":
		sprite := p.Sprites[fmt.Sprintf("%s", items[2])]
		repl = sprite.Dimensions(items[3].Value)
		p.Mark(items[0].Pos, items[4].Pos+len(items[4].Value), repl)
	case "sprite-file":
		if items[2].Type != SUB {
			log.Fatalf("%s must be followed by variable, was: %s",
				cmd.Value, items[2].Value)
		}
		if items[3].Type != FILE {
			log.Fatalf("sprite-file must be followed by "+
				"sprite-variable, was: %s",
				items[3].Type)
		}
		repl := p.Sprites[fmt.Sprintf("%s", items[2])].
			File(items[3].String())
		p.Mark(items[0].Pos, items[4].Pos+len(items[4].Value), repl)
		return []byte(repl), eoc
	case "image-height", "image-width":
		if items[2].Type == FILE {
			name := items[2].Value
			img := ImageList{
				ImageDir:  p.ImageDir,
				BuildDir:  p.BuildDir,
				GenImgDir: p.GenImgDir,
			}
			img.Decode(name)
			var d int
			if cmd.Value == "image-width" {
				d = img.ImageWidth(0)
			} else if cmd.Value == "image-height" {
				d = img.ImageHeight(0)
			}
			repl = fmt.Sprintf("%dpx", d)
			p.Mark(items[0].Pos, items[3].Pos+len(items[3].Value), repl)
			return []byte(repl), eoc
		}
		if items[2].Type != CMD {
			log.Fatalf("%s first arg must be sprite-file, was: %s",
				cmd.Value, items[2].Value)
		}
		if items[4].Type != SUB {
			log.Fatalf("%s must be followed by variable, was: %s",
				cmd.Value, items[4].Type)
		}
		// Resolve variable
		sprite := p.Sprites[items[4].Value]
		var pix int
		if cmd.Value == "image-width" {
			pix = sprite.SImageWidth(items[5].Value)
		} else if cmd.Value == "image-height" {
			pix = sprite.SImageHeight(items[5].Value)
		}
		repl := fmt.Sprintf("%dpx", pix)
		p.Mark(items[0].Pos, items[7].Pos+len(items[6].Value), repl)
	case "inline-image":
		var (
			img ImageList
			ok  bool
		)
		name := fmt.Sprintf("%s", items[2])
		if img, ok = p.InlineImgs[name]; !ok {
			img = ImageList{
				ImageDir:  p.ImageDir,
				BuildDir:  p.BuildDir,
				GenImgDir: p.GenImgDir,
			}
			img.Decode(name)
			img.Combine()
			_, err := img.Export()
			if err != nil {
				log.Printf("Failed to save sprite: %s", name)
				log.Println(err)
			}
			p.InlineImgs[name] = img
		}

		repl := img.Inline()
		p.Mark(items[0].Pos, items[3].Pos+len(items[3].Value), repl)
	case "image-url":
		repl := p.ImageUrl(items)
		p.Mark(items[0].Pos, items[3].Pos+len(items[3].Value), repl)
	default:
		fmt.Println("No comprende:", items[0])
	}

	return []byte(""), eoc
}

// Import recursively resolves all imports.  It lexes the input
// adding the tokens to the Parser object.
// TODO: Convert this to byte slice in/out
func (p *Parser) GetItems(pwd, input string) ([]Item, string, error) {

	var (
		status    []Item
		importing bool
		output    []byte
		pos       int
		last      *Item
	)

	lex := New(func(lex *Lexer) StateFn {
		return lex.Action()
	}, input)

	for {
		item := lex.Next()
		err := item.Error()
		//fmt.Println(item.Type, item.Value)
		if err != nil {
			return nil, string(output),
				fmt.Errorf("Error: %v (pos %d)", err, item.Pos)
		}
		switch item.Type {
		case ItemEOF:
			output = append(output, input[pos:]...)
			return status, string(output), nil
		case IMPORT:
			output = append(output, input[pos:item.Pos]...)
			last = item
			importing = true
		case INCLUDE, CMT:
			output = append(output, input[pos:item.Pos]...)
			pos = item.Pos
			status = append(status, *item)
		default:
			if importing {

				pwd, contents, err := p.ImportPath(pwd, fmt.Sprintf("%s", *item))
				if err != nil {
					return nil, "", err
				}
				//Eat the semicolon
				item := lex.Next()
				pos = item.Pos + len(item.Value)
				if item.Type != SEMIC {
					panic("@import statement must be followed by ;")
				}

				moreTokens, moreOutput, err := p.GetItems(
					pwd,
					contents)
				// If importing was successful, each token must be moved forward
				// by the position of the @import call that made it available.
				for i, _ := range moreTokens {
					moreTokens[i].Pos += last.Pos
				}

				if err != nil {
					panic(err)
				}
				output = append(output, moreOutput...)
				status = append(status, moreTokens...)
				importing = false
			} else {
				output = append(output, input[pos:item.Pos]...)
				pos = item.Pos
				status = append(status, *item)
			}
		}
	}
}
<commit_msg>Do not panic, just fail<commit_after>package sprite_sass

import (
	"bytes"
	"fmt"
	"io"
	"log"
)

func init() {
	log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
}

type Replace struct {
	Start, End int
	Value      []byte
}

type Parser struct {
	Idx, shift                    int
	Chop                          []Replace
	Pwd, Input                    string
	BuildDir, ImageDir, GenImgDir string
	Includes                      []string
	Items                         []Item
	Output                        []byte
	InlineImgs, Sprites           map[string]ImageList
	Vars                          map[string]string
}

func NewParser() *Parser {
	return &Parser{}
}

// Parser reads the tokens from the lexer and performs
// conversions and/or substitutions for sprite*() calls.
//
// Parser creates a map of all variables and sprites
// (created via sprite-map calls).
func (p *Parser) Start(in io.Reader, pkgdir string) ([]byte, error) {
	p.Vars = make(map[string]string)
	p.Sprites = make(map[string]ImageList)
	p.InlineImgs = make(map[string]ImageList)
	if p.ImageDir == "" {
		p.ImageDir = pkgdir
	}
	buf := bytes.NewBuffer(make([]byte, 0, bytes.MinRead))
	buf.ReadFrom(in)

	// This pass resolves all the imports, but positions will
	// be off due to @import calls
	items, input, err := p.GetItems(pkgdir, string(buf.Bytes()))
	if err != nil {
		return []byte(""), err
	}
	// This call will have valid token positions
	items, input, err = p.GetItems(pkgdir, input)

	p.Input = input
	p.Items = items
	if err != nil {
		panic(err)
	}
	// DEBUG
	// for _, item := range p.Items {
	// 	fmt.Printf("%s %s\n", item.Type, item)
	// }
	p.Parse(p.Items)

	p.Output = []byte(p.Input)
	p.Replace()
	// fmt.Printf("out: % #v\n", p.Sprites)
	return p.Output, nil
}

// Find Paren that matches the current (
func RParen(items []Item) (int, int) {
	if len(items) == 0 {
		return 0, 0
	}
	if items[0].Type != LPAREN {
		panic("Expected: ( was: " + items[0].Value)
	}
	pos := 1
	match := 1
	nest := false
	nestPos := 0

	for match != 0 && pos < len(items) {
		switch items[pos].Type {
		case LPAREN:
			match++
		case RPAREN:
			match--
		}
		if match > 1 {
			if !nest {
				nestPos = pos
			}
			// Nested command must be resolved
			nest = true
		}
		pos++
	}

	return pos, nestPos
}

func RBracket(items []Item, pos int) (int, int) {
	if items[pos].Type != LBRACKET && items[pos].Type != INTP {
		panic("Expected: { was: " + items[0].Value)
	}

	// Move to next item and set match to 1
	pos++
	match := 1
	nest := false
	nestPos := 0
	for match != 0 && pos < len(items) {
		switch items[pos].Type {
		case LBRACKET, INTP:
			match++
		case RBRACKET:
			match--
		}
		if match > 1 {
			if !nest {
				nestPos = pos
			}
			// Nested command must be resolved
			nest = true
		}
		pos++
	}
	return pos, nestPos
}

func (p *Parser) Parse(items []Item) []byte {
	var (
		out []byte
		eoc int
	)
	_ = eoc
	if len(items) == 0 {
		return []byte("")
	}
	j := 1
	item := items[0]
	switch item.Type {
	case VAR:
		if items[1].Value != ":" {
			log.Fatal(": expected after variable declaration")
		}
		for j < len(items) && items[j].Type != SEMIC {
			j++
		}
		// Eliminate variables for known commands
		switch items[2].Value {
		case "sprite-file":
			p.Mark(item.Pos, items[j].Pos+len(items[j].Value), "")
		}
		if items[2].Type != CMDVAR {
			// Hackery for empty sass maps
			val := string(p.Parse(items[2:j]))
			// TODO: $var: $anothervar doesnt work
			// setting other things like $var: darken(#123, 10%)
			if val != "()" && val != "" {
				// fmt.Println("SETTING", item, val)
				p.Vars[item.String()] = val
			}
		} else if items[2].Value == "sprite-map" {
			// Special parsing of sprite-maps
			p.Mark(items[0].Pos,
				items[j].Pos+len(items[j].Value), "")
			imgs := ImageList{
				ImageDir:  p.ImageDir,
				BuildDir:  p.BuildDir,
				GenImgDir: p.GenImgDir,
			}
			name := fmt.Sprintf("%s", items[0])
			glob := fmt.Sprintf("%s", items[4])
			imgs.Decode(glob)
			imgs.Vertical = true
			imgs.Combine()
			p.Sprites[name] = imgs
			//TODO: Generate filename
			_, err := imgs.Export()
			if err != nil {
				log.Printf("Failed to save sprite: %s", name)
				log.Println(err)
			}
		}
	case INTP:
		pos, _ := RBracket(items, 0)
		j = pos
		val := items[1].Value
		if val, ok := p.Vars[val]; ok {
			p.Mark(item.Pos, items[2].Pos+len(items[2].Value), val)
		}
	case SUB:
		break
		val, ok := p.Vars[item.Value]
		// Do not replace if nothing was found
		if !ok {
			val = item.Value
		}
		_ = val
		//p.Mark(item.Pos, item.Pos+len(item.Value), val)
	case CMD:
		for j < len(items) && items[j].Type != SEMIC {
			j++
		}
		out, eoc = p.Command(items[0:j])
	case TEXT:
		out = append(out, item.Value...)
	case MIXIN, FUNC, IF, ELSE, EACH:
		// Ignore the entire mixin and move to the next line
		lpos := 0
		for {
			if items[lpos].Type == LBRACKET {
				break
			}
			lpos++
		}
		pos, _ := RBracket(items, lpos)
		for i := 0; i < pos; i++ {
			out = append(out, items[i].Value...)
		}
		// fmt.Println(">>", item.Type, items[lpos:pos], "<<")
		j = pos
	default:
		if item.Type == INCLUDE {
			// Eat @include if command after is understood
			if Lookup(items[1].Value) > -1 {
				p.Mark(item.Pos, items[1].Pos, "")
			}
		}
		out = append(out, item.Value...)
	}

	return append(out, p.Parse(items[j:])...)
}

// Passed sass-command( args...)
func (p *Parser) Command(items []Item) ([]byte, int) {

	i := 0
	_ = i
	cmd := items[0]
	repl := ""
	if len(items) == 0 {
		panic(items)
	}
	eoc, nPos := RParen(items[1:])
	// Determine our offset from the source items
	if false && nPos != 0 {
		rightPos, _ := RParen(items[nPos:])
		p.Command(items[nPos:rightPos])
	}

	switch cmd.Value {
	case "sprite":
		//Capture sprite
		sprite := p.Sprites[fmt.Sprintf("%s", items[2])]
		pos, _ := RParen(items[1:])
		//Capture filename
		name := fmt.Sprintf("%s", items[3])
		repl = sprite.CSS(name)
		p.Mark(items[0].Pos, items[pos].Pos+len(items[pos].Value), repl)
	case "sprite-height":
		sprite := p.Sprites[fmt.Sprintf("%s", items[2])]
		repl = fmt.Sprintf("%dpx", sprite.SImageHeight(items[3].String()))
		p.Mark(cmd.Pos, items[eoc].Pos+len(items[eoc].Value), repl)
	case "sprite-width":
		sprite := p.Sprites[fmt.Sprintf("%s", items[2])]
		repl = fmt.Sprintf("%dpx",
			sprite.SImageWidth(items[3].String()))
		p.Mark(cmd.Pos, items[eoc].Pos+len(items[eoc].Value), repl)
	case "sprite-dimensions":
		sprite := p.Sprites[fmt.Sprintf("%s", items[2])]
		repl = sprite.Dimensions(items[3].Value)
		p.Mark(items[0].Pos, items[4].Pos+len(items[4].Value), repl)
	case "sprite-file":
		if items[2].Type != SUB {
			log.Fatalf("%s must be followed by variable, was: %s",
				cmd.Value, items[2].Value)
		}
		if items[3].Type != FILE {
			log.Fatalf("sprite-file must be followed by "+
				"sprite-variable, was: %s",
				items[3].Type)
		}
		repl := p.Sprites[fmt.Sprintf("%s", items[2])].
			File(items[3].String())
		p.Mark(items[0].Pos, items[4].Pos+len(items[4].Value), repl)
		return []byte(repl), eoc
	case "image-height", "image-width":
		if items[2].Type == FILE {
			name := items[2].Value
			img := ImageList{
				ImageDir:  p.ImageDir,
				BuildDir:  p.BuildDir,
				GenImgDir: p.GenImgDir,
			}
			img.Decode(name)
			var d int
			if cmd.Value == "image-width" {
				d = img.ImageWidth(0)
			} else if cmd.Value == "image-height" {
				d = img.ImageHeight(0)
			}
			repl = fmt.Sprintf("%dpx", d)
			p.Mark(items[0].Pos, items[3].Pos+len(items[3].Value), repl)
			return []byte(repl), eoc
		}
		if items[2].Type != CMD {
			log.Fatalf("%s first arg must be sprite-file, was: %s",
				cmd.Value, items[2].Value)
		}
		if items[4].Type != SUB {
			log.Fatalf("%s must be followed by variable, was: %s",
				cmd.Value, items[4].Type)
		}
		// Resolve variable
		sprite := p.Sprites[items[4].Value]
		var pix int
		if cmd.Value == "image-width" {
			pix = sprite.SImageWidth(items[5].Value)
		} else if cmd.Value == "image-height" {
			pix = sprite.SImageHeight(items[5].Value)
		}
		repl := fmt.Sprintf("%dpx", pix)
		p.Mark(items[0].Pos, items[7].Pos+len(items[6].Value), repl)
	case "inline-image":
		var (
			img ImageList
			ok  bool
		)
		name := fmt.Sprintf("%s", items[2])
		if img, ok = p.InlineImgs[name]; !ok {
			img = ImageList{
				ImageDir:  p.ImageDir,
				BuildDir:  p.BuildDir,
				GenImgDir: p.GenImgDir,
			}
			img.Decode(name)
			img.Combine()
			_, err := img.Export()
			if err != nil {
				log.Printf("Failed to save sprite: %s", name)
				log.Println(err)
			}
			p.InlineImgs[name] = img
		}

		repl := img.Inline()
		p.Mark(items[0].Pos, items[3].Pos+len(items[3].Value), repl)
	case "image-url":
		repl := p.ImageUrl(items)
		p.Mark(items[0].Pos, items[3].Pos+len(items[3].Value), repl)
	default:
		fmt.Println("No comprende:", items[0])
	}

	return []byte(""), eoc
}

// Import recursively resolves all imports.  It lexes the input
// adding the tokens to the Parser object.
// TODO: Convert this to byte slice in/out
func (p *Parser) GetItems(pwd, input string) ([]Item, string, error) {

	var (
		status    []Item
		importing bool
		output    []byte
		pos       int
		last      *Item
	)

	lex := New(func(lex *Lexer) StateFn {
		return lex.Action()
	}, input)

	for {
		item := lex.Next()
		err := item.Error()
		//fmt.Println(item.Type, item.Value)
		if err != nil {
			return nil, string(output),
				fmt.Errorf("Error: %v (pos %d)", err, item.Pos)
		}
		switch item.Type {
		case ItemEOF:
			output = append(output, input[pos:]...)
			return status, string(output), nil
		case IMPORT:
			output = append(output, input[pos:item.Pos]...)
			last = item
			importing = true
		case INCLUDE, CMT:
			output = append(output, input[pos:item.Pos]...)
			pos = item.Pos
			status = append(status, *item)
		default:
			if importing {

				pwd, contents, err := p.ImportPath(pwd, fmt.Sprintf("%s", *item))
				if err != nil {
					return nil, "", err
				}
				//Eat the semicolon
				item := lex.Next()
				pos = item.Pos + len(item.Value)
				if item.Type != SEMIC {
					panic("@import statement must be followed by ;")
				}

				moreTokens, moreOutput, err := p.GetItems(
					pwd,
					contents)
				// If importing was successful, each token must be moved forward
				// by the position of the @import call that made it available.
				for i, _ := range moreTokens {
					moreTokens[i].Pos += last.Pos
				}

				if err != nil {
					return nil, "", err
				}
				output = append(output, moreOutput...)
				status = append(status, moreTokens...)
				importing = false
			} else {
				output = append(output, input[pos:item.Pos]...)
				pos = item.Pos
				status = append(status, *item)
			}
		}
	}
}
<|endoftext|>


<commit_before>package plugin

// Repo represents a version control repository.
type Repo struct {
	Owner       string `json:"owner"`
	Name        string `json:"name"`
	FullName    string `json:"full_name"`
	Avatar      string `json:"avatar_url"`
	Link        string `json:"link_url"`
	Clone       string `json:"clone_url"`
	Branch      string `json:"default_branch"`
	Timeout     int64  `json:"timeout"`
	IsPrivate   bool   `json:"private"`
	IsTrusted   bool   `json:"trusted"`
	AllowPull   bool   `json:"allow_pr"`
	AllowPush   bool   `json:"allow_push"`
	AllowDeploy bool   `json:"allow_deploys"`
	AllowTag    bool   `json:"allow_tags"`
}

// System provides important information about the Drone
// server to the plugin.
type System struct {
	Version string   `json:"version"`
	Link    string   `json:"link_url"`
	Plugins []string `json:"plugins"`
	Globals []string `json:"globals"`
}

// Workspace defines the build's workspace inside the
// container. This helps the plugin locate the source
// code directory.
type Workspace struct {
	Root string `json:"root"`
	Path string `json:"path"`

	Netrc *Netrc   `json:"netrc"`
	Keys  *Keypair `json:"keys"`
}

// Keypair represents an RSA public and private key assigned to a
// repository. It may be used to clone private repositories, or as
// a deployment key.
type Keypair struct {
	Public  string `json:"public"`
	Private string `json:"private"`
}

// Netrc defines a default .netrc file that should be injected
// into the build environment. It will be used to authorize access
// to https resources, such as git+https clones.
type Netrc struct {
	Machine  string `json:"machine"`
	Login    string `json:"login"`
	Password string `json:"user"`
}

// Build represents the process of compiling and testing a changeset,
// typically triggered by the remote system (ie GitHub).
type Build struct {
	Number    int    `json:"number"`
	Event     string `json:"event"`
	Status    string `json:"status"`
	Created   int64  `json:"created_at"`
	Started   int64  `json:"started_at"`
	Finished  int64  `json:"finished_at"`
	Commit    string `json:"commit"`
	Branch    string `json:"branch"`
	Ref       string `json:"ref"`
	Refspec   string `json:"refspec"`
	Remote    string `json:"remote"`
	Title     string `json:"title"`
	Message   string `json:"message"`
	Timestamp string `json:"timestamp"`
	Author    string `json:"author"`
	Avatar    string `json:"author_avatar"`
	Email     string `json:"author_email"`
	Link      string `json:"link_url"`
}

// Job represents a single job that is being executed as part
// of a Build.
type Job struct {
	ID       int64  `json:"id"`
	Number   int    `json:"number"`
	Status   string `json:"status"`
	ExitCode int    `json:"exit_code"`
	Started  int64  `json:"started_at"`
	Finished int64  `json:"finished_at"`

	Environment map[string]string `json:"environment"`
}
<commit_msg>fixed timestamp type<commit_after>package plugin

// Repo represents a version control repository.
type Repo struct {
	Owner       string `json:"owner"`
	Name        string `json:"name"`
	FullName    string `json:"full_name"`
	Avatar      string `json:"avatar_url"`
	Link        string `json:"link_url"`
	Clone       string `json:"clone_url"`
	Branch      string `json:"default_branch"`
	Timeout     int64  `json:"timeout"`
	IsPrivate   bool   `json:"private"`
	IsTrusted   bool   `json:"trusted"`
	AllowPull   bool   `json:"allow_pr"`
	AllowPush   bool   `json:"allow_push"`
	AllowDeploy bool   `json:"allow_deploys"`
	AllowTag    bool   `json:"allow_tags"`
}

// System provides important information about the Drone
// server to the plugin.
type System struct {
	Version string   `json:"version"`
	Link    string   `json:"link_url"`
	Plugins []string `json:"plugins"`
	Globals []string `json:"globals"`
}

// Workspace defines the build's workspace inside the
// container. This helps the plugin locate the source
// code directory.
type Workspace struct {
	Root string `json:"root"`
	Path string `json:"path"`

	Netrc *Netrc   `json:"netrc"`
	Keys  *Keypair `json:"keys"`
}

// Keypair represents an RSA public and private key assigned to a
// repository. It may be used to clone private repositories, or as
// a deployment key.
type Keypair struct {
	Public  string `json:"public"`
	Private string `json:"private"`
}

// Netrc defines a default .netrc file that should be injected
// into the build environment. It will be used to authorize access
// to https resources, such as git+https clones.
type Netrc struct {
	Machine  string `json:"machine"`
	Login    string `json:"login"`
	Password string `json:"user"`
}

// Build represents the process of compiling and testing a changeset,
// typically triggered by the remote system (ie GitHub).
type Build struct {
	Number    int    `json:"number"`
	Event     string `json:"event"`
	Status    string `json:"status"`
	Enqueued  int64  `json:"enqueued_at"`
	Created   int64  `json:"created_at"`
	Started   int64  `json:"started_at"`
	Finished  int64  `json:"finished_at"`
	Commit    string `json:"commit"`
	Branch    string `json:"branch"`
	Ref       string `json:"ref"`
	Refspec   string `json:"refspec"`
	Remote    string `json:"remote"`
	Title     string `json:"title"`
	Message   string `json:"message"`
	Timestamp int64  `json:"timestamp"`
	Author    string `json:"author"`
	Avatar    string `json:"author_avatar"`
	Email     string `json:"author_email"`
	Link      string `json:"link_url"`
}

// Job represents a single job that is being executed as part
// of a Build.
type Job struct {
	ID       int64  `json:"id"`
	Number   int    `json:"number"`
	Status   string `json:"status"`
	ExitCode int    `json:"exit_code"`
	Enqueued int64  `json:"enqueued_at"`
	Started  int64  `json:"started_at"`
	Finished int64  `json:"finished_at"`

	Environment map[string]string `json:"environment"`
}
<|endoftext|>