<commit_before>// Copyright 2012 Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"
)

const (
	defaultKeepAlive = "5m"
)

var (
	// End of stream (or scan)
	EOS = errors.New("EOS")

	// No ScrollId
	ErrNoScrollId = errors.New("elastic: No scrollId")
)

// ScanService manages a cursor through documents in ElasticSearch.
type ScanService struct {
	client     *Client
	indices    []string
	types      []string
	keepAlive  string
	query      Query
	size       *int
	pretty     bool
	debug      bool
}

func NewScanService(client *Client) *ScanService {
	builder := &ScanService{
		client:  client,
		query:   NewMatchAllQuery(),
		debug:   false,
		pretty:  false,
	}
	return builder
}

func (s *ScanService) Index(index string) *ScanService {
	if s.indices == nil {
		s.indices = make([]string, 0)
	}
	s.indices = append(s.indices, index)
	return s
}

func (s *ScanService) Indices(indices ...string) *ScanService {
	if s.indices == nil {
		s.indices = make([]string, 0)
	}
	s.indices = append(s.indices, indices...)
	return s
}

func (s *ScanService) Type(typ string) *ScanService {
	if s.types == nil {
		s.types= make([]string, 0)
	}
	s.types = append(s.types, typ)
	return s
}

func (s *ScanService) Types(types...string) *ScanService {
	if s.types == nil {
		s.types= make([]string, 0)
	}
	s.types = append(s.types, types...)
	return s
}

// Scroll is an alias for KeepAlive, the time to keep
// the cursor alive (e.g. "5m" for 5 minutes).
func (s *ScanService) Scroll(keepAlive string) *ScanService {
	s.keepAlive = keepAlive
	return s
}

// KeepAlive sets the maximum time the cursor will be
// available before expiration (e.g. "5m" for 5 minutes).
func (s *ScanService) KeepAlive(keepAlive string) *ScanService {
	s.keepAlive = keepAlive
	return s
}

func (s *ScanService) Query(query Query) *ScanService {
	s.query = query
	return s
}

func (s *ScanService) Pretty(pretty bool) *ScanService {
	s.pretty = pretty
	return s
}

func (s *ScanService) Debug(debug bool) *ScanService {
	s.debug = debug
	return s
}

func (s *ScanService) Size(size int) *ScanService {
	s.size = &size
	return s
}

func (s *ScanService) Do() (*scanCursor, error) {
	// Build url
	urls := "/"

	// Indices part
	indexPart := make([]string, 0)
	for _, index := range s.indices {
		indexPart = append(indexPart, cleanPathString(index))
	}
	urls += strings.Join(indexPart, ",")

	// Types
	typesPart := make([]string, 0)
	for _, typ := range s.types {
		typesPart = append(typesPart, cleanPathString(typ))
	}
	if len(typesPart) > 0 {
		urls += "/"
		urls += strings.Join(typesPart, ",")
	}

	// Search
	urls += "/_search"

	// Parameters
	params := make(url.Values)
	params.Set("search_type", "scan")
	if s.pretty {
		params.Set("pretty", fmt.Sprintf("%v", s.pretty))
	}
	if s.keepAlive != "" {
		params.Set("scroll", s.keepAlive)
	} else {
		params.Set("scroll", defaultKeepAlive)
	}
	if s.size != nil && *s.size > 0 {
		params.Set("size", fmt.Sprintf("%d", *s.size))
	}
	urls += "?" + params.Encode()

	// Set up a new request
	req, err := s.client.NewRequest("POST", urls)
	if err != nil {
		return nil, err
	}

	// Set body
	body := make(map[string]interface{})

	// Query
	if s.query != nil {
		body["query"] = s.query.Source()
	}

	req.SetBodyJson(body)

	if s.debug {
		out, _ := httputil.DumpRequestOut((*http.Request)(req), true)
		fmt.Printf("%s\n", string(out))
	}

	// Get response
	res, err := s.client.c.Do((*http.Request)(req))
	if err != nil {
		return nil, err
	}
	if err := checkResponse(res); err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if s.debug {
		out, _ := httputil.DumpResponse(res, true)
		fmt.Printf("%s\n", string(out))
	}

	searchResult := new(SearchResult)
	if err := json.NewDecoder(res.Body).Decode(searchResult); err != nil {
		return nil, err
	}

	cursor := newScanCursor(s.client, s.keepAlive, s.pretty, s.debug, searchResult)

	return cursor, nil
}

// scanCursor represents a single page of results from
// an Elasticsearch Scan operation.
type scanCursor struct {
	Results      *SearchResult

	client       *Client
	keepAlive    string
	pretty       bool
	debug        bool
	currentPage  int
}

// newScanCursor returns a new initialized instance
// of scanCursor.
func newScanCursor(client *Client, keepAlive string, pretty, debug bool, searchResult *SearchResult) *scanCursor {
	return &scanCursor{
		client: client,
		keepAlive: keepAlive,
		pretty: pretty,
		debug: debug,
		Results: searchResult,
	}
}

// TotalHits is a convenience method that returns the number
// of hits the cursor will iterate through.
func (c *scanCursor) TotalHits() int64 {
	if c.Results.Hits == nil {
		return 0
	}
	return c.Results.Hits.TotalHits
}

// Next returns the next search result or nil when all
// documents have been scanned.
//
// Usage:
//
//   for {
//     res, err := cursor.Next()
//     if err == elastic.EOS {
//       // End of stream (or scan)
//       break
//     }
//     if err != nil {
//       // Handle error
//     }
//     // Work with res
//   }
//
func (c *scanCursor) Next() (*SearchResult, error) {
	if c.currentPage > 0 {
		if c.Results.Hits == nil || len(c.Results.Hits.Hits) == 0 || c.Results.Hits.TotalHits == 0 {
			return nil, EOS
		}
	}
	if c.Results.ScrollId == "" {
		return nil, ErrNoScrollId
	}

	// Build url
	urls := "/_search/scroll"

	// Parameters
	params := make(url.Values)
	if c.pretty {
		params.Set("pretty", fmt.Sprintf("%v", c.pretty))
	}
	if c.keepAlive != "" {
		params.Set("scroll", c.keepAlive)
	} else {
		params.Set("scroll", defaultKeepAlive)
	}
	urls += "?" + params.Encode()

	// Set up a new request
	req, err := c.client.NewRequest("POST", urls)
	if err != nil {
		return nil, err
	}

	// Set body
	req.SetBodyString(c.Results.ScrollId)

	if c.debug {
		out, _ := httputil.DumpRequestOut((*http.Request)(req), true)
		log.Printf("%s\n", string(out))
	}

	// Get response
	res, err := c.client.c.Do((*http.Request)(req))
	if err != nil {
		return nil, err
	}
	if err := checkResponse(res); err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if c.debug {
		out, _ := httputil.DumpResponse(res, true)
		log.Printf("%s\n", string(out))
	}

	if err := json.NewDecoder(res.Body).Decode(c.Results); err != nil {
		return nil, err
	}

	c.currentPage += 1

	return c.Results, nil
}

<commit_msg>Open ScanCursor class<commit_after>// Copyright 2012 Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"
)

const (
	defaultKeepAlive = "5m"
)

var (
	// End of stream (or scan)
	EOS = errors.New("EOS")

	// No ScrollId
	ErrNoScrollId = errors.New("elastic: No scrollId")
)

// ScanService manages a cursor through documents in ElasticSearch.
type ScanService struct {
	client    *Client
	indices   []string
	types     []string
	keepAlive string
	query     Query
	size      *int
	pretty    bool
	debug     bool
}

func NewScanService(client *Client) *ScanService {
	builder := &ScanService{
		client: client,
		query:  NewMatchAllQuery(),
		debug:  false,
		pretty: false,
	}
	return builder
}

func (s *ScanService) Index(index string) *ScanService {
	if s.indices == nil {
		s.indices = make([]string, 0)
	}
	s.indices = append(s.indices, index)
	return s
}

func (s *ScanService) Indices(indices ...string) *ScanService {
	if s.indices == nil {
		s.indices = make([]string, 0)
	}
	s.indices = append(s.indices, indices...)
	return s
}

func (s *ScanService) Type(typ string) *ScanService {
	if s.types == nil {
		s.types = make([]string, 0)
	}
	s.types = append(s.types, typ)
	return s
}

func (s *ScanService) Types(types ...string) *ScanService {
	if s.types == nil {
		s.types = make([]string, 0)
	}
	s.types = append(s.types, types...)
	return s
}

// Scroll is an alias for KeepAlive, the time to keep
// the cursor alive (e.g. "5m" for 5 minutes).
func (s *ScanService) Scroll(keepAlive string) *ScanService {
	s.keepAlive = keepAlive
	return s
}

// KeepAlive sets the maximum time the cursor will be
// available before expiration (e.g. "5m" for 5 minutes).
func (s *ScanService) KeepAlive(keepAlive string) *ScanService {
	s.keepAlive = keepAlive
	return s
}

func (s *ScanService) Query(query Query) *ScanService {
	s.query = query
	return s
}

func (s *ScanService) Pretty(pretty bool) *ScanService {
	s.pretty = pretty
	return s
}

func (s *ScanService) Debug(debug bool) *ScanService {
	s.debug = debug
	return s
}

func (s *ScanService) Size(size int) *ScanService {
	s.size = &size
	return s
}

func (s *ScanService) Do() (*ScanCursor, error) {
	// Build url
	urls := "/"

	// Indices part
	indexPart := make([]string, 0)
	for _, index := range s.indices {
		indexPart = append(indexPart, cleanPathString(index))
	}
	urls += strings.Join(indexPart, ",")

	// Types
	typesPart := make([]string, 0)
	for _, typ := range s.types {
		typesPart = append(typesPart, cleanPathString(typ))
	}
	if len(typesPart) > 0 {
		urls += "/"
		urls += strings.Join(typesPart, ",")
	}

	// Search
	urls += "/_search"

	// Parameters
	params := make(url.Values)
	params.Set("search_type", "scan")
	if s.pretty {
		params.Set("pretty", fmt.Sprintf("%v", s.pretty))
	}
	if s.keepAlive != "" {
		params.Set("scroll", s.keepAlive)
	} else {
		params.Set("scroll", defaultKeepAlive)
	}
	if s.size != nil && *s.size > 0 {
		params.Set("size", fmt.Sprintf("%d", *s.size))
	}
	urls += "?" + params.Encode()

	// Set up a new request
	req, err := s.client.NewRequest("POST", urls)
	if err != nil {
		return nil, err
	}

	// Set body
	body := make(map[string]interface{})

	// Query
	if s.query != nil {
		body["query"] = s.query.Source()
	}

	req.SetBodyJson(body)

	if s.debug {
		out, _ := httputil.DumpRequestOut((*http.Request)(req), true)
		fmt.Printf("%s\n", string(out))
	}

	// Get response
	res, err := s.client.c.Do((*http.Request)(req))
	if err != nil {
		return nil, err
	}
	if err := checkResponse(res); err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if s.debug {
		out, _ := httputil.DumpResponse(res, true)
		fmt.Printf("%s\n", string(out))
	}

	searchResult := new(SearchResult)
	if err := json.NewDecoder(res.Body).Decode(searchResult); err != nil {
		return nil, err
	}

	cursor := NewScanCursor(s.client, s.keepAlive, s.pretty, s.debug, searchResult)

	return cursor, nil
}

// scanCursor represents a single page of results from
// an Elasticsearch Scan operation.
type ScanCursor struct {
	Results *SearchResult

	client      *Client
	keepAlive   string
	pretty      bool
	debug       bool
	currentPage int
}

// newScanCursor returns a new initialized instance
// of scanCursor.
func NewScanCursor(client *Client, keepAlive string, pretty, debug bool, searchResult *SearchResult) *ScanCursor {
	return &ScanCursor{
		client:    client,
		keepAlive: keepAlive,
		pretty:    pretty,
		debug:     debug,
		Results:   searchResult,
	}
}

// TotalHits is a convenience method that returns the number
// of hits the cursor will iterate through.
func (c *ScanCursor) TotalHits() int64 {
	if c.Results.Hits == nil {
		return 0
	}
	return c.Results.Hits.TotalHits
}

// Next returns the next search result or nil when all
// documents have been scanned.
//
// Usage:
//
//   for {
//     res, err := cursor.Next()
//     if err == elastic.EOS {
//       // End of stream (or scan)
//       break
//     }
//     if err != nil {
//       // Handle error
//     }
//     // Work with res
//   }
//
func (c *ScanCursor) Next() (*SearchResult, error) {
	if c.currentPage > 0 {
		if c.Results.Hits == nil || len(c.Results.Hits.Hits) == 0 || c.Results.Hits.TotalHits == 0 {
			return nil, EOS
		}
	}
	if c.Results.ScrollId == "" {
		return nil, ErrNoScrollId
	}

	// Build url
	urls := "/_search/scroll"

	// Parameters
	params := make(url.Values)
	if c.pretty {
		params.Set("pretty", fmt.Sprintf("%v", c.pretty))
	}
	if c.keepAlive != "" {
		params.Set("scroll", c.keepAlive)
	} else {
		params.Set("scroll", defaultKeepAlive)
	}
	urls += "?" + params.Encode()

	// Set up a new request
	req, err := c.client.NewRequest("POST", urls)
	if err != nil {
		return nil, err
	}

	// Set body
	req.SetBodyString(c.Results.ScrollId)

	if c.debug {
		out, _ := httputil.DumpRequestOut((*http.Request)(req), true)
		log.Printf("%s\n", string(out))
	}

	// Get response
	res, err := c.client.c.Do((*http.Request)(req))
	if err != nil {
		return nil, err
	}
	if err := checkResponse(res); err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if c.debug {
		out, _ := httputil.DumpResponse(res, true)
		log.Printf("%s\n", string(out))
	}

	if err := json.NewDecoder(res.Body).Decode(c.Results); err != nil {
		return nil, err
	}

	c.currentPage += 1

	return c.Results, nil
}
<|endoftext|>


<commit_before>// written by Daniel Oaks <daniel@danieloaks.net>
// released under the ISC license

package ircmap

import (
	"errors"
	"strings"

	"golang.org/x/text/secure/precis"

	"github.com/DanielOaks/go-idn/idna2003/stringprep"
)

// MappingType values represent the types of IRC casemapping we support.
type MappingType int

const (
	// NONE represents no casemapping.
	NONE MappingType = 0 + iota

	// ASCII represents the traditional "ascii" casemapping.
	ASCII

	// RFC1459 represents the casemapping defined by "rfc1459"
	RFC1459

	// RFC3454 represents the UTF-8 nameprep casefolding as used by mammon-ircd.
	RFC3454

	// RFC7613 represents the UTF-8 casefolding currently being drafted by me
	// with the IRCv3 WG.
	RFC7613
)

var (
	// Mappings is a mapping of ISUPPORT CASEMAP strings to our MappingTypes.
	Mappings = map[string]MappingType{
		"ascii":   ASCII,
		"rfc1459": RFC1459,
		"rfc3454": RFC3454,
		"rfc7613": RFC7613,
	}
)

// ChannelPrefixes are the allowed prefixes for channels, used in casefolding.
var ChannelPrefixes = map[byte]bool{
	'#': true,
	'&': true,
}

// rfc1459Fold casefolds only the special chars defined by RFC1459 -- the
// others are handled by the strings.ToLower earlier.
func rfc1459Fold(r rune) rune {
	if '[' <= r && r <= ']' {
		r += '{' - '['
	}
	return r
}

var (
	// ErrCouldNotStabilize indicates that we could not stabilize the input string.
	ErrCouldNotStabilize = errors.New("Could not stabilize string while casefolding")
)

// Each pass of PRECIS casefolding is a composition of idempotent operations,
// but not idempotent itself. Therefore, the spec says "do it four times and hope
// it converges" (lolwtf). Golang's PRECIS implementation has a "repeat" option,
// which provides this functionality, but unfortunately it's not exposed publicly.
func iterateFolding(profile *precis.Profile, oldStr string) (str string, err error) {
	str = oldStr
	// follow the stabilizing rules laid out here:
	// https://tools.ietf.org/html/draft-ietf-precis-7564bis-10.html#section-7
	for i := 0; i < 4; i++ {
		str, err = profile.CompareKey(str)
		if err != nil {
			return "", err
		}
		if oldStr == str {
			break
		}
		oldStr = str
	}
	if oldStr != str {
		return "", ErrCouldNotStabilize
	}
	return str, nil
}

// PrecisCasefold returns a casefolded string, without doing any name or channel character checks.
func PrecisCasefold(str string) (string, error) {
	return iterateFolding(precis.UsernameCaseMapped, str)
}

// Casefold returns a string, lowercased/casefolded according to the given
// mapping as defined by this package (or an error if the given string is not
// valid in the chosen mapping).
func Casefold(mapping MappingType, input string) (string, error) {
	var out string
	var err error

	if mapping == ASCII || mapping == RFC1459 {
		// strings.ToLower ONLY replaces a-z, no unicode stuff so we're safe
		// to use that here without any issues.
		out = strings.ToLower(input)

		if mapping == RFC1459 {
			out = strings.Map(rfc1459Fold, out)
		}
	} else if mapping == RFC3454 {
		out, err = stringprep.Nameprep(input)
	} else if mapping == RFC7613 {
		// skip channel prefixes to avoid bidi rule (as per spec)
		var start int
		for start = 0; start < len(input) && ChannelPrefixes[input[start]]; start++ {
		}

		lowered, err := PrecisCasefold(input[start:])
		if err != nil {
			return "", err
		}

		return input[:start] + lowered, err
	}

	return out, err
}
<commit_msg>ircmap: Add a few more chan prefixes<commit_after>// written by Daniel Oaks <daniel@danieloaks.net>
// released under the ISC license

package ircmap

import (
	"errors"
	"strings"

	"golang.org/x/text/secure/precis"

	"github.com/DanielOaks/go-idn/idna2003/stringprep"
)

// MappingType values represent the types of IRC casemapping we support.
type MappingType int

const (
	// NONE represents no casemapping.
	NONE MappingType = 0 + iota

	// ASCII represents the traditional "ascii" casemapping.
	ASCII

	// RFC1459 represents the casemapping defined by "rfc1459"
	RFC1459

	// RFC3454 represents the UTF-8 nameprep casefolding as used by mammon-ircd.
	RFC3454

	// RFC7613 represents the UTF-8 casefolding currently being drafted by me
	// with the IRCv3 WG.
	RFC7613
)

var (
	// Mappings is a mapping of ISUPPORT CASEMAP strings to our MappingTypes.
	Mappings = map[string]MappingType{
		"ascii":   ASCII,
		"rfc1459": RFC1459,
		"rfc3454": RFC3454,
		"rfc7613": RFC7613,
	}
)

// ChannelPrefixes are the allowed prefixes for channels, used in casefolding.
var ChannelPrefixes = map[byte]bool{
	// standard, well-used
	'#': true,
	'&': true,
	// standard, not well-used
	'!': true,
	'+': true,
	// znc uses for partylines
	'~': true,
}

// rfc1459Fold casefolds only the special chars defined by RFC1459 -- the
// others are handled by the strings.ToLower earlier.
func rfc1459Fold(r rune) rune {
	if '[' <= r && r <= ']' {
		r += '{' - '['
	}
	return r
}

var (
	// ErrCouldNotStabilize indicates that we could not stabilize the input string.
	ErrCouldNotStabilize = errors.New("Could not stabilize string while casefolding")
)

// Each pass of PRECIS casefolding is a composition of idempotent operations,
// but not idempotent itself. Therefore, the spec says "do it four times and hope
// it converges" (lolwtf). Golang's PRECIS implementation has a "repeat" option,
// which provides this functionality, but unfortunately it's not exposed publicly.
func iterateFolding(profile *precis.Profile, oldStr string) (str string, err error) {
	str = oldStr
	// follow the stabilizing rules laid out here:
	// https://tools.ietf.org/html/draft-ietf-precis-7564bis-10.html#section-7
	for i := 0; i < 4; i++ {
		str, err = profile.CompareKey(str)
		if err != nil {
			return "", err
		}
		if oldStr == str {
			break
		}
		oldStr = str
	}
	if oldStr != str {
		return "", ErrCouldNotStabilize
	}
	return str, nil
}

// PrecisCasefold returns a casefolded string, without doing any name or channel character checks.
func PrecisCasefold(str string) (string, error) {
	return iterateFolding(precis.UsernameCaseMapped, str)
}

// Casefold returns a string, lowercased/casefolded according to the given
// mapping as defined by this package (or an error if the given string is not
// valid in the chosen mapping).
func Casefold(mapping MappingType, input string) (string, error) {
	var out string
	var err error

	if mapping == ASCII || mapping == RFC1459 {
		// strings.ToLower ONLY replaces a-z, no unicode stuff so we're safe
		// to use that here without any issues.
		out = strings.ToLower(input)

		if mapping == RFC1459 {
			out = strings.Map(rfc1459Fold, out)
		}
	} else if mapping == RFC3454 {
		out, err = stringprep.Nameprep(input)
	} else if mapping == RFC7613 {
		// skip channel prefixes to avoid bidi rule (as per spec)
		var start int
		for start = 0; start < len(input) && ChannelPrefixes[input[start]]; start++ {
		}

		lowered, err := PrecisCasefold(input[start:])
		if err != nil {
			return "", err
		}

		return input[:start] + lowered, err
	}

	return out, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/atotto/clipboard"
	"github.com/google/go-github/github"
)

func main() {
	if len(os.Args) != 2 {
		fmt.Println("Usage: go run seec.go <sha1>")
		os.Exit(0)
	}
	sha1 := os.Args[1]
	client := github.NewClient(nil)
	commit, _, err := client.Git.GetCommit("git", "git", sha1)
	if err != nil {
		fmt.Printf("Unable to get commit '%s': err '%v'\n", sha1, err)
		os.Exit(1)
	}
	if len(commit.Parents) != 2 {
		fmt.Printf("Sha1 '%s' has '%d' parent(s) instead of 2\n", sha1, len(commit.Parents))
	}
	clogin := login(*commit.Author.Email, *commit.Author.Name, client)
	parent := commit.Parents[1]
	var pcommit *github.Commit
	for pcommit == nil {
		pcommit, _, err = client.Git.GetCommit("git", "git", *parent.SHA)
		if err != nil {
			fmt.Printf("Unable to get parent commit '%s': err '%v'\n", parent.SHA, err)
			os.Exit(1)
		}
		// fmt.Printf("pcommit '%+v', len %d\n", pcommit, len(pcommit.Parents))
		if len(pcommit.Parents) == 2 {
			parent = pcommit.Parents[1]
			pcommit = nil
		} else {
			break
		}
	}
	plogin := login(*pcommit.Author.Email, *pcommit.Author.Name, client)

	res := fmt.Sprintf("See [commit %s](https://github.com/git/git/commit/%s) by [%s (`%s`)](https://github.com/%s), %s.  \n",
		(*pcommit.SHA)[:7], *pcommit.SHA,
		*pcommit.Author.Name, plogin, plogin, pcommit.Author.Date.Format("02 Jan 2006"))
	res = res + fmt.Sprintf("(Merged by [%s -- `%s` --](https://github.com/%s) in [commit %s](https://github.com/git/git/commit/%s), %s)  ",
		*commit.Author.Name, clogin, clogin,
		sha1[:7], sha1, commit.Committer.Date.Format("02 Jan 2006"))
	res = collect(res, *pcommit.Message, "Helped-by:", client)
	fmt.Println(res)
	clipboard.WriteAll(res)
	fmt.Println("(Copied to the clipboard)")
}

func login(email string, name string, client *github.Client) string {
	opts := &github.SearchOptions{Order: "desc"}
	var res *github.UsersSearchResult
	var err error
	if email != "" {
		res, _, err = client.Search.Users(email, opts)
		if err != nil {
			fmt.Printf("Unable to search user '%s': err '%v'", email, err)
			os.Exit(1)
		}
	}
	if res == nil || *res.Total == 0 {
		name = strings.Replace(name, "-", " ", -1)
		res, _, err = client.Search.Users(name, opts)
		if err != nil {
			fmt.Printf("Unable to search user '%s': err '%v'", name, err)
			os.Exit(1)
		}
	}
	if res == nil || *res.Total == 0 {
		return ""
	}
	return *res.Users[0].Login
}

func collect(res, msg, activity string, client *github.Client) string {
	re := regexp.MustCompile(fmt.Sprintf(`%s:\s+([^<\r\n])+<([^>\r\n])+>`, activity))
	activitymsg := activity
	first := true
	fmt.Printf("re='%v'\n%s\n", re.String(), msg)
	for _, resc := range re.FindAllStringSubmatch(msg, -1) {
		if len(resc) != 2 {
			continue
		}
		name := resc[0]
		email := resc[1]
		login := login(email, name, client)
		if !first {
			activitymsg = activitymsg + ", "
		}
		if login == "" {
			activitymsg = activitymsg + fmt.Sprintf("%s <%s>", name, email)
			first = false
			continue
		}
		activitymsg = activitymsg + fmt.Sprintf("[%s (`%s`)](https://github.com/%s)", name, login, login)
		first = false
	}
	if !first {
		res = res + "\n" + activitymsg + "  "
	}
	return res
}
<commit_msg>seec.go: search in Issues for user GitHub account<commit_after>package main

import (
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/atotto/clipboard"
	"github.com/google/go-github/github"
)

func main() {
	if len(os.Args) != 2 {
		fmt.Println("Usage: go run seec.go <sha1>")
		os.Exit(0)
	}
	sha1 := os.Args[1]
	client := github.NewClient(nil)
	commit, _, err := client.Git.GetCommit("git", "git", sha1)
	if err != nil {
		fmt.Printf("Unable to get commit '%s': err '%v'\n", sha1, err)
		os.Exit(1)
	}
	if len(commit.Parents) != 2 {
		fmt.Printf("Sha1 '%s' has '%d' parent(s) instead of 2\n", sha1, len(commit.Parents))
	}
	clogin := login(*commit.Author.Email, *commit.Author.Name, client)
	parent := commit.Parents[1]
	var pcommit *github.Commit
	for pcommit == nil {
		pcommit, _, err = client.Git.GetCommit("git", "git", *parent.SHA)
		if err != nil {
			fmt.Printf("Unable to get parent commit '%s': err '%v'\n", parent.SHA, err)
			os.Exit(1)
		}
		// fmt.Printf("pcommit '%+v', len %d\n", pcommit, len(pcommit.Parents))
		if len(pcommit.Parents) == 2 {
			parent = pcommit.Parents[1]
			pcommit = nil
		} else {
			break
		}
	}
	plogin := login(*pcommit.Author.Email, *pcommit.Author.Name, client)

	res := fmt.Sprintf("See [commit %s](https://github.com/git/git/commit/%s) by [%s (`%s`)](https://github.com/%s), %s.  \n",
		(*pcommit.SHA)[:7], *pcommit.SHA,
		*pcommit.Author.Name, plogin, plogin, pcommit.Author.Date.Format("02 Jan 2006"))
	res = res + fmt.Sprintf("(Merged by [%s -- `%s` --](https://github.com/%s) in [commit %s](https://github.com/git/git/commit/%s), %s)  ",
		*commit.Author.Name, clogin, clogin,
		sha1[:7], sha1, commit.Committer.Date.Format("02 Jan 2006"))
	res = collect(res, *pcommit.Message, "Helped-by:", client)
	fmt.Println(res)
	clipboard.WriteAll(res)
	fmt.Println("(Copied to the clipboard)")
}

func login(email string, name string, client *github.Client) string {
	opts := &github.SearchOptions{Order: "desc"}
	var res *github.UsersSearchResult
	var err error
	if email != "" {
		res, _, err = client.Search.Users(email, opts)
		if err != nil {
			fmt.Printf("Unable to search user '%s': err '%v'", email, err)
			os.Exit(1)
		}
	}
	if res == nil || *res.Total == 0 {
		name = strings.Replace(name, "-", " ", -1)
		res, _, err = client.Search.Users(name, opts)
		if err != nil {
			fmt.Printf("Unable to search user '%s': err '%v'", name, err)
			os.Exit(1)
		}
	}
	if res == nil || *res.Total == 0 {
		var resIssues *github.IssuesSearchResult
		issueSearch := fmt.Sprintf(`"Signed-off-by: %s <%s>"`, name, email)
		resIssues, _, err = client.Search.Issues(issueSearch, opts)
		if err != nil {
			fmt.Printf("Unable to search issue '%s': err '%v'", issueSearch, err)
			os.Exit(1)
		}
		if resIssues == nil || *resIssues.Total == 0 {
			return ""
		}
		issue := resIssues.Issues[0]
		return *issue.User.Login
	}
	if res == nil || *res.Total == 0 {
		return ""
	}
	return *res.Users[0].Login
}

func collect(res, msg, activity string, client *github.Client) string {
	re := regexp.MustCompile(fmt.Sprintf(`%s:\s+([^<\r\n])+<([^>\r\n])+>`, activity))
	activitymsg := activity
	first := true
	fmt.Printf("re='%v'\n%s\n", re.String(), msg)
	for _, resc := range re.FindAllStringSubmatch(msg, -1) {
		if len(resc) != 2 {
			continue
		}
		name := resc[0]
		email := resc[1]
		login := login(email, name, client)
		if !first {
			activitymsg = activitymsg + ", "
		}
		if login == "" {
			activitymsg = activitymsg + fmt.Sprintf("%s <%s>", name, email)
			first = false
			continue
		}
		activitymsg = activitymsg + fmt.Sprintf("[%s (`%s`)](https://github.com/%s)", name, login, login)
		first = false
	}
	if !first {
		res = res + "\n" + activitymsg + "  "
	}
	return res
}
<|endoftext|>


<commit_before>package segygo

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"unsafe"
)

const Version = "0.1"
const SEGY_DESC_HDR_LEN = 3200
const SEGY_BIN_HDR_LEN = 400
const SEGY_TRACE_HDR_LEN = 240

type BinHeader struct {
	Jobid, Lino, Reno                                                                                int32
	Ntrpr, Nart                                                                                int16
	Hdt, Dto, Hns, Nso                                                                                uint16
	Format, Fold, Tsort, Vscode, Hsfs, Hsfe, Hslen, Hstyp, Schn, Hstas, Hstae, Htatyp, Hcorr, Bgrcv, Rcvm, Mfeet, Polyt, Vgpol int16
	Hunass                                                                                [170]int16 // unassigned
}

type TraceHeader struct {
	Tracel                                                               int32
	Tracer                                                               int32
	Fldr                                                                 int32
	Tracf                                                                int32
	Ep                                                                   int32
	CDP                                                                  int32
	CDPT                                                                 int32
	Trid                                                                 int16
	Nvs                                                                  int16
	Nhs                                                                  int16
	Duse                                                                 int16
	Offset                                                               int32
	Gelev                                                                int32
	Selev                                                                int32
	Sdepth                                                               int32
	Gdel                                                                 int32
	Sdel                                                                 int32
	SwDep                                                                int32
	GwDep                                                                int32
	Scalel                                                               int16
	Scalco                                                               int16
	Sx                                                                   int32
	Sy                                                                   int32
	Gx                                                                   int32
	Gy                                                                   int32
	CoUnit                                                               int16
	WeVel                                                                int16
	SweVel                                                               int16
	Sut, Gut, Sstat, Gstat, Tstat, Laga, Lagb, Delrt, Muts, Mute         int16
	Ns, Dt                                                               uint16
	Gain, Igc, Igi, Corr, Sfs, Sfe, Slen, Styp, Stas, Stae, Tatyp        int16
	Afilf, Afils, NoFilf, NoFils, Lcf, Hcf, Lcs, Hcs, Year, Day          int16
	Hour, Minute, Sec, Timbas, Trwf, Grnors, Grnofr, Grnlof, Gaps, Otrav int16
	D1, F1, D2, F2, Ungpow, Unscale                                      float32
	Ntr                                                                  int32
	Mark, Shortpad                                                       int16
	Unass                                                                [14]int16 // unassigned short array
}

type Trace struct {
	TraceHeader
	Data []float32
}

type SegyFile struct {
	Filename string
	Header   BinHeader
	NrTraces int64
	file     *os.File
}

func OpenFile(filename string) (SegyFile, error) {
	var s SegyFile
	var binHdr BinHeader
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		return s, err
	}

	s.Filename = filename

	accum := []byte{}
	accum = append(accum, b...)

	accum2 := accum[3200:]
	r := bytes.NewReader(accum2)
	fmt.Println("Number of bytes:", r.Len())

	if err = binary.Read(r, binary.BigEndian, &binHdr); err != nil {
		fmt.Println("Error reading segy file (bigendian). ", err)
		return s, err
	}

	// Open and store the os.File object in our struct
	file, err := os.Open(s.Filename)
	s.file = file

	s.Header = binHdr
	s.NrTraces = s.GetNrTraces()

	return s, err
}

func (s *SegyFile) GetNrTraces() int64 {
	fi, err := s.file.Stat()
	if err != nil {
		fmt.Println("unable to get Stat()")
		log.Fatal(err)
	}
	size := fi.Size()
	nSamples := s.Header.Hns
	txtAndBinarySize := int64(SEGY_DESC_HDR_LEN + SEGY_BIN_HDR_LEN)
	nTraces := ((size - txtAndBinarySize) / (int64(SEGY_TRACE_HDR_LEN) + int64(nSamples)*int64(unsafe.Sizeof(float32(1)))))

	return nTraces
}

func (s *SegyFile) GetNrSamples() int32 {
	return int32(s.Header.Hns)
}

func (s *SegyFile) ReadTrace() (Trace, error) {
	// First read the TraceHeader
	//data := []byte{}
	//data = append(data,
	trace := Trace{}
	traceBuff := make([]float32, s.GetNrSamples())
	byteBuff := make([]byte, s.GetNrSamples()*4)
	trace.Data = traceBuff

	trcHdrBuff := make([]byte, SEGY_TRACE_HDR_LEN)
	bytesRead, err := s.file.Read(trcHdrBuff)
	if err != nil {
		log.Fatal(err)
		return trace, err
	}

	trcHdrReader := bytes.NewReader(trcHdrBuff)
	err = binary.Read(trcHdrReader, binary.BigEndian, &trace.TraceHeader)
	if err != nil {
		log.Fatal(err)
		return trace, err
	}

	bytesRead, err = s.file.Read(byteBuff)
	if err != nil {
		log.Fatal(err)
		return trace, err
	}

	for i := range trace.Data {
		trace.Data[i] = float32(binary.BigEndian.Uint32(byteBuff[i*4 : (i+1)*4]))
	}

	log.Println("ReadTrace read ", bytesRead, " bytes")

	// Then figure out the size of the data, and read it
	return trace, nil
}
<commit_msg>using proper go logger<commit_after>package segygo

import (
	"bytes"
	"encoding/binary"
	"io/ioutil"
	//"log"
	"os"
	"unsafe"
	"reflect"
	"github.com/op/go-logging"
)

var log = logging.MustGetLogger("segygo")
var format = logging.MustStringFormatter(
	`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
)

const Version = "0.1"
const SEGY_DESC_HDR_LEN = 3200
const SEGY_BIN_HDR_LEN = 400
const SEGY_TRACE_HDR_LEN = 240

type BinHeader struct {
	Jobid, Lino, Reno                                                                                int32
	Ntrpr, Nart                                                                                int16
	Hdt, Dto, Hns, Nso                                                                                uint16
	Format, Fold, Tsort, Vscode, Hsfs, Hsfe, Hslen, Hstyp, Schn, Hstas, Hstae, Htatyp, Hcorr, Bgrcv, Rcvm, Mfeet, Polyt, Vgpol int16
	Hunass                                                                                [170]int16 // unassigned
}

type TraceHeader struct {
	Tracel                                                               int32
	Tracer                                                               int32
	Fldr                                                                 int32
	Tracf                                                                int32
	Ep                                                                   int32
	CDP                                                                  int32
	CDPT                                                                 int32
	Trid                                                                 int16
	Nvs                                                                  int16
	Nhs                                                                  int16
	Duse                                                                 int16
	Offset                                                               int32
	Gelev                                                                int32
	Selev                                                                int32
	Sdepth                                                               int32
	Gdel                                                                 int32
	Sdel                                                                 int32
	SwDep                                                                int32
	GwDep                                                                int32
	Scalel                                                               int16
	Scalco                                                               int16
	Sx                                                                   int32
	Sy                                                                   int32
	Gx                                                                   int32
	Gy                                                                   int32
	CoUnit                                                               int16
	WeVel                                                                int16
	SweVel                                                               int16
	Sut, Gut, Sstat, Gstat, Tstat, Laga, Lagb, Delrt, Muts, Mute         int16
	Ns, Dt                                                               uint16
	Gain, Igc, Igi, Corr, Sfs, Sfe, Slen, Styp, Stas, Stae, Tatyp        int16
	Afilf, Afils, NoFilf, NoFils, Lcf, Hcf, Lcs, Hcs, Year, Day          int16
	Hour, Minute, Sec, Timbas, Trwf, Grnors, Grnofr, Grnlof, Gaps, Otrav int16
	D1, F1, D2, F2, Ungpow, Unscale                                      float32
	Ntr                                                                  int32
	Mark, Shortpad                                                       int16
	Unass                                                                [14]int16 // unassigned short array
}

type Trace struct {
	TraceHeader
	Data []float32
}

type SegyFile struct {
	Filename string
	Header   BinHeader
	NrTraces int64
	file     *os.File
	Position int64
}

func OpenFile(filename string) (SegyFile, error) {
	// Setup proper logging
	//backend1 := logging.NewLogBackend(os.Stdout, "", 0)
	//backend1Formatter := logging.NewBackendFormatter(backend1, format)
	//backend1Leveled := logging.AddModuleLevel(backend1)
	//backend1Leveled.SetLevel(logging.INFO, "")
	//logging.SetBackend(backend1Leveled, backend1Formatter)

	var s SegyFile
	var binHdr BinHeader
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		return s, err
	}

	s.Filename = filename

	accum := []byte{}
	accum = append(accum, b...)

	accum2 := accum[3200:]
	r := bytes.NewReader(accum2)
	//fmt.Println("Number of bytes:", r.Len())
	log.Infof("Number of bytes: %d", r.Len())

	if err = binary.Read(r, binary.BigEndian, &binHdr); err != nil {
		log.Errorf("Error reading segy file (bigendian). %s", err)
		return s, err
	}

	// Open and store the os.File object in our struct
	file, err := os.Open(s.Filename)
	s.file = file

	s.Header = binHdr
	s.NrTraces = s.GetNrTraces()

	return s, err
}

func (s *SegyFile) GetNrTraces() int64 {
	fi, err := s.file.Stat()
	if err != nil {
		log.Error("unable to get Stat()")
		log.Fatal(err)
	}
	size := fi.Size()
	nSamples := s.Header.Hns
	txtAndBinarySize := int64(SEGY_DESC_HDR_LEN + SEGY_BIN_HDR_LEN)
	nTraces := ((size - txtAndBinarySize) / (int64(SEGY_TRACE_HDR_LEN) + int64(nSamples)*int64(unsafe.Sizeof(float32(1)))))

	return nTraces
}

func (s *SegyFile) GetNrSamples() int32 {
	return int32(s.Header.Hns)
}

func (s *SegyFile) PrintBinaryHeader() {
	v := reflect.ValueOf(s.Header)
	for i := 0; i < v.NumField(); i++ {
		log.Infof("name = %s, value = %d", v.Type().Field(i).Name, v.Field(i).Interface())
	}
}

func (s *SegyFile) ReadTrace() (Trace, error) {
	trace := Trace{}
	traceBuff := make([]float32, s.GetNrSamples())
	byteBuff := make([]byte, s.GetNrSamples()*4)
	trace.Data = traceBuff

	trcHdrBuff := make([]byte, SEGY_TRACE_HDR_LEN)
	bytesRead, err := s.file.Read(trcHdrBuff)
	if err != nil {
		log.Fatal(err)
		return trace, err
	}

	trcHdrReader := bytes.NewReader(trcHdrBuff)
	err = binary.Read(trcHdrReader, binary.BigEndian, &trace.TraceHeader)
	if err != nil {
		log.Fatal(err)
		return trace, err
	}

	bytesRead, err = s.file.Read(byteBuff)
	if err != nil {
		log.Fatal(err)
		return trace, err
	}

	if bytesRead == 0 {
		log.Infof("No bytes read for trace #", s.Position)
	}

	for i := range trace.Data {
		trace.Data[i] = float32(binary.BigEndian.Uint32(byteBuff[i*4 : (i+1)*4]))
	}

	//log.Println("ReadTrace read ", bytesRead, " bytes")

	// Then figure out the size of the data, and read it
	return trace, nil
}
<|endoftext|>


<commit_before>package gomail

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/mail"
	"strings"
)

// Sender is the interface that wraps the Send method.
//
// Send sends an email to the given addresses.
type Sender interface {
	Send(from string, to []string, msg io.Reader) error
}

// SendCloser is the interface that groups the Send and Close methods.
type SendCloser interface {
	Sender
	Close() error
}

// A SendFunc is a function that sends emails to the given adresses.
// The SendFunc type is an adapter to allow the use of ordinary functions as
// email senders. If f is a function with the appropriate signature, SendFunc(f)
// is a Sender object that calls f.
type SendFunc func(from string, to []string, msg io.Reader) error

// Send calls f(from, to, msg).
func (f SendFunc) Send(from string, to []string, msg io.Reader) error {
	return f(from, to, msg)
}

// Send sends emails using the given Sender.
func Send(s Sender, msg ...*Message) error {
	for i, m := range msg {
		if err := send(s, m); err != nil {
			return fmt.Errorf("gomail: could not send email %d: %v", i+1, err)
		}
	}

	return nil
}

func send(s Sender, msg *Message) error {
	message := msg.Export()

	from, err := getFrom(message)
	if err != nil {
		return err
	}
	recipients, bcc, err := getRecipients(message)
	if err != nil {
		return err
	}

	h := flattenHeader(message, "")
	body, err := ioutil.ReadAll(message.Body)
	if err != nil {
		return err
	}

	mail := bytes.NewReader(append(h, body...))
	if err := s.Send(from, recipients, mail); err != nil {
		return err
	}

	for _, to := range bcc {
		h = flattenHeader(message, to)
		mail = bytes.NewReader(append(h, body...))
		if err := s.Send(from, []string{to}, mail); err != nil {
			return err
		}
	}

	return nil
}

func flattenHeader(msg *mail.Message, bcc string) []byte {
	buf := getBuffer()
	defer putBuffer(buf)

	for field, value := range msg.Header {
		if field != "Bcc" {
			buf.WriteString(field)
			buf.WriteString(": ")
			buf.WriteString(strings.Join(value, ", "))
			buf.WriteString("\r\n")
		} else if bcc != "" {
			for _, to := range value {
				if strings.Contains(to, bcc) {
					buf.WriteString(field)
					buf.WriteString(": ")
					buf.WriteString(to)
					buf.WriteString("\r\n")
				}
			}
		}
	}
	buf.WriteString("\r\n")

	return buf.Bytes()
}

func getFrom(msg *mail.Message) (string, error) {
	from := msg.Header.Get("Sender")
	if from == "" {
		from = msg.Header.Get("From")
		if from == "" {
			return "", errors.New("mailer: invalid message, \"From\" field is absent")
		}
	}

	return parseAddress(from)
}

func getRecipients(msg *mail.Message) (recipients, bcc []string, err error) {
	for _, field := range []string{"Bcc", "To", "Cc"} {
		if addresses, ok := msg.Header[field]; ok {
			for _, addr := range addresses {
				switch field {
				case "Bcc":
					bcc, err = addAdress(bcc, addr)
				default:
					recipients, err = addAdress(recipients, addr)
				}
				if err != nil {
					return recipients, bcc, err
				}
			}
		}
	}

	return recipients, bcc, nil
}

func addAdress(list []string, addr string) ([]string, error) {
	addr, err := parseAddress(addr)
	if err != nil {
		return list, err
	}
	for _, a := range list {
		if addr == a {
			return list, nil
		}
	}

	return append(list, addr), nil
}

func parseAddress(field string) (string, error) {
	a, err := mail.ParseAddress(field)
	if a == nil {
		return "", err
	}

	return a.Address, err
}
<commit_msg>Fixed an error message<commit_after>package gomail

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/mail"
	"strings"
)

// Sender is the interface that wraps the Send method.
//
// Send sends an email to the given addresses.
type Sender interface {
	Send(from string, to []string, msg io.Reader) error
}

// SendCloser is the interface that groups the Send and Close methods.
type SendCloser interface {
	Sender
	Close() error
}

// A SendFunc is a function that sends emails to the given adresses.
// The SendFunc type is an adapter to allow the use of ordinary functions as
// email senders. If f is a function with the appropriate signature, SendFunc(f)
// is a Sender object that calls f.
type SendFunc func(from string, to []string, msg io.Reader) error

// Send calls f(from, to, msg).
func (f SendFunc) Send(from string, to []string, msg io.Reader) error {
	return f(from, to, msg)
}

// Send sends emails using the given Sender.
func Send(s Sender, msg ...*Message) error {
	for i, m := range msg {
		if err := send(s, m); err != nil {
			return fmt.Errorf("gomail: could not send email %d: %v", i+1, err)
		}
	}

	return nil
}

func send(s Sender, msg *Message) error {
	message := msg.Export()

	from, err := getFrom(message)
	if err != nil {
		return err
	}
	recipients, bcc, err := getRecipients(message)
	if err != nil {
		return err
	}

	h := flattenHeader(message, "")
	body, err := ioutil.ReadAll(message.Body)
	if err != nil {
		return err
	}

	mail := bytes.NewReader(append(h, body...))
	if err := s.Send(from, recipients, mail); err != nil {
		return err
	}

	for _, to := range bcc {
		h = flattenHeader(message, to)
		mail = bytes.NewReader(append(h, body...))
		if err := s.Send(from, []string{to}, mail); err != nil {
			return err
		}
	}

	return nil
}

func flattenHeader(msg *mail.Message, bcc string) []byte {
	buf := getBuffer()
	defer putBuffer(buf)

	for field, value := range msg.Header {
		if field != "Bcc" {
			buf.WriteString(field)
			buf.WriteString(": ")
			buf.WriteString(strings.Join(value, ", "))
			buf.WriteString("\r\n")
		} else if bcc != "" {
			for _, to := range value {
				if strings.Contains(to, bcc) {
					buf.WriteString(field)
					buf.WriteString(": ")
					buf.WriteString(to)
					buf.WriteString("\r\n")
				}
			}
		}
	}
	buf.WriteString("\r\n")

	return buf.Bytes()
}

func getFrom(msg *mail.Message) (string, error) {
	from := msg.Header.Get("Sender")
	if from == "" {
		from = msg.Header.Get("From")
		if from == "" {
			return "", errors.New("gomail: invalid message, \"From\" field is absent")
		}
	}

	return parseAddress(from)
}

func getRecipients(msg *mail.Message) (recipients, bcc []string, err error) {
	for _, field := range []string{"Bcc", "To", "Cc"} {
		if addresses, ok := msg.Header[field]; ok {
			for _, addr := range addresses {
				switch field {
				case "Bcc":
					bcc, err = addAdress(bcc, addr)
				default:
					recipients, err = addAdress(recipients, addr)
				}
				if err != nil {
					return recipients, bcc, err
				}
			}
		}
	}

	return recipients, bcc, nil
}

func addAdress(list []string, addr string) ([]string, error) {
	addr, err := parseAddress(addr)
	if err != nil {
		return list, err
	}
	for _, a := range list {
		if addr == a {
			return list, nil
		}
	}

	return append(list, addr), nil
}

func parseAddress(field string) (string, error) {
	a, err := mail.ParseAddress(field)
	if a == nil {
		return "", err
	}

	return a.Address, err
}
<|endoftext|>


<commit_before>package launcher

import (
	"os"
	"os/user"
	"path"
	"strings"

	"fmt"

	"github.com/ellcrys/util"
	"github.com/goware/urlx"
)

// Go defines a deployment helper for go cocoon.
type Go struct {
	name        string
	image       string
	userHome    string
	imgGoPath   string
	repoURL     string
	buildParams map[string]interface{}
}

// NewGo returns a new instance a golang
// cocoon code deployment helper.
func NewGo() *Go {
	g := &Go{
		name:      "go",
		image:     "ncodes/launch-go",
		userHome:  os.Getenv("HOME"),
		imgGoPath: "/go",
	}
	g.setUserHomeDir()
	return g
}

// GetName returns the name for the language
func (g *Go) GetName() string {
	return g.name
}

// GetImage returns the image name to deploy cocoon code on
func (g *Go) GetImage() string {
	return g.image
}

// setUserHomeDir sets the user home directory
func (g *Go) setUserHomeDir() error {
	usr, err := user.Current()
	if err != nil {
		return err
	}
	g.userHome = usr.HomeDir
	return nil
}

// GetDownloadDestination returns the location to save
// the downloaded go cocoon code to.
func (g *Go) GetDownloadDestination(url string) string {
	g.repoURL = url
	u, _ := urlx.Parse(url)
	repoID := strings.Trim(u.Path, "/")
	return path.Join(g.userHome, "/ccode/source", repoID)
}

// GetMountDestination returns the location in
// the container where the source code root will be mounted
func (g *Go) GetMountDestination(url string) string {
	u, _ := urlx.Parse(url)
	repoID := strings.Trim(u.Path, "/")
	return path.Join(g.imgGoPath, "src/github.com/", repoID)
}

// RequiresBuild returns true if cocoon codes written in
// go langugage requires a build process.
// During development, If RUN_ROOT_BIN env is set, it will return false as
// the run command will find and find the ccode binary in the repo root.
func (g *Go) RequiresBuild() bool {
	if len(os.Getenv("RUN_ROOT_BIN")) > 0 {
		return false
	}
	return true
}

// SetBuildParams sets and validates build parameters
func (g *Go) SetBuildParams(buildParams map[string]interface{}) error {
	g.buildParams = buildParams
	if pkgMgr := g.buildParams["pkg_mgr"]; pkgMgr != nil {
		if !util.InStringSlice([]string{"glide"}, pkgMgr.(string)) {
			return fmt.Errorf("invalid pkg_mgr value in build script")
		}
	}
	return nil
}

// GetBuildScript will return the script required
// to create an executable
func (g *Go) GetBuildScript() string {

	// run known package manager fetch routine
	pkgFetchCmd := ""
	if pkgMgr := g.buildParams["pkg_mgr"]; pkgMgr != nil {
		switch pkgMgr {
		case "glide":
			pkgFetchCmd = "glide install"
		}
	}

	return strings.Join([]string{pkgFetchCmd, "go build -v -o /bin/ccode"}, " && ")
}

// GetRunScript returns the script required to start the
// cocoon code accodeording to the build and installation process
// of the language. If RUN_ROOT_BIN env is set, it will run the
// ccode binary located in the mount destination.
func (g *Go) GetRunScript() []string {
	script := []string{strings.Join([]string{"ccode"}, " && ")}

	// run ccode in the repo root if RUN_ROOT_BIN is set (for development only)
	if len(os.Getenv("RUN_ROOT_BIN")) > 0 {
		script = []string{"bash", "-c", fmt.Sprintf("cd %s && ./ccode", g.GetMountDestination(g.repoURL))}
	}

	return script
}
<commit_msg>removes empty commands in final command slice<commit_after>package launcher

import (
	"os"
	"os/user"
	"path"
	"strings"

	"fmt"

	"github.com/ellcrys/util"
	"github.com/goware/urlx"
)

// Go defines a deployment helper for go cocoon.
type Go struct {
	name        string
	image       string
	userHome    string
	imgGoPath   string
	repoURL     string
	buildParams map[string]interface{}
}

// NewGo returns a new instance a golang
// cocoon code deployment helper.
func NewGo() *Go {
	g := &Go{
		name:      "go",
		image:     "ncodes/launch-go",
		userHome:  os.Getenv("HOME"),
		imgGoPath: "/go",
	}
	g.setUserHomeDir()
	return g
}

// GetName returns the name for the language
func (g *Go) GetName() string {
	return g.name
}

// GetImage returns the image name to deploy cocoon code on
func (g *Go) GetImage() string {
	return g.image
}

// setUserHomeDir sets the user home directory
func (g *Go) setUserHomeDir() error {
	usr, err := user.Current()
	if err != nil {
		return err
	}
	g.userHome = usr.HomeDir
	return nil
}

// GetDownloadDestination returns the location to save
// the downloaded go cocoon code to.
func (g *Go) GetDownloadDestination(url string) string {
	g.repoURL = url
	u, _ := urlx.Parse(url)
	repoID := strings.Trim(u.Path, "/")
	return path.Join(g.userHome, "/ccode/source", repoID)
}

// GetMountDestination returns the location in
// the container where the source code root will be mounted
func (g *Go) GetMountDestination(url string) string {
	u, _ := urlx.Parse(url)
	repoID := strings.Trim(u.Path, "/")
	return path.Join(g.imgGoPath, "src/github.com/", repoID)
}

// RequiresBuild returns true if cocoon codes written in
// go langugage requires a build process.
// During development, If RUN_ROOT_BIN env is set, it will return false as
// the run command will find and find the ccode binary in the repo root.
func (g *Go) RequiresBuild() bool {
	if len(os.Getenv("RUN_ROOT_BIN")) > 0 {
		return false
	}
	return true
}

// SetBuildParams sets and validates build parameters
func (g *Go) SetBuildParams(buildParams map[string]interface{}) error {
	g.buildParams = buildParams
	if pkgMgr := g.buildParams["pkg_mgr"]; pkgMgr != nil {
		if !util.InStringSlice([]string{"glide"}, pkgMgr.(string)) {
			return fmt.Errorf("invalid pkg_mgr value in build script")
		}
	}
	return nil
}

// GetBuildScript will return the script required
// to create an executable
func (g *Go) GetBuildScript() string {

	// run known package manager fetch routine
	pkgFetchCmd := ""
	if pkgMgr := g.buildParams["pkg_mgr"]; pkgMgr != nil {
		switch pkgMgr {
		case "glide":
			pkgFetchCmd = "glide install"
		}
	}

	return strings.Join(util.RemoveEmptyInStringSlice([]string{pkgFetchCmd, "go build -v -o /bin/ccode"}), " && ")
}

// GetRunScript returns the script required to start the
// cocoon code accodeording to the build and installation process
// of the language. If RUN_ROOT_BIN env is set, it will run the
// ccode binary located in the mount destination.
func (g *Go) GetRunScript() []string {
	script := []string{strings.Join([]string{"ccode"}, " && ")}

	// run ccode in the repo root if RUN_ROOT_BIN is set (for development only)
	if len(os.Getenv("RUN_ROOT_BIN")) > 0 {
		script = []string{"bash", "-c", fmt.Sprintf("cd %s && ./ccode", g.GetMountDestination(g.repoURL))}
	}

	return script
}
<|endoftext|>


<commit_before>package kiwi

// This file consists of Sink related structures and functions.
// Outputs accepts incoming log records from Loggers, check them with filters
// and write to output streams if checks passed.

/* Copyright (c) 2016-2019, Alexander I.Grafov <grafov@gmail.com>
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of kvlog nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ॐ तारे तुत्तारे तुरे स्व */

import (
	"io"
	"sync"
	"sync/atomic"
	"time"
)

// States of the sink.
const (
	sinkClosed int32 = iota - 1
	sinkStopped
	sinkActive
)

// Sinks accepts records through the chanels.
// Each sink has its own channel.
var collector struct {
	sync.RWMutex
	sinks []*Sink
	count int
}

// FlushAll should wait for all the sinks to be flushed. It does
// nothing currently. It has left for compatibility with old API.
func FlushAll() {
	return
}

type (
	// Sink used for filtering incoming log records from all logger instances
	// and decides how to filter them. Each output wraps its own io.Writer.
	// Sink methods are safe for concurrent usage.
	Sink struct {
		id     uint
		In     chan chain
		close  chan struct{}
		writer io.Writer
		format Formatter
		state  *int32

		sync.RWMutex
		positiveFilters map[string]Filter
		negativeFilters map[string]Filter
		hiddenKeys      map[string]bool
	}
	chain struct {
		wg    *sync.WaitGroup
		pairs []*Pair
	}
)

// SinkTo creates a new sink for an arbitrary number of loggers.
// There are any number of sinks may be created for saving incoming log
// records to different places.
// The sink requires explicit start with Start() before usage.
// That allows firstly setup filters before sink will really accept any records.
func SinkTo(w io.Writer, fn Formatter) *Sink {
	collector.RLock()
	for i, sink := range collector.sinks {
		if sink.writer == w {
			collector.sinks[i].format = fn
			collector.RUnlock()
			return collector.sinks[i]
		}
	}
	collector.RUnlock()
	var (
		state = sinkStopped
		sink  = &Sink{
			In:              make(chan chain, 16),
			close:           make(chan struct{}),
			format:          fn,
			state:           &state,
			writer:          w,
			positiveFilters: make(map[string]Filter),
			negativeFilters: make(map[string]Filter),
			hiddenKeys:      make(map[string]bool),
		}
	)
	collector.Lock()
	sink.id = uint(collector.count)
	collector.sinks = append(collector.sinks, sink)
	collector.count++
	collector.Unlock()
	go processSink(sink)
	return sink
}

// WithKey sets restriction for records output.
// Only the records WITH any of the keys will be passed to output.
func (s *Sink) WithKey(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			s.positiveFilters[key] = &keyFilter{}
			delete(s.negativeFilters, key)
		}
		s.Unlock()
	}
	return s
}

// WithoutKey sets restriction for records output.
// Only the records WITHOUT any of the keys will be passed to output.
func (s *Sink) WithoutKey(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			s.negativeFilters[key] = &keyFilter{}
			delete(s.positiveFilters, key)
		}
		s.Unlock()
	}
	return s
}

// WithValue sets restriction for records output.
// A record passed to output if the key equal one of any of the listed values.
func (s *Sink) WithValue(key string, vals ...string) *Sink {
	if len(vals) == 0 {
		return s.WithKey(key)
	}
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		s.positiveFilters[key] = &valsFilter{Vals: vals}
		delete(s.negativeFilters, key)
		s.Unlock()
	}
	return s
}

// WithoutValue sets restriction for records output.
func (s *Sink) WithoutValue(key string, vals ...string) *Sink {
	if len(vals) == 0 {
		return s.WithoutKey(key)
	}
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		s.negativeFilters[key] = &valsFilter{Vals: vals}
		delete(s.positiveFilters, key)
		s.Unlock()
	}
	return s
}

// WithInt64Range sets restriction for records output.
func (s *Sink) WithInt64Range(key string, from, to int64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = &int64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithoutInt64Range sets restriction for records output.
func (s *Sink) WithoutInt64Range(key string, from, to int64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.positiveFilters, key)
		s.negativeFilters[key] = &int64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithFloat64Range sets restriction for records output.
func (s *Sink) WithFloat64Range(key string, from, to float64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = &float64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithoutFloat64Range sets restriction for records output.
func (s *Sink) WithoutFloat64Range(key string, from, to float64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.positiveFilters, key)
		s.negativeFilters[key] = &float64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithTimeRange sets restriction for records output.
func (s *Sink) WithTimeRange(key string, from, to time.Time) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = &timeRangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithoutTimeRange sets restriction for records output.
func (s *Sink) WithoutTimeRange(key string, from, to time.Time) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.positiveFilters, key)
		s.negativeFilters[key] = &timeRangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithFilter setup custom filtering function for values for a specific key.
// Custom filter should realize Filter interface. All custom filters treated
// as positive filters. So if the filter returns true then it will be passed.
func (s *Sink) WithFilter(key string, customFilter Filter) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = customFilter
		s.Unlock()
	}
	return s
}

// Reset all filters for the keys for the output.
func (s *Sink) Reset(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			delete(s.positiveFilters, key)
			delete(s.negativeFilters, key)
		}
		s.Unlock()
	}
	return s
}

// Hide keys from the output. Other keys in record will be displayed
// but not hidden keys.
func (s *Sink) Hide(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			s.hiddenKeys[key] = true
		}
		s.Unlock()
	}
	return s
}

// Unhide previously hidden keys. They will be displayed in the output again.
func (s *Sink) Unhide(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			delete(s.hiddenKeys, key)
		}
		s.Unlock()
	}
	return s
}

// Stop stops writing to the output.
func (s *Sink) Stop() *Sink {
	atomic.StoreInt32(s.state, sinkStopped)
	return s
}

// Start writing to the output.
// After creation of a new sink it will paused and you need explicitly start it.
// It allows setup the filters before the sink will accepts any records.
func (s *Sink) Start() *Sink {
	atomic.StoreInt32(s.state, sinkActive)
	return s
}

// Close closes the sink. It flushes records for the sink before closing.
func (s *Sink) Close() {
	return
	if atomic.LoadInt32(s.state) > sinkClosed {
		atomic.StoreInt32(s.state, sinkClosed)
		s.close <- struct{}{}
		collector.Lock()
		collector.count--
		collector.sinks = append(collector.sinks[0:s.id], collector.sinks[s.id+1:]...)
		collector.Unlock()
	}
}

// Flush waits that all previously sent to the output records
// worked. It does nothing currently. It has left for compatibility
// with old API.
func (s *Sink) Flush() *Sink {
	return s
}

func processSink(s *Sink) {
	var (
		record chain
		ok     bool
	)
	for {
		select {
		case record, ok = <-s.In:
			if !ok {
				return
			}
			if atomic.LoadInt32(s.state) < sinkActive {
				record.wg.Done()
				continue
			}
			s.RLock()
			var filter Filter
			for _, pair := range record.pairs {
				// Negative conditions have highest priority
				if filter, ok = s.negativeFilters[pair.Key]; ok {
					if filter.Check(pair.Key, pair.Val) {
						goto skipRecord
					}
				}
				// At last check for positive conditions
				if filter, ok = s.positiveFilters[pair.Key]; ok {
					if !filter.Check(pair.Key, pair.Val) {
						goto skipRecord
					}
				}
			}
			s.formatRecord(record.pairs)
		skipRecord:
			s.RUnlock()
			record.wg.Done()
		case <-s.close:
			s.Lock()
			s.positiveFilters = nil
			s.negativeFilters = nil
			s.hiddenKeys = nil
			s.Unlock()
			record.wg.Done()
			return
		}
	}
}

func (s *Sink) formatRecord(record []*Pair) {
	s.format.Begin()
	for _, pair := range record {
		if ok := s.hiddenKeys[pair.Key]; ok {
			continue
		}
		s.format.Pair(pair.Key, pair.Val, pair.Type)
	}
	s.writer.Write(s.format.Finish())
}

func sinkRecord(rec []*Pair) {
	var wg sync.WaitGroup
	collector.RLock()
	for _, s := range collector.sinks {
		if atomic.LoadInt32(s.state) == sinkActive {
			wg.Add(1)
			s.In <- chain{&wg, rec}
		}
	}
	collector.RUnlock()
	wg.Wait()
}
<commit_msg>Move obsoleted functions to separate file<commit_after>package kiwi

// This file consists of Sink related structures and functions.
// Outputs accepts incoming log records from Loggers, check them with filters
// and write to output streams if checks passed.

/* Copyright (c) 2016-2019, Alexander I.Grafov <grafov@gmail.com>
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of kvlog nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ॐ तारे तुत्तारे तुरे स्व */

import (
	"io"
	"sync"
	"sync/atomic"
	"time"
)

// States of the sink.
const (
	sinkClosed int32 = iota - 1
	sinkStopped
	sinkActive
)

// Sinks accepts records through the chanels.
// Each sink has its own channel.
var collector struct {
	sync.RWMutex
	sinks []*Sink
	count int
}

type (
	// Sink used for filtering incoming log records from all logger instances
	// and decides how to filter them. Each output wraps its own io.Writer.
	// Sink methods are safe for concurrent usage.
	Sink struct {
		id     uint
		In     chan chain
		close  chan struct{}
		writer io.Writer
		format Formatter
		state  *int32

		sync.RWMutex
		positiveFilters map[string]Filter
		negativeFilters map[string]Filter
		hiddenKeys      map[string]bool
	}
	chain struct {
		wg    *sync.WaitGroup
		pairs []*Pair
	}
)

// SinkTo creates a new sink for an arbitrary number of loggers.
// There are any number of sinks may be created for saving incoming log
// records to different places.
// The sink requires explicit start with Start() before usage.
// That allows firstly setup filters before sink will really accept any records.
func SinkTo(w io.Writer, fn Formatter) *Sink {
	collector.RLock()
	for i, sink := range collector.sinks {
		if sink.writer == w {
			collector.sinks[i].format = fn
			collector.RUnlock()
			return collector.sinks[i]
		}
	}
	collector.RUnlock()
	var (
		state = sinkStopped
		sink  = &Sink{
			In:              make(chan chain, 16),
			close:           make(chan struct{}),
			format:          fn,
			state:           &state,
			writer:          w,
			positiveFilters: make(map[string]Filter),
			negativeFilters: make(map[string]Filter),
			hiddenKeys:      make(map[string]bool),
		}
	)
	collector.Lock()
	sink.id = uint(collector.count)
	collector.sinks = append(collector.sinks, sink)
	collector.count++
	collector.Unlock()
	go processSink(sink)
	return sink
}

// WithKey sets restriction for records output.
// Only the records WITH any of the keys will be passed to output.
func (s *Sink) WithKey(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			s.positiveFilters[key] = &keyFilter{}
			delete(s.negativeFilters, key)
		}
		s.Unlock()
	}
	return s
}

// WithoutKey sets restriction for records output.
// Only the records WITHOUT any of the keys will be passed to output.
func (s *Sink) WithoutKey(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			s.negativeFilters[key] = &keyFilter{}
			delete(s.positiveFilters, key)
		}
		s.Unlock()
	}
	return s
}

// WithValue sets restriction for records output.
// A record passed to output if the key equal one of any of the listed values.
func (s *Sink) WithValue(key string, vals ...string) *Sink {
	if len(vals) == 0 {
		return s.WithKey(key)
	}
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		s.positiveFilters[key] = &valsFilter{Vals: vals}
		delete(s.negativeFilters, key)
		s.Unlock()
	}
	return s
}

// WithoutValue sets restriction for records output.
func (s *Sink) WithoutValue(key string, vals ...string) *Sink {
	if len(vals) == 0 {
		return s.WithoutKey(key)
	}
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		s.negativeFilters[key] = &valsFilter{Vals: vals}
		delete(s.positiveFilters, key)
		s.Unlock()
	}
	return s
}

// WithInt64Range sets restriction for records output.
func (s *Sink) WithInt64Range(key string, from, to int64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = &int64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithoutInt64Range sets restriction for records output.
func (s *Sink) WithoutInt64Range(key string, from, to int64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.positiveFilters, key)
		s.negativeFilters[key] = &int64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithFloat64Range sets restriction for records output.
func (s *Sink) WithFloat64Range(key string, from, to float64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = &float64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithoutFloat64Range sets restriction for records output.
func (s *Sink) WithoutFloat64Range(key string, from, to float64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.positiveFilters, key)
		s.negativeFilters[key] = &float64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithTimeRange sets restriction for records output.
func (s *Sink) WithTimeRange(key string, from, to time.Time) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = &timeRangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithoutTimeRange sets restriction for records output.
func (s *Sink) WithoutTimeRange(key string, from, to time.Time) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.positiveFilters, key)
		s.negativeFilters[key] = &timeRangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithFilter setup custom filtering function for values for a specific key.
// Custom filter should realize Filter interface. All custom filters treated
// as positive filters. So if the filter returns true then it will be passed.
func (s *Sink) WithFilter(key string, customFilter Filter) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = customFilter
		s.Unlock()
	}
	return s
}

// Reset all filters for the keys for the output.
func (s *Sink) Reset(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			delete(s.positiveFilters, key)
			delete(s.negativeFilters, key)
		}
		s.Unlock()
	}
	return s
}

// Hide keys from the output. Other keys in record will be displayed
// but not hidden keys.
func (s *Sink) Hide(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			s.hiddenKeys[key] = true
		}
		s.Unlock()
	}
	return s
}

// Unhide previously hidden keys. They will be displayed in the output again.
func (s *Sink) Unhide(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			delete(s.hiddenKeys, key)
		}
		s.Unlock()
	}
	return s
}

// Stop stops writing to the output.
func (s *Sink) Stop() *Sink {
	atomic.StoreInt32(s.state, sinkStopped)
	return s
}

// Start writing to the output.
// After creation of a new sink it will paused and you need explicitly start it.
// It allows setup the filters before the sink will accepts any records.
func (s *Sink) Start() *Sink {
	atomic.StoreInt32(s.state, sinkActive)
	return s
}

// Close closes the sink. It flushes records for the sink before closing.
func (s *Sink) Close() {
	return
	if atomic.LoadInt32(s.state) > sinkClosed {
		atomic.StoreInt32(s.state, sinkClosed)
		s.close <- struct{}{}
		collector.Lock()
		collector.count--
		collector.sinks = append(collector.sinks[0:s.id], collector.sinks[s.id+1:]...)
		collector.Unlock()
	}
}

func processSink(s *Sink) {
	var (
		record chain
		ok     bool
	)
	for {
		select {
		case record, ok = <-s.In:
			if !ok {
				return
			}
			if atomic.LoadInt32(s.state) < sinkActive {
				record.wg.Done()
				continue
			}
			s.RLock()
			var filter Filter
			for _, pair := range record.pairs {
				// Negative conditions have highest priority
				if filter, ok = s.negativeFilters[pair.Key]; ok {
					if filter.Check(pair.Key, pair.Val) {
						goto skipRecord
					}
				}
				// At last check for positive conditions
				if filter, ok = s.positiveFilters[pair.Key]; ok {
					if !filter.Check(pair.Key, pair.Val) {
						goto skipRecord
					}
				}
			}
			s.formatRecord(record.pairs)
		skipRecord:
			s.RUnlock()
			record.wg.Done()
		case <-s.close:
			s.Lock()
			s.positiveFilters = nil
			s.negativeFilters = nil
			s.hiddenKeys = nil
			s.Unlock()
			record.wg.Done()
			return
		}
	}
}

func (s *Sink) formatRecord(record []*Pair) {
	s.format.Begin()
	for _, pair := range record {
		if ok := s.hiddenKeys[pair.Key]; ok {
			continue
		}
		s.format.Pair(pair.Key, pair.Val, pair.Type)
	}
	s.writer.Write(s.format.Finish())
}

func sinkRecord(rec []*Pair) {
	var wg sync.WaitGroup
	collector.RLock()
	for _, s := range collector.sinks {
		if atomic.LoadInt32(s.state) == sinkActive {
			wg.Add(1)
			s.In <- chain{&wg, rec}
		}
	}
	collector.RUnlock()
	wg.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file implements primitive functions.

package golisp

import (
	"fmt"
	"strings"
)

type PrimitiveFunction struct {
	Name         string
	Special      bool
	NumberOfArgs string
	Body         func(d *Data, env *SymbolTableFrame) (*Data, error)
	IsRestricted bool
}

func MakePrimitiveFunction(name string, argCount string, function func(*Data, *SymbolTableFrame) (*Data, error)) {
	f := &PrimitiveFunction{Name: name, Special: false, NumberOfArgs: argCount, Body: function, IsRestricted: false}
	sym := Global.Intern(name)
	Global.BindTo(sym, PrimitiveWithNameAndFunc(name, f))
}

func MakeRestrictedPrimitiveFunction(name string, argCount string, function func(*Data, *SymbolTableFrame) (*Data, error)) {
	f := &PrimitiveFunction{Name: name, Special: false, NumberOfArgs: argCount, Body: function, IsRestricted: true}
	sym := Global.Intern(name)
	Global.BindTo(sym, PrimitiveWithNameAndFunc(name, f))
}

func MakeSpecialForm(name string, argCount string, function func(*Data, *SymbolTableFrame) (*Data, error)) {
	f := &PrimitiveFunction{Name: name, Special: true, NumberOfArgs: argCount, Body: function, IsRestricted: false}
	sym := Global.Intern(name)
	Global.BindTo(sym, PrimitiveWithNameAndFunc(name, f))
}

func MakeRestrictedSpecialForm(name string, argCount string, function func(*Data, *SymbolTableFrame) (*Data, error)) {
	f := &PrimitiveFunction{Name: name, Special: true, NumberOfArgs: argCount, Body: function, IsRestricted: true}
	sym := Global.Intern(name)
	Global.BindTo(sym, PrimitiveWithNameAndFunc(name, f))
}

func (self *PrimitiveFunction) String() string {
	return fmt.Sprintf("<prim: %s, %v>", self.Name, self.Body)
}

func (self *PrimitiveFunction) checkArgumentCount(argCount int) bool {
	if self.NumberOfArgs == "*" {
		return true
	}

	for _, term := range strings.Split(self.NumberOfArgs, "|") {
		var intTerm int
		n, _ := fmt.Sscanf(term, "%d", &intTerm)
		if n == 1 && argCount == intTerm {
			return true
		}
		n, _ = fmt.Sscanf(term, ">=%d", &intTerm)
		if n == 1 && argCount >= intTerm {
			return true
		}
		var lo int
		var hi int
		n, _ = fmt.Sscanf(term, "(%d,%d)", &lo, &hi)
		if n == 2 && lo <= argCount && argCount <= hi {
			return true
		}
	}
	return false
}

func (self *PrimitiveFunction) Apply(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	if self.IsRestricted && env.IsRestricted {
		err = fmt.Errorf("The %s primitive is restricted from execution in this environment\n", self.Name)
		return
	}

	if !self.checkArgumentCount(Length(args)) {
		err = fmt.Errorf("Wrong number of args to %s. Expected %s but got %d.\n", self.Name, self.NumberOfArgs, Length(args))
		return
	}

	argArray := make([]*Data, 0)
	var argValue *Data
	for a := args; NotNilP(a); a = Cdr(a) {
		if self.Special {
			argValue = Car(a)
		} else {
			argValue, err = Eval(Car(a), env)
			if err != nil {
				return
			}
		}

		argArray = append(argArray, argValue)
	}

	localGuid := ProfileGUID
	ProfileGUID++
	ProfileEnter("prim", self.Name, localGuid)

	result, err = (self.Body)(ArrayToList(argArray), env)

	ProfileExit("prim", self.Name, localGuid)

	return
}

func (self *PrimitiveFunction) ApplyWithoutEval(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	if self.Special {
		return self.Apply(args, env)
	} else {
		return self.Apply(QuoteAll(args), env)
	}
}
<commit_msg>Report special forms separately from primitives.<commit_after>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file implements primitive functions.

package golisp

import (
	"fmt"
	"strings"
)

type PrimitiveFunction struct {
	Name         string
	Special      bool
	NumberOfArgs string
	Body         func(d *Data, env *SymbolTableFrame) (*Data, error)
	IsRestricted bool
}

func MakePrimitiveFunction(name string, argCount string, function func(*Data, *SymbolTableFrame) (*Data, error)) {
	f := &PrimitiveFunction{Name: name, Special: false, NumberOfArgs: argCount, Body: function, IsRestricted: false}
	sym := Global.Intern(name)
	Global.BindTo(sym, PrimitiveWithNameAndFunc(name, f))
}

func MakeRestrictedPrimitiveFunction(name string, argCount string, function func(*Data, *SymbolTableFrame) (*Data, error)) {
	f := &PrimitiveFunction{Name: name, Special: false, NumberOfArgs: argCount, Body: function, IsRestricted: true}
	sym := Global.Intern(name)
	Global.BindTo(sym, PrimitiveWithNameAndFunc(name, f))
}

func MakeSpecialForm(name string, argCount string, function func(*Data, *SymbolTableFrame) (*Data, error)) {
	f := &PrimitiveFunction{Name: name, Special: true, NumberOfArgs: argCount, Body: function, IsRestricted: false}
	sym := Global.Intern(name)
	Global.BindTo(sym, PrimitiveWithNameAndFunc(name, f))
}

func MakeRestrictedSpecialForm(name string, argCount string, function func(*Data, *SymbolTableFrame) (*Data, error)) {
	f := &PrimitiveFunction{Name: name, Special: true, NumberOfArgs: argCount, Body: function, IsRestricted: true}
	sym := Global.Intern(name)
	Global.BindTo(sym, PrimitiveWithNameAndFunc(name, f))
}

func (self *PrimitiveFunction) String() string {
	return fmt.Sprintf("<prim: %s, %v>", self.Name, self.Body)
}

func (self *PrimitiveFunction) checkArgumentCount(argCount int) bool {
	if self.NumberOfArgs == "*" {
		return true
	}

	for _, term := range strings.Split(self.NumberOfArgs, "|") {
		var intTerm int
		n, _ := fmt.Sscanf(term, "%d", &intTerm)
		if n == 1 && argCount == intTerm {
			return true
		}
		n, _ = fmt.Sscanf(term, ">=%d", &intTerm)
		if n == 1 && argCount >= intTerm {
			return true
		}
		var lo int
		var hi int
		n, _ = fmt.Sscanf(term, "(%d,%d)", &lo, &hi)
		if n == 2 && lo <= argCount && argCount <= hi {
			return true
		}
	}
	return false
}

func (self *PrimitiveFunction) Apply(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	if self.IsRestricted && env.IsRestricted {
		err = fmt.Errorf("The %s primitive is restricted from execution in this environment\n", self.Name)
		return
	}

	if !self.checkArgumentCount(Length(args)) {
		err = fmt.Errorf("Wrong number of args to %s. Expected %s but got %d.\n", self.Name, self.NumberOfArgs, Length(args))
		return
	}

	argArray := make([]*Data, 0)
	var argValue *Data
	for a := args; NotNilP(a); a = Cdr(a) {
		if self.Special {
			argValue = Car(a)
		} else {
			argValue, err = Eval(Car(a), env)
			if err != nil {
				return
			}
		}

		argArray = append(argArray, argValue)
	}

	localGuid := ProfileGUID
	ProfileGUID++
	fType := "prim"
	if self.Special {
		fType = "form"
	}

	ProfileEnter(fType, self.Name, localGuid)

	result, err = (self.Body)(ArrayToList(argArray), env)

	ProfileExit(fType, self.Name, localGuid)

	return
}

func (self *PrimitiveFunction) ApplyWithoutEval(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	if self.Special {
		return self.Apply(args, env)
	} else {
		return self.Apply(QuoteAll(args), env)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
	"mime"
	"os"
	"path/filepath"
	"text/template"
	"time"
)

var (
	MsgCopyingFile  = "Copying File: %s"
	MsgGenerateFile = "Generating Page: %s"
	MsgUploadFile   = "Uploading: %s"
	MsgUsingConfig  = "Loading Config: %s"
)

type Site struct {
	Src  string // Directory where Jekyll will look to transform files
	Dest string // Directory where Jekyll will write files to
	Conf Config // Configuration date from the _config.yml file

	posts []Page             // Posts thet need to be generated
	pages []Page             // Pages that need to be generated
	files []string           // Static files to get copied to the destination
	templ *template.Template // Compiled templates
}

func NewSite(src, dest string) (*Site, error) {

	// Parse the _config.yml file
	path := filepath.Join(src, "_config.yml")
	conf, err := ParseConfig(path)
	logf(MsgUsingConfig, path)
	if err != nil {
		return nil, err
	}

	site := Site{
		Src:  src,
		Dest: dest,
		Conf: conf,
	}

	// Recursively process all files in the source directory
	// and parse pages, posts, templates, etc
	if err := site.read(); err != nil {
		return nil, err
	}

	return &site, nil
}

// Reloads the site into memory
func (s *Site) Reload() error {
	s.posts = []Page{}
	s.pages = []Page{}
	s.files = []string{}
	s.templ = nil
	return s.read()
}

// Prepares the source directory for site generation
func (s *Site) Prep() error {
	return os.MkdirAll(s.Dest, 0755)
}

// Removes the existing site (typically in _site).
func (s *Site) Clear() error {
	return os.RemoveAll(s.Dest)
}

// Generates a static website based on Jekyll standard layout.
func (s *Site) Generate() error {

	// Remove previously generated site, and then (re)create the
	// destination directory
	if err := s.Clear(); err != nil {
		return err
	}
	if err := s.Prep(); err != nil {
		return err
	}

	// Generate all Pages and Posts and static files
	if err := s.writePages(); err != nil {
		return err
	}
	if err := s.writeStatic(); err != nil {
		return err
	}

	return nil
}

// Deploys a site to S3.
func (s *Site) Deploy(user, pass, url string) error {

	auth := aws.Auth{AccessKey: user, SecretKey: pass}
	b := s3.New(auth, aws.USEast).Bucket(url)

	// walks _site directory and uploads file to S3
	walker := func(fn string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}

		rel, _ := filepath.Rel(s.Dest, fn)
		typ := mime.TypeByExtension(filepath.Ext(rel))
		content, err := ioutil.ReadFile(fn)
		logf(MsgUploadFile, rel)
		if err != nil {
			return err
		}

		// try to upload the file ... sometimes this fails due to amazon
		// issues. If so, we'll re-try
		if err := b.Put(rel, content, typ, s3.PublicRead); err != nil {
			time.Sleep(100 * time.Millisecond) // sleep so that we don't immediately retry
			return b.Put(rel, content, typ, s3.PublicRead)
		}

		// file upload was a success, return nil
		return nil
	}

	return filepath.Walk(s.Dest, walker)
}

// Helper function to traverse the source directory and identify all posts,
// projects, templates, etc and parse.
func (s *Site) read() error {

	// Lists of templates (_layouts, _includes) that we find that
	// will need to be compiled
	layouts := []string{}

	// func to walk the jekyll directory structure
	walker := func(fn string, fi os.FileInfo, err error) error {
		rel, _ := filepath.Rel(s.Src, fn)
		switch {
		case err != nil:
			return nil

		// Ignore directories
		case fi.IsDir():
			return nil

		// Ignore Hidden or Temp files
		// (starting with . or ending with ~)
		case isHiddenOrTemp(rel):
			return nil

		// Parse Templates
		case isTemplate(rel):
			layouts = append(layouts, fn)

		// Parse Posts
		case isPost(rel):
			post, err := ParsePost(rel)
			if err != nil {
				return err
			}
			// TODO: this is a hack to get the posts in rev chronological order
			s.posts = append([]Page{post}, s.posts...) //s.posts, post)

		// Parse Pages
		case isPage(rel):
			page, err := ParsePage(rel)
			if err != nil {
				return err
			}
			s.pages = append(s.pages, page)

		// Move static files, no processing required
		case isStatic(rel):
			s.files = append(s.files, rel)
		}
		return nil
	}

	// Walk the diretory recursively to get a list of all posts,
	// pages, templates and static files.
	err := filepath.Walk(s.Src, walker)
	if err != nil {
		return err
	}

	// Compile all templates found, if any
	if len(layouts) > 0 {
		s.templ, err = template.New("layouts").Funcs(funcMap).ParseFiles(layouts...)
		if err != nil {
			return err
		}
	}

	// Add the posts, timestamp, etc to the Site Params
	s.Conf.Set("posts", s.posts)
	s.Conf.Set("time", time.Now())
	s.calculateTags()
	s.calculateCategories()

	return nil
}

// Helper function to write all pages and posts to the destination directory
// during site generation.
func (s *Site) writePages() error {

	// There is really no difference between a Page and a Post (other than
	// initial parsing) so we can combine the lists and use the same rendering
	// code for both.
	pages := []Page{}
	pages = append(pages, s.pages...)
	pages = append(pages, s.posts...)

	for _, page := range pages {
		url := page.GetUrl()
		layout := page.GetLayout()

		// is the layout provided? or is it nil /empty?
		//layoutNil := layout == "" || layout == "nil"

		// make sure the posts's parent dir exists
		d := filepath.Join(s.Dest, filepath.Dir(url))
		f := filepath.Join(s.Dest, url)
		if err := os.MkdirAll(d, 0755); err != nil {
			return err
		}

		// if markdown, need to convert to html
		// otherwise just convert raw html to a string
		//var content string
		//if isMarkdown(page.GetExt()) {
		//	content = string(blackfriday.MarkdownCommon(raw))
		//} else {
		//	content = string(raw)
		//}

		//data passed in to each template
		data := map[string]interface{}{
			"site": s.Conf,
			"page": page,
		}

		// treat all non-markdown pages as templates
		content := page.GetContent()
		if isMarkdown(page.GetExt()) == false {
			// this code will add the page to the list of templates,
			// will execute the template, and then set the content
			// to the rendered template

			if s.templ == nil {
				return fmt.Errorf("No templates defined for page: %s", url)
			}

			t, err := s.templ.New(url).Parse(content)
			if err != nil {
				return err
			}
			var buf bytes.Buffer
			err = t.ExecuteTemplate(&buf, url, data)
			if err != nil {
				return err
			}
			content = buf.String()
		}

		// add document body to the map
		data["content"] = content

		// write the template to a buffer
		// NOTE: if template is nil or empty, then we should parse the
		//       content as if it were a template
		var buf bytes.Buffer
		if layout == "" || layout == "nil" {
			//t, err := s.templ.New(url).Parse(content);
			//if err != nil { return err }
			//err = t.ExecuteTemplate(&buf, url, data);
			//if err != nil { return err }

			buf.WriteString(content)
		} else {
			layout = appendExt(layout, ".html")
			s.templ.ExecuteTemplate(&buf, layout, data)
		}

		logf(MsgGenerateFile, url)
		if err := ioutil.WriteFile(f, buf.Bytes(), 0644); err != nil {
			return err
		}
	}

	return nil
}

// Helper function to write all static files to the destination directory
// during site generation. This will also take care of creating any parent
// directories, if necessary.
func (s *Site) writeStatic() error {

	for _, file := range s.files {
		from := filepath.Join(s.Src, file)
		to := filepath.Join(s.Dest, file)
		logf(MsgCopyingFile, file)
		if err := copyTo(from, to); err != nil {
			return err
		}
	}

	return nil
}

// Helper function to aggregate a list of all categories and their
// related posts.
func (s *Site) calculateCategories() {

	categories := make(map[string][]Page)
	for _, post := range s.posts {
		for _, category := range post.GetCategories() {
			if posts, ok := categories[category]; ok == false {
				categories[category] = append(posts, post)
			} else {
				categories[category] = []Page{post}
			}
		}
	}

	s.Conf.Set("categories", categories)
}

// Helper function to aggregate a list of all tags and their
// related posts.
func (s *Site) calculateTags() {

	tags := make(map[string][]Page)
	for _, post := range s.posts {
		for _, tag := range post.GetTags() {
			if posts, ok := tags[tag]; ok == false {
				tags[tag] = append(posts, post)
			} else {
				tags[tag] = []Page{post}
			}
		}
	}

	s.Conf.Set("tags", tags)
}
<commit_msg>Skip reading hidden directories<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
	"mime"
	"os"
	"path/filepath"
	"text/template"
	"time"
)

var (
	MsgCopyingFile  = "Copying File: %s"
	MsgGenerateFile = "Generating Page: %s"
	MsgUploadFile   = "Uploading: %s"
	MsgUsingConfig  = "Loading Config: %s"
)

type Site struct {
	Src  string // Directory where Jekyll will look to transform files
	Dest string // Directory where Jekyll will write files to
	Conf Config // Configuration date from the _config.yml file

	posts []Page             // Posts thet need to be generated
	pages []Page             // Pages that need to be generated
	files []string           // Static files to get copied to the destination
	templ *template.Template // Compiled templates
}

func NewSite(src, dest string) (*Site, error) {

	// Parse the _config.yml file
	path := filepath.Join(src, "_config.yml")
	conf, err := ParseConfig(path)
	logf(MsgUsingConfig, path)
	if err != nil {
		return nil, err
	}

	site := Site{
		Src:  src,
		Dest: dest,
		Conf: conf,
	}

	// Recursively process all files in the source directory
	// and parse pages, posts, templates, etc
	if err := site.read(); err != nil {
		return nil, err
	}

	return &site, nil
}

// Reloads the site into memory
func (s *Site) Reload() error {
	s.posts = []Page{}
	s.pages = []Page{}
	s.files = []string{}
	s.templ = nil
	return s.read()
}

// Prepares the source directory for site generation
func (s *Site) Prep() error {
	return os.MkdirAll(s.Dest, 0755)
}

// Removes the existing site (typically in _site).
func (s *Site) Clear() error {
	return os.RemoveAll(s.Dest)
}

// Generates a static website based on Jekyll standard layout.
func (s *Site) Generate() error {

	// Remove previously generated site, and then (re)create the
	// destination directory
	if err := s.Clear(); err != nil {
		return err
	}
	if err := s.Prep(); err != nil {
		return err
	}

	// Generate all Pages and Posts and static files
	if err := s.writePages(); err != nil {
		return err
	}
	if err := s.writeStatic(); err != nil {
		return err
	}

	return nil
}

// Deploys a site to S3.
func (s *Site) Deploy(user, pass, url string) error {

	auth := aws.Auth{AccessKey: user, SecretKey: pass}
	b := s3.New(auth, aws.USEast).Bucket(url)

	// walks _site directory and uploads file to S3
	walker := func(fn string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}

		rel, _ := filepath.Rel(s.Dest, fn)
		typ := mime.TypeByExtension(filepath.Ext(rel))
		content, err := ioutil.ReadFile(fn)
		logf(MsgUploadFile, rel)
		if err != nil {
			return err
		}

		// try to upload the file ... sometimes this fails due to amazon
		// issues. If so, we'll re-try
		if err := b.Put(rel, content, typ, s3.PublicRead); err != nil {
			time.Sleep(100 * time.Millisecond) // sleep so that we don't immediately retry
			return b.Put(rel, content, typ, s3.PublicRead)
		}

		// file upload was a success, return nil
		return nil
	}

	return filepath.Walk(s.Dest, walker)
}

// Helper function to traverse the source directory and identify all posts,
// projects, templates, etc and parse.
func (s *Site) read() error {

	// Lists of templates (_layouts, _includes) that we find that
	// will need to be compiled
	layouts := []string{}

	// func to walk the jekyll directory structure
	walker := func(fn string, fi os.FileInfo, err error) error {
		rel, _ := filepath.Rel(s.Src, fn)
		switch {
		case err != nil:
			return nil

		case fi.IsDir() && isHiddenOrTemp(fn):
			return filepath.SkipDir

		// Ignore directories
		case fi.IsDir():
			return nil

		// Ignore Hidden or Temp files
		// (starting with . or ending with ~)
		case isHiddenOrTemp(rel):
			return nil

		// Parse Templates
		case isTemplate(rel):
			layouts = append(layouts, fn)

		// Parse Posts
		case isPost(rel):
			post, err := ParsePost(rel)
			if err != nil {
				return err
			}
			// TODO: this is a hack to get the posts in rev chronological order
			s.posts = append([]Page{post}, s.posts...) //s.posts, post)

		// Parse Pages
		case isPage(rel):
			page, err := ParsePage(rel)
			if err != nil {
				return err
			}
			s.pages = append(s.pages, page)

		// Move static files, no processing required
		case isStatic(rel):
			s.files = append(s.files, rel)
		}
		return nil
	}

	// Walk the diretory recursively to get a list of all posts,
	// pages, templates and static files.
	err := filepath.Walk(s.Src, walker)
	if err != nil {
		return err
	}

	// Compile all templates found, if any
	if len(layouts) > 0 {
		s.templ, err = template.New("layouts").Funcs(funcMap).ParseFiles(layouts...)
		if err != nil {
			return err
		}
	}

	// Add the posts, timestamp, etc to the Site Params
	s.Conf.Set("posts", s.posts)
	s.Conf.Set("time", time.Now())
	s.calculateTags()
	s.calculateCategories()

	return nil
}

// Helper function to write all pages and posts to the destination directory
// during site generation.
func (s *Site) writePages() error {

	// There is really no difference between a Page and a Post (other than
	// initial parsing) so we can combine the lists and use the same rendering
	// code for both.
	pages := []Page{}
	pages = append(pages, s.pages...)
	pages = append(pages, s.posts...)

	for _, page := range pages {
		url := page.GetUrl()
		layout := page.GetLayout()

		// is the layout provided? or is it nil /empty?
		//layoutNil := layout == "" || layout == "nil"

		// make sure the posts's parent dir exists
		d := filepath.Join(s.Dest, filepath.Dir(url))
		f := filepath.Join(s.Dest, url)
		if err := os.MkdirAll(d, 0755); err != nil {
			return err
		}

		// if markdown, need to convert to html
		// otherwise just convert raw html to a string
		//var content string
		//if isMarkdown(page.GetExt()) {
		//	content = string(blackfriday.MarkdownCommon(raw))
		//} else {
		//	content = string(raw)
		//}

		//data passed in to each template
		data := map[string]interface{}{
			"site": s.Conf,
			"page": page,
		}

		// treat all non-markdown pages as templates
		content := page.GetContent()
		if isMarkdown(page.GetExt()) == false {
			// this code will add the page to the list of templates,
			// will execute the template, and then set the content
			// to the rendered template

			if s.templ == nil {
				return fmt.Errorf("No templates defined for page: %s", url)
			}

			t, err := s.templ.New(url).Parse(content)
			if err != nil {
				return err
			}
			var buf bytes.Buffer
			err = t.ExecuteTemplate(&buf, url, data)
			if err != nil {
				return err
			}
			content = buf.String()
		}

		// add document body to the map
		data["content"] = content

		// write the template to a buffer
		// NOTE: if template is nil or empty, then we should parse the
		//       content as if it were a template
		var buf bytes.Buffer
		if layout == "" || layout == "nil" {
			//t, err := s.templ.New(url).Parse(content);
			//if err != nil { return err }
			//err = t.ExecuteTemplate(&buf, url, data);
			//if err != nil { return err }

			buf.WriteString(content)
		} else {
			layout = appendExt(layout, ".html")
			s.templ.ExecuteTemplate(&buf, layout, data)
		}

		logf(MsgGenerateFile, url)
		if err := ioutil.WriteFile(f, buf.Bytes(), 0644); err != nil {
			return err
		}
	}

	return nil
}

// Helper function to write all static files to the destination directory
// during site generation. This will also take care of creating any parent
// directories, if necessary.
func (s *Site) writeStatic() error {

	for _, file := range s.files {
		from := filepath.Join(s.Src, file)
		to := filepath.Join(s.Dest, file)
		logf(MsgCopyingFile, file)
		if err := copyTo(from, to); err != nil {
			return err
		}
	}

	return nil
}

// Helper function to aggregate a list of all categories and their
// related posts.
func (s *Site) calculateCategories() {

	categories := make(map[string][]Page)
	for _, post := range s.posts {
		for _, category := range post.GetCategories() {
			if posts, ok := categories[category]; ok == false {
				categories[category] = append(posts, post)
			} else {
				categories[category] = []Page{post}
			}
		}
	}

	s.Conf.Set("categories", categories)
}

// Helper function to aggregate a list of all tags and their
// related posts.
func (s *Site) calculateTags() {

	tags := make(map[string][]Page)
	for _, post := range s.posts {
		for _, tag := range post.GetTags() {
			if posts, ok := tags[tag]; ok == false {
				tags[tag] = append(posts, post)
			} else {
				tags[tag] = []Page{post}
			}
		}
	}

	s.Conf.Set("tags", tags)
}
<|endoftext|>